NAMESTORE: Allow service-side record set filtering. Fixes #7193

This commit enables zone iteration APIs which allow you to set a record
set filter to determine which records should be returned or not.
In particular filtering of private records and maintenance records
(TOMBSTONE) for zonemaster.

4 files changed, 241 insertions, 90 deletions

diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index 1bfcec76b..0a3dfea25 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -85,6 +85,11 @@ struct ZoneIteration
   struct GNUNET_IDENTITY_PrivateKey zone;
 
   /**
+   * The record set filter
+   */
+  enum GNUNET_GNSRECORD_Filter filter;
+
+  /**
    * Last sequence number in the zone iteration used to address next
    * result of the zone iteration in the store
    *
@@ -181,6 +186,11 @@ struct ZoneMonitor
   struct GNUNET_IDENTITY_PrivateKey zone;
 
   /**
+   * The record set filter
+   */
+  enum GNUNET_GNSRECORD_Filter filter;
+
+  /**
    * Task active during initial iteration.
    */
   struct GNUNET_SCHEDULER_Task *task;
@@ -717,38 +727,63 @@ merge_with_nick_records (const struct GNUNET_GNSRECORD_Data *nick_rd,
  * @param name name
  * @param rd_count number of records in @a rd
  * @param rd array of records
+ * @param filter record set filter
  */
 static void
-send_lookup_response (struct NamestoreClient *nc,
-                      uint32_t request_id,
-                      const struct GNUNET_IDENTITY_PrivateKey *zone_key,
-                      const char *name,
-                      unsigned int rd_count,
-                      const struct GNUNET_GNSRECORD_Data *rd)
+send_lookup_response_with_filter (struct NamestoreClient *nc,
+                                  uint32_t request_id,
+                                  const struct
+                                  GNUNET_IDENTITY_PrivateKey *zone_key,
+                                  const char *name,
+                                  unsigned int rd_count,
+                                  const struct GNUNET_GNSRECORD_Data *rd,
+                                  enum GNUNET_GNSRECORD_Filter filter)
 {
   struct GNUNET_MQ_Envelope *env;
   struct RecordResultMessage *zir_msg;
   struct GNUNET_GNSRECORD_Data *nick;
   struct GNUNET_GNSRECORD_Data *res;
+  struct GNUNET_GNSRECORD_Data rd_nf[rd_count];
+  struct GNUNET_TIME_Absolute block_exp = GNUNET_TIME_UNIT_ZERO_ABS;;
   unsigned int res_count;
+  unsigned int rd_nf_count;
   size_t name_len;
   ssize_t rd_ser_len;
   char *name_tmp;
   char *rd_ser;
+  char *emsg;
 
   nick = get_nick_record (nc, zone_key);
   GNUNET_assert (-1 != GNUNET_GNSRECORD_records_get_size (rd_count, rd));
 
+  if (GNUNET_OK != GNUNET_GNSRECORD_normalize_record_set (name,
+                                                          rd,
+                                                          rd_count,
+                                                          rd_nf,
+                                                          &rd_nf_count,
+                                                          &block_exp,
+                                                          filter,
+                                                          &emsg))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", emsg);
+    GNUNET_free (emsg);
+    GNUNET_assert (0);
+  }
+
+  /**
+   * FIXME if we ever support GNUNET_NAMESTORE_OMIT_PUBLIC,
+   * we need to omit adding this public record here
+   */
   if ((NULL != nick) && (0 != strcmp (name, GNUNET_GNS_EMPTY_LABEL_AT)))
   {
     nick->flags =
       (nick->flags | GNUNET_GNSRECORD_RF_PRIVATE) ^ GNUNET_GNSRECORD_RF_PRIVATE;
-    merge_with_nick_records (nick, rd_count, rd, &res_count, &res);
+    merge_with_nick_records (nick, rd_nf_count, rd_nf, &res_count, &res);
   }
   else
   {
-    res_count = rd_count;
-    res = (struct GNUNET_GNSRECORD_Data *) rd;
+    res_count = rd_nf_count;
+    res = (struct GNUNET_GNSRECORD_Data *) rd_nf;
   }
   if (NULL != nick)
     GNUNET_free (nick);
@@ -782,6 +817,7 @@ send_lookup_response (struct NamestoreClient *nc,
   zir_msg->rd_count = htons (res_count);
   zir_msg->rd_len = htons ((uint16_t) rd_ser_len);
   zir_msg->private_key = *zone_key;
+  zir_msg->expire = GNUNET_TIME_absolute_hton (block_exp);
   name_tmp = (char *) &zir_msg[1];
   GNUNET_memcpy (name_tmp, name, name_len);
   rd_ser = &name_tmp[name_len];
@@ -796,10 +832,33 @@ send_lookup_response (struct NamestoreClient *nc,
                             1,
                             GNUNET_NO);
   GNUNET_MQ_send (nc->mq, env);
-  if (rd != res)
+  if (rd_nf != res)
     GNUNET_free (res);
 }
 
+/**
+ * Generate a `struct LookupNameResponseMessage` and send it to the
+ * given client using the given notification context.
+ *
+ * @param nc client to unicast to
+ * @param request_id request ID to use
+ * @param zone_key zone key of the zone
+ * @param name name
+ * @param rd_count number of records in @a rd
+ * @param rd array of records
+ */
+static void
+send_lookup_response (struct NamestoreClient *nc,
+                      uint32_t request_id,
+                      const struct
+                      GNUNET_IDENTITY_PrivateKey *zone_key,
+                      const char *name,
+                      unsigned int rd_count,
+                      const struct GNUNET_GNSRECORD_Data *rd)
+{
+  send_lookup_response_with_filter (nc, request_id, zone_key, name,
+                                    rd_count, rd, GNUNET_GNSRECORD_FILTER_NONE);
+}
 
 /**
  * Send response to the store request to the client.
@@ -995,7 +1054,7 @@ refresh_block (struct NamestoreClient *nc,
   cop->nc = nc;
   cop->zi = zi;
   if (NULL != zi)
-    zi->cache_ops ++;
+    zi->cache_ops++;
   cop->rid = rid;
   GNUNET_CONTAINER_DLL_insert (cop_head, cop_tail, cop);
   cop->qe = GNUNET_NAMECACHE_block_cache (namecache,
@@ -1081,12 +1140,13 @@ continue_store_activity (struct StoreActivity *sa)
                   "Notifying monitor about changes under label `%s'\n",
                   sa->conv_name);
       zm->limit--;
-      send_lookup_response (zm->nc,
-                            0,
-                            &rp_msg->private_key,
-                            sa->conv_name,
-                            rd_count,
-                            rd);
+      send_lookup_response_with_filter (zm->nc,
+                                        0,
+                                        &rp_msg->private_key,
+                                        sa->conv_name,
+                                        rd_count,
+                                        rd,
+                                        zm->filter);
       sa->zm_pos = zm->next;
     }
     /* great, done with the monitors, unpack (again) for refresh_block operation */
@@ -1454,7 +1514,7 @@ handle_record_lookup (void *cls, const struct LabelLookupMessage *ll_msg)
   if (GNUNET_YES == rlc.found)
     llr_msg->found = htons (GNUNET_YES);
   else if (GNUNET_SYSERR == res)
-   llr_msg->found = htons (GNUNET_SYSERR);
+    llr_msg->found = htons (GNUNET_SYSERR);
   else
     llr_msg->found = htons (GNUNET_NO);
   GNUNET_memcpy (&llr_msg[1], conv_name, name_len);
@@ -1531,13 +1591,15 @@ get_block_exp_existing (void *cls,
   unsigned int rd_pub_count;
   char *emsg;
 
-  if (GNUNET_OK != GNUNET_GNSRECORD_convert_records_for_export (label,
-                                                                rd,
-                                                                rd_count,
-                                                                rd_pub,
-                                                                &rd_pub_count,
-                                                                exp,
-                                                                &emsg))
+  if (GNUNET_OK !=
+      GNUNET_GNSRECORD_normalize_record_set (label,
+                                             rd,
+                                             rd_count,
+                                             rd_pub,
+                                             &rd_pub_count,
+                                             exp,
+                                             GNUNET_GNSRECORD_FILTER_OMIT_PRIVATE,
+                                             &emsg))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "%s\n", emsg);
@@ -1673,14 +1735,15 @@ handle_record_store (void *cls, const struct RecordStoreMessage *rp_msg)
           have_nick = GNUNET_YES;
         }
       }
-      if (GNUNET_OK != GNUNET_GNSRECORD_normalize_record_set (conv_name,
-                                                              rd_clean,
-                                                              rd_clean_off,
-                                                              rd_nf,
-                                                              &rd_nf_count,
-                                                              &new_block_exp,
-                                                              GNUNET_YES,
-                                                              &emsg))
+      if (GNUNET_OK !=
+          GNUNET_GNSRECORD_normalize_record_set (conv_name,
+                                                 rd_clean,
+                                                 rd_clean_off,
+                                                 rd_nf,
+                                                 &rd_nf_count,
+                                                 &new_block_exp,
+                                                 GNUNET_GNSRECORD_FILTER_NONE,
+                                                 &emsg))
       {
         send_store_response (nc, GNUNET_SYSERR, emsg, rid);
         GNUNET_free (emsg);
@@ -1991,12 +2054,13 @@ zone_iterate_proc (void *cls,
   }
   proc->limit--;
   proc->zi->seq = seq;
-  send_lookup_response (proc->zi->nc,
-                        proc->zi->request_id,
-                        zone_key,
-                        name,
-                        rd_count,
-                        rd);
+  send_lookup_response_with_filter (proc->zi->nc,
+                                    proc->zi->request_id,
+                                    zone_key,
+                                    name,
+                                    rd_count,
+                                    rd,
+                                    proc->zi->filter);
 
 
   do_refresh_block = GNUNET_NO;
@@ -2077,6 +2141,7 @@ handle_iteration_start (void *cls,
               "Received ZONE_ITERATION_START message\n");
   zi = GNUNET_new (struct ZoneIteration);
   zi->request_id = ntohl (zis_msg->gns_header.r_id);
+  zi->filter = ntohs (zis_msg->filter);
   zi->offset = 0;
   zi->nc = nc;
   zi->zone = zis_msg->zone;
@@ -2281,6 +2346,7 @@ handle_monitor_start (void *cls, const struct ZoneMonitorStartMessage *zis_msg)
   zm->nc = nc;
   zm->zone = zis_msg->zone;
   zm->limit = 1;
+  zm->filter = ntohs (zis_msg->filter);
   zm->in_first_iteration = (GNUNET_YES == ntohl (zis_msg->iterate_first));
   GNUNET_CONTAINER_DLL_insert (monitor_head, monitor_tail, zm);
   GNUNET_SERVICE_client_mark_monitor (nc->client);
diff --git a/src/namestore/namestore.h b/src/namestore/namestore.h
index 8aaba180f..ad02ffb48 100644
--- a/src/namestore/namestore.h
+++ b/src/namestore/namestore.h
@@ -280,6 +280,12 @@ struct RecordResultMessage
   struct GNUNET_NAMESTORE_Header gns_header;
 
   /**
+   * Expiration time if the record result (if any).
+   * Takes TOMBSTONEs into account.
+   */
+  struct GNUNET_TIME_AbsoluteNBO expire;
+
+  /**
    * Name length
    */
   uint16_t name_len GNUNET_PACKED;
@@ -376,6 +382,17 @@ struct ZoneMonitorStartMessage
   uint32_t iterate_first GNUNET_PACKED;
 
   /**
+   * Record set filter control flags.
+   * See GNUNET_NAMESTORE_Filter enum.
+   */
+  uint16_t filter;
+
+  /**
+   * Reserved for alignment
+   */
+  uint16_t reserved;
+
+  /**
    * Zone key.
    */
   struct GNUNET_IDENTITY_PrivateKey zone;
@@ -420,6 +437,17 @@ struct ZoneIterationStartMessage
    * Zone key.  All zeros for "all zones".
    */
   struct GNUNET_IDENTITY_PrivateKey zone;
+
+  /**
+   * Record set filter control flags.
+   * See GNUNET_NAMESTORE_Filter enum.
+   */
+  uint16_t filter;
+
+  /**
+   * Reserved for alignment
+   */
+  uint16_t reserved;
 };
 
 
diff --git a/src/namestore/namestore_api.c b/src/namestore/namestore_api.c
index 71d969022..51ee25acf 100644
--- a/src/namestore/namestore_api.c
+++ b/src/namestore/namestore_api.c
@@ -83,6 +83,11 @@ struct GNUNET_NAMESTORE_QueueEntry
   GNUNET_NAMESTORE_RecordMonitor proc;
 
   /**
+   * Function to call with the records we get back; or NULL.
+   */
+  GNUNET_NAMESTORE_RecordSetMonitor proc2;
+
+  /**
    * Closure for @e proc.
    */
   void *proc_cls;
@@ -151,6 +156,11 @@ struct GNUNET_NAMESTORE_ZoneIterator
   GNUNET_NAMESTORE_RecordMonitor proc;
 
   /**
+   * The continuation to call with the results
+   */
+  GNUNET_NAMESTORE_RecordSetMonitor proc2;
+
+  /**
    * Closure for @e proc.
    */
   void *proc_cls;
@@ -630,6 +640,9 @@ handle_record_result (void *cls, const struct RecordResultMessage *msg)
     {
       if (NULL != ze->proc)
         ze->proc (ze->proc_cls, &msg->private_key, name, rd_count, rd);
+      if (NULL != ze->proc2)
+        ze->proc2 (ze->proc_cls, &msg->private_key, name,
+                   rd_count, rd, GNUNET_TIME_absolute_ntoh (msg->expire));
       return;
     }
   }
@@ -1272,25 +1285,6 @@ GNUNET_NAMESTORE_zone_to_name (
 }
 
 
-/**
- * Starts a new zone iteration (used to periodically PUT all of our
- * records into our DHT). This MUST lock the struct GNUNET_NAMESTORE_Handle
- * for any other calls than #GNUNET_NAMESTORE_zone_iterator_next and
- * #GNUNET_NAMESTORE_zone_iteration_stop. @a proc will be called once
- * immediately, and then again after
- * #GNUNET_NAMESTORE_zone_iterator_next is invoked.
- *
- * @param h handle to the namestore
- * @param zone zone to access, NULL for all zones
- * @param error_cb function to call on error (i.e. disconnect)
- * @param error_cb_cls closure for @a error_cb
- * @param proc function to call on each name from the zone; it
- *        will be called repeatedly with a value (if available)
- * @param proc_cls closure for @a proc
- * @param finish_cb function to call on completion
- * @param finish_cb_cls closure for @a finish_cb
- * @return an iterator handle to use for iteration
- */
 struct GNUNET_NAMESTORE_ZoneIterator *
 GNUNET_NAMESTORE_zone_iteration_start (
   struct GNUNET_NAMESTORE_Handle *h,
@@ -1332,15 +1326,50 @@ GNUNET_NAMESTORE_zone_iteration_start (
   return it;
 }
 
+struct GNUNET_NAMESTORE_ZoneIterator *
+GNUNET_NAMESTORE_zone_iteration_start2 (
+  struct GNUNET_NAMESTORE_Handle *h,
+  const struct GNUNET_IDENTITY_PrivateKey *zone,
+  GNUNET_SCHEDULER_TaskCallback error_cb,
+  void *error_cb_cls,
+  GNUNET_NAMESTORE_RecordSetMonitor proc,
+  void *proc_cls,
+  GNUNET_SCHEDULER_TaskCallback finish_cb,
+  void *finish_cb_cls,
+  enum GNUNET_GNSRECORD_Filter filter)
+{
+  struct GNUNET_NAMESTORE_ZoneIterator *it;
+  struct GNUNET_MQ_Envelope *env;
+  struct ZoneIterationStartMessage *msg;
+  uint32_t rid;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "Sending ZONE_ITERATION_START message\n");
+  rid = get_op_id (h);
+  it = GNUNET_new (struct GNUNET_NAMESTORE_ZoneIterator);
+  it->h = h;
+  it->error_cb = error_cb;
+  it->error_cb_cls = error_cb_cls;
+  it->finish_cb = finish_cb;
+  it->finish_cb_cls = finish_cb_cls;
+  it->proc2 = proc;
+  it->proc_cls = proc_cls;
+  it->op_id = rid;
+  if (NULL != zone)
+    it->zone = *zone;
+  GNUNET_CONTAINER_DLL_insert_tail (h->z_head, h->z_tail, it);
+  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_START);
+  msg->gns_header.r_id = htonl (rid);
+  msg->filter = htons ((uint16_t) filter);
+  if (NULL != zone)
+    msg->zone = *zone;
+  if (NULL == h->mq)
+    it->env = env;
+  else
+    GNUNET_MQ_send (h->mq, env);
+  return it;
+}
+
 
-/**
- * Calls the record processor specified in #GNUNET_NAMESTORE_zone_iteration_start
- * for the next record.
- *
- * @param it the iterator
- * @param limit number of records to return to the iterator in one shot
- *         (before #GNUNET_NAMESTORE_zone_iterator_next is to be called again)
- */
 void
 GNUNET_NAMESTORE_zone_iterator_next (struct GNUNET_NAMESTORE_ZoneIterator *it,
                                      uint64_t limit)
diff --git a/src/namestore/namestore_api_monitor.c b/src/namestore/namestore_api_monitor.c
index 6670e54ce..968d7ed58 100644
--- a/src/namestore/namestore_api_monitor.c
+++ b/src/namestore/namestore_api_monitor.c
@@ -65,6 +65,16 @@ struct GNUNET_NAMESTORE_ZoneMonitor
   GNUNET_NAMESTORE_RecordMonitor monitor;
 
   /**
+   * Function to call on events.
+   */
+  GNUNET_NAMESTORE_RecordSetMonitor monitor2;
+
+  /**
+   * Record set filter for this monitor
+   */
+  enum GNUNET_GNSRECORD_Filter filter;
+
+  /**
    * Closure for @e monitor.
    */
   void *monitor_cls;
@@ -213,7 +223,11 @@ handle_result (void *cls, const struct RecordResultMessage *lrm)
     GNUNET_assert (
       GNUNET_OK ==
       GNUNET_GNSRECORD_records_deserialize (rd_len, rd_ser_tmp, rd_count, rd));
-    zm->monitor (zm->monitor_cls, &lrm->private_key, name_tmp, rd_count, rd);
+    if (NULL != zm->monitor2)
+      zm->monitor2 (zm->monitor_cls, &lrm->private_key, name_tmp,
+                    rd_count, rd, GNUNET_TIME_absolute_ntoh (lrm->expire));
+    else
+      zm->monitor (zm->monitor_cls, &lrm->private_key, name_tmp, rd_count, rd);
   }
 }
 
@@ -272,33 +286,11 @@ reconnect (struct GNUNET_NAMESTORE_ZoneMonitor *zm)
   env = GNUNET_MQ_msg (sm, GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_START);
   sm->iterate_first = htonl (zm->iterate_first);
   sm->zone = zm->zone;
+  sm->filter = htons ((uint16_t) zm->filter);
   GNUNET_MQ_send (zm->mq, env);
 }
 
 
-/**
- * Begin monitoring a zone for changes.  If @a iterate_first is set,
- * we Will first call the @a monitor function on all existing records
- * in the selected zone(s).  In any case, we will call @a sync and
- * afterwards call @a monitor whenever a record changes.
- *
- * @param cfg configuration to use to connect to namestore
- * @param zone zone to monitor
- * @param iterate_first #GNUNET_YES to first iterate over all existing records,
- *                      #GNUNET_NO to only return changes that happen from now
- * on
- * @param error_cb function to call on error (i.e. disconnect); note that
- *         unlike the other error callbacks in this API, a call to this
- *         function does NOT destroy the monitor handle, it merely signals
- *         that monitoring is down. You need to still explicitly call
- *         #GNUNET_NAMESTORE_zone_monitor_stop().
- * @param error_cb_cls closure for @a error_cb
- * @param monitor function to call on zone changes
- * @param monitor_cls closure for @a monitor
- * @param sync_cb function called when we're in sync with the namestore
- * @param cls closure for @a sync_cb
- * @return handle to stop monitoring
- */
 struct GNUNET_NAMESTORE_ZoneMonitor *
 GNUNET_NAMESTORE_zone_monitor_start (
   const struct GNUNET_CONFIGURATION_Handle *cfg,
@@ -333,6 +325,42 @@ GNUNET_NAMESTORE_zone_monitor_start (
   return zm;
 }
 
+struct GNUNET_NAMESTORE_ZoneMonitor *
+GNUNET_NAMESTORE_zone_monitor_start2 (
+  const struct GNUNET_CONFIGURATION_Handle *cfg,
+  const struct GNUNET_IDENTITY_PrivateKey *zone,
+  int iterate_first,
+  GNUNET_SCHEDULER_TaskCallback error_cb,
+  void *error_cb_cls,
+  GNUNET_NAMESTORE_RecordSetMonitor monitor,
+  void *monitor_cls,
+  GNUNET_SCHEDULER_TaskCallback sync_cb,
+  void *sync_cb_cls,
+  enum GNUNET_GNSRECORD_Filter filter)
+{
+  struct GNUNET_NAMESTORE_ZoneMonitor *zm;
+
+  zm = GNUNET_new (struct GNUNET_NAMESTORE_ZoneMonitor);
+  if (NULL != zone)
+    zm->zone = *zone;
+  zm->iterate_first = iterate_first;
+  zm->error_cb = error_cb;
+  zm->error_cb_cls = error_cb_cls;
+  zm->monitor2 = monitor;
+  zm->monitor_cls = monitor_cls;
+  zm->sync_cb = sync_cb;
+  zm->sync_cb_cls = sync_cb_cls;
+  zm->cfg = cfg;
+  zm->filter = filter;
+  reconnect (zm);
+  if (NULL == zm->mq)
+  {
+    GNUNET_free (zm);
+    return NULL;
+  }
+  return zm;
+}
+
 
 /**
  * Calls the monitor processor specified in #GNUNET_NAMESTORE_zone_monitor_start