extend namestore API to enable faster iterations by returning more than one result at a time

16 files changed, 355 insertions, 177 deletions

diff --git a/src/namestore/gnunet-namestore-fcfsd.c b/src/namestore/gnunet-namestore-fcfsd.c
index ddd609918..6e45f8227 100644
--- a/src/namestore/gnunet-namestore-fcfsd.c
+++ b/src/namestore/gnunet-namestore-fcfsd.c
@@ -326,13 +326,15 @@ iterate_cb (void *cls,
 
   if (1 != rd_len)
   {
-    GNUNET_NAMESTORE_zone_iterator_next (zr->list_it);
+    GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
+                                         1);
     return;
   }
 
   if (GNUNET_GNSRECORD_TYPE_PKEY != rd->record_type)
   {
-    GNUNET_NAMESTORE_zone_iterator_next (zr->list_it);
+    GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
+                                         1);
     return;
   }
 
@@ -343,7 +345,8 @@ iterate_cb (void *cls,
   if (NULL == pkey)
   {
     GNUNET_break (0);
-    GNUNET_NAMESTORE_zone_iterator_next (zr->list_it);
+    GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
+                                         1);
     return;
   }
   if (bytes_free < (strlen (name) + strlen (pkey) + 40))
@@ -359,7 +362,8 @@ iterate_cb (void *cls,
            name,
            pkey);
   zr->write_offset = strlen (zr->zoneinfo);
-  GNUNET_NAMESTORE_zone_iterator_next (zr->list_it);
+  GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
+                                       1);
   GNUNET_free (pkey);
 }
 
diff --git a/src/namestore/gnunet-namestore.c b/src/namestore/gnunet-namestore.c
index 660737595..7b8312b46 100644
--- a/src/namestore/gnunet-namestore.c
+++ b/src/namestore/gnunet-namestore.c
@@ -398,7 +398,8 @@ display_record (void *cls,
   if ( (NULL != name) &&
        (0 != strcmp (name, rname)) )
   {
-    GNUNET_NAMESTORE_zone_iterator_next (list_it);
+    GNUNET_NAMESTORE_zone_iterator_next (list_it,
+                                         1);
     return;
   }
   FPRINTF (stdout,
@@ -441,7 +442,8 @@ display_record (void *cls,
     GNUNET_free (s);
   }
   FPRINTF (stdout, "%s", "\n");
-  GNUNET_NAMESTORE_zone_iterator_next (list_it);
+  GNUNET_NAMESTORE_zone_iterator_next (list_it,
+                                       1);
 }
 
 
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index cc41f071d..2fffbeba7 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -83,8 +83,8 @@ struct ZoneIteration
    * Offset of the zone iteration used to address next result of the zone
    * iteration in the store
    *
-   * Initialy set to 0 in handle_iteration_start
-   * Incremented with by every call to handle_iteration_next
+   * Initialy set to 0 in #handle_iteration_start
+   * Incremented with by every call to #handle_iteration_next
    */
   uint32_t offset;
 
@@ -1065,6 +1065,7 @@ handle_record_store (void *cls,
           GSN_database->iterate_records (GSN_database->cls,
                                          &rp_msg->private_key,
                                          0,
+                                         1,
                                          NULL,
                                          0)) )
     {
@@ -1323,6 +1324,11 @@ struct ZoneIterationProcResult
   struct ZoneIteration *zi;
 
   /**
+   * Number of results left to be returned in this iteration.
+   */
+  uint64_t limit;
+
+  /**
    * Iteration result: iteration done?
    * #IT_SUCCESS_MORE_AVAILABLE:  if there may be more results overall but
    * we got one for now and have sent it to the client
@@ -1361,14 +1367,18 @@ zone_iterate_proc (void *cls,
     proc->res_iteration_finished = IT_SUCCESS_NOT_MORE_RESULTS_AVAILABLE;
     return;
   }
-  if ((NULL == zone_key) || (NULL == name))
+  if ( (NULL == zone_key) ||
+       (NULL == name) )
   {
     /* what is this!? should never happen */
     proc->res_iteration_finished = IT_START;
     GNUNET_break (0);
     return;
   }
-  proc->res_iteration_finished = IT_SUCCESS_MORE_AVAILABLE;
+  GNUNET_assert (proc->limit > 0);
+  proc->limit--;
+  if (0 == proc->limit)
+    proc->res_iteration_finished = IT_SUCCESS_MORE_AVAILABLE;
   send_lookup_response (proc->zi->nc,
                         proc->zi->request_id,
                         zone_key,
@@ -1389,7 +1399,6 @@ zone_iterate_proc (void *cls,
                    name,
                    rd_count,
                    rd);
-
 }
 
 
@@ -1397,18 +1406,23 @@ zone_iterate_proc (void *cls,
  * Perform the next round of the zone iteration.
  *
  * @param zi zone iterator to process
+ * @param limit number of results to return in one pass
  */
 static void
-run_zone_iteration_round (struct ZoneIteration *zi)
+run_zone_iteration_round (struct ZoneIteration *zi,
+                          uint64_t limit)
 {
   struct ZoneIterationProcResult proc;
   struct GNUNET_MQ_Envelope *env;
   struct RecordResultMessage *rrm;
   int ret;
 
-  memset (&proc, 0, sizeof (proc));
+  memset (&proc,
+          0,
+          sizeof (proc));
   proc.zi = zi;
   proc.res_iteration_finished = IT_START;
+  proc.limit = limit;
   while (IT_START == proc.res_iteration_finished)
   {
     if (GNUNET_SYSERR ==
@@ -1419,6 +1433,7 @@ run_zone_iteration_round (struct ZoneIteration *zi)
                                               ? NULL
                                               : &zi->zone,
                                               zi->offset,
+                                              limit,
                                               &zone_iterate_proc,
                                               &proc)))
     {
@@ -1433,7 +1448,8 @@ run_zone_iteration_round (struct ZoneIteration *zi)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "More results available\n");
-    return; /* more results later */
+    return; /* more results later after we get the
+               #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT message */
   }
   /* send empty response to indicate end of list */
   env = GNUNET_MQ_msg (rrm,
@@ -1472,7 +1488,8 @@ handle_iteration_start (void *cls,
   GNUNET_CONTAINER_DLL_insert (nc->op_head,
                                nc->op_tail,
                                zi);
-  run_zone_iteration_round (zi);
+  run_zone_iteration_round (zi,
+                            1);
   GNUNET_SERVICE_client_continue (nc->client);
 }
 
@@ -1525,6 +1542,7 @@ handle_iteration_next (void *cls,
   struct NamestoreClient *nc = cls;
   struct ZoneIteration *zi;
   uint32_t rid;
+  uint64_t limit;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received ZONE_ITERATION_NEXT message\n");
@@ -1533,6 +1551,7 @@ handle_iteration_next (void *cls,
                             1,
                             GNUNET_NO);
   rid = ntohl (zis_msg->gns_header.r_id);
+  limit = GNUNET_ntohll (zis_msg->limit);
   for (zi = nc->op_head; NULL != zi; zi = zi->next)
     if (zi->request_id == rid)
       break;
@@ -1542,7 +1561,8 @@ handle_iteration_next (void *cls,
     GNUNET_SERVICE_client_drop (nc->client);
     return;
   }
-  run_zone_iteration_round (zi);
+  run_zone_iteration_round (zi,
+                            limit);
   GNUNET_SERVICE_client_continue (nc->client);
 }
 
@@ -1665,6 +1685,7 @@ monitor_next (void *cls)
                                        ? NULL
                                        : &zm->zone,
                                        zm->offset++,
+                                       1,
                                        &monitor_iterate_cb,
                                        zm);
   if (GNUNET_SYSERR == ret)
diff --git a/src/namestore/namestore.h b/src/namestore/namestore.h
index ec7f28704..b398af8a9 100644
--- a/src/namestore/namestore.h
+++ b/src/namestore/namestore.h
@@ -345,6 +345,14 @@ struct ZoneIterationNextMessage
    * Type will be #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT
    */
   struct GNUNET_NAMESTORE_Header gns_header;
+
+  /**
+   * Number of records to return to the iterator in one shot
+   * (before #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT
+   * should be send again). In NBO.
+   */
+  uint64_t limit;
+
 };
 
 
diff --git a/src/namestore/namestore_api.c b/src/namestore/namestore_api.c
index 05bdf17da..65d3d75d0 100644
--- a/src/namestore/namestore_api.c
+++ b/src/namestore/namestore_api.c
@@ -1281,9 +1281,12 @@ GNUNET_NAMESTORE_zone_iteration_start (struct GNUNET_NAMESTORE_Handle *h,
  * for the next record.
  *
  * @param it the iterator
+ * @param limit number of records to return to the iterator in one shot
+ *         (before #GNUNET_NAMESTORE_zone_iterator_next is to be called again)
  */
 void
-GNUNET_NAMESTORE_zone_iterator_next (struct GNUNET_NAMESTORE_ZoneIterator *it)
+GNUNET_NAMESTORE_zone_iterator_next (struct GNUNET_NAMESTORE_ZoneIterator *it,
+                                     uint64_t limit)
 {
   struct GNUNET_NAMESTORE_Handle *h = it->h;
   struct ZoneIterationNextMessage *msg;
@@ -1294,6 +1297,7 @@ GNUNET_NAMESTORE_zone_iterator_next (struct GNUNET_NAMESTORE_ZoneIterator *it)
   env = GNUNET_MQ_msg (msg,
                        GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT);
   msg->gns_header.r_id = htonl (it->op_id);
+  msg->limit = GNUNET_htonll (limit);
   GNUNET_MQ_send (h->mq,
                   env);
 }
diff --git a/src/namestore/plugin_namestore_flat.c b/src/namestore/plugin_namestore_flat.c
index 305fe7ba1..e9ed1cc4f 100644
--- a/src/namestore/plugin_namestore_flat.c
+++ b/src/namestore/plugin_namestore_flat.c
@@ -86,6 +86,7 @@ struct Plugin
 
 };
 
+
 struct FlatFileEntry
 {
   /**
@@ -501,9 +502,7 @@ namestore_flat_lookup_records (void *cls,
   size_t key_len;
 
   if (NULL == zone)
-  {
     return GNUNET_SYSERR;
-  }
   key_len = strlen (label) + sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey);
   key = GNUNET_malloc (key_len);
   GNUNET_memcpy (key,
@@ -532,30 +531,77 @@ namestore_flat_lookup_records (void *cls,
 }
 
 
+/**
+ * Closure for #iterate_zones.
+ */
+struct IterateContext
+{
+  /**
+   * How many more records should we skip before returning results?
+   */
+  uint64_t offset;
+
+  /**
+   * How many more records should we return?
+   */
+  uint64_t limit;
+
+  /**
+   * Target zone.
+   */
+  const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone;
+
+  /**
+   * Function to call on each record.
+   */
+  GNUNET_NAMESTORE_RecordIterator iter;
+
+  /**
+   * Closure for @e iter.
+   */
+  void *iter_cls;
+
+};
+
+
+/**
+ * Helper function for #namestore_flat_iterate_records().
+ *
+ * @param cls a `struct IterateContext`
+ * @param key unused
+ * @param value a `struct FlatFileEntry`
+ * @return #GNUNET_YES to continue the iteration
+ */
 static int
 iterate_zones (void *cls,
                const struct GNUNET_HashCode *key,
                void *value)
 {
-  struct Plugin *plugin = cls;
+  struct IterateContext *ic = cls;
   struct FlatFileEntry *entry = value;
 
   (void) key;
-  if ((plugin->target_offset > plugin->offset) ||
-      ( (NULL != plugin->iter_zone) &&
-        (0 != memcmp (entry->private_key,
-                      plugin->iter_zone,
-                      sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))))) {
-    plugin->offset++;
+  if (0 == ic->limit)
+    return GNUNET_NO;
+  if ( (NULL != it->zone) &&
+       (0 != memcmp (entry->private_key,
+                     ic->zone,
+                     sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))) )
+    return GNUNET_YES;
+  if (ic->offset > 0)
+  {
+    ic->offset--;
     return GNUNET_YES;
   }
-  plugin->iter (plugin->iter_cls,
-                entry->private_key,
-                entry->label,
-                entry->record_count,
-                entry->record_data);
-  plugin->iter_result_found = GNUNET_YES;
-  return GNUNET_NO;
+  ic->iter (ic->iter_cls,
+            entry->private_key,
+            entry->label,
+            entry->record_count,
+            entry->record_data);
+  ic->limit--;
+  if (0 == ic->limit)
+    return GNUNET_NO;
+  return GNUNET_YES;
 }
 
 
@@ -566,31 +612,31 @@ iterate_zones (void *cls,
  * @param cls closure (internal context for the plugin)
  * @param zone hash of public key of the zone, NULL to iterate over all zones
  * @param offset offset in the list of all matching records
+ * @param limit maximum number of results to return to @a iter
  * @param iter function to call with the result
  * @param iter_cls closure for @a iter
- * @return #GNUNET_OK on success, #GNUNET_NO if there were no results, #GNUNET_SYSERR on error
+ * @return #GNUNET_OK on success, #GNUNET_NO if there were no more results, #GNUNET_SYSERR on error
  */
 static int
 namestore_flat_iterate_records (void *cls,
                                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
                                 uint64_t offset,
+                                uint64_t limit,
                                 GNUNET_NAMESTORE_RecordIterator iter,
                                 void *iter_cls)
 {
   struct Plugin *plugin = cls;
+  struct IterateContext ic;
 
-  /* FIXME: maybe use separate closure to better handle
-     recursive calls? */
-  plugin->target_offset = offset;
-  plugin->offset = 0;
-  plugin->iter = iter;
-  plugin->iter_cls = iter_cls;
-  plugin->iter_zone = zone;
-  plugin->iter_result_found = GNUNET_NO;
+  ic.offset = offset;
+  ic.limit = limit;
+  ic.iter = iter;
+  ic.iter_cls = iter_cls;
+  ic.zone = zone;
   GNUNET_CONTAINER_multihashmap_iterate (plugin->hm,
                                          &iterate_zones,
-                                         plugin);
-  return plugin->iter_result_found;
+                                         &ic);
+  return (0 == ic.limit) ? GNUNET_OK : GNUNET_NO;
 }
 
 
diff --git a/src/namestore/plugin_namestore_postgres.c b/src/namestore/plugin_namestore_postgres.c
index e38fcafb1..4a24ddf88 100644
--- a/src/namestore/plugin_namestore_postgres.c
+++ b/src/namestore/plugin_namestore_postgres.c
@@ -149,22 +149,28 @@ database_setup (struct Plugin *plugin)
     struct GNUNET_PQ_PreparedStatement ps[] = {
       GNUNET_PQ_make_prepare ("store_records",
                               "INSERT INTO ns097records (zone_private_key, pkey, rvalue, record_count, record_data, label) VALUES "
-                              "($1, $2, $3, $4, $5, $6)", 6),
+                              "($1, $2, $3, $4, $5, $6)",
+                              6),
       GNUNET_PQ_make_prepare ("delete_records",
                               "DELETE FROM ns097records "
-                              "WHERE zone_private_key=$1 AND label=$2", 2),
+                              "WHERE zone_private_key=$1 AND label=$2",
+                              2),
       GNUNET_PQ_make_prepare ("zone_to_name",
                               "SELECT record_count,record_data,label FROM ns097records"
-                              " WHERE zone_private_key=$1 AND pkey=$2", 2),
+                              " WHERE zone_private_key=$1 AND pkey=$2",
+                              2),
       GNUNET_PQ_make_prepare ("iterate_zone",
                               "SELECT record_count,record_data,label FROM ns097records "
-                              "WHERE zone_private_key=$1 ORDER BY rvalue LIMIT 1 OFFSET $2", 2),
+                              "WHERE zone_private_key=$1 ORDER BY rvalue OFFSET $2 LIMIT $3",
+                              3),
       GNUNET_PQ_make_prepare ("iterate_all_zones",
                               "SELECT record_count,record_data,label,zone_private_key"
-                              " FROM ns097records ORDER BY rvalue LIMIT 1 OFFSET $1", 1),
+                              " FROM ns097records ORDER BY rvalue OFFSET $1 LIMIT $2",
+                              2),
       GNUNET_PQ_make_prepare ("lookup_label",
                               "SELECT record_count,record_data,label "
-                              "FROM ns097records WHERE zone_private_key=$1 AND label=$2", 2),
+                              "FROM ns097records WHERE zone_private_key=$1 AND label=$2",
+                              2),
       GNUNET_PQ_PREPARED_STATEMENT_END
     };
 
@@ -278,6 +284,12 @@ struct ParserContext
    * Zone key, NULL if part of record.
    */
   const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key;
+
+  /**
+   * Number of results still to return (counted down by
+   * number of results given to iterator).
+   */
+  uint64_t limit;
 };
 
 
@@ -360,6 +372,7 @@ parse_result_call_iterator (void *cls,
     }
     GNUNET_PQ_cleanup_result (rs);
   }
+  pc->limit -= num_results;
 }
 
 
@@ -410,14 +423,16 @@ namestore_postgres_lookup_records (void *cls,
  * @param cls closure (internal context for the plugin)
  * @param zone hash of public key of the zone, NULL to iterate over all zones
  * @param offset offset in the list of all matching records
+ * @param limit maximum number of results to fetch
  * @param iter function to call with the result
  * @param iter_cls closure for @a iter
- * @return #GNUNET_OK on success, #GNUNET_NO if there were no results, #GNUNET_SYSERR on error
+ * @return #GNUNET_OK on success, #GNUNET_NO if there were no more results, #GNUNET_SYSERR on error
  */
 static int
 namestore_postgres_iterate_records (void *cls,
                                     const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
                                     uint64_t offset,
+                                    uint64_t limit,
                                     GNUNET_NAMESTORE_RecordIterator iter,
                                     void *iter_cls)
 {
@@ -428,10 +443,12 @@ namestore_postgres_iterate_records (void *cls,
   pc.iter = iter;
   pc.iter_cls = iter_cls;
   pc.zone_key = zone;
+  pc.limit = limit;
   if (NULL == zone)
   {
     struct GNUNET_PQ_QueryParam params_without_zone[] = {
       GNUNET_PQ_query_param_uint64 (&offset),
+      GNUNET_PQ_query_param_uint64 (&limit),
       GNUNET_PQ_query_param_end
     };
 
@@ -446,6 +463,7 @@ namestore_postgres_iterate_records (void *cls,
     struct GNUNET_PQ_QueryParam params_with_zone[] = {
       GNUNET_PQ_query_param_auto_from_type (zone),
       GNUNET_PQ_query_param_uint64 (&offset),
+      GNUNET_PQ_query_param_uint64 (&limit),
       GNUNET_PQ_query_param_end
     };
 
@@ -458,9 +476,9 @@ namestore_postgres_iterate_records (void *cls,
   if (res < 0)
     return GNUNET_SYSERR;
 
-  if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res)
+  if ( (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res) ||
+       (pc.limit > 0) )
     return GNUNET_NO;
-
   return GNUNET_OK;
 }
 
diff --git a/src/namestore/plugin_namestore_sqlite.c b/src/namestore/plugin_namestore_sqlite.c
index 5ad84688c..f905787b5 100644
--- a/src/namestore/plugin_namestore_sqlite.c
+++ b/src/namestore/plugin_namestore_sqlite.c
@@ -302,12 +302,12 @@ database_setup (struct Plugin *plugin)
         sq_prepare (plugin->dbh,
                     "SELECT record_count,record_data,label"
                     " FROM ns097records WHERE zone_private_key=?"
-                    " ORDER BY rvalue LIMIT 1 OFFSET ?",
+                    " ORDER BY rvalue LIMIT ? OFFSET ?",
                     &plugin->iterate_zone)) ||
        (SQLITE_OK !=
         sq_prepare (plugin->dbh,
                     "SELECT record_count,record_data,label,zone_private_key"
-                    " FROM ns097records ORDER BY rvalue LIMIT 1 OFFSET ?",
+                    " FROM ns097records ORDER BY rvalue LIMIT ? OFFSET ?",
                     &plugin->iterate_all_zones))  ||
        (SQLITE_OK !=
         sq_prepare (plugin->dbh,
@@ -526,16 +526,18 @@ namestore_sqlite_store_records (void *cls,
  * @param plugin plugin context
  * @param stmt to run (and then clean up)
  * @param zone_key private key of the zone
+ * @param limit maximum number of results to fetch
  * @param iter iterator to call with the result
  * @param iter_cls closure for @a iter
  * @return #GNUNET_OK on success, #GNUNET_NO if there were no results, #GNUNET_SYSERR on error
  */
 static int
-get_record_and_call_iterator (struct Plugin *plugin,
-                              sqlite3_stmt *stmt,
-                              const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key,
-                              GNUNET_NAMESTORE_RecordIterator iter,
-                              void *iter_cls)
+get_records_and_call_iterator (struct Plugin *plugin,
+                               sqlite3_stmt *stmt,
+                               const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key,
+                               uint64_t limit,
+                               GNUNET_NAMESTORE_RecordIterator iter,
+                               void *iter_cls)
 {
   uint32_t record_count;
   size_t data_size;
@@ -545,74 +547,86 @@ get_record_and_call_iterator (struct Plugin *plugin,
   int ret;
   int sret;
 
-  ret = GNUNET_NO;
-  if (SQLITE_ROW == (sret = sqlite3_step (stmt)))
+  ret = GNUNET_OK;
+  for (uint64_t i = 0;i<limit ; i++)
   {
-    struct GNUNET_SQ_ResultSpec rs[] = {
-      GNUNET_SQ_result_spec_uint32 (&record_count),
-      GNUNET_SQ_result_spec_variable_size (&data, &data_size),
-      GNUNET_SQ_result_spec_string (&label),
-      GNUNET_SQ_result_spec_end
-    };
-    struct GNUNET_SQ_ResultSpec rsx[] = {
-      GNUNET_SQ_result_spec_uint32 (&record_count),
-      GNUNET_SQ_result_spec_variable_size (&data, &data_size),
-      GNUNET_SQ_result_spec_string (&label),
-      GNUNET_SQ_result_spec_auto_from_type (&zk),
-      GNUNET_SQ_result_spec_end
-    };
-
-    if (NULL == zone_key)
-    {
-      zone_key = &zk;
-      ret = GNUNET_SQ_extract_result (stmt,
-                                      rsx);
-    }
-    else
-    {
-      ret = GNUNET_SQ_extract_result (stmt,
-                                      rs);
-    }
-    if ( (GNUNET_OK != ret) ||
-         (record_count > 64 * 1024) )
+    if (SQLITE_ROW == (sret = sqlite3_step (stmt)))
     {
-      /* sanity check, don't stack allocate far too much just
-         because database might contain a large value here */
-      GNUNET_break (0);
-      ret = GNUNET_SYSERR;
+      struct GNUNET_SQ_ResultSpec rs[] = {
+        GNUNET_SQ_result_spec_uint32 (&record_count),
+        GNUNET_SQ_result_spec_variable_size (&data, &data_size),
+        GNUNET_SQ_result_spec_string (&label),
+        GNUNET_SQ_result_spec_end
+      };
+      struct GNUNET_SQ_ResultSpec rsx[] = {
+        GNUNET_SQ_result_spec_uint32 (&record_count),
+        GNUNET_SQ_result_spec_variable_size (&data, &data_size),
+        GNUNET_SQ_result_spec_string (&label),
+        GNUNET_SQ_result_spec_auto_from_type (&zk),
+        GNUNET_SQ_result_spec_end
+      };
+
+      if (NULL == zone_key)
+      {
+        zone_key = &zk;
+        ret = GNUNET_SQ_extract_result (stmt,
+                                        rsx);
+      }
+      else
+      {
+        ret = GNUNET_SQ_extract_result (stmt,
+                                        rs);
+      }
+      if ( (GNUNET_OK != ret) ||
+           (record_count > 64 * 1024) )
+      {
+        /* sanity check, don't stack allocate far too much just
+           because database might contain a large value here */
+        GNUNET_break (0);
+        ret = GNUNET_SYSERR;
+        break;
+      }
+      else
+      {
+        struct GNUNET_GNSRECORD_Data rd[record_count];
+
+        if (GNUNET_OK !=
+            GNUNET_GNSRECORD_records_deserialize (data_size,
+                                                  data,
+                                                  record_count,
+                                                  rd))
+        {
+          GNUNET_break (0);
+          ret = GNUNET_SYSERR;
+          break;
+        }
+        else
+        {
+          if (NULL != iter)
+            iter (iter_cls,
+                  zone_key,
+                  label,
+                  record_count,
+                  rd);
+        }
+      }
+      GNUNET_SQ_cleanup_result (rs);
     }
     else
     {
-      struct GNUNET_GNSRECORD_Data rd[record_count];
-
-      if (GNUNET_OK !=
-          GNUNET_GNSRECORD_records_deserialize (data_size,
-                                                data,
-                                                record_count,
-                                                rd))
+      if (SQLITE_DONE != sret)
       {
-        GNUNET_break (0);
-        ret = GNUNET_SYSERR;
+        LOG_SQLITE (plugin,
+                    GNUNET_ERROR_TYPE_ERROR,
+                    "sqlite_step");
+        ret = GNUNET_SYSERR;
       }
       else
       {
-        if (NULL != iter)
-          iter (iter_cls,
-                zone_key,
-                label,
-                record_count,
-                rd);
-        ret = GNUNET_YES;
+        ret = GNUNET_NO;
       }
+      break;
     }
-    GNUNET_SQ_cleanup_result (rs);
-  }
-  else
-  {
-    if (SQLITE_DONE != sret)
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR,
-                  "sqlite_step");
   }
   GNUNET_SQ_reset (plugin->dbh,
                    stmt);
@@ -656,11 +670,12 @@ namestore_sqlite_lookup_records (void *cls,
                      plugin->lookup_label);
     return GNUNET_SYSERR;
   }
-  return get_record_and_call_iterator (plugin,
-                                       plugin->lookup_label,
-                                       zone,
-                                       iter,
-                                       iter_cls);
+  return get_records_and_call_iterator (plugin,
+                                        plugin->lookup_label,
+                                        zone,
+                                        1,
+                                        iter,
+                                        iter_cls);
 }
 
 
@@ -671,14 +686,16 @@ namestore_sqlite_lookup_records (void *cls,
  * @param cls closure (internal context for the plugin)
  * @param zone hash of public key of the zone, NULL to iterate over all zones
  * @param offset offset in the list of all matching records
+ * @param limit maximum number of results to return
  * @param iter function to call with the result
  * @param iter_cls closure for @a iter
- * @return #GNUNET_OK on success, #GNUNET_NO if there were no results, #GNUNET_SYSERR on error
+ * @return #GNUNET_OK on success, #GNUNET_NO if there were no more results, #GNUNET_SYSERR on error
  */
 static int
 namestore_sqlite_iterate_records (void *cls,
                                   const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
                                   uint64_t offset,
+                                  uint64_t limit,
                                   GNUNET_NAMESTORE_RecordIterator iter,
                                   void *iter_cls)
 {
@@ -689,6 +706,7 @@ namestore_sqlite_iterate_records (void *cls,
   if (NULL == zone)
   {
     struct GNUNET_SQ_QueryParam params[] = {
+      GNUNET_SQ_query_param_uint64 (&limit),
       GNUNET_SQ_query_param_uint64 (&offset),
       GNUNET_SQ_query_param_end
     };
@@ -701,6 +719,7 @@ namestore_sqlite_iterate_records (void *cls,
   {
     struct GNUNET_SQ_QueryParam params[] = {
       GNUNET_SQ_query_param_auto_from_type (zone),
+      GNUNET_SQ_query_param_uint64 (&limit),
       GNUNET_SQ_query_param_uint64 (&offset),
       GNUNET_SQ_query_param_end
     };
@@ -718,11 +737,12 @@ namestore_sqlite_iterate_records (void *cls,
                      stmt);
     return GNUNET_SYSERR;
   }
-  return get_record_and_call_iterator (plugin,
-                                       stmt,
-                                       zone,
-                                       iter,
-                                       iter_cls);
+  return get_records_and_call_iterator (plugin,
+                                        stmt,
+                                        zone,
+                                        limit,
+                                        iter,
+                                        iter_cls);
 }
 
 
@@ -764,11 +784,12 @@ namestore_sqlite_zone_to_name (void *cls,
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Performing reverse lookup for `%s'\n",
        GNUNET_GNSRECORD_z2s (value_zone));
-  return get_record_and_call_iterator (plugin,
-                                       plugin->zone_to_name,
-                                       zone,
-                                       iter,
-                                       iter_cls);
+  return get_records_and_call_iterator (plugin,
+                                        plugin->zone_to_name,
+                                        zone,
+                                        1,
+                                        iter,
+                                        iter_cls);
 }
 
 
@@ -787,7 +808,9 @@ libgnunet_plugin_namestore_sqlite_init (void *cls)
 
   if (NULL != plugin.cfg)
     return NULL;                /* can only initialize once! */
-  memset (&plugin, 0, sizeof (struct Plugin));
+  memset (&plugin,
+          0,
+          sizeof (struct Plugin));
   plugin.cfg = cfg;
   if (GNUNET_OK != database_setup (&plugin))
   {
diff --git a/src/namestore/plugin_rest_namestore.c b/src/namestore/plugin_rest_namestore.c
index aedb39159..fd1528a1d 100644
--- a/src/namestore/plugin_rest_namestore.c
+++ b/src/namestore/plugin_rest_namestore.c
@@ -452,7 +452,8 @@ namestore_list_response (void *cls,
                 "%s does not match %s\n",
                 rname,
                 handle->name);
-    GNUNET_NAMESTORE_zone_iterator_next (handle->list_it);
+    GNUNET_NAMESTORE_zone_iterator_next (handle->list_it,
+                                         1);
     return;
   }
 
@@ -483,7 +484,8 @@ namestore_list_response (void *cls,
   }
 
   json_decref (result_array);
-  GNUNET_NAMESTORE_zone_iterator_next (handle->list_it);
+  GNUNET_NAMESTORE_zone_iterator_next (handle->list_it,
+                                       1);
 }
 
 
diff --git a/src/namestore/test_namestore_api.conf b/src/namestore/test_namestore_api.conf
index a1a674d89..29ff90b5d 100644
--- a/src/namestore/test_namestore_api.conf
+++ b/src/namestore/test_namestore_api.conf
@@ -1,8 +1,19 @@
+@INLINE@ ../../contrib/no_forcestart.conf
+@INLINE@ ../../contrib/no_autostart_above_core.conf
+
 [PATHS]
 GNUNET_TEST_HOME = /tmp/test-gnunet-namestore/
 
 [namestore]
 DATABASE = sqlite
+AUTOSTART = YES
+
+[namecache]
+DATABASE = sqlite
+AUTOSTART = YES
+
+[identity]
+AUTOSTART = YES
 
 [namestore-sqlite]
 FILENAME = $GNUNET_TEST_HOME/namestore/sqlite_test.db
diff --git a/src/namestore/test_namestore_api_monitoring_existing.c b/src/namestore/test_namestore_api_monitoring_existing.c
index cd1838b5c..374ad44dd 100644
--- a/src/namestore/test_namestore_api_monitoring_existing.c
+++ b/src/namestore/test_namestore_api_monitoring_existing.c
@@ -71,18 +71,18 @@ do_shutdown ()
 
   if (NULL != ns_ops[0])
   {
-        GNUNET_NAMESTORE_cancel(ns_ops[0]);
-        ns_ops[0] = NULL;
+    GNUNET_NAMESTORE_cancel(ns_ops[0]);
+    ns_ops[0] = NULL;
   }
   if (NULL != ns_ops[1])
   {
-        GNUNET_NAMESTORE_cancel(ns_ops[1]);
-        ns_ops[1] = NULL;
+    GNUNET_NAMESTORE_cancel(ns_ops[1]);
+    ns_ops[1] = NULL;
   }
   if (NULL != ns_ops[2])
   {
-        GNUNET_NAMESTORE_cancel(ns_ops[2]);
-        ns_ops[2] = NULL;
+    GNUNET_NAMESTORE_cancel(ns_ops[2]);
+    ns_ops[2] = NULL;
   }
 
   if (NULL != nsh)
@@ -274,18 +274,20 @@ put_cont (void *cls, int32_t success, const char *emsg)
 static struct GNUNET_GNSRECORD_Data *
 create_record (unsigned int count)
 {
-  unsigned int c;
   struct GNUNET_GNSRECORD_Data * rd;
 
-  rd = GNUNET_malloc (count * sizeof (struct GNUNET_GNSRECORD_Data));
-  for (c = 0; c < count; c++)
+  rd = GNUNET_new_array (count,
+                         struct GNUNET_GNSRECORD_Data);
+  for (unsigned int c = 0; c < count; c++)
   {
     rd[c].expiration_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS).abs_value_us;
     rd[c].record_type = 1111;
     rd[c].data_size = 50;
     rd[c].data = GNUNET_malloc(50);
     rd[c].flags = 0;
-    memset ((char *) rd[c].data, 'a', 50);
+    memset ((char *) rd[c].data,
+            'a',
+            50);
   }
   return rd;
 }
@@ -300,7 +302,10 @@ run (void *cls,
 
   directory = NULL;
   GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONFIGURATION_get_value_string(mycfg, "PATHS", "GNUNET_TEST_HOME", &directory));
+                 GNUNET_CONFIGURATION_get_value_string (mycfg,
+                                                        "PATHS",
+                                                        "GNUNET_TEST_HOME",
+                                                        &directory));
   GNUNET_DISK_directory_remove (directory);
 
   res = 1;
diff --git a/src/namestore/test_namestore_api_zone_iteration.c b/src/namestore/test_namestore_api_zone_iteration.c
index 8960be55d..1343d42bf 100644
--- a/src/namestore/test_namestore_api_zone_iteration.c
+++ b/src/namestore/test_namestore_api_zone_iteration.c
@@ -276,7 +276,8 @@ zone_proc (void *cls,
     returned_records ++;
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Telling namestore to send the next result\n");
-    GNUNET_NAMESTORE_zone_iterator_next (zi);
+    GNUNET_NAMESTORE_zone_iterator_next (zi,
+                                         1);
   }
   else
   {
diff --git a/src/namestore/test_namestore_api_zone_iteration_nick.c b/src/namestore/test_namestore_api_zone_iteration_nick.c
index 791702f97..395d1854f 100644
--- a/src/namestore/test_namestore_api_zone_iteration_nick.c
+++ b/src/namestore/test_namestore_api_zone_iteration_nick.c
@@ -245,7 +245,8 @@ zone_proc (void *cls,
     returned_records ++;
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Telling namestore to send the next result\n");
-    GNUNET_NAMESTORE_zone_iterator_next (zi);
+    GNUNET_NAMESTORE_zone_iterator_next (zi,
+                                         1);
   }
   else
   {
diff --git a/src/namestore/test_namestore_api_zone_iteration_specific_zone.c b/src/namestore/test_namestore_api_zone_iteration_specific_zone.c
index c5ae927b0..d222bf16c 100644
--- a/src/namestore/test_namestore_api_zone_iteration_specific_zone.c
+++ b/src/namestore/test_namestore_api_zone_iteration_specific_zone.c
@@ -238,7 +238,8 @@ zone_proc (void *cls,
     returned_records ++;
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Telling namestore to send the next result\n");
-    GNUNET_NAMESTORE_zone_iterator_next (zi);
+    GNUNET_NAMESTORE_zone_iterator_next (zi,
+                                         1);
   }
   else
   {
diff --git a/src/namestore/test_namestore_api_zone_iteration_stop.c b/src/namestore/test_namestore_api_zone_iteration_stop.c
index a5f040150..d23a5f4cb 100644
--- a/src/namestore/test_namestore_api_zone_iteration_stop.c
+++ b/src/namestore/test_namestore_api_zone_iteration_stop.c
@@ -271,7 +271,8 @@ zone_proc (void *cls,
     returned_records ++;
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Telling namestore to send the next result\n");
-    GNUNET_NAMESTORE_zone_iterator_next (zi);
+    GNUNET_NAMESTORE_zone_iterator_next (zi,
+                                         1);
   }
   else
   {
diff --git a/src/namestore/test_plugin_namestore.c b/src/namestore/test_plugin_namestore.c
index aa5d10809..d591eb325 100644
--- a/src/namestore/test_plugin_namestore.c
+++ b/src/namestore/test_plugin_namestore.c
@@ -47,8 +47,12 @@ unload_plugin (struct GNUNET_NAMESTORE_PluginFunctions *api)
 {
   char *libname;
 
-  GNUNET_asprintf (&libname, "libgnunet_plugin_namestore_%s", plugin_name);
-  GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
+  GNUNET_asprintf (&libname,
+                   "libgnunet_plugin_namestore_%s",
+                   plugin_name);
+  GNUNET_break (NULL ==
+                GNUNET_PLUGIN_unload (libname,
+                                      api));
   GNUNET_free (libname);
 }
 
@@ -65,12 +69,17 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
   struct GNUNET_NAMESTORE_PluginFunctions *ret;
   char *libname;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' namestore plugin\n"),
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Loading `%s' namestore plugin\n"),
               plugin_name);
-  GNUNET_asprintf (&libname, "libgnunet_plugin_namestore_%s", plugin_name);
+  GNUNET_asprintf (&libname,
+                   "libgnunet_plugin_namestore_%s",
+                   plugin_name);
   if (NULL == (ret = GNUNET_PLUGIN_load (libname, (void*) cfg)))
   {
-    FPRINTF (stderr, "Failed to load plugin `%s'!\n", plugin_name);
+    FPRINTF (stderr,
+             "Failed to load plugin `%s'!\n",
+             plugin_name);
     GNUNET_free (libname);
     return NULL;
   }
@@ -81,21 +90,22 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
 
 static void
 test_record (void *cls,
-                                                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key,
-                                                 const char *label,
-                                                 unsigned int rd_count,
-                                                 const struct GNUNET_GNSRECORD_Data *rd)
+             const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key,
+             const char *label,
+             unsigned int rd_count,
+             const struct GNUNET_GNSRECORD_Data *rd)
 {
   int *idp = cls;
   int id = *idp;
   struct GNUNET_CRYPTO_EcdsaPrivateKey tzone_private_key;
   char tname[64];
   unsigned int trd_count = 1 + (id % 1024);
-  unsigned int i;
 
-  GNUNET_snprintf (tname, sizeof (tname),
-                   "a%u", (unsigned int ) id);
-  for (i=0;i<trd_count;i++)
+  GNUNET_snprintf (tname,
+                   sizeof (tname),
+                   "a%u",
+                   (unsigned int ) id);
+  for (unsigned int i=0;i<trd_count;i++)
   {
     GNUNET_assert (rd[i].data_size == id % 10);
     GNUNET_assert (0 == memcmp ("Hello World", rd[i].data, id % 10));
@@ -104,31 +114,39 @@ test_record (void *cls,
   }
   memset (&tzone_private_key, (id % 241), sizeof (tzone_private_key));
   GNUNET_assert (0 == strcmp (label, tname));
-  GNUNET_assert (0 == memcmp (&tzone_private_key, private_key, sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey)));
+  GNUNET_assert (0 == memcmp (&tzone_private_key,
+                              private_key,
+                              sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey)));
 }
 
 
 static void
-get_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp, int id)
+get_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp,
+            int id)
 {
-  GNUNET_assert (GNUNET_OK == nsp->iterate_records (nsp->cls,
-                                            NULL, 0, &test_record, &id));
+  GNUNET_assert (GNUNET_OK ==
+                 nsp->iterate_records (nsp->cls,
+                                       NULL,
+                                       0,
+                                       1,
+                                       &test_record,
+                                       &id));
 }
 
 
 static void
-put_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp, int id)
+put_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp,
+            int id)
 {
   struct GNUNET_CRYPTO_EcdsaPrivateKey zone_private_key;
   char label[64];
   unsigned int rd_count = 1 + (id % 1024);
   struct GNUNET_GNSRECORD_Data rd[rd_count];
   struct GNUNET_CRYPTO_EcdsaSignature signature;
-  unsigned int i;
 
   GNUNET_snprintf (label, sizeof (label),
                    "a%u", (unsigned int ) id);
-  for (i=0;i<rd_count;i++)
+  for (unsigned int i=0;i<rd_count;i++)
   {
     rd[i].data = "Hello World";
     rd[i].data_size = id % 10;
@@ -147,7 +165,9 @@ put_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp, int id)
 
 
 static void
-run (void *cls, char *const *args, const char *cfgfile,
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   struct GNUNET_NAMESTORE_PluginFunctions *nsp;
@@ -169,7 +189,8 @@ run (void *cls, char *const *args, const char *cfgfile,
 
 
 int
-main (int argc, char *argv[])
+main (int argc,
+      char *argv[])
 {
   char cfg_name[128];
   char *const xargv[] = {
@@ -187,12 +208,21 @@ main (int argc, char *argv[])
                     "WARNING",
                     NULL);
   plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
-  GNUNET_snprintf (cfg_name, sizeof (cfg_name), "test_plugin_namestore_%s.conf",
+  GNUNET_snprintf (cfg_name,
+                   sizeof (cfg_name),
+                   "test_plugin_namestore_%s.conf",
                    plugin_name);
-  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv,
-                      "test-plugin-namestore", "nohelp", options, &run, NULL);
+  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1,
+                      xargv,
+                      "test-plugin-namestore",
+                      "nohelp",
+                      options,
+                      &run,
+                      NULL);
   if (ok != 0)
-    FPRINTF (stderr, "Missed some testcases: %d\n", ok);
+    FPRINTF (stderr,
+             "Missed some testcases: %d\n",
+             ok);
   //GNUNET_DISK_directory_remove ("/tmp/gnunet-test-plugin-namestore-sqlite");
   return ok;
 }