-Merge branch 'master' of ssh://gnunet.org/gnunet into gsoc2018/rest_api

17 files changed, 529 insertions, 103 deletions

diff --git a/src/curl/curl.c b/src/curl/curl.c
index 2d0b09d92..cdd39ab8e 100644
--- a/src/curl/curl.c
+++ b/src/curl/curl.c
@@ -448,6 +448,25 @@ download_get_result (struct DownloadBuffer *db,
 
 
 /**
+ * Add custom request header.
+ *
+ * @param ctx cURL context.
+ * @param header header string; will be given to the context AS IS.
+ * @return #GNUNET_OK if no errors occurred, #GNUNET_SYSERR otherwise.
+ */
+int
+GNUNET_CURL_append_header (struct GNUNET_CURL_Context *ctx,
+                           const char *header)
+{
+  ctx->json_header = curl_slist_append (ctx->json_header,
+                                        header);
+  if (NULL == ctx->json_header)
+    return GNUNET_SYSERR;
+
+  return GNUNET_OK;
+}
+
+/**
  * Run the main event loop for the Taler interaction.
  *
  * @param ctx the library context
diff --git a/src/datacache/plugin_datacache_heap.c b/src/datacache/plugin_datacache_heap.c
index 2a08fc81b..494d1ae17 100644
--- a/src/datacache/plugin_datacache_heap.c
+++ b/src/datacache/plugin_datacache_heap.c
@@ -314,6 +314,9 @@ get_cb (void *cls,
   if ( (get_ctx->type != val->type) &&
        (GNUNET_BLOCK_TYPE_ANY != get_ctx->type) )
     return GNUNET_OK;
+  if (0 ==
+      GNUNET_TIME_absolute_get_remaining (val->discard_time).rel_value_us)
+    return GNUNET_OK;
   if (NULL != get_ctx->iter)
     ret = get_ctx->iter (get_ctx->iter_cls,
                          key,
diff --git a/src/datacache/plugin_datacache_postgres.c b/src/datacache/plugin_datacache_postgres.c
index 6eeeb5873..ea87acc1f 100644
--- a/src/datacache/plugin_datacache_postgres.c
+++ b/src/datacache/plugin_datacache_postgres.c
@@ -82,12 +82,12 @@ init_connection (struct Plugin *plugin)
   struct GNUNET_PQ_PreparedStatement ps[] = {
     GNUNET_PQ_make_prepare ("getkt",
                             "SELECT discard_time,type,value,path FROM gn011dc "
-                            "WHERE key=$1 AND type=$2",
-                            2),
+                            "WHERE key=$1 AND type=$2 AND discard_time >= $3",
+                            3),
     GNUNET_PQ_make_prepare ("getk",
                             "SELECT discard_time,type,value,path FROM gn011dc "
-                            "WHERE key=$1",
-                            1),
+                            "WHERE key=$1 AND discard_time >= $2",
+                            2),
     GNUNET_PQ_make_prepare ("getex",
                             "SELECT length(value) AS len,oid,key FROM gn011dc"
                             " WHERE discard_time < $1"
@@ -97,18 +97,15 @@ init_connection (struct Plugin *plugin)
                             "SELECT length(value) AS len,oid,key FROM gn011dc"
                             " ORDER BY prox ASC, discard_time ASC LIMIT 1",
                             0),
-    GNUNET_PQ_make_prepare ("getp",
-                            "SELECT length(value) AS len,oid,key FROM gn011dc "
-                            "ORDER BY discard_time ASC LIMIT 1",
-                            0),
     GNUNET_PQ_make_prepare ("get_random",
-                            "SELECT discard_time,type,value,path,key FROM gn011dc "
-                            "ORDER BY key ASC LIMIT 1 OFFSET $1",
-                            1),
+                            "SELECT discard_time,type,value,path,key FROM gn011dc"
+                            " WHERE discard_time >= $1"
+                            " ORDER BY key ASC LIMIT 1 OFFSET $2",
+                            2),
     GNUNET_PQ_make_prepare ("get_closest",
                             "SELECT discard_time,type,value,path,key FROM gn011dc "
-                            "WHERE key>=$1 ORDER BY key ASC LIMIT $2",
-                            1),
+                            "WHERE key>=$1 AND discard_time >= $2 ORDER BY key ASC LIMIT $3",
+                            3),
     GNUNET_PQ_make_prepare ("delrow",
                             "DELETE FROM gn011dc WHERE oid=$1",
                             1),
@@ -313,18 +310,22 @@ postgres_plugin_get (void *cls,
 {
   struct Plugin *plugin = cls;
   uint32_t type32 = (uint32_t) type;
+  struct GNUNET_TIME_Absolute now;
   struct GNUNET_PQ_QueryParam paramk[] = {
     GNUNET_PQ_query_param_auto_from_type (key),
+    GNUNET_PQ_query_param_absolute_time (&now),
     GNUNET_PQ_query_param_end
   };
   struct GNUNET_PQ_QueryParam paramkt[] = {
     GNUNET_PQ_query_param_auto_from_type (key),
     GNUNET_PQ_query_param_uint32 (&type32),
+    GNUNET_PQ_query_param_absolute_time (&now),
     GNUNET_PQ_query_param_end
   };
   enum GNUNET_DB_QueryStatus res;
   struct HandleResultContext hr_ctx;
 
+  now = GNUNET_TIME_absolute_get ();
   hr_ctx.iter = iter;
   hr_ctx.iter_cls = iter_cls;
   hr_ctx.key = key;
@@ -427,6 +428,7 @@ postgres_plugin_get_random (void *cls,
 {
   struct Plugin *plugin = cls;
   uint32_t off;
+  struct GNUNET_TIME_Absolute now;
   struct GNUNET_TIME_Absolute expiration_time;
   size_t data_size;
   void *data;
@@ -436,6 +438,7 @@ postgres_plugin_get_random (void *cls,
   uint32_t type;
   enum GNUNET_DB_QueryStatus res;
   struct GNUNET_PQ_QueryParam params[] = {
+    GNUNET_PQ_query_param_absolute_time (&now),
     GNUNET_PQ_query_param_uint32 (&off),
     GNUNET_PQ_query_param_end
   };
@@ -459,6 +462,7 @@ postgres_plugin_get_random (void *cls,
     return 0;
   if (NULL == iter)
     return 1;
+  now = GNUNET_TIME_absolute_get ();
   off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
                                   plugin->num_items);
   res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
@@ -620,8 +624,10 @@ postgres_plugin_get_closest (void *cls,
 {
   struct Plugin *plugin = cls;
   uint32_t num_results32 = (uint32_t) num_results;
+  struct GNUNET_TIME_Absolute now;
   struct GNUNET_PQ_QueryParam params[] = {
     GNUNET_PQ_query_param_auto_from_type (key),
+    GNUNET_PQ_query_param_absolute_time (&now),
     GNUNET_PQ_query_param_uint32 (&num_results32),
     GNUNET_PQ_query_param_end
   };
@@ -630,6 +636,7 @@ postgres_plugin_get_closest (void *cls,
 
   erc.iter = iter;
   erc.iter_cls = iter_cls;
+  now = GNUNET_TIME_absolute_get ();
   res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
                                               "get_closest",
                                               params,
diff --git a/src/datacache/test_datacache.c b/src/datacache/test_datacache.c
index 12edb62f8..50e45012d 100644
--- a/src/datacache/test_datacache.c
+++ b/src/datacache/test_datacache.c
@@ -44,6 +44,11 @@ checkIt (void *cls,
          unsigned int path_len,
          const struct GNUNET_PeerIdentity *path)
 {
+  (void) key;
+  (void) type;
+  (void) exp;
+  (void) path_len;
+  (void) path;
   if (size != sizeof (struct GNUNET_HashCode))
   {
     GNUNET_break (0);
@@ -59,17 +64,22 @@ checkIt (void *cls,
 
 
 static void
-run (void *cls, char *const *args, const char *cfgfile,
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   struct GNUNET_DATACACHE_Handle *h;
   struct GNUNET_HashCode k;
   struct GNUNET_HashCode n;
   struct GNUNET_TIME_Absolute exp;
-  unsigned int i;
 
+  (void) cls;
+  (void) args;
+  (void) cfgfile;
   ok = 0;
-  h = GNUNET_DATACACHE_create (cfg, "testcache");
+  h = GNUNET_DATACACHE_create (cfg,
+                               "testcache");
   if (h == NULL)
   {
     FPRINTF (stderr,
@@ -81,7 +91,7 @@ run (void *cls, char *const *args, const char *cfgfile,
   exp = GNUNET_TIME_absolute_get ();
   exp.abs_value_us += 5 * 60 * 1000 * 1000LL;
   memset (&k, 0, sizeof (struct GNUNET_HashCode));
-  for (i = 0; i < 100; i++)
+  for (unsigned int i = 0; i < 100; i++)
   {
     GNUNET_CRYPTO_hash (&k, sizeof (struct GNUNET_HashCode), &n);
     ASSERT (GNUNET_OK ==
@@ -93,26 +103,43 @@ run (void *cls, char *const *args, const char *cfgfile,
                                   0, NULL));
     k = n;
   }
-  memset (&k, 0, sizeof (struct GNUNET_HashCode));
-  for (i = 0; i < 100; i++)
+  memset (&k,
+          0,
+          sizeof (struct GNUNET_HashCode));
+  for (unsigned int i = 0; i < 100; i++)
   {
-    GNUNET_CRYPTO_hash (&k, sizeof (struct GNUNET_HashCode), &n);
-    ASSERT (1 == GNUNET_DATACACHE_get (h, &k, 1 + i % 16, &checkIt, &n));
+    GNUNET_CRYPTO_hash (&k,
+                        sizeof (struct GNUNET_HashCode),
+                        &n);
+    ASSERT (1 == GNUNET_DATACACHE_get (h,
+                                       &k,
+                                       1 + i % 16,
+                                       &checkIt,
+                                       &n));
     k = n;
   }
 
-  memset (&k, 42, sizeof (struct GNUNET_HashCode));
-  GNUNET_CRYPTO_hash (&k, sizeof (struct GNUNET_HashCode), &n);
+  memset (&k,
+          42,
+          sizeof (struct GNUNET_HashCode));
+  GNUNET_CRYPTO_hash (&k,
+                      sizeof (struct GNUNET_HashCode),
+                      &n);
   ASSERT (GNUNET_OK ==
           GNUNET_DATACACHE_put (h,
                                 &k,
                                 GNUNET_YES,
                                 sizeof (struct GNUNET_HashCode),
-                                (const char *) &n, 792,
+                                (const char *) &n,
+                                792,
                                 GNUNET_TIME_UNIT_FOREVER_ABS,
-                                0, NULL));
-  ASSERT (0 != GNUNET_DATACACHE_get (h, &k, 792, &checkIt, &n));
-
+                                0,
+                                NULL));
+  ASSERT (0 != GNUNET_DATACACHE_get (h,
+                                     &k,
+                                     792,
+                                     &checkIt,
+                                     &n));
   GNUNET_DATACACHE_destroy (h);
   ASSERT (ok == 0);
   return;
@@ -137,16 +164,26 @@ main (int argc, char *argv[])
     GNUNET_GETOPT_OPTION_END
   };
 
+  (void) argc;
   GNUNET_log_setup ("test-datacache",
                     "WARNING",
                     NULL);
   plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
-  GNUNET_snprintf (cfg_name, sizeof (cfg_name), "test_datacache_data_%s.conf",
+  GNUNET_snprintf (cfg_name,
+                   sizeof (cfg_name),
+                   "test_datacache_data_%s.conf",
                    plugin_name);
-  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv,
-                      "test-datacache", "nohelp", options, &run, NULL);
+  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1,
+                      xargv,
+                      "test-datacache",
+                      "nohelp",
+                      options,
+                      &run,
+                      NULL);
   if ( (0 != ok) && (77 != ok) )
-    FPRINTF (stderr, "Missed some testcases: %d\n", ok);
+    FPRINTF (stderr,
+             "Missed some testcases: %d\n",
+             ok);
   return ok;
 }
 
diff --git a/src/datacache/test_datacache_quota.c b/src/datacache/test_datacache_quota.c
index 3d02a7244..21e373608 100644
--- a/src/datacache/test_datacache_quota.c
+++ b/src/datacache/test_datacache_quota.c
@@ -41,7 +41,9 @@ static const char *plugin_name;
  * some of the data from the last iteration is still there.
  */
 static void
-run (void *cls, char *const *args, const char *cfgfile,
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   struct GNUNET_DATACACHE_Handle *h;
@@ -50,8 +52,12 @@ run (void *cls, char *const *args, const char *cfgfile,
   char buf[3200];
   struct GNUNET_TIME_Absolute exp;
 
+  (void) cls;
+  (void) args;
+  (void) cfgfile;
   ok = 0;
-  h = GNUNET_DATACACHE_create (cfg, "testcache");
+  h = GNUNET_DATACACHE_create (cfg,
+                               "testcache");
 
   if (h == NULL)
   {
@@ -112,7 +118,8 @@ FAILURE:
 
 
 int
-main (int argc, char *argv[])
+main (int argc,
+      char *argv[])
 {
   char cfg_name[128];
   char *const xargv[] = {
@@ -125,17 +132,27 @@ main (int argc, char *argv[])
     GNUNET_GETOPT_OPTION_END
   };
 
+  (void) argc;
   GNUNET_log_setup ("test-datacache-quota",
                     "WARNING",
                     NULL);
 
   plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
-  GNUNET_snprintf (cfg_name, sizeof (cfg_name), "test_datacache_data_%s.conf",
+  GNUNET_snprintf (cfg_name,
+                   sizeof (cfg_name),
+                   "test_datacache_data_%s.conf",
                    plugin_name);
-  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv,
-                      "test-datacache-quota", "nohelp", options, &run, NULL);
+  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1,
+                      xargv,
+                      "test-datacache-quota",
+                      "nohelp",
+                      options,
+                      &run,
+                      NULL);
   if (0 != ok)
-    FPRINTF (stderr, "Missed some testcases: %d\n", ok);
+    FPRINTF (stderr,
+             "Missed some testcases: %d\n",
+             ok);
   return ok;
 }
 
diff --git a/src/dht/gnunet-service-dht_datacache.c b/src/dht/gnunet-service-dht_datacache.c
index 1f01387ff..7ad9aa728 100644
--- a/src/dht/gnunet-service-dht_datacache.c
+++ b/src/dht/gnunet-service-dht_datacache.c
@@ -171,6 +171,11 @@ datacache_get_iterator (void *cls,
   struct GetRequestContext *ctx = cls;
   enum GNUNET_BLOCK_EvaluationResult eval;
 
+  if (0 == GNUNET_TIME_absolute_get_remaining (exp).rel_value_us)
+  {
+    GNUNET_break (0); /* why does datacache return expired values? */
+    return GNUNET_OK; /* skip expired record */
+  }
   if ( (NULL == data) &&
        (0 == data_size) )
     data = &non_null; /* point anywhere, but not to NULL */
diff --git a/src/dht/gnunet-service-dht_neighbours.c b/src/dht/gnunet-service-dht_neighbours.c
index 9c1395782..56639448f 100644
--- a/src/dht/gnunet-service-dht_neighbours.c
+++ b/src/dht/gnunet-service-dht_neighbours.c
@@ -468,6 +468,7 @@ free_connect_info (void *cls,
 {
   struct ConnectInfo *ci = value;
 
+ (void) cls;
   GNUNET_assert (GNUNET_YES ==
                  GNUNET_CONTAINER_multipeermap_remove (all_desired_peers,
                                                        peer,
@@ -579,7 +580,10 @@ update_desire_strength (void *cls,
                         const struct GNUNET_PeerIdentity *pid,
                         void *value)
 {
-  try_connect (pid, NULL);
+  (void) cls;
+  (void) value;
+  try_connect (pid,
+               NULL);
   return GNUNET_YES;
 }
 
@@ -616,6 +620,8 @@ add_known_to_bloom (void *cls,
   struct GNUNET_BLOCK_Group *bg = cls;
   struct GNUNET_HashCode key_hash;
 
+ (void) cls;
+ (void) value;
   GNUNET_CRYPTO_hash (key,
                       sizeof (struct GNUNET_PeerIdentity),
                       &key_hash);
@@ -643,7 +649,8 @@ send_find_peer_message (void *cls)
   struct GNUNET_BLOCK_Group *bg;
   struct GNUNET_CONTAINER_BloomFilter *peer_bf;
 
-  find_peer_task = NULL;
+ (void) cls;
+ find_peer_task = NULL;
   if (newly_found_peers > bucket_size)
   {
     /* If we are finding many peers already, no need to send out our request right now! */
@@ -716,7 +723,8 @@ handle_core_connect (void *cls,
 {
   struct PeerInfo *pi;
 
-  /* Check for connect to self message */
+ (void) cls;
+ /* Check for connect to self message */
   if (0 == memcmp (&my_identity,
                    peer,
                    sizeof (struct GNUNET_PeerIdentity)))
@@ -739,13 +747,13 @@ handle_core_connect (void *cls,
                       &pi->phash);
   pi->peer_bucket = find_bucket (&pi->phash);
   GNUNET_assert ( (pi->peer_bucket >= 0) &&
-                  (pi->peer_bucket < MAX_BUCKETS) );
+                  ((unsigned int) pi->peer_bucket < MAX_BUCKETS) );
   GNUNET_CONTAINER_DLL_insert_tail (k_buckets[pi->peer_bucket].head,
                                     k_buckets[pi->peer_bucket].tail,
                                     pi);
   k_buckets[pi->peer_bucket].peers_size++;
   closest_bucket = GNUNET_MAX (closest_bucket,
-                               pi->peer_bucket);
+                               (unsigned int) pi->peer_bucket);
   GNUNET_assert (GNUNET_OK ==
                  GNUNET_CONTAINER_multipeermap_put (all_connected_peers,
                                                     pi->id,
@@ -783,6 +791,7 @@ handle_core_disconnect (void *cls,
 {
   struct PeerInfo *to_remove = internal_cls;
 
+  (void) cls;
   /* Check for disconnect from self message */
   if (NULL == to_remove)
     return;
@@ -946,23 +955,23 @@ GDS_am_closest_peer (const struct GNUNET_HashCode *key,
   int bits;
   int other_bits;
   int bucket_num;
-  int count;
   struct PeerInfo *pos;
 
-  if (0 == memcmp (&my_identity_hash, key, sizeof (struct GNUNET_HashCode)))
+  if (0 == memcmp (&my_identity_hash,
+                   key,
+                   sizeof (struct GNUNET_HashCode)))
     return GNUNET_YES;
   bucket_num = find_bucket (key);
   GNUNET_assert (bucket_num >= 0);
   bits = GNUNET_CRYPTO_hash_matching_bits (&my_identity_hash,
                                            key);
   pos = k_buckets[bucket_num].head;
-  count = 0;
-  while ((NULL != pos) && (count < bucket_size))
+  while (NULL != pos) 
   {
-    if ((NULL != bloom) &&
-        (GNUNET_YES ==
-         GNUNET_CONTAINER_bloomfilter_test (bloom,
-                                            &pos->phash)))
+    if ( (NULL != bloom) &&
+         (GNUNET_YES ==
+          GNUNET_CONTAINER_bloomfilter_test (bloom,
+                                             &pos->phash)) )
     {
       pos = pos->next;
       continue;                 /* Skip already checked entries */
@@ -1594,6 +1603,7 @@ static void
 core_init (void *cls,
            const struct GNUNET_PeerIdentity *identity)
 {
+  (void) cls;
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "CORE called, I am %s\n",
               GNUNET_i2s (identity));
@@ -1619,6 +1629,7 @@ check_dht_p2p_put (void *cls,
   uint32_t putlen;
   uint16_t msize;
 
+  (void) cls;
   msize = ntohs (put->header.size);
   putlen = ntohl (put->put_path_length);
   if ((msize <
@@ -1654,7 +1665,17 @@ handle_dht_p2p_put (void *cls,
   struct GNUNET_CONTAINER_BloomFilter *bf;
   struct GNUNET_HashCode test_key;
   int forwarded;
+  struct GNUNET_TIME_Absolute exp_time;
 
+  exp_time = GNUNET_TIME_absolute_ntoh (put->expiration_time);
+  if (0 == GNUNET_TIME_absolute_get_remaining (exp_time).rel_value_us)
+  {
+    GNUNET_STATISTICS_update (GDS_stats,
+                              gettext_noop ("# Expired PUTs discarded"),
+                              1,
+                              GNUNET_NO);
+    return;
+  }
   msize = ntohs (put->header.size);
   putlen = ntohl (put->put_path_length);
   GNUNET_STATISTICS_update (GDS_stats,
@@ -1790,7 +1811,7 @@ handle_dht_p2p_put (void *cls,
       putlen = 0;
 
     /* give to local clients */
-    GDS_CLIENTS_handle_reply (GNUNET_TIME_absolute_ntoh (put->expiration_time),
+    GDS_CLIENTS_handle_reply (exp_time,
                               &put->key,
                               0,
                               NULL,
@@ -1802,7 +1823,7 @@ handle_dht_p2p_put (void *cls,
     /* store locally */
     if ((0 != (options & GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE)) ||
         (GDS_am_closest_peer (&put->key, bf)))
-      GDS_DATACACHE_handle_put (GNUNET_TIME_absolute_ntoh (put->expiration_time),
+      GDS_DATACACHE_handle_put (exp_time,
                                 &put->key,
                                 putlen,
                                 pp,
@@ -1813,7 +1834,7 @@ handle_dht_p2p_put (void *cls,
     forwarded = GDS_NEIGHBOURS_handle_put (ntohl (put->type),
                                            options,
                                            ntohl (put->desired_replication_level),
-                                           GNUNET_TIME_absolute_ntoh (put->expiration_time),
+                                           exp_time,
                                            ntohl (put->hop_count),
                                            bf,
                                            &put->key,
@@ -1830,7 +1851,7 @@ handle_dht_p2p_put (void *cls,
                              ntohl (put->hop_count),
                              ntohl (put->desired_replication_level),
                              putlen, pp,
-                             GNUNET_TIME_absolute_ntoh (put->expiration_time),
+                             exp_time,
                              &put->key,
                              payload,
                              payload_size);
@@ -1907,9 +1928,9 @@ handle_find_peer (const struct GNUNET_PeerIdentity *sender,
                    sizeof (struct GNUNET_HashCode)))
     bucket_idx = closest_bucket;
   else
-    bucket_idx = GNUNET_MIN (closest_bucket,
+    bucket_idx = GNUNET_MIN ((int) closest_bucket,
                              find_bucket (key));
-  if (bucket_idx == GNUNET_SYSERR)
+  if (bucket_idx < 0)
     return;
   bucket = &k_buckets[bucket_idx];
   if (bucket->peers_size == 0)
@@ -2016,6 +2037,7 @@ check_dht_p2p_get (void *cls,
   uint32_t xquery_size;
   uint16_t msize;
 
+  (void) cls;
   msize = ntohs (get->header.size);
   xquery_size = ntohl (get->xquery_size);
   if (msize < sizeof (struct PeerGetMessage) + xquery_size)
@@ -2203,6 +2225,7 @@ check_dht_p2p_result (void *cls,
   uint32_t put_path_length;
   uint16_t msize;
 
+  (void) cls;
   msize = ntohs (prm->header.size);
   put_path_length = ntohl (prm->put_path_length);
   get_path_length = ntohl (prm->get_path_length);
@@ -2316,8 +2339,18 @@ handle_dht_p2p_result (void *cls,
   uint16_t msize;
   size_t data_size;
   enum GNUNET_BLOCK_Type type;
+  struct GNUNET_TIME_Absolute exp_time;
 
   /* parse and validate message */
+  exp_time = GNUNET_TIME_absolute_ntoh (prm->expiration_time);
+  if (0 == GNUNET_TIME_absolute_get_remaining (exp_time).rel_value_us)
+  {
+    GNUNET_STATISTICS_update (GDS_stats,
+                              gettext_noop ("# Expired results discarded"),
+                              1,
+                              GNUNET_NO);
+    return;
+  }
   msize = ntohs (prm->header.size);
   put_path_length = ntohl (prm->put_path_length);
   get_path_length = ntohl (prm->get_path_length);
@@ -2391,7 +2424,6 @@ handle_dht_p2p_result (void *cls,
                    h);
   }
 
-
   /* First, check if 'peer' is already on the path, and if
      so, truncate it instead of expanding. */
   for (unsigned int i=0;i<=get_path_length;i++)
@@ -2399,7 +2431,7 @@ handle_dht_p2p_result (void *cls,
                      peer->id,
                      sizeof (struct GNUNET_PeerIdentity)))
     {
-      process_reply_with_path (GNUNET_TIME_absolute_ntoh (prm->expiration_time),
+      process_reply_with_path (exp_time,
                                &prm->key,
                                i,
                                get_path,
@@ -2420,7 +2452,7 @@ handle_dht_p2p_result (void *cls,
                    get_path_length * sizeof (struct GNUNET_PeerIdentity));
     xget_path[get_path_length] = *peer->id;
 
-    process_reply_with_path (GNUNET_TIME_absolute_ntoh (prm->expiration_time),
+    process_reply_with_path (exp_time,
                              &prm->key,
                              get_path_length + 1,
                              xget_path,
diff --git a/src/gns/gnunet-service-gns_resolver.c b/src/gns/gnunet-service-gns_resolver.c
index a90cc4c0e..54c3cba23 100644
--- a/src/gns/gnunet-service-gns_resolver.c
+++ b/src/gns/gnunet-service-gns_resolver.c
@@ -1377,6 +1377,10 @@ vpn_allocation_cb (void *cls,
     }
   }
   GNUNET_assert (i < vpn_ctx->rd_count);
+  if (0 == vpn_ctx->rd_count)
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _("VPN returned empty result for `%s'\n"),
+                rh->name);
   handle_gns_resolution_result (rh,
                                 vpn_ctx->rd_count,
                                 rd);
@@ -1859,7 +1863,8 @@ handle_gns_resolution_result (void *cls,
   if (0 == rd_count)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                _("GNS lookup failed (zero records found)\n"));
+                _("GNS lookup failed (zero records found for `%s')\n"),
+                rh->name);
     fail_resolution (rh);
     return;
   }
@@ -2370,6 +2375,11 @@ handle_dht_response (void *cls,
     fail_resolution (rh);
     return;
   }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Decrypting DHT block of size %u for `%s', expires %s\n",
+              ntohl (block->purpose.size),
+              rh->name,
+              GNUNET_STRINGS_absolute_time_to_string (exp));
   if (GNUNET_OK !=
       GNUNET_GNSRECORD_block_decrypt (block,
                                       &ac->authority_info.gns_authority,
@@ -2450,6 +2460,10 @@ handle_gns_namecache_resolution_result (void *cls,
 {
   struct GNS_ResolverHandle *rh = cls;
 
+  if (0 == rd_count)
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _("GNS namecache returned empty result for `%s'\n"),
+                rh->name);
   handle_gns_resolution_result (rh,
                                 rd_count,
                                 rd);
diff --git a/src/gnsrecord/gnsrecord_crypto.c b/src/gnsrecord/gnsrecord_crypto.c
index 0752086fe..295d31100 100644
--- a/src/gnsrecord/gnsrecord_crypto.c
+++ b/src/gnsrecord/gnsrecord_crypto.c
@@ -377,6 +377,8 @@ GNUNET_GNSRECORD_block_decrypt (const struct GNUNET_GNSRECORD_Block *block,
                  (0 == (rd[k].flags & GNUNET_GNSRECORD_RF_SHADOW_RECORD)) )
             {
               include_record = GNUNET_NO; /* We have a non-expired, non-shadow record of the same type */
+              GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                          "Ignoring shadow record\n");
               break;
             }
           }
@@ -395,6 +397,16 @@ GNUNET_GNSRECORD_block_decrypt (const struct GNUNET_GNSRECORD_Block *block,
             rd[j] = rd[i];
           j++;
         }
+        else
+        {
+          struct GNUNET_TIME_Absolute at;
+
+          at.abs_value_us = rd[i].expiration_time;
+          GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                      "Excluding record that expired %s (%llu ago)\n",
+                      GNUNET_STRINGS_absolute_time_to_string (at),
+                      (unsigned long long) rd[i].expiration_time - now.abs_value_us);
+        }
       }
       rd_count = j;
       if (NULL != proc)
diff --git a/src/include/gnunet_curl_lib.h b/src/include/gnunet_curl_lib.h
index 17e9aeea4..2b2442a52 100644
--- a/src/include/gnunet_curl_lib.h
+++ b/src/include/gnunet_curl_lib.h
@@ -99,6 +99,17 @@ GNUNET_CURL_get_select_info (struct GNUNET_CURL_Context *ctx,
 
 
 /**
+ * Add custom request header.
+ *
+ * @param ctx cURL context.
+ * @param header header string; will be given to the context AS IS.
+ * @return #GNUNET_OK if no errors occurred, #GNUNET_SYSERR otherwise.
+ */
+int
+GNUNET_CURL_append_header (struct GNUNET_CURL_Context *ctx,
+                           const char *header);
+
+/**
  * Run the main event loop for the CURL interaction.
  *
  * @param ctx the library context
diff --git a/src/my/my.c b/src/my/my.c
index b9d5c8424..2f4cd3ba3 100644
--- a/src/my/my.c
+++ b/src/my/my.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet
-     Copyright (C) 2016 Inria & GNUnet e.V.
+     Copyright (C) 2016, 2018 GNUnet e.V.
 
      GNUnet is free software: you can redistribute it and/or modify it
      under the terms of the GNU Affero General Public License as published
@@ -43,19 +43,20 @@ GNUNET_MY_exec_prepared (struct GNUNET_MYSQL_Context *mc,
 {
   const struct GNUNET_MY_QueryParam *p;
   unsigned int num;
-  unsigned int i;
   MYSQL_STMT *stmt;
 
   num = 0;
-  for (i=0;NULL != params[i].conv;i++)
+  for (unsigned int i=0;NULL != params[i].conv;i++)
     num += params[i].num_params;
   {
     MYSQL_BIND qbind[num];
     unsigned int off;
 
-    memset (qbind, 0, sizeof(qbind));
+    memset (qbind,
+            0,
+            sizeof(qbind));
     off = 0;
-    for (i=0;NULL != (p = &params[i])->conv;i++)
+    for (unsigned int i=0;NULL != (p = &params[i])->conv;i++)
     {
       if (GNUNET_OK !=
           p->conv (p->conv_cls,
@@ -111,9 +112,7 @@ void
 GNUNET_MY_cleanup_query (struct GNUNET_MY_QueryParam *qp,
                          MYSQL_BIND *qbind)
 {
-  unsigned int i;
-
-  for (i=0; NULL != qp[i].conv ;i++)
+  for (unsigned int i=0; NULL != qp[i].conv ;i++)
     if (NULL != qp[i].cleaner)
       qp[i].cleaner (qp[i].conv_cls,
                      &qbind[i]);
@@ -136,7 +135,6 @@ GNUNET_MY_extract_result (struct GNUNET_MYSQL_StatementHandle *sh,
                           struct GNUNET_MY_ResultSpec *rs)
 {
   unsigned int num_fields;
-  unsigned int i;
   int ret;
   MYSQL_STMT *stmt;
 
@@ -153,7 +151,7 @@ GNUNET_MY_extract_result (struct GNUNET_MYSQL_StatementHandle *sh,
   }
 
   num_fields = 0;
-  for (i=0;NULL != rs[i].pre_conv;i++)
+  for (unsigned int i=0;NULL != rs[i].pre_conv;i++)
     num_fields += rs[i].num_fields;
 
   if (mysql_stmt_field_count (stmt) != num_fields)
@@ -169,7 +167,7 @@ GNUNET_MY_extract_result (struct GNUNET_MYSQL_StatementHandle *sh,
 
     memset (result, 0, sizeof (MYSQL_BIND) * num_fields);
     field_off = 0;
-    for (i=0;NULL != rs[i].pre_conv;i++)
+    for (unsigned int i=0;NULL != rs[i].pre_conv;i++)
     {
       struct GNUNET_MY_ResultSpec *rp = &rs[i];
 
@@ -221,7 +219,7 @@ GNUNET_MY_extract_result (struct GNUNET_MYSQL_StatementHandle *sh,
       return GNUNET_SYSERR;
     }
     field_off = 0;
-    for (i=0;NULL != rs[i].post_conv;i++)
+    for (unsigned int i=0;NULL != rs[i].post_conv;i++)
     {
       struct GNUNET_MY_ResultSpec *rp = &rs[i];
 
@@ -256,9 +254,7 @@ GNUNET_MY_extract_result (struct GNUNET_MYSQL_StatementHandle *sh,
 void
 GNUNET_MY_cleanup_result (struct GNUNET_MY_ResultSpec *rs)
 {
-  unsigned int i;
-
-  for (i=0;NULL != rs[i].post_conv;i++)
+  for (unsigned int i=0;NULL != rs[i].post_conv;i++)
     if (NULL != rs[i].cleaner)
       rs[i].cleaner (rs[i].conv_cls,
                      &rs[i]);
diff --git a/src/my/my_query_helper.c b/src/my/my_query_helper.c
index c4516715a..b2a893a44 100644
--- a/src/my/my_query_helper.c
+++ b/src/my/my_query_helper.c
@@ -37,6 +37,7 @@ static void
 my_clean_query (void *cls,
                 MYSQL_BIND *qbind)
 {
+  (void) cls;
   GNUNET_free (qbind[0].buffer);
 }
 
@@ -54,6 +55,7 @@ my_conv_fixed_size (void *cls,
                     const struct GNUNET_MY_QueryParam *qp,
                     MYSQL_BIND *qbind)
 {
+  (void) cls;
   GNUNET_assert (1 == qp->num_params);
   qbind->buffer = (void *) qp->data;
   qbind->buffer_length = qp->data_len;
@@ -99,12 +101,11 @@ my_conv_string (void *cls,
                 const struct GNUNET_MY_QueryParam *qp,
                 MYSQL_BIND *qbind)
 {
+  (void) cls;
   GNUNET_assert (1 == qp->num_params);
-
   qbind->buffer = (void *) qp->data;
   qbind->buffer_length = qp->data_len;
   qbind->buffer_type = MYSQL_TYPE_STRING;
-
   return 1;
 }
 
@@ -142,10 +143,12 @@ my_conv_uint16 (void *cls,
                 const struct GNUNET_MY_QueryParam *qp,
                 MYSQL_BIND *qbind)
 {
+  (void) cls;
   GNUNET_assert (1 == qp->num_params);
   qbind->buffer = (void *) qp->data;
   qbind->buffer_length = sizeof (uint16_t);
   qbind->buffer_type = MYSQL_TYPE_SHORT;
+  qbind->is_unsigned = 1;
   return 1;
 }
 
@@ -184,11 +187,12 @@ my_conv_uint32 (void *cls,
                 const struct GNUNET_MY_QueryParam *qp,
                 MYSQL_BIND *qbind)
 {
+  (void) cls;
   GNUNET_assert (1 == qp->num_params);
   qbind->buffer = (void *) qp->data;
   qbind->buffer_length = sizeof(uint32_t);
   qbind->buffer_type = MYSQL_TYPE_LONG;
-
+  qbind->is_unsigned = 1;
   return 1;
 }
 
@@ -227,10 +231,12 @@ my_conv_uint64 (void *cls,
                 const struct GNUNET_MY_QueryParam *qp,
                 MYSQL_BIND * qbind)
 {
+  (void) cls;
   GNUNET_assert (1 == qp->num_params);
   qbind->buffer = (void *) qp->data;
   qbind->buffer_length = sizeof (uint64_t);
   qbind->buffer_type = MYSQL_TYPE_LONGLONG;
+  qbind->is_unsigned = 1;
   return 1;
 }
 
@@ -273,14 +279,13 @@ my_conv_rsa_public_key (void *cls,
   char *buf;
   size_t buf_size;
 
+  (void) cls;
   GNUNET_assert(1 == qp->num_params);
-
-  buf_size = GNUNET_CRYPTO_rsa_public_key_encode (rsa, &buf);
-
+  buf_size = GNUNET_CRYPTO_rsa_public_key_encode (rsa,
+                                                  &buf);
   qbind->buffer = (void *) buf;
   qbind->buffer_length = buf_size;
   qbind->buffer_type = MYSQL_TYPE_BLOB;
-
   return 1;
 }
 
@@ -325,14 +330,13 @@ my_conv_rsa_signature (void *cls,
   char *buf;
   size_t buf_size;
 
+  (void) cls;
   GNUNET_assert(1 == qp->num_params);
-
   buf_size = GNUNET_CRYPTO_rsa_signature_encode (sig,
                                                  &buf);
   qbind->buffer = (void *) buf;
   qbind->buffer_length = buf_size;
   qbind->buffer_type = MYSQL_TYPE_BLOB;
-
   return 1;
 }
 
diff --git a/src/pq/pq_query_helper.c b/src/pq/pq_query_helper.c
index 799f82ebe..98f697b5d 100644
--- a/src/pq/pq_query_helper.c
+++ b/src/pq/pq_query_helper.c
@@ -50,6 +50,8 @@ qconv_fixed (void *cls,
              void *scratch[],
              unsigned int scratch_length)
 {
+  (void) scratch;
+  (void) scratch_length;
   GNUNET_break (NULL == cls);
   if (1 != param_length)
     return -1;
@@ -117,6 +119,8 @@ qconv_uint16 (void *cls,
   const uint16_t *u_hbo = data;
   uint16_t *u_nbo;
 
+  (void) scratch;
+  (void) scratch_length;
   GNUNET_break (NULL == cls);
   if (1 != param_length)
     return -1;
@@ -172,6 +176,8 @@ qconv_uint32 (void *cls,
   const uint32_t *u_hbo = data;
   uint32_t *u_nbo;
 
+  (void) scratch;
+  (void) scratch_length;
   GNUNET_break (NULL == cls);
   if (1 != param_length)
     return -1;
@@ -227,6 +233,8 @@ qconv_uint64 (void *cls,
   const uint64_t *u_hbo = data;
   uint64_t *u_nbo;
 
+  (void) scratch;
+  (void) scratch_length;
   GNUNET_break (NULL == cls);
   if (1 != param_length)
     return -1;
@@ -371,6 +379,51 @@ GNUNET_PQ_query_param_rsa_signature (const struct GNUNET_CRYPTO_RsaSignature *x)
 
 
 /**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param[out] param_values SQL data to set
+ * @param[out] param_lengths SQL length data to set
+ * @param[out] param_formats SQL format data to set
+ * @param param_length number of entries available in the @a param_values, @a param_lengths and @a param_formats arrays
+ * @param[out] scratch buffer for dynamic allocations (to be done via #GNUNET_malloc()
+ * @param scratch_length number of entries left in @a scratch
+ * @return -1 on error, number of offsets used in @a scratch otherwise
+ */
+static int
+qconv_abs_time (void *cls,
+                const void *data,
+                size_t data_len,
+                void *param_values[],
+                int param_lengths[],
+                int param_formats[],
+                unsigned int param_length,
+                void *scratch[],
+                unsigned int scratch_length)
+{
+  const struct GNUNET_TIME_Absolute *u = data;
+  struct GNUNET_TIME_Absolute abs;
+  uint64_t *u_nbo;
+
+  GNUNET_break (NULL == cls);
+  if (1 != param_length)
+    return -1;
+  abs = *u;
+  if (abs.abs_value_us > INT64_MAX)
+    abs.abs_value_us = INT64_MAX;
+  u_nbo = GNUNET_new (uint64_t);
+  scratch[0] = u_nbo;
+  *u_nbo = GNUNET_htonll (abs.abs_value_us);
+  param_values[0] = (void *) u_nbo;
+  param_lengths[0] = sizeof (uint64_t);
+  param_formats[0] = 1;
+  return 1;
+}
+
+
+/**
  * Generate query parameter for an absolute time value.
  * The database must store a 64-bit integer.
  *
@@ -380,7 +433,10 @@ GNUNET_PQ_query_param_rsa_signature (const struct GNUNET_CRYPTO_RsaSignature *x)
 struct GNUNET_PQ_QueryParam
 GNUNET_PQ_query_param_absolute_time (const struct GNUNET_TIME_Absolute *x)
 {
-  return GNUNET_PQ_query_param_uint64 (&x->abs_value_us);
+  struct GNUNET_PQ_QueryParam res =
+    { &qconv_abs_time, NULL, x, sizeof (*x), 1 };
+
+  return res;
 }
 
 
diff --git a/src/pq/pq_result_helper.c b/src/pq/pq_result_helper.c
index 3805b8a8d..dc1a1554f 100644
--- a/src/pq/pq_result_helper.c
+++ b/src/pq/pq_result_helper.c
@@ -38,6 +38,7 @@ clean_varsize_blob (void *cls,
 {
   void **dst = rd;
 
+  (void) cls;
   if (NULL != *dst)
   {
     GNUNET_free (*dst);
@@ -72,6 +73,7 @@ extract_varsize_blob (void *cls,
   void *idst;
   int fnum;
 
+  (void) cls;
   *dst_size = 0;
   *((void **) dst) = NULL;
 
@@ -154,6 +156,7 @@ extract_fixed_blob (void *cls,
   const char *res;
   int fnum;
 
+  (void) cls;
   fnum = PQfnumber (result,
                     fname);
   if (fnum < 0)
@@ -237,6 +240,7 @@ extract_rsa_public_key (void *cls,
   const char *res;
   int fnum;
 
+  (void) cls;
   *pk = NULL;
   fnum = PQfnumber (result,
                     fname);
@@ -284,6 +288,7 @@ clean_rsa_public_key (void *cls,
 {
   struct GNUNET_CRYPTO_RsaPublicKey **pk = rd;
 
+  (void) cls;
   if (NULL != *pk)
   {
     GNUNET_CRYPTO_rsa_public_key_free (*pk);
@@ -338,6 +343,7 @@ extract_rsa_signature (void *cls,
   const char *res;
   int fnum;
 
+  (void) cls;
   *sig = NULL;
   fnum = PQfnumber (result,
                     fname);
@@ -385,6 +391,7 @@ clean_rsa_signature (void *cls,
 {
   struct GNUNET_CRYPTO_RsaSignature **sig = rd;
 
+  (void) cls;
   if (NULL != *sig)
   {
     GNUNET_CRYPTO_rsa_signature_free (*sig);
@@ -439,6 +446,7 @@ extract_string (void *cls,
   const char *res;
   int fnum;
 
+  (void) cls;
   *str = NULL;
   fnum = PQfnumber (result,
                     fname);
@@ -486,6 +494,7 @@ clean_string (void *cls,
 {
   char **str = rd;
 
+  (void) cls;
   if (NULL != *str)
   {
     GNUNET_free (*str);
@@ -515,6 +524,71 @@ GNUNET_PQ_result_spec_string (const char *name,
 
 
 /**
+ * Extract data from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param int row to extract data from
+ * @param fname name (or prefix) of the fields to extract from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_abs_time (void *cls,
+                  PGresult *result,
+                  int row,
+                  const char *fname,
+                  size_t *dst_size,
+                  void *dst)
+{
+  struct GNUNET_TIME_Absolute *udst = dst;
+  const int64_t *res;
+  int fnum;
+
+  (void) cls;
+  fnum = PQfnumber (result,
+                    fname);
+  if (fnum < 0)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  if (PQgetisnull (result,
+                   row,
+                   fnum))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_assert (NULL != dst);
+  if (sizeof (struct GNUNET_TIME_Absolute) != *dst_size)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  if (sizeof (int64_t) !=
+      PQgetlength (result,
+                   row,
+                   fnum))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  res = (int64_t *) PQgetvalue (result,
+                                row,
+                                fnum);
+  if (INT64_MAX == *res)
+    *udst = GNUNET_TIME_UNIT_FOREVER_ABS;
+  else
+    udst->abs_value_us = GNUNET_ntohll ((uint64_t) *res);
+  return GNUNET_OK;
+}
+
+
+/**
  * Absolute time expected.
  *
  * @param name name of the field in the table
@@ -525,8 +599,12 @@ struct GNUNET_PQ_ResultSpec
 GNUNET_PQ_result_spec_absolute_time (const char *name,
                                      struct GNUNET_TIME_Absolute *at)
 {
-  return GNUNET_PQ_result_spec_uint64 (name,
-                                       &at->abs_value_us);
+  struct GNUNET_PQ_ResultSpec res =
+    { &extract_abs_time,
+      NULL,
+      NULL,
+      (void *) at, sizeof (*at), (name), NULL };
+  return res;
 }
 
 
@@ -572,6 +650,7 @@ extract_uint16 (void *cls,
   const uint16_t *res;
   int fnum;
 
+  (void) cls;
   fnum = PQfnumber (result,
                     fname);
   if (fnum < 0)
@@ -653,6 +732,7 @@ extract_uint32 (void *cls,
   const uint32_t *res;
   int fnum;
 
+  (void) cls;
   fnum = PQfnumber (result,
                     fname);
   if (fnum < 0)
@@ -734,6 +814,7 @@ extract_uint64 (void *cls,
   const uint64_t *res;
   int fnum;
 
+  (void) cls;
   fnum = PQfnumber (result,
                     fname);
   if (fnum < 0)
diff --git a/src/rps/gnunet-rps-profiler.c b/src/rps/gnunet-rps-profiler.c
index 5ef42187f..4a7a89b15 100644
--- a/src/rps/gnunet-rps-profiler.c
+++ b/src/rps/gnunet-rps-profiler.c
@@ -42,6 +42,11 @@
 static uint32_t num_peers;
 
 /**
+ * @brief numer of bits required to represent the largest peer id
+ */
+static unsigned bits_needed;
+
+/**
  * How long do we run the test?
  * In seconds.
  */
@@ -1698,6 +1703,7 @@ profiler_reply_handle (void *cls,
   char *file_name;
   char *file_name_dh;
   char *file_name_dhr;
+  char *file_name_dhru;
   unsigned int i;
   struct PendingReply *pending_rep = (struct PendingReply *) cls;
 
@@ -1706,6 +1712,7 @@ profiler_reply_handle (void *cls,
   file_name = "/tmp/rps/received_ids";
   file_name_dh = "/tmp/rps/diehard_input";
   file_name_dhr = "/tmp/rps/diehard_input_raw";
+  file_name_dhru = "/tmp/rps/diehard_input_raw_aligned";
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "[%s] got %" PRIu64 " peers:\n",
               GNUNET_i2s (rps_peer->peer_id),
@@ -1725,8 +1732,12 @@ profiler_reply_handle (void *cls,
              "%" PRIu32 "\n",
              (uint32_t) rcv_rps_peer->index);
     to_file_raw (file_name_dhr,
-                 &rcv_rps_peer->index,
+                (char *) &rcv_rps_peer->index,
                  sizeof (uint32_t));
+    to_file_raw_unaligned (file_name_dhru,
+                          (char *) &rcv_rps_peer->index,
+                           sizeof (uint32_t),
+                           bits_needed);
   }
   default_reply_handle (cls, n, recv_peers);
 }
@@ -2626,6 +2637,14 @@ run (void *cls,
   GNUNET_DISK_directory_create ("/tmp/rps/");
   timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, timeout_s);
 
+  /* Compute number of bits for representing largest peer id */
+  for (bits_needed = 1; (bits_needed << 1) < num_peers - 1; bits_needed++)
+    ;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+            "Need %u bits to represent largest peer id %" PRIu32 "\n",
+             bits_needed,
+             num_peers - 1);
+
   rps_peers = GNUNET_new_array (num_peers, struct RPSPeer);
   peer_map = GNUNET_CONTAINER_multipeermap_create (num_peers, GNUNET_NO);
   rps_peer_ids = GNUNET_new_array (num_peers, struct GNUNET_PeerIdentity);
diff --git a/src/rps/rps-test_util.c b/src/rps/rps-test_util.c
index ea55deac5..9a1dfe0d8 100644
--- a/src/rps/rps-test_util.c
+++ b/src/rps/rps-test_util.c
@@ -29,13 +29,26 @@
 
 #include <inttypes.h>
 
-#define LOG(kind, ...) GNUNET_log_from(kind,"rps-sampler",__VA_ARGS__)
+#define LOG(kind, ...) GNUNET_log_from(kind,"rps-test_util",__VA_ARGS__)
 
 #ifndef TO_FILE
 #define TO_FILE
 #endif /* TO_FILE */
 
 #ifdef TO_FILE
+
+#define min(x,y) ((x) > (y) ? (y) : (x))
+
+/**
+ * @brief buffer for storing the unaligned bits for the next write
+ */
+static char buf_unaligned;
+
+/**
+ * @brief number of bits in unaligned buffer
+ */
+static unsigned num_bits_buf_unaligned;
+
 void
 to_file_ (const char *file_name, char *line)
 {
@@ -108,11 +121,10 @@ to_file_ (const char *file_name, char *line)
 }
 
 void
-to_file_raw (const char *file_name, void *buf, size_t size_buf)
+to_file_raw (const char *file_name, const char *buf, size_t size_buf)
 {
   struct GNUNET_DISK_FileHandle *f;
-  size_t size2;
-
+  size_t size_written;
 
   if (NULL == (f = GNUNET_DISK_file_open (file_name,
                                           GNUNET_DISK_OPEN_APPEND |
@@ -129,13 +141,13 @@ to_file_raw (const char *file_name, void *buf, size_t size_buf)
     return;
   }
 
-  size2 = GNUNET_DISK_file_write (f, buf, size_buf);
-  if (size_buf != size2)
+  size_written = GNUNET_DISK_file_write (f, buf, size_buf);
+  if (size_buf != size_written)
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Unable to write to file! (Size: %u, size2: %u)\n",
+         "Unable to write to file! (Size: %u, size_written: %u)\n",
          size_buf,
-         size2);
+         size_written);
 
     if (GNUNET_YES != GNUNET_DISK_file_close (f))
       LOG (GNUNET_ERROR_TYPE_WARNING,
@@ -143,15 +155,110 @@ to_file_raw (const char *file_name, void *buf, size_t size_buf)
 
     return;
   }
+}
 
-  //if (512 < size_buf)
+void
+to_file_raw_unaligned (const char *file_name,
+                       const char *buf,
+                       size_t size_buf,
+                       unsigned bits_needed)
+{
+  // TODO endianness!
+  GNUNET_assert (size_buf >= (bits_needed/8));
+  //if (0 == num_bits_buf_unaligned)
   //{
-  //  GNUNET_free (output_buffer_p);
+  //  if (0 == (bits_needed % 8))
+  //  {
+  //    to_file_raw (file_name, buf, size_buf);
+  //    return;
+  //  }
+  //  to_file_raw (file_name, buf, size_buf - 1);
+  //  buf_unaligned = buf[size_buf - 1];
+  //  num_bits_buf_unaligned = bits_needed % 8;
+  //  return;
   //}
 
-  //if (GNUNET_YES != GNUNET_DISK_file_close (f))
-  //  LOG (GNUNET_ERROR_TYPE_WARNING,
-  //       "Unable to close file\n");
+  char buf_write[size_buf + 1];
+  const unsigned bytes_iter = (0 != bits_needed % 8?
+                               (bits_needed/8)+1:
+                               bits_needed/8);
+  // TODO what if no iteration happens?
+  unsigned size_buf_write = 0;
+  buf_write[0] = buf_unaligned;
+  /* Iterate over input bytes */
+  for (unsigned i = 0; i < bytes_iter; i++)
+  {
+    /* Number of bits needed in this iteration - 8 for all except last iter */
+    unsigned num_bits_needed_iter;
+    /* Mask for bits to actually use */
+    unsigned mask_bits_needed_iter;
+    char byte_input;
+    /* Number of bits needed to align unaligned byte */
+    unsigned num_bits_to_align;
+    /* Number of bits that are to be moved */
+    unsigned num_bits_to_move;
+    /* Mask for bytes to be moved */
+    char mask_input_to_move;
+    /* Masked bits to be moved */
+    char bits_to_move;
+    /* The amount of bits needed to fit the bits to shift to the nearest spot */
+    unsigned distance_shift_bits;
+    /* Shifted bits on the move */
+    char bits_moving;
+    /* (unaligned) byte being filled with bits */
+    char byte_to_fill;
+    /* mask for needed bits of the input byte that have not been moved */
+    char mask_input_leftover;
+    /* needed bits of the input byte that have not been moved */
+    char byte_input_leftover;
+    unsigned num_bits_leftover;
+    unsigned num_bits_discard;
+    char byte_unaligned_new;
+
+    if ( (bits_needed - (i * 8)) <= 8)
+    {
+      /* last iteration */
+      num_bits_needed_iter = bits_needed - (i * 8);
+    }
+    else
+    {
+      num_bits_needed_iter = 8;
+    }
+    mask_bits_needed_iter = ((char) 1 << num_bits_needed_iter) - 1;
+    byte_input = buf[i];
+    byte_input &= mask_bits_needed_iter;
+    num_bits_to_align = 8 - num_bits_buf_unaligned;
+    num_bits_to_move  = min (num_bits_to_align, num_bits_needed_iter);
+    mask_input_to_move = ((char) 1 << num_bits_to_move) - 1;
+    bits_to_move = byte_input & mask_input_to_move;
+    distance_shift_bits = num_bits_buf_unaligned;
+    bits_moving = bits_to_move << distance_shift_bits;
+    byte_to_fill = buf_unaligned | bits_moving;
+    if (num_bits_buf_unaligned + num_bits_needed_iter > 8)
+    {
+      /* buf_unaligned was aligned by filling
+       * -> can be written to storage */
+      buf_write[i] = byte_to_fill;
+      size_buf_write++;
+
+      /* store the leftover, unaligned bits in buffer */
+      mask_input_leftover = mask_bits_needed_iter & (~ mask_input_to_move);
+      byte_input_leftover = byte_input & mask_input_leftover;
+      num_bits_leftover = num_bits_needed_iter - num_bits_to_move;
+      num_bits_discard = 8 - num_bits_needed_iter;
+      byte_unaligned_new = byte_input_leftover >> num_bits_to_move;
+      buf_unaligned = byte_unaligned_new;
+      num_bits_buf_unaligned = num_bits_leftover % 8;
+    }
+    else
+    {
+      /* unaligned buffer still unaligned but 'fuller' */
+      buf_unaligned = byte_to_fill;
+      num_bits_buf_unaligned = (num_bits_buf_unaligned + bits_needed) % 8;
+    }
+  }
+  to_file_raw (file_name, buf_write, size_buf_write);
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "\n");
 }
 
 char *
diff --git a/src/rps/rps-test_util.h b/src/rps/rps-test_util.h
index d42422750..577a2b0a7 100644
--- a/src/rps/rps-test_util.h
+++ b/src/rps/rps-test_util.h
@@ -75,7 +75,13 @@ store_prefix_file_name (const struct GNUNET_PeerIdentity *peer,
     const char *prefix);
 
 void
-to_file_raw (const char *file_name, void *buf, size_t size_buf);
+to_file_raw (const char *file_name, const char *buf, size_t size_buf);
+
+void
+to_file_raw_unaligned (const char *file_name,
+                       const char *buf,
+                       size_t size_buf,
+                       unsigned bits_needed);
 
 #endif /* RPS_TEST_UTIL_H */
 /* end of gnunet-service-rps.c */