ensure datacache does not return expired records, fixig pq behavior with respect to FOREVER absolute time

6 files changed, 242 insertions, 41 deletions

diff --git a/src/datacache/plugin_datacache_heap.c b/src/datacache/plugin_datacache_heap.c
index 2a08fc81b..494d1ae17 100644
--- a/src/datacache/plugin_datacache_heap.c
+++ b/src/datacache/plugin_datacache_heap.c
@@ -314,6 +314,9 @@ get_cb (void *cls,
   if ( (get_ctx->type != val->type) &&
        (GNUNET_BLOCK_TYPE_ANY != get_ctx->type) )
     return GNUNET_OK;
+  if (0 ==
+      GNUNET_TIME_absolute_get_remaining (val->discard_time).rel_value_us)
+    return GNUNET_OK;
   if (NULL != get_ctx->iter)
     ret = get_ctx->iter (get_ctx->iter_cls,
                          key,
diff --git a/src/datacache/plugin_datacache_postgres.c b/src/datacache/plugin_datacache_postgres.c
index 6eeeb5873..ea87acc1f 100644
--- a/src/datacache/plugin_datacache_postgres.c
+++ b/src/datacache/plugin_datacache_postgres.c
@@ -82,12 +82,12 @@ init_connection (struct Plugin *plugin)
   struct GNUNET_PQ_PreparedStatement ps[] = {
     GNUNET_PQ_make_prepare ("getkt",
                             "SELECT discard_time,type,value,path FROM gn011dc "
-                            "WHERE key=$1 AND type=$2",
-                            2),
+                            "WHERE key=$1 AND type=$2 AND discard_time >= $3",
+                            3),
     GNUNET_PQ_make_prepare ("getk",
                             "SELECT discard_time,type,value,path FROM gn011dc "
-                            "WHERE key=$1",
-                            1),
+                            "WHERE key=$1 AND discard_time >= $2",
+                            2),
     GNUNET_PQ_make_prepare ("getex",
                             "SELECT length(value) AS len,oid,key FROM gn011dc"
                             " WHERE discard_time < $1"
@@ -97,18 +97,15 @@ init_connection (struct Plugin *plugin)
                             "SELECT length(value) AS len,oid,key FROM gn011dc"
                             " ORDER BY prox ASC, discard_time ASC LIMIT 1",
                             0),
-    GNUNET_PQ_make_prepare ("getp",
-                            "SELECT length(value) AS len,oid,key FROM gn011dc "
-                            "ORDER BY discard_time ASC LIMIT 1",
-                            0),
     GNUNET_PQ_make_prepare ("get_random",
-                            "SELECT discard_time,type,value,path,key FROM gn011dc "
-                            "ORDER BY key ASC LIMIT 1 OFFSET $1",
-                            1),
+                            "SELECT discard_time,type,value,path,key FROM gn011dc"
+                            " WHERE discard_time >= $1"
+                            " ORDER BY key ASC LIMIT 1 OFFSET $2",
+                            2),
     GNUNET_PQ_make_prepare ("get_closest",
                             "SELECT discard_time,type,value,path,key FROM gn011dc "
-                            "WHERE key>=$1 ORDER BY key ASC LIMIT $2",
-                            1),
+                            "WHERE key>=$1 AND discard_time >= $2 ORDER BY key ASC LIMIT $3",
+                            3),
     GNUNET_PQ_make_prepare ("delrow",
                             "DELETE FROM gn011dc WHERE oid=$1",
                             1),
@@ -313,18 +310,22 @@ postgres_plugin_get (void *cls,
 {
   struct Plugin *plugin = cls;
   uint32_t type32 = (uint32_t) type;
+  struct GNUNET_TIME_Absolute now;
   struct GNUNET_PQ_QueryParam paramk[] = {
     GNUNET_PQ_query_param_auto_from_type (key),
+    GNUNET_PQ_query_param_absolute_time (&now),
     GNUNET_PQ_query_param_end
   };
   struct GNUNET_PQ_QueryParam paramkt[] = {
     GNUNET_PQ_query_param_auto_from_type (key),
     GNUNET_PQ_query_param_uint32 (&type32),
+    GNUNET_PQ_query_param_absolute_time (&now),
     GNUNET_PQ_query_param_end
   };
   enum GNUNET_DB_QueryStatus res;
   struct HandleResultContext hr_ctx;
 
+  now = GNUNET_TIME_absolute_get ();
   hr_ctx.iter = iter;
   hr_ctx.iter_cls = iter_cls;
   hr_ctx.key = key;
@@ -427,6 +428,7 @@ postgres_plugin_get_random (void *cls,
 {
   struct Plugin *plugin = cls;
   uint32_t off;
+  struct GNUNET_TIME_Absolute now;
   struct GNUNET_TIME_Absolute expiration_time;
   size_t data_size;
   void *data;
@@ -436,6 +438,7 @@ postgres_plugin_get_random (void *cls,
   uint32_t type;
   enum GNUNET_DB_QueryStatus res;
   struct GNUNET_PQ_QueryParam params[] = {
+    GNUNET_PQ_query_param_absolute_time (&now),
     GNUNET_PQ_query_param_uint32 (&off),
     GNUNET_PQ_query_param_end
   };
@@ -459,6 +462,7 @@ postgres_plugin_get_random (void *cls,
     return 0;
   if (NULL == iter)
     return 1;
+  now = GNUNET_TIME_absolute_get ();
   off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
                                   plugin->num_items);
   res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
@@ -620,8 +624,10 @@ postgres_plugin_get_closest (void *cls,
 {
   struct Plugin *plugin = cls;
   uint32_t num_results32 = (uint32_t) num_results;
+  struct GNUNET_TIME_Absolute now;
   struct GNUNET_PQ_QueryParam params[] = {
     GNUNET_PQ_query_param_auto_from_type (key),
+    GNUNET_PQ_query_param_absolute_time (&now),
     GNUNET_PQ_query_param_uint32 (&num_results32),
     GNUNET_PQ_query_param_end
   };
@@ -630,6 +636,7 @@ postgres_plugin_get_closest (void *cls,
 
   erc.iter = iter;
   erc.iter_cls = iter_cls;
+  now = GNUNET_TIME_absolute_get ();
   res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
                                               "get_closest",
                                               params,
diff --git a/src/datacache/test_datacache.c b/src/datacache/test_datacache.c
index 12edb62f8..50e45012d 100644
--- a/src/datacache/test_datacache.c
+++ b/src/datacache/test_datacache.c
@@ -44,6 +44,11 @@ checkIt (void *cls,
          unsigned int path_len,
          const struct GNUNET_PeerIdentity *path)
 {
+  (void) key;
+  (void) type;
+  (void) exp;
+  (void) path_len;
+  (void) path;
   if (size != sizeof (struct GNUNET_HashCode))
   {
     GNUNET_break (0);
@@ -59,17 +64,22 @@ checkIt (void *cls,
 
 
 static void
-run (void *cls, char *const *args, const char *cfgfile,
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   struct GNUNET_DATACACHE_Handle *h;
   struct GNUNET_HashCode k;
   struct GNUNET_HashCode n;
   struct GNUNET_TIME_Absolute exp;
-  unsigned int i;
 
+  (void) cls;
+  (void) args;
+  (void) cfgfile;
   ok = 0;
-  h = GNUNET_DATACACHE_create (cfg, "testcache");
+  h = GNUNET_DATACACHE_create (cfg,
+                               "testcache");
   if (h == NULL)
   {
     FPRINTF (stderr,
@@ -81,7 +91,7 @@ run (void *cls, char *const *args, const char *cfgfile,
   exp = GNUNET_TIME_absolute_get ();
   exp.abs_value_us += 5 * 60 * 1000 * 1000LL;
   memset (&k, 0, sizeof (struct GNUNET_HashCode));
-  for (i = 0; i < 100; i++)
+  for (unsigned int i = 0; i < 100; i++)
   {
     GNUNET_CRYPTO_hash (&k, sizeof (struct GNUNET_HashCode), &n);
     ASSERT (GNUNET_OK ==
@@ -93,26 +103,43 @@ run (void *cls, char *const *args, const char *cfgfile,
                                   0, NULL));
     k = n;
   }
-  memset (&k, 0, sizeof (struct GNUNET_HashCode));
-  for (i = 0; i < 100; i++)
+  memset (&k,
+          0,
+          sizeof (struct GNUNET_HashCode));
+  for (unsigned int i = 0; i < 100; i++)
   {
-    GNUNET_CRYPTO_hash (&k, sizeof (struct GNUNET_HashCode), &n);
-    ASSERT (1 == GNUNET_DATACACHE_get (h, &k, 1 + i % 16, &checkIt, &n));
+    GNUNET_CRYPTO_hash (&k,
+                        sizeof (struct GNUNET_HashCode),
+                        &n);
+    ASSERT (1 == GNUNET_DATACACHE_get (h,
+                                       &k,
+                                       1 + i % 16,
+                                       &checkIt,
+                                       &n));
     k = n;
   }
 
-  memset (&k, 42, sizeof (struct GNUNET_HashCode));
-  GNUNET_CRYPTO_hash (&k, sizeof (struct GNUNET_HashCode), &n);
+  memset (&k,
+          42,
+          sizeof (struct GNUNET_HashCode));
+  GNUNET_CRYPTO_hash (&k,
+                      sizeof (struct GNUNET_HashCode),
+                      &n);
   ASSERT (GNUNET_OK ==
           GNUNET_DATACACHE_put (h,
                                 &k,
                                 GNUNET_YES,
                                 sizeof (struct GNUNET_HashCode),
-                                (const char *) &n, 792,
+                                (const char *) &n,
+                                792,
                                 GNUNET_TIME_UNIT_FOREVER_ABS,
-                                0, NULL));
-  ASSERT (0 != GNUNET_DATACACHE_get (h, &k, 792, &checkIt, &n));
-
+                                0,
+                                NULL));
+  ASSERT (0 != GNUNET_DATACACHE_get (h,
+                                     &k,
+                                     792,
+                                     &checkIt,
+                                     &n));
   GNUNET_DATACACHE_destroy (h);
   ASSERT (ok == 0);
   return;
@@ -137,16 +164,26 @@ main (int argc, char *argv[])
     GNUNET_GETOPT_OPTION_END
   };
 
+  (void) argc;
   GNUNET_log_setup ("test-datacache",
                     "WARNING",
                     NULL);
   plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
-  GNUNET_snprintf (cfg_name, sizeof (cfg_name), "test_datacache_data_%s.conf",
+  GNUNET_snprintf (cfg_name,
+                   sizeof (cfg_name),
+                   "test_datacache_data_%s.conf",
                    plugin_name);
-  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv,
-                      "test-datacache", "nohelp", options, &run, NULL);
+  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1,
+                      xargv,
+                      "test-datacache",
+                      "nohelp",
+                      options,
+                      &run,
+                      NULL);
   if ( (0 != ok) && (77 != ok) )
-    FPRINTF (stderr, "Missed some testcases: %d\n", ok);
+    FPRINTF (stderr,
+             "Missed some testcases: %d\n",
+             ok);
   return ok;
 }
 
diff --git a/src/datacache/test_datacache_quota.c b/src/datacache/test_datacache_quota.c
index 3d02a7244..21e373608 100644
--- a/src/datacache/test_datacache_quota.c
+++ b/src/datacache/test_datacache_quota.c
@@ -41,7 +41,9 @@ static const char *plugin_name;
  * some of the data from the last iteration is still there.
  */
 static void
-run (void *cls, char *const *args, const char *cfgfile,
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   struct GNUNET_DATACACHE_Handle *h;
@@ -50,8 +52,12 @@ run (void *cls, char *const *args, const char *cfgfile,
   char buf[3200];
   struct GNUNET_TIME_Absolute exp;
 
+  (void) cls;
+  (void) args;
+  (void) cfgfile;
   ok = 0;
-  h = GNUNET_DATACACHE_create (cfg, "testcache");
+  h = GNUNET_DATACACHE_create (cfg,
+                               "testcache");
 
   if (h == NULL)
   {
@@ -112,7 +118,8 @@ FAILURE:
 
 
 int
-main (int argc, char *argv[])
+main (int argc,
+      char *argv[])
 {
   char cfg_name[128];
   char *const xargv[] = {
@@ -125,17 +132,27 @@ main (int argc, char *argv[])
     GNUNET_GETOPT_OPTION_END
   };
 
+  (void) argc;
   GNUNET_log_setup ("test-datacache-quota",
                     "WARNING",
                     NULL);
 
   plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
-  GNUNET_snprintf (cfg_name, sizeof (cfg_name), "test_datacache_data_%s.conf",
+  GNUNET_snprintf (cfg_name,
+                   sizeof (cfg_name),
+                   "test_datacache_data_%s.conf",
                    plugin_name);
-  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv,
-                      "test-datacache-quota", "nohelp", options, &run, NULL);
+  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1,
+                      xargv,
+                      "test-datacache-quota",
+                      "nohelp",
+                      options,
+                      &run,
+                      NULL);
   if (0 != ok)
-    FPRINTF (stderr, "Missed some testcases: %d\n", ok);
+    FPRINTF (stderr,
+             "Missed some testcases: %d\n",
+             ok);
   return ok;
 }
 
diff --git a/src/pq/pq_query_helper.c b/src/pq/pq_query_helper.c
index 799f82ebe..98f697b5d 100644
--- a/src/pq/pq_query_helper.c
+++ b/src/pq/pq_query_helper.c
@@ -50,6 +50,8 @@ qconv_fixed (void *cls,
              void *scratch[],
              unsigned int scratch_length)
 {
+  (void) scratch;
+  (void) scratch_length;
   GNUNET_break (NULL == cls);
   if (1 != param_length)
     return -1;
@@ -117,6 +119,8 @@ qconv_uint16 (void *cls,
   const uint16_t *u_hbo = data;
   uint16_t *u_nbo;
 
+  (void) scratch;
+  (void) scratch_length;
   GNUNET_break (NULL == cls);
   if (1 != param_length)
     return -1;
@@ -172,6 +176,8 @@ qconv_uint32 (void *cls,
   const uint32_t *u_hbo = data;
   uint32_t *u_nbo;
 
+  (void) scratch;
+  (void) scratch_length;
   GNUNET_break (NULL == cls);
   if (1 != param_length)
     return -1;
@@ -227,6 +233,8 @@ qconv_uint64 (void *cls,
   const uint64_t *u_hbo = data;
   uint64_t *u_nbo;
 
+  (void) scratch;
+  (void) scratch_length;
   GNUNET_break (NULL == cls);
   if (1 != param_length)
     return -1;
@@ -371,6 +379,51 @@ GNUNET_PQ_query_param_rsa_signature (const struct GNUNET_CRYPTO_RsaSignature *x)
 
 
 /**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param[out] param_values SQL data to set
+ * @param[out] param_lengths SQL length data to set
+ * @param[out] param_formats SQL format data to set
+ * @param param_length number of entries available in the @a param_values, @a param_lengths and @a param_formats arrays
+ * @param[out] scratch buffer for dynamic allocations (to be done via #GNUNET_malloc()
+ * @param scratch_length number of entries left in @a scratch
+ * @return -1 on error, number of offsets used in @a scratch otherwise
+ */
+static int
+qconv_abs_time (void *cls,
+                const void *data,
+                size_t data_len,
+                void *param_values[],
+                int param_lengths[],
+                int param_formats[],
+                unsigned int param_length,
+                void *scratch[],
+                unsigned int scratch_length)
+{
+  const struct GNUNET_TIME_Absolute *u = data;
+  struct GNUNET_TIME_Absolute abs;
+  uint64_t *u_nbo;
+
+  GNUNET_break (NULL == cls);
+  if (1 != param_length)
+    return -1;
+  abs = *u;
+  if (abs.abs_value_us > INT64_MAX)
+    abs.abs_value_us = INT64_MAX;
+  u_nbo = GNUNET_new (uint64_t);
+  scratch[0] = u_nbo;
+  *u_nbo = GNUNET_htonll (abs.abs_value_us);
+  param_values[0] = (void *) u_nbo;
+  param_lengths[0] = sizeof (uint64_t);
+  param_formats[0] = 1;
+  return 1;
+}
+
+
+/**
  * Generate query parameter for an absolute time value.
  * The database must store a 64-bit integer.
  *
@@ -380,7 +433,10 @@ GNUNET_PQ_query_param_rsa_signature (const struct GNUNET_CRYPTO_RsaSignature *x)
 struct GNUNET_PQ_QueryParam
 GNUNET_PQ_query_param_absolute_time (const struct GNUNET_TIME_Absolute *x)
 {
-  return GNUNET_PQ_query_param_uint64 (&x->abs_value_us);
+  struct GNUNET_PQ_QueryParam res =
+    { &qconv_abs_time, NULL, x, sizeof (*x), 1 };
+
+  return res;
 }
 
 
diff --git a/src/pq/pq_result_helper.c b/src/pq/pq_result_helper.c
index 3805b8a8d..dc1a1554f 100644
--- a/src/pq/pq_result_helper.c
+++ b/src/pq/pq_result_helper.c
@@ -38,6 +38,7 @@ clean_varsize_blob (void *cls,
 {
   void **dst = rd;
 
+  (void) cls;
   if (NULL != *dst)
   {
     GNUNET_free (*dst);
@@ -72,6 +73,7 @@ extract_varsize_blob (void *cls,
   void *idst;
   int fnum;
 
+  (void) cls;
   *dst_size = 0;
   *((void **) dst) = NULL;
 
@@ -154,6 +156,7 @@ extract_fixed_blob (void *cls,
   const char *res;
   int fnum;
 
+  (void) cls;
   fnum = PQfnumber (result,
                     fname);
   if (fnum < 0)
@@ -237,6 +240,7 @@ extract_rsa_public_key (void *cls,
   const char *res;
   int fnum;
 
+  (void) cls;
   *pk = NULL;
   fnum = PQfnumber (result,
                     fname);
@@ -284,6 +288,7 @@ clean_rsa_public_key (void *cls,
 {
   struct GNUNET_CRYPTO_RsaPublicKey **pk = rd;
 
+  (void) cls;
   if (NULL != *pk)
   {
     GNUNET_CRYPTO_rsa_public_key_free (*pk);
@@ -338,6 +343,7 @@ extract_rsa_signature (void *cls,
   const char *res;
   int fnum;
 
+  (void) cls;
   *sig = NULL;
   fnum = PQfnumber (result,
                     fname);
@@ -385,6 +391,7 @@ clean_rsa_signature (void *cls,
 {
   struct GNUNET_CRYPTO_RsaSignature **sig = rd;
 
+  (void) cls;
   if (NULL != *sig)
   {
     GNUNET_CRYPTO_rsa_signature_free (*sig);
@@ -439,6 +446,7 @@ extract_string (void *cls,
   const char *res;
   int fnum;
 
+  (void) cls;
   *str = NULL;
   fnum = PQfnumber (result,
                     fname);
@@ -486,6 +494,7 @@ clean_string (void *cls,
 {
   char **str = rd;
 
+  (void) cls;
   if (NULL != *str)
   {
     GNUNET_free (*str);
@@ -515,6 +524,71 @@ GNUNET_PQ_result_spec_string (const char *name,
 
 
 /**
+ * Extract data from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param int row to extract data from
+ * @param fname name (or prefix) of the fields to extract from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_abs_time (void *cls,
+                  PGresult *result,
+                  int row,
+                  const char *fname,
+                  size_t *dst_size,
+                  void *dst)
+{
+  struct GNUNET_TIME_Absolute *udst = dst;
+  const int64_t *res;
+  int fnum;
+
+  (void) cls;
+  fnum = PQfnumber (result,
+                    fname);
+  if (fnum < 0)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  if (PQgetisnull (result,
+                   row,
+                   fnum))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_assert (NULL != dst);
+  if (sizeof (struct GNUNET_TIME_Absolute) != *dst_size)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  if (sizeof (int64_t) !=
+      PQgetlength (result,
+                   row,
+                   fnum))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  res = (int64_t *) PQgetvalue (result,
+                                row,
+                                fnum);
+  if (INT64_MAX == *res)
+    *udst = GNUNET_TIME_UNIT_FOREVER_ABS;
+  else
+    udst->abs_value_us = GNUNET_ntohll ((uint64_t) *res);
+  return GNUNET_OK;
+}
+
+
+/**
  * Absolute time expected.
  *
  * @param name name of the field in the table
@@ -525,8 +599,12 @@ struct GNUNET_PQ_ResultSpec
 GNUNET_PQ_result_spec_absolute_time (const char *name,
                                      struct GNUNET_TIME_Absolute *at)
 {
-  return GNUNET_PQ_result_spec_uint64 (name,
-                                       &at->abs_value_us);
+  struct GNUNET_PQ_ResultSpec res =
+    { &extract_abs_time,
+      NULL,
+      NULL,
+      (void *) at, sizeof (*at), (name), NULL };
+  return res;
 }
 
 
@@ -572,6 +650,7 @@ extract_uint16 (void *cls,
   const uint16_t *res;
   int fnum;
 
+  (void) cls;
   fnum = PQfnumber (result,
                     fname);
   if (fnum < 0)
@@ -653,6 +732,7 @@ extract_uint32 (void *cls,
   const uint32_t *res;
   int fnum;
 
+  (void) cls;
   fnum = PQfnumber (result,
                     fname);
   if (fnum < 0)
@@ -734,6 +814,7 @@ extract_uint64 (void *cls,
   const uint64_t *res;
   int fnum;
 
+  (void) cls;
   fnum = PQfnumber (result,
                     fname);
   if (fnum < 0)