ensure all plugins properly use BF, move shared logic to shared library

9 files changed, 127 insertions, 194 deletions

diff --git a/src/block/bg_bf.c b/src/block/bg_bf.c
index 9c4dc9060..1a17ec84e 100644
--- a/src/block/bg_bf.c
+++ b/src/block/bg_bf.c
@@ -232,4 +232,36 @@ GNUNET_BLOCK_GROUP_bf_test_and_set (struct GNUNET_BLOCK_Group *bg,
 }
 
 
+/**
+ * How many bytes should a bloomfilter be if we have already seen
+ * entry_count responses?  Sized so that do not have to
+ * re-size the filter too often (to keep it cheap).
+ *
+ * Since other peers will also add entries but not resize the filter,
+ * we should generally pick a slightly larger size than what the
+ * strict math would suggest.
+ *
+ * @param entry_count expected number of entries in the Bloom filter
+ * @param k number of bits set per entry
+ * @return must be a power of two and smaller or equal to 2^15.
+ */
+size_t
+GNUNET_BLOCK_GROUP_compute_bloomfilter_size (unsigned int entry_count,
+                                             unsigned int k)
+{
+  size_t size;
+  unsigned int ideal = (entry_count * k) / 4;
+  uint16_t max = 1 << 15;
+
+  if (entry_count > max)
+    return max;
+  size = 8;
+  while ((size < max) && (size < ideal))
+    size *= 2;
+  if (size > max)
+    return max;
+  return size;
+}
+
+
 /* end of bg_bf.c */
diff --git a/src/block/plugin_block_template.c b/src/block/plugin_block_template.c
index 87adae7e9..b714b6858 100644
--- a/src/block/plugin_block_template.c
+++ b/src/block/plugin_block_template.c
@@ -44,37 +44,6 @@
 
 
 /**
- * How many bytes should a bloomfilter be if we have already seen
- * entry_count responses?  Note that #GNUNET_CONSTANTS_BLOOMFILTER_K
- * gives us the number of bits set per entry.  Furthermore, we should
- * not re-size the filter too often (to keep it cheap).
- *
- * Since other peers will also add entries but not resize the filter,
- * we should generally pick a slightly larger size than what the
- * strict math would suggest.
- *
- * @param entry_count expected number of entries in the Bloom filter
- * @return must be a power of two and smaller or equal to 2^15.
- */
-static size_t
-compute_bloomfilter_size (unsigned int entry_count)
-{
-  size_t size;
-  unsigned int ideal = (entry_count * BLOOMFILTER_K) / 4;
-  uint16_t max = 1 << 15;
-
-  if (entry_count > max)
-    return max;
-  size = 8;
-  while ((size < max) && (size < ideal))
-    size *= 2;
-  if (size > max)
-    return max;
-  return size;
-}
-
-
-/**
  * Create a new block group.
  *
  * @param ctx block context in which the block group is created
@@ -100,7 +69,8 @@ block_plugin_template_create_group (void *cls,
   guard = va_arg (va, const char *);
   if (0 == strcmp (guard,
                    "seen-set-size"))
-    bf_size = compute_bloomfilter_size (va_arg (va, unsigned int));
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
   else if (0 == strcmp (guard,
                         "filter-size"))
     bf_size = va_arg (va, unsigned int);
diff --git a/src/block/plugin_block_test.c b/src/block/plugin_block_test.c
index 31112e5dd..b9f71cacb 100644
--- a/src/block/plugin_block_test.c
+++ b/src/block/plugin_block_test.c
@@ -42,37 +42,6 @@
 
 
 /**
- * How many bytes should a bloomfilter be if we have already seen
- * entry_count responses?  Note that #GNUNET_CONSTANTS_BLOOMFILTER_K
- * gives us the number of bits set per entry.  Furthermore, we should
- * not re-size the filter too often (to keep it cheap).
- *
- * Since other peers will also add entries but not resize the filter,
- * we should generally pick a slightly larger size than what the
- * strict math would suggest.
- *
- * @param entry_count expected number of entries in the Bloom filter
- * @return must be a power of two and smaller or equal to 2^15.
- */
-static size_t
-compute_bloomfilter_size (unsigned int entry_count)
-{
-  size_t size;
-  unsigned int ideal = (entry_count * BLOOMFILTER_K) / 4;
-  uint16_t max = 1 << 15;
-
-  if (entry_count > max)
-    return max;
-  size = 8;
-  while ((size < max) && (size < ideal))
-    size *= 2;
-  if (size > max)
-    return max;
-  return size;
-}
-
-
-/**
  * Create a new block group.
  *
  * @param ctx block context in which the block group is created
@@ -98,7 +67,8 @@ block_plugin_test_create_group (void *cls,
   guard = va_arg (va, const char *);
   if (0 == strcmp (guard,
                    "seen-set-size"))
-    bf_size = compute_bloomfilter_size (va_arg (va, unsigned int));
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
   else if (0 == strcmp (guard,
                         "filter-size"))
     bf_size = va_arg (va, unsigned int);
diff --git a/src/dht/plugin_block_dht.c b/src/dht/plugin_block_dht.c
index 29c08dd50..74498746c 100644
--- a/src/dht/plugin_block_dht.c
+++ b/src/dht/plugin_block_dht.c
@@ -41,37 +41,6 @@
 
 
 /**
- * How many bytes should a bloomfilter be if we have already seen
- * entry_count responses?  Note that #GNUNET_CONSTANTS_BLOOMFILTER_K
- * gives us the number of bits set per entry.  Furthermore, we should
- * not re-size the filter too often (to keep it cheap).
- *
- * Since other peers will also add entries but not resize the filter,
- * we should generally pick a slightly larger size than what the
- * strict math would suggest.
- *
- * @param entry_count expected number of entries in the Bloom filter
- * @return must be a power of two and smaller or equal to 2^15.
- */
-static size_t
-compute_bloomfilter_size (unsigned int entry_count)
-{
-  size_t size;
-  unsigned int ideal = (entry_count * BLOOMFILTER_K) / 4;
-  uint16_t max = 1 << 15;
-
-  if (entry_count > max)
-    return max;
-  size = 8;
-  while ((size < max) && (size < ideal))
-    size *= 2;
-  if (size > max)
-    return max;
-  return size;
-}
-
-
-/**
  * Create a new block group.
  *
  * @param ctx block context in which the block group is created
@@ -97,7 +66,8 @@ block_plugin_dht_create_group (void *cls,
   guard = va_arg (va, const char *);
   if (0 == strcmp (guard,
                    "seen-set-size"))
-    bf_size = compute_bloomfilter_size (va_arg (va, unsigned int));
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
   else if (0 == strcmp (guard,
                         "filter-size"))
     bf_size = va_arg (va, unsigned int);
diff --git a/src/dns/plugin_block_dns.c b/src/dns/plugin_block_dns.c
index 65da0de63..fe416e450 100644
--- a/src/dns/plugin_block_dns.c
+++ b/src/dns/plugin_block_dns.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2013, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -31,6 +31,61 @@
 #include "gnunet_block_plugin.h"
 #include "block_dns.h"
 #include "gnunet_signatures.h"
+#include "gnunet_block_group_lib.h"
+
+
+/**
+ * Number of bits we set per entry in the bloomfilter.
+ * Do not change!
+ */
+#define BLOOMFILTER_K 16
+
+
+/**
+ * Create a new block group.
+ *
+ * @param ctx block context in which the block group is created
+ * @param type type of the block for which we are creating the group
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @param va variable arguments specific to @a type
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+static struct GNUNET_BLOCK_Group *
+block_plugin_dns_create_group (void *cls,
+                               enum GNUNET_BLOCK_Type type,
+                               uint32_t nonce,
+                               const void *raw_data,
+                               size_t raw_data_size,
+                               va_list va)
+{
+  unsigned int bf_size;
+  const char *guard;
+
+  guard = va_arg (va, const char *);
+  if (0 == strcmp (guard,
+                   "seen-set-size"))
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
+  else if (0 == strcmp (guard,
+                        "filter-size"))
+    bf_size = va_arg (va, unsigned int);
+  else
+  {
+    GNUNET_break (0);
+    bf_size = 8;
+  }
+  GNUNET_break (NULL == va_arg (va, const char *));
+  return GNUNET_BLOCK_GROUP_bf_create (cls,
+                                       bf_size,
+                                       BLOOMFILTER_K,
+                                       type,
+                                       nonce,
+                                       raw_data,
+                                       raw_data_size);
+}
 
 
 /**
@@ -60,6 +115,7 @@ block_plugin_dns_evaluate (void *cls,
                            size_t reply_block_size)
 {
   const struct GNUNET_DNS_Advertisement *ad;
+  struct GNUNET_HashCode phash;
 
   switch (type)
   {
@@ -101,6 +157,13 @@ block_plugin_dns_evaluate (void *cls,
       GNUNET_break_op (0);
       return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
     }
+    GNUNET_CRYPTO_hash (reply_block,
+                        reply_block_size,
+                        &phash);
+    if (GNUNET_YES ==
+        GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
+                                            &phash))
+      return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
     return GNUNET_BLOCK_EVALUATION_OK_MORE;
   default:
     return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
@@ -147,6 +210,7 @@ libgnunet_plugin_block_dns_init (void *cls)
   api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
   api->evaluate = &block_plugin_dns_evaluate;
   api->get_key = &block_plugin_dns_get_key;
+  api->create_group = &block_plugin_dns_create_group;
   api->types = types;
   return api;
 }
diff --git a/src/fs/plugin_block_fs.c b/src/fs/plugin_block_fs.c
index 996c06dcb..b6749b418 100644
--- a/src/fs/plugin_block_fs.c
+++ b/src/fs/plugin_block_fs.c
@@ -39,37 +39,6 @@
 
 
 /**
- * How many bytes should a bloomfilter be if we have already seen
- * entry_count responses?  Note that #GNUNET_CONSTANTS_BLOOMFILTER_K
- * gives us the number of bits set per entry.  Furthermore, we should
- * not re-size the filter too often (to keep it cheap).
- *
- * Since other peers will also add entries but not resize the filter,
- * we should generally pick a slightly larger size than what the
- * strict math would suggest.
- *
- * @param entry_count expected number of entries in the Bloom filter
- * @return must be a power of two and smaller or equal to 2^15.
- */
-static size_t
-compute_bloomfilter_size (unsigned int entry_count)
-{
-  size_t size;
-  unsigned int ideal = (entry_count * BLOOMFILTER_K) / 4;
-  uint16_t max = 1 << 15;
-
-  if (entry_count > max)
-    return max;
-  size = 8;
-  while ((size < max) && (size < ideal))
-    size *= 2;
-  if (size > max)
-    return max;
-  return size;
-}
-
-
-/**
  * Create a new block group.
  *
  * @param ctx block context in which the block group is created
@@ -111,7 +80,8 @@ block_plugin_fs_create_group (void *cls,
     }
     else
     {
-      size = compute_bloomfilter_size (va_arg (va, unsigned int));
+      size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                          BLOOMFILTER_K);
     }
     if (0 == size)
       size = raw_data_size; /* not for us to determine, use what we got! */
diff --git a/src/gns/plugin_block_gns.c b/src/gns/plugin_block_gns.c
index 8ce2d85da..2ef48f094 100644
--- a/src/gns/plugin_block_gns.c
+++ b/src/gns/plugin_block_gns.c
@@ -44,37 +44,6 @@
 
 
 /**
- * How many bytes should a bloomfilter be if we have already seen
- * entry_count responses?  Note that #GNUNET_CONSTANTS_BLOOMFILTER_K
- * gives us the number of bits set per entry.  Furthermore, we should
- * not re-size the filter too often (to keep it cheap).
- *
- * Since other peers will also add entries but not resize the filter,
- * we should generally pick a slightly larger size than what the
- * strict math would suggest.
- *
- * @param entry_count expected number of entries in the Bloom filter
- * @return must be a power of two and smaller or equal to 2^15.
- */
-static size_t
-compute_bloomfilter_size (unsigned int entry_count)
-{
-  size_t size;
-  unsigned int ideal = (entry_count * BLOOMFILTER_K) / 4;
-  uint16_t max = 1 << 15;
-
-  if (entry_count > max)
-    return max;
-  size = 8;
-  while ((size < max) && (size < ideal))
-    size *= 2;
-  if (size > max)
-    return max;
-  return size;
-}
-
-
-/**
  * Create a new block group.
  *
  * @param ctx block context in which the block group is created
@@ -100,7 +69,8 @@ block_plugin_gns_create_group (void *cls,
   guard = va_arg (va, const char *);
   if (0 == strcmp (guard,
                    "seen-set-size"))
-    bf_size = compute_bloomfilter_size (va_arg (va, unsigned int));
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
   else if (0 == strcmp (guard,
                         "filter-size"))
     bf_size = va_arg (va, unsigned int);
diff --git a/src/include/gnunet_block_group_lib.h b/src/include/gnunet_block_group_lib.h
index a1ea807f6..3a3dfb2e2 100644
--- a/src/include/gnunet_block_group_lib.h
+++ b/src/include/gnunet_block_group_lib.h
@@ -44,6 +44,24 @@ extern "C"
 
 
 /**
+ * How many bytes should a bloomfilter be if we have already seen
+ * entry_count responses?  Sized so that do not have to
+ * re-size the filter too often (to keep it cheap).
+ *
+ * Since other peers will also add entries but not resize the filter,
+ * we should generally pick a slightly larger size than what the
+ * strict math would suggest.
+ *
+ * @param entry_count expected number of entries in the Bloom filter
+ * @param k number of bits set per entry
+ * @return must be a power of two and smaller or equal to 2^15.
+ */
+size_t
+GNUNET_BLOCK_GROUP_compute_bloomfilter_size (unsigned int entry_count,
+                                             unsigned int k);
+
+
+/**
  * Create a new block group that filters duplicates using a Bloom filter.
  *
  * @param ctx block context in which the block group is created
diff --git a/src/regex/plugin_block_regex.c b/src/regex/plugin_block_regex.c
index 19335e815..a345dfedb 100644
--- a/src/regex/plugin_block_regex.c
+++ b/src/regex/plugin_block_regex.c
@@ -31,7 +31,6 @@
 #include "gnunet_signatures.h"
 
 
-
 /**
  * Number of bits we set per entry in the bloomfilter.
  * Do not change!
@@ -46,37 +45,6 @@
 
 
 /**
- * How many bytes should a bloomfilter be if we have already seen
- * entry_count responses?  Note that #GNUNET_CONSTANTS_BLOOMFILTER_K
- * gives us the number of bits set per entry.  Furthermore, we should
- * not re-size the filter too often (to keep it cheap).
- *
- * Since other peers will also add entries but not resize the filter,
- * we should generally pick a slightly larger size than what the
- * strict math would suggest.
- *
- * @param entry_count expected number of entries in the Bloom filter
- * @return must be a power of two and smaller or equal to 2^15.
- */
-static size_t
-compute_bloomfilter_size (unsigned int entry_count)
-{
-  size_t size;
-  unsigned int ideal = (entry_count * BLOOMFILTER_K) / 4;
-  uint16_t max = 1 << 15;
-
-  if (entry_count > max)
-    return max;
-  size = 8;
-  while ((size < max) && (size < ideal))
-    size *= 2;
-  if (size > max)
-    return max;
-  return size;
-}
-
-
-/**
  * Create a new block group.
  *
  * @param ctx block context in which the block group is created
@@ -102,7 +70,8 @@ block_plugin_regex_create_group (void *cls,
   guard = va_arg (va, const char *);
   if (0 == strcmp (guard,
                    "seen-set-size"))
-    bf_size = compute_bloomfilter_size (va_arg (va, unsigned int));
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
   else if (0 == strcmp (guard,
                         "filter-size"))
     bf_size = va_arg (va, unsigned int);