add logic for strata compression

10 files changed, 305 insertions, 66 deletions

diff --git a/src/include/gnunet_protocols.h b/src/include/gnunet_protocols.h
index a06c8ad67..a47af7f59 100644
--- a/src/include/gnunet_protocols.h
+++ b/src/include/gnunet_protocols.h
@@ -1925,6 +1925,11 @@ extern "C"
 #define GNUNET_MESSAGE_TYPE_SET_ITER_DONE 589
 
 /**
+ * Compressed strata estimator.
+ */
+#define GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC 590
+
+/**
  * Information about the element count for intersection
  */
 #define GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO 591
diff --git a/src/set/gnunet-service-set.c b/src/set/gnunet-service-set.c
index cfddef6fb..441637a93 100644
--- a/src/set/gnunet-service-set.c
+++ b/src/set/gnunet-service-set.c
@@ -975,7 +975,7 @@ again:
     GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
     set->iter = NULL;
     set->iteration_id++;
-    
+
     GNUNET_assert (set->content->iterator_count > 0);
     set->content->iterator_count -= 1;
 
@@ -1113,6 +1113,13 @@ handle_client_create_set (void *cls,
   }
   set->operation = ntohl (msg->operation);
   set->state = set->vt->create ();
+  if (NULL == set->state)
+  {
+    /* initialization failed (i.e. out of memory) */
+    GNUNET_free (set);
+    GNUNET_SERVER_client_disconnect (client);
+    return;
+  }
   set->content = GNUNET_new (struct SetContent);
   set->content->refcount = 1;
   set->content->elements = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_YES);
@@ -1497,7 +1504,7 @@ handle_client_copy_lazy_connect (void *cls,
     {
       found = GNUNET_YES;
       break;
-    } 
+    }
   }
 
   if (GNUNET_NO == found)
diff --git a/src/set/gnunet-service-set.h b/src/set/gnunet-service-set.h
index 982967744..f8a2d88ba 100644
--- a/src/set/gnunet-service-set.h
+++ b/src/set/gnunet-service-set.h
@@ -126,7 +126,7 @@ struct OperationSpecification
  * Signature of functions that create the implementation-specific
  * state for a set supporting a specific operation.
  *
- * @return a set state specific to the supported operation
+ * @return a set state specific to the supported operation, NULL on error
  */
 typedef struct SetState *
 (*CreateImpl) (void);
diff --git a/src/set/gnunet-service-set_intersection.c b/src/set/gnunet-service-set_intersection.c
index af0ce1988..7d172df29 100644
--- a/src/set/gnunet-service-set_intersection.c
+++ b/src/set/gnunet-service-set_intersection.c
@@ -59,6 +59,7 @@ enum IntersectionOperationPhase
    * client.
    */
   PHASE_FINISHED
+
 };
 
 
diff --git a/src/set/gnunet-service-set_union.c b/src/set/gnunet-service-set_union.c
index 1c5dfe3fb..bdcd4352e 100644
--- a/src/set/gnunet-service-set_union.c
+++ b/src/set/gnunet-service-set_union.c
@@ -39,10 +39,12 @@
  * Number of IBFs in a strata estimator.
  */
 #define SE_STRATA_COUNT 32
+
 /**
  * Size of the IBFs in the strata estimator.
  */
 #define SE_IBF_SIZE 80
+
 /**
  * The hash num parameter for the difference digests and strata estimators.
  */
@@ -119,7 +121,7 @@ enum UnionOperationPhase
    * In the ultimate phase, we wait until
    * our demands are satisfied and then
    * quit (sending another DONE message). */
-  PHASE_DONE,
+  PHASE_DONE
 };
 
 
@@ -216,7 +218,7 @@ struct SendElementClosure
 /**
  * Extra state required for efficient set union.
  */
-  struct SetState
+struct SetState
 {
   /**
    * The strata estimator is only generated once for
@@ -500,8 +502,9 @@ init_key_to_element_iterator (void *cls,
  *
  * @param op the union operation
  * @param size size of the ibf to create
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
  */
-static void
+static int
 prepare_ibf (struct Operation *op,
              uint32_t size)
 {
@@ -517,9 +520,16 @@ prepare_ibf (struct Operation *op,
   if (NULL != op->state->local_ibf)
     ibf_destroy (op->state->local_ibf);
   op->state->local_ibf = ibf_create (size, SE_IBF_HASH_NUM);
+  if (NULL == op->state->local_ibf)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to allocate local IBF\n");
+    return GNUNET_SYSERR;
+  }
   GNUNET_CONTAINER_multihashmap32_iterate (op->state->key_to_element,
                                            &prepare_ibf_iterator,
                                            op);
+  return GNUNET_OK;
 }
 
 
@@ -530,15 +540,21 @@ prepare_ibf (struct Operation *op,
  *
  * @param op the union operation
  * @param ibf_order order of the ibf to send, size=2^order
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
  */
-static void
+static int
 send_ibf (struct Operation *op,
           uint16_t ibf_order)
 {
   unsigned int buckets_sent = 0;
   struct InvertibleBloomFilter *ibf;
 
-  prepare_ibf (op, 1<<ibf_order);
+  if (GNUNET_OK !=
+      prepare_ibf (op, 1<<ibf_order))
+  {
+    /* allocation failed */
+    return GNUNET_SYSERR;
+  }
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "sending ibf of size %u\n",
@@ -583,6 +599,7 @@ send_ibf (struct Operation *op,
   /* The other peer must decode the IBF, so
    * we're passive. */
   op->state->phase = PHASE_INVENTORY_PASSIVE;
+  return GNUNET_OK;
 }
 
 
@@ -594,13 +611,27 @@ send_ibf (struct Operation *op,
 static void
 send_strata_estimator (struct Operation *op)
 {
+  const struct StrataEstimator *se = op->state->se;
   struct GNUNET_MQ_Envelope *ev;
   struct GNUNET_MessageHeader *strata_msg;
-
+  char *buf;
+  size_t len;
+  uint16_t type;
+
+  buf = GNUNET_malloc (se->strata_count * IBF_BUCKET_SIZE * se->ibf_size);
+  len = strata_estimator_write (op->state->se,
+                                buf);
+  if (len < se->strata_count * IBF_BUCKET_SIZE * se->ibf_size)
+    type = GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC;
+  else
+    type = GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE;
   ev = GNUNET_MQ_msg_header_extra (strata_msg,
-                                   SE_STRATA_COUNT * IBF_BUCKET_SIZE * SE_IBF_SIZE,
-                                   GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE);
-  strata_estimator_write (op->state->se, &strata_msg[1]);
+                                   len,
+                                   type);
+  memcpy (&strata_msg[1],
+          buf,
+          len);
+  GNUNET_free (buf);
   GNUNET_MQ_send (op->mq,
                   ev);
   op->state->phase = PHASE_EXPECT_IBF;
@@ -636,16 +667,19 @@ get_order_from_difference (unsigned int diff)
  *
  * @param cls the union operation
  * @param mh the message
+ * @param is_compressed #GNUNET_YES if the estimator is compressed
  * @return #GNUNET_SYSERR if the tunnel should be disconnected,
  *         #GNUNET_OK otherwise
  */
 static int
 handle_p2p_strata_estimator (void *cls,
-                             const struct GNUNET_MessageHeader *mh)
+                             const struct GNUNET_MessageHeader *mh,
+                             int is_compressed)
 {
   struct Operation *op = cls;
   struct StrataEstimator *remote_se;
   int diff;
+  size_t len;
 
   if (op->state->phase != PHASE_EXPECT_SE)
   {
@@ -653,9 +687,10 @@ handle_p2p_strata_estimator (void *cls,
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-  if (ntohs (mh->size) !=
-      SE_STRATA_COUNT * SE_IBF_SIZE * IBF_BUCKET_SIZE +
-      sizeof (struct GNUNET_MessageHeader))
+  len = ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader);
+  if ( (GNUNET_NO == is_compressed) &&
+       (len !=
+        SE_STRATA_COUNT * SE_IBF_SIZE * IBF_BUCKET_SIZE) )
   {
     fail_union_operation (op);
     GNUNET_break (0);
@@ -664,7 +699,22 @@ handle_p2p_strata_estimator (void *cls,
   remote_se = strata_estimator_create (SE_STRATA_COUNT,
                                        SE_IBF_SIZE,
                                        SE_IBF_HASH_NUM);
-  strata_estimator_read (&mh[1], remote_se);
+  if (NULL == remote_se)
+  {
+    /* insufficient resources, fail */
+    fail_union_operation (op);
+    return GNUNET_SYSERR;
+  }
+  if (GNUNET_OK !=
+      strata_estimator_read (&mh[1],
+                             len,
+                             is_compressed,
+                             remote_se))
+  {
+    /* decompression failed */
+    fail_union_operation (op);
+    return GNUNET_SYSERR;
+  }
   GNUNET_assert (NULL != op->state->se);
   diff = strata_estimator_difference (remote_se,
                                       op->state->se);
@@ -675,8 +725,16 @@ handle_p2p_strata_estimator (void *cls,
        "got se diff=%d, using ibf size %d\n",
        diff,
        1<<get_order_from_difference (diff));
-  send_ibf (op,
-            get_order_from_difference (diff));
+  if (GNUNET_OK !=
+      send_ibf (op,
+                get_order_from_difference (diff)))
+  {
+    /* Internal error, best we can do is shut the connection */
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to send IBF, closing connection\n");
+    fail_union_operation (op);
+    return GNUNET_SYSERR;
+  }
   return GNUNET_OK;
 }
 
@@ -744,8 +802,9 @@ send_offers_for_key (struct Operation *op,
  * send the appropriate offers and inquiries.
  *
  * @param op union operation
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
  */
-static void
+static int
 decode_and_send (struct Operation *op)
 {
   struct IBF_Key key;
@@ -756,7 +815,12 @@ decode_and_send (struct Operation *op)
 
   GNUNET_assert (PHASE_INVENTORY_ACTIVE == op->state->phase);
 
-  prepare_ibf (op, op->state->remote_ibf->size);
+  if (GNUNET_OK !=
+      prepare_ibf (op, op->state->remote_ibf->size))
+  {
+    /* allocation failed */
+    return GNUNET_SYSERR;
+  }
   diff_ibf = ibf_dup (op->state->local_ibf);
   ibf_subtract (diff_ibf, op->state->remote_ibf);
 
@@ -811,7 +875,16 @@ decode_and_send (struct Operation *op)
                                   "# of IBF retries",
                                   1,
                                   GNUNET_NO);
-        send_ibf (op, next_order);
+        if (GNUNET_OK !=
+            send_ibf (op, next_order))
+        {
+          /* Internal error, best we can do is shut the connection */
+          GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                      "Failed to send IBF, closing connection\n");
+          fail_union_operation (op);
+          ibf_destroy (diff_ibf);
+          return GNUNET_SYSERR;
+        }
       }
       else
       {
@@ -822,6 +895,9 @@ decode_and_send (struct Operation *op)
         // XXX: Send the whole set, element-by-element
         LOG (GNUNET_ERROR_TYPE_ERROR,
              "set union failed: reached ibf limit\n");
+        fail_union_operation (op);
+        ibf_destroy (diff_ibf);
+        return GNUNET_SYSERR;
       }
       break;
     }
@@ -867,6 +943,7 @@ decode_and_send (struct Operation *op)
     }
   }
   ibf_destroy (diff_ibf);
+  return GNUNET_OK;
 }
 
 
@@ -905,6 +982,13 @@ handle_p2p_ibf (void *cls,
          "Creating new ibf of size %u\n",
          1 << msg->order);
     op->state->remote_ibf = ibf_create (1<<msg->order, SE_IBF_HASH_NUM);
+    if (NULL == op->state->remote_ibf)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Failed to parse remote IBF, closing connection\n");
+      fail_union_operation (op);
+      return GNUNET_SYSERR;
+    }
     op->state->ibf_buckets_received = 0;
     if (0 != ntohs (msg->offset))
     {
@@ -957,7 +1041,14 @@ handle_p2p_ibf (void *cls,
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "received full ibf\n");
     op->state->phase = PHASE_INVENTORY_ACTIVE;
-    decode_and_send (op);
+    if (GNUNET_OK !=
+        decode_and_send (op))
+    {
+      /* Internal error, best we can do is shut down */
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Failed to decode IBF, closing connection\n");
+      return GNUNET_SYSERR;
+    }
   }
   return GNUNET_OK;
 }
@@ -1080,7 +1171,6 @@ handle_p2p_elements (void *cls,
     fail_union_operation (op);
     return;
   }
-
   if (ntohs (mh->size) < sizeof (struct GNUNET_SET_ElementMessage))
   {
     GNUNET_break_op (0);
@@ -1088,7 +1178,7 @@ handle_p2p_elements (void *cls,
     return;
   }
 
-  emsg = (struct GNUNET_SET_ElementMessage *) mh;
+  emsg = (const struct GNUNET_SET_ElementMessage *) mh;
 
   element_size = ntohs (mh->size) - sizeof (struct GNUNET_SET_ElementMessage);
   ee = GNUNET_malloc (sizeof (struct ElementEntry) + element_size);
@@ -1099,7 +1189,10 @@ handle_p2p_elements (void *cls,
   ee->remote = GNUNET_YES;
   GNUNET_SET_element_hash (&ee->element, &ee->element_hash);
 
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_remove (op->state->demanded_hashes, &ee->element_hash, NULL))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_remove (op->state->demanded_hashes,
+                                            &ee->element_hash,
+                                            NULL))
   {
     /* We got something we didn't demand, since it's not in our map. */
     GNUNET_break_op (0);
@@ -1194,10 +1287,12 @@ handle_p2p_inquiry (void *cls,
 }
 
 
-
+/**
+ * FIXME
+ */
 static void
 handle_p2p_demand (void *cls,
-                    const struct GNUNET_MessageHeader *mh)
+                   const struct GNUNET_MessageHeader *mh)
 {
   struct Operation *op = cls;
   struct ElementEntry *ee;
@@ -1303,12 +1398,16 @@ handle_p2p_offer (void *cls,
     struct ElementEntry *ee;
     struct GNUNET_MessageHeader *demands;
     struct GNUNET_MQ_Envelope *ev;
-    ee = GNUNET_CONTAINER_multihashmap_get (op->spec->set->content->elements, hash);
+
+    ee = GNUNET_CONTAINER_multihashmap_get (op->spec->set->content->elements,
+                                            hash);
     if (NULL != ee)
       if (GNUNET_YES == _GSS_is_element_of_operation (ee, op))
         continue;
 
-    if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (op->state->demanded_hashes, hash))
+    if (GNUNET_YES ==
+        GNUNET_CONTAINER_multihashmap_contains (op->state->demanded_hashes,
+                                                hash))
     {
       LOG (GNUNET_ERROR_TYPE_DEBUG,
            "Skipped sending duplicate demand\n");
@@ -1324,7 +1423,9 @@ handle_p2p_offer (void *cls,
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "[OP %x] Requesting element (hash %s)\n",
          (void *) op, GNUNET_h2s (hash));
-    ev = GNUNET_MQ_msg_header_extra (demands, sizeof (struct GNUNET_HashCode), GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND);
+    ev = GNUNET_MQ_msg_header_extra (demands,
+                                     sizeof (struct GNUNET_HashCode),
+                                     GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND);
     *(struct GNUNET_HashCode *) &demands[1] = *hash;
     GNUNET_MQ_send (op->mq, ev);
   }
@@ -1466,7 +1567,7 @@ union_accept (struct Operation *op)
  * We maintain one strata estimator per set and then manipulate it over the
  * lifetime of the set, as recreating a strata estimator would be expensive.
  *
- * @return the newly created set
+ * @return the newly created set, NULL on error
  */
 static struct SetState *
 union_set_create (void)
@@ -1478,6 +1579,13 @@ union_set_create (void)
   set_state = GNUNET_new (struct SetState);
   set_state->se = strata_estimator_create (SE_STRATA_COUNT,
                                            SE_IBF_SIZE, SE_IBF_HASH_NUM);
+  if (NULL == set_state->se)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to allocate strata estimator\n");
+    GNUNET_free (set_state);
+    return NULL;
+  }
   return set_state;
 }
 
@@ -1549,7 +1657,9 @@ union_handle_p2p_message (struct Operation *op,
     case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF:
       return handle_p2p_ibf (op, mh);
     case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE:
-      return handle_p2p_strata_estimator (op, mh);
+      return handle_p2p_strata_estimator (op, mh, GNUNET_NO);
+    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC:
+      return handle_p2p_strata_estimator (op, mh, GNUNET_YES);
     case GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS:
       handle_p2p_elements (op, mh);
       break;
diff --git a/src/set/gnunet-service-set_union_strata_estimator.c b/src/set/gnunet-service-set_union_strata_estimator.c
index 38b32961c..e94d52650 100644
--- a/src/set/gnunet-service-set_union_strata_estimator.c
+++ b/src/set/gnunet-service-set_union_strata_estimator.c
@@ -21,6 +21,7 @@
  * @file set/gnunet-service-set_union_strata_estimator.c
  * @brief invertible bloom filter
  * @author Florian Dold
+ * @author Christian Grothoff
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
@@ -29,12 +30,20 @@
 
 
 /**
+ * Should we try compressing the strata estimator? This will
+ * break compatibility with the 0.10.1-network.
+ */
+#define FAIL_10_1_COMPATIBILTIY 0
+
+
+/**
  * Write the given strata estimator to the buffer.
  *
  * @param se strata estimator to serialize
- * @param buf buffer to write to, must be of appropriate size
+ * @param[out] buf buffer to write to, must be of appropriate size
+ * @return number of bytes written to @a buf
  */
-void
+size_t
 strata_estimator_write (const struct StrataEstimator *se,
                         void *buf)
 {
@@ -43,9 +52,32 @@ strata_estimator_write (const struct StrataEstimator *se,
   GNUNET_assert (NULL != se);
   for (i = 0; i < se->strata_count; i++)
   {
-    ibf_write_slice (se->strata[i], 0, se->ibf_size, buf);
+    ibf_write_slice (se->strata[i],
+                     0,
+                     se->ibf_size,
+                     buf);
     buf += se->ibf_size * IBF_BUCKET_SIZE;
   }
+  osize = se->ibf_size * IBF_BUCKET_SIZE * se->strata_count;
+#if FAIL_10_1_COMPATIBILTIY
+  {
+    size_t osize;
+    char *cbuf;
+    size_t nsize;
+
+    if (GNUNET_YES ==
+        GNUNET_try_compression (buf,
+                                osize,
+                                &cbuf,
+                                &nsize))
+    {
+      memcpy (buf, cbuf, nsize);
+      osize = nsize;
+      GNUNET_free (cbuf);
+    }
+  }
+#endif
+  return osize;
 }
 
 
@@ -54,19 +86,50 @@ strata_estimator_write (const struct StrataEstimator *se,
  * estimator.  The strata estimator must already be allocated.
  *
  * @param buf buffer to read from
- * @param se strata estimator to write to
+ * @param buf_len number of bytes in @a buf
+ * @param is_compressed is the data compressed?
+ * @param[out] se strata estimator to write to
+ * @return #GNUNET_OK on success
  */
-void
+int
 strata_estimator_read (const void *buf,
+                       size_t buf_len,
+                       int is_compressed,
                        struct StrataEstimator *se)
 {
   unsigned int i;
+  size_t osize;
+  char *dbuf;
 
+  dbuf = NULL;
+  if (GNUNET_YES == is_compressed)
+  {
+    osize = se->ibf_size * IBF_BUCKET_SIZE * se->strata_count;
+    dbuf = GNUNET_decompress (buf,
+                              buf_len,
+                              osize);
+    if (NULL == dbuf)
+    {
+      GNUNET_break_op (0); /* bad compressed input data */
+      return GNUNET_SYSERR;
+    }
+    buf = dbuf;
+    buf_len = osize;
+  }
+
+  if (buf_len != se->strata_count * se->ibf_size * IBF_BUCKET_SIZE)
+  {
+    GNUNET_break (0); /* very odd error */
+    GNUNET_free_non_null (dbuf);
+    return GNUNET_SYSERR;
+  }
   for (i = 0; i < se->strata_count; i++)
   {
     ibf_read_slice (buf, 0, se->ibf_size, se->strata[i]);
     buf += se->ibf_size * IBF_BUCKET_SIZE;
   }
+  GNUNET_free_non_null (dbuf);
+  return GNUNET_OK;
 }
 
 
@@ -118,7 +181,7 @@ strata_estimator_remove (struct StrataEstimator *se,
  * @param strata_count number of stratas, that is, number of ibfs in the estimator
  * @param ibf_size size of each ibf stratum
  * @param ibf_hashnum hashnum parameter of each ibf
- * @return a freshly allocated, empty strata estimator
+ * @return a freshly allocated, empty strata estimator, NULL on error
  */
 struct StrataEstimator *
 strata_estimator_create (unsigned int strata_count,
@@ -127,14 +190,24 @@ strata_estimator_create (unsigned int strata_count,
 {
   struct StrataEstimator *se;
   unsigned int i;
+  unsigned int j;
 
-  /* fixme: allocate everything in one chunk */
   se = GNUNET_new (struct StrataEstimator);
   se->strata_count = strata_count;
   se->ibf_size = ibf_size;
-  se->strata = GNUNET_malloc (sizeof (struct InvertibleBloomFilter *) * strata_count);
+  se->strata = GNUNET_new_array (strata_count,
+                                 struct InvertibleBloomFilter *);
   for (i = 0; i < strata_count; i++)
+  {
     se->strata[i] = ibf_create (ibf_size, ibf_hashnum);
+    if (NULL == se->strata[i])
+    {
+      for (j = 0; j < i; j++)
+        ibf_destroy (se->strata[i]);
+      GNUNET_free (se);
+      return NULL;
+    }
+  }
   return se;
 }
 
@@ -226,4 +299,3 @@ strata_estimator_destroy (struct StrataEstimator *se)
   GNUNET_free (se->strata);
   GNUNET_free (se);
 }
-
diff --git a/src/set/gnunet-service-set_union_strata_estimator.h b/src/set/gnunet-service-set_union_strata_estimator.h
index 2b59e8ed3..8c50e2c9f 100644
--- a/src/set/gnunet-service-set_union_strata_estimator.h
+++ b/src/set/gnunet-service-set_union_strata_estimator.h
@@ -66,9 +66,10 @@ struct StrataEstimator
  * Write the given strata estimator to the buffer.
  *
  * @param se strata estimator to serialize
- * @param buf buffer to write to, must be of appropriate size
+ * @param[out] buf buffer to write to, must be of appropriate size
+ * @return number of bytes written to @a buf
  */
-void
+size_t
 strata_estimator_write (const struct StrataEstimator *se,
                         void *buf);
 
@@ -78,10 +79,15 @@ strata_estimator_write (const struct StrataEstimator *se,
  * estimator.  The strata estimator must already be allocated.
  *
  * @param buf buffer to read from
- * @param se strata estimator to write to
+ * @param buf_len number of bytes in @a buf
+ * @param is_compressed is the data compressed?
+ * @param[out] se strata estimator to write to
+ * @return #GNUNET_OK on success
  */
-void
+int
 strata_estimator_read (const void *buf,
+                       size_t buf_len,
+                       int is_compressed,
                        struct StrataEstimator *se);
 
 
@@ -91,7 +97,7 @@ strata_estimator_read (const void *buf,
  * @param strata_count number of stratas, that is, number of ibfs in the estimator
  * @param ibf_size size of each ibf stratum
  * @param ibf_hashnum hashnum parameter of each ibf
- * @return a freshly allocated, empty strata estimator
+ * @return a freshly allocated, empty strata estimator, NULL on error
  */
 struct StrataEstimator *
 strata_estimator_create (unsigned int strata_count,
@@ -161,4 +167,3 @@ strata_estimator_dup (struct StrataEstimator *se);
 #endif
 
 #endif
-
diff --git a/src/set/gnunet-set-ibf-profiler.c b/src/set/gnunet-set-ibf-profiler.c
index 86004d96e..412a9120e 100644
--- a/src/set/gnunet-set-ibf-profiler.c
+++ b/src/set/gnunet-set-ibf-profiler.c
@@ -102,7 +102,9 @@ remove_iterator (void *cls,
 
 
 static void
-run (void *cls, char *const *args, const char *cfgfile,
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   struct GNUNET_HashCode id;
@@ -171,6 +173,15 @@ run (void *cls, char *const *args, const char *cfgfile,
 
   ibf_a = ibf_create (ibf_size, hash_num);
   ibf_b = ibf_create (ibf_size, hash_num);
+  if ( (NULL == ibf_a) ||
+       (NULL == ibf_b) )
+  {
+    /* insufficient memory */
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+
 
   printf ("generated sets\n");
 
@@ -229,6 +240,7 @@ run (void *cls, char *const *args, const char *cfgfile,
          asize + bsize);
 }
 
+
 int
 main (int argc, char **argv)
 {
@@ -255,4 +267,3 @@ main (int argc, char **argv)
                       options, &run, NULL, GNUNET_YES);
   return 0;
 }
-
diff --git a/src/set/ibf.c b/src/set/ibf.c
index 19d15a5b2..0e8e1839e 100644
--- a/src/set/ibf.c
+++ b/src/set/ibf.c
@@ -41,7 +41,6 @@
 struct IBF_Key
 ibf_key_from_hashcode (const struct GNUNET_HashCode *hash)
 {
-  /* FIXME: endianess */
   return *(struct IBF_Key *) hash;
 }
 
@@ -53,11 +52,13 @@ ibf_key_from_hashcode (const struct GNUNET_HashCode *hash)
  * @param dst hashcode to store the result in
  */
 void
-ibf_hashcode_from_key (struct IBF_Key key, struct GNUNET_HashCode *dst)
+ibf_hashcode_from_key (struct IBF_Key key,
+                       struct GNUNET_HashCode *dst)
 {
   struct IBF_Key *p;
   unsigned int i;
   const unsigned int keys_per_hashcode = sizeof (struct GNUNET_HashCode) / sizeof (struct IBF_Key);
+
   p = (struct IBF_Key *) dst;
   for (i = 0; i < keys_per_hashcode; i++)
     *p++ = key;
@@ -69,7 +70,7 @@ ibf_hashcode_from_key (struct IBF_Key key, struct GNUNET_HashCode *dst)
  *
  * @param size number of IBF buckets
  * @param hash_num number of buckets one element is hashed in
- * @return the newly created invertible bloom filter
+ * @return the newly created invertible bloom filter, NULL on error
  */
 struct InvertibleBloomFilter *
 ibf_create (uint32_t size, uint8_t hash_num)
@@ -80,20 +81,40 @@ ibf_create (uint32_t size, uint8_t hash_num)
 
   ibf = GNUNET_new (struct InvertibleBloomFilter);
   ibf->count = GNUNET_malloc_large (size * sizeof (uint8_t));
+  if (NULL == ibf->count)
+  {
+    GNUNET_free (ibf);
+    return NULL;
+  }
   ibf->key_sum = GNUNET_malloc_large (size * sizeof (struct IBF_Key));
+  if (NULL == ibf->key_sum)
+  {
+    GNUNET_free (ibf->count);
+    GNUNET_free (ibf);
+    return NULL;
+  }
   ibf->key_hash_sum = GNUNET_malloc_large (size * sizeof (struct IBF_KeyHash));
+  if (NULL == ibf->key_hash_sum)
+  {
+    GNUNET_free (ibf->key_sum);
+    GNUNET_free (ibf->count);
+    GNUNET_free (ibf);
+    return NULL;
+  }
   ibf->size = size;
   ibf->hash_num = hash_num;
 
   return ibf;
 }
 
+
 /**
  * Store unique bucket indices for the specified key in dst.
  */
-static inline void
+static void
 ibf_get_indices (const struct InvertibleBloomFilter *ibf,
-                 struct IBF_Key key, int *dst)
+                 struct IBF_Key key,
+                 int *dst)
 {
   uint32_t filled;
   uint32_t i;
@@ -371,4 +392,3 @@ ibf_destroy (struct InvertibleBloomFilter *ibf)
   GNUNET_free (ibf->count);
   GNUNET_free (ibf);
 }
-
diff --git a/src/set/ibf.h b/src/set/ibf.h
index 831aa6b82..616cca5f6 100644
--- a/src/set/ibf.h
+++ b/src/set/ibf.h
@@ -123,7 +123,10 @@ struct InvertibleBloomFilter
  * @param buf buffer to write the data to
  */
 void
-ibf_write_slice (const struct InvertibleBloomFilter *ibf, uint32_t start, uint32_t count, void *buf);
+ibf_write_slice (const struct InvertibleBloomFilter *ibf,
+                 uint32_t start,
+                 uint32_t count,
+                 void *buf);
 
 
 /**
@@ -132,10 +135,13 @@ ibf_write_slice (const struct InvertibleBloomFilter *ibf, uint32_t start, uint32
  * @param buf pointer to the buffer to read from
  * @param start which bucket to start at
  * @param count how many buckets to read
- * @param ibf the ibf to read from
+ * @param ibf the ibf to write to
  */
 void
-ibf_read_slice (const void *buf, uint32_t start, uint32_t count, struct InvertibleBloomFilter *ibf);
+ibf_read_slice (const void *buf,
+                uint32_t start,
+                uint32_t count,
+                struct InvertibleBloomFilter *ibf);
 
 
 /**
@@ -164,7 +170,7 @@ ibf_hashcode_from_key (struct IBF_Key key, struct GNUNET_HashCode *dst);
  *
  * @param size number of IBF buckets
  * @param hash_num number of buckets one element is hashed in, usually 3 or 4
- * @return the newly created invertible bloom filter
+ * @return the newly created invertible bloom filter, NULL on error
  */
 struct InvertibleBloomFilter *
 ibf_create (uint32_t size, uint8_t hash_num);
@@ -198,7 +204,8 @@ ibf_remove (struct InvertibleBloomFilter *ibf, struct IBF_Key key);
  * @param ibf2 IBF that will be subtracted from ibf1
  */
 void
-ibf_subtract (struct InvertibleBloomFilter *ibf1, const struct InvertibleBloomFilter *ibf2);
+ibf_subtract (struct InvertibleBloomFilter *ibf1,
+              const struct InvertibleBloomFilter *ibf2);
 
 
 /**
@@ -209,12 +216,14 @@ ibf_subtract (struct InvertibleBloomFilter *ibf1, const struct InvertibleBloomFi
  *                 A negative sign indicates that the element was recovered
  *                 resides in an IBF that was previously subtracted from.
  * @param ret_id receives the hash code of the decoded element, if successful
- * @return GNUNET_YES if decoding an element was successful,
- *         GNUNET_NO if the IBF is empty,
- *         GNUNET_SYSERR if the decoding has failed
+ * @return #GNUNET_YES if decoding an element was successful,
+ *         #GNUNET_NO if the IBF is empty,
+ *         #GNUNET_SYSERR if the decoding has failed
  */
 int
-ibf_decode (struct InvertibleBloomFilter *ibf, int *ret_side, struct IBF_Key *ret_id);
+ibf_decode (struct InvertibleBloomFilter *ibf,
+            int *ret_side,
+            struct IBF_Key *ret_id);
 
 
 /**
@@ -244,4 +253,3 @@ ibf_destroy (struct InvertibleBloomFilter *ibf);
 #endif
 
 #endif
-