syn

11 files changed, 812 insertions, 576 deletions

diff --git a/src/fs/fs.h b/src/fs/fs.h
index 4743a594b..c66249613 100644
--- a/src/fs/fs.h
+++ b/src/fs/fs.h
@@ -26,11 +26,33 @@
 #ifndef FS_H
 #define FS_H
 
+#include "gnunet_datastore_service.h"
+#include "gnunet_fs_service.h"
+
 /**
  * Size of the individual blocks used for file-sharing.
  */
 #define GNUNET_FS_DBLOCK_SIZE (32*1024)
 
+
+/**
+ * Pick a multiple of 2 here to achive 8-byte alignment!
+ * We also probably want DBlocks to have (roughly) the
+ * same size as IBlocks.  With SHA-512, the optimal
+ * value is 32768 byte / 128 byte = 256
+ * (128 byte = 2 * 512 bits).  DO NOT CHANGE!
+ */
+#define GNUNET_FS_CHK_PER_INODE 256
+
+
+/**
+ * Maximum size for a file to be considered for
+ * inlining in a directory.
+ */
+#define GNUNET_FS_MAX_INLINE_SIZE 65536
+
+
+
 /**
  * @brief content hash key
  */
@@ -51,7 +73,7 @@ struct FileIdentifier
   /**
    * Total size of the file in bytes. (network byte order (!))
    */
-  unsigned long long file_length;
+  uint64_t file_length;
 
   /**
    * Query and key of the top GNUNET_EC_IBlock.
@@ -187,6 +209,12 @@ struct GNUNET_FS_FileInformation
   struct GNUNET_FS_Uri *keywords;
 
   /**
+   * CHK for this file or directory. NULL if
+   * we have not yet computed it.
+   */
+  struct GNUNET_FS_Uri *chk_uri;
+
+  /**
    * At what time should the content expire?
    */
   struct GNUNET_TIME_Absolute expirationTime;
@@ -198,6 +226,31 @@ struct GNUNET_FS_FileInformation
   char *serialization;
 
   /**
+   * In-memory cache of the current CHK tree.
+   * This struct will contain the CHK values
+   * from the root to the currently processed
+   * node in the tree as identified by 
+   * "current_depth" and "publish_offset".
+   * The "chktree" will be initially NULL,
+   * then allocated to a sufficient number of
+   * entries for the size of the file and
+   * finally freed once the upload is complete.
+   */
+  struct ContentHashKey *chk_tree;
+  
+  /**
+   * Number of entries in "chk_tree".
+   */
+  unsigned int chk_tree_depth;
+
+  /**
+   * Depth in the CHK-tree at which we are
+   * currently publishing.  0 is the root
+   * of the tree.
+   */
+  unsigned int current_depth;
+
+  /**
    * How many bytes of this file or directory have been
    * published so far?
    */
@@ -257,6 +310,12 @@ struct GNUNET_FS_FileInformation
        */
       uint64_t dir_size;
 
+      /**
+       * Pointer to the data for the directory (or NULL if not
+       * available).
+       */
+      char *dir_data;
+
     } dir;
 
   } data;
@@ -358,6 +417,23 @@ struct GNUNET_FS_PublishContext
    * if the upload has completed.
    */
   GNUNET_SCHEDULER_TaskIdentifier upload_task;
+
+  /**
+   * Current position in the file-tree for the
+   * upload.
+   */
+  struct GNUNET_FS_FileInformation *fi_pos;
+
+  /**
+   * Connection to the datastore service.
+   */
+  struct GNUNET_DATASTORE_Handle *dsh;
+
+  /**
+   * Space reservation ID with datastore service
+   * for this upload.
+   */
+  int rid;
 };
 
 
diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c
index 3eb3af50d..e5a9f963f 100644
--- a/src/fs/fs_directory.c
+++ b/src/fs/fs_directory.c
@@ -25,7 +25,6 @@
  *
  * TODO:
  * - add support for embedded file data (use padding room!)
- * - add directory builder API to gnunet_fs_service
  * - modify directory builder API to support incremental
  *   generation of directories (to allow directories that
  *   would not fit into memory to be created)
@@ -38,6 +37,9 @@
 #include "gnunet_fs_service.h"
 #include "fs.h"
 
+#ifndef EXTRACTOR_GNUNET_FULL_DATA
+#define EXTRACTOR_GNUNET_FULL_DATA 137
+#endif
 
 /**
  * Does the meta-data claim that this is a directory?
@@ -215,6 +217,7 @@ GNUNET_FS_directory_list_contents (size_t size,
           return; /* malformed ! */
         }
       pos += mdSize;
+      // EXTRACTOR_GNUNET_FULL_DATA
       /* FIXME: add support for embedded data */
       filename = GNUNET_CONTAINER_meta_data_get_by_type (md,
                                                          EXTRACTOR_FILENAME);
@@ -231,14 +234,150 @@ GNUNET_FS_directory_list_contents (size_t size,
     }
 }
 
+/**
+ * Entries in the directory (builder).
+ */
+struct BuilderEntry
+{
+  /**
+   * This is a linked list.
+   */
+  struct BuilderEntry *next;
+  
+  /**
+   * Length of this entry.
+   */
+  size_t len;
+};
 
-void
-GNUNET_FS_directory_create ()
+/**
+ * Internal state of a directory builder.
+ */
+struct GNUNET_FS_DirectoryBuilder
 {
+  /**
+   * Meta-data for the directory itself.
+   */
+  struct GNUNET_CONTAINER_MetaData *meta;
+
+  /**
+   * Head of linked list of entries.
+   */
+  struct BuilderEntry *head;
+
+  /**
+   * Number of entires in the directory.
+   */
+  unsigned int count;
+};
+
+
+/**
+ * Create a directory builder.
+ * 
+ * @param mdir metadata for the directory
+ */
+struct GNUNET_FS_DirectoryBuilder *
+GNUNET_FS_directory_builder_create (const struct GNUNET_CONTAINER_MetaData *mdir)
+{
+  struct GNUNET_FS_DirectoryBuilder *ret;
+
+  ret = GNUNET_malloc(sizeof(struct GNUNET_FS_DirectoryBuilder));
+  ret->meta = GNUNET_CONTAINER_meta_data_duplicate (mdir);
+  GNUNET_FS_meta_data_make_directory (ret->meta);
+  return ret;
 }
 
 
-#if 0
+/**
+ * Add an entry to a directory.
+ * 
+ * @param bld directory to extend
+ * @param uri uri of the entry (must not be a KSK)
+ * @param md metadata of the entry
+ * @param data raw data of the entry, can be NULL, otherwise
+ *        data must point to exactly the number of bytes specified
+ *        by the uri which must be of type LOC or CHK
+ */
+void
+GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld,
+                                 const struct GNUNET_FS_Uri *uri,
+                                 const struct GNUNET_CONTAINER_MetaData *md,
+                                 const void *data)
+{
+  struct BuilderEntry *e;
+  uint64_t fsize;
+  uint32_t big;
+  size_t mds;
+  size_t mdxs;
+  char *uris;
+  char *ser;
+  size_t slen;
+  struct GNUNET_CONTAINER_MetaData *meta;
+  const struct GNUNET_CONTAINER_MetaData *meta_use;
+
+  GNUNET_assert (! GNUNET_FS_uri_ksk_test (uri));
+  if (NULL != data)
+    if (GNUNET_FS_uri_chk_test (uri))
+      fsize = GNUNET_FS_uri_chk_get_size (uri);
+    else
+      fsize = GNUNET_FS_uri_chk_get_size (GNUNET_FS_uri_loc_get_uri (uri));
+  else
+    fsize = 0; /* not given */
+  if (fsize > GNUNET_FS_MAX_INLINE_SIZE)
+    fsize = 0; /* too large */
+  if (memchr (data, fsize, '\0')) // FIXME: check memchr args!
+    fsize = 0; /* must not have 0's in data! */
+  uris = GNUNET_FS_uri_to_string (uri);
+  slen = strlen (uris) + 1;
+  mds =
+    GNUNET_CONTAINER_meta_data_get_serialized_size (md,
+                                                    GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);  
+  meta_use = md;
+  meta = NULL;
+  if (fsize > 0)
+    {
+      meta = GNUNET_CONTAINER_meta_data_duplicate (md);
+      GNUNET_CONTAINER_meta_data_insert (meta,
+                                         EXTRACTOR_GNUNET_FULL_DATA,
+                                         data);
+      mdxs =
+        GNUNET_CONTAINER_meta_data_get_serialized_size (meta,
+                                                        GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);  
+      if ( (slen + sizeof (uint32_t) + mdxs - 1) / GNUNET_FS_DBLOCK_SIZE ==
+           (slen + sizeof (uint32_t) + mds - 1) / GNUNET_FS_DBLOCK_SIZE)
+        {
+          /* adding full data would not cause us to cross
+             additional blocks, so add it! */
+          meta_use = meta;
+          mds = mdxs;
+        }
+    }
+
+  if (mds > GNUNET_MAX_MALLOC_CHECKED / 2)
+    mds = GNUNET_MAX_MALLOC_CHECKED / 2;
+  e = GNUNET_malloc (sizeof(struct BuilderEntry) + 
+                     slen + mds + sizeof (uint32_t));
+  ser = (char*) &e[1];
+  memcpy (ser, uris, slen);
+  GNUNET_free (uris);
+  ret = GNUNET_CONTAINER_meta_data_serialize (meta_use,
+                                              &ser[slen + sizeof(uint32_t)],
+                                              mds,
+                                              GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
+  if (NULL != meta)
+    GNUNET_CONTAINER_meta_data_destroy (meta);
+  if (ret == -1)
+    mds = 0;
+  else
+    mds = ret;
+  big = htonl (mds);
+  memcpy (&ser[slen], &big, sizeof (uint32_t));
+  e->len = slen + sizeof (uint32_t) + mds;
+  e->next = bld->head;
+  bld->head = e;
+  bld->count++;
+}
 
 
 /**
@@ -246,11 +385,11 @@ GNUNET_FS_directory_create ()
  * data, return the end position of that data
  * after alignment to the GNUNET_FS_DBLOCK_SIZE.
  */
-static uint64_t
-do_align (uint64_t start_position, 
-          uint64_t end_position)
+static size_t
+do_align (size_t start_position, 
+          size_t end_position)
 {
-  uint64_t align;
+  size_t align;
   
   align = (end_position / GNUNET_FS_DBLOCK_SIZE) * GNUNET_FS_DBLOCK_SIZE;
   if ((start_position < align) && (end_position > align))
@@ -269,19 +408,19 @@ do_align (uint64_t start_position,
  * @param perm the permutation of the blocks (updated)
  */
 static void
-block_align (uint64_t start,
+block_align (size_t start,
              unsigned int count, 
-             const uint64_t *sizes,
+             const size_t *sizes,
              unsigned int *perm)
 {
   unsigned int i;
   unsigned int j;
   unsigned int tmp;
   unsigned int best;
-  int64_t badness;
-  uint64_t cpos;
-  uint64_t cend;
-  int64_t cbad;
+  ssize_t badness;
+  size_t cpos;
+  size_t cend;
+  ssize_t cbad;
   unsigned int cval;
 
   cpos = start;
@@ -334,135 +473,94 @@ block_align (uint64_t start,
 
 
 /**
- * Create a directory.  We allow packing more than one variable
- * size entry into one block (and an entry could also span more
- * than one block), but an entry that is smaller than a single
- * block will never cross the block boundary.  This is done to
- * allow processing entries of a directory already even if the
- * download is still partial.<p>
- *
- * The first block begins with the directories MAGIC signature,
- * followed by the meta-data about the directory itself.<p>
+ * Finish building the directory.  Frees the
+ * builder context and returns the directory
+ * in-memory.
  *
- * After that, the directory consists of block-aligned pairs
- * of URIs (0-terminated strings) and serialized meta-data.
- *
- * @param data pointer set to the beginning of the directory
- * @param len set to number of bytes in data
- * @param count number of entries in uris and mds
- * @param uris URIs of the files in the directory
- * @param mds meta-data for the files (must match
- *        respective values at same offset in in uris)
- * @param mdir meta-data for the directory
- * @return GNUNET_OK on success, GNUNET_SYSERR on error
+ * @param bld directory to finish
+ * @param rsize set to the number of bytes needed
+ * @param rdata set to the encoded directory
  */
-int
-GNUNET_FS_directory_create (char **data,
-                            size_t *len,
-                            unsigned int count,
-                            const struct GNUNET_FS_Uri **uris,
-                            const struct GNUNET_CONTAINER_MetaData **mds,
-                            const struct GNUNET_CONTAINER_MetaData *mdir)
+void
+GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
+                                    size_t *rsize,
+                                    void **rdata)
 {
+  char *data;
+  size_t *sizes;
+  unsigned int *perm;
   unsigned int i;
   unsigned int j;
-  uint64_t psize;
-  uint64_t size;
-  uint64_t pos;
-  char **ucs;
-  int ret;
-  uint64_t *sizes;
-  unsigned int *perm;
-
-  for (i = 0; i < count; i++)
-    {
-      if (GNUNET_FS_uri_test_ksk (fis[i].uri))
-        {
-          GNUNET_break (0);
-          return GNUNET_SYSERR; /* illegal in directory! */
-        }
-    }
-  ucs = GNUNET_malloc (sizeof (char *) * count);
-  size = 8 + sizeof (unsigned int);
-  size += GNUNET_meta_data_get_serialized_size (meta, GNUNET_SERIALIZE_FULL);
-  sizes = GNUNET_malloc (count * sizeof (unsigned long long));
-  perm = GNUNET_malloc (count * sizeof (int));
-  for (i = 0; i < count; i++)
+  struct BuilderEntry *pos;
+  struct BuilderEntry **bes;
+  size_t size;
+  size_t psize;
+  size_t off;
+  ssize_t ret;
+  uint32_t big;
+
+  size = 8 + sizeof (uint32_t);
+  size += GNUNET_meta_data_get_serialized_size (bld->meta, 
+                                                GNUNET_SERIALIZE_FULL);
+  if (bld->count > 0)
     {
-      perm[i] = i;
-      ucs[i] = GNUNET_FS_uri_to_string (fis[i].uri);
-      GNUNET_assert (ucs[i] != NULL);
-      psize =
-        GNUNET_meta_data_get_serialized_size (fis[i].meta,
-                                              GNUNET_SERIALIZE_FULL);
-      if (psize == -1)
-        {
-          GNUNET_break (0);
-          GNUNET_free (sizes);
-          GNUNET_free (perm);
-          while (i >= 0)
-            GNUNET_free (ucs[i--]);
-          GNUNET_free (ucs);
-          return GNUNET_SYSERR;
-        }
-      sizes[i] = psize + sizeof (unsigned int) + strlen (ucs[i]) + 1;
-    }
-  /* permutate entries to minimize alignment cost */
-  block_align (size, count, sizes, perm);
+      sizes = GNUNET_malloc (bld->count * sizeof (size_t));
+      perm = GNUNET_malloc (bld->count * sizeof (unsigned int));
+      bes = GNUNET_malloc (bld->count * sizeof (struct BuilderEntry *));
+      pos = bld->head;
+      for (i = 0; i < bld->count; i++)
+        {
+          perm[i] = i;
+          bes[i] = pos;
+          sizes[i] = pos->size;
+          pos = pos->next;
+        }
+    }  
+  block_align (size,
+               bld->count,
+               sizes,
+               perm);
 
   /* compute final size with alignment */
-  for (i = 0; i < count; i++)
+  for (i = 0; i < bld->count; i++)
     {
       psize = size;
       size += sizes[perm[i]];
       size = do_align (psize, size);
     }
-  *len = size;
-  *data = GNUNET_malloc (size);
-  memset (*data, 0, size);
-
-  pos = 8;
-  memcpy (*data, GNUNET_DIRECTORY_MAGIC, 8);
+  *rsize = size;
+  data = GNUNET_malloc (size);
+  *rdata = data;
+  memcpy (data, GNUNET_DIRECTORY_MAGIC, 8);
+  off = 8;
 
   ret = GNUNET_CONTAINER_meta_data_serialize (meta,
-                                              &(*data)[pos +
-                                                       sizeof (unsigned int)],
-                                              size - pos - sizeof (unsigned int),
+                                              &(*data)[off +
+                                                       sizeof (uint32_t)],
+                                              size - pos - sizeof (uint32_t),
                                               GNUNET_SERIALIZE_FULL);
-  GNUNET_assert (ret != GNUNET_SYSERR);
-  ret = htonl (ret);
-  memcpy (&(*data)[pos], &ret, sizeof (unsigned int));
-  pos += ntohl (ret) + sizeof (unsigned int);
-
+  GNUNET_assert (ret != -1);
+  big = htonl (ret);  
+  memcpy (&(*data)[8], &big, sizeof (uint32_t));
+  pos += sizeof (uint32_t) + ret;
   for (j = 0; j < count; j++)
     {
       i = perm[j];
       psize = pos;
       pos += sizes[i];
       pos = do_align (psize, pos);
-      pos -= sizes[i];          /* go back to beginning */
-      memcpy (&(*data)[pos], ucs[i], strlen (ucs[i]) + 1);
-      pos += strlen (ucs[i]) + 1;
-      GNUNET_free (ucs[i]);
-      ret = GNUNET_CONTAINER_meta_data_serialize (mds[i],
-                                                  &(*data)[pos +
-                                                           sizeof (unsigned int)],
-                                                  size - pos -
-                                                  sizeof (unsigned int),
-                                                  GNUNET_SERIALIZE_FULL);
-      GNUNET_assert (ret != GNUNET_SYSERR);
-      ret = htonl (ret);
-      memcpy (&(*data)[pos], &ret, sizeof (unsigned int));
-      pos += ntohl (ret) + sizeof (unsigned int);
+      memcpy (&data[pos - sizes[i]], 
+              &(bes[i])[1],
+              sizes[i]);
+      GNUNET_free (bes[i]);
     }
   GNUNET_free (sizes);
   GNUNET_free (perm);
-  GNUNET_free (ucs);
-  GNUNET_assert (pos == size);
-  return GNUNET_OK;
+  GNUNET_free (bes);
+  GNUNET_assert (pos == size);  
+  GNUNET_CONTAINER_meta_data_destroy (bld->meta);
+  GNUNET_free (bld);
 }
 
 
-#endif 
-
 /* end of fs_directory.c */
diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c
index 540b518dc..60bcc2529 100644
--- a/src/fs/fs_file_information.c
+++ b/src/fs/fs_file_information.c
@@ -804,6 +804,7 @@ GNUNET_FS_file_information_destroy (struct GNUNET_FS_FileInformation *fi,
                &fi->priority,
                &fi->expirationTime,
                &fi->client_info);
+      GNUNET_free_non_null (fi->data.dir.dir_data);
       GNUNET_free (fi->data.dir.dirname);
     }
   else
@@ -821,7 +822,7 @@ GNUNET_FS_file_information_destroy (struct GNUNET_FS_FileInformation *fi,
                &fi->expirationTime,
                &fi->client_info);
     }
-
+  GNUNET_free_non_null (fi->chk_tree);
   /* clean up serialization */
   if (0 != UNLINK (fi->serialization))
     GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c
index 7696dd940..0078d0d32 100644
--- a/src/fs/fs_publish.c
+++ b/src/fs/fs_publish.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     (C) 2001, 2002, 2003, 2004, 2005, 2006, 2009 Christian Grothoff (and other contributing authors)
+     (C) 2009 Christian Grothoff (and other contributing authors)
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -24,15 +24,394 @@
  * @see http://gnunet.org/encoding.php3
  * @author Krista Bennett
  * @author Christian Grothoff
+ *
+ * TODO:
+ * - directory creation
+ * - KBlocks
+ * - SBlocks
+ * - indexing support
+ * - calling of progress function
+ * - handling of IO errors (emsg)
+ * - code-sharing with unindex
+ * - datastore reservation support
+ * - persistence support
  */
 
 #include "platform.h"
+#include "gnunet_constants.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_fs_service.h"
 #include "fs.h"
 
 #define DEBUG_PUBLISH GNUNET_YES
 
+/**
+ * Main function that performs the upload.
+ * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
+ * @param tc task context
+ */
+static void
+do_upload (void *cls,
+           const struct GNUNET_SCHEDULER_TaskContext *tc);
+
+
+/**
+ * Context for "ds_put_cont".
+ */
+struct PutContCtx
+{
+  /**
+   * Publishing context for which the datastore
+   * PUT request was executed.
+   */
+  struct GNUNET_FS_PublishContext *sc;
+
+  /**
+   * Specific file with the block.
+   */
+  struct GNUNET_FS_FileInformation *p;
+
+  /**
+   * Function to run next, if any (can be NULL).
+   */
+  GNUNET_SCHEDULER_Task cont;
+};
+
+/**
+ * Function called by the datastore API with
+ * the result from the PUT request.
+ *
+ * @param cls our closure
+ * @param success GNUNET_OK on success
+ * @param msg error message (or NULL)
+ */
+static void
+ds_put_cont (void *cls,
+             int success,
+             const char *msg)
+{
+  struct PutContCtx *pcc = cls;
+
+  if (GNUNET_OK != success)
+    {
+      // FIXME: call progress CB with error
+      // FIXME: update pcc->p to indicate abort
+      GNUNET_FS_file_information_sync (pcc->p);
+      return;
+    }
+  GNUNET_FS_file_information_sync (pcc->p);
+  if (NULL != pcc->cont)
+    pcc->sc->upload_task 
+      = GNUNET_SCHEDULER_add_delayed (pcc->sc->h->sched,
+                                      GNUNET_NO,
+                                      GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
+                                      GNUNET_SCHEDULER_NO_TASK,
+                                      GNUNET_TIME_UNIT_ZERO,
+                                      pcc->cont,
+                                      pcc->sc);
+  GNUNET_free (pcc);
+}
+
+
+/**
+ * We need to publish a specific block.  Do it.  Then continue with
+ * the main task.
+ *
+ * @param sc overall upload data
+ * @param p file that the block belongs to (needed for options!)
+ * @param blk encoded block to publish
+ * @param blk_size size of the block
+ * @param blk_type type of the block
+ * @param cont function to run when done
+ */
+static void
+publish_block (struct GNUNET_FS_PublishContext *sc,
+               struct GNUNET_FS_FileInformation *p,
+               const void* blk,
+               uint16_t blk_size,
+               uint32_t blk_type,
+               GNUNET_SCHEDULER_Task cont)
+{
+  struct GNUNET_HashCode key;
+
+  // FIXME: GNUNET_FS_get_key (blk_type, blk, blk_size, &key);
+  // (or add "key" as argument to reduce hashing?)
+  dpc_cls = GNUNET_malloc(sizeof(struct PutContCtx));
+  dpc_cls->cont = cont;
+  dpc_cls->sc = sc;
+  dpc_cls->p = p;
+  // FIXME: need to do something to "sc" to mark
+  // that "sc" can not be freed right now due to this
+  // pending, scheduled operation for which we don't have
+  // a task ID!  
+  GNUNET_DATASTORE_put (sc->dsh,
+                        sc->rid,
+                        &key,
+                        blk_size,
+                        blk_type,
+                        p->priority,
+                        p->anonymity,
+                        p->expirationTime,
+                        GNUNET_CONSTANTS_SERVICE_TIMEOUT,
+                        &ds_put_cont,
+                        dpc_cls);
+}
+
+
+/**
+ * We are almost done publishing the structure,
+ * add SBlocks (if needed).
+ *
+ * @param sc overall upload data
+ */
+static void
+publish_sblock (struct GNUNET_FS_PublishContext *sc)
+{
+  struct GNUNET_FS_FileInformation *p;
+  p = sc->fi;
+
+  // FIXME: build sblock & call publish_block!
+  
+  // FIXME: continuation should
+  // be releasing the datastore reserve
+  // (once implemented)
+}
+
+
+/**
+ * We have uploaded a file or directory; now publish
+ * the KBlocks in the global keyword space so that
+ * it can be found.  Then continue with the
+ * main task.
+ *
+ * @param sc overall upload data
+ * @param p specific file or directory for which kblocks
+ *          should be created
+ */
+static void
+publish_kblocks (struct GNUNET_FS_PublishContext *sc,
+                 struct GNUNET_FS_FileInformation *p)
+{
+  // FIXME: build all kblocks
+  // call publish_kblock on each
+  // last continuation should then call the main continuation again
+}
+
+
+/**
+ * Compute the depth of the CHK tree.
+ *
+ * @param flen file length for which to compute the depth
+ * @return depth of the tree
+ */
+static unsigned int
+compute_depth (uint64_t flen)
+{
+  unsigned int treeDepth;
+  uint64_t fl;
+
+  treeDepth = 1;
+  fl = GNUNET_FS_DBLOCK_SIZE;
+  while (fl < flen)
+    {
+      treeDepth++;
+      if (fl * GNUNET_FS_CHK_PER_INODE < fl)
+        {
+          /* integer overflow, this is a HUGE file... */
+          return treeDepth;
+        }
+      fl = fl * GNUNET_FS_CHK_PER_INODE;
+    }
+  return treeDepth;
+}
+
+
+/**
+ * Compute the size of the current IBlock.
+ *
+ * @param height height of the IBlock in the tree (aka overall
+ *               number of tree levels minus depth); 0 == DBlock
+ * @param offset current offset in the overall file
+ * @return size of the corresponding IBlock
+ */
+static uint16_t 
+compute_iblock_size (unsigned int height,
+                     uint64_t offset)
+{
+  unsigned int ret;
+  unsigned int i;
+  uint64_t mod;
+  uint64_t bds;
+
+  GNUNET_assert (height > 0);
+  bds = GNUNET_FS_DBLOCK_SIZE; /* number of bytes each CHK at level "i"
+                                  corresponds to */
+  for (i=0;i<height;i++)
+    bds *= GNUNET_FS_CHK_PER_INODE;
+  mod = offset % bds;
+  if (0 == mod)
+    {
+      /* we were triggered at the end of a full block */
+      ret = GNUNET_FS_CHK_PER_INODE;
+    }
+  else
+    {
+      /* we were triggered at the end of the file */
+      bds /= GNUNET_FS_CHK_PER_INODE;
+      ret = mod / bds;
+      if (0 != mod % bds)
+        ret++; 
+    }
+  return (uint16_t) (ret * sizeof(struct ContentHashKey));
+}
+
+
+/**
+ * Compute the offset of the CHK for the
+ * current block in the IBlock above.
+ *
+ * @param height height of the IBlock in the tree (aka overall
+ *               number of tree levels minus depth); 0 == DBlock
+ * @param offset current offset in the overall file
+ * @return (array of CHKs') offset in the above IBlock
+ */
+static unsigned int
+compute_chk_offset (unsigned int height,
+                    uint64_t offset)
+{
+  uint64_t bds;
+  unsigned  int ret;
+
+  bds = GNUNET_FS_DBLOCK_SIZE; /* number of bytes each CHK at level "i"
+                                  corresponds to */
+  for (i=0;i<height;i++)
+    bds *= GNUNET_FS_CHK_PER_INODE;
+  GNUNET_assert (0 == (offset % bds));
+  ret = offset / bds;
+  return ret % GNUNET_FS_CHK_PER_INODE; 
+}
+
+
+/**
+ * We are uploading a file or directory; load (if necessary) the next
+ * block into memory, encrypt it and send it to the FS service.  Then
+ * continue with the main task.
+ *
+ * @param sc overall upload data
+ * @param p specific file or directory for which kblocks
+ *          should be created
+ */
+static void
+publish_content (struct GNUNET_FS_PublishContext *sc,
+                 struct GNUNET_FS_FileInformation *p)
+{
+  struct ContentHashKey *chk;
+  const void *pt_block;
+  uint16_t pt_size;
+  char *emsg;
+  char iob[GNUNET_FS_DBLOCK_SIZE];
+  char enc[GNUNET_FS_DBLOCK_SIZE];
+  struct GNUNET_CRYPTO_AesSessionKey sk;
+  struct GNUNET_CRYPTO_AesInitializationVector iv;
+  uint64_t size;
+  unsigned int off;
+
+  // FIXME: figure out how to share this code
+  // with unindex!
+  size = (p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
+  if (NULL == p->chk_tree)
+    {
+      if (p->is_directory)
+        {
+          /* FIXME: create function to create directory
+             and use that API here! */
+          GNUNET_FS_directory_create (&p->data.dir.dir_size,
+                                      &p->data.dir.dir_data,
+                                      p->meta,
+                                      &directory_entry_lister,
+                                      p->data.dir.entries);
+          size = p->data.dir.data_size;
+        }
+      p->chk_tree_depth = compute_depth (size);
+      p->chk_tree = GNUNET_malloc (p->chk_tree_depth * 
+                                   sizeof (struct ContentHashKey) *
+                                   GNUNET_FS_CHK_PER_INODE);
+      p->current_depth = p->chk_tree_depth;
+    }
+  if (p->current_depth == p->chk_tree_depth)
+    {
+      if (p->is_directory)
+        {
+          pt_size = GNUNET_MIN(GNUNET_FS_DBLOCK_SIZE,
+                               p->data.dir.dir_size - p->publish_offset);
+          pt_block = &p->data.dir.dir_data[p->publish_offset];
+        }
+      else
+        {
+          pt_size = GNUNET_MIN(GNUNET_FS_DBLOCK_SIZE,
+                               p->data.file.file_size - p->publish_offset);
+          p->data.file.reader (p->data.file.reader_cls,
+                               p->publish_offset,
+                               pt_size,
+                               iob,
+                               &emsg);
+          pt_block = iob;
+        }
+    }
+  else
+    {
+      pt_size = compute_iblock_size (p->chk_tree_depth - p->current_depth,
+                                     p->publish_offset); 
+      pt_block = &p->chk_tree[p->current_depth *
+                              GNUNET_FS_CHK_PER_INODE];
+    }
+  off = compute_chk_offset (p->chk_tree_depth - p->current_depth,
+                            p->publish_offset);
+  chk = &p->chk_tree[(p->current_depth-1)*GNUNET_FS_CHK_PER_INODE+off];
+  GNUNET_CRYPTO_hash (pt_block, pt_size, &chk->key);
+  GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv);
+  GNUNET_CRYPTO_aes_encrypt (pt_block,
+                             pt_size,
+                             &sk,
+                             &iv,
+                             enc);
+  // NOTE: this call (and progress below) is all that really differs
+  // between publish/unindex!  Parameterize & move this code!
+  // FIXME: something around here would need to change
+  // for indexing!
+  publish_block (sc, p, enc, pt_size, 
+                 (p->current_depth == p->chk_tree_depth) 
+                 ? GNUNET_DATASTORE_BLOCKTYPE_DBLOCK 
+                 : GNUNET_DATASTORE_BLOCKTYPE_IBLOCK,
+                 &do_upload);
+  // FIXME: should call progress function somewhere here!
+  GNUNET_CRYPTO_hash (enc, pt_size, &chk->query);
+  if (p->current_depth == p->chk_tree_depth) 
+    { 
+      p->publish_offset += pt_size;
+      if ( (p->publish_offset == size) ||
+           (0 == p->publish_offset % (GNUNET_FS_CHK_PER_INODE * GNUNET_FS_DBLOCK_SIZE) ) )
+        p->current_depth--;
+    }
+  else
+    {
+      if ( (off == GNUNET_FS_CHK_PER_INODE) ||
+           (p->publish_offset == size) )
+        p->current_depth--;
+      else
+        p->current_depth = p->chk_tree_depth;
+    }
+  if (0 == p->current_depth)
+    {
+      p->chk_uri = GNUNET_malloc (sizeof(struct GNUNET_FS_Uri));
+      p->chk_uri.type = chk;
+      p->chk_uri.data.chk.chk = p->chk_tree[0];
+      p->chk_uri.data.chk.file_length = size;
+      GNUNET_free (p->chk_tree);
+      p->chk_tree = NULL;
+    }
+}
+
 
 /**
  * Main function that performs the upload.
@@ -44,11 +423,36 @@ do_upload (void *cls,
            const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct GNUNET_FS_PublishContext *sc = cls;
+  struct GNUNET_FS_FileInformation *p;
 
   sc->upload_task = GNUNET_SCHEDULER_NO_TASK;  
-
-  // FIXME: find next block, process, schedule
-  // transmission to FS service
+  p = sc->fi_pos;
+  if (NULL == p)
+    {
+      /* upload of entire hierarchy complete,
+         publish namespace entries */
+      publish_sblock (sc);
+      return;
+    }
+  if (NULL != p->chk_uri)
+    {
+      /* move on to next file */
+      if (NULL != p->next)
+        sc->fi_pos = p->next;
+      else
+        sc->fi_pos = p->dir;
+      /* upload of "p" complete, publish KBlocks! */
+      publish_kblocks (sc, p);
+      return;
+    }
+  if (p->do_index)
+    {
+      // FIXME: need to pre-compute hash over
+      // the entire file and ask FS to prepare
+      // for indexing!
+      return;
+    }
+  publish_content (sc, p);
 }
 
 
@@ -75,8 +479,15 @@ GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h,
                          const char *nuid)
 {
   struct GNUNET_FS_PublishContext *ret;
+  struct GNUNET_FS_FileInformation *p;
+  struct GNUNET_DATASTORE_Handle *dsh;
 
+  dsh = GNUNET_DATASTORE_connect (h->cfg,
+                                  h->sched);
+  if (NULL == dsh)
+    return NULL;
   ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext));
+  ret->dsh = dsh;
   ret->h = h;
   ret->client_ctx = ctx;
   ret->fi = fi;
@@ -90,6 +501,17 @@ GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h,
         ret->nuid = GNUNET_strdup (nuid);
     }
   // FIXME: make upload persistent!
+
+  /* find first leaf, DFS */
+  p = ret->fi;
+  while ( (p->is_directory) &&
+          (NULL != p->data.dir.entries) )
+    p = p->data.dir.entries;          
+  ret->fi_pos = p;
+
+  // FIXME: calculate space needed for "fi"
+  // and reserve as first task (then trigger
+  // "do_upload" from that continuation)!
   ret->upload_task 
     = GNUNET_SCHEDULER_add_delayed (h->sched,
                                     GNUNET_NO,
@@ -120,429 +542,8 @@ GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *sc)
   GNUNET_FS_namespace_delete (sc->namespace, GNUNET_NO);
   GNUNET_free_non_null (sc->nid);  
   GNUNET_free_non_null (sc->nuid);
+  GNUNET_DATASTORE_disconnect (sc->dsh);
   GNUNET_free (sc);
 }
 
-
-#if 0
-
-/**
- * Append the given key and query to the iblock[level].  If
- * iblock[level] is already full, compute its chk and push it to
- * level+1 and clear the level.  iblocks is guaranteed to be big
- * enough.
- */
-static int
-pushBlock (struct GNUNET_ClientServerConnection *sock,
-           const GNUNET_EC_ContentHashKey * chk,
-           unsigned int level,
-           GNUNET_DatastoreValue ** iblocks,
-           unsigned int prio, GNUNET_CronTime expirationTime)
-{
-  unsigned int size;
-  unsigned int present;
-  GNUNET_DatastoreValue *value;
-  GNUNET_EC_DBlock *db;
-  GNUNET_EC_ContentHashKey ichk;
-
-  size = ntohl (iblocks[level]->size);
-  GNUNET_GE_ASSERT (NULL, size > sizeof (GNUNET_DatastoreValue));
-  size -= sizeof (GNUNET_DatastoreValue);
-  GNUNET_GE_ASSERT (NULL,
-                    size - sizeof (GNUNET_EC_DBlock) <=
-                    GNUNET_ECRS_IBLOCK_SIZE);
-  present =
-    (size - sizeof (GNUNET_EC_DBlock)) / sizeof (GNUNET_EC_ContentHashKey);
-  db = (GNUNET_EC_DBlock *) & iblocks[level][1];
-  if (present == GNUNET_ECRS_CHK_PER_INODE)
-    {
-      GNUNET_EC_file_block_get_key (db, size, &ichk.key);
-      GNUNET_EC_file_block_get_query (db, size, &ichk.query);
-      if (GNUNET_OK != pushBlock (sock,
-                                  &ichk, level + 1, iblocks, prio,
-                                  expirationTime))
-        return GNUNET_SYSERR;
-      GNUNET_EC_file_block_encode (db, size, &ichk.query, &value);
-      if (value == NULL)
-        {
-          GNUNET_GE_BREAK (NULL, 0);
-          return GNUNET_SYSERR;
-        }
-      value->priority = htonl (prio);
-      value->expiration_time = GNUNET_htonll (expirationTime);
-      if (GNUNET_OK != GNUNET_FS_insert (sock, value))
-        {
-          GNUNET_free (value);
-          return GNUNET_SYSERR;
-        }
-      GNUNET_free (value);
-      size = sizeof (GNUNET_EC_DBlock); /* type */
-    }
-  /* append GNUNET_EC_ContentHashKey */
-  memcpy (&((char *) db)[size], chk, sizeof (GNUNET_EC_ContentHashKey));
-  size += sizeof (GNUNET_EC_ContentHashKey) + sizeof (GNUNET_DatastoreValue);
-  GNUNET_GE_ASSERT (NULL, size < GNUNET_MAX_BUFFER_SIZE);
-  iblocks[level]->size = htonl (size);
-
-  return GNUNET_OK;
-}
-
-/**
- * Index or insert a file.
- *
- * @param priority what is the priority for OUR node to
- *   keep this file available?  Use 0 for maximum anonymity and
- *   minimum reliability...
- * @param doIndex GNUNET_YES for index, GNUNET_NO for insertion,
- *         GNUNET_SYSERR for simulation
- * @param uri set to the URI of the uploaded file
- * @return GNUNET_SYSERR if the upload failed (i.e. not enough space
- *  or gnunetd not running)
- */
-int
-GNUNET_ECRS_file_upload (struct GNUNET_GE_Context *ectx,
-                         struct GNUNET_GC_Configuration *cfg,
-                         const char *filename,
-                         int doIndex,
-                         unsigned int anonymityLevel,
-                         unsigned int priority,
-                         GNUNET_CronTime expirationTime,
-                         GNUNET_ECRS_UploadProgressCallback upcb,
-                         void *upcbClosure,
-                         GNUNET_ECRS_TestTerminate tt,
-                         void *ttClosure, struct GNUNET_ECRS_URI **uri)
-{
-  unsigned long long filesize;
-  unsigned long long pos;
-  unsigned int treedepth;
-  int fd;
-  int i;
-  int ret;
-  unsigned int size;
-  GNUNET_DatastoreValue **iblocks;
-  GNUNET_DatastoreValue *dblock;
-  GNUNET_EC_DBlock *db;
-  GNUNET_DatastoreValue *value;
-  struct GNUNET_ClientServerConnection *sock;
-  GNUNET_HashCode fileId;
-  GNUNET_EC_ContentHashKey mchk;
-  GNUNET_CronTime eta;
-  GNUNET_CronTime start;
-  GNUNET_CronTime now;
-  GNUNET_EC_FileIdentifier fid;
-#if DEBUG_UPLOAD
-  GNUNET_EncName enc;
-#endif
-
-  GNUNET_GE_ASSERT (ectx, cfg != NULL);
-  start = GNUNET_get_time ();
-  memset (&mchk, 0, sizeof (GNUNET_EC_ContentHashKey));
-  if (GNUNET_YES != GNUNET_disk_file_test (ectx, filename))
-    {
-      GNUNET_GE_LOG (ectx,
-                     GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
-                     _("`%s' is not a file.\n"), filename);
-      return GNUNET_SYSERR;
-    }
-  if (GNUNET_OK !=
-      GNUNET_disk_file_size (ectx, filename, &filesize, GNUNET_YES))
-    {
-      GNUNET_GE_LOG (ectx,
-                     GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
-                     _("Cannot get size of file `%s'"), filename);
-
-      return GNUNET_SYSERR;
-    }
-  sock = GNUNET_client_connection_create (ectx, cfg);
-  if (sock == NULL)
-    {
-      GNUNET_GE_LOG (ectx,
-                     GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
-                     _("Failed to connect to gnunetd."));
-      return GNUNET_SYSERR;
-    }
-  eta = 0;
-  if (upcb != NULL)
-    upcb (filesize, 0, eta, upcbClosure);
-  if (doIndex == GNUNET_YES)
-    {
-      if (GNUNET_SYSERR == GNUNET_hash_file (ectx, filename, &fileId))
-        {
-          GNUNET_GE_LOG (ectx,
-                         GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
-                         _("Cannot hash `%s'.\n"), filename);
-
-          GNUNET_client_connection_destroy (sock);
-          return GNUNET_SYSERR;
-        }
-      if (GNUNET_YES == GNUNET_FS_test_indexed (sock, &fileId))
-        {
-          /* file already indexed; simulate only to get the URI! */
-          doIndex = GNUNET_SYSERR;
-        }
-    }
-  if (doIndex == GNUNET_YES)
-    {
-      now = GNUNET_get_time ();
-      eta = now + 2 * (now - start);
-      /* very rough estimate: GNUNET_hash reads once through the file,
-         we'll do that once more and write it.  But of course
-         the second read may be cached, and we have the encryption,
-         so a factor of two is really, really just a rough estimate */
-      start = now;
-      /* reset the counter since the formula later does not
-         take the time for GNUNET_hash_file into account */
-
-      switch (GNUNET_FS_prepare_to_index (sock, &fileId, filename))
-        {
-        case GNUNET_SYSERR:
-          GNUNET_GE_LOG (ectx,
-                         GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
-                         _("Initialization for indexing file `%s' failed.\n"),
-                         filename);
-          GNUNET_client_connection_destroy (sock);
-          return GNUNET_SYSERR;
-        case GNUNET_NO:
-          GNUNET_GE_LOG (ectx,
-                         GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
-                         _
-                         ("Indexing file `%s' failed. Suggestion: try to insert the file.\n"),
-                         filename);
-          GNUNET_client_connection_destroy (sock);
-          return GNUNET_SYSERR;
-        default:
-          break;
-        }
-    }
-  treedepth = GNUNET_ECRS_compute_depth (filesize);
-  fd = GNUNET_disk_file_open (ectx, filename, O_RDONLY | O_LARGEFILE);
-  if (fd == -1)
-    {
-      GNUNET_GE_LOG (ectx,
-                     GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
-                     _("Cannot open file `%s': `%s'"), filename,
-                     STRERROR (errno));
-
-      GNUNET_client_connection_destroy (sock);
-      return GNUNET_SYSERR;
-    }
-
-  dblock =
-    GNUNET_malloc (sizeof (GNUNET_DatastoreValue) + GNUNET_ECRS_DBLOCK_SIZE +
-                   sizeof (GNUNET_EC_DBlock));
-  dblock->size =
-    htonl (sizeof (GNUNET_DatastoreValue) + GNUNET_ECRS_DBLOCK_SIZE +
-           sizeof (GNUNET_EC_DBlock));
-  dblock->anonymity_level = htonl (anonymityLevel);
-  dblock->priority = htonl (priority);
-  dblock->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA);
-  dblock->expiration_time = GNUNET_htonll (expirationTime);
-  db = (GNUNET_EC_DBlock *) & dblock[1];
-  db->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA);
-  iblocks =
-    GNUNET_malloc (sizeof (GNUNET_DatastoreValue *) * (treedepth + 1));
-  for (i = 0; i <= treedepth; i++)
-    {
-      iblocks[i] =
-        GNUNET_malloc (sizeof (GNUNET_DatastoreValue) +
-                       GNUNET_ECRS_IBLOCK_SIZE + sizeof (GNUNET_EC_DBlock));
-      iblocks[i]->size =
-        htonl (sizeof (GNUNET_DatastoreValue) + sizeof (GNUNET_EC_DBlock));
-      iblocks[i]->anonymity_level = htonl (anonymityLevel);
-      iblocks[i]->priority = htonl (priority);
-      iblocks[i]->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA);
-      iblocks[i]->expiration_time = GNUNET_htonll (expirationTime);
-      ((GNUNET_EC_DBlock *) & iblocks[i][1])->type =
-        htonl (GNUNET_ECRS_BLOCKTYPE_DATA);
-    }
-
-  pos = 0;
-  while (pos < filesize)
-    {
-      if (upcb != NULL)
-        upcb (filesize, pos, eta, upcbClosure);
-      if (tt != NULL)
-        if (GNUNET_OK != tt (ttClosure))
-          goto FAILURE;
-      size = GNUNET_ECRS_DBLOCK_SIZE;
-      if (size > filesize - pos)
-        {
-          size = filesize - pos;
-          memset (&db[1], 0, GNUNET_ECRS_DBLOCK_SIZE);
-        }
-      GNUNET_GE_ASSERT (ectx,
-                        sizeof (GNUNET_DatastoreValue) + size +
-                        sizeof (GNUNET_EC_DBlock) < GNUNET_MAX_BUFFER_SIZE);
-      dblock->size =
-        htonl (sizeof (GNUNET_DatastoreValue) + size +
-               sizeof (GNUNET_EC_DBlock));
-      if (size != READ (fd, &db[1], size))
-        {
-          GNUNET_GE_LOG_STRERROR_FILE (ectx,
-                                       GNUNET_GE_ERROR | GNUNET_GE_BULK |
-                                       GNUNET_GE_ADMIN | GNUNET_GE_USER,
-                                       "READ", filename);
-          goto FAILURE;
-        }
-      if (tt != NULL)
-        if (GNUNET_OK != tt (ttClosure))
-          goto FAILURE;
-      GNUNET_EC_file_block_get_key (db, size + sizeof (GNUNET_EC_DBlock),
-                                    &mchk.key);
-      GNUNET_EC_file_block_get_query (db, size + sizeof (GNUNET_EC_DBlock),
-                                      &mchk.query);
-#if DEBUG_UPLOAD
-      GNUNET_hash_to_enc (&mchk.query, &enc);
-      fprintf (stderr,
-               "Query for current block of size %u is `%s'\n", size,
-               (const char *) &enc);
-#endif
-      if (doIndex == GNUNET_YES)
-        {
-          if (GNUNET_SYSERR == GNUNET_FS_index (sock, &fileId, dblock, pos))
-            {
-              GNUNET_GE_LOG (ectx,
-                             GNUNET_GE_ERROR | GNUNET_GE_BULK |
-                             GNUNET_GE_USER,
-                             _
-                             ("Indexing data of file `%s' failed at position %llu.\n"),
-                             filename, pos);
-              goto FAILURE;
-            }
-        }
-      else
-        {
-          value = NULL;
-          if (GNUNET_OK !=
-              GNUNET_EC_file_block_encode (db,
-                                           size + sizeof (GNUNET_EC_DBlock),
-                                           &mchk.query, &value))
-            {
-              GNUNET_GE_BREAK (ectx, 0);
-              goto FAILURE;
-            }
-          GNUNET_GE_ASSERT (ectx, value != NULL);
-          *value = *dblock;     /* copy options! */
-          if ((doIndex == GNUNET_NO) &&
-              (GNUNET_OK != (ret = GNUNET_FS_insert (sock, value))))
-            {
-              GNUNET_GE_BREAK (ectx, ret == GNUNET_NO);
-              GNUNET_free (value);
-              goto FAILURE;
-            }
-          GNUNET_free (value);
-        }
-      pos += size;
-      now = GNUNET_get_time ();
-      if (pos > 0)
-        {
-          eta = (GNUNET_CronTime) (start +
-                                   (((double) (now - start) / (double) pos))
-                                   * (double) filesize);
-        }
-      if (GNUNET_OK != pushBlock (sock, &mchk, 0,       /* dblocks are on level 0 */
-                                  iblocks, priority, expirationTime))
-        goto FAILURE;
-    }
-  if (tt != NULL)
-    if (GNUNET_OK != tt (ttClosure))
-      goto FAILURE;
-#if DEBUG_UPLOAD
-  GNUNET_GE_LOG (ectx,
-                 GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
-                 "Tree depth is %u, walking up tree.\n", treedepth);
-#endif
-  for (i = 0; i < treedepth; i++)
-    {
-      size = ntohl (iblocks[i]->size) - sizeof (GNUNET_DatastoreValue);
-      GNUNET_GE_ASSERT (ectx, size < GNUNET_MAX_BUFFER_SIZE);
-      if (size == sizeof (GNUNET_EC_DBlock))
-        {
-#if DEBUG_UPLOAD
-          GNUNET_GE_LOG (ectx,
-                         GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
-                         "Level %u is empty\n", i);
-#endif
-          continue;
-        }
-      db = (GNUNET_EC_DBlock *) & iblocks[i][1];
-      GNUNET_EC_file_block_get_key (db, size, &mchk.key);
-#if DEBUG_UPLOAD
-      GNUNET_GE_LOG (ectx,
-                     GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
-                     "Computing query for %u bytes content.\n", size);
-#endif
-      GNUNET_EC_file_block_get_query (db, size, &mchk.query);
-#if DEBUG_UPLOAD
-      IF_GELOG (ectx,
-                GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
-                GNUNET_hash_to_enc (&mchk.query, &enc));
-      GNUNET_GE_LOG (ectx,
-                     GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
-                     "Query for current block at level %u is `%s'.\n", i,
-                     &enc);
-#endif
-      if (GNUNET_OK != pushBlock (sock,
-                                  &mchk, i + 1, iblocks, priority,
-                                  expirationTime))
-        {
-          GNUNET_GE_BREAK (ectx, 0);
-          goto FAILURE;
-        }
-      GNUNET_EC_file_block_encode (db, size, &mchk.query, &value);
-      if (value == NULL)
-        {
-          GNUNET_GE_BREAK (ectx, 0);
-          goto FAILURE;
-        }
-      value->expiration_time = GNUNET_htonll (expirationTime);
-      value->priority = htonl (priority);
-      if ((doIndex != GNUNET_SYSERR) &&
-          (GNUNET_SYSERR == GNUNET_FS_insert (sock, value)))
-        {
-          GNUNET_GE_BREAK (ectx, 0);
-          GNUNET_free (value);
-          goto FAILURE;
-        }
-      GNUNET_free (value);
-      GNUNET_free (iblocks[i]);
-      iblocks[i] = NULL;
-    }
-#if DEBUG_UPLOAD
-  IF_GELOG (ectx,
-            GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
-            GNUNET_hash_to_enc (&mchk.query, &enc));
-  GNUNET_GE_LOG (ectx, GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
-                 "Query for top block is %s\n", &enc);
-#endif
-  /* build URI */
-  fid.file_length = GNUNET_htonll (filesize);
-  db = (GNUNET_EC_DBlock *) & iblocks[treedepth][1];
-
-  fid.chk = *(GNUNET_EC_ContentHashKey *) & (db[1]);
-  *uri = GNUNET_malloc (sizeof (URI));
-  (*uri)->type = chk;
-  (*uri)->data.fi = fid;
-
-  /* free resources */
-  GNUNET_free_non_null (iblocks[treedepth]);
-  GNUNET_free (iblocks);
-  GNUNET_free (dblock);
-  if (upcb != NULL)
-    upcb (filesize, filesize, eta, upcbClosure);
-  CLOSE (fd);
-  GNUNET_client_connection_destroy (sock);
-  return GNUNET_OK;
-FAILURE:
-  for (i = 0; i <= treedepth; i++)
-    GNUNET_free_non_null (iblocks[i]);
-  GNUNET_free (iblocks);
-  GNUNET_free (dblock);
-  CLOSE (fd);
-  GNUNET_client_connection_destroy (sock);
-  return GNUNET_SYSERR;
-}
-
-#endif 
-
 /* end of fs_publish.c */
diff --git a/src/include/gnunet_constants.h b/src/include/gnunet_constants.h
index 2c917cc59..bb122eb64 100644
--- a/src/include/gnunet_constants.h
+++ b/src/include/gnunet_constants.h
@@ -55,6 +55,13 @@ extern "C"
  */
 #define GNUNET_CONSTANTS_EXEC_WAIT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 100)
 
+/**
+ * After how long do we consider a service irresponsive
+ * even if we assume that the service commonly does not
+ * respond instantly (DNS, Database, etc.).
+ */
+#define GNUNET_CONSTANTS_SERVICE_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 10)
+
 
 #if 0                           /* keep Emacsens' auto-indent happy */
 {
diff --git a/src/include/gnunet_container_lib.h b/src/include/gnunet_container_lib.h
index 291b99fa2..e2a0787c8 100644
--- a/src/include/gnunet_container_lib.h
+++ b/src/include/gnunet_container_lib.h
@@ -305,8 +305,8 @@ int GNUNET_CONTAINER_meta_data_extract_from_file (struct
 
 enum GNUNET_CONTAINER_MetaDataSerializationOptions
 {
-  GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL = GNUNET_NO,
-  GNUNET_CONTAINER_META_DATA_SERIALIZE_PART = GNUNET_YES,
+  GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL = 0,
+  GNUNET_CONTAINER_META_DATA_SERIALIZE_PART = 1,
   GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS = 2
 };
 
@@ -323,10 +323,11 @@ enum GNUNET_CONTAINER_MetaDataSerializationOptions
  *         GNUNET_SYSERR on error (typically: not enough
  *         space)
  */
-int GNUNET_CONTAINER_meta_data_serialize (const struct
-                                          GNUNET_CONTAINER_MetaData *md,
-                                          char *target, unsigned int size,
-                                          enum
+ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct
+                                              GNUNET_CONTAINER_MetaData *md,
+                                              char *target, 
+                                              size_t size,
+                                              enum
                                           GNUNET_CONTAINER_MetaDataSerializationOptions
                                           opt);
 
@@ -337,12 +338,12 @@ int GNUNET_CONTAINER_meta_data_serialize (const struct
  *        meta-data to match the size constraint,
  *        possibly discarding some data?
  */
-unsigned int GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
-                                                             GNUNET_CONTAINER_MetaData
-                                                             *md,
-                                                             enum
-                                                             GNUNET_CONTAINER_MetaDataSerializationOptions
-                                                             opt);
+ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
+                                                        GNUNET_CONTAINER_MetaData
+                                                        *md,
+                                                        enum
+                                                        GNUNET_CONTAINER_MetaDataSerializationOptions
+                                                        opt);
 
 /**
  * Deserialize meta-data.  Initializes md.
@@ -352,7 +353,7 @@ unsigned int GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
  */
 struct GNUNET_CONTAINER_MetaData
   *GNUNET_CONTAINER_meta_data_deserialize (const char *input,
-                                           unsigned int size);
+                                           size_t size);
 
 /**
  * Does the meta-data claim that this is a directory?
diff --git a/src/include/gnunet_crypto_lib.h b/src/include/gnunet_crypto_lib.h
index 361d244e2..168c79b2a 100644
--- a/src/include/gnunet_crypto_lib.h
+++ b/src/include/gnunet_crypto_lib.h
@@ -394,7 +394,7 @@ void GNUNET_CRYPTO_hash_xor (const GNUNET_HashCode * a,
 /**
  * Convert a hashcode into a key.
  */
-void GNUNET_CRYPTO_hash_to_AES_key (const GNUNET_HashCode * hc,
+void GNUNET_CRYPTO_hash_to_aes_key (const GNUNET_HashCode * hc,
                                     struct GNUNET_CRYPTO_AesSessionKey *skey,
                                     struct
                                     GNUNET_CRYPTO_AesInitializationVector
diff --git a/src/include/gnunet_datastore_service.h b/src/include/gnunet_datastore_service.h
index 28419126f..3805dea97 100644
--- a/src/include/gnunet_datastore_service.h
+++ b/src/include/gnunet_datastore_service.h
@@ -41,6 +41,12 @@ extern "C"
 #endif
 #endif
 
+#define GNUNET_DATASTORE_BLOCKTYPE_ANY 0
+#define GNUNET_DATASTORE_BLOCKTYPE_DBLOCK 1
+#define GNUNET_DATASTORE_BLOCKTYPE_IBLOCK 2
+#define GNUNET_DATASTORE_BLOCKTYPE_KBLOCK 3
+#define GNUNET_DATASTORE_BLOCKTYPE_SBLOCK 4
+#define GNUNET_DATASTORE_BLOCKTYPE_SKBLOCK 5
 
 /**
  * Handle to the datastore service.
diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h
index 4664c4cee..3c71d40e7 100644
--- a/src/include/gnunet_fs_service.h
+++ b/src/include/gnunet_fs_service.h
@@ -2169,6 +2169,52 @@ GNUNET_FS_directory_list_contents (size_t size,
                                    void *dep_cls);
 
 
+/**
+ * Opaque handle to a directory builder.
+ */
+struct GNUNET_FS_DirectoryBuilder;
+
+/**
+ * Create a directory builder.
+ * 
+ * @param mdir metadata for the directory
+ */
+struct GNUNET_FS_DirectoryBuilder *
+GNUNET_FS_directory_builder_create (const struct GNUNET_CONTAINER_MetaData *mdir);
+
+
+/**
+ * Add an entry to a directory.
+ * 
+ * @param bld directory to extend
+ * @param uri uri of the entry (must not be a KSK)
+ * @param md metadata of the entry
+ * @param data raw data of the entry, can be NULL, otherwise
+ *        data must point to exactly the number of bytes specified
+ *        by the uri
+ */
+void
+GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld,
+                                 const struct GNUNET_FS_Uri *uri,
+                                 const struct GNUNET_CONTAINER_MetaData *md,
+                                 const void *data);
+                 
+
+/**
+ * Finish building the directory.  Frees the
+ * builder context and returns the directory
+ * in-memory.
+ *
+ * @param bld directory to finish
+ * @param size set to the number of bytes needed
+ * @param data set to the encoded directory
+ */
+void
+GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
+                                    size_t *size,
+                                    void **data);
+
+
 #if 0                           /* keep Emacsens' auto-indent happy */
 {
 #endif
diff --git a/src/util/container_meta_data.c b/src/util/container_meta_data.c
index 6265fc142..d80c5fd2b 100644
--- a/src/util/container_meta_data.c
+++ b/src/util/container_meta_data.c
@@ -442,9 +442,9 @@ struct MetaDataHeader
  *         GNUNET_SYSERR on error (typically: not enough
  *         space)
  */
-int
+ssize_t
 GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData
-                                      *md, char *target, unsigned int max,
+                                      *md, char *target, size_t max,
                                       enum
                                       GNUNET_CONTAINER_MetaDataSerializationOptions
                                       part)
@@ -463,7 +463,7 @@ GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData
   while (1)
     {
       size = sizeof (struct MetaDataHeader);
-      size += sizeof (unsigned int) * ic;
+      size += sizeof (uint32_t) * ic;
       for (i = 0; i < ic; i++)
         size += 1 + strlen (md->items[i].data);
       while (size % 8 != 0)
@@ -472,8 +472,8 @@ GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData
       hdr->version = htonl (md == NULL ? 1 : 0);
       hdr->entries = htonl (ic);
       for (i = 0; i < ic; i++)
-        ((unsigned int *) &hdr[1])[i] =
-          htonl ((unsigned int) md->items[i].type);
+        ((uint32_t *) &hdr[1])[i] =
+          htonl ((uint32_t) md->items[i].type);
       pos = sizeof (struct MetaDataHeader);
       pos += sizeof (unsigned int) * ic;
       for (i = 0; i < ic; i++)
@@ -533,7 +533,7 @@ GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData
  * serialized form.  The estimate MAY be higher
  * than what is strictly needed.
  */
-unsigned int
+ssize_t
 GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
                                                 GNUNET_CONTAINER_MetaData *md,
                                                 enum
@@ -549,7 +549,7 @@ GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
 
   ic = md ? md->itemCount : 0;
   size = sizeof (struct MetaDataHeader);
-  size += sizeof (unsigned int) * ic;
+  size += sizeof (uint32_t) * ic;
   for (i = 0; i < ic; i++)
     size += 1 + strlen (md->items[i].data);
   while (size % 8 != 0)
@@ -558,9 +558,9 @@ GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
   hdr->version = htonl (md == NULL ? 1 : 0);
   hdr->entries = htonl (ic);
   for (i = 0; i < ic; i++)
-    ((unsigned int *) &hdr[1])[i] = htonl ((unsigned int) md->items[i].type);
+    ((uint32_t *) &hdr[1])[i] = htonl ((uint32_t) md->items[i].type);
   pos = sizeof (struct MetaDataHeader);
-  pos += sizeof (unsigned int) * ic;
+  pos += sizeof (uint32_t) * ic;
   for (i = 0; i < ic; i++)
     {
       len = strlen (md->items[i].data) + 1;
@@ -590,7 +590,7 @@ GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
  *         bad format)
  */
 struct GNUNET_CONTAINER_MetaData *
-GNUNET_CONTAINER_meta_data_deserialize (const char *input, unsigned int size)
+GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size)
 {
   struct GNUNET_CONTAINER_MetaData *md;
   const struct MetaDataHeader *hdr;
@@ -599,9 +599,9 @@ GNUNET_CONTAINER_meta_data_deserialize (const char *input, unsigned int size)
   const char *cdata;
   uint32_t dataSize;
   int compressed;
-  int i;
-  unsigned int pos;
-  int len;
+  uint32_t i;
+  size_t pos;
+  size_t len;
   uint32_t version;
 
   if (size < sizeof (struct MetaDataHeader))
@@ -651,7 +651,7 @@ GNUNET_CONTAINER_meta_data_deserialize (const char *input, unsigned int size)
         }
     }
 
-  if ((sizeof (unsigned int) * ic + ic) > dataSize)
+  if ((sizeof (uint32_t) * ic + ic) > dataSize)
     {
       GNUNET_break (0);
       goto FAILURE;
@@ -665,12 +665,12 @@ GNUNET_CONTAINER_meta_data_deserialize (const char *input, unsigned int size)
   md = GNUNET_CONTAINER_meta_data_create ();
   GNUNET_array_grow (md->items, md->itemCount, ic);
   i = 0;
-  pos = sizeof (unsigned int) * ic;
+  pos = sizeof (uint32_t) * ic;
   while ((pos < dataSize) && (i < ic))
     {
       len = strlen (&cdata[pos]) + 1;
       md->items[i].type = (EXTRACTOR_KeywordType)
-        ntohl (MAKE_UNALIGNED (((const unsigned int *) cdata)[i]));
+        ntohl (MAKE_UNALIGNED (((const uint32_t *) cdata)[i]));
       md->items[i].data = GNUNET_strdup (&cdata[pos]);
       pos += len;
       i++;
diff --git a/src/util/crypto_hash.c b/src/util/crypto_hash.c
index fc4d24320..a8843b5b7 100644
--- a/src/util/crypto_hash.c
+++ b/src/util/crypto_hash.c
@@ -709,7 +709,7 @@ GNUNET_CRYPTO_hash_xor (const GNUNET_HashCode * a,
  * Convert a hashcode into a key.
  */
 void
-GNUNET_CRYPTO_hash_to_AES_key (const GNUNET_HashCode * hc,
+GNUNET_CRYPTO_hash_to_aes_key (const GNUNET_HashCode * hc,
                                struct GNUNET_CRYPTO_AesSessionKey *skey,
                                struct GNUNET_CRYPTO_AesInitializationVector
                                *iv)