12 files changed, 350 insertions, 206 deletions

diff --git a/TODO b/TODO
index 5b3b4d48a..24c12a1bf 100644
--- a/TODO
+++ b/TODO
@@ -13,7 +13,7 @@ away), in order in which they will likely be done:
 * UPNP
 
 
-PHASE #2: (Goal: recover basic file-sharing functionality)
+Urgent items (before announcing ng.gnunet.org):
 * TESTING (needed for DV, DHT, Topology)
   - implement library for local testing
     + modify configuration to allow controlling
@@ -37,10 +37,10 @@ PHASE #2: (Goal: recover basic file-sharing functionality)
   - implement testcases
 * FS (basic anonymous FS only)
   - implement basic FS library
-    - download API
-    - gnunet-search (binary)
-    - gnunet-download (binary)
-  - API FIX: change type for anonymity/priority from unsigned int to uint32_t!
+    - gnunet-publish (progress CB, shutdown, URI args)
+    - gnunet-unindex (progress CB, shutdown, error checking)
+    - gnunet-search (start search, progress CB, shutdown, options)
+    - gnunet-download (start download, progress CB, shutdown, options)
   - design network structs (P2P)
     + query
     + response
@@ -57,21 +57,18 @@ PHASE #2: (Goal: recover basic file-sharing functionality)
       ~ download
       ~ search
       ~ unindex
-      ~ namespaces
-      ~ collection
     + directory API
 * new webpage
-  - lcov
-  - buildbot
-  - migrate Mantis?
-  - download links on Drupal?
-  - run peer
-  - configure hostlist
-  - install on proper server
+  - prevent?
+  - migrate Mantis!?
+  - Doxygen generation
+  - run peer => have a 0.9.x hostlist
+  - improve basic documentation (configure, dependencies, what works, etc.)
+  - write minimal welcome/introduction to 0.9.x-development/webpage/status
 => Deploy(able) development network
 
 
-PHASE #3: (Goal: ready for pre-release) [completion-goal: end of 2009]
+0.9.0pre0:
 * Module features to implement:
   - advanced FS API parts
     + namespaces
@@ -80,8 +77,9 @@ PHASE #3: (Goal: ready for pre-release) [completion-goal: end of 2009]
     + collection
     + location URIs (publish, search, download)
     + persistence support (publish, unindex, search, download)
-    + datastore reservation (publishing)
     + indexing: index-failure-cleanup
+    + download: management/bounding of parallel downloads (for recursive dl.)
+    + datastore reservation (publishing)
   - implement adv. FS testcases 
     + insert: sblocks, loc uris
     + download: loc uris
@@ -103,6 +101,9 @@ PHASE #3: (Goal: ready for pre-release) [completion-goal: end of 2009]
   - testing (RC-pre0)
     + implement library for distributed testing [Nate]
     + implement testcases for distributed testing [Nate]
+
+0.9.0pre1:
+* Module features to implement:
   - tbench (RC-pre1)
     + good to have for DV evaluation!
   - DV (RC-pre1)
@@ -112,6 +113,12 @@ PHASE #3: (Goal: ready for pre-release) [completion-goal: end of 2009]
     + implement DV transport plugin [Nate & CG]
     + implement testcases [Nate]
     + implement performance tests [Nate]
+* GUIs to implement:
+  - gtk (RC-pre1)
+    + how to integrate scheduler with GTK event loop!
+
+0.9.0pre2:
+* Module features to implement:
   - tracekit (RC-pre2)
     + good to have for DV/DHT evaluation!
   - DHT (RC-pre2)
@@ -120,25 +127,21 @@ PHASE #3: (Goal: ready for pre-release) [completion-goal: end of 2009]
     + implement testcases
     + implement performance tests
 * GUIs to implement:
-  - gtk (RC-pre1)
-    + how to integrate scheduler with GTK event loop!
   - fuse (RC-pre2)
-  - qt (RC-pre3)
-    + see discussions @ FISL about integration with event loop!
 * Plugins to implement:
   - UDP backend (RC-pre2)
     + Fragmentation library 
     + actual plugin
   - HTTP backend (RC-pre2)
-* Determine RC bugs and fix those!
-
-=> PRE-RELEASE
 
+0.9.0pre3:
+* GUIs to implement:
+  - qt (RC-pre3)
+    + see discussions @ FISL about integration with event loop!
+* Determine RC bugs and fix those!
 
-PHASE #4: [completion-goal: mid 2010]
+0.9.0:
 * Documentation
-  - Doxygen generation
-  - lcov generation
   - update man pages
   - update webpage documentation
 * new webpage:
@@ -161,13 +164,10 @@ PHASE #4: [completion-goal: mid 2010]
   - vpn
 * Determine RC bugs and fix those!
 
-=> 0.9.0 RELEASE
-
-
 
-
-Post 0.9.0 features:
-* SMTP transport backend
-* HTTPS transport backend
-  - improved HTTPS support in MHD
-  - actual plugin
+0.9.x:
+* Plugins to implement:
+  - SMTP transport backend
+  - HTTPS transport backend
+    + improved HTTPS support in MHD
+    + actual plugin
diff --git a/src/Makefile.am b/src/Makefile.am
index 9b3aa46ab..7e92437c6 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -23,5 +23,6 @@ SUBDIRS = \
   core \
   testing \
   $(HOSTLIST_DIR) \
-  topology 
+  topology \
+  fs
 
diff --git a/src/fs/fs.h b/src/fs/fs.h
index 50090c6fd..70e02360c 100644
--- a/src/fs/fs.h
+++ b/src/fs/fs.h
@@ -837,6 +837,16 @@ struct GNUNET_FS_DownloadContext
   struct GNUNET_FS_Uri *uri;
 
   /**
+   * Known meta-data for the file (can be NULL).
+   */
+  struct GNUNET_CONTAINER_MetaData *meta;
+
+  /**
+   * Error message, NULL if we're doing OK.
+   */
+  char *emsg;
+
+  /**
    * Where are we writing the data (name of the
    * file, can be NULL!).
    */
diff --git a/src/fs/fs_collection.c b/src/fs/fs_collection.c
index 01c7e3298..f94ef250d 100644
--- a/src/fs/fs_collection.c
+++ b/src/fs/fs_collection.c
@@ -139,12 +139,12 @@ typedef struct CollectionData
   /**
    * Anonymity level for the collection. (NBO)
    */
-  unsigned int anonymityLevel;
+  uint32_t anonymityLevel;
 
   /**
    * Priority of the collection (NBO).
    */
-  unsigned int priority;
+  uint32_t priority;
 
   /**
    * Has this collection changed since the last publication?
@@ -465,8 +465,8 @@ GNUNET_CO_done ()
  *        is destroyed (i.e. on exit from the UI).
  */
 int
-GNUNET_CO_collection_start (unsigned int anonymityLevel,
-                            unsigned int prio,
+GNUNET_CO_collection_start (uint32_t anonymityLevel,
+                            uint32_t prio,
                             const struct GNUNET_MetaData *meta)
 {
   struct GNUNET_ECRS_URI *advertisement;
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c
index c5bfec52c..145e16818 100644
--- a/src/fs/fs_download.c
+++ b/src/fs/fs_download.c
@@ -23,10 +23,11 @@
  * @author Christian Grothoff
  *
  * TODO:
- * - offset calculations
- * - callback signaling
- * - check if blocks exist already (can wait)
- * - location URI suppport (can wait)
+ * - location URI suppport (can wait, easy)
+ * - check if blocks exist already (can wait, easy)
+ * - handle recursive downloads (need directory & 
+ *   fs-level download-parallelism management, can wait)
+ * - check if iblocks can be computed from existing blocks (can wait, hard)
  * - persistence (can wait)
  */
 #include "platform.h"
@@ -46,6 +47,7 @@
  * to the DBLOCKS) and its depth, return the
  * offset where we would store this block
  * in the file.
+
  * 
  * @param fsize overall file size
  * @param off offset of the block in the file
@@ -58,29 +60,48 @@
  */
 static uint64_t
 compute_disk_offset (uint64_t fsize,
-                      uint64_t off,
-                      unsigned int depth,
-                      unsigned int treedepth)
+                     uint64_t off,
+                     unsigned int depth,
+                     unsigned int treedepth)
 {
+  unsigned int i;
+  uint64_t lsize; /* what is the size of all IBlocks for level "i"? */
+  uint64_t loff; /* where do IBlocks for level "i" start? */
+  unsigned int ioff; /* which IBlock corresponds to "off" at level "i"? */
+  
   if (depth == treedepth)
     return off;
-  return 42; // FIXME
+  /* first IBlocks start at the end of file, rounded up
+     to full DBLOCK_SIZE */
+  loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
+  lsize = ( (fsize + DBLOCK_SIZE-1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
+  GNUNET_assert (0 == (off % DBLOCK_SIZE));
+  ioff = (off / DBLOCK_SIZE);
+  for (i=treedepth-1;i>depth;i--)
+    {
+      loff += lsize;
+      lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
+      GNUNET_assert (lsize > 0);
+      GNUNET_assert (0 == (ioff % CHK_PER_INODE));
+      ioff /= CHK_PER_INODE;
+    }
+  return loff + ioff * sizeof (struct ContentHashKey);
 }
 
+
 /**
- * Given a file of the specified treedepth and 
- * a block at the given offset and depth,
- * calculate the offset for the CHK at
- * the given index.
+ * Given a file of the specified treedepth and a block at the given
+ * offset and depth, calculate the offset for the CHK at the given
+ * index.
  *
  * @param offset the offset of the first
  *        DBLOCK in the subtree of the 
  *        identified IBLOCK
  * @param depth the depth of the IBLOCK in the tree
  * @param treedepth overall depth of the tree
- * @param i which CHK in the IBLOCK are we 
+ * @param k which CHK in the IBLOCK are we 
  *        talking about
- * @return offset if i=0, otherwise an appropriately
+ * @return offset if k=0, otherwise an appropriately
  *         larger value (i.e., if depth = treedepth-1,
  *         the returned value should be offset+DBLOCK_SIZE)
  */
@@ -88,12 +109,47 @@ static uint64_t
 compute_dblock_offset (uint64_t offset,
                        unsigned int depth,
                        unsigned int treedepth,
-                       unsigned int i)
+                       unsigned int k)
 {
-  GNUNET_assert (depth < treedepth);
-  if (i == 0)
+  unsigned int i;
+  uint64_t lsize; /* what is the size of the sum of all DBlocks 
+                     that a CHK at level i corresponds to? */
+
+  if (depth == treedepth)
     return offset;
-  return 42; // FIXME
+  lsize = DBLOCK_SIZE;
+  for (i=treedepth-1;i>depth;i--)
+    lsize *= CHK_PER_INODE;
+  return offset + i * lsize;
+}
+
+
+/**
+ * Fill in all of the generic fields for 
+ * a download event.
+ *
+ * @param pc structure to fill in
+ * @param dc overall download context
+ */
+static void
+make_download_status (struct GNUNET_FS_ProgressInfo *pi,
+                      struct GNUNET_FS_DownloadContext *dc)
+{
+  pi->value.download.dc = dc;
+  pi->value.download.cctx
+    = dc->client_info;
+  pi->value.download.pctx
+    = (dc->parent == NULL) ? NULL : dc->parent->client_info;
+  pi->value.download.uri 
+    = dc->uri;
+  pi->value.download.length
+    = dc->length;
+  pi->value.download.duration
+    = GNUNET_TIME_absolute_get_duration (dc->start_time);
+  pi->value.download.completed
+    = dc->completed;
+  pi->value.download.anonymity
+    = dc->anonymity;
 }
 
 
@@ -167,6 +223,50 @@ try_reconnect (struct GNUNET_FS_DownloadContext *dc);
 
 
 /**
+ * Compute how many bytes of data should be stored in
+ * the specified node.
+ *
+ * @param fsize overall file size
+ * @param off offset of the node
+ * @param depth depth of the node
+ * @return number of bytes stored in this node
+ */
+static size_t
+calculate_block_size (uint64_t fsize,
+                      unsigned int totaldepth,
+                      uint64_t offset,
+                      unsigned int depth)
+{
+  unsigned int i;
+  size_t ret;
+  uint64_t rsize;
+  uint64_t epos;
+  unsigned int chks;
+
+  GNUNET_assert (offset < fsize);
+  if (depth == totaldepth)
+    {
+      ret = DBLOCK_SIZE;
+      if (offset + ret > fsize)
+        ret = (size_t) (fsize - offset);
+      return ret;
+    }
+
+  rsize = DBLOCK_SIZE;
+  for (i = totaldepth-1; i > depth; i--)
+    rsize *= CHK_PER_INODE;
+  epos = offset + rsize * CHK_PER_INODE;
+  GNUNET_assert (epos > offset);
+  if (epos > fsize)
+    epos = fsize;
+  /* round up when computing #CHKs in our IBlock */
+  chks = (epos - offset + rsize - 1) / rsize;
+  GNUNET_assert (chks <= CHK_PER_INODE);
+  return chks * sizeof (struct ContentHashKey);
+}
+
+
+/**
  * Process a search result.
  *
  * @param sc our search context
@@ -180,6 +280,7 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
                 const void *data,
                 size_t size)
 {
+  struct GNUNET_FS_ProgressInfo pi;
   GNUNET_HashCode query;
   struct DownloadRequest *sm;
   struct GNUNET_CRYPTO_AesSessionKey skey;
@@ -189,8 +290,8 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
   size_t app;
   unsigned int i;
   struct ContentHashKey *chk;
+  char *emsg;
 
-  // FIXME: check that size is as big as expected, otherwise ignore!!!
   GNUNET_CRYPTO_hash (data, size, &query);
   sm = GNUNET_CONTAINER_multihashmap_get (dc->active,
                                           &query);
@@ -199,6 +300,23 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
       GNUNET_break (0);
       return;
     }
+  if (size != calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
+                                    dc->treedepth,
+                                    sm->offset,
+                                    sm->depth))
+    {
+      dc->emsg = GNUNET_strdup ("Internal error or bogus download URI");
+      /* signal error */
+      pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
+      make_download_status (&pi, dc);
+      pi.value.download.specifics.error.message = dc->emsg;
+      dc->client_info = dc->h->upcb (dc->h->upcb_cls,
+                                     &pi);
+      /* abort all pending requests */
+      GNUNET_CLIENT_disconnect (dc->client);
+      dc->client = NULL;
+      return;
+    }
   GNUNET_assert (GNUNET_YES ==
                  GNUNET_CONTAINER_multihashmap_remove (dc->active,
                                                        &query,
@@ -218,16 +336,44 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
                                  sm->offset,
                                  sm->depth,
                                  dc->treedepth);
-      GNUNET_assert (off  != 
-                     GNUNET_DISK_file_seek (dc->handle,
-                                            off,
-                                            GNUNET_DISK_SEEK_SET) );
-      GNUNET_DISK_file_write (dc->handle,
-                              pt,
-                              size);
-    }
-  // FIXME: make persistent
+      emsg = NULL;
+      if ( (off  != 
+            GNUNET_DISK_file_seek (dc->handle,
+                                   off,
+                                   GNUNET_DISK_SEEK_SET) ) )
+        GNUNET_asprintf (&emsg,
+                         _("Failed to seek to offset %llu in file `%s': %s\n"),
+                         (unsigned long long) off,
+                         dc->filename,
+                         STRERROR (errno));
+      else if (size !=
+               GNUNET_DISK_file_write (dc->handle,
+                                       pt,
+                                       size))
+        GNUNET_asprintf (&emsg,
+                         _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
+                         (unsigned int) size,
+                         (unsigned long long) off,
+                         dc->filename,
+                         STRERROR (errno));
+      if (NULL != emsg)
+        {
+          dc->emsg = emsg;
+          // FIXME: make persistent
+
+          /* signal error */
+          pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
+          make_download_status (&pi, dc);
+          pi.value.download.specifics.error.message = emsg;
+          dc->client_info = dc->h->upcb (dc->h->upcb_cls,
+                                         &pi);
 
+          /* abort all pending requests */
+          GNUNET_CLIENT_disconnect (dc->client);
+          dc->client = NULL;
+          return;
+        }
+    }
   if (sm->depth == dc->treedepth) 
     {
       app = size;
@@ -247,7 +393,37 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
         }
       dc->completed += app;
     }
-  // FIXME: call progress callback
+
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
+  make_download_status (&pi, dc);
+  pi.value.download.specifics.progress.data = pt;
+  pi.value.download.specifics.progress.offset = sm->offset;
+  pi.value.download.specifics.progress.data_len = size;
+  pi.value.download.specifics.progress.depth = sm->depth;
+  dc->client_info = dc->h->upcb (dc->h->upcb_cls,
+                                 &pi);
+  GNUNET_assert (dc->completed <= dc->length);
+  if (dc->completed == dc->length)
+    {
+      /* truncate file to size (since we store IBlocks at the end) */
+      if (dc->handle != NULL)
+        {
+          GNUNET_DISK_file_close (dc->handle);
+          dc->handle = NULL;
+          if (0 != truncate (dc->filename,
+                             GNUNET_ntohll (dc->uri->data.chk.file_length)))
+            GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                      "truncate",
+                                      dc->filename);
+        }
+      /* signal completion */
+      pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
+      make_download_status (&pi, dc);
+      dc->client_info = dc->h->upcb (dc->h->upcb_cls,
+                                     &pi);
+      GNUNET_assert (sm->depth == dc->treedepth);
+    }
+  // FIXME: make persistent
   if (sm->depth == dc->treedepth) 
     return;
   GNUNET_assert (0 == (size % sizeof(struct ContentHashKey)));
@@ -458,6 +634,7 @@ try_reconnect (struct GNUNET_FS_DownloadContext *dc)
  *
  * @param h handle to the file sharing subsystem
  * @param uri the URI of the file (determines what to download); CHK or LOC URI
+ * @param meta known metadata for the file (can be NULL)
  * @param filename where to store the file, maybe NULL (then no file is
  *        created on disk and data must be grabbed from the callbacks)
  * @param offset at what offset should we start the download (typically 0)
@@ -471,6 +648,7 @@ try_reconnect (struct GNUNET_FS_DownloadContext *dc)
 struct GNUNET_FS_DownloadContext *
 GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h,
                                const struct GNUNET_FS_Uri *uri,
+                               const struct GNUNET_CONTAINER_MetaData *meta,
                                const char *filename,
                                uint64_t offset,
                                uint64_t length,
@@ -478,6 +656,7 @@ GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h,
                                enum GNUNET_FS_DownloadOptions options,
                                struct GNUNET_FS_DownloadContext *parent)
 {
+  struct GNUNET_FS_ProgressInfo pi;
   struct GNUNET_FS_DownloadContext *dc;
   struct GNUNET_CLIENT_Connection *client;
 
@@ -499,6 +678,7 @@ GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h,
   dc->client = client;
   dc->parent = parent;
   dc->uri = GNUNET_FS_uri_dup (uri);
+  dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
   if (NULL != filename)
     {
       dc->filename = GNUNET_strdup (filename);
@@ -519,6 +699,7 @@ GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h,
                       _("Download failed: could not open file `%s': %s\n"),
                       dc->filename,
                       STRERROR (errno));
+          GNUNET_CONTAINER_meta_data_destroy (dc->meta);
           GNUNET_FS_uri_destroy (dc->uri);
           GNUNET_free (dc->filename);
           GNUNET_CLIENT_disconnect (dc->client);
@@ -547,7 +728,12 @@ GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h,
                          &receive_results,
                          dc,
                          GNUNET_TIME_UNIT_FOREVER_REL);
-  // FIXME: signal download start
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
+  make_download_status (&pi, dc);
+  pi.value.download.specifics.start.meta = meta;
+  dc->client_info = dc->h->upcb (dc->h->upcb_cls,
+                                 &pi);
+
   return dc;
 }
 
@@ -580,9 +766,14 @@ void
 GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *dc,
                               int do_delete)
 {
-  // FIXME: make unpersistent
-  // FIXME: signal download end
-  
+  struct GNUNET_FS_ProgressInfo pi;
+
+  // FIXME: make unpersistent  
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
+  make_download_status (&pi, dc);
+  dc->client_info = dc->h->upcb (dc->h->upcb_cls,
+                                 &pi);
+
   if (GNUNET_SCHEDULER_NO_TASK != dc->task)
     GNUNET_SCHEDULER_cancel (dc->h->sched,
                              dc->task);
@@ -594,7 +785,8 @@ GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *dc,
   GNUNET_CONTAINER_multihashmap_destroy (dc->active);
   if (dc->filename != NULL)
     {
-      GNUNET_DISK_file_close (dc->handle);
+      if (NULL != dc->handle)
+        GNUNET_DISK_file_close (dc->handle);
       if ( (dc->completed != dc->length) &&
            (GNUNET_YES == do_delete) )
         {
@@ -605,75 +797,15 @@ GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *dc,
         }
       GNUNET_free (dc->filename);
     }
+  GNUNET_CONTAINER_meta_data_destroy (dc->meta);
   GNUNET_FS_uri_destroy (dc->uri);
   GNUNET_free (dc);
 }
 
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 #if 0
 
-/**
- * Compute how many bytes of data are stored in
- * this node.
- */
-static unsigned int
-get_node_size (const struct Node *node)
-{
-  unsigned int i;
-  unsigned int ret;
-  unsigned long long rsize;
-  unsigned long long spos;
-  unsigned long long epos;
-
-  GNUNET_GE_ASSERT (node->ctx->ectx, node->offset < node->ctx->total);
-  if (node->level == 0)
-    {
-      ret = GNUNET_ECRS_DBLOCK_SIZE;
-      if (node->offset + (unsigned long long) ret > node->ctx->total)
-        ret = (unsigned int) (node->ctx->total - node->offset);
-#if DEBUG_DOWNLOAD
-      GNUNET_GE_LOG (node->ctx->rm->ectx,
-                     GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
-                     "Node at offset %llu and level %d has size %u\n",
-                     node->offset, node->level, ret);
-#endif
-      return ret;
-    }
-  rsize = GNUNET_ECRS_DBLOCK_SIZE;
-  for (i = 0; i < node->level - 1; i++)
-    rsize *= GNUNET_ECRS_CHK_PER_INODE;
-  spos = rsize * (node->offset / sizeof (GNUNET_EC_ContentHashKey));
-  epos = spos + rsize * GNUNET_ECRS_CHK_PER_INODE;
-  if (epos > node->ctx->total)
-    epos = node->ctx->total;
-  ret = (epos - spos) / rsize;
-  if (ret * rsize < epos - spos)
-    ret++;                      /* need to round up! */
-#if DEBUG_DOWNLOAD
-  GNUNET_GE_LOG (node->ctx->rm->ectx,
-                 GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
-                 "Node at offset %llu and level %d has size %u\n",
-                 node->offset, node->level,
-                 ret * sizeof (GNUNET_EC_ContentHashKey));
-#endif
-  return ret * sizeof (GNUNET_EC_ContentHashKey);
-}
 
 /**
  * Check if self block is already present on the drive.  If the block
diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c
index 5de9cbeb8..e173b1149 100644
--- a/src/fs/fs_file_information.c
+++ b/src/fs/fs_file_information.c
@@ -192,8 +192,8 @@ GNUNET_FS_file_information_create_from_file (void *client_info,
                                              const struct GNUNET_FS_Uri *keywords,
                                              const struct GNUNET_CONTAINER_MetaData *meta,
                                              int do_index,
-                                             unsigned int anonymity,
-                                             unsigned int priority,
+                                             uint32_t anonymity,
+                                             uint32_t priority,
                                              struct GNUNET_TIME_Absolute expirationTime)
 {
   struct FileInfo *fi;
@@ -280,8 +280,8 @@ GNUNET_FS_file_information_create_from_data (void *client_info,
                                              const struct GNUNET_FS_Uri *keywords,
                                              const struct GNUNET_CONTAINER_MetaData *meta,
                                              int do_index,
-                                             unsigned int anonymity,
-                                             unsigned int priority,
+                                             uint32_t anonymity,
+                                             uint32_t priority,
                                              struct GNUNET_TIME_Absolute expirationTime)
 {
   return GNUNET_FS_file_information_create_from_reader (client_info,
@@ -323,8 +323,8 @@ GNUNET_FS_file_information_create_from_reader (void *client_info,
                                                const struct GNUNET_FS_Uri *keywords,
                                                const struct GNUNET_CONTAINER_MetaData *meta,
                                                int do_index,
-                                               unsigned int anonymity,
-                                               unsigned int priority,
+                                               uint32_t anonymity,
+                                               uint32_t priority,
                                                struct GNUNET_TIME_Absolute expirationTime)
 {
   struct GNUNET_FS_FileInformation *ret;
@@ -387,12 +387,12 @@ struct DirScanCls
   /**
    * Desired anonymity level.
    */
-  unsigned int anonymity;
+  uint32_t anonymity;
 
   /**
    * Desired publishing priority.
    */
-  unsigned int priority;
+  uint32_t priority;
 
   /**
    * Expiration time for publication.
@@ -499,8 +499,8 @@ int
 GNUNET_FS_directory_scanner_default (void *cls,
                                      const char *dirname,
                                      int do_index,
-                                     unsigned int anonymity,
-                                     unsigned int priority,
+                                     uint32_t anonymity,
+                                     uint32_t priority,
                                      struct GNUNET_TIME_Absolute expirationTime,
                                      GNUNET_FS_FileProcessor proc,
                                      void *proc_cls,
@@ -593,8 +593,8 @@ GNUNET_FS_file_information_create_from_directory (void *client_info,
                                                   GNUNET_FS_DirectoryScanner scanner,
                                                   void *scanner_cls,
                                                   int do_index,
-                                                  unsigned int anonymity,
-                                                  unsigned int priority,
+                                                  uint32_t anonymity,
+                                                  uint32_t priority,
                                                   struct GNUNET_TIME_Absolute expirationTime,
                                                   char **emsg)
 {
@@ -656,8 +656,8 @@ struct GNUNET_FS_FileInformation *
 GNUNET_FS_file_information_create_empty_directory (void *client_info,
                                                    const struct GNUNET_CONTAINER_MetaData *meta,
                                                    const struct GNUNET_FS_Uri *keywords,
-                                                   unsigned int anonymity,
-                                                   unsigned int priority,
+                                                   uint32_t anonymity,
+                                                   uint32_t priority,
                                                    struct GNUNET_TIME_Absolute expirationTime)
 {
   struct GNUNET_FS_FileInformation *ret;
diff --git a/src/fs/fs_namespace.c b/src/fs/fs_namespace.c
index fb45d99e9..64763d164 100644
--- a/src/fs/fs_namespace.c
+++ b/src/fs/fs_namespace.c
@@ -46,9 +46,9 @@
 struct GNUNET_FS_Uri *
 GNUNET_FS_namespace_advertise (struct GNUNET_FS_Handle *h,
                                struct GNUNET_FS_Namespace *namespace,
-                               const struct GNUNET_MetaData *meta,
-                               unsigned int anonymity,
-                               unsigned int priority,
+                               const struct GNUNET_CONTAINER_MetaData *meta,
+                               uint32_t anonymity,
+                               uint32_t priority,
                                struct GNUNET_TIME_Absolute expiration,
                                const struct GNUNET_FS_Uri *advertisementURI,
                                const char *rootEntry)
@@ -284,9 +284,9 @@ write_namespace_key (struct GNUNET_GC_Configuration *cfg,
 struct GNUNET_ECRS_URI *
 GNUNET_ECRS_namespace_create (struct GNUNET_GE_Context *ectx,
                               struct GNUNET_GC_Configuration *cfg,
-                              const struct GNUNET_MetaData *meta,
-                              unsigned int anonymityLevel,
-                              unsigned int priority,
+                              const struct GNUNET_CONTAINER_MetaData *meta,
+                              uint32_t anonymityLevel,
+                              uint32_t priority,
                               GNUNET_CronTime expiration,
                               const struct GNUNET_ECRS_URI *advertisementURI,
                               const char *rootEntry)
@@ -519,8 +519,8 @@ struct GNUNET_ECRS_URI *
 GNUNET_ECRS_namespace_add_content (struct GNUNET_GE_Context *ectx,
                                    struct GNUNET_GC_Configuration *cfg,
                                    const GNUNET_HashCode * pid,
-                                   unsigned int anonymityLevel,
-                                   unsigned int priority,
+                                   uint32_t anonymityLevel,
+                                   uint32_t priority,
                                    GNUNET_CronTime expiration,
                                    const char *thisId,
                                    const char *nextId,
diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c
index 422984ecb..d58ea3f37 100644
--- a/src/fs/fs_publish.c
+++ b/src/fs/fs_publish.c
@@ -467,7 +467,7 @@ static void
 block_proc (void *cls,
             const GNUNET_HashCode *query,
             uint64_t offset,
-            unsigned int type,
+            uint32_t type,
             const void *block,
             uint16_t block_size)
 {
@@ -909,8 +909,8 @@ fip_signal_start(void *cls,
                  uint64_t length,
                  struct GNUNET_CONTAINER_MetaData *meta,
                  struct GNUNET_FS_Uri **uri,
-                 unsigned int *anonymity,
-                 unsigned int *priority,
+                 uint32_t *anonymity,
+                 uint32_t *priority,
                  struct GNUNET_TIME_Absolute *expirationTime,
                  void **client_info)
 {
@@ -1021,8 +1021,8 @@ fip_signal_stop(void *cls,
                 uint64_t length,
                 struct GNUNET_CONTAINER_MetaData *meta,
                 struct GNUNET_FS_Uri **uri,
-                unsigned int *anonymity,
-                unsigned int *priority,
+                uint32_t *anonymity,
+                uint32_t *priority,
                 struct GNUNET_TIME_Absolute *expirationTime,
                 void **client_info)
 {
@@ -1135,12 +1135,12 @@ struct PublishKskContext
   /**
    * Anonymity level for the KBlocks.
    */
-  unsigned int anonymity;
+  uint32_t anonymity;
 
   /**
    * Priority for the KBlocks.
    */
-  unsigned int priority;
+  uint32_t priority;
 };
 
 
@@ -1284,8 +1284,8 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
                        struct GNUNET_CONTAINER_MetaData *meta,
                        struct GNUNET_FS_Uri *uri,
                        struct GNUNET_TIME_Absolute expirationTime,
-                       unsigned int anonymity,
-                       unsigned int priority,
+                       uint32_t anonymity,
+                       uint32_t priority,
                        enum GNUNET_FS_PublishOptions options,
                        GNUNET_FS_PublishContinuation cont,
                        void *cont_cls)
@@ -1443,8 +1443,8 @@ GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h,
                        struct GNUNET_CONTAINER_MetaData *meta,
                        struct GNUNET_FS_Uri *uri,
                        struct GNUNET_TIME_Absolute expirationTime,
-                       unsigned int anonymity,
-                       unsigned int priority,
+                       uint32_t anonymity,
+                       uint32_t priority,
                        enum GNUNET_FS_PublishOptions options,
                        GNUNET_FS_PublishContinuation cont,
                        void *cont_cls)
diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c
index 109a76791..fd7dcf6d9 100644
--- a/src/fs/fs_search.c
+++ b/src/fs/fs_search.c
@@ -274,7 +274,7 @@ process_ksk_result (struct GNUNET_FS_SearchContext *sc,
 static struct GNUNET_FS_SearchContext *
 search_start (struct GNUNET_FS_Handle *h,
               const struct GNUNET_FS_Uri *uri,
-              unsigned int anonymity,
+              uint32_t anonymity,
               struct GNUNET_FS_SearchContext *parent);
 
 
@@ -738,7 +738,7 @@ try_reconnect (struct GNUNET_FS_SearchContext *sc)
 static struct GNUNET_FS_SearchContext *
 search_start (struct GNUNET_FS_Handle *h,
               const struct GNUNET_FS_Uri *uri,
-              unsigned int anonymity,
+              uint32_t anonymity,
               struct GNUNET_FS_SearchContext *parent)
 {
   struct GNUNET_FS_SearchContext *sc;
@@ -830,7 +830,7 @@ search_start (struct GNUNET_FS_Handle *h,
 struct GNUNET_FS_SearchContext *
 GNUNET_FS_search_start (struct GNUNET_FS_Handle *h,
                         const struct GNUNET_FS_Uri *uri,
-                        unsigned int anonymity)
+                        uint32_t anonymity)
 {
   return search_start (h, uri, anonymity, NULL);
 }
diff --git a/src/fs/fs_tree.h b/src/fs/fs_tree.h
index 4186a4e21..bfbd7019b 100644
--- a/src/fs/fs_tree.h
+++ b/src/fs/fs_tree.h
@@ -67,7 +67,7 @@ struct GNUNET_FS_TreeEncoder;
 typedef void (*GNUNET_FS_TreeBlockProcessor)(void *cls,
                                              const GNUNET_HashCode *query,
                                              uint64_t offset,
-                                             unsigned int type,
+                                             uint32_t type,
                                              const void *block,
                                              uint16_t block_size);
                                              
diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c
index 23779bc97..03dae383f 100644
--- a/src/fs/fs_unindex.c
+++ b/src/fs/fs_unindex.c
@@ -200,7 +200,7 @@ static void
 unindex_process (void *cls,
                  const GNUNET_HashCode *query,
                  uint64_t offset,
-                 unsigned int type,
+                 uint32_t type,
                  const void *block,
                  uint16_t block_size)
 {
diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h
index fb6b7f133..ee10a0156 100644
--- a/src/include/gnunet_fs_service.h
+++ b/src/include/gnunet_fs_service.h
@@ -415,7 +415,7 @@ GNUNET_FS_getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext*
  * the metadata must be passed as the "scls" argument.
  *
  * @param ctx command line processor context
- * @param scls must be of type "struct GNUNET_MetaData **"
+ * @param scls must be of type "struct GNUNET_CONTAINER_MetaData **"
  * @param option name of the option (typically 'k')
  * @param value command line argument given
  * @return GNUNET_OK on success
@@ -864,12 +864,11 @@ struct GNUNET_FS_ProgressInfo
       const struct GNUNET_FS_Uri *uri;
       
       /**
-       * How large is the file overall?  For directories,
-       * this is only the size of the directory itself,
-       * not of the other files contained within the 
-       * directory.
+       * How large is the download overall?  This
+       * is NOT necessarily the size from the
+       * URI since we may be doing a partial download.
        */
-      uint64_t size;
+      uint64_t length;
 
       /**
        * At what time do we expect to finish the download?
@@ -938,8 +937,8 @@ struct GNUNET_FS_ProgressInfo
           /**
            * Known metadata for the download.
            */
-          const struct GNUNET_MetaData *meta;
-
+          const struct GNUNET_CONTAINER_MetaData *meta;
+          
         } start;
 
         /**
@@ -951,7 +950,7 @@ struct GNUNET_FS_ProgressInfo
           /**
            * Known metadata for the download.
            */
-          const struct GNUNET_MetaData *meta;
+          const struct GNUNET_CONTAINER_MetaData *meta;
 
           /**
            * Error message, NULL if we have not encountered any error yet.
@@ -1449,8 +1448,8 @@ typedef int (*GNUNET_FS_FileInformationProcessor)(void *cls,
                                                   uint64_t length,
                                                   struct GNUNET_CONTAINER_MetaData *meta,
                                                   struct GNUNET_FS_Uri **uri,
-                                                  unsigned int *anonymity,
-                                                  unsigned int *priority,
+                                                  uint32_t *anonymity,
+                                                  uint32_t *priority,
                                                   struct GNUNET_TIME_Absolute *expirationTime,
                                                   void **client_info);
 
@@ -1513,8 +1512,8 @@ GNUNET_FS_file_information_create_from_file (void *client_info,
                                              const struct GNUNET_FS_Uri *keywords,
                                              const struct GNUNET_CONTAINER_MetaData *meta,
                                              int do_index,
-                                             unsigned int anonymity,
-                                             unsigned int priority,
+                                             uint32_t anonymity,
+                                             uint32_t priority,
                                              struct GNUNET_TIME_Absolute expirationTime);
 
 
@@ -1543,8 +1542,8 @@ GNUNET_FS_file_information_create_from_data (void *client_info,
                                              const struct GNUNET_FS_Uri *keywords,
                                              const struct GNUNET_CONTAINER_MetaData *meta,
                                              int do_index,
-                                             unsigned int anonymity,
-                                             unsigned int priority,
+                                             uint32_t anonymity,
+                                             uint32_t priority,
                                              struct GNUNET_TIME_Absolute expirationTime);
 
 
@@ -1597,8 +1596,8 @@ GNUNET_FS_file_information_create_from_reader (void *client_info,
                                                const struct GNUNET_FS_Uri *keywords,
                                                const struct GNUNET_CONTAINER_MetaData *meta,
                                                int do_index,
-                                               unsigned int anonymity,
-                                               unsigned int priority,
+                                               uint32_t anonymity,
+                                               uint32_t priority,
                                                struct GNUNET_TIME_Absolute expirationTime);
 
 
@@ -1634,8 +1633,8 @@ typedef void (*GNUNET_FS_FileProcessor)(void *cls,
 typedef int (*GNUNET_FS_DirectoryScanner)(void *cls,
                                           const char *dirname,
                                           int do_index,
-                                          unsigned int anonymity,
-                                          unsigned int priority,
+                                          uint32_t anonymity,
+                                          uint32_t priority,
                                           struct GNUNET_TIME_Absolute expirationTime,
                                           GNUNET_FS_FileProcessor proc,
                                           void *proc_cls,
@@ -1669,8 +1668,8 @@ int
 GNUNET_FS_directory_scanner_default (void *cls,
                                      const char *dirname,
                                      int do_index,
-                                     unsigned int anonymity,
-                                     unsigned int priority,
+                                     uint32_t anonymity,
+                                     uint32_t priority,
                                      struct GNUNET_TIME_Absolute expirationTime,
                                      GNUNET_FS_FileProcessor proc,
                                      void *proc_cls,
@@ -1705,8 +1704,8 @@ GNUNET_FS_file_information_create_from_directory (void *client_info,
                                                   GNUNET_FS_DirectoryScanner scanner,
                                                   void *scanner_cls,
                                                   int do_index,
-                                                  unsigned int anonymity,
-                                                  unsigned int priority,
+                                                  uint32_t anonymity,
+                                                  uint32_t priority,
                                                   struct GNUNET_TIME_Absolute expirationTime,
                                                   char **emsg);
 
@@ -1731,8 +1730,8 @@ struct GNUNET_FS_FileInformation *
 GNUNET_FS_file_information_create_empty_directory (void *client_info,
                                                    const struct GNUNET_CONTAINER_MetaData *meta,
                                                    const struct GNUNET_FS_Uri *keywords,
-                                                   unsigned int anonymity,
-                                                   unsigned int priority,
+                                                   uint32_t anonymity,
+                                                   uint32_t priority,
                                                    struct GNUNET_TIME_Absolute expirationTime);
 
 
@@ -1874,8 +1873,8 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
                        struct GNUNET_CONTAINER_MetaData *meta,
                        struct GNUNET_FS_Uri *uri,
                        struct GNUNET_TIME_Absolute expirationTime,
-                       unsigned int anonymity,
-                       unsigned int priority,
+                       uint32_t anonymity,
+                       uint32_t priority,
                        enum GNUNET_FS_PublishOptions options,
                        GNUNET_FS_PublishContinuation cont,
                        void *cont_cls);
@@ -1904,8 +1903,8 @@ GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h,
                        struct GNUNET_CONTAINER_MetaData *meta,
                        struct GNUNET_FS_Uri *uri,
                        struct GNUNET_TIME_Absolute expirationTime,
-                       unsigned int anonymity,
-                       unsigned int priority,
+                       uint32_t anonymity,
+                       uint32_t priority,
                        enum GNUNET_FS_PublishOptions options,
                        GNUNET_FS_PublishContinuation cont,
                        void *cont_cls);
@@ -1983,9 +1982,9 @@ GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc);
 struct GNUNET_FS_Uri *
 GNUNET_FS_namespace_advertise (struct GNUNET_FS_Handle *h,
                                struct GNUNET_FS_Namespace *namespace,
-                               const struct GNUNET_MetaData *meta,
-                               unsigned int anonymity,
-                               unsigned int priority,
+                               const struct GNUNET_CONTAINER_MetaData *meta,
+                               uint32_t anonymity,
+                               uint32_t priority,
                                struct GNUNET_TIME_Absolute expiration,
                                const struct GNUNET_FS_Uri *advertisementURI,
                                const char *rootEntry);
@@ -2092,7 +2091,7 @@ GNUNET_FS_namespace_list_updateable (struct GNUNET_FS_Namespace *namespace,
 struct GNUNET_FS_SearchContext *
 GNUNET_FS_search_start (struct GNUNET_FS_Handle *h,
                         const struct GNUNET_FS_Uri *uri,
-                        unsigned int anonymity);
+                        uint32_t anonymity);
 
 
 /**
@@ -2165,6 +2164,7 @@ enum GNUNET_FS_DownloadOptions
  *
  * @param h handle to the file sharing subsystem
  * @param uri the URI of the file (determines what to download); CHK or LOC URI
+ * @param meta known metadata for the file (can be NULL)
  * @param filename where to store the file, maybe NULL (then no file is
  *        created on disk and data must be grabbed from the callbacks)
  * @param offset at what offset should we start the download (typically 0)
@@ -2178,6 +2178,7 @@ enum GNUNET_FS_DownloadOptions
 struct GNUNET_FS_DownloadContext *
 GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h,
                                const struct GNUNET_FS_Uri *uri,
+                               const struct GNUNET_CONTAINER_MetaData *meta,
                                const char *filename,
                                uint64_t offset,
                                uint64_t length,