7 files changed, 280 insertions, 99 deletions

diff --git a/src/arm/Makefile.am b/src/arm/Makefile.am
index f9c2a3289..0f6ef4237 100644
--- a/src/arm/Makefile.am
+++ b/src/arm/Makefile.am
@@ -53,7 +53,8 @@ check_PROGRAMS = \
 check_SCRIPTS = \
  test_gnunet_arm.sh
 
-TESTS = $(check_PROGRAMS) $(check_SCRIPTS)
+TESTS = $(check_PROGRAMS) 
+#$(check_SCRIPTS)
 
 test_arm_api_SOURCES = \
  test_arm_api.c
diff --git a/src/fs/fs.h b/src/fs/fs.h
index e3b3bc99d..77a73c3d6 100644
--- a/src/fs/fs.h
+++ b/src/fs/fs.h
@@ -955,6 +955,26 @@ struct GNUNET_FS_DownloadContext
   struct GNUNET_FS_DownloadContext *parent;
 
   /**
+   * Head of list of child downloads.
+   */
+  struct GNUNET_FS_DownloadContext *child_head;
+
+  /**
+   * Tail of list of child downloads.
+   */
+  struct GNUNET_FS_DownloadContext *child_tail;
+
+  /**
+   * Previous download belonging to the same parent.
+   */
+  struct GNUNET_FS_DownloadContext *prev;
+
+  /**
+   * Next download belonging to the same parent.
+   */
+  struct GNUNET_FS_DownloadContext *next;
+
+  /**
    * Context kept for the client.
    */
   void *client_info;
@@ -982,6 +1002,13 @@ struct GNUNET_FS_DownloadContext
   char *filename;
 
   /**
+   * Where are we writing the data temporarily (name of the
+   * file, can be NULL!); used if we do not have a permanent
+   * name and we are a directory and we do a recursive download.
+   */
+  char *temp_filename;
+
+  /**
    * Map of active requests (those waiting
    * for a response).  The key is the hash
    * of the encryped block (aka query).
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c
index 3de192e12..46759f495 100644
--- a/src/fs/fs_download.c
+++ b/src/fs/fs_download.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors)
+     (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors)
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -25,6 +25,8 @@
  * TODO:
  * - handle recursive downloads (need directory & 
  *   fs-level download-parallelism management)
+ * - handle recursive downloads where directory file is
+ *   NOT saved on disk (need temporary file instead then!)
  * - location URI suppport (can wait, easy)
  * - check if blocks exist already (can wait, easy)
  * - check if iblocks can be computed from existing blocks (can wait, hard)
@@ -39,15 +41,12 @@
 #define DEBUG_DOWNLOAD GNUNET_NO
 
 /**
- * We're storing the IBLOCKS after the
- * DBLOCKS on disk (so that we only have
- * to truncate the file once we're done).
+ * We're storing the IBLOCKS after the DBLOCKS on disk (so that we
+ * only have to truncate the file once we're done).
  *
- * Given the offset of a block (with respect
- * to the DBLOCKS) and its depth, return the
- * offset where we would store this block
- * in the file.
-
+ * Given the offset of a block (with respect to the DBLOCKS) and its
+ * depth, return the offset where we would store this block in the
+ * file.
  * 
  * @param fsize overall file size
  * @param off offset of the block in the file
@@ -245,8 +244,7 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
                                                   GNUNET_CONSTANTS_SERVICE_TIMEOUT,
                                                   GNUNET_NO,
                                                   &transmit_download_request,
-                                                  dc); 
-
+                                                  dc);
 }
 
 
@@ -341,6 +339,129 @@ struct ProcessResultClosure
 
 
 /**
+ * We found an entry in a directory.  Check if the respective child
+ * already exists and if not create the respective child download.
+ *
+ * @param cls the parent download
+ * @param filename name of the file in the directory
+ * @param uri URI of the file (CHK or LOC)
+ * @param meta meta data of the file
+ * @param length number of bytes in data
+ * @param data contents of the file (or NULL if they were not inlined)
+ */
+static void 
+trigger_recursive_download (void *cls,
+                            const char *filename,
+                            const struct GNUNET_FS_Uri *uri,
+                            const struct GNUNET_CONTAINER_MetaData *meta,
+                            size_t length,
+                            const void *data)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;  
+  struct GNUNET_FS_DownloadContext *cpos;
+
+  cpos = dc->child_head;
+  while (cpos != NULL)
+    {
+      if (0 == strcmp (cpos->filename,
+                       filename))
+        {
+          GNUNET_break_op (GNUNET_FS_uri_test_equal (uri,
+                                                     cpos->uri));
+          break;
+        }
+      cpos = cpos->next;
+    }
+  if (cpos != NULL)
+    return; /* already exists */
+  if (data != NULL)
+    {
+      /* determine on-disk filename, write data! */
+      GNUNET_break (0); // FIXME: not implemented
+    }
+  GNUNET_FS_download_start (dc->h,
+                            uri,
+                            meta,
+                            filename, /* FIXME: prepend directory name! */
+                            0,
+                            GNUNET_FS_uri_chk_get_file_size (uri),
+                            dc->anonymity,
+                            dc->options,
+                            NULL,
+                            dc);
+}
+
+
+/**
+ * We're done downloading a directory.  Open the file and
+ * trigger all of the (remaining) child downloads.
+ *
+ * @param dc context of download that just completed
+ */
+static void
+full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
+{
+  size_t size;
+  uint64_t size64;
+  void *data;
+  struct GNUNET_DISK_FileHandle *h;
+  struct GNUNET_DISK_MapHandle *m;
+  
+  size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
+  size = (size_t) size64;
+  if (size64 != (uint64_t) size)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
+      return;
+    }
+  if (dc->filename != NULL)
+    {
+      h = GNUNET_DISK_file_open (dc->filename,
+                                 GNUNET_DISK_OPEN_READ,
+                                 GNUNET_DISK_PERM_NONE);
+    }
+  else
+    {
+      /* FIXME: need to initialize (and use) temp_filename
+         in various places in order for this assertion to
+         not fail; right now, it will always fail! */
+      GNUNET_assert (dc->temp_filename != NULL);
+      h = GNUNET_DISK_file_open (dc->temp_filename,
+                                 GNUNET_DISK_OPEN_READ,
+                                 GNUNET_DISK_PERM_NONE);
+    }
+  if (h == NULL)
+    return; /* oops */
+  data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
+  if (data == NULL)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("Directory too large for system address space\n"));
+    }
+  else
+    {
+      GNUNET_FS_directory_list_contents (size,
+                                         data,
+                                         0,
+                                         &trigger_recursive_download,
+                                         dc);         
+      GNUNET_DISK_file_unmap (m);
+    }
+  GNUNET_DISK_file_close (h);
+  if (dc->filename == NULL)
+    {
+      if (0 != UNLINK (dc->temp_filename))
+        GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                  "unlink",
+                                  dc->temp_filename);
+      GNUNET_free (dc->temp_filename);
+      dc->temp_filename = NULL;
+    }
+}
+
+
+/**
  * Iterator over entries in the pending requests in the 'active' map for the
  * reply that we just got.
  *
@@ -484,6 +605,17 @@ process_result_with_request (void *cls,
           app -= (sm->offset + prc->size) - (dc->offset + dc->length);
         }
       dc->completed += app;
+
+      if ( (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
+           (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) )
+        {
+          GNUNET_FS_directory_list_contents (prc->size,
+                                             pt,
+                                             off,
+                                             &trigger_recursive_download,
+                                             dc);         
+        }
+
     }
 
   pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
@@ -513,11 +645,18 @@ process_result_with_request (void *cls,
                                       "truncate",
                                       dc->filename);
         }
-      /* signal completion */
-      pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
-      make_download_status (&pi, dc);
-      dc->client_info = dc->h->upcb (dc->h->upcb_cls,
-                                     &pi);
+
+      if ( (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
+           (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) )
+        full_recursive_download (dc);
+      if (dc->child_head == NULL)
+        {
+          /* signal completion */
+          pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
+          make_download_status (&pi, dc);
+          dc->client_info = dc->h->upcb (dc->h->upcb_cls,
+                                         &pi);
+        }
       GNUNET_assert (sm->depth == dc->treedepth);
     }
   // FIXME: make persistent
@@ -836,11 +975,6 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
       GNUNET_break (0);
       return NULL;
     }
-  client = GNUNET_CLIENT_connect (h->sched,
-                                  "fs",
-                                  h->cfg);
-  if (NULL == client)
-    return NULL;
   // FIXME: add support for "loc" URIs!
 #if DEBUG_DOWNLOAD
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -850,8 +984,13 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
 #endif
   dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
   dc->h = h;
-  dc->client = client;
   dc->parent = parent;
+  if (parent != NULL)
+    {
+      GNUNET_CONTAINER_DLL_insert (parent->child_head,
+                                   parent->child_tail,
+                                   dc);
+    }
   dc->uri = GNUNET_FS_uri_dup (uri);
   dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
   dc->client_info = cctx;
@@ -897,6 +1036,12 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
               dc->treedepth);
 #endif
   // FIXME: make persistent
+  
+  // FIXME: bound parallelism here!
+  client = GNUNET_CLIENT_connect (h->sched,
+                                  "fs",
+                                  h->cfg);
+  dc->client = client;
   schedule_block_download (dc, 
                            &dc->uri->data.chk.chk,
                            0, 
@@ -945,7 +1090,15 @@ GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc,
 {
   struct GNUNET_FS_ProgressInfo pi;
 
+  while (NULL != dc->child_head)
+    GNUNET_FS_download_stop (dc->child_head, 
+                             do_delete);
   // FIXME: make unpersistent  
+  if (dc->parent != NULL)
+    GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
+                                 dc->parent->child_tail,
+                                 dc);
+  
   pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
   make_download_status (&pi, dc);
   dc->client_info = dc->h->upcb (dc->h->upcb_cls,
diff --git a/src/fs/gnunet-download.c b/src/fs/gnunet-download.c
index 63895d4b2..6f50922be 100644
--- a/src/fs/gnunet-download.c
+++ b/src/fs/gnunet-download.c
@@ -24,9 +24,6 @@
  * @author Krista Bennett
  * @author James Blackwell
  * @author Igor Wronsky
- *
- * TODO:
- * - download-directory option support (do_directory)
  */
 #include "platform.h"
 #include "gnunet_fs_service.h"
@@ -51,8 +48,6 @@ static unsigned int parallelism = 16;
 
 static int do_recursive;
 
-static int do_directory;
-
 static char *filename;
 
 
@@ -133,15 +128,8 @@ progress_cb (void *cls,
                info->value.download.filename,
                s);
       GNUNET_free (s);
-      if (do_directory)
-        {
-          GNUNET_break (0); //FIXME: not implemented
-        }
-      else
-        {
-          if (info->value.download.dc == dc)
-            GNUNET_SCHEDULER_shutdown (sched);
-        }
+      if (info->value.download.dc == dc)
+        GNUNET_SCHEDULER_shutdown (sched);
       break;
     case GNUNET_FS_STATUS_DOWNLOAD_STOPPED: 
       if (info->value.download.dc == dc)
@@ -181,31 +169,24 @@ run (void *cls,
   enum GNUNET_FS_DownloadOptions options;
 
   sched = s;
-  if (do_directory)
+  uri = GNUNET_FS_uri_parse (args[0],
+                             &emsg);
+  if (NULL == uri)
     {
-      GNUNET_break (0); //FIXME: not implemented    
+      fprintf (stderr,
+               _("Failed to parse URI: %s\n"),
+               emsg);
+      GNUNET_free (emsg);
+      ret = 1;
+      return;
     }
-  else
+  if (! GNUNET_FS_uri_test_chk (uri))
     {
-      uri = GNUNET_FS_uri_parse (args[0],
-                                 &emsg);
-      if (NULL == uri)
-        {
-          fprintf (stderr,
-                   _("Failed to parse URI: %s\n"),
-                   emsg);
-          GNUNET_free (emsg);
-          ret = 1;
-          return;
-        }
-      if (! GNUNET_FS_uri_test_chk (uri))
-        {
-          fprintf (stderr,
-                   "Only CHK URIs supported right now.\n");
-          ret = 1;
-          GNUNET_FS_uri_destroy (uri);
-          return;                
-        }
+      fprintf (stderr,
+               "Only CHK URIs supported right now.\n");
+      ret = 1;
+      GNUNET_FS_uri_destroy (uri);
+      return;            
     }
   if (NULL == filename)
     {
@@ -237,29 +218,22 @@ run (void *cls,
   options = GNUNET_FS_DOWNLOAD_OPTION_NONE;
   if (do_recursive)
     options |= GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE;
-  if (do_directory)
+  dc = GNUNET_FS_download_start (ctx,
+                                 uri,
+                                 NULL,
+                                 filename,
+                                 0,
+                                 GNUNET_FS_uri_chk_get_file_size (uri),
+                                 anonymity,
+                                 options,
+                                 NULL,
+                                 NULL);
+  GNUNET_FS_uri_destroy (uri);
+  if (dc == NULL)
     {
-      GNUNET_break (0); //FIXME: not implemented
-    }
-  else
-    {  
-      dc = GNUNET_FS_download_start (ctx,
-                                     uri,
-                                     NULL,
-                                     filename,
-                                     0,
-                                     GNUNET_FS_uri_chk_get_file_size (uri),
-                                     anonymity,
-                                     options,
-                                     NULL,
-                                     NULL);
-      GNUNET_FS_uri_destroy (uri);
-      if (dc == NULL)
-        {
-          GNUNET_FS_stop (ctx);
-          ctx = NULL;
-          return;
-        }
+      GNUNET_FS_stop (ctx);
+      ctx = NULL;
+      return;
     }
   GNUNET_SCHEDULER_add_delayed (sched,
                                 GNUNET_TIME_UNIT_FOREVER_REL,
@@ -275,10 +249,6 @@ static struct GNUNET_GETOPT_CommandLineOption options[] = {
   {'a', "anonymity", "LEVEL",
    gettext_noop ("set the desired LEVEL of receiver-anonymity"),
    1, &GNUNET_GETOPT_set_uint, &anonymity},
-  {'d', "directory", NULL,
-   gettext_noop
-   ("download a GNUnet directory that has already been downloaded.  Requires that a filename of an existing file is specified instead of the URI.  The download will only download the top-level files in the directory unless the `-R' option is also specified."),
-   0, &GNUNET_GETOPT_set_one, &do_directory},
   {'D', "delete-incomplete", NULL,
    gettext_noop ("delete incomplete downloads (when aborted with CTRL-C)"),
    0, &GNUNET_GETOPT_set_one, &delete_incomplete},
diff --git a/src/fs/gnunet-publish.c b/src/fs/gnunet-publish.c
index 1e0884361..70964590d 100644
--- a/src/fs/gnunet-publish.c
+++ b/src/fs/gnunet-publish.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other contributing authors)
+     (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009, 2010 Christian Grothoff (and other contributing authors)
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -68,6 +68,8 @@ static int extract_only;
 
 static int do_disable_creation_time;
 
+static GNUNET_SCHEDULER_TaskIdentifier kill_task;
+
 
 static void 
 do_stop_task (void *cls,
@@ -124,6 +126,12 @@ progress_cb (void *cls,
       fprintf (stderr,
                _("Error publishing: %s.\n"),
                info->value.publish.specifics.error.message);
+      if (kill_task != GNUNET_SCHEDULER_NO_TASK)
+        {
+          GNUNET_SCHEDULER_cancel (sched,
+                                   kill_task);
+          kill_task = GNUNET_SCHEDULER_NO_TASK;
+        }
       GNUNET_SCHEDULER_add_continuation (sched,
                                          &do_stop_task,
                                          NULL,
@@ -133,11 +141,24 @@ progress_cb (void *cls,
       fprintf (stdout,
                _("Publishing `%s' done.\n"),
                info->value.publish.filename);
+      s = GNUNET_FS_uri_to_string (info->value.publish.specifics.completed.chk_uri);
+      fprintf (stdout,
+               _("URI is `%s'.\n"),
+               s);
+      GNUNET_free (s);
       if (info->value.publish.pctx == NULL)
-        GNUNET_SCHEDULER_add_continuation (sched,
-                                           &do_stop_task,
-                                           NULL,
-                                           GNUNET_SCHEDULER_REASON_PREREQ_DONE);
+        {
+          if (kill_task != GNUNET_SCHEDULER_NO_TASK)
+            {
+              GNUNET_SCHEDULER_cancel (sched,
+                                       kill_task);
+              kill_task = GNUNET_SCHEDULER_NO_TASK;
+            }
+          GNUNET_SCHEDULER_add_continuation (sched,
+                                             &do_stop_task,
+                                             NULL,
+                                             GNUNET_SCHEDULER_REASON_PREREQ_DONE);
+        }
       break;
     case GNUNET_FS_STATUS_PUBLISH_STOPPED: 
       GNUNET_break (NULL == pc);
@@ -547,6 +568,10 @@ run (void *cls,
       ret = 1;
       return;
     }
+  kill_task = GNUNET_SCHEDULER_add_delayed (sched,
+                                            GNUNET_TIME_UNIT_FOREVER_REL,
+                                            &do_stop_task,
+                                            NULL);
 }
 
 
diff --git a/src/fs/gnunet-service-fs_indexing.c b/src/fs/gnunet-service-fs_indexing.c
index ca9a226a2..525da1a91 100644
--- a/src/fs/gnunet-service-fs_indexing.c
+++ b/src/fs/gnunet-service-fs_indexing.c
@@ -22,9 +22,6 @@
  * @file fs/gnunet-service-fs_indexing.c
  * @brief program that provides indexing functions of the file-sharing service
  * @author Christian Grothoff
- *
- * TODO:
- * - indexed files/blocks not removed on errors
  */
 #include "platform.h"
 #include <float.h>
@@ -180,7 +177,7 @@ read_index_list ()
     }
   if (GNUNET_NO == GNUNET_DISK_file_test (fn))
     {
-      /* no index info  yet */
+      /* no index info yet */
       GNUNET_free (fn);
       return;
     }
@@ -596,8 +593,12 @@ GNUNET_FS_handle_on_demand_block (const GNUNET_HashCode * key,
                   STRERROR (errno));
       if (fh != NULL)
         GNUNET_DISK_file_close (fh);
-      /* FIXME: if this happens often, we need
-         to remove the OnDemand block from the DS! */
+      GNUNET_FS_drq_remove (key,
+                            size,
+                            data,
+                            &remove_cont,
+                            NULL,
+                            GNUNET_TIME_UNIT_FOREVER_REL);
       return GNUNET_SYSERR;
     }
   GNUNET_DISK_file_close (fh);
@@ -621,8 +622,12 @@ GNUNET_FS_handle_on_demand_block (const GNUNET_HashCode * key,
                   _("Indexed file `%s' changed at offset %llu\n"),
                   fn,
                   (unsigned long long) off);
-      /* FIXME: if this happens often, we need
-         to remove the OnDemand block from the DS! */
+      GNUNET_FS_drq_remove (key,
+                            size,
+                            data,
+                            &remove_cont,
+                            NULL,
+                            GNUNET_TIME_UNIT_FOREVER_REL);
       return GNUNET_SYSERR;
     }
 #if DEBUG_FS
diff --git a/src/include/gnunet_disk_lib.h b/src/include/gnunet_disk_lib.h
index 7b6898293..fcb58ffef 100644
--- a/src/include/gnunet_disk_lib.h
+++ b/src/include/gnunet_disk_lib.h
@@ -312,7 +312,7 @@ GNUNET_DISK_mktemp (const char *t);
  * @param fn file name to be opened
  * @param flags opening flags, a combination of GNUNET_DISK_OPEN_xxx bit flags
  * @param perm permissions for the newly created file, use
- *             GNUNET_DISK_PERM_USER_NONE if a file could not be created by this
+ *             GNUNET_DISK_PERM_NONE if a file could not be created by this
  *             call (because of flags)
  * @return IO handle on success, NULL on error
  */