towards having download

2 files changed, 508 insertions, 33 deletions

diff --git a/src/fs/fs.h b/src/fs/fs.h
index ed925d9c8..3dc60a5dc 100644
--- a/src/fs/fs.h
+++ b/src/fs/fs.h
@@ -337,12 +337,12 @@ struct GNUNET_FS_FileInformation
   /**
    * Desired anonymity level.
    */
-  unsigned int anonymity;
+  uint32_t anonymity;
 
   /**
    * Desired priority (for keeping the content in the DB).
    */
-  unsigned int priority;
+  uint32_t priority;
 
 };
 
@@ -428,12 +428,6 @@ struct GNUNET_FS_PublishContext
   char *nuid;
 
   /**
-   * ID of the task performing the upload. NO_TASK
-   * if the upload has completed.
-   */
-  GNUNET_SCHEDULER_TaskIdentifier upload_task;
-
-  /**
    * Our own client handle for the FS service;
    * only briefly used when we start to index a
    * file, otherwise NULL.
@@ -441,6 +435,23 @@ struct GNUNET_FS_PublishContext
   struct GNUNET_CLIENT_Connection *client;
 
   /**
+   * Current position in the file-tree for the
+   * upload.
+   */
+  struct GNUNET_FS_FileInformation *fi_pos;
+
+  /**
+   * Connection to the datastore service.
+   */
+  struct GNUNET_DATASTORE_Handle *dsh;
+
+  /**
+   * ID of the task performing the upload. NO_TASK
+   * if the upload has completed.
+   */
+  GNUNET_SCHEDULER_TaskIdentifier upload_task;
+
+  /**
    * Typically GNUNET_NO.  Set to GNUNET_YES if
    * "upload_task" is GNUNET_SCHEDULER_NO_TASK
    * and we're waiting for a response from the
@@ -461,17 +472,6 @@ struct GNUNET_FS_PublishContext
   enum GNUNET_FS_PublishOptions options;
 
   /**
-   * Current position in the file-tree for the
-   * upload.
-   */
-  struct GNUNET_FS_FileInformation *fi_pos;
-
-  /**
-   * Connection to the datastore service.
-   */
-  struct GNUNET_DATASTORE_Handle *dsh;
-
-  /**
    * Space reservation ID with datastore service
    * for this upload.
    */
@@ -765,10 +765,127 @@ struct GNUNET_FS_SearchContext
 
 
 /**
+ * Information about an active download request.
+ */ 
+struct DownloadRequest
+{
+  /**
+   * While pending, we keep all download requests
+   * in a linked list.
+   */
+  struct DownloadRequest *next;
+
+  /**
+   * CHK for the request.
+   */
+  struct ContentHashKey chk;
+
+  /**
+   * Offset of the corresponding block.
+   */
+  uint64_t offset;
+
+  /**
+   * Depth of the corresponding block in the tree.
+   */
+  unsigned int depth;
+
+  /**
+   * Set if this request is currently in the linked list of pending
+   * requests.  Needed in case we get a response for a request that we
+   * have not yet send (due to FS bug or two blocks with identical
+   * content); in this case, we would need to remove the block from
+   * the pending list (and need a fast way to check if the block is on
+   * it).
+   */
+  int is_pending;
+
+};
+
+
+/**
  * Context for controlling a download.
  */
 struct GNUNET_FS_DownloadContext
 {
+  
+  /**
+   * Global FS context.
+   */ 
+  struct GNUNET_FS_Handle *h;
+  
+  /**
+   * Connection to the FS service.
+   */
+  struct GNUNET_CLIENT_Connection *client;
+
+  /**
+   * Parent download (used when downloading files
+   * in directories).
+   */
+  struct GNUNET_FS_DownloadContext *parent;
+
+  /**
+   * Context kept for the client.
+   */
+  void *client_info;
+
+  /**
+   * URI that identifies the file that
+   * we are downloading.
+   */
+  struct GNUNET_FS_Uri *uri;
+
+  /**
+   * Where are we writing the data (name of the
+   * file, can be NULL!).
+   */
+  char *filename;
+
+  /**
+   * Map of active requests (those waiting
+   * for a response).  The key is the hash
+   * of the encryped block (aka query).
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *active;
+
+  /**
+   * Linked list of pending requests.
+   */
+  struct DownloadRequest *pending;
+
+  /**
+   * ID of a task that is using this struct
+   * and that must be cancelled when the download
+   * is being stopped (if not GNUNET_SCHEDULER_NO_TASK).
+   * Used for the task that adds some artificial
+   * delay when trying to reconnect to the FS
+   * service.
+   */
+  GNUNET_SCHEDULER_TaskIdentifier task;
+
+  /**
+   * What is the first offset that we're interested
+   * in?
+   */
+  uint64_t offset;
+
+  /**
+   * How many bytes starting from offset are desired?
+   * This is NOT the overall length of the file!
+   */
+  uint64_t length;
+
+  /**
+   * Desired level of anonymity.
+   */
+  uint32_t anonymity;
+
+  /**
+   * Options for the download.
+   */
+  enum GNUNET_FS_DownloadOptions options;
+
 };
 
 struct GNUNET_FS_Namespace
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c
index f1954ea89..06c4672a2 100644
--- a/src/fs/fs_download.c
+++ b/src/fs/fs_download.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008 Christian Grothoff (and other contributing authors)
+     (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors)
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -19,17 +19,288 @@
 */
 /**
  * @file fs/fs_download.c
- * @brief DOWNLOAD helper methods (which do the real work).
+ * @brief download methods
  * @author Christian Grothoff
+ *
+ * TODO:
+ * - process replies
+ * - callback signaling
+ * - location URI suppport (can wait)
+ * - persistence (can wait)
  */
-
 #include "platform.h"
+#include "gnunet_constants.h"
 #include "gnunet_fs_service.h"
 #include "fs.h"
 
 #define DEBUG_DOWNLOAD GNUNET_YES
 
 
+/**
+ * Schedule the download of the specified
+ * block in the tree.
+ *
+ * @param dc overall download this block belongs to
+ * @param chk content-hash-key of the block
+ * @param offset offset of the block in the file
+ *         (for IBlocks, the offset is the lowest
+ *          offset of any DBlock in the subtree under
+ *          the IBlock)
+ * @param depth depth of the block, 0 is the root of the tree
+ */
+static void
+schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
+                         const struct ContentHashKey *chk,
+                         uint64_t offset,
+                         unsigned int depth)
+{
+  struct DownloadRequest *sm;
+
+  sm = GNUNET_malloc (sizeof (struct DownloadRequest));
+  sm->chk = *chk;
+  sm->offset = offset;
+  sm->depth = depth;
+  sm->is_pending = GNUNET_YES;
+  sm->next = dc->pending;
+  dc->pending = sm;
+  GNUNET_CONTAINER_multihashmap_put (dc->active,
+                                     &chk->query,
+                                     sm,
+                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+}
+
+
+/**
+ * We've lost our connection with the FS service.
+ * Re-establish it and re-transmit all of our
+ * pending requests.
+ *
+ * @param dc download context that is having trouble
+ */
+static void
+try_reconnect (struct GNUNET_FS_DownloadContext *dc);
+
+
+/**
+ * Process a search result.
+ *
+ * @param sc our search context
+ * @param type type of the result
+ * @param data the (encrypted) response
+ * @param size size of data
+ */
+static void
+process_result (struct GNUNET_FS_DownloadContext *dc,
+                uint32_t type,
+                const void *data,
+                size_t size)
+{
+  GNUNET_HashCode query;
+  struct DownloadRequest *sm;
+  struct GNUNET_CRYPTO_AesSessionKey skey;
+  struct GNUNET_CRYPTO_AesInitializationVector iv;
+  char pt[size];
+
+  GNUNET_CRYPTO_hash (data, size, &query);
+  sm = GNUNET_CONTAINER_multihashmap_get (dc->active,
+                                          &query);
+  if (NULL == sm)
+    {
+      GNUNET_break (0);
+      return;
+    }
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (dc->active,
+                                                       &query,
+                                                       sm));
+  GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
+  GNUNET_CRYPTO_aes_decrypt (data,
+                             size,
+                             &skey,
+                             &iv,
+                             pt);
+  // FIXME: save to disk
+  // FIXME: make persistent
+  // FIXME: call progress callback
+  // FIXME: trigger next block (if applicable)  
+}
+
+
+/**
+ * Type of a function to call when we receive a message
+ * from the service.
+ *
+ * @param cls closure
+ * @param msg message received, NULL on timeout or fatal error
+ */
+static void 
+receive_results (void *cls,
+                 const struct GNUNET_MessageHeader * msg)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  const struct ContentMessage *cm;
+  uint16_t msize;
+
+  if ( (NULL == msg) ||
+       (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_CONTENT) ||
+       (ntohs (msg->size) <= sizeof (struct ContentMessage)) )
+    {
+      try_reconnect (dc);
+      return;
+    }
+  msize = ntohs (msg->size);
+  cm = (const struct ContentMessage*) msg;
+  process_result (dc, 
+                  ntohl (cm->type),
+                  &cm[1],
+                  msize - sizeof (struct ContentMessage));
+  /* continue receiving */
+  GNUNET_CLIENT_receive (dc->client,
+                         &receive_results,
+                         dc,
+                         GNUNET_TIME_UNIT_FOREVER_REL);
+}
+
+
+
+/**
+ * We're ready to transmit a search request to the
+ * file-sharing service.  Do it.  If there is 
+ * more than one request pending, try to send 
+ * multiple or request another transmission.
+ *
+ * @param cls closure
+ * @param size number of bytes available in buf
+ * @param buf where the callee should write the message
+ * @return number of bytes written to buf
+ */
+static size_t
+transmit_download_request (void *cls,
+                           size_t size, 
+                           void *buf)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  size_t msize;
+  struct SearchMessage *sm;
+
+  if (NULL == buf)
+    {
+      try_reconnect (dc);
+      return 0;
+    }
+  GNUNET_assert (size >= sizeof (struct SearchMessage));
+  msize = 0;
+  sm = buf;
+  while ( (dc->pending == NULL) &&
+          (size > msize + sizeof (struct SearchMessage)) )
+    {
+      memset (sm, 0, sizeof (struct SearchMessage));
+      sm->header.size = htons (sizeof (struct SearchMessage));
+      sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
+      sm->anonymity_level = htonl (dc->anonymity);
+      // FIXME: support 'loc' URIs (set sm->target) 
+      sm->query = dc->pending->chk.query;
+      dc->pending->is_pending = GNUNET_NO;
+      dc->pending = dc->pending->next;
+      msize += sizeof (struct SearchMessage);
+      sm++;
+    }
+  return msize;
+}
+
+
+/**
+ * Reconnect to the FS service and transmit
+ * our queries NOW.
+ *
+ * @param cls our download context
+ * @param tc unused
+ */
+static void
+do_reconnect (void *cls,
+              const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct GNUNET_CLIENT_Connection *client;
+  
+  dc->task = GNUNET_SCHEDULER_NO_TASK;
+  client = GNUNET_CLIENT_connect (dc->h->sched,
+                                  "fs",
+                                  dc->h->cfg);
+  if (NULL == client)
+    {
+      try_reconnect (dc);
+      return;
+    }
+  dc->client = client;
+  GNUNET_CLIENT_notify_transmit_ready (client,
+                                       sizeof (struct SearchMessage),
+                                       GNUNET_CONSTANTS_SERVICE_TIMEOUT,
+                                       &transmit_download_request,
+                                       dc);  
+  GNUNET_CLIENT_receive (client,
+                         &receive_results,
+                         dc,
+                         GNUNET_TIME_UNIT_FOREVER_REL);
+}
+
+
+/**
+ * Add entries that are not yet pending back to
+ * the pending list.
+ *
+ * @param cls our download context
+ * @param key unused
+ * @param entry entry of type "struct DownloadRequest"
+ * @return GNUNET_OK
+ */
+static int
+retry_entry (void *cls,
+             const GNUNET_HashCode *key,
+             void *entry)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct DownloadRequest *dr = entry;
+
+  if (! dr->is_pending)
+    {
+      dr->next = dc->pending;
+      dr->is_pending = GNUNET_YES;
+      dc->pending = entry;
+    }
+  return GNUNET_OK;
+}
+
+
+/**
+ * We've lost our connection with the FS service.
+ * Re-establish it and re-transmit all of our
+ * pending requests.
+ *
+ * @param dc download context that is having trouble
+ */
+static void
+try_reconnect (struct GNUNET_FS_DownloadContext *dc)
+{
+  
+  if (NULL != dc->client)
+    {
+      GNUNET_CONTAINER_multihashmap_iterate (dc->active,
+                                             &retry_entry,
+                                             dc);
+      GNUNET_CLIENT_disconnect (dc->client);
+      dc->client = NULL;
+    }
+  dc->task
+    = GNUNET_SCHEDULER_add_delayed (dc->h->sched,
+                                    GNUNET_NO,
+                                    GNUNET_SCHEDULER_PRIORITY_IDLE,
+                                    GNUNET_SCHEDULER_NO_TASK,
+                                    GNUNET_TIME_UNIT_SECONDS,
+                                    &do_reconnect,
+                                    dc);
+}
+
 
 /**
  * Download parts of a file.  Note that this will store
@@ -50,9 +321,7 @@
  * @param offset at what offset should we start the download (typically 0)
  * @param length how many bytes should be downloaded starting at offset
  * @param anonymity anonymity level to use for the download
- * @param no_temporaries set to GNUNET_YES to disallow generation of temporary files
- * @param recursive should this be a recursive download (useful for directories
- *        to automatically trigger download of files in the directories)
+ * @param options various options
  * @param parent parent download to associate this download with (use NULL
  *        for top-level downloads; useful for manually-triggered recursive downloads)
  * @return context that can be used to control this download
@@ -61,31 +330,120 @@ struct GNUNET_FS_DownloadContext *
 GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h,
                                const struct GNUNET_FS_Uri *uri,
                                const char *filename,
-                               unsigned long long offset,
-                               unsigned long long length,
-                               unsigned int anonymity,
-                               int no_temporaries,      
-                               int recursive,
+                               uint64_t offset,
+                               uint64_t length,
+                               uint32_t anonymity,
+                               enum GNUNET_FS_DownloadOptions options,
                                struct GNUNET_FS_DownloadContext *parent)
 {
-  return NULL;
+  struct GNUNET_FS_DownloadContext *dc;
+  struct GNUNET_CLIENT_Connection *client;
+
+  client = GNUNET_CLIENT_connect (h->sched,
+                                  "fs",
+                                  h->cfg);
+  if (NULL == client)
+    return NULL;
+  // FIXME: add support for "loc" URIs!
+  GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
+  if ( (dc->offset + dc->length < dc->offset) ||
+       (dc->offset + dc->length > uri->data.chk.file_length) )
+    {
+      GNUNET_break (0);
+      return NULL;
+    }
+  dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
+  dc->h = h;
+  dc->client = client;
+  dc->parent = parent;
+  dc->uri = GNUNET_FS_uri_dup (uri);
+  dc->filename = (NULL == filename) ? NULL : GNUNET_strdup (filename);
+  dc->offset = offset;
+  dc->length = length;
+  dc->anonymity = anonymity;
+  dc->options = options;
+  dc->active = GNUNET_CONTAINER_multihashmap_create (1 + (length / DBLOCK_SIZE));
+  // FIXME: make persistent
+  schedule_block_download (dc, 
+                           &dc->uri->data.chk.chk,
+                           0, 
+                           0);
+  GNUNET_CLIENT_notify_transmit_ready (client,
+                                       sizeof (struct SearchMessage),
+                                       GNUNET_CONSTANTS_SERVICE_TIMEOUT,
+                                       &transmit_download_request,
+                                       dc);  
+  GNUNET_CLIENT_receive (client,
+                         &receive_results,
+                         dc,
+                         GNUNET_TIME_UNIT_FOREVER_REL);
+  // FIXME: signal download start
+  return dc;
+}
+
+
+/**
+ * Free entries in the map.
+ *
+ * @param cls unused (NULL)
+ * @param key unused
+ * @param entry entry of type "struct DownloadRequest" which is freed
+ * @return GNUNET_OK
+ */
+static int
+free_entry (void *cls,
+            const GNUNET_HashCode *key,
+            void *entry)
+{
+  GNUNET_free (entry);
+  return GNUNET_OK;
 }
 
+
 /**
  * Stop a download (aborts if download is incomplete).
  *
- * @param rm handle for the download
+ * @param dc handle for the download
  * @param do_delete delete files of incomplete downloads
  */
 void
-GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *rm,
+GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *dc,
                               int do_delete)
 {
+  // FIXME: make unpersistent
+  // FIXME: signal download end
+  
+  if (GNUNET_SCHEDULER_NO_TASK != dc->task)
+    GNUNET_SCHEDULER_cancel (dc->h->sched,
+                             dc->task);
+  if (NULL != dc->client)
+    GNUNET_CLIENT_disconnect (dc->client);
+  GNUNET_CONTAINER_multihashmap_iterate (dc->active,
+                                         &free_entry,
+                                         NULL);
+  GNUNET_CONTAINER_multihashmap_destroy (dc->active);
+  GNUNET_FS_uri_destroy (dc->uri);
+  GNUNET_free_non_null (dc->filename);
+  GNUNET_free (dc);
 }
 
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 #if 0
 
 /**