finishing initial coding of fs-publish serialization/deserialization

3 files changed, 233 insertions, 118 deletions

diff --git a/src/fs/fs.c b/src/fs/fs.c
index 837f59a10..e7a52d235 100644
--- a/src/fs/fs.c
+++ b/src/fs/fs.c
@@ -20,7 +20,7 @@
 
 /**
  * @file fs/fs.c
- * @brief main FS functions
+ * @brief main FS functions (master initialization, serialization, deserialization, shared code)
  * @author Christian Grothoff
  */
 
@@ -214,6 +214,116 @@ GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh)
 }
 
 
+
+
+/**
+ * Closure for "data_reader_file".
+ */
+struct FileInfo
+{
+  /**
+   * Name of the file to read.
+   */
+  char *filename;
+
+  /**
+   * File descriptor, NULL if it has not yet been opened.
+   */
+  struct GNUNET_DISK_FileHandle *fd;
+};
+
+
+/**
+ * Function that provides data by reading from a file.
+ *
+ * @param cls closure (points to the file information)
+ * @param offset offset to read from; it is possible
+ *            that the caller might need to go backwards
+ *            a bit at times
+ * @param max maximum number of bytes that should be 
+ *            copied to buf; readers are not allowed
+ *            to provide less data unless there is an error;
+ *            a value of "0" will be used at the end to allow
+ *            the reader to clean up its internal state
+ * @param buf where the reader should write the data
+ * @param emsg location for the reader to store an error message
+ * @return number of bytes written, usually "max", 0 on error
+ */
+size_t
+GNUNET_FS_data_reader_file_(void *cls, 
+                            uint64_t offset,
+                            size_t max, 
+                            void *buf,
+                            char **emsg)
+{
+  struct FileInfo *fi = cls;
+  ssize_t ret;
+
+  if (max == 0)
+    {
+      if (fi->fd != NULL)
+        GNUNET_DISK_file_close (fi->fd);
+      GNUNET_free (fi->filename);
+      GNUNET_free (fi);
+      return 0;
+    }  
+  if (fi->fd == NULL)
+    {
+      fi->fd = GNUNET_DISK_file_open (fi->filename,
+                                      GNUNET_DISK_OPEN_READ,
+                                      GNUNET_DISK_PERM_NONE);
+      if (fi->fd == NULL)
+        {
+          GNUNET_asprintf (emsg, 
+                           _("Could not open file `%s': %s"),
+                           fi->filename,
+                           STRERROR (errno));
+          return 0;
+        }
+    }
+  GNUNET_DISK_file_seek (fi->fd, offset, GNUNET_DISK_SEEK_SET);
+  ret = GNUNET_DISK_file_read (fi->fd, buf, max);
+  if (ret == -1)
+    {
+      GNUNET_asprintf (emsg, 
+                       _("Could not read file `%s': %s"),
+                       fi->filename,
+                       STRERROR (errno));
+      return 0;
+    }
+  if (ret != max)
+    {
+      GNUNET_asprintf (emsg, 
+                       _("Short read reading from file `%s'!"),
+                       fi->filename);
+      return 0;
+    }
+  return max;
+}
+
+
+/**
+ * Create the closure for the 'GNUNET_FS_data_reader_file_' callback.
+ *
+ * @param filename file to read
+ * @return closure to use, NULL on error
+ */
+void *
+GNUNET_FS_make_file_reader_context_ (const char *filename)
+{
+  struct FileInfo *fi;
+
+  fi = GNUNET_malloc (sizeof(struct FileInfo));
+  fi->filename = GNUNET_STRINGS_filename_expand (filename);
+  if (fi->filename == NULL)
+    {
+      GNUNET_free (fi);
+      return NULL;
+    }
+  return fi;
+}
+
+
 /**
  * Return the full filename where we would store state information
  * (for serialization/deserialization).
@@ -422,6 +532,8 @@ deserialize_fi_node (struct GNUNET_FS_Handle *h,
          files (!?) */
       break;
     case 1: /* file-index, no hash */
+      if (NULL == ret->filename)
+        goto cleanup;
       if (GNUNET_OK !=
           GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size))
         goto cleanup;
@@ -429,11 +541,12 @@ deserialize_fi_node (struct GNUNET_FS_Handle *h,
       ret->data.file.do_index = GNUNET_YES;
       ret->data.file.have_hash = GNUNET_NO;
       ret->data.file.index_start_confirmed = GNUNET_NO;
-      /* FIXME: what's our approach for dealing with the
-         'reader' and 'reader_cls' fields? 
-         (should be easy for indexing since we must have a file) */
+      ret->data.file.reader = &GNUNET_FS_data_reader_file_;
+      ret->data.file.reader_cls = GNUNET_FS_make_file_reader_context_ (ret->filename);
       break;
     case 2: /* file-index-with-hash */
+      if (NULL == ret->filename)
+        goto cleanup;
       if ( (GNUNET_OK !=
             GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) ||
            (GNUNET_OK !=
@@ -443,23 +556,24 @@ deserialize_fi_node (struct GNUNET_FS_Handle *h,
       ret->data.file.do_index = GNUNET_YES;
       ret->data.file.have_hash = GNUNET_YES;
       ret->data.file.index_start_confirmed = GNUNET_NO;
-      /* FIXME: what's our approach for dealing with the
-         'reader' and 'reader_cls' fields? 
-         (should be easy for indexing since we must have a file) */
+      ret->data.file.reader = &GNUNET_FS_data_reader_file_;
+      ret->data.file.reader_cls = GNUNET_FS_make_file_reader_context_ (ret->filename);
       break;
     case 3: /* file-index-with-hash-confirmed */
+      if (NULL == ret->filename)
+        goto cleanup;
       if ( (GNUNET_OK !=
             GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) ||
            (GNUNET_OK !=
             GNUNET_BIO_read (rh, "fileid", &ret->data.file.file_id, sizeof (GNUNET_HashCode))) )
         goto cleanup;
+
       ret->is_directory = GNUNET_NO;
       ret->data.file.do_index = GNUNET_YES;
       ret->data.file.have_hash = GNUNET_YES;
       ret->data.file.index_start_confirmed = GNUNET_YES;
-      /* FIXME: what's our approach for dealing with the
-         'reader' and 'reader_cls' fields? 
-         (should be easy for indexing since we must have a file) */
+      ret->data.file.reader = &GNUNET_FS_data_reader_file_;
+      ret->data.file.reader_cls = GNUNET_FS_make_file_reader_context_ (ret->filename);
       break;
     case 4: /* directory */
       if ( (GNUNET_OK !=
@@ -507,7 +621,6 @@ deserialize_fi_node (struct GNUNET_FS_Handle *h,
   GNUNET_free_non_null (filename);
   GNUNET_FS_file_information_destroy (ret, NULL, NULL);
   return NULL;
-   
 }
 
 
@@ -581,7 +694,8 @@ get_serialization_short_name (const char *fullname)
 
 
 /**
- * Create a new random name for serialization.
+ * Create a new random name for serialization.  Also checks if persistence
+ * is enabled and returns NULL if not.
  *
  * @param h master context
  * @param ext component of the path 
@@ -595,7 +709,8 @@ make_serialization_file_name (struct GNUNET_FS_Handle *h,
   char *dn;
   char *ret;
 
-  /* FIXME: check if persistence option was set! */
+  if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE))
+    return NULL; /* persistence not requested */
   dn = get_serialization_file_name (h, ext, "");
   fn = GNUNET_DISK_mktemp (dn);
   GNUNET_free (dn);
@@ -608,6 +723,44 @@ make_serialization_file_name (struct GNUNET_FS_Handle *h,
 
 
 /**
+ * Copy all of the data from the reader to the write handle.
+ *
+ * @param wh write handle
+ * @param fi file with reader
+ * @return GNUNET_OK on success
+ */
+static int
+copy_from_reader (struct GNUNET_BIO_WriteHandle *wh,
+                  struct GNUNET_FS_FileInformation * fi)
+{
+  char buf[32 * 1024];
+  uint64_t off;
+  size_t ret;
+  char *emsg;
+
+  emsg = NULL;
+  off = 0;
+  while (off < fi->data.file.file_size)
+    {
+      ret = fi->data.file.reader (fi->data.file.reader_cls,
+                                  off, sizeof (buf),
+                                  buf,
+                                  &emsg);
+      if (ret == 0)
+        {
+          GNUNET_free (emsg);
+          return GNUNET_SYSERR;
+        }
+      if (GNUNET_OK != 
+          GNUNET_BIO_write (wh, buf, ret))
+        return GNUNET_SYSERR;
+      off += ret;
+    }
+  return GNUNET_OK;
+}
+
+
+/**
  * Create a temporary file on disk to store the current
  * state of "fi" in.
  *
@@ -622,7 +775,6 @@ GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation * fi)
   char *ksks;
   char *chks;
 
-  /* FIXME: check if persistence option was set! */
   if (NULL == fi->serialization)    
     fi->serialization = make_serialization_file_name (fi->h, "publish-fi");
   if (NULL == fi->serialization)
@@ -684,27 +836,28 @@ GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation * fi)
       if (GNUNET_OK !=
           GNUNET_BIO_write_int64 (wh, fi->data.file.file_size))
         goto cleanup;
-      /* FIXME: what's our approach for dealing with the
-         'reader' and 'reader_cls' fields?  I guess the only
-         good way would be to dump "small" files into 
-         'rh' and to not support serialization of "large"
-         files (!?) */
+      if ( (GNUNET_NO == fi->is_published) &&
+           (NULL == fi->filename) )     
+        if (GNUNET_OK != 
+            copy_from_reader (wh, fi))
+          goto cleanup;
       break;
     case 1: /* file-index, no hash */
+      if (NULL == fi->filename)
+        goto cleanup;
       if (GNUNET_OK !=
           GNUNET_BIO_write_int64 (wh, fi->data.file.file_size))
         goto cleanup;
       break;
     case 2: /* file-index-with-hash */
     case 3: /* file-index-with-hash-confirmed */
+      if (NULL == fi->filename)
+        goto cleanup;
       if ( (GNUNET_OK !=
             GNUNET_BIO_write_int64 (wh, fi->data.file.file_size)) ||
            (GNUNET_OK !=
             GNUNET_BIO_write (wh, &fi->data.file.file_id, sizeof (GNUNET_HashCode))) )
         goto cleanup;
-      /* FIXME: what's our approach for dealing with the
-         'reader' and 'reader_cls' fields? 
-         (should be easy for indexing since we must have a file) */
       break;
     case 4: /* directory */
       if ( (GNUNET_OK !=
diff --git a/src/fs/fs.h b/src/fs/fs.h
index fa36b8f02..953e1e51b 100644
--- a/src/fs/fs.h
+++ b/src/fs/fs.h
@@ -431,11 +431,6 @@ struct GNUNET_FS_FileInformation
   } data;
 
   /**
-   * Is this struct for a file or directory?
-   */
-  int is_directory;
-
-  /**
    * Desired anonymity level.
    */
   uint32_t anonymity;
@@ -445,6 +440,16 @@ struct GNUNET_FS_FileInformation
    */
   uint32_t priority;
 
+  /**
+   * Is this struct for a file or directory?
+   */
+  int is_directory;
+
+  /**
+   * Are we done publishing this file?
+   */
+  int is_published;
+
 };
 
 
@@ -563,6 +568,39 @@ void
 GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh);
 
 
+/**
+ * Function that provides data by reading from a file.
+ *
+ * @param cls closure (points to the file information)
+ * @param offset offset to read from; it is possible
+ *            that the caller might need to go backwards
+ *            a bit at times
+ * @param max maximum number of bytes that should be 
+ *            copied to buf; readers are not allowed
+ *            to provide less data unless there is an error;
+ *            a value of "0" will be used at the end to allow
+ *            the reader to clean up its internal state
+ * @param buf where the reader should write the data
+ * @param emsg location for the reader to store an error message
+ * @return number of bytes written, usually "max", 0 on error
+ */
+size_t
+GNUNET_FS_data_reader_file_(void *cls, 
+                            uint64_t offset,
+                            size_t max, 
+                            void *buf,
+                            char **emsg);
+
+
+/**
+ * Create the closure for the 'GNUNET_FS_data_reader_file_' callback.
+ *
+ * @param filename file to read
+ * @return closure to use
+ */
+void *
+GNUNET_FS_make_file_reader_context_ (const char *filename);
+
 
 /**
  * Notification of FS that a search probe has made progress.
diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c
index 2d0194741..df5593f75 100644
--- a/src/fs/fs_file_information.c
+++ b/src/fs/fs_file_information.c
@@ -55,92 +55,6 @@ GNUNET_FS_file_information_get_id (struct GNUNET_FS_FileInformation *s)
 
 
 /**
- * Closure for "data_reader_file".
- */
-struct FileInfo
-{
-  /**
-   * Name of the file to read.
-   */
-  char *filename;
-
-  /**
-   * File descriptor, NULL if it has not yet been opened.
-   */
-  struct GNUNET_DISK_FileHandle *fd;
-};
-
-
-/**
- * Function that provides data by reading from a file.
- *
- * @param cls closure (points to the file information)
- * @param offset offset to read from; it is possible
- *            that the caller might need to go backwards
- *            a bit at times
- * @param max maximum number of bytes that should be 
- *            copied to buf; readers are not allowed
- *            to provide less data unless there is an error;
- *            a value of "0" will be used at the end to allow
- *            the reader to clean up its internal state
- * @param buf where the reader should write the data
- * @param emsg location for the reader to store an error message
- * @return number of bytes written, usually "max", 0 on error
- */
-static size_t
-data_reader_file(void *cls, 
-                 uint64_t offset,
-                 size_t max, 
-                 void *buf,
-                 char **emsg)
-{
-  struct FileInfo *fi = cls;
-  ssize_t ret;
-
-  if (max == 0)
-    {
-      if (fi->fd != NULL)
-        GNUNET_DISK_file_close (fi->fd);
-      GNUNET_free (fi->filename);
-      GNUNET_free (fi);
-      return 0;
-    }  
-  if (fi->fd == NULL)
-    {
-      fi->fd = GNUNET_DISK_file_open (fi->filename,
-                                      GNUNET_DISK_OPEN_READ,
-                                      GNUNET_DISK_PERM_NONE);
-      if (fi->fd == NULL)
-        {
-          GNUNET_asprintf (emsg, 
-                           _("Could not open file `%s': %s"),
-                           fi->filename,
-                           STRERROR (errno));
-          return 0;
-        }
-    }
-  GNUNET_DISK_file_seek (fi->fd, offset, GNUNET_DISK_SEEK_SET);
-  ret = GNUNET_DISK_file_read (fi->fd, buf, max);
-  if (ret == -1)
-    {
-      GNUNET_asprintf (emsg, 
-                       _("Could not read file `%s': %s"),
-                       fi->filename,
-                       STRERROR (errno));
-      return 0;
-    }
-  if (ret != max)
-    {
-      GNUNET_asprintf (emsg, 
-                       _("Short read reading from file `%s'!"),
-                       fi->filename);
-      return 0;
-    }
-  return max;
-}
-
-
-/**
  * Create an entry for a file in a publish-structure.
  *
  * @param h handle to the file sharing subsystem
@@ -182,17 +96,16 @@ GNUNET_FS_file_information_create_from_file (struct GNUNET_FS_Handle *h,
                                 filename);
       return NULL;
     }
-  fi = GNUNET_malloc (sizeof(struct FileInfo));
-  fi->filename = GNUNET_STRINGS_filename_expand (filename);
-  if (fi->filename == NULL)
+  fi = GNUNET_FS_make_file_reader_context_ (filename);
+  if (fi == NULL)
     {
-      GNUNET_free (fi);
+      GNUNET_break (0);
       return NULL;
     }
   ret = GNUNET_FS_file_information_create_from_reader (h,
                                                        client_info,
                                                        sbuf.st_size,
-                                                       &data_reader_file,
+                                                       &GNUNET_FS_data_reader_file_,
                                                        fi,
                                                        keywords,
                                                        meta,
@@ -284,6 +197,11 @@ GNUNET_FS_file_information_create_from_data (struct GNUNET_FS_Handle *h,
                                              uint32_t priority,
                                              struct GNUNET_TIME_Absolute expirationTime)
 {
+  if (GNUNET_YES == do_index)        
+    {
+      GNUNET_break (0);
+      return NULL;
+    }
   return GNUNET_FS_file_information_create_from_reader (h,
                                                         client_info,
                                                         length,
@@ -333,6 +251,12 @@ GNUNET_FS_file_information_create_from_reader (struct GNUNET_FS_Handle *h,
 {
   struct GNUNET_FS_FileInformation *ret;
 
+  if ( (GNUNET_YES == do_index) &&
+       (reader != &GNUNET_FS_data_reader_file_) )
+    {
+      GNUNET_break (0);
+      return NULL;
+    }
   ret = GNUNET_malloc (sizeof (struct GNUNET_FS_FileInformation));
   ret->h = h;
   ret->client_info = client_info;