6 files changed, 112 insertions, 59 deletions

diff --git a/TODO b/TODO
index e65b3b940..0b24b0347 100644
--- a/TODO
+++ b/TODO
@@ -1,6 +1,7 @@
 0.9.0pre1:
 * FS: [CG]
-  - persistence support (publish, unindex, search, download)
+  - persistence support (unindex, search, download)
+  - persistence testing (publish)
   - gnunet-service-fs (hot-path routing, load-based routing, nitpicks)  
   - [gnunet-service-fs.c:208]: member 'LocalGetContext::results_bf_size' is never used
   - [gnunet-service-fs.c:501]: member 'PendingRequest::used_pids_size' is never used
diff --git a/src/fs/fs.c b/src/fs/fs.c
index e7a52d235..062870cd6 100644
--- a/src/fs/fs.c
+++ b/src/fs/fs.c
@@ -325,6 +325,42 @@ GNUNET_FS_make_file_reader_context_ (const char *filename)
 
 
 /**
+ * Function that provides data by copying from a buffer.
+ *
+ * @param cls closure (points to the buffer)
+ * @param offset offset to read from; it is possible
+ *            that the caller might need to go backwards
+ *            a bit at times
+ * @param max maximum number of bytes that should be 
+ *            copied to buf; readers are not allowed
+ *            to provide less data unless there is an error;
+ *            a value of "0" will be used at the end to allow
+ *            the reader to clean up its internal state
+ * @param buf where the reader should write the data
+ * @param emsg location for the reader to store an error message
+ * @return number of bytes written, usually "max", 0 on error
+ */
+size_t
+GNUNET_FS_data_reader_copy_ (void *cls, 
+                             uint64_t offset,
+                             size_t max, 
+                             void *buf,
+                             char **emsg)
+{
+  char *data = cls;
+
+  if (max == 0)
+    {
+      GNUNET_free_non_null (data);
+      return 0;
+    }  
+  memcpy (buf, &data[offset], max);
+  return max;
+}
+
+
+
+/**
  * Return the full filename where we would store state information
  * (for serialization/deserialization).
  *
@@ -525,11 +561,24 @@ deserialize_fi_node (struct GNUNET_FS_Handle *h,
       ret->data.file.do_index = GNUNET_NO;
       ret->data.file.have_hash = GNUNET_NO;
       ret->data.file.index_start_confirmed = GNUNET_NO;
-      /* FIXME: what's our approach for dealing with the
-         'reader' and 'reader_cls' fields?  I guess the only
-         good way would be to dump "small" files into 
-         'rh' and to not support serialization of "large"
-         files (!?) */
+      if (GNUNET_NO == ret->is_published) 
+        {
+          if (NULL == ret->filename)
+            {
+              ret->data.file.reader = &GNUNET_FS_data_reader_copy_;
+              ret->data.file.reader_cls = GNUNET_malloc_large (ret->data.file.file_size);
+              if (ret->data.file.reader_cls == NULL)
+                goto cleanup;
+              if (GNUNET_OK !=
+                  GNUNET_BIO_read (rh, "file-data", ret->data.file.reader_cls, ret->data.file.file_size))
+                goto cleanup;
+            }      
+          else
+            {
+              ret->data.file.reader = &GNUNET_FS_data_reader_file_;
+              ret->data.file.reader_cls = GNUNET_FS_make_file_reader_context_ (ret->filename);
+            }
+        }
       break;
     case 1: /* file-index, no hash */
       if (NULL == ret->filename)
diff --git a/src/fs/fs.h b/src/fs/fs.h
index 953e1e51b..2e870bb3b 100644
--- a/src/fs/fs.h
+++ b/src/fs/fs.h
@@ -602,6 +602,30 @@ void *
 GNUNET_FS_make_file_reader_context_ (const char *filename);
 
 
+
+/**
+ * Function that provides data by copying from a buffer.
+ *
+ * @param cls closure (points to the buffer)
+ * @param offset offset to read from; it is possible
+ *            that the caller might need to go backwards
+ *            a bit at times
+ * @param max maximum number of bytes that should be 
+ *            copied to buf; readers are not allowed
+ *            to provide less data unless there is an error;
+ *            a value of "0" will be used at the end to allow
+ *            the reader to clean up its internal state
+ * @param buf where the reader should write the data
+ * @param emsg location for the reader to store an error message
+ * @return number of bytes written, usually "max", 0 on error
+ */
+size_t
+GNUNET_FS_data_reader_copy_(void *cls, 
+                            uint64_t offset,
+                            size_t max, 
+                            void *buf,
+                            char **emsg);
+
 /**
  * Notification of FS that a search probe has made progress.
  * This function is used INSTEAD of the client's event handler
diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c
index df5593f75..df53bcc1c 100644
--- a/src/fs/fs_file_information.c
+++ b/src/fs/fs_file_information.c
@@ -131,41 +131,6 @@ GNUNET_FS_file_information_create_from_file (struct GNUNET_FS_Handle *h,
 
 
 /**
- * Function that provides data by copying from a buffer.
- *
- * @param cls closure (points to the buffer)
- * @param offset offset to read from; it is possible
- *            that the caller might need to go backwards
- *            a bit at times
- * @param max maximum number of bytes that should be 
- *            copied to buf; readers are not allowed
- *            to provide less data unless there is an error;
- *            a value of "0" will be used at the end to allow
- *            the reader to clean up its internal state
- * @param buf where the reader should write the data
- * @param emsg location for the reader to store an error message
- * @return number of bytes written, usually "max", 0 on error
- */
-static size_t
-data_reader_copy(void *cls, 
-                 uint64_t offset,
-                 size_t max, 
-                 void *buf,
-                 char **emsg)
-{
-  char *data = cls;
-
-  if (max == 0)
-    {
-      GNUNET_free (data);
-      return 0;
-    }  
-  memcpy (buf, &data[offset], max);
-  return max;
-}
-
-
-/**
  * Create an entry for a file in a publish-structure.
  *
  * @param h handle to the file sharing subsystem
@@ -205,7 +170,7 @@ GNUNET_FS_file_information_create_from_data (struct GNUNET_FS_Handle *h,
   return GNUNET_FS_file_information_create_from_reader (h,
                                                         client_info,
                                                         length,
-                                                        &data_reader_copy,
+                                                        &GNUNET_FS_data_reader_copy_,
                                                         data,
                                                         keywords,
                                                         meta,
diff --git a/src/include/gnunet_common.h b/src/include/gnunet_common.h
index d7842a519..cb2ed4c0c 100644
--- a/src/include/gnunet_common.h
+++ b/src/include/gnunet_common.h
@@ -322,7 +322,7 @@ unsigned long long GNUNET_htonll (unsigned long long n);
  *
  * @param size the number of bytes to allocate, must be
  *        smaller than 40 MB.
- * @return pointer to size bytes of memory
+ * @return pointer to size bytes of memory, never NULL (!)
  */
 #define GNUNET_malloc(size) GNUNET_xmalloc_(size, __FILE__, __LINE__)
 
@@ -331,7 +331,7 @@ unsigned long long GNUNET_htonll (unsigned long long n);
  * The memory will be zero'ed out.
  *
  * @param size the number of bytes to allocate
- * @return pointer to size bytes of memory
+ * @return pointer to size bytes of memory, NULL if we do not have enough memory
  */
 #define GNUNET_malloc_large(size) GNUNET_xmalloc_unchecked_(size, __FILE__, __LINE__)
 
@@ -452,16 +452,16 @@ void *GNUNET_xmalloc_ (size_t size, const char *filename, int linenumber);
 
 
 /**
- * Allocate memory.  This function does not check if the
- * allocation request is within reasonable bounds, allowing
- * allocations larger than 40 MB.  If you don't expect the
- * possibility of very large allocations, use GNUNET_malloc instead.
- * The memory will be zero'ed out.
+ * Allocate memory.  This function does not check if the allocation
+ * request is within reasonable bounds, allowing allocations larger
+ * than 40 MB.  If you don't expect the possibility of very large
+ * allocations, use GNUNET_malloc instead.  The memory will be zero'ed
+ * out.
  *
  * @param size number of bytes to allocate
  * @param filename where is this call being made (for debugging)
  * @param linenumber line where this call is being made (for debugging)
- * @return allocated memory, never NULL
+ * @return pointer to size bytes of memory, NULL if we do not have enough memory
  */
 void *GNUNET_xmalloc_unchecked_ (size_t size,
                                  const char *filename, int linenumber);
diff --git a/src/util/common_allocation.c b/src/util/common_allocation.c
index 5be7caaa7..e62c12d08 100644
--- a/src/util/common_allocation.c
+++ b/src/util/common_allocation.c
@@ -47,19 +47,36 @@ static LONG mem_used = 0;
  *  this function (or GNUNET_malloc) to allocate more than several MB
  *  of memory, if you are possibly needing a very large chunk use
  *  GNUNET_xmalloc_unchecked_ instead.
- * @param filename where in the code was the call to GNUNET_array_grow
- * @param linenumber where in the code was the call to GNUNET_array_grow
+ * @param filename where in the code was the call to GNUNET_malloc
+ * @param linenumber where in the code was the call to GNUNET_malloc
  * @return pointer to size bytes of memory
  */
 void *
 GNUNET_xmalloc_ (size_t size, const char *filename, int linenumber)
 {
+  void *ret;
   /* As a security precaution, we generally do not allow very large
      allocations using the default 'GNUNET_malloc' macro */
   GNUNET_assert_at (size <= GNUNET_MAX_MALLOC_CHECKED, filename, linenumber);
-  return GNUNET_xmalloc_unchecked_ (size, filename, linenumber);
+  ret = GNUNET_xmalloc_unchecked_ (size, filename, linenumber);
+  if (ret == NULL)
+    {
+      GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "malloc");
+      abort ();
+    }
+  return ret;
 }
 
+
+/**
+ * Wrapper around malloc. Allocates size bytes of memory.
+ * The memory will be zero'ed out.
+ *
+ * @param size the number of bytes to allocate
+ * @param filename where in the code was the call to GNUNET_malloc_large
+ * @param linenumber where in the code was the call to GNUNET_malloc_large
+ * @return pointer to size bytes of memory, NULL if we do not have enough memory
+ */
 void *
 GNUNET_xmalloc_unchecked_ (size_t size, const char *filename, int linenumber)
 {
@@ -74,10 +91,7 @@ GNUNET_xmalloc_unchecked_ (size_t size, const char *filename, int linenumber)
   GNUNET_assert_at (size < INT_MAX, filename, linenumber);
   result = malloc (size);
   if (result == NULL)
-    {
-      GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "malloc");
-      abort ();
-    }
+    return NULL;
   memset (result, 0, size);
 
 #ifdef W32_MEM_LIMIT
@@ -148,8 +162,8 @@ GNUNET_xfree_ (void *ptr, const char *filename, int linenumber)
  * Dup a string (same semantics as strdup).
  *
  * @param str the string to dup
- * @param filename where in the code was the call to GNUNET_array_grow
- * @param linenumber where in the code was the call to GNUNET_array_grow
+ * @param filename where in the code was the call to GNUNET_strdup
+ * @param linenumber where in the code was the call to GNUNET_strdup
  * @return strdup(str)
  */
 char *