fixing indexing and unindexing issues

8 files changed, 100 insertions, 36 deletions

diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am
index 38db1bc58..59361b03e 100644
--- a/src/fs/Makefile.am
+++ b/src/fs/Makefile.am
@@ -118,6 +118,7 @@ TESTS = \
  test_fs_download \
  test_fs_search \
  test_fs_start_stop \
+ test_fs_unindex \
  test_fs_uri
 # $(check_PROGRAMS)
 
diff --git a/src/fs/fs.h b/src/fs/fs.h
index 2cac304a3..acfc8b1c6 100644
--- a/src/fs/fs.h
+++ b/src/fs/fs.h
@@ -367,6 +367,18 @@ struct GNUNET_FS_FileInformation
        */
       int do_index;
 
+      /**
+       * Is "file_id" already valid?  Set to GNUNET_YES
+       * once the hash has been calculated.
+       */
+      int have_hash;
+
+      /**
+       * Has the service confirmed our INDEX_START request?
+       * GNUNET_YES if this step has been completed.
+       */
+      int index_start_confirmed;
+
     } file;
 
     /**
diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c
index f4f4fb567..330ddc2fc 100644
--- a/src/fs/fs_file_information.c
+++ b/src/fs/fs_file_information.c
@@ -202,6 +202,7 @@ GNUNET_FS_file_information_create_from_file (void *client_info,
 {
   struct FileInfo *fi;
   struct stat sbuf;
+  struct GNUNET_FS_FileInformation *ret;
 
   if (0 != STAT (filename, &sbuf))
     {
@@ -212,16 +213,18 @@ GNUNET_FS_file_information_create_from_file (void *client_info,
     }
   fi = GNUNET_malloc (sizeof(struct FileInfo));
   fi->filename = GNUNET_strdup (filename);
-  return GNUNET_FS_file_information_create_from_reader (client_info,
-                                                        sbuf.st_size,
-                                                        &data_reader_file,
-                                                        fi,
-                                                        keywords,
-                                                        meta,
-                                                        do_index,
-                                                        anonymity,
-                                                        priority,
-                                                        expirationTime);
+  ret = GNUNET_FS_file_information_create_from_reader (client_info,
+                                                       sbuf.st_size,
+                                                       &data_reader_file,
+                                                       fi,
+                                                       keywords,
+                                                       meta,
+                                                       do_index,
+                                                       anonymity,
+                                                       priority,
+                                                       expirationTime);
+  ret->data.file.filename = GNUNET_strdup (filename);
+  return ret;
 }
 
 
diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c
index 34a13f55c..a64f0d15e 100644
--- a/src/fs/fs_publish.c
+++ b/src/fs/fs_publish.c
@@ -482,13 +482,6 @@ block_proc (void *cls,
   struct PutContCtx * dpc_cls;
   struct OnDemandBlock odb;
 
-#if DEBUG_PUBLISH
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Publishing block `%s' for offset %llu with size %u\n",
-              GNUNET_h2s (query),
-              (unsigned long long) offset,
-              (unsigned int) block_size);
-#endif
   p = sc->fi_pos;
   if (NULL == sc->dsh)
     {
@@ -510,10 +503,17 @@ block_proc (void *cls,
   dpc_cls->cont_cls = sc;
   dpc_cls->sc = sc;
   dpc_cls->p = p;
-  if ( (p->is_directory) &&
-       (p->data.file.do_index) &&
+  if ( (! p->is_directory) &&
+       (GNUNET_YES == p->data.file.do_index) &&
        (type == GNUNET_DATASTORE_BLOCKTYPE_DBLOCK) )
     {
+#if DEBUG_PUBLISH
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Indexing block `%s' for offset %llu with index size %u\n",
+                  GNUNET_h2s (query),
+                  (unsigned long long) offset,
+                  sizeof (struct OnDemandBlock));
+#endif
       odb.offset = offset;
       odb.file_id = p->data.file.file_id;
       GNUNET_DATASTORE_put (sc->dsh,
@@ -530,6 +530,13 @@ block_proc (void *cls,
                             dpc_cls);     
       return;
     }
+#if DEBUG_PUBLISH
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Publishing block `%s' for offset %llu with size %u\n",
+              GNUNET_h2s (query),
+              (unsigned long long) offset,
+              (unsigned int) block_size);
+#endif
   GNUNET_DATASTORE_put (sc->dsh,
                         sc->rid,
                         query,
@@ -702,6 +709,7 @@ process_index_start_response (void *cls,
       publish_content (sc);
       return;
     }
+  p->data.file.index_start_confirmed = GNUNET_YES;
   /* success! continue with indexing */
   publish_content (sc);
 }
@@ -738,6 +746,11 @@ hash_for_index_cb (void *cls,
       publish_content (sc);
       return;
     }
+  if (GNUNET_YES == p->data.file.index_start_confirmed)
+    {
+      publish_content (sc);
+      return;
+    }
   slen = strlen (p->data.file.filename) + 1;
   if (slen > GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage))
     {
@@ -749,6 +762,12 @@ hash_for_index_cb (void *cls,
       publish_content (sc);
       return;
     }
+#if DEBUG_PUBLISH
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Hash of indexed file `%s' is `%s'\n",
+              p->data.file.filename,
+              GNUNET_h2s (res));
+#endif
   client = GNUNET_CLIENT_connect (sc->h->sched,
                                   "fs",
                                   sc->h->cfg);
@@ -763,6 +782,7 @@ hash_for_index_cb (void *cls,
       return;
     }
   p->data.file.file_id = *res;
+  p->data.file.have_hash = GNUNET_YES;
   ism = GNUNET_malloc (sizeof(struct IndexStartMessage) +
                        slen);
   ism->header.size = htons(sizeof(struct IndexStartMessage) +
@@ -776,6 +796,13 @@ hash_for_index_cb (void *cls,
       ism->device = htonl (dev);
       ism->inode = GNUNET_htonll(ino);
     }
+  else
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _("Failed to get file identifiers for `%s'\n"),
+                  p->data.file.filename);
+    }
+  ism->file_id = *res;
   memcpy (&ism[1],
           p->data.file.filename,
           slen);
@@ -877,13 +904,17 @@ do_upload (void *cls,
           publish_content (sc);
           return;
         }      
-      GNUNET_CRYPTO_hash_file (sc->h->sched,
-                               GNUNET_SCHEDULER_PRIORITY_IDLE,
-                               GNUNET_NO,
-                               p->data.file.filename,
-                               HASHING_BLOCKSIZE,
-                               &hash_for_index_cb,
-                               sc);
+      if (p->data.file.have_hash)
+        hash_for_index_cb (sc,
+                           &p->data.file.file_id);
+      else
+        GNUNET_CRYPTO_hash_file (sc->h->sched,
+                                 GNUNET_SCHEDULER_PRIORITY_IDLE,
+                                 GNUNET_NO,
+                                 p->data.file.filename,
+                                 HASHING_BLOCKSIZE,
+                                 &hash_for_index_cb,
+                                 sc);
       return;
     }
   publish_content (sc);
diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c
index 10faf4cf9..3d39cb932 100644
--- a/src/fs/fs_unindex.c
+++ b/src/fs/fs_unindex.c
@@ -20,12 +20,9 @@
 
 /**
  * @file fs/fs_unindex.c
- * @author Krista Bennett
+ * @author Krista Grothoff
  * @author Christian Grothoff
  * @brief Unindex file.
- *
- * TODO:
- * - code cleanup (share more with upload.c)
  */
 #include "platform.h"
 #include "gnunet_constants.h"
diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c
index dc97b2cb8..5148c4675 100644
--- a/src/fs/gnunet-service-fs.c
+++ b/src/fs/gnunet-service-fs.c
@@ -990,8 +990,14 @@ hash_for_index_val (void *cls,
                      sizeof(GNUNET_HashCode))) )
     {
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                  _("Hash mismatch trying to index file `%s'\n"),
-                  ii->filename);
+                  _("Hash mismatch trying to index file `%s' which has hash `%s'\n"),
+                  ii->filename,
+                  GNUNET_h2s (res));
+#if DEBUG_FS
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Wanted `%s'\n",
+                  GNUNET_h2s (&ii->file_id));
+#endif
       GNUNET_SERVER_transmit_context_append (ii->tc,
                                              NULL, 0,
                                              GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED);
@@ -1058,6 +1064,14 @@ handle_index_start (void *cls,
       signal_index_ok (ii);
       return;
     }
+#if DEBUG_FS
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n",
+              (unsigned long long) ino,
+              (unsigned long long) myino,
+              (unsigned int) dev,
+              (unsigned int) mydev);
+#endif
   /* slow validation, need to hash full file (again) */
   GNUNET_CRYPTO_hash_file (sched,
                            GNUNET_SCHEDULER_PRIORITY_IDLE,
@@ -1165,7 +1179,13 @@ handle_unindex (void *cls,
         }
       pos = next;
     }
-  if (GNUNET_YES == found)
+#if DEBUG_FS
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Client requested unindexing of file `%s': %s\n",
+              GNUNET_h2s (&um->file_id),
+              found ? "found" : "not found");
+#endif
+  if (GNUNET_YES == found)    
     write_index_list ();
   tc = GNUNET_SERVER_transmit_context_create (client);
   GNUNET_SERVER_transmit_context_append (tc,
diff --git a/src/fs/test_fs_download.c b/src/fs/test_fs_download.c
index 1d1813c56..9b85e1ca1 100644
--- a/src/fs/test_fs_download.c
+++ b/src/fs/test_fs_download.c
@@ -116,7 +116,7 @@ progress_cb (void *cls,
     case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
       printf ("Publishing complete, %llu kbps.\n",
               (unsigned long long) (FILESIZE * 1000 / (1+GNUNET_TIME_absolute_get_duration (start).value) / 1024));
-      fn = GNUNET_DISK_mktemp ("gnunet-download-test-dstXXXXXX");
+      fn = GNUNET_DISK_mktemp ("gnunet-download-test-dst");
       start = GNUNET_TIME_absolute_get ();
       download = GNUNET_FS_download_start (fs,
                                            event->value.publish.specifics.completed.chk_uri,
diff --git a/src/fs/test_fs_unindex.c b/src/fs/test_fs_unindex.c
index 0b7173115..3dcf7b7f8 100644
--- a/src/fs/test_fs_unindex.c
+++ b/src/fs/test_fs_unindex.c
@@ -29,7 +29,7 @@
 #include "gnunet_arm_service.h"
 #include "gnunet_fs_service.h"
 
-#define VERBOSE GNUNET_NO
+#define VERBOSE GNUNET_YES
 
 #define START_ARM GNUNET_YES
 
@@ -248,7 +248,7 @@ run (void *cls,
 
   sched = s;
   setup_peer (&p1, "test_fs_unindex_data.conf");
-  fn = GNUNET_DISK_mktemp ("gnunet-unindex-test-dstXXXXXX");
+  fn = GNUNET_DISK_mktemp ("gnunet-unindex-test-dst");
   fs = GNUNET_FS_start (sched,
                         cfg,
                         "test-fs-unindex",