-towards external meta scanner process

3 files changed, 292 insertions, 72 deletions

diff --git a/src/fs/fs_dirmetascan.c b/src/fs/fs_dirmetascan.c
index 6d97f1553..69c7770db 100644
--- a/src/fs/fs_dirmetascan.c
+++ b/src/fs/fs_dirmetascan.c
@@ -68,11 +68,20 @@ struct GNUNET_FS_DirScanner
   /**
    * After the scan is finished, it will contain a pointer to the
    * top-level directory entry in the directory tree built by the
-   * scanner.  Must only be manipulated by the thread for the
-   * duration of the thread's runtime.
+   * scanner. 
    */
   struct GNUNET_FS_ShareTreeItem *toplevel;
 
+  /**
+   * Current position during processing.
+   */
+  struct GNUNET_FS_ShareTreeItem *pos;
+
+  /**
+   * Arguments for helper.
+   */
+  char *args[4];
+
 };
 
 
@@ -121,6 +130,76 @@ GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
 
 
 /**
+ * Move in the directory from the given position to the next file
+ * in DFS traversal.
+ *
+ * @param pos current position
+ * @return next file, NULL for none
+ */
+static struct GNUNET_FS_ShareTreeItem *
+advance (struct GNUNET_FS_ShareTreeItem *pos)
+{
+  int moved;
+  
+  GNUNET_assert (NULL != pos);
+  moved = 0; /* must not terminate, even on file, otherwise "normal" */
+  while ( (pos->is_directory) ||
+          (0 == moved) )
+  {
+    if ( (moved != -1) &&
+         (NULL != pos->children_head) )
+    {
+      pos = pos->children_head;
+      moved = 1; /* can terminate if file */
+      continue;
+    }
+    if (NULL != pos->next)
+    {
+      pos = pos->next;
+      moved = 1; /* can terminate if file */
+      continue;
+    }
+    if (NULL != pos->parent)
+    {
+      pos = pos->parent;
+      moved = -1; /* force move to 'next' or 'parent' */
+      continue;
+    }
+    /* no more options, end of traversal */
+    return NULL;
+  }
+  return pos;
+}
+
+
+/**
+ * Add another child node to the tree.
+ *
+ * @param parent parent of the child, NULL for top level
+ * @param filename name of the file or directory
+ * @param is_directory GNUNET_YES for directories
+ * @return new entry that was just created
+ */
+static struct GNUNET_FS_ShareTreeItem *
+expand_tree (struct GNUNET_FS_ShareTreeItem *parent,
+             const char *filename,
+             int is_directory)
+{
+  struct GNUNET_FS_ShareTreeItem *chld;
+
+  chld = GNUNET_malloc (sizeof (struct GNUNET_FS_ShareTreeItem));
+  chld->parent = parent;
+  chld->filename = GNUNET_strdup (filename);
+  chld->is_directory = is_directory;
+  if (NULL != parent)
+      GNUNET_CONTAINER_DLL_insert (parent->children_head,
+                                   parent->children_tail,
+                                   chld);  
+  return chld;
+}
+
+
+/**
  * Called every time there is data to read from the scanner.
  * Calls the scanner progress handler.
  *
@@ -134,35 +213,145 @@ process_helper_msgs (void *cls,
                      const struct GNUNET_MessageHeader *msg)
 {
   struct GNUNET_FS_DirScanner *ds = cls;
-  ds++;
-#if 0
-  enum GNUNET_FS_DirScannerProgressUpdateReason reason;
-  size_t filename_len;
-  int is_directory;
-  char *filename;
-
-  /* Process message. If message is malformed or can't be read, end the scanner */
-  /* read successfully, notify client about progress */
+  const char *filename;
+  size_t left;
+
+  left = ntohs (msg->size) - sizeof (struct GNUNET_MessageHeader);
+  filename = (const char*) &msg[1];
+  switch (ntohs (msg->type))
+  {
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE:
+    if (filename[left-1] != '\0')
+    {
+      GNUNET_break (0);
+      break;
+    }
+    ds->progress_callback (ds->progress_callback_cls, 
+                           filename, GNUNET_NO,
+                           GNUNET_FS_DIRSCANNER_FILE_START);
+    expand_tree (ds->pos,
+                 filename, GNUNET_NO);
+    return;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY:
+    if (filename[left-1] != '\0')
+    {
+      GNUNET_break (0);
+      break;
+    }
+    if (0 == strcmp ("..", filename))
+    {
+      if (NULL == ds->pos)
+      {
+        GNUNET_break (0);
+        break;
+      }
+      ds->pos = ds->pos->parent;
+      return;
+    }
+    ds->progress_callback (ds->progress_callback_cls, 
+                           filename, GNUNET_YES,
+                           GNUNET_FS_DIRSCANNER_FILE_START);
+    ds->pos = expand_tree (ds->pos,
+                           filename, GNUNET_YES);
+    if (NULL == ds->toplevel)
+      ds->toplevel = ds->pos;
+    return;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR:
+    break;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE:
+    if (filename[left-1] != '\0')
+      break;
+    ds->progress_callback (ds->progress_callback_cls, 
+                           filename, GNUNET_SYSERR,
+                           GNUNET_FS_DIRSCANNER_FILE_IGNORED);
+    return;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE:
+    if (0 != left)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    ds->progress_callback (ds->progress_callback_cls, 
+                           NULL, GNUNET_SYSERR,
+                           GNUNET_FS_DIRSCANNER_ALL_COUNTED);
+    ds->pos = ds->toplevel;
+    if (ds->pos->is_directory)
+      ds->pos = advance (ds->pos);
+    return;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA:
+    {
+      size_t nlen;
+      const char *end;
+      
+      if (NULL == ds->pos)
+      {
+        GNUNET_break (0);
+        break;
+      }
+      end = memchr (filename, 0, left);
+      if (NULL == end)
+      {
+        GNUNET_break (0);
+        break;
+      }
+      end++;
+      nlen = end - filename;
+      left -= nlen;
+      if (0 != strcmp (filename,
+                       ds->pos->filename))
+      {
+        GNUNET_break (0);
+        break;
+      }
+      ds->progress_callback (ds->progress_callback_cls, 
+                             filename, GNUNET_YES,
+                             GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED);
+      if (0 < left)
+      {
+        ds->pos->meta = GNUNET_CONTAINER_meta_data_deserialize (end, left);
+        if (NULL == ds->pos->meta)
+        {
+          GNUNET_break (0);
+          break;
+        }
+        /* having full filenames is too dangerous; always make sure we clean them up */
+        ds->pos->short_filename = GNUNET_strdup (GNUNET_STRINGS_get_short_name (filename));
+        GNUNET_CONTAINER_meta_data_delete (ds->pos->meta, 
+                                           EXTRACTOR_METATYPE_FILENAME,
+                                           NULL, 0);
+        GNUNET_CONTAINER_meta_data_insert (ds->pos->meta, "<libgnunetfs>",
+                                           EXTRACTOR_METATYPE_FILENAME,
+                                           EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+                                           ds->pos->short_filename, 
+                                           strlen (ds->pos->short_filename) + 1);
+      }
+      ds->pos = advance (ds->pos);      
+      return;
+    }
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED:
+    if (NULL != ds->pos)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    if (0 != left)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    ds->progress_callback (ds->progress_callback_cls, 
+                           NULL, GNUNET_SYSERR,
+                           GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
+    
+    return;
+  default:
+    GNUNET_break (0);
+    break;
+  }
   ds->progress_callback (ds->progress_callback_cls, 
-                         ds, 
-                         filename,
-                         is_directory, 
-                         reason);
-  GNUNET_free (filename);
-
-
-  /* having full filenames is too dangerous; always make sure we clean them up */
-  item->short_filename = GNUNET_strdup (GNUNET_STRINGS_get_short_name (filename));
-
-  GNUNET_CONTAINER_meta_data_delete (item->meta, 
-                                     EXTRACTOR_METATYPE_FILENAME,
-                                     NULL, 0);
-  GNUNET_CONTAINER_meta_data_insert (item->meta, "<libgnunetfs>",
-                                     EXTRACTOR_METATYPE_FILENAME,
-                                     EXTRACTOR_METAFORMAT_UTF8, "text/plain",
-                                     item->short_filename, 
-                                     strlen (item->short_filename) + 1);
-#endif
+                         NULL, GNUNET_SYSERR,
+                         GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
+
 }
 
 
@@ -185,7 +374,6 @@ GNUNET_FS_directory_scan_start (const char *filename,
   struct stat sbuf;
   char *filename_expanded;
   struct GNUNET_FS_DirScanner *ds;
-  char *args[4];
 
   if (0 != STAT (filename, &sbuf))
     return NULL;
@@ -199,13 +387,16 @@ GNUNET_FS_directory_scan_start (const char *filename,
   ds->progress_callback = cb;
   ds->progress_callback_cls = cb_cls;
   ds->filename_expanded = filename_expanded;
-  ds->ex_arg = GNUNET_strdup ((disable_extractor) ? "-" : ex);
-  args[0] = "gnunet-helper-fs-publish";
-  args[1] = ds->filename_expanded;
-  args[2] = ds->ex_arg;
-  args[3] = NULL;
+  if (disable_extractor)  
+    ds->ex_arg = GNUNET_strdup ("-");
+  else 
+    ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL;
+  ds->args[0] = "gnunet-helper-fs-publish";
+  ds->args[1] = ds->filename_expanded;
+  ds->args[2] = ds->ex_arg;
+  ds->args[3] = NULL;
   ds->helper = GNUNET_HELPER_start ("gnunet-helper-fs-publish",
-                                    args,
+                                    ds->args,
                                     &process_helper_msgs,
                                     ds);
   if (NULL == ds->helper)
diff --git a/src/fs/gnunet-helper-fs-publish.c b/src/fs/gnunet-helper-fs-publish.c
index 21ec19fe9..7accf79ac 100644
--- a/src/fs/gnunet-helper-fs-publish.c
+++ b/src/fs/gnunet-helper-fs-publish.c
@@ -223,9 +223,10 @@ scan_callback (void *cls,
     return GNUNET_SYSERR;
   }
   chld->parent = rc->parent;
-  GNUNET_CONTAINER_DLL_insert (rc->parent->children_head,
-                               rc->parent->children_tail,
-                               chld);
+  if (NULL != rc->parent)
+    GNUNET_CONTAINER_DLL_insert (rc->parent->children_head,
+                                 rc->parent->children_tail,
+                                 chld);
   return GNUNET_OK;
 }
 
@@ -282,6 +283,10 @@ preprocess_file (const char *filename,
       free_tree (item);
       return GNUNET_SYSERR;
     }
+    if (GNUNET_OK !=
+        write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY,
+                       "..", 3))
+      return GNUNET_SYSERR;
   }
   *dst = item;
   return GNUNET_OK;
@@ -379,7 +384,8 @@ int main(int argc,
   }
   filename_expanded = argv[1];
   ex = argv[2];
-  if (0 != strcmp (ex, "-"))
+  if ( (ex == NULL) ||
+       (0 != strcmp (ex, "-")) )
   {
     plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
     if (NULL != ex)
diff --git a/src/fs/gnunet-publish.c b/src/fs/gnunet-publish.c
index 05bd66e31..9cc55a290 100644
--- a/src/fs/gnunet-publish.c
+++ b/src/fs/gnunet-publish.c
@@ -425,50 +425,67 @@ directory_trim_complete ()
 }
 
 
+/**
+ * Function called by the directory scanner as we build the tree
+ * that we will need to publish later.
+ *
+ * @param cls closure
+ * @param filename which file we are making progress on
+ * @param is_directory GNUNET_YES if this is a directory,
+ *                     GNUNET_NO if this is a file
+ *                     GNUNET_SYSERR if it is neither (or unknown)
+ * @param reason kind of progress we are making
+ */
 static void
-directory_scan_cb (void *cls, struct GNUNET_FS_DirScanner *ds,
+directory_scan_cb (void *cls, 
                    const char *filename, 
                    int is_directory,
                    enum GNUNET_FS_DirScannerProgressUpdateReason reason)
 {
   switch (reason)
   {
-    case GNUNET_FS_DIRSCANNER_FILE_START:
+  case GNUNET_FS_DIRSCANNER_FILE_START:
+    if (verbose > 1)
+    {
       if (is_directory)
         FPRINTF (stdout, _("Scanning directory `%s'.\n"), filename);
       else
         FPRINTF (stdout, _("Scanning file `%s'.\n"), filename);      
-      break;
-    case GNUNET_FS_DIRSCANNER_ALL_COUNTED:
+    }
+    break;
+  case GNUNET_FS_DIRSCANNER_FILE_IGNORED:
+    FPRINTF (stderr, 
+             _("There was trouble processing file `%s', skipping it.\n"),
+             filename);
+    break;
+  case GNUNET_FS_DIRSCANNER_ALL_COUNTED:
+    if (verbose)
       FPRINTF (stdout, "%s", _("Preprocessing complete.\n"));      
-      break;
-    case GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED:
+    break;
+  case GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED:
+    if (verbose > 2)
       FPRINTF (stdout, _("Extracting meta data from file `%s' complete.\n"), filename);      
-      break;
-    case GNUNET_FS_DIRSCANNER_DOES_NOT_EXIST:
-      FPRINTF (stdout, 
-               _("There was trouble processing file `%s', skipping it.\n"),
-               filename);
-      break;
-    case GNUNET_FS_DIRSCANNER_FINISHED:
-      FPRINTF (stdout, "%s", _("Scanner has finished.\n"));
-      directory_scan_result = GNUNET_FS_directory_scan_get_result (ds);
-      ds = NULL;
-      GNUNET_FS_share_tree_trim (directory_scan_result);
-      directory_trim_complete ();
-      break;
-    case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR:
-      FPRINTF (stdout, "%s", _("Internal error scanning directory.\n"));
-      GNUNET_FS_directory_scan_abort (ds);
-      ds = NULL;
-      if (namespace != NULL)
-        GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
-      GNUNET_FS_stop (ctx);
-      ret = 1;
-      break;
-    default:
-      GNUNET_assert (0);
-      break;
+    break;
+  case GNUNET_FS_DIRSCANNER_FINISHED:
+    if (verbose > 1)
+      FPRINTF (stdout, "%s", _("Meta data extraction has finished.\n"));
+    directory_scan_result = GNUNET_FS_directory_scan_get_result (ds);
+    ds = NULL;
+    GNUNET_FS_share_tree_trim (directory_scan_result);
+    directory_trim_complete ();
+    break;
+  case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR:
+    FPRINTF (stdout, "%s", _("Internal error scanning directory.\n"));
+    GNUNET_FS_directory_scan_abort (ds);
+    ds = NULL;
+    if (namespace != NULL)
+      GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
+    GNUNET_FS_stop (ctx);
+    ret = 1;
+    break;
+  default:
+    GNUNET_assert (0);
+    break;
   }
   fflush (stdout);
 }
@@ -587,6 +604,12 @@ run (void *cls, char *const *args, const char *cfgfile,
                                        disable_extractor, 
                                        ex, 
                                        &directory_scan_cb, NULL);
+  if (NULL == ds)
+  {
+    FPRINTF (stderr,
+             "%s", _("Failed to start meta directory scanner.  Is gnunet-helper-publish-fs installed?\n"));
+    return;
+  }
   kill_task =
       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &do_stop_task,
                                     NULL);