-LRN: make directory scanner run via scheduler

3 files changed, 228 insertions, 12 deletions

diff --git a/src/fs/gnunet-fs-gtk-event_handler.c b/src/fs/gnunet-fs-gtk-event_handler.c
index 4658d23e..11a73be4 100644
--- a/src/fs/gnunet-fs-gtk-event_handler.c
+++ b/src/fs/gnunet-fs-gtk-event_handler.c
@@ -1215,11 +1215,17 @@ handle_publish_stop (struct PublishEntry *ent)
     /* get piter from parent */
     path = gtk_tree_row_reference_get_path (ent->rr);
     tm = gtk_tree_row_reference_get_model (ent->rr);
-    if (TRUE != gtk_tree_model_get_iter (tm, &iter, path))
-      GNUNET_break (0);
-    else
-      (void) gtk_tree_store_remove (GTK_TREE_STORE (tm), &iter);
-    gtk_tree_path_free (path);
+    /* This is a child of a directory, and we've had that directory
+     * freed already
+     */
+    if (path != NULL)
+    {
+      if (TRUE != gtk_tree_model_get_iter (tm, &iter, path))
+        GNUNET_break (0);
+      else
+        (void) gtk_tree_store_remove (GTK_TREE_STORE (tm), &iter);
+      gtk_tree_path_free (path);
+    }
   }
   gtk_tree_row_reference_free (ent->rr);
   if (ent->uri != NULL)
diff --git a/src/fs/gnunet-fs-gtk-main_window_adv_pseudonym.c b/src/fs/gnunet-fs-gtk-main_window_adv_pseudonym.c
index 50fdf4d1..b3075eb1 100644
--- a/src/fs/gnunet-fs-gtk-main_window_adv_pseudonym.c
+++ b/src/fs/gnunet-fs-gtk-main_window_adv_pseudonym.c
@@ -203,7 +203,7 @@ GNUNET_GTK_select_pseudonym_dialog_response_cb (GtkDialog * dialog,
   nds->bo.anonymity_level = 1;
   nds->fip =
       GNUNET_FS_file_information_create_empty_directory (NULL, NULL, NULL, meta,
-                                                         &nds->bo);
+                                                         &nds->bo, NULL);
   GNUNET_CONTAINER_meta_data_destroy (meta);
   GNUNET_FS_GTK_edit_publish_dialog (transient, nds->do_index,
                                      nds->short_fn, nds->bo, nds->fip, FALSE, anon_liststore,
diff --git a/src/fs/gnunet-fs-gtk-main_window_file_publish.c b/src/fs/gnunet-fs-gtk-main_window_file_publish.c
index 6791e61b..ee74ec64 100644
--- a/src/fs/gnunet-fs-gtk-main_window_file_publish.c
+++ b/src/fs/gnunet-fs-gtk-main_window_file_publish.c
@@ -321,7 +321,7 @@ create_dir_at_iter (struct MainPublishingDialogContext *ctx, const char *name,
   row_reference = gtk_tree_row_reference_new (ctx->file_info_treemodel, path);
   gtk_tree_path_free (path);
   fi = GNUNET_FS_file_information_create_empty_directory
-      (GNUNET_FS_GTK_get_fs_handle (), row_reference, NULL, meta, bo);
+      (GNUNET_FS_GTK_get_fs_handle (), row_reference, NULL, meta, bo, name);
   GNUNET_CONTAINER_meta_data_destroy (meta);
   gtk_tree_store_set (GTK_TREE_STORE (ctx->file_info_treemodel), pos, 0, MARKER_DIR_FILE_SIZE, 1, (gboolean) GNUNET_NO,
                       2, name, 3, (guint) bo->anonymity_level, 4,
@@ -391,6 +391,29 @@ struct AddDirContext
   GtkTreeIter *parent;
 
   /**
+   * Pointer to the context of the parent directory, or NULL
+   * for top-level.
+   */
+  struct AddDirContext *parent_ctx;
+
+  /**
+   * Directory iterator of the parent directory, or NULL
+   * for top-level.
+   */
+  struct GNUNET_DISK_DirectoryIterator *parent_iter;
+
+  /**
+   * Filename parent directory was processing (filename of THIS directory)
+   * when it started child directory iterator. NULL for top-level.
+   */
+  char *parent_filename;
+
+  /**
+   * Publication data for this directory.
+   */
+  struct PublishData *pd;
+
+  /**
    * Master publication dialog context (used to access treestore)
    */
   struct MainPublishingDialogContext *ctx;
@@ -585,7 +608,7 @@ add_entry_to_ts (GtkTreeStore * ts, GtkTreeIter * iter, const char *filename,
     GNUNET_FS_uri_ksk_add_keyword (ksk_uri, GNUNET_FS_DIRECTORY_MIME,
                                    GNUNET_NO);
     fi = GNUNET_FS_file_information_create_empty_directory
-        (GNUNET_FS_GTK_get_fs_handle (), row_reference, ksk_uri, meta, bo);
+        (GNUNET_FS_GTK_get_fs_handle (), row_reference, ksk_uri, meta, bo, filename);
   }
   else
   {
@@ -796,13 +819,188 @@ scan_directory (void *cls, const char *filename)
   return GNUNET_OK;
 }
 
+static void
+add_dir_callback (void *cls, struct GNUNET_DISK_DirectoryIterator * di,
+    const char *filename, const char *dirname);
+
 
 static void
-add_dir_finish (struct AddDirContext *scan_ctx)
+child_dir_finished_publishing (struct AddDirContext *adc)
 {
-  GNUNET_free (scan_ctx);
+  const char *ss;
+  const char *short_fn;
+  const char *user;
+  GNUNET_HashCode hc;
+
+  GNUNET_CONTAINER_multihashmap_destroy (adc->metamap);
+  if (adc->parent_ctx)
+    adc->parent_ctx->dir_entry_count += 1;
+  short_fn = adc->parent_filename;
+  while ( (NULL != (ss = strstr (short_fn, DIR_SEPARATOR_STR))) && 
+   (ss[1] != '\0') )
+      short_fn = 1 + ss;
+  user = getenv ("USER");
+  if ((user == NULL) || (0 != strncasecmp (user, short_fn, strlen(user))))
+  {
+    /* only use filename if it doesn't match $USER */
+    GNUNET_CONTAINER_meta_data_insert (adc->pd->meta, "<gnunet-gtk>",
+        EXTRACTOR_METATYPE_FILENAME,
+        EXTRACTOR_METAFORMAT_UTF8,
+        "text/plain", short_fn,
+        strlen (short_fn) + 1);
+    GNUNET_CONTAINER_meta_data_insert (adc->pd->meta, "<gnunet-gtk>",
+        EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
+        EXTRACTOR_METAFORMAT_UTF8,
+        "text/plain", short_fn,
+        strlen (short_fn) + 1);
+  }
+  if (adc->parent_ctx != NULL)
+  {
+    if (adc->parent_ctx->metamap != NULL)
+    {
+      GNUNET_CRYPTO_hash (adc->parent_filename, strlen (adc->parent_filename), &hc);
+      GNUNET_CONTAINER_multihashmap_put (adc->parent_ctx->metamap, &hc, adc->pd,
+                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
+      /* FIXME: what if this put fails? I think it actually can... Why unique only? */
+      GNUNET_FS_uri_ksk_get_keywords (adc->pd->ksk_uri, &add_to_keyword_counter, adc->parent_ctx->keywordcounter);
+    }
+    else
+    {
+      GNUNET_assert (adc->parent_ctx->keywordcounter == NULL);
+      add_entry_to_ts (GTK_TREE_STORE (adc->ctx->file_info_treemodel),
+          &adc->pd->iter, adc->parent_filename, &adc->bo, adc->do_index,
+          adc->pd->ksk_uri, NULL, adc->pd->meta);
+    }
+  }
+  GNUNET_FS_uri_destroy (adc->exclude_ksk);
+  if (adc->parent_ctx != NULL)
+    add_dir_callback (adc->parent_ctx, adc->parent_iter,
+      adc->parent_filename, NULL);
+  GNUNET_free_non_null (adc->parent_filename);
+  GNUNET_free (adc);
 }
 
+static void
+publish_dir_callback (void *cls, struct GNUNET_DISK_DirectoryIterator *di,
+    const char *filename, const char *dirname)
+{
+  struct AddDirContext *adc = cls;
+  struct PublishData *pd;
+  GNUNET_HashCode hc;
+  struct stat sbuf;
+
+  GNUNET_CRYPTO_hash (filename, strlen (filename), &hc);
+  pd = GNUNET_CONTAINER_multihashmap_get (adc->metamap, &hc);
+  /* stat() is needed here to match the stat() call add_dir_callback() does */
+  if ((pd != NULL) && (0 == STAT (filename, &sbuf)))
+  {
+    add_entry_to_ts (GTK_TREE_STORE (adc->ctx->file_info_treemodel),
+        &pd->iter, filename, &adc->bo, adc->do_index,
+        pd->ksk_uri, adc->exclude_ksk, pd->meta);
+  }
+  if (pd != NULL)
+  {
+    GNUNET_CONTAINER_multihashmap_remove (adc->metamap, &hc, pd);
+    GNUNET_free (pd);
+  }
+  if (GNUNET_DISK_directory_iterator_next (di, GNUNET_NO) != GNUNET_YES)
+  {
+    child_dir_finished_publishing (adc);
+  }
+}
+
+static void
+child_dir_finished_scanning (struct AddDirContext *adc)
+{
+  adc->pd->ksk_uri = process_keywords (adc);
+  adc->pd->meta = GNUNET_CONTAINER_meta_data_create ();
+  adc->exclude_ksk = GNUNET_FS_uri_dup (adc->pd->ksk_uri);
+  if (GNUNET_YES != GNUNET_DISK_directory_iterator_start (GNUNET_SCHEDULER_PRIORITY_IDLE,
+      adc->parent_filename, &publish_dir_callback, adc))
+    child_dir_finished_publishing (adc);
+}
+
+/**
+ * Function called by directory iterator.
+ *
+ * @param cls closure
+ * @param di argument to pass to "GNUNET_DISK_directory_iterator_next" to
+ *           get called on the next entry (or finish cleanly);
+ *           NULL on error (will be the last call in that case)
+ * @param filename complete filename (absolute path)
+ * @param dirname directory name (absolute path)
+ */
+static void
+add_dir_callback (void *cls, struct GNUNET_DISK_DirectoryIterator * di,
+    const char *filename, const char *dirname)
+{
+  struct AddDirContext *adc = cls, *child_adc;
+  struct stat sbuf;
+  int will_continue;
+
+  /* dirname == NULL is a special case when add_dir_callback() is called by
+   * the child directory iterator callback.
+   */
+  if (dirname == NULL)
+  {
+    if (di == NULL)
+      return;
+    else
+      will_continue = GNUNET_DISK_directory_iterator_next (di, GNUNET_NO);
+  }
+  else if (0 != STAT (filename, &sbuf))
+  {
+    GNUNET_assert (di != NULL);
+    will_continue = GNUNET_DISK_directory_iterator_next (di, GNUNET_NO);
+  }
+  else
+  {
+    if (!S_ISDIR (sbuf.st_mode))
+    {
+      GNUNET_assert (adc->metamap != NULL);
+      extract_file (adc, filename);
+      will_continue = GNUNET_DISK_directory_iterator_next (di, GNUNET_NO);
+    }
+    else
+    {
+      child_adc = GNUNET_malloc (sizeof (struct AddDirContext));
+      will_continue = GNUNET_DISK_directory_iterator_start (GNUNET_SCHEDULER_PRIORITY_IDLE,
+          filename, &add_dir_callback, child_adc);
+      if (will_continue != GNUNET_YES)
+      {
+        /* Ignore empty directories for now */
+        GNUNET_free (child_adc);
+        will_continue = GNUNET_DISK_directory_iterator_next (di, GNUNET_NO);
+      }
+      else
+      {
+        child_adc->metamap = GNUNET_CONTAINER_multihashmap_create (1024);
+        child_adc->keywordcounter = GNUNET_CONTAINER_multihashmap_create (1024);
+        child_adc->dir_entry_count = 0;
+        child_adc->parent_ctx = adc;
+        child_adc->parent_iter = di;
+        child_adc->bo = adc->bo;
+        child_adc->do_index = adc->do_index;
+        child_adc->ctx = adc->ctx;
+        child_adc->pd = GNUNET_malloc (sizeof (struct PublishData));
+        child_adc->parent_filename = GNUNET_strdup (filename);
+        gtk_tree_store_insert_before (GTK_TREE_STORE (adc->ctx->file_info_treemodel),
+            &child_adc->pd->iter, adc->parent, NULL);
+        child_adc->parent = &child_adc->pd->iter;
+        /* Don't advance directory iterator here, child iterator will call
+         * us with dirname == NULL for that later.
+         */
+      }
+    }
+  }
+  if (will_continue != GNUNET_YES)
+  {
+    child_dir_finished_scanning (adc);
+  }
+  return;
+}
+
+
 /**
  * Add a directory to the tree model.
  *
@@ -816,6 +1014,7 @@ add_dir (struct MainPublishingDialogContext *ctx, const char *filename,
 {
   struct stat sbuf;
   struct AddDirContext *scan_ctx;
+  char *filename_expanded;
 
   if (0 != STAT (filename, &sbuf))
     return;
@@ -824,12 +1023,17 @@ add_dir (struct MainPublishingDialogContext *ctx, const char *filename,
     GNUNET_break (0);
     return;
   }
+  filename_expanded = GNUNET_STRINGS_filename_expand (filename);
+  if (filename_expanded == NULL)
+    return;
   scan_ctx = GNUNET_malloc (sizeof (struct AddDirContext));
   scan_ctx->bo = *bo;
   scan_ctx->do_index = do_index;
   scan_ctx->ctx = ctx;
-  scan_directory (scan_ctx, filename);
-  add_dir_finish (scan_ctx);
+
+  /* just make sure that dirname is not NULL, it's not really used */
+  add_dir_callback (scan_ctx, NULL, filename_expanded, filename_expanded);
+  GNUNET_free (filename_expanded);
 }
 
 
@@ -1198,15 +1402,21 @@ master_publish_edit_publish_dialog_cb (gpointer cls, int do_index,
                                        gint ret)
 {
   struct EditPublishContext *cbargs = cls;
+  struct GNUNET_FileInformation *fi;
+  const char *filename;
 
   GNUNET_assert (NULL == root);
   if (ret == GTK_RESPONSE_OK)
+  {
     gtk_tree_store_set (GTK_TREE_STORE (cbargs->tm), &cbargs->iter, 1, do_index,
                         2, short_fn, 3, (guint) bo->anonymity_level, 4,
                         (guint) bo->content_priority, 
                         6, (guint64) bo->expiration_time.abs_value,
                         7, (guint) bo->replication_level,                       
                         -1);
+    gtk_tree_model_get (cbargs->tm, &cbargs->iter, 5, &fi);
+    GNUNET_FS_file_information_set_filename (fi, short_fn);
+  }
   GNUNET_free (cbargs);
 }