/*
This file is part of GNUnet
Copyright (C) 2005-2012 GNUnet e.V.
GNUnet is free software: you can redistribute it and/or modify it
under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License,
or (at your option) any later version.
GNUnet is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see .
SPDX-License-Identifier: AGPL3.0-or-later
*/
/**
* @file fs/fs_dirmetascan.c
* @brief code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem'
* from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'.
* @author LRN
* @author Christian Grothoff
*/
#include "platform.h"
#include "gnunet_fs_service.h"
#include "gnunet_scheduler_lib.h"
#include
/**
* An opaque structure a pointer to which is returned to the
* caller to be used to control the scanner.
*/
struct GNUNET_FS_DirScanner
{
/**
* Helper process.
*/
struct GNUNET_HELPER_Handle *helper;
/**
* Expanded filename (as given by the scan initiator).
* The scanner thread stores a copy here, and frees it when it finishes.
*/
char *filename_expanded;
/**
* Second argument to helper process.
*/
char *ex_arg;
/**
* The function that will be called every time there's a progress
* message.
*/
GNUNET_FS_DirScannerProgressCallback progress_callback;
/**
* A closure for progress_callback.
*/
void *progress_callback_cls;
/**
* After the scan is finished, it will contain a pointer to the
* top-level directory entry in the directory tree built by the
* scanner.
*/
struct GNUNET_FS_ShareTreeItem *toplevel;
/**
* Current position during processing.
*/
struct GNUNET_FS_ShareTreeItem *pos;
/**
* Task scheduled when we are done.
*/
struct GNUNET_SCHEDULER_Task *stop_task;
/**
* Arguments for helper.
*/
char *args[4];
};
/**
* Abort the scan. Must not be called from within the progress_callback
* function.
*
* @param ds directory scanner structure
*/
void
GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds)
{
/* terminate helper */
if (NULL != ds->helper)
GNUNET_HELPER_stop (ds->helper, GNUNET_NO);
/* free resources */
if (NULL != ds->toplevel)
GNUNET_FS_share_tree_free (ds->toplevel);
if (NULL != ds->stop_task)
GNUNET_SCHEDULER_cancel (ds->stop_task);
GNUNET_free_non_null (ds->ex_arg);
GNUNET_free (ds->filename_expanded);
GNUNET_free (ds);
}
/**
* Obtain the result of the scan after the scan has signalled
* completion. Must not be called prior to completion. The 'ds' is
* freed as part of this call.
*
* @param ds directory scanner structure
* @return the results of the scan (a directory tree)
*/
struct GNUNET_FS_ShareTreeItem *
GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
{
struct GNUNET_FS_ShareTreeItem *result;
/* check that we're actually done */
GNUNET_assert (NULL == ds->helper);
/* preserve result */
result = ds->toplevel;
ds->toplevel = NULL;
GNUNET_FS_directory_scan_abort (ds);
return result;
}
/**
* Move in the directory from the given position to the next file
* in DFS traversal.
*
* @param pos current position
* @return next file, NULL for none
*/
static struct GNUNET_FS_ShareTreeItem *
advance (struct GNUNET_FS_ShareTreeItem *pos)
{
int moved;
GNUNET_assert (NULL != pos);
moved = 0; /* must not terminate, even on file, otherwise "normal" */
while ((pos->is_directory == GNUNET_YES) || (0 == moved))
{
if ((moved != -1) && (NULL != pos->children_head))
{
pos = pos->children_head;
moved = 1; /* can terminate if file */
continue;
}
if (NULL != pos->next)
{
pos = pos->next;
moved = 1; /* can terminate if file */
continue;
}
if (NULL != pos->parent)
{
pos = pos->parent;
moved = -1; /* force move to 'next' or 'parent' */
continue;
}
/* no more options, end of traversal */
return NULL;
}
return pos;
}
/**
* Add another child node to the tree.
*
* @param parent parent of the child, NULL for top level
* @param filename name of the file or directory
* @param is_directory GNUNET_YES for directories
* @return new entry that was just created
*/
static struct GNUNET_FS_ShareTreeItem *
expand_tree (struct GNUNET_FS_ShareTreeItem *parent,
const char *filename,
int is_directory)
{
struct GNUNET_FS_ShareTreeItem *chld;
size_t slen;
chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
chld->parent = parent;
chld->filename = GNUNET_strdup (filename);
GNUNET_asprintf (&chld->short_filename,
"%s%s",
GNUNET_STRINGS_get_short_name (filename),
is_directory == GNUNET_YES ? "/" : "");
/* make sure we do not end with '//' */
slen = strlen (chld->short_filename);
if ((slen >= 2) && (chld->short_filename[slen - 1] == '/') &&
(chld->short_filename[slen - 2] == '/'))
chld->short_filename[slen - 1] = '\0';
chld->is_directory = is_directory;
if (NULL != parent)
GNUNET_CONTAINER_DLL_insert (parent->children_head,
parent->children_tail,
chld);
return chld;
}
/**
* Task run last to shut everything down.
*
* @param cls the 'struct GNUNET_FS_DirScanner'
*/
static void
finish_scan (void *cls)
{
struct GNUNET_FS_DirScanner *ds = cls;
ds->stop_task = NULL;
if (NULL != ds->helper)
{
GNUNET_HELPER_stop (ds->helper, GNUNET_NO);
ds->helper = NULL;
}
ds->progress_callback (ds->progress_callback_cls,
NULL,
GNUNET_SYSERR,
GNUNET_FS_DIRSCANNER_FINISHED);
}
/**
* Called every time there is data to read from the scanner.
* Calls the scanner progress handler.
*
* @param cls the closure (directory scanner object)
* @param msg message from the helper process
* @return #GNUNET_OK on success,
* #GNUNET_NO to stop further processing (no error)
* #GNUNET_SYSERR to stop further processing with error
*/
static int
process_helper_msgs (void *cls, const struct GNUNET_MessageHeader *msg)
{
struct GNUNET_FS_DirScanner *ds = cls;
const char *filename;
size_t left;
#if 0
fprintf (stderr,
"DMS parses %u-byte message of type %u\n",
(unsigned int) ntohs (msg->size),
(unsigned int) ntohs (msg->type));
#endif
left = ntohs (msg->size) - sizeof(struct GNUNET_MessageHeader);
filename = (const char *) &msg[1];
switch (ntohs (msg->type))
{
case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE:
if (filename[left - 1] != '\0')
{
GNUNET_break (0);
break;
}
ds->progress_callback (ds->progress_callback_cls,
filename,
GNUNET_NO,
GNUNET_FS_DIRSCANNER_FILE_START);
if (NULL == ds->toplevel)
{
ds->toplevel = expand_tree (ds->pos, filename, GNUNET_NO);
}
else
{
GNUNET_assert (NULL != ds->pos);
(void) expand_tree (ds->pos, filename, GNUNET_NO);
}
return GNUNET_OK;
case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY:
if (filename[left - 1] != '\0')
{
GNUNET_break (0);
break;
}
if (0 == strcmp ("..", filename))
{
if (NULL == ds->pos)
{
GNUNET_break (0);
break;
}
ds->pos = ds->pos->parent;
return GNUNET_OK;
}
ds->progress_callback (ds->progress_callback_cls,
filename,
GNUNET_YES,
GNUNET_FS_DIRSCANNER_FILE_START);
ds->pos = expand_tree (ds->pos, filename, GNUNET_YES);
if (NULL == ds->toplevel)
ds->toplevel = ds->pos;
return GNUNET_OK;
case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR:
break;
case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE:
if ('\0' != filename[left - 1])
break;
ds->progress_callback (ds->progress_callback_cls,
filename,
GNUNET_SYSERR,
GNUNET_FS_DIRSCANNER_FILE_IGNORED);
return GNUNET_OK;
case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE:
if (0 != left)
{
GNUNET_break (0);
break;
}
if (NULL == ds->toplevel)
break;
ds->progress_callback (ds->progress_callback_cls,
NULL,
GNUNET_SYSERR,
GNUNET_FS_DIRSCANNER_ALL_COUNTED);
ds->pos = ds->toplevel;
if (GNUNET_YES == ds->pos->is_directory)
ds->pos = advance (ds->pos);
return GNUNET_OK;
case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA: {
size_t nlen;
const char *end;
if (NULL == ds->pos)
{
GNUNET_break (0);
break;
}
end = memchr (filename, 0, left);
if (NULL == end)
{
GNUNET_break (0);
break;
}
end++;
nlen = end - filename;
left -= nlen;
if (0 != strcmp (filename, ds->pos->filename))
{
GNUNET_break (0);
break;
}
ds->progress_callback (ds->progress_callback_cls,
filename,
GNUNET_YES,
GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED);
if (0 < left)
{
ds->pos->meta = GNUNET_CONTAINER_meta_data_deserialize (end, left);
if (NULL == ds->pos->meta)
{
GNUNET_break (0);
break;
}
/* having full filenames is too dangerous; always make sure we clean them up */
GNUNET_CONTAINER_meta_data_delete (ds->pos->meta,
EXTRACTOR_METATYPE_FILENAME,
NULL,
0);
/* instead, put in our 'safer' original filename */
GNUNET_CONTAINER_meta_data_insert (ds->pos->meta,
"",
EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
EXTRACTOR_METAFORMAT_UTF8,
"text/plain",
ds->pos->short_filename,
strlen (ds->pos->short_filename)
+ 1);
}
ds->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_meta_data (
ds->pos->meta);
ds->pos = advance (ds->pos);
return GNUNET_OK;
}
case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED:
if (NULL != ds->pos)
{
GNUNET_break (0);
break;
}
if (0 != left)
{
GNUNET_break (0);
break;
}
if (NULL == ds->toplevel)
break;
ds->stop_task = GNUNET_SCHEDULER_add_now (&finish_scan, ds);
return GNUNET_OK;
default:
GNUNET_break (0);
break;
}
ds->progress_callback (ds->progress_callback_cls,
NULL,
GNUNET_SYSERR,
GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
return GNUNET_OK;
}
/**
* Function called if our helper process died.
*
* @param cls the 'struct GNUNET_FS_DirScanner' callback.
*/
static void
helper_died_cb (void *cls)
{
struct GNUNET_FS_DirScanner *ds = cls;
ds->helper = NULL;
if (NULL != ds->stop_task)
return; /* normal death, was finished */
ds->progress_callback (ds->progress_callback_cls,
NULL,
GNUNET_SYSERR,
GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
}
/**
* Start a directory scanner thread.
*
* @param filename name of the directory to scan
* @param disable_extractor #GNUNET_YES to not run libextractor on files (only
* build a tree)
* @param ex if not NULL, must be a list of extra plugins for extractor
* @param cb the callback to call when there are scanning progress messages
* @param cb_cls closure for 'cb'
* @return directory scanner object to be used for controlling the scanner
*/
struct GNUNET_FS_DirScanner *
GNUNET_FS_directory_scan_start (const char *filename,
int disable_extractor,
const char *ex,
GNUNET_FS_DirScannerProgressCallback cb,
void *cb_cls)
{
struct stat sbuf;
char *filename_expanded;
struct GNUNET_FS_DirScanner *ds;
if (0 != stat (filename, &sbuf))
return NULL;
filename_expanded = GNUNET_STRINGS_filename_expand (filename);
if (NULL == filename_expanded)
return NULL;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Starting to scan directory `%s'\n",
filename_expanded);
ds = GNUNET_new (struct GNUNET_FS_DirScanner);
ds->progress_callback = cb;
ds->progress_callback_cls = cb_cls;
ds->filename_expanded = filename_expanded;
if (disable_extractor)
ds->ex_arg = GNUNET_strdup ("-");
else
ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL;
ds->args[0] = "gnunet-helper-fs-publish";
ds->args[1] = ds->filename_expanded;
ds->args[2] = ds->ex_arg;
ds->args[3] = NULL;
ds->helper = GNUNET_HELPER_start (GNUNET_NO,
"gnunet-helper-fs-publish",
ds->args,
&process_helper_msgs,
&helper_died_cb,
ds);
if (NULL == ds->helper)
{
GNUNET_free (filename_expanded);
GNUNET_free (ds);
return NULL;
}
return ds;
}
/* end of fs_dirmetascan.c */