/* This file is part of GNUnet. Copyright (C) 2001--2012 GNUnet e.V. GNUnet is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . SPDX-License-Identifier: AGPL3.0-or-later */ /** * @file fs/gnunet-auto-share.c * @brief automatically publish files on GNUnet * @author Christian Grothoff * * TODO: * - support loading meta data / keywords from resource file * - add stability timer (a la buildbot) */ #include "platform.h" #include "gnunet_util_lib.h" #define MAX_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS, 4) #define MIN_DELAY GNUNET_TIME_UNIT_MINUTES /** * Item in our work queue (or in the set of files/directories * we have successfully published). */ struct WorkItem { /** * PENDING Work is kept in a linked list. */ struct WorkItem *prev; /** * PENDING Work is kept in a linked list. */ struct WorkItem *next; /** * Filename of the work item. */ char *filename; /** * Unique identity for this work item (used to detect * if we need to do the work again). */ struct GNUNET_HashCode id; }; /** * Global return value from 'main'. */ static int ret; /** * Are we running 'verbosely'? */ static unsigned int verbose; /** * Configuration to use. */ static const struct GNUNET_CONFIGURATION_Handle *cfg; /** * Name of the configuration file. */ static char *cfg_filename; /** * Disable extractor option to use for publishing. */ static int disable_extractor; /** * Disable creation time option to use for publishing. */ static int do_disable_creation_time; /** * Handle for the main task that does scanning and working. */ static struct GNUNET_SCHEDULER_Task *run_task; /** * Anonymity level option to use for publishing. */ static unsigned int anonymity_level = 1; /** * Content priority option to use for publishing. */ static unsigned int content_priority = 365; /** * Replication level option to use for publishing. */ static unsigned int replication_level = 1; /** * Top-level directory we monitor to auto-publish. */ static const char *dir_name; /** * Head of linked list of files still to publish. */ static struct WorkItem *work_head; /** * Tail of linked list of files still to publish. */ static struct WorkItem *work_tail; /** * Map from the hash of the filename (!) to a `struct WorkItem` * that was finished. */ static struct GNUNET_CONTAINER_MultiHashMap *work_finished; /** * Set to #GNUNET_YES if we are shutting down. */ static int do_shutdown; /** * Start time of the current round; used to determine how long * one iteration takes (which influences how fast we schedule * the next one). */ static struct GNUNET_TIME_Absolute start_time; /** * Pipe used to communicate 'gnunet-publish' completion (SIGCHLD) via signal. */ static struct GNUNET_DISK_PipeHandle *sigpipe; /** * Handle to the 'gnunet-publish' process that we executed. */ static struct GNUNET_OS_Process *publish_proc; /** * Compute the name of the state database file we will use. */ static char * get_state_file () { char *ret; GNUNET_asprintf (&ret, "%s%s.auto-share", dir_name, (DIR_SEPARATOR == dir_name[strlen(dir_name)-1]) ? "" : DIR_SEPARATOR_STR); return ret; } /** * Load the set of #work_finished items from disk. */ static void load_state () { char *fn; struct GNUNET_BIO_ReadHandle *rh; uint32_t n; struct GNUNET_HashCode id; struct WorkItem *wi; char *emsg; emsg = NULL; fn = get_state_file (); rh = GNUNET_BIO_read_open (fn); GNUNET_free (fn); if (NULL == rh) return; fn = NULL; if (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &n)) goto error; while (n-- > 0) { if ( (GNUNET_OK != GNUNET_BIO_read_string (rh, "filename", &fn, 1024)) || (GNUNET_OK != GNUNET_BIO_read (rh, "id", &id, sizeof (struct GNUNET_HashCode))) ) goto error; wi = GNUNET_new (struct WorkItem); wi->id = id; wi->filename = fn; GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Loaded serialization ID for `%s' is `%s'\n", wi->filename, GNUNET_h2s (&id)); fn = NULL; GNUNET_CRYPTO_hash (wi->filename, strlen (wi->filename), &id); GNUNET_break (GNUNET_OK == GNUNET_CONTAINER_multihashmap_put (work_finished, &id, wi, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); } if (GNUNET_OK == GNUNET_BIO_read_close (rh, &emsg)) return; rh = NULL; error: GNUNET_free_non_null (fn); if (NULL != rh) (void) GNUNET_BIO_read_close (rh, &emsg); GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to load state: %s\n"), emsg); GNUNET_free_non_null (emsg); } /** * Write work item from the #work_finished map to the given write handle. * * @param cls the `struct GNUNET_BIO_WriteHandle *` * @param key key of the item in the map (unused) * @param value the `struct WorkItem` to write * @return #GNUNET_OK to continue to iterate (if write worked) */ static int write_item (void *cls, const struct GNUNET_HashCode *key, void *value) { struct GNUNET_BIO_WriteHandle *wh = cls; struct WorkItem *wi = value; GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Saving serialization ID of file `%s' with value `%s'\n", wi->filename, GNUNET_h2s (&wi->id)); if ( (GNUNET_OK != GNUNET_BIO_write_string (wh, wi->filename)) || (GNUNET_OK != GNUNET_BIO_write (wh, &wi->id, sizeof (struct GNUNET_HashCode))) ) return GNUNET_SYSERR; /* write error, abort iteration */ return GNUNET_OK; } /** * Save the set of #work_finished items on disk. */ static void save_state () { uint32_t n; struct GNUNET_BIO_WriteHandle *wh; char *fn; n = GNUNET_CONTAINER_multihashmap_size (work_finished); fn = get_state_file (); wh = GNUNET_BIO_write_open (fn); if (NULL == wh) { GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to save state to file %s\n"), fn); GNUNET_free (fn); return; } if (GNUNET_OK != GNUNET_BIO_write_int32 (wh, n)) { (void) GNUNET_BIO_write_close (wh); GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to save state to file %s\n"), fn); GNUNET_free (fn); return; } (void) GNUNET_CONTAINER_multihashmap_iterate (work_finished, &write_item, wh); if (GNUNET_OK != GNUNET_BIO_write_close (wh)) GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to save state to file %s\n"), fn); GNUNET_free (fn); } /** * Task run on shutdown. Serializes our current state to disk. * * @param cls closure, unused */ static void do_stop_task (void *cls) { do_shutdown = GNUNET_YES; if (NULL != publish_proc) { GNUNET_OS_process_kill (publish_proc, SIGKILL); return; } if (NULL != run_task) { GNUNET_SCHEDULER_cancel (run_task); run_task = NULL; } } /** * Decide what the next task is (working or scanning) and schedule it. */ static void schedule_next_task (void); /** * Task triggered whenever we receive a SIGCHLD (child * process died). * * @param cls the `struct WorkItem` we were working on */ static void maint_child_death (void *cls) { struct WorkItem *wi = cls; struct GNUNET_HashCode key; enum GNUNET_OS_ProcessStatusType type; unsigned long code; int ret; char c; const struct GNUNET_DISK_FileHandle *pr; const struct GNUNET_SCHEDULER_TaskContext *tc; run_task = NULL; pr = GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_READ); tc = GNUNET_SCHEDULER_get_task_context (); if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_READ_READY)) { /* shutdown scheduled us, someone else will kill child, we should just try again */ run_task = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL, pr, &maint_child_death, wi); return; } /* consume the signal */ GNUNET_break (0 < GNUNET_DISK_file_read (pr, &c, sizeof (c))); ret = GNUNET_OS_process_status (publish_proc, &type, &code); GNUNET_assert (GNUNET_SYSERR != ret); if (GNUNET_NO == ret) { /* process still running? Then where did the SIGCHLD come from? Well, let's declare it spurious (kernel bug?) and keep rolling. */ GNUNET_break (0); run_task = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL, pr, &maint_child_death, wi); return; } GNUNET_assert (GNUNET_OK == ret); GNUNET_OS_process_destroy (publish_proc); publish_proc = NULL; if (GNUNET_YES == do_shutdown) { GNUNET_free (wi->filename); GNUNET_free (wi); return; } if ( (GNUNET_OS_PROCESS_EXITED == type) && (0 == code) ) { GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Publication of `%s' done\n"), wi->filename); GNUNET_CRYPTO_hash (wi->filename, strlen (wi->filename), &key); GNUNET_break (GNUNET_OK == GNUNET_CONTAINER_multihashmap_put (work_finished, &key, wi, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); } else { GNUNET_CONTAINER_DLL_insert_tail (work_head, work_tail, wi); } save_state (); schedule_next_task (); } /** * Signal handler called for SIGCHLD. Triggers the * respective handler by writing to the trigger pipe. */ static void sighandler_child_death () { static char c; int old_errno = errno; /* back-up errno */ GNUNET_break (1 == GNUNET_DISK_file_write (GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_WRITE), &c, sizeof (c))); errno = old_errno; /* restore errno */ } /** * Function called to process work items. * * @param cls closure, NULL */ static void work (void *cls) { static char *argv[14]; static char anon_level[20]; static char content_prio[20]; static char repl_level[20]; struct WorkItem *wi; const struct GNUNET_DISK_FileHandle *pr; int argc; run_task = NULL; wi = work_head; GNUNET_CONTAINER_DLL_remove (work_head, work_tail, wi); argc = 0; argv[argc++] = "gnunet-publish"; if (verbose) argv[argc++] = "-V"; if (disable_extractor) argv[argc++] = "-D"; if (do_disable_creation_time) argv[argc++] = "-d"; argv[argc++] = "-c"; argv[argc++] = cfg_filename; GNUNET_snprintf (anon_level, sizeof (anon_level), "%u", anonymity_level); argv[argc++] = "-a"; argv[argc++] = anon_level; GNUNET_snprintf (content_prio, sizeof (content_prio), "%u", content_priority); argv[argc++] = "-p"; argv[argc++] = content_prio; GNUNET_snprintf (repl_level, sizeof (repl_level), "%u", replication_level); argv[argc++] = "-r"; argv[argc++] = repl_level; argv[argc++] = wi->filename; argv[argc] = NULL; GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Publishing `%s'\n"), wi->filename); GNUNET_assert (NULL == publish_proc); publish_proc = GNUNET_OS_start_process_vap (GNUNET_YES, 0, NULL, NULL, NULL, "gnunet-publish", argv); if (NULL == publish_proc) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("Failed to run `%s'\n"), "gnunet-publish"); GNUNET_CONTAINER_DLL_insert (work_head, work_tail, wi); run_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, &work, NULL); return; } pr = GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_READ); run_task = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL, pr, &maint_child_death, wi); } /** * Recursively scan the given file/directory structure to determine * a unique ID that represents the current state of the hierarchy. * * @param cls where to store the unique ID we are computing * @param filename file to scan * @return #GNUNET_OK (always) */ static int determine_id (void *cls, const char *filename) { struct GNUNET_HashCode *id = cls; struct stat sbuf; struct GNUNET_HashCode fx[2]; struct GNUNET_HashCode ft; if (0 != STAT (filename, &sbuf)) { GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "stat", filename); return GNUNET_OK; } GNUNET_CRYPTO_hash (filename, strlen (filename), &fx[0]); if (!S_ISDIR (sbuf.st_mode)) { uint64_t fattr[2]; fattr[0] = GNUNET_htonll (sbuf.st_size); fattr[0] = GNUNET_htonll (sbuf.st_mtime); GNUNET_CRYPTO_hash (fattr, sizeof (fattr), &fx[1]); } else { memset (&fx[1], 1, sizeof (struct GNUNET_HashCode)); GNUNET_DISK_directory_scan (filename, &determine_id, &fx[1]); } /* use hash here to make hierarchical structure distinct from all files on the same level */ GNUNET_CRYPTO_hash (fx, sizeof (fx), &ft); /* use XOR here so that order of the files in the directory does not matter! */ GNUNET_CRYPTO_hash_xor (&ft, id, id); return GNUNET_OK; } /** * Function called with a filename (or directory name) to publish * (if it has changed since the last time we published it). This function * is called for the top-level files only. * * @param cls closure, NULL * @param filename complete filename (absolute path) * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR during shutdown */ static int add_file (void *cls, const char *filename) { struct WorkItem *wi; struct GNUNET_HashCode key; struct GNUNET_HashCode id; if (GNUNET_YES == do_shutdown) return GNUNET_SYSERR; if ( (NULL != strstr (filename, "/.auto-share")) || (NULL != strstr (filename, "\\.auto-share")) ) return GNUNET_OK; /* skip internal file */ GNUNET_CRYPTO_hash (filename, strlen (filename), &key); wi = GNUNET_CONTAINER_multihashmap_get (work_finished, &key); memset (&id, 0, sizeof (struct GNUNET_HashCode)); determine_id (&id, filename); if (NULL != wi) { if (0 == memcmp (&id, &wi->id, sizeof (struct GNUNET_HashCode))) return GNUNET_OK; /* skip: we did this one already */ /* contents changed, need to re-do the directory... */ GNUNET_assert (GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove (work_finished, &key, wi)); } else { wi = GNUNET_new (struct WorkItem); wi->filename = GNUNET_strdup (filename); } wi->id = id; GNUNET_CONTAINER_DLL_insert (work_head, work_tail, wi); if (GNUNET_YES == do_shutdown) return GNUNET_SYSERR; return GNUNET_OK; } /** * Periodically run task to update our view of the directory to share. * * @param cls NULL */ static void scan (void *cls) { run_task = NULL; start_time = GNUNET_TIME_absolute_get (); (void) GNUNET_DISK_directory_scan (dir_name, &add_file, NULL); schedule_next_task (); } /** * Decide what the next task is (working or scanning) and schedule it. */ static void schedule_next_task () { struct GNUNET_TIME_Relative delay; if (GNUNET_YES == do_shutdown) return; GNUNET_assert (NULL == run_task); if (NULL == work_head) { /* delay by at most 4h, at least 1s, and otherwise in between depending on how long it took to scan */ delay = GNUNET_TIME_absolute_get_duration (start_time); delay = GNUNET_TIME_relative_saturating_multiply (delay, 100); delay = GNUNET_TIME_relative_min (delay, MAX_DELAY); delay = GNUNET_TIME_relative_max (delay, MIN_DELAY); run_task = GNUNET_SCHEDULER_add_delayed (delay, &scan, NULL); } else { run_task = GNUNET_SCHEDULER_add_now (&work, NULL); } } /** * Main function that will be run by the scheduler. * * @param cls closure * @param args remaining command-line arguments * @param cfgfile name of the configuration file used (for saving, can be NULL!) * @param c configuration */ static void run (void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *c) { /* check arguments */ if ( (NULL == args[0]) || (NULL != args[1]) || (GNUNET_YES != GNUNET_DISK_directory_test (args[0], GNUNET_YES)) ) { printf (_("You must specify one and only one directory name for automatic publication.\n")); ret = -1; return; } cfg_filename = GNUNET_strdup (cfgfile); cfg = c; dir_name = args[0]; work_finished = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO); load_state (); run_task = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, &scan, NULL); GNUNET_SCHEDULER_add_shutdown (&do_stop_task, NULL); } /** * Free memory associated with the work item from the work_finished map. * * @param cls NULL (unused) * @param key key of the item in the map (unused) * @param value the `struct WorkItem` to free * @return #GNUNET_OK to continue to iterate */ static int free_item (void *cls, const struct GNUNET_HashCode *key, void *value) { struct WorkItem *wi = value; GNUNET_free (wi->filename); GNUNET_free (wi); return GNUNET_OK; } /** * The main function to automatically publish content to GNUnet. * * @param argc number of arguments from the command line * @param argv command line arguments * @return 0 ok, 1 on error */ int main (int argc, char *const *argv) { struct GNUNET_GETOPT_CommandLineOption options[] = { GNUNET_GETOPT_option_uint ('a', "anonymity", "LEVEL", gettext_noop ("set the desired LEVEL of sender-anonymity"), &anonymity_level), GNUNET_GETOPT_option_flag ('d', "disable-creation-time", gettext_noop ("disable adding the creation time to the metadata of the uploaded file"), &do_disable_creation_time), GNUNET_GETOPT_option_flag ('D', "disable-extractor", gettext_noop ("do not use libextractor to add keywords or metadata"), &disable_extractor), GNUNET_GETOPT_option_uint ('p', "priority", "PRIORITY", gettext_noop ("specify the priority of the content"), &content_priority), GNUNET_GETOPT_option_uint ('r', "replication", "LEVEL", gettext_noop ("set the desired replication LEVEL"), &replication_level), GNUNET_GETOPT_option_verbose (&verbose), GNUNET_GETOPT_OPTION_END }; struct WorkItem *wi; int ok; struct GNUNET_SIGNAL_Context *shc_chld; if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) return 2; sigpipe = GNUNET_DISK_pipe (GNUNET_NO, GNUNET_NO, GNUNET_NO, GNUNET_NO); GNUNET_assert (NULL != sigpipe); shc_chld = GNUNET_SIGNAL_handler_install (GNUNET_SIGCHLD, &sighandler_child_death); ok = (GNUNET_OK == GNUNET_PROGRAM_run (argc, argv, "gnunet-auto-share [OPTIONS] FILENAME", gettext_noop ("Automatically publish files from a directory on GNUnet"), options, &run, NULL)) ? ret : 1; if (NULL != work_finished) { (void) GNUNET_CONTAINER_multihashmap_iterate (work_finished, &free_item, NULL); GNUNET_CONTAINER_multihashmap_destroy (work_finished); } while (NULL != (wi = work_head)) { GNUNET_CONTAINER_DLL_remove (work_head, work_tail, wi); GNUNET_free (wi->filename); GNUNET_free (wi); } GNUNET_SIGNAL_handler_uninstall (shc_chld); shc_chld = NULL; GNUNET_DISK_pipe_close (sigpipe); sigpipe = NULL; GNUNET_free_non_null (cfg_filename); cfg_filename = NULL; GNUNET_free ((void*) argv); return ok; } /* end of gnunet-auto-share.c */