From 0443654040fb277df95aae28a7b2a54a4f0a73bf Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Tue, 15 Nov 2011 09:38:48 +0000 Subject: dead code elimination, splitting fs.h into fs.h and fs_api.h --- src/fs/Makefile.am | 2 +- src/fs/fs.c | 2784 --------------------------------------- src/fs/fs.h | 2001 +--------------------------- src/fs/fs_api.c | 2784 +++++++++++++++++++++++++++++++++++++++ src/fs/fs_api.h | 1907 +++++++++++++++++++++++++++ src/fs/fs_directory.c | 2 +- src/fs/fs_download.c | 2 +- src/fs/fs_file_information.c | 2 +- src/fs/fs_getopt.c | 2 +- src/fs/fs_list_indexed.c | 2 +- src/fs/fs_misc.c | 2 +- src/fs/fs_namespace.c | 14 +- src/fs/fs_publish.c | 7 +- src/fs/fs_search.c | 7 +- src/fs/fs_test_lib.c | 2 +- src/fs/fs_tree.h | 2 +- src/fs/fs_unindex.c | 2 +- src/fs/fs_uri.c | 2 +- src/fs/gnunet-service-fs.c | 2 - src/fs/gnunet-service-fs.h | 23 +- src/fs/gnunet-service-fs_cp.c | 8 + src/fs/gnunet-service-fs_cp.h | 30 + src/fs/gnunet-service-fs_pr.c | 15 + src/fs/gnunet-service-fs_push.c | 11 + src/fs/test_fs_directory.c | 2 +- src/fs/test_fs_uri.c | 2 +- 26 files changed, 4801 insertions(+), 4818 deletions(-) delete mode 100644 src/fs/fs.c create mode 100644 src/fs/fs_api.c create mode 100644 src/fs/fs_api.h (limited to 'src') diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am index 8cc0f9bf3..b1f15afa3 100644 --- a/src/fs/Makefile.am +++ b/src/fs/Makefile.am @@ -20,7 +20,7 @@ lib_LTLIBRARIES = libgnunetfs.la noinst_LIBRARIES = libgnunetfstest.a libgnunetfs_la_SOURCES = \ - fs.c fs.h \ + fs_api.c fs_api.h fs.h \ fs_directory.c \ fs_download.c \ fs_file_information.c \ diff --git a/src/fs/fs.c b/src/fs/fs.c deleted file mode 100644 index 83c0bf271..000000000 --- a/src/fs/fs.c +++ /dev/null @@ -1,2784 +0,0 @@ -/* - This file is part of GNUnet. - (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010, 2011 Christian Grothoff (and other contributing authors) - - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 59 Temple Place - Suite 330, - Boston, MA 02111-1307, USA. -*/ - -/** - * @file fs/fs.c - * @brief main FS functions (master initialization, serialization, deserialization, shared code) - * @author Christian Grothoff - */ - -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_fs_service.h" -#include "fs.h" -#include "fs_tree.h" - - -/** - * Start the given job (send signal, remove from pending queue, update - * counters and state). - * - * @param qe job to start - */ -static void -start_job (struct GNUNET_FS_QueueEntry *qe) -{ - GNUNET_assert (NULL == qe->client); - qe->client = GNUNET_CLIENT_connect ("fs", qe->h->cfg); - if (qe->client == NULL) - { - GNUNET_break (0); - return; - } - qe->start (qe->cls, qe->client); - qe->start_times++; - qe->h->active_blocks += qe->blocks; - qe->start_time = GNUNET_TIME_absolute_get (); - GNUNET_CONTAINER_DLL_remove (qe->h->pending_head, qe->h->pending_tail, qe); - GNUNET_CONTAINER_DLL_insert_after (qe->h->running_head, qe->h->running_tail, - qe->h->running_tail, qe); -} - - -/** - * Stop the given job (send signal, remove from active queue, update - * counters and state). - * - * @param qe job to stop - */ -static void -stop_job (struct GNUNET_FS_QueueEntry *qe) -{ - qe->client = NULL; - qe->stop (qe->cls); - qe->h->active_downloads--; - qe->h->active_blocks -= qe->blocks; - qe->run_time = - GNUNET_TIME_relative_add (qe->run_time, - GNUNET_TIME_absolute_get_duration - (qe->start_time)); - GNUNET_CONTAINER_DLL_remove (qe->h->running_head, qe->h->running_tail, qe); - GNUNET_CONTAINER_DLL_insert_after (qe->h->pending_head, qe->h->pending_tail, - qe->h->pending_tail, qe); -} - - -/** - * Process the jobs in the job queue, possibly starting some - * and stopping others. - * - * @param cls the 'struct GNUNET_FS_Handle' - * @param tc scheduler context - */ -static void -process_job_queue (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) -{ - struct GNUNET_FS_Handle *h = cls; - struct GNUNET_FS_QueueEntry *qe; - struct GNUNET_FS_QueueEntry *next; - struct GNUNET_TIME_Relative run_time; - struct GNUNET_TIME_Relative restart_at; - struct GNUNET_TIME_Relative rst; - struct GNUNET_TIME_Absolute end_time; - - h->queue_job = GNUNET_SCHEDULER_NO_TASK; - next = h->pending_head; - while (NULL != (qe = next)) - { - next = qe->next; - if (h->running_head == NULL) - { - start_job (qe); - continue; - } - if ((qe->blocks + h->active_blocks <= h->max_parallel_requests) && - (h->active_downloads + 1 <= h->max_parallel_downloads)) - { - start_job (qe); - continue; - } - } - if (h->pending_head == NULL) - return; /* no need to stop anything */ - restart_at = GNUNET_TIME_UNIT_FOREVER_REL; - next = h->running_head; - while (NULL != (qe = next)) - { - next = qe->next; - run_time = - GNUNET_TIME_relative_multiply (h->avg_block_latency, - qe->blocks * qe->start_times); - end_time = GNUNET_TIME_absolute_add (qe->start_time, run_time); - rst = GNUNET_TIME_absolute_get_remaining (end_time); - restart_at = GNUNET_TIME_relative_min (rst, restart_at); - if (rst.rel_value > 0) - continue; - stop_job (qe); - } - h->queue_job = - GNUNET_SCHEDULER_add_delayed (restart_at, &process_job_queue, h); -} - - -/** - * Add a job to the queue. - * - * @param h handle to the overall FS state - * @param start function to call to begin the job - * @param stop function to call to pause the job, or on dequeue (if the job was running) - * @param cls closure for start and stop - * @param blocks number of blocks this jobs uses - * @return queue handle - */ -struct GNUNET_FS_QueueEntry * -GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, GNUNET_FS_QueueStart start, - GNUNET_FS_QueueStop stop, void *cls, unsigned int blocks) -{ - struct GNUNET_FS_QueueEntry *qe; - - qe = GNUNET_malloc (sizeof (struct GNUNET_FS_QueueEntry)); - qe->h = h; - qe->start = start; - qe->stop = stop; - qe->cls = cls; - qe->queue_time = GNUNET_TIME_absolute_get (); - qe->blocks = blocks; - GNUNET_CONTAINER_DLL_insert_after (h->pending_head, h->pending_tail, - h->pending_tail, qe); - if (h->queue_job != GNUNET_SCHEDULER_NO_TASK) - GNUNET_SCHEDULER_cancel (h->queue_job); - h->queue_job = GNUNET_SCHEDULER_add_now (&process_job_queue, h); - return qe; -} - - -/** - * Dequeue a job from the queue. - * @param qh handle for the job - */ -void -GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh) -{ - struct GNUNET_FS_Handle *h; - - h = qh->h; - if (qh->client != NULL) - stop_job (qh); - GNUNET_CONTAINER_DLL_remove (h->pending_head, h->pending_tail, qh); - GNUNET_free (qh); - if (h->queue_job != GNUNET_SCHEDULER_NO_TASK) - GNUNET_SCHEDULER_cancel (h->queue_job); - h->queue_job = GNUNET_SCHEDULER_add_now (&process_job_queue, h); -} - - -/** - * Create a top-level activity entry. - * - * @param h global fs handle - * @param ssf suspend signal function to use - * @param ssf_cls closure for ssf - * @return fresh top-level activity handle - */ -struct TopLevelActivity * -GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf, - void *ssf_cls) -{ - struct TopLevelActivity *ret; - - ret = GNUNET_malloc (sizeof (struct TopLevelActivity)); - ret->ssf = ssf; - ret->ssf_cls = ssf_cls; - GNUNET_CONTAINER_DLL_insert (h->top_head, h->top_tail, ret); - return ret; -} - - -/** - * Destroy a top-level activity entry. - * - * @param h global fs handle - * @param top top level activity entry - */ -void -GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top) -{ - GNUNET_CONTAINER_DLL_remove (h->top_head, h->top_tail, top); - GNUNET_free (top); -} - - - -/** - * Closure for "data_reader_file". - */ -struct FileInfo -{ - /** - * Name of the file to read. - */ - char *filename; - - /** - * File descriptor, NULL if it has not yet been opened. - */ - struct GNUNET_DISK_FileHandle *fd; -}; - - -/** - * Function that provides data by reading from a file. - * - * @param cls closure (points to the file information) - * @param offset offset to read from; it is possible - * that the caller might need to go backwards - * a bit at times - * @param max maximum number of bytes that should be - * copied to buf; readers are not allowed - * to provide less data unless there is an error; - * a value of "0" will be used at the end to allow - * the reader to clean up its internal state - * @param buf where the reader should write the data - * @param emsg location for the reader to store an error message - * @return number of bytes written, usually "max", 0 on error - */ -size_t -GNUNET_FS_data_reader_file_ (void *cls, uint64_t offset, size_t max, void *buf, - char **emsg) -{ - struct FileInfo *fi = cls; - ssize_t ret; - - if (max == 0) - { - if (fi->fd != NULL) - GNUNET_DISK_file_close (fi->fd); - GNUNET_free (fi->filename); - GNUNET_free (fi); - return 0; - } - if (fi->fd == NULL) - { - fi->fd = - GNUNET_DISK_file_open (fi->filename, GNUNET_DISK_OPEN_READ, - GNUNET_DISK_PERM_NONE); - if (fi->fd == NULL) - { - GNUNET_asprintf (emsg, _("Could not open file `%s': %s"), fi->filename, - STRERROR (errno)); - return 0; - } - } - GNUNET_DISK_file_seek (fi->fd, offset, GNUNET_DISK_SEEK_SET); - ret = GNUNET_DISK_file_read (fi->fd, buf, max); - if (ret == -1) - { - GNUNET_asprintf (emsg, _("Could not read file `%s': %s"), fi->filename, - STRERROR (errno)); - return 0; - } - if (ret != max) - { - GNUNET_asprintf (emsg, _("Short read reading from file `%s'!"), - fi->filename); - return 0; - } - return max; -} - - -/** - * Create the closure for the 'GNUNET_FS_data_reader_file_' callback. - * - * @param filename file to read - * @return closure to use, NULL on error - */ -void * -GNUNET_FS_make_file_reader_context_ (const char *filename) -{ - struct FileInfo *fi; - - fi = GNUNET_malloc (sizeof (struct FileInfo)); - fi->filename = GNUNET_STRINGS_filename_expand (filename); - if (fi->filename == NULL) - { - GNUNET_free (fi); - return NULL; - } - return fi; -} - - -/** - * Function that provides data by copying from a buffer. - * - * @param cls closure (points to the buffer) - * @param offset offset to read from; it is possible - * that the caller might need to go backwards - * a bit at times - * @param max maximum number of bytes that should be - * copied to buf; readers are not allowed - * to provide less data unless there is an error; - * a value of "0" will be used at the end to allow - * the reader to clean up its internal state - * @param buf where the reader should write the data - * @param emsg location for the reader to store an error message - * @return number of bytes written, usually "max", 0 on error - */ -size_t -GNUNET_FS_data_reader_copy_ (void *cls, uint64_t offset, size_t max, void *buf, - char **emsg) -{ - char *data = cls; - - if (max == 0) - { - GNUNET_free_non_null (data); - return 0; - } - memcpy (buf, &data[offset], max); - return max; -} - - -/** - * Return the full filename where we would store state information - * (for serialization/deserialization). - * - * @param h master context - * @param ext component of the path - * @param ent entity identifier (or emtpy string for the directory) - * @return NULL on error - */ -static char * -get_serialization_file_name (struct GNUNET_FS_Handle *h, const char *ext, - const char *ent) -{ - char *basename; - char *ret; - - if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) - return NULL; /* persistence not requested */ - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_filename (h->cfg, "fs", "STATE_DIR", - &basename)) - return NULL; - GNUNET_asprintf (&ret, "%s%s%s%s%s%s%s", basename, DIR_SEPARATOR_STR, - h->client_name, DIR_SEPARATOR_STR, ext, DIR_SEPARATOR_STR, - ent); - GNUNET_free (basename); - return ret; -} - - -/** - * Return the full filename where we would store state information - * (for serialization/deserialization) that is associated with a - * parent operation. - * - * @param h master context - * @param ext component of the path - * @param uni name of the parent operation - * @param ent entity identifier (or emtpy string for the directory) - * @return NULL on error - */ -static char * -get_serialization_file_name_in_dir (struct GNUNET_FS_Handle *h, const char *ext, - const char *uni, const char *ent) -{ - char *basename; - char *ret; - - if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) - return NULL; /* persistence not requested */ - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_filename (h->cfg, "fs", "STATE_DIR", - &basename)) - return NULL; - GNUNET_asprintf (&ret, "%s%s%s%s%s%s%s.dir%s%s", basename, DIR_SEPARATOR_STR, - h->client_name, DIR_SEPARATOR_STR, ext, DIR_SEPARATOR_STR, - uni, DIR_SEPARATOR_STR, ent); - GNUNET_free (basename); - return ret; -} - - -/** - * Return a read handle for deserialization. - * - * @param h master context - * @param ext component of the path - * @param ent entity identifier (or emtpy string for the directory) - * @return NULL on error - */ -static struct GNUNET_BIO_ReadHandle * -get_read_handle (struct GNUNET_FS_Handle *h, const char *ext, const char *ent) -{ - char *fn; - struct GNUNET_BIO_ReadHandle *ret; - - fn = get_serialization_file_name (h, ext, ent); - if (fn == NULL) - return NULL; - ret = GNUNET_BIO_read_open (fn); - GNUNET_free (fn); - return ret; -} - - -/** - * Return a write handle for serialization. - * - * @param h master context - * @param ext component of the path - * @param ent entity identifier (or emtpy string for the directory) - * @return NULL on error - */ -static struct GNUNET_BIO_WriteHandle * -get_write_handle (struct GNUNET_FS_Handle *h, const char *ext, const char *ent) -{ - char *fn; - struct GNUNET_BIO_WriteHandle *ret; - - fn = get_serialization_file_name (h, ext, ent); - if (fn == NULL) - { - return NULL; - } - ret = GNUNET_BIO_write_open (fn); - if (ret == NULL) - GNUNET_break (0); - GNUNET_free (fn); - return ret; -} - - -/** - * Return a write handle for serialization. - * - * @param h master context - * @param ext component of the path - * @param uni name of parent - * @param ent entity identifier (or emtpy string for the directory) - * @return NULL on error - */ -static struct GNUNET_BIO_WriteHandle * -get_write_handle_in_dir (struct GNUNET_FS_Handle *h, const char *ext, - const char *uni, const char *ent) -{ - char *fn; - struct GNUNET_BIO_WriteHandle *ret; - - fn = get_serialization_file_name_in_dir (h, ext, uni, ent); - if (fn == NULL) - return NULL; - ret = GNUNET_BIO_write_open (fn); - GNUNET_free (fn); - return ret; -} - - -/** - * Remove serialization/deserialization file from disk. - * - * @param h master context - * @param ext component of the path - * @param ent entity identifier - */ -void -GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, const char *ext, - const char *ent) -{ - char *filename; - - if ((NULL == ent) || (0 == strlen (ent))) - { - GNUNET_break (0); - return; - } - filename = get_serialization_file_name (h, ext, ent); - if (filename != NULL) - { - if (0 != UNLINK (filename)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); - GNUNET_free (filename); - } -} - - -/** - * Remove serialization/deserialization file from disk. - * - * @param h master context - * @param ext component of the path - * @param uni parent name - * @param ent entity identifier - */ -static void -remove_sync_file_in_dir (struct GNUNET_FS_Handle *h, const char *ext, - const char *uni, const char *ent) -{ - char *filename; - - if ((NULL == ent) || (0 == strlen (ent))) - { - GNUNET_break (0); - return; - } - filename = get_serialization_file_name_in_dir (h, ext, uni, ent); - if (filename != NULL) - { - if (0 != UNLINK (filename)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); - GNUNET_free (filename); - } -} - - -/** - * Remove serialization/deserialization directory from disk. - * - * @param h master context - * @param ext component of the path - * @param uni unique name of parent - */ -void -GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, const char *ext, - const char *uni) -{ - char *dn; - - if (uni == NULL) - return; - dn = get_serialization_file_name_in_dir (h, ext, uni, ""); - if (dn == NULL) - return; - if ((GNUNET_OK == GNUNET_DISK_directory_test (dn)) && - (GNUNET_OK != GNUNET_DISK_directory_remove (dn))) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "rmdir", dn); - GNUNET_free (dn); -} - - -/** - * Serialize a 'start_time'. Since we use start-times to - * calculate the duration of some operation, we actually - * do not serialize the absolute time but the (relative) - * duration since the start time. When we then - * deserialize the start time, we take the current time and - * subtract that duration so that we get again an absolute - * time stamp that will result in correct performance - * calculations. - * - * @param wh handle for writing - * @param timestamp time to serialize - * @return GNUNET_OK on success - */ -static int -write_start_time (struct GNUNET_BIO_WriteHandle *wh, - struct GNUNET_TIME_Absolute timestamp) -{ - struct GNUNET_TIME_Relative dur; - - dur = GNUNET_TIME_absolute_get_duration (timestamp); - return GNUNET_BIO_write_int64 (wh, dur.rel_value); -} - - -/** - * Serialize a 'start_time'. Since we use start-times to - * calculate the duration of some operation, we actually - * do not serialize the absolute time but the (relative) - * duration since the start time. When we then - * deserialize the start time, we take the current time and - * subtract that duration so that we get again an absolute - * time stamp that will result in correct performance - * calculations. - * - * @param rh handle for reading - * @param timestamp where to write the deserialized timestamp - * @return GNUNET_OK on success - */ -static int -read_start_time (struct GNUNET_BIO_ReadHandle *rh, - struct GNUNET_TIME_Absolute *timestamp) -{ - struct GNUNET_TIME_Relative dur; - - if (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dur.rel_value)) - return GNUNET_SYSERR; - *timestamp = GNUNET_TIME_absolute_subtract (GNUNET_TIME_absolute_get (), dur); - return GNUNET_OK; -} - - -/** - * Using the given serialization filename, try to deserialize - * the file-information tree associated with it. - * - * @param h master context - * @param filename name of the file (without directory) with - * the infromation - * @return NULL on error - */ -static struct GNUNET_FS_FileInformation * -deserialize_file_information (struct GNUNET_FS_Handle *h, const char *filename); - - -/** - * Using the given serialization filename, try to deserialize - * the file-information tree associated with it. - * - * @param h master context - * @param fn name of the file (without directory) with - * the infromation - * @param rh handle for reading - * @return NULL on error - */ -static struct GNUNET_FS_FileInformation * -deserialize_fi_node (struct GNUNET_FS_Handle *h, const char *fn, - struct GNUNET_BIO_ReadHandle *rh) -{ - struct GNUNET_FS_FileInformation *ret; - struct GNUNET_FS_FileInformation *nxt; - char b; - char *ksks; - char *chks; - char *filename; - uint32_t dsize; - - if (GNUNET_OK != GNUNET_BIO_read (rh, "status flag", &b, sizeof (b))) - { - GNUNET_break (0); - return NULL; - } - ret = GNUNET_malloc (sizeof (struct GNUNET_FS_FileInformation)); - ret->h = h; - ksks = NULL; - chks = NULL; - filename = NULL; - if ((GNUNET_OK != GNUNET_BIO_read_meta_data (rh, "metadata", &ret->meta)) || - (GNUNET_OK != GNUNET_BIO_read_string (rh, "ksk-uri", &ksks, 32 * 1024)) || - ((ksks != NULL) && - (NULL == (ret->keywords = GNUNET_FS_uri_parse (ksks, NULL)))) || - (GNUNET_YES != GNUNET_FS_uri_test_ksk (ret->keywords)) || - (GNUNET_OK != GNUNET_BIO_read_string (rh, "chk-uri", &chks, 1024)) || - ((chks != NULL) && - ((NULL == (ret->chk_uri = GNUNET_FS_uri_parse (chks, NULL))) || - (GNUNET_YES != GNUNET_FS_uri_test_chk (ret->chk_uri)))) || - (GNUNET_OK != read_start_time (rh, &ret->start_time)) || - (GNUNET_OK != GNUNET_BIO_read_string (rh, "emsg", &ret->emsg, 16 * 1024)) - || (GNUNET_OK != - GNUNET_BIO_read_string (rh, "fn", &ret->filename, 16 * 1024)) || - (GNUNET_OK != - GNUNET_BIO_read_int64 (rh, &ret->bo.expiration_time.abs_value)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &ret->bo.anonymity_level)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &ret->bo.content_priority)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &ret->bo.replication_level))) - { - GNUNET_break (0); - goto cleanup; - } - switch (b) - { - case 0: /* file-insert */ - if (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) - { - GNUNET_break (0); - goto cleanup; - } - ret->is_directory = GNUNET_NO; - ret->data.file.do_index = GNUNET_NO; - ret->data.file.have_hash = GNUNET_NO; - ret->data.file.index_start_confirmed = GNUNET_NO; - if (GNUNET_NO == ret->is_published) - { - if (NULL == ret->filename) - { - ret->data.file.reader = &GNUNET_FS_data_reader_copy_; - ret->data.file.reader_cls = - GNUNET_malloc_large (ret->data.file.file_size); - if (ret->data.file.reader_cls == NULL) - goto cleanup; - if (GNUNET_OK != - GNUNET_BIO_read (rh, "file-data", ret->data.file.reader_cls, - ret->data.file.file_size)) - { - GNUNET_break (0); - goto cleanup; - } - } - else - { - ret->data.file.reader = &GNUNET_FS_data_reader_file_; - ret->data.file.reader_cls = - GNUNET_FS_make_file_reader_context_ (ret->filename); - } - } - break; - case 1: /* file-index, no hash */ - if (NULL == ret->filename) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) - { - GNUNET_break (0); - goto cleanup; - } - ret->is_directory = GNUNET_NO; - ret->data.file.do_index = GNUNET_YES; - ret->data.file.have_hash = GNUNET_NO; - ret->data.file.index_start_confirmed = GNUNET_NO; - ret->data.file.reader = &GNUNET_FS_data_reader_file_; - ret->data.file.reader_cls = - GNUNET_FS_make_file_reader_context_ (ret->filename); - break; - case 2: /* file-index-with-hash */ - if (NULL == ret->filename) - { - GNUNET_break (0); - goto cleanup; - } - if ((GNUNET_OK != GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) || - (GNUNET_OK != - GNUNET_BIO_read (rh, "fileid", &ret->data.file.file_id, - sizeof (GNUNET_HashCode)))) - { - GNUNET_break (0); - goto cleanup; - } - ret->is_directory = GNUNET_NO; - ret->data.file.do_index = GNUNET_YES; - ret->data.file.have_hash = GNUNET_YES; - ret->data.file.index_start_confirmed = GNUNET_NO; - ret->data.file.reader = &GNUNET_FS_data_reader_file_; - ret->data.file.reader_cls = - GNUNET_FS_make_file_reader_context_ (ret->filename); - break; - case 3: /* file-index-with-hash-confirmed */ - if (NULL == ret->filename) - { - GNUNET_break (0); - goto cleanup; - } - if ((GNUNET_OK != GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) || - (GNUNET_OK != - GNUNET_BIO_read (rh, "fileid", &ret->data.file.file_id, - sizeof (GNUNET_HashCode)))) - { - GNUNET_break (0); - goto cleanup; - } - ret->is_directory = GNUNET_NO; - ret->data.file.do_index = GNUNET_YES; - ret->data.file.have_hash = GNUNET_YES; - ret->data.file.index_start_confirmed = GNUNET_YES; - ret->data.file.reader = &GNUNET_FS_data_reader_file_; - ret->data.file.reader_cls = - GNUNET_FS_make_file_reader_context_ (ret->filename); - break; - case 4: /* directory */ - ret->is_directory = GNUNET_YES; - if ((GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dsize)) || - (NULL == (ret->data.dir.dir_data = GNUNET_malloc_large (dsize))) || - (GNUNET_OK != - GNUNET_BIO_read (rh, "dir-data", ret->data.dir.dir_data, dsize)) || - (GNUNET_OK != - GNUNET_BIO_read_string (rh, "ent-filename", &filename, 16 * 1024))) - { - GNUNET_break (0); - goto cleanup; - } - ret->data.dir.dir_size = (uint32_t) dsize; - if (filename != NULL) - { - ret->data.dir.entries = deserialize_file_information (h, filename); - GNUNET_free (filename); - filename = NULL; - nxt = ret->data.dir.entries; - while (nxt != NULL) - { - nxt->dir = ret; - nxt = nxt->next; - } - } - break; - default: - GNUNET_break (0); - goto cleanup; - } - ret->serialization = GNUNET_strdup (fn); - if (GNUNET_OK != - GNUNET_BIO_read_string (rh, "nxt-filename", &filename, 16 * 1024)) - { - GNUNET_break (0); - goto cleanup; - } - if (filename != NULL) - { - ret->next = deserialize_file_information (h, filename); - GNUNET_free (filename); - filename = NULL; - } - GNUNET_free_non_null (ksks); - GNUNET_free_non_null (chks); - return ret; -cleanup: - GNUNET_free_non_null (ksks); - GNUNET_free_non_null (chks); - GNUNET_free_non_null (filename); - GNUNET_FS_file_information_destroy (ret, NULL, NULL); - return NULL; -} - - -/** - * Using the given serialization filename, try to deserialize - * the file-information tree associated with it. - * - * @param h master context - * @param filename name of the file (without directory) with - * the infromation - * @return NULL on error - */ -static struct GNUNET_FS_FileInformation * -deserialize_file_information (struct GNUNET_FS_Handle *h, const char *filename) -{ - struct GNUNET_FS_FileInformation *ret; - struct GNUNET_BIO_ReadHandle *rh; - char *emsg; - - rh = get_read_handle (h, GNUNET_FS_SYNC_PATH_FILE_INFO, filename); - if (rh == NULL) - return NULL; - ret = deserialize_fi_node (h, filename, rh); - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failed to resume publishing information `%s': %s\n"), - filename, emsg); - GNUNET_free (emsg); - } - if (ret == NULL) - { - if (0 != UNLINK (filename)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); - } - return ret; -} - - -/** - * Given a serialization name (full absolute path), return the - * basename of the file (without the path), which must only - * consist of the 6 random characters. - * - * @param fullname name to extract the basename from - * @return copy of the basename, NULL on error - */ -static char * -get_serialization_short_name (const char *fullname) -{ - const char *end; - const char *nxt; - - end = NULL; - nxt = fullname; - /* FIXME: we could do this faster since we know - * the length of 'end'... */ - while ('\0' != *nxt) - { - if (DIR_SEPARATOR == *nxt) - end = nxt + 1; - nxt++; - } - if ((end == NULL) || (strlen (end) == 0)) - { - GNUNET_break (0); - return NULL; - } - GNUNET_break (6 == strlen (end)); - return GNUNET_strdup (end); -} - - -/** - * Create a new random name for serialization. Also checks if persistence - * is enabled and returns NULL if not. - * - * @param h master context - * @param ext component of the path - * @return NULL on errror - */ -static char * -make_serialization_file_name (struct GNUNET_FS_Handle *h, const char *ext) -{ - char *fn; - char *dn; - char *ret; - - if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) - return NULL; /* persistence not requested */ - dn = get_serialization_file_name (h, ext, ""); - if (dn == NULL) - return NULL; - if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dn)) - { - GNUNET_free (dn); - return NULL; - } - fn = GNUNET_DISK_mktemp (dn); - GNUNET_free (dn); - if (fn == NULL) - return NULL; /* epic fail */ - ret = get_serialization_short_name (fn); - GNUNET_free (fn); - return ret; -} - - -/** - * Create a new random name for serialization. Also checks if persistence - * is enabled and returns NULL if not. - * - * @param h master context - * @param ext component of the path - * @param uni name of parent - * @return NULL on errror - */ -static char * -make_serialization_file_name_in_dir (struct GNUNET_FS_Handle *h, - const char *ext, const char *uni) -{ - char *fn; - char *dn; - char *ret; - - if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) - return NULL; /* persistence not requested */ - dn = get_serialization_file_name_in_dir (h, ext, uni, ""); - if (dn == NULL) - return NULL; - if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dn)) - { - GNUNET_free (dn); - return NULL; - } - fn = GNUNET_DISK_mktemp (dn); - GNUNET_free (dn); - if (fn == NULL) - return NULL; /* epic fail */ - ret = get_serialization_short_name (fn); - GNUNET_free (fn); - return ret; -} - - -/** - * Copy all of the data from the reader to the write handle. - * - * @param wh write handle - * @param fi file with reader - * @return GNUNET_OK on success - */ -static int -copy_from_reader (struct GNUNET_BIO_WriteHandle *wh, - struct GNUNET_FS_FileInformation *fi) -{ - char buf[32 * 1024]; - uint64_t off; - size_t ret; - size_t left; - char *emsg; - - emsg = NULL; - off = 0; - while (off < fi->data.file.file_size) - { - left = GNUNET_MIN (sizeof (buf), fi->data.file.file_size - off); - ret = - fi->data.file.reader (fi->data.file.reader_cls, off, left, buf, &emsg); - if (ret == 0) - { - GNUNET_free (emsg); - return GNUNET_SYSERR; - } - if (GNUNET_OK != GNUNET_BIO_write (wh, buf, ret)) - return GNUNET_SYSERR; - off += ret; - } - return GNUNET_OK; -} - - -/** - * Create a temporary file on disk to store the current - * state of "fi" in. - * - * @param fi file information to sync with disk - */ -void -GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *fi) -{ - char *fn; - struct GNUNET_BIO_WriteHandle *wh; - char b; - char *ksks; - char *chks; - - if (NULL == fi->serialization) - fi->serialization = - make_serialization_file_name (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO); - if (NULL == fi->serialization) - return; - wh = get_write_handle (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO, - fi->serialization); - if (wh == NULL) - { - GNUNET_free (fi->serialization); - fi->serialization = NULL; - return; - } - if (GNUNET_YES == fi->is_directory) - b = 4; - else if (GNUNET_YES == fi->data.file.index_start_confirmed) - b = 3; - else if (GNUNET_YES == fi->data.file.have_hash) - b = 2; - else if (GNUNET_YES == fi->data.file.do_index) - b = 1; - else - b = 0; - if (fi->keywords != NULL) - ksks = GNUNET_FS_uri_to_string (fi->keywords); - else - ksks = NULL; - if (fi->chk_uri != NULL) - chks = GNUNET_FS_uri_to_string (fi->chk_uri); - else - chks = NULL; - if ((GNUNET_OK != GNUNET_BIO_write (wh, &b, sizeof (b))) || - (GNUNET_OK != GNUNET_BIO_write_meta_data (wh, fi->meta)) || - (GNUNET_OK != GNUNET_BIO_write_string (wh, ksks)) || - (GNUNET_OK != GNUNET_BIO_write_string (wh, chks)) || - (GNUNET_OK != write_start_time (wh, fi->start_time)) || - (GNUNET_OK != GNUNET_BIO_write_string (wh, fi->emsg)) || - (GNUNET_OK != GNUNET_BIO_write_string (wh, fi->filename)) || - (GNUNET_OK != - GNUNET_BIO_write_int64 (wh, fi->bo.expiration_time.abs_value)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, fi->bo.anonymity_level)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, fi->bo.content_priority)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, fi->bo.replication_level))) - { - GNUNET_break (0); - goto cleanup; - } - GNUNET_free_non_null (chks); - chks = NULL; - GNUNET_free_non_null (ksks); - ksks = NULL; - - switch (b) - { - case 0: /* file-insert */ - if (GNUNET_OK != GNUNET_BIO_write_int64 (wh, fi->data.file.file_size)) - { - GNUNET_break (0); - goto cleanup; - } - if ((GNUNET_NO == fi->is_published) && (NULL == fi->filename)) - if (GNUNET_OK != copy_from_reader (wh, fi)) - { - GNUNET_break (0); - goto cleanup; - } - break; - case 1: /* file-index, no hash */ - if (NULL == fi->filename) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_OK != GNUNET_BIO_write_int64 (wh, fi->data.file.file_size)) - { - GNUNET_break (0); - goto cleanup; - } - break; - case 2: /* file-index-with-hash */ - case 3: /* file-index-with-hash-confirmed */ - if (NULL == fi->filename) - { - GNUNET_break (0); - goto cleanup; - } - if ((GNUNET_OK != GNUNET_BIO_write_int64 (wh, fi->data.file.file_size)) || - (GNUNET_OK != - GNUNET_BIO_write (wh, &fi->data.file.file_id, - sizeof (GNUNET_HashCode)))) - { - GNUNET_break (0); - goto cleanup; - } - break; - case 4: /* directory */ - if ((GNUNET_OK != GNUNET_BIO_write_int32 (wh, fi->data.dir.dir_size)) || - (GNUNET_OK != - GNUNET_BIO_write (wh, fi->data.dir.dir_data, - (uint32_t) fi->data.dir.dir_size)) || - (GNUNET_OK != - GNUNET_BIO_write_string (wh, - (fi->data.dir.entries == - NULL) ? NULL : fi->data.dir. - entries->serialization))) - { - GNUNET_break (0); - goto cleanup; - } - break; - default: - GNUNET_assert (0); - goto cleanup; - } - if (GNUNET_OK != - GNUNET_BIO_write_string (wh, - (fi->next != - NULL) ? fi->next->serialization : NULL)) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_OK != GNUNET_BIO_write_close (wh)) - { - wh = NULL; - GNUNET_break (0); - goto cleanup; - } - return; /* done! */ -cleanup: - if (wh != NULL) - (void) GNUNET_BIO_write_close (wh); - GNUNET_free_non_null (chks); - GNUNET_free_non_null (ksks); - fn = get_serialization_file_name (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO, - fi->serialization); - if (NULL != fn) - { - if (0 != UNLINK (fn)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn); - GNUNET_free (fn); - } - GNUNET_free (fi->serialization); - fi->serialization = NULL; -} - - - -/** - * Find the entry in the file information struct where the - * serialization filename matches the given name. - * - * @param pos file information to search - * @param srch filename to search for - * @return NULL if srch was not found in this subtree - */ -static struct GNUNET_FS_FileInformation * -find_file_position (struct GNUNET_FS_FileInformation *pos, const char *srch) -{ - struct GNUNET_FS_FileInformation *r; - - while (pos != NULL) - { - if (0 == strcmp (srch, pos->serialization)) - return pos; - if (pos->is_directory) - { - r = find_file_position (pos->data.dir.entries, srch); - if (r != NULL) - return r; - } - pos = pos->next; - } - return NULL; -} - - -/** - * Signal the FS's progress function that we are resuming - * an upload. - * - * @param cls closure (of type "struct GNUNET_FS_PublishContext*") - * @param fi the entry in the publish-structure - * @param length length of the file or directory - * @param meta metadata for the file or directory (can be modified) - * @param uri pointer to the keywords that will be used for this entry (can be modified) - * @param bo block options (can be modified) - * @param do_index should we index? - * @param client_info pointer to client context set upon creation (can be modified) - * @return GNUNET_OK to continue (always) - */ -static int -fip_signal_resume (void *cls, struct GNUNET_FS_FileInformation *fi, - uint64_t length, struct GNUNET_CONTAINER_MetaData *meta, - struct GNUNET_FS_Uri **uri, - struct GNUNET_FS_BlockOptions *bo, int *do_index, - void **client_info) -{ - struct GNUNET_FS_PublishContext *sc = cls; - struct GNUNET_FS_ProgressInfo pi; - - pi.status = GNUNET_FS_STATUS_PUBLISH_RESUME; - pi.value.publish.specifics.resume.message = sc->fi->emsg; - pi.value.publish.specifics.resume.chk_uri = sc->fi->chk_uri; - *client_info = GNUNET_FS_publish_make_status_ (&pi, sc, fi, 0); - return GNUNET_OK; -} - - -/** - * Function called with a filename of serialized publishing operation - * to deserialize. - * - * @param cls the 'struct GNUNET_FS_Handle*' - * @param filename complete filename (absolute path) - * @return GNUNET_OK (continue to iterate) - */ -static int -deserialize_publish_file (void *cls, const char *filename) -{ - struct GNUNET_FS_Handle *h = cls; - struct GNUNET_BIO_ReadHandle *rh; - struct GNUNET_FS_PublishContext *pc; - int32_t options; - int32_t all_done; - char *fi_root; - char *ns; - char *fi_pos; - char *emsg; - - pc = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext)); - pc->h = h; - pc->serialization = get_serialization_short_name (filename); - fi_root = NULL; - fi_pos = NULL; - ns = NULL; - rh = GNUNET_BIO_read_open (filename); - if (rh == NULL) - { - GNUNET_break (0); - goto cleanup; - } - if ((GNUNET_OK != GNUNET_BIO_read_string (rh, "publish-nid", &pc->nid, 1024)) - || (GNUNET_OK != - GNUNET_BIO_read_string (rh, "publish-nuid", &pc->nuid, 1024)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &options)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &all_done)) || - (GNUNET_OK != - GNUNET_BIO_read_string (rh, "publish-firoot", &fi_root, 128)) || - (GNUNET_OK != GNUNET_BIO_read_string (rh, "publish-fipos", &fi_pos, 128)) - || (GNUNET_OK != GNUNET_BIO_read_string (rh, "publish-ns", &ns, 1024))) - { - GNUNET_break (0); - goto cleanup; - } - pc->options = options; - pc->all_done = all_done; - if (NULL == fi_root) - { - GNUNET_break (0); - goto cleanup; - } - pc->fi = deserialize_file_information (h, fi_root); - if (pc->fi == NULL) - { - GNUNET_break (0); - goto cleanup; - } - if (ns != NULL) - { - pc->namespace = GNUNET_FS_namespace_create (h, ns); - if (pc->namespace == NULL) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ - ("Failed to recover namespace `%s', cannot resume publishing operation.\n"), - ns); - goto cleanup; - } - } - if ((0 == (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) && - (GNUNET_YES != pc->all_done)) - { - pc->dsh = GNUNET_DATASTORE_connect (h->cfg); - if (NULL == pc->dsh) - goto cleanup; - } - if (fi_pos != NULL) - { - pc->fi_pos = find_file_position (pc->fi, fi_pos); - GNUNET_free (fi_pos); - fi_pos = NULL; - if (pc->fi_pos == NULL) - { - /* failed to find position for resuming, outch! Will start from root! */ - GNUNET_break (0); - if (pc->all_done != GNUNET_YES) - pc->fi_pos = pc->fi; - } - } - GNUNET_free (fi_root); - fi_root = NULL; - /* generate RESUME event(s) */ - GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_resume, pc); - - /* re-start publishing (if needed)... */ - if (pc->all_done != GNUNET_YES) - { - GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task); - pc->upload_task = - GNUNET_SCHEDULER_add_with_priority - (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, pc); - } - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failure while resuming publishing operation `%s': %s\n"), - filename, emsg); - GNUNET_free (emsg); - } - GNUNET_free_non_null (ns); - pc->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, pc); - return GNUNET_OK; -cleanup: - GNUNET_free_non_null (pc->nid); - GNUNET_free_non_null (pc->nuid); - GNUNET_free_non_null (fi_root); - GNUNET_free_non_null (fi_pos); - GNUNET_free_non_null (ns); - if ((rh != NULL) && (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failed to resume publishing operation `%s': %s\n"), filename, - emsg); - GNUNET_free (emsg); - } - if (pc->fi != NULL) - GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL); - if (0 != UNLINK (filename)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); - GNUNET_free (pc->serialization); - GNUNET_free (pc); - return GNUNET_OK; -} - - -/** - * Synchronize this publishing struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param pc the struct to sync - */ -void -GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc) -{ - struct GNUNET_BIO_WriteHandle *wh; - - if (NULL == pc->serialization) - pc->serialization = - make_serialization_file_name (pc->h, - GNUNET_FS_SYNC_PATH_MASTER_PUBLISH); - if (NULL == pc->serialization) - return; - if (NULL == pc->fi) - return; - if (NULL == pc->fi->serialization) - { - GNUNET_break (0); - return; - } - wh = get_write_handle (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, - pc->serialization); - if (wh == NULL) - { - GNUNET_break (0); - goto cleanup; - } - if ((GNUNET_OK != GNUNET_BIO_write_string (wh, pc->nid)) || - (GNUNET_OK != GNUNET_BIO_write_string (wh, pc->nuid)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pc->options)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pc->all_done)) || - (GNUNET_OK != GNUNET_BIO_write_string (wh, pc->fi->serialization)) || - (GNUNET_OK != - GNUNET_BIO_write_string (wh, - (pc->fi_pos == - NULL) ? NULL : pc->fi_pos->serialization)) || - (GNUNET_OK != - GNUNET_BIO_write_string (wh, - (pc->namespace == - NULL) ? NULL : pc->namespace->name))) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_OK != GNUNET_BIO_write_close (wh)) - { - wh = NULL; - GNUNET_break (0); - goto cleanup; - } - return; -cleanup: - if (wh != NULL) - (void) GNUNET_BIO_write_close (wh); - GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, - pc->serialization); - GNUNET_free (pc->serialization); - pc->serialization = NULL; -} - - -/** - * Synchronize this unindex struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param uc the struct to sync - */ -void -GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc) -{ - struct GNUNET_BIO_WriteHandle *wh; - - if (NULL == uc->serialization) - uc->serialization = - make_serialization_file_name (uc->h, - GNUNET_FS_SYNC_PATH_MASTER_UNINDEX); - if (NULL == uc->serialization) - return; - wh = get_write_handle (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, - uc->serialization); - if (wh == NULL) - { - GNUNET_break (0); - goto cleanup; - } - if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uc->filename)) || - (GNUNET_OK != GNUNET_BIO_write_int64 (wh, uc->file_size)) || - (GNUNET_OK != write_start_time (wh, uc->start_time)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) uc->state)) || - ((uc->state == UNINDEX_STATE_FS_NOTIFY) && - (GNUNET_OK != - GNUNET_BIO_write (wh, &uc->file_id, sizeof (GNUNET_HashCode)))) || - ((uc->state == UNINDEX_STATE_ERROR) && - (GNUNET_OK != GNUNET_BIO_write_string (wh, uc->emsg)))) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_OK != GNUNET_BIO_write_close (wh)) - { - wh = NULL; - GNUNET_break (0); - goto cleanup; - } - return; -cleanup: - if (wh != NULL) - (void) GNUNET_BIO_write_close (wh); - GNUNET_FS_remove_sync_file_ (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, - uc->serialization); - GNUNET_free (uc->serialization); - uc->serialization = NULL; -} - - -/** - * Serialize a download request. - * - * @param wh the 'struct GNUNET_BIO_WriteHandle*' - * @param dr the 'struct DownloadRequest' - * @return GNUNET_YES on success, GNUNET_NO on error - */ -static int -write_download_request (struct GNUNET_BIO_WriteHandle *wh, - struct DownloadRequest *dr) -{ - unsigned int i; - - if ((GNUNET_OK != GNUNET_BIO_write_int32 (wh, dr->state)) || - (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dr->offset)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, dr->num_children)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, dr->depth))) - return GNUNET_NO; - if ((dr->state == BRS_CHK_SET) && - (GNUNET_OK != - GNUNET_BIO_write (wh, &dr->chk, sizeof (struct ContentHashKey)))) - return GNUNET_NO; - for (i = 0; i < dr->num_children; i++) - if (GNUNET_NO == write_download_request (wh, dr->children[i])) - return GNUNET_NO; - return GNUNET_YES; -} - - -/** - * Read a download request tree. - * - * @param rh stream to read from - * @return value the 'struct DownloadRequest', NULL on error - */ -static struct DownloadRequest * -read_download_request (struct GNUNET_BIO_ReadHandle *rh) -{ - struct DownloadRequest *dr; - unsigned int i; - - dr = GNUNET_malloc (sizeof (struct DownloadRequest)); - - if ((GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dr->state)) || - (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dr->offset)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dr->num_children)) || - (dr->num_children > CHK_PER_INODE) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dr->depth)) || ((dr->depth == 0) - && - (dr->num_children - > 0)) || - ((dr->depth > 0) && (dr->num_children == 0))) - { - GNUNET_break (0); - dr->num_children = 0; - goto cleanup; - } - if (dr->num_children > 0) - dr->children = - GNUNET_malloc (dr->num_children * sizeof (struct ContentHashKey)); - switch (dr->state) - { - case BRS_INIT: - case BRS_RECONSTRUCT_DOWN: - case BRS_RECONSTRUCT_META_UP: - case BRS_RECONSTRUCT_UP: - break; - case BRS_CHK_SET: - if (GNUNET_OK != - GNUNET_BIO_read (rh, "chk", &dr->chk, sizeof (struct ContentHashKey))) - goto cleanup; - break; - case BRS_DOWNLOAD_DOWN: - case BRS_DOWNLOAD_UP: - case BRS_ERROR: - break; - default: - GNUNET_break (0); - goto cleanup; - } - for (i = 0; i < dr->num_children; i++) - { - if (NULL == (dr->children[i] = read_download_request (rh))) - goto cleanup; - dr->children[i]->parent = dr; - } - return dr; -cleanup: - GNUNET_FS_free_download_request_ (dr); - return NULL; -} - - -/** - * Compute the name of the sync file (or directory) for the given download - * context. - * - * @param dc download context to compute for - * @param uni unique filename to use, use "" for the directory name - * @param ext extension to use, use ".dir" for our own subdirectory - * @return the expanded file name, NULL for none - */ -static char * -get_download_sync_filename (struct GNUNET_FS_DownloadContext *dc, - const char *uni, const char *ext) -{ - char *par; - char *epar; - - if (dc->parent == NULL) - return get_serialization_file_name (dc->h, - (dc->search != - NULL) ? - GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD : - GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD, - uni); - if (dc->parent->serialization == NULL) - return NULL; - par = get_download_sync_filename (dc->parent, dc->parent->serialization, ""); - if (par == NULL) - return NULL; - GNUNET_asprintf (&epar, "%s.dir%s%s%s", par, DIR_SEPARATOR_STR, uni, ext); - GNUNET_free (par); - return epar; -} - - -/** - * Synchronize this download struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param dc the struct to sync - */ -void -GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc) -{ - struct GNUNET_BIO_WriteHandle *wh; - char *uris; - char *fn; - char *dir; - - if (NULL == dc->serialization) - { - dir = get_download_sync_filename (dc, "", ""); - if (dir == NULL) - return; - if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dir)) - { - GNUNET_free (dir); - return; - } - fn = GNUNET_DISK_mktemp (dir); - GNUNET_free (dir); - if (fn == NULL) - return; - dc->serialization = get_serialization_short_name (fn); - } - else - { - fn = get_download_sync_filename (dc, dc->serialization, ""); - if (fn == NULL) - { - GNUNET_free (dc->serialization); - dc->serialization = NULL; - GNUNET_free (fn); - return; - } - } - wh = GNUNET_BIO_write_open (fn); - if (wh == NULL) - { - GNUNET_free (dc->serialization); - dc->serialization = NULL; - GNUNET_free (fn); - return; - } - GNUNET_assert ((GNUNET_YES == GNUNET_FS_uri_test_chk (dc->uri)) || - (GNUNET_YES == GNUNET_FS_uri_test_loc (dc->uri))); - uris = GNUNET_FS_uri_to_string (dc->uri); - if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uris)) || - (GNUNET_OK != GNUNET_BIO_write_meta_data (wh, dc->meta)) || - (GNUNET_OK != GNUNET_BIO_write_string (wh, dc->emsg)) || - (GNUNET_OK != GNUNET_BIO_write_string (wh, dc->filename)) || - (GNUNET_OK != GNUNET_BIO_write_string (wh, dc->temp_filename)) || - (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dc->old_file_size)) || - (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dc->offset)) || - (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dc->length)) || - (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dc->completed)) || - (GNUNET_OK != write_start_time (wh, dc->start_time)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, dc->anonymity)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) dc->options)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) dc->has_finished))) - { - GNUNET_break (0); - goto cleanup; - } - if (NULL == dc->emsg) - { - GNUNET_assert (dc->top_request != NULL); - if (GNUNET_YES != write_download_request (wh, dc->top_request)) - { - GNUNET_break (0); - goto cleanup; - } - } - GNUNET_free_non_null (uris); - uris = NULL; - if (GNUNET_OK != GNUNET_BIO_write_close (wh)) - { - wh = NULL; - GNUNET_break (0); - goto cleanup; - } - GNUNET_free (fn); - return; -cleanup: - if (NULL != wh) - (void) GNUNET_BIO_write_close (wh); - GNUNET_free_non_null (uris); - if (0 != UNLINK (fn)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn); - GNUNET_free (fn); - GNUNET_free (dc->serialization); - dc->serialization = NULL; -} - - -/** - * Synchronize this search result with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param sr the struct to sync - */ -void -GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr) -{ - struct GNUNET_BIO_WriteHandle *wh; - char *uris; - - uris = NULL; - if (NULL == sr->serialization) - sr->serialization = - make_serialization_file_name_in_dir (sr->sc->h, - (sr->sc->psearch_result == - NULL) ? - GNUNET_FS_SYNC_PATH_MASTER_SEARCH : - GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sr->sc->serialization); - if (NULL == sr->serialization) - return; - wh = get_write_handle_in_dir (sr->sc->h, - (sr->sc->psearch_result == - NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : - GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sr->sc->serialization, sr->serialization); - if (wh == NULL) - { - GNUNET_break (0); - goto cleanup; - } - uris = GNUNET_FS_uri_to_string (sr->uri); - if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uris)) || - (GNUNET_OK != - GNUNET_BIO_write_string (wh, - sr->download != - NULL ? sr->download->serialization : NULL)) || - (GNUNET_OK != - GNUNET_BIO_write_string (wh, - sr->update_search != - NULL ? sr->update_search->serialization : NULL)) - || (GNUNET_OK != GNUNET_BIO_write_meta_data (wh, sr->meta)) || - (GNUNET_OK != GNUNET_BIO_write (wh, &sr->key, sizeof (GNUNET_HashCode))) - || (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sr->mandatory_missing)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sr->optional_support)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sr->availability_success)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sr->availability_trials))) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_OK != GNUNET_BIO_write_close (wh)) - { - wh = NULL; - GNUNET_break (0); - goto cleanup; - } - GNUNET_free_non_null (uris); - return; -cleanup: - GNUNET_free_non_null (uris); - if (wh != NULL) - (void) GNUNET_BIO_write_close (wh); - remove_sync_file_in_dir (sr->sc->h, - (sr->sc->psearch_result == - NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : - GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sr->sc->serialization, sr->serialization); - GNUNET_free (sr->serialization); - sr->serialization = NULL; -} - - -/** - * Synchronize this search struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param sc the struct to sync - */ -void -GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc) -{ - struct GNUNET_BIO_WriteHandle *wh; - char *uris; - char in_pause; - const char *category; - - category = - (sc->psearch_result == - NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : - GNUNET_FS_SYNC_PATH_CHILD_SEARCH; - if (NULL == sc->serialization) - sc->serialization = make_serialization_file_name (sc->h, category); - if (NULL == sc->serialization) - return; - uris = NULL; - wh = get_write_handle (sc->h, category, sc->serialization); - if (wh == NULL) - { - GNUNET_break (0); - goto cleanup; - } - GNUNET_assert ((GNUNET_YES == GNUNET_FS_uri_test_ksk (sc->uri)) || - (GNUNET_YES == GNUNET_FS_uri_test_sks (sc->uri))); - uris = GNUNET_FS_uri_to_string (sc->uri); - in_pause = (sc->task != GNUNET_SCHEDULER_NO_TASK) ? 'r' : '\0'; - if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uris)) || - (GNUNET_OK != write_start_time (wh, sc->start_time)) || - (GNUNET_OK != GNUNET_BIO_write_string (wh, sc->emsg)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) sc->options)) || - (GNUNET_OK != GNUNET_BIO_write (wh, &in_pause, sizeof (in_pause))) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sc->anonymity))) - { - GNUNET_break (0); - goto cleanup; - } - GNUNET_free (uris); - uris = NULL; - if (GNUNET_OK != GNUNET_BIO_write_close (wh)) - { - wh = NULL; - GNUNET_break (0); - goto cleanup; - } - return; -cleanup: - if (wh != NULL) - (void) GNUNET_BIO_write_close (wh); - GNUNET_free_non_null (uris); - GNUNET_FS_remove_sync_file_ (sc->h, category, sc->serialization); - GNUNET_free (sc->serialization); - sc->serialization = NULL; -} - - -/** - * Function called with a filename of serialized unindexing operation - * to deserialize. - * - * @param cls the 'struct GNUNET_FS_Handle*' - * @param filename complete filename (absolute path) - * @return GNUNET_OK (continue to iterate) - */ -static int -deserialize_unindex_file (void *cls, const char *filename) -{ - struct GNUNET_FS_Handle *h = cls; - struct GNUNET_BIO_ReadHandle *rh; - struct GNUNET_FS_UnindexContext *uc; - struct GNUNET_FS_ProgressInfo pi; - char *emsg; - uint32_t state; - - uc = GNUNET_malloc (sizeof (struct GNUNET_FS_UnindexContext)); - uc->h = h; - uc->serialization = get_serialization_short_name (filename); - rh = GNUNET_BIO_read_open (filename); - if (rh == NULL) - { - GNUNET_break (0); - goto cleanup; - } - if ((GNUNET_OK != - GNUNET_BIO_read_string (rh, "unindex-fn", &uc->filename, 10 * 1024)) || - (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &uc->file_size)) || - (GNUNET_OK != read_start_time (rh, &uc->start_time)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &state))) - { - GNUNET_break (0); - goto cleanup; - } - uc->state = (enum UnindexState) state; - switch (state) - { - case UNINDEX_STATE_HASHING: - break; - case UNINDEX_STATE_FS_NOTIFY: - if (GNUNET_OK != - GNUNET_BIO_read (rh, "unindex-hash", &uc->file_id, - sizeof (GNUNET_HashCode))) - { - GNUNET_break (0); - goto cleanup; - } - break; - case UNINDEX_STATE_DS_REMOVE: - break; - case UNINDEX_STATE_COMPLETE: - break; - case UNINDEX_STATE_ERROR: - if (GNUNET_OK != - GNUNET_BIO_read_string (rh, "unindex-emsg", &uc->emsg, 10 * 1024)) - { - GNUNET_break (0); - goto cleanup; - } - break; - default: - GNUNET_break (0); - goto cleanup; - } - uc->top = GNUNET_FS_make_top (h, &GNUNET_FS_unindex_signal_suspend_, uc); - pi.status = GNUNET_FS_STATUS_UNINDEX_RESUME; - pi.value.unindex.specifics.resume.message = uc->emsg; - GNUNET_FS_unindex_make_status_ (&pi, uc, - (uc->state == - UNINDEX_STATE_COMPLETE) ? uc->file_size : 0); - switch (uc->state) - { - case UNINDEX_STATE_HASHING: - uc->fhc = - GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, uc->filename, - HASHING_BLOCKSIZE, - &GNUNET_FS_unindex_process_hash_, uc); - break; - case UNINDEX_STATE_FS_NOTIFY: - uc->state = UNINDEX_STATE_HASHING; - GNUNET_FS_unindex_process_hash_ (uc, &uc->file_id); - break; - case UNINDEX_STATE_DS_REMOVE: - GNUNET_FS_unindex_do_remove_ (uc); - break; - case UNINDEX_STATE_COMPLETE: - case UNINDEX_STATE_ERROR: - /* no need to resume any operation, we were done */ - break; - default: - break; - } - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failure while resuming unindexing operation `%s': %s\n"), - filename, emsg); - GNUNET_free (emsg); - } - return GNUNET_OK; -cleanup: - GNUNET_free_non_null (uc->filename); - if ((rh != NULL) && (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failed to resume unindexing operation `%s': %s\n"), filename, - emsg); - GNUNET_free (emsg); - } - if (uc->serialization != NULL) - GNUNET_FS_remove_sync_file_ (h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, - uc->serialization); - GNUNET_free_non_null (uc->serialization); - GNUNET_free (uc); - return GNUNET_OK; -} - - -/** - * Deserialize a download. - * - * @param h overall context - * @param rh file to deserialize from - * @param parent parent download - * @param search associated search - * @param serialization name under which the search was serialized - */ -static void -deserialize_download (struct GNUNET_FS_Handle *h, - struct GNUNET_BIO_ReadHandle *rh, - struct GNUNET_FS_DownloadContext *parent, - struct GNUNET_FS_SearchResult *search, - const char *serialization); - - -/** - * Deserialize a search. - * - * @param h overall context - * @param rh file to deserialize from - * @param psearch_result parent search result - * @param serialization name under which the search was serialized - */ -static struct GNUNET_FS_SearchContext * -deserialize_search (struct GNUNET_FS_Handle *h, - struct GNUNET_BIO_ReadHandle *rh, - struct GNUNET_FS_SearchResult *psearch_result, - const char *serialization); - - -/** - * Function called with a filename of serialized search result - * to deserialize. - * - * @param cls the 'struct GNUNET_FS_SearchContext*' - * @param filename complete filename (absolute path) - * @return GNUNET_OK (continue to iterate) - */ -static int -deserialize_search_result (void *cls, const char *filename) -{ - struct GNUNET_FS_SearchContext *sc = cls; - char *ser; - char *uris; - char *emsg; - char *download; - char *update_srch; - struct GNUNET_BIO_ReadHandle *rh; - struct GNUNET_BIO_ReadHandle *drh; - struct GNUNET_FS_SearchResult *sr; - - ser = get_serialization_short_name (filename); - rh = GNUNET_BIO_read_open (filename); - if (rh == NULL) - { - if (ser != NULL) - { - remove_sync_file_in_dir (sc->h, - (sc->psearch_result == - NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : - GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sc->serialization, ser); - GNUNET_free (ser); - } - return GNUNET_OK; - } - emsg = NULL; - uris = NULL; - download = NULL; - update_srch = NULL; - sr = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchResult)); - sr->serialization = ser; - if ((GNUNET_OK != GNUNET_BIO_read_string (rh, "result-uri", &uris, 10 * 1024)) - || (NULL == (sr->uri = GNUNET_FS_uri_parse (uris, &emsg))) || - (GNUNET_OK != GNUNET_BIO_read_string (rh, "download-lnk", &download, 16)) - || (GNUNET_OK != - GNUNET_BIO_read_string (rh, "search-lnk", &update_srch, 16)) || - (GNUNET_OK != GNUNET_BIO_read_meta_data (rh, "result-meta", &sr->meta)) || - (GNUNET_OK != - GNUNET_BIO_read (rh, "result-key", &sr->key, sizeof (GNUNET_HashCode))) - || (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sr->mandatory_missing)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sr->optional_support)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sr->availability_success)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sr->availability_trials))) - { - GNUNET_break (0); - goto cleanup; - } - GNUNET_free (uris); - if (download != NULL) - { - drh = get_read_handle (sc->h, GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD, download); - if (drh != NULL) - { - deserialize_download (sc->h, drh, NULL, sr, download); - if (GNUNET_OK != GNUNET_BIO_read_close (drh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failed to resume sub-download `%s': %s\n"), download, - emsg); - GNUNET_free (emsg); - } - } - GNUNET_free (download); - } - if (update_srch != NULL) - { - drh = - get_read_handle (sc->h, GNUNET_FS_SYNC_PATH_CHILD_SEARCH, update_srch); - if (drh != NULL) - { - deserialize_search (sc->h, drh, sr, update_srch); - if (GNUNET_OK != GNUNET_BIO_read_close (drh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failed to resume sub-search `%s': %s\n"), update_srch, - emsg); - GNUNET_free (emsg); - } - } - GNUNET_free (update_srch); - } - GNUNET_CONTAINER_multihashmap_put (sc->master_result_map, &sr->key, sr, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failure while resuming search operation `%s': %s\n"), - filename, emsg); - GNUNET_free (emsg); - } - return GNUNET_OK; -cleanup: - GNUNET_free_non_null (download); - GNUNET_free_non_null (emsg); - GNUNET_free_non_null (uris); - GNUNET_free_non_null (update_srch); - if (sr->uri != NULL) - GNUNET_FS_uri_destroy (sr->uri); - if (sr->meta != NULL) - GNUNET_CONTAINER_meta_data_destroy (sr->meta); - GNUNET_free (sr->serialization); - GNUNET_free (sr); - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failure while resuming search operation `%s': %s\n"), - filename, emsg); - GNUNET_free (emsg); - } - return GNUNET_OK; -} - - -/** - * Send the 'resume' signal to the callback; also actually - * resume the download (put it in the queue). Does this - * recursively for the top-level download and all child - * downloads. - * - * @param dc download to resume - */ -static void -signal_download_resume (struct GNUNET_FS_DownloadContext *dc) -{ - struct GNUNET_FS_DownloadContext *dcc; - struct GNUNET_FS_ProgressInfo pi; - - pi.status = GNUNET_FS_STATUS_DOWNLOAD_RESUME; - pi.value.download.specifics.resume.meta = dc->meta; - pi.value.download.specifics.resume.message = dc->emsg; - GNUNET_FS_download_make_status_ (&pi, dc); - dcc = dc->child_head; - while (NULL != dcc) - { - signal_download_resume (dcc); - dcc = dcc->next; - } - if (dc->pending_head != NULL) - GNUNET_FS_download_start_downloading_ (dc); -} - - -/** - * Signal resuming of a search to our clients (for the - * top level search and all sub-searches). - * - * @param sc search being resumed - */ -static void -signal_search_resume (struct GNUNET_FS_SearchContext *sc); - - -/** - * Iterator over search results signaling resume to the client for - * each result. - * - * @param cls closure, the 'struct GNUNET_FS_SearchContext' - * @param key current key code - * @param value value in the hash map, the 'struct GNUNET_FS_SearchResult' - * @return GNUNET_YES (we should continue to iterate) - */ -static int -signal_result_resume (void *cls, const GNUNET_HashCode * key, void *value) -{ - struct GNUNET_FS_SearchContext *sc = cls; - struct GNUNET_FS_ProgressInfo pi; - struct GNUNET_FS_SearchResult *sr = value; - - if (0 == sr->mandatory_missing) - { - pi.status = GNUNET_FS_STATUS_SEARCH_RESUME_RESULT; - pi.value.search.specifics.resume_result.meta = sr->meta; - pi.value.search.specifics.resume_result.uri = sr->uri; - pi.value.search.specifics.resume_result.result = sr; - pi.value.search.specifics.resume_result.availability_rank = - 2 * sr->availability_success - sr->availability_trials; - pi.value.search.specifics.resume_result.availability_certainty = - sr->availability_trials; - pi.value.search.specifics.resume_result.applicability_rank = - sr->optional_support; - sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc); - } - if (sr->download != NULL) - { - signal_download_resume (sr->download); - } - else - { - GNUNET_FS_search_start_probe_ (sr); - } - if (sr->update_search != NULL) - signal_search_resume (sr->update_search); - return GNUNET_YES; -} - - -/** - * Free memory allocated by the search context and its children - * - * @param sc search context to free - */ -static void -free_search_context (struct GNUNET_FS_SearchContext *sc); - - -/** - * Iterator over search results freeing each. - * - * @param cls closure, the 'struct GNUNET_FS_SearchContext' - * @param key current key code - * @param value value in the hash map, the 'struct GNUNET_FS_SearchResult' - * @return GNUNET_YES (we should continue to iterate) - */ -static int -free_result (void *cls, const GNUNET_HashCode * key, void *value) -{ - struct GNUNET_FS_SearchResult *sr = value; - - if (sr->update_search != NULL) - { - free_search_context (sr->update_search); - GNUNET_assert (NULL == sr->update_search); - } - GNUNET_CONTAINER_meta_data_destroy (sr->meta); - GNUNET_FS_uri_destroy (sr->uri); - GNUNET_free (sr); - return GNUNET_YES; -} - - -/** - * Free memory allocated by the search context and its children - * - * @param sc search context to free - */ -static void -free_search_context (struct GNUNET_FS_SearchContext *sc) -{ - if (sc->serialization != NULL) - { - GNUNET_FS_remove_sync_file_ (sc->h, - (sc->psearch_result == - NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : - GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sc->serialization); - GNUNET_FS_remove_sync_dir_ (sc->h, - (sc->psearch_result == - NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : - GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sc->serialization); - } - GNUNET_free_non_null (sc->serialization); - GNUNET_free_non_null (sc->emsg); - if (sc->uri != NULL) - GNUNET_FS_uri_destroy (sc->uri); - if (sc->master_result_map != NULL) - { - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, &free_result, - sc); - GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map); - } - GNUNET_free (sc); -} - - -/** - * Function called with a filename of serialized sub-download - * to deserialize. - * - * @param cls the 'struct GNUNET_FS_DownloadContext*' (parent) - * @param filename complete filename (absolute path) - * @return GNUNET_OK (continue to iterate) - */ -static int -deserialize_subdownload (void *cls, const char *filename) -{ - struct GNUNET_FS_DownloadContext *parent = cls; - char *ser; - char *emsg; - struct GNUNET_BIO_ReadHandle *rh; - - ser = get_serialization_short_name (filename); - rh = GNUNET_BIO_read_open (filename); - if (rh == NULL) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ - ("Failed to resume sub-download `%s': could not open file `%s'\n"), - ser, filename); - GNUNET_free (ser); - return GNUNET_OK; - } - deserialize_download (parent->h, rh, parent, NULL, ser); - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failed to resume sub-download `%s': %s\n"), ser, emsg); - GNUNET_free (emsg); - } - GNUNET_free (ser); - return GNUNET_OK; -} - - -/** - * Free this download context and all of its descendants. - * (only works during deserialization since not all possible - * state it taken care of). - * - * @param dc context to free - */ -static void -free_download_context (struct GNUNET_FS_DownloadContext *dc) -{ - struct GNUNET_FS_DownloadContext *dcc; - - if (dc->meta != NULL) - GNUNET_CONTAINER_meta_data_destroy (dc->meta); - if (dc->uri != NULL) - GNUNET_FS_uri_destroy (dc->uri); - GNUNET_free_non_null (dc->temp_filename); - GNUNET_free_non_null (dc->emsg); - GNUNET_free_non_null (dc->filename); - GNUNET_free_non_null (dc->serialization); - while (NULL != (dcc = dc->child_head)) - { - GNUNET_CONTAINER_DLL_remove (dc->child_head, dc->child_tail, dcc); - free_download_context (dcc); - } - GNUNET_FS_free_download_request_ (dc->top_request); - if (NULL != dc->active) - GNUNET_CONTAINER_multihashmap_destroy (dc->active); - GNUNET_free (dc); -} - - -/** - * Deserialize a download. - * - * @param h overall context - * @param rh file to deserialize from - * @param parent parent download - * @param search associated search - * @param serialization name under which the search was serialized - */ -static void -deserialize_download (struct GNUNET_FS_Handle *h, - struct GNUNET_BIO_ReadHandle *rh, - struct GNUNET_FS_DownloadContext *parent, - struct GNUNET_FS_SearchResult *search, - const char *serialization) -{ - struct GNUNET_FS_DownloadContext *dc; - char *emsg; - char *uris; - char *dn; - uint32_t options; - uint32_t status; - - uris = NULL; - emsg = NULL; - dc = GNUNET_malloc (sizeof (struct GNUNET_FS_DownloadContext)); - dc->parent = parent; - dc->h = h; - dc->serialization = GNUNET_strdup (serialization); - if ((GNUNET_OK != - GNUNET_BIO_read_string (rh, "download-uri", &uris, 10 * 1024)) || - (NULL == (dc->uri = GNUNET_FS_uri_parse (uris, &emsg))) || - ((GNUNET_YES != GNUNET_FS_uri_test_chk (dc->uri)) && - (GNUNET_YES != GNUNET_FS_uri_test_loc (dc->uri))) || - (GNUNET_OK != GNUNET_BIO_read_meta_data (rh, "download-meta", &dc->meta)) - || (GNUNET_OK != - GNUNET_BIO_read_string (rh, "download-emsg", &dc->emsg, 10 * 1024)) || - (GNUNET_OK != - GNUNET_BIO_read_string (rh, "download-fn", &dc->filename, 10 * 1024)) || - (GNUNET_OK != - GNUNET_BIO_read_string (rh, "download-tfn", &dc->temp_filename, - 10 * 1024)) || - (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dc->old_file_size)) || - (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dc->offset)) || - (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dc->length)) || - (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dc->completed)) || - (GNUNET_OK != read_start_time (rh, &dc->start_time)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dc->anonymity)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &options)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &status))) - { - GNUNET_break (0); - goto cleanup; - } - dc->options = (enum GNUNET_FS_DownloadOptions) options; - dc->active = - GNUNET_CONTAINER_multihashmap_create (1 + 2 * (dc->length / DBLOCK_SIZE)); - dc->has_finished = (int) status; - dc->treedepth = - GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size (dc->uri)); - if (GNUNET_FS_uri_test_loc (dc->uri)) - GNUNET_assert (GNUNET_OK == - GNUNET_FS_uri_loc_get_peer_identity (dc->uri, &dc->target)); - if (dc->emsg == NULL) - { - dc->top_request = read_download_request (rh); - if (dc->top_request == NULL) - { - GNUNET_break (0); - goto cleanup; - } - } - dn = get_download_sync_filename (dc, dc->serialization, ".dir"); - if (dn != NULL) - { - if (GNUNET_YES == GNUNET_DISK_directory_test (dn)) - GNUNET_DISK_directory_scan (dn, &deserialize_subdownload, dc); - GNUNET_free (dn); - } - if (parent != NULL) - { - GNUNET_abort (); // for debugging for now - FIXME - GNUNET_CONTAINER_DLL_insert (parent->child_head, parent->child_tail, dc); - } - if (search != NULL) - { - dc->search = search; - search->download = dc; - } - if ((parent == NULL) && (search == NULL)) - { - dc->top = - GNUNET_FS_make_top (dc->h, &GNUNET_FS_download_signal_suspend_, dc); - signal_download_resume (dc); - } - GNUNET_free (uris); - dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); - return; -cleanup: - GNUNET_free_non_null (uris); - GNUNET_free_non_null (emsg); - free_download_context (dc); -} - - -/** - * Signal resuming of a search to our clients (for the - * top level search and all sub-searches). - * - * @param sc search being resumed - */ -static void -signal_search_resume (struct GNUNET_FS_SearchContext *sc) -{ - struct GNUNET_FS_ProgressInfo pi; - - pi.status = GNUNET_FS_STATUS_SEARCH_RESUME; - pi.value.search.specifics.resume.message = sc->emsg; - pi.value.search.specifics.resume.is_paused = - (sc->client == NULL) ? GNUNET_YES : GNUNET_NO; - sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc); - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &signal_result_resume, sc); - -} - - -/** - * Deserialize a search. - * - * @param h overall context - * @param rh file to deserialize from - * @param psearch_result parent search result - * @param serialization name under which the search was serialized - */ -static struct GNUNET_FS_SearchContext * -deserialize_search (struct GNUNET_FS_Handle *h, - struct GNUNET_BIO_ReadHandle *rh, - struct GNUNET_FS_SearchResult *psearch_result, - const char *serialization) -{ - struct GNUNET_FS_SearchContext *sc; - char *emsg; - char *uris; - char *dn; - uint32_t options; - char in_pause; - - if ((psearch_result != NULL) && (psearch_result->update_search != NULL)) - { - GNUNET_break (0); - return NULL; - } - uris = NULL; - emsg = NULL; - sc = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchContext)); - if (psearch_result != NULL) - { - sc->psearch_result = psearch_result; - psearch_result->update_search = sc; - } - sc->h = h; - sc->serialization = GNUNET_strdup (serialization); - if ((GNUNET_OK != GNUNET_BIO_read_string (rh, "search-uri", &uris, 10 * 1024)) - || (NULL == (sc->uri = GNUNET_FS_uri_parse (uris, &emsg))) || - ((GNUNET_YES != GNUNET_FS_uri_test_ksk (sc->uri)) && - (GNUNET_YES != GNUNET_FS_uri_test_sks (sc->uri))) || - (GNUNET_OK != read_start_time (rh, &sc->start_time)) || - (GNUNET_OK != - GNUNET_BIO_read_string (rh, "search-emsg", &sc->emsg, 10 * 1024)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &options)) || - (GNUNET_OK != - GNUNET_BIO_read (rh, "search-pause", &in_pause, sizeof (in_pause))) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sc->anonymity))) - { - GNUNET_break (0); - goto cleanup; - } - sc->options = (enum GNUNET_FS_SearchOptions) options; - sc->master_result_map = GNUNET_CONTAINER_multihashmap_create (16); - dn = get_serialization_file_name_in_dir (h, - (sc->psearch_result == - NULL) ? - GNUNET_FS_SYNC_PATH_MASTER_SEARCH : - GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sc->serialization, ""); - if (dn != NULL) - { - if (GNUNET_YES == GNUNET_DISK_directory_test (dn)) - GNUNET_DISK_directory_scan (dn, &deserialize_search_result, sc); - GNUNET_free (dn); - } - if (('\0' == in_pause) && - (GNUNET_OK != GNUNET_FS_search_start_searching_ (sc))) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ - ("Could not resume running search, will resume as paused search\n")); - } - signal_search_resume (sc); - GNUNET_free (uris); - return sc; -cleanup: - GNUNET_free_non_null (emsg); - free_search_context (sc); - GNUNET_free_non_null (uris); - return NULL; -} - - -/** - * Function called with a filename of serialized search operation - * to deserialize. - * - * @param cls the 'struct GNUNET_FS_Handle*' - * @param filename complete filename (absolute path) - * @return GNUNET_OK (continue to iterate) - */ -static int -deserialize_search_file (void *cls, const char *filename) -{ - struct GNUNET_FS_Handle *h = cls; - char *ser; - char *emsg; - struct GNUNET_BIO_ReadHandle *rh; - struct GNUNET_FS_SearchContext *sc; - - ser = get_serialization_short_name (filename); - rh = GNUNET_BIO_read_open (filename); - if (rh == NULL) - { - if (ser != NULL) - { - GNUNET_FS_remove_sync_file_ (h, GNUNET_FS_SYNC_PATH_MASTER_SEARCH, ser); - GNUNET_free (ser); - } - return GNUNET_OK; - } - sc = deserialize_search (h, rh, NULL, ser); - if (sc != NULL) - sc->top = GNUNET_FS_make_top (h, &GNUNET_FS_search_signal_suspend_, sc); - GNUNET_free (ser); - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failure while resuming search operation `%s': %s\n"), - filename, emsg); - GNUNET_free (emsg); - } - return GNUNET_OK; -} - - -/** - * Function called with a filename of serialized download operation - * to deserialize. - * - * @param cls the 'struct GNUNET_FS_Handle*' - * @param filename complete filename (absolute path) - * @return GNUNET_OK (continue to iterate) - */ -static int -deserialize_download_file (void *cls, const char *filename) -{ - struct GNUNET_FS_Handle *h = cls; - char *ser; - char *emsg; - struct GNUNET_BIO_ReadHandle *rh; - - ser = get_serialization_short_name (filename); - rh = GNUNET_BIO_read_open (filename); - if (rh == NULL) - { - if (0 != UNLINK (filename)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); - GNUNET_free (ser); - return GNUNET_OK; - } - deserialize_download (h, rh, NULL, NULL, ser); - GNUNET_free (ser); - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failure while resuming download operation `%s': %s\n"), - filename, emsg); - GNUNET_free (emsg); - } - return GNUNET_OK; -} - - -/** - * Deserialize informatin about pending operations. - * - * @param master_path which master directory should be scanned - * @param proc function to call for each entry (will get 'h' for 'cls') - * @param h the 'struct GNUNET_FS_Handle*' - */ -static void -deserialization_master (const char *master_path, GNUNET_FileNameCallback proc, - struct GNUNET_FS_Handle *h) -{ - char *dn; - - dn = get_serialization_file_name (h, master_path, ""); - if (dn == NULL) - return; - if (GNUNET_YES == GNUNET_DISK_directory_test (dn)) - GNUNET_DISK_directory_scan (dn, proc, h); - GNUNET_free (dn); -} - - -/** - * Setup a connection to the file-sharing service. - * - * @param cfg configuration to use - * @param client_name unique identifier for this client - * @param upcb function to call to notify about FS actions - * @param upcb_cls closure for upcb - * @param flags specific attributes for fs-operations - * @param ... list of optional options, terminated with GNUNET_FS_OPTIONS_END - * @return NULL on error - */ -struct GNUNET_FS_Handle * -GNUNET_FS_start (const struct GNUNET_CONFIGURATION_Handle *cfg, - const char *client_name, GNUNET_FS_ProgressCallback upcb, - void *upcb_cls, enum GNUNET_FS_Flags flags, ...) -{ - struct GNUNET_FS_Handle *ret; - enum GNUNET_FS_OPTIONS opt; - va_list ap; - - ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Handle)); - ret->cfg = cfg; - ret->client_name = GNUNET_strdup (client_name); - ret->upcb = upcb; - ret->upcb_cls = upcb_cls; - ret->flags = flags; - ret->max_parallel_downloads = 1; - ret->max_parallel_requests = 1; - ret->avg_block_latency = GNUNET_TIME_UNIT_MINUTES; /* conservative starting point */ - va_start (ap, flags); - while (GNUNET_FS_OPTIONS_END != (opt = va_arg (ap, enum GNUNET_FS_OPTIONS))) - { - switch (opt) - { - case GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM: - ret->max_parallel_downloads = va_arg (ap, unsigned int); - - break; - case GNUNET_FS_OPTIONS_REQUEST_PARALLELISM: - ret->max_parallel_requests = va_arg (ap, unsigned int); - - break; - default: - GNUNET_break (0); - GNUNET_free (ret->client_name); - GNUNET_free (ret); - va_end (ap); - return NULL; - } - } - va_end (ap); - if (0 != (GNUNET_FS_FLAGS_PERSISTENCE & flags)) - { - deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, - &deserialize_publish_file, ret); - deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_SEARCH, - &deserialize_search_file, ret); - deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD, - &deserialize_download_file, ret); - deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, - &deserialize_unindex_file, ret); - } - return ret; -} - - -/** - * Close our connection with the file-sharing service. - * The callback given to GNUNET_FS_start will no longer be - * called after this function returns. - * - * @param h handle that was returned from GNUNET_FS_start - */ -void -GNUNET_FS_stop (struct GNUNET_FS_Handle *h) -{ - while (h->top_head != NULL) - h->top_head->ssf (h->top_head->ssf_cls); - if (h->queue_job != GNUNET_SCHEDULER_NO_TASK) - GNUNET_SCHEDULER_cancel (h->queue_job); - GNUNET_free (h->client_name); - GNUNET_free (h); -} - - -/* end of fs.c */ diff --git a/src/fs/fs.h b/src/fs/fs.h index b5fdb1cc7..89a375022 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h @@ -34,202 +34,18 @@ #include "block_fs.h" -/** - * Maximum number of outgoing messages we queue per peer. - * - * Performance measurements for 2 peer setup for 50 MB file - * (with MAX_DATASTORE_QUEUE = 1 and RETRY_PROBABILITY_INV = 1): - * - * 2: 1700 kb/s, 1372 kb/s - * 8: 2117 kb/s, 1284 kb/s, 1112 kb/s - * 16: 3500 kb/s, 3200 kb/s, 3388 kb/s - * 32: 3441 kb/s, 3163 kb/s, 3277 kb/s - * 128: 1700 kb/s; 2010 kb/s, 3383 kb/s, 1156 kb/s - * - * Conclusion: 16 seems to be a pretty good value (stable - * and high performance, no excessive memory use). - */ -#define MAX_QUEUE_PER_PEER 16 - -/** - * Maximum size of the datastore queue for P2P operations. - * Needs to be large enough to queue MAX_QUEUE_PER_PEER - * operations for roughly the number of active (connected) - * peers. - */ -#define MAX_DATASTORE_QUEUE (16 * MAX_QUEUE_PER_PEER) - -/** - * Maximum number of blocks we keep in memory for migration. - */ -#define MAX_MIGRATION_QUEUE 8 - -/** - * Blocks are at most migrated to this number of peers - * plus one, each time they are fetched from the database. - */ -#define MIGRATION_LIST_SIZE 2 - -/** - * To how many peers do we forward each migration block ultimately? - * This number must be smaller or equal to MIGRATION_LIST_SIZE. Using - * a smaller value allows for variation in available bandwidth (for - * migration) between the peers. - */ -#define MIGRATION_TARGET_COUNT 2 - -/** - * Ratio for moving average delay calculation. The previous - * average goes in with a factor of (n-1) into the calculation. - * Must be > 0. - */ -#define RUNAVG_DELAY_N 16 - /** * Size of the individual blocks used for file-sharing. */ #define DBLOCK_SIZE (32*1024) /** - * Maximum legal size for a kblock. - */ -#define MAX_KBLOCK_SIZE (60 * 1024) - -/** - * Maximum legal size for an sblock. - */ -#define MAX_SBLOCK_SIZE (60 * 1024) - -/** - * Maximum legal size for an nblock. - */ -#define MAX_NBLOCK_SIZE (60 * 1024) - -/** - * Pick a multiple of 2 here to achive 8-byte alignment! - * We also probably want DBlocks to have (roughly) the - * same size as IBlocks. With SHA-512, the optimal - * value is 32768 byte / 128 byte = 256 - * (128 byte = 2 * 512 bits). DO NOT CHANGE! - */ -#define CHK_PER_INODE 256 - - -/** - * Maximum size for a file to be considered for - * inlining in a directory. - */ -#define MAX_INLINE_SIZE 65536 - - -/** - * Blocksize to use when hashing files - * for indexing (blocksize for IO, not for - * the DBlocks). Larger blocksizes can - * be more efficient but will be more disruptive - * as far as the scheduler is concerned. + * Blocksize to use when hashing files for indexing (blocksize for IO, + * not for the DBlocks). Larger blocksizes can be more efficient but + * will be more disruptive as far as the scheduler is concerned. */ #define HASHING_BLOCKSIZE (1024 * 128) -/** - * Number of availability trials we perform per search result. - */ -#define AVAILABILITY_TRIALS_MAX 8 - -/** - * Length of the P2P success tracker. Note that - * having a very long list can also hurt performance. - */ -#define P2P_SUCCESS_LIST_SIZE 8 - - -/** - * Length of the CS-2-P success tracker. Note that - * having a very long list can also hurt performance. - */ -#define CS2P_SUCCESS_LIST_SIZE 8 - -/** - * How long are we willing to wait for the datastore to be ready to - * process a request for a query without priority? - */ -#define BASIC_DATASTORE_REQUEST_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5) - - -/** - * How long are we willing to wait for the core to be ready to - * transmit a reply to the target peer (if we can not transmit - * until then, we will discard the reply). - */ -#define ACCEPTABLE_REPLY_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5) - - -/** - * Bandwidth value of an (effectively) 0-priority query. - */ -#define QUERY_BANDWIDTH_VALUE 1 - -/** - * Bandwidth value of a 0-priority content (must be - * fairly high compared to query since content is - * typically significantly larger -- and more valueable - * since it can take many queries to get one piece of - * content). - */ -#define CONTENT_BANDWIDTH_VALUE 800 - -/** - * By which amount do we decrement the TTL for simple forwarding / - * indirection of the query; in milli-seconds. Set somewhat in - * accordance to your network latency (above the time it'll take you - * to send a packet and get a reply). - */ -#define TTL_DECREMENT 5000 - -/** - * Until which load do we consider the peer idle and do not - * charge at all? (should be larger than GNUNET_IDLE_LOAD_THRESHOLD used - * by the rest of the code)! - */ -#define IDLE_LOAD_THRESHOLD ((100 + GNUNET_CONSTANTS_IDLE_LOAD_THRESHOLD) / 2) - -/** - * Name of the directory with top-level searches. - */ -#define GNUNET_FS_SYNC_PATH_MASTER_SEARCH "search" - -/** - * Name of the directory with sub-searches (namespace-updates). - */ -#define GNUNET_FS_SYNC_PATH_CHILD_SEARCH "search-child" - -/** - * Name of the directory with master downloads (not associated - * with search or part of another download). - */ -#define GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD "download" - -/** - * Name of the directory with downloads that are part of another - * download or a search. - */ -#define GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD "download-child" - -/** - * Name of the directory with publishing operations. - */ -#define GNUNET_FS_SYNC_PATH_MASTER_PUBLISH "publish" - -/** - * Name of the directory with files that are being published - */ -#define GNUNET_FS_SYNC_PATH_FILE_INFO "publish-file" - -/** - * Name of the directory with unindex operations. - */ -#define GNUNET_FS_SYNC_PATH_MASTER_UNINDEX "unindex" - /** * @brief content hash key @@ -248,1817 +64,6 @@ struct ContentHashKey }; -/** - * @brief complete information needed - * to download a file. - */ -struct FileIdentifier -{ - - /** - * Total size of the file in bytes. (network byte order (!)) - */ - uint64_t file_length; - - /** - * Query and key of the top GNUNET_EC_IBlock. - */ - struct ContentHashKey chk; - -}; - - -/** - * Information about a file and its location - * (peer claiming to share the file). - */ -struct Location -{ - /** - * Information about the shared file. - */ - struct FileIdentifier fi; - - /** - * Identity of the peer sharing the file. - */ - struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded peer; - - /** - * Time when this location URI expires. - */ - struct GNUNET_TIME_Absolute expirationTime; - - /** - * RSA signature over the GNUNET_EC_FileIdentifier, - * GNUNET_hash of the peer and expiration time. - */ - struct GNUNET_CRYPTO_RsaSignature contentSignature; - -}; - -/** - * Types of URIs. - */ -enum uri_types -{ - /** - * Content-hash-key (simple file). - */ - chk, - - /** - * Signed key space (file in namespace). - */ - sks, - - /** - * Keyword search key (query with keywords). - */ - ksk, - - /** - * Location (chk with identity of hosting peer). - */ - loc -}; - -/** - * A Universal Resource Identifier (URI), opaque. - */ -struct GNUNET_FS_Uri -{ - /** - * Type of the URI. - */ - enum uri_types type; - - union - { - struct - { - /** - * Keywords start with a '+' if they are - * mandatory (in which case the '+' is NOT - * part of the keyword) and with a - * simple space if they are optional - * (in which case the space is ALSO not - * part of the actual keyword). - * - * Double-quotes to protect spaces and - * %-encoding are NOT used internally - * (only in URI-strings). - */ - char **keywords; - - /** - * Size of the keywords array. - */ - unsigned int keywordCount; - } ksk; - - struct - { - /** - * Hash of the public key for the namespace. - */ - GNUNET_HashCode namespace; - - /** - * Human-readable identifier chosen for this - * entry in the namespace. - */ - char *identifier; - } sks; - - /** - * Information needed to retrieve a file (content-hash-key - * plus file size). - */ - struct FileIdentifier chk; - - /** - * Information needed to retrieve a file including signed - * location (identity of a peer) of the content. - */ - struct Location loc; - } data; - -}; - - -/** - * Information for a file or directory that is - * about to be published. - */ -struct GNUNET_FS_FileInformation -{ - - /** - * Files in a directory are kept as a linked list. - */ - struct GNUNET_FS_FileInformation *next; - - /** - * If this is a file in a directory, "dir" refers to - * the directory; otherwise NULL. - */ - struct GNUNET_FS_FileInformation *dir; - - /** - * Handle to the master context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Pointer kept for the client. - */ - void *client_info; - - /** - * Metadata to use for the file. - */ - struct GNUNET_CONTAINER_MetaData *meta; - - /** - * Keywords to use for KBlocks. - */ - struct GNUNET_FS_Uri *keywords; - - /** - * CHK for this file or directory. NULL if - * we have not yet computed it. - */ - struct GNUNET_FS_Uri *chk_uri; - - /** - * Block options for the file. - */ - struct GNUNET_FS_BlockOptions bo; - - /** - * At what time did we start this upload? - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * Under what filename is this struct serialized - * (for operational persistence). Should be determined - * using 'mktemp'. - */ - char *serialization; - - /** - * Encoder being used to publish this file. - */ - struct GNUNET_FS_TreeEncoder *te; - - /** - * Error message (non-NULL if this operation failed). - */ - char *emsg; - - /** - * Name of the file or directory (must be an absolute path). - */ - char *filename; - - /** - * Data describing either the file or the directory. - */ - union - { - - /** - * Data for a file. - */ - struct - { - - /** - * Function that can be used to read the data for the file. - */ - GNUNET_FS_DataReader reader; - - /** - * Closure for reader. - */ - void *reader_cls; - - /** - * If this file is being indexed, this value is set to the hash - * over the entire file (when the indexing process is started). - * Otherwise this field is not used. - */ - GNUNET_HashCode file_id; - - /** - * Size of the file (in bytes). - */ - uint64_t file_size; - - /** - * Should the file be indexed or inserted? - */ - int do_index; - - /** - * Is "file_id" already valid? Set to GNUNET_YES once the hash - * has been calculated. - */ - int have_hash; - - /** - * Has the service confirmed our INDEX_START request? - * GNUNET_YES if this step has been completed. - */ - int index_start_confirmed; - - } file; - - /** - * Data for a directory. - */ - struct - { - - /** - * Linked list of entries in the directory. - */ - struct GNUNET_FS_FileInformation *entries; - - /** - * Size of the directory itself (in bytes); 0 if the - * size has not yet been calculated. - */ - size_t dir_size; - - /** - * Pointer to the data for the directory (or NULL if not - * available). - */ - void *dir_data; - - } dir; - - } data; - - /** - * Is this struct for a file or directory? - */ - int is_directory; - - /** - * Are we done publishing this file? - */ - int is_published; - -}; - - -/** - * The job is now ready to run and should use the given client - * handle to communicate with the FS service. - * - * @param cls closure - * @param client handle to use for FS communication - */ -typedef void (*GNUNET_FS_QueueStart) (void *cls, - struct GNUNET_CLIENT_Connection * client); - - -/** - * The job must now stop to run and should destry the client handle as - * soon as possible (ideally prior to returning). - */ -typedef void (*GNUNET_FS_QueueStop) (void *cls); - - -/** - * Entry in the job queue. - */ -struct GNUNET_FS_QueueEntry -{ - /** - * This is a linked list. - */ - struct GNUNET_FS_QueueEntry *next; - - /** - * This is a linked list. - */ - struct GNUNET_FS_QueueEntry *prev; - - /** - * Function to call when the job is started. - */ - GNUNET_FS_QueueStart start; - - /** - * Function to call when the job needs to stop (or is done / dequeued). - */ - GNUNET_FS_QueueStop stop; - - /** - * Closure for start and stop. - */ - void *cls; - - /** - * Handle to FS primary context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Client handle, or NULL if job is not running. - */ - struct GNUNET_CLIENT_Connection *client; - - /** - * Time the job was originally queued. - */ - struct GNUNET_TIME_Absolute queue_time; - - /** - * Time the job was started last. - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * Total amount of time the job has been running (except for the - * current run). - */ - struct GNUNET_TIME_Relative run_time; - - /** - * How many blocks do the active downloads have? - */ - unsigned int blocks; - - /** - * How often have we (re)started this download? - */ - unsigned int start_times; - -}; - - - - -/** - * Information we store for each search result. - */ -struct GNUNET_FS_SearchResult -{ - - /** - * Search context this result belongs to. - */ - struct GNUNET_FS_SearchContext *sc; - - /** - * URI to which this search result refers to. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Metadata for the search result. - */ - struct GNUNET_CONTAINER_MetaData *meta; - - /** - * Client info for this search result. - */ - void *client_info; - - /** - * ID of a job that is currently probing this results' availability - * (NULL if we are not currently probing). - */ - struct GNUNET_FS_DownloadContext *probe_ctx; - - /** - * ID of an associated download based on this search result (or - * NULL for none). - */ - struct GNUNET_FS_DownloadContext *download; - - /** - * If this search result triggered an update search, this field - * links to the update search. - */ - struct GNUNET_FS_SearchContext *update_search; - - /** - * Name under which this search result is stored on disk. - */ - char *serialization; - - /** - * Key for the search result - */ - GNUNET_HashCode key; - - /** - * ID of the task that will clean up the probe_ctx should it not - * complete on time (and that will need to be cancelled if we clean - * up the search result before then). - */ - GNUNET_SCHEDULER_TaskIdentifier probe_cancel_task; - - /** - * When did the current probe become active? - */ - struct GNUNET_TIME_Absolute probe_active_time; - - /** - * How much longer should we run the current probe before giving up? - */ - struct GNUNET_TIME_Relative remaining_probe_time; - - /** - * Number of mandatory keywords for which we have NOT yet found the - * search result; when this value hits zero, the search result is - * given to the callback. - */ - uint32_t mandatory_missing; - - /** - * Number of optional keywords under which this result was also - * found. - */ - uint32_t optional_support; - - /** - * Number of availability tests that have succeeded for this result. - */ - uint32_t availability_success; - - /** - * Number of availability trials that we have performed for this - * search result. - */ - uint32_t availability_trials; - -}; - - -/** - * Add a job to the queue. - * - * @param h handle to the overall FS state - * @param start function to call to begin the job - * @param stop function to call to pause the job, or on dequeue (if the job was running) - * @param cls closure for start and stop - * @param blocks number of blocks this download has - * @return queue handle - */ -struct GNUNET_FS_QueueEntry * -GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, GNUNET_FS_QueueStart start, - GNUNET_FS_QueueStop stop, void *cls, unsigned int blocks); - - -/** - * Dequeue a job from the queue. - * @param qh handle for the job - */ -void -GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh); - - -/** - * Function that provides data by reading from a file. - * - * @param cls closure (points to the file information) - * @param offset offset to read from; it is possible - * that the caller might need to go backwards - * a bit at times - * @param max maximum number of bytes that should be - * copied to buf; readers are not allowed - * to provide less data unless there is an error; - * a value of "0" will be used at the end to allow - * the reader to clean up its internal state - * @param buf where the reader should write the data - * @param emsg location for the reader to store an error message - * @return number of bytes written, usually "max", 0 on error - */ -size_t -GNUNET_FS_data_reader_file_ (void *cls, uint64_t offset, size_t max, void *buf, - char **emsg); - - -/** - * Create the closure for the 'GNUNET_FS_data_reader_file_' callback. - * - * @param filename file to read - * @return closure to use - */ -void * -GNUNET_FS_make_file_reader_context_ (const char *filename); - - - -/** - * Function that provides data by copying from a buffer. - * - * @param cls closure (points to the buffer) - * @param offset offset to read from; it is possible - * that the caller might need to go backwards - * a bit at times - * @param max maximum number of bytes that should be - * copied to buf; readers are not allowed - * to provide less data unless there is an error; - * a value of "0" will be used at the end to allow - * the reader to clean up its internal state - * @param buf where the reader should write the data - * @param emsg location for the reader to store an error message - * @return number of bytes written, usually "max", 0 on error - */ -size_t -GNUNET_FS_data_reader_copy_ (void *cls, uint64_t offset, size_t max, void *buf, - char **emsg); - -/** - * Notification of FS that a search probe has made progress. - * This function is used INSTEAD of the client's event handler - * for downloads where the GNUNET_FS_DOWNLOAD_IS_PROBE flag is set. - * - * @param cls closure, always NULL (!), actual closure - * is in the client-context of the info struct - * @param info details about the event, specifying the event type - * and various bits about the event - * @return client-context (for the next progress call - * for this operation; should be set to NULL for - * SUSPEND and STOPPED events). The value returned - * will be passed to future callbacks in the respective - * field in the GNUNET_FS_ProgressInfo struct. - */ -void * -GNUNET_FS_search_probe_progress_ (void *cls, - const struct GNUNET_FS_ProgressInfo *info); - - -/** - * Main function that performs the upload. - * - * @param cls "struct GNUNET_FS_PublishContext" identifies the upload - * @param tc task context - */ -void -GNUNET_FS_publish_main_ (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc); - - -/** - * Function called once the hash of the file - * that is being unindexed has been computed. - * - * @param cls closure, unindex context - * @param file_id computed hash, NULL on error - */ -void -GNUNET_FS_unindex_process_hash_ (void *cls, const GNUNET_HashCode * file_id); - - -/** - * Fill in all of the generic fields for a publish event and call the - * callback. - * - * @param pi structure to fill in - * @param pc overall publishing context - * @param p file information for the file being published - * @param offset where in the file are we so far - * @return value returned from callback - */ -void * -GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_PublishContext *pc, - const struct GNUNET_FS_FileInformation *p, - uint64_t offset); - - -/** - * Fill in all of the generic fields for a download event and call the - * callback. - * - * @param pi structure to fill in - * @param dc overall download context - */ -void -GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_DownloadContext *dc); - - -/** - * Task that creates the initial (top-level) download - * request for the file. - * - * @param cls the 'struct GNUNET_FS_DownloadContext' - * @param tc scheduler context - */ -void -GNUNET_FS_download_start_task_ (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc); - - - -/** - * Fill in all of the generic fields for - * an unindex event and call the callback. - * - * @param pi structure to fill in - * @param uc overall unindex context - * @param offset where we are in the file (for progress) - */ -void -GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_UnindexContext *uc, - uint64_t offset); - -/** - * Fill in all of the generic fields for a search event and - * call the callback. - * - * @param pi structure to fill in - * @param sc overall search context - * @return value returned by the callback - */ -void * -GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_SearchContext *sc); - - -/** - * Connect to the datastore and remove the blocks. - * - * @param uc context for the unindex operation. - */ -void -GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc); - -/** - * Build the request and actually initiate the search using the - * GNUnet FS service. - * - * @param sc search context - * @return GNUNET_OK on success, GNUNET_SYSERR on error - */ -int -GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc); - -/** - * Start the downloading process (by entering the queue). - * - * @param dc our download context - */ -void -GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc); - - -/** - * Start download probes for the given search result. - * - * @param sr the search result - */ -void -GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr); - -/** - * Remove serialization/deserialization file from disk. - * - * @param h master context - * @param ext component of the path - * @param ent entity identifier - */ -void -GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, const char *ext, - const char *ent); - - -/** - * Remove serialization/deserialization directory from disk. - * - * @param h master context - * @param ext component of the path - * @param uni unique name of parent - */ -void -GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, const char *ext, - const char *uni); - - -/** - * Synchronize this file-information struct with its mirror - * on disk. Note that all internal FS-operations that change - * file information data should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param fi the struct to sync - */ -void -GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *f); - -/** - * Synchronize this publishing struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param pc the struct to sync - */ -void -GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc); - -/** - * Synchronize this unindex struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param uc the struct to sync - */ -void -GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc); - -/** - * Synchronize this search struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param sc the struct to sync - */ -void -GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc); - -/** - * Synchronize this search result with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param sr the struct to sync - */ -void -GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr); - -/** - * Synchronize this download struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param dc the struct to sync - */ -void -GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc); - -/** - * Create SUSPEND event for the given publish operation - * and then clean up our state (without stop signal). - * - * @param cls the 'struct GNUNET_FS_PublishContext' to signal for - */ -void -GNUNET_FS_publish_signal_suspend_ (void *cls); - -/** - * Create SUSPEND event for the given search operation - * and then clean up our state (without stop signal). - * - * @param cls the 'struct GNUNET_FS_SearchContext' to signal for - */ -void -GNUNET_FS_search_signal_suspend_ (void *cls); - -/** - * Create SUSPEND event for the given download operation - * and then clean up our state (without stop signal). - * - * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for - */ -void -GNUNET_FS_download_signal_suspend_ (void *cls); - -/** - * Create SUSPEND event for the given unindex operation - * and then clean up our state (without stop signal). - * - * @param cls the 'struct GNUNET_FS_UnindexContext' to signal for - */ -void -GNUNET_FS_unindex_signal_suspend_ (void *cls); - -/** - * Function signature of the functions that can be called - * to trigger suspend signals and clean-up for top-level - * activities. - * - * @param cls closure - */ -typedef void (*SuspendSignalFunction) (void *cls); - -/** - * We track all of the top-level activities of FS - * so that we can signal 'suspend' on shutdown. - */ -struct TopLevelActivity -{ - /** - * This is a doubly-linked list. - */ - struct TopLevelActivity *next; - - /** - * This is a doubly-linked list. - */ - struct TopLevelActivity *prev; - - /** - * Function to call for suspend-signalling and clean up. - */ - SuspendSignalFunction ssf; - - /** - * Closure for 'ssf' (some struct GNUNET_FS_XXXHandle*) - */ - void *ssf_cls; -}; - - -/** - * Create a top-level activity entry. - * - * @param h global fs handle - * @param ssf suspend signal function to use - * @param ssf_cls closure for ssf - * @return fresh top-level activity handle - */ -struct TopLevelActivity * -GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf, - void *ssf_cls); - - -/** - * Destroy a top-level activity entry. - * - * @param h global fs handle - * @param top top level activity entry - */ -void -GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top); - - - -/** - * Master context for most FS operations. - */ -struct GNUNET_FS_Handle -{ - /** - * Configuration to use. - */ - const struct GNUNET_CONFIGURATION_Handle *cfg; - - /** - * Name of our client. - */ - char *client_name; - - /** - * Function to call with updates on our progress. - */ - GNUNET_FS_ProgressCallback upcb; - - /** - * Closure for upcb. - */ - void *upcb_cls; - - /** - * Head of DLL of top-level activities. - */ - struct TopLevelActivity *top_head; - - /** - * Tail of DLL of top-level activities. - */ - struct TopLevelActivity *top_tail; - - /** - * Head of DLL of running jobs. - */ - struct GNUNET_FS_QueueEntry *running_head; - - /** - * Tail of DLL of running jobs. - */ - struct GNUNET_FS_QueueEntry *running_tail; - - /** - * Head of DLL of pending jobs. - */ - struct GNUNET_FS_QueueEntry *pending_head; - - /** - * Tail of DLL of pending jobs. - */ - struct GNUNET_FS_QueueEntry *pending_tail; - - /** - * Task that processes the jobs in the running and pending queues - * (and moves jobs around as needed). - */ - GNUNET_SCHEDULER_TaskIdentifier queue_job; - - /** - * Average time we take for a single request to be satisfied. - * FIXME: not yet calcualted properly... - */ - struct GNUNET_TIME_Relative avg_block_latency; - - /** - * How many actual downloads do we have running right now? - */ - unsigned int active_downloads; - - /** - * How many blocks do the active downloads have? - */ - unsigned int active_blocks; - - /** - * General flags. - */ - enum GNUNET_FS_Flags flags; - - /** - * Maximum number of parallel downloads. - */ - unsigned int max_parallel_downloads; - - /** - * Maximum number of parallel requests. - */ - unsigned int max_parallel_requests; - -}; - - -/** - * Handle for controlling a publication process. - */ -struct GNUNET_FS_PublishContext -{ - /** - * Handle to the global fs context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Our top-level activity entry (if we are top-level, otherwise NULL). - */ - struct TopLevelActivity *top; - - /** - * File-structure that is being shared. - */ - struct GNUNET_FS_FileInformation *fi; - - /** - * Namespace that we are publishing in, NULL if we have no namespace. - */ - struct GNUNET_FS_Namespace *namespace; - - /** - * ID of the content in the namespace, NULL if we have no namespace. - */ - char *nid; - - /** - * ID for future updates, NULL if we have no namespace or no updates. - */ - char *nuid; - - /** - * Filename used for serializing information about this operation - * (should be determined using 'mktemp'). - */ - char *serialization; - - /** - * Our own client handle for the FS service; only briefly used when - * we start to index a file, otherwise NULL. - */ - struct GNUNET_CLIENT_Connection *client; - - /** - * Current position in the file-tree for the upload. - */ - struct GNUNET_FS_FileInformation *fi_pos; - - /** - * Non-null if we are currently hashing a file. - */ - struct GNUNET_CRYPTO_FileHashContext *fhc; - - /** - * Connection to the datastore service. - */ - struct GNUNET_DATASTORE_Handle *dsh; - - /** - * Queue entry for reservation/unreservation. - */ - struct GNUNET_DATASTORE_QueueEntry *qre; - - /** - * ID of the task performing the upload. NO_TASK if the upload has - * completed. - */ - GNUNET_SCHEDULER_TaskIdentifier upload_task; - - /** - * Storage space to reserve for the operation. - */ - uint64_t reserve_space; - - /** - * Overall number of entries to reserve for the - * publish operation. - */ - uint32_t reserve_entries; - - /** - * Typically GNUNET_NO. Set to GNUNET_YES if "upload_task" is - * GNUNET_SCHEDULER_NO_TASK and we're waiting for a response from - * the datastore service (in which case this struct must not be - * freed until we have that response). If someone tries to stop the - * download for good during this period, "in_network_wait" is set to - * GNUNET_SYSERR which will cause the struct to be destroyed right - * after we have the reply (or timeout) from the datastore service. - */ - int in_network_wait; - - /** - * Options for publishing. - */ - enum GNUNET_FS_PublishOptions options; - - /** - * Space reservation ID with datastore service - * for this upload. - */ - int rid; - - /** - * Set to GNUNET_YES if all processing has completed. - */ - int all_done; -}; - - -/** - * Phases of unindex processing (state machine). - */ -enum UnindexState -{ - /** - * We're currently hashing the file. - */ - UNINDEX_STATE_HASHING = 0, - - /** - * We're telling the datastore to delete - * the respective entries. - */ - UNINDEX_STATE_DS_REMOVE = 1, - - /** - * We're notifying the FS service about - * the unindexing. - */ - UNINDEX_STATE_FS_NOTIFY = 2, - - /** - * We're done. - */ - UNINDEX_STATE_COMPLETE = 3, - - /** - * We've encountered a fatal error. - */ - UNINDEX_STATE_ERROR = 4 -}; - - -/** - * Handle for controlling an unindexing operation. - */ -struct GNUNET_FS_UnindexContext -{ - - /** - * Global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Our top-level activity entry. - */ - struct TopLevelActivity *top; - - /** - * Name of the file that we are unindexing. - */ - char *filename; - - /** - * Short name under which we are serializing the state of this operation. - */ - char *serialization; - - /** - * Connection to the FS service, only valid during the - * UNINDEX_STATE_FS_NOTIFY phase. - */ - struct GNUNET_CLIENT_Connection *client; - - /** - * Connection to the datastore service, only valid during the - * UNINDEX_STATE_DS_NOTIFY phase. - */ - struct GNUNET_DATASTORE_Handle *dsh; - - /** - * Pointer kept for the client. - */ - void *client_info; - - /** - * Merkle-ish tree encoder context. - */ - struct GNUNET_FS_TreeEncoder *tc; - - /** - * Handle used to read the file. - */ - struct GNUNET_DISK_FileHandle *fh; - - /** - * Error message, NULL on success. - */ - char *emsg; - - /** - * Context for hashing of the file. - */ - struct GNUNET_CRYPTO_FileHashContext *fhc; - - /** - * Overall size of the file. - */ - uint64_t file_size; - - /** - * When did we start? - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * Hash of the file's contents (once computed). - */ - GNUNET_HashCode file_id; - - /** - * Current operatinonal phase. - */ - enum UnindexState state; - -}; - - -/** - * Information we keep for each keyword in - * a keyword search. - */ -struct SearchRequestEntry -{ - /** - * Hash of the original keyword, also known as the - * key (for decrypting the KBlock). - */ - GNUNET_HashCode key; - - /** - * Hash of the public key, also known as the query. - */ - GNUNET_HashCode query; - - /** - * Map that contains a "struct GNUNET_FS_SearchResult" for each result that - * was found under this keyword. Note that the entries will point - * to the same locations as those in the master result map (in - * "struct GNUNET_FS_SearchContext"), so they should not be freed. - * The key for each entry is the XOR of the key and query in the CHK - * URI (as a unique identifier for the search result). - */ - struct GNUNET_CONTAINER_MultiHashMap *results; - - /** - * Is this keyword a mandatory keyword - * (started with '+')? - */ - int mandatory; - -}; - - -/** - * Handle for controlling a search. - */ -struct GNUNET_FS_SearchContext -{ - /** - * Handle to the global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Our top-level activity entry (if we are top-level, otherwise NULL). - */ - struct TopLevelActivity *top; - - /** - * List of keywords that we're looking for. - */ - struct GNUNET_FS_Uri *uri; - - /** - * For update-searches, link to the search result that triggered - * the update search; otherwise NULL. - */ - struct GNUNET_FS_SearchResult *psearch_result; - - /** - * Connection to the FS service. - */ - struct GNUNET_CLIENT_Connection *client; - - /** - * Pointer we keep for the client. - */ - void *client_info; - - /** - * Name of the file on disk we use for persistence. - */ - char *serialization; - - /** - * Error message (non-NULL if this operation failed). - */ - char *emsg; - - /** - * Map that contains a "struct GNUNET_FS_SearchResult" for each result that - * was found in the search. The key for each entry is the XOR of - * the key and query in the CHK URI (as a unique identifier for the - * search result). - */ - struct GNUNET_CONTAINER_MultiHashMap *master_result_map; - - /** - * Per-keyword information for a keyword search. This array will - * have exactly as many entries as there were keywords. - */ - struct SearchRequestEntry *requests; - - /** - * When did we start? - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * ID of a task that is using this struct and that must be cancelled - * when the search is being stopped (if not - * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some - * artificial delay when trying to reconnect to the FS service. - */ - GNUNET_SCHEDULER_TaskIdentifier task; - - /** - * How many of the entries in the search request - * map have been passed to the service so far? - */ - unsigned int search_request_map_offset; - - /** - * How many of the keywords in the KSK - * map have been passed to the service so far? - */ - unsigned int keyword_offset; - - /** - * Anonymity level for the search. - */ - uint32_t anonymity; - - /** - * Number of mandatory keywords in this query. - */ - uint32_t mandatory_count; - - /** - * Options for the search. - */ - enum GNUNET_FS_SearchOptions options; -}; - - -/** - * FSM for possible states a block can go through. The typical - * order of progression is linear through the states, alternatives - * are documented in the comments. - */ -enum BlockRequestState -{ - /** - * Initial state, block has only been allocated (since it is - * relevant to the overall download request). - */ - BRS_INIT = 0, - - /** - * We've checked the block on the path down the tree, and the - * content on disk did match the desired CHK, but not all - * the way down, so at the bottom some blocks will still - * need to be reconstructed). - */ - BRS_RECONSTRUCT_DOWN = 1, - - /** - * We've calculated the CHK bottom-up based on the meta data. - * This may work, but if it did we have to write the meta data to - * disk at the end (and we still need to check against the - * CHK set on top). - */ - BRS_RECONSTRUCT_META_UP = 2, - - /** - * We've calculated the CHK bottom-up based on what we have on - * disk, which may not be what the desired CHK is. If the - * reconstructed CHKs match whatever comes from above, we're - * done with the respective subtree. - */ - BRS_RECONSTRUCT_UP = 3, - - /** - * We've determined the real, desired CHK for this block - * (full tree reconstruction failed), request is now pending. - * If the CHK that bubbled up through reconstruction did match - * the top-level request, the state machine for the subtree - * would have moved to BRS_DOWNLOAD_UP. - */ - BRS_CHK_SET = 4, - - /** - * We've successfully downloaded this block, but the children - * still need to be either downloaded or verified (download - * request propagates down). If the download fails, the - * state machine for this block may move to - * BRS_DOWNLOAD_ERROR instead. - */ - BRS_DOWNLOAD_DOWN = 5, - - /** - * This block and all of its children have been downloaded - * successfully (full completion propagates up). - */ - BRS_DOWNLOAD_UP = 6, - - /** - * We got a block back that matched the query but did not hash to - * the key (malicious publisher or hash collision); this block - * can never be downloaded (error propagates up). - */ - BRS_ERROR = 7 -}; - - -/** - * Information about an active download request. - */ -struct DownloadRequest -{ - /** - * While pending, we keep all download requests in a doubly-linked list. - */ - struct DownloadRequest *next; - - /** - * While pending, we keep all download requests in a doubly-linked list. - */ - struct DownloadRequest *prev; - - /** - * Parent in the CHK-tree. - */ - struct DownloadRequest *parent; - - /** - * Array (!) of child-requests, or NULL for the bottom of the tree. - */ - struct DownloadRequest **children; - - /** - * CHK for the request for this block (set during reconstruction - * to what we have on disk, later to what we want to have). - */ - struct ContentHashKey chk; - - /** - * Offset of the corresponding block. Specifically, first (!) byte of - * the first DBLOCK in the subtree induced by block represented by - * this request. - */ - uint64_t offset; - - /** - * Number of entries in 'children' array. - */ - unsigned int num_children; - - /** - * Depth of the corresponding block in the tree. 0==DBLOCKs. - */ - unsigned int depth; - - /** - * State in the FSM. - */ - enum BlockRequestState state; - - /** - * GNUNET_YES if this entry is in the pending list. - */ - int is_pending; - -}; - - -/** - * (recursively) free download request structure - * - * @param dr request to free - */ -void -GNUNET_FS_free_download_request_ (struct DownloadRequest *dr); - - -/** - * Context for controlling a download. - */ -struct GNUNET_FS_DownloadContext -{ - - /** - * Global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Our top-level activity entry (if we are top-level, otherwise NULL). - */ - struct TopLevelActivity *top; - - /** - * Connection to the FS service. - */ - struct GNUNET_CLIENT_Connection *client; - - /** - * Parent download (used when downloading files - * in directories). - */ - struct GNUNET_FS_DownloadContext *parent; - - /** - * Associated search (used when downloading files - * based on search results), or NULL for none. - */ - struct GNUNET_FS_SearchResult *search; - - /** - * Head of list of child downloads. - */ - struct GNUNET_FS_DownloadContext *child_head; - - /** - * Tail of list of child downloads. - */ - struct GNUNET_FS_DownloadContext *child_tail; - - /** - * Previous download belonging to the same parent. - */ - struct GNUNET_FS_DownloadContext *prev; - - /** - * Next download belonging to the same parent. - */ - struct GNUNET_FS_DownloadContext *next; - - /** - * Context kept for the client. - */ - void *client_info; - - /** - * URI that identifies the file that we are downloading. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Known meta-data for the file (can be NULL). - */ - struct GNUNET_CONTAINER_MetaData *meta; - - /** - * Error message, NULL if we're doing OK. - */ - char *emsg; - - /** - * Random portion of filename we use for syncing state of this - * download. - */ - char *serialization; - - /** - * Where are we writing the data (name of the - * file, can be NULL!). - */ - char *filename; - - /** - * Where are we writing the data temporarily (name of the - * file, can be NULL!); used if we do not have a permanent - * name and we are a directory and we do a recursive download. - */ - char *temp_filename; - - /** - * Our entry in the job queue. - */ - struct GNUNET_FS_QueueEntry *job_queue; - - /** - * Non-NULL if we are currently having a request for - * transmission pending with the client handle. - */ - struct GNUNET_CLIENT_TransmitHandle *th; - - /** - * Tree encoder used for the reconstruction. - */ - struct GNUNET_FS_TreeEncoder *te; - - /** - * File handle for reading data from an existing file - * (to pass to tree encoder). - */ - struct GNUNET_DISK_FileHandle *rfh; - - /** - * Map of active requests (those waiting for a response). The key - * is the hash of the encryped block (aka query). - */ - struct GNUNET_CONTAINER_MultiHashMap *active; - - /** - * Head of linked list of pending requests. - */ - struct DownloadRequest *pending_head; - - /** - * Head of linked list of pending requests. - */ - struct DownloadRequest *pending_tail; - - /** - * Top-level download request. - */ - struct DownloadRequest *top_request; - - /** - * Identity of the peer having the content, or all-zeros - * if we don't know of such a peer. - */ - struct GNUNET_PeerIdentity target; - - /** - * ID of a task that is using this struct and that must be cancelled - * when the download is being stopped (if not - * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some - * artificial delay when trying to reconnect to the FS service or - * the task processing incrementally the data on disk, or the - * task requesting blocks, etc. - */ - GNUNET_SCHEDULER_TaskIdentifier task; - - /** - * What is the first offset that we're interested - * in? - */ - uint64_t offset; - - /** - * How many bytes starting from offset are desired? - * This is NOT the overall length of the file! - */ - uint64_t length; - - /** - * How many bytes have we already received within - * the specified range (DBlocks only). - */ - uint64_t completed; - - /** - * What was the size of the file on disk that we're downloading - * before we started? Used to detect if there is a point in - * checking an existing block on disk for matching the desired - * content. 0 if the file did not exist already. - */ - uint64_t old_file_size; - - /** - * Time download was started. - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * Desired level of anonymity. - */ - uint32_t anonymity; - - /** - * The depth of the file-tree. - */ - unsigned int treedepth; - - /** - * Options for the download. - */ - enum GNUNET_FS_DownloadOptions options; - - /** - * Flag set upon transitive completion (includes child downloads). - * This flag is only set to GNUNET_YES for directories where all - * child-downloads have also completed (and signalled completion). - */ - int has_finished; - - /** - * Have we started the receive continuation yet? - */ - int in_receive; - -}; - - -/** - * Information about an (updateable) node in the - * namespace. - */ -struct NamespaceUpdateNode -{ - /** - * Identifier for this node. - */ - char *id; - - /** - * Identifier of children of this node. - */ - char *update; - - /** - * Metadata for this entry. - */ - struct GNUNET_CONTAINER_MetaData *md; - - /** - * URI of this entry in the namespace. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Namespace update generation ID. Used to ensure - * freshness of the tree_id. - */ - unsigned int nug; - - /** - * TREE this entry belongs to (if nug is current). - */ - unsigned int tree_id; - -}; - - -struct GNUNET_FS_Namespace -{ - - /** - * Handle to the FS service context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Array with information about nodes in the namespace. - */ - struct NamespaceUpdateNode **update_nodes; - - /** - * Private key for the namespace. - */ - struct GNUNET_CRYPTO_RsaPrivateKey *key; - - /** - * Hash map mapping identifiers of update nodes - * to the update nodes (initialized on-demand). - */ - struct GNUNET_CONTAINER_MultiHashMap *update_map; - - /** - * Name of the file with the private key. - */ - char *filename; - - /** - * Name of the namespace. - */ - char *name; - - /** - * Size of the update nodes array. - */ - unsigned int update_node_count; - - /** - * Reference counter. - */ - unsigned int rc; - - /** - * Generator for unique nug numbers. - */ - unsigned int nug_gen; -}; - - /** * Message sent from a GNUnet (fs) publishing activity to the * gnunet-fs-service to initiate indexing of a file. The service is diff --git a/src/fs/fs_api.c b/src/fs/fs_api.c new file mode 100644 index 000000000..ce4b3b887 --- /dev/null +++ b/src/fs/fs_api.c @@ -0,0 +1,2784 @@ +/* + This file is part of GNUnet. + (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_api.c + * @brief main FS functions (master initialization, serialization, deserialization, shared code) + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_fs_service.h" +#include "fs_api.h" +#include "fs_tree.h" + + +/** + * Start the given job (send signal, remove from pending queue, update + * counters and state). + * + * @param qe job to start + */ +static void +start_job (struct GNUNET_FS_QueueEntry *qe) +{ + GNUNET_assert (NULL == qe->client); + qe->client = GNUNET_CLIENT_connect ("fs", qe->h->cfg); + if (qe->client == NULL) + { + GNUNET_break (0); + return; + } + qe->start (qe->cls, qe->client); + qe->start_times++; + qe->h->active_blocks += qe->blocks; + qe->start_time = GNUNET_TIME_absolute_get (); + GNUNET_CONTAINER_DLL_remove (qe->h->pending_head, qe->h->pending_tail, qe); + GNUNET_CONTAINER_DLL_insert_after (qe->h->running_head, qe->h->running_tail, + qe->h->running_tail, qe); +} + + +/** + * Stop the given job (send signal, remove from active queue, update + * counters and state). + * + * @param qe job to stop + */ +static void +stop_job (struct GNUNET_FS_QueueEntry *qe) +{ + qe->client = NULL; + qe->stop (qe->cls); + qe->h->active_downloads--; + qe->h->active_blocks -= qe->blocks; + qe->run_time = + GNUNET_TIME_relative_add (qe->run_time, + GNUNET_TIME_absolute_get_duration + (qe->start_time)); + GNUNET_CONTAINER_DLL_remove (qe->h->running_head, qe->h->running_tail, qe); + GNUNET_CONTAINER_DLL_insert_after (qe->h->pending_head, qe->h->pending_tail, + qe->h->pending_tail, qe); +} + + +/** + * Process the jobs in the job queue, possibly starting some + * and stopping others. + * + * @param cls the 'struct GNUNET_FS_Handle' + * @param tc scheduler context + */ +static void +process_job_queue (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_Handle *h = cls; + struct GNUNET_FS_QueueEntry *qe; + struct GNUNET_FS_QueueEntry *next; + struct GNUNET_TIME_Relative run_time; + struct GNUNET_TIME_Relative restart_at; + struct GNUNET_TIME_Relative rst; + struct GNUNET_TIME_Absolute end_time; + + h->queue_job = GNUNET_SCHEDULER_NO_TASK; + next = h->pending_head; + while (NULL != (qe = next)) + { + next = qe->next; + if (h->running_head == NULL) + { + start_job (qe); + continue; + } + if ((qe->blocks + h->active_blocks <= h->max_parallel_requests) && + (h->active_downloads + 1 <= h->max_parallel_downloads)) + { + start_job (qe); + continue; + } + } + if (h->pending_head == NULL) + return; /* no need to stop anything */ + restart_at = GNUNET_TIME_UNIT_FOREVER_REL; + next = h->running_head; + while (NULL != (qe = next)) + { + next = qe->next; + run_time = + GNUNET_TIME_relative_multiply (h->avg_block_latency, + qe->blocks * qe->start_times); + end_time = GNUNET_TIME_absolute_add (qe->start_time, run_time); + rst = GNUNET_TIME_absolute_get_remaining (end_time); + restart_at = GNUNET_TIME_relative_min (rst, restart_at); + if (rst.rel_value > 0) + continue; + stop_job (qe); + } + h->queue_job = + GNUNET_SCHEDULER_add_delayed (restart_at, &process_job_queue, h); +} + + +/** + * Add a job to the queue. + * + * @param h handle to the overall FS state + * @param start function to call to begin the job + * @param stop function to call to pause the job, or on dequeue (if the job was running) + * @param cls closure for start and stop + * @param blocks number of blocks this jobs uses + * @return queue handle + */ +struct GNUNET_FS_QueueEntry * +GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, GNUNET_FS_QueueStart start, + GNUNET_FS_QueueStop stop, void *cls, unsigned int blocks) +{ + struct GNUNET_FS_QueueEntry *qe; + + qe = GNUNET_malloc (sizeof (struct GNUNET_FS_QueueEntry)); + qe->h = h; + qe->start = start; + qe->stop = stop; + qe->cls = cls; + qe->queue_time = GNUNET_TIME_absolute_get (); + qe->blocks = blocks; + GNUNET_CONTAINER_DLL_insert_after (h->pending_head, h->pending_tail, + h->pending_tail, qe); + if (h->queue_job != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel (h->queue_job); + h->queue_job = GNUNET_SCHEDULER_add_now (&process_job_queue, h); + return qe; +} + + +/** + * Dequeue a job from the queue. + * @param qh handle for the job + */ +void +GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh) +{ + struct GNUNET_FS_Handle *h; + + h = qh->h; + if (qh->client != NULL) + stop_job (qh); + GNUNET_CONTAINER_DLL_remove (h->pending_head, h->pending_tail, qh); + GNUNET_free (qh); + if (h->queue_job != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel (h->queue_job); + h->queue_job = GNUNET_SCHEDULER_add_now (&process_job_queue, h); +} + + +/** + * Create a top-level activity entry. + * + * @param h global fs handle + * @param ssf suspend signal function to use + * @param ssf_cls closure for ssf + * @return fresh top-level activity handle + */ +struct TopLevelActivity * +GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf, + void *ssf_cls) +{ + struct TopLevelActivity *ret; + + ret = GNUNET_malloc (sizeof (struct TopLevelActivity)); + ret->ssf = ssf; + ret->ssf_cls = ssf_cls; + GNUNET_CONTAINER_DLL_insert (h->top_head, h->top_tail, ret); + return ret; +} + + +/** + * Destroy a top-level activity entry. + * + * @param h global fs handle + * @param top top level activity entry + */ +void +GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top) +{ + GNUNET_CONTAINER_DLL_remove (h->top_head, h->top_tail, top); + GNUNET_free (top); +} + + + +/** + * Closure for "data_reader_file". + */ +struct FileInfo +{ + /** + * Name of the file to read. + */ + char *filename; + + /** + * File descriptor, NULL if it has not yet been opened. + */ + struct GNUNET_DISK_FileHandle *fd; +}; + + +/** + * Function that provides data by reading from a file. + * + * @param cls closure (points to the file information) + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually "max", 0 on error + */ +size_t +GNUNET_FS_data_reader_file_ (void *cls, uint64_t offset, size_t max, void *buf, + char **emsg) +{ + struct FileInfo *fi = cls; + ssize_t ret; + + if (max == 0) + { + if (fi->fd != NULL) + GNUNET_DISK_file_close (fi->fd); + GNUNET_free (fi->filename); + GNUNET_free (fi); + return 0; + } + if (fi->fd == NULL) + { + fi->fd = + GNUNET_DISK_file_open (fi->filename, GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE); + if (fi->fd == NULL) + { + GNUNET_asprintf (emsg, _("Could not open file `%s': %s"), fi->filename, + STRERROR (errno)); + return 0; + } + } + GNUNET_DISK_file_seek (fi->fd, offset, GNUNET_DISK_SEEK_SET); + ret = GNUNET_DISK_file_read (fi->fd, buf, max); + if (ret == -1) + { + GNUNET_asprintf (emsg, _("Could not read file `%s': %s"), fi->filename, + STRERROR (errno)); + return 0; + } + if (ret != max) + { + GNUNET_asprintf (emsg, _("Short read reading from file `%s'!"), + fi->filename); + return 0; + } + return max; +} + + +/** + * Create the closure for the 'GNUNET_FS_data_reader_file_' callback. + * + * @param filename file to read + * @return closure to use, NULL on error + */ +void * +GNUNET_FS_make_file_reader_context_ (const char *filename) +{ + struct FileInfo *fi; + + fi = GNUNET_malloc (sizeof (struct FileInfo)); + fi->filename = GNUNET_STRINGS_filename_expand (filename); + if (fi->filename == NULL) + { + GNUNET_free (fi); + return NULL; + } + return fi; +} + + +/** + * Function that provides data by copying from a buffer. + * + * @param cls closure (points to the buffer) + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually "max", 0 on error + */ +size_t +GNUNET_FS_data_reader_copy_ (void *cls, uint64_t offset, size_t max, void *buf, + char **emsg) +{ + char *data = cls; + + if (max == 0) + { + GNUNET_free_non_null (data); + return 0; + } + memcpy (buf, &data[offset], max); + return max; +} + + +/** + * Return the full filename where we would store state information + * (for serialization/deserialization). + * + * @param h master context + * @param ext component of the path + * @param ent entity identifier (or emtpy string for the directory) + * @return NULL on error + */ +static char * +get_serialization_file_name (struct GNUNET_FS_Handle *h, const char *ext, + const char *ent) +{ + char *basename; + char *ret; + + if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) + return NULL; /* persistence not requested */ + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (h->cfg, "fs", "STATE_DIR", + &basename)) + return NULL; + GNUNET_asprintf (&ret, "%s%s%s%s%s%s%s", basename, DIR_SEPARATOR_STR, + h->client_name, DIR_SEPARATOR_STR, ext, DIR_SEPARATOR_STR, + ent); + GNUNET_free (basename); + return ret; +} + + +/** + * Return the full filename where we would store state information + * (for serialization/deserialization) that is associated with a + * parent operation. + * + * @param h master context + * @param ext component of the path + * @param uni name of the parent operation + * @param ent entity identifier (or emtpy string for the directory) + * @return NULL on error + */ +static char * +get_serialization_file_name_in_dir (struct GNUNET_FS_Handle *h, const char *ext, + const char *uni, const char *ent) +{ + char *basename; + char *ret; + + if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) + return NULL; /* persistence not requested */ + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (h->cfg, "fs", "STATE_DIR", + &basename)) + return NULL; + GNUNET_asprintf (&ret, "%s%s%s%s%s%s%s.dir%s%s", basename, DIR_SEPARATOR_STR, + h->client_name, DIR_SEPARATOR_STR, ext, DIR_SEPARATOR_STR, + uni, DIR_SEPARATOR_STR, ent); + GNUNET_free (basename); + return ret; +} + + +/** + * Return a read handle for deserialization. + * + * @param h master context + * @param ext component of the path + * @param ent entity identifier (or emtpy string for the directory) + * @return NULL on error + */ +static struct GNUNET_BIO_ReadHandle * +get_read_handle (struct GNUNET_FS_Handle *h, const char *ext, const char *ent) +{ + char *fn; + struct GNUNET_BIO_ReadHandle *ret; + + fn = get_serialization_file_name (h, ext, ent); + if (fn == NULL) + return NULL; + ret = GNUNET_BIO_read_open (fn); + GNUNET_free (fn); + return ret; +} + + +/** + * Return a write handle for serialization. + * + * @param h master context + * @param ext component of the path + * @param ent entity identifier (or emtpy string for the directory) + * @return NULL on error + */ +static struct GNUNET_BIO_WriteHandle * +get_write_handle (struct GNUNET_FS_Handle *h, const char *ext, const char *ent) +{ + char *fn; + struct GNUNET_BIO_WriteHandle *ret; + + fn = get_serialization_file_name (h, ext, ent); + if (fn == NULL) + { + return NULL; + } + ret = GNUNET_BIO_write_open (fn); + if (ret == NULL) + GNUNET_break (0); + GNUNET_free (fn); + return ret; +} + + +/** + * Return a write handle for serialization. + * + * @param h master context + * @param ext component of the path + * @param uni name of parent + * @param ent entity identifier (or emtpy string for the directory) + * @return NULL on error + */ +static struct GNUNET_BIO_WriteHandle * +get_write_handle_in_dir (struct GNUNET_FS_Handle *h, const char *ext, + const char *uni, const char *ent) +{ + char *fn; + struct GNUNET_BIO_WriteHandle *ret; + + fn = get_serialization_file_name_in_dir (h, ext, uni, ent); + if (fn == NULL) + return NULL; + ret = GNUNET_BIO_write_open (fn); + GNUNET_free (fn); + return ret; +} + + +/** + * Remove serialization/deserialization file from disk. + * + * @param h master context + * @param ext component of the path + * @param ent entity identifier + */ +void +GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, const char *ext, + const char *ent) +{ + char *filename; + + if ((NULL == ent) || (0 == strlen (ent))) + { + GNUNET_break (0); + return; + } + filename = get_serialization_file_name (h, ext, ent); + if (filename != NULL) + { + if (0 != UNLINK (filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); + GNUNET_free (filename); + } +} + + +/** + * Remove serialization/deserialization file from disk. + * + * @param h master context + * @param ext component of the path + * @param uni parent name + * @param ent entity identifier + */ +static void +remove_sync_file_in_dir (struct GNUNET_FS_Handle *h, const char *ext, + const char *uni, const char *ent) +{ + char *filename; + + if ((NULL == ent) || (0 == strlen (ent))) + { + GNUNET_break (0); + return; + } + filename = get_serialization_file_name_in_dir (h, ext, uni, ent); + if (filename != NULL) + { + if (0 != UNLINK (filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); + GNUNET_free (filename); + } +} + + +/** + * Remove serialization/deserialization directory from disk. + * + * @param h master context + * @param ext component of the path + * @param uni unique name of parent + */ +void +GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, const char *ext, + const char *uni) +{ + char *dn; + + if (uni == NULL) + return; + dn = get_serialization_file_name_in_dir (h, ext, uni, ""); + if (dn == NULL) + return; + if ((GNUNET_OK == GNUNET_DISK_directory_test (dn)) && + (GNUNET_OK != GNUNET_DISK_directory_remove (dn))) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "rmdir", dn); + GNUNET_free (dn); +} + + +/** + * Serialize a 'start_time'. Since we use start-times to + * calculate the duration of some operation, we actually + * do not serialize the absolute time but the (relative) + * duration since the start time. When we then + * deserialize the start time, we take the current time and + * subtract that duration so that we get again an absolute + * time stamp that will result in correct performance + * calculations. + * + * @param wh handle for writing + * @param timestamp time to serialize + * @return GNUNET_OK on success + */ +static int +write_start_time (struct GNUNET_BIO_WriteHandle *wh, + struct GNUNET_TIME_Absolute timestamp) +{ + struct GNUNET_TIME_Relative dur; + + dur = GNUNET_TIME_absolute_get_duration (timestamp); + return GNUNET_BIO_write_int64 (wh, dur.rel_value); +} + + +/** + * Serialize a 'start_time'. Since we use start-times to + * calculate the duration of some operation, we actually + * do not serialize the absolute time but the (relative) + * duration since the start time. When we then + * deserialize the start time, we take the current time and + * subtract that duration so that we get again an absolute + * time stamp that will result in correct performance + * calculations. + * + * @param rh handle for reading + * @param timestamp where to write the deserialized timestamp + * @return GNUNET_OK on success + */ +static int +read_start_time (struct GNUNET_BIO_ReadHandle *rh, + struct GNUNET_TIME_Absolute *timestamp) +{ + struct GNUNET_TIME_Relative dur; + + if (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dur.rel_value)) + return GNUNET_SYSERR; + *timestamp = GNUNET_TIME_absolute_subtract (GNUNET_TIME_absolute_get (), dur); + return GNUNET_OK; +} + + +/** + * Using the given serialization filename, try to deserialize + * the file-information tree associated with it. + * + * @param h master context + * @param filename name of the file (without directory) with + * the infromation + * @return NULL on error + */ +static struct GNUNET_FS_FileInformation * +deserialize_file_information (struct GNUNET_FS_Handle *h, const char *filename); + + +/** + * Using the given serialization filename, try to deserialize + * the file-information tree associated with it. + * + * @param h master context + * @param fn name of the file (without directory) with + * the infromation + * @param rh handle for reading + * @return NULL on error + */ +static struct GNUNET_FS_FileInformation * +deserialize_fi_node (struct GNUNET_FS_Handle *h, const char *fn, + struct GNUNET_BIO_ReadHandle *rh) +{ + struct GNUNET_FS_FileInformation *ret; + struct GNUNET_FS_FileInformation *nxt; + char b; + char *ksks; + char *chks; + char *filename; + uint32_t dsize; + + if (GNUNET_OK != GNUNET_BIO_read (rh, "status flag", &b, sizeof (b))) + { + GNUNET_break (0); + return NULL; + } + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_FileInformation)); + ret->h = h; + ksks = NULL; + chks = NULL; + filename = NULL; + if ((GNUNET_OK != GNUNET_BIO_read_meta_data (rh, "metadata", &ret->meta)) || + (GNUNET_OK != GNUNET_BIO_read_string (rh, "ksk-uri", &ksks, 32 * 1024)) || + ((ksks != NULL) && + (NULL == (ret->keywords = GNUNET_FS_uri_parse (ksks, NULL)))) || + (GNUNET_YES != GNUNET_FS_uri_test_ksk (ret->keywords)) || + (GNUNET_OK != GNUNET_BIO_read_string (rh, "chk-uri", &chks, 1024)) || + ((chks != NULL) && + ((NULL == (ret->chk_uri = GNUNET_FS_uri_parse (chks, NULL))) || + (GNUNET_YES != GNUNET_FS_uri_test_chk (ret->chk_uri)))) || + (GNUNET_OK != read_start_time (rh, &ret->start_time)) || + (GNUNET_OK != GNUNET_BIO_read_string (rh, "emsg", &ret->emsg, 16 * 1024)) + || (GNUNET_OK != + GNUNET_BIO_read_string (rh, "fn", &ret->filename, 16 * 1024)) || + (GNUNET_OK != + GNUNET_BIO_read_int64 (rh, &ret->bo.expiration_time.abs_value)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &ret->bo.anonymity_level)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &ret->bo.content_priority)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &ret->bo.replication_level))) + { + GNUNET_break (0); + goto cleanup; + } + switch (b) + { + case 0: /* file-insert */ + if (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) + { + GNUNET_break (0); + goto cleanup; + } + ret->is_directory = GNUNET_NO; + ret->data.file.do_index = GNUNET_NO; + ret->data.file.have_hash = GNUNET_NO; + ret->data.file.index_start_confirmed = GNUNET_NO; + if (GNUNET_NO == ret->is_published) + { + if (NULL == ret->filename) + { + ret->data.file.reader = &GNUNET_FS_data_reader_copy_; + ret->data.file.reader_cls = + GNUNET_malloc_large (ret->data.file.file_size); + if (ret->data.file.reader_cls == NULL) + goto cleanup; + if (GNUNET_OK != + GNUNET_BIO_read (rh, "file-data", ret->data.file.reader_cls, + ret->data.file.file_size)) + { + GNUNET_break (0); + goto cleanup; + } + } + else + { + ret->data.file.reader = &GNUNET_FS_data_reader_file_; + ret->data.file.reader_cls = + GNUNET_FS_make_file_reader_context_ (ret->filename); + } + } + break; + case 1: /* file-index, no hash */ + if (NULL == ret->filename) + { + GNUNET_break (0); + goto cleanup; + } + if (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) + { + GNUNET_break (0); + goto cleanup; + } + ret->is_directory = GNUNET_NO; + ret->data.file.do_index = GNUNET_YES; + ret->data.file.have_hash = GNUNET_NO; + ret->data.file.index_start_confirmed = GNUNET_NO; + ret->data.file.reader = &GNUNET_FS_data_reader_file_; + ret->data.file.reader_cls = + GNUNET_FS_make_file_reader_context_ (ret->filename); + break; + case 2: /* file-index-with-hash */ + if (NULL == ret->filename) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) || + (GNUNET_OK != + GNUNET_BIO_read (rh, "fileid", &ret->data.file.file_id, + sizeof (GNUNET_HashCode)))) + { + GNUNET_break (0); + goto cleanup; + } + ret->is_directory = GNUNET_NO; + ret->data.file.do_index = GNUNET_YES; + ret->data.file.have_hash = GNUNET_YES; + ret->data.file.index_start_confirmed = GNUNET_NO; + ret->data.file.reader = &GNUNET_FS_data_reader_file_; + ret->data.file.reader_cls = + GNUNET_FS_make_file_reader_context_ (ret->filename); + break; + case 3: /* file-index-with-hash-confirmed */ + if (NULL == ret->filename) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) || + (GNUNET_OK != + GNUNET_BIO_read (rh, "fileid", &ret->data.file.file_id, + sizeof (GNUNET_HashCode)))) + { + GNUNET_break (0); + goto cleanup; + } + ret->is_directory = GNUNET_NO; + ret->data.file.do_index = GNUNET_YES; + ret->data.file.have_hash = GNUNET_YES; + ret->data.file.index_start_confirmed = GNUNET_YES; + ret->data.file.reader = &GNUNET_FS_data_reader_file_; + ret->data.file.reader_cls = + GNUNET_FS_make_file_reader_context_ (ret->filename); + break; + case 4: /* directory */ + ret->is_directory = GNUNET_YES; + if ((GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dsize)) || + (NULL == (ret->data.dir.dir_data = GNUNET_malloc_large (dsize))) || + (GNUNET_OK != + GNUNET_BIO_read (rh, "dir-data", ret->data.dir.dir_data, dsize)) || + (GNUNET_OK != + GNUNET_BIO_read_string (rh, "ent-filename", &filename, 16 * 1024))) + { + GNUNET_break (0); + goto cleanup; + } + ret->data.dir.dir_size = (uint32_t) dsize; + if (filename != NULL) + { + ret->data.dir.entries = deserialize_file_information (h, filename); + GNUNET_free (filename); + filename = NULL; + nxt = ret->data.dir.entries; + while (nxt != NULL) + { + nxt->dir = ret; + nxt = nxt->next; + } + } + break; + default: + GNUNET_break (0); + goto cleanup; + } + ret->serialization = GNUNET_strdup (fn); + if (GNUNET_OK != + GNUNET_BIO_read_string (rh, "nxt-filename", &filename, 16 * 1024)) + { + GNUNET_break (0); + goto cleanup; + } + if (filename != NULL) + { + ret->next = deserialize_file_information (h, filename); + GNUNET_free (filename); + filename = NULL; + } + GNUNET_free_non_null (ksks); + GNUNET_free_non_null (chks); + return ret; +cleanup: + GNUNET_free_non_null (ksks); + GNUNET_free_non_null (chks); + GNUNET_free_non_null (filename); + GNUNET_FS_file_information_destroy (ret, NULL, NULL); + return NULL; +} + + +/** + * Using the given serialization filename, try to deserialize + * the file-information tree associated with it. + * + * @param h master context + * @param filename name of the file (without directory) with + * the infromation + * @return NULL on error + */ +static struct GNUNET_FS_FileInformation * +deserialize_file_information (struct GNUNET_FS_Handle *h, const char *filename) +{ + struct GNUNET_FS_FileInformation *ret; + struct GNUNET_BIO_ReadHandle *rh; + char *emsg; + + rh = get_read_handle (h, GNUNET_FS_SYNC_PATH_FILE_INFO, filename); + if (rh == NULL) + return NULL; + ret = deserialize_fi_node (h, filename, rh); + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume publishing information `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + if (ret == NULL) + { + if (0 != UNLINK (filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); + } + return ret; +} + + +/** + * Given a serialization name (full absolute path), return the + * basename of the file (without the path), which must only + * consist of the 6 random characters. + * + * @param fullname name to extract the basename from + * @return copy of the basename, NULL on error + */ +static char * +get_serialization_short_name (const char *fullname) +{ + const char *end; + const char *nxt; + + end = NULL; + nxt = fullname; + /* FIXME: we could do this faster since we know + * the length of 'end'... */ + while ('\0' != *nxt) + { + if (DIR_SEPARATOR == *nxt) + end = nxt + 1; + nxt++; + } + if ((end == NULL) || (strlen (end) == 0)) + { + GNUNET_break (0); + return NULL; + } + GNUNET_break (6 == strlen (end)); + return GNUNET_strdup (end); +} + + +/** + * Create a new random name for serialization. Also checks if persistence + * is enabled and returns NULL if not. + * + * @param h master context + * @param ext component of the path + * @return NULL on errror + */ +static char * +make_serialization_file_name (struct GNUNET_FS_Handle *h, const char *ext) +{ + char *fn; + char *dn; + char *ret; + + if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) + return NULL; /* persistence not requested */ + dn = get_serialization_file_name (h, ext, ""); + if (dn == NULL) + return NULL; + if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dn)) + { + GNUNET_free (dn); + return NULL; + } + fn = GNUNET_DISK_mktemp (dn); + GNUNET_free (dn); + if (fn == NULL) + return NULL; /* epic fail */ + ret = get_serialization_short_name (fn); + GNUNET_free (fn); + return ret; +} + + +/** + * Create a new random name for serialization. Also checks if persistence + * is enabled and returns NULL if not. + * + * @param h master context + * @param ext component of the path + * @param uni name of parent + * @return NULL on errror + */ +static char * +make_serialization_file_name_in_dir (struct GNUNET_FS_Handle *h, + const char *ext, const char *uni) +{ + char *fn; + char *dn; + char *ret; + + if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) + return NULL; /* persistence not requested */ + dn = get_serialization_file_name_in_dir (h, ext, uni, ""); + if (dn == NULL) + return NULL; + if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dn)) + { + GNUNET_free (dn); + return NULL; + } + fn = GNUNET_DISK_mktemp (dn); + GNUNET_free (dn); + if (fn == NULL) + return NULL; /* epic fail */ + ret = get_serialization_short_name (fn); + GNUNET_free (fn); + return ret; +} + + +/** + * Copy all of the data from the reader to the write handle. + * + * @param wh write handle + * @param fi file with reader + * @return GNUNET_OK on success + */ +static int +copy_from_reader (struct GNUNET_BIO_WriteHandle *wh, + struct GNUNET_FS_FileInformation *fi) +{ + char buf[32 * 1024]; + uint64_t off; + size_t ret; + size_t left; + char *emsg; + + emsg = NULL; + off = 0; + while (off < fi->data.file.file_size) + { + left = GNUNET_MIN (sizeof (buf), fi->data.file.file_size - off); + ret = + fi->data.file.reader (fi->data.file.reader_cls, off, left, buf, &emsg); + if (ret == 0) + { + GNUNET_free (emsg); + return GNUNET_SYSERR; + } + if (GNUNET_OK != GNUNET_BIO_write (wh, buf, ret)) + return GNUNET_SYSERR; + off += ret; + } + return GNUNET_OK; +} + + +/** + * Create a temporary file on disk to store the current + * state of "fi" in. + * + * @param fi file information to sync with disk + */ +void +GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *fi) +{ + char *fn; + struct GNUNET_BIO_WriteHandle *wh; + char b; + char *ksks; + char *chks; + + if (NULL == fi->serialization) + fi->serialization = + make_serialization_file_name (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO); + if (NULL == fi->serialization) + return; + wh = get_write_handle (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO, + fi->serialization); + if (wh == NULL) + { + GNUNET_free (fi->serialization); + fi->serialization = NULL; + return; + } + if (GNUNET_YES == fi->is_directory) + b = 4; + else if (GNUNET_YES == fi->data.file.index_start_confirmed) + b = 3; + else if (GNUNET_YES == fi->data.file.have_hash) + b = 2; + else if (GNUNET_YES == fi->data.file.do_index) + b = 1; + else + b = 0; + if (fi->keywords != NULL) + ksks = GNUNET_FS_uri_to_string (fi->keywords); + else + ksks = NULL; + if (fi->chk_uri != NULL) + chks = GNUNET_FS_uri_to_string (fi->chk_uri); + else + chks = NULL; + if ((GNUNET_OK != GNUNET_BIO_write (wh, &b, sizeof (b))) || + (GNUNET_OK != GNUNET_BIO_write_meta_data (wh, fi->meta)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, ksks)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, chks)) || + (GNUNET_OK != write_start_time (wh, fi->start_time)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, fi->emsg)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, fi->filename)) || + (GNUNET_OK != + GNUNET_BIO_write_int64 (wh, fi->bo.expiration_time.abs_value)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, fi->bo.anonymity_level)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, fi->bo.content_priority)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, fi->bo.replication_level))) + { + GNUNET_break (0); + goto cleanup; + } + GNUNET_free_non_null (chks); + chks = NULL; + GNUNET_free_non_null (ksks); + ksks = NULL; + + switch (b) + { + case 0: /* file-insert */ + if (GNUNET_OK != GNUNET_BIO_write_int64 (wh, fi->data.file.file_size)) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_NO == fi->is_published) && (NULL == fi->filename)) + if (GNUNET_OK != copy_from_reader (wh, fi)) + { + GNUNET_break (0); + goto cleanup; + } + break; + case 1: /* file-index, no hash */ + if (NULL == fi->filename) + { + GNUNET_break (0); + goto cleanup; + } + if (GNUNET_OK != GNUNET_BIO_write_int64 (wh, fi->data.file.file_size)) + { + GNUNET_break (0); + goto cleanup; + } + break; + case 2: /* file-index-with-hash */ + case 3: /* file-index-with-hash-confirmed */ + if (NULL == fi->filename) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != GNUNET_BIO_write_int64 (wh, fi->data.file.file_size)) || + (GNUNET_OK != + GNUNET_BIO_write (wh, &fi->data.file.file_id, + sizeof (GNUNET_HashCode)))) + { + GNUNET_break (0); + goto cleanup; + } + break; + case 4: /* directory */ + if ((GNUNET_OK != GNUNET_BIO_write_int32 (wh, fi->data.dir.dir_size)) || + (GNUNET_OK != + GNUNET_BIO_write (wh, fi->data.dir.dir_data, + (uint32_t) fi->data.dir.dir_size)) || + (GNUNET_OK != + GNUNET_BIO_write_string (wh, + (fi->data.dir.entries == + NULL) ? NULL : fi->data.dir. + entries->serialization))) + { + GNUNET_break (0); + goto cleanup; + } + break; + default: + GNUNET_assert (0); + goto cleanup; + } + if (GNUNET_OK != + GNUNET_BIO_write_string (wh, + (fi->next != + NULL) ? fi->next->serialization : NULL)) + { + GNUNET_break (0); + goto cleanup; + } + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + wh = NULL; + GNUNET_break (0); + goto cleanup; + } + return; /* done! */ +cleanup: + if (wh != NULL) + (void) GNUNET_BIO_write_close (wh); + GNUNET_free_non_null (chks); + GNUNET_free_non_null (ksks); + fn = get_serialization_file_name (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO, + fi->serialization); + if (NULL != fn) + { + if (0 != UNLINK (fn)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn); + GNUNET_free (fn); + } + GNUNET_free (fi->serialization); + fi->serialization = NULL; +} + + + +/** + * Find the entry in the file information struct where the + * serialization filename matches the given name. + * + * @param pos file information to search + * @param srch filename to search for + * @return NULL if srch was not found in this subtree + */ +static struct GNUNET_FS_FileInformation * +find_file_position (struct GNUNET_FS_FileInformation *pos, const char *srch) +{ + struct GNUNET_FS_FileInformation *r; + + while (pos != NULL) + { + if (0 == strcmp (srch, pos->serialization)) + return pos; + if (pos->is_directory) + { + r = find_file_position (pos->data.dir.entries, srch); + if (r != NULL) + return r; + } + pos = pos->next; + } + return NULL; +} + + +/** + * Signal the FS's progress function that we are resuming + * an upload. + * + * @param cls closure (of type "struct GNUNET_FS_PublishContext*") + * @param fi the entry in the publish-structure + * @param length length of the file or directory + * @param meta metadata for the file or directory (can be modified) + * @param uri pointer to the keywords that will be used for this entry (can be modified) + * @param bo block options (can be modified) + * @param do_index should we index? + * @param client_info pointer to client context set upon creation (can be modified) + * @return GNUNET_OK to continue (always) + */ +static int +fip_signal_resume (void *cls, struct GNUNET_FS_FileInformation *fi, + uint64_t length, struct GNUNET_CONTAINER_MetaData *meta, + struct GNUNET_FS_Uri **uri, + struct GNUNET_FS_BlockOptions *bo, int *do_index, + void **client_info) +{ + struct GNUNET_FS_PublishContext *sc = cls; + struct GNUNET_FS_ProgressInfo pi; + + pi.status = GNUNET_FS_STATUS_PUBLISH_RESUME; + pi.value.publish.specifics.resume.message = sc->fi->emsg; + pi.value.publish.specifics.resume.chk_uri = sc->fi->chk_uri; + *client_info = GNUNET_FS_publish_make_status_ (&pi, sc, fi, 0); + return GNUNET_OK; +} + + +/** + * Function called with a filename of serialized publishing operation + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_Handle*' + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_publish_file (void *cls, const char *filename) +{ + struct GNUNET_FS_Handle *h = cls; + struct GNUNET_BIO_ReadHandle *rh; + struct GNUNET_FS_PublishContext *pc; + int32_t options; + int32_t all_done; + char *fi_root; + char *ns; + char *fi_pos; + char *emsg; + + pc = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext)); + pc->h = h; + pc->serialization = get_serialization_short_name (filename); + fi_root = NULL; + fi_pos = NULL; + ns = NULL; + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != GNUNET_BIO_read_string (rh, "publish-nid", &pc->nid, 1024)) + || (GNUNET_OK != + GNUNET_BIO_read_string (rh, "publish-nuid", &pc->nuid, 1024)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &options)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &all_done)) || + (GNUNET_OK != + GNUNET_BIO_read_string (rh, "publish-firoot", &fi_root, 128)) || + (GNUNET_OK != GNUNET_BIO_read_string (rh, "publish-fipos", &fi_pos, 128)) + || (GNUNET_OK != GNUNET_BIO_read_string (rh, "publish-ns", &ns, 1024))) + { + GNUNET_break (0); + goto cleanup; + } + pc->options = options; + pc->all_done = all_done; + if (NULL == fi_root) + { + GNUNET_break (0); + goto cleanup; + } + pc->fi = deserialize_file_information (h, fi_root); + if (pc->fi == NULL) + { + GNUNET_break (0); + goto cleanup; + } + if (ns != NULL) + { + pc->namespace = GNUNET_FS_namespace_create (h, ns); + if (pc->namespace == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Failed to recover namespace `%s', cannot resume publishing operation.\n"), + ns); + goto cleanup; + } + } + if ((0 == (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) && + (GNUNET_YES != pc->all_done)) + { + pc->dsh = GNUNET_DATASTORE_connect (h->cfg); + if (NULL == pc->dsh) + goto cleanup; + } + if (fi_pos != NULL) + { + pc->fi_pos = find_file_position (pc->fi, fi_pos); + GNUNET_free (fi_pos); + fi_pos = NULL; + if (pc->fi_pos == NULL) + { + /* failed to find position for resuming, outch! Will start from root! */ + GNUNET_break (0); + if (pc->all_done != GNUNET_YES) + pc->fi_pos = pc->fi; + } + } + GNUNET_free (fi_root); + fi_root = NULL; + /* generate RESUME event(s) */ + GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_resume, pc); + + /* re-start publishing (if needed)... */ + if (pc->all_done != GNUNET_YES) + { + GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority + (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, pc); + } + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming publishing operation `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + GNUNET_free_non_null (ns); + pc->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, pc); + return GNUNET_OK; +cleanup: + GNUNET_free_non_null (pc->nid); + GNUNET_free_non_null (pc->nuid); + GNUNET_free_non_null (fi_root); + GNUNET_free_non_null (fi_pos); + GNUNET_free_non_null (ns); + if ((rh != NULL) && (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume publishing operation `%s': %s\n"), filename, + emsg); + GNUNET_free (emsg); + } + if (pc->fi != NULL) + GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL); + if (0 != UNLINK (filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); + GNUNET_free (pc->serialization); + GNUNET_free (pc); + return GNUNET_OK; +} + + +/** + * Synchronize this publishing struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param pc the struct to sync + */ +void +GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc) +{ + struct GNUNET_BIO_WriteHandle *wh; + + if (NULL == pc->serialization) + pc->serialization = + make_serialization_file_name (pc->h, + GNUNET_FS_SYNC_PATH_MASTER_PUBLISH); + if (NULL == pc->serialization) + return; + if (NULL == pc->fi) + return; + if (NULL == pc->fi->serialization) + { + GNUNET_break (0); + return; + } + wh = get_write_handle (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, + pc->serialization); + if (wh == NULL) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != GNUNET_BIO_write_string (wh, pc->nid)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, pc->nuid)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pc->options)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pc->all_done)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, pc->fi->serialization)) || + (GNUNET_OK != + GNUNET_BIO_write_string (wh, + (pc->fi_pos == + NULL) ? NULL : pc->fi_pos->serialization)) || + (GNUNET_OK != + GNUNET_BIO_write_string (wh, + (pc->namespace == + NULL) ? NULL : pc->namespace->name))) + { + GNUNET_break (0); + goto cleanup; + } + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + wh = NULL; + GNUNET_break (0); + goto cleanup; + } + return; +cleanup: + if (wh != NULL) + (void) GNUNET_BIO_write_close (wh); + GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, + pc->serialization); + GNUNET_free (pc->serialization); + pc->serialization = NULL; +} + + +/** + * Synchronize this unindex struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param uc the struct to sync + */ +void +GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc) +{ + struct GNUNET_BIO_WriteHandle *wh; + + if (NULL == uc->serialization) + uc->serialization = + make_serialization_file_name (uc->h, + GNUNET_FS_SYNC_PATH_MASTER_UNINDEX); + if (NULL == uc->serialization) + return; + wh = get_write_handle (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, + uc->serialization); + if (wh == NULL) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uc->filename)) || + (GNUNET_OK != GNUNET_BIO_write_int64 (wh, uc->file_size)) || + (GNUNET_OK != write_start_time (wh, uc->start_time)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) uc->state)) || + ((uc->state == UNINDEX_STATE_FS_NOTIFY) && + (GNUNET_OK != + GNUNET_BIO_write (wh, &uc->file_id, sizeof (GNUNET_HashCode)))) || + ((uc->state == UNINDEX_STATE_ERROR) && + (GNUNET_OK != GNUNET_BIO_write_string (wh, uc->emsg)))) + { + GNUNET_break (0); + goto cleanup; + } + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + wh = NULL; + GNUNET_break (0); + goto cleanup; + } + return; +cleanup: + if (wh != NULL) + (void) GNUNET_BIO_write_close (wh); + GNUNET_FS_remove_sync_file_ (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, + uc->serialization); + GNUNET_free (uc->serialization); + uc->serialization = NULL; +} + + +/** + * Serialize a download request. + * + * @param wh the 'struct GNUNET_BIO_WriteHandle*' + * @param dr the 'struct DownloadRequest' + * @return GNUNET_YES on success, GNUNET_NO on error + */ +static int +write_download_request (struct GNUNET_BIO_WriteHandle *wh, + struct DownloadRequest *dr) +{ + unsigned int i; + + if ((GNUNET_OK != GNUNET_BIO_write_int32 (wh, dr->state)) || + (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dr->offset)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, dr->num_children)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, dr->depth))) + return GNUNET_NO; + if ((dr->state == BRS_CHK_SET) && + (GNUNET_OK != + GNUNET_BIO_write (wh, &dr->chk, sizeof (struct ContentHashKey)))) + return GNUNET_NO; + for (i = 0; i < dr->num_children; i++) + if (GNUNET_NO == write_download_request (wh, dr->children[i])) + return GNUNET_NO; + return GNUNET_YES; +} + + +/** + * Read a download request tree. + * + * @param rh stream to read from + * @return value the 'struct DownloadRequest', NULL on error + */ +static struct DownloadRequest * +read_download_request (struct GNUNET_BIO_ReadHandle *rh) +{ + struct DownloadRequest *dr; + unsigned int i; + + dr = GNUNET_malloc (sizeof (struct DownloadRequest)); + + if ((GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dr->state)) || + (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dr->offset)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dr->num_children)) || + (dr->num_children > CHK_PER_INODE) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dr->depth)) || ((dr->depth == 0) + && + (dr->num_children + > 0)) || + ((dr->depth > 0) && (dr->num_children == 0))) + { + GNUNET_break (0); + dr->num_children = 0; + goto cleanup; + } + if (dr->num_children > 0) + dr->children = + GNUNET_malloc (dr->num_children * sizeof (struct ContentHashKey)); + switch (dr->state) + { + case BRS_INIT: + case BRS_RECONSTRUCT_DOWN: + case BRS_RECONSTRUCT_META_UP: + case BRS_RECONSTRUCT_UP: + break; + case BRS_CHK_SET: + if (GNUNET_OK != + GNUNET_BIO_read (rh, "chk", &dr->chk, sizeof (struct ContentHashKey))) + goto cleanup; + break; + case BRS_DOWNLOAD_DOWN: + case BRS_DOWNLOAD_UP: + case BRS_ERROR: + break; + default: + GNUNET_break (0); + goto cleanup; + } + for (i = 0; i < dr->num_children; i++) + { + if (NULL == (dr->children[i] = read_download_request (rh))) + goto cleanup; + dr->children[i]->parent = dr; + } + return dr; +cleanup: + GNUNET_FS_free_download_request_ (dr); + return NULL; +} + + +/** + * Compute the name of the sync file (or directory) for the given download + * context. + * + * @param dc download context to compute for + * @param uni unique filename to use, use "" for the directory name + * @param ext extension to use, use ".dir" for our own subdirectory + * @return the expanded file name, NULL for none + */ +static char * +get_download_sync_filename (struct GNUNET_FS_DownloadContext *dc, + const char *uni, const char *ext) +{ + char *par; + char *epar; + + if (dc->parent == NULL) + return get_serialization_file_name (dc->h, + (dc->search != + NULL) ? + GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD : + GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD, + uni); + if (dc->parent->serialization == NULL) + return NULL; + par = get_download_sync_filename (dc->parent, dc->parent->serialization, ""); + if (par == NULL) + return NULL; + GNUNET_asprintf (&epar, "%s.dir%s%s%s", par, DIR_SEPARATOR_STR, uni, ext); + GNUNET_free (par); + return epar; +} + + +/** + * Synchronize this download struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param dc the struct to sync + */ +void +GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc) +{ + struct GNUNET_BIO_WriteHandle *wh; + char *uris; + char *fn; + char *dir; + + if (NULL == dc->serialization) + { + dir = get_download_sync_filename (dc, "", ""); + if (dir == NULL) + return; + if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dir)) + { + GNUNET_free (dir); + return; + } + fn = GNUNET_DISK_mktemp (dir); + GNUNET_free (dir); + if (fn == NULL) + return; + dc->serialization = get_serialization_short_name (fn); + } + else + { + fn = get_download_sync_filename (dc, dc->serialization, ""); + if (fn == NULL) + { + GNUNET_free (dc->serialization); + dc->serialization = NULL; + GNUNET_free (fn); + return; + } + } + wh = GNUNET_BIO_write_open (fn); + if (wh == NULL) + { + GNUNET_free (dc->serialization); + dc->serialization = NULL; + GNUNET_free (fn); + return; + } + GNUNET_assert ((GNUNET_YES == GNUNET_FS_uri_test_chk (dc->uri)) || + (GNUNET_YES == GNUNET_FS_uri_test_loc (dc->uri))); + uris = GNUNET_FS_uri_to_string (dc->uri); + if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uris)) || + (GNUNET_OK != GNUNET_BIO_write_meta_data (wh, dc->meta)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, dc->emsg)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, dc->filename)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, dc->temp_filename)) || + (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dc->old_file_size)) || + (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dc->offset)) || + (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dc->length)) || + (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dc->completed)) || + (GNUNET_OK != write_start_time (wh, dc->start_time)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, dc->anonymity)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) dc->options)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) dc->has_finished))) + { + GNUNET_break (0); + goto cleanup; + } + if (NULL == dc->emsg) + { + GNUNET_assert (dc->top_request != NULL); + if (GNUNET_YES != write_download_request (wh, dc->top_request)) + { + GNUNET_break (0); + goto cleanup; + } + } + GNUNET_free_non_null (uris); + uris = NULL; + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + wh = NULL; + GNUNET_break (0); + goto cleanup; + } + GNUNET_free (fn); + return; +cleanup: + if (NULL != wh) + (void) GNUNET_BIO_write_close (wh); + GNUNET_free_non_null (uris); + if (0 != UNLINK (fn)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn); + GNUNET_free (fn); + GNUNET_free (dc->serialization); + dc->serialization = NULL; +} + + +/** + * Synchronize this search result with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param sr the struct to sync + */ +void +GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr) +{ + struct GNUNET_BIO_WriteHandle *wh; + char *uris; + + uris = NULL; + if (NULL == sr->serialization) + sr->serialization = + make_serialization_file_name_in_dir (sr->sc->h, + (sr->sc->psearch_result == + NULL) ? + GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sr->sc->serialization); + if (NULL == sr->serialization) + return; + wh = get_write_handle_in_dir (sr->sc->h, + (sr->sc->psearch_result == + NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sr->sc->serialization, sr->serialization); + if (wh == NULL) + { + GNUNET_break (0); + goto cleanup; + } + uris = GNUNET_FS_uri_to_string (sr->uri); + if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uris)) || + (GNUNET_OK != + GNUNET_BIO_write_string (wh, + sr->download != + NULL ? sr->download->serialization : NULL)) || + (GNUNET_OK != + GNUNET_BIO_write_string (wh, + sr->update_search != + NULL ? sr->update_search->serialization : NULL)) + || (GNUNET_OK != GNUNET_BIO_write_meta_data (wh, sr->meta)) || + (GNUNET_OK != GNUNET_BIO_write (wh, &sr->key, sizeof (GNUNET_HashCode))) + || (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sr->mandatory_missing)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sr->optional_support)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sr->availability_success)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sr->availability_trials))) + { + GNUNET_break (0); + goto cleanup; + } + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + wh = NULL; + GNUNET_break (0); + goto cleanup; + } + GNUNET_free_non_null (uris); + return; +cleanup: + GNUNET_free_non_null (uris); + if (wh != NULL) + (void) GNUNET_BIO_write_close (wh); + remove_sync_file_in_dir (sr->sc->h, + (sr->sc->psearch_result == + NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sr->sc->serialization, sr->serialization); + GNUNET_free (sr->serialization); + sr->serialization = NULL; +} + + +/** + * Synchronize this search struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param sc the struct to sync + */ +void +GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc) +{ + struct GNUNET_BIO_WriteHandle *wh; + char *uris; + char in_pause; + const char *category; + + category = + (sc->psearch_result == + NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH; + if (NULL == sc->serialization) + sc->serialization = make_serialization_file_name (sc->h, category); + if (NULL == sc->serialization) + return; + uris = NULL; + wh = get_write_handle (sc->h, category, sc->serialization); + if (wh == NULL) + { + GNUNET_break (0); + goto cleanup; + } + GNUNET_assert ((GNUNET_YES == GNUNET_FS_uri_test_ksk (sc->uri)) || + (GNUNET_YES == GNUNET_FS_uri_test_sks (sc->uri))); + uris = GNUNET_FS_uri_to_string (sc->uri); + in_pause = (sc->task != GNUNET_SCHEDULER_NO_TASK) ? 'r' : '\0'; + if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uris)) || + (GNUNET_OK != write_start_time (wh, sc->start_time)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, sc->emsg)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) sc->options)) || + (GNUNET_OK != GNUNET_BIO_write (wh, &in_pause, sizeof (in_pause))) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sc->anonymity))) + { + GNUNET_break (0); + goto cleanup; + } + GNUNET_free (uris); + uris = NULL; + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + wh = NULL; + GNUNET_break (0); + goto cleanup; + } + return; +cleanup: + if (wh != NULL) + (void) GNUNET_BIO_write_close (wh); + GNUNET_free_non_null (uris); + GNUNET_FS_remove_sync_file_ (sc->h, category, sc->serialization); + GNUNET_free (sc->serialization); + sc->serialization = NULL; +} + + +/** + * Function called with a filename of serialized unindexing operation + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_Handle*' + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_unindex_file (void *cls, const char *filename) +{ + struct GNUNET_FS_Handle *h = cls; + struct GNUNET_BIO_ReadHandle *rh; + struct GNUNET_FS_UnindexContext *uc; + struct GNUNET_FS_ProgressInfo pi; + char *emsg; + uint32_t state; + + uc = GNUNET_malloc (sizeof (struct GNUNET_FS_UnindexContext)); + uc->h = h; + uc->serialization = get_serialization_short_name (filename); + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != + GNUNET_BIO_read_string (rh, "unindex-fn", &uc->filename, 10 * 1024)) || + (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &uc->file_size)) || + (GNUNET_OK != read_start_time (rh, &uc->start_time)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &state))) + { + GNUNET_break (0); + goto cleanup; + } + uc->state = (enum UnindexState) state; + switch (state) + { + case UNINDEX_STATE_HASHING: + break; + case UNINDEX_STATE_FS_NOTIFY: + if (GNUNET_OK != + GNUNET_BIO_read (rh, "unindex-hash", &uc->file_id, + sizeof (GNUNET_HashCode))) + { + GNUNET_break (0); + goto cleanup; + } + break; + case UNINDEX_STATE_DS_REMOVE: + break; + case UNINDEX_STATE_COMPLETE: + break; + case UNINDEX_STATE_ERROR: + if (GNUNET_OK != + GNUNET_BIO_read_string (rh, "unindex-emsg", &uc->emsg, 10 * 1024)) + { + GNUNET_break (0); + goto cleanup; + } + break; + default: + GNUNET_break (0); + goto cleanup; + } + uc->top = GNUNET_FS_make_top (h, &GNUNET_FS_unindex_signal_suspend_, uc); + pi.status = GNUNET_FS_STATUS_UNINDEX_RESUME; + pi.value.unindex.specifics.resume.message = uc->emsg; + GNUNET_FS_unindex_make_status_ (&pi, uc, + (uc->state == + UNINDEX_STATE_COMPLETE) ? uc->file_size : 0); + switch (uc->state) + { + case UNINDEX_STATE_HASHING: + uc->fhc = + GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, uc->filename, + HASHING_BLOCKSIZE, + &GNUNET_FS_unindex_process_hash_, uc); + break; + case UNINDEX_STATE_FS_NOTIFY: + uc->state = UNINDEX_STATE_HASHING; + GNUNET_FS_unindex_process_hash_ (uc, &uc->file_id); + break; + case UNINDEX_STATE_DS_REMOVE: + GNUNET_FS_unindex_do_remove_ (uc); + break; + case UNINDEX_STATE_COMPLETE: + case UNINDEX_STATE_ERROR: + /* no need to resume any operation, we were done */ + break; + default: + break; + } + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming unindexing operation `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + return GNUNET_OK; +cleanup: + GNUNET_free_non_null (uc->filename); + if ((rh != NULL) && (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume unindexing operation `%s': %s\n"), filename, + emsg); + GNUNET_free (emsg); + } + if (uc->serialization != NULL) + GNUNET_FS_remove_sync_file_ (h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, + uc->serialization); + GNUNET_free_non_null (uc->serialization); + GNUNET_free (uc); + return GNUNET_OK; +} + + +/** + * Deserialize a download. + * + * @param h overall context + * @param rh file to deserialize from + * @param parent parent download + * @param search associated search + * @param serialization name under which the search was serialized + */ +static void +deserialize_download (struct GNUNET_FS_Handle *h, + struct GNUNET_BIO_ReadHandle *rh, + struct GNUNET_FS_DownloadContext *parent, + struct GNUNET_FS_SearchResult *search, + const char *serialization); + + +/** + * Deserialize a search. + * + * @param h overall context + * @param rh file to deserialize from + * @param psearch_result parent search result + * @param serialization name under which the search was serialized + */ +static struct GNUNET_FS_SearchContext * +deserialize_search (struct GNUNET_FS_Handle *h, + struct GNUNET_BIO_ReadHandle *rh, + struct GNUNET_FS_SearchResult *psearch_result, + const char *serialization); + + +/** + * Function called with a filename of serialized search result + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_SearchContext*' + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_search_result (void *cls, const char *filename) +{ + struct GNUNET_FS_SearchContext *sc = cls; + char *ser; + char *uris; + char *emsg; + char *download; + char *update_srch; + struct GNUNET_BIO_ReadHandle *rh; + struct GNUNET_BIO_ReadHandle *drh; + struct GNUNET_FS_SearchResult *sr; + + ser = get_serialization_short_name (filename); + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + if (ser != NULL) + { + remove_sync_file_in_dir (sc->h, + (sc->psearch_result == + NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sc->serialization, ser); + GNUNET_free (ser); + } + return GNUNET_OK; + } + emsg = NULL; + uris = NULL; + download = NULL; + update_srch = NULL; + sr = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchResult)); + sr->serialization = ser; + if ((GNUNET_OK != GNUNET_BIO_read_string (rh, "result-uri", &uris, 10 * 1024)) + || (NULL == (sr->uri = GNUNET_FS_uri_parse (uris, &emsg))) || + (GNUNET_OK != GNUNET_BIO_read_string (rh, "download-lnk", &download, 16)) + || (GNUNET_OK != + GNUNET_BIO_read_string (rh, "search-lnk", &update_srch, 16)) || + (GNUNET_OK != GNUNET_BIO_read_meta_data (rh, "result-meta", &sr->meta)) || + (GNUNET_OK != + GNUNET_BIO_read (rh, "result-key", &sr->key, sizeof (GNUNET_HashCode))) + || (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sr->mandatory_missing)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sr->optional_support)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sr->availability_success)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sr->availability_trials))) + { + GNUNET_break (0); + goto cleanup; + } + GNUNET_free (uris); + if (download != NULL) + { + drh = get_read_handle (sc->h, GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD, download); + if (drh != NULL) + { + deserialize_download (sc->h, drh, NULL, sr, download); + if (GNUNET_OK != GNUNET_BIO_read_close (drh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume sub-download `%s': %s\n"), download, + emsg); + GNUNET_free (emsg); + } + } + GNUNET_free (download); + } + if (update_srch != NULL) + { + drh = + get_read_handle (sc->h, GNUNET_FS_SYNC_PATH_CHILD_SEARCH, update_srch); + if (drh != NULL) + { + deserialize_search (sc->h, drh, sr, update_srch); + if (GNUNET_OK != GNUNET_BIO_read_close (drh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume sub-search `%s': %s\n"), update_srch, + emsg); + GNUNET_free (emsg); + } + } + GNUNET_free (update_srch); + } + GNUNET_CONTAINER_multihashmap_put (sc->master_result_map, &sr->key, sr, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming search operation `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + return GNUNET_OK; +cleanup: + GNUNET_free_non_null (download); + GNUNET_free_non_null (emsg); + GNUNET_free_non_null (uris); + GNUNET_free_non_null (update_srch); + if (sr->uri != NULL) + GNUNET_FS_uri_destroy (sr->uri); + if (sr->meta != NULL) + GNUNET_CONTAINER_meta_data_destroy (sr->meta); + GNUNET_free (sr->serialization); + GNUNET_free (sr); + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming search operation `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + return GNUNET_OK; +} + + +/** + * Send the 'resume' signal to the callback; also actually + * resume the download (put it in the queue). Does this + * recursively for the top-level download and all child + * downloads. + * + * @param dc download to resume + */ +static void +signal_download_resume (struct GNUNET_FS_DownloadContext *dc) +{ + struct GNUNET_FS_DownloadContext *dcc; + struct GNUNET_FS_ProgressInfo pi; + + pi.status = GNUNET_FS_STATUS_DOWNLOAD_RESUME; + pi.value.download.specifics.resume.meta = dc->meta; + pi.value.download.specifics.resume.message = dc->emsg; + GNUNET_FS_download_make_status_ (&pi, dc); + dcc = dc->child_head; + while (NULL != dcc) + { + signal_download_resume (dcc); + dcc = dcc->next; + } + if (dc->pending_head != NULL) + GNUNET_FS_download_start_downloading_ (dc); +} + + +/** + * Signal resuming of a search to our clients (for the + * top level search and all sub-searches). + * + * @param sc search being resumed + */ +static void +signal_search_resume (struct GNUNET_FS_SearchContext *sc); + + +/** + * Iterator over search results signaling resume to the client for + * each result. + * + * @param cls closure, the 'struct GNUNET_FS_SearchContext' + * @param key current key code + * @param value value in the hash map, the 'struct GNUNET_FS_SearchResult' + * @return GNUNET_YES (we should continue to iterate) + */ +static int +signal_result_resume (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct GNUNET_FS_SearchContext *sc = cls; + struct GNUNET_FS_ProgressInfo pi; + struct GNUNET_FS_SearchResult *sr = value; + + if (0 == sr->mandatory_missing) + { + pi.status = GNUNET_FS_STATUS_SEARCH_RESUME_RESULT; + pi.value.search.specifics.resume_result.meta = sr->meta; + pi.value.search.specifics.resume_result.uri = sr->uri; + pi.value.search.specifics.resume_result.result = sr; + pi.value.search.specifics.resume_result.availability_rank = + 2 * sr->availability_success - sr->availability_trials; + pi.value.search.specifics.resume_result.availability_certainty = + sr->availability_trials; + pi.value.search.specifics.resume_result.applicability_rank = + sr->optional_support; + sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc); + } + if (sr->download != NULL) + { + signal_download_resume (sr->download); + } + else + { + GNUNET_FS_search_start_probe_ (sr); + } + if (sr->update_search != NULL) + signal_search_resume (sr->update_search); + return GNUNET_YES; +} + + +/** + * Free memory allocated by the search context and its children + * + * @param sc search context to free + */ +static void +free_search_context (struct GNUNET_FS_SearchContext *sc); + + +/** + * Iterator over search results freeing each. + * + * @param cls closure, the 'struct GNUNET_FS_SearchContext' + * @param key current key code + * @param value value in the hash map, the 'struct GNUNET_FS_SearchResult' + * @return GNUNET_YES (we should continue to iterate) + */ +static int +free_result (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct GNUNET_FS_SearchResult *sr = value; + + if (sr->update_search != NULL) + { + free_search_context (sr->update_search); + GNUNET_assert (NULL == sr->update_search); + } + GNUNET_CONTAINER_meta_data_destroy (sr->meta); + GNUNET_FS_uri_destroy (sr->uri); + GNUNET_free (sr); + return GNUNET_YES; +} + + +/** + * Free memory allocated by the search context and its children + * + * @param sc search context to free + */ +static void +free_search_context (struct GNUNET_FS_SearchContext *sc) +{ + if (sc->serialization != NULL) + { + GNUNET_FS_remove_sync_file_ (sc->h, + (sc->psearch_result == + NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sc->serialization); + GNUNET_FS_remove_sync_dir_ (sc->h, + (sc->psearch_result == + NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sc->serialization); + } + GNUNET_free_non_null (sc->serialization); + GNUNET_free_non_null (sc->emsg); + if (sc->uri != NULL) + GNUNET_FS_uri_destroy (sc->uri); + if (sc->master_result_map != NULL) + { + GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, &free_result, + sc); + GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map); + } + GNUNET_free (sc); +} + + +/** + * Function called with a filename of serialized sub-download + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_DownloadContext*' (parent) + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_subdownload (void *cls, const char *filename) +{ + struct GNUNET_FS_DownloadContext *parent = cls; + char *ser; + char *emsg; + struct GNUNET_BIO_ReadHandle *rh; + + ser = get_serialization_short_name (filename); + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Failed to resume sub-download `%s': could not open file `%s'\n"), + ser, filename); + GNUNET_free (ser); + return GNUNET_OK; + } + deserialize_download (parent->h, rh, parent, NULL, ser); + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume sub-download `%s': %s\n"), ser, emsg); + GNUNET_free (emsg); + } + GNUNET_free (ser); + return GNUNET_OK; +} + + +/** + * Free this download context and all of its descendants. + * (only works during deserialization since not all possible + * state it taken care of). + * + * @param dc context to free + */ +static void +free_download_context (struct GNUNET_FS_DownloadContext *dc) +{ + struct GNUNET_FS_DownloadContext *dcc; + + if (dc->meta != NULL) + GNUNET_CONTAINER_meta_data_destroy (dc->meta); + if (dc->uri != NULL) + GNUNET_FS_uri_destroy (dc->uri); + GNUNET_free_non_null (dc->temp_filename); + GNUNET_free_non_null (dc->emsg); + GNUNET_free_non_null (dc->filename); + GNUNET_free_non_null (dc->serialization); + while (NULL != (dcc = dc->child_head)) + { + GNUNET_CONTAINER_DLL_remove (dc->child_head, dc->child_tail, dcc); + free_download_context (dcc); + } + GNUNET_FS_free_download_request_ (dc->top_request); + if (NULL != dc->active) + GNUNET_CONTAINER_multihashmap_destroy (dc->active); + GNUNET_free (dc); +} + + +/** + * Deserialize a download. + * + * @param h overall context + * @param rh file to deserialize from + * @param parent parent download + * @param search associated search + * @param serialization name under which the search was serialized + */ +static void +deserialize_download (struct GNUNET_FS_Handle *h, + struct GNUNET_BIO_ReadHandle *rh, + struct GNUNET_FS_DownloadContext *parent, + struct GNUNET_FS_SearchResult *search, + const char *serialization) +{ + struct GNUNET_FS_DownloadContext *dc; + char *emsg; + char *uris; + char *dn; + uint32_t options; + uint32_t status; + + uris = NULL; + emsg = NULL; + dc = GNUNET_malloc (sizeof (struct GNUNET_FS_DownloadContext)); + dc->parent = parent; + dc->h = h; + dc->serialization = GNUNET_strdup (serialization); + if ((GNUNET_OK != + GNUNET_BIO_read_string (rh, "download-uri", &uris, 10 * 1024)) || + (NULL == (dc->uri = GNUNET_FS_uri_parse (uris, &emsg))) || + ((GNUNET_YES != GNUNET_FS_uri_test_chk (dc->uri)) && + (GNUNET_YES != GNUNET_FS_uri_test_loc (dc->uri))) || + (GNUNET_OK != GNUNET_BIO_read_meta_data (rh, "download-meta", &dc->meta)) + || (GNUNET_OK != + GNUNET_BIO_read_string (rh, "download-emsg", &dc->emsg, 10 * 1024)) || + (GNUNET_OK != + GNUNET_BIO_read_string (rh, "download-fn", &dc->filename, 10 * 1024)) || + (GNUNET_OK != + GNUNET_BIO_read_string (rh, "download-tfn", &dc->temp_filename, + 10 * 1024)) || + (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dc->old_file_size)) || + (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dc->offset)) || + (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dc->length)) || + (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dc->completed)) || + (GNUNET_OK != read_start_time (rh, &dc->start_time)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dc->anonymity)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &options)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &status))) + { + GNUNET_break (0); + goto cleanup; + } + dc->options = (enum GNUNET_FS_DownloadOptions) options; + dc->active = + GNUNET_CONTAINER_multihashmap_create (1 + 2 * (dc->length / DBLOCK_SIZE)); + dc->has_finished = (int) status; + dc->treedepth = + GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size (dc->uri)); + if (GNUNET_FS_uri_test_loc (dc->uri)) + GNUNET_assert (GNUNET_OK == + GNUNET_FS_uri_loc_get_peer_identity (dc->uri, &dc->target)); + if (dc->emsg == NULL) + { + dc->top_request = read_download_request (rh); + if (dc->top_request == NULL) + { + GNUNET_break (0); + goto cleanup; + } + } + dn = get_download_sync_filename (dc, dc->serialization, ".dir"); + if (dn != NULL) + { + if (GNUNET_YES == GNUNET_DISK_directory_test (dn)) + GNUNET_DISK_directory_scan (dn, &deserialize_subdownload, dc); + GNUNET_free (dn); + } + if (parent != NULL) + { + GNUNET_abort (); // for debugging for now - FIXME + GNUNET_CONTAINER_DLL_insert (parent->child_head, parent->child_tail, dc); + } + if (search != NULL) + { + dc->search = search; + search->download = dc; + } + if ((parent == NULL) && (search == NULL)) + { + dc->top = + GNUNET_FS_make_top (dc->h, &GNUNET_FS_download_signal_suspend_, dc); + signal_download_resume (dc); + } + GNUNET_free (uris); + dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); + return; +cleanup: + GNUNET_free_non_null (uris); + GNUNET_free_non_null (emsg); + free_download_context (dc); +} + + +/** + * Signal resuming of a search to our clients (for the + * top level search and all sub-searches). + * + * @param sc search being resumed + */ +static void +signal_search_resume (struct GNUNET_FS_SearchContext *sc) +{ + struct GNUNET_FS_ProgressInfo pi; + + pi.status = GNUNET_FS_STATUS_SEARCH_RESUME; + pi.value.search.specifics.resume.message = sc->emsg; + pi.value.search.specifics.resume.is_paused = + (sc->client == NULL) ? GNUNET_YES : GNUNET_NO; + sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc); + GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, + &signal_result_resume, sc); + +} + + +/** + * Deserialize a search. + * + * @param h overall context + * @param rh file to deserialize from + * @param psearch_result parent search result + * @param serialization name under which the search was serialized + */ +static struct GNUNET_FS_SearchContext * +deserialize_search (struct GNUNET_FS_Handle *h, + struct GNUNET_BIO_ReadHandle *rh, + struct GNUNET_FS_SearchResult *psearch_result, + const char *serialization) +{ + struct GNUNET_FS_SearchContext *sc; + char *emsg; + char *uris; + char *dn; + uint32_t options; + char in_pause; + + if ((psearch_result != NULL) && (psearch_result->update_search != NULL)) + { + GNUNET_break (0); + return NULL; + } + uris = NULL; + emsg = NULL; + sc = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchContext)); + if (psearch_result != NULL) + { + sc->psearch_result = psearch_result; + psearch_result->update_search = sc; + } + sc->h = h; + sc->serialization = GNUNET_strdup (serialization); + if ((GNUNET_OK != GNUNET_BIO_read_string (rh, "search-uri", &uris, 10 * 1024)) + || (NULL == (sc->uri = GNUNET_FS_uri_parse (uris, &emsg))) || + ((GNUNET_YES != GNUNET_FS_uri_test_ksk (sc->uri)) && + (GNUNET_YES != GNUNET_FS_uri_test_sks (sc->uri))) || + (GNUNET_OK != read_start_time (rh, &sc->start_time)) || + (GNUNET_OK != + GNUNET_BIO_read_string (rh, "search-emsg", &sc->emsg, 10 * 1024)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &options)) || + (GNUNET_OK != + GNUNET_BIO_read (rh, "search-pause", &in_pause, sizeof (in_pause))) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sc->anonymity))) + { + GNUNET_break (0); + goto cleanup; + } + sc->options = (enum GNUNET_FS_SearchOptions) options; + sc->master_result_map = GNUNET_CONTAINER_multihashmap_create (16); + dn = get_serialization_file_name_in_dir (h, + (sc->psearch_result == + NULL) ? + GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sc->serialization, ""); + if (dn != NULL) + { + if (GNUNET_YES == GNUNET_DISK_directory_test (dn)) + GNUNET_DISK_directory_scan (dn, &deserialize_search_result, sc); + GNUNET_free (dn); + } + if (('\0' == in_pause) && + (GNUNET_OK != GNUNET_FS_search_start_searching_ (sc))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Could not resume running search, will resume as paused search\n")); + } + signal_search_resume (sc); + GNUNET_free (uris); + return sc; +cleanup: + GNUNET_free_non_null (emsg); + free_search_context (sc); + GNUNET_free_non_null (uris); + return NULL; +} + + +/** + * Function called with a filename of serialized search operation + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_Handle*' + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_search_file (void *cls, const char *filename) +{ + struct GNUNET_FS_Handle *h = cls; + char *ser; + char *emsg; + struct GNUNET_BIO_ReadHandle *rh; + struct GNUNET_FS_SearchContext *sc; + + ser = get_serialization_short_name (filename); + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + if (ser != NULL) + { + GNUNET_FS_remove_sync_file_ (h, GNUNET_FS_SYNC_PATH_MASTER_SEARCH, ser); + GNUNET_free (ser); + } + return GNUNET_OK; + } + sc = deserialize_search (h, rh, NULL, ser); + if (sc != NULL) + sc->top = GNUNET_FS_make_top (h, &GNUNET_FS_search_signal_suspend_, sc); + GNUNET_free (ser); + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming search operation `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + return GNUNET_OK; +} + + +/** + * Function called with a filename of serialized download operation + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_Handle*' + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_download_file (void *cls, const char *filename) +{ + struct GNUNET_FS_Handle *h = cls; + char *ser; + char *emsg; + struct GNUNET_BIO_ReadHandle *rh; + + ser = get_serialization_short_name (filename); + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + if (0 != UNLINK (filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); + GNUNET_free (ser); + return GNUNET_OK; + } + deserialize_download (h, rh, NULL, NULL, ser); + GNUNET_free (ser); + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming download operation `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + return GNUNET_OK; +} + + +/** + * Deserialize informatin about pending operations. + * + * @param master_path which master directory should be scanned + * @param proc function to call for each entry (will get 'h' for 'cls') + * @param h the 'struct GNUNET_FS_Handle*' + */ +static void +deserialization_master (const char *master_path, GNUNET_FileNameCallback proc, + struct GNUNET_FS_Handle *h) +{ + char *dn; + + dn = get_serialization_file_name (h, master_path, ""); + if (dn == NULL) + return; + if (GNUNET_YES == GNUNET_DISK_directory_test (dn)) + GNUNET_DISK_directory_scan (dn, proc, h); + GNUNET_free (dn); +} + + +/** + * Setup a connection to the file-sharing service. + * + * @param cfg configuration to use + * @param client_name unique identifier for this client + * @param upcb function to call to notify about FS actions + * @param upcb_cls closure for upcb + * @param flags specific attributes for fs-operations + * @param ... list of optional options, terminated with GNUNET_FS_OPTIONS_END + * @return NULL on error + */ +struct GNUNET_FS_Handle * +GNUNET_FS_start (const struct GNUNET_CONFIGURATION_Handle *cfg, + const char *client_name, GNUNET_FS_ProgressCallback upcb, + void *upcb_cls, enum GNUNET_FS_Flags flags, ...) +{ + struct GNUNET_FS_Handle *ret; + enum GNUNET_FS_OPTIONS opt; + va_list ap; + + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Handle)); + ret->cfg = cfg; + ret->client_name = GNUNET_strdup (client_name); + ret->upcb = upcb; + ret->upcb_cls = upcb_cls; + ret->flags = flags; + ret->max_parallel_downloads = 1; + ret->max_parallel_requests = 1; + ret->avg_block_latency = GNUNET_TIME_UNIT_MINUTES; /* conservative starting point */ + va_start (ap, flags); + while (GNUNET_FS_OPTIONS_END != (opt = va_arg (ap, enum GNUNET_FS_OPTIONS))) + { + switch (opt) + { + case GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM: + ret->max_parallel_downloads = va_arg (ap, unsigned int); + + break; + case GNUNET_FS_OPTIONS_REQUEST_PARALLELISM: + ret->max_parallel_requests = va_arg (ap, unsigned int); + + break; + default: + GNUNET_break (0); + GNUNET_free (ret->client_name); + GNUNET_free (ret); + va_end (ap); + return NULL; + } + } + va_end (ap); + if (0 != (GNUNET_FS_FLAGS_PERSISTENCE & flags)) + { + deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, + &deserialize_publish_file, ret); + deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_SEARCH, + &deserialize_search_file, ret); + deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD, + &deserialize_download_file, ret); + deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, + &deserialize_unindex_file, ret); + } + return ret; +} + + +/** + * Close our connection with the file-sharing service. + * The callback given to GNUNET_FS_start will no longer be + * called after this function returns. + * + * @param h handle that was returned from GNUNET_FS_start + */ +void +GNUNET_FS_stop (struct GNUNET_FS_Handle *h) +{ + while (h->top_head != NULL) + h->top_head->ssf (h->top_head->ssf_cls); + if (h->queue_job != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel (h->queue_job); + GNUNET_free (h->client_name); + GNUNET_free (h); +} + + +/* end of fs.c */ diff --git a/src/fs/fs_api.h b/src/fs/fs_api.h new file mode 100644 index 000000000..f358047aa --- /dev/null +++ b/src/fs/fs_api.h @@ -0,0 +1,1907 @@ +/* + This file is part of GNUnet. + (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_api.h + * @brief shared definitions for the FS library + * @author Igor Wronsky, Christian Grothoff + */ +#ifndef FS_API_H +#define FS_API_H + +#include "gnunet_constants.h" +#include "gnunet_datastore_service.h" +#include "gnunet_dht_service.h" +#include "gnunet_fs_service.h" +#include "gnunet_block_lib.h" +#include "block_fs.h" +#include "fs.h" + +/** + * Size of the individual blocks used for file-sharing. + */ +#define DBLOCK_SIZE (32*1024) + +/** + * Pick a multiple of 2 here to achive 8-byte alignment! We also + * probably want DBlocks to have (roughly) the same size as IBlocks. + * With SHA-512, the optimal value is 32768 byte / 128 byte = 256 (128 + * byte = 2 * 512 bits). DO NOT CHANGE! + */ +#define CHK_PER_INODE 256 + +/** + * Maximum size for a file to be considered for inlining in a + * directory. + */ +#define MAX_INLINE_SIZE 65536 + +/** + * Name of the directory with top-level searches. + */ +#define GNUNET_FS_SYNC_PATH_MASTER_SEARCH "search" + +/** + * Name of the directory with sub-searches (namespace-updates). + */ +#define GNUNET_FS_SYNC_PATH_CHILD_SEARCH "search-child" + +/** + * Name of the directory with master downloads (not associated + * with search or part of another download). + */ +#define GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD "download" + +/** + * Name of the directory with downloads that are part of another + * download or a search. + */ +#define GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD "download-child" + +/** + * Name of the directory with publishing operations. + */ +#define GNUNET_FS_SYNC_PATH_MASTER_PUBLISH "publish" + +/** + * Name of the directory with files that are being published + */ +#define GNUNET_FS_SYNC_PATH_FILE_INFO "publish-file" + +/** + * Name of the directory with unindex operations. + */ +#define GNUNET_FS_SYNC_PATH_MASTER_UNINDEX "unindex" + + +/** + * @brief complete information needed + * to download a file. + */ +struct FileIdentifier +{ + + /** + * Total size of the file in bytes. (network byte order (!)) + */ + uint64_t file_length; + + /** + * Query and key of the top GNUNET_EC_IBlock. + */ + struct ContentHashKey chk; + +}; + + +/** + * Information about a file and its location + * (peer claiming to share the file). + */ +struct Location +{ + /** + * Information about the shared file. + */ + struct FileIdentifier fi; + + /** + * Identity of the peer sharing the file. + */ + struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded peer; + + /** + * Time when this location URI expires. + */ + struct GNUNET_TIME_Absolute expirationTime; + + /** + * RSA signature over the GNUNET_EC_FileIdentifier, + * GNUNET_hash of the peer and expiration time. + */ + struct GNUNET_CRYPTO_RsaSignature contentSignature; + +}; + +/** + * Types of URIs. + */ +enum uri_types +{ + /** + * Content-hash-key (simple file). + */ + chk, + + /** + * Signed key space (file in namespace). + */ + sks, + + /** + * Keyword search key (query with keywords). + */ + ksk, + + /** + * Location (chk with identity of hosting peer). + */ + loc +}; + +/** + * A Universal Resource Identifier (URI), opaque. + */ +struct GNUNET_FS_Uri +{ + /** + * Type of the URI. + */ + enum uri_types type; + + union + { + struct + { + /** + * Keywords start with a '+' if they are + * mandatory (in which case the '+' is NOT + * part of the keyword) and with a + * simple space if they are optional + * (in which case the space is ALSO not + * part of the actual keyword). + * + * Double-quotes to protect spaces and + * %-encoding are NOT used internally + * (only in URI-strings). + */ + char **keywords; + + /** + * Size of the keywords array. + */ + unsigned int keywordCount; + } ksk; + + struct + { + /** + * Hash of the public key for the namespace. + */ + GNUNET_HashCode namespace; + + /** + * Human-readable identifier chosen for this + * entry in the namespace. + */ + char *identifier; + } sks; + + /** + * Information needed to retrieve a file (content-hash-key + * plus file size). + */ + struct FileIdentifier chk; + + /** + * Information needed to retrieve a file including signed + * location (identity of a peer) of the content. + */ + struct Location loc; + } data; + +}; + + +/** + * Information for a file or directory that is + * about to be published. + */ +struct GNUNET_FS_FileInformation +{ + + /** + * Files in a directory are kept as a linked list. + */ + struct GNUNET_FS_FileInformation *next; + + /** + * If this is a file in a directory, "dir" refers to + * the directory; otherwise NULL. + */ + struct GNUNET_FS_FileInformation *dir; + + /** + * Handle to the master context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Pointer kept for the client. + */ + void *client_info; + + /** + * Metadata to use for the file. + */ + struct GNUNET_CONTAINER_MetaData *meta; + + /** + * Keywords to use for KBlocks. + */ + struct GNUNET_FS_Uri *keywords; + + /** + * CHK for this file or directory. NULL if + * we have not yet computed it. + */ + struct GNUNET_FS_Uri *chk_uri; + + /** + * Block options for the file. + */ + struct GNUNET_FS_BlockOptions bo; + + /** + * At what time did we start this upload? + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Under what filename is this struct serialized + * (for operational persistence). Should be determined + * using 'mktemp'. + */ + char *serialization; + + /** + * Encoder being used to publish this file. + */ + struct GNUNET_FS_TreeEncoder *te; + + /** + * Error message (non-NULL if this operation failed). + */ + char *emsg; + + /** + * Name of the file or directory (must be an absolute path). + */ + char *filename; + + /** + * Data describing either the file or the directory. + */ + union + { + + /** + * Data for a file. + */ + struct + { + + /** + * Function that can be used to read the data for the file. + */ + GNUNET_FS_DataReader reader; + + /** + * Closure for reader. + */ + void *reader_cls; + + /** + * If this file is being indexed, this value is set to the hash + * over the entire file (when the indexing process is started). + * Otherwise this field is not used. + */ + GNUNET_HashCode file_id; + + /** + * Size of the file (in bytes). + */ + uint64_t file_size; + + /** + * Should the file be indexed or inserted? + */ + int do_index; + + /** + * Is "file_id" already valid? Set to GNUNET_YES once the hash + * has been calculated. + */ + int have_hash; + + /** + * Has the service confirmed our INDEX_START request? + * GNUNET_YES if this step has been completed. + */ + int index_start_confirmed; + + } file; + + /** + * Data for a directory. + */ + struct + { + + /** + * Linked list of entries in the directory. + */ + struct GNUNET_FS_FileInformation *entries; + + /** + * Size of the directory itself (in bytes); 0 if the + * size has not yet been calculated. + */ + size_t dir_size; + + /** + * Pointer to the data for the directory (or NULL if not + * available). + */ + void *dir_data; + + } dir; + + } data; + + /** + * Is this struct for a file or directory? + */ + int is_directory; + + /** + * Are we done publishing this file? + */ + int is_published; + +}; + + +/** + * The job is now ready to run and should use the given client + * handle to communicate with the FS service. + * + * @param cls closure + * @param client handle to use for FS communication + */ +typedef void (*GNUNET_FS_QueueStart) (void *cls, + struct GNUNET_CLIENT_Connection * client); + + +/** + * The job must now stop to run and should destry the client handle as + * soon as possible (ideally prior to returning). + */ +typedef void (*GNUNET_FS_QueueStop) (void *cls); + + +/** + * Entry in the job queue. + */ +struct GNUNET_FS_QueueEntry +{ + /** + * This is a linked list. + */ + struct GNUNET_FS_QueueEntry *next; + + /** + * This is a linked list. + */ + struct GNUNET_FS_QueueEntry *prev; + + /** + * Function to call when the job is started. + */ + GNUNET_FS_QueueStart start; + + /** + * Function to call when the job needs to stop (or is done / dequeued). + */ + GNUNET_FS_QueueStop stop; + + /** + * Closure for start and stop. + */ + void *cls; + + /** + * Handle to FS primary context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Client handle, or NULL if job is not running. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Time the job was originally queued. + */ + struct GNUNET_TIME_Absolute queue_time; + + /** + * Time the job was started last. + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Total amount of time the job has been running (except for the + * current run). + */ + struct GNUNET_TIME_Relative run_time; + + /** + * How many blocks do the active downloads have? + */ + unsigned int blocks; + + /** + * How often have we (re)started this download? + */ + unsigned int start_times; + +}; + + + + +/** + * Information we store for each search result. + */ +struct GNUNET_FS_SearchResult +{ + + /** + * Search context this result belongs to. + */ + struct GNUNET_FS_SearchContext *sc; + + /** + * URI to which this search result refers to. + */ + struct GNUNET_FS_Uri *uri; + + /** + * Metadata for the search result. + */ + struct GNUNET_CONTAINER_MetaData *meta; + + /** + * Client info for this search result. + */ + void *client_info; + + /** + * ID of a job that is currently probing this results' availability + * (NULL if we are not currently probing). + */ + struct GNUNET_FS_DownloadContext *probe_ctx; + + /** + * ID of an associated download based on this search result (or + * NULL for none). + */ + struct GNUNET_FS_DownloadContext *download; + + /** + * If this search result triggered an update search, this field + * links to the update search. + */ + struct GNUNET_FS_SearchContext *update_search; + + /** + * Name under which this search result is stored on disk. + */ + char *serialization; + + /** + * Key for the search result + */ + GNUNET_HashCode key; + + /** + * ID of the task that will clean up the probe_ctx should it not + * complete on time (and that will need to be cancelled if we clean + * up the search result before then). + */ + GNUNET_SCHEDULER_TaskIdentifier probe_cancel_task; + + /** + * When did the current probe become active? + */ + struct GNUNET_TIME_Absolute probe_active_time; + + /** + * How much longer should we run the current probe before giving up? + */ + struct GNUNET_TIME_Relative remaining_probe_time; + + /** + * Number of mandatory keywords for which we have NOT yet found the + * search result; when this value hits zero, the search result is + * given to the callback. + */ + uint32_t mandatory_missing; + + /** + * Number of optional keywords under which this result was also + * found. + */ + uint32_t optional_support; + + /** + * Number of availability tests that have succeeded for this result. + */ + uint32_t availability_success; + + /** + * Number of availability trials that we have performed for this + * search result. + */ + uint32_t availability_trials; + +}; + + +/** + * Add a job to the queue. + * + * @param h handle to the overall FS state + * @param start function to call to begin the job + * @param stop function to call to pause the job, or on dequeue (if the job was running) + * @param cls closure for start and stop + * @param blocks number of blocks this download has + * @return queue handle + */ +struct GNUNET_FS_QueueEntry * +GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, GNUNET_FS_QueueStart start, + GNUNET_FS_QueueStop stop, void *cls, unsigned int blocks); + + +/** + * Dequeue a job from the queue. + * @param qh handle for the job + */ +void +GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh); + + +/** + * Function that provides data by reading from a file. + * + * @param cls closure (points to the file information) + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually "max", 0 on error + */ +size_t +GNUNET_FS_data_reader_file_ (void *cls, uint64_t offset, size_t max, void *buf, + char **emsg); + + +/** + * Create the closure for the 'GNUNET_FS_data_reader_file_' callback. + * + * @param filename file to read + * @return closure to use + */ +void * +GNUNET_FS_make_file_reader_context_ (const char *filename); + + + +/** + * Function that provides data by copying from a buffer. + * + * @param cls closure (points to the buffer) + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually "max", 0 on error + */ +size_t +GNUNET_FS_data_reader_copy_ (void *cls, uint64_t offset, size_t max, void *buf, + char **emsg); + +/** + * Notification of FS that a search probe has made progress. + * This function is used INSTEAD of the client's event handler + * for downloads where the GNUNET_FS_DOWNLOAD_IS_PROBE flag is set. + * + * @param cls closure, always NULL (!), actual closure + * is in the client-context of the info struct + * @param info details about the event, specifying the event type + * and various bits about the event + * @return client-context (for the next progress call + * for this operation; should be set to NULL for + * SUSPEND and STOPPED events). The value returned + * will be passed to future callbacks in the respective + * field in the GNUNET_FS_ProgressInfo struct. + */ +void * +GNUNET_FS_search_probe_progress_ (void *cls, + const struct GNUNET_FS_ProgressInfo *info); + + +/** + * Main function that performs the upload. + * + * @param cls "struct GNUNET_FS_PublishContext" identifies the upload + * @param tc task context + */ +void +GNUNET_FS_publish_main_ (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc); + + +/** + * Function called once the hash of the file + * that is being unindexed has been computed. + * + * @param cls closure, unindex context + * @param file_id computed hash, NULL on error + */ +void +GNUNET_FS_unindex_process_hash_ (void *cls, const GNUNET_HashCode * file_id); + + +/** + * Fill in all of the generic fields for a publish event and call the + * callback. + * + * @param pi structure to fill in + * @param pc overall publishing context + * @param p file information for the file being published + * @param offset where in the file are we so far + * @return value returned from callback + */ +void * +GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_PublishContext *pc, + const struct GNUNET_FS_FileInformation *p, + uint64_t offset); + + +/** + * Fill in all of the generic fields for a download event and call the + * callback. + * + * @param pi structure to fill in + * @param dc overall download context + */ +void +GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_DownloadContext *dc); + + +/** + * Task that creates the initial (top-level) download + * request for the file. + * + * @param cls the 'struct GNUNET_FS_DownloadContext' + * @param tc scheduler context + */ +void +GNUNET_FS_download_start_task_ (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc); + + + +/** + * Fill in all of the generic fields for + * an unindex event and call the callback. + * + * @param pi structure to fill in + * @param uc overall unindex context + * @param offset where we are in the file (for progress) + */ +void +GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_UnindexContext *uc, + uint64_t offset); + +/** + * Fill in all of the generic fields for a search event and + * call the callback. + * + * @param pi structure to fill in + * @param sc overall search context + * @return value returned by the callback + */ +void * +GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_SearchContext *sc); + + +/** + * Connect to the datastore and remove the blocks. + * + * @param uc context for the unindex operation. + */ +void +GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc); + +/** + * Build the request and actually initiate the search using the + * GNUnet FS service. + * + * @param sc search context + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +int +GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc); + +/** + * Start the downloading process (by entering the queue). + * + * @param dc our download context + */ +void +GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc); + + +/** + * Start download probes for the given search result. + * + * @param sr the search result + */ +void +GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr); + +/** + * Remove serialization/deserialization file from disk. + * + * @param h master context + * @param ext component of the path + * @param ent entity identifier + */ +void +GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, const char *ext, + const char *ent); + + +/** + * Remove serialization/deserialization directory from disk. + * + * @param h master context + * @param ext component of the path + * @param uni unique name of parent + */ +void +GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, const char *ext, + const char *uni); + + +/** + * Synchronize this file-information struct with its mirror + * on disk. Note that all internal FS-operations that change + * file information data should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param fi the struct to sync + */ +void +GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *f); + +/** + * Synchronize this publishing struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param pc the struct to sync + */ +void +GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc); + +/** + * Synchronize this unindex struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param uc the struct to sync + */ +void +GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc); + +/** + * Synchronize this search struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param sc the struct to sync + */ +void +GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc); + +/** + * Synchronize this search result with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param sr the struct to sync + */ +void +GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr); + +/** + * Synchronize this download struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param dc the struct to sync + */ +void +GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc); + +/** + * Create SUSPEND event for the given publish operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_PublishContext' to signal for + */ +void +GNUNET_FS_publish_signal_suspend_ (void *cls); + +/** + * Create SUSPEND event for the given search operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_SearchContext' to signal for + */ +void +GNUNET_FS_search_signal_suspend_ (void *cls); + +/** + * Create SUSPEND event for the given download operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for + */ +void +GNUNET_FS_download_signal_suspend_ (void *cls); + +/** + * Create SUSPEND event for the given unindex operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_UnindexContext' to signal for + */ +void +GNUNET_FS_unindex_signal_suspend_ (void *cls); + +/** + * Function signature of the functions that can be called + * to trigger suspend signals and clean-up for top-level + * activities. + * + * @param cls closure + */ +typedef void (*SuspendSignalFunction) (void *cls); + +/** + * We track all of the top-level activities of FS + * so that we can signal 'suspend' on shutdown. + */ +struct TopLevelActivity +{ + /** + * This is a doubly-linked list. + */ + struct TopLevelActivity *next; + + /** + * This is a doubly-linked list. + */ + struct TopLevelActivity *prev; + + /** + * Function to call for suspend-signalling and clean up. + */ + SuspendSignalFunction ssf; + + /** + * Closure for 'ssf' (some struct GNUNET_FS_XXXHandle*) + */ + void *ssf_cls; +}; + + +/** + * Create a top-level activity entry. + * + * @param h global fs handle + * @param ssf suspend signal function to use + * @param ssf_cls closure for ssf + * @return fresh top-level activity handle + */ +struct TopLevelActivity * +GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf, + void *ssf_cls); + + +/** + * Destroy a top-level activity entry. + * + * @param h global fs handle + * @param top top level activity entry + */ +void +GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top); + + + +/** + * Master context for most FS operations. + */ +struct GNUNET_FS_Handle +{ + /** + * Configuration to use. + */ + const struct GNUNET_CONFIGURATION_Handle *cfg; + + /** + * Name of our client. + */ + char *client_name; + + /** + * Function to call with updates on our progress. + */ + GNUNET_FS_ProgressCallback upcb; + + /** + * Closure for upcb. + */ + void *upcb_cls; + + /** + * Head of DLL of top-level activities. + */ + struct TopLevelActivity *top_head; + + /** + * Tail of DLL of top-level activities. + */ + struct TopLevelActivity *top_tail; + + /** + * Head of DLL of running jobs. + */ + struct GNUNET_FS_QueueEntry *running_head; + + /** + * Tail of DLL of running jobs. + */ + struct GNUNET_FS_QueueEntry *running_tail; + + /** + * Head of DLL of pending jobs. + */ + struct GNUNET_FS_QueueEntry *pending_head; + + /** + * Tail of DLL of pending jobs. + */ + struct GNUNET_FS_QueueEntry *pending_tail; + + /** + * Task that processes the jobs in the running and pending queues + * (and moves jobs around as needed). + */ + GNUNET_SCHEDULER_TaskIdentifier queue_job; + + /** + * Average time we take for a single request to be satisfied. + * FIXME: not yet calcualted properly... + */ + struct GNUNET_TIME_Relative avg_block_latency; + + /** + * How many actual downloads do we have running right now? + */ + unsigned int active_downloads; + + /** + * How many blocks do the active downloads have? + */ + unsigned int active_blocks; + + /** + * General flags. + */ + enum GNUNET_FS_Flags flags; + + /** + * Maximum number of parallel downloads. + */ + unsigned int max_parallel_downloads; + + /** + * Maximum number of parallel requests. + */ + unsigned int max_parallel_requests; + +}; + + +/** + * Handle for controlling a publication process. + */ +struct GNUNET_FS_PublishContext +{ + /** + * Handle to the global fs context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Our top-level activity entry (if we are top-level, otherwise NULL). + */ + struct TopLevelActivity *top; + + /** + * File-structure that is being shared. + */ + struct GNUNET_FS_FileInformation *fi; + + /** + * Namespace that we are publishing in, NULL if we have no namespace. + */ + struct GNUNET_FS_Namespace *namespace; + + /** + * ID of the content in the namespace, NULL if we have no namespace. + */ + char *nid; + + /** + * ID for future updates, NULL if we have no namespace or no updates. + */ + char *nuid; + + /** + * Filename used for serializing information about this operation + * (should be determined using 'mktemp'). + */ + char *serialization; + + /** + * Our own client handle for the FS service; only briefly used when + * we start to index a file, otherwise NULL. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Current position in the file-tree for the upload. + */ + struct GNUNET_FS_FileInformation *fi_pos; + + /** + * Non-null if we are currently hashing a file. + */ + struct GNUNET_CRYPTO_FileHashContext *fhc; + + /** + * Connection to the datastore service. + */ + struct GNUNET_DATASTORE_Handle *dsh; + + /** + * Queue entry for reservation/unreservation. + */ + struct GNUNET_DATASTORE_QueueEntry *qre; + + /** + * ID of the task performing the upload. NO_TASK if the upload has + * completed. + */ + GNUNET_SCHEDULER_TaskIdentifier upload_task; + + /** + * Storage space to reserve for the operation. + */ + uint64_t reserve_space; + + /** + * Overall number of entries to reserve for the + * publish operation. + */ + uint32_t reserve_entries; + + /** + * Typically GNUNET_NO. Set to GNUNET_YES if "upload_task" is + * GNUNET_SCHEDULER_NO_TASK and we're waiting for a response from + * the datastore service (in which case this struct must not be + * freed until we have that response). If someone tries to stop the + * download for good during this period, "in_network_wait" is set to + * GNUNET_SYSERR which will cause the struct to be destroyed right + * after we have the reply (or timeout) from the datastore service. + */ + int in_network_wait; + + /** + * Options for publishing. + */ + enum GNUNET_FS_PublishOptions options; + + /** + * Space reservation ID with datastore service + * for this upload. + */ + int rid; + + /** + * Set to GNUNET_YES if all processing has completed. + */ + int all_done; +}; + + +/** + * Phases of unindex processing (state machine). + */ +enum UnindexState +{ + /** + * We're currently hashing the file. + */ + UNINDEX_STATE_HASHING = 0, + + /** + * We're telling the datastore to delete + * the respective entries. + */ + UNINDEX_STATE_DS_REMOVE = 1, + + /** + * We're notifying the FS service about + * the unindexing. + */ + UNINDEX_STATE_FS_NOTIFY = 2, + + /** + * We're done. + */ + UNINDEX_STATE_COMPLETE = 3, + + /** + * We've encountered a fatal error. + */ + UNINDEX_STATE_ERROR = 4 +}; + + +/** + * Handle for controlling an unindexing operation. + */ +struct GNUNET_FS_UnindexContext +{ + + /** + * Global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Our top-level activity entry. + */ + struct TopLevelActivity *top; + + /** + * Name of the file that we are unindexing. + */ + char *filename; + + /** + * Short name under which we are serializing the state of this operation. + */ + char *serialization; + + /** + * Connection to the FS service, only valid during the + * UNINDEX_STATE_FS_NOTIFY phase. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Connection to the datastore service, only valid during the + * UNINDEX_STATE_DS_NOTIFY phase. + */ + struct GNUNET_DATASTORE_Handle *dsh; + + /** + * Pointer kept for the client. + */ + void *client_info; + + /** + * Merkle-ish tree encoder context. + */ + struct GNUNET_FS_TreeEncoder *tc; + + /** + * Handle used to read the file. + */ + struct GNUNET_DISK_FileHandle *fh; + + /** + * Error message, NULL on success. + */ + char *emsg; + + /** + * Context for hashing of the file. + */ + struct GNUNET_CRYPTO_FileHashContext *fhc; + + /** + * Overall size of the file. + */ + uint64_t file_size; + + /** + * When did we start? + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Hash of the file's contents (once computed). + */ + GNUNET_HashCode file_id; + + /** + * Current operatinonal phase. + */ + enum UnindexState state; + +}; + + +/** + * Information we keep for each keyword in + * a keyword search. + */ +struct SearchRequestEntry +{ + /** + * Hash of the original keyword, also known as the + * key (for decrypting the KBlock). + */ + GNUNET_HashCode key; + + /** + * Hash of the public key, also known as the query. + */ + GNUNET_HashCode query; + + /** + * Map that contains a "struct GNUNET_FS_SearchResult" for each result that + * was found under this keyword. Note that the entries will point + * to the same locations as those in the master result map (in + * "struct GNUNET_FS_SearchContext"), so they should not be freed. + * The key for each entry is the XOR of the key and query in the CHK + * URI (as a unique identifier for the search result). + */ + struct GNUNET_CONTAINER_MultiHashMap *results; + + /** + * Is this keyword a mandatory keyword + * (started with '+')? + */ + int mandatory; + +}; + + +/** + * Handle for controlling a search. + */ +struct GNUNET_FS_SearchContext +{ + /** + * Handle to the global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Our top-level activity entry (if we are top-level, otherwise NULL). + */ + struct TopLevelActivity *top; + + /** + * List of keywords that we're looking for. + */ + struct GNUNET_FS_Uri *uri; + + /** + * For update-searches, link to the search result that triggered + * the update search; otherwise NULL. + */ + struct GNUNET_FS_SearchResult *psearch_result; + + /** + * Connection to the FS service. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Pointer we keep for the client. + */ + void *client_info; + + /** + * Name of the file on disk we use for persistence. + */ + char *serialization; + + /** + * Error message (non-NULL if this operation failed). + */ + char *emsg; + + /** + * Map that contains a "struct GNUNET_FS_SearchResult" for each result that + * was found in the search. The key for each entry is the XOR of + * the key and query in the CHK URI (as a unique identifier for the + * search result). + */ + struct GNUNET_CONTAINER_MultiHashMap *master_result_map; + + /** + * Per-keyword information for a keyword search. This array will + * have exactly as many entries as there were keywords. + */ + struct SearchRequestEntry *requests; + + /** + * When did we start? + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * ID of a task that is using this struct and that must be cancelled + * when the search is being stopped (if not + * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some + * artificial delay when trying to reconnect to the FS service. + */ + GNUNET_SCHEDULER_TaskIdentifier task; + + /** + * How many of the entries in the search request + * map have been passed to the service so far? + */ + unsigned int search_request_map_offset; + + /** + * How many of the keywords in the KSK + * map have been passed to the service so far? + */ + unsigned int keyword_offset; + + /** + * Anonymity level for the search. + */ + uint32_t anonymity; + + /** + * Number of mandatory keywords in this query. + */ + uint32_t mandatory_count; + + /** + * Options for the search. + */ + enum GNUNET_FS_SearchOptions options; +}; + + +/** + * FSM for possible states a block can go through. The typical + * order of progression is linear through the states, alternatives + * are documented in the comments. + */ +enum BlockRequestState +{ + /** + * Initial state, block has only been allocated (since it is + * relevant to the overall download request). + */ + BRS_INIT = 0, + + /** + * We've checked the block on the path down the tree, and the + * content on disk did match the desired CHK, but not all + * the way down, so at the bottom some blocks will still + * need to be reconstructed). + */ + BRS_RECONSTRUCT_DOWN = 1, + + /** + * We've calculated the CHK bottom-up based on the meta data. + * This may work, but if it did we have to write the meta data to + * disk at the end (and we still need to check against the + * CHK set on top). + */ + BRS_RECONSTRUCT_META_UP = 2, + + /** + * We've calculated the CHK bottom-up based on what we have on + * disk, which may not be what the desired CHK is. If the + * reconstructed CHKs match whatever comes from above, we're + * done with the respective subtree. + */ + BRS_RECONSTRUCT_UP = 3, + + /** + * We've determined the real, desired CHK for this block + * (full tree reconstruction failed), request is now pending. + * If the CHK that bubbled up through reconstruction did match + * the top-level request, the state machine for the subtree + * would have moved to BRS_DOWNLOAD_UP. + */ + BRS_CHK_SET = 4, + + /** + * We've successfully downloaded this block, but the children + * still need to be either downloaded or verified (download + * request propagates down). If the download fails, the + * state machine for this block may move to + * BRS_DOWNLOAD_ERROR instead. + */ + BRS_DOWNLOAD_DOWN = 5, + + /** + * This block and all of its children have been downloaded + * successfully (full completion propagates up). + */ + BRS_DOWNLOAD_UP = 6, + + /** + * We got a block back that matched the query but did not hash to + * the key (malicious publisher or hash collision); this block + * can never be downloaded (error propagates up). + */ + BRS_ERROR = 7 +}; + + +/** + * Information about an active download request. + */ +struct DownloadRequest +{ + /** + * While pending, we keep all download requests in a doubly-linked list. + */ + struct DownloadRequest *next; + + /** + * While pending, we keep all download requests in a doubly-linked list. + */ + struct DownloadRequest *prev; + + /** + * Parent in the CHK-tree. + */ + struct DownloadRequest *parent; + + /** + * Array (!) of child-requests, or NULL for the bottom of the tree. + */ + struct DownloadRequest **children; + + /** + * CHK for the request for this block (set during reconstruction + * to what we have on disk, later to what we want to have). + */ + struct ContentHashKey chk; + + /** + * Offset of the corresponding block. Specifically, first (!) byte of + * the first DBLOCK in the subtree induced by block represented by + * this request. + */ + uint64_t offset; + + /** + * Number of entries in 'children' array. + */ + unsigned int num_children; + + /** + * Depth of the corresponding block in the tree. 0==DBLOCKs. + */ + unsigned int depth; + + /** + * State in the FSM. + */ + enum BlockRequestState state; + + /** + * GNUNET_YES if this entry is in the pending list. + */ + int is_pending; + +}; + + +/** + * (recursively) free download request structure + * + * @param dr request to free + */ +void +GNUNET_FS_free_download_request_ (struct DownloadRequest *dr); + + +/** + * Context for controlling a download. + */ +struct GNUNET_FS_DownloadContext +{ + + /** + * Global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Our top-level activity entry (if we are top-level, otherwise NULL). + */ + struct TopLevelActivity *top; + + /** + * Connection to the FS service. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Parent download (used when downloading files + * in directories). + */ + struct GNUNET_FS_DownloadContext *parent; + + /** + * Associated search (used when downloading files + * based on search results), or NULL for none. + */ + struct GNUNET_FS_SearchResult *search; + + /** + * Head of list of child downloads. + */ + struct GNUNET_FS_DownloadContext *child_head; + + /** + * Tail of list of child downloads. + */ + struct GNUNET_FS_DownloadContext *child_tail; + + /** + * Previous download belonging to the same parent. + */ + struct GNUNET_FS_DownloadContext *prev; + + /** + * Next download belonging to the same parent. + */ + struct GNUNET_FS_DownloadContext *next; + + /** + * Context kept for the client. + */ + void *client_info; + + /** + * URI that identifies the file that we are downloading. + */ + struct GNUNET_FS_Uri *uri; + + /** + * Known meta-data for the file (can be NULL). + */ + struct GNUNET_CONTAINER_MetaData *meta; + + /** + * Error message, NULL if we're doing OK. + */ + char *emsg; + + /** + * Random portion of filename we use for syncing state of this + * download. + */ + char *serialization; + + /** + * Where are we writing the data (name of the + * file, can be NULL!). + */ + char *filename; + + /** + * Where are we writing the data temporarily (name of the + * file, can be NULL!); used if we do not have a permanent + * name and we are a directory and we do a recursive download. + */ + char *temp_filename; + + /** + * Our entry in the job queue. + */ + struct GNUNET_FS_QueueEntry *job_queue; + + /** + * Non-NULL if we are currently having a request for + * transmission pending with the client handle. + */ + struct GNUNET_CLIENT_TransmitHandle *th; + + /** + * Tree encoder used for the reconstruction. + */ + struct GNUNET_FS_TreeEncoder *te; + + /** + * File handle for reading data from an existing file + * (to pass to tree encoder). + */ + struct GNUNET_DISK_FileHandle *rfh; + + /** + * Map of active requests (those waiting for a response). The key + * is the hash of the encryped block (aka query). + */ + struct GNUNET_CONTAINER_MultiHashMap *active; + + /** + * Head of linked list of pending requests. + */ + struct DownloadRequest *pending_head; + + /** + * Head of linked list of pending requests. + */ + struct DownloadRequest *pending_tail; + + /** + * Top-level download request. + */ + struct DownloadRequest *top_request; + + /** + * Identity of the peer having the content, or all-zeros + * if we don't know of such a peer. + */ + struct GNUNET_PeerIdentity target; + + /** + * ID of a task that is using this struct and that must be cancelled + * when the download is being stopped (if not + * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some + * artificial delay when trying to reconnect to the FS service or + * the task processing incrementally the data on disk, or the + * task requesting blocks, etc. + */ + GNUNET_SCHEDULER_TaskIdentifier task; + + /** + * What is the first offset that we're interested + * in? + */ + uint64_t offset; + + /** + * How many bytes starting from offset are desired? + * This is NOT the overall length of the file! + */ + uint64_t length; + + /** + * How many bytes have we already received within + * the specified range (DBlocks only). + */ + uint64_t completed; + + /** + * What was the size of the file on disk that we're downloading + * before we started? Used to detect if there is a point in + * checking an existing block on disk for matching the desired + * content. 0 if the file did not exist already. + */ + uint64_t old_file_size; + + /** + * Time download was started. + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Desired level of anonymity. + */ + uint32_t anonymity; + + /** + * The depth of the file-tree. + */ + unsigned int treedepth; + + /** + * Options for the download. + */ + enum GNUNET_FS_DownloadOptions options; + + /** + * Flag set upon transitive completion (includes child downloads). + * This flag is only set to GNUNET_YES for directories where all + * child-downloads have also completed (and signalled completion). + */ + int has_finished; + + /** + * Have we started the receive continuation yet? + */ + int in_receive; + +}; + + +/** + * Information about an (updateable) node in the + * namespace. + */ +struct NamespaceUpdateNode +{ + /** + * Identifier for this node. + */ + char *id; + + /** + * Identifier of children of this node. + */ + char *update; + + /** + * Metadata for this entry. + */ + struct GNUNET_CONTAINER_MetaData *md; + + /** + * URI of this entry in the namespace. + */ + struct GNUNET_FS_Uri *uri; + + /** + * Namespace update generation ID. Used to ensure + * freshness of the tree_id. + */ + unsigned int nug; + + /** + * TREE this entry belongs to (if nug is current). + */ + unsigned int tree_id; + +}; + + +struct GNUNET_FS_Namespace +{ + + /** + * Handle to the FS service context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Array with information about nodes in the namespace. + */ + struct NamespaceUpdateNode **update_nodes; + + /** + * Private key for the namespace. + */ + struct GNUNET_CRYPTO_RsaPrivateKey *key; + + /** + * Hash map mapping identifiers of update nodes + * to the update nodes (initialized on-demand). + */ + struct GNUNET_CONTAINER_MultiHashMap *update_map; + + /** + * Name of the file with the private key. + */ + char *filename; + + /** + * Name of the namespace. + */ + char *name; + + /** + * Size of the update nodes array. + */ + unsigned int update_node_count; + + /** + * Reference counter. + */ + unsigned int rc; + + /** + * Generator for unique nug numbers. + */ + unsigned int nug_gen; +}; + + +#endif + +/* end of fs_api.h */ diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c index f43fe14a7..d5f73623a 100644 --- a/src/fs/fs_directory.c +++ b/src/fs/fs_directory.c @@ -34,7 +34,7 @@ */ #include "platform.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" /** * String that is used to indicate that a file diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index af889d804..09785f6dd 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c @@ -28,7 +28,7 @@ #include "platform.h" #include "gnunet_constants.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #include "fs_tree.h" #define DEBUG_DOWNLOAD GNUNET_EXTRA_LOGGING diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c index bb84b948e..b094ed7d5 100644 --- a/src/fs/fs_file_information.c +++ b/src/fs/fs_file_information.c @@ -30,7 +30,7 @@ #include "platform.h" #include #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #include "fs_tree.h" diff --git a/src/fs/fs_getopt.c b/src/fs/fs_getopt.c index b50d0624b..03747742b 100644 --- a/src/fs/fs_getopt.c +++ b/src/fs/fs_getopt.c @@ -25,7 +25,7 @@ */ #include "platform.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" /* ******************** command-line option parsing API ******************** */ diff --git a/src/fs/fs_list_indexed.c b/src/fs/fs_list_indexed.c index 85e12db32..0896de347 100644 --- a/src/fs/fs_list_indexed.c +++ b/src/fs/fs_list_indexed.c @@ -28,7 +28,7 @@ #include "gnunet_constants.h" #include "gnunet_fs_service.h" #include "gnunet_protocols.h" -#include "fs.h" +#include "fs_api.h" /** diff --git a/src/fs/fs_misc.c b/src/fs/fs_misc.c index 6bd4e59f8..d4afc0d55 100644 --- a/src/fs/fs_misc.c +++ b/src/fs/fs_misc.c @@ -25,7 +25,7 @@ #include "platform.h" #include "gnunet_constants.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" /** diff --git a/src/fs/fs_namespace.c b/src/fs/fs_namespace.c index 47ca6bad1..9d754009b 100644 --- a/src/fs/fs_namespace.c +++ b/src/fs/fs_namespace.c @@ -28,10 +28,22 @@ #include "gnunet_signatures.h" #include "gnunet_util_lib.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #define DEBUG_NAMESPACE GNUNET_EXTRA_LOGGING + +/** + * Maximum legal size for an sblock. + */ +#define MAX_SBLOCK_SIZE (60 * 1024) + +/** + * Maximum legal size for an nblock. + */ +#define MAX_NBLOCK_SIZE (60 * 1024) + + /** * Return the name of the directory in which we store * our local namespaces (or rather, their public keys). diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c index 4f27abae4..9a834ee86 100644 --- a/src/fs/fs_publish.c +++ b/src/fs/fs_publish.c @@ -31,11 +31,16 @@ #include "gnunet_signatures.h" #include "gnunet_util_lib.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #include "fs_tree.h" #define DEBUG_PUBLISH GNUNET_EXTRA_LOGGING +/** + * Maximum legal size for a kblock. + */ +#define MAX_KBLOCK_SIZE (60 * 1024) + /** * Fill in all of the generic fields for diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c index 93a787477..b5152cc66 100644 --- a/src/fs/fs_search.c +++ b/src/fs/fs_search.c @@ -28,10 +28,15 @@ #include "gnunet_constants.h" #include "gnunet_fs_service.h" #include "gnunet_protocols.h" -#include "fs.h" +#include "fs_api.h" #define DEBUG_SEARCH GNUNET_EXTRA_LOGGING +/** + * Number of availability trials we perform per search result. + */ +#define AVAILABILITY_TRIALS_MAX 8 + /** * Fill in all of the generic fields for a search event and * call the callback. diff --git a/src/fs/fs_test_lib.c b/src/fs/fs_test_lib.c index 1c15c6f60..06ab01f8d 100644 --- a/src/fs/fs_test_lib.c +++ b/src/fs/fs_test_lib.c @@ -27,7 +27,7 @@ * @author Christian Grothoff */ #include "platform.h" -#include "fs.h" +#include "fs_api.h" #include "fs_test_lib.h" #include "gnunet_testing_lib.h" diff --git a/src/fs/fs_tree.h b/src/fs/fs_tree.h index 56c32a390..5b1c2024f 100644 --- a/src/fs/fs_tree.h +++ b/src/fs/fs_tree.h @@ -32,7 +32,7 @@ #ifndef GNUNET_FS_TREE_H #define GNUNET_FS_TREE_H -#include "fs.h" +#include "fs_api.h" /** * Compute the depth of the CHK tree. diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c index 8dab62051..79b7e8c73 100644 --- a/src/fs/fs_unindex.c +++ b/src/fs/fs_unindex.c @@ -28,7 +28,7 @@ #include "gnunet_constants.h" #include "gnunet_fs_service.h" #include "gnunet_protocols.h" -#include "fs.h" +#include "fs_api.h" #include "fs_tree.h" #define DEBUG_UNINDEX GNUNET_EXTRA_LOGGING diff --git a/src/fs/fs_uri.c b/src/fs/fs_uri.c index 48fabc1fb..62fd51304 100644 --- a/src/fs/fs_uri.c +++ b/src/fs/fs_uri.c @@ -81,7 +81,7 @@ #include "platform.h" #include "gnunet_fs_service.h" #include "gnunet_signatures.h" -#include "fs.h" +#include "fs_api.h" /** diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c index daffce444..48b7bca29 100644 --- a/src/fs/gnunet-service-fs.c +++ b/src/fs/gnunet-service-fs.c @@ -25,8 +25,6 @@ * * To use: * - consider re-issue GSF_dht_lookup_ after non-DHT reply received - * - implement 'SUPPORT_DELAYS' - * */ #include "platform.h" #include diff --git a/src/fs/gnunet-service-fs.h b/src/fs/gnunet-service-fs.h index e4efbb8f5..7efeedd5f 100644 --- a/src/fs/gnunet-service-fs.h +++ b/src/fs/gnunet-service-fs.h @@ -37,26 +37,13 @@ #define DEBUG_FS_CLIENT GNUNET_EXTRA_LOGGING - /** - * Should we introduce random latency in processing? Required for proper - * implementation of GAP, but can be disabled for performance evaluation of - * the basic routing algorithm. - * - * Note that with delays enabled, performance can be significantly lower - * (several orders of magnitude in 2-peer test runs); if you want to - * measure throughput of other components, set this to NO. Also, you - * might want to consider changing 'RETRY_PROBABILITY_INV' to 1 for - * a rather wasteful mode of operation (that might still get the highest - * throughput overall). - * - * Performance measurements (for 50 MB file, 2 peers): - * - * - Without delays: 3300 kb/s - * - With delays: 101 kb/s + * By which amount do we decrement the TTL for simple forwarding / + * indirection of the query; in milli-seconds. Set somewhat in + * accordance to your network latency (above the time it'll take you + * to send a packet and get a reply). */ -#define SUPPORT_DELAYS GNUNET_NO - +#define TTL_DECREMENT 5000 /** * At what frequency should our datastore load decrease diff --git a/src/fs/gnunet-service-fs_cp.c b/src/fs/gnunet-service-fs_cp.c index d4c81d509..9d2385378 100644 --- a/src/fs/gnunet-service-fs_cp.c +++ b/src/fs/gnunet-service-fs_cp.c @@ -32,6 +32,14 @@ #include "gnunet-service-fs_pr.h" #include "gnunet-service-fs_push.h" + +/** + * Ratio for moving average delay calculation. The previous + * average goes in with a factor of (n-1) into the calculation. + * Must be > 0. + */ +#define RUNAVG_DELAY_N 16 + /** * How often do we flush trust values to disk? */ diff --git a/src/fs/gnunet-service-fs_cp.h b/src/fs/gnunet-service-fs_cp.h index a2cc5232c..11be4c6f1 100644 --- a/src/fs/gnunet-service-fs_cp.h +++ b/src/fs/gnunet-service-fs_cp.h @@ -30,6 +30,36 @@ #include "gnunet-service-fs.h" +/** + * Maximum number of outgoing messages we queue per peer. + * + * Performance measurements for 2 peer setup for 50 MB file + * (with MAX_DATASTORE_QUEUE = 1 and RETRY_PROBABILITY_INV = 1): + * + * 2: 1700 kb/s, 1372 kb/s + * 8: 2117 kb/s, 1284 kb/s, 1112 kb/s + * 16: 3500 kb/s, 3200 kb/s, 3388 kb/s + * 32: 3441 kb/s, 3163 kb/s, 3277 kb/s + * 128: 1700 kb/s; 2010 kb/s, 3383 kb/s, 1156 kb/s + * + * Conclusion: 16 seems to be a pretty good value (stable + * and high performance, no excessive memory use). + */ +#define MAX_QUEUE_PER_PEER 16 + +/** + * Length of the P2P success tracker. Note that having a very long + * list can also hurt performance. + */ +#define P2P_SUCCESS_LIST_SIZE 8 + +/** + * Length of the CS-2-P success tracker. Note that + * having a very long list can also hurt performance. + */ +#define CS2P_SUCCESS_LIST_SIZE 8 + + /** * Performance data kept for a peer. */ diff --git a/src/fs/gnunet-service-fs_pr.c b/src/fs/gnunet-service-fs_pr.c index ac183a980..3b43b4c5f 100644 --- a/src/fs/gnunet-service-fs_pr.c +++ b/src/fs/gnunet-service-fs_pr.c @@ -31,6 +31,21 @@ #include "gnunet-service-fs_pe.h" #include "gnunet-service-fs_pr.h" +/** + * Maximum size of the datastore queue for P2P operations. Needs to + * be large enough to queue MAX_QUEUE_PER_PEER operations for roughly + * the number of active (connected) peers. + */ +#define MAX_DATASTORE_QUEUE (16 * MAX_QUEUE_PER_PEER) + +/** + * Bandwidth value of a 0-priority content (must be fairly high + * compared to query since content is typically significantly larger + * -- and more valueable since it can take many queries to get one + * piece of content). + */ +#define CONTENT_BANDWIDTH_VALUE 800 + /** * Hard limit on the number of results we may get from the datastore per query. */ diff --git a/src/fs/gnunet-service-fs_push.c b/src/fs/gnunet-service-fs_push.c index 515446117..4fde823a1 100644 --- a/src/fs/gnunet-service-fs_push.c +++ b/src/fs/gnunet-service-fs_push.c @@ -33,6 +33,17 @@ #define DEBUG_FS_MIGRATION GNUNET_EXTRA_LOGGING +/** + * Maximum number of blocks we keep in memory for migration. + */ +#define MAX_MIGRATION_QUEUE 8 + +/** + * Blocks are at most migrated to this number of peers + * plus one, each time they are fetched from the database. + */ +#define MIGRATION_LIST_SIZE 2 + /** * How long must content remain valid for us to consider it for migration? * If content will expire too soon, there is clearly no point in pushing diff --git a/src/fs/test_fs_directory.c b/src/fs/test_fs_directory.c index ac581bd6f..81012e764 100644 --- a/src/fs/test_fs_directory.c +++ b/src/fs/test_fs_directory.c @@ -28,7 +28,7 @@ #include #include "gnunet_util_lib.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #define ABORT() { fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); return 1; } diff --git a/src/fs/test_fs_uri.c b/src/fs/test_fs_uri.c index 693bff1d6..72744351b 100644 --- a/src/fs/test_fs_uri.c +++ b/src/fs/test_fs_uri.c @@ -27,7 +27,7 @@ #include "platform.h" #include "gnunet_util_lib.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #define ABORT() { fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); return 1; } -- cgit v1.2.3