From d83a3d2ac2a33d43239bbd5703574872081ba81f Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 28 Apr 2010 12:40:41 +0000 Subject: added option to restrict search to local-only --- TODO | 5 ++--- doc/man/gnunet-download.1 | 17 +++++++++++++++++ doc/man/gnunet-search.1 | 6 +++++- src/fs/fs.h | 12 ++++++++---- src/fs/fs_download.c | 4 ++++ src/fs/fs_search.c | 18 +++++++++++++++++- src/fs/gnunet-download.c | 6 ++++++ src/fs/gnunet-search.c | 10 ++++++++++ src/fs/gnunet-service-fs.c | 13 ++++++++++++- src/fs/test_fs_namespace.c | 4 ++-- src/fs/test_fs_search.c | 1 + src/include/gnunet_fs_service.h | 30 ++++++++++++++++++++++++++++-- 12 files changed, 112 insertions(+), 14 deletions(-) diff --git a/TODO b/TODO index fb7326dd1..e65b3b940 100644 --- a/TODO +++ b/TODO @@ -1,8 +1,5 @@ 0.9.0pre1: * FS: [CG] - - Allow checking of presence of search results and/or content via command-line tools - (add options to gnunet-search / gnunet-download to limit search to local peer) - [needed for full persistence support...] - persistence support (publish, unindex, search, download) - gnunet-service-fs (hot-path routing, load-based routing, nitpicks) - [gnunet-service-fs.c:208]: member 'LocalGetContext::results_bf_size' is never used @@ -174,6 +171,8 @@ Optimizations: (theoretically reduces overhead; bounds message queue size) - merge multiple iteration requests over "all" peers in the queue (theoretically reduces overhead; bounds messgae queue size) +* FS: + - use different queue prioritization for probe-downloads vs. normal downloads (!?) Minor features: * TCP: diff --git a/doc/man/gnunet-download.1 b/doc/man/gnunet-download.1 index b0ad09b73..2da931554 100644 --- a/doc/man/gnunet-download.1 +++ b/doc/man/gnunet-download.1 @@ -7,43 +7,60 @@ gnunet\-download \- a command line interface for downloading files from GNUnet .SH DESCRIPTION .PP Download files from GNUnet. + .TP \fB\-a \fILEVEL\fR, \fB\-\-anonymity=LEVEL\fR set desired level of receiver anonymity. Default is 1. + .TP \fB\-c \fIFILENAME\fR, \fB\-\-config=FILENAME\fR use config file (defaults: ~/.gnunet/gnunet.conf) + .TP \fB\-D, \fB\-\-delete\-incomplete\fR causes gnunet\-download to delete incomplete downloads when aborted with CTRL\-C. Note that complete files that are part of an incomplete recursive download will not be deleted even with this option. Without this option, terminating gnunet\-download with a signal will cause incomplete downloads to stay on disk. If gnunet\-download runs to (normal) completion finishing the download, this option has no effect. + .TP \fB\-h\fR, \fB\-\-help\fR print help page + .TP \fB\-H \fIHOSTNAME\fR, \fB\-\-host=\fIHOSTNAME\fR on which host is gnunetd running (default: localhost). You can also specify a port using the syntax HOSTNAME:PORT. The default port is 2087. + .TP \fB\-L \fILOGLEVEL\fR, \fB\-\-loglevel=LOGLEVEL\fR Change the loglevel. Possible values for LOGLEVEL are ERROR, WARNING, INFO and DEBUG. + +.TP +\fB\-n\fR, \fB\-\-no-network\fR +Only search locally, do not forward requests to other peers. + .TP \fB\-o \fIFILENAME\fR, \fB\-\-output=FILENAME\fR write the file to FILENAME. Hint: when recursively downloading a directory, append a '/' to the end of the FILENAME to create a directory of that name. If no FILENAME is specified, gnunet\-download constructs a temporary ID from the URI of the file. The final filename is constructed based on meta\-data extracted using libextractor (if available). + .TP \fB\-p \fIDOWNLOADS\fR, \fB\-\-parallelism=DOWNLOADS\fR set the maximum number of parallel downloads that is allowed. More parallel downloads can, to some extent, improve the overall time to download content. However, parallel downloads also take more memory (see also option \-r which can be used to limit memory utilization) and more sockets. GNUnet typically only supports 1024 sockets per process, and each parallel download requires a new socket. Hence it is not possible to run more than about 1000 downloads in parallel. This option is used to limit the number of files that are downloaded in parallel (\-r can be used to limit the number of blocks that are concurrently requested). As a result, the value only matters for recursive downloads. The default value is 32. Note that the overall limit of 1000 downloads applies to gnunet\-service\-fs as well and using multiple gnunet\-download processes hence does not increase this limit. + .TP \fB\-r \fIREQUESTS\fR, \fB\-\-request-parallelism=REQUESTS\fR set the maximum number of parallel requests that is allowed. If multiple files are downloaded, gnunet\-download will not run them in parallel if this would cause the number of pending requests to possibly exceed the given value. This is useful since, for example, downloading dozens of multi\-gigabyte files in parallel could exhaust memory resources and would hardly improve performance. Note that the limit only applies to this specific process and that other download activities by other processes are not included in this limit. Consider raising this limit for large recursive downloads with many large files if memory and network bandwidth are not fully utilized and if the parallelism limit (\-p option) is not reached. This option also only matters for recursive downloads. The default value is 4092. + .TP \fB\-R\fR, \fB\-\-recursive\fR download directories recursively (and in parallel); note that the URI must belong to a GNUnet directory and that the filename given must end with a '/' \-\- otherwise, only the file corresponding to the URI will be downloaded. Note that in addition to using '-R', you must also specify a filename ending in '.gnd' so that the code realizes that the top-level file is a directory (since we have no meta data). + .TP \fB\-v\fR, \fB\-\-version\fR print the version number + .TP \fB\-V\fR, \fB\-\-verbose\fR print progress information + .SH NOTES The GNUNET_URI is typically obtained from gnunet\-search. gnunet\-gtk can also be used instead of gnunet\-download. If you ever have to abort a download, you can at any time continue it by re\-issuing gnunet\-download with the same filename. In that case GNUnet will not download blocks again that are already present. GNUnets file\-encoding will ensure file integrity, even if the existing file was not downloaded from GNUnet in the first place. Temporary information will be appended to the target file until the download is completed. diff --git a/doc/man/gnunet-search.1 b/doc/man/gnunet-search.1 index 5eefaa099..526097733 100644 --- a/doc/man/gnunet-search.1 +++ b/doc/man/gnunet-search.1 @@ -1,4 +1,4 @@ -.TH GNUNET-SEARCH "1" "6 Sep 2009" "GNUnet" +.TH GNUNET-SEARCH "1" "28 Apr 2010" "GNUnet" .SH NAME gnunet\-search \- a command line interface to search for content on GNUnet .SH SYNOPSIS @@ -37,6 +37,10 @@ ERROR, WARNING, INFO and DEBUG. \fB\-o \fIFILENAME\fR, \fB\-\-output=\fIFILENAME\fR Writes a GNUnet directory containing all of the search results to FILENAME. +.TP +\fB\-n\fR, \fB\-\-no-network\fR +Only search locally, do not forward requests to other peers. + .TP \fB\-v\fR, \fB\-\-version\fR print the version number diff --git a/src/fs/fs.h b/src/fs/fs.h index 0a656e1e9..a6a12b9cd 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h @@ -1079,7 +1079,10 @@ struct GNUNET_FS_SearchContext */ uint32_t mandatory_count; - + /** + * Options for the search. + */ + enum GNUNET_FS_SearchOptions options; }; @@ -1439,9 +1442,10 @@ struct SearchMessage struct GNUNET_MessageHeader header; /** - * Should be zero. + * Bitmask with options. Zero for no options, one for loopback-only. + * Other bits are currently not defined. */ - int32_t reserved GNUNET_PACKED; + int32_t options GNUNET_PACKED; /** * Type of the content that we're looking for. @@ -1536,7 +1540,7 @@ struct GetMessage uint32_t priority GNUNET_PACKED; /** - * Relative time to live in GNUNET_CRON_MILLISECONDS (network byte order) + * Relative time to live in MILLISECONDS (network byte order) */ int32_t ttl GNUNET_PACKED; diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index 14e6266f8..fe9c18aa1 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c @@ -1150,6 +1150,10 @@ transmit_download_request (void *cls, memset (sm, 0, sizeof (struct SearchMessage)); sm->header.size = htons (sizeof (struct SearchMessage)); sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH); + if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY)) + sm->options = htonl (1); + else + sm->options = htonl (0); if (dc->pending->depth == dc->treedepth) sm->type = htonl (GNUNET_BLOCK_TYPE_DBLOCK); else diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c index 2c8022237..efe5bf34a 100644 --- a/src/fs/fs_search.c +++ b/src/fs/fs_search.c @@ -458,6 +458,7 @@ process_ksk_result (struct GNUNET_FS_SearchContext *sc, * @param uri specifies the search parameters; can be * a KSK URI or an SKS URI. * @param anonymity desired level of anonymity + * @param options options for the search * @param cctx client context * @param parent parent search (for namespace update searches) * @return context that can be used to control the search @@ -466,6 +467,7 @@ static struct GNUNET_FS_SearchContext * search_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, uint32_t anonymity, + enum GNUNET_FS_SearchOptions options, void *cctx, struct GNUNET_FS_SearchContext *parent); @@ -523,6 +525,7 @@ process_sks_result (struct GNUNET_FS_SearchContext *sc, search_start (sc->h, &uu, sc->anonymity, + sc->options, NULL, sc); } @@ -938,6 +941,10 @@ transmit_search_request (void *cls, { sm[i].header.size = htons (sizeof (struct SearchMessage)); sm[i].header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH); + if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY)) + sm[i].options = htonl (1); + else + sm[i].options = htonl (0); sm[i].type = htonl (GNUNET_BLOCK_TYPE_ANY); sm[i].anonymity_level = htonl (sc->anonymity); sm[i].query = sc->requests[i].query; @@ -952,6 +959,10 @@ transmit_search_request (void *cls, memset (sm, 0, msize); sm->header.size = htons (sizeof (struct SearchMessage)); sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH); + if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY)) + sm->options = htonl (1); + else + sm->options = htonl (0); sm->type = htonl (GNUNET_BLOCK_TYPE_SBLOCK); sm->anonymity_level = htonl (sc->anonymity); sm->target = sc->uri->data.sks.namespace; @@ -1042,6 +1053,7 @@ try_reconnect (struct GNUNET_FS_SearchContext *sc) * @param uri specifies the search parameters; can be * a KSK URI or an SKS URI. * @param anonymity desired level of anonymity + * @param options options for the search * @param cctx initial value for the client context * @param parent parent search (for namespace update searches) * @return context that can be used to control the search @@ -1050,6 +1062,7 @@ static struct GNUNET_FS_SearchContext * search_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, uint32_t anonymity, + enum GNUNET_FS_SearchOptions options, void *cctx, struct GNUNET_FS_SearchContext *parent) { @@ -1085,6 +1098,7 @@ search_start (struct GNUNET_FS_Handle *h, return NULL; sc = GNUNET_malloc (sizeof(struct GNUNET_FS_SearchContext)); sc->h = h; + sc->options = options; sc->uri = GNUNET_FS_uri_dup (uri); sc->anonymity = anonymity; sc->start_time = GNUNET_TIME_absolute_get (); @@ -1139,6 +1153,7 @@ search_start (struct GNUNET_FS_Handle *h, * @param uri specifies the search parameters; can be * a KSK URI or an SKS URI. * @param anonymity desired level of anonymity + * @param options options for the search * @param cctx initial value for the client context * @return context that can be used to control the search */ @@ -1146,9 +1161,10 @@ struct GNUNET_FS_SearchContext * GNUNET_FS_search_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, uint32_t anonymity, + enum GNUNET_FS_SearchOptions options, void *cctx) { - return search_start (h, uri, anonymity, cctx, NULL); + return search_start (h, uri, anonymity, options, cctx, NULL); } diff --git a/src/fs/gnunet-download.c b/src/fs/gnunet-download.c index bc98d0275..fde254d7c 100644 --- a/src/fs/gnunet-download.c +++ b/src/fs/gnunet-download.c @@ -52,6 +52,7 @@ static int do_recursive; static char *filename; +static int local_only; static void cleanup_task (void *cls, @@ -225,6 +226,8 @@ run (void *cls, options = GNUNET_FS_DOWNLOAD_OPTION_NONE; if (do_recursive) options |= GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE; + if (local_only) + options |= GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY; dc = GNUNET_FS_download_start (ctx, uri, NULL, @@ -259,6 +262,9 @@ static struct GNUNET_GETOPT_CommandLineOption options[] = { {'D', "delete-incomplete", NULL, gettext_noop ("delete incomplete downloads (when aborted with CTRL-C)"), 0, &GNUNET_GETOPT_set_one, &delete_incomplete}, + {'n', "no-network", NULL, + gettext_noop ("only search the local peer (no P2P network search)"), + 1, &GNUNET_GETOPT_set_uint, &local_only}, {'o', "output", "FILENAME", gettext_noop ("write the file to FILENAME"), 1, &GNUNET_GETOPT_set_string, &filename}, diff --git a/src/fs/gnunet-search.c b/src/fs/gnunet-search.c index 64955a3ee..0226a340e 100644 --- a/src/fs/gnunet-search.c +++ b/src/fs/gnunet-search.c @@ -46,6 +46,8 @@ static unsigned int anonymity = 1; static int verbose; +static int local_only; + /** * Type of a function that libextractor calls for each * meta data item found. @@ -231,6 +233,7 @@ run (void *cls, { struct GNUNET_FS_Uri *uri; unsigned int argc; + enum GNUNET_FS_SearchOptions options; sched = s; argc = 0; @@ -266,9 +269,13 @@ run (void *cls, } if (output_filename != NULL) db = GNUNET_FS_directory_builder_create (NULL); + options = GNUNET_FS_SEARCH_OPTION_NONE; + if (local_only) + options |= GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY; sc = GNUNET_FS_search_start (ctx, uri, anonymity, + options, NULL); GNUNET_FS_uri_destroy (uri); if (NULL == sc) @@ -293,6 +300,9 @@ static struct GNUNET_GETOPT_CommandLineOption options[] = { {'a', "anonymity", "LEVEL", gettext_noop ("set the desired LEVEL of receiver-anonymity"), 1, &GNUNET_GETOPT_set_uint, &anonymity}, + {'n', "no-network", NULL, + gettext_noop ("only search the local peer (no P2P network search)"), + 1, &GNUNET_GETOPT_set_uint, &local_only}, {'o', "output", "PREFIX", gettext_noop ("write search results to file starting with PREFIX"), diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c index 41be75e98..84a4f1d65 100644 --- a/src/fs/gnunet-service-fs.c +++ b/src/fs/gnunet-service-fs.c @@ -545,7 +545,12 @@ struct PendingRequest /** * Remove this request after transmission of the current response. */ - int do_remove; + int16_t do_remove; + + /** + * GNUNET_YES if we should not forward this request to other peers. + */ + int16_t local_only; }; @@ -1600,6 +1605,8 @@ forward_request_task (void *cls, #endif return; /* already pending */ } + if (GNUNET_YES == pr->local_only) + return; /* configured to not do P2P search */ /* (1) select target */ psc.pr = pr; psc.target_score = DBL_MIN; @@ -2783,6 +2790,10 @@ handle_start_search (void *cls, pr->anonymity_level = ntohl (sm->anonymity_level); refresh_bloomfilter (pr); pr->query = sm->query; + if (0 == (1 & ntohl (sm->options))) + pr->local_only = GNUNET_NO; + else + pr->local_only = GNUNET_YES; switch (type) { case GNUNET_BLOCK_TYPE_DBLOCK: diff --git a/src/fs/test_fs_namespace.c b/src/fs/test_fs_namespace.c index b57f19bc9..b154ce799 100644 --- a/src/fs/test_fs_namespace.c +++ b/src/fs/test_fs_namespace.c @@ -250,8 +250,8 @@ publish_cont (void *cls, GNUNET_free (msg); return; } - ksk_search = GNUNET_FS_search_start (fs, ksk_uri, 1, "ksk_search"); - sks_search = GNUNET_FS_search_start (fs, sks_uri, 1, "sks_search"); + ksk_search = GNUNET_FS_search_start (fs, ksk_uri, 1, GNUNET_FS_SEARCH_OPTION_NONE, "ksk_search"); + sks_search = GNUNET_FS_search_start (fs, sks_uri, 1, GNUNET_FS_SEARCH_OPTION_NONE, "sks_search"); GNUNET_FS_uri_destroy (sks_uri); } diff --git a/src/fs/test_fs_search.c b/src/fs/test_fs_search.c index 5f5f7f62a..32f885814 100644 --- a/src/fs/test_fs_search.c +++ b/src/fs/test_fs_search.c @@ -115,6 +115,7 @@ progress_cb (void *cls, search = GNUNET_FS_search_start (fs, kuri, 1, + GNUNET_FS_SEARCH_OPTION_NONE, "search"); GNUNET_FS_uri_destroy (kuri); GNUNET_assert (search != NULL); diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h index e3c99f3e9..acd059c96 100644 --- a/src/include/gnunet_fs_service.h +++ b/src/include/gnunet_fs_service.h @@ -2192,6 +2192,25 @@ GNUNET_FS_namespace_list_updateable (struct GNUNET_FS_Namespace *namespace, void *ip_cls); +/** + * Options for searching. Compatible options + * can be OR'ed together. + */ +enum GNUNET_FS_SearchOptions + { + /** + * No options (use defaults for everything). + */ + GNUNET_FS_SEARCH_OPTION_NONE = 0, + + /** + * Only search the local host, do not search remote systems (no P2P) + */ + GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY = 1 + + }; + + /** * Start search for content. * @@ -2199,6 +2218,7 @@ GNUNET_FS_namespace_list_updateable (struct GNUNET_FS_Namespace *namespace, * @param uri specifies the search parameters; can be * a KSK URI or an SKS URI. * @param anonymity desired level of anonymity + * @param options options for the search * @param cctx initial value for the client context * @return context that can be used to control the search */ @@ -2206,6 +2226,7 @@ struct GNUNET_FS_SearchContext * GNUNET_FS_search_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, uint32_t anonymity, + enum GNUNET_FS_SearchOptions options, void *cctx); @@ -2248,18 +2269,23 @@ enum GNUNET_FS_DownloadOptions * No options (use defaults for everything). */ GNUNET_FS_DOWNLOAD_OPTION_NONE = 0, + + /** + * Only download from the local host, do not access remote systems (no P2P) + */ + GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY = 1, /** * Do a recursive download (that is, automatically trigger the * download of files in directories). */ - GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE = 1, + GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE = 2, /** * Do not append temporary data to * the target file (for the IBlocks). */ - GNUNET_FS_DOWNLOAD_NO_TEMPORARIES = 2, + GNUNET_FS_DOWNLOAD_NO_TEMPORARIES = 4, /** * Internal option used to flag this download as a 'probe' for a -- cgit v1.2.3