From 579d9473bb75072303789599b23be9b0203336fc Mon Sep 17 00:00:00 2001 From: Martin Schanzenbach Date: Thu, 19 Oct 2023 11:55:21 +0200 Subject: BUILD: Move fs to contrib/service --- src/fs/.gitignore | 43 - src/fs/Makefile.am | 531 ---- src/fs/fs.conf.in | 63 - src/fs/fs.h | 397 --- src/fs/fs_api.c | 3321 --------------------- src/fs/fs_api.h | 1941 ------------ src/fs/fs_directory.c | 676 ----- src/fs/fs_dirmetascan.c | 496 --- src/fs/fs_download.c | 2339 --------------- src/fs/fs_file_information.c | 407 --- src/fs/fs_getopt.c | 274 -- src/fs/fs_list_indexed.c | 219 -- src/fs/fs_misc.c | 165 - src/fs/fs_namespace.c | 804 ----- src/fs/fs_publish.c | 1625 ---------- src/fs/fs_publish_ksk.c | 255 -- src/fs/fs_publish_ublock.c | 299 -- src/fs/fs_publish_ublock.h | 109 - src/fs/fs_search.c | 1826 ----------- src/fs/fs_sharetree.c | 457 --- src/fs/fs_test_lib.c | 630 ---- src/fs/fs_test_lib.h | 102 - src/fs/fs_test_lib_data.conf | 17 - src/fs/fs_tree.c | 452 --- src/fs/fs_tree.h | 217 -- src/fs/fs_unindex.c | 902 ------ src/fs/fs_uri.c | 2045 ------------- src/fs/gnunet-auto-share.c | 791 ----- src/fs/gnunet-daemon-fsprofiler.c | 673 ----- src/fs/gnunet-directory.c | 212 -- src/fs/gnunet-download.c | 385 --- src/fs/gnunet-fs-profiler.c | 245 -- src/fs/gnunet-fs.c | 191 -- src/fs/gnunet-helper-fs-publish.c | 579 ---- src/fs/gnunet-publish.c | 1009 ------- src/fs/gnunet-search.c | 801 ----- src/fs/gnunet-service-fs.c | 1378 --------- src/fs/gnunet-service-fs.h | 304 -- src/fs/gnunet-service-fs_cadet.h | 168 -- src/fs/gnunet-service-fs_cadet_client.c | 728 ----- src/fs/gnunet-service-fs_cadet_server.c | 545 ---- src/fs/gnunet-service-fs_cp.c | 1659 ---------- src/fs/gnunet-service-fs_cp.h | 415 --- src/fs/gnunet-service-fs_indexing.c | 495 --- src/fs/gnunet-service-fs_indexing.h | 120 - src/fs/gnunet-service-fs_pe.c | 814 ----- src/fs/gnunet-service-fs_pe.h | 95 - src/fs/gnunet-service-fs_pr.c | 1887 ------------ src/fs/gnunet-service-fs_pr.h | 422 --- src/fs/gnunet-service-fs_push.c | 672 ----- src/fs/gnunet-service-fs_push.h | 66 - src/fs/gnunet-service-fs_put.c | 290 -- src/fs/gnunet-service-fs_put.h | 46 - src/fs/gnunet-unindex.c | 206 -- src/fs/meson.build | 141 - src/fs/meta_data.c | 1229 -------- src/fs/perf_gnunet_service_fs_p2p.c | 370 --- src/fs/perf_gnunet_service_fs_p2p.conf | 7 - src/fs/perf_gnunet_service_fs_p2p_respect.c | 480 --- src/fs/plugin_block_fs.c | 337 --- src/fs/test_fs.c | 262 -- src/fs/test_fs_data.conf | 7 - src/fs/test_fs_defaults.conf | 47 - src/fs/test_fs_directory.c | 186 -- src/fs/test_fs_download.c | 368 --- src/fs/test_fs_download_data.conf | 10 - src/fs/test_fs_download_indexed.conf | 10 - src/fs/test_fs_download_persistence.c | 351 --- src/fs/test_fs_file_information.c | 163 - src/fs/test_fs_file_information_data.conf | 7 - src/fs/test_fs_getopt.c | 37 - src/fs/test_fs_list_indexed.c | 265 -- src/fs/test_fs_list_indexed_data.conf | 10 - src/fs/test_fs_meta_data.c | 492 --- src/fs/test_fs_namespace.c | 320 -- src/fs/test_fs_namespace_data.conf | 7 - src/fs/test_fs_namespace_list_updateable.c | 175 -- src/fs/test_fs_publish.c | 251 -- src/fs/test_fs_publish_data.conf | 10 - src/fs/test_fs_publish_persistence.c | 322 -- src/fs/test_fs_search.c | 252 -- src/fs/test_fs_search_data.conf | 7 - src/fs/test_fs_search_persistence.c | 318 -- src/fs/test_fs_search_probes.c | 258 -- src/fs/test_fs_search_with_and.c | 272 -- src/fs/test_fs_start_stop.c | 64 - src/fs/test_fs_test_lib.c | 181 -- src/fs/test_fs_unindex.c | 237 -- src/fs/test_fs_unindex_data.conf | 7 - src/fs/test_fs_unindex_persistence.c | 307 -- src/fs/test_fs_uri.c | 340 --- src/fs/test_gnunet_fs_idx.py.in | 113 - src/fs/test_gnunet_fs_idx_data.conf | 7 - src/fs/test_gnunet_fs_psd.py.in | 149 - src/fs/test_gnunet_fs_psd_data.conf | 7 - src/fs/test_gnunet_fs_rec.py.in | 171 -- src/fs/test_gnunet_fs_rec_data.conf | 7 - src/fs/test_gnunet_fs_rec_data.tgz | Bin 17822 -> 0 bytes src/fs/test_gnunet_service_fs_migration.c | 223 -- src/fs/test_gnunet_service_fs_migration_data.conf | 10 - src/fs/test_gnunet_service_fs_p2p.c | 167 -- src/fs/test_gnunet_service_fs_p2p_cadet.conf | 20 - src/fs/test_plugin_block_fs.c | 86 - src/fs/test_pseudonym_data.conf | 6 - 104 files changed, 44881 deletions(-) delete mode 100644 src/fs/.gitignore delete mode 100644 src/fs/Makefile.am delete mode 100644 src/fs/fs.conf.in delete mode 100644 src/fs/fs.h delete mode 100644 src/fs/fs_api.c delete mode 100644 src/fs/fs_api.h delete mode 100644 src/fs/fs_directory.c delete mode 100644 src/fs/fs_dirmetascan.c delete mode 100644 src/fs/fs_download.c delete mode 100644 src/fs/fs_file_information.c delete mode 100644 src/fs/fs_getopt.c delete mode 100644 src/fs/fs_list_indexed.c delete mode 100644 src/fs/fs_misc.c delete mode 100644 src/fs/fs_namespace.c delete mode 100644 src/fs/fs_publish.c delete mode 100644 src/fs/fs_publish_ksk.c delete mode 100644 src/fs/fs_publish_ublock.c delete mode 100644 src/fs/fs_publish_ublock.h delete mode 100644 src/fs/fs_search.c delete mode 100644 src/fs/fs_sharetree.c delete mode 100644 src/fs/fs_test_lib.c delete mode 100644 src/fs/fs_test_lib.h delete mode 100644 src/fs/fs_test_lib_data.conf delete mode 100644 src/fs/fs_tree.c delete mode 100644 src/fs/fs_tree.h delete mode 100644 src/fs/fs_unindex.c delete mode 100644 src/fs/fs_uri.c delete mode 100644 src/fs/gnunet-auto-share.c delete mode 100644 src/fs/gnunet-daemon-fsprofiler.c delete mode 100644 src/fs/gnunet-directory.c delete mode 100644 src/fs/gnunet-download.c delete mode 100644 src/fs/gnunet-fs-profiler.c delete mode 100644 src/fs/gnunet-fs.c delete mode 100644 src/fs/gnunet-helper-fs-publish.c delete mode 100644 src/fs/gnunet-publish.c delete mode 100644 src/fs/gnunet-search.c delete mode 100644 src/fs/gnunet-service-fs.c delete mode 100644 src/fs/gnunet-service-fs.h delete mode 100644 src/fs/gnunet-service-fs_cadet.h delete mode 100644 src/fs/gnunet-service-fs_cadet_client.c delete mode 100644 src/fs/gnunet-service-fs_cadet_server.c delete mode 100644 src/fs/gnunet-service-fs_cp.c delete mode 100644 src/fs/gnunet-service-fs_cp.h delete mode 100644 src/fs/gnunet-service-fs_indexing.c delete mode 100644 src/fs/gnunet-service-fs_indexing.h delete mode 100644 src/fs/gnunet-service-fs_pe.c delete mode 100644 src/fs/gnunet-service-fs_pe.h delete mode 100644 src/fs/gnunet-service-fs_pr.c delete mode 100644 src/fs/gnunet-service-fs_pr.h delete mode 100644 src/fs/gnunet-service-fs_push.c delete mode 100644 src/fs/gnunet-service-fs_push.h delete mode 100644 src/fs/gnunet-service-fs_put.c delete mode 100644 src/fs/gnunet-service-fs_put.h delete mode 100644 src/fs/gnunet-unindex.c delete mode 100644 src/fs/meson.build delete mode 100644 src/fs/meta_data.c delete mode 100644 src/fs/perf_gnunet_service_fs_p2p.c delete mode 100644 src/fs/perf_gnunet_service_fs_p2p.conf delete mode 100644 src/fs/perf_gnunet_service_fs_p2p_respect.c delete mode 100644 src/fs/plugin_block_fs.c delete mode 100644 src/fs/test_fs.c delete mode 100644 src/fs/test_fs_data.conf delete mode 100644 src/fs/test_fs_defaults.conf delete mode 100644 src/fs/test_fs_directory.c delete mode 100644 src/fs/test_fs_download.c delete mode 100644 src/fs/test_fs_download_data.conf delete mode 100644 src/fs/test_fs_download_indexed.conf delete mode 100644 src/fs/test_fs_download_persistence.c delete mode 100644 src/fs/test_fs_file_information.c delete mode 100644 src/fs/test_fs_file_information_data.conf delete mode 100644 src/fs/test_fs_getopt.c delete mode 100644 src/fs/test_fs_list_indexed.c delete mode 100644 src/fs/test_fs_list_indexed_data.conf delete mode 100644 src/fs/test_fs_meta_data.c delete mode 100644 src/fs/test_fs_namespace.c delete mode 100644 src/fs/test_fs_namespace_data.conf delete mode 100644 src/fs/test_fs_namespace_list_updateable.c delete mode 100644 src/fs/test_fs_publish.c delete mode 100644 src/fs/test_fs_publish_data.conf delete mode 100644 src/fs/test_fs_publish_persistence.c delete mode 100644 src/fs/test_fs_search.c delete mode 100644 src/fs/test_fs_search_data.conf delete mode 100644 src/fs/test_fs_search_persistence.c delete mode 100644 src/fs/test_fs_search_probes.c delete mode 100644 src/fs/test_fs_search_with_and.c delete mode 100644 src/fs/test_fs_start_stop.c delete mode 100644 src/fs/test_fs_test_lib.c delete mode 100644 src/fs/test_fs_unindex.c delete mode 100644 src/fs/test_fs_unindex_data.conf delete mode 100644 src/fs/test_fs_unindex_persistence.c delete mode 100644 src/fs/test_fs_uri.c delete mode 100755 src/fs/test_gnunet_fs_idx.py.in delete mode 100644 src/fs/test_gnunet_fs_idx_data.conf delete mode 100755 src/fs/test_gnunet_fs_psd.py.in delete mode 100644 src/fs/test_gnunet_fs_psd_data.conf delete mode 100755 src/fs/test_gnunet_fs_rec.py.in delete mode 100644 src/fs/test_gnunet_fs_rec_data.conf delete mode 100644 src/fs/test_gnunet_fs_rec_data.tgz delete mode 100644 src/fs/test_gnunet_service_fs_migration.c delete mode 100644 src/fs/test_gnunet_service_fs_migration_data.conf delete mode 100644 src/fs/test_gnunet_service_fs_p2p.c delete mode 100644 src/fs/test_gnunet_service_fs_p2p_cadet.conf delete mode 100644 src/fs/test_plugin_block_fs.c delete mode 100644 src/fs/test_pseudonym_data.conf (limited to 'src/fs') diff --git a/src/fs/.gitignore b/src/fs/.gitignore deleted file mode 100644 index f0e2a4f7b..000000000 --- a/src/fs/.gitignore +++ /dev/null @@ -1,43 +0,0 @@ -gnunet-unindex -gnunet-auto-share -gnunet-daemon-fsprofiler -gnunet-directory -gnunet-download -gnunet-fs -gnunet-fs-profiler -gnunet-helper-fs-publish -gnunet-publish -gnunet-search -gnunet-service-fs -test_fs_directory -test_fs_download -test_fs_download_cadet -test_fs_download_indexed -test_fs_download_persistence -test_fs_file_information -test_fs_getopt -test_fs_list_indexed -test_fs_namespace -test_fs_namespace_list_updateable -test_fs_publish -test_fs_publish_persistence -test_fs_search -test_fs_search_persistence -test_fs_search_probes -test_fs_search_with_and -test_fs_start_stop -test_fs_test_lib -test_fs_unindex -test_fs_unindex_persistence -test_fs_uri -test_gnunet_fs_idx.py -test_gnunet_fs_psd.py -test_gnunet_fs_rec.py -test_gnunet_service_fs_migration -test_gnunet_service_fs_p2p -test_gnunet_service_fs_p2p_cadet -test_plugin_block_fs -perf_gnunet_service_fs_p2p -perf_gnunet_service_fs_p2p_index -perf_gnunet_service_fs_p2p_respect -rdir.gnd diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am deleted file mode 100644 index 38d75c0dc..000000000 --- a/src/fs/Makefile.am +++ /dev/null @@ -1,531 +0,0 @@ -# This Makefile.am is in the public domain -AM_CPPFLAGS = -I$(top_srcdir)/src/include - -if USE_COVERAGE - AM_CFLAGS = --coverage -O0 - XLIB = -lgcov -endif - -pkgcfgdir= $(pkgdatadir)/config.d/ - -libexecdir= $(pkglibdir)/libexec/ - -pkgcfg_DATA = \ - fs.conf - -plugindir = $(libdir)/gnunet - - -lib_LTLIBRARIES = libgnunetfs.la - -plugin_LTLIBRARIES = \ - libgnunet_plugin_block_fs.la - -libgnunetfs_la_SOURCES = \ - fs_api.c fs_api.h fs.h \ - fs_directory.c \ - fs_dirmetascan.c \ - fs_download.c \ - fs_file_information.c \ - fs_getopt.c \ - fs_list_indexed.c \ - fs_publish.c \ - fs_publish_ksk.c \ - fs_publish_ublock.c fs_publish_ublock.h \ - fs_misc.c \ - fs_namespace.c \ - fs_search.c \ - fs_sharetree.c \ - fs_tree.c fs_tree.h \ - fs_unindex.c \ - fs_uri.c \ - meta_data.c - -libgnunetfs_la_LIBADD = \ - $(top_builddir)/src/service/datastore/libgnunetdatastore.la \ - $(top_builddir)/src/service/statistics/libgnunetstatistics.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la \ - $(GN_LIBINTL) $(XLIB) $(LIBGCRYPT_LIBS) -lunistring - -if HAVE_LIBEXTRACTOR -libgnunetfs_la_LIBADD += \ - -lextractor -endif - -libgnunetfs_la_LDFLAGS = \ - $(GN_LIB_LDFLAGS) \ - -version-info 3:1:1 - - -libexec_PROGRAMS = \ - gnunet-helper-fs-publish \ - gnunet-service-fs - -noinst_PROGRAMS = \ - gnunet-daemon-fsprofiler - -bin_PROGRAMS = \ - gnunet-auto-share \ - gnunet-directory \ - gnunet-download \ - gnunet-publish \ - gnunet-search \ - gnunet-fs \ - gnunet-unindex - -gnunet_directory_SOURCES = \ - gnunet-directory.c -gnunet_directory_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la \ - $(GN_LIBINTL) - -if HAVE_LIBEXTRACTOR -gnunet_directory_LDADD += \ - -lextractor -endif - -gnunet_fs_SOURCES = \ - gnunet-fs.c -gnunet_fs_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la \ - $(GN_LIBINTL) - -if HAVE_LIBEXTRACTOR -gnunet_fs_LDADD += \ - -lextractor -endif - -gnunet_download_SOURCES = \ - gnunet-download.c -gnunet_download_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la \ - $(GN_LIBINTL) - -gnunet_publish_SOURCES = \ - gnunet-publish.c -gnunet_publish_LDADD = \ - $(top_builddir)/src/service/identity/libgnunetidentity.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la \ - $(GN_LIBINTL) - -if HAVE_LIBEXTRACTOR -gnunet_publish_LDADD += \ - -lextractor -endif - -gnunet_auto_share_SOURCES = \ - gnunet-auto-share.c -gnunet_auto_share_LDADD = \ - $(top_builddir)/src/lib/util/libgnunetutil.la \ - $(GN_LIBINTL) - -if HAVE_LIBEXTRACTOR -gnunet_auto_share_LDADD += \ - -lextractor -endif - -gnunet_helper_fs_publish_SOURCES = \ - gnunet-helper-fs-publish.c -gnunet_helper_fs_publish_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la \ - $(GN_LIBINTL) - -if HAVE_LIBEXTRACTOR -gnunet_helper_fs_publish_LDADD += \ - -lextractor -endif - -gnunet_search_SOURCES = \ - gnunet-search.c -gnunet_search_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la \ - $(GN_LIBINTL) - -if HAVE_LIBEXTRACTOR -gnunet_search_LDADD += \ - -lextractor -endif - - -gnunet_daemon_fsprofiler_SOURCES = \ - gnunet-daemon-fsprofiler.c -gnunet_daemon_fsprofiler_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/service/statistics/libgnunetstatistics.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la \ - $(GN_LIBINTL) - -gnunet_service_fs_SOURCES = \ - gnunet-service-fs.c gnunet-service-fs.h \ - gnunet-service-fs_cp.c gnunet-service-fs_cp.h \ - gnunet-service-fs_indexing.c gnunet-service-fs_indexing.h \ - gnunet-service-fs_pe.c gnunet-service-fs_pe.h \ - gnunet-service-fs_pr.c gnunet-service-fs_pr.h \ - gnunet-service-fs_push.c gnunet-service-fs_push.h \ - gnunet-service-fs_put.c gnunet-service-fs_put.h \ - gnunet-service-fs_cadet_client.c gnunet-service-fs_cadet.h \ - gnunet-service-fs_cadet_server.c -gnunet_service_fs_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/service/dht/libgnunetdht.la \ - $(top_builddir)/src/lib/block/libgnunetblock.la \ - $(top_builddir)/src/service/datastore/libgnunetdatastore.la \ - $(top_builddir)/src/service/statistics/libgnunetstatistics.la \ - $(top_builddir)/src/service/cadet/libgnunetcadet.la \ - $(top_builddir)/src/service/core/libgnunetcore.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la \ - $(top_builddir)/src/service/peerstore/libgnunetpeerstore.la \ - $(GN_LIBINTL) -lm - -gnunet_unindex_SOURCES = \ - gnunet-unindex.c -gnunet_unindex_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la \ - $(GN_LIBINTL) - -libgnunet_plugin_block_fs_la_SOURCES = \ - plugin_block_fs.c -libgnunet_plugin_block_fs_la_LIBADD = \ - $(top_builddir)/src/lib/block/libgnunetblockgroup.la \ - $(top_builddir)/src/lib/block/libgnunetblock.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la \ - $(LTLIBINTL) -libgnunet_plugin_block_fs_la_LDFLAGS = \ - $(GN_PLUGIN_LDFLAGS) - -check_PROGRAMS = \ - test_plugin_block_fs \ - test_fs_directory \ - test_fs_download \ - test_fs_download_cadet \ - test_fs_download_indexed \ - test_fs_download_persistence \ - test_fs_file_information \ - test_fs_getopt \ - test_fs_list_indexed \ - test_fs_namespace \ - test_fs_namespace_list_updateable \ - test_fs_publish \ - test_fs_publish_persistence \ - test_fs_search \ - test_fs_search_with_and \ - test_fs_search_probes \ - test_fs_search_persistence \ - test_fs_start_stop \ - test_fs_test_lib \ - test_fs_unindex \ - test_fs_unindex_persistence \ - test_fs_uri \ - test_fs_meta_data \ - test_gnunet_service_fs_migration \ - $(FS_BENCHMARKS) - -test_plugin_block_fs_SOURCES = \ - test_plugin_block_fs.c -test_plugin_block_fs_LDADD = \ - $(top_builddir)/src/lib/block/libgnunetblock.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -if HAVE_PYTHON -check_SCRIPTS = \ - test_gnunet_fs_rec.py \ - test_gnunet_fs_idx.py - -if HAVE_LIBEXTRACTOR -check_SCRIPTS += \ - test_gnunet_fs_psd.py -endif -endif - - -if ENABLE_TEST_RUN -AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME; -TESTS = test_fs_directory \ - test_fs_file_information \ - test_fs_namespace \ - test_fs_namespace_list_updateable \ - test_fs_search \ - test_fs_search_with_and \ - test_fs_search_probes \ - test_fs_search_persistence \ - test_fs_start_stop \ - test_fs_uri \ - test_fs_meta_data - # $(check_SCRIPTS) -endif - - -test_fs_directory_SOURCES = \ - test_fs_directory.c -test_fs_directory_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -if HAVE_LIBEXTRACTOR -test_fs_directory_LDADD += \ - -lextractor -endif - - -test_fs_download_SOURCES = \ - test_fs_download.c -test_fs_download_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_download_indexed_SOURCES = \ - test_fs_download.c -test_fs_download_indexed_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_download_cadet_SOURCES = \ - test_fs_download.c -test_fs_download_cadet_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_download_persistence_SOURCES = \ - test_fs_download_persistence.c -test_fs_download_persistence_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_file_information_SOURCES = \ - test_fs_file_information.c -test_fs_file_information_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -if HAVE_LIBEXTRACTOR -test_fs_file_information_LDADD += \ - -lextractor -endif - - -test_fs_getopt_SOURCES = \ - test_fs_getopt.c -test_fs_getopt_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_list_indexed_SOURCES = \ - test_fs_list_indexed.c -test_fs_list_indexed_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_namespace_SOURCES = \ - test_fs_namespace.c -test_fs_namespace_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_namespace_list_updateable_SOURCES = \ - test_fs_namespace_list_updateable.c -test_fs_namespace_list_updateable_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_publish_SOURCES = \ - test_fs_publish.c -test_fs_publish_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_publish_persistence_SOURCES = \ - test_fs_publish_persistence.c -test_fs_publish_persistence_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_search_SOURCES = \ - test_fs_search.c -test_fs_search_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_search_with_and_SOURCES = \ - test_fs_search_with_and.c -test_fs_search_with_and_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_search_probes_SOURCES = \ - test_fs_search_probes.c -test_fs_search_probes_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_search_persistence_SOURCES = \ - test_fs_search_persistence.c -test_fs_search_persistence_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_start_stop_SOURCES = \ - test_fs_start_stop.c -test_fs_start_stop_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_unindex_SOURCES = \ - test_fs_unindex.c -test_fs_unindex_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_unindex_persistence_SOURCES = \ - test_fs_unindex_persistence.c -test_fs_unindex_persistence_LDADD = \ - $(top_builddir)/src/service/testing/libgnunettesting.la \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -test_fs_meta_data_SOURCES = \ - test_fs_meta_data.c -test_fs_meta_data_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - - -test_fs_uri_SOURCES = \ - test_fs_uri.c -test_fs_uri_LDADD = \ - libgnunetfs.la \ - $(top_builddir)/src/lib/util/libgnunetutil.la - -# TNG -#test_fs_test_lib_SOURCES = \ -# test_fs_test_lib.c -#test_fs_test_lib_LDADD = \ -# libgnunetfstest.a \ -# $(top_builddir)/src/testbed/libgnunettestbed.la \ -# libgnunetfs.la \ -# $(top_builddir)/src/lib/util/libgnunetutil.la - -#test_gnunet_service_fs_p2p_SOURCES = \ -# test_gnunet_service_fs_p2p.c -#test_gnunet_service_fs_p2p_LDADD = \ -# libgnunetfstest.a \ -# $(top_builddir)/src/testbed/libgnunettestbed.la \ -# libgnunetfs.la \ -# $(top_builddir)/src/lib/util/libgnunetutil.la -# -#test_gnunet_service_fs_p2p_cadet_SOURCES = \ -# test_gnunet_service_fs_p2p.c -#test_gnunet_service_fs_p2p_cadet_LDADD = \ -# libgnunetfstest.a \ -# $(top_builddir)/src/testbed/libgnunettestbed.la \ -# libgnunetfs.la \ -# $(top_builddir)/src/lib/util/libgnunetutil.la -# -#test_gnunet_service_fs_migration_SOURCES = \ -# test_gnunet_service_fs_migration.c -#test_gnunet_service_fs_migration_LDADD = \ -# libgnunetfstest.a \ -# $(top_builddir)/src/testbed/libgnunettestbed.la \ -# libgnunetfs.la \ -# $(top_builddir)/src/lib/util/libgnunetutil.la -# -#perf_gnunet_service_fs_p2p_SOURCES = \ -# perf_gnunet_service_fs_p2p.c -#perf_gnunet_service_fs_p2p_LDADD = \ -# libgnunetfstest.a \ -# $(top_builddir)/src/service/statistics/libgnunetstatistics.la \ -# $(top_builddir)/src/testbed/libgnunettestbed.la \ -# libgnunetfs.la \ -# $(top_builddir)/src/lib/util/libgnunetutil.la -# -#perf_gnunet_service_fs_p2p_index_SOURCES = \ -# perf_gnunet_service_fs_p2p.c -#perf_gnunet_service_fs_p2p_index_LDADD = \ -# libgnunetfstest.a \ -# $(top_builddir)/src/service/statistics/libgnunetstatistics.la \ -# $(top_builddir)/src/testbed/libgnunettestbed.la \ -# libgnunetfs.la \ -# $(top_builddir)/src/lib/util/libgnunetutil.la -# -#perf_gnunet_service_fs_p2p_dht_SOURCES = \ -# perf_gnunet_service_fs_p2p.c -#perf_gnunet_service_fs_p2p_dht_LDADD = \ -# libgnunetfstest.a \ -# $(top_builddir)/src/service/statistics/libgnunetstatistics.la \ -# $(top_builddir)/src/testbed/libgnunettestbed.la \ -# libgnunetfs.la \ -# $(top_builddir)/src/lib/util/libgnunetutil.la -# -#perf_gnunet_service_fs_p2p_respect_SOURCES = \ -# perf_gnunet_service_fs_p2p_respect.c -#perf_gnunet_service_fs_p2p_respect_LDADD = \ -# libgnunetfstest.a \ -# $(top_builddir)/src/service/statistics/libgnunetstatistics.la \ -# $(top_builddir)/src/testbed/libgnunettestbed.la \ -# libgnunetfs.la \ -# $(top_builddir)/src/lib/util/libgnunetutil.la - -test_gnunet_fs_psd.py: test_gnunet_fs_psd.py.in Makefile - $(AWK) -v bdir="$(bindir)" -v py="$(PYTHON)" -v awkay="$(AWK_BINARY)" -v pfx="$(prefix)" -v prl="$(PERL)" -v sysconfdirectory="$(sysconfdir)" -v pkgdatadirectory="$(pkgdatadir)" -f $(top_srcdir)/bin/dosubst.awk < $(srcdir)/test_gnunet_fs_psd.py.in > test_gnunet_fs_psd.py - chmod +x test_gnunet_fs_psd.py - -test_gnunet_fs_rec.py: test_gnunet_fs_rec.py.in Makefile - $(AWK) -v bdir="$(bindir)" -v py="$(PYTHON)" -v awkay="$(AWK_BINARY)" -v pfx="$(prefix)" -v prl="$(PERL)" -v sysconfdirectory="$(sysconfdir)" -v pkgdatadirectory="$(pkgdatadir)" -f $(top_srcdir)/bin/dosubst.awk < $(srcdir)/test_gnunet_fs_rec.py.in > test_gnunet_fs_rec.py - chmod +x test_gnunet_fs_rec.py - -test_gnunet_fs_ns.py: test_gnunet_fs_ns.py.in Makefile - $(AWK) -v bdir="$(bindir)" -v py="$(PYTHON)" -v awkay="$(AWK_BINARY)" -v pfx="$(prefix)" -v prl="$(PERL)" -v sysconfdirectory="$(sysconfdir)" -v pkgdatadirectory="$(pkgdatadir)" -f $(top_srcdir)/bin/dosubst.awk < $(srcdir)/test_gnunet_fs_ns.py.in > test_gnunet_fs_ns.py - chmod +x test_gnunet_fs_ns.py - -test_gnunet_fs_idx.py: test_gnunet_fs_idx.py.in Makefile - $(AWK) -v bdir="$(bindir)" -v py="$(PYTHON)" -v awkay="$(AWK_BINARY)" -v pfx="$(prefix)" -v prl="$(PERL)" -v sysconfdirectory="$(sysconfdir)" -v pkgdatadirectory="$(pkgdatadir)" -f $(top_srcdir)/bin/dosubst.awk < $(srcdir)/test_gnunet_fs_idx.py.in > test_gnunet_fs_idx.py - chmod +x test_gnunet_fs_idx.py - - -EXTRA_DIST = \ - fs_test_lib_data.conf \ - perf_gnunet_service_fs_p2p.conf \ - test_fs_data.conf \ - test_fs_defaults.conf \ - test_fs_download_data.conf \ - test_fs_download_indexed.conf \ - test_fs_file_information_data.conf \ - test_fs_list_indexed_data.conf \ - test_fs_namespace_data.conf \ - test_fs_publish_data.conf \ - test_fs_search_data.conf \ - test_fs_unindex_data.conf \ - test_gnunet_fs_idx_data.conf \ - test_gnunet_fs_psd_data.conf \ - test_gnunet_fs_rec_data.conf \ - test_gnunet_fs_rec_data.tgz \ - test_gnunet_fs_psd.py.in \ - test_gnunet_fs_rec.py.in \ - test_gnunet_fs_idx.py.in \ - test_gnunet_service_fs_migration_data.conf \ - test_gnunet_service_fs_p2p_cadet.conf \ - test_pseudonym_data.conf - -CLEANFILES = $(check_SCRIPTS) diff --git a/src/fs/fs.conf.in b/src/fs/fs.conf.in deleted file mode 100644 index be02619bf..000000000 --- a/src/fs/fs.conf.in +++ /dev/null @@ -1,63 +0,0 @@ -[fs] -RUN_PER_USER = YES -START_ON_DEMAND = @START_ON_DEMAND@ -IMMEDIATE_START = YES -INDEXDB = $GNUNET_DATA_HOME/fs/idxinfo.lst -RESPECT = $GNUNET_DATA_HOME/fs/credit/ -STATE_DIR = $GNUNET_DATA_HOME/fs/persistence/ -UPDATE_DIR = $GNUNET_DATA_HOME/fs/updates/ -@UNIXONLY@ PORT = 2094 -HOSTNAME = localhost -BINARY = gnunet-service-fs -ACCEPT_FROM = 127.0.0.1; -ACCEPT_FROM6 = ::1; - -# PREFIX = valgrind - -# Do we introduce artificial delays? (may improve anonymity) -DELAY = YES - -# Do we cache content from other nodes? (may improve anonymity) -CONTENT_CACHING = YES - -# Do we send unsolicited data to other nodes if we have excess bandwidth? -# (may improve anonymity, probably not a good idea if content_caching is NO) -CONTENT_PUSHING = YES - -UNIXPATH = $GNUNET_USER_RUNTIME_DIR/gnunet-service-fs.sock - -# Do we require users that want to access file-sharing to run this process -# (usually not a good idea) -UNIX_MATCH_UID = NO - -# Do we require users that want to access file-sharing to be in the 'gnunet' group? -UNIX_MATCH_GID = YES - -# Maximum number of requests this peer tracks (important for -# memory consumption; 2k RAM/request is not unusual) -MAX_PENDING_REQUESTS = 65536 - -# How many requests do we have at most waiting in the queue towards -# the datastore? (important for memory consumption) -DATASTORE_QUEUE_SIZE = 32 - -# Maximum frequency we're allowed to poll the datastore -# for content for migration (can be used to reduce -# GNUnet's disk-IO rate) -MIN_MIGRATION_DELAY = 100 ms - -# For how many neighbouring peers should we allocate hash maps? -EXPECTED_NEIGHBOUR_COUNT = 128 - -# Disable anonymous file-sharing (but keep non-anonymous transfers)? -# This option is mostly for testing. -DISABLE_ANON_TRANSFER = NO - -# Maximum number of non-anonymous transfers this peer will support -# at the same time. Excessive values mostly have the problem that -# the service might use more memory, so we need to bound this at -# some reasonable level. And if we have a very, very large -# number, we probably won't have enough bandwidth to support them -# well anyway, so better have a moderate cap. -MAX_CADET_CLIENTS = 128 - diff --git a/src/fs/fs.h b/src/fs/fs.h deleted file mode 100644 index c3bae65d2..000000000 --- a/src/fs/fs.h +++ /dev/null @@ -1,397 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2003--2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs.h - * @brief definitions for the entire fs module - * @author Igor Wronsky, Christian Grothoff - */ -#ifndef FS_H -#define FS_H - -#include "gnunet_constants.h" -#include "gnunet_datastore_service.h" -#include "gnunet_dht_service.h" - -#include "gnunet_fs_service.h" -#include "gnunet_block_lib.h" -#include "block_fs.h" - - -/** - * Size of the individual blocks used for file-sharing. - */ -#define DBLOCK_SIZE (32 * 1024) - -/** - * Blocksize to use when hashing files for indexing (blocksize for IO, - * not for the DBlocks). Larger blocksizes can be more efficient but - * will be more disruptive as far as the scheduler is concerned. - */ -#define HASHING_BLOCKSIZE (1024 * 128) - - -/** - * @brief content hash key - */ -struct ContentHashKey -{ - /** - * Hash of the original content, used for encryption. - */ - struct GNUNET_HashCode key; - - /** - * Hash of the encrypted content, used for querying. - */ - struct GNUNET_HashCode query; -}; - - -GNUNET_NETWORK_STRUCT_BEGIN - - -/** - * Message sent from a GNUnet (fs) publishing activity to sign - * a LOC URI. - */ -struct RequestLocSignatureMessage -{ - /** - * Message type will be #GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGN. - */ - struct GNUNET_MessageHeader header; - - /** - * Requested signature purpose. For now, always - * #GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT. - */ - uint32_t purpose GNUNET_PACKED; - - /** - * Requested expiration time. - */ - struct GNUNET_TIME_AbsoluteNBO expiration_time; - - /** - * Information about the shared file (to be signed). - */ - struct ContentHashKey chk; - - /** - * Size of the shared file (to be signed). - */ - uint64_t file_length; -}; - - -/** - * Message sent from the service with the signed LOC URI. - */ -struct ResponseLocSignatureMessage -{ - /** - * Message type will be - * #GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGNATURE. - */ - struct GNUNET_MessageHeader header; - - /** - * Purpose of the generated signature. For now, always - * #GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT. - */ - uint32_t purpose GNUNET_PACKED; - - /** - * Expiration time that was actually used (rounded!). - */ - struct GNUNET_TIME_AbsoluteNBO expiration_time; - - /** - * The requested signature. - */ - struct GNUNET_CRYPTO_EddsaSignature signature; - - /** - * Identity of the peer sharing the file. - */ - struct GNUNET_PeerIdentity peer; -}; - - -/** - * Message sent from a GNUnet (fs) publishing activity to the - * gnunet-fs-service to initiate indexing of a file. The service is - * supposed to check if the specified file is available and has the - * same cryptographic hash. It should then respond with either a - * confirmation or a denial. - * - * On OSes where this works, it is considered acceptable if the - * service only checks that the path, device and inode match (it can - * then be assumed that the hash will also match without actually - * computing it; this is an optimization that should be safe given - * that the client is not our adversary). - */ -struct IndexStartMessage -{ - /** - * Message type will be #GNUNET_MESSAGE_TYPE_FS_INDEX_START. - */ - struct GNUNET_MessageHeader header; - - /** - * For alignment. - */ - uint32_t reserved GNUNET_PACKED; - - /** - * ID of device containing the file, as seen by the client. This - * device ID is obtained using a call like "statvfs" (and converting - * the "f_fsid" field to a 32-bit big-endian number). Use 0 if the - * OS does not support this, in which case the service must do a - * full hash recomputation. - */ - uint64_t device GNUNET_PACKED; - - /** - * Inode of the file on the given device, as seen by the client - * ("st_ino" field from "struct stat"). Use 0 if the OS does not - * support this, in which case the service must do a full hash - * recomputation. - */ - uint64_t inode GNUNET_PACKED; - - /** - * Hash of the file that we would like to index. - */ - struct GNUNET_HashCode file_id; - - /* this is followed by a 0-terminated - * filename of a file with the hash - * "file_id" as seen by the client */ -}; - - -/** - * Message send by FS service in response to a request - * asking for a list of all indexed files. - */ -struct IndexInfoMessage -{ - /** - * Message type will be - * #GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY. - */ - struct GNUNET_MessageHeader header; - - /** - * Always zero. - */ - uint32_t reserved GNUNET_PACKED; - - /** - * Hash of the indexed file. - */ - struct GNUNET_HashCode file_id; - - /* this is followed by a 0-terminated - * filename of a file with the hash - * "file_id" as seen by the client */ -}; - - -/** - * Message sent from a GNUnet (fs) unindexing activity to the - * gnunet-service-fs to indicate that a file will be unindexed. The - * service is supposed to remove the file from the list of indexed - * files and response with a confirmation message (even if the file - * was already not on the list). - */ -struct UnindexMessage -{ - /** - * Message type will be #GNUNET_MESSAGE_TYPE_FS_UNINDEX. - */ - struct GNUNET_MessageHeader header; - - /** - * Always zero. - */ - uint32_t reserved GNUNET_PACKED; - - /** - * Hash of the file that we will unindex. - */ - struct GNUNET_HashCode file_id; -}; - - -/** - * No options. - */ -#define SEARCH_MESSAGE_OPTION_NONE 0 - -/** - * Only search the local datastore (no network) - */ -#define SEARCH_MESSAGE_OPTION_LOOPBACK_ONLY 1 - -/** - * Request is too large to fit in 64k format. The list of - * already-known search results will be continued in another message - * for the same type/query/target and additional already-known results - * following this one). - */ -#define SEARCH_MESSAGE_OPTION_CONTINUED 2 - - -/** - * Message sent from a GNUnet (fs) search activity to the - * gnunet-service-fs to start a search. - */ -struct SearchMessage -{ - /** - * Message type will be #GNUNET_MESSAGE_TYPE_FS_START_SEARCH. - */ - struct GNUNET_MessageHeader header; - - /** - * Bitmask with options. Zero for no options, one for - * loopback-only, two for 'to be continued' (with a second search - * message for the same type/query/target and additional - * already-known results following this one). See - * SEARCH_MESSAGE_OPTION_ defines. - * - * Other bits are currently not defined. - */ - uint32_t options GNUNET_PACKED; - - /** - * Type of the content that we're looking for. - */ - uint32_t type GNUNET_PACKED; - - /** - * Desired anonymity level, big-endian. - */ - uint32_t anonymity_level GNUNET_PACKED; - - /** - * If the request is for a DBLOCK or IBLOCK, this is the identity of - * the peer that is known to have a response. Set to all-zeros if - * such a target is not known (note that even if OUR anonymity - * level is >0 we may happen to know the responder's identity; - * nevertheless, we should probably not use it for a DHT-lookup - * or similar blunt actions in order to avoid exposing ourselves). - *

- * Otherwise, "target" must be all zeros. - */ - struct GNUNET_PeerIdentity target; - - /** - * Hash of the public key for UBLOCKs; Hash of - * the CHK-encoded block for DBLOCKS and IBLOCKS. - */ - struct GNUNET_HashCode query; - - /* this is followed by the hash codes of already-known - * results (which should hence be excluded from what - * the service returns); naturally, this only applies - * to queries that can have multiple results (UBLOCKS). - */ -}; - - -/** - * Response from FS service with a result for a previous FS search. - * Note that queries for DBLOCKS and IBLOCKS that have received a - * single response are considered done. This message is transmitted - * between peers. - */ -struct PutMessage -{ - /** - * Message type will be #GNUNET_MESSAGE_TYPE_FS_PUT. - */ - struct GNUNET_MessageHeader header; - - /** - * Type of the block (in big endian). Should never be zero. - */ - uint32_t type GNUNET_PACKED; - - /** - * When does this result expire? - */ - struct GNUNET_TIME_AbsoluteNBO expiration; - - /* this is followed by the actual encrypted content */ -}; - -/** - * Response from FS service with a result for a previous FS search. - * Note that queries for DBLOCKS and IBLOCKS that have received a - * single response are considered done. This message is transmitted - * between the service and a client. - */ -struct ClientPutMessage -{ - /** - * Message type will be #GNUNET_MESSAGE_TYPE_FS_PUT. - */ - struct GNUNET_MessageHeader header; - - /** - * Type of the block (in big endian). Should never be zero. - */ - uint32_t type GNUNET_PACKED; - - /** - * When does this result expire? - */ - struct GNUNET_TIME_AbsoluteNBO expiration; - - /** - * When was the last time we've tried to download this block? - * (FOREVER if unknown/not relevant) - */ - struct GNUNET_TIME_AbsoluteNBO last_transmission; - - /** - * How often did we transmit this query before getting an - * answer (estimate). - */ - uint32_t num_transmissions; - - /** - * How much respect did we offer (in total) before getting an - * answer (estimate). - */ - uint32_t respect_offered; - - /* this is followed by the actual encrypted content */ -}; -GNUNET_NETWORK_STRUCT_END - - -#endif - -/* end of fs.h */ diff --git a/src/fs/fs_api.c b/src/fs/fs_api.c deleted file mode 100644 index 627c58004..000000000 --- a/src/fs/fs_api.c +++ /dev/null @@ -1,3321 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2001--2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_api.c - * @brief main FS functions (master initialization, serialization, deserialization, shared code) - * @author Christian Grothoff - */ - -#include "platform.h" -#include "gnunet_util_lib.h" - -#include "gnunet_fs_service.h" -#include "fs_api.h" -#include "fs_tree.h" - -/** - * How many block requests can we have outstanding in parallel at a time by default? - */ -#define DEFAULT_MAX_PARALLEL_REQUESTS (1024 * 10) - -/** - * How many downloads can we have outstanding in parallel at a time by default? - */ -#define DEFAULT_MAX_PARALLEL_DOWNLOADS 16 - -/** - * Start the given job (send signal, remove from pending queue, update - * counters and state). - * - * @param qe job to start - */ -static void -start_job (struct GNUNET_FS_QueueEntry *qe) -{ - qe->active = GNUNET_YES; - qe->start (qe->cls); - qe->start_times++; - qe->h->active_blocks += qe->blocks; - qe->h->active_downloads++; - qe->start_time = GNUNET_TIME_absolute_get (); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting job %p (%u active)\n", - qe, - qe->h->active_downloads); - GNUNET_CONTAINER_DLL_remove (qe->h->pending_head, qe->h->pending_tail, qe); - GNUNET_CONTAINER_DLL_insert_after (qe->h->running_head, - qe->h->running_tail, - qe->h->running_tail, - qe); -} - - -/** - * Stop the given job (send signal, remove from active queue, update - * counters and state). - * - * @param qe job to stop - */ -static void -stop_job (struct GNUNET_FS_QueueEntry *qe) -{ - qe->active = GNUNET_NO; - qe->stop (qe->cls); - GNUNET_assert (0 < qe->h->active_downloads); - qe->h->active_downloads--; - qe->h->active_blocks -= qe->blocks; - qe->run_time = GNUNET_TIME_relative_add (qe->run_time, - GNUNET_TIME_absolute_get_duration ( - qe->start_time)); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Stopping job %p (%u active)\n", - qe, - qe->h->active_downloads); - GNUNET_CONTAINER_DLL_remove (qe->h->running_head, qe->h->running_tail, qe); - GNUNET_CONTAINER_DLL_insert_after (qe->h->pending_head, - qe->h->pending_tail, - qe->h->pending_tail, - qe); -} - - -/** - * Process the jobs in the job queue, possibly starting some - * and stopping others. - * - * @param cls the `struct GNUNET_FS_Handle *` - */ -static void -process_job_queue (void *cls) -{ - struct GNUNET_FS_Handle *h = cls; - struct GNUNET_FS_QueueEntry *qe; - struct GNUNET_FS_QueueEntry *next; - struct GNUNET_TIME_Relative run_time; - struct GNUNET_TIME_Relative restart_at; - struct GNUNET_TIME_Relative rst; - struct GNUNET_TIME_Absolute end_time; - unsigned int num_downloads_waiting; - unsigned int num_downloads_active; - unsigned int num_downloads_expired; - unsigned int num_probes_active; - unsigned int num_probes_waiting; - unsigned int num_probes_expired; - int num_probes_change; - int num_downloads_change; - int block_limit_hit; - - h->queue_job = NULL; - /* restart_at will be set to the time when it makes sense to - re-evaluate the job queue (unless, of course, jobs complete - or are added, then we'll be triggered immediately */ - restart_at = GNUNET_TIME_UNIT_FOREVER_REL; - /* first, calculate some basic statistics on pending jobs */ - num_probes_waiting = 0; - num_downloads_waiting = 0; - for (qe = h->pending_head; NULL != qe; qe = qe->next) - { - switch (qe->priority) - { - case GNUNET_FS_QUEUE_PRIORITY_PROBE: - num_probes_waiting++; - break; - - case GNUNET_FS_QUEUE_PRIORITY_NORMAL: - num_downloads_waiting++; - break; - - default: - GNUNET_break (0); - break; - } - } - /* now, calculate some basic statistics on running jobs */ - num_probes_active = 0; - num_probes_expired = 0; - num_downloads_active = 0; - num_downloads_expired = 0; - next = h->running_head; - while (NULL != (qe = next)) - { - next = qe->next; - switch (qe->priority) - { - case GNUNET_FS_QUEUE_PRIORITY_PROBE: - run_time = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 2); - end_time = GNUNET_TIME_absolute_add (qe->start_time, run_time); - rst = GNUNET_TIME_absolute_get_remaining (end_time); - if (0 == rst.rel_value_us) - { - num_probes_expired++; - stop_job (qe); - } - else - { - num_probes_active++; - restart_at = GNUNET_TIME_relative_min (rst, restart_at); - } - break; - - case GNUNET_FS_QUEUE_PRIORITY_NORMAL: - run_time = - GNUNET_TIME_relative_saturating_multiply (h->avg_block_latency, - qe->blocks * qe->start_times); - end_time = GNUNET_TIME_absolute_add (qe->start_time, run_time); - rst = GNUNET_TIME_absolute_get_remaining (end_time); - if (0 == rst.rel_value_us) - { - num_downloads_expired++; - stop_job (qe); - } - else - { - num_downloads_active++; - restart_at = GNUNET_TIME_relative_min (rst, restart_at); - } - break; - - default: - GNUNET_break (0); - break; - } - } - GNUNET_break (h->active_downloads == - num_downloads_active + num_probes_active); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "PA: %u, PE: %u, PW: %u; DA: %u, DE: %u, DW: %u\n", - num_probes_active, - num_probes_expired, - num_probes_waiting, - num_downloads_active, - num_downloads_expired, - num_downloads_waiting); - GNUNET_break (h->active_downloads + num_probes_active <= - h->max_parallel_downloads); - /* calculate start/stop decisions */ - if (h->active_downloads + num_downloads_waiting > h->max_parallel_downloads) - { - /* stop as many probes as there are downloads and probes */ - num_probes_change = -GNUNET_MIN (num_probes_active, num_downloads_waiting); - /* start as many downloads as there are free slots, including those - we just opened up */ - num_downloads_change = - h->max_parallel_downloads - h->active_downloads - num_probes_change; - } - else - { - /* start all downloads (we can) */ - num_downloads_change = num_downloads_waiting; - /* also start probes if there is room, but use a lower cap of (mpd/4) + 1 */ - if (1 + h->max_parallel_downloads / 4 >= - (h->active_downloads + num_downloads_change)) - num_probes_change = - GNUNET_MIN (num_probes_waiting, - (1 + h->max_parallel_downloads / 4) - - (h->active_downloads + num_downloads_change)); - else - num_probes_change = 0; - } - GNUNET_break (num_downloads_change <= num_downloads_waiting); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Changing %d probes and %d/%u/%u downloads\n", - num_probes_change, - num_downloads_change, - (unsigned int) h->active_downloads, - (unsigned int) h->max_parallel_downloads); - /* actually stop probes */ - next = h->running_head; - while (NULL != (qe = next)) - { - next = qe->next; - if (GNUNET_FS_QUEUE_PRIORITY_PROBE != qe->priority) - continue; - if (num_probes_change < 0) - { - stop_job (qe); - num_probes_change++; - if (0 == num_probes_change) - break; - } - } - GNUNET_break (0 <= num_probes_change); - - /* start some more tasks if we now have empty slots */ - block_limit_hit = GNUNET_NO; - next = h->pending_head; - while ((NULL != (qe = next)) && - ((num_probes_change > 0) || (num_downloads_change > 0))) - { - next = qe->next; - switch (qe->priority) - { - case GNUNET_FS_QUEUE_PRIORITY_PROBE: - if (num_probes_change > 0) - { - start_job (qe); - num_probes_change--; - run_time = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 2); - restart_at = GNUNET_TIME_relative_min (run_time, restart_at); - } - break; - - case GNUNET_FS_QUEUE_PRIORITY_NORMAL: - if ((num_downloads_change > 0) && - ((qe->blocks + h->active_blocks <= h->max_parallel_requests) || - ((qe->blocks > h->max_parallel_requests) && - (0 == h->active_downloads)))) - { - start_job (qe); - num_downloads_change--; - } - else if (num_downloads_change > 0) - block_limit_hit = GNUNET_YES; - break; - - default: - GNUNET_break (0); - break; - } - } - GNUNET_break ((0 == num_downloads_change) || (GNUNET_YES == block_limit_hit)); - GNUNET_break (0 == num_probes_change); - - GNUNET_log ( - GNUNET_ERROR_TYPE_DEBUG, - "AD: %u, MP: %u; %d probes and %d downloads to start, will run again in %s\n", - h->active_downloads, - h->max_parallel_requests, - num_probes_change, - num_downloads_change, - GNUNET_STRINGS_relative_time_to_string (restart_at, GNUNET_YES)); - - /* make sure we run again, callbacks might have - already re-scheduled the job, so cancel such - an operation (if it exists) */ - if (NULL != h->queue_job) - GNUNET_SCHEDULER_cancel (h->queue_job); - h->queue_job = - GNUNET_SCHEDULER_add_delayed (restart_at, &process_job_queue, h); -} - - -struct GNUNET_FS_QueueEntry * -GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, - GNUNET_SCHEDULER_TaskCallback start, - GNUNET_SCHEDULER_TaskCallback stop, - void *cls, - unsigned int blocks, - enum GNUNET_FS_QueuePriority priority) -{ - struct GNUNET_FS_QueueEntry *qe; - - qe = GNUNET_new (struct GNUNET_FS_QueueEntry); - qe->h = h; - qe->start = start; - qe->stop = stop; - qe->cls = cls; - qe->queue_time = GNUNET_TIME_absolute_get (); - qe->blocks = blocks; - qe->priority = priority; - GNUNET_CONTAINER_DLL_insert_after (h->pending_head, - h->pending_tail, - h->pending_tail, - qe); - if (NULL != h->queue_job) - GNUNET_SCHEDULER_cancel (h->queue_job); - h->queue_job = GNUNET_SCHEDULER_add_now (&process_job_queue, h); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Queueing job %p\n", qe); - return qe; -} - - -/** - * Dequeue a job from the queue. - * - * @param qe handle for the job - */ -void -GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qe) -{ - struct GNUNET_FS_Handle *h; - - h = qe->h; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Dequeueing job %p\n", qe); - if (GNUNET_YES == qe->active) - stop_job (qe); - GNUNET_CONTAINER_DLL_remove (h->pending_head, h->pending_tail, qe); - GNUNET_free (qe); - if (NULL != h->queue_job) - GNUNET_SCHEDULER_cancel (h->queue_job); - h->queue_job = GNUNET_SCHEDULER_add_now (&process_job_queue, h); -} - - -/** - * Create a top-level activity entry. - * - * @param h global fs handle - * @param ssf suspend signal function to use - * @param ssf_cls closure for @a ssf - * @return fresh top-level activity handle - */ -struct TopLevelActivity * -GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, - SuspendSignalFunction ssf, - void *ssf_cls) -{ - struct TopLevelActivity *ret; - - ret = GNUNET_new (struct TopLevelActivity); - ret->ssf = ssf; - ret->ssf_cls = ssf_cls; - GNUNET_CONTAINER_DLL_insert (h->top_head, h->top_tail, ret); - return ret; -} - - -/** - * Destroy a top-level activity entry. - * - * @param h global fs handle - * @param top top level activity entry - */ -void -GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top) -{ - GNUNET_CONTAINER_DLL_remove (h->top_head, h->top_tail, top); - GNUNET_free (top); -} - - -/** - * Closure for #GNUNET_FS_data_reader_file_(). - */ -struct FileInfo -{ - /** - * Name of the file to read. - */ - char *filename; - - /** - * File descriptor, NULL if it has not yet been opened. - */ - struct GNUNET_DISK_FileHandle *fd; -}; - - -/** - * Function that provides data by reading from a file. - * - * @param cls closure with the `struct FileInfo *` - * @param offset offset to read from; it is possible - * that the caller might need to go backwards - * a bit at times; set to `UINT64_MAX` to tell - * the reader that we won't be reading for a while - * (used to close the file descriptor but NOT fully - * clean up the reader's state); in this case, - * a value of '0' for @a max should be ignored - * @param max maximum number of bytes that should be - * copied to @a buf; readers are not allowed - * to provide less data unless there is an error; - * a value of "0" will be used at the end to allow - * the reader to clean up its internal state - * @param buf where the reader should write the data - * @param emsg location for the reader to store an error message - * @return number of bytes written, usually @a max, 0 on error - */ -size_t -GNUNET_FS_data_reader_file_ (void *cls, - uint64_t offset, - size_t max, - void *buf, - char **emsg) -{ - struct FileInfo *fi = cls; - ssize_t ret; - - if (UINT64_MAX == offset) - { - if (NULL != fi->fd) - { - GNUNET_DISK_file_close (fi->fd); - fi->fd = NULL; - } - return 0; - } - if (0 == max) - { - if (NULL != fi->fd) - GNUNET_DISK_file_close (fi->fd); - GNUNET_free (fi->filename); - GNUNET_free (fi); - return 0; - } - if (NULL == fi->fd) - { - fi->fd = GNUNET_DISK_file_open (fi->filename, - GNUNET_DISK_OPEN_READ, - GNUNET_DISK_PERM_NONE); - if (NULL == fi->fd) - { - GNUNET_asprintf (emsg, - _ ("Could not open file `%s': %s"), - fi->filename, - strerror (errno)); - return 0; - } - } - if ((GNUNET_SYSERR == - GNUNET_DISK_file_seek (fi->fd, offset, GNUNET_DISK_SEEK_SET)) || - (-1 == (ret = GNUNET_DISK_file_read (fi->fd, buf, max)))) - { - GNUNET_asprintf (emsg, - _ ("Could not read file `%s': %s"), - fi->filename, - strerror (errno)); - return 0; - } - if (ret != max) - { - GNUNET_asprintf (emsg, - _ ("Short read reading from file `%s'!"), - fi->filename); - return 0; - } - return max; -} - - -void * -GNUNET_FS_make_file_reader_context_ (const char *filename) -{ - struct FileInfo *fi; - - fi = GNUNET_new (struct FileInfo); - fi->filename = GNUNET_STRINGS_filename_expand (filename); - if (NULL == fi->filename) - { - GNUNET_free (fi); - return NULL; - } - return fi; -} - - -/** - * Function that provides data by copying from a buffer. - * - * @param cls closure (points to the buffer) - * @param offset offset to read from; it is possible - * that the caller might need to go backwards - * a bit at times; set to `UINT64_MAX` to tell - * the reader that we won't be reading for a while - * (used to close the file descriptor but NOT fully - * clean up the reader's state); in this case, - * a value of '0' for @a max should be ignored - * @param max maximum number of bytes that should be - * copied to @a buf; readers are not allowed - * to provide less data unless there is an error; - * a value of "0" will be used at the end to allow - * the reader to clean up its internal state - * @param buf where the reader should write the data - * @param emsg location for the reader to store an error message - * @return number of bytes written, usually @a max, 0 on error - */ -size_t -GNUNET_FS_data_reader_copy_ (void *cls, - uint64_t offset, - size_t max, - void *buf, - char **emsg) -{ - char *data = cls; - - if (UINT64_MAX == offset) - return 0; - if (0 == max) - { - GNUNET_free (data); - return 0; - } - GNUNET_memcpy (buf, &data[offset], max); - return max; -} - - -/** - * Return the full filename where we would store state information - * (for serialization/deserialization). - * - * @param h master context - * @param ext component of the path - * @param ent entity identifier (or empty string for the directory) - * @return NULL on error - */ -static char * -get_serialization_file_name (struct GNUNET_FS_Handle *h, - const char *ext, - const char *ent) -{ - char *basename; - char *ret; - - if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) - return NULL; /* persistence not requested */ - if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (h->cfg, - "fs", - "STATE_DIR", - &basename)) - return NULL; - GNUNET_asprintf (&ret, - "%s%s%s%s%s%s%s", - basename, - DIR_SEPARATOR_STR, - h->client_name, - DIR_SEPARATOR_STR, - ext, - DIR_SEPARATOR_STR, - ent); - GNUNET_free (basename); - return ret; -} - - -/** - * Return the full filename where we would store state information - * (for serialization/deserialization) that is associated with a - * parent operation. - * - * @param h master context - * @param ext component of the path - * @param uni name of the parent operation - * @param ent entity identifier (or empty string for the directory) - * @return NULL on error - */ -static char * -get_serialization_file_name_in_dir (struct GNUNET_FS_Handle *h, - const char *ext, - const char *uni, - const char *ent) -{ - char *basename; - char *ret; - - if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) - return NULL; /* persistence not requested */ - if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (h->cfg, - "fs", - "STATE_DIR", - &basename)) - return NULL; - GNUNET_asprintf (&ret, - "%s%s%s%s%s%s%s.dir%s%s", - basename, - DIR_SEPARATOR_STR, - h->client_name, - DIR_SEPARATOR_STR, - ext, - DIR_SEPARATOR_STR, - uni, - DIR_SEPARATOR_STR, - ent); - GNUNET_free (basename); - return ret; -} - - -/** - * Return a read handle for deserialization. - * - * @param h master context - * @param ext component of the path - * @param ent entity identifier (or empty string for the directory) - * @return NULL on error - */ -static struct GNUNET_BIO_ReadHandle * -get_read_handle (struct GNUNET_FS_Handle *h, const char *ext, const char *ent) -{ - char *fn; - struct GNUNET_BIO_ReadHandle *ret; - - fn = get_serialization_file_name (h, ext, ent); - if (NULL == fn) - return NULL; - ret = GNUNET_BIO_read_open_file (fn); - GNUNET_free (fn); - return ret; -} - - -/** - * Return a write handle for serialization. - * - * @param h master context - * @param ext component of the path - * @param ent entity identifier (or empty string for the directory) - * @return NULL on error - */ -static struct GNUNET_BIO_WriteHandle * -get_write_handle (struct GNUNET_FS_Handle *h, const char *ext, const char *ent) -{ - char *fn; - struct GNUNET_BIO_WriteHandle *ret; - - fn = get_serialization_file_name (h, ext, ent); - if (NULL == fn) - return NULL; - ret = GNUNET_BIO_write_open_file (fn); - GNUNET_break (NULL != ret); - GNUNET_free (fn); - return ret; -} - - -/** - * Return a write handle for serialization. - * - * @param h master context - * @param ext component of the path - * @param uni name of parent - * @param ent entity identifier (or empty string for the directory) - * @return NULL on error - */ -static struct GNUNET_BIO_WriteHandle * -get_write_handle_in_dir (struct GNUNET_FS_Handle *h, - const char *ext, - const char *uni, - const char *ent) -{ - char *fn; - struct GNUNET_BIO_WriteHandle *ret; - - fn = get_serialization_file_name_in_dir (h, ext, uni, ent); - if (NULL == fn) - return NULL; - ret = GNUNET_BIO_write_open_file (fn); - GNUNET_free (fn); - return ret; -} - - -/** - * Remove serialization/deserialization file from disk. - * - * @param h master context - * @param ext component of the path - * @param ent entity identifier - */ -void -GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, - const char *ext, - const char *ent) -{ - char *filename; - - if ((NULL == ent) || (0 == strlen (ent))) - { - GNUNET_break (0); - return; - } - filename = get_serialization_file_name (h, ext, ent); - if (NULL != filename) - { - if ((0 != unlink (filename)) && (ENOENT != errno)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); - GNUNET_free (filename); - } -} - - -/** - * Remove serialization/deserialization file from disk. - * - * @param h master context - * @param ext component of the path - * @param uni parent name - * @param ent entity identifier - */ -static void -remove_sync_file_in_dir (struct GNUNET_FS_Handle *h, - const char *ext, - const char *uni, - const char *ent) -{ - char *filename; - - if ((NULL == ent) || (0 == strlen (ent))) - { - GNUNET_break (0); - return; - } - filename = get_serialization_file_name_in_dir (h, ext, uni, ent); - if (NULL == filename) - return; - if (0 != unlink (filename)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); - GNUNET_free (filename); -} - - -/** - * Remove serialization/deserialization directory from disk. - * - * @param h master context - * @param ext component of the path - * @param uni unique name of parent - */ -void -GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, - const char *ext, - const char *uni) -{ - char *dn; - - if (NULL == uni) - return; - dn = get_serialization_file_name_in_dir (h, ext, uni, ""); - if (NULL == dn) - return; - if ((GNUNET_YES == GNUNET_DISK_directory_test (dn, GNUNET_YES)) && - (GNUNET_OK != GNUNET_DISK_directory_remove (dn))) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "rmdir", dn); - GNUNET_free (dn); -} - - -/** - * Serialize a start-time. Since we use start-times to - * calculate the duration of some operation, we actually - * do not serialize the absolute time but the (relative) - * duration since the start time. When we then - * deserialize the start time, we take the current time and - * subtract that duration so that we get again an absolute - * time stamp that will result in correct performance - * calculations. - * - * @param wh handle for writing - * @param timestamp time to serialize - * @return #GNUNET_OK on success - */ -static int -write_start_time (struct GNUNET_BIO_WriteHandle *wh, - struct GNUNET_TIME_Absolute timestamp) -{ - struct GNUNET_TIME_Relative dur; - - dur = GNUNET_TIME_absolute_get_duration (timestamp); - return GNUNET_BIO_write_int64 (wh, "start time", dur.rel_value_us); -} - - -/** - * Deserialize a start-time. Since we use start-times to - * calculate the duration of some operation, we actually - * do not serialize the absolute time but the (relative) - * duration since the start time. Thus, when we then - * deserialize the start time, we take the current time and - * subtract that duration so that we get again an absolute - * time stamp that will result in correct performance - * calculations. - * - * @param rh handle for reading - * @param timestamp where to write the deserialized timestamp - * @return #GNUNET_OK on success - */ -static int -read_start_time (struct GNUNET_BIO_ReadHandle *rh, - struct GNUNET_TIME_Absolute *timestamp) -{ - struct GNUNET_TIME_Relative dur; - - if (GNUNET_OK != GNUNET_BIO_read_int64 (rh, "start time", - (int64_t *) &dur.rel_value_us)) - return GNUNET_SYSERR; - *timestamp = GNUNET_TIME_absolute_subtract (GNUNET_TIME_absolute_get (), dur); - return GNUNET_OK; -} - - -/** - * Using the given serialization filename, try to deserialize - * the file-information tree associated with it. - * - * @param h master context - * @param filename name of the file (without directory) with - * the information - * @return NULL on error - */ -static struct GNUNET_FS_FileInformation * -deserialize_file_information (struct GNUNET_FS_Handle *h, const char *filename); - - -/** - * Using the given serialization filename, try to deserialize - * the file-information tree associated with it. - * - * @param h master context - * @param fn name of the file (without directory) with - * the information - * @param rh handle for reading - * @return NULL on error - */ -static struct GNUNET_FS_FileInformation * -deserialize_fi_node (struct GNUNET_FS_Handle *h, - const char *fn, - struct GNUNET_BIO_ReadHandle *rh) -{ - struct GNUNET_FS_FileInformation *ret; - struct GNUNET_FS_FileInformation *nxt; - char b; - char *ksks; - char *chks; - char *skss; - char *filename; - uint32_t dsize; - - if (GNUNET_OK != GNUNET_BIO_read (rh, "status flag", &b, sizeof(b))) - { - GNUNET_break (0); - return NULL; - } - ret = GNUNET_new (struct GNUNET_FS_FileInformation); - ret->h = h; - ksks = NULL; - chks = NULL; - skss = NULL; - filename = NULL; - if ((GNUNET_OK != GNUNET_FS_read_meta_data (rh, "metadata", &ret->meta)) || - (GNUNET_OK != GNUNET_BIO_read_string (rh, "ksk-uri", &ksks, 32 * 1024)) || - ((NULL != ksks) && - ((NULL == (ret->keywords = GNUNET_FS_uri_parse (ksks, NULL))) || - (GNUNET_YES != GNUNET_FS_uri_test_ksk (ret->keywords)))) || - (GNUNET_OK != GNUNET_BIO_read_string (rh, "chk-uri", &chks, 1024)) || - ((NULL != chks) && - ((NULL == (ret->chk_uri = GNUNET_FS_uri_parse (chks, NULL))) || - (GNUNET_YES != GNUNET_FS_uri_test_chk (ret->chk_uri)))) || - (GNUNET_OK != GNUNET_BIO_read_string (rh, "sks-uri", &skss, 1024)) || - ((NULL != skss) && - ((NULL == (ret->sks_uri = GNUNET_FS_uri_parse (skss, NULL))) || - (GNUNET_YES != GNUNET_FS_uri_test_sks (ret->sks_uri)))) || - (GNUNET_OK != read_start_time (rh, &ret->start_time)) || - (GNUNET_OK != - GNUNET_BIO_read_string (rh, "emsg", &ret->emsg, 16 * 1024)) || - (GNUNET_OK != - GNUNET_BIO_read_string (rh, "fn", &ret->filename, 16 * 1024)) || - (GNUNET_OK != - GNUNET_BIO_read_int64 ( - rh, - "expiration time", - (int64_t *) &ret->bo.expiration_time.abs_value_us)) || - (GNUNET_OK != GNUNET_BIO_read_int32 ( - rh, - "anonymity level", - (int32_t *) &ret->bo.anonymity_level)) || - (GNUNET_OK != GNUNET_BIO_read_int32 ( - rh, - "content priority", - (int32_t *) &ret->bo.content_priority)) || - (GNUNET_OK != GNUNET_BIO_read_int32 ( - rh, - "replication level", - (int32_t *) &ret->bo.replication_level))) - { - GNUNET_break (0); - goto cleanup; - } - switch (b) - { - case 0: /* file-insert */ - if (GNUNET_OK != GNUNET_BIO_read_int64 ( - rh, - "file size", - (int64_t *) &ret->data.file.file_size)) - { - GNUNET_break (0); - goto cleanup; - } - ret->is_directory = GNUNET_NO; - ret->data.file.do_index = GNUNET_NO; - ret->data.file.have_hash = GNUNET_NO; - ret->data.file.index_start_confirmed = GNUNET_NO; - if (GNUNET_NO == ret->is_published) - { - if (NULL == ret->filename) - { - ret->data.file.reader = &GNUNET_FS_data_reader_copy_; - ret->data.file.reader_cls = - GNUNET_malloc_large (ret->data.file.file_size); - if (ret->data.file.reader_cls == NULL) - goto cleanup; - if (GNUNET_OK != GNUNET_BIO_read (rh, - "file-data", - ret->data.file.reader_cls, - ret->data.file.file_size)) - { - GNUNET_break (0); - goto cleanup; - } - } - else - { - ret->data.file.reader = &GNUNET_FS_data_reader_file_; - ret->data.file.reader_cls = - GNUNET_FS_make_file_reader_context_ (ret->filename); - } - } - break; - - case 1: /* file-index, no hash */ - if (NULL == ret->filename) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_OK != GNUNET_BIO_read_int64 ( - rh, - "file size", - (int64_t *) &ret->data.file.file_size)) - { - GNUNET_break (0); - goto cleanup; - } - ret->is_directory = GNUNET_NO; - ret->data.file.do_index = GNUNET_YES; - ret->data.file.have_hash = GNUNET_NO; - ret->data.file.index_start_confirmed = GNUNET_NO; - ret->data.file.reader = &GNUNET_FS_data_reader_file_; - ret->data.file.reader_cls = - GNUNET_FS_make_file_reader_context_ (ret->filename); - break; - - case 2: /* file-index-with-hash */ - if (NULL == ret->filename) - { - GNUNET_break (0); - goto cleanup; - } - if ((GNUNET_OK != GNUNET_BIO_read_int64 ( - rh, - "file size", - (int64_t *) &ret->data.file.file_size)) || - (GNUNET_OK != GNUNET_BIO_read (rh, - "fileid", - &ret->data.file.file_id, - sizeof(struct GNUNET_HashCode)))) - { - GNUNET_break (0); - goto cleanup; - } - ret->is_directory = GNUNET_NO; - ret->data.file.do_index = GNUNET_YES; - ret->data.file.have_hash = GNUNET_YES; - ret->data.file.index_start_confirmed = GNUNET_NO; - ret->data.file.reader = &GNUNET_FS_data_reader_file_; - ret->data.file.reader_cls = - GNUNET_FS_make_file_reader_context_ (ret->filename); - break; - - case 3: /* file-index-with-hash-confirmed */ - if (NULL == ret->filename) - { - GNUNET_break (0); - goto cleanup; - } - if ((GNUNET_OK != GNUNET_BIO_read_int64 ( - rh, - "file size", - (int64_t *) &ret->data.file.file_size)) || - (GNUNET_OK != GNUNET_BIO_read (rh, - "fileid", - &ret->data.file.file_id, - sizeof(struct GNUNET_HashCode)))) - { - GNUNET_break (0); - goto cleanup; - } - ret->is_directory = GNUNET_NO; - ret->data.file.do_index = GNUNET_YES; - ret->data.file.have_hash = GNUNET_YES; - ret->data.file.index_start_confirmed = GNUNET_YES; - ret->data.file.reader = &GNUNET_FS_data_reader_file_; - ret->data.file.reader_cls = - GNUNET_FS_make_file_reader_context_ (ret->filename); - break; - - case 4: /* directory */ - ret->is_directory = GNUNET_YES; - if ((GNUNET_OK != GNUNET_BIO_read_int32 (rh, "dsize", - (int32_t *) &dsize)) || - (GNUNET_OK != - GNUNET_BIO_read_int64 ( - rh, - "contents completed", - (int64_t *) &ret->data.dir.contents_completed)) || - (GNUNET_OK != - GNUNET_BIO_read_int64 ( - rh, - "contents size", - (int64_t *) &ret->data.dir.contents_size)) || - (NULL == (ret->data.dir.dir_data = GNUNET_malloc_large (dsize))) || - (GNUNET_OK != - GNUNET_BIO_read (rh, "dir-data", ret->data.dir.dir_data, dsize)) || - (GNUNET_OK != - GNUNET_BIO_read_string (rh, "ent-filename", &filename, 16 * 1024))) - { - GNUNET_break (0); - goto cleanup; - } - ret->data.dir.dir_size = (uint32_t) dsize; - if (NULL != filename) - { - ret->data.dir.entries = deserialize_file_information (h, filename); - GNUNET_free (filename); - filename = NULL; - nxt = ret->data.dir.entries; - while (NULL != nxt) - { - nxt->dir = ret; - nxt = nxt->next; - } - } - break; - - default: - GNUNET_break (0); - goto cleanup; - } - ret->serialization = GNUNET_strdup (fn); - if (GNUNET_OK != - GNUNET_BIO_read_string (rh, "nxt-filename", &filename, 16 * 1024)) - { - GNUNET_break (0); - goto cleanup; - } - if (NULL != filename) - { - ret->next = deserialize_file_information (h, filename); - GNUNET_free (filename); - filename = NULL; - } - GNUNET_free (ksks); - GNUNET_free (skss); - GNUNET_free (chks); - return ret; -cleanup: - GNUNET_free (ksks); - GNUNET_free (chks); - GNUNET_free (skss); - GNUNET_free (filename); - GNUNET_FS_file_information_destroy (ret, NULL, NULL); - return NULL; -} - - -/** - * Using the given serialization filename, try to deserialize - * the file-information tree associated with it. - * - * @param h master context - * @param filename name of the file (without directory) with - * the information - * @return NULL on error - */ -static struct GNUNET_FS_FileInformation * -deserialize_file_information (struct GNUNET_FS_Handle *h, const char *filename) -{ - struct GNUNET_FS_FileInformation *ret; - struct GNUNET_BIO_ReadHandle *rh; - char *emsg; - char *fn; - - rh = get_read_handle (h, GNUNET_FS_SYNC_PATH_FILE_INFO, filename); - if (NULL == rh) - return NULL; - ret = deserialize_fi_node (h, filename, rh); - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to resume publishing information `%s': %s\n"), - filename, - emsg); - GNUNET_free (emsg); - } - if (NULL == ret) - { - fn = - get_serialization_file_name (h, GNUNET_FS_SYNC_PATH_FILE_INFO, filename); - if (NULL != fn) - { - if (0 != unlink (fn)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn); - GNUNET_free (fn); - } - } - return ret; -} - - -/** - * Given a serialization name (full absolute path), return the - * basename of the file (without the path), which must only - * consist of the 6 random characters. - * - * @param fullname name to extract the basename from - * @return copy of the basename, NULL on error - */ -static char * -get_serialization_short_name (const char *fullname) -{ - const char *end; - const char *nxt; - - end = NULL; - nxt = fullname; - /* FIXME: we could do this faster since we know - * the length of 'end'... */ - while ('\0' != *nxt) - { - if (DIR_SEPARATOR == *nxt) - end = nxt + 1; - nxt++; - } - if ((NULL == end) || (0 == strlen (end))) - { - GNUNET_break (0); - return NULL; - } - GNUNET_break (6 == strlen (end)); - return GNUNET_strdup (end); -} - - -/** - * Create a new random name for serialization. Also checks if persistence - * is enabled and returns NULL if not. - * - * @param h master context - * @param ext component of the path - * @return NULL on error - */ -static char * -make_serialization_file_name (struct GNUNET_FS_Handle *h, const char *ext) -{ - char *fn; - char *dn; - char *ret; - - if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) - return NULL; /* persistence not requested */ - dn = get_serialization_file_name (h, ext, ""); - if (NULL == dn) - return NULL; - if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dn)) - { - GNUNET_free (dn); - return NULL; - } - fn = GNUNET_DISK_mktemp (dn); - GNUNET_free (dn); - if (NULL == fn) - return NULL; /* epic fail */ - ret = get_serialization_short_name (fn); - GNUNET_free (fn); - return ret; -} - - -/** - * Create a new random name for serialization. Also checks if persistence - * is enabled and returns NULL if not. - * - * @param h master context - * @param ext component of the path - * @param uni name of parent - * @return NULL on error - */ -static char * -make_serialization_file_name_in_dir (struct GNUNET_FS_Handle *h, - const char *ext, - const char *uni) -{ - char *fn; - char *dn; - char *ret; - - if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) - return NULL; /* persistence not requested */ - dn = get_serialization_file_name_in_dir (h, ext, uni, ""); - if (NULL == dn) - return NULL; - if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dn)) - { - GNUNET_free (dn); - return NULL; - } - fn = GNUNET_DISK_mktemp (dn); - GNUNET_free (dn); - if (NULL == fn) - return NULL; /* epic fail */ - ret = get_serialization_short_name (fn); - GNUNET_free (fn); - return ret; -} - - -/** - * Copy all of the data from the reader to the write handle. - * - * @param wh write handle - * @param fi file with reader - * @return #GNUNET_OK on success - */ -static int -copy_from_reader (struct GNUNET_BIO_WriteHandle *wh, - struct GNUNET_FS_FileInformation *fi) -{ - char buf[32 * 1024]; - uint64_t off; - size_t ret; - size_t left; - char *emsg; - - emsg = NULL; - off = 0; - while (off < fi->data.file.file_size) - { - left = GNUNET_MIN (sizeof(buf), fi->data.file.file_size - off); - ret = - fi->data.file.reader (fi->data.file.reader_cls, off, left, buf, &emsg); - if (0 == ret) - { - GNUNET_free (emsg); - return GNUNET_SYSERR; - } - if (GNUNET_OK != GNUNET_BIO_write (wh, "copied from reader", buf, ret)) - return GNUNET_SYSERR; - off += ret; - } - return GNUNET_OK; -} - - -/** - * Create a temporary file on disk to store the current - * state of @a fi in. - * - * @param fi file information to sync with disk - */ -void -GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *fi) -{ - char *fn; - struct GNUNET_BIO_WriteHandle *wh; - char b; - char *ksks; - char *chks; - char *skss; - - if (NULL == fi->serialization) - fi->serialization = - make_serialization_file_name (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO); - if (NULL == fi->serialization) - return; - wh = - get_write_handle (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO, fi->serialization); - if (NULL == wh) - { - GNUNET_free (fi->serialization); - fi->serialization = NULL; - return; - } - if (GNUNET_YES == fi->is_directory) - b = 4; - else if (GNUNET_YES == fi->data.file.index_start_confirmed) - b = 3; - else if (GNUNET_YES == fi->data.file.have_hash) - b = 2; - else if (GNUNET_YES == fi->data.file.do_index) - b = 1; - else - b = 0; - if (NULL != fi->keywords) - ksks = GNUNET_FS_uri_to_string (fi->keywords); - else - ksks = NULL; - if (NULL != fi->chk_uri) - chks = GNUNET_FS_uri_to_string (fi->chk_uri); - else - chks = NULL; - if (NULL != fi->sks_uri) - skss = GNUNET_FS_uri_to_string (fi->sks_uri); - else - skss = NULL; - struct GNUNET_BIO_WriteSpec ws1[] = { - GNUNET_BIO_write_spec_object ("b", &b, sizeof (b)), - GNUNET_FS_write_spec_meta_data ("meta", fi->meta), - GNUNET_BIO_write_spec_string ("ksks", ksks), - GNUNET_BIO_write_spec_string ("chks", chks), - GNUNET_BIO_write_spec_string ("skss", skss), - GNUNET_BIO_write_spec_end (), - }; - struct GNUNET_BIO_WriteSpec ws2[] = { - GNUNET_BIO_write_spec_string ("emsg", fi->emsg), - GNUNET_BIO_write_spec_string ("filename", fi->filename), - GNUNET_BIO_write_spec_int64 ( - "expiration time", - (int64_t *) &fi->bo.expiration_time.abs_value_us), - GNUNET_BIO_write_spec_int32 ( - "anonymity level", - (int32_t *) &fi->bo.anonymity_level), - GNUNET_BIO_write_spec_int32 ( - "content priority", - (int32_t *) &fi->bo.content_priority), - GNUNET_BIO_write_spec_int32 ( - "replication level", - (int32_t *) &fi->bo.replication_level), - GNUNET_BIO_write_spec_end (), - }; - if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws1)) || - (GNUNET_OK != write_start_time (wh, fi->start_time)) || - (GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws2))) - { - GNUNET_break (0); - goto cleanup; - } - GNUNET_free (chks); - chks = NULL; - GNUNET_free (ksks); - ksks = NULL; - GNUNET_free (skss); - skss = NULL; - - switch (b) - { - case 0: /* file-insert */ - if (GNUNET_OK != GNUNET_BIO_write_int64 (wh, "file size", - fi->data.file.file_size)) - { - GNUNET_break (0); - goto cleanup; - } - if ((GNUNET_NO == fi->is_published) && (NULL == fi->filename)) - if (GNUNET_OK != copy_from_reader (wh, fi)) - { - GNUNET_break (0); - goto cleanup; - } - break; - - case 1: /* file-index, no hash */ - if (NULL == fi->filename) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_OK != GNUNET_BIO_write_int64 (wh, "file size", - fi->data.file.file_size)) - { - GNUNET_break (0); - goto cleanup; - } - break; - - case 2: /* file-index-with-hash */ - case 3: /* file-index-with-hash-confirmed */ - if (NULL == fi->filename) - { - GNUNET_break (0); - goto cleanup; - } - if ((GNUNET_OK != GNUNET_BIO_write_int64 (wh, "file size", - fi->data.file.file_size)) || - (GNUNET_OK != GNUNET_BIO_write (wh, - "file id", - &fi->data.file.file_id, - sizeof(struct GNUNET_HashCode)))) - { - GNUNET_break (0); - goto cleanup; - } - break; - - case 4: /* directory */ - if ((NULL != fi->data.dir.entries) && - (NULL == fi->data.dir.entries->serialization)) - GNUNET_FS_file_information_sync_ (fi->data.dir.entries); - struct GNUNET_BIO_WriteSpec ws[] = { - GNUNET_BIO_write_spec_int32 ("dir size", - (int32_t *) &fi->data.dir.dir_size), - GNUNET_BIO_write_spec_int64 ( - "contents completed", - (int64_t *) &fi->data.dir.contents_completed), - GNUNET_BIO_write_spec_int64 ("contents size", - (int64_t *) &fi->data.dir.contents_size), - GNUNET_BIO_write_spec_object ("dir data", - fi->data.dir.dir_data, - (uint32_t) fi->data.dir.dir_size), - GNUNET_BIO_write_spec_string ("dir entries", - (fi->data.dir.entries == NULL) - ? NULL - : fi->data.dir.entries->serialization), - GNUNET_BIO_write_spec_end (), - }; - if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws))) - { - GNUNET_break (0); - goto cleanup; - } - break; - - default: - GNUNET_assert (0); - goto cleanup; - } - if ((NULL != fi->next) && (NULL == fi->next->serialization)) - GNUNET_FS_file_information_sync_ (fi->next); - if (GNUNET_OK != GNUNET_BIO_write_string (wh, - "serialization", - (fi->next != NULL) - ? fi->next->serialization - : NULL)) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL)) - { - wh = NULL; - GNUNET_break (0); - goto cleanup; - } - return; /* done! */ -cleanup: - if (NULL != wh) - (void) GNUNET_BIO_write_close (wh, NULL); - GNUNET_free (chks); - GNUNET_free (ksks); - GNUNET_free (skss); - fn = get_serialization_file_name (fi->h, - GNUNET_FS_SYNC_PATH_FILE_INFO, - fi->serialization); - if (NULL != fn) - { - if (0 != unlink (fn)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn); - GNUNET_free (fn); - } - GNUNET_free (fi->serialization); - fi->serialization = NULL; -} - - -/** - * Find the entry in the file information struct where the - * serialization filename matches the given name. - * - * @param pos file information to search - * @param srch filename to search for - * @return NULL if srch was not found in this subtree - */ -static struct GNUNET_FS_FileInformation * -find_file_position (struct GNUNET_FS_FileInformation *pos, const char *srch) -{ - struct GNUNET_FS_FileInformation *r; - - while (NULL != pos) - { - if (0 == strcmp (srch, pos->serialization)) - return pos; - if ((GNUNET_YES == pos->is_directory) && - (NULL != (r = find_file_position (pos->data.dir.entries, srch)))) - return r; - pos = pos->next; - } - return NULL; -} - - -/** - * Signal the FS's progress function that we are resuming - * an upload. - * - * @param cls closure (of type `struct GNUNET_FS_PublishContext *`, for the parent (!)) - * @param fi the entry in the publish-structure - * @param length length of the file or directory - * @param meta metadata for the file or directory (can be modified) - * @param uri pointer to the keywords that will be used for this entry (can be modified) - * @param bo block options (can be modified) - * @param do_index should we index? - * @param client_info pointer to client context set upon creation (can be modified) - * @return #GNUNET_OK to continue (always) - */ -static int -fip_signal_resume (void *cls, - struct GNUNET_FS_FileInformation *fi, - uint64_t length, - struct GNUNET_FS_MetaData *meta, - struct GNUNET_FS_Uri **uri, - struct GNUNET_FS_BlockOptions *bo, - int *do_index, - void **client_info) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_ProgressInfo pi; - - if (GNUNET_YES == pc->skip_next_fi_callback) - { - pc->skip_next_fi_callback = GNUNET_NO; - return GNUNET_OK; - } - pi.status = GNUNET_FS_STATUS_PUBLISH_RESUME; - pi.value.publish.specifics.resume.message = fi->emsg; - pi.value.publish.specifics.resume.chk_uri = fi->chk_uri; - *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0); - if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) - { - /* process entries in directory */ - pc->skip_next_fi_callback = GNUNET_YES; - GNUNET_FS_file_information_inspect (fi, &fip_signal_resume, pc); - } - return GNUNET_OK; -} - - -/** - * Function called with a filename of serialized publishing operation - * to deserialize. - * - * @param cls the `struct GNUNET_FS_Handle *` - * @param filename complete filename (absolute path) - * @return #GNUNET_OK (continue to iterate) - */ -static int -deserialize_publish_file (void *cls, const char *filename) -{ - struct GNUNET_FS_Handle *h = cls; - struct GNUNET_BIO_ReadHandle *rh; - struct GNUNET_FS_PublishContext *pc; - int32_t options; - int32_t all_done; - int32_t have_ns; - char *fi_root; - struct GNUNET_CRYPTO_EcdsaPrivateKey ns; - char *fi_pos; - char *emsg; - - pc = GNUNET_new (struct GNUNET_FS_PublishContext); - pc->h = h; - pc->serialization = get_serialization_short_name (filename); - fi_root = NULL; - fi_pos = NULL; - rh = GNUNET_BIO_read_open_file (filename); - if (NULL == rh) - { - GNUNET_break (0); - goto cleanup; - } - struct GNUNET_BIO_ReadSpec rs[] = { - GNUNET_BIO_read_spec_string ("publish-nid", &pc->nid, 1024), - GNUNET_BIO_read_spec_string ("publish-nuid", &pc->nuid, 1024), - GNUNET_BIO_read_spec_int32 ("options", &options), - GNUNET_BIO_read_spec_int32 ("all done", &all_done), - GNUNET_BIO_read_spec_int32 ("have ns", &have_ns), - GNUNET_BIO_read_spec_string ("publish-firoot", &fi_root, 128), - GNUNET_BIO_read_spec_string ("publish-fipos", &fi_pos, 128), - GNUNET_BIO_read_spec_end (), - }; - if ((GNUNET_OK != GNUNET_BIO_read_spec_commit (rh, rs)) || - ((GNUNET_YES == have_ns) && - (GNUNET_OK != GNUNET_BIO_read (rh, "publish-ns", &ns, sizeof(ns))))) - { - GNUNET_break (0); - goto cleanup; - } - pc->options = options; - pc->all_done = all_done; - if (NULL == fi_root) - { - GNUNET_break (0); - goto cleanup; - } - pc->fi = deserialize_file_information (h, fi_root); - if (NULL == pc->fi) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_YES == have_ns) - { - pc->ns = GNUNET_new (struct GNUNET_CRYPTO_EcdsaPrivateKey); - *pc->ns = ns; - } - if ((0 == (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) && - (GNUNET_YES != pc->all_done)) - { - pc->dsh = GNUNET_DATASTORE_connect (h->cfg); - if (NULL == pc->dsh) - goto cleanup; - } - if (NULL != fi_pos) - { - pc->fi_pos = find_file_position (pc->fi, fi_pos); - GNUNET_free (fi_pos); - fi_pos = NULL; - if (NULL == pc->fi_pos) - { - /* failed to find position for resuming, outch! Will start from root! */ - GNUNET_break (0); - if (GNUNET_YES != pc->all_done) - pc->fi_pos = pc->fi; - } - } - GNUNET_free (fi_root); - fi_root = NULL; - /* generate RESUME event(s) */ - GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_resume, pc); - - /* re-start publishing (if needed)... */ - if (GNUNET_YES != pc->all_done) - { - GNUNET_assert (NULL == pc->upload_task); - pc->upload_task = - GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, - &GNUNET_FS_publish_main_, - pc); - } - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failure while resuming publishing operation `%s': %s\n"), - filename, - emsg); - GNUNET_free (emsg); - } - pc->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, pc); - return GNUNET_OK; -cleanup: - GNUNET_free (pc->nid); - GNUNET_free (pc->nuid); - GNUNET_free (fi_root); - GNUNET_free (fi_pos); - if ((NULL != rh) && (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to resume publishing operation `%s': %s\n"), - filename, - emsg); - GNUNET_free (emsg); - } - if (NULL != pc->fi) - GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL); - if (0 != unlink (filename)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); - GNUNET_free (pc->serialization); - GNUNET_free (pc); - return GNUNET_OK; -} - - -/** - * Synchronize this publishing struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param pc the struct to sync - */ -void -GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc) -{ - struct GNUNET_BIO_WriteHandle *wh; - int32_t have_ns; - - if (NULL == pc->serialization) - pc->serialization = - make_serialization_file_name (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH); - if (NULL == pc->serialization) - return; - if (NULL == pc->fi) - return; - if (NULL == pc->fi->serialization) - { - GNUNET_break (0); - return; - } - wh = get_write_handle (pc->h, - GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, - pc->serialization); - if (NULL == wh) - { - GNUNET_break (0); - goto cleanup; - } - have_ns = (NULL != pc->ns) ? GNUNET_YES : GNUNET_NO; - struct GNUNET_BIO_WriteSpec ws[] = { - GNUNET_BIO_write_spec_string ("nid", pc->nid), - GNUNET_BIO_write_spec_string ("nuid", pc->nuid), - GNUNET_BIO_write_spec_int32 ("options", (int32_t *) &pc->options), - GNUNET_BIO_write_spec_int32 ("all done", &pc->all_done), - GNUNET_BIO_write_spec_int32 ("have ns", &have_ns), - GNUNET_BIO_write_spec_string ("serialization", pc->fi->serialization), - GNUNET_BIO_write_spec_string ("pos serialization", (NULL == pc->fi_pos) - ? NULL - : pc->fi_pos->serialization), - GNUNET_BIO_read_spec_end () - }; - if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws)) || - ((NULL != pc->ns) && - (GNUNET_OK != - GNUNET_BIO_write (wh, - "ns", - pc->ns, - sizeof(struct GNUNET_CRYPTO_EcdsaPrivateKey))))) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL)) - { - wh = NULL; - GNUNET_break (0); - goto cleanup; - } - return; -cleanup: - if (NULL != wh) - (void) GNUNET_BIO_write_close (wh, NULL); - GNUNET_FS_remove_sync_file_ (pc->h, - GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, - pc->serialization); - GNUNET_free (pc->serialization); - pc->serialization = NULL; -} - - -/** - * Synchronize this unindex struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param uc the struct to sync - */ -void -GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc) -{ - struct GNUNET_BIO_WriteHandle *wh; - char *uris; - - if (NULL == uc->serialization) - uc->serialization = - make_serialization_file_name (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX); - if (NULL == uc->serialization) - return; - wh = get_write_handle (uc->h, - GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, - uc->serialization); - if (NULL == wh) - { - GNUNET_break (0); - goto cleanup; - } - if (NULL != uc->ksk_uri) - uris = GNUNET_FS_uri_to_string (uc->ksk_uri); - else - uris = NULL; - struct GNUNET_BIO_WriteSpec ws1[] = { - GNUNET_BIO_write_spec_string ("filename", uc->filename), - GNUNET_BIO_write_spec_int64 ("file size", (int64_t *) &uc->file_size), - GNUNET_BIO_write_spec_end (), - }; - struct GNUNET_BIO_WriteSpec ws2[] = { - GNUNET_BIO_write_spec_int32 ("state", (int32_t *) &uc->state), - GNUNET_BIO_write_spec_object ("hashkey", &uc->chk, - sizeof (struct ContentHashKey)), - GNUNET_BIO_write_spec_string ("uris", uris), - GNUNET_BIO_write_spec_int32 ("ksk offset", (int32_t *) &uc->ksk_offset), - GNUNET_BIO_write_spec_end (), - }; - if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws1)) || - (GNUNET_OK != write_start_time (wh, uc->start_time)) || - (GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws2)) || - ((uc->state == UNINDEX_STATE_FS_NOTIFY) && - (GNUNET_OK != GNUNET_BIO_write (wh, - "file id", - &uc->file_id, - sizeof(struct GNUNET_HashCode)))) || - ((uc->state == UNINDEX_STATE_ERROR) && - (GNUNET_OK != GNUNET_BIO_write_string (wh, "emsg", uc->emsg)))) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL)) - { - wh = NULL; - GNUNET_break (0); - goto cleanup; - } - return; -cleanup: - if (NULL != wh) - (void) GNUNET_BIO_write_close (wh, NULL); - GNUNET_FS_remove_sync_file_ (uc->h, - GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, - uc->serialization); - GNUNET_free (uc->serialization); - uc->serialization = NULL; -} - - -/** - * Serialize a download request. - * - * @param wh handle for writing the download request to disk - * @param dr the the request to write to disk - * @return #GNUNET_YES on success, #GNUNET_NO on error - */ -static int -write_download_request (struct GNUNET_BIO_WriteHandle *wh, - struct DownloadRequest *dr) -{ - unsigned int i; - struct GNUNET_BIO_WriteSpec ws[] = { - GNUNET_BIO_write_spec_int32 ("state", (int32_t *) &dr->state), - GNUNET_BIO_write_spec_int64 ("offset", (int64_t *) &dr->offset), - GNUNET_BIO_write_spec_int32 ("num children", (int32_t *) &dr->num_children), - GNUNET_BIO_write_spec_int32 ("depth", (int32_t *) &dr->depth), - GNUNET_BIO_write_spec_end (), - }; - - if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws))) - return GNUNET_NO; - if ((BRS_CHK_SET == dr->state) && - (GNUNET_OK != - GNUNET_BIO_write (wh, "hashkey", - &dr->chk, sizeof(struct ContentHashKey)))) - return GNUNET_NO; - for (i = 0; i < dr->num_children; i++) - if (GNUNET_NO == write_download_request (wh, dr->children[i])) - return GNUNET_NO; - return GNUNET_YES; -} - - -/** - * Read a download request tree. - * - * @param rh cadet to read from - * @return value the download request read from disk, NULL on error - */ -static struct DownloadRequest * -read_download_request (struct GNUNET_BIO_ReadHandle *rh) -{ - struct DownloadRequest *dr; - unsigned int i; - - dr = GNUNET_new (struct DownloadRequest); - struct GNUNET_BIO_ReadSpec rs[] = { - GNUNET_BIO_read_spec_int32 ("state", (int32_t *) &dr->state), - GNUNET_BIO_read_spec_int64 ("offset", (int64_t *) &dr->offset), - GNUNET_BIO_read_spec_int32 ("num children", (int32_t *) &dr->num_children), - GNUNET_BIO_read_spec_end (), - }; - if ((GNUNET_OK != GNUNET_BIO_read_spec_commit (rh, rs)) || - (dr->num_children > CHK_PER_INODE) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "depth", - (int32_t *) &dr->depth)) || - ((0 == dr->depth) && (dr->num_children > 0)) || - ((dr->depth > 0) && (0 == dr->num_children))) - { - GNUNET_break (0); - dr->num_children = 0; - goto cleanup; - } - if (dr->num_children > 0) - dr->children = - GNUNET_malloc (dr->num_children * sizeof(struct DownloadRequest *)); - switch (dr->state) - { - case BRS_INIT: - case BRS_RECONSTRUCT_DOWN: - case BRS_RECONSTRUCT_META_UP: - case BRS_RECONSTRUCT_UP: - break; - - case BRS_CHK_SET: - if (GNUNET_OK != - GNUNET_BIO_read (rh, "chk", &dr->chk, sizeof(struct ContentHashKey))) - goto cleanup; - break; - - case BRS_DOWNLOAD_DOWN: - case BRS_DOWNLOAD_UP: - case BRS_ERROR: - break; - - default: - GNUNET_break (0); - goto cleanup; - } - for (i = 0; i < dr->num_children; i++) - { - if (NULL == (dr->children[i] = read_download_request (rh))) - goto cleanup; - dr->children[i]->parent = dr; - } - return dr; -cleanup: - GNUNET_FS_free_download_request_ (dr); - return NULL; -} - - -/** - * Compute the name of the sync file (or directory) for the given download - * context. - * - * @param dc download context to compute for - * @param uni unique filename to use, use "" for the directory name - * @param ext extension to use, use ".dir" for our own subdirectory - * @return the expanded file name, NULL for none - */ -static char * -get_download_sync_filename (struct GNUNET_FS_DownloadContext *dc, - const char *uni, - const char *ext) -{ - char *par; - char *epar; - - if (dc->parent == NULL) - return get_serialization_file_name (dc->h, - (dc->search != NULL) - ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD - : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD, - uni); - if (NULL == dc->parent->serialization) - return NULL; - par = get_download_sync_filename (dc->parent, dc->parent->serialization, ""); - if (NULL == par) - return NULL; - GNUNET_asprintf (&epar, "%s.dir%s%s%s", par, DIR_SEPARATOR_STR, uni, ext); - GNUNET_free (par); - return epar; -} - - -/** - * Synchronize this download struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param dc the struct to sync - */ -void -GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc) -{ - struct GNUNET_BIO_WriteHandle *wh; - char *uris; - char *fn; - char *dir; - - if (0 != (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE)) - return; /* we don't sync probes */ - if (NULL == dc->serialization) - { - dir = get_download_sync_filename (dc, "", ""); - if (NULL == dir) - return; - if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dir)) - { - GNUNET_free (dir); - return; - } - fn = GNUNET_DISK_mktemp (dir); - GNUNET_free (dir); - if (NULL == fn) - return; - dc->serialization = get_serialization_short_name (fn); - } - else - { - fn = get_download_sync_filename (dc, dc->serialization, ""); - if (NULL == fn) - { - GNUNET_free (dc->serialization); - dc->serialization = NULL; - GNUNET_free (fn); - return; - } - } - wh = GNUNET_BIO_write_open_file (fn); - if (NULL == wh) - { - GNUNET_free (dc->serialization); - dc->serialization = NULL; - GNUNET_free (fn); - return; - } - GNUNET_assert ((GNUNET_YES == GNUNET_FS_uri_test_chk (dc->uri)) || - (GNUNET_YES == GNUNET_FS_uri_test_loc (dc->uri))); - uris = GNUNET_FS_uri_to_string (dc->uri); - struct GNUNET_BIO_WriteSpec ws1[] = { - GNUNET_BIO_write_spec_string ("uris", uris), - GNUNET_FS_write_spec_meta_data ("metadata", dc->meta), - GNUNET_BIO_write_spec_string ("emsg", dc->emsg), - GNUNET_BIO_write_spec_string ("filename", dc->filename), - GNUNET_BIO_write_spec_string ("temp filename", dc->temp_filename), - GNUNET_BIO_write_spec_int64 ("old file size", - (int64_t *) &dc->old_file_size), - GNUNET_BIO_write_spec_int64 ("offset", (int64_t *) &dc->offset), - GNUNET_BIO_write_spec_int64 ("length", (int64_t *) &dc->length), - GNUNET_BIO_write_spec_int64 ("completed", (int64_t *) &dc->completed), - GNUNET_BIO_write_spec_end (), - }; - struct GNUNET_BIO_WriteSpec ws2[] = { - GNUNET_BIO_write_spec_int32 ("anonymity", (int32_t *) &dc->anonymity), - GNUNET_BIO_write_spec_int32 ("options", (int32_t *) &dc->options), - GNUNET_BIO_write_spec_int32 ("has finished", (int32_t *) &dc->has_finished), - GNUNET_BIO_write_spec_end (), - }; - if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws1)) || - (GNUNET_OK != write_start_time (wh, dc->start_time)) || - (GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws2))) - { - GNUNET_break (0); - goto cleanup; - } - if (NULL == dc->emsg) - { - GNUNET_assert (dc->top_request != NULL); - if (GNUNET_YES != write_download_request (wh, dc->top_request)) - { - GNUNET_break (0); - goto cleanup; - } - } - GNUNET_free (uris); - uris = NULL; - if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL)) - { - wh = NULL; - GNUNET_break (0); - goto cleanup; - } - GNUNET_free (fn); - return; -cleanup: - if (NULL != wh) - (void) GNUNET_BIO_write_close (wh, NULL); - GNUNET_free (uris); - if (0 != unlink (fn)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn); - GNUNET_free (fn); - GNUNET_free (dc->serialization); - dc->serialization = NULL; -} - - -/** - * Synchronize this search result with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param sr the struct to sync - */ -void -GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr) -{ - struct GNUNET_BIO_WriteHandle *wh; - char *uris; - - if (NULL == sr->sc) - return; - uris = NULL; - if (NULL == sr->serialization) - sr->serialization = - make_serialization_file_name_in_dir (sr->h, - (sr->sc->psearch_result == NULL) - ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH - : GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sr->sc->serialization); - if (NULL == sr->serialization) - return; - wh = get_write_handle_in_dir (sr->h, - (sr->sc->psearch_result == NULL) - ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH - : GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sr->sc->serialization, - sr->serialization); - if (NULL == wh) - { - GNUNET_break (0); - goto cleanup; - } - uris = GNUNET_FS_uri_to_string (sr->uri); - struct GNUNET_BIO_WriteSpec ws[] = { - GNUNET_BIO_write_spec_string ("uris", uris), - GNUNET_BIO_write_spec_string ("download serialization", - (sr->download != NULL) - ? sr->download->serialization - : NULL), - GNUNET_BIO_write_spec_string ("update search serialization", - (sr->update_search != NULL) - ? sr->update_search->serialization - : NULL), - GNUNET_FS_write_spec_meta_data ("metadata", sr->meta), - GNUNET_BIO_write_spec_object ("key", &sr->key, - sizeof(struct GNUNET_HashCode)), - GNUNET_BIO_write_spec_int32 ("mandatory missing", - (int32_t *) &sr->mandatory_missing), - GNUNET_BIO_write_spec_int32 ("optional support", - (int32_t *) &sr->optional_support), - GNUNET_BIO_write_spec_int32 ("availability success", - (int32_t *) &sr->availability_success), - GNUNET_BIO_write_spec_int32 ("availability trials", - (int32_t *) &sr->availability_trials), - GNUNET_BIO_write_spec_end (), - }; - if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws))) - { - GNUNET_break (0); - goto cleanup; - } - if ((NULL != sr->uri) && (GNUNET_FS_URI_KSK == sr->sc->uri->type) && - (GNUNET_OK != - GNUNET_BIO_write (wh, - "keyword bitmap", - sr->keyword_bitmap, - (sr->sc->uri->data.ksk.keywordCount + 7) / 8))) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL)) - { - wh = NULL; - GNUNET_break (0); - goto cleanup; - } - GNUNET_free (uris); - return; -cleanup: - GNUNET_free (uris); - if (NULL != wh) - (void) GNUNET_BIO_write_close (wh, NULL); - remove_sync_file_in_dir (sr->h, - (NULL == sr->sc->psearch_result) - ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH - : GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sr->sc->serialization, - sr->serialization); - GNUNET_free (sr->serialization); - sr->serialization = NULL; -} - - -/** - * Synchronize this search struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param sc the struct to sync - */ -void -GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc) -{ - struct GNUNET_BIO_WriteHandle *wh; - char *uris; - char in_pause; - const char *category; - - category = (NULL == sc->psearch_result) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH - : GNUNET_FS_SYNC_PATH_CHILD_SEARCH; - if (NULL == sc->serialization) - sc->serialization = make_serialization_file_name (sc->h, category); - if (NULL == sc->serialization) - return; - uris = NULL; - wh = get_write_handle (sc->h, category, sc->serialization); - if (NULL == wh) - { - GNUNET_break (0); - goto cleanup; - } - GNUNET_assert ((GNUNET_YES == GNUNET_FS_uri_test_ksk (sc->uri)) || - (GNUNET_YES == GNUNET_FS_uri_test_sks (sc->uri))); - uris = GNUNET_FS_uri_to_string (sc->uri); - in_pause = (sc->task != NULL) ? 'r' : '\0'; - if ((GNUNET_OK != GNUNET_BIO_write_string (wh, "uris", uris)) || - (GNUNET_OK != write_start_time (wh, sc->start_time)) || - (GNUNET_OK != GNUNET_BIO_write_string (wh, "emsg", sc->emsg)) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, "options", - (uint32_t) sc->options)) || - (GNUNET_OK != GNUNET_BIO_write (wh, "in pause", - &in_pause, sizeof(in_pause))) || - (GNUNET_OK != GNUNET_BIO_write_int32 (wh, "anonymity", sc->anonymity))) - { - GNUNET_break (0); - goto cleanup; - } - GNUNET_free (uris); - uris = NULL; - if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL)) - { - wh = NULL; - GNUNET_break (0); - goto cleanup; - } - return; -cleanup: - if (NULL != wh) - (void) GNUNET_BIO_write_close (wh, NULL); - GNUNET_free (uris); - GNUNET_FS_remove_sync_file_ (sc->h, category, sc->serialization); - GNUNET_free (sc->serialization); - sc->serialization = NULL; -} - - -/** - * Function called with a filename of serialized unindexing operation - * to deserialize. - * - * @param cls the `struct GNUNET_FS_Handle *` - * @param filename complete filename (absolute path) - * @return #GNUNET_OK (continue to iterate) - */ -static int -deserialize_unindex_file (void *cls, const char *filename) -{ - struct GNUNET_FS_Handle *h = cls; - struct GNUNET_BIO_ReadHandle *rh; - struct GNUNET_FS_UnindexContext *uc; - struct GNUNET_FS_ProgressInfo pi; - char *emsg; - char *uris; - uint32_t state; - - uc = GNUNET_new (struct GNUNET_FS_UnindexContext); - uc->h = h; - uc->serialization = get_serialization_short_name (filename); - rh = GNUNET_BIO_read_open_file (filename); - if (NULL == rh) - { - GNUNET_break (0); - goto cleanup; - } - uris = NULL; - if ((GNUNET_OK != - GNUNET_BIO_read_string (rh, "unindex-fn", &uc->filename, 10 * 1024)) || - (GNUNET_OK != GNUNET_BIO_read_int64 (rh, "file size", - (int64_t *) &uc->file_size)) || - (GNUNET_OK != read_start_time (rh, &uc->start_time)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "state", - (int32_t *) &state)) || - (GNUNET_OK != - GNUNET_BIO_read (rh, "uri", &uc->chk, sizeof(struct ContentHashKey))) || - (GNUNET_OK != - GNUNET_BIO_read_string (rh, "unindex-kskuri", &uris, 10 * 1024)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "ksk offset", - (int32_t *) &uc->ksk_offset))) - { - GNUNET_free (uris); - GNUNET_break (0); - goto cleanup; - } - if (NULL != uris) - { - uc->ksk_uri = GNUNET_FS_uri_parse (uris, &emsg); - GNUNET_free (uris); - if (NULL == uc->ksk_uri) - { - GNUNET_break (0); - GNUNET_free (emsg); - goto cleanup; - } - } - if ((uc->ksk_offset > 0) && - ((NULL == uc->ksk_uri) || - (uc->ksk_offset > uc->ksk_uri->data.ksk.keywordCount))) - { - GNUNET_break (0); - goto cleanup; - } - uc->state = (enum UnindexState) state; - switch (state) - { - case UNINDEX_STATE_HASHING: - break; - - case UNINDEX_STATE_FS_NOTIFY: - if (GNUNET_OK != GNUNET_BIO_read (rh, - "unindex-hash", - &uc->file_id, - sizeof(struct GNUNET_HashCode))) - { - GNUNET_break (0); - goto cleanup; - } - break; - - case UNINDEX_STATE_DS_REMOVE: - case UNINDEX_STATE_EXTRACT_KEYWORDS: - case UNINDEX_STATE_DS_REMOVE_KBLOCKS: - break; - - case UNINDEX_STATE_COMPLETE: - break; - - case UNINDEX_STATE_ERROR: - if (GNUNET_OK != - GNUNET_BIO_read_string (rh, "unindex-emsg", &uc->emsg, 10 * 1024)) - { - GNUNET_break (0); - goto cleanup; - } - break; - - default: - GNUNET_break (0); - goto cleanup; - } - uc->top = GNUNET_FS_make_top (h, &GNUNET_FS_unindex_signal_suspend_, uc); - pi.status = GNUNET_FS_STATUS_UNINDEX_RESUME; - pi.value.unindex.specifics.resume.message = uc->emsg; - GNUNET_FS_unindex_make_status_ (&pi, - uc, - (uc->state == UNINDEX_STATE_COMPLETE) - ? uc->file_size - : 0); - switch (uc->state) - { - case UNINDEX_STATE_HASHING: - uc->fhc = GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, - uc->filename, - HASHING_BLOCKSIZE, - &GNUNET_FS_unindex_process_hash_, - uc); - break; - - case UNINDEX_STATE_FS_NOTIFY: - uc->state = UNINDEX_STATE_HASHING; - GNUNET_FS_unindex_process_hash_ (uc, &uc->file_id); - break; - - case UNINDEX_STATE_DS_REMOVE: - GNUNET_FS_unindex_do_remove_ (uc); - break; - - case UNINDEX_STATE_EXTRACT_KEYWORDS: - GNUNET_FS_unindex_do_extract_keywords_ (uc); - break; - - case UNINDEX_STATE_DS_REMOVE_KBLOCKS: - GNUNET_FS_unindex_do_remove_kblocks_ (uc); - break; - - case UNINDEX_STATE_COMPLETE: - case UNINDEX_STATE_ERROR: - /* no need to resume any operation, we were done */ - break; - - default: - break; - } - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failure while resuming unindexing operation `%s': %s\n"), - filename, - emsg); - GNUNET_free (emsg); - } - return GNUNET_OK; -cleanup: - GNUNET_free (uc->filename); - if ((NULL != rh) && (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to resume unindexing operation `%s': %s\n"), - filename, - emsg); - GNUNET_free (emsg); - } - if (NULL != uc->serialization) - GNUNET_FS_remove_sync_file_ (h, - GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, - uc->serialization); - GNUNET_free (uc->serialization); - GNUNET_free (uc); - return GNUNET_OK; -} - - -/** - * Deserialize a download. - * - * @param h overall context - * @param rh file to deserialize from - * @param parent parent download - * @param search associated search - * @param serialization name under which the search was serialized - */ -static void -deserialize_download (struct GNUNET_FS_Handle *h, - struct GNUNET_BIO_ReadHandle *rh, - struct GNUNET_FS_DownloadContext *parent, - struct GNUNET_FS_SearchResult *search, - const char *serialization); - - -/** - * Deserialize a search. - * - * @param h overall context - * @param rh file to deserialize from - * @param psearch_result parent search result - * @param serialization name under which the search was serialized - */ -static struct GNUNET_FS_SearchContext * -deserialize_search (struct GNUNET_FS_Handle *h, - struct GNUNET_BIO_ReadHandle *rh, - struct GNUNET_FS_SearchResult *psearch_result, - const char *serialization); - - -/** - * Function called with a filename of serialized search result - * to deserialize. - * - * @param cls the `struct GNUNET_FS_SearchContext *` - * @param filename complete filename (absolute path) - * @return #GNUNET_OK (continue to iterate) - */ -static int -deserialize_search_result (void *cls, const char *filename) -{ - struct GNUNET_FS_SearchContext *sc = cls; - char *serialized; - char *uris; - char *emsg; - char *download; - char *update_srch; - struct GNUNET_BIO_ReadHandle *rh; - struct GNUNET_BIO_ReadHandle *drh; - struct GNUNET_FS_SearchResult *sr; - - serialized = get_serialization_short_name (filename); - rh = GNUNET_BIO_read_open_file (filename); - if (NULL == rh) - { - if (NULL != serialized) - { - remove_sync_file_in_dir (sc->h, - (NULL == sc->psearch_result) - ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH - : GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sc->serialization, - serialized); - GNUNET_free (serialized); - } - return GNUNET_OK; - } - emsg = NULL; - uris = NULL; - download = NULL; - update_srch = NULL; - sr = GNUNET_new (struct GNUNET_FS_SearchResult); - sr->h = sc->h; - sr->sc = sc; - sr->serialization = serialized; - if ((GNUNET_OK != - GNUNET_BIO_read_string (rh, "result-uri", &uris, 10 * 1024)) || - (NULL == (sr->uri = GNUNET_FS_uri_parse (uris, &emsg))) || - (GNUNET_OK != - GNUNET_BIO_read_string (rh, "download-lnk", &download, 16)) || - (GNUNET_OK != - GNUNET_BIO_read_string (rh, "search-lnk", &update_srch, 16)) || - (GNUNET_OK != GNUNET_FS_read_meta_data (rh, "result-meta", &sr->meta)) || - (GNUNET_OK != GNUNET_BIO_read (rh, - "result-key", - &sr->key, - sizeof(struct GNUNET_HashCode))) || - (GNUNET_OK != GNUNET_BIO_read_int32 ( - rh, - "mandatory missing", - (int32_t *) &sr->mandatory_missing)) || - (GNUNET_OK != GNUNET_BIO_read_int32 ( - rh, - "optional support", - (int32_t *) &sr->optional_support)) || - (GNUNET_OK != GNUNET_BIO_read_int32 ( - rh, - "availability success", - (int32_t *) &sr->availability_success)) || - (GNUNET_OK != GNUNET_BIO_read_int32 ( - rh, - "availability trials", - (int32_t *) &sr->availability_trials))) - { - GNUNET_break (0); - goto cleanup; - } - if (GNUNET_FS_URI_KSK == sr->sc->uri->type) - { - sr->keyword_bitmap = GNUNET_malloc ( - (sr->sc->uri->data.ksk.keywordCount + 7) / 8); /* round up, count bits */ - if (GNUNET_OK != - GNUNET_BIO_read (rh, - "keyword-bitmap", - sr->keyword_bitmap, - (sr->sc->uri->data.ksk.keywordCount + 7) / 8)) - { - GNUNET_break (0); - goto cleanup; - } - } - GNUNET_free (uris); - if (NULL != download) - { - drh = get_read_handle (sc->h, GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD, download); - if (NULL != drh) - { - deserialize_download (sc->h, drh, NULL, sr, download); - if (GNUNET_OK != GNUNET_BIO_read_close (drh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to resume sub-download `%s': %s\n"), - download, - emsg); - GNUNET_free (emsg); - } - } - GNUNET_free (download); - } - if (NULL != update_srch) - { - drh = - get_read_handle (sc->h, GNUNET_FS_SYNC_PATH_CHILD_SEARCH, update_srch); - if (NULL != drh) - { - deserialize_search (sc->h, drh, sr, update_srch); - if (GNUNET_OK != GNUNET_BIO_read_close (drh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to resume sub-search `%s': %s\n"), - update_srch, - emsg); - GNUNET_free (emsg); - } - } - GNUNET_free (update_srch); - } - GNUNET_break (GNUNET_YES == GNUNET_CONTAINER_multihashmap_put ( - sc->master_result_map, - &sr->key, - sr, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE)); - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failure while resuming search operation `%s': %s\n"), - filename, - emsg); - GNUNET_free (emsg); - } - return GNUNET_OK; -cleanup: - GNUNET_free (download); - GNUNET_free (emsg); - GNUNET_free (uris); - GNUNET_free (update_srch); - if (NULL != sr->uri) - GNUNET_FS_uri_destroy (sr->uri); - if (NULL != sr->meta) - GNUNET_FS_meta_data_destroy (sr->meta); - GNUNET_free (sr->serialization); - GNUNET_free (sr); - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failure while resuming search operation `%s': %s\n"), - filename, - emsg); - GNUNET_free (emsg); - } - return GNUNET_OK; -} - - -/** - * Send the 'resume' signal to the callback; also actually - * resume the download (put it in the queue). Does this - * recursively for the top-level download and all child - * downloads. - * - * @param dc download to resume - */ -static void -signal_download_resume (struct GNUNET_FS_DownloadContext *dc) -{ - struct GNUNET_FS_DownloadContext *dcc; - struct GNUNET_FS_ProgressInfo pi; - - pi.status = GNUNET_FS_STATUS_DOWNLOAD_RESUME; - pi.value.download.specifics.resume.meta = dc->meta; - pi.value.download.specifics.resume.message = dc->emsg; - GNUNET_FS_download_make_status_ (&pi, dc); - dcc = dc->child_head; - while (NULL != dcc) - { - signal_download_resume (dcc); - dcc = dcc->next; - } -} - - -/** - * Signal resuming of a search to our clients (for the - * top level search and all sub-searches). - * - * @param sc search being resumed - */ -static void -signal_search_resume (struct GNUNET_FS_SearchContext *sc); - - -/** - * Iterator over search results signaling resume to the client for - * each result. - * - * @param cls closure, the `struct GNUNET_FS_SearchContext *` - * @param key current key code - * @param value value in the hash map, the `struct GNUNET_FS_SearchResult *` - * @return #GNUNET_YES (we should continue to iterate) - */ -static int -signal_result_resume (void *cls, const struct GNUNET_HashCode *key, void *value) -{ - struct GNUNET_FS_SearchContext *sc = cls; - struct GNUNET_FS_ProgressInfo pi; - struct GNUNET_FS_SearchResult *sr = value; - - if (0 == sr->mandatory_missing) - { - pi.status = GNUNET_FS_STATUS_SEARCH_RESUME_RESULT; - pi.value.search.specifics.resume_result.meta = sr->meta; - pi.value.search.specifics.resume_result.uri = sr->uri; - pi.value.search.specifics.resume_result.result = sr; - pi.value.search.specifics.resume_result.availability_rank = - 2 * sr->availability_success - sr->availability_trials; - pi.value.search.specifics.resume_result.availability_certainty = - sr->availability_trials; - pi.value.search.specifics.resume_result.applicability_rank = - sr->optional_support; - sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc); - } - if (NULL != sr->download) - { - signal_download_resume (sr->download); - } - else - { - GNUNET_FS_search_start_probe_ (sr); - } - if (NULL != sr->update_search) - signal_search_resume (sr->update_search); - return GNUNET_YES; -} - - -/** - * Free memory allocated by the search context and its children - * - * @param sc search context to free - */ -static void -free_search_context (struct GNUNET_FS_SearchContext *sc); - - -/** - * Iterator over search results freeing each. - * - * @param cls closure, the `struct GNUNET_FS_SearchContext *` - * @param key current key code - * @param value value in the hash map, the `struct GNUNET_FS_SearchResult *` - * @return #GNUNET_YES (we should continue to iterate) - */ -static int -free_result (void *cls, const struct GNUNET_HashCode *key, void *value) -{ - struct GNUNET_FS_SearchResult *sr = value; - - if (NULL != sr->update_search) - { - free_search_context (sr->update_search); - GNUNET_assert (NULL == sr->update_search); - } - GNUNET_FS_meta_data_destroy (sr->meta); - GNUNET_FS_uri_destroy (sr->uri); - GNUNET_free (sr); - return GNUNET_YES; -} - - -/** - * Free memory allocated by the search context and its children - * - * @param sc search context to free - */ -static void -free_search_context (struct GNUNET_FS_SearchContext *sc) -{ - if (NULL != sc->serialization) - { - GNUNET_FS_remove_sync_file_ (sc->h, - (sc->psearch_result == NULL) - ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH - : GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sc->serialization); - GNUNET_FS_remove_sync_dir_ (sc->h, - (sc->psearch_result == NULL) - ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH - : GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sc->serialization); - } - GNUNET_free (sc->serialization); - GNUNET_free (sc->emsg); - if (NULL != sc->uri) - GNUNET_FS_uri_destroy (sc->uri); - if (NULL != sc->master_result_map) - { - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &free_result, - sc); - GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map); - } - GNUNET_free (sc); -} - - -/** - * Function called with a filename of serialized sub-download - * to deserialize. - * - * @param cls the `struct GNUNET_FS_DownloadContext *` (parent) - * @param filename complete filename (absolute path) - * @return #GNUNET_OK (continue to iterate) - */ -static int -deserialize_subdownload (void *cls, const char *filename) -{ - struct GNUNET_FS_DownloadContext *parent = cls; - char *serialized; - char *emsg; - struct GNUNET_BIO_ReadHandle *rh; - - serialized = get_serialization_short_name (filename); - rh = GNUNET_BIO_read_open_file (filename); - if (NULL == rh) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ( - "Failed to resume sub-download `%s': could not open file `%s'\n"), - serialized, - filename); - GNUNET_free (serialized); - return GNUNET_OK; - } - deserialize_download (parent->h, rh, parent, NULL, serialized); - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to resume sub-download `%s': %s\n"), - serialized, - emsg); - GNUNET_free (emsg); - } - GNUNET_free (serialized); - return GNUNET_OK; -} - - -/** - * Free this download context and all of its descendants. - * (only works during deserialization since not all possible - * state it taken care of). - * - * @param dc context to free - */ -static void -free_download_context (struct GNUNET_FS_DownloadContext *dc) -{ - struct GNUNET_FS_DownloadContext *dcc; - - if (NULL != dc->meta) - GNUNET_FS_meta_data_destroy (dc->meta); - if (NULL != dc->uri) - GNUNET_FS_uri_destroy (dc->uri); - GNUNET_free (dc->temp_filename); - GNUNET_free (dc->emsg); - GNUNET_free (dc->filename); - GNUNET_free (dc->serialization); - while (NULL != (dcc = dc->child_head)) - { - GNUNET_CONTAINER_DLL_remove (dc->child_head, dc->child_tail, dcc); - free_download_context (dcc); - } - GNUNET_FS_free_download_request_ (dc->top_request); - if (NULL != dc->active) - GNUNET_CONTAINER_multihashmap_destroy (dc->active); - GNUNET_free (dc); -} - - -/** - * Deserialize a download. - * - * @param h overall context - * @param rh file to deserialize from - * @param parent parent download - * @param search associated search - * @param serialization name under which the search was serialized - */ -static void -deserialize_download (struct GNUNET_FS_Handle *h, - struct GNUNET_BIO_ReadHandle *rh, - struct GNUNET_FS_DownloadContext *parent, - struct GNUNET_FS_SearchResult *search, - const char *serialization) -{ - struct GNUNET_FS_DownloadContext *dc; - char *emsg; - char *uris; - char *dn; - uint32_t options; - uint32_t status; - - uris = NULL; - emsg = NULL; - dc = GNUNET_new (struct GNUNET_FS_DownloadContext); - dc->parent = parent; - dc->h = h; - dc->serialization = GNUNET_strdup (serialization); - struct GNUNET_BIO_ReadSpec rs[] = { - GNUNET_FS_read_spec_meta_data ("download-meta", &dc->meta), - GNUNET_BIO_read_spec_string ("download-emsg", &dc->emsg, 10 * 1024), - GNUNET_BIO_read_spec_string ("download-fn", &dc->filename, 10 * 1024), - GNUNET_BIO_read_spec_string ("download-tfn", - &dc->temp_filename, 10 * 1024), - GNUNET_BIO_read_spec_int64 ("old file size", - (int64_t *) &dc->old_file_size), - GNUNET_BIO_read_spec_int64 ("offset", - (int64_t *) &dc->offset), - GNUNET_BIO_read_spec_int64 ("length", - (int64_t *) &dc->length), - GNUNET_BIO_read_spec_int64 ("completed", - (int64_t *) &dc->completed), - GNUNET_BIO_read_spec_end (), - }; - if ((GNUNET_OK != - GNUNET_BIO_read_string (rh, "download-uri", &uris, 10 * 1024)) || - (NULL == (dc->uri = GNUNET_FS_uri_parse (uris, &emsg))) || - ((GNUNET_YES != GNUNET_FS_uri_test_chk (dc->uri)) && - (GNUNET_YES != GNUNET_FS_uri_test_loc (dc->uri))) || - (GNUNET_OK != GNUNET_BIO_read_spec_commit (rh, rs)) || - (GNUNET_OK != read_start_time (rh, &dc->start_time)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "anonymity", - (int32_t *) &dc->anonymity)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "options", - (int32_t *) &options)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "status", - (int32_t *) &status))) - { - GNUNET_break (0); - goto cleanup; - } - dc->options = (enum GNUNET_FS_DownloadOptions) options; - dc->active = - GNUNET_CONTAINER_multihashmap_create (1 + 2 * (dc->length / DBLOCK_SIZE), - GNUNET_NO); - dc->has_finished = (int) status; - dc->treedepth = - GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size (dc->uri)); - if (GNUNET_FS_uri_test_loc (dc->uri)) - GNUNET_assert (GNUNET_OK == - GNUNET_FS_uri_loc_get_peer_identity (dc->uri, &dc->target)); - if (NULL == dc->emsg) - { - dc->top_request = read_download_request (rh); - if (NULL == dc->top_request) - { - GNUNET_break (0); - goto cleanup; - } - } - dn = get_download_sync_filename (dc, dc->serialization, ".dir"); - if (NULL != dn) - { - if (GNUNET_YES == GNUNET_DISK_directory_test (dn, GNUNET_YES)) - GNUNET_DISK_directory_scan (dn, &deserialize_subdownload, dc); - GNUNET_free (dn); - } - if (NULL != parent) - { - GNUNET_CONTAINER_DLL_insert (parent->child_head, parent->child_tail, dc); - } - if (NULL != search) - { - dc->search = search; - search->download = dc; - } - if ((NULL == parent) && (NULL == search)) - { - dc->top = - GNUNET_FS_make_top (dc->h, &GNUNET_FS_download_signal_suspend_, dc); - signal_download_resume (dc); - } - GNUNET_free (uris); - GNUNET_assert (NULL == dc->job_queue); - dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); - return; -cleanup: - GNUNET_free (uris); - GNUNET_free (emsg); - free_download_context (dc); -} - - -/** - * Signal resuming of a search to our clients (for the - * top level search and all sub-searches). - * - * @param sc search being resumed - */ -static void -signal_search_resume (struct GNUNET_FS_SearchContext *sc) -{ - struct GNUNET_FS_ProgressInfo pi; - - pi.status = GNUNET_FS_STATUS_SEARCH_RESUME; - pi.value.search.specifics.resume.message = sc->emsg; - pi.value.search.specifics.resume.is_paused = - (NULL == sc->mq) ? GNUNET_YES : GNUNET_NO; - sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc); - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &signal_result_resume, - sc); -} - - -/** - * Deserialize a search. - * - * @param h overall context - * @param rh file to deserialize from - * @param psearch_result parent search result - * @param serialization name under which the search was serialized - */ -static struct GNUNET_FS_SearchContext * -deserialize_search (struct GNUNET_FS_Handle *h, - struct GNUNET_BIO_ReadHandle *rh, - struct GNUNET_FS_SearchResult *psearch_result, - const char *serialization) -{ - struct GNUNET_FS_SearchContext *sc; - char *emsg; - char *uris; - char *dn; - uint32_t options; - char in_pause; - - if ((NULL != psearch_result) && (NULL != psearch_result->update_search)) - { - GNUNET_break (0); - return NULL; - } - uris = NULL; - emsg = NULL; - sc = GNUNET_new (struct GNUNET_FS_SearchContext); - if (NULL != psearch_result) - { - sc->psearch_result = psearch_result; - psearch_result->update_search = sc; - } - sc->h = h; - sc->serialization = GNUNET_strdup (serialization); - if ((GNUNET_OK != - GNUNET_BIO_read_string (rh, "search-uri", &uris, 10 * 1024)) || - (NULL == (sc->uri = GNUNET_FS_uri_parse (uris, &emsg))) || - ((GNUNET_YES != GNUNET_FS_uri_test_ksk (sc->uri)) && - (GNUNET_YES != GNUNET_FS_uri_test_sks (sc->uri))) || - (GNUNET_OK != read_start_time (rh, &sc->start_time)) || - (GNUNET_OK != - GNUNET_BIO_read_string (rh, "search-emsg", &sc->emsg, 10 * 1024)) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "options", - (int32_t *) &options)) || - (GNUNET_OK != - GNUNET_BIO_read (rh, "search-pause", &in_pause, sizeof(in_pause))) || - (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "anonymity", - (int32_t *) &sc->anonymity))) - { - GNUNET_break (0); - goto cleanup; - } - sc->options = (enum GNUNET_FS_SearchOptions) options; - sc->master_result_map = GNUNET_CONTAINER_multihashmap_create (16, GNUNET_NO); - dn = get_serialization_file_name_in_dir (h, - (NULL == sc->psearch_result) - ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH - : GNUNET_FS_SYNC_PATH_CHILD_SEARCH, - sc->serialization, - ""); - if (NULL != dn) - { - if (GNUNET_YES == GNUNET_DISK_directory_test (dn, GNUNET_YES)) - GNUNET_DISK_directory_scan (dn, &deserialize_search_result, sc); - GNUNET_free (dn); - } - if (('\0' == in_pause) && - (GNUNET_OK != GNUNET_FS_search_start_searching_ (sc))) - { - GNUNET_log ( - GNUNET_ERROR_TYPE_WARNING, - _ ("Could not resume running search, will resume as paused search\n")); - } - signal_search_resume (sc); - GNUNET_free (uris); - return sc; -cleanup: - GNUNET_free (emsg); - free_search_context (sc); - GNUNET_free (uris); - return NULL; -} - - -/** - * Function called with a filename of serialized search operation - * to deserialize. - * - * @param cls the `struct GNUNET_FS_Handle *` - * @param filename complete filename (absolute path) - * @return #GNUNET_OK (continue to iterate) - */ -static int -deserialize_search_file (void *cls, const char *filename) -{ - struct GNUNET_FS_Handle *h = cls; - char *set; - char *emsg; - struct GNUNET_BIO_ReadHandle *rh; - struct GNUNET_FS_SearchContext *sc; - struct stat buf; - - if (0 != stat (filename, &buf)) - { - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "stat", filename); - return GNUNET_OK; - } - if (S_ISDIR (buf.st_mode)) - return GNUNET_OK; /* skip directories */ - set = get_serialization_short_name (filename); - rh = GNUNET_BIO_read_open_file (filename); - if (NULL == rh) - { - if (NULL != set) - { - GNUNET_FS_remove_sync_file_ (h, GNUNET_FS_SYNC_PATH_MASTER_SEARCH, set); - GNUNET_free (set); - } - return GNUNET_OK; - } - sc = deserialize_search (h, rh, NULL, set); - if (NULL != sc) - sc->top = GNUNET_FS_make_top (h, &GNUNET_FS_search_signal_suspend_, sc); - GNUNET_free (set); - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failure while resuming search operation `%s': %s\n"), - filename, - emsg); - GNUNET_free (emsg); - } - return GNUNET_OK; -} - - -/** - * Function called with a filename of serialized download operation - * to deserialize. - * - * @param cls the `struct GNUNET_FS_Handle *` - * @param filename complete filename (absolute path) - * @return #GNUNET_OK (continue to iterate) - */ -static int -deserialize_download_file (void *cls, const char *filename) -{ - struct GNUNET_FS_Handle *h = cls; - char *set; - char *emsg; - struct GNUNET_BIO_ReadHandle *rh; - - set = get_serialization_short_name (filename); - rh = GNUNET_BIO_read_open_file (filename); - if (NULL == rh) - { - if (0 != unlink (filename)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, - "unlink", - filename); - GNUNET_free (set); - return GNUNET_OK; - } - deserialize_download (h, rh, NULL, NULL, set); - GNUNET_free (set); - if (GNUNET_OK != - GNUNET_BIO_read_close (rh, - &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - "Failure while resuming download operation `%s': %s\n", - filename, - emsg); - GNUNET_free (emsg); - } - return GNUNET_OK; -} - - -/** - * Deserialize information about pending operations. - * - * @param master_path which master directory should be scanned - * @param proc function to call for each entry (will get @a h for 'cls') - * @param h the `struct GNUNET_FS_Handle *` - */ -static void -deserialization_master (const char *master_path, - GNUNET_FileNameCallback proc, - struct GNUNET_FS_Handle *h) -{ - char *dn; - - dn = get_serialization_file_name (h, master_path, ""); - if (NULL == dn) - return; - if (GNUNET_YES == - GNUNET_DISK_directory_test (dn, - GNUNET_YES)) - GNUNET_DISK_directory_scan (dn, - proc, - h); - GNUNET_free (dn); -} - - -struct GNUNET_FS_Handle * -GNUNET_FS_start (const struct GNUNET_CONFIGURATION_Handle *cfg, - const char *client_name, - GNUNET_FS_ProgressCallback upcb, - void *upcb_cls, - enum GNUNET_FS_Flags flags, - ...) -{ - struct GNUNET_FS_Handle *ret; - enum GNUNET_FS_OPTIONS opt; - va_list ap; - - ret = GNUNET_new (struct GNUNET_FS_Handle); - ret->cfg = cfg; - ret->client_name = GNUNET_strdup (client_name); - ret->upcb = upcb; - ret->upcb_cls = upcb_cls; - ret->flags = flags; - ret->max_parallel_downloads = DEFAULT_MAX_PARALLEL_DOWNLOADS; - ret->max_parallel_requests = DEFAULT_MAX_PARALLEL_REQUESTS; - ret->avg_block_latency = - GNUNET_TIME_UNIT_MINUTES; /* conservative starting point */ - va_start (ap, flags); - while (GNUNET_FS_OPTIONS_END != - (opt = GNUNET_VA_ARG_ENUM (ap, GNUNET_FS_OPTIONS))) - { - switch (opt) - { - case GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM: - ret->max_parallel_downloads = va_arg (ap, unsigned int); - - break; - - case GNUNET_FS_OPTIONS_REQUEST_PARALLELISM: - ret->max_parallel_requests = va_arg (ap, unsigned int); - - break; - - default: - GNUNET_break (0); - GNUNET_free (ret->client_name); - GNUNET_free (ret); - va_end (ap); - return NULL; - } - } - va_end (ap); - if (0 != (GNUNET_FS_FLAGS_PERSISTENCE & flags)) - { - deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, - &deserialize_publish_file, - ret); - deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_SEARCH, - &deserialize_search_file, - ret); - deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD, - &deserialize_download_file, - ret); - deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, - &deserialize_unindex_file, - ret); - } - return ret; -} - - -void -GNUNET_FS_stop (struct GNUNET_FS_Handle *h) -{ - while (NULL != h->top_head) - h->top_head->ssf (h->top_head->ssf_cls); - if (NULL != h->queue_job) - GNUNET_SCHEDULER_cancel (h->queue_job); - GNUNET_free (h->client_name); - GNUNET_free (h); -} - - -/* end of fs_api.c */ diff --git a/src/fs/fs_api.h b/src/fs/fs_api.h deleted file mode 100644 index fdda91928..000000000 --- a/src/fs/fs_api.h +++ /dev/null @@ -1,1941 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_api.h - * @brief shared definitions for the FS library - * @author Igor Wronsky, Christian Grothoff - */ -#ifndef FS_API_H -#define FS_API_H - -#include "gnunet_constants.h" -#include "gnunet_datastore_service.h" -#include "gnunet_dht_service.h" - -#include "gnunet_fs_service.h" -#include "gnunet_block_lib.h" -#include "block_fs.h" -#include "fs.h" - -/** - * Pick a multiple of 2 here to achieve 8-byte alignment! We also - * probably want DBlocks to have (roughly) the same size as IBlocks. - * With SHA-512, the optimal value is 32768 byte / 128 byte = 256 (128 - * byte = 2 * 512 bits). DO NOT CHANGE! - */ -#define CHK_PER_INODE 256 - -/** - * Maximum size for a file to be considered for inlining in a - * directory. - */ -#define MAX_INLINE_SIZE 65536 - -/** - * Name of the directory with top-level searches. - */ -#define GNUNET_FS_SYNC_PATH_MASTER_SEARCH "search" - -/** - * Name of the directory with sub-searches (namespace-updates). - */ -#define GNUNET_FS_SYNC_PATH_CHILD_SEARCH "search-child" - -/** - * Name of the directory with master downloads (not associated - * with search or part of another download). - */ -#define GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD "download" - -/** - * Name of the directory with downloads that are part of another - * download or a search. - */ -#define GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD "download-child" - -/** - * Name of the directory with publishing operations. - */ -#define GNUNET_FS_SYNC_PATH_MASTER_PUBLISH "publish" - -/** - * Name of the directory with files that are being published - */ -#define GNUNET_FS_SYNC_PATH_FILE_INFO "publish-file" - -/** - * Name of the directory with unindex operations. - */ -#define GNUNET_FS_SYNC_PATH_MASTER_UNINDEX "unindex" - - -/** - * @brief complete information needed - * to download a file. - */ -struct FileIdentifier -{ - /** - * Total size of the file in bytes. (network byte order (!)) - */ - uint64_t file_length; - - /** - * Query and key of the top GNUNET_EC_IBlock. - */ - struct ContentHashKey chk; -}; - - -/** - * Information about a file and its location - * (peer claiming to share the file). - */ -struct Location -{ - /** - * Information about the shared file. - */ - struct FileIdentifier fi; - - /** - * Identity of the peer sharing the file. - */ - struct GNUNET_PeerIdentity peer; - - /** - * Time when this location URI expires. - */ - struct GNUNET_TIME_Absolute expirationTime; - - /** - * Signature over the GNUNET_EC_FileIdentifier, - * peer identity and expiration time. - */ - struct GNUNET_CRYPTO_EddsaSignature contentSignature; -}; - -/** - * Types of URIs. - */ -enum GNUNET_FS_UriType -{ - /** - * Content-hash-key (simple file). - */ - GNUNET_FS_URI_CHK, - - /** - * Signed key space (file in namespace). - */ - GNUNET_FS_URI_SKS, - - /** - * Keyword search key (query with keywords). - */ - GNUNET_FS_URI_KSK, - - /** - * Location (chk with identity of hosting peer). - */ - GNUNET_FS_URI_LOC -}; - - -/** - * A Universal Resource Identifier (URI), opaque. - */ -struct GNUNET_FS_Uri -{ - /** - * Type of the URI. - */ - enum GNUNET_FS_UriType type; - - union - { - struct - { - /** - * Keywords start with a '+' if they are mandatory (in which - * case the '+' is NOT part of the keyword) and with a simple - * space if they are optional (in which case the space is ALSO - * not part of the actual keyword). - * - * Double-quotes to protect spaces and %-encoding are NOT used - * internally (only in URI-strings). - */ - char **keywords; - - /** - * Size of the keywords array. - */ - unsigned int keywordCount; - } ksk; - - struct - { - /** - * Identifier of the namespace. - */ - struct GNUNET_CRYPTO_EcdsaPublicKey ns; - - /** - * Human-readable identifier chosen for this entry in the - * namespace. - */ - char *identifier; - } sks; - - /** - * Information needed to retrieve a file (content-hash-key - * plus file size). - */ - struct FileIdentifier chk; - - /** - * Information needed to retrieve a file including signed - * location (identity of a peer) of the content. - */ - struct Location loc; - } data; -}; - - -/** - * Information for a file or directory that is - * about to be published. - */ -struct GNUNET_FS_FileInformation -{ - /** - * Files in a directory are kept as a linked list. - */ - struct GNUNET_FS_FileInformation *next; - - /** - * If this is a file in a directory, "dir" refers to - * the directory; otherwise NULL. - */ - struct GNUNET_FS_FileInformation *dir; - - /** - * Handle to the master context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Pointer kept for the client. - */ - void *client_info; - - /** - * Metadata to use for the file. - */ - struct GNUNET_FS_MetaData *meta; - - /** - * Keywords to use for KBlocks. - */ - struct GNUNET_FS_Uri *keywords; - - /** - * CHK for this file or directory. NULL if - * we have not yet computed it. - */ - struct GNUNET_FS_Uri *chk_uri; - - /** - * SKS URI for this file or directory. NULL if - * we have not yet computed it. - */ - struct GNUNET_FS_Uri *sks_uri; - - /** - * Block options for the file. - */ - struct GNUNET_FS_BlockOptions bo; - - /** - * At what time did we start this upload? - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * Under what filename is this struct serialized - * (for operational persistence). Should be determined - * using 'mktemp'. - */ - char *serialization; - - /** - * Encoder being used to publish this file. - */ - struct GNUNET_FS_TreeEncoder *te; - - /** - * Error message (non-NULL if this operation failed). - */ - char *emsg; - - /** - * Name of the file or directory (must be an absolute path). - */ - char *filename; - - /** - * Data describing either the file or the directory. - */ - union - { - /** - * Data for a file. - */ - struct - { - /** - * Function that can be used to read the data for the file. - */ - GNUNET_FS_DataReader reader; - - /** - * Closure for reader. - */ - void *reader_cls; - - /** - * If this file is being indexed, this value is set to the hash - * over the entire file (when the indexing process is started). - * Otherwise this field is not used. - */ - struct GNUNET_HashCode file_id; - - /** - * Size of the file (in bytes). - */ - uint64_t file_size; - - /** - * Should the file be indexed or inserted? - */ - int do_index; - - /** - * Is "file_id" already valid? Set to #GNUNET_YES once the hash - * has been calculated. - */ - int have_hash; - - /** - * Has the service confirmed our INDEX_START request? - * #GNUNET_YES if this step has been completed. - */ - int index_start_confirmed; - } file; - - /** - * Data for a directory. - */ - struct - { - /** - * Linked list of entries in the directory. - */ - struct GNUNET_FS_FileInformation *entries; - - /** - * Size of the directory itself (in bytes); 0 if the - * size has not yet been calculated. - */ - size_t dir_size; - - /** - * Pointer to the data for the directory (or NULL if not - * available). - */ - void *dir_data; - - /** - * How much of the directory have we published (relative to @e contents_size). - */ - uint64_t contents_completed; - - /** - * Sum of all of the sizes of all of the files in the directory. - */ - uint64_t contents_size; - } dir; - } data; - - /** - * Is this struct for a file or directory? - */ - int is_directory; - - /** - * Are we done publishing this file? - */ - int is_published; -}; - - -/** - * Priorities for the queue. - */ -enum GNUNET_FS_QueuePriority -{ - /** - * This is a probe (low priority). - */ - GNUNET_FS_QUEUE_PRIORITY_PROBE, - - /** - * Default priority. - */ - GNUNET_FS_QUEUE_PRIORITY_NORMAL -}; - - -/** - * Entry in the job queue. - */ -struct GNUNET_FS_QueueEntry -{ - /** - * This is a linked list. - */ - struct GNUNET_FS_QueueEntry *next; - - /** - * This is a linked list. - */ - struct GNUNET_FS_QueueEntry *prev; - - /** - * Function to call when the job is started. - */ - GNUNET_SCHEDULER_TaskCallback start; - - /** - * Function to call when the job needs to stop (or is done / dequeued). - */ - GNUNET_SCHEDULER_TaskCallback stop; - - /** - * Closure for start and stop. - */ - void *cls; - - /** - * Handle to FS primary context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Message queue handle, or NULL if job is not running. - */ - struct GNUNET_MQ_Handle *mq; - - /** - * Time the job was originally queued. - */ - struct GNUNET_TIME_Absolute queue_time; - - /** - * Time the job was started last. - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * Total amount of time the job has been running (except for the - * current run). - */ - struct GNUNET_TIME_Relative run_time; - - /** - * How many blocks do the active downloads have? - */ - unsigned int blocks; - - /** - * How important is this download? - */ - enum GNUNET_FS_QueuePriority priority; - - /** - * How often have we (re)started this download? - */ - unsigned int start_times; - - /** - * #GNUNET_YES if the job is active now. - */ - int active; -}; - - -/** - * Information we store for each search result. - */ -struct GNUNET_FS_SearchResult -{ - /** - * File-sharing context this result belongs to. - */ - struct GNUNET_FS_Handle *h; - - /** - * Kept in a DLL while probing. - */ - struct GNUNET_FS_SearchResult *next; - - /** - * Kept in a DLL while probing. - */ - struct GNUNET_FS_SearchResult *prev; - - /** - * Search context this result belongs to; can be NULL - * for probes that come from a directory result. - */ - struct GNUNET_FS_SearchContext *sc; - - /** - * URI to which this search result refers to. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Metadata for the search result. - */ - struct GNUNET_FS_MetaData *meta; - - /** - * Client info for this search result. - */ - void *client_info; - - /** - * ID of a job that is currently probing this results' availability - * (NULL if we are not currently probing). - */ - struct GNUNET_FS_DownloadContext *probe_ctx; - - /** - * ID of an associated download based on this search result (or - * NULL for none). - */ - struct GNUNET_FS_DownloadContext *download; - - /** - * If this search result triggered an update search, this field - * links to the update search. - */ - struct GNUNET_FS_SearchContext *update_search; - - /** - * Name under which this search result is stored on disk. - */ - char *serialization; - - /** - * Bitmap that specifies precisely which keywords have been matched already. - */ - uint8_t *keyword_bitmap; - - /** - * Key for the search result based on the URI. - */ - struct GNUNET_HashCode key; - - /** - * ID of the task that will clean up the probe_ctx should it not - * complete on time (and that will need to be cancelled if we clean - * up the search result before then). - */ - struct GNUNET_SCHEDULER_Task *probe_cancel_task; - - /** - * When did the current probe become active? - */ - struct GNUNET_TIME_Absolute probe_active_time; - - /** - * How much longer should we run the current probe before giving up? - */ - struct GNUNET_TIME_Relative remaining_probe_time; - - /** - * Anonymity level to use for probes using this search result. - */ - uint32_t anonymity; - - /** - * Number of mandatory keywords for which we have NOT yet found the - * search result; when this value hits zero, the search result is - * given to the callback. - */ - uint32_t mandatory_missing; - - /** - * Number of optional keywords under which this result was also - * found. - */ - uint32_t optional_support; - - /** - * Number of availability tests that have succeeded for this result. - */ - uint32_t availability_success; - - /** - * Number of availability trials that we have performed for this - * search result. - */ - uint32_t availability_trials; -}; - - -/** - * Add a job to the queue. - * - * @param h handle to the overall FS state - * @param start function to call to begin the job - * @param stop function to call to pause the job, or on dequeue (if the job was running) - * @param cls closure for start and stop - * @param blocks number of blocks this download has - * @param priority how important is this download - * @return queue handle - */ -struct GNUNET_FS_QueueEntry * -GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, - GNUNET_SCHEDULER_TaskCallback start, - GNUNET_SCHEDULER_TaskCallback stop, - void *cls, - unsigned int blocks, - enum GNUNET_FS_QueuePriority priority); - - -/** - * Dequeue a job from the queue. - * - * @param qe handle for the job - */ -void -GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qe); - - -/** - * Function that provides data by reading from a file. - * - * @param cls closure (points to the file information) - * @param offset offset to read from; it is possible - * that the caller might need to go backwards - * a bit at times - * @param max maximum number of bytes that should be - * copied to @a buf; readers are not allowed - * to provide less data unless there is an error; - * a value of "0" will be used at the end to allow - * the reader to clean up its internal state - * @param buf where the reader should write the data - * @param emsg location for the reader to store an error message - * @return number of bytes written, usually "max", 0 on error - */ -size_t -GNUNET_FS_data_reader_file_ (void *cls, - uint64_t offset, - size_t max, - void *buf, - char **emsg); - - -/** - * Create the closure for the #GNUNET_FS_data_reader_file_() callback. - * - * @param filename file to read - * @return closure to use, NULL on error - */ -void * -GNUNET_FS_make_file_reader_context_ (const char *filename); - - -/** - * Function that provides data by copying from a buffer. - * - * @param cls closure (points to the buffer) - * @param offset offset to read from; it is possible - * that the caller might need to go backwards - * a bit at times - * @param max maximum number of bytes that should be - * copied to @a buf; readers are not allowed - * to provide less data unless there is an error; - * a value of "0" will be used at the end to allow - * the reader to clean up its internal state - * @param buf where the reader should write the data - * @param emsg location for the reader to store an error message - * @return number of bytes written, usually @a max, 0 on error - */ -size_t -GNUNET_FS_data_reader_copy_ (void *cls, - uint64_t offset, - size_t max, - void *buf, - char **emsg); - - -/** - * Notification of FS that a search probe has made progress. - * This function is used INSTEAD of the client's event handler - * for downloads where the #GNUNET_FS_DOWNLOAD_IS_PROBE flag is set. - * - * @param cls closure, always NULL (!), actual closure - * is in the client-context of the info struct - * @param info details about the event, specifying the event type - * and various bits about the event - * @return client-context (for the next progress call - * for this operation; should be set to NULL for - * SUSPEND and STOPPED events). The value returned - * will be passed to future callbacks in the respective - * field in the `struct GNUNET_FS_ProgressInfo`. - */ -void * -GNUNET_FS_search_probe_progress_ (void *cls, - const struct GNUNET_FS_ProgressInfo *info); - - -/** - * Main function that performs the upload. - * - * @param cls `struct GNUNET_FS_PublishContext` identifies the upload - */ -void -GNUNET_FS_publish_main_ (void *cls); - - -/** - * Function called once the hash of the file - * that is being unindexed has been computed. - * - * @param cls closure, unindex context - * @param file_id computed hash, NULL on error - */ -void -GNUNET_FS_unindex_process_hash_ (void *cls, - const struct GNUNET_HashCode *file_id); - - -/** - * Extract the keywords for KBlock removal - * - * @param uc context for the unindex operation. - */ -void -GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc); - - -/** - * If necessary, connect to the datastore and remove the KBlocks. - * - * @param uc context for the unindex operation. - */ -void -GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc); - - -/** - * Fill in all of the generic fields for a publish event and call the - * callback. - * - * @param pi structure to fill in - * @param pc overall publishing context - * @param p file information for the file being published - * @param offset where in the file are we so far - * @return value returned from callback - */ -void * -GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_PublishContext *pc, - const struct GNUNET_FS_FileInformation *p, - uint64_t offset); - - -/** - * Fill in all of the generic fields for a download event and call the - * callback. - * - * @param pi structure to fill in - * @param dc overall download context - */ -void -GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_DownloadContext *dc); - - -/** - * Task that creates the initial (top-level) download - * request for the file. - * - * @param cls the 'struct GNUNET_FS_DownloadContext' - */ -void -GNUNET_FS_download_start_task_ (void *cls); - - -/** - * Fill in all of the generic fields for - * an unindex event and call the callback. - * - * @param pi structure to fill in - * @param uc overall unindex context - * @param offset where we are in the file (for progress) - */ -void -GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_UnindexContext *uc, - uint64_t offset); - -/** - * Fill in all of the generic fields for a search event and - * call the callback. - * - * @param pi structure to fill in - * @param h file-sharing handle - * @param sc overall search context - * @return value returned by the callback - */ -void * -GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_Handle *h, - struct GNUNET_FS_SearchContext *sc); - - -/** - * Connect to the datastore and remove the blocks. - * - * @param uc context for the unindex operation. - */ -void -GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc); - -/** - * Build the request and actually initiate the search using the - * GNUnet FS service. - * - * @param sc search context - * @return #GNUNET_OK on success, #GNUNET_SYSERR on error - */ -int -GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc); - -/** - * Start the downloading process (by entering the queue). - * - * @param dc our download context - */ -void -GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc); - - -/** - * Start download probes for the given search result. - * - * @param sr the search result - */ -void -GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr); - - -/** - * Remove serialization/deserialization file from disk. - * - * @param h master context - * @param ext component of the path - * @param ent entity identifier - */ -void -GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, - const char *ext, - const char *ent); - - -/** - * Remove serialization/deserialization directory from disk. - * - * @param h master context - * @param ext component of the path - * @param uni unique name of parent - */ -void -GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, - const char *ext, - const char *uni); - - -/** - * Synchronize this file-information struct with its mirror - * on disk. Note that all internal FS-operations that change - * file information data should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param fi the struct to sync - */ -void -GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *f); - - -/** - * Synchronize this publishing struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param pc the struct to sync - */ -void -GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc); - - -/** - * Synchronize this unindex struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param uc the struct to sync - */ -void -GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc); - - -/** - * Synchronize this search struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param sc the struct to sync - */ -void -GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc); - - -/** - * Synchronize this search result with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param sr the struct to sync - */ -void -GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr); - - -/** - * Synchronize this download struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param dc the struct to sync - */ -void -GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc); - - -/** - * Create SUSPEND event for the given publish operation - * and then clean up our state (without stop signal). - * - * @param cls the `struct GNUNET_FS_PublishContext` to signal for - */ -void -GNUNET_FS_publish_signal_suspend_ (void *cls); - - -/** - * Create SUSPEND event for the given search operation - * and then clean up our state (without stop signal). - * - * @param cls the 'struct GNUNET_FS_SearchContext' to signal for - */ -void -GNUNET_FS_search_signal_suspend_ (void *cls); - - -/** - * Create SUSPEND event for the given download operation - * and then clean up our state (without stop signal). - * - * @param cls the `struct GNUNET_FS_DownloadContext` to signal for - */ -void -GNUNET_FS_download_signal_suspend_ (void *cls); - - -/** - * Create SUSPEND event for the given unindex operation - * and then clean up our state (without stop signal). - * - * @param cls the `struct GNUNET_FS_UnindexContext` to signal for - */ -void -GNUNET_FS_unindex_signal_suspend_ (void *cls); - - -/** - * Function signature of the functions that can be called - * to trigger suspend signals and clean-up for top-level - * activities. - * - * @param cls closure - */ -typedef void (*SuspendSignalFunction) (void *cls); - -/** - * We track all of the top-level activities of FS - * so that we can signal 'suspend' on shutdown. - */ -struct TopLevelActivity -{ - /** - * This is a doubly-linked list. - */ - struct TopLevelActivity *next; - - /** - * This is a doubly-linked list. - */ - struct TopLevelActivity *prev; - - /** - * Function to call for suspend-signalling and clean up. - */ - SuspendSignalFunction ssf; - - /** - * Closure for 'ssf' (some struct GNUNET_FS_XXXHandle*) - */ - void *ssf_cls; -}; - - -/** - * Create a top-level activity entry. - * - * @param h global fs handle - * @param ssf suspend signal function to use - * @param ssf_cls closure for @a ssf - * @return fresh top-level activity handle - */ -struct TopLevelActivity * -GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, - SuspendSignalFunction ssf, - void *ssf_cls); - - -/** - * Destroy a top-level activity entry. - * - * @param h global fs handle - * @param top top level activity entry - */ -void -GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, - struct TopLevelActivity *top); - - -/** - * Master context for most FS operations. - */ -struct GNUNET_FS_Handle -{ - /** - * Configuration to use. - */ - const struct GNUNET_CONFIGURATION_Handle *cfg; - - /** - * Name of our client. - */ - char *client_name; - - /** - * Function to call with updates on our progress. - */ - GNUNET_FS_ProgressCallback upcb; - - /** - * Closure for upcb. - */ - void *upcb_cls; - - /** - * Head of DLL of top-level activities. - */ - struct TopLevelActivity *top_head; - - /** - * Tail of DLL of top-level activities. - */ - struct TopLevelActivity *top_tail; - - /** - * Head of DLL of running jobs. - */ - struct GNUNET_FS_QueueEntry *running_head; - - /** - * Tail of DLL of running jobs. - */ - struct GNUNET_FS_QueueEntry *running_tail; - - /** - * Head of DLL of pending jobs. - */ - struct GNUNET_FS_QueueEntry *pending_head; - - /** - * Tail of DLL of pending jobs. - */ - struct GNUNET_FS_QueueEntry *pending_tail; - - /** - * Head of active probes. - */ - struct GNUNET_FS_SearchResult *probes_head; - - /** - * Tail of active probes. - */ - struct GNUNET_FS_SearchResult *probes_tail; - - /** - * Task that processes the jobs in the running and pending queues - * (and moves jobs around as needed). - */ - struct GNUNET_SCHEDULER_Task *queue_job; - - /** - * Task we use to report periodically to the application that - * certain search probes (from @e probes_head) are still running. - */ - struct GNUNET_SCHEDULER_Task *probe_ping_task; - - /** - * Average time we take for a single request to be satisfied. - * FIXME: not yet calculated properly... - */ - struct GNUNET_TIME_Relative avg_block_latency; - - /** - * How many actual downloads do we have running right now? - */ - unsigned int active_downloads; - - /** - * How many blocks do the active downloads have? - */ - unsigned int active_blocks; - - /** - * General flags. - */ - enum GNUNET_FS_Flags flags; - - /** - * Maximum number of parallel downloads. - */ - unsigned int max_parallel_downloads; - - /** - * Maximum number of parallel requests. - */ - unsigned int max_parallel_requests; -}; - - -/** - * Handle for controlling a publication process. - */ -struct GNUNET_FS_PublishContext -{ - /** - * Handle to the global fs context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Our top-level activity entry (if we are top-level, otherwise NULL). - */ - struct TopLevelActivity *top; - - /** - * File-structure that is being shared. - */ - struct GNUNET_FS_FileInformation *fi; - - /** - * Namespace that we are publishing in, NULL if we have no namespace. - */ - struct GNUNET_CRYPTO_EcdsaPrivateKey *ns; - - /** - * ID of the content in the namespace, NULL if we have no namespace. - */ - char *nid; - - /** - * ID for future updates, NULL if we have no namespace or no updates. - */ - char *nuid; - - /** - * Filename used for serializing information about this operation - * (should be determined using 'mktemp'). - */ - char *serialization; - - /** - * Our own message queue for the FS service; only briefly used when - * we start to index a file, otherwise NULL. - */ - struct GNUNET_MQ_Handle *mq; - - /** - * Current position in the file-tree for the upload. - */ - struct GNUNET_FS_FileInformation *fi_pos; - - /** - * Non-null if we are currently hashing a file. - */ - struct GNUNET_CRYPTO_FileHashContext *fhc; - - /** - * Connection to the datastore service. - */ - struct GNUNET_DATASTORE_Handle *dsh; - - /** - * Queue entry for reservation/unreservation. - */ - struct GNUNET_DATASTORE_QueueEntry *qre; - - /** - * Context for SKS publishing operation that is part of this publishing operation - * (NULL if not active). - */ - struct GNUNET_FS_PublishSksContext *sks_pc; - - /** - * Context for KSK publishing operation that is part of this publishing operation - * (NULL if not active). - */ - struct GNUNET_FS_PublishKskContext *ksk_pc; - - /** - * ID of the task performing the upload. NO_TASK if the upload has - * completed. - */ - struct GNUNET_SCHEDULER_Task *upload_task; - - /** - * Storage space to reserve for the operation. - */ - uint64_t reserve_space; - - /** - * Overall number of entries to reserve for the - * publish operation. - */ - uint32_t reserve_entries; - - /** - * Options for publishing. - */ - enum GNUNET_FS_PublishOptions options; - - /** - * Space reservation ID with datastore service - * for this upload. - */ - int rid; - - /** - * Set to #GNUNET_YES if we were able to publish any block. - * (and thus unindexing on error might make sense). - */ - int any_done; - - /** - * Set to #GNUNET_YES if all processing has completed. - */ - int all_done; - - /** - * Flag set to #GNUNET_YES if the next callback from - * #GNUNET_FS_file_information_inspect should be skipped because it - * is for the directory which was already processed with the parent. - */ - int skip_next_fi_callback; -}; - - -/** - * Phases of unindex processing (state machine). - */ -enum UnindexState -{ - /** - * We're currently hashing the file. - */ - UNINDEX_STATE_HASHING = 0, - - /** - * We're telling the datastore to delete - * the respective DBlocks and IBlocks. - */ - UNINDEX_STATE_DS_REMOVE = 1, - - /** - * Find out which keywords apply. - */ - UNINDEX_STATE_EXTRACT_KEYWORDS = 2, - - /** - * We're telling the datastore to remove KBlocks. - */ - UNINDEX_STATE_DS_REMOVE_KBLOCKS = 3, - - /** - * We're notifying the FS service about - * the unindexing. - */ - UNINDEX_STATE_FS_NOTIFY = 4, - - /** - * We're done. - */ - UNINDEX_STATE_COMPLETE = 5, - - /** - * We've encountered a fatal error. - */ - UNINDEX_STATE_ERROR = 6 -}; - - -/** - * Handle for controlling an unindexing operation. - */ -struct GNUNET_FS_UnindexContext -{ - /** - * The content hash key of the last block we processed, will in the - * end be set to the CHK from the URI. Used to remove the KBlocks. - */ - struct ContentHashKey chk; - - /** - * Global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Our top-level activity entry. - */ - struct TopLevelActivity *top; - - /** - * Directory scanner to find keywords (KBlock removal). - */ - struct GNUNET_FS_DirScanner *dscan; - - /** - * Keywords found (telling us which KBlocks to remove). - */ - struct GNUNET_FS_Uri *ksk_uri; - - /** - * Current offset in KSK removal. - */ - uint32_t ksk_offset; - - /** - * Name of the file that we are unindexing. - */ - char *filename; - - /** - * Short name under which we are serializing the state of this operation. - */ - char *serialization; - - /** - * Connection to the FS service, only valid during the - * #UNINDEX_STATE_FS_NOTIFY phase. - */ - struct GNUNET_MQ_Handle *mq; - - /** - * Connection to the datastore service, only valid during the - * UNINDEX_STATE_DS_NOTIFY phase. - */ - struct GNUNET_DATASTORE_Handle *dsh; - - /** - * Pointer kept for the client. - */ - void *client_info; - - /** - * Merkle-ish tree encoder context. - */ - struct GNUNET_FS_TreeEncoder *tc; - - /** - * Handle used to read the file. - */ - struct GNUNET_DISK_FileHandle *fh; - - /** - * Handle to datastore 'get_key' operation issued for - * obtaining KBlocks. - */ - struct GNUNET_DATASTORE_QueueEntry *dqe; - - /** - * Current key for decrypting UBLocks from 'get_key' operation. - */ - struct GNUNET_HashCode ukey; - - /** - * Current query of 'get_key' operation. - */ - struct GNUNET_HashCode uquery; - - /** - * Error message, NULL on success. - */ - char *emsg; - - /** - * Context for hashing of the file. - */ - struct GNUNET_CRYPTO_FileHashContext *fhc; - - /** - * Overall size of the file. - */ - uint64_t file_size; - - /** - * When did we start? - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * Hash of the file's contents (once computed). - */ - struct GNUNET_HashCode file_id; - - /** - * Current operatinonal phase. - */ - enum UnindexState state; -}; - - -/** - * Information we keep for each keyword in a keyword search. - */ -struct SearchRequestEntry -{ - /** - * Hash of the public key, also known as the query. - */ - struct GNUNET_HashCode uquery; - - /** - * Derived public key, hashes to 'uquery'. - */ - struct GNUNET_CRYPTO_EcdsaPublicKey dpub; - - /** - * The original keyword, used to derive the - * key (for decrypting the UBlock). - */ - char *keyword; - - /** - * Map that contains a "struct GNUNET_FS_SearchResult" for each result that - * was found under this keyword. Note that the entries will point - * to the same locations as those in the master result map (in - * "struct GNUNET_FS_SearchContext"), so they should not be freed. - * The key for each entry is the XOR of the key and query in the CHK - * URI (as a unique identifier for the search result). - */ - struct GNUNET_CONTAINER_MultiHashMap *results; - - /** - * Is this keyword a mandatory keyword - * (started with '+')? - */ - int mandatory; -}; - - -/** - * Handle for controlling a search. - */ -struct GNUNET_FS_SearchContext -{ - /** - * Handle to the global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Our top-level activity entry (if we are top-level, otherwise NULL). - */ - struct TopLevelActivity *top; - - /** - * List of keywords that we're looking for. - */ - struct GNUNET_FS_Uri *uri; - - /** - * For update-searches, link to the search result that triggered - * the update search; otherwise NULL. - */ - struct GNUNET_FS_SearchResult *psearch_result; - - /** - * Connection to the FS service. - */ - struct GNUNET_MQ_Handle *mq; - - /** - * Pointer we keep for the client. - */ - void *client_info; - - /** - * Name of the file on disk we use for persistence. - */ - char *serialization; - - /** - * Error message (non-NULL if this operation failed). - */ - char *emsg; - - /** - * Map that contains a `struct GNUNET_FS_SearchResult` for each result that - * was found in the search. The key for each entry is the XOR of - * the key and query in the CHK URI (as a unique identifier for the - * search result). - */ - struct GNUNET_CONTAINER_MultiHashMap *master_result_map; - - /** - * Per-keyword information for a keyword search. This array will - * have exactly as many entries as there were keywords. - */ - struct SearchRequestEntry *requests; - - /** - * When did we start? - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * How long to wait before we try to reconnect to FS service? - */ - struct GNUNET_TIME_Relative reconnect_backoff; - - /** - * ID of a task that is using this struct and that must be cancelled - * when the search is being stopped (if not - * NULL). Used for the task that adds some - * artificial delay when trying to reconnect to the FS service. - */ - struct GNUNET_SCHEDULER_Task *task; - - /** - * Anonymity level for the search. - */ - uint32_t anonymity; - - /** - * Number of mandatory keywords in this query. - */ - uint32_t mandatory_count; - - /** - * Options for the search. - */ - enum GNUNET_FS_SearchOptions options; -}; - - -/** - * FSM for possible states a block can go through. The typical - * order of progression is linear through the states, alternatives - * are documented in the comments. - */ -enum BlockRequestState -{ - /** - * Initial state, block has only been allocated (since it is - * relevant to the overall download request). - */ - BRS_INIT = 0, - - /** - * We've checked the block on the path down the tree, and the - * content on disk did match the desired CHK, but not all - * the way down, so at the bottom some blocks will still - * need to be reconstructed). - */ - BRS_RECONSTRUCT_DOWN = 1, - - /** - * We've calculated the CHK bottom-up based on the meta data. - * This may work, but if it did we have to write the meta data to - * disk at the end (and we still need to check against the - * CHK set on top). - */ - BRS_RECONSTRUCT_META_UP = 2, - - /** - * We've calculated the CHK bottom-up based on what we have on - * disk, which may not be what the desired CHK is. If the - * reconstructed CHKs match whatever comes from above, we're - * done with the respective subtree. - */ - BRS_RECONSTRUCT_UP = 3, - - /** - * We've determined the real, desired CHK for this block - * (full tree reconstruction failed), request is now pending. - * If the CHK that bubbled up through reconstruction did match - * the top-level request, the state machine for the subtree - * would have moved to BRS_DOWNLOAD_UP. - */ - BRS_CHK_SET = 4, - - /** - * We've successfully downloaded this block, but the children - * still need to be either downloaded or verified (download - * request propagates down). If the download fails, the - * state machine for this block may move to - * BRS_DOWNLOAD_ERROR instead. - */ - BRS_DOWNLOAD_DOWN = 5, - - /** - * This block and all of its children have been downloaded - * successfully (full completion propagates up). - */ - BRS_DOWNLOAD_UP = 6, - - /** - * We got a block back that matched the query but did not hash to - * the key (malicious publisher or hash collision); this block - * can never be downloaded (error propagates up). - */ - BRS_ERROR = 7 -}; - - -/** - * Information about an active download request. - */ -struct DownloadRequest -{ - /** - * Parent in the CHK-tree. - */ - struct DownloadRequest *parent; - - /** - * Array (!) of child-requests, or NULL for the bottom of the tree. - */ - struct DownloadRequest **children; - - /** - * CHK for the request for this block (set during reconstruction - * to what we have on disk, later to what we want to have). - */ - struct ContentHashKey chk; - - /** - * Offset of the corresponding block. Specifically, first (!) byte of - * the first DBLOCK in the subtree induced by block represented by - * this request. - */ - uint64_t offset; - - /** - * Number of entries in @e children array. - */ - unsigned int num_children; - - /** - * Depth of the corresponding block in the tree. 0==DBLOCKs. - */ - unsigned int depth; - - /** - * Offset of the CHK for this block in the parent block - */ - unsigned int chk_idx; - - /** - * State in the FSM. - */ - enum BlockRequestState state; -}; - - -/** - * (recursively) free download request structure - * - * @param dr request to free - */ -void -GNUNET_FS_free_download_request_ (struct DownloadRequest *dr); - - -/** - * Stop the ping task for this search result. - * - * @param sr result to start pinging for. - */ -void -GNUNET_FS_stop_probe_ping_task_ (struct GNUNET_FS_SearchResult *sr); - - -/** - * Context for controlling a download. - */ -struct GNUNET_FS_DownloadContext -{ - /** - * Global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Our top-level activity entry (if we are top-level, otherwise NULL). - */ - struct TopLevelActivity *top; - - /** - * Connection to the FS service. - */ - struct GNUNET_MQ_Handle *mq; - - /** - * Parent download (used when downloading files - * in directories). - */ - struct GNUNET_FS_DownloadContext *parent; - - /** - * Associated search (used when downloading files - * based on search results), or NULL for none. - */ - struct GNUNET_FS_SearchResult *search; - - /** - * Head of list of child downloads. - */ - struct GNUNET_FS_DownloadContext *child_head; - - /** - * Tail of list of child downloads. - */ - struct GNUNET_FS_DownloadContext *child_tail; - - /** - * Previous download belonging to the same parent. - */ - struct GNUNET_FS_DownloadContext *prev; - - /** - * Next download belonging to the same parent. - */ - struct GNUNET_FS_DownloadContext *next; - - /** - * Context kept for the client. - */ - void *client_info; - - /** - * URI that identifies the file that we are downloading. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Known meta-data for the file (can be NULL). - */ - struct GNUNET_FS_MetaData *meta; - - /** - * Error message, NULL if we're doing OK. - */ - char *emsg; - - /** - * Random portion of filename we use for syncing state of this - * download. - */ - char *serialization; - - /** - * Where are we writing the data (name of the - * file, can be NULL!). - */ - char *filename; - - /** - * Where are we writing the data temporarily (name of the - * file, can be NULL!); used if we do not have a permanent - * name and we are a directory and we do a recursive download. - */ - char *temp_filename; - - /** - * Our entry in the job queue. - */ - struct GNUNET_FS_QueueEntry *job_queue; - - /** - * Tree encoder used for the reconstruction. - */ - struct GNUNET_FS_TreeEncoder *te; - - /** - * File handle for reading data from an existing file - * (to pass to tree encoder). - */ - struct GNUNET_DISK_FileHandle *rfh; - - /** - * Map of active requests (those waiting for a response). The key - * is the hash of the encryped block (aka query). - */ - struct GNUNET_CONTAINER_MultiHashMap *active; - - /** - * Top-level download request. - */ - struct DownloadRequest *top_request; - - /** - * Identity of the peer having the content, or all-zeros - * if we don't know of such a peer. - */ - struct GNUNET_PeerIdentity target; - - /** - * ID of a task that is using this struct and that must be cancelled - * when the download is being stopped (if not - * NULL). Used for the task that adds some - * artificial delay when trying to reconnect to the FS service or - * the task processing incrementally the data on disk, or the - * task requesting blocks, etc. - */ - struct GNUNET_SCHEDULER_Task *task; - - /** - * What is the first offset that we're interested - * in? - */ - uint64_t offset; - - /** - * How many bytes starting from offset are desired? - * This is NOT the overall length of the file! - */ - uint64_t length; - - /** - * How many bytes have we already received within - * the specified range (DBlocks only). - */ - uint64_t completed; - - /** - * What was the size of the file on disk that we're downloading - * before we started? Used to detect if there is a point in - * checking an existing block on disk for matching the desired - * content. 0 if the file did not exist already. - */ - uint64_t old_file_size; - - /** - * Time download was started. - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * How long to wait before we try to reconnect to FS service? - */ - struct GNUNET_TIME_Relative reconnect_backoff; - - /** - * Desired level of anonymity. - */ - uint32_t anonymity; - - /** - * The depth of the file-tree. - */ - unsigned int treedepth; - - /** - * Options for the download. - */ - enum GNUNET_FS_DownloadOptions options; - - /** - * Flag set upon transitive completion (includes child downloads). - * This flag is only set to #GNUNET_YES for directories where all - * child-downloads have also completed (and signalled completion). - */ - int has_finished; - - /** - * Are we ready to issue requests (reconstructions are finished)? - */ - int issue_requests; -}; - - -#endif - -/* end of fs_api.h */ diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c deleted file mode 100644 index c693f9216..000000000 --- a/src/fs/fs_directory.c +++ /dev/null @@ -1,676 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2003, 2004, 2006, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_directory.c - * @brief Helper functions for building directories. - * @author Christian Grothoff - * - * TODO: - * - modify directory builder API to support incremental - * generation of directories (to allow directories that - * would not fit into memory to be created) - * - modify directory processor API to support incremental - * iteration over FULL directories (without missing entries) - * to allow access to directories that do not fit entirely - * into memory - */ -#include "platform.h" - -#include "gnunet_fs_service.h" -#include "fs_api.h" - -/** - * String that is used to indicate that a file - * is a GNUnet directory. - */ -#define GNUNET_DIRECTORY_MAGIC "\211GND\r\n\032\n" - - -/** - * Does the meta-data claim that this is a directory? - * Checks if the mime-type is that of a GNUnet directory. - * - * @return #GNUNET_YES if it is, #GNUNET_NO if it is not, #GNUNET_SYSERR if - * we have no mime-type information (treat as #GNUNET_NO) - */ -int -GNUNET_FS_meta_data_test_for_directory (const struct - GNUNET_FS_MetaData *md) -{ - char *mime; - int ret; - - if (NULL == md) - return GNUNET_SYSERR; - mime = GNUNET_FS_meta_data_get_by_type (md, - EXTRACTOR_METATYPE_MIMETYPE); - if (NULL == mime) - return GNUNET_SYSERR; - ret = (0 == strcasecmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES : - GNUNET_NO; - GNUNET_free (mime); - return ret; -} - - -/** - * Set the MIMETYPE information for the given - * metadata to "application/gnunet-directory". - * - * @param md metadata to add mimetype to - */ -void -GNUNET_FS_meta_data_make_directory (struct GNUNET_FS_MetaData *md) -{ - char *mime; - - mime = - GNUNET_FS_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE); - if (mime != NULL) - { - GNUNET_break (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME)); - GNUNET_free (mime); - return; - } - GNUNET_FS_meta_data_insert (md, "", - EXTRACTOR_METATYPE_MIMETYPE, - EXTRACTOR_METAFORMAT_UTF8, "text/plain", - GNUNET_FS_DIRECTORY_MIME, - strlen (GNUNET_FS_DIRECTORY_MIME) + 1); -} - - -/** - * Closure for 'find_full_data'. - */ -struct GetFullDataClosure -{ - /** - * Extracted binary meta data. - */ - void *data; - - /** - * Number of bytes stored in data. - */ - size_t size; -}; - - -/** - * Type of a function that libextractor calls for each - * meta data item found. - * - * @param cls closure (user-defined) - * @param plugin_name name of the plugin that produced this value; - * special values can be used (e.g. '<zlib>' for zlib being - * used in the main libextractor library and yielding - * meta data). - * @param type libextractor-type describing the meta data - * @param format basic format information about data - * @param data_mime_type mime-type of data (not of the original file); - * can be NULL (if mime-type is not known) - * @param data actual meta-data found - * @param data_len number of bytes in data - * @return 0 to continue extracting, 1 to abort - */ -static int -find_full_data (void *cls, const char *plugin_name, - enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, - const char *data_mime_type, const char *data, size_t data_len) -{ - struct GetFullDataClosure *gfdc = cls; - - if (type == EXTRACTOR_METATYPE_GNUNET_FULL_DATA) - { - gfdc->size = data_len; - if (data_len > 0) - { - gfdc->data = GNUNET_malloc (data_len); - GNUNET_memcpy (gfdc->data, data, data_len); - } - return 1; - } - return 0; -} - - -/** - * Iterate over all entries in a directory. Note that directories - * are structured such that it is possible to iterate over the - * individual blocks as well as over the entire directory. Thus - * a client can call this function on the buffer in the - * GNUNET_FS_ProgressCallback. Also, directories can optionally - * include the contents of (small) files embedded in the directory - * itself; for those files, the processor may be given the - * contents of the file directly by this function. - *

- * - * Note that this function maybe called on parts of directories. Thus - * parser errors should not be reported _at all_ (with GNUNET_break). - * Still, if some entries can be recovered despite these parsing - * errors, the function should try to do this. - * - * @param size number of bytes in data - * @param data pointer to the beginning of the directory - * @param offset offset of data in the directory - * @param dep function to call on each entry - * @param dep_cls closure for @a dep - * @return #GNUNET_OK if this could be a block in a directory, - * #GNUNET_NO if this could be part of a directory (but not 100% OK) - * #GNUNET_SYSERR if @a data does not represent a directory - */ -int -GNUNET_FS_directory_list_contents (size_t size, - const void *data, - uint64_t offset, - GNUNET_FS_DirectoryEntryProcessor dep, - void *dep_cls) -{ - struct GetFullDataClosure full_data; - const char *cdata = data; - char *emsg; - uint64_t pos; - uint64_t align; - uint32_t mdSize; - uint64_t epos; - struct GNUNET_FS_Uri *uri; - struct GNUNET_FS_MetaData *md; - char *filename; - - if ((offset == 0) && - ((size < 8 + sizeof(uint32_t)) || - (0 != memcmp (cdata, - GNUNET_FS_DIRECTORY_MAGIC, - 8)))) - return GNUNET_SYSERR; - pos = offset; - if (offset == 0) - { - GNUNET_memcpy (&mdSize, - &cdata[8], - sizeof(uint32_t)); - mdSize = ntohl (mdSize); - if (mdSize > size - 8 - sizeof(uint32_t)) - { - /* invalid size */ - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("MAGIC mismatch. This is not a GNUnet directory.\n")); - return GNUNET_SYSERR; - } - md = GNUNET_FS_meta_data_deserialize (&cdata[8 + sizeof(uint32_t)], - mdSize); - if (md == NULL) - { - GNUNET_break (0); - return GNUNET_SYSERR; /* malformed ! */ - } - dep (dep_cls, - NULL, - NULL, - md, - 0, - NULL); - GNUNET_FS_meta_data_destroy (md); - pos = 8 + sizeof(uint32_t) + mdSize; - } - while (pos < size) - { - /* find end of URI */ - if (cdata[pos] == '\0') - { - /* URI is never empty, must be end of block, - * skip to next alignment */ - align = ((pos / DBLOCK_SIZE) + 1) * DBLOCK_SIZE; - if (align == pos) - { - /* if we were already aligned, still skip a block! */ - align += DBLOCK_SIZE; - } - pos = align; - if (pos >= size) - { - /* malformed - or partial download... */ - break; - } - } - epos = pos; - while ((epos < size) && (cdata[epos] != '\0')) - epos++; - if (epos >= size) - return GNUNET_NO; /* malformed - or partial download */ - - uri = GNUNET_FS_uri_parse (&cdata[pos], &emsg); - pos = epos + 1; - if (NULL == uri) - { - GNUNET_free (emsg); - pos--; /* go back to '\0' to force going to next alignment */ - continue; - } - if (GNUNET_FS_uri_test_ksk (uri)) - { - GNUNET_FS_uri_destroy (uri); - GNUNET_break (0); - return GNUNET_NO; /* illegal in directory! */ - } - - GNUNET_memcpy (&mdSize, - &cdata[pos], - sizeof(uint32_t)); - mdSize = ntohl (mdSize); - pos += sizeof(uint32_t); - if (pos + mdSize > size) - { - GNUNET_FS_uri_destroy (uri); - return GNUNET_NO; /* malformed - or partial download */ - } - - md = GNUNET_FS_meta_data_deserialize (&cdata[pos], - mdSize); - if (NULL == md) - { - GNUNET_FS_uri_destroy (uri); - GNUNET_break (0); - return GNUNET_NO; /* malformed ! */ - } - pos += mdSize; - filename = - GNUNET_FS_meta_data_get_by_type (md, - EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); - full_data.size = 0; - full_data.data = NULL; - GNUNET_FS_meta_data_iterate (md, - &find_full_data, - &full_data); - if (NULL != dep) - { - dep (dep_cls, - filename, - uri, - md, - full_data.size, - full_data.data); - } - GNUNET_free (full_data.data); - GNUNET_free (filename); - GNUNET_FS_meta_data_destroy (md); - GNUNET_FS_uri_destroy (uri); - } - return GNUNET_OK; -} - - -/** - * Entries in the directory (builder). - */ -struct BuilderEntry -{ - /** - * This is a linked list. - */ - struct BuilderEntry *next; - - /** - * Length of this entry. - */ - size_t len; -}; - -/** - * Internal state of a directory builder. - */ -struct GNUNET_FS_DirectoryBuilder -{ - /** - * Meta-data for the directory itself. - */ - struct GNUNET_FS_MetaData *meta; - - /** - * Head of linked list of entries. - */ - struct BuilderEntry *head; - - /** - * Number of entries in the directory. - */ - unsigned int count; -}; - - -/** - * Create a directory builder. - * - * @param mdir metadata for the directory - */ -struct GNUNET_FS_DirectoryBuilder * -GNUNET_FS_directory_builder_create (const struct GNUNET_FS_MetaData - *mdir) -{ - struct GNUNET_FS_DirectoryBuilder *ret; - - ret = GNUNET_new (struct GNUNET_FS_DirectoryBuilder); - if (mdir != NULL) - ret->meta = GNUNET_FS_meta_data_duplicate (mdir); - else - ret->meta = GNUNET_FS_meta_data_create (); - GNUNET_FS_meta_data_make_directory (ret->meta); - return ret; -} - - -/** - * Add an entry to a directory. - * - * @param bld directory to extend - * @param uri uri of the entry (must not be a KSK) - * @param md metadata of the entry - * @param data raw data of the entry, can be NULL, otherwise - * data must point to exactly the number of bytes specified - * by the uri which must be of type LOC or CHK - */ -void -GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *md, - const void *data) -{ - struct GNUNET_FS_Uri *curi; - struct BuilderEntry *e; - uint64_t fsize; - uint32_t big; - ssize_t ret; - size_t mds; - size_t mdxs; - char *uris; - char *serialized; - char *sptr; - size_t slen; - struct GNUNET_FS_MetaData *meta; - const struct GNUNET_FS_MetaData *meta_use; - - GNUNET_assert (! GNUNET_FS_uri_test_ksk (uri)); - if (NULL != data) - { - GNUNET_assert (! GNUNET_FS_uri_test_sks (uri)); - if (GNUNET_FS_uri_test_chk (uri)) - { - fsize = GNUNET_FS_uri_chk_get_file_size (uri); - } - else - { - curi = GNUNET_FS_uri_loc_get_uri (uri); - GNUNET_assert (NULL != curi); - fsize = GNUNET_FS_uri_chk_get_file_size (curi); - GNUNET_FS_uri_destroy (curi); - } - } - else - { - fsize = 0; /* not given */ - } - if (fsize > MAX_INLINE_SIZE) - fsize = 0; /* too large */ - uris = GNUNET_FS_uri_to_string (uri); - slen = strlen (uris) + 1; - mds = GNUNET_FS_meta_data_get_serialized_size (md); - meta_use = md; - meta = NULL; - if (fsize > 0) - { - meta = GNUNET_FS_meta_data_duplicate (md); - GNUNET_FS_meta_data_insert (meta, "", - EXTRACTOR_METATYPE_GNUNET_FULL_DATA, - EXTRACTOR_METAFORMAT_BINARY, NULL, data, - fsize); - mdxs = GNUNET_FS_meta_data_get_serialized_size (meta); - if ((slen + sizeof(uint32_t) + mdxs - 1) / DBLOCK_SIZE == - (slen + sizeof(uint32_t) + mds - 1) / DBLOCK_SIZE) - { - /* adding full data would not cause us to cross - * additional blocks, so add it! */ - meta_use = meta; - mds = mdxs; - } - } - - if (mds > GNUNET_MAX_MALLOC_CHECKED / 2) - mds = GNUNET_MAX_MALLOC_CHECKED / 2; - e = GNUNET_malloc (sizeof(struct BuilderEntry) + slen + mds - + sizeof(uint32_t)); - serialized = (char *) &e[1]; - GNUNET_memcpy (serialized, uris, slen); - GNUNET_free (uris); - sptr = &serialized[slen + sizeof(uint32_t)]; - ret = - GNUNET_FS_meta_data_serialize (meta_use, &sptr, mds, - GNUNET_FS_META_DATA_SERIALIZE_PART); - if (NULL != meta) - GNUNET_FS_meta_data_destroy (meta); - if (ret == -1) - mds = 0; - else - mds = ret; - big = htonl (mds); - GNUNET_memcpy (&serialized[slen], &big, sizeof(uint32_t)); - e->len = slen + sizeof(uint32_t) + mds; - e->next = bld->head; - bld->head = e; - bld->count++; -} - - -/** - * Given the start and end position of a block of - * data, return the end position of that data - * after alignment to the DBLOCK_SIZE. - */ -static size_t -do_align (size_t start_position, size_t end_position) -{ - size_t align; - - align = (end_position / DBLOCK_SIZE) * DBLOCK_SIZE; - if ((start_position < align) && (end_position > align)) - return align + end_position - start_position; - return end_position; -} - - -/** - * Compute a permutation of the blocks to - * minimize the cost of alignment. Greedy packer. - * - * @param start starting position for the first block - * @param count size of the two arrays - * @param sizes the sizes of the individual blocks - * @param perm the permutation of the blocks (updated) - */ -static void -block_align (size_t start, unsigned int count, const size_t *sizes, - unsigned int *perm) -{ - unsigned int i; - unsigned int j; - unsigned int tmp; - unsigned int best; - ssize_t badness; - size_t cpos; - size_t cend; - ssize_t cbad; - unsigned int cval; - - cpos = start; - for (i = 0; i < count; i++) - { - start = cpos; - badness = 0x7FFFFFFF; - best = -1; - for (j = i; j < count; j++) - { - cval = perm[j]; - cend = cpos + sizes[cval]; - if (cpos % DBLOCK_SIZE == 0) - { - /* prefer placing the largest blocks first */ - cbad = -(cend % DBLOCK_SIZE); - } - else - { - if (cpos / DBLOCK_SIZE == cend / DBLOCK_SIZE) - { - /* Data fits into the same block! Prefer small left-overs! */ - cbad = DBLOCK_SIZE - cend % DBLOCK_SIZE; - } - else - { - /* Would have to waste space to re-align, add big factor, this - * case is a real loss (proportional to space wasted)! */ - cbad = DBLOCK_SIZE * (DBLOCK_SIZE - cpos % DBLOCK_SIZE); - } - } - if (cbad < badness) - { - best = j; - badness = cbad; - } - } - GNUNET_assert (best != -1); - tmp = perm[i]; - perm[i] = perm[best]; - perm[best] = tmp; - cpos += sizes[perm[i]]; - cpos = do_align (start, cpos); - } -} - - -/** - * Finish building the directory. Frees the - * builder context and returns the directory - * in-memory. - * - * @param bld directory to finish - * @param rsize set to the number of bytes needed - * @param rdata set to the encoded directory - * @return #GNUNET_OK on success - */ -int -GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, - size_t *rsize, - void **rdata) -{ - char *data; - char *sptr; - size_t *sizes; - unsigned int *perm; - unsigned int i; - unsigned int j; - struct BuilderEntry *pos; - struct BuilderEntry **bes; - size_t size; - size_t psize; - size_t off; - ssize_t ret; - uint32_t big; - - size = strlen (GNUNET_DIRECTORY_MAGIC) + sizeof(uint32_t); - size += GNUNET_FS_meta_data_get_serialized_size (bld->meta); - sizes = NULL; - perm = NULL; - bes = NULL; - if (0 < bld->count) - { - sizes = GNUNET_new_array (bld->count, - size_t); - perm = GNUNET_new_array (bld->count, - unsigned int); - bes = GNUNET_new_array (bld->count, - struct BuilderEntry *); - pos = bld->head; - for (i = 0; i < bld->count; i++) - { - perm[i] = i; - bes[i] = pos; - sizes[i] = pos->len; - pos = pos->next; - } - block_align (size, bld->count, sizes, perm); - /* compute final size with alignment */ - for (i = 0; i < bld->count; i++) - { - psize = size; - size += sizes[perm[i]]; - size = do_align (psize, size); - } - } - *rsize = size; - data = GNUNET_malloc_large (size); - if (data == NULL) - { - GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, - "malloc"); - *rsize = 0; - *rdata = NULL; - GNUNET_free (sizes); - GNUNET_free (perm); - GNUNET_free (bes); - return GNUNET_SYSERR; - } - *rdata = data; - GNUNET_memcpy (data, - GNUNET_DIRECTORY_MAGIC, - strlen (GNUNET_DIRECTORY_MAGIC)); - off = strlen (GNUNET_DIRECTORY_MAGIC); - - sptr = &data[off + sizeof(uint32_t)]; - ret = - GNUNET_FS_meta_data_serialize (bld->meta, - &sptr, - size - off - sizeof(uint32_t), - GNUNET_FS_META_DATA_SERIALIZE_FULL); - GNUNET_assert (ret != -1); - big = htonl (ret); - GNUNET_memcpy (&data[off], - &big, - sizeof(uint32_t)); - off += sizeof(uint32_t) + ret; - for (j = 0; j < bld->count; j++) - { - i = perm[j]; - psize = off; - off += sizes[i]; - off = do_align (psize, off); - GNUNET_memcpy (&data[off - sizes[i]], &(bes[i])[1], sizes[i]); - GNUNET_free (bes[i]); - } - GNUNET_free (sizes); - GNUNET_free (perm); - GNUNET_free (bes); - GNUNET_assert (off == size); - GNUNET_FS_meta_data_destroy (bld->meta); - GNUNET_free (bld); - return GNUNET_OK; -} - - -/* end of fs_directory.c */ diff --git a/src/fs/fs_dirmetascan.c b/src/fs/fs_dirmetascan.c deleted file mode 100644 index 2379e29ce..000000000 --- a/src/fs/fs_dirmetascan.c +++ /dev/null @@ -1,496 +0,0 @@ -/* - This file is part of GNUnet - Copyright (C) 2005-2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_dirmetascan.c - * @brief code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem' - * from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'. - * @author LRN - * @author Christian Grothoff - */ -#include "platform.h" - -#include "gnunet_fs_service.h" -#include "gnunet_scheduler_lib.h" -#include - - -/** - * An opaque structure a pointer to which is returned to the - * caller to be used to control the scanner. - */ -struct GNUNET_FS_DirScanner -{ - /** - * Helper process. - */ - struct GNUNET_HELPER_Handle *helper; - - /** - * Expanded filename (as given by the scan initiator). - * The scanner thread stores a copy here, and frees it when it finishes. - */ - char *filename_expanded; - - /** - * Second argument to helper process. - */ - char *ex_arg; - - /** - * The function that will be called every time there's a progress - * message. - */ - GNUNET_FS_DirScannerProgressCallback progress_callback; - - /** - * A closure for progress_callback. - */ - void *progress_callback_cls; - - /** - * After the scan is finished, it will contain a pointer to the - * top-level directory entry in the directory tree built by the - * scanner. - */ - struct GNUNET_FS_ShareTreeItem *toplevel; - - /** - * Current position during processing. - */ - struct GNUNET_FS_ShareTreeItem *pos; - - /** - * Task scheduled when we are done. - */ - struct GNUNET_SCHEDULER_Task *stop_task; - - /** - * Arguments for helper. - */ - char *args[4]; -}; - - -/** - * Abort the scan. Must not be called from within the progress_callback - * function. - * - * @param ds directory scanner structure - */ -void -GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds) -{ - /* terminate helper */ - if (NULL != ds->helper) - GNUNET_HELPER_stop (ds->helper, GNUNET_NO); - - /* free resources */ - if (NULL != ds->toplevel) - GNUNET_FS_share_tree_free (ds->toplevel); - if (NULL != ds->stop_task) - GNUNET_SCHEDULER_cancel (ds->stop_task); - GNUNET_free (ds->ex_arg); - GNUNET_free (ds->filename_expanded); - GNUNET_free (ds); -} - - -struct GNUNET_FS_ShareTreeItem * -GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds) -{ - struct GNUNET_FS_ShareTreeItem *result; - - /* check that we're actually done */ - GNUNET_assert (NULL == ds->helper); - /* preserve result */ - result = ds->toplevel; - ds->toplevel = NULL; - GNUNET_FS_directory_scan_abort (ds); - return result; -} - - -/** - * Move in the directory from the given position to the next file - * in DFS traversal. - * - * @param pos current position - * @return next file, NULL for none - */ -static struct GNUNET_FS_ShareTreeItem * -advance (struct GNUNET_FS_ShareTreeItem *pos) -{ - int moved; - - GNUNET_assert (NULL != pos); - moved = 0; /* must not terminate, even on file, otherwise "normal" */ - while ((pos->is_directory == GNUNET_YES) || (0 == moved)) - { - if ((moved != -1) && (NULL != pos->children_head)) - { - pos = pos->children_head; - moved = 1; /* can terminate if file */ - continue; - } - if (NULL != pos->next) - { - pos = pos->next; - moved = 1; /* can terminate if file */ - continue; - } - if (NULL != pos->parent) - { - pos = pos->parent; - moved = -1; /* force move to 'next' or 'parent' */ - continue; - } - /* no more options, end of traversal */ - return NULL; - } - return pos; -} - - -/** - * Add another child node to the tree. - * - * @param parent parent of the child, NULL for top level - * @param filename name of the file or directory - * @param is_directory GNUNET_YES for directories - * @return new entry that was just created - */ -static struct GNUNET_FS_ShareTreeItem * -expand_tree (struct GNUNET_FS_ShareTreeItem *parent, - const char *filename, - int is_directory) -{ - struct GNUNET_FS_ShareTreeItem *chld; - size_t slen; - - chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem); - chld->parent = parent; - chld->filename = GNUNET_strdup (filename); - GNUNET_asprintf (&chld->short_filename, - "%s%s", - GNUNET_STRINGS_get_short_name (filename), - is_directory == GNUNET_YES ? "/" : ""); - /* make sure we do not end with '//' */ - slen = strlen (chld->short_filename); - if ((slen >= 2) && (chld->short_filename[slen - 1] == '/') && - (chld->short_filename[slen - 2] == '/')) - chld->short_filename[slen - 1] = '\0'; - chld->is_directory = is_directory; - if (NULL != parent) - GNUNET_CONTAINER_DLL_insert (parent->children_head, - parent->children_tail, - chld); - return chld; -} - - -/** - * Task run last to shut everything down. - * - * @param cls the 'struct GNUNET_FS_DirScanner' - */ -static void -finish_scan (void *cls) -{ - struct GNUNET_FS_DirScanner *ds = cls; - - ds->stop_task = NULL; - if (NULL != ds->helper) - { - GNUNET_HELPER_stop (ds->helper, GNUNET_NO); - ds->helper = NULL; - } - ds->progress_callback (ds->progress_callback_cls, - NULL, - GNUNET_SYSERR, - GNUNET_FS_DIRSCANNER_FINISHED); -} - - -/** - * Called every time there is data to read from the scanner. - * Calls the scanner progress handler. - * - * @param cls the closure (directory scanner object) - * @param msg message from the helper process - * @return #GNUNET_OK on success, - * #GNUNET_NO to stop further processing (no error) - * #GNUNET_SYSERR to stop further processing with error - */ -static int -process_helper_msgs (void *cls, const struct GNUNET_MessageHeader *msg) -{ - struct GNUNET_FS_DirScanner *ds = cls; - const char *filename; - size_t left; - -#if 0 - fprintf (stderr, - "DMS parses %u-byte message of type %u\n", - (unsigned int) ntohs (msg->size), - (unsigned int) ntohs (msg->type)); -#endif - left = ntohs (msg->size) - sizeof(struct GNUNET_MessageHeader); - filename = (const char *) &msg[1]; - switch (ntohs (msg->type)) - { - case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE: - if (filename[left - 1] != '\0') - { - GNUNET_break (0); - break; - } - ds->progress_callback (ds->progress_callback_cls, - filename, - GNUNET_NO, - GNUNET_FS_DIRSCANNER_FILE_START); - if (NULL == ds->toplevel) - { - ds->toplevel = expand_tree (ds->pos, filename, GNUNET_NO); - } - else - { - GNUNET_assert (NULL != ds->pos); - (void) expand_tree (ds->pos, filename, GNUNET_NO); - } - return GNUNET_OK; - - case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY: - if (filename[left - 1] != '\0') - { - GNUNET_break (0); - break; - } - if (0 == strcmp ("..", filename)) - { - if (NULL == ds->pos) - { - GNUNET_break (0); - break; - } - ds->pos = ds->pos->parent; - return GNUNET_OK; - } - ds->progress_callback (ds->progress_callback_cls, - filename, - GNUNET_YES, - GNUNET_FS_DIRSCANNER_FILE_START); - ds->pos = expand_tree (ds->pos, filename, GNUNET_YES); - if (NULL == ds->toplevel) - ds->toplevel = ds->pos; - return GNUNET_OK; - - case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR: - break; - - case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE: - if ('\0' != filename[left - 1]) - break; - ds->progress_callback (ds->progress_callback_cls, - filename, - GNUNET_SYSERR, - GNUNET_FS_DIRSCANNER_FILE_IGNORED); - return GNUNET_OK; - - case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE: - if (0 != left) - { - GNUNET_break (0); - break; - } - if (NULL == ds->toplevel) - break; - ds->progress_callback (ds->progress_callback_cls, - NULL, - GNUNET_SYSERR, - GNUNET_FS_DIRSCANNER_ALL_COUNTED); - ds->pos = ds->toplevel; - if (GNUNET_YES == ds->pos->is_directory) - ds->pos = advance (ds->pos); - return GNUNET_OK; - - case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA: { - size_t nlen; - const char *end; - - if (NULL == ds->pos) - { - GNUNET_break (0); - break; - } - end = memchr (filename, 0, left); - if (NULL == end) - { - GNUNET_break (0); - break; - } - end++; - nlen = end - filename; - left -= nlen; - if (0 != strcmp (filename, ds->pos->filename)) - { - GNUNET_break (0); - break; - } - ds->progress_callback (ds->progress_callback_cls, - filename, - GNUNET_YES, - GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED); - if (0 < left) - { - ds->pos->meta = GNUNET_FS_meta_data_deserialize (end, left); - if (NULL == ds->pos->meta) - { - GNUNET_break (0); - break; - } - /* having full filenames is too dangerous; always make sure we clean them up */ - GNUNET_FS_meta_data_delete (ds->pos->meta, - EXTRACTOR_METATYPE_FILENAME, - NULL, - 0); - /* instead, put in our 'safer' original filename */ - GNUNET_FS_meta_data_insert (ds->pos->meta, - "", - EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, - EXTRACTOR_METAFORMAT_UTF8, - "text/plain", - ds->pos->short_filename, - strlen (ds->pos->short_filename) - + 1); - } - ds->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_meta_data ( - ds->pos->meta); - ds->pos = advance (ds->pos); - return GNUNET_OK; - } - - case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED: - if (NULL != ds->pos) - { - GNUNET_break (0); - break; - } - if (0 != left) - { - GNUNET_break (0); - break; - } - if (NULL == ds->toplevel) - break; - ds->stop_task = GNUNET_SCHEDULER_add_now (&finish_scan, ds); - return GNUNET_OK; - - default: - GNUNET_break (0); - break; - } - ds->progress_callback (ds->progress_callback_cls, - NULL, - GNUNET_SYSERR, - GNUNET_FS_DIRSCANNER_INTERNAL_ERROR); - return GNUNET_OK; -} - - -/** - * Function called if our helper process died. - * - * @param cls the 'struct GNUNET_FS_DirScanner' callback. - */ -static void -helper_died_cb (void *cls) -{ - struct GNUNET_FS_DirScanner *ds = cls; - - ds->helper = NULL; - if (NULL != ds->stop_task) - return; /* normal death, was finished */ - ds->progress_callback (ds->progress_callback_cls, - NULL, - GNUNET_SYSERR, - GNUNET_FS_DIRSCANNER_INTERNAL_ERROR); -} - - -/** - * Start a directory scanner thread. - * - * @param filename name of the directory to scan - * @param disable_extractor #GNUNET_YES to not run libextractor on files (only - * build a tree) - * @param ex if not NULL, must be a list of extra plugins for extractor - * @param cb the callback to call when there are scanning progress messages - * @param cb_cls closure for 'cb' - * @return directory scanner object to be used for controlling the scanner - */ -struct GNUNET_FS_DirScanner * -GNUNET_FS_directory_scan_start (const char *filename, - int disable_extractor, - const char *ex, - GNUNET_FS_DirScannerProgressCallback cb, - void *cb_cls) -{ - struct stat sbuf; - char *filename_expanded; - struct GNUNET_FS_DirScanner *ds; - - if (0 != stat (filename, &sbuf)) - return NULL; - filename_expanded = GNUNET_STRINGS_filename_expand (filename); - if (NULL == filename_expanded) - return NULL; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting to scan directory `%s'\n", - filename_expanded); - ds = GNUNET_new (struct GNUNET_FS_DirScanner); - ds->progress_callback = cb; - ds->progress_callback_cls = cb_cls; - ds->filename_expanded = filename_expanded; - if (disable_extractor) - ds->ex_arg = GNUNET_strdup ("-"); - else - ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL; - ds->args[0] = "gnunet-helper-fs-publish"; - ds->args[1] = ds->filename_expanded; - ds->args[2] = ds->ex_arg; - ds->args[3] = NULL; - ds->helper = GNUNET_HELPER_start (GNUNET_NO, - "gnunet-helper-fs-publish", - ds->args, - &process_helper_msgs, - &helper_died_cb, - ds); - if (NULL == ds->helper) - { - GNUNET_free (filename_expanded); - GNUNET_free (ds); - return NULL; - } - return ds; -} - - -/* end of fs_dirmetascan.c */ diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c deleted file mode 100644 index 2a21e4810..000000000 --- a/src/fs/fs_download.c +++ /dev/null @@ -1,2339 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2001-2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/fs_download.c - * @brief download methods - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_constants.h" - -#include "gnunet_fs_service.h" -#include "fs_api.h" -#include "fs_tree.h" - - -/** - * Determine if the given download (options and meta data) should cause - * use to try to do a recursive download. - */ -static int -is_recursive_download (struct GNUNET_FS_DownloadContext *dc) -{ - return (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) && - ((GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) || - ((NULL == dc->meta) && - ((NULL == dc->filename) || - ((strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) && - (NULL != strstr (dc->filename + strlen (dc->filename) - - strlen (GNUNET_FS_DIRECTORY_EXT), - GNUNET_FS_DIRECTORY_EXT)))))); -} - - -/** - * We're storing the IBLOCKS after the DBLOCKS on disk (so that we - * only have to truncate the file once we're done). - * - * Given the offset of a block (with respect to the DBLOCKS) and its - * depth, return the offset where we would store this block in the - * file. - * - * @param fsize overall file size - * @param off offset of the block in the file - * @param depth depth of the block in the tree, 0 for DBLOCK - * @return off for DBLOCKS (depth == treedepth), - * otherwise an offset past the end - * of the file that does not overlap - * with the range for any other block - */ -static uint64_t -compute_disk_offset (uint64_t fsize, uint64_t off, unsigned int depth) -{ - unsigned int i; - uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */ - uint64_t loff; /* where do IBlocks for depth "i" start? */ - unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */ - - if (0 == depth) - return off; - /* first IBlocks start at the end of file, rounded up - * to full DBLOCK_SIZE */ - loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE; - lsize = - ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof(struct ContentHashKey); - GNUNET_assert (0 == (off % DBLOCK_SIZE)); - ioff = (off / DBLOCK_SIZE); - for (i = 1; i < depth; i++) - { - loff += lsize; - lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE; - GNUNET_assert (lsize > 0); - GNUNET_assert (0 == (ioff % CHK_PER_INODE)); - ioff /= CHK_PER_INODE; - } - return loff + ioff * sizeof(struct ContentHashKey); -} - - -/** - * Fill in all of the generic fields for a download event and call the - * callback. - * - * @param pi structure to fill in - * @param dc overall download context - */ -void -GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_DownloadContext *dc) -{ - pi->value.download.dc = dc; - pi->value.download.cctx = dc->client_info; - pi->value.download.pctx = - (NULL == dc->parent) ? NULL : dc->parent->client_info; - pi->value.download.sctx = - (NULL == dc->search) ? NULL : dc->search->client_info; - pi->value.download.uri = dc->uri; - pi->value.download.filename = dc->filename; - pi->value.download.size = dc->length; - /* FIXME: Fix duration calculation to account for pauses */ - pi->value.download.duration = - GNUNET_TIME_absolute_get_duration (dc->start_time); - pi->value.download.completed = dc->completed; - pi->value.download.anonymity = dc->anonymity; - pi->value.download.eta = - GNUNET_TIME_calculate_eta (dc->start_time, dc->completed, dc->length); - pi->value.download.is_active = (NULL == dc->mq) ? GNUNET_NO : GNUNET_YES; - pi->fsh = dc->h; - if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE)) - dc->client_info = dc->h->upcb (dc->h->upcb_cls, pi); - else - dc->client_info = GNUNET_FS_search_probe_progress_ (NULL, pi); -} - - -/** - * Closure for iterator processing results. - */ -struct ProcessResultClosure -{ - /** - * Hash of data. - */ - struct GNUNET_HashCode query; - - /** - * Data found in P2P network. - */ - const void *data; - - /** - * Our download context. - */ - struct GNUNET_FS_DownloadContext *dc; - - /** - * When did we last transmit the request? - */ - struct GNUNET_TIME_Absolute last_transmission; - - /** - * Number of bytes in data. - */ - size_t size; - - /** - * Type of data. - */ - enum GNUNET_BLOCK_Type type; - - /** - * Flag to indicate if this block should be stored on disk. - */ - int do_store; - - /** - * how much respect did we offer to get this reply? - */ - uint32_t respect_offered; - - /** - * how often did we transmit the query? - */ - uint32_t num_transmissions; -}; - - -/** - * Iterator over entries in the pending requests in the 'active' map for the - * reply that we just got. - * - * @param cls closure (our `struct ProcessResultClosure`) - * @param key query for the given value / request - * @param value value in the hash map (a `struct DownloadRequest`) - * @return #GNUNET_YES (we should continue to iterate); unless serious error - */ -static int -process_result_with_request (void *cls, - const struct GNUNET_HashCode *key, - void *value); - - -/** - * We've found a matching block without downloading it. - * Encrypt it and pass it to our "receive" function as - * if we had received it from the network. - * - * @param dc download in question - * @param chk request this relates to - * @param dr request details - * @param block plaintext data matching request - * @param len number of bytes in block - * @param do_store should we still store the block on disk? - * @return GNUNET_OK on success - */ -static int -encrypt_existing_match (struct GNUNET_FS_DownloadContext *dc, - const struct ContentHashKey *chk, - struct DownloadRequest *dr, - const char *block, - size_t len, - int do_store) -{ - struct ProcessResultClosure prc; - char enc[len]; - struct GNUNET_CRYPTO_SymmetricSessionKey sk; - struct GNUNET_CRYPTO_SymmetricInitializationVector iv; - struct GNUNET_HashCode query; - - GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv); - if (-1 == GNUNET_CRYPTO_symmetric_encrypt (block, len, &sk, &iv, enc)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - GNUNET_CRYPTO_hash (enc, len, &query); - if (0 != memcmp (&query, &chk->query, sizeof(struct GNUNET_HashCode))) - { - GNUNET_break_op (0); - return GNUNET_SYSERR; - } - GNUNET_log ( - GNUNET_ERROR_TYPE_DEBUG, - "Matching %u byte block for `%s' at offset %llu already present, no need for download!\n", - (unsigned int) len, - dc->filename, - (unsigned long long) dr->offset); - /* already got it! */ - prc.dc = dc; - prc.data = enc; - prc.size = len; - prc.type = (0 == dr->depth) ? GNUNET_BLOCK_TYPE_FS_DBLOCK - : GNUNET_BLOCK_TYPE_FS_IBLOCK; - prc.query = chk->query; - prc.do_store = do_store; - prc.last_transmission = GNUNET_TIME_UNIT_FOREVER_ABS; - process_result_with_request (&prc, &chk->key, dr); - return GNUNET_OK; -} - - -/** - * We've lost our connection with the FS service. - * Re-establish it and re-transmit all of our - * pending requests. - * - * @param dc download context that is having trouble - */ -static void -try_reconnect (struct GNUNET_FS_DownloadContext *dc); - - -/** - * We found an entry in a directory. Check if the respective child - * already exists and if not create the respective child download. - * - * @param cls the parent download - * @param filename name of the file in the directory - * @param uri URI of the file (CHK or LOC) - * @param meta meta data of the file - * @param length number of bytes in data - * @param data contents of the file (or NULL if they were not inlined) - */ -static void -trigger_recursive_download (void *cls, - const char *filename, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *meta, - size_t length, - const void *data); - - -/** - * We're done downloading a directory. Open the file and - * trigger all of the (remaining) child downloads. - * - * @param dc context of download that just completed - */ -static void -full_recursive_download (struct GNUNET_FS_DownloadContext *dc) -{ - size_t size; - uint64_t size64; - void *data; - struct GNUNET_DISK_FileHandle *h; - struct GNUNET_DISK_MapHandle *m; - - size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri); - size = (size_t) size64; - if (size64 != (uint64_t) size) - { - GNUNET_log ( - GNUNET_ERROR_TYPE_ERROR, - _ ( - "Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n")); - return; - } - if (NULL != dc->filename) - { - h = GNUNET_DISK_file_open (dc->filename, - GNUNET_DISK_OPEN_READ, - GNUNET_DISK_PERM_NONE); - } - else - { - GNUNET_assert (NULL != dc->temp_filename); - h = GNUNET_DISK_file_open (dc->temp_filename, - GNUNET_DISK_OPEN_READ, - GNUNET_DISK_PERM_NONE); - } - if (NULL == h) - return; /* oops */ - data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size); - if (NULL == data) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Directory too large for system address space\n")); - } - else - { - if (GNUNET_OK != - GNUNET_FS_directory_list_contents (size, - data, - 0, - &trigger_recursive_download, - dc)) - { - GNUNET_log ( - GNUNET_ERROR_TYPE_WARNING, - _ ( - "Failed to access full directory contents of `%s' for recursive download\n"), - dc->filename); - } - GNUNET_DISK_file_unmap (m); - } - GNUNET_DISK_file_close (h); - if (NULL == dc->filename) - { - if (0 != unlink (dc->temp_filename)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, - "unlink", - dc->temp_filename); - GNUNET_free (dc->temp_filename); - dc->temp_filename = NULL; - } -} - - -/** - * Check if all child-downloads have completed (or trigger them if - * necessary) and once we're completely done, signal completion (and - * possibly recurse to parent). This function MUST be called when the - * download of a file itself is done or when the download of a file is - * done and then later a direct child download has completed (and - * hence this download may complete itself). - * - * @param dc download to check for completion of children - */ -static void -check_completed (struct GNUNET_FS_DownloadContext *dc) -{ - struct GNUNET_FS_ProgressInfo pi; - struct GNUNET_FS_DownloadContext *pos; - - /* first, check if we need to download children */ - if (is_recursive_download (dc)) - full_recursive_download (dc); - /* then, check if children are done already */ - for (pos = dc->child_head; NULL != pos; pos = pos->next) - { - if ((NULL == pos->emsg) && (pos->completed < pos->length)) - return; /* not done yet */ - if ((NULL != pos->child_head) && (pos->has_finished != GNUNET_YES)) - return; /* not transitively done yet */ - } - /* All of our children are done, so mark this download done */ - dc->has_finished = GNUNET_YES; - if (NULL != dc->job_queue) - { - GNUNET_FS_dequeue_ (dc->job_queue); - dc->job_queue = NULL; - } - if (NULL != dc->task) - { - GNUNET_SCHEDULER_cancel (dc->task); - dc->task = NULL; - } - if (NULL != dc->rfh) - { - GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (dc->rfh)); - dc->rfh = NULL; - } - GNUNET_FS_download_sync_ (dc); - - /* signal completion */ - pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED; - GNUNET_FS_download_make_status_ (&pi, dc); - - /* let parent know */ - if (NULL != dc->parent) - check_completed (dc->parent); -} - - -/** - * We got a block of plaintext data (from the meta data). - * Try it for upward reconstruction of the data. On success, - * the top-level block will move to state BRS_DOWNLOAD_UP. - * - * @param dc context for the download - * @param dr download request to match against - * @param data plaintext data, starting from the beginning of the file - * @param data_len number of bytes in data - */ -static void -try_match_block (struct GNUNET_FS_DownloadContext *dc, - struct DownloadRequest *dr, - const char *data, - size_t data_len) -{ - struct GNUNET_FS_ProgressInfo pi; - unsigned int i; - char enc[DBLOCK_SIZE]; - struct ContentHashKey chks[CHK_PER_INODE]; - struct ContentHashKey in_chk; - struct GNUNET_CRYPTO_SymmetricSessionKey sk; - struct GNUNET_CRYPTO_SymmetricInitializationVector iv; - size_t dlen; - struct DownloadRequest *drc; - struct GNUNET_DISK_FileHandle *fh; - int complete; - const char *fn; - const char *odata; - size_t odata_len; - - odata = data; - odata_len = data_len; - if (BRS_DOWNLOAD_UP == dr->state) - return; - if (dr->depth > 0) - { - if ((dc->offset > 0) || - (dc->length < GNUNET_ntohll (dc->uri->data.chk.file_length))) - { - /* NOTE: this test is not tight, but should suffice; the issue - here is that 'dr->num_children' may inherently only specify a - smaller range than what is in the original file; - thus, reconstruction of (some) inner blocks will fail. - FIXME: we might eventually want to write a tighter test to - maximize the circumstances under which we do succeed with - IBlock reconstruction. (need good tests though). */return; - } - complete = GNUNET_YES; - for (i = 0; i < dr->num_children; i++) - { - drc = dr->children[i]; - try_match_block (dc, drc, data, data_len); - if (drc->state != BRS_RECONSTRUCT_META_UP) - complete = GNUNET_NO; - else - chks[i] = drc->chk; - } - if (GNUNET_YES != complete) - return; - data = (const char *) chks; - dlen = dr->num_children * sizeof(struct ContentHashKey); - } - else - { - if (dr->offset > data_len) - return; /* oops */ - dlen = GNUNET_MIN (data_len - dr->offset, DBLOCK_SIZE); - } - GNUNET_CRYPTO_hash (&data[dr->offset], dlen, &in_chk.key); - GNUNET_CRYPTO_hash_to_aes_key (&in_chk.key, &sk, &iv); - if (-1 == - GNUNET_CRYPTO_symmetric_encrypt (&data[dr->offset], dlen, &sk, &iv, enc)) - { - GNUNET_break (0); - return; - } - GNUNET_CRYPTO_hash (enc, dlen, &in_chk.query); - switch (dr->state) - { - case BRS_INIT: - dr->chk = in_chk; - dr->state = BRS_RECONSTRUCT_META_UP; - break; - - case BRS_CHK_SET: - if (0 != memcmp (&in_chk, &dr->chk, sizeof(struct ContentHashKey))) - { - /* other peer provided bogus meta data */ - GNUNET_break_op (0); - break; - } - /* write block to disk */ - fn = (NULL != dc->filename) ? dc->filename : dc->temp_filename; - if (NULL != fn) - { - fh = GNUNET_DISK_file_open (fn, - GNUNET_DISK_OPEN_READWRITE - | GNUNET_DISK_OPEN_CREATE - | GNUNET_DISK_OPEN_TRUNCATE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE - | GNUNET_DISK_PERM_GROUP_READ - | GNUNET_DISK_PERM_OTHER_READ); - if (NULL == fh) - { - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "open", fn); - GNUNET_asprintf (&dc->emsg, - _ ("Failed to open file `%s' for writing"), - fn); - GNUNET_DISK_file_close (fh); - dr->state = BRS_ERROR; - pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; - pi.value.download.specifics.error.message = dc->emsg; - GNUNET_FS_download_make_status_ (&pi, dc); - return; - } - if (data_len != GNUNET_DISK_file_write (fh, odata, odata_len)) - { - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "write", fn); - GNUNET_asprintf (&dc->emsg, - _ ("Failed to open file `%s' for writing"), - fn); - GNUNET_DISK_file_close (fh); - dr->state = BRS_ERROR; - pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; - pi.value.download.specifics.error.message = dc->emsg; - GNUNET_FS_download_make_status_ (&pi, dc); - return; - } - GNUNET_DISK_file_close (fh); - } - /* signal success */ - dr->state = BRS_DOWNLOAD_UP; - dc->completed = dc->length; - GNUNET_FS_download_sync_ (dc); - pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS; - pi.value.download.specifics.progress.data = data; - pi.value.download.specifics.progress.offset = 0; - pi.value.download.specifics.progress.data_len = dlen; - pi.value.download.specifics.progress.depth = 0; - pi.value.download.specifics.progress.respect_offered = 0; - pi.value.download.specifics.progress.block_download_duration = - GNUNET_TIME_UNIT_ZERO; - GNUNET_FS_download_make_status_ (&pi, dc); - if ((NULL != dc->filename) && - (0 != truncate (dc->filename, - GNUNET_ntohll (dc->uri->data.chk.file_length)))) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, - "truncate", - dc->filename); - check_completed (dc); - break; - - default: - /* how did we get here? */ - GNUNET_break (0); - break; - } -} - - -/** - * Type of a function that libextractor calls for each - * meta data item found. If we find full data meta data, - * call 'try_match_block' on it. - * - * @param cls our 'struct GNUNET_FS_DownloadContext*' - * @param plugin_name name of the plugin that produced this value; - * special values can be used (e.g. '<zlib>' for zlib being - * used in the main libextractor library and yielding - * meta data). - * @param type libextractor-type describing the meta data - * @param format basic format information about data - * @param data_mime_type mime-type of data (not of the original file); - * can be NULL (if mime-type is not known) - * @param data actual meta-data found - * @param data_len number of bytes in data - * @return 0 to continue extracting, 1 to abort - */ -static int -match_full_data (void *cls, - const char *plugin_name, - enum EXTRACTOR_MetaType type, - enum EXTRACTOR_MetaFormat format, - const char *data_mime_type, - const char *data, - size_t data_len) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - - if (EXTRACTOR_METATYPE_GNUNET_FULL_DATA != type) - return 0; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Found %u bytes of FD!\n", - (unsigned int) data_len); - if (GNUNET_FS_uri_chk_get_file_size (dc->uri) != data_len) - { - GNUNET_break_op (0); - return 1; /* bogus meta data */ - } - try_match_block (dc, dc->top_request, data, data_len); - return 1; -} - - -/** - * Set the state of the given download request to - * BRS_DOWNLOAD_UP and propagate it up the tree. - * - * @param dr download request that is done - */ -static void -propagate_up (struct DownloadRequest *dr) -{ - unsigned int i; - - do - { - dr->state = BRS_DOWNLOAD_UP; - dr = dr->parent; - if (NULL == dr) - break; - for (i = 0; i < dr->num_children; i++) - if (dr->children[i]->state != BRS_DOWNLOAD_UP) - break; - } - while (i == dr->num_children); -} - - -/** - * Try top-down reconstruction. Before, the given request node - * must have the state BRS_CHK_SET. Afterwards, more nodes may - * have that state or advanced to BRS_DOWNLOAD_DOWN or even - * BRS_DOWNLOAD_UP. It is also possible to get BRS_ERROR on the - * top level. - * - * @param dc overall download this block belongs to - * @param dr block to reconstruct - */ -static void -try_top_down_reconstruction (struct GNUNET_FS_DownloadContext *dc, - struct DownloadRequest *dr) -{ - uint64_t off; - char block[DBLOCK_SIZE]; - struct GNUNET_HashCode key; - uint64_t total; - size_t len; - unsigned int i; - struct DownloadRequest *drc; - uint64_t child_block_size; - const struct ContentHashKey *chks; - int up_done; - - GNUNET_assert (NULL != dc->rfh); - GNUNET_assert (BRS_CHK_SET == dr->state); - total = GNUNET_FS_uri_chk_get_file_size (dc->uri); - GNUNET_assert (dr->depth < dc->treedepth); - len = GNUNET_FS_tree_calculate_block_size (total, dr->offset, dr->depth); - GNUNET_assert (len <= DBLOCK_SIZE); - off = compute_disk_offset (total, dr->offset, dr->depth); - if (dc->old_file_size < off + len) - return; /* failure */ - if (off != GNUNET_DISK_file_seek (dc->rfh, off, GNUNET_DISK_SEEK_SET)) - { - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "seek", dc->filename); - return; /* failure */ - } - if (len != GNUNET_DISK_file_read (dc->rfh, block, len)) - { - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "read", dc->filename); - return; /* failure */ - } - GNUNET_CRYPTO_hash (block, len, &key); - if (0 != memcmp (&key, &dr->chk.key, sizeof(struct GNUNET_HashCode))) - return; /* mismatch */ - if (GNUNET_OK != - encrypt_existing_match (dc, &dr->chk, dr, block, len, GNUNET_NO)) - { - /* hash matches but encrypted block does not, really bad */ - dr->state = BRS_ERROR; - /* propagate up */ - while (NULL != dr->parent) - { - dr = dr->parent; - dr->state = BRS_ERROR; - } - return; - } - /* block matches */ - dr->state = BRS_DOWNLOAD_DOWN; - - /* set CHKs for children */ - up_done = GNUNET_YES; - chks = (const struct ContentHashKey *) block; - for (i = 0; i < dr->num_children; i++) - { - drc = dr->children[i]; - GNUNET_assert (drc->offset >= dr->offset); - child_block_size = GNUNET_FS_tree_compute_tree_size (drc->depth); - GNUNET_assert (0 == (drc->offset - dr->offset) % child_block_size); - if (BRS_INIT == drc->state) - { - drc->state = BRS_CHK_SET; - drc->chk = chks[drc->chk_idx]; - try_top_down_reconstruction (dc, drc); - } - if (BRS_DOWNLOAD_UP != drc->state) - up_done = GNUNET_NO; /* children not all done */ - } - if (GNUNET_YES == up_done) - propagate_up (dr); /* children all done (or no children...) */ -} - - -/** - * Add entries to the message queue. - * - * @param cls our download context - * @param key unused - * @param entry entry of type `struct DownloadRequest` - * @return #GNUNET_OK - */ -static int -retry_entry (void *cls, const struct GNUNET_HashCode *key, void *entry) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - struct DownloadRequest *dr = entry; - struct SearchMessage *sm; - struct GNUNET_MQ_Envelope *env; - - env = GNUNET_MQ_msg (sm, GNUNET_MESSAGE_TYPE_FS_START_SEARCH); - if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY)) - sm->options = htonl (GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY); - else - sm->options = htonl (GNUNET_FS_SEARCH_OPTION_NONE); - if (0 == dr->depth) - sm->type = htonl (GNUNET_BLOCK_TYPE_FS_DBLOCK); - else - sm->type = htonl (GNUNET_BLOCK_TYPE_FS_IBLOCK); - sm->anonymity_level = htonl (dc->anonymity); - sm->target = dc->target; - sm->query = dr->chk.query; - GNUNET_MQ_send (dc->mq, env); - return GNUNET_OK; -} - - -/** - * Schedule the download of the specified block in the tree. - * - * @param dc overall download this block belongs to - * @param dr request to schedule - */ -static void -schedule_block_download (struct GNUNET_FS_DownloadContext *dc, - struct DownloadRequest *dr) -{ - unsigned int i; - - switch (dr->state) - { - case BRS_INIT: - GNUNET_assert (0); - break; - - case BRS_RECONSTRUCT_DOWN: - GNUNET_assert (0); - break; - - case BRS_RECONSTRUCT_META_UP: - GNUNET_assert (0); - break; - - case BRS_RECONSTRUCT_UP: - GNUNET_assert (0); - break; - - case BRS_CHK_SET: - /* normal case, start download */ - break; - - case BRS_DOWNLOAD_DOWN: - for (i = 0; i < dr->num_children; i++) - schedule_block_download (dc, dr->children[i]); - return; - - case BRS_DOWNLOAD_UP: - /* We're done! */ - return; - - case BRS_ERROR: - GNUNET_break (0); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Scheduling download at offset %llu and depth %u for `%s'\n", - (unsigned long long) dr->offset, - dr->depth, - GNUNET_h2s (&dr->chk.query)); - if (GNUNET_NO != GNUNET_CONTAINER_multihashmap_contains_value (dc->active, - &dr->chk.query, - dr)) - return; /* already active */ - GNUNET_CONTAINER_multihashmap_put (dc->active, - &dr->chk.query, - dr, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); - if (NULL == dc->mq) - return; /* download not active */ - retry_entry (dc, &dr->chk.query, dr); -} - - -#define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX - -/** - * We found an entry in a directory. Check if the respective child - * already exists and if not create the respective child download. - * - * @param cls the parent download - * @param filename name of the file in the directory - * @param uri URI of the file (CHK or LOC) - * @param meta meta data of the file - * @param length number of bytes in data - * @param data contents of the file (or NULL if they were not inlined) - */ -static void -trigger_recursive_download (void *cls, - const char *filename, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *meta, - size_t length, - const void *data) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - struct GNUNET_FS_DownloadContext *cpos; - char *temp_name; - char *fn; - char *us; - char *ext; - char *dn; - char *pos; - char *full_name; - char *sfn; - - if (NULL == uri) - return; /* entry for the directory itself */ - cpos = dc->child_head; - while (NULL != cpos) - { - if ((GNUNET_FS_uri_test_equal (uri, cpos->uri)) || - ((NULL != filename) && (0 == strcmp (cpos->filename, filename)))) - break; - cpos = cpos->next; - } - if (NULL != cpos) - return; /* already exists */ - fn = NULL; - if (NULL == filename) - { - fn = GNUNET_FS_meta_data_suggest_filename (meta); - if (NULL == fn) - { - us = GNUNET_FS_uri_to_string (uri); - fn = GNUNET_strdup (&us[strlen (GNUNET_FS_URI_CHK_PREFIX)]); - GNUNET_free (us); - } - else if ('.' == fn[0]) - { - ext = fn; - us = GNUNET_FS_uri_to_string (uri); - GNUNET_asprintf (&fn, - "%s%s", - &us[strlen (GNUNET_FS_URI_CHK_PREFIX)], - ext); - GNUNET_free (ext); - GNUNET_free (us); - } - /* change '\' to '/' (this should have happened - * during insertion, but malicious peers may - * not have done this) */ - while (NULL != (pos = strstr (fn, "\\"))) - *pos = '/'; - /* remove '../' everywhere (again, well-behaved - * peers don't do this, but don't trust that - * we did not get something nasty) */ - while (NULL != (pos = strstr (fn, "../"))) - { - pos[0] = '_'; - pos[1] = '_'; - pos[2] = '_'; - } - filename = fn; - } - if (NULL == dc->filename) - { - full_name = NULL; - } - else - { - dn = GNUNET_strdup (dc->filename); - GNUNET_break ( - (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) && - (NULL != strstr (dn + strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT), - GNUNET_FS_DIRECTORY_EXT))); - sfn = GNUNET_strdup (filename); - while ((strlen (sfn) > 0) && ('/' == filename[strlen (sfn) - 1])) - sfn[strlen (sfn) - 1] = '\0'; - if ((strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) && - (NULL != strstr (dn + strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT), - GNUNET_FS_DIRECTORY_EXT))) - dn[strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0'; - if ((GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) && - ((strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) || - (NULL == strstr (filename + strlen (filename) - - strlen (GNUNET_FS_DIRECTORY_EXT), - GNUNET_FS_DIRECTORY_EXT)))) - { - GNUNET_asprintf (&full_name, - "%s%s%s%s", - dn, - DIR_SEPARATOR_STR, - sfn, - GNUNET_FS_DIRECTORY_EXT); - } - else - { - GNUNET_asprintf (&full_name, "%s%s%s", dn, DIR_SEPARATOR_STR, sfn); - } - GNUNET_free (sfn); - GNUNET_free (dn); - } - if ((NULL != full_name) && - (GNUNET_OK != GNUNET_DISK_directory_create_for_file (full_name))) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ( - "Failed to create directory for recursive download of `%s'\n"), - full_name); - GNUNET_free (full_name); - GNUNET_free (fn); - return; - } - - temp_name = NULL; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Triggering recursive download of size %llu with %u bytes MD\n", - (unsigned long long) GNUNET_FS_uri_chk_get_file_size (uri), - (unsigned int) GNUNET_FS_meta_data_get_serialized_size ( - meta)); - GNUNET_FS_download_start (dc->h, - uri, - meta, - full_name, - temp_name, - 0, - GNUNET_FS_uri_chk_get_file_size (uri), - dc->anonymity, - dc->options, - NULL, - dc); - GNUNET_free (full_name); - GNUNET_free (temp_name); - GNUNET_free (fn); -} - - -/** - * (recursively) free download request structure - * - * @param dr request to free - */ -void -GNUNET_FS_free_download_request_ (struct DownloadRequest *dr) -{ - if (NULL == dr) - return; - for (unsigned int i = 0; i < dr->num_children; i++) - GNUNET_FS_free_download_request_ (dr->children[i]); - GNUNET_free (dr->children); - GNUNET_free (dr); -} - - -static int -process_result_with_request (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct ProcessResultClosure *prc = cls; - struct DownloadRequest *dr = value; - struct GNUNET_FS_DownloadContext *dc = prc->dc; - struct DownloadRequest *drc; - struct GNUNET_DISK_FileHandle *fh = NULL; - struct GNUNET_CRYPTO_SymmetricSessionKey skey; - struct GNUNET_CRYPTO_SymmetricInitializationVector iv; - char pt[prc->size]; - struct GNUNET_FS_ProgressInfo pi; - uint64_t off; - size_t bs; - size_t app; - int i; - struct ContentHashKey *chkarr; - - GNUNET_log ( - GNUNET_ERROR_TYPE_DEBUG, - "Received %u byte block `%s' matching pending request at depth %u and offset %llu/%llu\n", - (unsigned int) prc->size, - GNUNET_h2s (key), - dr->depth, - (unsigned long long) dr->offset, - (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length)); - bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll ( - dc->uri->data.chk.file_length), - dr->offset, - dr->depth); - if (prc->size != bs) - { - GNUNET_asprintf ( - &dc->emsg, - _ ( - "Internal error or bogus download URI (expected %lu bytes at depth %u and offset %llu/%llu, got %lu bytes)"), - bs, - dr->depth, - (unsigned long long) dr->offset, - (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length), - prc->size); - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "%s\n", dc->emsg); - while (NULL != dr->parent) - { - dr->state = BRS_ERROR; - dr = dr->parent; - } - dr->state = BRS_ERROR; - goto signal_error; - } - - (void) GNUNET_CONTAINER_multihashmap_remove (dc->active, &prc->query, dr); - GNUNET_CRYPTO_hash_to_aes_key (&dr->chk.key, &skey, &iv); - if (-1 == - GNUNET_CRYPTO_symmetric_decrypt (prc->data, prc->size, &skey, &iv, pt)) - { - GNUNET_break (0); - dc->emsg = GNUNET_strdup (_ ("internal error decrypting content")); - goto signal_error; - } - off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length), - dr->offset, - dr->depth); - /* save to disk */ - if ((GNUNET_YES == prc->do_store) && - ((NULL != dc->filename) || (is_recursive_download (dc))) && - ((dr->depth == dc->treedepth) || - (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)))) - { - fh = GNUNET_DISK_file_open (NULL != dc->filename ? dc->filename - : dc->temp_filename, - GNUNET_DISK_OPEN_READWRITE - | GNUNET_DISK_OPEN_CREATE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE - | GNUNET_DISK_PERM_GROUP_READ - | GNUNET_DISK_PERM_OTHER_READ); - if (NULL == fh) - { - GNUNET_asprintf (&dc->emsg, - _ ("Download failed: could not open file `%s': %s"), - dc->filename, - strerror (errno)); - goto signal_error; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Saving decrypted block to disk at offset %llu\n", - (unsigned long long) off); - if ((off != GNUNET_DISK_file_seek (fh, off, GNUNET_DISK_SEEK_SET))) - { - GNUNET_asprintf (&dc->emsg, - _ ("Failed to seek to offset %llu in file `%s': %s"), - (unsigned long long) off, - dc->filename, - strerror (errno)); - goto signal_error; - } - if (prc->size != GNUNET_DISK_file_write (fh, pt, prc->size)) - { - GNUNET_asprintf ( - &dc->emsg, - _ ("Failed to write block of %u bytes at offset %llu in file `%s': %s"), - (unsigned int) prc->size, - (unsigned long long) off, - dc->filename, - strerror (errno)); - goto signal_error; - } - GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh)); - fh = NULL; - } - - if (0 == dr->depth) - { - /* DBLOCK, update progress and try recursion if applicable */ - app = prc->size; - if (dr->offset < dc->offset) - { - /* starting offset begins in the middle of pt, - * do not count first bytes as progress */ - GNUNET_assert (app > (dc->offset - dr->offset)); - app -= (dc->offset - dr->offset); - } - if (dr->offset + prc->size > dc->offset + dc->length) - { - /* end of block is after relevant range, - * do not count last bytes as progress */ - GNUNET_assert (app > - (dr->offset + prc->size) - (dc->offset + dc->length)); - app -= (dr->offset + prc->size) - (dc->offset + dc->length); - } - dc->completed += app; - - /* do recursive download if option is set and either meta data - * says it is a directory or if no meta data is given AND filename - * ends in '.gnd' (top-level case) */ - if (is_recursive_download (dc)) - GNUNET_FS_directory_list_contents (prc->size, - pt, - off, - &trigger_recursive_download, - dc); - } - GNUNET_assert (dc->completed <= dc->length); - dr->state = BRS_DOWNLOAD_DOWN; - pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS; - pi.value.download.specifics.progress.data = pt; - pi.value.download.specifics.progress.offset = dr->offset; - pi.value.download.specifics.progress.data_len = prc->size; - pi.value.download.specifics.progress.depth = dr->depth; - pi.value.download.specifics.progress.respect_offered = prc->respect_offered; - pi.value.download.specifics.progress.num_transmissions = - prc->num_transmissions; - if (prc->last_transmission.abs_value_us != - GNUNET_TIME_UNIT_FOREVER_ABS.abs_value_us) - pi.value.download.specifics.progress.block_download_duration = - GNUNET_TIME_absolute_get_duration (prc->last_transmission); - else - pi.value.download.specifics.progress.block_download_duration = - GNUNET_TIME_UNIT_ZERO; /* found locally */ - GNUNET_FS_download_make_status_ (&pi, dc); - if (0 == dr->depth) - propagate_up (dr); - - if (dc->completed == dc->length) - { - /* download completed, signal */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Download completed, truncating file to desired length %llu\n", - (unsigned long long) GNUNET_ntohll ( - dc->uri->data.chk.file_length)); - /* truncate file to size (since we store IBlocks at the end) */ - if (NULL != dc->filename) - { - if (0 != truncate (dc->filename, - GNUNET_ntohll (dc->uri->data.chk.file_length))) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, - "truncate", - dc->filename); - } - GNUNET_assert (0 == dr->depth); - check_completed (dc); - } - if (0 == dr->depth) - { - /* bottom of the tree, no child downloads possible, just sync */ - GNUNET_FS_download_sync_ (dc); - return GNUNET_YES; - } - - GNUNET_log ( - GNUNET_ERROR_TYPE_DEBUG, - "Triggering downloads of children (this block was at depth %u and offset %llu)\n", - dr->depth, - (unsigned long long) dr->offset); - GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey))); - chkarr = (struct ContentHashKey *) pt; - for (i = dr->num_children - 1; i >= 0; i--) - { - drc = dr->children[i]; - switch (drc->state) - { - case BRS_INIT: - if ((drc->chk_idx + 1) * sizeof(struct ContentHashKey) > prc->size) - { - /* 'chkarr' does not have enough space for this chk_idx; - internal error! */ - GNUNET_break (0); - GNUNET_assert (0); - dc->emsg = GNUNET_strdup (_ ("internal error decoding tree")); - goto signal_error; - } - drc->chk = chkarr[drc->chk_idx]; - drc->state = BRS_CHK_SET; - if (GNUNET_YES == dc->issue_requests) - schedule_block_download (dc, drc); - break; - - case BRS_RECONSTRUCT_DOWN: - GNUNET_assert (0); - break; - - case BRS_RECONSTRUCT_META_UP: - GNUNET_assert (0); - break; - - case BRS_RECONSTRUCT_UP: - GNUNET_assert (0); - break; - - case BRS_CHK_SET: - GNUNET_assert (0); - break; - - case BRS_DOWNLOAD_DOWN: - GNUNET_assert (0); - break; - - case BRS_DOWNLOAD_UP: - GNUNET_assert (0); - break; - - case BRS_ERROR: - GNUNET_assert (0); - break; - - default: - GNUNET_assert (0); - break; - } - } - GNUNET_FS_download_sync_ (dc); - return GNUNET_YES; - -signal_error: - if (NULL != fh) - GNUNET_DISK_file_close (fh); - pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; - pi.value.download.specifics.error.message = dc->emsg; - GNUNET_FS_download_make_status_ (&pi, dc); - GNUNET_MQ_destroy (dc->mq); - dc->mq = NULL; - GNUNET_FS_free_download_request_ (dc->top_request); - dc->top_request = NULL; - if (NULL != dc->job_queue) - { - GNUNET_FS_dequeue_ (dc->job_queue); - dc->job_queue = NULL; - } - GNUNET_FS_download_sync_ (dc); - return GNUNET_NO; -} - - -/** - * Type of a function to call when we check the PUT message - * from the service. - * - * @param cls closure - * @param msg message received - */ -static int -check_put (void *cls, const struct ClientPutMessage *cm) -{ - /* any varsize length is OK */ - return GNUNET_OK; -} - - -/** - * Type of a function to call when we receive a message - * from the service. - * - * @param cls closure - * @param msg message received - */ -static void -handle_put (void *cls, const struct ClientPutMessage *cm) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - uint16_t msize = ntohs (cm->header.size) - sizeof(*cm); - struct ProcessResultClosure prc; - - prc.dc = dc; - prc.data = &cm[1]; - prc.last_transmission = GNUNET_TIME_absolute_ntoh (cm->last_transmission); - prc.size = msize; - prc.type = ntohl (cm->type); - prc.do_store = GNUNET_YES; - prc.respect_offered = ntohl (cm->respect_offered); - prc.num_transmissions = ntohl (cm->num_transmissions); - GNUNET_CRYPTO_hash (prc.data, msize, &prc.query); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Received result for query `%s' from FS service\n", - GNUNET_h2s (&prc.query)); - GNUNET_CONTAINER_multihashmap_get_multiple (dc->active, - &prc.query, - &process_result_with_request, - &prc); -} - - -/** - * Generic error handler, called with the appropriate error code and - * the same closure specified at the creation of the message queue. - * Not every message queue implementation supports an error handler. - * - * @param cls closure with the `struct GNUNET_FS_DownloadContext *` - * @param error error code - */ -static void -download_mq_error_handler (void *cls, enum GNUNET_MQ_Error error) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - - if (NULL != dc->mq) - { - GNUNET_MQ_destroy (dc->mq); - dc->mq = NULL; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Transmitting download request failed, trying to reconnect\n"); - try_reconnect (dc); -} - - -/** - * Reconnect to the FS service and transmit our queries NOW. - * - * @param cls our download context - */ -static void -do_reconnect (void *cls) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - struct GNUNET_MQ_MessageHandler handlers[] = - { GNUNET_MQ_hd_var_size (put, - GNUNET_MESSAGE_TYPE_FS_PUT, - struct ClientPutMessage, - dc), - GNUNET_MQ_handler_end () }; - - dc->task = NULL; - dc->mq = GNUNET_CLIENT_connect (dc->h->cfg, - "fs", - handlers, - &download_mq_error_handler, - dc); - if (NULL == dc->mq) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - "Connecting to `%s'-service failed, will try again.\n", - "FS"); - try_reconnect (dc); - return; - } - GNUNET_CONTAINER_multihashmap_iterate (dc->active, &retry_entry, dc); -} - - -/** - * We've lost our connection with the FS service. - * Re-establish it and re-transmit all of our - * pending requests. - * - * @param dc download context that is having trouble - */ -static void -try_reconnect (struct GNUNET_FS_DownloadContext *dc) -{ - if (NULL != dc->mq) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Moving all requests back to pending list\n"); - GNUNET_MQ_destroy (dc->mq); - dc->mq = NULL; - } - if (0 == dc->reconnect_backoff.rel_value_us) - dc->reconnect_backoff = GNUNET_TIME_UNIT_MILLISECONDS; - else - dc->reconnect_backoff = GNUNET_TIME_STD_BACKOFF (dc->reconnect_backoff); - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Will try to reconnect in %s\n", - GNUNET_STRINGS_relative_time_to_string (dc->reconnect_backoff, - GNUNET_YES)); - GNUNET_break (NULL != dc->job_queue); - dc->task = - GNUNET_SCHEDULER_add_delayed (dc->reconnect_backoff, &do_reconnect, dc); -} - - -/** - * We're allowed to ask the FS service for our blocks. Start the download. - * - * @param cls the 'struct GNUNET_FS_DownloadContext' - */ -static void -activate_fs_download (void *cls) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - struct GNUNET_FS_ProgressInfo pi; - - GNUNET_assert (NULL == dc->mq); - GNUNET_assert (NULL != dc->active); - do_reconnect (dc); - if (NULL != dc->mq) - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download activated\n"); - pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE; - GNUNET_FS_download_make_status_ (&pi, dc); -} - - -/** - * We must stop to ask the FS service for our blocks. Pause the download. - * - * @param cls the `struct GNUNET_FS_DownloadContext` - */ -static void -deactivate_fs_download (void *cls) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - struct GNUNET_FS_ProgressInfo pi; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download deactivated\n"); - if (NULL != dc->mq) - { - GNUNET_MQ_destroy (dc->mq); - dc->mq = NULL; - } - pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE; - GNUNET_FS_download_make_status_ (&pi, dc); -} - - -/** - * (recursively) Create a download request structure. - * - * @param parent parent of the current entry - * @param chk_idx index of the chk for this block in the parent block - * @param depth depth of the current entry, 0 are the DBLOCKs, - * top level block is 'dc->treedepth - 1' - * @param dr_offset offset in the original file this block maps to - * (as in, offset of the first byte of the first DBLOCK - * in the subtree rooted in the returned download request tree) - * @param file_start_offset desired starting offset for the download - * in the original file; requesting tree should not contain - * DBLOCKs prior to the file_start_offset - * @param desired_length desired number of bytes the user wanted to access - * (from file_start_offset). Resulting tree should not contain - * DBLOCKs after file_start_offset + file_length. - * @return download request tree for the given range of DBLOCKs at - * the specified depth - */ -static struct DownloadRequest * -create_download_request (struct DownloadRequest *parent, - unsigned int chk_idx, - unsigned int depth, - uint64_t dr_offset, - uint64_t file_start_offset, - uint64_t desired_length) -{ - struct DownloadRequest *dr; - unsigned int i; - unsigned int head_skip; - uint64_t child_block_size; - - dr = GNUNET_new (struct DownloadRequest); - dr->parent = parent; - dr->depth = depth; - dr->offset = dr_offset; - dr->chk_idx = chk_idx; - if (0 == depth) - return dr; - child_block_size = GNUNET_FS_tree_compute_tree_size (depth - 1); - - /* calculate how many blocks at this level are not interesting - * from the start (rounded down), either because of the requested - * file offset or because this IBlock is further along */ - if (dr_offset < file_start_offset) - { - head_skip = (file_start_offset - dr_offset) / child_block_size; - } - else - { - head_skip = 0; - } - - /* calculate index of last block at this level that is interesting (rounded up) */ - dr->num_children = - (file_start_offset + desired_length - dr_offset) / child_block_size; - if (dr->num_children * child_block_size < - file_start_offset + desired_length - dr_offset) - dr->num_children++; /* round up */ - GNUNET_assert (dr->num_children > head_skip); - dr->num_children -= head_skip; - if (dr->num_children > CHK_PER_INODE) - dr->num_children = CHK_PER_INODE; /* cap at max */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Block at offset %llu and depth %u has %u children\n", - (unsigned long long) dr_offset, - depth, - dr->num_children); - - /* now we can get the total number of *interesting* children for this block */ - - /* why else would we have gotten here to begin with? (that'd be a bad logic error) */ - GNUNET_assert (dr->num_children > 0); - - dr->children = GNUNET_new_array (dr->num_children, struct DownloadRequest *); - for (i = 0; i < dr->num_children; i++) - { - dr->children[i] = - create_download_request (dr, - i + head_skip, - depth - 1, - dr_offset + (i + head_skip) * child_block_size, - file_start_offset, - desired_length); - } - return dr; -} - - -/** - * Continuation after a possible attempt to reconstruct - * the current IBlock from the existing file. - * - * @param cls the 'struct ReconstructContext' - */ -static void -reconstruct_cont (void *cls) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - - /* clean up state from tree encoder */ - if (NULL != dc->task) - { - GNUNET_SCHEDULER_cancel (dc->task); - dc->task = NULL; - } - if (NULL != dc->rfh) - { - GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (dc->rfh)); - dc->rfh = NULL; - } - /* start "normal" download */ - dc->issue_requests = GNUNET_YES; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting normal download\n"); - schedule_block_download (dc, dc->top_request); -} - - -/** - * Task requesting the next block from the tree encoder. - * - * @param cls the 'struct GNUJNET_FS_DownloadContext' we're processing - */ -static void -get_next_block (void *cls) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - - dc->task = NULL; - GNUNET_FS_tree_encoder_next (dc->te); -} - - -/** - * Function called asking for the current (encoded) - * block to be processed. After processing the - * client should either call "GNUNET_FS_tree_encode_next" - * or (on error) "GNUNET_FS_tree_encode_finish". - * - * This function checks if the content on disk matches - * the expected content based on the URI. - * - * @param cls closure - * @param chk content hash key for the block - * @param offset offset of the block - * @param depth depth of the block, 0 for DBLOCK - * @param type type of the block (IBLOCK or DBLOCK) - * @param block the (encrypted) block - * @param block_size size of block (in bytes) - */ -static void -reconstruct_cb (void *cls, - const struct ContentHashKey *chk, - uint64_t offset, - unsigned int depth, - enum GNUNET_BLOCK_Type type, - const void *block, - uint16_t block_size) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - struct GNUNET_FS_ProgressInfo pi; - struct DownloadRequest *dr; - uint64_t blen; - unsigned int chld; - - /* find corresponding request entry */ - dr = dc->top_request; - while (dr->depth > depth) - { - GNUNET_assert (dr->num_children > 0); - blen = GNUNET_FS_tree_compute_tree_size (dr->depth - 1); - chld = (offset - dr->offset) / blen; - if (chld < dr->children[0]->chk_idx) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Block %u < %u irrelevant for our range\n", - chld, - dr->children[0]->chk_idx); - dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc); - return; /* irrelevant block */ - } - if (chld > dr->children[dr->num_children - 1]->chk_idx) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Block %u > %u irrelevant for our range\n", - chld, - dr->children[dr->num_children - 1]->chk_idx); - dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc); - return; /* irrelevant block */ - } - dr = dr->children[chld - dr->children[0]->chk_idx]; - } - GNUNET_log ( - GNUNET_ERROR_TYPE_DEBUG, - "Matched TE block with request at offset %llu and depth %u in state %d\n", - (unsigned long long) dr->offset, - dr->depth, - dr->state); - /* FIXME: this code needs more testing and might - need to handle more states... */ - switch (dr->state) - { - case BRS_INIT: - break; - - case BRS_RECONSTRUCT_DOWN: - break; - - case BRS_RECONSTRUCT_META_UP: - break; - - case BRS_RECONSTRUCT_UP: - break; - - case BRS_CHK_SET: - if (0 == memcmp (chk, &dr->chk, sizeof(struct ContentHashKey))) - { - GNUNET_log ( - GNUNET_ERROR_TYPE_DEBUG, - "Reconstruction succeeded, can use block at offset %llu, depth %u\n", - (unsigned long long) offset, - depth); - /* block matches, hence tree below matches; - * this request is done! */ - dr->state = BRS_DOWNLOAD_UP; - (void) GNUNET_CONTAINER_multihashmap_remove (dc->active, - &dr->chk.query, - dr); - /* calculate how many bytes of payload this block - * corresponds to */ - blen = GNUNET_FS_tree_compute_tree_size (dr->depth); - /* how many of those bytes are in the requested range? */ - blen = GNUNET_MIN (blen, dc->length + dc->offset - dr->offset); - /* signal progress */ - dc->completed += blen; - pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS; - pi.value.download.specifics.progress.data = NULL; - pi.value.download.specifics.progress.offset = offset; - pi.value.download.specifics.progress.data_len = 0; - pi.value.download.specifics.progress.depth = 0; - pi.value.download.specifics.progress.respect_offered = 0; - pi.value.download.specifics.progress.block_download_duration = - GNUNET_TIME_UNIT_ZERO; - GNUNET_FS_download_make_status_ (&pi, dc); - /* FIXME: duplicated code from 'process_result_with_request - refactor */ - if (dc->completed == dc->length) - { - /* download completed, signal */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Download completed, truncating file to desired length %llu\n", - (unsigned long long) GNUNET_ntohll ( - dc->uri->data.chk.file_length)); - /* truncate file to size (since we store IBlocks at the end) */ - if (NULL != dc->filename) - { - if (0 != truncate (dc->filename, - GNUNET_ntohll (dc->uri->data.chk.file_length))) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, - "truncate", - dc->filename); - } - } - } - else - GNUNET_log ( - GNUNET_ERROR_TYPE_DEBUG, - "Reconstruction failed, need to download block at offset %llu, depth %u\n", - (unsigned long long) offset, - depth); - break; - - case BRS_DOWNLOAD_DOWN: - break; - - case BRS_DOWNLOAD_UP: - break; - - case BRS_ERROR: - break; - - default: - GNUNET_assert (0); - break; - } - dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc); - if ((dr == dc->top_request) && (dr->state == BRS_DOWNLOAD_UP)) - check_completed (dc); -} - - -/** - * Function called by the tree encoder to obtain a block of plaintext - * data (for the lowest level of the tree). - * - * @param cls our 'struct ReconstructContext' - * @param offset identifies which block to get - * @param max (maximum) number of bytes to get; returning - * fewer will also cause errors - * @param buf where to copy the plaintext buffer - * @param emsg location to store an error message (on error) - * @return number of bytes copied to buf, 0 on error - */ -static size_t -fh_reader (void *cls, uint64_t offset, size_t max, void *buf, char **emsg) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - struct GNUNET_DISK_FileHandle *fh = dc->rfh; - ssize_t ret; - - if (NULL != emsg) - *emsg = NULL; - if (offset != GNUNET_DISK_file_seek (fh, offset, GNUNET_DISK_SEEK_SET)) - { - if (NULL != emsg) - *emsg = GNUNET_strdup (strerror (errno)); - return 0; - } - ret = GNUNET_DISK_file_read (fh, buf, max); - if (ret < 0) - { - if (NULL != emsg) - *emsg = GNUNET_strdup (strerror (errno)); - return 0; - } - return ret; -} - - -/** - * Task that creates the initial (top-level) download - * request for the file. - * - * @param cls the 'struct GNUNET_FS_DownloadContext' - */ -void -GNUNET_FS_download_start_task_ (void *cls) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - struct GNUNET_FS_ProgressInfo pi; - struct GNUNET_DISK_FileHandle *fh; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Start task running...\n"); - dc->task = NULL; - if (0 == dc->length) - { - /* no bytes required! */ - if (NULL != dc->filename) - { - fh = GNUNET_DISK_file_open (dc->filename, - GNUNET_DISK_OPEN_READWRITE - | GNUNET_DISK_OPEN_CREATE - | ((0 == - GNUNET_FS_uri_chk_get_file_size (dc->uri)) - ? GNUNET_DISK_OPEN_TRUNCATE - : 0), - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE - | GNUNET_DISK_PERM_GROUP_READ - | GNUNET_DISK_PERM_OTHER_READ); - GNUNET_DISK_file_close (fh); - } - GNUNET_FS_download_sync_ (dc); - pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; - pi.value.download.specifics.start.meta = dc->meta; - GNUNET_FS_download_make_status_ (&pi, dc); - check_completed (dc); - return; - } - if (NULL != dc->emsg) - return; - if (NULL == dc->top_request) - { - dc->top_request = create_download_request (NULL, - 0, - dc->treedepth - 1, - 0, - dc->offset, - dc->length); - dc->top_request->state = BRS_CHK_SET; - dc->top_request->chk = (dc->uri->type == GNUNET_FS_URI_CHK) - ? dc->uri->data.chk.chk - : dc->uri->data.loc.fi.chk; - /* signal start */ - GNUNET_FS_download_sync_ (dc); - if (NULL != dc->search) - GNUNET_FS_search_result_sync_ (dc->search); - pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; - pi.value.download.specifics.start.meta = dc->meta; - GNUNET_FS_download_make_status_ (&pi, dc); - } - GNUNET_FS_download_start_downloading_ (dc); - /* attempt reconstruction from disk */ - if (GNUNET_YES == GNUNET_DISK_file_test (dc->filename)) - dc->rfh = GNUNET_DISK_file_open (dc->filename, - GNUNET_DISK_OPEN_READ, - GNUNET_DISK_PERM_NONE); - if (dc->top_request->state == BRS_CHK_SET) - { - if (NULL != dc->rfh) - { - /* first, try top-down */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Trying top-down reconstruction for `%s'\n", - dc->filename); - try_top_down_reconstruction (dc, dc->top_request); - switch (dc->top_request->state) - { - case BRS_CHK_SET: - break; /* normal */ - - case BRS_DOWNLOAD_DOWN: - break; /* normal, some blocks already down */ - - case BRS_DOWNLOAD_UP: - /* already done entirely, party! */ - if (NULL != dc->rfh) - { - /* avoid hanging on to file handle longer than - * necessary */ - GNUNET_DISK_file_close (dc->rfh); - dc->rfh = NULL; - } - return; - - case BRS_ERROR: - GNUNET_asprintf (&dc->emsg, _ ("Invalid URI")); - GNUNET_FS_download_sync_ (dc); - pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; - pi.value.download.specifics.error.message = dc->emsg; - GNUNET_FS_download_make_status_ (&pi, dc); - return; - - default: - GNUNET_assert (0); - break; - } - } - } - /* attempt reconstruction from meta data */ - if ((GNUNET_FS_uri_chk_get_file_size (dc->uri) <= MAX_INLINE_SIZE) && - (NULL != dc->meta)) - { - GNUNET_log ( - GNUNET_ERROR_TYPE_DEBUG, - "Trying to find embedded meta data for download of size %llu with %u bytes MD\n", - (unsigned long long) GNUNET_FS_uri_chk_get_file_size (dc->uri), - (unsigned int) GNUNET_FS_meta_data_get_serialized_size (dc->meta)); - GNUNET_FS_meta_data_iterate (dc->meta, &match_full_data, dc); - if (BRS_DOWNLOAD_UP == dc->top_request->state) - { - if (NULL != dc->rfh) - { - /* avoid hanging on to file handle longer than - * necessary */ - GNUNET_DISK_file_close (dc->rfh); - dc->rfh = NULL; - } - return; /* finished, status update was already done for us */ - } - } - if (NULL != dc->rfh) - { - /* finally, actually run bottom-up */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Trying bottom-up reconstruction of file `%s'\n", - dc->filename); - dc->te = - GNUNET_FS_tree_encoder_create (dc->h, - GNUNET_FS_uri_chk_get_file_size (dc->uri), - dc, - &fh_reader, - &reconstruct_cb, - NULL, - &reconstruct_cont); - dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc); - } - else - { - /* simple, top-level download */ - dc->issue_requests = GNUNET_YES; - schedule_block_download (dc, dc->top_request); - } - if (BRS_DOWNLOAD_UP == dc->top_request->state) - check_completed (dc); -} - - -void -GNUNET_FS_download_signal_suspend_ (void *cls) -{ - struct GNUNET_FS_DownloadContext *dc = cls; - struct GNUNET_FS_ProgressInfo pi; - - if (NULL != dc->top) - GNUNET_FS_end_top (dc->h, dc->top); - while (NULL != dc->child_head) - GNUNET_FS_download_signal_suspend_ (dc->child_head); - if (NULL != dc->search) - { - dc->search->download = NULL; - dc->search = NULL; - } - if (NULL != dc->job_queue) - { - GNUNET_FS_dequeue_ (dc->job_queue); - dc->job_queue = NULL; - } - if (NULL != dc->parent) - GNUNET_CONTAINER_DLL_remove (dc->parent->child_head, - dc->parent->child_tail, - dc); - if (NULL != dc->task) - { - GNUNET_SCHEDULER_cancel (dc->task); - dc->task = NULL; - } - pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND; - GNUNET_FS_download_make_status_ (&pi, dc); - if (NULL != dc->te) - { - GNUNET_FS_tree_encoder_finish (dc->te, NULL); - dc->te = NULL; - } - if (NULL != dc->rfh) - { - GNUNET_DISK_file_close (dc->rfh); - dc->rfh = NULL; - } - GNUNET_FS_free_download_request_ (dc->top_request); - if (NULL != dc->active) - { - GNUNET_CONTAINER_multihashmap_destroy (dc->active); - dc->active = NULL; - } - GNUNET_free (dc->filename); - GNUNET_FS_meta_data_destroy (dc->meta); - GNUNET_FS_uri_destroy (dc->uri); - GNUNET_free (dc->temp_filename); - GNUNET_free (dc->serialization); - GNUNET_assert (NULL == dc->job_queue); - GNUNET_free (dc); -} - - -/** - * Helper function to setup the download context. - * - * @param h handle to the file sharing subsystem - * @param uri the URI of the file (determines what to download); CHK or LOC URI - * @param meta known metadata for the file (can be NULL) - * @param filename where to store the file, maybe NULL (then no file is - * created on disk and data must be grabbed from the callbacks) - * @param tempname where to store temporary file data, not used if filename is non-NULL; - * can be NULL (in which case we will pick a name if needed); the temporary file - * may already exist, in which case we will try to use the data that is there and - * if it is not what is desired, will overwrite it - * @param offset at what offset should we start the download (typically 0) - * @param length how many bytes should be downloaded starting at offset - * @param anonymity anonymity level to use for the download - * @param options various options - * @param cctx initial value for the client context for this download - * @return context that can be used to control this download - */ -struct GNUNET_FS_DownloadContext * -create_download_context (struct GNUNET_FS_Handle *h, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *meta, - const char *filename, - const char *tempname, - uint64_t offset, - uint64_t length, - uint32_t anonymity, - enum GNUNET_FS_DownloadOptions options, - void *cctx) -{ - struct GNUNET_FS_DownloadContext *dc; - - GNUNET_assert (GNUNET_FS_uri_test_chk (uri) || GNUNET_FS_uri_test_loc (uri)); - if ((offset + length < offset) || - (offset + length > GNUNET_FS_uri_chk_get_file_size (uri))) - { - GNUNET_break (0); - return NULL; - } - dc = GNUNET_new (struct GNUNET_FS_DownloadContext); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting download %p, %u bytes at offset %llu\n", - dc, - (unsigned int) length, - (unsigned long long) offset); - dc->h = h; - dc->uri = GNUNET_FS_uri_dup (uri); - dc->meta = GNUNET_FS_meta_data_duplicate (meta); - dc->client_info = cctx; - dc->start_time = GNUNET_TIME_absolute_get (); - if (NULL != filename) - { - dc->filename = GNUNET_strdup (filename); - if (GNUNET_YES == GNUNET_DISK_file_test (filename)) - GNUNET_break (GNUNET_OK == GNUNET_DISK_file_size (filename, - &dc->old_file_size, - GNUNET_YES, - GNUNET_YES)); - } - if (GNUNET_FS_uri_test_loc (dc->uri)) - GNUNET_assert (GNUNET_OK == - GNUNET_FS_uri_loc_get_peer_identity (dc->uri, &dc->target)); - dc->offset = offset; - dc->length = length; - dc->anonymity = anonymity; - dc->options = options; - dc->active = - GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE), - GNUNET_NO); - dc->treedepth = - GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size (dc->uri)); - if ((NULL == filename) && (is_recursive_download (dc))) - { - if (NULL != tempname) - dc->temp_filename = GNUNET_strdup (tempname); - else - dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp"); - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting download `%s' of %llu bytes with tree depth %u\n", - filename, - (unsigned long long) length, - dc->treedepth); - GNUNET_assert (NULL == dc->job_queue); - dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); - return dc; -} - - -struct GNUNET_FS_DownloadContext * -GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *meta, - const char *filename, - const char *tempname, - uint64_t offset, - uint64_t length, - uint32_t anonymity, - enum GNUNET_FS_DownloadOptions options, - void *cctx, - struct GNUNET_FS_DownloadContext *parent) -{ - struct GNUNET_FS_DownloadContext *dc; - - dc = create_download_context (h, - uri, - meta, - filename, - tempname, - offset, - length, - anonymity, - options, - cctx); - if (NULL == dc) - return NULL; - dc->parent = parent; - if (NULL != parent) - GNUNET_CONTAINER_DLL_insert (parent->child_head, parent->child_tail, dc); - else if (0 == (GNUNET_FS_DOWNLOAD_IS_PROBE & options)) - dc->top = - GNUNET_FS_make_top (dc->h, &GNUNET_FS_download_signal_suspend_, dc); - return dc; -} - - -struct GNUNET_FS_DownloadContext * -GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h, - struct GNUNET_FS_SearchResult *sr, - const char *filename, - const char *tempname, - uint64_t offset, - uint64_t length, - uint32_t anonymity, - enum GNUNET_FS_DownloadOptions options, - void *cctx) -{ - struct GNUNET_FS_DownloadContext *dc; - - if ((NULL == sr) || (NULL != sr->download)) - { - GNUNET_break (0); - return NULL; - } - dc = create_download_context (h, - sr->uri, - sr->meta, - filename, - tempname, - offset, - length, - anonymity, - options, - cctx); - if (NULL == dc) - return NULL; - dc->search = sr; - sr->download = dc; - if (NULL != sr->probe_ctx) - { - GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES); - sr->probe_ctx = NULL; - GNUNET_FS_stop_probe_ping_task_ (sr); - } - return dc; -} - - -/** - * Start the downloading process (by entering the queue). - * - * @param dc our download context - */ -void -GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc) -{ - if (dc->completed == dc->length) - return; - if (NULL != dc->mq) - return; /* already running */ - GNUNET_assert (NULL == dc->job_queue); - GNUNET_assert (NULL == dc->task); - GNUNET_assert (NULL != dc->active); - dc->job_queue = - GNUNET_FS_queue_ (dc->h, - &activate_fs_download, - &deactivate_fs_download, - dc, - (dc->length + DBLOCK_SIZE - 1) / DBLOCK_SIZE, - (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE)) - ? GNUNET_FS_QUEUE_PRIORITY_NORMAL - : GNUNET_FS_QUEUE_PRIORITY_PROBE); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Download %p put into queue as job %p\n", - dc, - dc->job_queue); -} - - -/** - * Suspend a download. - * - * @param dc handle for the download - */ -void -GNUNET_FS_download_suspend (struct GNUNET_FS_DownloadContext *dc) -{ - deactivate_fs_download (dc); -} - - -/** - * Resume a suspended download. - * - * @param dc handle for the download - */ -void -GNUNET_FS_download_resume (struct GNUNET_FS_DownloadContext *dc) -{ - struct GNUNET_FS_ProgressInfo pi; - - pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE; - GNUNET_FS_download_make_status_ (&pi, dc); - - GNUNET_assert (NULL == dc->task); - dc->job_queue = - GNUNET_FS_queue_ (dc->h, - &activate_fs_download, - &deactivate_fs_download, - dc, - (dc->length + DBLOCK_SIZE - 1) / DBLOCK_SIZE, - (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE)) - ? GNUNET_FS_QUEUE_PRIORITY_NORMAL - : GNUNET_FS_QUEUE_PRIORITY_PROBE); -} - - -/** - * Stop a download (aborts if download is incomplete). - * - * @param dc handle for the download - * @param do_delete delete files of incomplete downloads - */ -void -GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, int do_delete) -{ - struct GNUNET_FS_ProgressInfo pi; - int have_children; - int search_was_null; - - if (NULL != dc->top) - GNUNET_FS_end_top (dc->h, dc->top); - if (NULL != dc->task) - { - GNUNET_SCHEDULER_cancel (dc->task); - dc->task = NULL; - } - search_was_null = (NULL == dc->search); - if (NULL != dc->search) - { - dc->search->download = NULL; - GNUNET_FS_search_result_sync_ (dc->search); - dc->search = NULL; - } - if (NULL != dc->job_queue) - { - GNUNET_FS_dequeue_ (dc->job_queue); - dc->job_queue = NULL; - } - if (NULL != dc->te) - { - GNUNET_FS_tree_encoder_finish (dc->te, NULL); - dc->te = NULL; - } - have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO; - while (NULL != dc->child_head) - GNUNET_FS_download_stop (dc->child_head, do_delete); - if (NULL != dc->parent) - GNUNET_CONTAINER_DLL_remove (dc->parent->child_head, - dc->parent->child_tail, - dc); - if (NULL != dc->serialization) - GNUNET_FS_remove_sync_file_ (dc->h, - ((NULL != dc->parent) || (! search_was_null)) - ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD - : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD, - dc->serialization); - if ((GNUNET_YES == have_children) && (NULL == dc->parent)) - GNUNET_FS_remove_sync_dir_ (dc->h, - (! search_was_null) - ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD - : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD, - dc->serialization); - pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED; - GNUNET_FS_download_make_status_ (&pi, dc); - GNUNET_FS_free_download_request_ (dc->top_request); - dc->top_request = NULL; - if (NULL != dc->active) - { - GNUNET_CONTAINER_multihashmap_destroy (dc->active); - dc->active = NULL; - } - if (NULL != dc->filename) - { - if ((dc->completed != dc->length) && (GNUNET_YES == do_delete)) - { - if ((0 != unlink (dc->filename)) && (ENOENT != errno)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, - "unlink", - dc->filename); - } - GNUNET_free (dc->filename); - } - GNUNET_FS_meta_data_destroy (dc->meta); - GNUNET_FS_uri_destroy (dc->uri); - if (NULL != dc->temp_filename) - { - if (0 != unlink (dc->temp_filename)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, - "unlink", - dc->temp_filename); - GNUNET_free (dc->temp_filename); - } - GNUNET_free (dc->serialization); - GNUNET_assert (NULL == dc->job_queue); - GNUNET_free (dc); -} - - -/* end of fs_download.c */ diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c deleted file mode 100644 index f23b9da2a..000000000 --- a/src/fs/fs_file_information.c +++ /dev/null @@ -1,407 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009, 2011 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_file_information.c - * @brief Manage information for publishing directory hierarchies - * @author Christian Grothoff - */ -#include "platform.h" -#if HAVE_EXTRACTOR_H -#include -#endif -#include "gnunet_fs_service.h" -#include "fs_api.h" -#include "fs_tree.h" - - -/** - * Obtain the name under which this file information - * structure is stored on disk. Only works for top-level - * file information structures. - * - * @param s structure to get the filename for - * @return NULL on error, otherwise filename that - * can be used to read this fi-struct from disk. - */ -const char * -GNUNET_FS_file_information_get_id (struct GNUNET_FS_FileInformation *s) -{ - if (NULL != s->dir) - return NULL; - return s->serialization; -} - - -/** - * Obtain the filename from the file information structure. - * - * @param s structure to get the filename for - * @return "filename" field of the structure (can be NULL) - */ -const char * -GNUNET_FS_file_information_get_filename (const struct - GNUNET_FS_FileInformation *s) -{ - return s->filename; -} - - -/** - * Set the filename in the file information structure. - * If filename was already set, frees it before setting the new one. - * Makes a copy of the argument. - * - * @param s structure to get the filename for - * @param filename filename to set - */ -void -GNUNET_FS_file_information_set_filename (struct GNUNET_FS_FileInformation *s, - const char *filename) -{ - GNUNET_free (s->filename); - if (filename) - s->filename = GNUNET_strdup (filename); - else - s->filename = NULL; -} - - -struct GNUNET_FS_FileInformation * -GNUNET_FS_file_information_create_from_file ( - struct GNUNET_FS_Handle *h, - void *client_info, - const char *filename, - const struct GNUNET_FS_Uri *keywords, - const struct GNUNET_FS_MetaData *meta, - int do_index, - const struct GNUNET_FS_BlockOptions *bo) -{ - struct FileInfo *fi; - uint64_t fsize; - struct GNUNET_FS_FileInformation *ret; - const char *fn; - const char *ss; - - /* FIXME: should include_symbolic_links be GNUNET_NO or GNUNET_YES here? */ - if (GNUNET_OK != - GNUNET_DISK_file_size (filename, &fsize, GNUNET_NO, GNUNET_YES)) - { - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "stat", filename); - return NULL; - } - fi = GNUNET_FS_make_file_reader_context_ (filename); - if (NULL == fi) - { - GNUNET_break (0); - return NULL; - } - ret = - GNUNET_FS_file_information_create_from_reader (h, - client_info, - fsize, - &GNUNET_FS_data_reader_file_, - fi, - keywords, - meta, - do_index, - bo); - if (ret == NULL) - return NULL; - ret->h = h; - ret->filename = GNUNET_strdup (filename); - fn = filename; - while (NULL != (ss = strstr (fn, DIR_SEPARATOR_STR))) - fn = ss + 1; -/* FIXME: If we assume that on other platforms CRT is UTF-8-aware, then - * this should be changed to EXTRACTOR_METAFORMAT_UTF8 - */ - GNUNET_FS_meta_data_insert (ret->meta, - "", - EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, - EXTRACTOR_METAFORMAT_C_STRING, - "text/plain", - fn, - strlen (fn) + 1); - return ret; -} - - -struct GNUNET_FS_FileInformation * -GNUNET_FS_file_information_create_from_data ( - struct GNUNET_FS_Handle *h, - void *client_info, - uint64_t length, - void *data, - const struct GNUNET_FS_Uri *keywords, - const struct GNUNET_FS_MetaData *meta, - int do_index, - const struct GNUNET_FS_BlockOptions *bo) -{ - if (GNUNET_YES == do_index) - { - GNUNET_break (0); - return NULL; - } - return GNUNET_FS_file_information_create_from_reader (h, - client_info, - length, - & - GNUNET_FS_data_reader_copy_, - data, - keywords, - meta, - do_index, - bo); -} - - -struct GNUNET_FS_FileInformation * -GNUNET_FS_file_information_create_from_reader ( - struct GNUNET_FS_Handle *h, - void *client_info, - uint64_t length, - GNUNET_FS_DataReader reader, - void *reader_cls, - const struct GNUNET_FS_Uri *keywords, - const struct GNUNET_FS_MetaData *meta, - int do_index, - const struct GNUNET_FS_BlockOptions *bo) -{ - struct GNUNET_FS_FileInformation *ret; - - if ((GNUNET_YES == do_index) && (reader != &GNUNET_FS_data_reader_file_)) - { - GNUNET_break (0); - return NULL; - } - ret = GNUNET_new (struct GNUNET_FS_FileInformation); - ret->h = h; - ret->client_info = client_info; - ret->meta = GNUNET_FS_meta_data_duplicate (meta); - if (ret->meta == NULL) - ret->meta = GNUNET_FS_meta_data_create (); - ret->keywords = (keywords == NULL) ? NULL : GNUNET_FS_uri_dup (keywords); - ret->data.file.reader = reader; - ret->data.file.reader_cls = reader_cls; - ret->data.file.do_index = do_index; - ret->data.file.file_size = length; - ret->bo = *bo; - return ret; -} - - -/** - * Test if a given entry represents a directory. - * - * @param ent check if this FI represents a directory - * @return #GNUNET_YES if so, #GNUNET_NO if not - */ -int -GNUNET_FS_file_information_is_directory ( - const struct GNUNET_FS_FileInformation *ent) -{ - return ent->is_directory; -} - - -struct GNUNET_FS_FileInformation * -GNUNET_FS_file_information_create_empty_directory ( - struct GNUNET_FS_Handle *h, - void *client_info, - const struct GNUNET_FS_Uri *keywords, - const struct GNUNET_FS_MetaData *meta, - const struct GNUNET_FS_BlockOptions *bo, - const char *filename) -{ - struct GNUNET_FS_FileInformation *ret; - - ret = GNUNET_new (struct GNUNET_FS_FileInformation); - ret->h = h; - ret->client_info = client_info; - ret->meta = GNUNET_FS_meta_data_duplicate (meta); - ret->keywords = GNUNET_FS_uri_dup (keywords); - ret->bo = *bo; - ret->is_directory = GNUNET_YES; - if (filename != NULL) - ret->filename = GNUNET_strdup (filename); - return ret; -} - - -int -GNUNET_FS_file_information_add (struct GNUNET_FS_FileInformation *dir, - struct GNUNET_FS_FileInformation *ent) -{ - if ((ent->dir != NULL) || (ent->next != NULL) || - (dir->is_directory != GNUNET_YES)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - ent->dir = dir; - ent->next = dir->data.dir.entries; - dir->data.dir.entries = ent; - dir->data.dir.dir_size = 0; - return GNUNET_OK; -} - - -/** - * Inspect a file or directory in a publish-structure. Clients - * should never modify publish structures that were passed to - * #GNUNET_FS_publish_start already. When called on a directory, - * this function will FIRST call @a proc with information about - * the directory itself and then for each of the files in the - * directory (but not for files in subdirectories). When called - * on a file, @a proc will be called exactly once (with information - * about the specific file). - * - * @param dir the directory - * @param proc function to call on each entry - * @param proc_cls closure for @a proc - */ -void -GNUNET_FS_file_information_inspect (struct GNUNET_FS_FileInformation *dir, - GNUNET_FS_FileInformationProcessor proc, - void *proc_cls) -{ - struct GNUNET_FS_FileInformation *pos; - int no; - - no = GNUNET_NO; - if (GNUNET_OK != - proc (proc_cls, - dir, - (dir->is_directory == GNUNET_YES) ? dir->data.dir.dir_size - : dir->data.file.file_size, - dir->meta, - &dir->keywords, - &dir->bo, - (dir->is_directory == GNUNET_YES) ? &no : &dir->data.file.do_index, - &dir->client_info)) - return; - if (dir->is_directory != GNUNET_YES) - return; - pos = dir->data.dir.entries; - while (pos != NULL) - { - no = GNUNET_NO; - if (GNUNET_OK != - proc (proc_cls, - pos, - (pos->is_directory == GNUNET_YES) ? pos->data.dir.dir_size - : pos->data.file.file_size, - pos->meta, - &pos->keywords, - &pos->bo, - (pos->is_directory == GNUNET_YES) ? &no - : &pos->data.file.do_index, - &pos->client_info)) - break; - pos = pos->next; - } -} - - -/** - * Destroy publish-structure. Clients should never destroy publish - * structures that were passed to #GNUNET_FS_publish_start already. - * - * @param fi structure to destroy - * @param cleaner function to call on each entry in the structure - * (useful to clean up client_info); can be NULL; return - * values are ignored - * @param cleaner_cls closure for @a cleaner - */ -void -GNUNET_FS_file_information_destroy (struct GNUNET_FS_FileInformation *fi, - GNUNET_FS_FileInformationProcessor cleaner, - void *cleaner_cls) -{ - struct GNUNET_FS_FileInformation *pos; - int no; - - no = GNUNET_NO; - if (GNUNET_YES == fi->is_directory) - { - /* clean up directory */ - while (NULL != (pos = fi->data.dir.entries)) - { - fi->data.dir.entries = pos->next; - GNUNET_FS_file_information_destroy (pos, cleaner, cleaner_cls); - } - /* clean up client-info */ - if (NULL != cleaner) - cleaner (cleaner_cls, - fi, - fi->data.dir.dir_size, - fi->meta, - &fi->keywords, - &fi->bo, - &no, - &fi->client_info); - GNUNET_free (fi->data.dir.dir_data); - } - else - { - /* call clean-up function of the reader */ - if (NULL != fi->data.file.reader) - { - (void) fi->data.file.reader (fi->data.file.reader_cls, 0, 0, NULL, NULL); - fi->data.file.reader = NULL; - } - /* clean up client-info */ - if (NULL != cleaner) - cleaner (cleaner_cls, - fi, - fi->data.file.file_size, - fi->meta, - &fi->keywords, - &fi->bo, - &fi->data.file.do_index, - &fi->client_info); - } - GNUNET_free (fi->filename); - GNUNET_free (fi->emsg); - if (NULL != fi->sks_uri) - GNUNET_FS_uri_destroy (fi->sks_uri); - if (NULL != fi->chk_uri) - GNUNET_FS_uri_destroy (fi->chk_uri); - /* clean up serialization */ - if ((NULL != fi->serialization) && (0 != unlink (fi->serialization))) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, - "unlink", - fi->serialization); - if (NULL != fi->keywords) - GNUNET_FS_uri_destroy (fi->keywords); - if (NULL != fi->meta) - GNUNET_FS_meta_data_destroy (fi->meta); - GNUNET_free (fi->serialization); - if (NULL != fi->te) - { - GNUNET_FS_tree_encoder_finish (fi->te, NULL); - fi->te = NULL; - } - GNUNET_free (fi); -} - - -/* end of fs_file_information.c */ diff --git a/src/fs/fs_getopt.c b/src/fs/fs_getopt.c deleted file mode 100644 index 0135e2e05..000000000 --- a/src/fs/fs_getopt.c +++ /dev/null @@ -1,274 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_getopt.c - * @brief helper functions for command-line argument processing - * @author Igor Wronsky, Christian Grothoff - */ -#include "platform.h" - -#include "gnunet_fs_service.h" -#include "fs_api.h" - -/* ******************** command-line option parsing API ******************** */ - -/** - * Command-line option parser function that allows the user - * to specify one or more '-k' options with keywords. Each - * specified keyword will be added to the URI. A pointer to - * the URI must be passed as the "scls" argument. - * - * @param ctx command line processor context - * @param scls must be of type "struct GNUNET_FS_Uri **" - * @param option name of the option (typically 'k') - * @param value command line argument given - * @return #GNUNET_OK on success - */ -static int -getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx, - void *scls, - const char *option, - const char *value) -{ - struct GNUNET_FS_Uri **uri = scls; - struct GNUNET_FS_Uri *u = *uri; - char *val; - size_t slen; - - if (NULL == u) - { - u = GNUNET_new (struct GNUNET_FS_Uri); - *uri = u; - u->type = GNUNET_FS_URI_KSK; - u->data.ksk.keywordCount = 0; - u->data.ksk.keywords = NULL; - } - else - { - GNUNET_assert (GNUNET_FS_URI_KSK == u->type); - } - slen = strlen (value); - if (0 == slen) - return GNUNET_SYSERR; /* cannot be empty */ - if (value[0] == '+') - { - /* simply preserve the "mandatory" flag */ - if (slen < 2) - return GNUNET_SYSERR; /* empty keywords not allowed */ - if ((value[1] == '"') && (slen > 3) && (value[slen - 1] == '"')) - { - /* remove the quotes, keep the '+' */ - val = GNUNET_malloc (slen - 1); - val[0] = '+'; - GNUNET_memcpy (&val[1], - &value[2], - slen - 3); - val[slen - 2] = '\0'; - } - else - { - /* no quotes, just keep the '+' */ - val = GNUNET_strdup (value); - } - } - else - { - if ((value[0] == '"') && (slen > 2) && (value[slen - 1] == '"')) - { - /* remove the quotes, add a space */ - val = GNUNET_malloc (slen); - val[0] = ' '; - GNUNET_memcpy (&val[1], - &value[1], - slen - 2); - val[slen - 1] = '\0'; - } - else - { - /* add a space to indicate "not mandatory" */ - val = GNUNET_malloc (slen + 2); - strcpy (val, " "); - strcat (val, value); - } - } - GNUNET_array_append (u->data.ksk.keywords, - u->data.ksk.keywordCount, - val); - return GNUNET_OK; -} - - -/** - * Allow user to specify keywords. - * - * @param shortName short name of the option - * @param name long name of the option - * @param argumentHelp help text for the option argument - * @param description long help text for the option - * @param[out] topKeywords set to the desired value - */ -struct GNUNET_GETOPT_CommandLineOption -GNUNET_FS_GETOPT_KEYWORDS (char shortName, - const char *name, - const char *argumentHelp, - const char *description, - struct GNUNET_FS_Uri **topKeywords) -{ - struct GNUNET_GETOPT_CommandLineOption clo = { - .shortName = shortName, - .name = name, - .argumentHelp = argumentHelp, - .description = description, - .require_argument = 1, - .processor = &getopt_set_keywords, - .scls = (void *) topKeywords - }; - - return clo; -} - - -/** - * Command-line option parser function that allows the user to specify - * one or more '-m' options with metadata. Each specified entry of - * the form "type=value" will be added to the metadata. A pointer to - * the metadata must be passed as the "scls" argument. - * - * @param ctx command line processor context - * @param scls must be of type "struct GNUNET_MetaData **" - * @param option name of the option (typically 'k') - * @param value command line argument given - * @return #GNUNET_OK on success - */ -static int -getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx, - void *scls, - const char *option, - const char *value) -{ - struct GNUNET_FS_MetaData **mm = scls; - -#if HAVE_EXTRACTOR_H && HAVE_LIBEXTRACTOR - enum EXTRACTOR_MetaType type; - const char *typename; - const char *typename_i18n; -#endif - struct GNUNET_FS_MetaData *meta; - char *tmp; - - meta = *mm; - if (meta == NULL) - { - meta = GNUNET_FS_meta_data_create (); - *mm = meta; - } - - /* Use GNUNET_STRINGS_get_utf8_args() in main() to acquire utf-8-encoded - * commandline arguments, so that the following line is not needed. - */ - /*tmp = GNUNET_STRINGS_to_utf8 (value, strlen (value), locale_charset ());*/ - tmp = GNUNET_strdup (value); -#if HAVE_EXTRACTOR_H && HAVE_LIBEXTRACTOR - type = EXTRACTOR_metatype_get_max (); - while (type > 0) - { - type--; - typename = EXTRACTOR_metatype_to_string (type); - typename_i18n = dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, typename); - if ((strlen (tmp) >= strlen (typename) + 1) && - (tmp[strlen (typename)] == ':') && - (0 == strncmp (typename, tmp, strlen (typename)))) - { - GNUNET_FS_meta_data_insert (meta, "", type, - EXTRACTOR_METAFORMAT_UTF8, - "text/plain", - &tmp[strlen (typename) + 1], - strlen (&tmp[strlen (typename) + 1]) - + 1); - GNUNET_free (tmp); - tmp = NULL; - break; - } - if ((strlen (tmp) >= strlen (typename_i18n) + 1) && - (tmp[strlen (typename_i18n)] == ':') && - (0 == strncmp (typename_i18n, tmp, strlen (typename_i18n)))) - { - GNUNET_FS_meta_data_insert (meta, "", type, - EXTRACTOR_METAFORMAT_UTF8, - "text/plain", - &tmp[strlen (typename_i18n) + 1], - strlen (&tmp - [strlen (typename_i18n) + 1]) - + 1); - GNUNET_free (tmp); - tmp = NULL; - break; - } - } -#endif - - if (NULL != tmp) - { - GNUNET_FS_meta_data_insert (meta, "", - EXTRACTOR_METATYPE_UNKNOWN, - EXTRACTOR_METAFORMAT_UTF8, "text/plain", - tmp, strlen (tmp) + 1); - GNUNET_free (tmp); - printf (_ - ( - "Unknown metadata type in metadata option `%s'. Using metadata type `unknown' instead.\n"), - value); - } - return GNUNET_OK; -} - - -/** - * Allow user to specify metadata. - * - * @param shortName short name of the option - * @param name long name of the option - * @param argumentHelp help text for the option argument - * @param description long help text for the option - * @param[out] metadata set to the desired value - */ -struct GNUNET_GETOPT_CommandLineOption -GNUNET_FS_GETOPT_METADATA (char shortName, - const char *name, - const char *argumentHelp, - const char *description, - struct GNUNET_FS_MetaData **meta) -{ - struct GNUNET_GETOPT_CommandLineOption clo = { - .shortName = shortName, - .name = name, - .argumentHelp = argumentHelp, - .description = description, - .require_argument = 1, - .processor = &getopt_set_metadata, - .scls = (void *) meta - }; - - return clo; -} - - -/* end of fs_getopt.c */ diff --git a/src/fs/fs_list_indexed.c b/src/fs/fs_list_indexed.c deleted file mode 100644 index 78816cad1..000000000 --- a/src/fs/fs_list_indexed.c +++ /dev/null @@ -1,219 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2003, 2004, 2006, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_list_indexed.c - * @author Christian Grothoff - * @brief provide a list of all indexed files - */ - -#include "platform.h" -#include "gnunet_constants.h" - -#include "gnunet_fs_service.h" -#include "gnunet_protocols.h" -#include "fs_api.h" - - -/** - * Context for #GNUNET_FS_get_indexed_files(). - */ -struct GNUNET_FS_GetIndexedContext -{ - /** - * Connection to the FS service. - */ - struct GNUNET_MQ_Handle *mq; - - /** - * Function to call for each indexed file. - */ - GNUNET_FS_IndexedFileProcessor iterator; - - /** - * Closure for @e iterator. - */ - void *iterator_cls; - - /** - * Continuation to trigger at the end. - */ - GNUNET_SCHEDULER_TaskCallback cont; - - /** - * Closure for @e cont. - */ - void *cont_cls; -}; - - -/** - * Function called on each response from the FS - * service with information about indexed files. - * - * @param cls closure (of type `struct GNUNET_FS_GetIndexedContext *`) - * @param msg message with indexing information - */ -static void -handle_index_info_end (void *cls, - const struct GNUNET_MessageHeader *msg) -{ - struct GNUNET_FS_GetIndexedContext *gic = cls; - - (void) gic->iterator (gic->iterator_cls, - NULL, - NULL); - GNUNET_FS_get_indexed_files_cancel (gic); -} - - -/** - * Check validity of response from the FS - * service with information about indexed files. - * - * @param cls closure (of type `struct GNUNET_FS_GetIndexedContext *`) - * @param iim message with indexing information - */ -static int -check_index_info (void *cls, - const struct IndexInfoMessage *iim) -{ - uint16_t msize = ntohs (iim->header.size) - sizeof(*iim); - const char *filename; - - filename = (const char *) &iim[1]; - if (filename[msize - 1] != '\0') - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - return GNUNET_OK; -} - - -/** - * Function called on each response from the FS - * service with information about indexed files. - * - * @param cls closure (of type `struct GNUNET_FS_GetIndexedContext *`) - * @param iim message with indexing information - */ -static void -handle_index_info (void *cls, - const struct IndexInfoMessage *iim) -{ - struct GNUNET_FS_GetIndexedContext *gic = cls; - const char *filename; - - filename = (const char *) &iim[1]; - if (GNUNET_OK != - gic->iterator (gic->iterator_cls, - filename, - &iim->file_id)) - { - GNUNET_FS_get_indexed_files_cancel (gic); - return; - } -} - - -/** - * Generic error handler, called with the appropriate error code and - * the same closure specified at the creation of the message queue. - * Not every message queue implementation supports an error handler. - * - * @param cls closure with the `struct GNUNET_FS_GetIndexedContent *` - * @param error error code - */ -static void -mq_error_handler (void *cls, - enum GNUNET_MQ_Error error) -{ - struct GNUNET_FS_GetIndexedContext *gic = cls; - - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - "Failed to receive response from `%s' service (error code is %d).\n", - "fs", - error); - (void) gic->iterator (gic->iterator_cls, - NULL, - NULL); - GNUNET_FS_get_indexed_files_cancel (gic); -} - - -struct GNUNET_FS_GetIndexedContext * -GNUNET_FS_get_indexed_files (struct GNUNET_FS_Handle *h, - GNUNET_FS_IndexedFileProcessor iterator, - void *iterator_cls) -{ - struct GNUNET_FS_GetIndexedContext *gic - = GNUNET_new (struct GNUNET_FS_GetIndexedContext); - struct GNUNET_MQ_MessageHandler handlers[] = { - GNUNET_MQ_hd_fixed_size (index_info_end, - GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END, - struct GNUNET_MessageHeader, - gic), - GNUNET_MQ_hd_var_size (index_info, - GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY, - struct IndexInfoMessage, - gic), - GNUNET_MQ_handler_end () - }; - struct GNUNET_MQ_Envelope *env; - struct GNUNET_MessageHeader *msg; - - gic->mq = GNUNET_CLIENT_connect (h->cfg, - "fs", - handlers, - &mq_error_handler, - h); - if (NULL == gic->mq) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to not connect to `%s' service.\n"), - "fs"); - GNUNET_free (gic); - return NULL; - } - gic->iterator = iterator; - gic->iterator_cls = iterator_cls; - env = GNUNET_MQ_msg (msg, - GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_GET); - GNUNET_MQ_send (gic->mq, - env); - return gic; -} - - -/** - * Cancel iteration over all indexed files. - * - * @param gic operation to cancel - */ -void -GNUNET_FS_get_indexed_files_cancel (struct GNUNET_FS_GetIndexedContext *gic) -{ - GNUNET_MQ_destroy (gic->mq); - GNUNET_free (gic); -} - - -/* end of fs_list_indexed.c */ diff --git a/src/fs/fs_misc.c b/src/fs/fs_misc.c deleted file mode 100644 index a8e23f042..000000000 --- a/src/fs/fs_misc.c +++ /dev/null @@ -1,165 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2010, 2011, 2017 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/fs_misc.c - * @brief misc. functions related to file-sharing in general - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_constants.h" - -#include "gnunet_fs_service.h" -#include "fs_api.h" - - -/** - * Suggest a filename based on given metadata. - * - * @param md given meta data - * @return NULL if meta data is useless for suggesting a filename - */ -char * -GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_FS_MetaData - *md) -{ - static const char *mimeMap[][2] = { - { "application/bz2", ".bz2" }, - { "application/gnunet-directory", ".gnd" }, - { "application/java", ".class" }, - { "application/msword", ".doc" }, - { "application/nar", ".nar" }, - { "application/narinfo", ".narinfo" }, - { "application/ogg", ".ogg" }, - { "application/pdf", ".pdf" }, - { "application/pgp-keys", ".key" }, - { "application/pgp-signature", ".pgp" }, - { "application/postscript", ".ps" }, - { "application/rar", ".rar" }, - { "application/rtf", ".rtf" }, - { "application/xml", ".xml" }, - { "application/x-debian-package", ".deb" }, - { "application/x-dvi", ".dvi" }, - { "application/x-flac", ".flac" }, - { "application/x-gzip", ".gz" }, - { "application/x-java-archive", ".jar" }, - { "application/x-java-vm", ".class" }, - { "application/x-python-code", ".pyc" }, - { "application/x-redhat-package-manager", ".rpm" }, - { "application/x-rpm", ".rpm" }, - { "application/x-tar", ".tar" }, - { "application/x-tex-pk", ".pk" }, - { "application/x-texinfo", ".texinfo" }, - { "application/x-xcf", ".xcf" }, - { "application/x-xfig", ".xfig" }, - { "application/zip", ".zip" }, - - { "audio/midi", ".midi" }, - { "audio/mpeg", ".mp3" }, - { "audio/real", ".rm" }, - { "audio/x-wav", ".wav" }, - - { "image/gif", ".gif" }, - { "image/jpeg", ".jpg" }, - { "image/pcx", ".pcx" }, - { "image/png", ".png" }, - { "image/tiff", ".tiff" }, - { "image/x-ms-bmp", ".bmp" }, - { "image/x-xpixmap", ".xpm" }, - - { "text/css", ".css" }, - { "text/html", ".html" }, - { "text/plain", ".txt" }, - { "text/rtf", ".rtf" }, - { "text/x-c++hdr", ".h++" }, - { "text/x-c++src", ".c++" }, - { "text/x-chdr", ".h" }, - { "text/x-csrc", ".c" }, - { "text/x-java", ".java" }, - { "text/x-moc", ".moc" }, - { "text/x-pascal", ".pas" }, - { "text/x-perl", ".pl" }, - { "text/x-python", ".py" }, - { "text/x-tex", ".tex" }, - - { "video/avi", ".avi" }, - { "video/mpeg", ".mpeg" }, - { "video/quicktime", ".qt" }, - { "video/real", ".rm" }, - { "video/x-msvideo", ".avi" }, - { NULL, NULL }, - }; - char *ret; - unsigned int i; - char *mime; - char *base; - const char *ext; - - ret = - GNUNET_FS_meta_data_get_by_type (md, - EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); - if (ret != NULL) - return ret; - ext = NULL; - mime = - GNUNET_FS_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE); - if (mime != NULL) - { - i = 0; - while ((mimeMap[i][0] != NULL) && (0 != strcmp (mime, mimeMap[i][0]))) - i++; - if (mimeMap[i][1] == NULL) - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, - _ ("Did not find mime type `%s' in extension list.\n"), mime); - else - ext = mimeMap[i][1]; - GNUNET_free (mime); - } - base = - GNUNET_FS_meta_data_get_first_by_types (md, - EXTRACTOR_METATYPE_TITLE, - EXTRACTOR_METATYPE_BOOK_TITLE, - EXTRACTOR_METATYPE_ORIGINAL_TITLE, - EXTRACTOR_METATYPE_PACKAGE_NAME, - EXTRACTOR_METATYPE_URL, - EXTRACTOR_METATYPE_URI, - EXTRACTOR_METATYPE_DESCRIPTION, - EXTRACTOR_METATYPE_ISRC, - EXTRACTOR_METATYPE_JOURNAL_NAME, - EXTRACTOR_METATYPE_AUTHOR_NAME, - EXTRACTOR_METATYPE_SUBJECT, - EXTRACTOR_METATYPE_ALBUM, - EXTRACTOR_METATYPE_ARTIST, - EXTRACTOR_METATYPE_KEYWORDS, - EXTRACTOR_METATYPE_COMMENT, - EXTRACTOR_METATYPE_UNKNOWN, - -1); - if ((base == NULL) && (ext == NULL)) - return NULL; - if (base == NULL) - return GNUNET_strdup (ext); - if (ext == NULL) - return base; - GNUNET_asprintf (&ret, "%s%s", base, ext); - GNUNET_free (base); - return ret; -} - - -/* end of fs_misc.c */ diff --git a/src/fs/fs_namespace.c b/src/fs/fs_namespace.c deleted file mode 100644 index f8b7b91c0..000000000 --- a/src/fs/fs_namespace.c +++ /dev/null @@ -1,804 +0,0 @@ -/* - This file is part of GNUnet - Copyright (C) 2003-2013 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_namespace.c - * @brief publishing to namespaces, and tracking updateable entries - * for our namespaces - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_constants.h" -#include "gnunet_signatures.h" -#include "gnunet_util_lib.h" - -#include "gnunet_fs_service.h" -#include "fs_api.h" -#include "fs_publish_ublock.h" - - -/** - * Information about an (updateable) node in the - * namespace. - */ -struct NamespaceUpdateNode -{ - /** - * Identifier for this node. - */ - char *id; - - /** - * Identifier of children of this node. - */ - char *update; - - /** - * Metadata for this entry. - */ - struct GNUNET_FS_MetaData *md; - - /** - * URI of this entry in the namespace. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Namespace update generation ID. Used to ensure - * freshness of the tree_id. - */ - unsigned int nug; - - /** - * TREE this entry belongs to (if nug is current). - */ - unsigned int tree_id; -}; - - -/** - * Handle to update information for a namespace. - */ -struct GNUNET_FS_UpdateInformationGraph -{ - /** - * Handle to the FS service context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Array with information about nodes in the namespace. - */ - struct NamespaceUpdateNode **update_nodes; - - /** - * Private key for the namespace. - */ - struct GNUNET_CRYPTO_EcdsaPrivateKey ns; - - /** - * Hash map mapping identifiers of update nodes - * to the update nodes (initialized on-demand). - */ - struct GNUNET_CONTAINER_MultiHashMap *update_map; - - /** - * Size of the update nodes array. - */ - unsigned int update_node_count; - - /** - * Reference counter. - */ - unsigned int rc; - - /** - * Generator for unique nug numbers. - */ - unsigned int nug_gen; -}; - - -/** - * Return the name of the directory in which we store - * the update information graph for the given local namespace. - * - * @param h file-sharing handle - * @param ns namespace handle - * @return NULL on error, otherwise the name of the directory - */ -static char * -get_update_information_directory ( - struct GNUNET_FS_Handle *h, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns) -{ - char *dn; - char *ret; - struct GNUNET_CRYPTO_EcdsaPublicKey pub; - struct GNUNET_HashCode hc; - struct GNUNET_CRYPTO_HashAsciiEncoded enc; - - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_filename (h->cfg, "FS", "UPDATE_DIR", &dn)) - { - GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "fs", "UPDATE_DIR"); - return NULL; - } - GNUNET_CRYPTO_ecdsa_key_get_public (ns, &pub); - GNUNET_CRYPTO_hash (&pub, sizeof(pub), &hc); - GNUNET_CRYPTO_hash_to_enc (&hc, &enc); - GNUNET_asprintf (&ret, - "%s%s%s", - dn, - DIR_SEPARATOR_STR, - (const char *) enc.encoding); - GNUNET_free (dn); - return ret; -} - - -/** - * Release memory occupied by UIG datastructure. - * - * @param uig data structure to free - */ -static void -free_update_information_graph (struct GNUNET_FS_UpdateInformationGraph *uig) -{ - unsigned int i; - struct NamespaceUpdateNode *nsn; - - for (i = 0; i < uig->update_node_count; i++) - { - nsn = uig->update_nodes[i]; - GNUNET_FS_meta_data_destroy (nsn->md); - GNUNET_FS_uri_destroy (nsn->uri); - GNUNET_free (nsn->id); - GNUNET_free (nsn->update); - GNUNET_free (nsn); - } - GNUNET_array_grow (uig->update_nodes, uig->update_node_count, 0); - if (NULL != uig->update_map) - GNUNET_CONTAINER_multihashmap_destroy (uig->update_map); - GNUNET_free (uig); -} - - -/** - * Write a namespace's update node graph to a file. - * - * @param uig update information graph to dump - */ -static void -write_update_information_graph (struct GNUNET_FS_UpdateInformationGraph *uig) -{ - char *fn; - struct GNUNET_BIO_WriteHandle *wh; - unsigned int i; - struct NamespaceUpdateNode *n; - char *uris; - - fn = get_update_information_directory (uig->h, &uig->ns); - wh = GNUNET_BIO_write_open_file (fn); - if (NULL == wh) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Failed to open `%s' for writing: %s\n"), - fn, - strerror (errno)); - GNUNET_free (fn); - return; - } - if (GNUNET_OK != GNUNET_BIO_write_int32 (wh, - "fs-namespace-node-count", - uig->update_node_count)) - goto END; - for (i = 0; i < uig->update_node_count; i++) - { - n = uig->update_nodes[i]; - uris = GNUNET_FS_uri_to_string (n->uri); - struct GNUNET_BIO_WriteSpec ws[] = { - GNUNET_BIO_write_spec_string ("fs-namespace-node-id", n->id), - GNUNET_FS_write_spec_meta_data ("fs-namespace-node-meta", n->md), - GNUNET_BIO_write_spec_string ("fs-namespace-node-update", n->update), - GNUNET_BIO_write_spec_string ("fs-namespace-uris", uris), - GNUNET_BIO_write_spec_end (), - }; - if (GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws)) - { - GNUNET_free (uris); - break; - } - GNUNET_free (uris); - } - END: - if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL)) - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Failed to write `%s': %s\n"), - fn, - strerror (errno)); - GNUNET_free (fn); -} - - -/** - * Read the namespace update node graph from a file. - * - * @param h FS handle to use - * @param ns namespace to read - * @return update graph, never NULL - */ -static struct GNUNET_FS_UpdateInformationGraph * -read_update_information_graph (struct GNUNET_FS_Handle *h, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns) -{ - struct GNUNET_FS_UpdateInformationGraph *uig; - char *fn; - struct GNUNET_BIO_ReadHandle *rh; - unsigned int i; - struct NamespaceUpdateNode *n; - char *uris; - uint32_t count; - char *emsg; - - uig = GNUNET_new (struct GNUNET_FS_UpdateInformationGraph); - uig->h = h; - uig->ns = *ns; - fn = get_update_information_directory (h, ns); - if (GNUNET_YES != GNUNET_DISK_file_test (fn)) - { - GNUNET_free (fn); - return uig; - } - rh = GNUNET_BIO_read_open_file (fn); - if (NULL == rh) - { - GNUNET_free (fn); - return uig; - } - if (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "fs-namespace-count", - (int32_t *) &count)) - { - GNUNET_break (0); - goto END; - } - if (count > 1024 * 1024) - { - GNUNET_break (0); - goto END; - } - if (0 == count) - goto END; - uig->update_nodes = - GNUNET_malloc (count * sizeof(struct NamespaceUpdateNode *)); - - for (i = 0; i < count; i++) - { - n = GNUNET_new (struct NamespaceUpdateNode); - struct GNUNET_BIO_ReadSpec rs[] = { - GNUNET_BIO_read_spec_string ("identifier", &n->id, 1024), - GNUNET_FS_read_spec_meta_data ("meta", &n->md), - GNUNET_BIO_read_spec_string ("update-id", &n->update, 1024), - GNUNET_BIO_read_spec_string ("uri", &uris, 1024 * 2), - GNUNET_BIO_read_spec_end (), - }; - if (GNUNET_OK != GNUNET_BIO_read_spec_commit (rh, rs)) - { - GNUNET_break (0); - GNUNET_free (n->id); - GNUNET_free (n->update); - if (n->md != NULL) - GNUNET_FS_meta_data_destroy (n->md); - GNUNET_free (n); - break; - } - n->uri = GNUNET_FS_uri_parse (uris, &emsg); - GNUNET_free (uris); - if (n->uri == NULL) - { - GNUNET_break (0); - GNUNET_free (emsg); - GNUNET_free (n->id); - GNUNET_free (n->update); - GNUNET_FS_meta_data_destroy (n->md); - GNUNET_free (n); - break; - } - uig->update_nodes[i] = n; - } - uig->update_node_count = i; - END: - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Failed to read `%s': %s\n"), - fn, - emsg); - GNUNET_free (emsg); - } - GNUNET_free (fn); - return uig; -} - - -/** - * Context for the SKS publication. - */ -struct GNUNET_FS_PublishSksContext -{ - /** - * URI of the new entry in the namespace. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Namespace update node to add to namespace on success (or to be - * deleted if publishing failed). - */ - struct NamespaceUpdateNode *nsn; - - /** - * Namespace we're publishing to. - */ - struct GNUNET_CRYPTO_EcdsaPrivateKey ns; - - /** - * Handle to the datastore. - */ - struct GNUNET_DATASTORE_Handle *dsh; - - /** - * Handle to FS. - */ - struct GNUNET_FS_Handle *h; - - /** - * Function to call once we're done. - */ - GNUNET_FS_PublishContinuation cont; - - /** - * Closure for cont. - */ - void *cont_cls; - - /** - * Handle for our UBlock operation request. - */ - struct GNUNET_FS_PublishUblockContext *uc; -}; - - -/** - * Function called by the UBlock construction with - * the result from the PUT (UBlock) request. - * - * @param cls closure of type "struct GNUNET_FS_PublishSksContext*" - * @param msg error message (or NULL) - */ -static void -sks_publish_cont (void *cls, const char *msg) -{ - struct GNUNET_FS_PublishSksContext *psc = cls; - struct GNUNET_FS_UpdateInformationGraph *uig; - - psc->uc = NULL; - if (NULL != msg) - { - if (NULL != psc->cont) - psc->cont (psc->cont_cls, NULL, msg); - GNUNET_FS_publish_sks_cancel (psc); - return; - } - if (NULL != psc->nsn) - { - /* FIXME: this can be done much more - * efficiently by simply appending to the - * file and overwriting the 4-byte header */ - uig = read_update_information_graph (psc->h, &psc->ns); - GNUNET_array_append (uig->update_nodes, uig->update_node_count, psc->nsn); - psc->nsn = NULL; - write_update_information_graph (uig); - free_update_information_graph (uig); - } - if (NULL != psc->cont) - psc->cont (psc->cont_cls, psc->uri, NULL); - GNUNET_FS_publish_sks_cancel (psc); -} - - -struct GNUNET_FS_PublishSksContext * -GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns, - const char *identifier, - const char *update, - const struct GNUNET_FS_MetaData *meta, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_BlockOptions *bo, - enum GNUNET_FS_PublishOptions options, - GNUNET_FS_PublishContinuation cont, - void *cont_cls) -{ - struct GNUNET_FS_PublishSksContext *psc; - struct GNUNET_FS_Uri *sks_uri; - - sks_uri = GNUNET_new (struct GNUNET_FS_Uri); - sks_uri->type = GNUNET_FS_URI_SKS; - sks_uri->data.sks.identifier = GNUNET_strdup (identifier); - GNUNET_CRYPTO_ecdsa_key_get_public (ns, &sks_uri->data.sks.ns); - - psc = GNUNET_new (struct GNUNET_FS_PublishSksContext); - psc->h = h; - psc->uri = sks_uri; - psc->cont = cont; - psc->cont_cls = cont_cls; - psc->ns = *ns; - if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) - { - psc->dsh = GNUNET_DATASTORE_connect (h->cfg); - if (NULL == psc->dsh) - { - sks_publish_cont (psc, _ ("Failed to connect to datastore.")); - return NULL; - } - } - if (NULL != update) - { - psc->nsn = GNUNET_new (struct NamespaceUpdateNode); - psc->nsn->id = GNUNET_strdup (identifier); - psc->nsn->update = GNUNET_strdup (update); - psc->nsn->md = GNUNET_FS_meta_data_duplicate (meta); - psc->nsn->uri = GNUNET_FS_uri_dup (uri); - } - psc->uc = GNUNET_FS_publish_ublock_ (h, - psc->dsh, - identifier, - update, - ns, - meta, - uri, - bo, - options, - &sks_publish_cont, - psc); - return psc; -} - - -/** - * Abort the SKS publishing operation. - * - * @param psc context of the operation to abort. - */ -void -GNUNET_FS_publish_sks_cancel (struct GNUNET_FS_PublishSksContext *psc) -{ - if (NULL != psc->uc) - { - GNUNET_FS_publish_ublock_cancel_ (psc->uc); - psc->uc = NULL; - } - if (NULL != psc->dsh) - { - GNUNET_DATASTORE_disconnect (psc->dsh, GNUNET_NO); - psc->dsh = NULL; - } - GNUNET_FS_uri_destroy (psc->uri); - if (NULL != psc->nsn) - { - GNUNET_FS_meta_data_destroy (psc->nsn->md); - GNUNET_FS_uri_destroy (psc->nsn->uri); - GNUNET_free (psc->nsn->id); - GNUNET_free (psc->nsn->update); - GNUNET_free (psc->nsn); - } - GNUNET_free (psc); -} - - -/** - * Closure for 'process_update_node'. - */ -struct ProcessUpdateClosure -{ - /** - * Function to call for each node. - */ - GNUNET_FS_IdentifierProcessor ip; - - /** - * Closure for 'ip'. - */ - void *ip_cls; -}; - - -/** - * Call the iterator in the closure for each node. - * - * @param cls closure (of type 'struct ProcessUpdateClosure *') - * @param key current key code - * @param value value in the hash map (of type 'struct NamespaceUpdateNode *') - * @return GNUNET_YES if we should continue to - * iterate, - * GNUNET_NO if not. - */ -static int -process_update_node (void *cls, const struct GNUNET_HashCode *key, void *value) -{ - struct ProcessUpdateClosure *pc = cls; - struct NamespaceUpdateNode *nsn = value; - - pc->ip (pc->ip_cls, nsn->id, nsn->uri, nsn->md, nsn->update); - return GNUNET_YES; -} - - -/** - * Closure for 'find_trees'. - */ -struct FindTreeClosure -{ - /** - * UIG we are operating on. - */ - struct GNUNET_FS_UpdateInformationGraph *uig; - - /** - * Array with 'head's of TREEs. - */ - struct NamespaceUpdateNode **tree_array; - - /** - * Size of 'tree_array' - */ - unsigned int tree_array_size; - - /** - * Current generational ID used. - */ - unsigned int nug; - - /** - * Identifier for the current TREE, or UINT_MAX for none yet. - */ - unsigned int id; -}; - - -/** - * Find all nodes reachable from the current node (including the - * current node itself). If they are in no tree, add them to the - * current one. If they are the head of another tree, merge the - * trees. If they are in the middle of another tree, let them be. - * We can tell that a node is already in an tree by checking if - * its 'nug' field is set to the current 'nug' value. It is the - * head of an tree if it is in the 'tree_array' under its respective - * 'tree_id'. - * - * In short, we're trying to find the smallest number of tree to - * cover a directed graph. - * - * @param cls closure (of type 'struct FindTreeClosure') - * @param key current key code - * @param value value in the hash map - * @return GNUNET_YES if we should continue to - * iterate, - * GNUNET_NO if not. - */ -static int -find_trees (void *cls, const struct GNUNET_HashCode *key, void *value) -{ - struct FindTreeClosure *fc = cls; - struct NamespaceUpdateNode *nsn = value; - struct GNUNET_HashCode hc; - - if (nsn->nug == fc->nug) - { - if (UINT_MAX == nsn->tree_id) - return GNUNET_YES; /* circular */ - GNUNET_assert (nsn->tree_id < fc->tree_array_size); - if (fc->tree_array[nsn->tree_id] != nsn) - return GNUNET_YES; /* part of "another" (directed) TREE, - * and not root of it, end trace */ - if (nsn->tree_id == fc->id) - return GNUNET_YES; /* that's our own root (can this be?) */ - /* merge existing TREE, we have a root for both */ - fc->tree_array[nsn->tree_id] = NULL; - if (UINT_MAX == fc->id) - fc->id = nsn->tree_id; /* take over ID */ - } - else - { - nsn->nug = fc->nug; - nsn->tree_id = UINT_MAX; /* mark as undef */ - /* trace */ - GNUNET_CRYPTO_hash (nsn->update, strlen (nsn->update), &hc); - GNUNET_CONTAINER_multihashmap_get_multiple (fc->uig->update_map, - &hc, - &find_trees, - fc); - } - return GNUNET_YES; -} - - -/** - * List all of the identifiers in the namespace for which we could - * produce an update. Namespace updates form a graph where each node - * has a name. Each node can have any number of URI/meta-data entries - * which can each be linked to other nodes. Cycles are possible. - * - * Calling this function with "next_id" NULL will cause the library to - * call "ip" with a root for each strongly connected component of the - * graph (a root being a node from which all other nodes in the Tree - * are reachable). - * - * Calling this function with "next_id" being the name of a node will - * cause the library to call "ip" with all children of the node. Note - * that cycles within the final tree are possible (including self-loops). - * I know, odd definition of a tree, but the GUI will display an actual - * tree (GtkTreeView), so that's what counts for the term here. - * - * @param h fs handle to use - * @param ns namespace to inspect for updateable content - * @param next_id ID to look for; use NULL to look for tree roots - * @param ip function to call on each updateable identifier - * @param ip_cls closure for ip - */ -void -GNUNET_FS_namespace_list_updateable ( - struct GNUNET_FS_Handle *h, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns, - const char *next_id, - GNUNET_FS_IdentifierProcessor ip, - void *ip_cls) -{ - unsigned int i; - unsigned int nug; - struct GNUNET_HashCode hc; - struct NamespaceUpdateNode *nsn; - struct ProcessUpdateClosure pc; - struct FindTreeClosure fc; - struct GNUNET_FS_UpdateInformationGraph *uig; - - uig = read_update_information_graph (h, ns); - if (NULL == uig->update_nodes) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "No updateable nodes found for ID `%s'\n", - next_id); - free_update_information_graph (uig); - return; /* no nodes */ - } - uig->update_map = - GNUNET_CONTAINER_multihashmap_create (2 + 3 * uig->update_node_count / 4, - GNUNET_NO); - for (i = 0; i < uig->update_node_count; i++) - { - nsn = uig->update_nodes[i]; - GNUNET_CRYPTO_hash (nsn->id, strlen (nsn->id), &hc); - GNUNET_CONTAINER_multihashmap_put ( - uig->update_map, - &hc, - nsn, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); - } - if (NULL != next_id) - { - GNUNET_CRYPTO_hash (next_id, strlen (next_id), &hc); - pc.ip = ip; - pc.ip_cls = ip_cls; - GNUNET_CONTAINER_multihashmap_get_multiple (uig->update_map, - &hc, - &process_update_node, - &pc); - free_update_information_graph (uig); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Calculating TREEs to find roots of update trees\n"); - /* Find heads of TREEs in update graph */ - nug = ++uig->nug_gen; - fc.tree_array = NULL; - fc.tree_array_size = 0; - - for (i = 0; i < uig->update_node_count; i++) - { - nsn = uig->update_nodes[i]; - if (nsn->nug == nug) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "TREE of node `%s' is %u\n", - nsn->id, - nsn->nug); - continue; /* already placed in TREE */ - } - GNUNET_CRYPTO_hash (nsn->update, strlen (nsn->update), &hc); - nsn->nug = nug; - nsn->tree_id = UINT_MAX; - fc.id = UINT_MAX; - fc.nug = nug; - fc.uig = uig; - GNUNET_CONTAINER_multihashmap_get_multiple (uig->update_map, - &hc, - &find_trees, - &fc); - if (UINT_MAX == fc.id) - { - /* start new TREE */ - for (fc.id = 0; fc.id < fc.tree_array_size; fc.id++) - { - if (NULL == fc.tree_array[fc.id]) - { - fc.tree_array[fc.id] = nsn; - nsn->tree_id = fc.id; - break; - } - } - if (fc.id == fc.tree_array_size) - { - GNUNET_array_append (fc.tree_array, fc.tree_array_size, nsn); - nsn->tree_id = fc.id; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting new TREE %u with node `%s'\n", - nsn->tree_id, - nsn->id); - /* put all nodes with same identifier into this TREE */ - GNUNET_CRYPTO_hash (nsn->id, strlen (nsn->id), &hc); - fc.id = nsn->tree_id; - fc.nug = nug; - fc.uig = uig; - GNUNET_CONTAINER_multihashmap_get_multiple (uig->update_map, - &hc, - &find_trees, - &fc); - } - else - { - /* make head of TREE "id" */ - fc.tree_array[fc.id] = nsn; - nsn->tree_id = fc.id; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "TREE of node `%s' is %u\n", - nsn->id, - fc.id); - } - for (i = 0; i < fc.tree_array_size; i++) - { - nsn = fc.tree_array[i]; - if (NULL != nsn) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Root of TREE %u is node `%s'\n", - i, - nsn->id); - ip (ip_cls, nsn->id, nsn->uri, nsn->md, nsn->update); - } - } - GNUNET_array_grow (fc.tree_array, fc.tree_array_size, 0); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Done processing TREEs\n"); - free_update_information_graph (uig); -} - - -/* end of fs_namespace.c */ diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c deleted file mode 100644 index d1662c78b..000000000 --- a/src/fs/fs_publish.c +++ /dev/null @@ -1,1625 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009, 2010 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/fs_publish.c - * @brief publish a file or directory in GNUnet - * @see https://gnunet.org/encoding - * @author Krista Bennett - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_constants.h" -#include "gnunet_signatures.h" -#include "gnunet_util_lib.h" - -#include "gnunet_fs_service.h" -#include "fs_api.h" -#include "fs_tree.h" - - -/** - * Fill in all of the generic fields for - * a publish event and call the callback. - * - * @param pi structure to fill in - * @param pc overall publishing context - * @param p file information for the file being published - * @param offset where in the file are we so far - * @return value returned from callback - */ -void * -GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_PublishContext *pc, - const struct GNUNET_FS_FileInformation *p, - uint64_t offset) -{ - pi->value.publish.pc = pc; - pi->value.publish.fi = p; - pi->value.publish.cctx = p->client_info; - pi->value.publish.pctx = (NULL == p->dir) ? NULL : p->dir->client_info; - pi->value.publish.filename = p->filename; - pi->value.publish.size = - (GNUNET_YES == p->is_directory) ? p->data.dir.dir_size : - p->data.file.file_size; - pi->value.publish.eta = - GNUNET_TIME_calculate_eta (p->start_time, offset, - pi->value.publish.size); - pi->value.publish.completed = offset; - pi->value.publish.duration = - GNUNET_TIME_absolute_get_duration (p->start_time); - pi->value.publish.anonymity = p->bo.anonymity_level; - pi->fsh = pc->h; - return pc->h->upcb (pc->h->upcb_cls, pi); -} - - -/** - * Cleanup the publish context, we're done with it. - * - * @param pc struct to clean up - */ -static void -publish_cleanup (struct GNUNET_FS_PublishContext *pc) -{ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Cleaning up publish context (done!)\n"); - if (NULL != pc->fhc) - { - GNUNET_CRYPTO_hash_file_cancel (pc->fhc); - pc->fhc = NULL; - } - GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL); - GNUNET_free (pc->nid); - GNUNET_free (pc->nuid); - GNUNET_free (pc->serialization); - if (NULL != pc->dsh) - { - GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO); - pc->dsh = NULL; - } - if (NULL != pc->mq) - { - GNUNET_MQ_destroy (pc->mq); - pc->mq = NULL; - } - GNUNET_assert (NULL == pc->upload_task); - GNUNET_free (pc); -} - - -/** - * Function called by the datastore API with - * the result from the PUT request. - * - * @param cls the `struct GNUNET_FS_PublishContext *` - * @param success #GNUNET_OK on success - * @param min_expiration minimum expiration time required for content to be stored - * @param msg error message (or NULL) - */ -static void -ds_put_cont (void *cls, - int success, - struct GNUNET_TIME_Absolute min_expiration, - const char *msg) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_ProgressInfo pi; - - pc->qre = NULL; - if (GNUNET_SYSERR == success) - { - GNUNET_asprintf (&pc->fi_pos->emsg, - _ ("Publishing failed: %s"), - msg); - pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; - pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; - pi.value.publish.specifics.error.message = pc->fi_pos->emsg; - pc->fi_pos->client_info = - GNUNET_FS_publish_make_status_ (&pi, pc, pc->fi_pos, 0); - if ((GNUNET_YES != pc->fi_pos->is_directory) && - (NULL != pc->fi_pos->filename) && - (GNUNET_YES == pc->any_done) && - (GNUNET_YES == pc->fi_pos->data.file.do_index)) - { - /* run unindex to clean up */ - GNUNET_FS_unindex_start (pc->h, - pc->fi_pos->filename, - NULL); - } - return; - } - pc->any_done = GNUNET_YES; - GNUNET_assert (NULL == pc->upload_task); - pc->upload_task = - GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, - &GNUNET_FS_publish_main_, pc); -} - - -/** - * Generate the callback that signals clients - * that a file (or directory) has been completely - * published. - * - * @param p the completed upload - * @param pc context of the publication - */ -static void -signal_publish_completion (struct GNUNET_FS_FileInformation *p, - struct GNUNET_FS_PublishContext *pc) -{ - struct GNUNET_FS_ProgressInfo pi; - - pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED; - pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO; - pi.value.publish.specifics.completed.chk_uri = p->chk_uri; - pi.value.publish.specifics.completed.sks_uri = p->sks_uri; - p->client_info = - GNUNET_FS_publish_make_status_ (&pi, pc, p, - p->data.file.file_size); -} - - -/** - * Generate the callback that signals clients - * that a file (or directory) has encountered - * a problem during publication. - * - * @param p the upload that had trouble - * @param pc context of the publication - * @param emsg error message - */ -static void -signal_publish_error (struct GNUNET_FS_FileInformation *p, - struct GNUNET_FS_PublishContext *pc, - const char *emsg) -{ - struct GNUNET_FS_ProgressInfo pi; - - p->emsg = GNUNET_strdup (emsg); - pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; - pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; - pi.value.publish.specifics.error.message = emsg; - p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0); - if ((p->is_directory != GNUNET_YES) && - (NULL != p->filename) && - (GNUNET_YES == pc->any_done) && - (p->data.file.do_index == GNUNET_YES)) - { - /* run unindex to clean up */ - GNUNET_FS_unindex_start (pc->h, - p->filename, - NULL); - } -} - - -/** - * Datastore returns from reservation cancel request. - * - * @param cls the `struct GNUNET_FS_PublishContext *` - * @param success success code (not used) - * @param min_expiration minimum expiration time required for content to be stored - * @param msg error message (typically NULL, not used) - */ -static void -finish_release_reserve (void *cls, int success, - struct GNUNET_TIME_Absolute min_expiration, - const char *msg) -{ - struct GNUNET_FS_PublishContext *pc = cls; - - pc->qre = NULL; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Releasing reserve done!\n"); - signal_publish_completion (pc->fi, pc); - pc->all_done = GNUNET_YES; - GNUNET_FS_publish_sync_ (pc); -} - - -/** - * We've finished publishing the SBlock as part of a larger upload. - * Check the result and complete the larger upload. - * - * @param cls the `struct GNUNET_FS_PublishContext *` of the larger upload - * @param uri URI of the published SBlock - * @param emsg NULL on success, otherwise error message - */ -static void -publish_sblocks_cont (void *cls, - const struct GNUNET_FS_Uri *uri, - const char *emsg) -{ - struct GNUNET_FS_PublishContext *pc = cls; - - pc->sks_pc = NULL; - if (NULL != emsg) - { - signal_publish_error (pc->fi, pc, emsg); - GNUNET_FS_publish_sync_ (pc); - return; - } - if (NULL != uri) - { - /* sks publication, remember namespace URI */ - pc->fi->sks_uri = GNUNET_FS_uri_dup (uri); - } - GNUNET_assert (pc->qre == NULL); - if ((pc->dsh != NULL) && (pc->rid != 0)) - { - pc->qre = - GNUNET_DATASTORE_release_reserve (pc->dsh, pc->rid, UINT_MAX, UINT_MAX, - &finish_release_reserve, pc); - } - else - { - finish_release_reserve (pc, GNUNET_OK, GNUNET_TIME_UNIT_ZERO_ABS, NULL); - } -} - - -/** - * We are almost done publishing the structure, - * add SBlocks (if needed). - * - * @param pc overall upload data - */ -static void -publish_sblock (struct GNUNET_FS_PublishContext *pc) -{ - if (NULL != pc->ns) - pc->sks_pc = GNUNET_FS_publish_sks (pc->h, - pc->ns, - pc->nid, - pc->nuid, - pc->fi->meta, - pc->fi->chk_uri, - &pc->fi->bo, - pc->options, - &publish_sblocks_cont, pc); - else - publish_sblocks_cont (pc, NULL, NULL); -} - - -/** - * We've finished publishing a KBlock as part of a larger upload. - * Check the result and continue the larger upload. - * - * @param cls the `struct GNUNET_FS_PublishContext *` - * of the larger upload - * @param uri URI of the published blocks - * @param emsg NULL on success, otherwise error message - */ -static void -publish_kblocks_cont (void *cls, - const struct GNUNET_FS_Uri *uri, - const char *emsg) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_FileInformation *p = pc->fi_pos; - - pc->ksk_pc = NULL; - if (NULL != emsg) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Error uploading KSK blocks: %s\n", - emsg); - signal_publish_error (p, pc, emsg); - GNUNET_FS_file_information_sync_ (p); - GNUNET_FS_publish_sync_ (pc); - GNUNET_assert (NULL == pc->upload_task); - pc->upload_task = - GNUNET_SCHEDULER_add_with_priority - (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, - &GNUNET_FS_publish_main_, - pc); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "KSK blocks published, moving on to next file\n"); - if (NULL != p->dir) - signal_publish_completion (p, pc); - /* move on to next file */ - if (NULL != p->next) - pc->fi_pos = p->next; - else - pc->fi_pos = p->dir; - GNUNET_FS_publish_sync_ (pc); - GNUNET_assert (NULL == pc->upload_task); - pc->upload_task = - GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, - &GNUNET_FS_publish_main_, pc); -} - - -/** - * Function called by the tree encoder to obtain - * a block of plaintext data (for the lowest level - * of the tree). - * - * @param cls our publishing context - * @param offset identifies which block to get - * @param max (maximum) number of bytes to get; returning - * fewer will also cause errors - * @param buf where to copy the plaintext buffer - * @param emsg location to store an error message (on error) - * @return number of bytes copied to buf, 0 on error - */ -static size_t -block_reader (void *cls, - uint64_t offset, - size_t max, - void *buf, - char **emsg) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_FileInformation *p; - const char *dd; - size_t pt_size; - - p = pc->fi_pos; - if (GNUNET_YES == p->is_directory) - { - pt_size = GNUNET_MIN (max, p->data.dir.dir_size - offset); - dd = p->data.dir.dir_data; - GNUNET_memcpy (buf, &dd[offset], pt_size); - } - else - { - if (UINT64_MAX == offset) - { - if (&GNUNET_FS_data_reader_file_ == p->data.file.reader) - { - /* force closing the file to avoid keeping too many files open */ - p->data.file.reader (p->data.file.reader_cls, offset, 0, NULL, NULL); - } - return 0; - } - pt_size = GNUNET_MIN (max, p->data.file.file_size - offset); - if (0 == pt_size) - return 0; /* calling reader with pt_size==0 - * might free buf, so don't! */ - if (pt_size != - p->data.file.reader (p->data.file.reader_cls, offset, pt_size, buf, - emsg)) - return 0; - } - return pt_size; -} - - -/** - * The tree encoder has finished processing a - * file. Call it's finish method and deal with - * the final result. - * - * @param cls our publishing context - */ -static void -encode_cont (void *cls) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_FileInformation *p; - struct GNUNET_FS_ProgressInfo pi; - char *emsg; - uint64_t flen; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Finished with tree encoder\n"); - p = pc->fi_pos; - p->chk_uri = GNUNET_FS_tree_encoder_get_uri (p->te); - GNUNET_FS_file_information_sync_ (p); - GNUNET_FS_tree_encoder_finish (p->te, &emsg); - p->te = NULL; - if (NULL != emsg) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Error during tree walk: %s\n", - emsg); - GNUNET_asprintf (&p->emsg, - _ ("Publishing failed: %s"), - emsg); - GNUNET_free (emsg); - pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; - pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; - pi.value.publish.specifics.error.message = p->emsg; - p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0); - } - else - { - /* final progress event */ - GNUNET_assert (NULL != p->chk_uri); - flen = GNUNET_FS_uri_chk_get_file_size (p->chk_uri); - pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS; - pi.value.publish.specifics.progress.data = NULL; - pi.value.publish.specifics.progress.offset = flen; - pi.value.publish.specifics.progress.data_len = 0; - pi.value.publish.specifics.progress.depth = GNUNET_FS_compute_depth (flen); - p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, flen); - } - /* continue with main */ /* continue with main */ - GNUNET_assert (NULL == pc->upload_task); - pc->upload_task = - GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, - &GNUNET_FS_publish_main_, pc); -} - - -/** - * Function called asking for the current (encoded) - * block to be processed. After processing the - * client should either call #GNUNET_FS_tree_encoder_next - * or (on error) #GNUNET_FS_tree_encoder_finish. - * - * @param cls closure - * @param chk content hash key for the block - * @param offset offset of the block in the file - * @param depth depth of the block in the file, 0 for DBLOCK - * @param type type of the block (IBLOCK or DBLOCK) - * @param block the (encrypted) block - * @param block_size size of @a block (in bytes) - */ -static void -block_proc (void *cls, - const struct ContentHashKey *chk, - uint64_t offset, - unsigned int depth, - enum GNUNET_BLOCK_Type type, - const void *block, - uint16_t block_size) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_FileInformation *p; - struct OnDemandBlock odb; - - p = pc->fi_pos; - if (NULL == pc->dsh) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Waiting for datastore connection\n"); - GNUNET_assert (NULL == pc->upload_task); - pc->upload_task = - GNUNET_SCHEDULER_add_with_priority - (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, pc); - return; - } - - if ((GNUNET_YES != p->is_directory) && - (GNUNET_YES == p->data.file.do_index) && - (GNUNET_BLOCK_TYPE_FS_DBLOCK == type)) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Indexing block `%s' for offset %llu with index size %u\n", - GNUNET_h2s (&chk->query), - (unsigned long long) offset, - (unsigned int) sizeof(struct OnDemandBlock)); - odb.offset = GNUNET_htonll (offset); - odb.file_id = p->data.file.file_id; - GNUNET_assert (pc->qre == NULL); - pc->qre = - GNUNET_DATASTORE_put (pc->dsh, - (p->is_directory == GNUNET_YES) ? 0 : pc->rid, - &chk->query, - sizeof(struct OnDemandBlock), - &odb, - GNUNET_BLOCK_TYPE_FS_ONDEMAND, - p->bo.content_priority, - p->bo.anonymity_level, - p->bo.replication_level, - p->bo.expiration_time, - -2, 1, - &ds_put_cont, pc); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publishing block `%s' for offset %llu with size %u\n", - GNUNET_h2s (&chk->query), - (unsigned long long) offset, - (unsigned int) block_size); - GNUNET_assert (pc->qre == NULL); - pc->qre = - GNUNET_DATASTORE_put (pc->dsh, (p->is_directory == GNUNET_YES) ? 0 : - pc->rid, - &chk->query, - block_size, - block, - type, - p->bo.content_priority, - p->bo.anonymity_level, - p->bo.replication_level, - p->bo.expiration_time, - -2, 1, - &ds_put_cont, - pc); -} - - -/** - * Function called with information about our - * progress in computing the tree encoding. - * - * @param cls closure - * @param offset where are we in the file - * @param pt_block plaintext of the currently processed block - * @param pt_size size of @a pt_block - * @param depth depth of the block in the tree, 0 for DBLOCK - */ -static void -progress_proc (void *cls, uint64_t offset, - const void *pt_block, - size_t pt_size, - unsigned int depth) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_FileInformation *p; - struct GNUNET_FS_FileInformation *par; - struct GNUNET_FS_ProgressInfo pi; - - p = pc->fi_pos; - pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS; - pi.value.publish.specifics.progress.data = pt_block; - pi.value.publish.specifics.progress.offset = offset; - pi.value.publish.specifics.progress.data_len = pt_size; - pi.value.publish.specifics.progress.depth = depth; - p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, offset); - if ((0 != depth) || - (GNUNET_YES == p->is_directory)) - return; - while (NULL != (par = p->dir)) - { - p = par; - GNUNET_assert (GNUNET_YES == par->is_directory); - p->data.dir.contents_completed += pt_size; - pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY; - pi.value.publish.specifics.progress_directory.completed = - p->data.dir.contents_completed; - pi.value.publish.specifics.progress_directory.total = - p->data.dir.contents_size; - pi.value.publish.specifics.progress_directory.eta = - GNUNET_TIME_calculate_eta (p->start_time, - p - ->data.dir.contents_completed, - p - ->data.dir.contents_size); - p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0); - } -} - - -/** - * We are uploading a file or directory; load (if necessary) the next - * block into memory, encrypt it and send it to the FS service. Then - * continue with the main task. - * - * @param pc overall upload data - */ -static void -publish_content (struct GNUNET_FS_PublishContext *pc) -{ - struct GNUNET_FS_FileInformation *p; - char *emsg; - struct GNUNET_FS_DirectoryBuilder *db; - struct GNUNET_FS_FileInformation *dirpos; - void *raw_data; - uint64_t size; - - p = pc->fi_pos; - GNUNET_assert (NULL != p); - if (NULL == p->te) - { - if (GNUNET_YES == p->is_directory) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Creating directory\n"); - db = GNUNET_FS_directory_builder_create (p->meta); - dirpos = p->data.dir.entries; - while (NULL != dirpos) - { - if (GNUNET_YES == dirpos->is_directory) - { - raw_data = dirpos->data.dir.dir_data; - dirpos->data.dir.dir_data = NULL; - } - else - { - raw_data = NULL; - if ((dirpos->data.file.file_size < MAX_INLINE_SIZE) && - (dirpos->data.file.file_size > 0)) - { - raw_data = GNUNET_malloc (dirpos->data.file.file_size); - emsg = NULL; - if (dirpos->data.file.file_size != - dirpos->data.file.reader (dirpos->data.file.reader_cls, 0, - dirpos->data.file.file_size, raw_data, - &emsg)) - { - GNUNET_free (emsg); - GNUNET_free (raw_data); - raw_data = NULL; - } - dirpos->data.file.reader (dirpos->data.file.reader_cls, UINT64_MAX, - 0, 0, NULL); - } - } - GNUNET_FS_directory_builder_add (db, dirpos->chk_uri, dirpos->meta, - raw_data); - GNUNET_free (raw_data); - dirpos = dirpos->next; - } - GNUNET_free (p->data.dir.dir_data); - p->data.dir.dir_data = NULL; - p->data.dir.dir_size = 0; - GNUNET_FS_directory_builder_finish (db, &p->data.dir.dir_size, - &p->data.dir.dir_data); - GNUNET_FS_file_information_sync_ (p); - } - size = (GNUNET_YES == p->is_directory) ? p->data.dir.dir_size : - p->data.file.file_size; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Creating tree encoder\n"); - p->te = - GNUNET_FS_tree_encoder_create (pc->h, size, pc, &block_reader, - &block_proc, &progress_proc, - &encode_cont); - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Processing next block from tree\n"); - GNUNET_FS_tree_encoder_next (p->te); -} - - -/** - * Check the response from the "fs" service to our 'start index' - * request. - * - * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) - * @param msg the response we got - */ -static int -check_index_start_failed (void *cls, - const struct GNUNET_MessageHeader *msg) -{ - size_t msize = ntohs (msg->size) - sizeof(*msg); - const char *emsg = (const char *) &msg[1]; - - if (emsg[msize - 1] != '\0') - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - return GNUNET_OK; -} - - -/** - * Process the response from the "fs" service to our 'start index' - * request. - * - * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) - * @param msg the response we got - */ -static void -handle_index_start_failed (void *cls, - const struct GNUNET_MessageHeader *msg) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_FileInformation *p; - const char *emsg = (const char *) &msg[1]; - char *msgtxt; - - GNUNET_MQ_destroy (pc->mq); - pc->mq = NULL; - p = pc->fi_pos; - GNUNET_asprintf (&msgtxt, - _ ("Can not index file `%s': %s.\n"), - p->filename, - gettext (emsg)); - signal_publish_error (p, - pc, - msgtxt); - GNUNET_free (msgtxt); - GNUNET_FS_file_information_sync_ (p); - GNUNET_FS_publish_sync_ (pc); -} - - -/** - * Process the response from the "fs" service to our 'start index' - * request. - * - * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) - * @param msg the response we got - */ -static void -handle_index_start_ok (void *cls, - const struct GNUNET_MessageHeader *msg) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_FileInformation *p; - - GNUNET_MQ_destroy (pc->mq); - pc->mq = NULL; - p = pc->fi_pos; - p->data.file.index_start_confirmed = GNUNET_YES; - GNUNET_FS_file_information_sync_ (p); - publish_content (pc); -} - - -/** - * Generic error handler, called with the appropriate error code and - * the same closure specified at the creation of the message queue. - * Not every message queue implementation supports an error handler. - * - * @param cls closure with the `struct GNUNET_FS_PublishContext *` - * @param error error code - */ -static void -index_mq_error_handler (void *cls, - enum GNUNET_MQ_Error error) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_FileInformation *p; - - if (NULL != pc->mq) - { - GNUNET_MQ_destroy (pc->mq); - pc->mq = NULL; - } - p = pc->fi_pos; - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Can not index file `%s': %s. Will try to insert instead.\n"), - p->filename, - _ ("error on index-start request to `fs' service")); - p->data.file.do_index = GNUNET_NO; - GNUNET_FS_file_information_sync_ (p); - publish_content (pc); -} - - -/** - * Function called once the hash computation over an - * indexed file has completed. - * - * @param cls closure, our publishing context - * @param res resulting hash, NULL on error - */ -static void -hash_for_index_cb (void *cls, - const struct GNUNET_HashCode *res) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_MQ_MessageHandler handlers[] = { - GNUNET_MQ_hd_fixed_size (index_start_ok, - GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK, - struct GNUNET_MessageHeader, - pc), - GNUNET_MQ_hd_var_size (index_start_failed, - GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED, - struct GNUNET_MessageHeader, - pc), - GNUNET_MQ_handler_end () - }; - struct GNUNET_FS_FileInformation *p; - struct GNUNET_MQ_Envelope *env; - struct IndexStartMessage *ism; - size_t slen; - uint64_t dev; - uint64_t ino; - char *fn; - - pc->fhc = NULL; - p = pc->fi_pos; - if (NULL == res) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ( - "Can not index file `%s': %s. Will try to insert instead.\n"), - p->filename, - _ ("failed to compute hash")); - p->data.file.do_index = GNUNET_NO; - GNUNET_FS_file_information_sync_ (p); - publish_content (pc); - return; - } - if (GNUNET_YES == p->data.file.index_start_confirmed) - { - publish_content (pc); - return; - } - fn = GNUNET_STRINGS_filename_expand (p->filename); - GNUNET_assert (fn != NULL); - slen = strlen (fn) + 1; - if (slen >= - GNUNET_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ - ("Can not index file `%s': %s. Will try to insert instead.\n"), - fn, _ ("filename too long")); - GNUNET_free (fn); - p->data.file.do_index = GNUNET_NO; - GNUNET_FS_file_information_sync_ (p); - publish_content (pc); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Hash of indexed file `%s' is `%s'\n", - p->filename, - GNUNET_h2s (res)); - if (0 != (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) - { - p->data.file.file_id = *res; - p->data.file.have_hash = GNUNET_YES; - p->data.file.index_start_confirmed = GNUNET_YES; - GNUNET_FS_file_information_sync_ (p); - publish_content (pc); - GNUNET_free (fn); - return; - } - pc->mq = GNUNET_CLIENT_connect (pc->h->cfg, - "fs", - handlers, - &index_mq_error_handler, - pc); - if (NULL == pc->mq) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ( - "Can not index file `%s': %s. Will try to insert instead.\n"), - p->filename, - _ ("could not connect to `fs' service")); - p->data.file.do_index = GNUNET_NO; - publish_content (pc); - GNUNET_free (fn); - return; - } - if (p->data.file.have_hash != GNUNET_YES) - { - p->data.file.file_id = *res; - p->data.file.have_hash = GNUNET_YES; - GNUNET_FS_file_information_sync_ (p); - } - env = GNUNET_MQ_msg_extra (ism, - slen, - GNUNET_MESSAGE_TYPE_FS_INDEX_START); - if (GNUNET_OK == - GNUNET_DISK_file_get_identifiers (p->filename, - &dev, - &ino)) - { - ism->device = GNUNET_htonll (dev); - ism->inode = GNUNET_htonll (ino); - } - else - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - _ ("Failed to get file identifiers for `%s'\n"), - p->filename); - } - ism->file_id = *res; - GNUNET_memcpy (&ism[1], - fn, - slen); - GNUNET_free (fn); - GNUNET_MQ_send (pc->mq, - env); -} - - -/** - * We've computed the CHK/LOC URI, now publish the KSKs (if applicable). - * - * @param pc publishing context to do this for - */ -static void -publish_kblocks (struct GNUNET_FS_PublishContext *pc) -{ - struct GNUNET_FS_FileInformation *p; - - p = pc->fi_pos; - /* upload of "p" complete, publish KBlocks! */ - if (NULL != p->keywords) - { - pc->ksk_pc = GNUNET_FS_publish_ksk (pc->h, - p->keywords, - p->meta, - p->chk_uri, - &p->bo, - pc->options, - &publish_kblocks_cont, - pc); - } - else - { - publish_kblocks_cont (pc, p->chk_uri, NULL); - } -} - - -/** - * Process the response from the "fs" service to our LOC sign request. - * - * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) - * @param sig the response we got - */ -static void -handle_signature_response (void *cls, - const struct ResponseLocSignatureMessage *sig) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_FileInformation *p; - - p = pc->fi_pos; - p->chk_uri->type = GNUNET_FS_URI_LOC; - /* p->data.loc.fi kept from CHK before */ - p->chk_uri->data.loc.peer = sig->peer; - p->chk_uri->data.loc.expirationTime - = GNUNET_TIME_absolute_ntoh (sig->expiration_time); - p->chk_uri->data.loc.contentSignature = sig->signature; - GNUNET_FS_file_information_sync_ (p); - GNUNET_FS_publish_sync_ (pc); - publish_kblocks (pc); -} - - -/** - * Generic error handler, called with the appropriate error code and - * the same closure specified at the creation of the message queue. - * Not every message queue implementation supports an error handler. - * - * @param cls closure with the `struct GNUNET_FS_PublishContext *` - * @param error error code - */ -static void -loc_mq_error_handler (void *cls, - enum GNUNET_MQ_Error error) -{ - struct GNUNET_FS_PublishContext *pc = cls; - - if (NULL != pc->mq) - { - GNUNET_MQ_destroy (pc->mq); - pc->mq = NULL; - } - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Can not create LOC URI. Will continue with CHK instead.\n")); - publish_kblocks (pc); -} - - -/** - * We're publishing without anonymity. Contact the FS service - * to create a signed LOC URI for further processing, then - * continue with KSKs. - * - * @param pc the publishing context do to this for - */ -static void -create_loc_uri (struct GNUNET_FS_PublishContext *pc) -{ - struct GNUNET_MQ_MessageHandler handlers[] = { - GNUNET_MQ_hd_fixed_size (signature_response, - GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGNATURE, - struct ResponseLocSignatureMessage, - pc), - GNUNET_MQ_handler_end () - }; - struct GNUNET_MQ_Envelope *env; - struct RequestLocSignatureMessage *req; - struct GNUNET_FS_FileInformation *p; - - if (NULL != pc->mq) - GNUNET_MQ_destroy (pc->mq); - pc->mq = GNUNET_CLIENT_connect (pc->h->cfg, - "fs", - handlers, - &loc_mq_error_handler, - pc); - if (NULL == pc->mq) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ( - "Can not create LOC URI. Will continue with CHK instead.\n")); - publish_kblocks (pc); - return; - } - p = pc->fi_pos; - env = GNUNET_MQ_msg (req, - GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGN); - req->purpose = htonl (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT); - req->expiration_time = GNUNET_TIME_absolute_hton (p->bo.expiration_time); - req->chk = p->chk_uri->data.chk.chk; - req->file_length = GNUNET_htonll (p->chk_uri->data.chk.file_length); - GNUNET_MQ_send (pc->mq, - env); -} - - -/** - * Main function that performs the upload. - * - * @param cls `struct GNUNET_FS_PublishContext *` identifies the upload - */ -void -GNUNET_FS_publish_main_ (void *cls) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_ProgressInfo pi; - struct GNUNET_FS_FileInformation *p; - char *fn; - - pc->upload_task = NULL; - p = pc->fi_pos; - if (NULL == p) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publishing complete, now publishing SKS and KSK blocks.\n"); - /* upload of entire hierarchy complete, - * publish namespace entries */ - GNUNET_FS_publish_sync_ (pc); - publish_sblock (pc); - return; - } - /* find starting position */ - while ((GNUNET_YES == p->is_directory) && - (NULL != p->data.dir.entries) && - (NULL == p->emsg) && - (NULL == p->data.dir.entries->chk_uri)) - { - p = p->data.dir.entries; - pc->fi_pos = p; - GNUNET_FS_publish_sync_ (pc); - } - /* abort on error */ - if (NULL != p->emsg) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Error uploading: %s\n", - p->emsg); - /* error with current file, abort all - * related files as well! */ - while (NULL != p->dir) - { - fn = GNUNET_FS_meta_data_get_by_type (p->meta, - EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); - p = p->dir; - if (fn != NULL) - { - GNUNET_asprintf (&p->emsg, - _ ("Recursive upload failed at `%s': %s"), - fn, - p->emsg); - GNUNET_free (fn); - } - else - { - GNUNET_asprintf (&p->emsg, - _ ("Recursive upload failed: %s"), - p->emsg); - } - pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; - pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; - pi.value.publish.specifics.error.message = p->emsg; - p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0); - } - pc->all_done = GNUNET_YES; - GNUNET_FS_publish_sync_ (pc); - return; - } - /* handle completion */ - if (NULL != p->chk_uri) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "File upload complete, now publishing KSK blocks.\n"); - GNUNET_FS_publish_sync_ (pc); - - if ((0 == p->bo.anonymity_level) && - (GNUNET_YES != - GNUNET_FS_uri_test_loc (p->chk_uri))) - { - /* zero anonymity, box CHK URI in LOC URI */ - create_loc_uri (pc); - } - else - { - publish_kblocks (pc); - } - return; - } - if ((GNUNET_YES != p->is_directory) && (p->data.file.do_index)) - { - if (NULL == p->filename) - { - p->data.file.do_index = GNUNET_NO; - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ( - "Can not index file `%s': %s. Will try to insert instead.\n"), - "", - _ ("needs to be an actual file")); - GNUNET_FS_file_information_sync_ (p); - publish_content (pc); - return; - } - if (p->data.file.have_hash) - { - hash_for_index_cb (pc, &p->data.file.file_id); - } - else - { - p->start_time = GNUNET_TIME_absolute_get (); - pc->fhc = - GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, p->filename, - HASHING_BLOCKSIZE, &hash_for_index_cb, pc); - } - return; - } - publish_content (pc); -} - - -/** - * Signal the FS's progress function that we are starting - * an upload. - * - * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) - * @param fi the entry in the publish-structure - * @param length length of the file or directory - * @param meta metadata for the file or directory (can be modified) - * @param uri pointer to the keywords that will be used for this entry (can be modified) - * @param bo block options - * @param do_index should we index? - * @param client_info pointer to client context set upon creation (can be modified) - * @return #GNUNET_OK to continue (always) - */ -static int -fip_signal_start (void *cls, - struct GNUNET_FS_FileInformation *fi, - uint64_t length, - struct GNUNET_FS_MetaData *meta, - struct GNUNET_FS_Uri **uri, - struct GNUNET_FS_BlockOptions *bo, - int *do_index, - void **client_info) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_ProgressInfo pi; - unsigned int kc; - uint64_t left; - - if (GNUNET_YES == pc->skip_next_fi_callback) - { - pc->skip_next_fi_callback = GNUNET_NO; - return GNUNET_OK; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting publish operation\n"); - if (*do_index) - { - /* space for on-demand blocks */ - pc->reserve_space += - ((length + DBLOCK_SIZE - - 1) / DBLOCK_SIZE) * sizeof(struct OnDemandBlock); - } - else - { - /* space for DBlocks */ - pc->reserve_space += length; - } - /* entries for IBlocks and DBlocks, space for IBlocks */ - left = length; - while (1) - { - left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE; - pc->reserve_entries += left; - if (left <= 1) - break; - left = left * sizeof(struct ContentHashKey); - pc->reserve_space += left; - } - pc->reserve_entries++; - /* entries and space for keywords */ - if (NULL != *uri) - { - kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri); - pc->reserve_entries += kc; - pc->reserve_space += GNUNET_MAX_MESSAGE_SIZE * kc; - } - pi.status = GNUNET_FS_STATUS_PUBLISH_START; - *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0); - GNUNET_FS_file_information_sync_ (fi); - if ((fi->is_directory) && (fi->dir != NULL)) - { - /* We are a directory, and we are not top-level; process entries in directory */ - pc->skip_next_fi_callback = GNUNET_YES; - GNUNET_FS_file_information_inspect (fi, &fip_signal_start, pc); - } - return GNUNET_OK; -} - - -/** - * Actually signal the FS's progress function that we are suspending - * an upload. - * - * @param fi the entry in the publish-structure - * @param pc the publish context of which a file is being suspended - */ -static void -suspend_operation (struct GNUNET_FS_FileInformation *fi, - struct GNUNET_FS_PublishContext *pc) -{ - struct GNUNET_FS_ProgressInfo pi; - uint64_t off; - - if (NULL != pc->ksk_pc) - { - GNUNET_FS_publish_ksk_cancel (pc->ksk_pc); - pc->ksk_pc = NULL; - } - if (NULL != pc->sks_pc) - { - GNUNET_FS_publish_sks_cancel (pc->sks_pc); - pc->sks_pc = NULL; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Suspending publish operation\n"); - GNUNET_free (fi->serialization); - fi->serialization = NULL; - off = (NULL == fi->chk_uri) ? 0 : (GNUNET_YES == fi->is_directory) ? - fi->data.dir.dir_size : fi->data.file.file_size; - pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND; - GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off)); - if (NULL != pc->qre) - { - GNUNET_DATASTORE_cancel (pc->qre); - pc->qre = NULL; - } - if (NULL != pc->dsh) - { - GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO); - pc->dsh = NULL; - } - pc->rid = 0; -} - - -/** - * Signal the FS's progress function that we are suspending - * an upload. Performs the recursion. - * - * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) - * @param fi the entry in the publish-structure - * @param length length of the file or directory - * @param meta metadata for the file or directory (can be modified) - * @param uri pointer to the keywords that will be used for this entry (can be modified) - * @param bo block options - * @param do_index should we index? - * @param client_info pointer to client context set upon creation (can be modified) - * @return #GNUNET_OK to continue (always) - */ -static int -fip_signal_suspend (void *cls, - struct GNUNET_FS_FileInformation *fi, - uint64_t length, - struct GNUNET_FS_MetaData *meta, - struct GNUNET_FS_Uri **uri, - struct GNUNET_FS_BlockOptions *bo, - int *do_index, - void **client_info) -{ - struct GNUNET_FS_PublishContext *pc = cls; - - if (GNUNET_YES == pc->skip_next_fi_callback) - { - pc->skip_next_fi_callback = GNUNET_NO; - return GNUNET_OK; - } - if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) - { - /* process entries in directory */ - pc->skip_next_fi_callback = GNUNET_YES; - GNUNET_FS_file_information_inspect (fi, &fip_signal_suspend, pc); - } - suspend_operation (fi, pc); - *client_info = NULL; - return GNUNET_OK; -} - - -/** - * Create SUSPEND event for the given publish operation - * and then clean up our state (without stop signal). - * - * @param cls the `struct GNUNET_FS_PublishContext` to signal for - */ -void -GNUNET_FS_publish_signal_suspend_ (void *cls) -{ - struct GNUNET_FS_PublishContext *pc = cls; - - if (NULL != pc->upload_task) - { - GNUNET_SCHEDULER_cancel (pc->upload_task); - pc->upload_task = NULL; - } - pc->skip_next_fi_callback = GNUNET_YES; - GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_suspend, pc); - suspend_operation (pc->fi, pc); - GNUNET_FS_end_top (pc->h, pc->top); - pc->top = NULL; - publish_cleanup (pc); -} - - -/** - * We have gotten a reply for our space reservation request. - * Either fail (insufficient space) or start publishing for good. - * - * @param cls the `struct GNUNET_FS_PublishContext *` - * @param success positive reservation ID on success - * @param min_expiration minimum expiration time required for content to be stored - * @param msg error message on error, otherwise NULL - */ -static void -finish_reserve (void *cls, - int success, - struct GNUNET_TIME_Absolute min_expiration, - const char *msg) -{ - struct GNUNET_FS_PublishContext *pc = cls; - - pc->qre = NULL; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Reservation complete (%d)!\n", - success); - if ((msg != NULL) || (success <= 0)) - { - GNUNET_asprintf (&pc->fi->emsg, - _ ("Datastore failure: %s"), - msg); - signal_publish_error (pc->fi, pc, pc->fi->emsg); - return; - } - pc->rid = success; - GNUNET_assert (NULL == pc->upload_task); - pc->upload_task = - GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, - &GNUNET_FS_publish_main_, pc); -} - - -/** - * Calculate the total size of all of the files in the directory structure. - * - * @param fi file structure to traverse - */ -static uint64_t -compute_contents_size (struct GNUNET_FS_FileInformation *fi) -{ - struct GNUNET_FS_FileInformation *ent; - - if (GNUNET_YES != fi->is_directory) - return fi->data.file.file_size; - fi->data.dir.contents_size = 0; - for (ent = fi->data.dir.entries; NULL != ent; ent = ent->next) - fi->data.dir.contents_size += compute_contents_size (ent); - return fi->data.dir.contents_size; -} - - -/** - * Publish a file or directory. - * - * @param h handle to the file sharing subsystem - * @param fi information about the file or directory structure to publish - * @param ns namespace to publish the file in, NULL for no namespace - * @param nid identifier to use for the published content in the namespace - * (can be NULL, must be NULL if namespace is NULL) - * @param nuid update-identifier that will be used for future updates - * (can be NULL, must be NULL if namespace or nid is NULL) - * @param options options for the publication - * @return context that can be used to control the publish operation - */ -struct GNUNET_FS_PublishContext * -GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h, - struct GNUNET_FS_FileInformation *fi, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns, - const char *nid, - const char *nuid, - enum GNUNET_FS_PublishOptions options) -{ - struct GNUNET_FS_PublishContext *ret; - struct GNUNET_DATASTORE_Handle *dsh; - - GNUNET_assert (NULL != h); - compute_contents_size (fi); - if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) - { - dsh = GNUNET_DATASTORE_connect (h->cfg); - if (NULL == dsh) - return NULL; - } - else - { - dsh = NULL; - } - ret = GNUNET_new (struct GNUNET_FS_PublishContext); - ret->dsh = dsh; - ret->h = h; - ret->fi = fi; - if (NULL != ns) - { - ret->ns = GNUNET_new (struct GNUNET_CRYPTO_EcdsaPrivateKey); - *ret->ns = *ns; - GNUNET_assert (NULL != nid); - ret->nid = GNUNET_strdup (nid); - if (NULL != nuid) - ret->nuid = GNUNET_strdup (nuid); - } - ret->options = options; - /* signal start */ - GNUNET_FS_file_information_inspect (ret->fi, &fip_signal_start, ret); - ret->fi_pos = ret->fi; - ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret); - GNUNET_FS_publish_sync_ (ret); - if (NULL != ret->dsh) - { - GNUNET_assert (NULL == ret->qre); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - _ ( - "Reserving space for %u entries and %llu bytes for publication\n"), - (unsigned int) ret->reserve_entries, - (unsigned long long) ret->reserve_space); - ret->qre = - GNUNET_DATASTORE_reserve (ret->dsh, ret->reserve_space, - ret->reserve_entries, - &finish_reserve, - ret); - } - else - { - GNUNET_assert (NULL == ret->upload_task); - ret->upload_task = - GNUNET_SCHEDULER_add_with_priority - (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, ret); - } - return ret; -} - - -/** - * Signal the FS's progress function that we are stopping - * an upload. - * - * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) - * @param fi the entry in the publish-structure - * @param length length of the file or directory - * @param meta metadata for the file or directory (can be modified) - * @param uri pointer to the keywords that will be used for this entry (can be modified) - * @param bo block options (can be modified) - * @param do_index should we index? - * @param client_info pointer to client context set upon creation (can be modified) - * @return #GNUNET_OK to continue (always) - */ -static int -fip_signal_stop (void *cls, - struct GNUNET_FS_FileInformation *fi, - uint64_t length, - struct GNUNET_FS_MetaData *meta, - struct GNUNET_FS_Uri **uri, - struct GNUNET_FS_BlockOptions *bo, - int *do_index, void **client_info) -{ - struct GNUNET_FS_PublishContext *pc = cls; - struct GNUNET_FS_ProgressInfo pi; - uint64_t off; - - if (GNUNET_YES == pc->skip_next_fi_callback) - { - pc->skip_next_fi_callback = GNUNET_NO; - return GNUNET_OK; - } - if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) - { - /* process entries in directory first */ - pc->skip_next_fi_callback = GNUNET_YES; - GNUNET_FS_file_information_inspect (fi, &fip_signal_stop, pc); - } - if (NULL != fi->serialization) - { - GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_FILE_INFO, - fi->serialization); - GNUNET_free (fi->serialization); - fi->serialization = NULL; - } - off = (fi->chk_uri == NULL) ? 0 : length; - pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED; - GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off)); - *client_info = NULL; - return GNUNET_OK; -} - - -/** - * Stop an upload. Will abort incomplete uploads (but - * not remove blocks that have already been published) or - * simply clean up the state for completed uploads. - * Must NOT be called from within the event callback! - * - * @param pc context for the upload to stop - */ -void -GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc) -{ - struct GNUNET_FS_ProgressInfo pi; - uint64_t off; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publish stop called\n"); - GNUNET_FS_end_top (pc->h, pc->top); - if (NULL != pc->ksk_pc) - { - GNUNET_FS_publish_ksk_cancel (pc->ksk_pc); - pc->ksk_pc = NULL; - } - if (NULL != pc->sks_pc) - { - GNUNET_FS_publish_sks_cancel (pc->sks_pc); - pc->sks_pc = NULL; - } - if (NULL != pc->upload_task) - { - GNUNET_SCHEDULER_cancel (pc->upload_task); - pc->upload_task = NULL; - } - pc->skip_next_fi_callback = GNUNET_YES; - GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_stop, pc); - - if (NULL != pc->fi->serialization) - { - GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_FILE_INFO, - pc->fi->serialization); - GNUNET_free (pc->fi->serialization); - pc->fi->serialization = NULL; - } - off = (NULL == pc->fi->chk_uri) ? 0 : GNUNET_ntohll ( - pc->fi->chk_uri->data.chk.file_length); - - if (NULL != pc->serialization) - { - GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, - pc->serialization); - GNUNET_free (pc->serialization); - pc->serialization = NULL; - } - if (NULL != pc->qre) - { - GNUNET_DATASTORE_cancel (pc->qre); - pc->qre = NULL; - } - pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED; - GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, pc->fi, off)); - publish_cleanup (pc); -} - - -/* end of fs_publish.c */ diff --git a/src/fs/fs_publish_ksk.c b/src/fs/fs_publish_ksk.c deleted file mode 100644 index 3981ad335..000000000 --- a/src/fs/fs_publish_ksk.c +++ /dev/null @@ -1,255 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009, 2010, 2012, 2013 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_publish_ksk.c - * @brief publish a URI under a keyword in GNUnet - * @see https://gnunet.org/encoding and #2564 - * @author Krista Bennett - * @author Christian Grothoff - */ - -#include "platform.h" -#include "gnunet_constants.h" -#include "gnunet_signatures.h" -#include "gnunet_util_lib.h" - -#include "gnunet_fs_service.h" -#include "fs_api.h" -#include "fs_tree.h" -#include "fs_publish_ublock.h" - -/** - * Context for the KSK publication. - */ -struct GNUNET_FS_PublishKskContext -{ - /** - * Keywords to use. - */ - struct GNUNET_FS_Uri *ksk_uri; - - /** - * URI to publish. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Metadata to use. - */ - struct GNUNET_FS_MetaData *meta; - - /** - * Global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * UBlock publishing operation that is active. - */ - struct GNUNET_FS_PublishUblockContext *uc; - - /** - * Handle to the datastore, NULL if we are just simulating. - */ - struct GNUNET_DATASTORE_Handle *dsh; - - /** - * Current task. - */ - struct GNUNET_SCHEDULER_Task *ksk_task; - - /** - * Function to call once we're done. - */ - GNUNET_FS_PublishContinuation cont; - - /** - * Closure for cont. - */ - void *cont_cls; - - /** - * When should the KBlocks expire? - */ - struct GNUNET_FS_BlockOptions bo; - - /** - * Options to use. - */ - enum GNUNET_FS_PublishOptions options; - - /** - * Keyword that we are currently processing. - */ - unsigned int i; -}; - - -/** - * Continuation of #GNUNET_FS_publish_ksk() that performs - * the actual publishing operation (iterating over all - * of the keywords). - * - * @param cls closure of type `struct PublishKskContext *` - */ -static void -publish_ksk_cont (void *cls); - - -/** - * Function called by the datastore API with - * the result from the PUT request. - * - * @param cls closure of type `struct GNUNET_FS_PublishKskContext *` - * @param msg error message (or NULL) - */ -static void -kb_put_cont (void *cls, - const char *msg) -{ - struct GNUNET_FS_PublishKskContext *pkc = cls; - - pkc->uc = NULL; - if (NULL != msg) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "KBlock PUT operation failed: %s\n", msg); - pkc->cont (pkc->cont_cls, NULL, msg); - GNUNET_FS_publish_ksk_cancel (pkc); - return; - } - pkc->ksk_task = GNUNET_SCHEDULER_add_now (&publish_ksk_cont, pkc); -} - - -static void -publish_ksk_cont (void *cls) -{ - struct GNUNET_FS_PublishKskContext *pkc = cls; - const char *keyword; - - pkc->ksk_task = NULL; - if ((pkc->i == pkc->ksk_uri->data.ksk.keywordCount) || - (NULL == pkc->dsh)) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "KSK PUT operation complete\n"); - pkc->cont (pkc->cont_cls, pkc->ksk_uri, - NULL); - GNUNET_FS_publish_ksk_cancel (pkc); - return; - } - keyword = pkc->ksk_uri->data.ksk.keywords[pkc->i++]; - pkc->uc = GNUNET_FS_publish_ublock_ (pkc->h, - pkc->dsh, - keyword + 1 /* skip '+' */, - NULL, - GNUNET_CRYPTO_ecdsa_key_get_anonymous (), - pkc->meta, - pkc->uri, - &pkc->bo, - pkc->options, - &kb_put_cont, pkc); -} - - -/** - * Publish a CHK under various keywords on GNUnet. - * - * @param h handle to the file sharing subsystem - * @param ksk_uri keywords to use - * @param meta metadata to use - * @param uri URI to refer to in the KBlock - * @param bo per-block options - * @param options publication options - * @param cont continuation - * @param cont_cls closure for cont - * @return NULL on error ('cont' will still be called) - */ -struct GNUNET_FS_PublishKskContext * -GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, - const struct GNUNET_FS_Uri *ksk_uri, - const struct GNUNET_FS_MetaData *meta, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_BlockOptions *bo, - enum GNUNET_FS_PublishOptions options, - GNUNET_FS_PublishContinuation cont, void *cont_cls) -{ - struct GNUNET_FS_PublishKskContext *pkc; - - GNUNET_assert (NULL != uri); - pkc = GNUNET_new (struct GNUNET_FS_PublishKskContext); - pkc->h = h; - pkc->bo = *bo; - pkc->options = options; - pkc->cont = cont; - pkc->cont_cls = cont_cls; - pkc->meta = GNUNET_FS_meta_data_duplicate (meta); - if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) - { - pkc->dsh = GNUNET_DATASTORE_connect (h->cfg); - if (NULL == pkc->dsh) - { - cont (cont_cls, - NULL, - _ ("Could not connect to datastore.")); - GNUNET_free (pkc); - return NULL; - } - } - pkc->uri = GNUNET_FS_uri_dup (uri); - pkc->ksk_uri = GNUNET_FS_uri_dup (ksk_uri); - pkc->ksk_task = GNUNET_SCHEDULER_add_now (&publish_ksk_cont, pkc); - return pkc; -} - - -/** - * Abort the KSK publishing operation. - * - * @param pkc context of the operation to abort. - */ -void -GNUNET_FS_publish_ksk_cancel (struct GNUNET_FS_PublishKskContext *pkc) -{ - if (NULL != pkc->ksk_task) - { - GNUNET_SCHEDULER_cancel (pkc->ksk_task); - pkc->ksk_task = NULL; - } - if (NULL != pkc->uc) - { - GNUNET_FS_publish_ublock_cancel_ (pkc->uc); - pkc->uc = NULL; - } - if (NULL != pkc->dsh) - { - GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO); - pkc->dsh = NULL; - } - GNUNET_FS_meta_data_destroy (pkc->meta); - GNUNET_FS_uri_destroy (pkc->ksk_uri); - GNUNET_FS_uri_destroy (pkc->uri); - GNUNET_free (pkc); -} - - -/* end of fs_publish_ksk.c */ diff --git a/src/fs/fs_publish_ublock.c b/src/fs/fs_publish_ublock.c deleted file mode 100644 index ad12d9b08..000000000 --- a/src/fs/fs_publish_ublock.c +++ /dev/null @@ -1,299 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009, 2010, 2012, 2013 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_publish_ublock.c - * @brief publish a UBLOCK in GNUnet - * @see https://gnunet.org/encoding and #2564 - * @author Krista Bennett - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_constants.h" -#include "gnunet_signatures.h" -#include "fs_publish_ublock.h" -#include "fs_api.h" -#include "fs_tree.h" - - -/** - * Derive the key for symmetric encryption/decryption from - * the public key and the label. - * - * @param skey where to store symmetric key - * @param iv where to store the IV - * @param label label to use for key derivation - * @param pub public key to use for key derivation - */ -static void -derive_ublock_encryption_key (struct GNUNET_CRYPTO_SymmetricSessionKey *skey, - struct GNUNET_CRYPTO_SymmetricInitializationVector - *iv, - const char *label, - const struct GNUNET_CRYPTO_EcdsaPublicKey *pub) -{ - struct GNUNET_HashCode key; - - /* derive key from 'label' and public key of the namespace */ - GNUNET_assert (GNUNET_YES == - GNUNET_CRYPTO_kdf (&key, sizeof(key), - "UBLOCK-ENC", strlen ("UBLOCK-ENC"), - label, strlen (label), - pub, sizeof(*pub), - NULL, 0)); - GNUNET_CRYPTO_hash_to_aes_key (&key, skey, iv); -} - - -void -GNUNET_FS_ublock_decrypt_ (const void *input, - size_t input_len, - const struct GNUNET_CRYPTO_EcdsaPublicKey *ns, - const char *label, - void *output) -{ - struct GNUNET_CRYPTO_SymmetricInitializationVector iv; - struct GNUNET_CRYPTO_SymmetricSessionKey skey; - - derive_ublock_encryption_key (&skey, &iv, - label, ns); - GNUNET_CRYPTO_symmetric_decrypt (input, input_len, - &skey, &iv, - output); -} - - -/** - * Context for 'ublock_put_cont'. - */ -struct GNUNET_FS_PublishUblockContext -{ - /** - * Function to call when done. - */ - GNUNET_FS_UBlockContinuation cont; - - /** - * Closure of 'cont'. - */ - void *cont_cls; - - /** - * Handle for active datastore operation. - */ - struct GNUNET_DATASTORE_QueueEntry *qre; - - /** - * Task to run continuation asynchronously. - */ - struct GNUNET_SCHEDULER_Task *task; -}; - - -/** - * Continuation of #GNUNET_FS_publish_ublock_(). - * - * @param cls closure of type "struct GNUNET_FS_PublishUblockContext*" - * @param success #GNUNET_SYSERR on failure (including timeout/queue drop) - * #GNUNET_NO if content was already there - * #GNUNET_YES (or other positive value) on success - * @param min_expiration minimum expiration time required for 0-priority content to be stored - * by the datacache at this time, zero for unknown, forever if we have no - * space for 0-priority content - * @param msg NULL on success, otherwise an error message - */ -static void -ublock_put_cont (void *cls, - int32_t success, - struct GNUNET_TIME_Absolute min_expiration, - const char *msg) -{ - struct GNUNET_FS_PublishUblockContext *uc = cls; - - uc->qre = NULL; - uc->cont (uc->cont_cls, msg); - GNUNET_free (uc); -} - - -/** - * Run the continuation. - * - * @param cls the `struct GNUNET_FS_PublishUblockContext *` - */ -static void -run_cont (void *cls) -{ - struct GNUNET_FS_PublishUblockContext *uc = cls; - - uc->task = NULL; - uc->cont (uc->cont_cls, NULL); - GNUNET_free (uc); -} - - -struct GNUNET_FS_PublishUblockContext * -GNUNET_FS_publish_ublock_ (struct GNUNET_FS_Handle *h, - struct GNUNET_DATASTORE_Handle *dsh, - const char *label, - const char *ulabel, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns, - const struct GNUNET_FS_MetaData *meta, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_BlockOptions *bo, - enum GNUNET_FS_PublishOptions options, - GNUNET_FS_UBlockContinuation cont, void *cont_cls) -{ - struct GNUNET_FS_PublishUblockContext *uc; - struct GNUNET_HashCode query; - struct GNUNET_CRYPTO_SymmetricInitializationVector iv; - struct GNUNET_CRYPTO_SymmetricSessionKey skey; - struct GNUNET_CRYPTO_EcdsaPrivateKey *nsd; - struct GNUNET_CRYPTO_EcdsaPublicKey pub; - char *uris; - size_t size; - char *kbe; - char *sptr; - ssize_t mdsize; - size_t slen; - size_t ulen; - struct UBlock *ub_plain; - struct UBlock *ub_enc; - - /* compute ublock to publish */ - if (NULL == meta) - mdsize = 0; - else - mdsize = GNUNET_FS_meta_data_get_serialized_size (meta); - GNUNET_assert (mdsize >= 0); - uris = GNUNET_FS_uri_to_string (uri); - slen = strlen (uris) + 1; - if (NULL == ulabel) - ulen = 1; - else - ulen = strlen (ulabel) + 1; - size = mdsize + sizeof(struct UBlock) + slen + ulen; - if (size > MAX_UBLOCK_SIZE) - { - size = MAX_UBLOCK_SIZE; - mdsize = size - sizeof(struct UBlock) - (slen + ulen); - } - ub_plain = GNUNET_malloc (size); - kbe = (char *) &ub_plain[1]; - if (NULL != ulabel) - GNUNET_memcpy (kbe, ulabel, ulen); - kbe += ulen; - GNUNET_memcpy (kbe, uris, slen); - kbe += slen; - GNUNET_free (uris); - sptr = kbe; - if (NULL != meta) - mdsize = - GNUNET_FS_meta_data_serialize (meta, &sptr, mdsize, - GNUNET_FS_META_DATA_SERIALIZE_PART); - if (-1 == mdsize) - { - GNUNET_break (0); - GNUNET_free (ub_plain); - cont (cont_cls, _ ("Internal error.")); - return NULL; - } - size = sizeof(struct UBlock) + slen + mdsize + ulen; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publishing under identifier `%s'\n", - label); - /* get public key of the namespace */ - GNUNET_CRYPTO_ecdsa_key_get_public (ns, - &pub); - derive_ublock_encryption_key (&skey, &iv, - label, &pub); - - /* encrypt ublock */ - ub_enc = GNUNET_malloc (size); - GNUNET_CRYPTO_symmetric_encrypt (&ub_plain[1], - ulen + slen + mdsize, - &skey, &iv, - &ub_enc[1]); - GNUNET_free (ub_plain); - ub_enc->purpose.size = htonl (ulen + slen + mdsize - + sizeof(struct UBlock) - - sizeof(struct GNUNET_CRYPTO_EcdsaSignature)); - ub_enc->purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_FS_UBLOCK); - - /* derive signing-key from 'label' and public key of the namespace */ - nsd = GNUNET_CRYPTO_ecdsa_private_key_derive (ns, label, "fs-ublock"); - GNUNET_CRYPTO_ecdsa_key_get_public (nsd, - &ub_enc->verification_key); - GNUNET_assert (GNUNET_OK == - GNUNET_CRYPTO_ecdsa_sign_ (nsd, - &ub_enc->purpose, - &ub_enc->signature)); - GNUNET_CRYPTO_hash (&ub_enc->verification_key, - sizeof(ub_enc->verification_key), - &query); - GNUNET_free (nsd); - - uc = GNUNET_new (struct GNUNET_FS_PublishUblockContext); - uc->cont = cont; - uc->cont_cls = cont_cls; - if (NULL != dsh) - { - uc->qre = - GNUNET_DATASTORE_put (dsh, - 0, - &query, - ulen + slen + mdsize + sizeof(struct UBlock), - ub_enc, - GNUNET_BLOCK_TYPE_FS_UBLOCK, - bo->content_priority, - bo->anonymity_level, - bo->replication_level, - bo->expiration_time, - -2, 1, - &ublock_put_cont, uc); - } - else - { - uc->task = GNUNET_SCHEDULER_add_now (&run_cont, - uc); - } - GNUNET_free (ub_enc); - return uc; -} - - -/** - * Abort UBlock publishing operation. - * - * @param uc operation to abort. - */ -void -GNUNET_FS_publish_ublock_cancel_ (struct GNUNET_FS_PublishUblockContext *uc) -{ - if (NULL != uc->qre) - GNUNET_DATASTORE_cancel (uc->qre); - if (NULL != uc->task) - GNUNET_SCHEDULER_cancel (uc->task); - GNUNET_free (uc); -} - - -/* end of fs_publish_ublock.c */ diff --git a/src/fs/fs_publish_ublock.h b/src/fs/fs_publish_ublock.h deleted file mode 100644 index 4adffc6c1..000000000 --- a/src/fs/fs_publish_ublock.h +++ /dev/null @@ -1,109 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009, 2010, 2012, 2013 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_publish_ublock.h - * @brief publish a UBLOCK in GNUnet - * @see https://gnunet.org/encoding and #2564 - * @author Krista Bennett - * @author Christian Grothoff - */ -#ifndef FS_PUBLISH_UBLOCK_H -#define FS_PUBLISH_UBLOCK_H - -#include "gnunet_util_lib.h" -#include "gnunet_datastore_service.h" - -#include "gnunet_fs_service.h" -#include "gnunet_identity_service.h" - - -/** - * Decrypt the given UBlock, storing the result in output. - * - * @param input input data - * @param input_len number of bytes in @a input - * @param ns public key under which the UBlock was stored - * @param label label under which the UBlock was stored - * @param output where to write the result, has input_len bytes - */ -void -GNUNET_FS_ublock_decrypt_ (const void *input, - size_t input_len, - const struct GNUNET_CRYPTO_EcdsaPublicKey *ns, - const char *label, - void *output); - - -/** - * Context for 'ublock_put_cont'. - */ -struct GNUNET_FS_PublishUblockContext; - - -/** - * Signature of a function called as the continuation of a UBlock - * publication. - * - * @param cls closure - * @param emsg error message, NULL on success - */ -typedef void (*GNUNET_FS_UBlockContinuation) (void *cls, - const char *emsg); - - -/** - * Publish a UBlock. - * - * @param h handle to the file sharing subsystem - * @param dsh datastore handle to use for storage operation - * @param label identifier to use - * @param ulabel update label to use, may be an empty string for none - * @param ns namespace to publish in - * @param meta metadata to use - * @param uri URI to refer to in the UBlock - * @param bo per-block options - * @param options publication options - * @param cont continuation - * @param cont_cls closure for @a cont - * @return NULL on error (@a cont will still be called) - */ -struct GNUNET_FS_PublishUblockContext * -GNUNET_FS_publish_ublock_ (struct GNUNET_FS_Handle *h, - struct GNUNET_DATASTORE_Handle *dsh, - const char *label, - const char *ulabel, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns, - const struct GNUNET_FS_MetaData *meta, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_BlockOptions *bo, - enum GNUNET_FS_PublishOptions options, - GNUNET_FS_UBlockContinuation cont, void *cont_cls); - - -/** - * Abort UBlock publishing operation. - * - * @param uc operation to abort. - */ -void -GNUNET_FS_publish_ublock_cancel_ (struct GNUNET_FS_PublishUblockContext *uc); - -#endif diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c deleted file mode 100644 index 8b8c54c67..000000000 --- a/src/fs/fs_search.c +++ /dev/null @@ -1,1826 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2001-2014 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/fs_search.c - * @brief Helper functions for searching. - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_constants.h" - -#include "gnunet_fs_service.h" -#include "gnunet_protocols.h" -#include "fs_api.h" -#include "fs_publish_ublock.h" - - -/** - * Number of availability trials we perform per search result. - */ -#define AVAILABILITY_TRIALS_MAX 8 - -/** - * Fill in all of the generic fields for a search event and - * call the callback. - * - * @param pi structure to fill in - * @param h file-sharing handle - * @param sc overall search context - * @return value returned by the callback - */ -void * -GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_Handle *h, - struct GNUNET_FS_SearchContext *sc) -{ - void *ret; - - pi->value.search.sc = sc; - pi->value.search.cctx = (NULL != sc) ? sc->client_info : NULL; - pi->value.search.pctx = - ((NULL == sc) || (NULL == sc->psearch_result)) - ? NULL - : sc->psearch_result->client_info; - pi->value.search.query = (NULL != sc) ? sc->uri : NULL; - pi->value.search.duration = (NULL != sc) - ? GNUNET_TIME_absolute_get_duration ( - sc->start_time) - : GNUNET_TIME_UNIT_ZERO; - pi->value.search.anonymity = (NULL != sc) ? sc->anonymity : 0; - pi->fsh = h; - ret = h->upcb (h->upcb_cls, pi); - return ret; -} - - -/** - * Check if the given result is identical to the given URI. - * - * @param cls points to the URI we check against - * @param key not used - * @param value a `struct GNUNET_FS_SearchResult` who's URI we - * should compare with - * @return #GNUNET_SYSERR if the result is present, - * #GNUNET_OK otherwise - */ -static int -test_result_present (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - const struct GNUNET_FS_Uri *uri = cls; - struct GNUNET_FS_SearchResult *sr = value; - - if (GNUNET_FS_uri_test_equal (uri, sr->uri)) - return GNUNET_SYSERR; - return GNUNET_OK; -} - - -/** - * We've found a new CHK result. Let the client - * know about it. - * - * @param sc the search context - * @param sr the specific result - */ -static void -notify_client_chk_result (struct GNUNET_FS_SearchContext *sc, - struct GNUNET_FS_SearchResult *sr) -{ - struct GNUNET_FS_ProgressInfo pi; - - pi.status = GNUNET_FS_STATUS_SEARCH_RESULT; - pi.value.search.specifics.result.meta = sr->meta; - pi.value.search.specifics.result.uri = sr->uri; - pi.value.search.specifics.result.result = sr; - pi.value.search.specifics.result.applicability_rank = sr->optional_support; - sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc); -} - - -/** - * We've found new information about an existing CHK result. Let the - * client know about it. - * - * @param sc the search context - * @param sr the specific result - */ -static void -notify_client_chk_update (struct GNUNET_FS_SearchContext *sc, - struct GNUNET_FS_SearchResult *sr) -{ - struct GNUNET_FS_ProgressInfo pi; - - pi.status = GNUNET_FS_STATUS_SEARCH_UPDATE; - pi.value.search.specifics.update.cctx = sr->client_info; - pi.value.search.specifics.update.meta = sr->meta; - pi.value.search.specifics.update.uri = sr->uri; - pi.value.search.specifics.update.availability_rank = - 2 * sr->availability_success - sr->availability_trials; - pi.value.search.specifics.update.availability_certainty = - sr->availability_trials; - pi.value.search.specifics.update.applicability_rank = sr->optional_support; - pi.value.search.specifics.update.current_probe_time - = GNUNET_TIME_absolute_get_duration (sr->probe_active_time); - sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc); -} - - -/** - * Context for "get_result_present". - */ -struct GetResultContext -{ - /** - * The URI we're looking for. - */ - const struct GNUNET_FS_Uri *uri; - - /** - * Where to store a pointer to the search - * result struct if we found a match. - */ - struct GNUNET_FS_SearchResult *sr; -}; - - -/** - * Check if the given result is identical to the given URI and if so - * return it. - * - * @param cls a `struct GetResultContext` - * @param key not used - * @param value a `struct GNUNET_FS_SearchResult` who's URI we - * should compare with - * @return #GNUNET_OK - */ -static int -get_result_present (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct GetResultContext *grc = cls; - struct GNUNET_FS_SearchResult *sr = value; - - if (GNUNET_FS_uri_test_equal (grc->uri, sr->uri)) - grc->sr = sr; - return GNUNET_OK; -} - - -/** - * Signal result of last probe to client and then schedule next - * probe. - * - * @param sr search result to signal for - */ -static void -signal_probe_result (struct GNUNET_FS_SearchResult *sr) -{ - struct GNUNET_FS_ProgressInfo pi; - - pi.status = GNUNET_FS_STATUS_SEARCH_UPDATE; - pi.value.search.specifics.update.cctx = sr->client_info; - pi.value.search.specifics.update.meta = sr->meta; - pi.value.search.specifics.update.uri = sr->uri; - pi.value.search.specifics.update.availability_rank - = 2 * sr->availability_success - sr->availability_trials; - pi.value.search.specifics.update.availability_certainty - = sr->availability_trials; - pi.value.search.specifics.update.applicability_rank = sr->optional_support; - pi.value.search.specifics.update.current_probe_time - = GNUNET_TIME_absolute_get_duration (sr->probe_active_time); - sr->client_info = GNUNET_FS_search_make_status_ (&pi, sr->h, sr->sc); - GNUNET_FS_search_start_probe_ (sr); -} - - -/** - * Handle the case where we have failed to receive a response for our probe. - * - * @param cls our `struct GNUNET_FS_SearchResult *` - */ -static void -probe_failure_handler (void *cls) -{ - struct GNUNET_FS_SearchResult *sr = cls; - - sr->probe_cancel_task = NULL; - sr->availability_trials++; - GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES); - sr->probe_ctx = NULL; - GNUNET_FS_stop_probe_ping_task_ (sr); - GNUNET_FS_search_result_sync_ (sr); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Probe #%u for search result %p failed\n", - sr->availability_trials, - sr); - signal_probe_result (sr); -} - - -/** - * Handle the case where we have gotten a response for our probe. - * - * @param cls our `struct GNUNET_FS_SearchResult *` - */ -static void -probe_success_handler (void *cls) -{ - struct GNUNET_FS_SearchResult *sr = cls; - - sr->probe_cancel_task = NULL; - sr->availability_trials++; - sr->availability_success++; - GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES); - sr->probe_ctx = NULL; - GNUNET_FS_stop_probe_ping_task_ (sr); - GNUNET_FS_search_result_sync_ (sr); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Probe #%u for search result %p succeeded\n", - sr->availability_trials, - sr); - signal_probe_result (sr); -} - - -/** - * Notification of FS that a search probe has made progress. - * This function is used INSTEAD of the client's event handler - * for downloads where the #GNUNET_FS_DOWNLOAD_IS_PROBE flag is set. - * - * @param cls closure, always NULL (!), actual closure - * is in the client-context of the info struct - * @param info details about the event, specifying the event type - * and various bits about the event - * @return client-context (for the next progress call - * for this operation; should be set to NULL for - * SUSPEND and STOPPED events). The value returned - * will be passed to future callbacks in the respective - * field in the `struct GNUNET_FS_ProgressInfo`. - */ -void * -GNUNET_FS_search_probe_progress_ (void *cls, - const struct GNUNET_FS_ProgressInfo *info) -{ - struct GNUNET_FS_SearchResult *sr = info->value.download.cctx; - struct GNUNET_TIME_Relative dur; - - switch (info->status) - { - case GNUNET_FS_STATUS_DOWNLOAD_START: - /* ignore */ - break; - case GNUNET_FS_STATUS_DOWNLOAD_RESUME: - /* probes should never be resumed */ - GNUNET_assert (0); - break; - case GNUNET_FS_STATUS_DOWNLOAD_SUSPEND: - /* probes should never be suspended */ - GNUNET_break (0); - break; - case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: - /* ignore */ - break; - case GNUNET_FS_STATUS_DOWNLOAD_ERROR: - if (NULL != sr->probe_cancel_task) - { - GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); - sr->probe_cancel_task = NULL; - } - sr->probe_cancel_task = - GNUNET_SCHEDULER_add_delayed (sr->remaining_probe_time, - &probe_failure_handler, sr); - break; - case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED: - if (NULL != sr->probe_cancel_task) - { - GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); - sr->probe_cancel_task = NULL; - } - sr->probe_cancel_task = - GNUNET_SCHEDULER_add_now (&probe_success_handler, sr); - break; - case GNUNET_FS_STATUS_DOWNLOAD_STOPPED: - if (NULL != sr->probe_cancel_task) - { - GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); - sr->probe_cancel_task = NULL; - } - sr = NULL; - break; - case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: - if (NULL == sr->probe_cancel_task) - { - sr->probe_active_time = GNUNET_TIME_absolute_get (); - sr->probe_cancel_task = - GNUNET_SCHEDULER_add_delayed (sr->remaining_probe_time, - &probe_failure_handler, sr); - } - break; - case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: - if (NULL != sr->probe_cancel_task) - { - GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); - sr->probe_cancel_task = NULL; - } - dur = GNUNET_TIME_absolute_get_duration (sr->probe_active_time); - sr->remaining_probe_time = - GNUNET_TIME_relative_subtract (sr->remaining_probe_time, dur); - if (0 == sr->remaining_probe_time.rel_value_us) - sr->probe_cancel_task = - GNUNET_SCHEDULER_add_now (&probe_failure_handler, sr); - GNUNET_FS_search_result_sync_ (sr); - break; - - default: - GNUNET_break (0); - return NULL; - } - return sr; -} - - -/** - * Task run periodically to remind clients that a probe is active. - * - * @param cls the `struct GNUNET_FS_SearchResult` that we are probing for - */ -static void -probe_ping_task_cb (void *cls) -{ - struct GNUNET_FS_Handle *h = cls; - - for (struct GNUNET_FS_SearchResult *sr = h->probes_head; - NULL != sr; - sr = sr->next) - if (NULL != sr->probe_ctx->mq) - signal_probe_result (sr); - h->probe_ping_task - = GNUNET_SCHEDULER_add_delayed (GNUNET_FS_PROBE_UPDATE_FREQUENCY, - &probe_ping_task_cb, - h); -} - - -/** - * Start the ping task for this search result. - * - * @param sr result to start pinging for. - */ -static void -start_probe_ping_task (struct GNUNET_FS_SearchResult *sr) -{ - struct GNUNET_FS_Handle *h = sr->h; - - GNUNET_CONTAINER_DLL_insert (h->probes_head, - h->probes_tail, - sr); - if (NULL == h->probe_ping_task) - h->probe_ping_task - = GNUNET_SCHEDULER_add_now (&probe_ping_task_cb, - h); -} - - -/** - * Stop the ping task for this search result. - * - * @param sr result to start pinging for. - */ -void -GNUNET_FS_stop_probe_ping_task_ (struct GNUNET_FS_SearchResult *sr) -{ - struct GNUNET_FS_Handle *h = sr->h; - - GNUNET_CONTAINER_DLL_remove (h->probes_head, - h->probes_tail, - sr); - if (NULL == h->probes_head) - { - GNUNET_SCHEDULER_cancel (h->probe_ping_task); - h->probe_ping_task = NULL; - } -} - - -/** - * Start download probes for the given search result. - * - * @param sr the search result - */ -void -GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr) -{ - uint64_t off; - uint64_t len; - - if (NULL != sr->probe_ctx) - return; - if (NULL != sr->download) - return; - if (0 == (sr->h->flags & GNUNET_FS_FLAGS_DO_PROBES)) - return; - if (sr->availability_trials > AVAILABILITY_TRIALS_MAX) - return; - if ( (GNUNET_FS_URI_CHK != sr->uri->type) && - (GNUNET_FS_URI_LOC != sr->uri->type) ) - return; - len = GNUNET_FS_uri_chk_get_file_size (sr->uri); - if (0 == len) - return; - if ((len <= DBLOCK_SIZE) && (sr->availability_success > 0)) - return; - off = len / DBLOCK_SIZE; - if (off > 0) - off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, off); - off *= DBLOCK_SIZE; - if (len - off < DBLOCK_SIZE) - len = len - off; - else - len = DBLOCK_SIZE; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting probe #%u (at offset %llu) for search result %p\n", - sr->availability_trials + 1, - (unsigned long long) off, - sr); - sr->remaining_probe_time = - GNUNET_TIME_relative_saturating_multiply (sr->h->avg_block_latency, - 2 * (1 - + sr->availability_trials)); - sr->probe_ctx = - GNUNET_FS_download_start (sr->h, sr->uri, sr->meta, NULL, NULL, off, - len, sr->anonymity, - GNUNET_FS_DOWNLOAD_NO_TEMPORARIES - | GNUNET_FS_DOWNLOAD_IS_PROBE, sr, NULL); - start_probe_ping_task (sr); -} - - -/** - * Start download probes for the given search result. - * - * @param h file-sharing handle to use for the operation - * @param uri URI to probe - * @param meta meta data associated with the URI - * @param client_info client info pointer to use for associated events - * @param anonymity anonymity level to use for the probes - * @return the search result handle to access the probe activity - */ -struct GNUNET_FS_SearchResult * -GNUNET_FS_probe (struct GNUNET_FS_Handle *h, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *meta, - void *client_info, - uint32_t anonymity) -{ - struct GNUNET_FS_SearchResult *sr; - - GNUNET_assert (NULL != h); - GNUNET_assert (NULL != uri); - sr = GNUNET_new (struct GNUNET_FS_SearchResult); - sr->h = h; - sr->uri = GNUNET_FS_uri_dup (uri); - sr->meta = GNUNET_FS_meta_data_duplicate (meta); - sr->client_info = client_info; - sr->anonymity = anonymity; - GNUNET_FS_search_start_probe_ (sr); - return sr; -} - - -/** - * Stop probing activity associated with a search result. - * - * @param sr search result - */ -static void -GNUNET_FS_search_stop_probe_ (struct GNUNET_FS_SearchResult *sr) -{ - if (NULL != sr->probe_ctx) - { - GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES); - sr->probe_ctx = NULL; - GNUNET_FS_stop_probe_ping_task_ (sr); - } - if (NULL != sr->probe_cancel_task) - { - GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); - sr->probe_cancel_task = NULL; - } -} - - -/** - * Stop probe activity. Must ONLY be used on values - * returned from #GNUNET_FS_probe. - * - * @param sr search result to stop probing for (freed) - * @return the value of the 'client_info' pointer - */ -void * -GNUNET_FS_probe_stop (struct GNUNET_FS_SearchResult *sr) -{ - void *client_info; - - GNUNET_assert (NULL == sr->sc); - GNUNET_FS_search_stop_probe_ (sr); - GNUNET_FS_uri_destroy (sr->uri); - GNUNET_FS_meta_data_destroy (sr->meta); - client_info = sr->client_info; - GNUNET_free (sr); - return client_info; -} - - -/** - * We have received a KSK result. Check how it fits in with the - * overall query and notify the client accordingly. - * - * @param sc context for the overall query - * @param ent entry for the specific keyword - * @param uri the URI that was found - * @param meta metadata associated with the URI - * under the @a ent keyword - */ -static void -process_ksk_result (struct GNUNET_FS_SearchContext *sc, - struct SearchRequestEntry *ent, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *meta) -{ - struct GNUNET_HashCode key; - struct GNUNET_FS_SearchResult *sr; - struct GetResultContext grc; - int is_new; - unsigned int koff; - - /* check if new */ - GNUNET_assert (NULL != sc); - if (GNUNET_OK != - GNUNET_FS_uri_to_key (uri, - &key)) - { - GNUNET_break_op (0); - return; - } - if (GNUNET_SYSERR == - GNUNET_CONTAINER_multihashmap_get_multiple (ent->results, - &key, - &test_result_present, - (void *) uri)) - return; /* duplicate result */ - /* try to find search result in master map */ - grc.sr = NULL; - grc.uri = uri; - GNUNET_CONTAINER_multihashmap_get_multiple (sc->master_result_map, - &key, - &get_result_present, &grc); - sr = grc.sr; - is_new = (NULL == sr) || (sr->mandatory_missing > 0); - if (NULL == sr) - { - sr = GNUNET_new (struct GNUNET_FS_SearchResult); - sr->h = sc->h; - sr->sc = sc; - sr->anonymity = sc->anonymity; - sr->uri = GNUNET_FS_uri_dup (uri); - sr->meta = GNUNET_FS_meta_data_duplicate (meta); - sr->mandatory_missing = sc->mandatory_count; - sr->key = key; - sr->keyword_bitmap = GNUNET_malloc ((sc->uri->data.ksk.keywordCount + 7) - / 8); /* round up, count bits */ - GNUNET_CONTAINER_multihashmap_put (sc->master_result_map, &key, sr, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); - } - else - { - GNUNET_FS_meta_data_merge (sr->meta, meta); - } - GNUNET_break (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put (ent->results, - &sr->key, - sr, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - - koff = ent - sc->requests; - GNUNET_assert ((ent >= sc->requests) && - (koff < sc->uri->data.ksk.keywordCount)); - sr->keyword_bitmap[koff / 8] |= (1 << (koff % 8)); - /* check if mandatory satisfied */ - if (1 <= GNUNET_CONTAINER_multihashmap_size (ent->results)) - { - if (ent->mandatory) - { - GNUNET_break (sr->mandatory_missing > 0); - sr->mandatory_missing--; - } - else - { - sr->optional_support++; - } - } - if (0 != sr->mandatory_missing) - { - GNUNET_break (NULL == sr->client_info); - return; - } - if (is_new) - notify_client_chk_result (sc, sr); - else - notify_client_chk_update (sc, sr); - GNUNET_FS_search_result_sync_ (sr); - GNUNET_FS_search_start_probe_ (sr); -} - - -/** - * Start search for content, internal API. - * - * @param h handle to the file sharing subsystem - * @param uri specifies the search parameters; can be - * a KSK URI or an SKS URI. - * @param anonymity desired level of anonymity - * @param options options for the search - * @param cctx client context - * @param psearch parent search result (for namespace update searches) - * @return context that can be used to control the search - */ -static struct GNUNET_FS_SearchContext * -search_start (struct GNUNET_FS_Handle *h, - const struct GNUNET_FS_Uri *uri, - uint32_t anonymity, - enum GNUNET_FS_SearchOptions options, - void *cctx, - struct GNUNET_FS_SearchResult *psearch); - - -/** - * We have received an SKS result. Start searching for updates and - * notify the client if it is a new result. - * - * @param sc context for the overall query - * @param id_update identifier for updates, NULL for none - * @param uri the URI that was found - * @param meta metadata associated with the URI - */ -static void -process_sks_result (struct GNUNET_FS_SearchContext *sc, - const char *id_update, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *meta) -{ - struct GNUNET_FS_Uri uu; - struct GNUNET_HashCode key; - struct GNUNET_FS_SearchResult *sr; - - /* check if new */ - GNUNET_assert (NULL != sc); - if (GNUNET_OK != - GNUNET_FS_uri_to_key (uri, - &key)) - { - GNUNET_break (0); - return; - } - GNUNET_CRYPTO_hash_xor (&uri->data.chk.chk.key, - &uri->data.chk.chk.query, - &key); - if (GNUNET_SYSERR == - GNUNET_CONTAINER_multihashmap_get_multiple (sc->master_result_map, &key, - &test_result_present, - (void *) uri)) - return; /* duplicate result */ - sr = GNUNET_new (struct GNUNET_FS_SearchResult); - sr->h = sc->h; - sr->sc = sc; - sr->anonymity = sc->anonymity; - sr->uri = GNUNET_FS_uri_dup (uri); - sr->meta = GNUNET_FS_meta_data_duplicate (meta); - sr->key = key; - GNUNET_CONTAINER_multihashmap_put (sc->master_result_map, &key, sr, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); - GNUNET_FS_search_result_sync_ (sr); - GNUNET_FS_search_start_probe_ (sr); - /* notify client */ - if (0 == sr->mandatory_missing) - notify_client_chk_result (sc, sr); - else - GNUNET_break (NULL == sr->client_info); - /* search for updates */ - if (0 == strlen (id_update)) - return; /* no updates */ - uu.type = GNUNET_FS_URI_SKS; - uu.data.sks.ns = sc->uri->data.sks.ns; - uu.data.sks.identifier = GNUNET_strdup (id_update); - (void) search_start (sc->h, &uu, sc->anonymity, sc->options, NULL, sr); - GNUNET_free (uu.data.sks.identifier); -} - - -/** - * Decrypt a ublock using a 'keyword' as the passphrase. Given the - * KSK public key derived from the keyword, this function looks up - * the original keyword in the search context and decrypts the - * given ciphertext block. - * - * @param sc search context with the keywords - * @param dpub derived public key used for the search - * @param edata encrypted data - * @param edata_size number of bytes in @a edata (and @a data) - * @param data where to store the plaintext - * @return keyword index on success, #GNUNET_SYSERR on error (no such - * keyword, internal error) - */ -static int -decrypt_block_with_keyword (const struct GNUNET_FS_SearchContext *sc, - const struct GNUNET_CRYPTO_EcdsaPublicKey *dpub, - const void *edata, - size_t edata_size, - char *data) -{ - const struct GNUNET_CRYPTO_EcdsaPrivateKey *anon; - struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub; - unsigned int i; - - /* find key */ - for (i = 0; i < sc->uri->data.ksk.keywordCount; i++) - if (0 == memcmp (dpub, - &sc->requests[i].dpub, - sizeof(struct GNUNET_CRYPTO_EcdsaPublicKey))) - break; - if (i == sc->uri->data.ksk.keywordCount) - { - /* oops, does not match any of our keywords!? */ - GNUNET_break (0); - return GNUNET_SYSERR; - } - /* decrypt */ - anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous (); - GNUNET_CRYPTO_ecdsa_key_get_public (anon, &anon_pub); - GNUNET_FS_ublock_decrypt_ (edata, edata_size, - &anon_pub, - sc->requests[i].keyword, - data); - return i; -} - - -/** - * Process a keyword search result. The actual type of block is - * a UBlock; we know it is a keyword search result because that's - * what we were searching for. - * - * @param sc our search context - * @param ub the ublock with the keyword search result - * @param size size of @a ub - */ -static void -process_kblock (struct GNUNET_FS_SearchContext *sc, - const struct UBlock *ub, - size_t size) -{ - size_t j; - char pt[size - sizeof(struct UBlock)]; - const char *eos; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *uri; - char *emsg; - int i; - - if (-1 == (i = decrypt_block_with_keyword (sc, - &ub->verification_key, - &ub[1], - size - sizeof(struct UBlock), - pt))) - return; - /* parse; pt[0] is just '\0', so we skip over that */ - eos = memchr (&pt[1], '\0', sizeof(pt) - 1); - if (NULL == eos) - { - GNUNET_break_op (0); - return; - } - if (NULL == (uri = GNUNET_FS_uri_parse (&pt[1], &emsg))) - { - if (GNUNET_FS_VERSION > 0x00090400) - { - /* we broke this in 0x00090300, so don't bitch - too loudly just one version up... */ - GNUNET_break_op (0); /* ublock malformed */ - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Failed to parse URI `%s': %s\n"), - &pt[1], - emsg); - } - GNUNET_free (emsg); - return; - } - j = eos - pt + 1; - if (sizeof(pt) == j) - meta = GNUNET_FS_meta_data_create (); - else - meta = GNUNET_FS_meta_data_deserialize (&pt[j], sizeof(pt) - j); - if (NULL == meta) - { - GNUNET_break_op (0); /* ublock malformed */ - GNUNET_FS_uri_destroy (uri); - return; - } - process_ksk_result (sc, - &sc->requests[i], - uri, - meta); - - /* clean up */ - GNUNET_FS_meta_data_destroy (meta); - GNUNET_FS_uri_destroy (uri); -} - - -/** - * Process a namespace-search result. The actual type of block is - * a UBlock; we know it is a namespace search result because that's - * what we were searching for. - * - * @param sc our search context - * @param ub the ublock with a namespace result - * @param size size of @a ub - */ -static void -process_sblock (struct GNUNET_FS_SearchContext *sc, - const struct UBlock *ub, - size_t size) -{ - size_t len = size - sizeof(struct UBlock); - char pt[len]; - struct GNUNET_FS_Uri *uri; - struct GNUNET_FS_MetaData *meta; - const char *id; - const char *uris; - size_t off; - char *emsg; - - GNUNET_FS_ublock_decrypt_ (&ub[1], len, - &sc->uri->data.sks.ns, - sc->uri->data.sks.identifier, - pt); - /* parse */ - if (0 == (off = GNUNET_STRINGS_buffer_tokenize (pt, len, 2, &id, &uris))) - { - GNUNET_break_op (0); /* ublock malformed */ - return; - } - if (NULL == (meta = GNUNET_FS_meta_data_deserialize (&pt[off], len - - off))) - { - GNUNET_break_op (0); /* ublock malformed */ - return; - } - if (NULL == (uri = GNUNET_FS_uri_parse (uris, &emsg))) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Failed to parse URI `%s': %s\n"), - uris, emsg); - GNUNET_break_op (0); /* ublock malformed */ - GNUNET_free (emsg); - GNUNET_FS_meta_data_destroy (meta); - return; - } - /* process */ - process_sks_result (sc, id, uri, meta); - /* clean up */ - GNUNET_FS_uri_destroy (uri); - GNUNET_FS_meta_data_destroy (meta); -} - - -/** - * Shutdown any existing connection to the FS - * service and try to establish a fresh one - * (and then re-transmit our search request). - * - * @param sc the search to reconnec - */ -static void -try_reconnect (struct GNUNET_FS_SearchContext *sc); - - -/** - * We check a result message from the service. - * - * @param cls closure - * @param cm result message received - */ -static int -check_result (void *cls, - const struct ClientPutMessage *cm) -{ - /* payload of any variable size is OK */ - return GNUNET_OK; -} - - -/** - * We process a search result from the service. - * - * @param cls closure - * @param cm result message received - */ -static void -handle_result (void *cls, - const struct ClientPutMessage *cm) -{ - struct GNUNET_FS_SearchContext *sc = cls; - uint16_t msize = ntohs (cm->header.size) - sizeof(*cm); - enum GNUNET_BLOCK_Type type = ntohl (cm->type); - - if (GNUNET_TIME_absolute_get_duration (GNUNET_TIME_absolute_ntoh ( - cm->expiration)).rel_value_us > 0) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Result received has already expired.\n"); - return; /* result expired */ - } - switch (type) - { - case GNUNET_BLOCK_TYPE_FS_UBLOCK: - if (GNUNET_FS_URI_SKS == sc->uri->type) - process_sblock (sc, - (const struct UBlock *) &cm[1], - msize); - else - process_kblock (sc, - (const struct UBlock *) &cm[1], - msize); - break; - - case GNUNET_BLOCK_TYPE_ANY: - GNUNET_break (0); - break; - - case GNUNET_BLOCK_TYPE_FS_DBLOCK: - GNUNET_break (0); - break; - - case GNUNET_BLOCK_TYPE_FS_ONDEMAND: - GNUNET_break (0); - break; - - case GNUNET_BLOCK_TYPE_FS_IBLOCK: - GNUNET_break (0); - break; - - default: - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Got result with unknown block type `%d', ignoring"), - type); - break; - } -} - - -/** - * Schedule the transmission of the (next) search request - * to the service. - * - * @param sc context for the search - */ -static void -schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc); - - -/** - * Closure for #build_result_set(). - */ -struct MessageBuilderContext -{ - /** - * How many entries can we store to xoff. - */ - unsigned int put_cnt; - - /** - * How many entries should we skip. - */ - unsigned int skip_cnt; - - /** - * Where to store the keys. - */ - struct GNUNET_HashCode *xoff; - - /** - * Search context we are iterating for. - */ - struct GNUNET_FS_SearchContext *sc; - - /** - * Keyword offset the search result must match (0 for SKS) - */ - unsigned int keyword_offset; -}; - - -/** - * Iterating over the known results, pick those matching the given - * result range and store their keys at 'xoff'. - * - * @param cls the `struct MessageBuilderContext` - * @param key key for a result - * @param value the search result - * @return #GNUNET_OK to continue iterating - */ -static int -build_result_set (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct MessageBuilderContext *mbc = cls; - struct GNUNET_FS_SearchResult *sr = value; - - if ((NULL != sr->keyword_bitmap) && - (0 == (sr->keyword_bitmap[mbc->keyword_offset / 8] & (1 - << (mbc-> - keyword_offset - % 8))))) - return GNUNET_OK; /* have no match for this keyword yet */ - if (mbc->skip_cnt > 0) - { - mbc->skip_cnt--; - return GNUNET_OK; - } - if (0 == mbc->put_cnt) - return GNUNET_SYSERR; - mbc->xoff[--mbc->put_cnt] = *key; - - return GNUNET_OK; -} - - -/** - * Iterating over the known results, count those matching the given - * result range and increment put count for each. - * - * @param cls the `struct MessageBuilderContext` - * @param key key for a result - * @param value the search result - * @return #GNUNET_OK to continue iterating - */ -static int -find_result_set (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct MessageBuilderContext *mbc = cls; - struct GNUNET_FS_SearchResult *sr = value; - - if ((NULL != sr->keyword_bitmap) && - (0 == (sr->keyword_bitmap[mbc->keyword_offset / 8] & (1 - << (mbc-> - keyword_offset - % 8))))) - return GNUNET_OK; /* have no match for this keyword yet */ - mbc->put_cnt++; - return GNUNET_OK; -} - - -/** - * Schedule the transmission of the (next) search request - * to the service. - * - * @param sc context for the search - */ -static void -schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc) -{ - struct MessageBuilderContext mbc; - struct GNUNET_MQ_Envelope *env; - struct SearchMessage *sm; - struct GNUNET_CRYPTO_EcdsaPublicKey dpub; - unsigned int total_seen_results; /* total number of result hashes to send */ - uint32_t options; - unsigned int left; - unsigned int todo; - unsigned int fit; - unsigned int search_request_map_offset; - unsigned int keyword_offset; - int first_call; - - memset (&mbc, 0, sizeof(mbc)); - mbc.sc = sc; - if (GNUNET_FS_uri_test_ksk (sc->uri)) - { - /* This will calculate the result set size ONLY for - "keyword_offset == 0", so we will have to recalculate - it for the other keywords later! */ - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &find_result_set, - &mbc); - total_seen_results = mbc.put_cnt; - } - else - { - total_seen_results - = GNUNET_CONTAINER_multihashmap_size (sc->master_result_map); - } - search_request_map_offset = 0; - keyword_offset = 0; - first_call = GNUNET_YES; - while ((0 != (left = - (total_seen_results - search_request_map_offset))) || - (GNUNET_YES == first_call)) - { - first_call = GNUNET_NO; - options = SEARCH_MESSAGE_OPTION_NONE; - if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY)) - options |= SEARCH_MESSAGE_OPTION_LOOPBACK_ONLY; - - fit = (GNUNET_MAX_MESSAGE_SIZE - 1 - sizeof(*sm)) / sizeof(struct - GNUNET_HashCode); - todo = GNUNET_MIN (fit, - left); - env = GNUNET_MQ_msg_extra (sm, - sizeof(struct GNUNET_HashCode) * todo, - GNUNET_MESSAGE_TYPE_FS_START_SEARCH); - mbc.skip_cnt = search_request_map_offset; - mbc.xoff = (struct GNUNET_HashCode *) &sm[1]; - sm->type = htonl (GNUNET_BLOCK_TYPE_FS_UBLOCK); - sm->anonymity_level = htonl (sc->anonymity); - memset (&sm->target, - 0, - sizeof(struct GNUNET_PeerIdentity)); - - if (GNUNET_FS_uri_test_ksk (sc->uri)) - { - mbc.keyword_offset = keyword_offset; - /* calculate how many results we can send in this message */ - mbc.put_cnt = todo; - /* now build message */ - sm->query = sc->requests[keyword_offset].uquery; - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &build_result_set, - &mbc); - search_request_map_offset += todo; - GNUNET_assert (0 == mbc.put_cnt); - GNUNET_assert (total_seen_results >= search_request_map_offset); - if (total_seen_results != search_request_map_offset) - { - /* more requesting to be done... */ - sm->options = htonl (options | SEARCH_MESSAGE_OPTION_CONTINUED); - } - else - { - sm->options = htonl (options); - keyword_offset++; - if (sc->uri->data.ksk.keywordCount != keyword_offset) - { - /* more keywords => more requesting to be done... */ - first_call = GNUNET_YES; - search_request_map_offset = 0; - mbc.put_cnt = 0; - mbc.keyword_offset = keyword_offset; - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &find_result_set, - &mbc); - total_seen_results = mbc.put_cnt; - } - } - } - else - { - GNUNET_assert (GNUNET_FS_uri_test_sks (sc->uri)); - - GNUNET_CRYPTO_ecdsa_public_key_derive (&sc->uri->data.sks.ns, - sc->uri->data.sks.identifier, - "fs-ublock", - &dpub); - GNUNET_CRYPTO_hash (&dpub, - sizeof(dpub), - &sm->query); - mbc.put_cnt = todo; - mbc.keyword_offset = 0; - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &build_result_set, - &mbc); - GNUNET_assert (total_seen_results >= search_request_map_offset); - if (total_seen_results != search_request_map_offset) - { - /* more requesting to be done... */ - sm->options = htonl (options | SEARCH_MESSAGE_OPTION_CONTINUED); - } - else - { - sm->options = htonl (options); - } - } - GNUNET_MQ_send (sc->mq, - env); - } -} - - -/** - * Generic error handler, called with the appropriate error code and - * the same closure specified at the creation of the message queue. - * Not every message queue implementation supports an error handler. - * - * @param cls closure with the `struct GNUNET_FS_SearchContext *` - * @param error error code - */ -static void -search_mq_error_handler (void *cls, - enum GNUNET_MQ_Error error) -{ - struct GNUNET_FS_SearchContext *sc = cls; - - if (NULL != sc->mq) - { - GNUNET_MQ_destroy (sc->mq); - sc->mq = NULL; - } - try_reconnect (sc); -} - - -/** - * Reconnect to the FS service and transmit - * our queries NOW. - * - * @param cls our search context - */ -static void -do_reconnect (void *cls) -{ - struct GNUNET_FS_SearchContext *sc = cls; - struct GNUNET_MQ_MessageHandler handlers[] = { - GNUNET_MQ_hd_var_size (result, - GNUNET_MESSAGE_TYPE_FS_PUT, - struct ClientPutMessage, - sc), - GNUNET_MQ_handler_end () - }; - - sc->task = NULL; - sc->mq = GNUNET_CLIENT_connect (sc->h->cfg, - "fs", - handlers, - &search_mq_error_handler, - sc); - if (NULL == sc->mq) - { - try_reconnect (sc); - return; - } - schedule_transmit_search_request (sc); -} - - -/** - * Shutdown any existing connection to the FS - * service and try to establish a fresh one - * (and then re-transmit our search request). - * - * @param sc the search to reconnec - */ -static void -try_reconnect (struct GNUNET_FS_SearchContext *sc) -{ - if (NULL != sc->mq) - { - GNUNET_MQ_destroy (sc->mq); - sc->mq = NULL; - } - sc->reconnect_backoff = GNUNET_TIME_STD_BACKOFF (sc->reconnect_backoff); - sc->task = - GNUNET_SCHEDULER_add_delayed (sc->reconnect_backoff, - &do_reconnect, - sc); -} - - -/** - * Start search for content, internal API. - * - * @param h handle to the file sharing subsystem - * @param uri specifies the search parameters; can be - * a KSK URI or an SKS URI. - * @param anonymity desired level of anonymity - * @param options options for the search - * @param cctx initial value for the client context - * @param psearch parent search result (for namespace update searches) - * @return context that can be used to control the search - */ -static struct GNUNET_FS_SearchContext * -search_start (struct GNUNET_FS_Handle *h, - const struct GNUNET_FS_Uri *uri, - uint32_t anonymity, - enum GNUNET_FS_SearchOptions options, - void *cctx, - struct GNUNET_FS_SearchResult *psearch) -{ - struct GNUNET_FS_SearchContext *sc; - struct GNUNET_FS_ProgressInfo pi; - - sc = GNUNET_new (struct GNUNET_FS_SearchContext); - sc->h = h; - sc->options = options; - sc->uri = GNUNET_FS_uri_dup (uri); - sc->anonymity = anonymity; - sc->start_time = GNUNET_TIME_absolute_get (); - if (NULL != psearch) - { - sc->psearch_result = psearch; - psearch->update_search = sc; - } - sc->master_result_map = GNUNET_CONTAINER_multihashmap_create (16, GNUNET_NO); - sc->client_info = cctx; - if (GNUNET_OK != GNUNET_FS_search_start_searching_ (sc)) - { - GNUNET_FS_uri_destroy (sc->uri); - GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map); - GNUNET_free (sc); - return NULL; - } - GNUNET_FS_search_sync_ (sc); - pi.status = GNUNET_FS_STATUS_SEARCH_START; - sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc); - return sc; -} - - -/** - * Update the 'results' map for the individual keywords with the - * results from the 'global' result set. - * - * @param cls closure, the `struct GNUNET_FS_SearchContext *` - * @param key current key code - * @param value value in the hash map, the `struct GNUNET_FS_SearchResult *` - * @return #GNUNET_YES (we should continue to iterate) - */ -static int -update_sre_result_maps (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct GNUNET_FS_SearchContext *sc = cls; - struct GNUNET_FS_SearchResult *sr = value; - - for (unsigned int i = 0; i < sc->uri->data.ksk.keywordCount; i++) - { - if (0 != (sr->keyword_bitmap[i / 8] & (1 << (i % 8)))) - GNUNET_break (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put (sc->requests[i].results, - &sr->key, - sr, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - } - return GNUNET_YES; -} - - -int -GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc) -{ - unsigned int i; - const char *keyword; - const struct GNUNET_CRYPTO_EcdsaPrivateKey *anon; - struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub; - struct SearchRequestEntry *sre; - - GNUNET_assert (NULL == sc->mq); - if (GNUNET_FS_uri_test_ksk (sc->uri)) - { - GNUNET_assert (0 != sc->uri->data.ksk.keywordCount); - anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous (); - GNUNET_CRYPTO_ecdsa_key_get_public (anon, &anon_pub); - sc->requests - = GNUNET_new_array (sc->uri->data.ksk.keywordCount, - struct SearchRequestEntry); - - for (i = 0; i < sc->uri->data.ksk.keywordCount; i++) - { - keyword = &sc->uri->data.ksk.keywords[i][1]; - sre = &sc->requests[i]; - sre->keyword = GNUNET_strdup (keyword); - GNUNET_CRYPTO_ecdsa_public_key_derive (&anon_pub, - keyword, - "fs-ublock", - &sre->dpub); - GNUNET_CRYPTO_hash (&sre->dpub, - sizeof(struct GNUNET_CRYPTO_EcdsaPublicKey), - &sre->uquery); - sre->mandatory = (sc->uri->data.ksk.keywords[i][0] == '+'); - if (sre->mandatory) - sc->mandatory_count++; - sre->results = GNUNET_CONTAINER_multihashmap_create (4, GNUNET_NO); - } - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &update_sre_result_maps, - sc); - } - GNUNET_assert (NULL == sc->task); - do_reconnect (sc); - if (NULL == sc->mq) - { - GNUNET_SCHEDULER_cancel (sc->task); - sc->task = NULL; - return GNUNET_SYSERR; - } - return GNUNET_OK; -} - - -/** - * Freeze probes for the given search result. - * - * @param cls the global FS handle - * @param key the key for the search result (unused) - * @param value the search result to free - * @return #GNUNET_OK - */ -static int -search_result_freeze_probes (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct GNUNET_FS_SearchResult *sr = value; - - if (NULL != sr->probe_ctx) - { - GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES); - sr->probe_ctx = NULL; - GNUNET_FS_stop_probe_ping_task_ (sr); - } - if (NULL != sr->probe_cancel_task) - { - GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); - sr->probe_cancel_task = NULL; - } - if (NULL != sr->update_search) - GNUNET_FS_search_pause (sr->update_search); - return GNUNET_OK; -} - - -/** - * Resume probes for the given search result. - * - * @param cls the global FS handle - * @param key the key for the search result (unused) - * @param value the search result to free - * @return #GNUNET_OK - */ -static int -search_result_resume_probes (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct GNUNET_FS_SearchResult *sr = value; - - GNUNET_FS_search_start_probe_ (sr); - if (NULL != sr->update_search) - GNUNET_FS_search_continue (sr->update_search); - return GNUNET_OK; -} - - -/** - * Signal suspend and free the given search result. - * - * @param cls the global FS handle - * @param key the key for the search result (unused) - * @param value the search result to free - * @return #GNUNET_OK - */ -static int -search_result_suspend (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct GNUNET_FS_SearchContext *sc = cls; - struct GNUNET_FS_SearchResult *sr = value; - struct GNUNET_FS_ProgressInfo pi; - - if (NULL != sr->download) - { - GNUNET_FS_download_signal_suspend_ (sr->download); - sr->download = NULL; - } - if (NULL != sr->update_search) - { - GNUNET_FS_search_signal_suspend_ (sr->update_search); - sr->update_search = NULL; - } - GNUNET_FS_search_stop_probe_ (sr); - if (0 == sr->mandatory_missing) - { - /* client is aware of search result, notify about suspension event */ - pi.status = GNUNET_FS_STATUS_SEARCH_RESULT_SUSPEND; - pi.value.search.specifics.result_suspend.cctx = sr->client_info; - pi.value.search.specifics.result_suspend.meta = sr->meta; - pi.value.search.specifics.result_suspend.uri = sr->uri; - sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc); - } - GNUNET_break (NULL == sr->client_info); - GNUNET_free (sr->serialization); - GNUNET_FS_uri_destroy (sr->uri); - GNUNET_FS_meta_data_destroy (sr->meta); - GNUNET_free (sr->keyword_bitmap); - GNUNET_free (sr); - return GNUNET_OK; -} - - -void -GNUNET_FS_search_signal_suspend_ (void *cls) -{ - struct GNUNET_FS_SearchContext *sc = cls; - struct GNUNET_FS_ProgressInfo pi; - unsigned int i; - - GNUNET_FS_end_top (sc->h, sc->top); - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &search_result_suspend, sc); - pi.status = GNUNET_FS_STATUS_SEARCH_SUSPEND; - sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc); - GNUNET_break (NULL == sc->client_info); - if (sc->task != NULL) - { - GNUNET_SCHEDULER_cancel (sc->task); - sc->task = NULL; - } - if (NULL != sc->mq) - { - GNUNET_MQ_destroy (sc->mq); - sc->mq = NULL; - } - GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map); - if (NULL != sc->requests) - { - GNUNET_assert (GNUNET_FS_uri_test_ksk (sc->uri)); - for (i = 0; i < sc->uri->data.ksk.keywordCount; i++) - { - GNUNET_CONTAINER_multihashmap_destroy (sc->requests[i].results); - GNUNET_free (sc->requests[i].keyword); - } - } - GNUNET_free (sc->requests); - GNUNET_free (sc->emsg); - GNUNET_FS_uri_destroy (sc->uri); - GNUNET_free (sc->serialization); - GNUNET_free (sc); -} - - -/** - * Start search for content. - * - * @param h handle to the file sharing subsystem - * @param uri specifies the search parameters; can be - * a KSK URI or an SKS URI. - * @param anonymity desired level of anonymity - * @param options options for the search - * @param cctx initial value for the client context - * @return context that can be used to control the search - */ -struct GNUNET_FS_SearchContext * -GNUNET_FS_search_start (struct GNUNET_FS_Handle *h, - const struct GNUNET_FS_Uri *uri, uint32_t anonymity, - enum GNUNET_FS_SearchOptions options, void *cctx) -{ - struct GNUNET_FS_SearchContext *ret; - - ret = search_start (h, uri, anonymity, options, cctx, NULL); - if (NULL == ret) - return NULL; - ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_search_signal_suspend_, ret); - return ret; -} - - -/** - * Pause search. - * - * @param sc context for the search that should be paused - */ -void -GNUNET_FS_search_pause (struct GNUNET_FS_SearchContext *sc) -{ - struct GNUNET_FS_ProgressInfo pi; - - if (NULL != sc->task) - { - GNUNET_SCHEDULER_cancel (sc->task); - sc->task = NULL; - } - if (NULL != sc->mq) - { - GNUNET_MQ_destroy (sc->mq); - sc->mq = NULL; - } - GNUNET_FS_search_sync_ (sc); - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &search_result_freeze_probes, - sc); - pi.status = GNUNET_FS_STATUS_SEARCH_PAUSED; - sc->client_info = GNUNET_FS_search_make_status_ (&pi, - sc->h, - sc); -} - - -/** - * Continue paused search. - * - * @param sc context for the search that should be resumed - */ -void -GNUNET_FS_search_continue (struct GNUNET_FS_SearchContext *sc) -{ - struct GNUNET_FS_ProgressInfo pi; - - GNUNET_assert (NULL == sc->mq); - GNUNET_assert (NULL == sc->task); - do_reconnect (sc); - GNUNET_FS_search_sync_ (sc); - pi.status = GNUNET_FS_STATUS_SEARCH_CONTINUED; - sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc); - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &search_result_resume_probes, sc); -} - - -/** - * Signal stop for the given search result. - * - * @param cls the global FS handle - * @param key the key for the search result (unused) - * @param value the search result to free - * @return #GNUNET_OK - */ -static int -search_result_stop (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct GNUNET_FS_SearchContext *sc = cls; - struct GNUNET_FS_SearchResult *sr = value; - struct GNUNET_FS_ProgressInfo pi; - - GNUNET_FS_search_stop_probe_ (sr); - if (NULL != sr->download) - { - sr->download->search = NULL; - sr->download->top - = GNUNET_FS_make_top (sr->download->h, - &GNUNET_FS_download_signal_suspend_, - sr->download); - if (NULL != sr->download->serialization) - { - GNUNET_FS_remove_sync_file_ (sc->h, - GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD, - sr->download->serialization); - GNUNET_free (sr->download->serialization); - sr->download->serialization = NULL; - } - pi.status = GNUNET_FS_STATUS_DOWNLOAD_LOST_PARENT; - GNUNET_FS_download_make_status_ (&pi, - sr->download); - GNUNET_FS_download_sync_ (sr->download); - sr->download = NULL; - } - if (0 != sr->mandatory_missing) - { - /* client is unaware of search result as - it does not match required keywords */ - GNUNET_break (NULL == sr->client_info); - return GNUNET_OK; - } - pi.status = GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED; - pi.value.search.specifics.result_stopped.cctx = sr->client_info; - pi.value.search.specifics.result_stopped.meta = sr->meta; - pi.value.search.specifics.result_stopped.uri = sr->uri; - sr->client_info = GNUNET_FS_search_make_status_ (&pi, sr->h, sc); - return GNUNET_OK; -} - - -/** - * Free the given search result. - * - * @param cls the global FS handle - * @param key the key for the search result (unused) - * @param value the search result to free - * @return #GNUNET_OK - */ -static int -search_result_free (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct GNUNET_FS_SearchResult *sr = value; - - if (NULL != sr->update_search) - { - GNUNET_FS_search_stop (sr->update_search); - GNUNET_assert (NULL == sr->update_search); - } - GNUNET_break (NULL == sr->probe_ctx); - GNUNET_break (NULL == sr->probe_cancel_task); - GNUNET_break (NULL == sr->client_info); - GNUNET_free (sr->serialization); - GNUNET_FS_uri_destroy (sr->uri); - GNUNET_FS_meta_data_destroy (sr->meta); - GNUNET_free (sr->keyword_bitmap); - GNUNET_free (sr); - return GNUNET_OK; -} - - -/** - * Stop search for content. - * - * @param sc context for the search that should be stopped - */ -void -GNUNET_FS_search_stop (struct GNUNET_FS_SearchContext *sc) -{ - struct GNUNET_FS_ProgressInfo pi; - unsigned int i; - - if (NULL != sc->top) - GNUNET_FS_end_top (sc->h, sc->top); - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &search_result_stop, - sc); - if (NULL != sc->psearch_result) - sc->psearch_result->update_search = NULL; - if (NULL != sc->serialization) - { - GNUNET_FS_remove_sync_file_ (sc->h, - (NULL != sc->psearch_result) - ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH - : GNUNET_FS_SYNC_PATH_MASTER_SEARCH, - sc->serialization); - GNUNET_FS_remove_sync_dir_ (sc->h, - (NULL != sc->psearch_result) - ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH - : GNUNET_FS_SYNC_PATH_MASTER_SEARCH, - sc->serialization); - GNUNET_free (sc->serialization); - } - pi.status = GNUNET_FS_STATUS_SEARCH_STOPPED; - sc->client_info = GNUNET_FS_search_make_status_ (&pi, - sc->h, - sc); - GNUNET_break (NULL == sc->client_info); - if (NULL != sc->task) - { - GNUNET_SCHEDULER_cancel (sc->task); - sc->task = NULL; - } - if (NULL != sc->mq) - { - GNUNET_MQ_destroy (sc->mq); - sc->mq = NULL; - } - GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, - &search_result_free, sc); - GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map); - if (NULL != sc->requests) - { - GNUNET_assert (GNUNET_FS_uri_test_ksk (sc->uri)); - for (i = 0; i < sc->uri->data.ksk.keywordCount; i++) - { - GNUNET_CONTAINER_multihashmap_destroy (sc->requests[i].results); - GNUNET_free (sc->requests[i].keyword); - } - } - GNUNET_free (sc->requests); - GNUNET_free (sc->emsg); - GNUNET_FS_uri_destroy (sc->uri); - GNUNET_free (sc); -} - - -/* end of fs_search.c */ diff --git a/src/fs/fs_sharetree.c b/src/fs/fs_sharetree.c deleted file mode 100644 index 6c246a3ad..000000000 --- a/src/fs/fs_sharetree.c +++ /dev/null @@ -1,457 +0,0 @@ -/* - This file is part of GNUnet - Copyright (C) 2005-2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_sharetree.c - * @brief code to manipulate the 'struct GNUNET_FS_ShareTreeItem' tree - * @author LRN - * @author Christian Grothoff - */ -#include "platform.h" - -#include "gnunet_fs_service.h" -#include "gnunet_scheduler_lib.h" -#include - - -/** - * Entry for each unique keyword to track how often - * it occurred. Contains the keyword and the counter. - */ -struct KeywordCounter -{ - /** - * This is a doubly-linked list - */ - struct KeywordCounter *prev; - - /** - * This is a doubly-linked list - */ - struct KeywordCounter *next; - - /** - * Keyword that was found. - */ - const char *value; - - /** - * How many files have this keyword? - */ - unsigned int count; -}; - - -/** - * Aggregate information we keep for meta data in each directory. - */ -struct MetaCounter -{ - /** - * This is a doubly-linked list - */ - struct MetaCounter *prev; - - /** - * This is a doubly-linked list - */ - struct MetaCounter *next; - - /** - * Name of the plugin that provided that piece of metadata - */ - const char *plugin_name; - - /** - * MIME-type of the metadata itself - */ - const char *data_mime_type; - - /** - * The actual meta data. - */ - const char *data; - - /** - * Number of bytes in 'data'. - */ - size_t data_size; - - /** - * Type of the data - */ - enum EXTRACTOR_MetaType type; - - /** - * Format of the data - */ - enum EXTRACTOR_MetaFormat format; - - /** - * How many files have meta entries matching this value? - * (type and format do not have to match). - */ - unsigned int count; -}; - - -/** - * A structure that forms a singly-linked list that serves as a stack - * for metadata-processing function. - */ -struct TrimContext -{ - /** - * Map from the hash over the keyword to an 'struct KeywordCounter *' - * counter that says how often this keyword was - * encountered in the current directory. - */ - struct GNUNET_CONTAINER_MultiHashMap *keywordcounter; - - /** - * Map from the hash over the metadata to an 'struct MetaCounter *' - * counter that says how often this metadata was - * encountered in the current directory. - */ - struct GNUNET_CONTAINER_MultiHashMap *metacounter; - - /** - * Position we are currently manipulating. - */ - struct GNUNET_FS_ShareTreeItem *pos; - - /** - * Number of times an item has to be found to be moved to the parent. - */ - unsigned int move_threshold; -}; - - -/** - * Add the given keyword to the keyword statistics tracker. - * - * @param cls the multihashmap we store the keyword counters in - * @param keyword the keyword to count - * @param is_mandatory ignored - * @return always GNUNET_OK - */ -static int -add_to_keyword_counter (void *cls, const char *keyword, int is_mandatory) -{ - struct GNUNET_CONTAINER_MultiHashMap *mcm = cls; - struct KeywordCounter *cnt; - struct GNUNET_HashCode hc; - size_t klen; - - klen = strlen (keyword) + 1; - GNUNET_CRYPTO_hash (keyword, klen - 1, &hc); - cnt = GNUNET_CONTAINER_multihashmap_get (mcm, &hc); - if (cnt == NULL) - { - cnt = GNUNET_malloc (sizeof(struct KeywordCounter) + klen); - cnt->value = (const char *) &cnt[1]; - GNUNET_memcpy (&cnt[1], keyword, klen); - GNUNET_assert (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put (mcm, - &hc, cnt, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - } - cnt->count++; - return GNUNET_OK; -} - - -/** - * Function called on each meta data item. Increments the - * respective counter. - * - * @param cls the container multihashmap to update - * @param plugin_name name of the plugin that produced this value; - * special values can be used (e.g. '<zlib>' for zlib being - * used in the main libextractor library and yielding - * meta data). - * @param type libextractor-type describing the meta data - * @param format basic format information about data - * @param data_mime_type mime-type of data (not of the original file); - * can be NULL (if mime-type is not known) - * @param data actual meta-data found - * @param data_len number of bytes in data - * @return 0 to continue extracting / iterating - */ -static int -add_to_meta_counter (void *cls, const char *plugin_name, - enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat - format, - const char *data_mime_type, const char *data, size_t - data_len) -{ - struct GNUNET_CONTAINER_MultiHashMap *map = cls; - struct GNUNET_HashCode key; - struct MetaCounter *cnt; - - GNUNET_CRYPTO_hash (data, data_len, &key); - cnt = GNUNET_CONTAINER_multihashmap_get (map, &key); - if (NULL == cnt) - { - cnt = GNUNET_new (struct MetaCounter); - cnt->data = data; - cnt->data_size = data_len; - cnt->plugin_name = plugin_name; - cnt->type = type; - cnt->format = format; - cnt->data_mime_type = data_mime_type; - GNUNET_assert (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put (map, - &key, cnt, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - } - cnt->count++; - return 0; -} - - -/** - * Remove keywords above the threshold. - * - * @param cls the 'struct TrimContext' with the pos to remove the keywords from - * @param keyword the keyword to check - * @param is_mandatory ignored - * @return always GNUNET_OK - */ -static int -remove_high_frequency_keywords (void *cls, const char *keyword, int - is_mandatory) -{ - struct TrimContext *tc = cls; - struct KeywordCounter *counter; - struct GNUNET_HashCode hc; - size_t klen; - - klen = strlen (keyword) + 1; - GNUNET_CRYPTO_hash (keyword, klen - 1, &hc); - counter = GNUNET_CONTAINER_multihashmap_get (tc->keywordcounter, &hc); - GNUNET_assert (NULL != counter); - if (counter->count < tc->move_threshold) - return GNUNET_OK; - GNUNET_FS_uri_ksk_remove_keyword (tc->pos->ksk_uri, - counter->value); - return GNUNET_OK; -} - - -/** - * Move "frequent" keywords over to the target ksk uri, free the - * counters. - * - * @param cls the 'struct TrimContext' - * @param key key of the entry - * @param value the 'struct KeywordCounter' - * @return GNUNET_YES (always) - */ -static int -migrate_and_drop_keywords (void *cls, const struct GNUNET_HashCode *key, - void *value) -{ - struct TrimContext *tc = cls; - struct KeywordCounter *counter = value; - - if (counter->count >= tc->move_threshold) - { - if (NULL == tc->pos->ksk_uri) - tc->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_args (1, - &counter->value); - else - GNUNET_FS_uri_ksk_add_keyword (tc->pos->ksk_uri, counter->value, - GNUNET_NO); - } - GNUNET_assert (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (tc->keywordcounter, - key, - counter)); - GNUNET_free (counter); - return GNUNET_YES; -} - - -/** - * Copy "frequent" metadata items over to the - * target metadata container, free the counters. - * - * @param cls the 'struct TrimContext' - * @param key key of the entry - * @param value the 'struct KeywordCounter' - * @return GNUNET_YES (always) - */ -static int -migrate_and_drop_metadata (void *cls, const struct GNUNET_HashCode *key, - void *value) -{ - struct TrimContext *tc = cls; - struct MetaCounter *counter = value; - - if (counter->count >= tc->move_threshold) - { - if (NULL == tc->pos->meta) - tc->pos->meta = GNUNET_FS_meta_data_create (); - GNUNET_FS_meta_data_insert (tc->pos->meta, - counter->plugin_name, - counter->type, - counter->format, - counter->data_mime_type, counter->data, - counter->data_size); - } - GNUNET_assert (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (tc->metacounter, - key, - counter)); - GNUNET_free (counter); - return GNUNET_YES; -} - - -/** - * Process a share item tree, moving frequent keywords up and - * copying frequent metadata up. - * - * @param tc trim context with hash maps to use - * @param tree tree to trim - */ -static void -share_tree_trim (struct TrimContext *tc, - struct GNUNET_FS_ShareTreeItem *tree) -{ - struct GNUNET_FS_ShareTreeItem *pos; - unsigned int num_children; - - /* first, trim all children */ - num_children = 0; - for (pos = tree->children_head; NULL != pos; pos = pos->next) - { - share_tree_trim (tc, pos); - num_children++; - } - - /* consider adding filename to directory meta data */ - if (tree->is_directory == GNUNET_YES) - { - const char *user = getenv ("USER"); - if ((user == NULL) || - (0 != strncasecmp (user, tree->short_filename, strlen (user)))) - { - /* only use filename if it doesn't match $USER */ - if (NULL == tree->meta) - tree->meta = GNUNET_FS_meta_data_create (); - GNUNET_FS_meta_data_insert (tree->meta, "", - EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, - EXTRACTOR_METAFORMAT_UTF8, - "text/plain", tree->short_filename, - strlen (tree->short_filename) + 1); - } - } - - if (1 >= num_children) - return; /* nothing to trim */ - - /* now, count keywords and meta data in children */ - for (pos = tree->children_head; NULL != pos; pos = pos->next) - { - if (NULL != pos->meta) - GNUNET_FS_meta_data_iterate (pos->meta, &add_to_meta_counter, - tc->metacounter); - if (NULL != pos->ksk_uri) - GNUNET_FS_uri_ksk_get_keywords (pos->ksk_uri, &add_to_keyword_counter, - tc->keywordcounter); - } - - /* calculate threshold for moving keywords / meta data */ - tc->move_threshold = 1 + (num_children / 2); - - /* remove high-frequency keywords from children */ - for (pos = tree->children_head; NULL != pos; pos = pos->next) - { - tc->pos = pos; - if (NULL != pos->ksk_uri) - { - struct GNUNET_FS_Uri *ksk_uri_copy = GNUNET_FS_uri_dup (pos->ksk_uri); - GNUNET_FS_uri_ksk_get_keywords (ksk_uri_copy, - &remove_high_frequency_keywords, tc); - GNUNET_FS_uri_destroy (ksk_uri_copy); - } - } - - /* add high-frequency meta data and keywords to parent */ - tc->pos = tree; - GNUNET_CONTAINER_multihashmap_iterate (tc->keywordcounter, - &migrate_and_drop_keywords, - tc); - GNUNET_CONTAINER_multihashmap_iterate (tc->metacounter, - &migrate_and_drop_metadata, - tc); -} - - -/** - * Process a share item tree, moving frequent keywords up and - * copying frequent metadata up. - * - * @param toplevel toplevel directory in the tree, returned by the scanner - */ -void -GNUNET_FS_share_tree_trim (struct GNUNET_FS_ShareTreeItem *toplevel) -{ - struct TrimContext tc; - - if (toplevel == NULL) - return; - tc.keywordcounter = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO); - tc.metacounter = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO); - share_tree_trim (&tc, toplevel); - GNUNET_CONTAINER_multihashmap_destroy (tc.keywordcounter); - GNUNET_CONTAINER_multihashmap_destroy (tc.metacounter); -} - - -/** - * Release memory of a share item tree. - * - * @param toplevel toplevel of the tree to be freed - */ -void -GNUNET_FS_share_tree_free (struct GNUNET_FS_ShareTreeItem *toplevel) -{ - struct GNUNET_FS_ShareTreeItem *pos; - - while (NULL != (pos = toplevel->children_head)) - GNUNET_FS_share_tree_free (pos); - if (NULL != toplevel->parent) - GNUNET_CONTAINER_DLL_remove (toplevel->parent->children_head, - toplevel->parent->children_tail, - toplevel); - if (NULL != toplevel->meta) - GNUNET_FS_meta_data_destroy (toplevel->meta); - if (NULL != toplevel->ksk_uri) - GNUNET_FS_uri_destroy (toplevel->ksk_uri); - GNUNET_free (toplevel->filename); - GNUNET_free (toplevel->short_filename); - GNUNET_free (toplevel); -} - - -/* end fs_sharetree.c */ diff --git a/src/fs/fs_test_lib.c b/src/fs/fs_test_lib.c deleted file mode 100644 index f80a2859c..000000000 --- a/src/fs/fs_test_lib.c +++ /dev/null @@ -1,630 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2010, 2011, 2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_test_lib.c - * @brief library routines for testing FS publishing and downloading; - * this code is limited to flat files - * and no keywords (those functions can be tested with - * single-peer setups; this is for testing routing). - * @author Christian Grothoff - */ -#include "platform.h" -#include "fs_api.h" -#include "fs_test_lib.h" - - -#define CONTENT_LIFETIME GNUNET_TIME_UNIT_HOURS - - -/** - * Handle for a publishing operation started for testing FS. - */ -struct TestPublishOperation -{ - /** - * Handle for the operation to connect to the peer's 'fs' service. - */ - struct GNUNET_TESTBED_Operation *fs_op; - - /** - * Handle to the file sharing context using this daemon. - */ - struct GNUNET_FS_Handle *fs; - - /** - * Function to call when upload is done. - */ - GNUNET_FS_TEST_UriContinuation publish_cont; - - /** - * Closure for publish_cont. - */ - void *publish_cont_cls; - - /** - * Task to abort publishing (timeout). - */ - struct GNUNET_SCHEDULER_Task *publish_timeout_task; - - /** - * Seed for file generation. - */ - uint32_t publish_seed; - - /** - * Context for current publishing operation. - */ - struct GNUNET_FS_PublishContext *publish_context; - - /** - * Result URI. - */ - struct GNUNET_FS_Uri *publish_uri; - - /** - * Name of the temporary file used, or NULL for none. - */ - char *publish_tmp_file; - - /** - * Size of the file. - */ - uint64_t size; - - /** - * Anonymity level used. - */ - uint32_t anonymity; - - /** - * Verbosity level of the current operation. - */ - unsigned int verbose; - - /** - * Are we testing indexing? (YES: index, NO: insert, SYSERR: simulate) - */ - int do_index; -}; - - -/** - * Handle for a download operation started for testing FS. - */ -struct TestDownloadOperation -{ - /** - * Handle for the operation to connect to the peer's 'fs' service. - */ - struct GNUNET_TESTBED_Operation *fs_op; - - /** - * Handle to the file sharing context using this daemon. - */ - struct GNUNET_FS_Handle *fs; - - /** - * Handle to the daemon via testing. - */ - struct GNUNET_TESTING_Daemon *daemon; - - /** - * Function to call when download is done. - */ - GNUNET_SCHEDULER_TaskCallback download_cont; - - /** - * Closure for download_cont. - */ - void *download_cont_cls; - - /** - * URI to download. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Task to abort downloading (timeout). - */ - struct GNUNET_SCHEDULER_Task *download_timeout_task; - - /** - * Context for current download operation. - */ - struct GNUNET_FS_DownloadContext *download_context; - - /** - * Size of the file. - */ - uint64_t size; - - /** - * Anonymity level used. - */ - uint32_t anonymity; - - /** - * Seed for download verification. - */ - uint32_t download_seed; - - /** - * Verbosity level of the current operation. - */ - unsigned int verbose; -}; - - -/** - * Task scheduled to report on the completion of our publish operation. - * - * @param cls the publish operation context - */ -static void -report_uri (void *cls) -{ - struct TestPublishOperation *po = cls; - - GNUNET_FS_publish_stop (po->publish_context); - GNUNET_TESTBED_operation_done (po->fs_op); - po->publish_cont (po->publish_cont_cls, - po->publish_uri, - (GNUNET_YES == po->do_index) - ? po->publish_tmp_file - : NULL); - GNUNET_FS_uri_destroy (po->publish_uri); - if ((GNUNET_YES != po->do_index) && - (NULL != po->publish_tmp_file)) - (void) GNUNET_DISK_directory_remove (po->publish_tmp_file); - GNUNET_free (po->publish_tmp_file); - GNUNET_free (po); -} - - -/** - * Task scheduled to run when publish operation times out. - * - * @param cls the publish operation context - */ -static void -publish_timeout (void *cls) -{ - struct TestPublishOperation *po = cls; - - po->publish_timeout_task = NULL; - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Timeout while trying to publish data\n"); - GNUNET_TESTBED_operation_done (po->fs_op); - GNUNET_FS_publish_stop (po->publish_context); - po->publish_cont (po->publish_cont_cls, NULL, NULL); - (void) GNUNET_DISK_directory_remove (po->publish_tmp_file); - GNUNET_free (po->publish_tmp_file); - GNUNET_free (po); -} - - -/** - * Progress callback for file-sharing events while publishing. - * - * @param cls the publish operation context - * @param info information about the event - */ -static void * -publish_progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *info) -{ - struct TestPublishOperation *po = cls; - - switch (info->status) - { - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - GNUNET_SCHEDULER_cancel (po->publish_timeout_task); - po->publish_timeout_task = NULL; - po->publish_uri = - GNUNET_FS_uri_dup (info->value.publish.specifics.completed.chk_uri); - GNUNET_SCHEDULER_add_now (&report_uri, - po); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - if (po->verbose) - GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Publishing at %llu/%llu bytes\n", - (unsigned long long) info->value.publish.completed, - (unsigned long long) info->value.publish.size); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - break; - - case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: - if (po->verbose) - GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Download at %llu/%llu bytes\n", - (unsigned long long) info->value.download.completed, - (unsigned long long) info->value.download.size); - break; - - default: - break; - } - return NULL; -} - - -/** - * Generate test data for publishing test. - * - * @param cls pointer to uint32_t with publishing seed - * @param offset offset to generate data for - * @param max maximum number of bytes to generate - * @param buf where to write generated data - * @param emsg where to store error message (unused) - * @return number of bytes written to buf - */ -static size_t -file_generator (void *cls, - uint64_t offset, - size_t max, - void *buf, - char **emsg) -{ - uint32_t *publish_seed = cls; - uint64_t pos; - uint8_t *cbuf = buf; - int mod; - - if (emsg != NULL) - *emsg = NULL; - if (buf == NULL) - return 0; - for (pos = 0; pos < 8; pos++) - cbuf[pos] = (uint8_t) (offset >> pos * 8); - for (pos = 8; pos < max; pos++) - { - mod = (255 - (offset / 1024 / 32)); - if (mod == 0) - mod = 1; - cbuf[pos] = (uint8_t) ((offset * (*publish_seed)) % mod); - } - return max; -} - - -/** - * Connect adapter for publishing operation. - * - * @param cls the 'struct TestPublishOperation' - * @param cfg configuration of the peer to connect to; will be available until - * GNUNET_TESTBED_operation_done() is called on the operation returned - * from GNUNET_TESTBED_service_connect() - * @return service handle to return in 'op_result', NULL on error - */ -static void * -publish_connect_adapter (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg) -{ - struct TestPublishOperation *po = cls; - - return GNUNET_FS_start (cfg, - "fs-test-publish", - &publish_progress_cb, po, - GNUNET_FS_FLAGS_NONE, - GNUNET_FS_OPTIONS_END); -} - - -/** - * Adapter function called to destroy connection to file-sharing service. - * - * @param cls the 'struct GNUNET_FS_Handle' - * @param op_result unused (different for publish/download!) - */ -static void -fs_disconnect_adapter (void *cls, - void *op_result) -{ - struct GNUNET_FS_Handle *fs = op_result; - - GNUNET_FS_stop (fs); -} - - -/** - * Callback to be called when testbed has connected to the fs service - * - * @param cls the 'struct TestPublishOperation' - * @param op the operation that has been finished - * @param ca_result the 'struct GNUNET_FS_Handle ' (NULL on error) - * @param emsg error message in case the operation has failed; will be NULL if - * operation has executed successfully. - */ -static void -publish_fs_connect_complete_cb (void *cls, - struct GNUNET_TESTBED_Operation *op, - void *ca_result, - const char *emsg) -{ - struct TestPublishOperation *po = cls; - struct GNUNET_FS_FileInformation *fi; - struct GNUNET_DISK_FileHandle *fh; - char *em; - uint64_t off; - char buf[DBLOCK_SIZE]; - size_t bsize; - struct GNUNET_FS_BlockOptions bo; - - if (NULL == ca_result) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Failed to connect to FS for publishing: %s\n", emsg); - po->publish_cont (po->publish_cont_cls, - NULL, NULL); - GNUNET_TESTBED_operation_done (po->fs_op); - GNUNET_free (po); - return; - } - po->fs = ca_result; - - bo.expiration_time = GNUNET_TIME_relative_to_absolute (CONTENT_LIFETIME); - bo.anonymity_level = po->anonymity; - bo.content_priority = 42; - bo.replication_level = 1; - if (GNUNET_YES == po->do_index) - { - po->publish_tmp_file = GNUNET_DISK_mktemp ("fs-test-publish-index"); - GNUNET_assert (po->publish_tmp_file != NULL); - fh = GNUNET_DISK_file_open (po->publish_tmp_file, - GNUNET_DISK_OPEN_WRITE - | GNUNET_DISK_OPEN_CREATE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE); - GNUNET_assert (NULL != fh); - off = 0; - while (off < po->size) - { - bsize = GNUNET_MIN (sizeof(buf), po->size - off); - emsg = NULL; - GNUNET_assert (bsize == file_generator (&po->publish_seed, off, bsize, - buf, &em)); - GNUNET_assert (em == NULL); - GNUNET_assert (bsize == GNUNET_DISK_file_write (fh, buf, bsize)); - off += bsize; - } - GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_close (fh)); - fi = GNUNET_FS_file_information_create_from_file (po->fs, po, - po->publish_tmp_file, - NULL, NULL, po->do_index, - &bo); - GNUNET_assert (NULL != fi); - } - else - { - fi = GNUNET_FS_file_information_create_from_reader (po->fs, po, - po->size, - &file_generator, - &po->publish_seed, - NULL, NULL, - po->do_index, &bo); - GNUNET_assert (NULL != fi); - } - po->publish_context = - GNUNET_FS_publish_start (po->fs, fi, NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); -} - - -void -GNUNET_FS_TEST_publish (struct GNUNET_TESTBED_Peer *peer, - struct GNUNET_TIME_Relative timeout, uint32_t anonymity, - int do_index, uint64_t size, uint32_t seed, - unsigned int verbose, - GNUNET_FS_TEST_UriContinuation cont, void *cont_cls) -{ - struct TestPublishOperation *po; - - po = GNUNET_new (struct TestPublishOperation); - po->publish_cont = cont; - po->publish_cont_cls = cont_cls; - po->publish_seed = seed; - po->anonymity = anonymity; - po->size = size; - po->verbose = verbose; - po->do_index = do_index; - po->fs_op = GNUNET_TESTBED_service_connect (po, - peer, - "fs", - &publish_fs_connect_complete_cb, - po, - &publish_connect_adapter, - &fs_disconnect_adapter, - po); - po->publish_timeout_task = - GNUNET_SCHEDULER_add_delayed (timeout, &publish_timeout, po); -} - - -/* ************************** download ************************ */ - - -/** - * Task scheduled to run when download operation times out. - * - * @param cls the download operation context - */ -static void -download_timeout (void *cls) -{ - struct TestDownloadOperation *dop = cls; - - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Timeout while trying to download file\n"); - dop->download_timeout_task = NULL; - GNUNET_FS_download_stop (dop->download_context, - GNUNET_YES); - GNUNET_SCHEDULER_add_now (dop->download_cont, - dop->download_cont_cls); - GNUNET_TESTBED_operation_done (dop->fs_op); - GNUNET_FS_uri_destroy (dop->uri); - GNUNET_free (dop); -} - - -/** - * Task scheduled to report on the completion of our download operation. - * - * @param cls the download operation context - */ -static void -report_success (void *cls) -{ - struct TestDownloadOperation *dop = cls; - - GNUNET_FS_download_stop (dop->download_context, - GNUNET_YES); - GNUNET_SCHEDULER_add_now (dop->download_cont, - dop->download_cont_cls); - GNUNET_TESTBED_operation_done (dop->fs_op); - GNUNET_FS_uri_destroy (dop->uri); - GNUNET_free (dop); -} - - -/** - * Progress callback for file-sharing events while downloading. - * - * @param cls the download operation context - * @param info information about the event - */ -static void * -download_progress_cb (void *cls, - const struct GNUNET_FS_ProgressInfo *info) -{ - struct TestDownloadOperation *dop = cls; - - switch (info->status) - { - case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: - if (dop->verbose) - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Download at %llu/%llu bytes\n", - (unsigned long long) info->value.download.completed, - (unsigned long long) info->value.download.size); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED: - GNUNET_SCHEDULER_cancel (dop->download_timeout_task); - dop->download_timeout_task = NULL; - GNUNET_SCHEDULER_add_now (&report_success, dop); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: - case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: - break; - - /* FIXME: monitor data correctness during download progress */ - /* FIXME: do performance reports given sufficient verbosity */ - /* FIXME: advance timeout task to "immediate" on error */ - default: - break; - } - return NULL; -} - - -/** - * Connect adapter for download operation. - * - * @param cls the 'struct TestDownloadOperation' - * @param cfg configuration of the peer to connect to; will be available until - * GNUNET_TESTBED_operation_done() is called on the operation returned - * from GNUNET_TESTBED_service_connect() - * @return service handle to return in 'op_result', NULL on error - */ -static void * -download_connect_adapter (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg) -{ - struct TestPublishOperation *po = cls; - - return GNUNET_FS_start (cfg, - "fs-test-download", - &download_progress_cb, po, - GNUNET_FS_FLAGS_NONE, - GNUNET_FS_OPTIONS_END); -} - - -/** - * Callback to be called when testbed has connected to the fs service - * - * @param cls the 'struct TestPublishOperation' - * @param op the operation that has been finished - * @param ca_result the 'struct GNUNET_FS_Handle ' (NULL on error) - * @param emsg error message in case the operation has failed; will be NULL if - * operation has executed successfully. - */ -static void -download_fs_connect_complete_cb (void *cls, - struct GNUNET_TESTBED_Operation *op, - void *ca_result, - const char *emsg) -{ - struct TestDownloadOperation *dop = cls; - - dop->fs = ca_result; - GNUNET_assert (NULL != dop->fs); - dop->download_context = - GNUNET_FS_download_start (dop->fs, dop->uri, NULL, NULL, NULL, 0, dop->size, - dop->anonymity, GNUNET_FS_DOWNLOAD_OPTION_NONE, - NULL, NULL); -} - - -void -GNUNET_FS_TEST_download (struct GNUNET_TESTBED_Peer *peer, - struct GNUNET_TIME_Relative timeout, - uint32_t anonymity, uint32_t seed, - const struct GNUNET_FS_Uri *uri, unsigned int verbose, - GNUNET_SCHEDULER_TaskCallback cont, void *cont_cls) -{ - struct TestDownloadOperation *dop; - - dop = GNUNET_new (struct TestDownloadOperation); - dop->uri = GNUNET_FS_uri_dup (uri); - dop->size = GNUNET_FS_uri_chk_get_file_size (uri); - dop->verbose = verbose; - dop->anonymity = anonymity; - dop->download_cont = cont; - dop->download_cont_cls = cont_cls; - dop->download_seed = seed; - - dop->fs_op = GNUNET_TESTBED_service_connect (dop, - peer, - "fs", - &download_fs_connect_complete_cb, - dop, - &download_connect_adapter, - &fs_disconnect_adapter, - dop); - dop->download_timeout_task = - GNUNET_SCHEDULER_add_delayed (timeout, &download_timeout, dop); -} - - -/* end of fs_test_lib.c */ diff --git a/src/fs/fs_test_lib.h b/src/fs/fs_test_lib.h deleted file mode 100644 index 36244eb19..000000000 --- a/src/fs/fs_test_lib.h +++ /dev/null @@ -1,102 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2010, 2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_test_lib.h - * @brief library routines for testing FS publishing and downloading; - * this code is limited to flat files - * and no keywords (those functions can be tested with - * single-peer setups; this is for testing routing). - * @author Christian Grothoff - */ -#ifndef FS_TEST_LIB_H -#define FS_TEST_LIB_H - -#include "gnunet_util_lib.h" -#include "gnunet_fs_service.h" -#include "gnunet_testbed_service.h" - - -/** - * Function signature. - * - * @param cls closure (user defined) - * @param uri a URI, NULL for errors - * @param fn name of the file on disk to be removed upon - * completion, or NULL for inserted files (also NULL on error) - */ -typedef void -(*GNUNET_FS_TEST_UriContinuation) (void *cls, - const struct GNUNET_FS_Uri *uri, - const char *fn); - - -/** - * Publish a file at the given daemon. - * - * @param peer where to publish - * @param timeout if this operation cannot be completed within the - * given period, call the continuation with an error code - * @param anonymity option for publication - * @param do_index #GNUNET_YES for index, #GNUNET_NO for insertion, - * #GNUNET_SYSERR for simulation - * @param size size of the file to publish - * @param seed seed to use for file generation - * @param verbose how verbose to be in reporting - * @param cont function to call when done - * @param cont_cls closure for @a cont - */ -void -GNUNET_FS_TEST_publish (struct GNUNET_TESTBED_Peer *peer, - struct GNUNET_TIME_Relative timeout, - uint32_t anonymity, - int do_index, - uint64_t size, - uint32_t seed, - unsigned int verbose, - GNUNET_FS_TEST_UriContinuation cont, - void *cont_cls); - - -/** - * Perform test download. - * - * @param peer which peer to download from - * @param timeout if this operation cannot be completed within the - * given period, call the continuation with an error code - * @param anonymity option for download - * @param seed used for file validation - * @param uri URI of file to download (CHK/LOC only) - * @param verbose how verbose to be in reporting - * @param cont function to call when done - * @param cont_cls closure for @a cont - */ -void -GNUNET_FS_TEST_download (struct GNUNET_TESTBED_Peer *peer, - struct GNUNET_TIME_Relative timeout, - uint32_t anonymity, - uint32_t seed, - const struct GNUNET_FS_Uri *uri, - unsigned int verbose, - GNUNET_SCHEDULER_TaskCallback cont, - void *cont_cls); - - -#endif diff --git a/src/fs/fs_test_lib_data.conf b/src/fs/fs_test_lib_data.conf deleted file mode 100644 index c99674798..000000000 --- a/src/fs/fs_test_lib_data.conf +++ /dev/null @@ -1,17 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-fs-test-lib/ - -[ats] -WAN_QUOTA_IN = 3932160 -WAN_QUOTA_OUT = 3932160 - -[datastore] -QUOTA = 2 GB -#PLUGIN = heap -# -[fs] -IMMEDIATE_START = YES - -[testbed] -OVERLAY_TOPOLOGY = CLIQUE diff --git a/src/fs/fs_tree.c b/src/fs/fs_tree.c deleted file mode 100644 index 65f589966..000000000 --- a/src/fs/fs_tree.c +++ /dev/null @@ -1,452 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009-2011 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/fs_tree.c - * @brief Merkle-tree-ish-CHK file encoding for GNUnet - * @see http://gnunet.org/encoding.php3 - * @author Krista Bennett - * @author Christian Grothoff - */ -#include "platform.h" -#include "fs_tree.h" - - -/** - * Context for an ECRS-based file encoder that computes - * the Merkle-ish-CHK tree. - */ -struct GNUNET_FS_TreeEncoder -{ - /** - * Global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Closure for all callbacks. - */ - void *cls; - - /** - * Function to call on encrypted blocks. - */ - GNUNET_FS_TreeBlockProcessor proc; - - /** - * Function to call with progress information. - */ - GNUNET_FS_TreeProgressCallback progress; - - /** - * Function to call to receive input data. - */ - GNUNET_FS_DataReader reader; - - /** - * Function to call once we're done with processing. - */ - GNUNET_SCHEDULER_TaskCallback cont; - - /** - * Set to an error message (if we had an error). - */ - char *emsg; - - /** - * Set to the URI (upon successful completion) - */ - struct GNUNET_FS_Uri *uri; - - /** - * Overall file size. - */ - uint64_t size; - - /** - * How far are we? - */ - uint64_t publish_offset; - - /** - * How deep are we? Depth 0 is for the DBLOCKs. - */ - unsigned int current_depth; - - /** - * How deep is the tree? Always > 0. - */ - unsigned int chk_tree_depth; - - /** - * In-memory cache of the current CHK tree. - * This struct will contain the CHK values - * from the root to the currently processed - * node in the tree as identified by - * "current_depth" and "publish_offset". - * The "chktree" will be initially NULL, - * then allocated to a sufficient number of - * entries for the size of the file and - * finally freed once the upload is complete. - */ - struct ContentHashKey *chk_tree; - - /** - * Are we currently in 'GNUNET_FS_tree_encoder_next'? - * Flag used to prevent recursion. - */ - int in_next; -}; - - -/** - * Compute the depth of the CHK tree. - * - * @param flen file length for which to compute the depth - * @return depth of the tree, always > 0. A depth of 1 means only a DBLOCK. - */ -unsigned int -GNUNET_FS_compute_depth (uint64_t flen) -{ - unsigned int treeDepth; - uint64_t fl; - - treeDepth = 1; - fl = DBLOCK_SIZE; - while (fl < flen) - { - treeDepth++; - if (fl * CHK_PER_INODE < fl) - { - /* integer overflow, this is a HUGE file... */ - return treeDepth; - } - fl = fl * CHK_PER_INODE; - } - return treeDepth; -} - - -/** - * Calculate how many bytes of payload a block tree of the given - * depth MAY correspond to at most (this function ignores the fact that - * some blocks will only be present partially due to the total file - * size cutting some blocks off at the end). - * - * @param depth depth of the block. depth==0 is a DBLOCK. - * @return number of bytes of payload a subtree of this depth may correspond to - */ -uint64_t -GNUNET_FS_tree_compute_tree_size (unsigned int depth) -{ - uint64_t rsize; - unsigned int i; - - rsize = DBLOCK_SIZE; - for (i = 0; i < depth; i++) - rsize *= CHK_PER_INODE; - return rsize; -} - - -/** - * Compute the size of the current IBLOCK. The encoder is - * triggering the calculation of the size of an IBLOCK at the - * *end* (hence end_offset) of its construction. The IBLOCK - * maybe a full or a partial IBLOCK, and this function is to - * calculate how long it should be. - * - * @param depth depth of the IBlock in the tree, 0 would be a DBLOCK, - * must be > 0 (this function is for IBLOCKs only!) - * @param end_offset current offset in the payload (!) of the overall file, - * must be > 0 (since this function is called at the - * end of a block). - * @return size of the corresponding IBlock - */ -static uint16_t -GNUNET_FS_tree_compute_iblock_size (unsigned int depth, uint64_t end_offset) -{ - unsigned int ret; - uint64_t mod; - uint64_t bds; - - GNUNET_assert (depth > 0); - GNUNET_assert (end_offset > 0); - bds = GNUNET_FS_tree_compute_tree_size (depth); - mod = end_offset % bds; - if (0 == mod) - { - /* we were triggered at the end of a full block */ - ret = CHK_PER_INODE; - } - else - { - /* we were triggered at the end of the file */ - bds /= CHK_PER_INODE; - ret = mod / bds; - if (0 != mod % bds) - ret++; - } - return (uint16_t) (ret * sizeof(struct ContentHashKey)); -} - - -size_t -GNUNET_FS_tree_calculate_block_size (uint64_t fsize, uint64_t offset, - unsigned int depth) -{ - size_t ret; - uint64_t rsize; - uint64_t epos; - unsigned int chks; - - GNUNET_assert (fsize > 0); - GNUNET_assert (offset <= fsize); - if (depth == 0) - { - ret = DBLOCK_SIZE; - if ((offset + ret > fsize) || (offset + ret < offset)) - ret = (size_t) (fsize - offset); - return ret; - } - - rsize = GNUNET_FS_tree_compute_tree_size (depth - 1); - epos = offset + rsize * CHK_PER_INODE; - if ((epos < offset) || (epos > fsize)) - epos = fsize; - /* round up when computing #CHKs in our IBlock */ - chks = (epos - offset + rsize - 1) / rsize; - GNUNET_assert (chks <= CHK_PER_INODE); - return chks * sizeof(struct ContentHashKey); -} - - -/** - * Initialize a tree encoder. This function will call @a proc and - * "progress" on each block in the tree. Once all blocks have been - * processed, "cont" will be scheduled. The @a reader will be called - * to obtain the (plaintext) blocks for the file. Note that this - * function will not actually call @a proc. The client must - * call #GNUNET_FS_tree_encoder_next to trigger encryption (and - * calling of @a proc) for the each block. - * - * @param h the global FS context - * @param size overall size of the file to encode - * @param cls closure for reader, proc, progress and cont - * @param reader function to call to read plaintext data - * @param proc function to call on each encrypted block - * @param progress function to call with progress information - * @param cont function to call when done - */ -struct GNUNET_FS_TreeEncoder * -GNUNET_FS_tree_encoder_create (struct GNUNET_FS_Handle *h, uint64_t size, - void *cls, - GNUNET_FS_DataReader reader, - GNUNET_FS_TreeBlockProcessor proc, - GNUNET_FS_TreeProgressCallback progress, - GNUNET_SCHEDULER_TaskCallback cont) -{ - struct GNUNET_FS_TreeEncoder *te; - - te = GNUNET_new (struct GNUNET_FS_TreeEncoder); - te->h = h; - te->size = size; - te->cls = cls; - te->reader = reader; - te->proc = proc; - te->progress = progress; - te->cont = cont; - te->chk_tree_depth = GNUNET_FS_compute_depth (size); - te->chk_tree - = GNUNET_new_array (te->chk_tree_depth * CHK_PER_INODE, - struct ContentHashKey); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Created tree encoder for file with %llu bytes and depth %u\n", - (unsigned long long) size, - te->chk_tree_depth); - return te; -} - - -/** - * Compute the offset of the CHK for the - * current block in the IBlock above. - * - * @param depth depth of the IBlock in the tree (aka overall - * number of tree levels minus depth); 0 == DBlock - * @param end_offset current offset in the overall file, - * at the *beginning* of the block for DBLOCKs (depth==0), - * otherwise at the *end* of the block (exclusive) - * @return (array of CHKs') offset in the above IBlock - */ -static unsigned int -compute_chk_offset (unsigned int depth, uint64_t end_offset) -{ - uint64_t bds; - unsigned int ret; - - bds = GNUNET_FS_tree_compute_tree_size (depth); - if (depth > 0) - end_offset--; /* round down since for depth > 0 offset is at the END of the block */ - ret = end_offset / bds; - return ret % CHK_PER_INODE; -} - - -/** - * Encrypt the next block of the file (and call proc and progress - * accordingly; or of course "cont" if we have already completed - * encoding of the entire file). - * - * @param te tree encoder to use - */ -void -GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder *te) -{ - struct ContentHashKey *mychk; - const void *pt_block; - uint16_t pt_size; - char iob[DBLOCK_SIZE]; - char enc[DBLOCK_SIZE]; - struct GNUNET_CRYPTO_SymmetricSessionKey sk; - struct GNUNET_CRYPTO_SymmetricInitializationVector iv; - unsigned int off; - - GNUNET_assert (GNUNET_NO == te->in_next); - te->in_next = GNUNET_YES; - if (te->chk_tree_depth == te->current_depth) - { - off = CHK_PER_INODE * (te->chk_tree_depth - 1); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "TE done, reading CHK `%s' from %u\n", - GNUNET_h2s (&te->chk_tree[off].query), off); - te->uri = GNUNET_new (struct GNUNET_FS_Uri); - te->uri->type = GNUNET_FS_URI_CHK; - te->uri->data.chk.chk = te->chk_tree[off]; - te->uri->data.chk.file_length = GNUNET_htonll (te->size); - te->in_next = GNUNET_NO; - te->cont (te->cls); - return; - } - if (0 == te->current_depth) - { - /* read DBLOCK */ - pt_size = GNUNET_MIN (DBLOCK_SIZE, te->size - te->publish_offset); - if (pt_size != - te->reader (te->cls, te->publish_offset, pt_size, iob, &te->emsg)) - { - te->in_next = GNUNET_NO; - te->cont (te->cls); - return; - } - pt_block = iob; - } - else - { - pt_size = - GNUNET_FS_tree_compute_iblock_size (te->current_depth, - te->publish_offset); - pt_block = &te->chk_tree[(te->current_depth - 1) * CHK_PER_INODE]; - } - off = compute_chk_offset (te->current_depth, te->publish_offset); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "TE is at offset %llu and depth %u with block size %u and target-CHK-offset %u\n", - (unsigned long long) te->publish_offset, te->current_depth, - (unsigned int) pt_size, (unsigned int) off); - mychk = &te->chk_tree[te->current_depth * CHK_PER_INODE + off]; - GNUNET_CRYPTO_hash (pt_block, pt_size, &mychk->key); - GNUNET_CRYPTO_hash_to_aes_key (&mychk->key, &sk, &iv); - GNUNET_CRYPTO_symmetric_encrypt (pt_block, pt_size, &sk, &iv, enc); - GNUNET_CRYPTO_hash (enc, pt_size, &mychk->query); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "TE calculates query to be `%s', stored at %u\n", - GNUNET_h2s (&mychk->query), - te->current_depth * CHK_PER_INODE + off); - if (NULL != te->proc) - te->proc (te->cls, mychk, te->publish_offset, te->current_depth, - (0 == - te->current_depth) ? GNUNET_BLOCK_TYPE_FS_DBLOCK : - GNUNET_BLOCK_TYPE_FS_IBLOCK, enc, pt_size); - if (NULL != te->progress) - te->progress (te->cls, te->publish_offset, pt_block, pt_size, - te->current_depth); - if (0 == te->current_depth) - { - te->publish_offset += pt_size; - if ((te->publish_offset == te->size) || - (0 == te->publish_offset % (CHK_PER_INODE * DBLOCK_SIZE))) - te->current_depth++; - } - else - { - if ((off == CHK_PER_INODE) || (te->publish_offset == te->size)) - te->current_depth++; - else - te->current_depth = 0; - } - te->in_next = GNUNET_NO; -} - - -/** - * Get the resulting URI from the encoding. - * - * @param te the tree encoder to clean up - * @return uri set to the resulting URI (if encoding finished), NULL otherwise - */ -struct GNUNET_FS_Uri * -GNUNET_FS_tree_encoder_get_uri (struct GNUNET_FS_TreeEncoder *te) -{ - if (NULL != te->uri) - return GNUNET_FS_uri_dup (te->uri); - return NULL; -} - - -/** - * Clean up a tree encoder and return information - * about possible errors. - * - * @param te the tree encoder to clean up - * @param emsg set to an error message (if an error occurred - * within the tree encoder; if this function is called - * prior to completion and prior to an internal error, - * both "*emsg" will be set to NULL). - */ -void -GNUNET_FS_tree_encoder_finish (struct GNUNET_FS_TreeEncoder *te, - char **emsg) -{ - if (NULL != te->reader) - { - (void) te->reader (te->cls, UINT64_MAX, 0, 0, NULL); - te->reader = NULL; - } - GNUNET_assert (GNUNET_NO == te->in_next); - if (NULL != te->uri) - GNUNET_FS_uri_destroy (te->uri); - if (emsg != NULL) - *emsg = te->emsg; - else - GNUNET_free (te->emsg); - GNUNET_free (te->chk_tree); - GNUNET_free (te); -} - - -/* end of fs_tree.c */ diff --git a/src/fs/fs_tree.h b/src/fs/fs_tree.h deleted file mode 100644 index 1fb681d27..000000000 --- a/src/fs/fs_tree.h +++ /dev/null @@ -1,217 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_tree.h - * @brief Merkle-tree-ish-CHK file encoding for GNUnet - * @see https://gnunet.org/encoding - * @author Krista Bennett - * @author Christian Grothoff - * - * TODO: - * - decide if this API should be made public (gnunet_fs_service.h) - * or remain "internal" (but with exported symbols?) - */ -#ifndef GNUNET_FS_TREE_H -#define GNUNET_FS_TREE_H - -#include "fs_api.h" - -/** - * Compute the depth of the CHK tree. - * - * @param flen file length for which to compute the depth - * @return depth of the tree, always > 0. A depth of 1 means only a DBLOCK. - */ -unsigned int -GNUNET_FS_compute_depth (uint64_t flen); - - -/** - * Calculate how many bytes of payload a block tree of the given - * depth MAY correspond to at most (this function ignores the fact that - * some blocks will only be present partially due to the total file - * size cutting some blocks off at the end). - * - * @param depth depth of the block. depth==0 is a DBLOCK. - * @return number of bytes of payload a subtree of this depth may correspond to - */ -uint64_t -GNUNET_FS_tree_compute_tree_size (unsigned int depth); - - -/** - * Compute how many bytes of data should be stored in - * the specified block. - * - * @param fsize overall file size, must be > 0. - * @param offset offset in the original data corresponding - * to the beginning of the tree induced by the block; - * must be < fsize - * @param depth depth of the node in the tree, 0 for DBLOCK - * @return number of bytes stored in this node - */ -size_t -GNUNET_FS_tree_calculate_block_size (uint64_t fsize, uint64_t offset, - unsigned int depth); - - -/** - * Context for an ECRS-based file encoder that computes - * the Merkle-ish-CHK tree. - */ -struct GNUNET_FS_TreeEncoder; - - -/** - * Function called asking for the current (encoded) - * block to be processed. After processing the - * client should either call "GNUNET_FS_tree_encode_next" - * or (on error) "GNUNET_FS_tree_encode_finish". - * - * @param cls closure - * @param chk content hash key for the block - * @param offset offset of the block - * @param depth depth of the block, 0 for DBLOCKs - * @param type type of the block (IBLOCK or DBLOCK) - * @param block the (encrypted) block - * @param block_size size of block (in bytes) - */ -typedef void (*GNUNET_FS_TreeBlockProcessor) (void *cls, - const struct ContentHashKey *chk, - uint64_t offset, - unsigned int depth, - enum GNUNET_BLOCK_Type type, - const void *block, - uint16_t block_size); - - -/** - * Function called with information about our - * progress in computing the tree encoding. - * - * @param cls closure - * @param offset where are we in the file - * @param pt_block plaintext of the currently processed block - * @param pt_size size of pt_block - * @param depth depth of the block in the tree, 0 for DBLOCKS - */ -typedef void (*GNUNET_FS_TreeProgressCallback) (void *cls, uint64_t offset, - const void *pt_block, - size_t pt_size, - unsigned int depth); - - -/** - * Initialize a tree encoder. This function will call "proc" and - * "progress" on each block in the tree. Once all blocks have been - * processed, "cont" will be scheduled. The "reader" will be called - * to obtain the (plaintext) blocks for the file. Note that this - * function will actually never call "proc"; the "proc" function must - * be triggered by calling "GNUNET_FS_tree_encoder_next" to trigger - * encryption (and calling of "proc") for each block. - * - * @param h the global FS context - * @param size overall size of the file to encode - * @param cls closure for reader, proc, progress and cont - * @param reader function to call to read plaintext data - * @param proc function to call on each encrypted block - * @param progress function to call with progress information - * @param cont function to call when done - * @return tree encoder context - */ -struct GNUNET_FS_TreeEncoder * -GNUNET_FS_tree_encoder_create (struct GNUNET_FS_Handle *h, uint64_t size, - void *cls, GNUNET_FS_DataReader reader, - GNUNET_FS_TreeBlockProcessor proc, - GNUNET_FS_TreeProgressCallback progress, - GNUNET_SCHEDULER_TaskCallback cont); - - -/** - * Encrypt the next block of the file (and - * call proc and progress accordingly; or - * of course "cont" if we have already completed - * encoding of the entire file). - * - * @param te tree encoder to use - */ -void -GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder *te); - - -/** - * Get the resulting URI from the encoding. - * - * @param te the tree encoder to clean up - * @return uri set to the resulting URI (if encoding finished), NULL otherwise - */ -struct GNUNET_FS_Uri * -GNUNET_FS_tree_encoder_get_uri (struct GNUNET_FS_TreeEncoder *te); - - -/** - * Clean up a tree encoder and return information - * about possible errors. - * - * @param te the tree encoder to clean up - * @param emsg set to an error message (if an error occurred - * within the tree encoder; if this function is called - * prior to completion and prior to an internal error, - * both "*emsg" will be set to NULL). - */ -void -GNUNET_FS_tree_encoder_finish (struct GNUNET_FS_TreeEncoder *te, - char **emsg); - - -#if 0 -/* the functions below will be needed for persistence - but are not yet implemented -- FIXME... */ -/** - * Get data that would be needed to resume - * the encoding later. - * - * @param te encoding to resume - * @param data set to the resume data - * @param size set to the size of the resume data - */ -void -GNUNET_FS_tree_encoder_resume_get_data (const struct GNUNET_FS_TreeEncoder *te, - void **data, size_t *size); - - -/** - * Reset tree encoder to point previously - * obtained for resuming. - * - * @param te encoding to resume - * @param data the resume data - * @param size the size of the resume data - */ -void -GNUNET_FS_tree_encoder_resume (struct GNUNET_FS_TreeEncoder *te, - const void *data, size_t size); - -#endif - -#endif - -/* end of fs_tree.h */ diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c deleted file mode 100644 index 68ba667c4..000000000 --- a/src/fs/fs_unindex.c +++ /dev/null @@ -1,902 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2003--2013, 2016 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_unindex.c - * @author Krista Grothoff - * @author Christian Grothoff - * @brief Unindex file. - */ -#include "platform.h" -#include "gnunet_constants.h" - -#include "gnunet_fs_service.h" -#include "gnunet_protocols.h" -#include "fs_api.h" -#include "fs_tree.h" -#include "block_fs.h" -#include "fs_publish_ublock.h" - - -/** - * Function called by the tree encoder to obtain - * a block of plaintext data (for the lowest level - * of the tree). - * - * @param cls our publishing context - * @param offset identifies which block to get - * @param max (maximum) number of bytes to get; returning - * fewer will also cause errors - * @param buf where to copy the plaintext buffer - * @param emsg location to store an error message (on error) - * @return number of bytes copied to buf, 0 on error - */ -static size_t -unindex_reader (void *cls, - uint64_t offset, - size_t max, - void *buf, - char **emsg) -{ - struct GNUNET_FS_UnindexContext *uc = cls; - size_t pt_size; - - pt_size = GNUNET_MIN (max, uc->file_size - offset); - if (offset != GNUNET_DISK_file_seek (uc->fh, offset, GNUNET_DISK_SEEK_SET)) - { - *emsg = GNUNET_strdup (_ ("Failed to find given position in file")); - return 0; - } - if (pt_size != GNUNET_DISK_file_read (uc->fh, buf, pt_size)) - { - *emsg = GNUNET_strdup (_ ("Failed to read file")); - return 0; - } - return pt_size; -} - - -/** - * Fill in all of the generic fields for - * an unindex event and call the callback. - * - * @param pi structure to fill in - * @param uc overall unindex context - * @param offset where we are in the file (for progress) - */ -void -GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_UnindexContext *uc, - uint64_t offset) -{ - pi->value.unindex.uc = uc; - pi->value.unindex.cctx = uc->client_info; - pi->value.unindex.filename = uc->filename; - pi->value.unindex.size = uc->file_size; - pi->value.unindex.eta = - GNUNET_TIME_calculate_eta (uc->start_time, offset, uc->file_size); - pi->value.unindex.duration = - GNUNET_TIME_absolute_get_duration (uc->start_time); - pi->value.unindex.completed = offset; - pi->fsh = uc->h; - uc->client_info = uc->h->upcb (uc->h->upcb_cls, pi); -} - - -/** - * Function called with information about our - * progress in computing the tree encoding. - * - * @param cls closure - * @param offset where are we in the file - * @param pt_block plaintext of the currently processed block - * @param pt_size size of pt_block - * @param depth depth of the block in the tree, 0 for DBLOCK - */ -static void -unindex_progress (void *cls, - uint64_t offset, - const void *pt_block, - size_t pt_size, - unsigned int depth) -{ - struct GNUNET_FS_UnindexContext *uc = cls; - struct GNUNET_FS_ProgressInfo pi; - - pi.status = GNUNET_FS_STATUS_UNINDEX_PROGRESS; - pi.value.unindex.specifics.progress.data = pt_block; - pi.value.unindex.specifics.progress.offset = offset; - pi.value.unindex.specifics.progress.data_len = pt_size; - pi.value.unindex.specifics.progress.depth = depth; - GNUNET_FS_unindex_make_status_ (&pi, uc, offset); -} - - -/** - * We've encountered an error during - * unindexing. Signal the client. - * - * @param uc context for the failed unindexing operation - */ -static void -signal_unindex_error (struct GNUNET_FS_UnindexContext *uc) -{ - struct GNUNET_FS_ProgressInfo pi; - - pi.status = GNUNET_FS_STATUS_UNINDEX_ERROR; - pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL; - pi.value.unindex.specifics.error.message = uc->emsg; - GNUNET_FS_unindex_make_status_ (&pi, uc, 0); -} - - -/** - * Continuation called to notify client about result of the - * datastore removal operation. - * - * @param cls closure - * @param success #GNUNET_SYSERR on failure - * @param min_expiration minimum expiration time required for content to be stored - * @param msg NULL on success, otherwise an error message - */ -static void -process_cont (void *cls, - int success, - struct GNUNET_TIME_Absolute min_expiration, - const char *msg) -{ - struct GNUNET_FS_UnindexContext *uc = cls; - - if (success == GNUNET_SYSERR) - { - uc->emsg = GNUNET_strdup (msg); - signal_unindex_error (uc); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Datastore REMOVE operation succeeded\n"); - GNUNET_FS_tree_encoder_next (uc->tc); -} - - -/** - * Function called asking for the current (encoded) - * block to be processed. After processing the - * client should either call "GNUNET_FS_tree_encode_next" - * or (on error) "GNUNET_FS_tree_encode_finish". - * - * @param cls closure - * @param chk content hash key for the block (key for lookup in the datastore) - * @param offset offset of the block - * @param depth depth of the block, 0 for DBLOCK - * @param type type of the block (IBLOCK or DBLOCK) - * @param block the (encrypted) block - * @param block_size size of block (in bytes) - */ -static void -unindex_process (void *cls, - const struct ContentHashKey *chk, - uint64_t offset, - unsigned int depth, - enum GNUNET_BLOCK_Type type, - const void *block, - uint16_t block_size) -{ - struct GNUNET_FS_UnindexContext *uc = cls; - uint32_t size; - const void *data; - struct OnDemandBlock odb; - - if (type != GNUNET_BLOCK_TYPE_FS_DBLOCK) - { - size = block_size; - data = block; - } - else /* on-demand encoded DBLOCK */ - { - size = sizeof(struct OnDemandBlock); - odb.offset = GNUNET_htonll (offset); - odb.file_id = uc->file_id; - data = &odb; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Sending REMOVE request to DATASTORE service\n"); - GNUNET_DATASTORE_remove (uc->dsh, &chk->query, size, data, -2, 1, - &process_cont, uc); - uc->chk = *chk; -} - - -/** - * Function called with the response from the FS service to our - * unindexing request. - * - * @param cls closure, unindex context - * @param msg the response - */ -static void -handle_unindex_response (void *cls, - const struct GNUNET_MessageHeader *msg) -{ - struct GNUNET_FS_UnindexContext *uc = cls; - struct GNUNET_FS_ProgressInfo pi; - - if (NULL != uc->mq) - { - GNUNET_MQ_destroy (uc->mq); - uc->mq = NULL; - } - uc->state = UNINDEX_STATE_COMPLETE; - pi.status = GNUNET_FS_STATUS_UNINDEX_COMPLETED; - pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO; - GNUNET_FS_unindex_sync_ (uc); - GNUNET_FS_unindex_make_status_ (&pi, - uc, - uc->file_size); -} - - -/** - * Generic error handler, called with the appropriate error code and - * the same closure specified at the creation of the message queue. - * Not every message queue implementation supports an error handler. - * - * @param cls closure with the `struct GNUNET_FS_UnindexContext *` - * @param error error code - */ -static void -unindex_mq_error_handler (void *cls, - enum GNUNET_MQ_Error error) -{ - struct GNUNET_FS_UnindexContext *uc = cls; - - if (NULL != uc->mq) - { - GNUNET_MQ_destroy (uc->mq); - uc->mq = NULL; - } - uc->state = UNINDEX_STATE_ERROR; - uc->emsg = GNUNET_strdup (_ ("Error communicating with `fs' service.")); - GNUNET_FS_unindex_sync_ (uc); - signal_unindex_error (uc); -} - - -/** - * Function called when we are done with removing UBlocks. - * Disconnect from datastore and notify FS service about - * the unindex event. - * - * @param uc our unindexing context - */ -static void -unindex_finish (struct GNUNET_FS_UnindexContext *uc) -{ - struct GNUNET_MQ_MessageHandler handlers[] = { - GNUNET_MQ_hd_fixed_size (unindex_response, - GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK, - struct GNUNET_MessageHeader, - uc), - GNUNET_MQ_handler_end () - }; - char *emsg; - struct GNUNET_MQ_Envelope *env; - struct UnindexMessage *req; - - /* generate final progress message */ - unindex_progress (uc, - uc->file_size, - NULL, - 0, - 0); - GNUNET_FS_tree_encoder_finish (uc->tc, - &emsg); - uc->tc = NULL; - GNUNET_DISK_file_close (uc->fh); - uc->fh = NULL; - GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); - uc->dsh = NULL; - uc->state = UNINDEX_STATE_FS_NOTIFY; - GNUNET_FS_unindex_sync_ (uc); - uc->mq = GNUNET_CLIENT_connect (uc->h->cfg, - "fs", - handlers, - &unindex_mq_error_handler, - uc); - if (NULL == uc->mq) - { - uc->state = UNINDEX_STATE_ERROR; - uc->emsg = - GNUNET_strdup (_ ("Failed to connect to FS service for unindexing.")); - GNUNET_FS_unindex_sync_ (uc); - signal_unindex_error (uc); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Sending UNINDEX message to FS service\n"); - env = GNUNET_MQ_msg (req, - GNUNET_MESSAGE_TYPE_FS_UNINDEX); - req->reserved = 0; - req->file_id = uc->file_id; - GNUNET_MQ_send (uc->mq, - env); -} - - -/** - * Function called by the directory scanner as we extract keywords - * that we will need to remove UBlocks. - * - * @param cls the 'struct GNUNET_FS_UnindexContext *' - * @param filename which file we are making progress on - * @param is_directory #GNUNET_YES if this is a directory, - * #GNUNET_NO if this is a file - * #GNUNET_SYSERR if it is neither (or unknown) - * @param reason kind of progress we are making - */ -static void -unindex_directory_scan_cb (void *cls, - const char *filename, - int is_directory, - enum GNUNET_FS_DirScannerProgressUpdateReason reason) -{ - struct GNUNET_FS_UnindexContext *uc = cls; - static struct GNUNET_FS_ShareTreeItem *directory_scan_result; - - switch (reason) - { - case GNUNET_FS_DIRSCANNER_FINISHED: - directory_scan_result = GNUNET_FS_directory_scan_get_result (uc->dscan); - uc->dscan = NULL; - if (NULL != directory_scan_result->ksk_uri) - { - uc->ksk_uri = GNUNET_FS_uri_dup (directory_scan_result->ksk_uri); - uc->state = UNINDEX_STATE_DS_REMOVE_KBLOCKS; - GNUNET_FS_unindex_sync_ (uc); - GNUNET_FS_unindex_do_remove_kblocks_ (uc); - } - else - { - uc->emsg = GNUNET_strdup (_ ("Failed to get KSKs from directory scan.")); - GNUNET_FS_unindex_sync_ (uc); - unindex_finish (uc); - } - GNUNET_FS_share_tree_free (directory_scan_result); - break; - - case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR: - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Internal error scanning `%s'.\n"), - uc->filename); - GNUNET_FS_directory_scan_abort (uc->dscan); - uc->dscan = NULL; - uc->emsg = GNUNET_strdup (_ ("Failed to get KSKs from directory scan.")); - GNUNET_FS_unindex_sync_ (uc); - unindex_finish (uc); - break; - - default: - break; - } -} - - -/** - * If necessary, connect to the datastore and remove the UBlocks. - * - * @param uc context for the unindex operation. - */ -void -GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc) -{ - char *ex; - - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (uc->h->cfg, "FS", "EXTRACTORS", - &ex)) - ex = NULL; - uc->dscan = GNUNET_FS_directory_scan_start (uc->filename, - GNUNET_NO, ex, - &unindex_directory_scan_cb, - uc); - GNUNET_free (ex); -} - - -/** - * Continuation called to notify client about result of the remove - * operation for the UBlock. - * - * @param cls the 'struct GNUNET_FS_UnindexContext *' - * @param success GNUNET_SYSERR on failure (including timeout/queue drop) - * GNUNET_NO if content was already there - * GNUNET_YES (or other positive value) on success - * @param min_expiration minimum expiration time required for 0-priority content to be stored - * by the datacache at this time, zero for unknown, forever if we have no - * space for 0-priority content - * @param msg NULL on success, otherwise an error message - */ -static void -continue_after_remove (void *cls, - int32_t success, - struct GNUNET_TIME_Absolute min_expiration, - const char *msg) -{ - struct GNUNET_FS_UnindexContext *uc = cls; - - uc->dqe = NULL; - if (success != GNUNET_YES) - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to remove UBlock: %s\n"), - msg); - uc->ksk_offset++; - GNUNET_FS_unindex_do_remove_kblocks_ (uc); -} - - -/** - * Function called from datastore with result from us looking for - * a UBlock. There are four cases: - * 1) no result, means we move on to the next keyword - * 2) data hash is the same as an already seen data hash, means we move on to - * next keyword - * 3) UBlock for a different CHK, means we keep looking for more - * 4) UBlock is for our CHK, means we remove the block and then move - * on to the next keyword - * - * @param cls the 'struct GNUNET_FS_UnindexContext *' - * @param key key for the content - * @param size number of bytes in data - * @param data content stored - * @param type type of the content - * @param priority priority of the content - * @param anonymity anonymity-level for the content - * @param replication replication-level for the content - * @param expiration expiration time for the content - * @param uid unique identifier for the datum; - * maybe 0 if no unique identifier is available - */ -static void -process_kblock_for_unindex (void *cls, - const struct GNUNET_HashCode *key, - size_t size, - const void *data, - enum GNUNET_BLOCK_Type type, - uint32_t priority, - uint32_t anonymity, - uint32_t replication, - struct GNUNET_TIME_Absolute expiration, - uint64_t uid) -{ - struct GNUNET_FS_UnindexContext *uc = cls; - const struct UBlock *ub; - struct GNUNET_FS_Uri *chk_uri; - struct GNUNET_HashCode query; - - uc->dqe = NULL; - if (NULL == data) - { - /* no result */ - uc->ksk_offset++; - GNUNET_FS_unindex_do_remove_kblocks_ (uc); - return; - } - GNUNET_assert (GNUNET_BLOCK_TYPE_FS_UBLOCK == type); - if (size < sizeof(struct UBlock)) - { - GNUNET_break (0); - goto get_next; - } - ub = data; - GNUNET_CRYPTO_hash (&ub->verification_key, - sizeof(ub->verification_key), - &query); - if (0 != memcmp (&query, - key, - sizeof(struct GNUNET_HashCode))) - { - /* result does not match our keyword, skip */ - goto get_next; - } - { - char pt[size - sizeof(struct UBlock)]; - struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub; - const char *keyword; - - GNUNET_CRYPTO_ecdsa_key_get_public ( - GNUNET_CRYPTO_ecdsa_key_get_anonymous (), - &anon_pub); - keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1]; - GNUNET_FS_ublock_decrypt_ (&ub[1], size - sizeof(struct UBlock), - &anon_pub, - keyword, - pt); - if (NULL == memchr (&pt[1], 0, sizeof(pt) - 1)) - { - GNUNET_break_op (0); /* malformed UBlock */ - goto get_next; - } - chk_uri = GNUNET_FS_uri_parse (&pt[1], NULL); - if (NULL == chk_uri) - { - GNUNET_break_op (0); /* malformed UBlock */ - goto get_next; - } - } - if (0 != memcmp (&uc->chk, - &chk_uri->data.chk.chk, - sizeof(struct ContentHashKey))) - { - /* different CHK, ignore */ - GNUNET_FS_uri_destroy (chk_uri); - goto get_next; - } - GNUNET_FS_uri_destroy (chk_uri); - /* matches! */ - uc->dqe = GNUNET_DATASTORE_remove (uc->dsh, - key, - size, - data, - 0 /* priority */, - 1 /* queue size */, - &continue_after_remove, - uc); - return; -get_next: - uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh, - uid + 1 /* next_uid */, - false /* random */, - &uc->uquery, - GNUNET_BLOCK_TYPE_FS_UBLOCK, - 0 /* priority */, - 1 /* queue size */, - &process_kblock_for_unindex, - uc); -} - - -/** - * If necessary, connect to the datastore and remove the KBlocks. - * - * @param uc context for the unindex operation. - */ -void -GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc) -{ - const char *keyword; - const struct GNUNET_CRYPTO_EcdsaPrivateKey *anon; - struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub; - struct GNUNET_CRYPTO_EcdsaPublicKey dpub; - - if (NULL == uc->dsh) - uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg); - if (NULL == uc->dsh) - { - uc->state = UNINDEX_STATE_ERROR; - uc->emsg = GNUNET_strdup (_ ("Failed to connect to `datastore' service.")); - GNUNET_FS_unindex_sync_ (uc); - signal_unindex_error (uc); - return; - } - if ((NULL == uc->ksk_uri) || - (uc->ksk_offset >= uc->ksk_uri->data.ksk.keywordCount)) - { - unindex_finish (uc); - return; - } - anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous (); - GNUNET_CRYPTO_ecdsa_key_get_public (anon, - &anon_pub); - keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1]; - GNUNET_CRYPTO_ecdsa_public_key_derive (&anon_pub, - keyword, - "fs-ublock", - &dpub); - GNUNET_CRYPTO_hash (&dpub, - sizeof(dpub), - &uc->uquery); - uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh, - 0 /* next_uid */, - false /* random */, - &uc->uquery, - GNUNET_BLOCK_TYPE_FS_UBLOCK, - 0 /* priority */, - 1 /* queue size */, - &process_kblock_for_unindex, - uc); -} - - -/** - * Function called when the tree encoder has - * processed all blocks. Clean up. - * - * @param cls our unindexing context - */ -static void -unindex_extract_keywords (void *cls) -{ - struct GNUNET_FS_UnindexContext *uc = cls; - - uc->state = UNINDEX_STATE_EXTRACT_KEYWORDS; - GNUNET_FS_unindex_sync_ (uc); - GNUNET_FS_unindex_do_extract_keywords_ (uc); -} - - -/** - * Connect to the datastore and remove the blocks. - * - * @param uc context for the unindex operation. - */ -void -GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc) -{ - if (NULL == uc->dsh) - uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg); - if (NULL == uc->dsh) - { - uc->state = UNINDEX_STATE_ERROR; - uc->emsg = GNUNET_strdup (_ ("Failed to connect to `datastore' service.")); - GNUNET_FS_unindex_sync_ (uc); - signal_unindex_error (uc); - return; - } - uc->fh = - GNUNET_DISK_file_open (uc->filename, GNUNET_DISK_OPEN_READ, - GNUNET_DISK_PERM_NONE); - if (NULL == uc->fh) - { - GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); - uc->dsh = NULL; - uc->state = UNINDEX_STATE_ERROR; - uc->emsg = GNUNET_strdup (_ ("Failed to open file for unindexing.")); - GNUNET_FS_unindex_sync_ (uc); - signal_unindex_error (uc); - return; - } - uc->tc = - GNUNET_FS_tree_encoder_create (uc->h, - uc->file_size, - uc, - &unindex_reader, - &unindex_process, - &unindex_progress, - &unindex_extract_keywords); - GNUNET_FS_tree_encoder_next (uc->tc); -} - - -/** - * Function called once the hash of the file - * that is being unindexed has been computed. - * - * @param cls closure, unindex context - * @param file_id computed hash, NULL on error - */ -void -GNUNET_FS_unindex_process_hash_ (void *cls, - const struct GNUNET_HashCode *file_id) -{ - struct GNUNET_FS_UnindexContext *uc = cls; - - uc->fhc = NULL; - if (uc->state != UNINDEX_STATE_HASHING) - { - GNUNET_FS_unindex_stop (uc); - return; - } - if (file_id == NULL) - { - uc->state = UNINDEX_STATE_ERROR; - uc->emsg = GNUNET_strdup (_ ("Failed to compute hash of file.")); - GNUNET_FS_unindex_sync_ (uc); - signal_unindex_error (uc); - return; - } - uc->file_id = *file_id; - uc->state = UNINDEX_STATE_DS_REMOVE; - GNUNET_FS_unindex_sync_ (uc); - GNUNET_FS_unindex_do_remove_ (uc); -} - - -/** - * Create SUSPEND event for the given unindex operation - * and then clean up our state (without stop signal). - * - * @param cls the `struct GNUNET_FS_UnindexContext` to signal for - */ -void -GNUNET_FS_unindex_signal_suspend_ (void *cls) -{ - struct GNUNET_FS_UnindexContext *uc = cls; - struct GNUNET_FS_ProgressInfo pi; - - /* FIXME: lots of duplication with unindex_stop here! */ - if (uc->dscan != NULL) - { - GNUNET_FS_directory_scan_abort (uc->dscan); - uc->dscan = NULL; - } - if (NULL != uc->dqe) - { - GNUNET_DATASTORE_cancel (uc->dqe); - uc->dqe = NULL; - } - if (uc->fhc != NULL) - { - GNUNET_CRYPTO_hash_file_cancel (uc->fhc); - uc->fhc = NULL; - } - if (NULL != uc->ksk_uri) - { - GNUNET_FS_uri_destroy (uc->ksk_uri); - uc->ksk_uri = NULL; - } - if (NULL != uc->mq) - { - GNUNET_MQ_destroy (uc->mq); - uc->mq = NULL; - } - if (NULL != uc->dsh) - { - GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); - uc->dsh = NULL; - } - if (NULL != uc->tc) - { - GNUNET_FS_tree_encoder_finish (uc->tc, NULL); - uc->tc = NULL; - } - if (uc->fh != NULL) - { - GNUNET_DISK_file_close (uc->fh); - uc->fh = NULL; - } - GNUNET_FS_end_top (uc->h, uc->top); - pi.status = GNUNET_FS_STATUS_UNINDEX_SUSPEND; - GNUNET_FS_unindex_make_status_ (&pi, uc, - (uc->state == - UNINDEX_STATE_COMPLETE) ? uc->file_size : 0); - GNUNET_break (NULL == uc->client_info); - GNUNET_free (uc->filename); - GNUNET_free (uc->serialization); - GNUNET_free (uc->emsg); - GNUNET_free (uc); -} - - -/** - * Unindex a file. - * - * @param h handle to the file sharing subsystem - * @param filename file to unindex - * @param cctx initial value for the client context - * @return NULL on error, otherwise handle - */ -struct GNUNET_FS_UnindexContext * -GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h, - const char *filename, - void *cctx) -{ - struct GNUNET_FS_UnindexContext *uc; - struct GNUNET_FS_ProgressInfo pi; - uint64_t size; - - if (GNUNET_OK != - GNUNET_DISK_file_size (filename, - &size, - GNUNET_YES, - GNUNET_YES)) - return NULL; - uc = GNUNET_new (struct GNUNET_FS_UnindexContext); - uc->h = h; - uc->filename = GNUNET_strdup (filename); - uc->start_time = GNUNET_TIME_absolute_get (); - uc->file_size = size; - uc->client_info = cctx; - GNUNET_FS_unindex_sync_ (uc); - pi.status = GNUNET_FS_STATUS_UNINDEX_START; - pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL; - GNUNET_FS_unindex_make_status_ (&pi, uc, 0); - uc->fhc = - GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, - filename, - HASHING_BLOCKSIZE, - &GNUNET_FS_unindex_process_hash_, uc); - uc->top = GNUNET_FS_make_top (h, - &GNUNET_FS_unindex_signal_suspend_, - uc); - return uc; -} - - -/** - * Clean up after completion of an unindex operation. - * - * @param uc handle - */ -void -GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc) -{ - struct GNUNET_FS_ProgressInfo pi; - - if (NULL != uc->dscan) - { - GNUNET_FS_directory_scan_abort (uc->dscan); - uc->dscan = NULL; - } - if (NULL != uc->dqe) - { - GNUNET_DATASTORE_cancel (uc->dqe); - uc->dqe = NULL; - } - if (NULL != uc->fhc) - { - GNUNET_CRYPTO_hash_file_cancel (uc->fhc); - uc->fhc = NULL; - } - if (NULL != uc->mq) - { - GNUNET_MQ_destroy (uc->mq); - uc->mq = NULL; - } - if (NULL != uc->dsh) - { - GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); - uc->dsh = NULL; - } - if (NULL != uc->ksk_uri) - { - GNUNET_FS_uri_destroy (uc->ksk_uri); - uc->ksk_uri = NULL; - } - if (NULL != uc->tc) - { - GNUNET_FS_tree_encoder_finish (uc->tc, NULL); - uc->tc = NULL; - } - if (uc->fh != NULL) - { - GNUNET_DISK_file_close (uc->fh); - uc->fh = NULL; - } - GNUNET_FS_end_top (uc->h, uc->top); - if (uc->serialization != NULL) - { - GNUNET_FS_remove_sync_file_ (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, - uc->serialization); - GNUNET_free (uc->serialization); - uc->serialization = NULL; - } - pi.status = GNUNET_FS_STATUS_UNINDEX_STOPPED; - pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO; - GNUNET_FS_unindex_make_status_ (&pi, uc, - (uc->state == - UNINDEX_STATE_COMPLETE) ? uc->file_size : 0); - GNUNET_break (NULL == uc->client_info); - GNUNET_free (uc->emsg); - GNUNET_free (uc->filename); - GNUNET_free (uc); -} - - -/* end of fs_unindex.c */ diff --git a/src/fs/fs_uri.c b/src/fs/fs_uri.c deleted file mode 100644 index b0be0db4f..000000000 --- a/src/fs/fs_uri.c +++ /dev/null @@ -1,2045 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2003--2014 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/fs_uri.c - * @brief Parses and produces uri strings. - * @author Igor Wronsky, Christian Grothoff - * - * GNUnet URIs are of the general form "gnunet://MODULE/IDENTIFIER". - * The specific structure of "IDENTIFIER" depends on the module and - * maybe differentiated into additional subcategories if applicable. - * This module only deals with fs identifiers (MODULE = "fs"). - *

- * - * This module only parses URIs for the AFS module. The FS URIs fall - * into four categories, "chk", "sks", "ksk" and "loc". The first three - * categories were named in analogy (!) to Freenet, but they do NOT - * work in exactly the same way. They are very similar from the user's - * point of view (unique file identifier, subspace, keyword), but the - * implementation is rather different in pretty much every detail. - * The concrete URI formats are: - * - *

- * - * - * - * The encoding for hexadecimal values is defined in the hashing.c - * module in the gnunetutil library and discussed there. - * - */ -#include "platform.h" - -#include "gnunet_fs_service.h" -#include "gnunet_signatures.h" -#include "fs_api.h" -#include -#include -#include -#include -#include - - -int -GNUNET_FS_uri_to_key (const struct GNUNET_FS_Uri *uri, - struct GNUNET_HashCode *key) -{ - switch (uri->type) - { - case GNUNET_FS_URI_CHK: - *key = uri->data.chk.chk.query; - return GNUNET_OK; - - case GNUNET_FS_URI_SKS: - GNUNET_CRYPTO_hash (uri->data.sks.identifier, - strlen (uri->data.sks.identifier), - key); - return GNUNET_OK; - - case GNUNET_FS_URI_KSK: - if (uri->data.ksk.keywordCount > 0) - { - GNUNET_CRYPTO_hash (uri->data.ksk.keywords[0], - strlen (uri->data.ksk.keywords[0]), - key); - return GNUNET_OK; - } - else - { - memset (key, 0, sizeof(struct GNUNET_HashCode)); - return GNUNET_SYSERR; - } - break; - - case GNUNET_FS_URI_LOC: - GNUNET_CRYPTO_hash (&uri->data.loc.fi, - sizeof(struct FileIdentifier) - + sizeof(struct GNUNET_PeerIdentity), - key); - return GNUNET_OK; - - default: - memset (key, 0, sizeof(struct GNUNET_HashCode)); - return GNUNET_SYSERR; - } -} - - -/** - * Convert keyword URI to a human readable format - * (i.e. the search query that was used in the first place) - * - * @param uri ksk uri to convert to a string - * @return string with the keywords - */ -char * -GNUNET_FS_uri_ksk_to_string_fancy (const struct GNUNET_FS_Uri *uri) -{ - size_t n; - char *ret; - unsigned int i; - const char *keyword; - char **keywords; - unsigned int keywordCount; - - if ((NULL == uri) || (GNUNET_FS_URI_KSK != uri->type)) - { - GNUNET_break (0); - return NULL; - } - keywords = uri->data.ksk.keywords; - keywordCount = uri->data.ksk.keywordCount; - n = keywordCount + 1; - for (i = 0; i < keywordCount; i++) - { - keyword = keywords[i]; - n += strlen (keyword) - 1; - if (NULL != strstr (&keyword[1], " ")) - n += 2; - if (keyword[0] == '+') - n++; - } - ret = GNUNET_malloc (n); - strcpy (ret, ""); - for (i = 0; i < keywordCount; i++) - { - keyword = keywords[i]; - if (NULL != strstr (&keyword[1], " ")) - { - strcat (ret, "\""); - if (keyword[0] == '+') - strcat (ret, keyword); - else - strcat (ret, &keyword[1]); - strcat (ret, "\""); - } - else - { - if (keyword[0] == '+') - strcat (ret, keyword); - else - strcat (ret, &keyword[1]); - } - strcat (ret, " "); - } - return ret; -} - - -/** - * Given a keyword with %-encoding (and possibly quotes to protect - * spaces), return a copy of the keyword without %-encoding and - * without double-quotes (%22). Also, add a space at the beginning - * if there is not a '+'. - * - * @param in string with %-encoding - * @param emsg where to store the parser error message (if any) - * @return decoded string with leading space (or preserved plus) - */ -static char * -percent_decode_keyword (const char *in, char **emsg) -{ - char *out; - char *ret; - unsigned int rpos; - unsigned int wpos; - unsigned int hx; - - out = GNUNET_strdup (in); - rpos = 0; - wpos = 0; - while (out[rpos] != '\0') - { - if (out[rpos] == '%') - { - if (1 != sscanf (&out[rpos + 1], "%2X", &hx)) - { - GNUNET_free (out); - *emsg = GNUNET_strdup ( - _ ( /* xgettext:no-c-format */ - "Malformed KSK URI (`%' must be followed by HEX number)")); - return NULL; - } - rpos += 3; - if (hx == '"') - continue; /* skip double quote */ - out[wpos++] = (char) hx; - } - else - { - out[wpos++] = out[rpos++]; - } - } - out[wpos] = '\0'; - if (out[0] == '+') - { - ret = GNUNET_strdup (out); - } - else - { - /* need to prefix with space */ - ret = GNUNET_malloc (strlen (out) + 2); - strcpy (ret, " "); - strcat (ret, out); - } - GNUNET_free (out); - return ret; -} - - -#define GNUNET_FS_URI_KSK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_KSK_INFIX - -/** - * Parse a KSK URI. - * - * @param s an uri string - * @param emsg where to store the parser error message (if any) - * @return NULL on error, otherwise the KSK URI - */ -static struct GNUNET_FS_Uri * -uri_ksk_parse (const char *s, char **emsg) -{ - struct GNUNET_FS_Uri *ret; - char **keywords; - unsigned int pos; - int max; - int iret; - int i; - size_t slen; - char *dup; - int saw_quote; - - slen = strlen (s); - pos = strlen (GNUNET_FS_URI_KSK_PREFIX); - if ((slen <= pos) || (0 != strncmp (s, GNUNET_FS_URI_KSK_PREFIX, pos))) - return NULL; /* not KSK URI */ - if ((s[slen - 1] == '+') || (s[pos] == '+')) - { - *emsg = - GNUNET_strdup (_ ("Malformed KSK URI (must not begin or end with `+')")); - return NULL; - } - max = 1; - saw_quote = 0; - for (i = pos; i < slen; i++) - { - if ((s[i] == '%') && (&s[i] == strstr (&s[i], "%22"))) - { - saw_quote = (saw_quote + 1) % 2; - i += 3; - continue; - } - if ((s[i] == '+') && (saw_quote == 0)) - { - max++; - if (s[i - 1] == '+') - { - *emsg = GNUNET_strdup (_ ("Malformed KSK URI (`++' not allowed)")); - return NULL; - } - } - } - if (saw_quote == 1) - { - *emsg = GNUNET_strdup (_ ("Malformed KSK URI (quotes not balanced)")); - return NULL; - } - iret = max; - dup = GNUNET_strdup (s); - keywords = GNUNET_new_array (max, char *); - for (i = slen - 1; i >= (int) pos; i--) - { - if ((s[i] == '%') && (&s[i] == strstr (&s[i], "%22"))) - { - saw_quote = (saw_quote + 1) % 2; - continue; - } - if ((dup[i] == '+') && (saw_quote == 0)) - { - keywords[--max] = percent_decode_keyword (&dup[i + 1], emsg); - if (NULL == keywords[max]) - goto CLEANUP; - dup[i] = '\0'; - } - } - keywords[--max] = percent_decode_keyword (&dup[pos], emsg); - if (NULL == keywords[max]) - goto CLEANUP; - GNUNET_assert (0 == max); - GNUNET_free (dup); - ret = GNUNET_new (struct GNUNET_FS_Uri); - ret->type = GNUNET_FS_URI_KSK; - ret->data.ksk.keywordCount = iret; - ret->data.ksk.keywords = keywords; - return ret; - CLEANUP: - for (i = 0; i < max; i++) - GNUNET_free (keywords[i]); - GNUNET_free (keywords); - GNUNET_free (dup); - return NULL; -} - - -#define GNUNET_FS_URI_SKS_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_SKS_INFIX - -/** - * Parse an SKS URI. - * - * @param s an uri string - * @param emsg where to store the parser error message (if any) - * @return NULL on error, SKS URI otherwise - */ -static struct GNUNET_FS_Uri * -uri_sks_parse (const char *s, char **emsg) -{ - struct GNUNET_FS_Uri *ret; - struct GNUNET_CRYPTO_EcdsaPublicKey ns; - size_t pos; - char *end; - - pos = strlen (GNUNET_FS_URI_SKS_PREFIX); - if ((strlen (s) <= pos) || (0 != strncmp (s, GNUNET_FS_URI_SKS_PREFIX, pos))) - return NULL; /* not an SKS URI */ - end = strchr (&s[pos], '/'); - if ((NULL == end) || - (GNUNET_OK != GNUNET_STRINGS_string_to_data (&s[pos], - end - &s[pos], - &ns, - sizeof(ns)))) - { - *emsg = GNUNET_strdup (_ ("Malformed SKS URI (wrong syntax)")); - return NULL; /* malformed */ - } - end++; /* skip over '/' */ - ret = GNUNET_new (struct GNUNET_FS_Uri); - ret->type = GNUNET_FS_URI_SKS; - ret->data.sks.ns = ns; - ret->data.sks.identifier = GNUNET_strdup (end); - return ret; -} - - -#define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX - - -/** - * Parse a CHK URI. - * - * @param s an uri string - * @param emsg where to store the parser error message (if any) - * @return NULL on error, CHK URI otherwise - */ -static struct GNUNET_FS_Uri * -uri_chk_parse (const char *s, char **emsg) -{ - struct GNUNET_FS_Uri *ret; - struct FileIdentifier fi; - unsigned int pos; - unsigned long long flen; - size_t slen; - char h1[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)]; - char h2[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)]; - - slen = strlen (s); - pos = strlen (GNUNET_FS_URI_CHK_PREFIX); - if ((slen < pos + 2 * sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) + 1) || - (0 != strncmp (s, GNUNET_FS_URI_CHK_PREFIX, pos))) - return NULL; /* not a CHK URI */ - if ((s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] != '.') || - (s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) * 2 - 1] != '.')) - { - *emsg = GNUNET_strdup (_ ("Malformed CHK URI (wrong syntax)")); - return NULL; - } - GNUNET_memcpy (h1, &s[pos], sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)); - h1[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0'; - GNUNET_memcpy (h2, - &s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)], - sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)); - h2[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0'; - - if ((GNUNET_OK != GNUNET_CRYPTO_hash_from_string (h1, &fi.chk.key)) || - (GNUNET_OK != GNUNET_CRYPTO_hash_from_string (h2, &fi.chk.query)) || - (1 != - sscanf (&s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) * 2], - "%llu", - &flen))) - { - *emsg = GNUNET_strdup (_ ("Malformed CHK URI (failed to decode CHK)")); - return NULL; - } - fi.file_length = GNUNET_htonll (flen); - ret = GNUNET_new (struct GNUNET_FS_Uri); - ret->type = GNUNET_FS_URI_CHK; - ret->data.chk = fi; - return ret; -} - - -GNUNET_NETWORK_STRUCT_BEGIN -/** - * Structure that defines how the contents of a location URI must be - * assembled in memory to create or verify the signature of a location - * URI. - */ -struct LocUriAssembly -{ - /** - * What is being signed (rest of this struct). - */ - struct GNUNET_CRYPTO_EccSignaturePurpose purpose; - - /** - * Expiration time of the offer. - */ - struct GNUNET_TIME_AbsoluteNBO exptime; - - /** - * File being offered. - */ - struct FileIdentifier fi; - - /** - * Peer offering the file. - */ - struct GNUNET_PeerIdentity peer; -}; -GNUNET_NETWORK_STRUCT_END - - -#define GNUNET_FS_URI_LOC_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_LOC_INFIX - -#define SIGNATURE_ASCII_LENGTH 103 - -/** - * Parse a LOC URI. - * Also verifies validity of the location URI. - * - * @param s an uri string - * @param emsg where to store the parser error message (if any) - * @return NULL on error, valid LOC URI otherwise - */ -static struct GNUNET_FS_Uri * -uri_loc_parse (const char *s, char **emsg) -{ - struct GNUNET_FS_Uri *uri; - char h1[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)]; - char h2[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)]; - unsigned int pos; - unsigned int npos; - unsigned long long exptime; - unsigned long long flen; - struct GNUNET_TIME_Absolute et; - struct GNUNET_CRYPTO_EddsaSignature sig; - struct LocUriAssembly ass; - size_t slen; - - slen = strlen (s); - pos = strlen (GNUNET_FS_URI_LOC_PREFIX); - if ((slen < pos + 2 * sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) + 1) || - (0 != strncmp (s, GNUNET_FS_URI_LOC_PREFIX, pos))) - return NULL; /* not a LOC URI */ - if ((s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] != '.') || - (s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) * 2 - 1] != '.')) - { - *emsg = GNUNET_strdup (_ ("LOC URI malformed (wrong syntax)")); - return NULL; - } - GNUNET_memcpy (h1, &s[pos], sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)); - h1[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0'; - GNUNET_memcpy (h2, - &s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)], - sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)); - h2[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0'; - - if ((GNUNET_OK != GNUNET_CRYPTO_hash_from_string (h1, &ass.fi.chk.key)) || - (GNUNET_OK != GNUNET_CRYPTO_hash_from_string (h2, &ass.fi.chk.query)) || - (1 != - sscanf (&s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) * 2], - "%llu", - &flen))) - { - *emsg = GNUNET_strdup (_ ("LOC URI malformed (no CHK)")); - return NULL; - } - ass.fi.file_length = GNUNET_htonll (flen); - - npos = pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) * 2; - while ((s[npos] != '\0') && (s[npos] != '.')) - npos++; - if (s[npos] == '\0') - { - *emsg = GNUNET_strdup (_ ("LOC URI malformed (missing LOC)")); - goto ERR; - } - npos++; - if ((strlen (&s[npos]) <= GNUNET_CRYPTO_PKEY_ASCII_LENGTH + 1) || - ('.' != s[npos + GNUNET_CRYPTO_PKEY_ASCII_LENGTH])) - { - *emsg = - GNUNET_strdup (_ ("LOC URI malformed (wrong syntax for public key)")); - } - if ( - GNUNET_OK != - GNUNET_CRYPTO_eddsa_public_key_from_string (&s[npos], - GNUNET_CRYPTO_PKEY_ASCII_LENGTH, - &ass.peer.public_key)) - { - *emsg = - GNUNET_strdup (_ ("LOC URI malformed (could not decode public key)")); - goto ERR; - } - npos += GNUNET_CRYPTO_PKEY_ASCII_LENGTH; - if (s[npos++] != '.') - { - *emsg = GNUNET_strdup (_ ("LOC URI malformed (could not find signature)")); - goto ERR; - } - if ((strlen (&s[npos]) <= SIGNATURE_ASCII_LENGTH + 1) || - ('.' != s[npos + SIGNATURE_ASCII_LENGTH])) - { - *emsg = - GNUNET_strdup (_ ("LOC URI malformed (wrong syntax for signature)")); - goto ERR; - } - if (GNUNET_OK != - GNUNET_STRINGS_string_to_data (&s[npos], - SIGNATURE_ASCII_LENGTH, - &sig, - sizeof( - struct GNUNET_CRYPTO_EddsaSignature))) - { - *emsg = - GNUNET_strdup (_ ("LOC URI malformed (could not decode signature)")); - goto ERR; - } - npos += SIGNATURE_ASCII_LENGTH; - if (s[npos++] != '.') - { - *emsg = GNUNET_strdup ( - _ ("LOC URI malformed (wrong syntax for expiration time)")); - goto ERR; - } - if (1 != sscanf (&s[npos], "%llu", &exptime)) - { - *emsg = - GNUNET_strdup (_ ("LOC URI malformed (could not parse expiration time)")); - goto ERR; - } - ass.purpose.size = htonl (sizeof(struct LocUriAssembly)); - ass.purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT); - et.abs_value_us = exptime * 1000LL * 1000LL; - ass.exptime = GNUNET_TIME_absolute_hton (et); - if (GNUNET_OK != - GNUNET_CRYPTO_eddsa_verify (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT, - &ass, - &sig, - &ass.peer.public_key)) - { - *emsg = - GNUNET_strdup (_ ("LOC URI malformed (signature failed validation)")); - goto ERR; - } - uri = GNUNET_new (struct GNUNET_FS_Uri); - uri->type = GNUNET_FS_URI_LOC; - uri->data.loc.fi = ass.fi; - uri->data.loc.peer = ass.peer; - uri->data.loc.expirationTime = et; - uri->data.loc.contentSignature = sig; - - return uri; - ERR: - return NULL; -} - - -/** - * Convert a UTF-8 String to a URI. - * - * @param uri string to parse - * @param emsg where to store the parser error message (if any) - * @return NULL on error - */ -struct GNUNET_FS_Uri * -GNUNET_FS_uri_parse (const char *uri, char **emsg) -{ - struct GNUNET_FS_Uri *ret; - char *msg; - - if (NULL == uri) - { - GNUNET_break (0); - if (NULL != emsg) - *emsg = GNUNET_strdup (_ ("invalid argument")); - return NULL; - } - /** - * FIXME: Do we want to log this? - */ - msg = NULL; - if (NULL != (ret = uri_chk_parse (uri, &msg))) - return ret; - GNUNET_free (msg); - if (NULL != (ret = uri_ksk_parse (uri, &msg))) - return ret; - GNUNET_free (msg); - if (NULL != (ret = uri_sks_parse (uri, &msg))) - return ret; - GNUNET_free (msg); - if (NULL != (ret = uri_loc_parse (uri, &msg))) - return ret; - GNUNET_free (msg); - if (NULL != emsg) - *emsg = GNUNET_strdup (_ ("Unrecognized URI type")); - return NULL; -} - - -/** - * Free URI. - * - * @param uri uri to free - */ -void -GNUNET_FS_uri_destroy (struct GNUNET_FS_Uri *uri) -{ - unsigned int i; - - switch (uri->type) - { - case GNUNET_FS_URI_KSK: - for (i = 0; i < uri->data.ksk.keywordCount; i++) - GNUNET_free (uri->data.ksk.keywords[i]); - GNUNET_array_grow (uri->data.ksk.keywords, uri->data.ksk.keywordCount, 0); - break; - - case GNUNET_FS_URI_SKS: - GNUNET_free (uri->data.sks.identifier); - break; - - case GNUNET_FS_URI_LOC: - break; - - default: - /* do nothing */ - break; - } - GNUNET_free (uri); -} - - -/** - * How many keywords are ANDed in this keyword URI? - * - * @param uri ksk uri to get the number of keywords from - * @return 0 if this is not a keyword URI - */ -unsigned int -GNUNET_FS_uri_ksk_get_keyword_count (const struct GNUNET_FS_Uri *uri) -{ - if (uri->type != GNUNET_FS_URI_KSK) - return 0; - return uri->data.ksk.keywordCount; -} - - -int -GNUNET_FS_uri_ksk_get_keywords (const struct GNUNET_FS_Uri *uri, - GNUNET_FS_KeywordIterator iterator, - void *iterator_cls) -{ - unsigned int i; - char *keyword; - - if (uri->type != GNUNET_FS_URI_KSK) - return -1; - if (NULL == iterator) - return uri->data.ksk.keywordCount; - for (i = 0; i < uri->data.ksk.keywordCount; i++) - { - keyword = uri->data.ksk.keywords[i]; - /* first character of keyword indicates - * if it is mandatory or not */ - if (GNUNET_OK != iterator (iterator_cls, &keyword[1],(keyword[0] == '+') )) - return i; - } - return i; -} - - -/** - * Add the given keyword to the set of keywords represented by the URI. - * Does nothing if the keyword is already present. - * - * @param uri ksk uri to modify - * @param keyword keyword to add - * @param is_mandatory is this keyword mandatory? - */ -void -GNUNET_FS_uri_ksk_add_keyword (struct GNUNET_FS_Uri *uri, - const char *keyword, - int is_mandatory) -{ - unsigned int i; - const char *old; - char *n; - - GNUNET_assert (uri->type == GNUNET_FS_URI_KSK); - for (i = 0; i < uri->data.ksk.keywordCount; i++) - { - old = uri->data.ksk.keywords[i]; - if (0 == strcmp (&old[1], keyword)) - return; - } - GNUNET_asprintf (&n, is_mandatory ? "+%s" : " %s", keyword); - GNUNET_array_append (uri->data.ksk.keywords, uri->data.ksk.keywordCount, n); -} - - -/** - * Remove the given keyword from the set of keywords represented by the URI. - * Does nothing if the keyword is not present. - * - * @param uri ksk uri to modify - * @param keyword keyword to add - */ -void -GNUNET_FS_uri_ksk_remove_keyword (struct GNUNET_FS_Uri *uri, - const char *keyword) -{ - unsigned int i; - char *old; - - GNUNET_assert (uri->type == GNUNET_FS_URI_KSK); - for (i = 0; i < uri->data.ksk.keywordCount; i++) - { - old = uri->data.ksk.keywords[i]; - if (0 == strcmp (&old[1], keyword)) - { - uri->data.ksk.keywords[i] = - uri->data.ksk.keywords[uri->data.ksk.keywordCount - 1]; - GNUNET_array_grow (uri->data.ksk.keywords, - uri->data.ksk.keywordCount, - uri->data.ksk.keywordCount - 1); - GNUNET_free (old); - return; - } - } -} - - -/** - * Obtain the identity of the peer offering the data - * - * @param uri the location URI to inspect - * @param peer where to store the identify of the peer (presumably) offering the content - * @return #GNUNET_SYSERR if this is not a location URI, otherwise #GNUNET_OK - */ -int -GNUNET_FS_uri_loc_get_peer_identity (const struct GNUNET_FS_Uri *uri, - struct GNUNET_PeerIdentity *peer) -{ - if (uri->type != GNUNET_FS_URI_LOC) - return GNUNET_SYSERR; - *peer = uri->data.loc.peer; - return GNUNET_OK; -} - - -/** - * Obtain the expiration of the LOC URI. - * - * @param uri location URI to get the expiration from - * @return expiration time of the URI - */ -struct GNUNET_TIME_Absolute -GNUNET_FS_uri_loc_get_expiration (const struct GNUNET_FS_Uri *uri) -{ - GNUNET_assert (uri->type == GNUNET_FS_URI_LOC); - return uri->data.loc.expirationTime; -} - - -/** - * Obtain the URI of the content itself. - * - * @param uri location URI to get the content URI from - * @return NULL if argument is not a location URI - */ -struct GNUNET_FS_Uri * -GNUNET_FS_uri_loc_get_uri (const struct GNUNET_FS_Uri *uri) -{ - struct GNUNET_FS_Uri *ret; - - if (uri->type != GNUNET_FS_URI_LOC) - return NULL; - ret = GNUNET_new (struct GNUNET_FS_Uri); - ret->type = GNUNET_FS_URI_CHK; - ret->data.chk = uri->data.loc.fi; - return ret; -} - - -/** - * Construct a location URI (this peer will be used for the location). - * This function should only be called from within gnunet-service-fs, - * as it requires the peer's private key which is generally unavailable - * to processes directly under the user's control. However, for - * testing and as it logically fits under URIs, it is in this API. - * - * @param base_uri content offered by the sender - * @param sign_key private key of the peer - * @param expiration_time how long will the content be offered? - * @return the location URI, NULL on error - */ -struct GNUNET_FS_Uri * -GNUNET_FS_uri_loc_create (const struct GNUNET_FS_Uri *base_uri, - const struct GNUNET_CRYPTO_EddsaPrivateKey *sign_key, - struct GNUNET_TIME_Absolute expiration_time) -{ - struct GNUNET_FS_Uri *uri; - struct GNUNET_CRYPTO_EddsaPublicKey my_public_key; - struct LocUriAssembly ass; - struct GNUNET_TIME_Absolute et; - - if (GNUNET_FS_URI_CHK != base_uri->type) - return NULL; - /* we round expiration time to full seconds for SKS URIs */ - et.abs_value_us = (expiration_time.abs_value_us / 1000000LL) * 1000000LL; - GNUNET_CRYPTO_eddsa_key_get_public (sign_key, &my_public_key); - ass.purpose.size = htonl (sizeof(struct LocUriAssembly)); - ass.purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT); - ass.exptime = GNUNET_TIME_absolute_hton (et); - ass.fi = base_uri->data.chk; - ass.peer.public_key = my_public_key; - uri = GNUNET_new (struct GNUNET_FS_Uri); - uri->type = GNUNET_FS_URI_LOC; - uri->data.loc.fi = base_uri->data.chk; - uri->data.loc.expirationTime = et; - uri->data.loc.peer.public_key = my_public_key; - GNUNET_CRYPTO_eddsa_sign (sign_key, - &ass, - &uri->data.loc.contentSignature); - return uri; -} - - -/** - * Create an SKS URI from a namespace ID and an identifier. - * - * @param ns namespace ID - * @param id identifier - * @return an FS URI for the given namespace and identifier - */ -struct GNUNET_FS_Uri * -GNUNET_FS_uri_sks_create (const struct GNUNET_CRYPTO_EcdsaPublicKey *ns, - const char *id) -{ - struct GNUNET_FS_Uri *ns_uri; - - ns_uri = GNUNET_new (struct GNUNET_FS_Uri); - ns_uri->type = GNUNET_FS_URI_SKS; - ns_uri->data.sks.ns = *ns; - ns_uri->data.sks.identifier = GNUNET_strdup (id); - return ns_uri; -} - - -/** - * Merge the sets of keywords from two KSK URIs. - * (useful for merging the canonicalized keywords with - * the original keywords for sharing). - * - * @param u1 first uri - * @param u2 second uri - * @return merged URI, NULL on error - */ -struct GNUNET_FS_Uri * -GNUNET_FS_uri_ksk_merge (const struct GNUNET_FS_Uri *u1, - const struct GNUNET_FS_Uri *u2) -{ - struct GNUNET_FS_Uri *ret; - unsigned int kc; - unsigned int i; - unsigned int j; - int found; - const char *kp; - char **kl; - - if ((u1 == NULL) && (u2 == NULL)) - return NULL; - if (u1 == NULL) - return GNUNET_FS_uri_dup (u2); - if (u2 == NULL) - return GNUNET_FS_uri_dup (u1); - if ((u1->type != GNUNET_FS_URI_KSK) || (u2->type != GNUNET_FS_URI_KSK)) - { - GNUNET_break (0); - return NULL; - } - kc = u1->data.ksk.keywordCount; - kl = GNUNET_new_array (kc + u2->data.ksk.keywordCount, char *); - for (i = 0; i < u1->data.ksk.keywordCount; i++) - kl[i] = GNUNET_strdup (u1->data.ksk.keywords[i]); - for (i = 0; i < u2->data.ksk.keywordCount; i++) - { - kp = u2->data.ksk.keywords[i]; - found = 0; - for (j = 0; j < u1->data.ksk.keywordCount; j++) - if (0 == strcmp (kp + 1, kl[j] + 1)) - { - found = 1; - if (kp[0] == '+') - kl[j][0] = '+'; - break; - } - if (0 == found) - kl[kc++] = GNUNET_strdup (kp); - } - ret = GNUNET_new (struct GNUNET_FS_Uri); - ret->type = GNUNET_FS_URI_KSK; - ret->data.ksk.keywordCount = kc; - ret->data.ksk.keywords = kl; - return ret; -} - - -/** - * Duplicate URI. - * - * @param uri the URI to duplicate - * @return copy of the URI - */ -struct GNUNET_FS_Uri * -GNUNET_FS_uri_dup (const struct GNUNET_FS_Uri *uri) -{ - struct GNUNET_FS_Uri *ret; - unsigned int i; - - if (uri == NULL) - return NULL; - ret = GNUNET_new (struct GNUNET_FS_Uri); - GNUNET_memcpy (ret, uri, sizeof(struct GNUNET_FS_Uri)); - switch (ret->type) - { - case GNUNET_FS_URI_KSK: - if (ret->data.ksk.keywordCount >= - GNUNET_MAX_MALLOC_CHECKED / sizeof(char *)) - { - GNUNET_break (0); - GNUNET_free (ret); - return NULL; - } - if (ret->data.ksk.keywordCount > 0) - { - ret->data.ksk.keywords = - GNUNET_new_array (ret->data.ksk.keywordCount, char *); - for (i = 0; i < ret->data.ksk.keywordCount; i++) - ret->data.ksk.keywords[i] = GNUNET_strdup (uri->data.ksk.keywords[i]); - } - else - ret->data.ksk.keywords = NULL; /* just to be sure */ - break; - - case GNUNET_FS_URI_SKS: - ret->data.sks.identifier = GNUNET_strdup (uri->data.sks.identifier); - break; - - case GNUNET_FS_URI_LOC: - break; - - default: - break; - } - return ret; -} - - -/** - * Create an FS URI from a single user-supplied string of keywords. - * The string is broken up at spaces into individual keywords. - * Keywords that start with "+" are mandatory. Double-quotes can - * be used to prevent breaking up strings at spaces (and also - * to specify non-mandatory keywords starting with "+"). - * - * Keywords must contain a balanced number of double quotes and - * double quotes can not be used in the actual keywords (for - * example, the string '""foo bar""' will be turned into two - * "OR"ed keywords 'foo' and 'bar', not into '"foo bar"'. - * - * @param keywords the keyword string - * @param emsg where to store an error message - * @return an FS URI for the given keywords, NULL - * if keywords is not legal (i.e. empty). - */ -struct GNUNET_FS_Uri * -GNUNET_FS_uri_ksk_create (const char *keywords, char **emsg) -{ - char **keywordarr; - unsigned int num_Words; - int inWord; - char *pos; - struct GNUNET_FS_Uri *uri; - char *searchString; - int saw_quote; - - if (keywords == NULL) - { - *emsg = GNUNET_strdup (_ ("No keywords specified!\n")); - GNUNET_break (0); - return NULL; - } - searchString = GNUNET_strdup (keywords); - num_Words = 0; - inWord = 0; - saw_quote = 0; - pos = searchString; - while ('\0' != *pos) - { - if ((saw_quote == 0) && (isspace ((unsigned char) *pos))) - { - inWord = 0; - } - else if (0 == inWord) - { - inWord = 1; - ++num_Words; - } - if ('"' == *pos) - saw_quote = (saw_quote + 1) % 2; - pos++; - } - if (num_Words == 0) - { - GNUNET_free (searchString); - *emsg = GNUNET_strdup (_ ("No keywords specified!\n")); - return NULL; - } - if (saw_quote != 0) - { - GNUNET_free (searchString); - *emsg = GNUNET_strdup (_ ("Number of double-quotes not balanced!\n")); - return NULL; - } - keywordarr = GNUNET_new_array (num_Words, char *); - num_Words = 0; - inWord = 0; - pos = searchString; - while ('\0' != *pos) - { - if ((saw_quote == 0) && (isspace ((unsigned char) *pos))) - { - inWord = 0; - *pos = '\0'; - } - else if (0 == inWord) - { - keywordarr[num_Words] = pos; - inWord = 1; - ++num_Words; - } - if ('"' == *pos) - saw_quote = (saw_quote + 1) % 2; - pos++; - } - uri = - GNUNET_FS_uri_ksk_create_from_args (num_Words, (const char **) keywordarr); - GNUNET_free (keywordarr); - GNUNET_free (searchString); - return uri; -} - - -/** - * Create an FS URI from a user-supplied command line of keywords. - * Arguments should start with "+" to indicate mandatory - * keywords. - * - * @param argc number of keywords - * @param argv keywords (double quotes are not required for - * keywords containing spaces; however, double - * quotes are required for keywords starting with - * "+"); there is no mechanism for having double - * quotes in the actual keywords (if the user - * did specifically specify double quotes, the - * caller should convert each double quote - * into two single quotes). - * @return an FS URI for the given keywords, NULL - * if keywords is not legal (i.e. empty). - */ -struct GNUNET_FS_Uri * -GNUNET_FS_uri_ksk_create_from_args (unsigned int argc, const char **argv) -{ - unsigned int i; - struct GNUNET_FS_Uri *uri; - const char *keyword; - char *val; - const char *r; - char *w; - char *emsg; - - if (argc == 0) - return NULL; - /* allow URI to be given as one and only keyword and - * handle accordingly */ - emsg = NULL; - if ((argc == 1) && (strlen (argv[0]) > strlen (GNUNET_FS_URI_PREFIX)) && - (0 == strncmp (argv[0], - GNUNET_FS_URI_PREFIX, - strlen (GNUNET_FS_URI_PREFIX))) && - (NULL != (uri = GNUNET_FS_uri_parse (argv[0], &emsg)))) - return uri; - GNUNET_free (emsg); - uri = GNUNET_new (struct GNUNET_FS_Uri); - uri->type = GNUNET_FS_URI_KSK; - uri->data.ksk.keywordCount = argc; - uri->data.ksk.keywords = GNUNET_new_array (argc, char *); - for (i = 0; i < argc; i++) - { - keyword = argv[i]; - if (keyword[0] == '+') - val = GNUNET_strdup (keyword); - else - GNUNET_asprintf (&val, " %s", keyword); - r = val; - w = val; - while ('\0' != *r) - { - if ('"' == *r) - r++; - else - *(w++) = *(r++); - } - *w = '\0'; - uri->data.ksk.keywords[i] = val; - } - return uri; -} - - -/** - * Test if two URIs are equal. - * - * @param u1 one of the URIs - * @param u2 the other URI - * @return #GNUNET_YES if the URIs are equal - */ -int -GNUNET_FS_uri_test_equal (const struct GNUNET_FS_Uri *u1, - const struct GNUNET_FS_Uri *u2) -{ - int ret; - unsigned int i; - unsigned int j; - - GNUNET_assert (u1 != NULL); - GNUNET_assert (u2 != NULL); - if (u1->type != u2->type) - return GNUNET_NO; - switch (u1->type) - { - case GNUNET_FS_URI_CHK: - if (0 == - memcmp (&u1->data.chk, &u2->data.chk, sizeof(struct FileIdentifier))) - return GNUNET_YES; - return GNUNET_NO; - - case GNUNET_FS_URI_SKS: - if ((0 == memcmp (&u1->data.sks.ns, - &u2->data.sks.ns, - sizeof(struct GNUNET_CRYPTO_EcdsaPublicKey))) && - (0 == strcmp (u1->data.sks.identifier, u2->data.sks.identifier))) - - return GNUNET_YES; - return GNUNET_NO; - - case GNUNET_FS_URI_KSK: - if (u1->data.ksk.keywordCount != u2->data.ksk.keywordCount) - return GNUNET_NO; - for (i = 0; i < u1->data.ksk.keywordCount; i++) - { - ret = GNUNET_NO; - for (j = 0; j < u2->data.ksk.keywordCount; j++) - { - if (0 == strcmp (u1->data.ksk.keywords[i], u2->data.ksk.keywords[j])) - { - ret = GNUNET_YES; - break; - } - } - if (ret == GNUNET_NO) - return GNUNET_NO; - } - return GNUNET_YES; - - case GNUNET_FS_URI_LOC: - if (memcmp (&u1->data.loc, - &u2->data.loc, - sizeof(struct FileIdentifier) - + sizeof(struct GNUNET_CRYPTO_EcdsaPublicKey) - + sizeof(struct GNUNET_TIME_Absolute) - + sizeof(unsigned short) + sizeof(unsigned short)) != 0) - return GNUNET_NO; - return GNUNET_YES; - - default: - return GNUNET_NO; - } -} - - -/** - * Is this a namespace URI? - * - * @param uri the uri to check - * @return #GNUNET_YES if this is an SKS uri - */ -int -GNUNET_FS_uri_test_sks (const struct GNUNET_FS_Uri *uri) -{ - return uri->type == GNUNET_FS_URI_SKS; -} - - -/** - * Get the ID of a namespace from the given - * namespace URI. - * - * @param uri the uri to get the namespace ID from - * @param pseudonym where to store the ID of the namespace - * @return #GNUNET_OK on success - */ -int -GNUNET_FS_uri_sks_get_namespace (const struct GNUNET_FS_Uri *uri, - struct GNUNET_CRYPTO_EcdsaPublicKey *pseudonym) -{ - if (! GNUNET_FS_uri_test_sks (uri)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - *pseudonym = uri->data.sks.ns; - return GNUNET_OK; -} - - -/** - * Get the content identifier of an SKS URI. - * - * @param uri the sks uri - * @return NULL on error (not a valid SKS URI) - */ -char * -GNUNET_FS_uri_sks_get_content_id (const struct GNUNET_FS_Uri *uri) -{ - if (! GNUNET_FS_uri_test_sks (uri)) - { - GNUNET_break (0); - return NULL; - } - return GNUNET_strdup (uri->data.sks.identifier); -} - - -/** - * Is this a keyword URI? - * - * @param uri the uri - * @return #GNUNET_YES if this is a KSK uri - */ -int -GNUNET_FS_uri_test_ksk (const struct GNUNET_FS_Uri *uri) -{ -#if EXTRA_CHECKS - unsigned int i; - - if (uri->type == GNUNET_FS_URI_KSK) - { - for (i = 0; i < uri->data.ksk.keywordCount; i++) - GNUNET_assert (uri->data.ksk.keywords[i] != NULL); - } -#endif - return uri->type == GNUNET_FS_URI_KSK; -} - - -/** - * Is this a file (or directory) URI? - * - * @param uri the uri to check - * @return #GNUNET_YES if this is a CHK uri - */ -int -GNUNET_FS_uri_test_chk (const struct GNUNET_FS_Uri *uri) -{ - return uri->type == GNUNET_FS_URI_CHK; -} - - -/** - * What is the size of the file that this URI - * refers to? - * - * @param uri the CHK URI to inspect - * @return size of the file as specified in the CHK URI - */ -uint64_t -GNUNET_FS_uri_chk_get_file_size (const struct GNUNET_FS_Uri *uri) -{ - switch (uri->type) - { - case GNUNET_FS_URI_CHK: - return GNUNET_ntohll (uri->data.chk.file_length); - - case GNUNET_FS_URI_LOC: - return GNUNET_ntohll (uri->data.loc.fi.file_length); - - default: - GNUNET_assert (0); - } - return 0; /* unreachable */ -} - - -/** - * Is this a location URI? - * - * @param uri the uri to check - * @return #GNUNET_YES if this is a LOC uri - */ -int -GNUNET_FS_uri_test_loc (const struct GNUNET_FS_Uri *uri) -{ - return uri->type == GNUNET_FS_URI_LOC; -} - - -/** - * Add a keyword as non-mandatory (with ' '-prefix) to the - * given keyword list at offset 'index'. The array is - * guaranteed to be long enough. - * - * @param s keyword to add - * @param array array to add the keyword to - * @param index offset where to add the keyword - */ -static void -insert_non_mandatory_keyword (const char *s, char **array, int index) -{ - char *nkword; - - GNUNET_asprintf (&nkword, - " %s", /* space to mark as 'non mandatory' */ - s); - array[index] = nkword; -} - - -/** - * Test if the given keyword @a s is already present in the - * given array, ignoring the '+'-mandatory prefix in the array. - * - * @param s keyword to test - * @param array keywords to test against, with ' ' or '+' prefix to ignore - * @param array_length length of the @a array - * @return #GNUNET_YES if the keyword exists, #GNUNET_NO if not - */ -static int -find_duplicate (const char *s, const char **array, int array_length) -{ - int j; - - for (j = array_length - 1; j >= 0; j--) - if (0 == strcmp (&array[j][1], s)) - return GNUNET_YES; - return GNUNET_NO; -} - - -/** - * FIXME: comment - */ -static char * -normalize_metadata (enum EXTRACTOR_MetaFormat format, - const char *data, - size_t data_len) -{ - uint8_t *free_str = NULL; - uint8_t *str_to_normalize = (uint8_t *) data; - uint8_t *normalized; - size_t r_len; - - if (str_to_normalize == NULL) - return NULL; - /* Don't trust libextractor */ - if (format == EXTRACTOR_METAFORMAT_UTF8) - { - free_str = (uint8_t *) u8_check ((const uint8_t *) data, data_len); - if (free_str == NULL) - free_str = NULL; - else - format = EXTRACTOR_METAFORMAT_C_STRING; - } - if (format == EXTRACTOR_METAFORMAT_C_STRING) - { - free_str = u8_strconv_from_encoding (data, - locale_charset (), - iconveh_escape_sequence); - if (free_str == NULL) - return NULL; - } - - normalized = u8_tolower (str_to_normalize, - strlen ((char *) str_to_normalize), - NULL, - UNINORM_NFD, - NULL, - &r_len); - /* free_str is allocated by libunistring internally, use free() */ - if (free_str != NULL) - free (free_str); - if (normalized != NULL) - { - /* u8_tolower allocates a non-NULL-terminated string! */ - free_str = GNUNET_malloc (r_len + 1); - GNUNET_memcpy (free_str, normalized, r_len); - free_str[r_len] = '\0'; - free (normalized); - normalized = free_str; - } - return (char *) normalized; -} - - -/** - * Counts the number of UTF-8 characters (not bytes) in the string, - * returns that count. - */ -static size_t -u8_strcount (const uint8_t *s) -{ - size_t count; - ucs4_t c; - - GNUNET_assert (s != NULL); - if (s[0] == 0) - return 0; - for (count = 0; s != NULL; count++) - s = u8_next (&c, s); - return count - 1; -} - - -/** - * Break the filename up by matching [], () and {} pairs to make - * keywords. In case of nesting parentheses only the inner pair counts. - * You can't escape parentheses to scan something like "[blah\{foo]" to - * make a "blah{foo" keyword, this function is only a heuristic! - * - * @param s string to break down. - * @param array array to fill with enclosed tokens. If NULL, then tokens - * are only counted. - * @param index index at which to start filling the array (entries prior - * to it are used to check for duplicates). ignored if @a array == NULL. - * @return number of tokens counted (including duplicates), or number of - * tokens extracted (excluding duplicates). 0 if there are no - * matching parens in the string (when counting), or when all tokens - * were duplicates (when extracting). - */ -static int -get_keywords_from_parens (const char *s, char **array, int index) -{ - int count = 0; - char *open_paren; - char *close_paren; - char *ss; - char tmp; - - if (NULL == s) - return 0; - ss = GNUNET_strdup (s); - open_paren = ss - 1; - while (NULL != (open_paren = strpbrk (open_paren + 1, "[{("))) - { - int match = 0; - - close_paren = strpbrk (open_paren + 1, "]})"); - if (NULL == close_paren) - continue; - switch (open_paren[0]) - { - case '[': - if (']' == close_paren[0]) - match = 1; - break; - - case '{': - if ('}' == close_paren[0]) - match = 1; - break; - - case '(': - if (')' == close_paren[0]) - match = 1; - break; - - default: - break; - } - if (match && (close_paren - open_paren > 1)) - { - tmp = close_paren[0]; - close_paren[0] = '\0'; - /* Keywords must be at least 3 characters long */ - if (u8_strcount ((const uint8_t *) &open_paren[1]) <= 2) - { - close_paren[0] = tmp; - continue; - } - if (NULL != array) - { - char *normalized; - if (GNUNET_NO == find_duplicate ((const char *) &open_paren[1], - (const char **) array, - index + count)) - { - insert_non_mandatory_keyword ((const char *) &open_paren[1], - array, - index + count); - count++; - } - normalized = normalize_metadata (EXTRACTOR_METAFORMAT_UTF8, - &open_paren[1], - close_paren - &open_paren[1]); - if (normalized != NULL) - { - if (GNUNET_NO == find_duplicate ((const char *) normalized, - (const char **) array, - index + count)) - { - insert_non_mandatory_keyword ((const char *) normalized, - array, - index + count); - count++; - } - GNUNET_free (normalized); - } - } - else - count++; - close_paren[0] = tmp; - } - } - GNUNET_free (ss); - return count; -} - - -/** - * Where to break up keywords - */ -#define TOKENS "_. /-!?#&+@\"\'\\;:,()[]{}$<>|" - -/** - * Break the filename up by TOKENS to make - * keywords. - * - * @param s string to break down. - * @param array array to fill with tokens. If NULL, then tokens are only - * counted. - * @param index index at which to start filling the array (entries prior - * to it are used to check for duplicates). ignored if @a array == NULL. - * @return number of tokens (>1) counted (including duplicates), or number of - * tokens extracted (excluding duplicates). 0 if there are no - * separators in the string (when counting), or when all tokens were - * duplicates (when extracting). - */ -static int -get_keywords_from_tokens (const char *s, char **array, int index) -{ - char *p; - char *ss; - int seps = 0; - - ss = GNUNET_strdup (s); - for (p = strtok (ss, TOKENS); p != NULL; p = strtok (NULL, TOKENS)) - { - /* Keywords must be at least 3 characters long */ - if (u8_strcount ((const uint8_t *) p) <= 2) - continue; - if (NULL != array) - { - char *normalized; - if (GNUNET_NO == find_duplicate (p, (const char **) array, index + seps)) - { - insert_non_mandatory_keyword (p, array, index + seps); - seps++; - } - normalized = - normalize_metadata (EXTRACTOR_METAFORMAT_UTF8, p, strlen (p)); - if (normalized != NULL) - { - if (GNUNET_NO == find_duplicate ((const char *) normalized, - (const char **) array, - index + seps)) - { - insert_non_mandatory_keyword ((const char *) normalized, - array, - index + seps); - seps++; - } - GNUNET_free (normalized); - } - } - else - seps++; - } - GNUNET_free (ss); - return seps; -} - - -#undef TOKENS - - -/** - * Function called on each value in the meta data. - * Adds it to the URI. - * - * @param cls URI to update - * @param plugin_name name of the plugin that produced this value; - * special values can be used (e.g. '<zlib>' for zlib being - * used in the main libextractor library and yielding - * meta data). - * @param type libextractor-type describing the meta data - * @param format basic format information about data - * @param data_mime_type mime-type of data (not of the original file); - * can be NULL (if mime-type is not known) - * @param data actual meta-data found - * @param data_len number of bytes in @a data - * @return 0 (always) - */ -static int -gather_uri_data (void *cls, - const char *plugin_name, - enum EXTRACTOR_MetaType type, - enum EXTRACTOR_MetaFormat format, - const char *data_mime_type, - const char *data, - size_t data_len) -{ - struct GNUNET_FS_Uri *uri = cls; - char *normalized_data; - const char *sep; - - if ((format != EXTRACTOR_METAFORMAT_UTF8) && - (format != EXTRACTOR_METAFORMAT_C_STRING)) - return 0; - /* Keywords must be at least 3 characters long - * If given non-utf8 string it will, most likely, find it to be invalid, - * and will return the length of its valid part, skipping the keyword. - * If it does - fix the extractor, not this check! - */if (u8_strcount ((const uint8_t *) data) <= 2) - return 0; - if ((EXTRACTOR_METATYPE_MIMETYPE == type) && - (NULL != (sep = memchr (data, '/', data_len))) && (sep != data)) - { - char *xtra; - - GNUNET_asprintf (&xtra, "mimetype:%.*s", (int) (sep - data), data); - if (! find_duplicate (xtra, - (const char **) uri->data.ksk.keywords, - uri->data.ksk.keywordCount)) - { - insert_non_mandatory_keyword (xtra, - uri->data.ksk.keywords, - uri->data.ksk.keywordCount); - uri->data.ksk.keywordCount++; - } - GNUNET_free (xtra); - } - - normalized_data = normalize_metadata (format, data, data_len); - if (! find_duplicate (data, - (const char **) uri->data.ksk.keywords, - uri->data.ksk.keywordCount)) - { - insert_non_mandatory_keyword (data, - uri->data.ksk.keywords, - uri->data.ksk.keywordCount); - uri->data.ksk.keywordCount++; - } - if (NULL != normalized_data) - { - if (! find_duplicate (normalized_data, - (const char **) uri->data.ksk.keywords, - uri->data.ksk.keywordCount)) - { - insert_non_mandatory_keyword (normalized_data, - uri->data.ksk.keywords, - uri->data.ksk.keywordCount); - uri->data.ksk.keywordCount++; - } - GNUNET_free (normalized_data); - } - return 0; -} - - -/** - * Construct a keyword-URI from meta-data (take all entries - * in the meta-data and construct one large keyword URI - * that lists all keywords that can be found in the meta-data). - * - * @param md metadata to use - * @return NULL on error, otherwise a KSK URI - */ -struct GNUNET_FS_Uri * -GNUNET_FS_uri_ksk_create_from_meta_data ( - const struct GNUNET_FS_MetaData *md) -{ - struct GNUNET_FS_Uri *ret; - char *filename; - char *full_name = NULL; - char *ss; - int ent; - int tok_keywords = 0; - int paren_keywords = 0; - - if (NULL == md) - return NULL; - ret = GNUNET_new (struct GNUNET_FS_Uri); - ret->type = GNUNET_FS_URI_KSK; - ent = GNUNET_FS_meta_data_iterate (md, NULL, NULL); - if (ent > 0) - { - full_name = GNUNET_FS_meta_data_get_first_by_types ( - md, - EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, - -1); - if (NULL != full_name) - { - filename = full_name; - while (NULL != (ss = strstr (filename, DIR_SEPARATOR_STR))) - filename = ss + 1; - tok_keywords = get_keywords_from_tokens (filename, NULL, 0); - paren_keywords = get_keywords_from_parens (filename, NULL, 0); - } - /* x3 because there might be a normalized variant of every keyword, - plus theoretically one more for mime... */ - ret->data.ksk.keywords = - GNUNET_new_array ((ent + tok_keywords + paren_keywords) * 3, char *); - GNUNET_FS_meta_data_iterate (md, &gather_uri_data, ret); - } - if (tok_keywords > 0) - ret->data.ksk.keywordCount += - get_keywords_from_tokens (filename, - ret->data.ksk.keywords, - ret->data.ksk.keywordCount); - if (paren_keywords > 0) - ret->data.ksk.keywordCount += - get_keywords_from_parens (filename, - ret->data.ksk.keywords, - ret->data.ksk.keywordCount); - if (ent > 0) - GNUNET_free (full_name); - return ret; -} - - -/** - * In URI-encoding, does the given character - * need to be encoded using %-encoding? - */ -static int -needs_percent (char c) -{ - return(! ((isalnum ((unsigned char) c)) || (c == '-') || (c == '_') || - (c == '.') || (c == '~'))); -} - - -/** - * Convert a KSK URI to a string. - * - * @param uri the URI to convert - * @return NULL on error (i.e. keywordCount == 0) - */ -static char * -uri_ksk_to_string (const struct GNUNET_FS_Uri *uri) -{ - char **keywords; - unsigned int keywordCount; - size_t n; - char *ret; - unsigned int i; - unsigned int j; - unsigned int wpos; - size_t slen; - const char *keyword; - - if (uri->type != GNUNET_FS_URI_KSK) - return NULL; - keywords = uri->data.ksk.keywords; - keywordCount = uri->data.ksk.keywordCount; - n = keywordCount + strlen (GNUNET_FS_URI_PREFIX) - + strlen (GNUNET_FS_URI_KSK_INFIX) + 1; - for (i = 0; i < keywordCount; i++) - { - keyword = keywords[i]; - slen = strlen (keyword); - n += slen; - for (j = 0; j < slen; j++) - { - if ((j == 0) && (keyword[j] == ' ')) - { - n--; - continue; /* skip leading space */ - } - if (needs_percent (keyword[j])) - n += 2; /* will use %-encoding */ - } - } - ret = GNUNET_malloc (n); - strcpy (ret, GNUNET_FS_URI_PREFIX); - strcat (ret, GNUNET_FS_URI_KSK_INFIX); - wpos = strlen (ret); - for (i = 0; i < keywordCount; i++) - { - keyword = keywords[i]; - slen = strlen (keyword); - for (j = 0; j < slen; j++) - { - if ((j == 0) && (keyword[j] == ' ')) - continue; /* skip leading space */ - if (needs_percent (keyword[j])) - { - sprintf (&ret[wpos], "%%%02X", (unsigned char) keyword[j]); - wpos += 3; - } - else - { - ret[wpos++] = keyword[j]; - } - } - if (i != keywordCount - 1) - ret[wpos++] = '+'; - } - return ret; -} - - -/** - * Convert SKS URI to a string. - * - * @param uri sks uri to convert - * @return NULL on error - */ -static char * -uri_sks_to_string (const struct GNUNET_FS_Uri *uri) -{ - char *ret; - char buf[1024]; - - if (GNUNET_FS_URI_SKS != uri->type) - return NULL; - ret = - GNUNET_STRINGS_data_to_string (&uri->data.sks.ns, - sizeof(struct GNUNET_CRYPTO_EcdsaPublicKey), - buf, - sizeof(buf)); - GNUNET_assert (NULL != ret); - ret[0] = '\0'; - GNUNET_asprintf (&ret, - "%s%s%s/%s", - GNUNET_FS_URI_PREFIX, - GNUNET_FS_URI_SKS_INFIX, - buf, - uri->data.sks.identifier); - return ret; -} - - -/** - * Convert a CHK URI to a string. - * - * @param uri chk uri to convert - * @return NULL on error - */ -static char * -uri_chk_to_string (const struct GNUNET_FS_Uri *uri) -{ - const struct FileIdentifier *fi; - char *ret; - struct GNUNET_CRYPTO_HashAsciiEncoded keyhash; - struct GNUNET_CRYPTO_HashAsciiEncoded queryhash; - - if (uri->type != GNUNET_FS_URI_CHK) - return NULL; - fi = &uri->data.chk; - GNUNET_CRYPTO_hash_to_enc (&fi->chk.key, &keyhash); - GNUNET_CRYPTO_hash_to_enc (&fi->chk.query, &queryhash); - - GNUNET_asprintf (&ret, - "%s%s%s.%s.%llu", - GNUNET_FS_URI_PREFIX, - GNUNET_FS_URI_CHK_INFIX, - (const char *) &keyhash, - (const char *) &queryhash, - (unsigned long long) GNUNET_ntohll (fi->file_length)); - return ret; -} - - -/** - * Convert a LOC URI to a string. - * - * @param uri loc uri to convert - * @return NULL on error - */ -static char * -uri_loc_to_string (const struct GNUNET_FS_Uri *uri) -{ - char *ret; - struct GNUNET_CRYPTO_HashAsciiEncoded keyhash; - struct GNUNET_CRYPTO_HashAsciiEncoded queryhash; - char *peer_id; - char peer_sig[SIGNATURE_ASCII_LENGTH + 1]; - - GNUNET_CRYPTO_hash_to_enc (&uri->data.loc.fi.chk.key, &keyhash); - GNUNET_CRYPTO_hash_to_enc (&uri->data.loc.fi.chk.query, &queryhash); - peer_id = - GNUNET_CRYPTO_eddsa_public_key_to_string (&uri->data.loc.peer.public_key); - GNUNET_assert ( - NULL != - GNUNET_STRINGS_data_to_string (&uri->data.loc.contentSignature, - sizeof(struct GNUNET_CRYPTO_EddsaSignature), - peer_sig, - sizeof(peer_sig))); - GNUNET_asprintf (&ret, - "%s%s%s.%s.%llu.%s.%s.%llu", - GNUNET_FS_URI_PREFIX, - GNUNET_FS_URI_LOC_INFIX, - (const char *) &keyhash, - (const char *) &queryhash, - (unsigned long long) GNUNET_ntohll ( - uri->data.loc.fi.file_length), - peer_id, - peer_sig, - (unsigned long long) - uri->data.loc.expirationTime.abs_value_us - / 1000000LL); - GNUNET_free (peer_id); - return ret; -} - - -/** - * Convert a URI to a UTF-8 String. - * - * @param uri uri to convert to a string - * @return the UTF-8 string - */ -char * -GNUNET_FS_uri_to_string (const struct GNUNET_FS_Uri *uri) -{ - if (uri == NULL) - { - GNUNET_break (0); - return NULL; - } - switch (uri->type) - { - case GNUNET_FS_URI_KSK: - return uri_ksk_to_string (uri); - - case GNUNET_FS_URI_SKS: - return uri_sks_to_string (uri); - - case GNUNET_FS_URI_CHK: - return uri_chk_to_string (uri); - - case GNUNET_FS_URI_LOC: - return uri_loc_to_string (uri); - - default: - GNUNET_break (0); - return NULL; - } -} - - -/* end of fs_uri.c */ diff --git a/src/fs/gnunet-auto-share.c b/src/fs/gnunet-auto-share.c deleted file mode 100644 index f91e9d00d..000000000 --- a/src/fs/gnunet-auto-share.c +++ /dev/null @@ -1,791 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2001--2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/gnunet-auto-share.c - * @brief automatically publish files on GNUnet - * @author Christian Grothoff - * - * TODO: - * - support loading meta data / keywords from resource file - * - add stability timer (a la buildbot) - */ -#include "platform.h" -#include "gnunet_util_lib.h" - -#define MAX_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS, 4) - -#define MIN_DELAY GNUNET_TIME_UNIT_MINUTES - - -/** - * Item in our work queue (or in the set of files/directories - * we have successfully published). - */ -struct WorkItem -{ - /** - * PENDING Work is kept in a linked list. - */ - struct WorkItem *prev; - - /** - * PENDING Work is kept in a linked list. - */ - struct WorkItem *next; - - /** - * Filename of the work item. - */ - char *filename; - - /** - * Unique identity for this work item (used to detect - * if we need to do the work again). - */ - struct GNUNET_HashCode id; -}; - - -/** - * Global return value from 'main'. - */ -static int ret; - -/** - * Are we running 'verbosely'? - */ -static unsigned int verbose; - -/** - * Configuration to use. - */ -static const struct GNUNET_CONFIGURATION_Handle *cfg; - -/** - * Name of the configuration file. - */ -static char *cfg_filename; - -/** - * Disable extractor option to use for publishing. - */ -static int disable_extractor; - -/** - * Disable creation time option to use for publishing. - */ -static int do_disable_creation_time; - -/** - * Handle for the main task that does scanning and working. - */ -static struct GNUNET_SCHEDULER_Task *run_task; - -/** - * Anonymity level option to use for publishing. - */ -static unsigned int anonymity_level = 1; - -/** - * Content priority option to use for publishing. - */ -static unsigned int content_priority = 365; - -/** - * Replication level option to use for publishing. - */ -static unsigned int replication_level = 1; - -/** - * Top-level directory we monitor to auto-publish. - */ -static const char *dir_name; - -/** - * Head of linked list of files still to publish. - */ -static struct WorkItem *work_head; - -/** - * Tail of linked list of files still to publish. - */ -static struct WorkItem *work_tail; - -/** - * Map from the hash of the filename (!) to a `struct WorkItem` - * that was finished. - */ -static struct GNUNET_CONTAINER_MultiHashMap *work_finished; - -/** - * Set to #GNUNET_YES if we are shutting down. - */ -static int do_shutdown; - -/** - * Start time of the current round; used to determine how long - * one iteration takes (which influences how fast we schedule - * the next one). - */ -static struct GNUNET_TIME_Absolute start_time; - -/** - * Pipe used to communicate 'gnunet-publish' completion (SIGCHLD) via signal. - */ -static struct GNUNET_DISK_PipeHandle *sigpipe; - -/** - * Handle to the 'gnunet-publish' process that we executed. - */ -static struct GNUNET_OS_Process *publish_proc; - - -/** - * Compute the name of the state database file we will use. - */ -static char * -get_state_file () -{ - char *ret; - - GNUNET_asprintf (&ret, - "%s%s.auto-share", - dir_name, - (DIR_SEPARATOR == dir_name[strlen (dir_name) - 1]) - ? "" - : DIR_SEPARATOR_STR); - return ret; -} - - -/** - * Load the set of #work_finished items from disk. - */ -static void -load_state () -{ - char *fn; - struct GNUNET_BIO_ReadHandle *rh; - uint32_t n; - struct GNUNET_HashCode id; - struct WorkItem *wi; - char *emsg; - - emsg = NULL; - fn = get_state_file (); - rh = GNUNET_BIO_read_open_file (fn); - GNUNET_free (fn); - if (NULL == rh) - return; - fn = NULL; - if (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "number of files", - (int32_t *) &n)) - goto error; - while (n-- > 0) - { - struct GNUNET_BIO_ReadSpec rs[] = { - GNUNET_BIO_read_spec_string ("filename", &fn, 1024), - GNUNET_BIO_read_spec_object ("id", &id, sizeof(struct GNUNET_HashCode)), - GNUNET_BIO_read_spec_end (), - }; - if (GNUNET_OK != GNUNET_BIO_read_spec_commit (rh, rs)) - goto error; - wi = GNUNET_new (struct WorkItem); - wi->id = id; - wi->filename = fn; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Loaded serialization ID for `%s' is `%s'\n", - wi->filename, - GNUNET_h2s (&id)); - fn = NULL; - GNUNET_CRYPTO_hash (wi->filename, strlen (wi->filename), &id); - GNUNET_break (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put ( - work_finished, - &id, - wi, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - } - if (GNUNET_OK == GNUNET_BIO_read_close (rh, &emsg)) - return; - rh = NULL; -error: - GNUNET_free (fn); - if (NULL != rh) - (void) GNUNET_BIO_read_close (rh, &emsg); - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to load state: %s\n"), - emsg); - GNUNET_free (emsg); -} - - -/** - * Write work item from the #work_finished map to the given write handle. - * - * @param cls the `struct GNUNET_BIO_WriteHandle *` - * @param key key of the item in the map (unused) - * @param value the `struct WorkItem` to write - * @return #GNUNET_OK to continue to iterate (if write worked) - */ -static int -write_item (void *cls, const struct GNUNET_HashCode *key, void *value) -{ - struct GNUNET_BIO_WriteHandle *wh = cls; - struct WorkItem *wi = value; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Saving serialization ID of file `%s' with value `%s'\n", - wi->filename, - GNUNET_h2s (&wi->id)); - struct GNUNET_BIO_WriteSpec ws[] = { - GNUNET_BIO_write_spec_string ("auto-share-write-item-filename", - wi->filename), - GNUNET_BIO_write_spec_object ("id", &wi->id, sizeof(struct - GNUNET_HashCode)), - GNUNET_BIO_write_spec_end (), - }; - if (GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws)) - return GNUNET_SYSERR; /* write error, abort iteration */ - return GNUNET_OK; -} - - -/** - * Save the set of #work_finished items on disk. - */ -static void -save_state () -{ - uint32_t n; - struct GNUNET_BIO_WriteHandle *wh; - char *fn; - - n = GNUNET_CONTAINER_multihashmap_size (work_finished); - fn = get_state_file (); - wh = GNUNET_BIO_write_open_file (fn); - if (NULL == wh) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to save state to file %s\n"), - fn); - GNUNET_free (fn); - return; - } - if (GNUNET_OK != GNUNET_BIO_write_int32 (wh, "size of state", n)) - { - (void) GNUNET_BIO_write_close (wh, NULL); - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to save state to file %s\n"), - fn); - GNUNET_free (fn); - return; - } - (void) GNUNET_CONTAINER_multihashmap_iterate (work_finished, &write_item, wh); - if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL)) - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to save state to file %s\n"), - fn); - GNUNET_free (fn); -} - - -/** - * Task run on shutdown. Serializes our current state to disk. - * - * @param cls closure, unused - */ -static void -do_stop_task (void *cls) -{ - do_shutdown = GNUNET_YES; - if (NULL != publish_proc) - { - GNUNET_OS_process_kill (publish_proc, SIGKILL); - return; - } - if (NULL != run_task) - { - GNUNET_SCHEDULER_cancel (run_task); - run_task = NULL; - } -} - - -/** - * Decide what the next task is (working or scanning) and schedule it. - */ -static void -schedule_next_task (void); - - -/** - * Task triggered whenever we receive a SIGCHLD (child - * process died). - * - * @param cls the `struct WorkItem` we were working on - */ -static void -maint_child_death (void *cls) -{ - struct WorkItem *wi = cls; - struct GNUNET_HashCode key; - enum GNUNET_OS_ProcessStatusType type; - unsigned long code; - int ret; - char c; - const struct GNUNET_DISK_FileHandle *pr; - const struct GNUNET_SCHEDULER_TaskContext *tc; - - run_task = NULL; - pr = GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_READ); - tc = GNUNET_SCHEDULER_get_task_context (); - if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_READ_READY)) - { - /* shutdown scheduled us, someone else will kill child, - we should just try again */ - run_task = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL, - pr, - &maint_child_death, - wi); - return; - } - /* consume the signal */ - GNUNET_break (0 < GNUNET_DISK_file_read (pr, &c, sizeof(c))); - - ret = GNUNET_OS_process_status (publish_proc, &type, &code); - GNUNET_assert (GNUNET_SYSERR != ret); - if (GNUNET_NO == ret) - { - /* process still running? Then where did the SIGCHLD come from? - Well, let's declare it spurious (kernel bug?) and keep rolling. - */ - GNUNET_break (0); - run_task = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL, - pr, - &maint_child_death, - wi); - return; - } - GNUNET_assert (GNUNET_OK == ret); - - GNUNET_OS_process_destroy (publish_proc); - publish_proc = NULL; - - if (GNUNET_YES == do_shutdown) - { - GNUNET_free (wi->filename); - GNUNET_free (wi); - return; - } - if ((GNUNET_OS_PROCESS_EXITED == type) && (0 == code)) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - _ ("Publication of `%s' done\n"), - wi->filename); - GNUNET_CRYPTO_hash (wi->filename, strlen (wi->filename), &key); - GNUNET_break (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put ( - work_finished, - &key, - wi, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - } - else - { - GNUNET_CONTAINER_DLL_insert_tail (work_head, work_tail, wi); - } - save_state (); - schedule_next_task (); -} - - -/** - * Signal handler called for SIGCHLD. Triggers the - * respective handler by writing to the trigger pipe. - */ -static void -sighandler_child_death () -{ - static char c; - int old_errno = errno; /* back-up errno */ - - GNUNET_break ( - 1 == - GNUNET_DISK_file_write (GNUNET_DISK_pipe_handle (sigpipe, - GNUNET_DISK_PIPE_END_WRITE), - &c, - sizeof(c))); - errno = old_errno; /* restore errno */ -} - - -/** - * Function called to process work items. - * - * @param cls closure, NULL - */ -static void -work (void *cls) -{ - static char *argv[14]; - static char anon_level[20]; - static char content_prio[20]; - static char repl_level[20]; - struct WorkItem *wi; - const struct GNUNET_DISK_FileHandle *pr; - int argc; - - run_task = NULL; - wi = work_head; - GNUNET_CONTAINER_DLL_remove (work_head, work_tail, wi); - argc = 0; - argv[argc++] = "gnunet-publish"; - if (verbose) - argv[argc++] = "-V"; - if (disable_extractor) - argv[argc++] = "-D"; - if (do_disable_creation_time) - argv[argc++] = "-d"; - argv[argc++] = "-c"; - argv[argc++] = cfg_filename; - GNUNET_snprintf (anon_level, sizeof(anon_level), "%u", anonymity_level); - argv[argc++] = "-a"; - argv[argc++] = anon_level; - GNUNET_snprintf (content_prio, sizeof(content_prio), "%u", content_priority); - argv[argc++] = "-p"; - argv[argc++] = content_prio; - GNUNET_snprintf (repl_level, sizeof(repl_level), "%u", replication_level); - argv[argc++] = "-r"; - argv[argc++] = repl_level; - argv[argc++] = wi->filename; - argv[argc] = NULL; - GNUNET_log (GNUNET_ERROR_TYPE_INFO, _ ("Publishing `%s'\n"), wi->filename); - GNUNET_assert (NULL == publish_proc); - publish_proc = GNUNET_OS_start_process_vap (GNUNET_OS_USE_PIPE_CONTROL, - NULL, - NULL, - NULL, - "gnunet-publish", - argv); - if (NULL == publish_proc) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Failed to run `%s'\n"), - "gnunet-publish"); - GNUNET_CONTAINER_DLL_insert (work_head, work_tail, wi); - run_task = - GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, &work, NULL); - return; - } - pr = GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_READ); - run_task = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL, - pr, - &maint_child_death, - wi); -} - - -/** - * Recursively scan the given file/directory structure to determine - * a unique ID that represents the current state of the hierarchy. - * - * @param cls where to store the unique ID we are computing - * @param filename file to scan - * @return #GNUNET_OK (always) - */ -static int -determine_id (void *cls, const char *filename) -{ - struct GNUNET_HashCode *id = cls; - struct stat sbuf; - struct GNUNET_HashCode fx[2]; - struct GNUNET_HashCode ft; - - if (0 != stat (filename, &sbuf)) - { - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "stat", filename); - return GNUNET_OK; - } - GNUNET_CRYPTO_hash (filename, strlen (filename), &fx[0]); - if (! S_ISDIR (sbuf.st_mode)) - { - uint64_t fattr[2]; - - fattr[0] = GNUNET_htonll (sbuf.st_size); - fattr[0] = GNUNET_htonll (sbuf.st_mtime); - - GNUNET_CRYPTO_hash (fattr, sizeof(fattr), &fx[1]); - } - else - { - memset (&fx[1], 1, sizeof(struct GNUNET_HashCode)); - GNUNET_DISK_directory_scan (filename, &determine_id, &fx[1]); - } - /* use hash here to make hierarchical structure distinct from - all files on the same level */ - GNUNET_CRYPTO_hash (fx, sizeof(fx), &ft); - /* use XOR here so that order of the files in the directory - does not matter! */ - GNUNET_CRYPTO_hash_xor (&ft, id, id); - return GNUNET_OK; -} - - -/** - * Function called with a filename (or directory name) to publish - * (if it has changed since the last time we published it). This function - * is called for the top-level files only. - * - * @param cls closure, NULL - * @param filename complete filename (absolute path) - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR during shutdown - */ -static int -add_file (void *cls, const char *filename) -{ - struct WorkItem *wi; - struct GNUNET_HashCode key; - struct GNUNET_HashCode id; - - if (GNUNET_YES == do_shutdown) - return GNUNET_SYSERR; - if ((NULL != strstr (filename, "/.auto-share")) || - (NULL != strstr (filename, "\\.auto-share"))) - return GNUNET_OK; /* skip internal file */ - GNUNET_CRYPTO_hash (filename, strlen (filename), &key); - wi = GNUNET_CONTAINER_multihashmap_get (work_finished, &key); - memset (&id, 0, sizeof(struct GNUNET_HashCode)); - determine_id (&id, filename); - if (NULL != wi) - { - if (0 == memcmp (&id, &wi->id, sizeof(struct GNUNET_HashCode))) - return GNUNET_OK; /* skip: we did this one already */ - /* contents changed, need to re-do the directory... */ - GNUNET_assert ( - GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (work_finished, &key, wi)); - } - else - { - wi = GNUNET_new (struct WorkItem); - wi->filename = GNUNET_strdup (filename); - } - wi->id = id; - GNUNET_CONTAINER_DLL_insert (work_head, work_tail, wi); - if (GNUNET_YES == do_shutdown) - return GNUNET_SYSERR; - return GNUNET_OK; -} - - -/** - * Periodically run task to update our view of the directory to share. - * - * @param cls NULL - */ -static void -scan (void *cls) -{ - run_task = NULL; - start_time = GNUNET_TIME_absolute_get (); - (void) GNUNET_DISK_directory_scan (dir_name, &add_file, NULL); - schedule_next_task (); -} - - -/** - * Decide what the next task is (working or scanning) and schedule it. - */ -static void -schedule_next_task () -{ - struct GNUNET_TIME_Relative delay; - - if (GNUNET_YES == do_shutdown) - return; - GNUNET_assert (NULL == run_task); - if (NULL == work_head) - { - /* delay by at most 4h, at least 1s, and otherwise in between depending - on how long it took to scan */ - delay = GNUNET_TIME_absolute_get_duration (start_time); - delay = GNUNET_TIME_relative_saturating_multiply (delay, 100); - delay = GNUNET_TIME_relative_min (delay, MAX_DELAY); - delay = GNUNET_TIME_relative_max (delay, MIN_DELAY); - run_task = GNUNET_SCHEDULER_add_delayed (delay, &scan, NULL); - } - else - { - run_task = GNUNET_SCHEDULER_add_now (&work, NULL); - } -} - - -/** - * Main function that will be run by the scheduler. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param c configuration - */ -static void -run (void *cls, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *c) -{ - /* check arguments */ - if ((NULL == args[0]) || (NULL != args[1]) || - (GNUNET_YES != GNUNET_DISK_directory_test (args[0], GNUNET_YES))) - { - printf (_ ( - "You must specify one and only one directory name for automatic publication.\n")); - ret = -1; - return; - } - cfg_filename = GNUNET_strdup (cfgfile); - cfg = c; - dir_name = args[0]; - work_finished = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO); - load_state (); - run_task = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, - &scan, - NULL); - GNUNET_SCHEDULER_add_shutdown (&do_stop_task, NULL); -} - - -/** - * Free memory associated with the work item from the work_finished map. - * - * @param cls NULL (unused) - * @param key key of the item in the map (unused) - * @param value the `struct WorkItem` to free - * @return #GNUNET_OK to continue to iterate - */ -static int -free_item (void *cls, const struct GNUNET_HashCode *key, void *value) -{ - struct WorkItem *wi = value; - - GNUNET_free (wi->filename); - GNUNET_free (wi); - return GNUNET_OK; -} - - -/** - * The main function to automatically publish content to GNUnet. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, char *const *argv) -{ - struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_option_uint ('a', - "anonymity", - "LEVEL", - gettext_noop ( - "set the desired LEVEL of sender-anonymity"), - &anonymity_level), - - GNUNET_GETOPT_option_flag ( - 'd', - "disable-creation-time", - gettext_noop ( - "disable adding the creation time to the metadata of the uploaded file"), - &do_disable_creation_time), - - GNUNET_GETOPT_option_flag ( - 'D', - "disable-extractor", - gettext_noop ("do not use libextractor to add keywords or metadata"), - &disable_extractor), - - GNUNET_GETOPT_option_uint ('p', - "priority", - "PRIORITY", - gettext_noop ( - "specify the priority of the content"), - &content_priority), - - GNUNET_GETOPT_option_uint ('r', - "replication", - "LEVEL", - gettext_noop ( - "set the desired replication LEVEL"), - &replication_level), - - GNUNET_GETOPT_option_verbose (&verbose), - - GNUNET_GETOPT_OPTION_END - }; - struct WorkItem *wi; - int ok; - struct GNUNET_SIGNAL_Context *shc_chld; - - if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) - return 2; - sigpipe = GNUNET_DISK_pipe (GNUNET_DISK_PF_NONE); - GNUNET_assert (NULL != sigpipe); - shc_chld = - GNUNET_SIGNAL_handler_install (GNUNET_SIGCHLD, &sighandler_child_death); - ok = - (GNUNET_OK == - GNUNET_PROGRAM_run ( - argc, - argv, - "gnunet-auto-share [OPTIONS] FILENAME", - gettext_noop ("Automatically publish files from a directory on GNUnet"), - options, - &run, - NULL)) - ? ret - : 1; - if (NULL != work_finished) - { - (void) GNUNET_CONTAINER_multihashmap_iterate (work_finished, - &free_item, - NULL); - GNUNET_CONTAINER_multihashmap_destroy (work_finished); - } - while (NULL != (wi = work_head)) - { - GNUNET_CONTAINER_DLL_remove (work_head, work_tail, wi); - GNUNET_free (wi->filename); - GNUNET_free (wi); - } - GNUNET_SIGNAL_handler_uninstall (shc_chld); - shc_chld = NULL; - GNUNET_DISK_pipe_close (sigpipe); - sigpipe = NULL; - GNUNET_free (cfg_filename); - cfg_filename = NULL; - GNUNET_free_nz ((void *) argv); - return ok; -} - - -/* end of gnunet-auto-share.c */ diff --git a/src/fs/gnunet-daemon-fsprofiler.c b/src/fs/gnunet-daemon-fsprofiler.c deleted file mode 100644 index b99933cfa..000000000 --- a/src/fs/gnunet-daemon-fsprofiler.c +++ /dev/null @@ -1,673 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2012 Christian Grothoff - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-daemon-fsprofiler.c - * @brief daemon that publishes and downloads (random) files - * @author Christian Grothoff - * - * TODO: - * - how to signal driver that we're done? - */ -#include "platform.h" - -#include "gnunet_fs_service.h" -#include "gnunet_statistics_service.h" - -/** - * We use 'patterns' of the form (x,y,t) to specify desired download/publish - * activities of a peer. They are stored in a DLL. - */ -struct Pattern -{ - /** - * Kept in a DLL. - */ - struct Pattern *next; - - /** - * Kept in a DLL. - */ - struct Pattern *prev; - - /** - * Execution context for the pattern (FS-handle to the operation). - */ - void *ctx; - - /** - * Secondary execution context for the pattern (FS-handle to the operation). - */ - void *sctx; - - /** - * When did the operation start? - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * With how much delay should this operation be started? - */ - struct GNUNET_TIME_Relative delay; - - /** - * Task to run the operation. - */ - struct GNUNET_SCHEDULER_Task *task; - - /** - * Secondary task to run the operation. - */ - struct GNUNET_SCHEDULER_Task *stask; - - /** - * X-value. - */ - unsigned long long x; - - /** - * Y-value. - */ - unsigned long long y; -}; - - -/** - * Return value from 'main'. - */ -static int global_ret; - -/** - * Configuration we use. - */ -static const struct GNUNET_CONFIGURATION_Handle *cfg; - -/** - * Handle to the statistics service. - */ -static struct GNUNET_STATISTICS_Handle *stats_handle; - -/** - * Peer's FS handle. - */ -static struct GNUNET_FS_Handle *fs_handle; - -/** - * Unique number for this peer in the testbed. - */ -static unsigned long long my_peerid; - -/** - * Desired anonymity level. - */ -static unsigned long long anonymity_level; - -/** - * Desired replication level. - */ -static unsigned long long replication_level; - -/** - * String describing which publishing operations this peer should - * perform. The format is "(SIZE,SEED,TIME)*", for example: - * "(1,5,0)(7,3,13)" means to publish a file with 1 byte and - * seed/keyword 5 immediately and another file with 7 bytes and - * seed/keyword 3 after 13 ms. - */ -static char *publish_pattern; - -/** - * Head of the DLL of publish patterns. - */ -static struct Pattern *publish_head; - -/** - * Tail of the DLL of publish patterns. - */ -static struct Pattern *publish_tail; - -/** - * String describing which download operations this peer should - * perform. The format is "(KEYWORD,SIZE,DELAY)*"; for example, - * "(1,7,3)(3,8,8)" means to download one file of 7 bytes under - * keyword "1" starting the search after 3 ms; and another one of 8 - * bytes under keyword '3' starting after 8 ms. The file size is - * used to determine which search result(s) should be used or ignored. - */ -static char *download_pattern; - -/** - * Head of the DLL of publish patterns. - */ -static struct Pattern *download_head; - -/** - * Tail of the DLL of publish patterns. - */ -static struct Pattern *download_tail; - - -/** - * Parse a pattern string and store the corresponding - * 'struct Pattern' in the given head/tail. - * - * @param head where to store the head - * @param tail where to store the tail - * @param pattern pattern to parse - * @return GNUNET_OK on success - */ -static int -parse_pattern (struct Pattern **head, - struct Pattern **tail, - const char *pattern) -{ - struct Pattern *p; - unsigned long long x; - unsigned long long y; - unsigned long long t; - - while (3 == sscanf (pattern, - "(%llu,%llu,%llu)", - &x, &y, &t)) - { - p = GNUNET_new (struct Pattern); - p->x = x; - p->y = y; - p->delay.rel_value_us = (uint64_t) t; - GNUNET_CONTAINER_DLL_insert (*head, *tail, p); - pattern = strstr (pattern, ")"); - GNUNET_assert (NULL != pattern); - pattern++; - } - return (0 == strlen (pattern)) ? GNUNET_OK : GNUNET_SYSERR; -} - - -/** - * Create a KSK URI from a number. - * - * @param kval the number - * @return corresponding KSK URI - */ -static struct GNUNET_FS_Uri * -make_keywords (uint64_t kval) -{ - char kw[128]; - - GNUNET_snprintf (kw, sizeof(kw), - "%llu", (unsigned long long) kval); - return GNUNET_FS_uri_ksk_create (kw, NULL); -} - - -/** - * Create a file of the given length with a deterministic amount - * of data to be published under keyword 'kval'. - * - * @param length number of bytes in the file - * @param kval keyword value and seed for the data of the file - * @param ctx context to pass to 'fi' - * @return file information handle for the file - */ -static struct GNUNET_FS_FileInformation * -make_file (uint64_t length, - uint64_t kval, - void *ctx) -{ - struct GNUNET_FS_FileInformation *fi; - struct GNUNET_FS_BlockOptions bo; - char *data; - struct GNUNET_FS_Uri *keywords; - unsigned long long i; - uint64_t xor; - - data = NULL; /* to make compilers happy */ - if ((0 != length) && - (NULL == (data = GNUNET_malloc_large ((size_t) length)))) - return NULL; - /* initialize data with 'unique' data only depending on 'kval' and 'size', - making sure that blocks do not repeat */ - for (i = 0; i < length; i += 8) - { - xor = length ^ kval ^ (uint64_t) (i / 32 / 1024); - GNUNET_memcpy (&data[i], &xor, GNUNET_MIN (length - i, sizeof(uint64_t))); - } - bo.expiration_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS); - bo.anonymity_level = (uint32_t) anonymity_level; - bo.content_priority = 128; - bo.replication_level = (uint32_t) replication_level; - keywords = make_keywords (kval); - fi = GNUNET_FS_file_information_create_from_data (fs_handle, - ctx, - length, - data, keywords, - NULL, GNUNET_NO, &bo); - GNUNET_FS_uri_destroy (keywords); - return fi; -} - - -/** - * Task run during shutdown. - * - * @param cls unused - */ -static void -shutdown_task (void *cls) -{ - struct Pattern *p; - - while (NULL != (p = publish_head)) - { - if (NULL != p->task) - GNUNET_SCHEDULER_cancel (p->task); - if (NULL != p->ctx) - GNUNET_FS_publish_stop (p->ctx); - GNUNET_CONTAINER_DLL_remove (publish_head, publish_tail, p); - GNUNET_free (p); - } - while (NULL != (p = download_head)) - { - if (NULL != p->task) - GNUNET_SCHEDULER_cancel (p->task); - if (NULL != p->stask) - GNUNET_SCHEDULER_cancel (p->stask); - if (NULL != p->ctx) - GNUNET_FS_download_stop (p->ctx, GNUNET_YES); - if (NULL != p->sctx) - GNUNET_FS_search_stop (p->sctx); - GNUNET_CONTAINER_DLL_remove (download_head, download_tail, p); - GNUNET_free (p); - } - if (NULL != fs_handle) - { - GNUNET_FS_stop (fs_handle); - fs_handle = NULL; - } - if (NULL != stats_handle) - { - GNUNET_STATISTICS_destroy (stats_handle, GNUNET_YES); - stats_handle = NULL; - } -} - - -/** - * Task run when a publish operation should be stopped. - * - * @param cls the 'struct Pattern' of the publish operation to stop - */ -static void -publish_stop_task (void *cls) -{ - struct Pattern *p = cls; - - p->task = NULL; - GNUNET_FS_publish_stop (p->ctx); -} - - -/** - * Task run when a download operation should be stopped. - * - * @param cls the 'struct Pattern' of the download operation to stop - */ -static void -download_stop_task (void *cls) -{ - struct Pattern *p = cls; - - p->task = NULL; - GNUNET_FS_download_stop (p->ctx, GNUNET_YES); -} - - -/** - * Task run when a download operation should be stopped. - * - * @param cls the 'struct Pattern' of the download operation to stop - */ -static void -search_stop_task (void *cls) -{ - struct Pattern *p = cls; - - p->stask = NULL; - GNUNET_FS_search_stop (p->sctx); -} - - -/** - * Notification of FS to a client about the progress of an - * operation. Callbacks of this type will be used for uploads, - * downloads and searches. Some of the arguments depend a bit - * in their meaning on the context in which the callback is used. - * - * @param cls closure - * @param info details about the event, specifying the event type - * and various bits about the event - * @return client-context (for the next progress call - * for this operation; should be set to NULL for - * SUSPEND and STOPPED events). The value returned - * will be passed to future callbacks in the respective - * field in the GNUNET_FS_ProgressInfo struct. - */ -static void * -progress_cb (void *cls, - const struct GNUNET_FS_ProgressInfo *info) -{ - struct Pattern *p; - const struct GNUNET_FS_Uri *uri; - - switch (info->status) - { - case GNUNET_FS_STATUS_PUBLISH_START: - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - p = info->value.publish.cctx; - return p; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - p = info->value.publish.cctx; - return p; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Publishing failed\n"); - GNUNET_STATISTICS_update (stats_handle, - "# failed publish operations", 1, GNUNET_NO); - p = info->value.publish.cctx; - p->task = GNUNET_SCHEDULER_add_now (&publish_stop_task, p); - return p; - - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - p = info->value.publish.cctx; - GNUNET_STATISTICS_update (stats_handle, - "# publishing time (ms)", - (long long) GNUNET_TIME_absolute_get_duration ( - p->start_time).rel_value_us / 1000LL, - GNUNET_NO); - p->task = GNUNET_SCHEDULER_add_now (&publish_stop_task, p); - return p; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - p = info->value.publish.cctx; - p->ctx = NULL; - GNUNET_CONTAINER_DLL_remove (publish_head, publish_tail, p); - GNUNET_free (p); - return NULL; - - case GNUNET_FS_STATUS_DOWNLOAD_START: - case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: - case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: - case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: - p = info->value.download.cctx; - return p; - - case GNUNET_FS_STATUS_DOWNLOAD_ERROR: - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Download failed\n"); - GNUNET_STATISTICS_update (stats_handle, - "# failed downloads", 1, GNUNET_NO); - p = info->value.download.cctx; - p->task = GNUNET_SCHEDULER_add_now (&download_stop_task, p); - return p; - - case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED: - p = info->value.download.cctx; - GNUNET_STATISTICS_update (stats_handle, - "# download time (ms)", - (long long) GNUNET_TIME_absolute_get_duration ( - p->start_time).rel_value_us / 1000LL, - GNUNET_NO); - p->task = GNUNET_SCHEDULER_add_now (&download_stop_task, p); - return p; - - case GNUNET_FS_STATUS_DOWNLOAD_STOPPED: - p = info->value.download.cctx; - p->ctx = NULL; - if (NULL == p->sctx) - { - GNUNET_CONTAINER_DLL_remove (download_head, download_tail, p); - GNUNET_free (p); - } - return NULL; - - case GNUNET_FS_STATUS_SEARCH_START: - case GNUNET_FS_STATUS_SEARCH_RESULT_NAMESPACE: - p = info->value.search.cctx; - return p; - - case GNUNET_FS_STATUS_SEARCH_RESULT: - p = info->value.search.cctx; - uri = info->value.search.specifics.result.uri; - if (GNUNET_YES != GNUNET_FS_uri_test_chk (uri)) - return NULL; /* not what we want */ - if (p->y != GNUNET_FS_uri_chk_get_file_size (uri)) - return NULL; /* not what we want */ - GNUNET_STATISTICS_update (stats_handle, - "# search time (ms)", - (long long) GNUNET_TIME_absolute_get_duration ( - p->start_time).rel_value_us / 1000LL, - GNUNET_NO); - p->start_time = GNUNET_TIME_absolute_get (); - p->ctx = GNUNET_FS_download_start (fs_handle, uri, - NULL, NULL, NULL, - 0, GNUNET_FS_uri_chk_get_file_size (uri), - anonymity_level, - GNUNET_FS_DOWNLOAD_NO_TEMPORARIES, - p, - NULL); - p->stask = GNUNET_SCHEDULER_add_now (&search_stop_task, p); - return NULL; - - case GNUNET_FS_STATUS_SEARCH_UPDATE: - case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED: - return NULL; /* don't care */ - - case GNUNET_FS_STATUS_SEARCH_ERROR: - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Search failed\n"); - GNUNET_STATISTICS_update (stats_handle, - "# failed searches", 1, GNUNET_NO); - p = info->value.search.cctx; - p->stask = GNUNET_SCHEDULER_add_now (&search_stop_task, p); - return p; - - case GNUNET_FS_STATUS_SEARCH_STOPPED: - p = info->value.search.cctx; - p->sctx = NULL; - if (NULL == p->ctx) - { - GNUNET_CONTAINER_DLL_remove (download_head, download_tail, p); - GNUNET_free (p); - } - return NULL; - - default: - /* unexpected event during profiling */ - GNUNET_break (0); - return NULL; - } -} - - -/** - * Start publish operation. - * - * @param cls the 'struct Pattern' specifying the operation to perform - */ -static void -start_publish (void *cls) -{ - struct Pattern *p = cls; - struct GNUNET_FS_FileInformation *fi; - - p->task = NULL; - fi = make_file (p->x, p->y, p); - p->start_time = GNUNET_TIME_absolute_get (); - p->ctx = GNUNET_FS_publish_start (fs_handle, - fi, - NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); -} - - -/** - * Start download operation. - * - * @param cls the 'struct Pattern' specifying the operation to perform - */ -static void -start_download (void *cls) -{ - struct Pattern *p = cls; - struct GNUNET_FS_Uri *keywords; - - p->task = NULL; - keywords = make_keywords (p->x); - p->start_time = GNUNET_TIME_absolute_get (); - p->sctx = GNUNET_FS_search_start (fs_handle, keywords, - anonymity_level, - GNUNET_FS_SEARCH_OPTION_NONE, - p); -} - - -/** - * @brief Main function that will be run by the scheduler. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param cfg_ configuration - */ -static void -run (void *cls, char *const *args GNUNET_UNUSED, - const char *cfgfile GNUNET_UNUSED, - const struct GNUNET_CONFIGURATION_Handle *cfg_) -{ - char myoptname[128]; - struct Pattern *p; - - cfg = cfg_; - /* Scheduled the task to clean up when shutdown is called */ - GNUNET_SCHEDULER_add_shutdown (&shutdown_task, - NULL); - - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_number (cfg, - "TESTBED", "PEERID", - &my_peerid)) - { - GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, - "TESTBED", "PEERID"); - global_ret = GNUNET_SYSERR; - GNUNET_SCHEDULER_shutdown (); - return; - } - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_number (cfg, - "FSPROFILER", "ANONYMITY_LEVEL", - &anonymity_level)) - anonymity_level = 1; - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_number (cfg, - "FSPROFILER", "REPLICATION_LEVEL", - &replication_level)) - replication_level = 1; - GNUNET_snprintf (myoptname, sizeof(myoptname), - "DOWNLOAD-PATTERN-%llu", my_peerid); - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (cfg, - "FSPROFILER", myoptname, - &download_pattern)) - download_pattern = GNUNET_strdup (""); - GNUNET_snprintf (myoptname, sizeof(myoptname), - "PUBLISH-PATTERN-%llu", my_peerid); - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (cfg, - "FSPROFILER", myoptname, - &publish_pattern)) - publish_pattern = GNUNET_strdup (""); - if ((GNUNET_OK != - parse_pattern (&download_head, - &download_tail, - download_pattern)) || - (GNUNET_OK != - parse_pattern (&publish_head, - &publish_tail, - publish_pattern))) - { - GNUNET_SCHEDULER_shutdown (); - return; - } - - stats_handle = GNUNET_STATISTICS_create ("fsprofiler", cfg); - fs_handle = - GNUNET_FS_start (cfg, - "fsprofiler", - &progress_cb, NULL, - GNUNET_FS_FLAGS_NONE, - GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM, 1, - GNUNET_FS_OPTIONS_REQUEST_PARALLELISM, 1, - GNUNET_FS_OPTIONS_END); - if (NULL == fs_handle) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Could not acquire FS handle. Exiting.\n"); - global_ret = GNUNET_SYSERR; - GNUNET_SCHEDULER_shutdown (); - return; - } - for (p = publish_head; NULL != p; p = p->next) - p->task = GNUNET_SCHEDULER_add_delayed (p->delay, - &start_publish, p); - for (p = download_head; NULL != p; p = p->next) - p->task = GNUNET_SCHEDULER_add_delayed (p->delay, - &start_download, p); -} - - -/** - * Program that performs various "random" FS activities. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, char *const *argv) -{ - static const struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_OPTION_END - }; - - if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) - return 2; - return (GNUNET_OK == - GNUNET_PROGRAM_run (argc, argv, "gnunet-daemon-fsprofiler", - gettext_noop - ( - "Daemon to use file-sharing to measure its performance."), - options, &run, NULL)) ? global_ret : 1; -} - - -/* end of gnunet-daemon-fsprofiler.c */ diff --git a/src/fs/gnunet-directory.c b/src/fs/gnunet-directory.c deleted file mode 100644 index ab9f2905a..000000000 --- a/src/fs/gnunet-directory.c +++ /dev/null @@ -1,212 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/gnunet-directory.c - * @brief display content of GNUnet directories - * @author Christian Grothoff - */ -#include "platform.h" - -#include "gnunet_fs_service.h" - -static int ret; - -/** - * Print a meta data entry. - * - * @param cls closure (unused) - * @param plugin_name name of the plugin that generated the meta data - * @param type type of the keyword - * @param format format of data - * @param data_mime_type mime type of data - * @param data value of the meta data - * @param data_size number of bytes in @a data - * @return always 0 (to continue iterating) - */ -static int -item_printer (void *cls, - const char *plugin_name, - enum EXTRACTOR_MetaType type, - enum EXTRACTOR_MetaFormat format, - const char *data_mime_type, - const char *data, - size_t data_size) -{ - if (type == EXTRACTOR_METATYPE_GNUNET_FULL_DATA) - { - printf (_ ("\t\n"), - (unsigned int) data_size); - return 0; - } - if ((format != EXTRACTOR_METAFORMAT_UTF8) && - (format != EXTRACTOR_METAFORMAT_C_STRING)) - return 0; - if (type == EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME) - return 0; -#if HAVE_LIBEXTRACTOR - printf ("\t%20s: %s\n", - dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, - EXTRACTOR_metatype_to_string (type)), - data); -#else - printf ("\t%20d: %s\n", type, data); -#endif - return 0; -} - - -/** - * Print an entry in a directory. - * - * @param cls closure (not used) - * @param filename name of the file in the directory - * @param uri URI of the file - * @param meta metadata for the file; metadata for - * the directory if everything else is NULL/zero - * @param length length of the available data for the file - * (of type size_t since data must certainly fit - * into memory; if files are larger than size_t - * permits, then they will certainly not be - * embedded with the directory itself). - * @param data data available for the file (length bytes) - */ -static void -print_entry (void *cls, - const char *filename, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *meta, - size_t length, - const void *data) -{ - char *string; - char *name; - - name = GNUNET_FS_meta_data_get_by_type ( - meta, - EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); - if (uri == NULL) - { - printf (_ ("Directory `%s' meta data:\n"), name ? name : ""); - GNUNET_FS_meta_data_iterate (meta, &item_printer, NULL); - printf ("\n"); - printf (_ ("Directory `%s' contents:\n"), name ? name : ""); - GNUNET_free (name); - return; - } - string = GNUNET_FS_uri_to_string (uri); - printf ("%s (%s):\n", name ? name : "", string); - GNUNET_free (string); - GNUNET_FS_meta_data_iterate (meta, &item_printer, NULL); - printf ("\n"); - GNUNET_free (name); -} - - -/** - * Main function that will be run by the scheduler. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param cfg configuration - */ -static void -run (void *cls, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *cfg) -{ - struct GNUNET_DISK_MapHandle *map; - struct GNUNET_DISK_FileHandle *h; - void *data; - size_t len; - uint64_t size; - const char *filename; - int i; - - if (NULL == args[0]) - { - fprintf (stderr, "%s", _ ("You must specify a filename to inspect.\n")); - ret = 1; - return; - } - i = 0; - while (NULL != (filename = args[i++])) - { - if ((GNUNET_OK != - GNUNET_DISK_file_size (filename, &size, GNUNET_YES, GNUNET_YES)) || - (NULL == (h = GNUNET_DISK_file_open (filename, - GNUNET_DISK_OPEN_READ, - GNUNET_DISK_PERM_NONE)))) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Failed to read directory `%s'\n"), - filename); - ret = 1; - continue; - } - len = (size_t) size; - data = GNUNET_DISK_file_map (h, &map, GNUNET_DISK_MAP_TYPE_READ, len); - GNUNET_assert (NULL != data); - if (GNUNET_OK != - GNUNET_FS_directory_list_contents (len, data, 0, &print_entry, NULL)) - fprintf (stdout, _ ("`%s' is not a GNUnet directory\n"), filename); - else - printf ("\n"); - GNUNET_DISK_file_unmap (map); - GNUNET_DISK_file_close (h); - } -} - - -/** - * The main function to inspect GNUnet directories. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, char *const *argv) -{ - static struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_OPTION_END - }; - - if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) - return 2; - - ret = (GNUNET_OK == - GNUNET_PROGRAM_run (argc, - argv, - "gnunet-directory [OPTIONS] FILENAME", - gettext_noop ( - "Display contents of a GNUnet directory"), - options, - &run, - NULL)) - ? ret - : 1; - GNUNET_free_nz ((void *) argv); - return ret; -} - - -/* end of gnunet-directory.c */ diff --git a/src/fs/gnunet-download.c b/src/fs/gnunet-download.c deleted file mode 100644 index 4694077e9..000000000 --- a/src/fs/gnunet-download.c +++ /dev/null @@ -1,385 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/gnunet-download.c - * @brief downloading for files on GNUnet - * @author Christian Grothoff - * @author Krista Bennett - * @author James Blackwell - * @author Igor Wronsky - */ -#include "platform.h" - -#include "gnunet_fs_service.h" - -static int ret; - -static unsigned int verbose; - -static int delete_incomplete; - -static const struct GNUNET_CONFIGURATION_Handle *cfg; - -static struct GNUNET_FS_Handle *ctx; - -static struct GNUNET_FS_DownloadContext *dc; - -static unsigned int anonymity = 1; - -static unsigned int parallelism = 16; - -static unsigned int request_parallelism = 4092; - -static int do_recursive; - -static char *filename; - -static int local_only; - - -static void -cleanup_task (void *cls) -{ - GNUNET_FS_stop (ctx); - ctx = NULL; -} - - -static void -shutdown_task (void *cls) -{ - if (NULL != dc) - { - GNUNET_FS_download_stop (dc, delete_incomplete); - dc = NULL; - } -} - - -/** - * Display progress bar (if tty). - * - * @param x current position in the download - * @param n total size of the download - * @param w desired number of steps in the progress bar - */ -static void -display_bar (unsigned long long x, unsigned long long n, unsigned int w) -{ - char buf[w + 20]; - unsigned int p; - unsigned int endeq; - float ratio_complete; - - if (0 == isatty (1)) - return; - ratio_complete = x / (float) n; - endeq = ratio_complete * w; - GNUNET_snprintf (buf, sizeof(buf), "%3d%% [", (int) (ratio_complete * 100)); - for (p = 0; p < endeq; p++) - strcat (buf, "="); - for (p = endeq; p < w; p++) - strcat (buf, " "); - strcat (buf, "]\r"); - printf ("%s", buf); - fflush (stdout); -} - - -/** - * Called by FS client to give information about the progress of an - * operation. - * - * @param cls closure - * @param info details about the event, specifying the event type - * and various bits about the event - * @return client-context (for the next progress call - * for this operation; should be set to NULL for - * SUSPEND and STOPPED events). The value returned - * will be passed to future callbacks in the respective - * field in the `struct GNUNET_FS_ProgressInfo` - */ -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *info) -{ - char *s; - const char *s2; - char *t; - - switch (info->status) - { - case GNUNET_FS_STATUS_DOWNLOAD_START: - if (verbose > 1) - fprintf (stderr, - _ ("Starting download `%s'.\n"), - info->value.download.filename); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: - if (verbose) - { - s = GNUNET_strdup ( - GNUNET_STRINGS_relative_time_to_string (info->value.download.eta, - GNUNET_YES)); - if (info->value.download.specifics.progress.block_download_duration - .rel_value_us == GNUNET_TIME_UNIT_FOREVER_REL.rel_value_us) - s2 = _ (""); - else - s2 = GNUNET_STRINGS_relative_time_to_string (info->value.download - .specifics.progress - .block_download_duration, - GNUNET_YES); - t = GNUNET_STRINGS_byte_size_fancy ( - info->value.download.completed * 1000LL - / (info->value.download.duration.rel_value_us + 1)); - fprintf ( - stdout, - _ ( - "Downloading `%s' at %llu/%llu (%s remaining, %s/s). Block took %s to download\n"), - info->value.download.filename, - (unsigned long long) info->value.download.completed, - (unsigned long long) info->value.download.size, - s, - t, - s2); - GNUNET_free (s); - GNUNET_free (t); - } - else - { - display_bar (info->value.download.completed, - info->value.download.size, - 60); - } - break; - - case GNUNET_FS_STATUS_DOWNLOAD_ERROR: - if (0 != isatty (1)) - fprintf (stdout, "\n"); - fprintf (stderr, - _ ("Error downloading: %s.\n"), - info->value.download.specifics.error.message); - GNUNET_SCHEDULER_shutdown (); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED: - s = GNUNET_STRINGS_byte_size_fancy ( - info->value.download.completed * 1000 - / (info->value.download.duration.rel_value_us + 1)); - if (0 != isatty (1)) - fprintf (stdout, "\n"); - fprintf (stdout, - _ ("Downloading `%s' done (%s/s).\n"), - info->value.download.filename, - s); - GNUNET_free (s); - if (info->value.download.dc == dc) - GNUNET_SCHEDULER_shutdown (); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_STOPPED: - if (info->value.download.dc == dc) - GNUNET_SCHEDULER_add_now (&cleanup_task, NULL); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: - case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: - break; - - default: - fprintf (stderr, _ ("Unexpected status: %d\n"), info->status); - break; - } - return NULL; -} - - -/** - * Main function that will be run by the scheduler. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param c configuration - */ -static void -run (void *cls, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *c) -{ - struct GNUNET_FS_Uri *uri; - char *emsg; - enum GNUNET_FS_DownloadOptions options; - - if (NULL == args[0]) - { - fprintf (stderr, "%s", _ ("You need to specify a URI argument.\n")); - return; - } - uri = GNUNET_FS_uri_parse (args[0], &emsg); - if (NULL == uri) - { - fprintf (stderr, _ ("Failed to parse URI: %s\n"), emsg); - GNUNET_free (emsg); - ret = 1; - return; - } - if ((! GNUNET_FS_uri_test_chk (uri)) && (! GNUNET_FS_uri_test_loc (uri))) - { - fprintf (stderr, "%s", _ ("Only CHK or LOC URIs supported.\n")); - ret = 1; - GNUNET_FS_uri_destroy (uri); - return; - } - if (NULL == filename) - { - fprintf (stderr, "%s", _ ("Target filename must be specified.\n")); - ret = 1; - GNUNET_FS_uri_destroy (uri); - return; - } - cfg = c; - ctx = GNUNET_FS_start (cfg, - "gnunet-download", - &progress_cb, - NULL, - GNUNET_FS_FLAGS_NONE, - GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM, - parallelism, - GNUNET_FS_OPTIONS_REQUEST_PARALLELISM, - request_parallelism, - GNUNET_FS_OPTIONS_END); - if (NULL == ctx) - { - fprintf (stderr, _ ("Could not initialize `%s' subsystem.\n"), "FS"); - GNUNET_FS_uri_destroy (uri); - ret = 1; - return; - } - options = GNUNET_FS_DOWNLOAD_OPTION_NONE; - if (do_recursive) - options |= GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE; - if (local_only) - options |= GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY; - dc = GNUNET_FS_download_start (ctx, - uri, - NULL, - filename, - NULL, - 0, - GNUNET_FS_uri_chk_get_file_size (uri), - anonymity, - options, - NULL, - NULL); - GNUNET_FS_uri_destroy (uri); - if (dc == NULL) - { - GNUNET_FS_stop (ctx); - ctx = NULL; - return; - } - GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL); -} - - -/** - * The main function to download GNUnet. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, char *const *argv) -{ - struct GNUNET_GETOPT_CommandLineOption options[] = - { GNUNET_GETOPT_option_uint ('a', - "anonymity", - "LEVEL", - gettext_noop ( - "set the desired LEVEL of receiver-anonymity"), - &anonymity), - - GNUNET_GETOPT_option_flag ( - 'D', - "delete-incomplete", - gettext_noop ("delete incomplete downloads (when aborted with CTRL-C)"), - &delete_incomplete), - - GNUNET_GETOPT_option_flag ( - 'n', - "no-network", - gettext_noop ("only search the local peer (no P2P network search)"), - &local_only), - GNUNET_GETOPT_option_string ('o', - "output", - "FILENAME", - gettext_noop ("write the file to FILENAME"), - &filename), - GNUNET_GETOPT_option_uint ( - 'p', - "parallelism", - "DOWNLOADS", - gettext_noop ( - "set the maximum number of parallel downloads that is allowed"), - ¶llelism), - GNUNET_GETOPT_option_uint ( - 'r', - "request-parallelism", - "REQUESTS", - gettext_noop ( - "set the maximum number of parallel requests for blocks that is allowed"), - &request_parallelism), - GNUNET_GETOPT_option_flag ('R', - "recursive", - gettext_noop ( - "download a GNUnet directory recursively"), - &do_recursive), - GNUNET_GETOPT_option_increment_uint ( - 'V', - "verbose", - gettext_noop ("be verbose (print progress information)"), - &verbose), - GNUNET_GETOPT_OPTION_END }; - - if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) - return 2; - - ret = - (GNUNET_OK == - GNUNET_PROGRAM_run ( - argc, - argv, - "gnunet-download [OPTIONS] URI", - gettext_noop ( - "Download files from GNUnet using a GNUnet CHK or LOC URI (gnunet://fs/chk/...)"), - options, - &run, - NULL)) - ? ret - : 1; - GNUNET_free_nz ((void *) argv); - return ret; -} - - -/* end of gnunet-download.c */ diff --git a/src/fs/gnunet-fs-profiler.c b/src/fs/gnunet-fs-profiler.c deleted file mode 100644 index 62da46834..000000000 --- a/src/fs/gnunet-fs-profiler.c +++ /dev/null @@ -1,245 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-fs-profiler.c - * @brief tool to benchmark/profile file-sharing - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testbed_service.h" - -/** - * Final status code. - */ -static int ret; - -/** - * Data file with the hosts for the testbed. - */ -static char *host_filename; - -/** - * Number of peers to run in the experiment. - */ -static unsigned int num_peers; - -/** - * After how long do we abort the test? - */ -static struct GNUNET_TIME_Relative timeout; - -/** - * Handle to the task run during termination. - */ -static struct GNUNET_SCHEDULER_Task *terminate_taskid; - - -/** - * Function called after we've collected the statistics. - * - * @param cls NULL - * @param op the operation that has been finished - * @param emsg error message in case the operation has failed; will be NULL if - * operation has executed successfully. - */ -static void -shutdown_task (void *cls, - struct GNUNET_TESTBED_Operation *op, - const char *emsg) -{ - if (NULL != emsg) - fprintf (stderr, - "Error collecting statistics: %s\n", - emsg); - GNUNET_SCHEDULER_shutdown (); -} - - -/** - * Callback function to process statistic values from all peers. - * Prints them out. - * - * @param cls closure - * @param peer the peer the statistic belong to - * @param subsystem name of subsystem that created the statistic - * @param name the name of the datum - * @param value the current value - * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not - * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration - */ -static int -process_stats (void *cls, - const struct GNUNET_TESTBED_Peer *peer, - const char *subsystem, - const char *name, - uint64_t value, - int is_persistent) -{ - fprintf (stdout, - "%p-%s: %s = %llu\n", - peer, - subsystem, - name, - (unsigned long long) value); - return GNUNET_OK; -} - - -/** - * Task run on shutdown to terminate. Triggers printing out - * all statistics. - * - * @param cls NULL - */ -static void -terminate_task (void *cls) -{ - if (NULL != terminate_taskid) - { - GNUNET_SCHEDULER_cancel (terminate_taskid); - terminate_taskid = NULL; - } - GNUNET_TESTBED_get_statistics (0, NULL, - NULL, NULL, - &process_stats, - &shutdown_task, - NULL); -} - - -/** - * Task run on timeout to terminate. Triggers printing out - * all statistics. - * - * @param cls NULL - */ -static void -timeout_task (void *cls) -{ - terminate_taskid = NULL; - GNUNET_SCHEDULER_shutdown (); -} - - -/** - * Signature of a main function for a testcase. - * - * @param cls closure - * @param h the run handle - * @param num_peers number of peers in 'peers' - * @param peers handle to peers run in the testbed - * @param links_succeeded the number of overlay link connection attempts that - * succeeded - * @param links_failed the number of overlay link connection attempts that - * failed - */ -static void -test_master (void *cls, - struct GNUNET_TESTBED_RunHandle *h, - unsigned int num_peers, - struct GNUNET_TESTBED_Peer **peers, - unsigned int links_succeeded, - unsigned int links_failed) -{ - // const struct GNUNET_CONFIGURATION_Handle *cfg = cls; - // FIXME: enable clients to signal 'completion' before timeout; - // in that case, run the 'terminate_task' "immediately" - - if (0 != timeout.rel_value_us) - terminate_taskid = GNUNET_SCHEDULER_add_delayed (timeout, - &timeout_task, - NULL); - GNUNET_SCHEDULER_add_shutdown (&terminate_task, - NULL); -} - - -/** - * Main function that will be run by the scheduler. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param cfg configuration - */ -static void -run (void *cls, char *const *args, const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *cfg) -{ - GNUNET_TESTBED_run (host_filename, - cfg, - num_peers, - 0, NULL, NULL, - &test_master, (void *) cfg); -} - - -/** - * Program to run a file-sharing testbed. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, char *const *argv) -{ - struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_option_uint ('n', - "num-peers", - "COUNT", - gettext_noop ( - "run the experiment with COUNT peers"), - &num_peers), - - GNUNET_GETOPT_option_string ('H', - "hosts", - "HOSTFILE", - gettext_noop ( - "specifies name of a file with the HOSTS the testbed should use"), - &host_filename), - - GNUNET_GETOPT_option_relative_time ('t', - "timeout", - "DELAY", - gettext_noop ( - "automatically terminate experiment after DELAY"), - &timeout), - - GNUNET_GETOPT_OPTION_END - }; - - if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) - return 2; - - ret = (GNUNET_OK == - GNUNET_PROGRAM_run (argc, argv, "gnunet-fs-profiler", - gettext_noop ( - "run a testbed to measure file-sharing performance"), - options, &run, - NULL)) ? ret : 1; - GNUNET_free_nz ((void *) argv); - return ret; -} - - -/* end of gnunet-fs-profiler.c */ diff --git a/src/fs/gnunet-fs.c b/src/fs/gnunet-fs.c deleted file mode 100644 index 21e3c4a40..000000000 --- a/src/fs/gnunet-fs.c +++ /dev/null @@ -1,191 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2011 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/gnunet-fs.c - * @brief special file-sharing functions - * @author Christian Grothoff - */ -#include "platform.h" - -#include "gnunet_fs_service.h" - -/** - * Return value. - */ -static int ret; - -/** - * Handle to FS service. - */ -static struct GNUNET_FS_Handle *fs; - -/** - * Handle for the index listing operation. - */ -static struct GNUNET_FS_GetIndexedContext *gic; - -/** - * Option -i given? - */ -static int list_indexed_files; - -/** - * Option -v given? - */ -static unsigned int verbose; - - -/** - * Print indexed filenames to stdout. - * - * @param cls closure - * @param filename the name of the file - * @param file_id hash of the contents of the indexed file - * @return #GNUNET_OK to continue iteration - */ -static enum GNUNET_GenericReturnValue -print_indexed (void *cls, - const char *filename, - const struct GNUNET_HashCode *file_id) -{ - if (NULL == filename) - { - gic = NULL; - GNUNET_SCHEDULER_shutdown (); - return GNUNET_OK; - } - if (verbose) - fprintf (stdout, - "%s: %s\n", - GNUNET_h2s (file_id), - filename); - else - fprintf (stdout, - "%s\n", - filename); - return GNUNET_OK; -} - - -/** - * Function run on shutdown. - * - * @param cls NULL - */ -static void -do_shutdown (void *cls) -{ - (void) cls; - if (NULL != gic) - { - GNUNET_FS_get_indexed_files_cancel (gic); - gic = NULL; - } - if (NULL != fs) - { - GNUNET_FS_stop (fs); - fs = NULL; - } -} - - -/** - * Main function that will be run by the scheduler. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param cfg configuration - */ -static void -run (void *cls, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *cfg) -{ - if (! list_indexed_files) - return; - GNUNET_SCHEDULER_add_shutdown (&do_shutdown, - NULL); - fs = GNUNET_FS_start (cfg, - "gnunet-fs", - NULL, - NULL, - GNUNET_FS_FLAGS_NONE, - GNUNET_FS_OPTIONS_END); - if (NULL == fs) - { - ret = 1; - return; - } - gic = GNUNET_FS_get_indexed_files (fs, - &print_indexed, - NULL); - if (NULL == gic) - { - ret = 2; - GNUNET_SCHEDULER_shutdown (); - return; - } -} - - -/** - * The main function to access special file-sharing functions. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, - char *const *argv) -{ - struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_option_flag ('i', - "list-indexed", - gettext_noop ( - "print a list of all indexed files"), - &list_indexed_files), - - GNUNET_GETOPT_option_verbose (&verbose), - GNUNET_GETOPT_OPTION_END - }; - - if (GNUNET_OK != - GNUNET_STRINGS_get_utf8_args (argc, argv, - &argc, &argv)) - return 2; - ret = (GNUNET_OK == - GNUNET_PROGRAM_run (argc, - argv, - "gnunet-fs [OPTIONS]", - gettext_noop ("Special file-sharing operations"), - options, - &run, - NULL)) - ? ret - : 1; - GNUNET_free_nz ((void *) argv); - return ret; -} - - -/* end of gnunet-fs.c */ diff --git a/src/fs/gnunet-helper-fs-publish.c b/src/fs/gnunet-helper-fs-publish.c deleted file mode 100644 index 0e07b79dc..000000000 --- a/src/fs/gnunet-helper-fs-publish.c +++ /dev/null @@ -1,579 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file src/fs/gnunet-helper-fs-publish.c - * @brief Tool to help extract meta data asynchronously - * @author Christian Grothoff - * - * This program will scan a directory for files with meta data - * and report the results to stdout. - */ -#include "platform.h" - -#include "gnunet_fs_service.h" - - -/** - * A node of a directory tree. - */ -struct ScanTreeNode -{ - /** - * This is a doubly-linked list - */ - struct ScanTreeNode *next; - - /** - * This is a doubly-linked list - */ - struct ScanTreeNode *prev; - - /** - * Parent of this node, NULL for top-level entries. - */ - struct ScanTreeNode *parent; - - /** - * This is a doubly-linked tree - * NULL for files and empty directories - */ - struct ScanTreeNode *children_head; - - /** - * This is a doubly-linked tree - * NULL for files and empty directories - */ - struct ScanTreeNode *children_tail; - - /** - * Name of the file/directory - */ - char *filename; - - /** - * Size of the file (if it is a file), in bytes. - * At the moment it is set to 0 for directories. - */ - uint64_t file_size; - - /** - * #GNUNET_YES if this is a directory - */ - int is_directory; -}; - - -#if HAVE_LIBEXTRACTOR -/** - * List of libextractor plugins to use for extracting. - */ -static struct EXTRACTOR_PluginList *plugins; -#endif - -/** - * File descriptor we use for IPC with the parent. - */ -static int output_stream; - - -#if HAVE_LIBEXTRACTOR -/** - * Add meta data that libextractor finds to our meta data - * container. - * - * @param cls closure, our meta data container - * @param plugin_name name of the plugin that produced this value; - * special values can be used (e.g. '<zlib>' for zlib being - * used in the main libextractor library and yielding - * meta data). - * @param type libextractor-type describing the meta data - * @param format basic format information about data - * @param data_mime_type mime-type of data (not of the original file); - * can be NULL (if mime-type is not known) - * @param data actual meta-data found - * @param data_len number of bytes in @a data - * @return always 0 to continue extracting - */ -static int -add_to_md (void *cls, - const char *plugin_name, - enum EXTRACTOR_MetaType type, - enum EXTRACTOR_MetaFormat format, - const char *data_mime_type, - const char *data, - size_t data_len) -{ - struct GNUNET_FS_MetaData *md = cls; - - if (((EXTRACTOR_METAFORMAT_UTF8 == format) || - (EXTRACTOR_METAFORMAT_C_STRING == format)) && - ('\0' != data[data_len - 1])) - { - char zdata[data_len + 1]; - GNUNET_memcpy (zdata, data, data_len); - zdata[data_len] = '\0'; - (void) GNUNET_FS_meta_data_insert (md, - plugin_name, - type, - format, - data_mime_type, - zdata, - data_len + 1); - } - else - { - (void) GNUNET_FS_meta_data_insert (md, - plugin_name, - type, - format, - data_mime_type, - data, - data_len); - } - return 0; -} - - -#endif - - -/** - * Free memory of the @a tree structure - * - * @param tree tree to free - */ -static void -free_tree (struct ScanTreeNode *tree) -{ - struct ScanTreeNode *pos; - - while (NULL != (pos = tree->children_head)) - free_tree (pos); - if (NULL != tree->parent) - GNUNET_CONTAINER_DLL_remove (tree->parent->children_head, - tree->parent->children_tail, - tree); - GNUNET_free (tree->filename); - GNUNET_free (tree); -} - - -/** - * Write @a size bytes from @a buf into the #output_stream. - * - * @param buf buffer with data to write - * @param size number of bytes to write - * @return #GNUNET_OK on success, #GNUNET_SYSERR on error - */ -static int -write_all (const void *buf, size_t size) -{ - const char *cbuf = buf; - size_t total; - ssize_t wr; - - total = 0; - do - { - wr = write (output_stream, &cbuf[total], size - total); - if (wr > 0) - total += wr; - } - while ((wr > 0) && (total < size)); - if (wr <= 0) - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Failed to write to stdout: %s\n", - strerror (errno)); - return (total == size) ? GNUNET_OK : GNUNET_SYSERR; -} - - -/** - * Write message to the master process. - * - * @param message_type message type to use - * @param data data to append, NULL for none - * @param data_length number of bytes in @a data - * @return #GNUNET_SYSERR to stop scanning (the pipe was broken somehow) - */ -static int -write_message (uint16_t message_type, const char *data, size_t data_length) -{ - struct GNUNET_MessageHeader hdr; - -#if 0 - fprintf (stderr, - "Helper sends %u-byte message of type %u\n", - (unsigned int) (sizeof(struct GNUNET_MessageHeader) + data_length), - (unsigned int) message_type); -#endif - hdr.type = htons (message_type); - hdr.size = htons (sizeof(struct GNUNET_MessageHeader) + data_length); - if ((GNUNET_OK != write_all (&hdr, sizeof(hdr))) || - (GNUNET_OK != write_all (data, data_length))) - return GNUNET_SYSERR; - return GNUNET_OK; -} - - -/** - * Function called to (recursively) add all of the files in the - * directory to the tree. Called by the directory scanner to initiate - * the scan. Does NOT yet add any metadata. - * - * @param filename file or directory to scan - * @param dst where to store the resulting share tree item; - * NULL is stored in @a dst upon recoverable errors (#GNUNET_OK is returned) - * @return #GNUNET_OK on success, #GNUNET_SYSERR on error - */ -static int -preprocess_file (const char *filename, struct ScanTreeNode **dst); - - -/** - * Closure for the 'scan_callback' - */ -struct RecursionContext -{ - /** - * Parent to add the files to. - */ - struct ScanTreeNode *parent; - - /** - * Flag to set to GNUNET_YES on serious errors. - */ - int stop; -}; - - -/** - * Function called by the directory iterator to (recursively) add all - * of the files in the directory to the tree. Called by the directory - * scanner to initiate the scan. Does NOT yet add any metadata. - * - * @param cls the `struct RecursionContext` - * @param filename file or directory to scan - * @return #GNUNET_OK on success, #GNUNET_SYSERR on error - */ -static int -scan_callback (void *cls, const char *filename) -{ - struct RecursionContext *rc = cls; - struct ScanTreeNode *chld; - - if (GNUNET_OK != preprocess_file (filename, &chld)) - { - rc->stop = GNUNET_YES; - return GNUNET_SYSERR; - } - if (NULL == chld) - return GNUNET_OK; - chld->parent = rc->parent; - GNUNET_CONTAINER_DLL_insert (rc->parent->children_head, - rc->parent->children_tail, - chld); - return GNUNET_OK; -} - - -/** - * Function called to (recursively) add all of the files in the - * directory to the tree. Called by the directory scanner to initiate - * the scan. Does NOT yet add any metadata. - * - * @param filename file or directory to scan - * @param dst where to store the resulting share tree item; - * NULL is stored in @a dst upon recoverable errors (#GNUNET_OK is returned) - * @return #GNUNET_OK on success, #GNUNET_SYSERR on error - */ -static int -preprocess_file (const char *filename, struct ScanTreeNode **dst) -{ - struct ScanTreeNode *item; - struct stat sbuf; - uint64_t fsize = 0; - - if ((0 != stat (filename, &sbuf)) || - ((! S_ISDIR (sbuf.st_mode)) && - (GNUNET_OK != - GNUNET_DISK_file_size (filename, &fsize, GNUNET_NO, GNUNET_YES)))) - { - /* If the file doesn't exist (or is not stat-able for any other reason) - skip it (but report it), but do continue. */ - if (GNUNET_OK != - write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, - filename, - strlen (filename) + 1)) - return GNUNET_SYSERR; - /* recoverable error, store 'NULL' in *dst */ - *dst = NULL; - return GNUNET_OK; - } - - /* Report the progress */ - if ( - GNUNET_OK != - write_message (S_ISDIR (sbuf.st_mode) - ? GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY - : GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, - filename, - strlen (filename) + 1)) - return GNUNET_SYSERR; - item = GNUNET_new (struct ScanTreeNode); - item->filename = GNUNET_strdup (filename); - item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO; - item->file_size = fsize; - if (GNUNET_YES == item->is_directory) - { - struct RecursionContext rc; - - rc.parent = item; - rc.stop = GNUNET_NO; - GNUNET_DISK_directory_scan (filename, &scan_callback, &rc); - if ( - (GNUNET_YES == rc.stop) || - (GNUNET_OK != - write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, - "..", - 3))) - { - free_tree (item); - return GNUNET_SYSERR; - } - } - *dst = item; - return GNUNET_OK; -} - - -/** - * Extract metadata from files. - * - * @param item entry we are processing - * @return #GNUNET_OK on success, #GNUNET_SYSERR on fatal errors - */ -static int -extract_files (struct ScanTreeNode *item) -{ - struct GNUNET_FS_MetaData *meta; - ssize_t size; - size_t slen; - - if (GNUNET_YES == item->is_directory) - { - /* for directories, we simply only descent, no extraction, no - progress reporting */ - struct ScanTreeNode *pos; - - for (pos = item->children_head; NULL != pos; pos = pos->next) - if (GNUNET_OK != extract_files (pos)) - return GNUNET_SYSERR; - return GNUNET_OK; - } - - /* this is the expensive operation, *afterwards* we'll check for aborts */ - meta = GNUNET_FS_meta_data_create (); -#if HAVE_LIBEXTRACTOR - EXTRACTOR_extract (plugins, item->filename, NULL, 0, &add_to_md, meta); -#endif - slen = strlen (item->filename) + 1; - size = GNUNET_FS_meta_data_get_serialized_size (meta); - if (-1 == size) - { - /* no meta data */ - GNUNET_FS_meta_data_destroy (meta); - if (GNUNET_OK != - write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, - item->filename, - slen)) - return GNUNET_SYSERR; - return GNUNET_OK; - } - else if (size > (UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen)) - { - /* We can't transfer more than 64k bytes in one message. */ - size = UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen; - } - { - char buf[size + slen]; - char *dst = &buf[slen]; - - GNUNET_memcpy (buf, item->filename, slen); - size = GNUNET_FS_meta_data_serialize ( - meta, - &dst, - size, - GNUNET_FS_META_DATA_SERIALIZE_PART); - if (size < 0) - { - GNUNET_break (0); - size = 0; - } - GNUNET_FS_meta_data_destroy (meta); - if (GNUNET_OK != - write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, - buf, - slen + size)) - return GNUNET_SYSERR; - } - return GNUNET_OK; -} - - -/** - * Install a signal handler to ignore SIGPIPE. - */ -static void -ignore_sigpipe () -{ - struct sigaction oldsig; - struct sigaction sig; - - memset (&sig, 0, sizeof(struct sigaction)); - sig.sa_handler = SIG_IGN; - sigemptyset (&sig.sa_mask); -#ifdef SA_INTERRUPT - sig.sa_flags = SA_INTERRUPT; /* SunOS */ -#else - sig.sa_flags = SA_RESTART; -#endif - if (0 != sigaction (SIGPIPE, &sig, &oldsig)) - fprintf (stderr, - "Failed to install SIGPIPE handler: %s\n", - strerror (errno)); -} - - -/** - * Turn the given file descriptor in to '/dev/null'. - * - * @param fd fd to bind to /dev/null - * @param flags flags to use (O_RDONLY or O_WRONLY) - */ -static void -make_dev_zero (int fd, int flags) -{ - int z; - - GNUNET_assert (0 == close (fd)); - z = open ("/dev/null", flags); - GNUNET_assert (-1 != z); - if (z == fd) - return; - GNUNET_break (fd == dup2 (z, fd)); - GNUNET_assert (0 == close (z)); -} - - -/** - * Main function of the helper process to extract meta data. - * - * @param argc should be 3 - * @param argv [0] our binary name - * [1] name of the file or directory to process - * [2] "-" to disable extraction, NULL for defaults, - * otherwise custom plugins to load from LE - * @return 0 on success - */ -int -main (int argc, char *const *argv) -{ - const char *filename_expanded; - const char *ex; - struct ScanTreeNode *root; - - ignore_sigpipe (); - /* move stdout to some other FD for IPC, bind - stdout/stderr to /dev/null */ - output_stream = dup (1); - make_dev_zero (1, O_WRONLY); - make_dev_zero (2, O_WRONLY); - - /* parse command line */ - if ((3 != argc) && (2 != argc)) - { - fprintf (stderr, - "%s", - "gnunet-helper-fs-publish needs exactly one or two arguments\n"); - return 1; - } - filename_expanded = argv[1]; - ex = argv[2]; - if ((NULL == ex) || (0 != strcmp (ex, "-"))) - { -#if HAVE_LIBEXTRACTOR - plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY); - if (NULL != ex) - plugins = EXTRACTOR_plugin_add_config (plugins, - ex, - EXTRACTOR_OPTION_DEFAULT_POLICY); -#endif - } - - /* scan tree to find out how much work there is to be done */ - if (GNUNET_OK != preprocess_file (filename_expanded, &root)) - { - (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, NULL, 0); -#if HAVE_LIBEXTRACTOR - EXTRACTOR_plugin_remove_all (plugins); -#endif - return 2; - } - /* signal that we're done counting files, so that a percentage of - progress can now be calculated */ - if (GNUNET_OK != - write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, - NULL, - 0)) - { -#if HAVE_LIBEXTRACTOR - EXTRACTOR_plugin_remove_all (plugins); -#endif - return 3; - } - if (NULL != root) - { - if (GNUNET_OK != extract_files (root)) - { - (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, - NULL, - 0); - free_tree (root); -#if HAVE_LIBEXTRACTOR - EXTRACTOR_plugin_remove_all (plugins); -#endif - return 4; - } - free_tree (root); - } - /* enable "clean" shutdown by telling parent that we are done */ - (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, - NULL, - 0); -#if HAVE_LIBEXTRACTOR - EXTRACTOR_plugin_remove_all (plugins); -#endif - return 0; -} - - -/* end of gnunet-helper-fs-publish.c */ diff --git a/src/fs/gnunet-publish.c b/src/fs/gnunet-publish.c deleted file mode 100644 index 7a87130de..000000000 --- a/src/fs/gnunet-publish.c +++ /dev/null @@ -1,1009 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2001-2013 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/gnunet-publish.c - * @brief publishing files on GNUnet - * @author Christian Grothoff - * @author Krista Bennett - * @author James Blackwell - * @author Igor Wronsky - */ -#include "platform.h" - -#include "gnunet_fs_service.h" -#include "gnunet_identity_service.h" - -/** - * Global return value from #main(). - */ -static int ret; - -/** - * Command line option 'verbose' set - */ -static unsigned int verbose; - -/** - * Handle to our configuration. - */ -static const struct GNUNET_CONFIGURATION_Handle *cfg; - -/** - * Handle for interaction with file-sharing service. - */ -static struct GNUNET_FS_Handle *ctx; - -/** - * Handle to FS-publishing operation. - */ -static struct GNUNET_FS_PublishContext *pc; - -/** - * Meta-data provided via command-line option. - */ -static struct GNUNET_FS_MetaData *meta; - -/** - * Keywords provided via command-line option. - */ -static struct GNUNET_FS_Uri *topKeywords; - -/** - * Options we set for published blocks. - */ -static struct GNUNET_FS_BlockOptions bo = { { 0LL }, 1, 365, 1 }; - -/** - * Value of URI provided on command-line (when not publishing - * a file but just creating UBlocks to refer to an existing URI). - */ -static char *uri_string; - -/** - * Value of URI provided on command-line (when not publishing - * a file but just creating UBlocks to refer to an existing URI); - * parsed version of 'uri_string'. - */ -static struct GNUNET_FS_Uri *uri; - -/** - * Command-line option for namespace publishing: identifier for updates - * to this publication. - */ -static char *next_id; - -/** - * Command-line option for namespace publishing: identifier for this - * publication. - */ -static char *this_id; - -/** - * Command-line option identifying the pseudonym to use for the publication. - */ -static char *pseudonym; - -/** - * Command-line option for 'inserting' - */ -static int do_insert; - -/** - * Command-line option to disable meta data extraction. - */ -static int disable_extractor; - -/** - * Command-line option to merely simulate publishing operation. - */ -static int do_simulate; - -/** - * Command-line option to only perform meta data extraction, but not publish. - */ -static int extract_only; - -/** - * Command-line option to disable adding creation time. - */ -static int enable_creation_time; - -/** - * Handle to the directory scanner (for recursive insertions). - */ -static struct GNUNET_FS_DirScanner *ds; - -/** - * Which namespace do we publish to? NULL if we do not publish to - * a namespace. - */ -static struct GNUNET_IDENTITY_Ego *namespace; - -/** - * Handle to identity service. - */ -static struct GNUNET_IDENTITY_Handle *identity; - - -/** - * We are finished with the publishing operation, clean up all - * FS state. - * - * @param cls NULL - */ -static void -do_stop_task (void *cls) -{ - struct GNUNET_FS_PublishContext *p; - - if (NULL != ds) - { - GNUNET_FS_directory_scan_abort (ds); - ds = NULL; - } - if (NULL != identity) - { - GNUNET_IDENTITY_disconnect (identity); - identity = NULL; - } - if (NULL != pc) - { - p = pc; - pc = NULL; - GNUNET_FS_publish_stop (p); - } - if (NULL != ctx) - { - GNUNET_FS_stop (ctx); - ctx = NULL; - } - if (NULL != meta) - { - GNUNET_FS_meta_data_destroy (meta); - meta = NULL; - } - if (NULL != uri) - { - GNUNET_FS_uri_destroy (uri); - uri = NULL; - } -} - - -/** - * Called by FS client to give information about the progress of an - * operation. - * - * @param cls closure - * @param info details about the event, specifying the event type - * and various bits about the event - * @return client-context (for the next progress call - * for this operation; should be set to NULL for - * SUSPEND and STOPPED events). The value returned - * will be passed to future callbacks in the respective - * field in the GNUNET_FS_ProgressInfo struct. - */ -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *info) -{ - const char *s; - char *suri; - - switch (info->status) - { - case GNUNET_FS_STATUS_PUBLISH_START: - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - if (verbose) - { - s = GNUNET_STRINGS_relative_time_to_string (info->value.publish.eta, - GNUNET_YES); - fprintf (stdout, - _ ("Publishing `%s' at %llu/%llu (%s remaining)\n"), - info->value.publish.filename, - (unsigned long long) info->value.publish.completed, - (unsigned long long) info->value.publish.size, - s); - } - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - if (verbose) - { - s = GNUNET_STRINGS_relative_time_to_string (info->value.publish.specifics - .progress_directory.eta, - GNUNET_YES); - fprintf (stdout, - _ ("Publishing `%s' at %llu/%llu (%s remaining)\n"), - info->value.publish.filename, - (unsigned long long) - info->value.publish.specifics.progress_directory.completed, - (unsigned long long) - info->value.publish.specifics.progress_directory.total, - s); - } - break; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - fprintf (stderr, - _ ("Error publishing: %s.\n"), - info->value.publish.specifics.error.message); - ret = 1; - GNUNET_SCHEDULER_shutdown (); - break; - - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - fprintf (stdout, - _ ("Publishing `%s' done.\n"), - info->value.publish.filename); - suri = - GNUNET_FS_uri_to_string (info->value.publish.specifics.completed.chk_uri); - fprintf (stdout, _ ("URI is `%s'.\n"), suri); - GNUNET_free (suri); - if (NULL != info->value.publish.specifics.completed.sks_uri) - { - suri = GNUNET_FS_uri_to_string ( - info->value.publish.specifics.completed.sks_uri); - fprintf (stdout, _ ("Namespace URI is `%s'.\n"), suri); - GNUNET_free (suri); - } - if (NULL == info->value.publish.pctx) - { - ret = 0; - GNUNET_SCHEDULER_shutdown (); - } - break; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - GNUNET_break (NULL == pc); - return NULL; - - case GNUNET_FS_STATUS_UNINDEX_START: - fprintf (stderr, "%s", _ ("Starting cleanup after abort\n")); - return NULL; - - case GNUNET_FS_STATUS_UNINDEX_PROGRESS: - return NULL; - - case GNUNET_FS_STATUS_UNINDEX_COMPLETED: - fprintf (stderr, "%s", _ ("Cleanup after abort completed.\n")); - GNUNET_FS_unindex_stop (info->value.unindex.uc); - return NULL; - - case GNUNET_FS_STATUS_UNINDEX_ERROR: - fprintf (stderr, "%s", _ ("Cleanup after abort failed.\n")); - GNUNET_FS_unindex_stop (info->value.unindex.uc); - return NULL; - - case GNUNET_FS_STATUS_UNINDEX_STOPPED: - return NULL; - - default: - fprintf (stderr, _ ("Unexpected status: %d\n"), info->status); - return NULL; - } - return ""; /* non-null */ -} - - -/** - * Print metadata entries (except binary - * metadata and the filename). - * - * @param cls closure - * @param plugin_name name of the plugin that generated the meta data - * @param type type of the meta data - * @param format format of data - * @param data_mime_type mime type of @a data - * @param data value of the meta data - * @param data_size number of bytes in @a data - * @return always 0 - */ -static int -meta_printer (void *cls, - const char *plugin_name, - enum EXTRACTOR_MetaType type, - enum EXTRACTOR_MetaFormat format, - const char *data_mime_type, - const char *data, - size_t data_size) -{ - if ((EXTRACTOR_METAFORMAT_UTF8 != format) && - (EXTRACTOR_METAFORMAT_C_STRING != format)) - return 0; - if (EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME == type) - return 0; -#if HAVE_LIBEXTRACTOR - fprintf (stdout, "\t%s - %s\n", EXTRACTOR_metatype_to_string (type), data); -#else - fprintf (stdout, "\t%d - %s\n", type, data); -#endif - return 0; -} - - -/** - * Iterator printing keywords - * - * @param cls closure - * @param keyword the keyword - * @param is_mandatory is the keyword mandatory (in a search) - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to abort - */ -static int -keyword_printer (void *cls, const char *keyword, int is_mandatory) -{ - fprintf (stdout, "\t%s\n", keyword); - return GNUNET_OK; -} - - -/** - * Function called on all entries before the publication. This is - * where we perform modifications to the default based on command-line - * options. - * - * @param cls closure - * @param fi the entry in the publish-structure - * @param length length of the file or directory - * @param m metadata for the file or directory (can be modified) - * @param uri pointer to the keywords that will be used for this entry (can be modified) - * @param bo block options - * @param do_index should we index? - * @param client_info pointer to client context set upon creation (can be modified) - * @return #GNUNET_OK to continue, #GNUNET_NO to remove - * this entry from the directory, #GNUNET_SYSERR - * to abort the iteration - */ -static int -publish_inspector (void *cls, - struct GNUNET_FS_FileInformation *fi, - uint64_t length, - struct GNUNET_FS_MetaData *m, - struct GNUNET_FS_Uri **uri, - struct GNUNET_FS_BlockOptions *bo, - int *do_index, - void **client_info) -{ - char *fn; - char *fs; - struct GNUNET_FS_Uri *new_uri; - - if (cls == fi) - return GNUNET_OK; - if ((disable_extractor) && (NULL != *uri)) - { - GNUNET_FS_uri_destroy (*uri); - *uri = NULL; - } - if (NULL != topKeywords) - { - if (NULL != *uri) - { - new_uri = GNUNET_FS_uri_ksk_merge (topKeywords, *uri); - GNUNET_FS_uri_destroy (*uri); - *uri = new_uri; - GNUNET_FS_uri_destroy (topKeywords); - } - else - { - *uri = topKeywords; - } - topKeywords = NULL; - } - if (NULL != meta) - { - GNUNET_FS_meta_data_merge (m, meta); - GNUNET_FS_meta_data_destroy (meta); - meta = NULL; - } - if (enable_creation_time) - GNUNET_FS_meta_data_add_publication_date (m); - if (extract_only) - { - fn = GNUNET_FS_meta_data_get_by_type ( - m, - EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); - fs = GNUNET_STRINGS_byte_size_fancy (length); - fprintf (stdout, _ ("Meta data for file `%s' (%s)\n"), fn, fs); - GNUNET_FS_meta_data_iterate (m, &meta_printer, NULL); - fprintf (stdout, _ ("Keywords for file `%s' (%s)\n"), fn, fs); - GNUNET_free (fn); - GNUNET_free (fs); - if (NULL != *uri) - GNUNET_FS_uri_ksk_get_keywords (*uri, &keyword_printer, NULL); - fprintf (stdout, "%s", "\n"); - } - if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (m)) - GNUNET_FS_file_information_inspect (fi, &publish_inspector, fi); - return GNUNET_OK; -} - - -/** - * Function called upon completion of the publishing - * of the UBLOCK for the SKS URI. As this is the last - * step, stop our interaction with FS (clean up). - * - * @param cls NULL (closure) - * @param sks_uri URI for the block that was published - * @param emsg error message, NULL on success - */ -static void -uri_sks_continuation (void *cls, - const struct GNUNET_FS_Uri *sks_uri, - const char *emsg) -{ - if (NULL != emsg) - { - fprintf (stderr, "%s\n", emsg); - ret = 1; - } - GNUNET_SCHEDULER_shutdown (); -} - - -/** - * Function called upon completion of the publishing - * of the UBLOCK for the KSK URI. Continue with - * publishing the SKS URI (if applicable) or clean up. - * - * @param cls NULL (closure) - * @param ksk_uri URI for the block that was published - * @param emsg error message, NULL on success - */ -static void -uri_ksk_continuation (void *cls, - const struct GNUNET_FS_Uri *ksk_uri, - const char *emsg) -{ - const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv; - const struct GNUNET_CRYPTO_PrivateKey *pk; - - if (NULL != emsg) - { - fprintf (stderr, "%s\n", emsg); - ret = 1; - } - if (NULL == namespace) - { - GNUNET_SCHEDULER_shutdown (); - return; - } - pk = GNUNET_IDENTITY_ego_get_private_key (namespace); - if (GNUNET_PUBLIC_KEY_TYPE_ECDSA != ntohl (pk->type)) - return; - priv = &pk->ecdsa_key; - GNUNET_FS_publish_sks (ctx, - priv, - this_id, - next_id, - meta, - uri, - &bo, - GNUNET_FS_PUBLISH_OPTION_NONE, - &uri_sks_continuation, - NULL); -} - - -/** - * Iterate over the results from the directory scan and extract - * the desired information for the publishing operation. - * - * @param item root with the data from the directory scan - * @return handle with the information for the publishing operation - */ -static struct GNUNET_FS_FileInformation * -get_file_information (struct GNUNET_FS_ShareTreeItem *item) -{ - struct GNUNET_FS_FileInformation *fi; - struct GNUNET_FS_FileInformation *fic; - struct GNUNET_FS_ShareTreeItem *child; - - if (GNUNET_YES == item->is_directory) - { - if (NULL == item->meta) - item->meta = GNUNET_FS_meta_data_create (); - GNUNET_FS_meta_data_delete (item->meta, - EXTRACTOR_METATYPE_MIMETYPE, - NULL, - 0); - GNUNET_FS_meta_data_make_directory (item->meta); - if (NULL == item->ksk_uri) - { - const char *mime = GNUNET_FS_DIRECTORY_MIME; - item->ksk_uri = GNUNET_FS_uri_ksk_create_from_args (1, &mime); - } - else - GNUNET_FS_uri_ksk_add_keyword (item->ksk_uri, - GNUNET_FS_DIRECTORY_MIME, - GNUNET_NO); - fi = GNUNET_FS_file_information_create_empty_directory (ctx, - NULL, - item->ksk_uri, - item->meta, - &bo, - item->filename); - for (child = item->children_head; child; child = child->next) - { - fic = get_file_information (child); - GNUNET_break (GNUNET_OK == GNUNET_FS_file_information_add (fi, fic)); - } - } - else - { - fi = GNUNET_FS_file_information_create_from_file (ctx, - NULL, - item->filename, - item->ksk_uri, - item->meta, - ! do_insert, - &bo); - } - return fi; -} - - -/** - * We've finished scanning the directory and optimized the meta data. - * Begin the publication process. - * - * @param directory_scan_result result from the directory scan, freed in this function - */ -static void -directory_trim_complete (struct GNUNET_FS_ShareTreeItem *directory_scan_result) -{ - struct GNUNET_FS_FileInformation *fi; - const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv; - const struct GNUNET_CRYPTO_PrivateKey *pk; - - fi = get_file_information (directory_scan_result); - GNUNET_FS_share_tree_free (directory_scan_result); - if (NULL == fi) - { - fprintf (stderr, "%s", _ ("Could not publish\n")); - ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - GNUNET_FS_file_information_inspect (fi, &publish_inspector, NULL); - if (extract_only) - { - GNUNET_FS_file_information_destroy (fi, NULL, NULL); - GNUNET_SCHEDULER_shutdown (); - return; - } - priv = NULL; - if (NULL != namespace) - { - pk = GNUNET_IDENTITY_ego_get_private_key (namespace); - GNUNET_assert (GNUNET_PUBLIC_KEY_TYPE_ECDSA == ntohl (pk->type)); - priv = &pk->ecdsa_key; - } - pc = GNUNET_FS_publish_start (ctx, - fi, - priv, - this_id, - next_id, - (do_simulate) - ? GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY - : GNUNET_FS_PUBLISH_OPTION_NONE); - if (NULL == pc) - { - fprintf (stderr, "%s", _ ("Could not start publishing.\n")); - ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } -} - - -/** - * Function called by the directory scanner as we build the tree - * that we will need to publish later. - * - * @param cls closure - * @param filename which file we are making progress on - * @param is_directory #GNUNET_YES if this is a directory, - * #GNUNET_NO if this is a file - * #GNUNET_SYSERR if it is neither (or unknown) - * @param reason kind of progress we are making - */ -static void -directory_scan_cb (void *cls, - const char *filename, - int is_directory, - enum GNUNET_FS_DirScannerProgressUpdateReason reason) -{ - struct GNUNET_FS_ShareTreeItem *directory_scan_result; - - switch (reason) - { - case GNUNET_FS_DIRSCANNER_FILE_START: - if (verbose > 1) - { - if (is_directory == GNUNET_YES) - fprintf (stdout, _ ("Scanning directory `%s'.\n"), filename); - else - fprintf (stdout, _ ("Scanning file `%s'.\n"), filename); - } - break; - - case GNUNET_FS_DIRSCANNER_FILE_IGNORED: - fprintf (stderr, - _ ("There was trouble processing file `%s', skipping it.\n"), - filename); - break; - - case GNUNET_FS_DIRSCANNER_ALL_COUNTED: - if (verbose) - fprintf (stdout, "%s", _ ("Preprocessing complete.\n")); - break; - - case GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED: - if (verbose > 2) - fprintf (stdout, - _ ("Extracting meta data from file `%s' complete.\n"), - filename); - break; - - case GNUNET_FS_DIRSCANNER_FINISHED: - if (verbose > 1) - fprintf (stdout, "%s", _ ("Meta data extraction has finished.\n")); - directory_scan_result = GNUNET_FS_directory_scan_get_result (ds); - ds = NULL; - GNUNET_FS_share_tree_trim (directory_scan_result); - directory_trim_complete (directory_scan_result); - break; - - case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR: - fprintf (stdout, "%s", _ ("Error scanning directory.\n")); - ret = 1; - GNUNET_SCHEDULER_shutdown (); - break; - - default: - GNUNET_assert (0); - break; - } - fflush (stdout); -} - - -/** - * Continuation proceeding with initialization after identity subsystem - * has been initialized. - * - * @param args0 filename to publish - */ -static void -identity_continuation (const char *args0) -{ - char *ex; - char *emsg; - - if ((NULL != pseudonym) && (NULL == namespace)) - { - fprintf (stderr, _ ("Selected pseudonym `%s' unknown\n"), pseudonym); - ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - if (NULL != uri_string) - { - emsg = NULL; - if (NULL == (uri = GNUNET_FS_uri_parse (uri_string, &emsg))) - { - fprintf (stderr, _ ("Failed to parse URI: %s\n"), emsg); - GNUNET_free (emsg); - ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - GNUNET_FS_publish_ksk (ctx, - topKeywords, - meta, - uri, - &bo, - GNUNET_FS_PUBLISH_OPTION_NONE, - &uri_ksk_continuation, - NULL); - return; - } - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (cfg, "FS", "EXTRACTORS", &ex)) - ex = NULL; - if (0 != access (args0, R_OK)) - { - fprintf (stderr, - _ ("Failed to access `%s': %s\n"), - args0, - strerror (errno)); - GNUNET_free (ex); - return; - } - ds = GNUNET_FS_directory_scan_start (args0, - disable_extractor, - ex, - &directory_scan_cb, - NULL); - if (NULL == ds) - { - fprintf ( - stderr, - "%s", - _ ( - "Failed to start meta directory scanner. Is gnunet-helper-publish-fs installed?\n")); - GNUNET_free (ex); - return; - } - GNUNET_free (ex); -} - - -/** - * Function called by identity service with known pseudonyms. - * - * @param cls closure with 'const char *' of filename to publish - * @param ego ego handle - * @param ctx context for application to store data for this ego - * (during the lifetime of this process, initially NULL) - * @param name name assigned by the user for this ego, - * NULL if the user just deleted the ego and it - * must thus no longer be used - */ -static void -identity_cb (void *cls, - struct GNUNET_IDENTITY_Ego *ego, - void **ctx, - const char *name) -{ - const char *args0 = cls; - - if (NULL == ego) - { - identity_continuation (args0); - return; - } - if (NULL == name) - return; - if (0 == strcmp (name, pseudonym)) - namespace = ego; -} - - -/** - * Main function that will be run by the scheduler. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param c configuration - */ -static void -run (void *cls, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *c) -{ - /* check arguments */ - if ((NULL != uri_string) && (extract_only)) - { - printf (_ ("Cannot extract metadata from a URI!\n")); - ret = -1; - return; - } - if (((NULL == uri_string) || (extract_only)) && - ((NULL == args[0]) || (NULL != args[1]))) - { - printf (_ ("You must specify one and only one filename for insertion.\n")); - ret = -1; - return; - } - if ((NULL != uri_string) && (NULL != args[0])) - { - printf (_ ("You must NOT specify an URI and a filename.\n")); - ret = -1; - return; - } - if (NULL != pseudonym) - { - if (NULL == this_id) - { - fprintf (stderr, - _ ("Option `%s' is required when using option `%s'.\n"), - "-t", - "-P"); - ret = -1; - return; - } - } - else - { /* ordinary insertion checks */ - if (NULL != next_id) - { - fprintf (stderr, - _ ("Option `%s' makes no sense without option `%s'.\n"), - "-N", - "-P"); - ret = -1; - return; - } - if (NULL != this_id) - { - fprintf (stderr, - _ ("Option `%s' makes no sense without option `%s'.\n"), - "-t", - "-P"); - ret = -1; - return; - } - } - cfg = c; - ctx = GNUNET_FS_start (cfg, - "gnunet-publish", - &progress_cb, - NULL, - GNUNET_FS_FLAGS_NONE, - GNUNET_FS_OPTIONS_END); - if (NULL == ctx) - { - fprintf (stderr, _ ("Could not initialize `%s' subsystem.\n"), "FS"); - ret = 1; - return; - } - GNUNET_SCHEDULER_add_shutdown (&do_stop_task, NULL); - if (NULL != pseudonym) - identity = GNUNET_IDENTITY_connect (cfg, &identity_cb, args[0]); - else - identity_continuation (args[0]); -} - - -/** - * The main function to publish content to GNUnet. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, char *const *argv) -{ - struct GNUNET_GETOPT_CommandLineOption options[] = - { GNUNET_GETOPT_option_uint ('a', - "anonymity", - "LEVEL", - gettext_noop ( - "set the desired LEVEL of sender-anonymity"), - &bo.anonymity_level), - GNUNET_GETOPT_option_flag ( - 'D', - "disable-extractor", - gettext_noop ("do not use libextractor to add keywords or metadata"), - &disable_extractor), - GNUNET_GETOPT_option_flag ('E', - "enable-creation-time", - gettext_noop ( - "enable adding the creation time to the " - "metadata of the uploaded file"), - &enable_creation_time), - GNUNET_GETOPT_option_flag ('e', - "extract", - gettext_noop ( - "print list of extracted keywords that would " - "be used, but do not perform upload"), - &extract_only), - GNUNET_FS_GETOPT_KEYWORDS ( - 'k', - "key", - "KEYWORD", - gettext_noop ( - "add an additional keyword for the top-level " - "file or directory (this option can be specified multiple times)"), - &topKeywords), - GNUNET_FS_GETOPT_METADATA ( - 'm', - "meta", - "TYPE:VALUE", - gettext_noop ("set the meta-data for the given TYPE to the given VALUE"), - &meta), - GNUNET_GETOPT_option_flag ( - 'n', - "noindex", - gettext_noop ("do not index, perform full insertion (stores " - "entire file in encrypted form in GNUnet database)"), - &do_insert), - GNUNET_GETOPT_option_string ( - 'N', - "next", - "ID", - gettext_noop ("specify ID of an updated version to be " - "published in the future (for namespace insertions only)"), - &next_id), - GNUNET_GETOPT_option_uint ('p', - "priority", - "PRIORITY", - gettext_noop ( - "specify the priority of the content"), - &bo.content_priority), - GNUNET_GETOPT_option_string ('P', - "pseudonym", - "NAME", - gettext_noop ( - "publish the files under the pseudonym " - "NAME (place file into namespace)"), - &pseudonym), - GNUNET_GETOPT_option_uint ('r', - "replication", - "LEVEL", - gettext_noop ( - "set the desired replication LEVEL"), - &bo.replication_level), - GNUNET_GETOPT_option_flag ('s', - "simulate-only", - gettext_noop ( - "only simulate the process but do not do " - "any actual publishing (useful to compute URIs)"), - &do_simulate), - GNUNET_GETOPT_option_string ('t', - "this", - "ID", - gettext_noop ( - "set the ID of this version of the publication " - "(for namespace insertions only)"), - &this_id), - GNUNET_GETOPT_option_string ( - 'u', - "uri", - "URI", - gettext_noop ( - "URI to be published (can be used instead of passing a " - "file to add keywords to the file with the respective URI)"), - &uri_string), - - GNUNET_GETOPT_option_verbose (&verbose), - - GNUNET_GETOPT_OPTION_END }; - - bo.expiration_time = - GNUNET_TIME_year_to_time (GNUNET_TIME_get_current_year () + 2); - - if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) - return 2; - ret = - (GNUNET_OK == - GNUNET_PROGRAM_run (argc, - argv, - "gnunet-publish [OPTIONS] FILENAME", - gettext_noop ("Publish a file or directory on GNUnet"), - options, - &run, - NULL)) - ? ret - : 1; - GNUNET_free_nz ((void *) argv); - return ret; -} - - -/* end of gnunet-publish.c */ diff --git a/src/fs/gnunet-search.c b/src/fs/gnunet-search.c deleted file mode 100644 index a72cf97cc..000000000 --- a/src/fs/gnunet-search.c +++ /dev/null @@ -1,801 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009, 2022 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/gnunet-search.c - * @brief searching for files on GNUnet - * @author Christian Grothoff - * @author Krista Bennett - * @author James Blackwell - * @author Igor Wronsky - * @author madmurphy - */ -#include "platform.h" -#include -#include -#include - -#include "gnunet_fs_service.h" - - -#define GNUNET_SEARCH_log(kind, ...) \ - GNUNET_log_from (kind, "gnunet-search", __VA_ARGS__) - - -/* The default settings that we use for the printed output */ - -#define DEFAULT_DIR_FORMAT "#%n:\ngnunet-download -o \"%f\" -R %u\n\n" -#define HELP_DEFAULT_DIR_FORMAT "#%n:\\ngnunet-download -o \"%f\" -R %u\\n\\n" -#define DEFAULT_FILE_FORMAT "#%n:\ngnunet-download -o \"%f\" %u\n\n" -#define HELP_DEFAULT_FILE_FORMAT "#%n:\\ngnunet-download -o \"%f\" %u\\n\\n" -#define VERB_DEFAULT_DIR_FORMAT DEFAULT_DIR_FORMAT "%a\n" -#define VERB_DEFAULT_FILE_FORMAT DEFAULT_FILE_FORMAT "%a\n" - -#if HAVE_LIBEXTRACTOR -#define DEFAULT_META_FORMAT " %t: %p\n" -#define HELP_DEFAULT_META_FORMAT " %t: %p\\n" -#define HELP_EXTRACTOR_TEXTADD ", %t" -#else -#define DEFAULT_META_FORMAT " MetaType #%i: %p\n" -#define HELP_DEFAULT_META_FORMAT " MetaType #%i: %p\\n" -#define HELP_EXTRACTOR_TEXTADD "" -#endif - -#define GENERIC_DIRECTORY_NAME "collection" -#define GENERIC_FILE_NAME "no-name" -#define GENERIC_FILE_MIMETYPE "application/octet-stream" - - -enum GNUNET_SEARCH_MetadataPrinterFlags -{ - METADATA_PRINTER_FLAG_NONE = 0, - METADATA_PRINTER_FLAG_ONE_RUN = 1, - METADATA_PRINTER_FLAG_HAVE_TYPE = 2 -}; - - -struct GNUNET_SEARCH_MetadataPrinterInfo -{ - unsigned int counter; - unsigned int flags; - int type; -}; - - -static int ret; - -static const struct GNUNET_CONFIGURATION_Handle *cfg; - -static struct GNUNET_FS_Handle *ctx; - -static struct GNUNET_FS_SearchContext *sc; - -static char *output_filename; - -static char *format_string; - -static char *dir_format_string; - -static char *meta_format_string; - -static struct GNUNET_FS_DirectoryBuilder *db; - -static unsigned int anonymity = 1; - -/** - * Timeout for the search, 0 means to wait for CTRL-C. - */ -static struct GNUNET_TIME_Relative timeout; - -static unsigned int results_limit; - -static unsigned int results; - -static unsigned int verbose; - -static int bookmark_only; - -static int local_only; - -static int silent_mode; - -static struct GNUNET_SCHEDULER_Task *tt; - -static int stop_searching; - - -/** - * Print the escape sequence at the beginning of a string. - * - * @param esc a string that **must** begin with a backslash (the function only - * assumes that it does, but does not check) - * @return the fragment that follows what has been printed - * @author madmurphy - * - * If `"\\nfoo"` is passed as argument, this function prints a new line and - * returns `"foo"` - */ -static const char * -print_escape_sequence (const char *const esc) -{ - unsigned int probe; - const char *cursor = esc + 1; - char tmp; - switch (*cursor) - { - /* Trivia */ - case '\\': putchar ('\\'); return cursor + 1; - case 'a': putchar ('\a'); return cursor + 1; - case 'b': putchar ('\b'); return cursor + 1; - case 'e': putchar ('\x1B'); return cursor + 1; - case 'f': putchar ('\f'); return cursor + 1; - case 'n': putchar ('\n'); return cursor + 1; - case 'r': putchar ('\r'); return cursor + 1; - case 't': putchar ('\t'); return cursor + 1; - case 'v': putchar ('\v'); return cursor + 1; - - /* Possibly hexadecimal code point */ - case 'x': - probe = 0; - while (probe < 256 && isxdigit ((tmp = *++cursor))) - probe = (probe << 4) + tmp - (tmp > 96 ? 87 : tmp > 64 ? 55 : 48); - goto maybe_codepoint; - - /* Possibly octal code point */ - case '0': case '1': case '2': case '3': - case '4': case '5': case '6': case '7': - probe = *cursor++ - 48; - do probe = (probe << 3) + *cursor++ - 48; - while (probe < 256 && cursor < esc + 4 && *cursor > 47 && *cursor < 56); - goto maybe_codepoint; - - /* Boredom */ - case '\0': putchar ('\\'); return cursor; - default: printf ("\\%c", *cursor); return cursor + 1; - } - - maybe_codepoint: - if (probe < 256) - putchar (probe); - else - fwrite (esc, 1, cursor - esc, stdout); - return cursor; -} - - -/** - * Type of a function that libextractor calls for each - * meta data item found. - * - * @param cls closure (user-defined, used for the iteration info) - * @param plugin_name name of the plugin that produced this value; - * special values can be used (e.g. '<zlib>' for zlib being - * used in the main libextractor library and yielding - * meta data). - * @param type libextractor-type describing the meta data - * @param format basic format information about data - * @param data_mime_type mime-type of data (not of the original file); - * can be NULL (if mime-type is not known) - * @param data actual meta-data found - * @param data_size number of bytes in @a data - * @return 0 to continue extracting, 1 to abort - */ -static int -item_printer (void *const cls, - const char *const plugin_name, - const enum EXTRACTOR_MetaType type, - const enum EXTRACTOR_MetaFormat format, - const char *const data_mime_type, - const char *const data, - const size_t data_size) -{ -#define info ((struct GNUNET_SEARCH_MetadataPrinterInfo *) cls) - if ((format != EXTRACTOR_METAFORMAT_UTF8 && - format != EXTRACTOR_METAFORMAT_C_STRING) || - type == EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME) - return 0; - info->counter++; - if ((info->flags & METADATA_PRINTER_FLAG_HAVE_TYPE) && type != info->type) - return 0; - - const char *cursor = meta_format_string; - const char *next_spec = strchr (cursor, '%'); - const char *next_esc = strchr (cursor, '\\'); - - parse_format: - - /* If an escape sequence exists before the next format specifier... */ - if (next_esc && (! next_spec || next_esc < next_spec)) - { - if (next_esc > cursor) - fwrite (cursor, 1, next_esc - cursor, stdout); - - cursor = print_escape_sequence (next_esc); - next_esc = strchr (cursor, '\\'); - goto parse_format; - } - - /* If a format specifier exists before the next escape sequence... */ - if (next_spec && (! next_esc || next_spec < next_esc)) - { - if (next_spec > cursor) - fwrite (cursor, 1, next_spec - cursor, stdout); - - switch (*++next_spec) - { - case '%': putchar ('%'); break; - case 'i': printf ("%d", type); break; - case 'l': printf ("%lu", (long unsigned int) data_size); break; - case 'n': printf ("%u", info->counter); break; - case 'p': printf ("%s", data); break; -#if HAVE_LIBEXTRACTOR - case 't': - printf ("%s", - dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, - EXTRACTOR_metatype_to_string (type))); - break; -#endif - case 'w': printf ("%s", plugin_name); break; - case '\0': putchar ('%'); return 0; - default: printf ("%%%c", *next_spec); break; - } - cursor = next_spec + 1; - next_spec = strchr (cursor, '%'); - goto parse_format; - } - - if (*cursor) - printf ("%s", cursor); - - return info->flags & METADATA_PRINTER_FLAG_ONE_RUN; -#undef info -} - - -/** - * Print a search result according to the current formats - * - * @param filename the filename for this result - * @param uri the `struct GNUNET_FS_Uri` this result refers to - * @param metadata the `struct GNUNET_FS_MetaData` associated with this - result - * @param resultnum the result number - * @param is_directory GNUNET_YES if this is a directory, otherwise GNUNET_NO - * @author madmurphy - */ -static void -print_search_result (const char *const filename, - const struct GNUNET_FS_Uri *const uri, - const struct GNUNET_FS_MetaData *const metadata, - const unsigned int resultnum, - const int is_directory) -{ - - const char *cursor = GNUNET_YES == is_directory ? - dir_format_string - : format_string; - - const char *next_spec = strchr (cursor, '%'); - const char *next_esc = strchr (cursor, '\\'); - char *placeholder; - struct GNUNET_SEARCH_MetadataPrinterInfo info; - - parse_format: - /* If an escape sequence exists before the next format specifier... */ - if (next_esc && (! next_spec || next_esc < next_spec)) - { - if (next_esc > cursor) - fwrite (cursor, 1, next_esc - cursor, stdout); - - cursor = print_escape_sequence (next_esc); - next_esc = strchr (cursor, '\\'); - goto parse_format; - } - - /* If a format specifier exists before the next escape sequence... */ - if (next_spec && (! next_esc || next_spec < next_esc)) - { - if (next_spec > cursor) - fwrite (cursor, 1, next_spec - cursor, stdout); - - switch (*++next_spec) - { - /* All metadata fields */ - case 'a': - info.flags = METADATA_PRINTER_FLAG_NONE; - - iterate_meta: - info.counter = 0; - GNUNET_FS_meta_data_iterate (metadata, &item_printer, &info); - break; - /* File's name */ - case 'f': - if (GNUNET_YES == is_directory) - { - printf ("%s%s", filename, GNUNET_FS_DIRECTORY_EXT); - break; - } - printf ("%s", filename); - break; - /* Only the first metadata field */ - case 'j': - info.flags = METADATA_PRINTER_FLAG_ONE_RUN; - goto iterate_meta; - /* File name's length */ - case 'l': - printf ("%lu", - (long unsigned int) (GNUNET_YES == is_directory ? - strlen (filename) - + (sizeof(GNUNET_FS_DIRECTORY_EXT) - 1) - : - strlen (filename))); - break; - /* File's mime type */ - case 'm': - if (GNUNET_YES == is_directory) - { - printf ("%s", GNUNET_FS_DIRECTORY_MIME); - break; - } - placeholder = GNUNET_FS_meta_data_get_by_type ( - metadata, - EXTRACTOR_METATYPE_MIMETYPE); - printf ("%s", placeholder ? placeholder : GENERIC_FILE_MIMETYPE); - GNUNET_free (placeholder); - break; - /* Result number */ - case 'n': printf ("%u", resultnum); break; - /* File's size */ - case 's': - printf ("%" PRIu64, GNUNET_FS_uri_chk_get_file_size (uri)); - break; - /* File's URI */ - case 'u': - placeholder = GNUNET_FS_uri_to_string (uri); - printf ("%s", placeholder); - GNUNET_free (placeholder); - break; - - /* We can add as many cases as we want here... */ - - /* Handle `%123#a` and `%123#j` (e.g. `%5#j` is a book title) */ - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - cursor = next_spec; - info.type = *cursor - 48; - while (isdigit (*++cursor) && info.type < (INT_MAX - *cursor + 48) / 10) - info.type = info.type * 10 + *cursor - 48; - if (info.type == 0 || *cursor != '#') - goto not_a_specifier; - switch (*++cursor) - { - /* All metadata fields of type `info.type` */ - case 'a': - next_spec = cursor; - info.flags = METADATA_PRINTER_FLAG_HAVE_TYPE; - goto iterate_meta; - - /* Only the first metadata field of type `info.type` */ - case 'j': - next_spec = cursor; - info.flags = METADATA_PRINTER_FLAG_HAVE_TYPE - | METADATA_PRINTER_FLAG_ONE_RUN; - goto iterate_meta; - } - goto not_a_specifier; - - /* All other cases */ - case '%': putchar ('%'); break; - case '\0': putchar ('%'); return; - - not_a_specifier: - default: printf ("%%%c", *next_spec); break; - } - cursor = next_spec + 1; - next_spec = strchr (cursor, '%'); - goto parse_format; - } - - if (*cursor) - printf ("%s", cursor); -} - - -static void -clean_task (void *const cls) -{ - size_t dsize; - void *ddata; - - GNUNET_FS_stop (ctx); - ctx = NULL; - if (output_filename == NULL) - return; - if (GNUNET_OK != - GNUNET_FS_directory_builder_finish (db, &dsize, &ddata)) - { - GNUNET_break (0); - GNUNET_free (output_filename); - return; - } - (void) GNUNET_DISK_directory_remove (output_filename); - if (GNUNET_OK != - GNUNET_DISK_fn_write (output_filename, - ddata, - dsize, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE)) - { - GNUNET_SEARCH_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Failed to write directory with search results to " - "`%s'\n"), - output_filename); - } - GNUNET_free (ddata); - GNUNET_free (output_filename); -} - - -/** - * Called by FS client to give information about the progress of an - * operation. - * - * @param cls closure - * @param info details about the event, specifying the event type - * and various bits about the event - * @return client-context (for the next progress call - * for this operation; should be set to NULL for - * SUSPEND and STOPPED events). The value returned - * will be passed to future callbacks in the respective - * field in the GNUNET_FS_ProgressInfo struct. - */ -static void * -progress_cb (void *const cls, - const struct GNUNET_FS_ProgressInfo *const info) -{ - static unsigned int cnt; - int is_directory; - char *filename; - - switch (info->status) - { - case GNUNET_FS_STATUS_SEARCH_START: - break; - - case GNUNET_FS_STATUS_SEARCH_RESULT: - if (stop_searching) - break; - - if (db != NULL) - GNUNET_FS_directory_builder_add ( - db, - info->value.search.specifics.result.uri, - info->value.search.specifics.result.meta, - NULL); - - if (silent_mode) - break; - - cnt++; - filename = GNUNET_FS_meta_data_get_by_type ( - info->value.search.specifics.result.meta, - EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); - is_directory = GNUNET_FS_meta_data_test_for_directory ( - info->value.search.specifics.result.meta); - if (NULL != filename) - { - while ((filename[0] != '\0') && ('/' == filename[strlen (filename) - 1])) - filename[strlen (filename) - 1] = '\0'; - GNUNET_DISK_filename_canonicalize (filename); - } - print_search_result (filename ? - filename - : is_directory ? - GENERIC_DIRECTORY_NAME - : - GENERIC_FILE_NAME, - info->value.search.specifics.result.uri, - info->value.search.specifics.result.meta, - cnt, - is_directory); - fflush (stdout); - GNUNET_free (filename); - results++; - if ((results_limit > 0) && (results >= results_limit)) - { - GNUNET_SCHEDULER_shutdown (); - /* otherwise the function might keep printing results for a while... */ - stop_searching = GNUNET_YES; - } - break; - - case GNUNET_FS_STATUS_SEARCH_UPDATE: - case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED: - /* ignore */ - break; - - case GNUNET_FS_STATUS_SEARCH_ERROR: - GNUNET_SEARCH_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Error searching: %s.\n"), - info->value.search.specifics.error.message); - GNUNET_SCHEDULER_shutdown (); - break; - - case GNUNET_FS_STATUS_SEARCH_STOPPED: - GNUNET_SCHEDULER_add_now (&clean_task, NULL); - break; - - default: - GNUNET_SEARCH_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Unexpected status: %d\n"), - info->status); - break; - } - return NULL; -} - - -static void -shutdown_task (void *const cls) -{ - if (sc != NULL) - { - GNUNET_FS_search_stop (sc); - sc = NULL; - } -} - - -static void -timeout_task (void *const cls) -{ - tt = NULL; - stop_searching = GNUNET_YES; - GNUNET_SCHEDULER_shutdown (); -} - - -/** - * Main function that will be run by the scheduler. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param cfgarg configuration - */ -static void -run (void *const cls, - char *const *const args, - const char *const cfgfile, - const struct GNUNET_CONFIGURATION_Handle *const cfgarg) -{ - struct GNUNET_FS_Uri *uri; - unsigned int argc; - enum GNUNET_FS_SearchOptions options; - - if (silent_mode && bookmark_only) - { - fprintf (stderr, - _ ("Conflicting options --bookmark-only and --silent.\n")); - ret = 1; - return; - } - if (bookmark_only && output_filename) - { - fprintf (stderr, - _ ("Conflicting options --bookmark-only and --output.\n")); - ret = 1; - return; - } - if (silent_mode && ! output_filename) - { - fprintf (stderr, _ ("An output file is mandatory for silent mode.\n")); - ret = 1; - return; - } - if (NULL == dir_format_string) - dir_format_string = format_string ? format_string - : verbose ? VERB_DEFAULT_DIR_FORMAT - : DEFAULT_DIR_FORMAT; - if (NULL == format_string) - format_string = verbose ? VERB_DEFAULT_FILE_FORMAT - : DEFAULT_FILE_FORMAT; - if (NULL == meta_format_string) - meta_format_string = DEFAULT_META_FORMAT; - argc = 0; - while (NULL != args[argc]) - argc++; - uri = GNUNET_FS_uri_ksk_create_from_args (argc, (const char **) args); - if (NULL == uri) - { - fprintf (stderr, - "%s", - _ ("Could not create keyword URI from arguments.\n")); - ret = 1; - return; - } - if (! GNUNET_FS_uri_test_ksk (uri) && ! GNUNET_FS_uri_test_sks (uri)) - { - fprintf (stderr, - "%s", - _ ("Invalid URI. Valid URIs for searching are keyword query " - "URIs\n(\"gnunet://fs/ksk/...\") and namespace content URIs " - "(\"gnunet://fs/sks/...\").\n")); - GNUNET_FS_uri_destroy (uri); - ret = 1; - return; - } - if (bookmark_only) - { - char *bmstr = GNUNET_FS_uri_to_string (uri); - printf ("%s\n", bmstr); - GNUNET_free (bmstr); - GNUNET_FS_uri_destroy (uri); - ret = 0; - return; - } - cfg = cfgarg; - ctx = GNUNET_FS_start (cfg, - "gnunet-search", - &progress_cb, - NULL, - GNUNET_FS_FLAGS_NONE, - GNUNET_FS_OPTIONS_END); - if (NULL == ctx) - { - fprintf (stderr, _ ("Could not initialize the `%s` subsystem.\n"), "FS"); - GNUNET_FS_uri_destroy (uri); - ret = 1; - return; - } - if (output_filename != NULL) - db = GNUNET_FS_directory_builder_create (NULL); - options = GNUNET_FS_SEARCH_OPTION_NONE; - if (local_only) - options |= GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY; - sc = GNUNET_FS_search_start (ctx, uri, anonymity, options, NULL); - GNUNET_FS_uri_destroy (uri); - if (NULL == sc) - { - fprintf (stderr, "%s", _ ("Could not start searching.\n")); - GNUNET_FS_stop (ctx); - ret = 1; - return; - } - if (0 != timeout.rel_value_us) - tt = GNUNET_SCHEDULER_add_delayed (timeout, &timeout_task, NULL); - GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL); -} - - -/** - * The main function to search GNUnet. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, an error number on error - */ -int -main (int argc, char *const *argv) -{ - struct GNUNET_GETOPT_CommandLineOption options[] = - { GNUNET_GETOPT_option_uint ( - 'a', - "anonymity", - "LEVEL", - gettext_noop ("set the desired LEVEL of receiver-anonymity (default: " - "1)"), - &anonymity), - GNUNET_GETOPT_option_flag ( - 'b', - "bookmark-only", - gettext_noop ("do not search, print only the URI that points to this " - "search"), - &bookmark_only), - GNUNET_GETOPT_option_string ( - 'F', - "dir-printf", - "FORMAT", - gettext_noop ("write search results for directories according to " - "FORMAT; accepted placeholders are: %a, %f, %j, %l, %m, " - "%n, %s; defaults to the value of --printf when omitted " - "or to `" HELP_DEFAULT_DIR_FORMAT "` if --printf is " - "omitted too"), - &dir_format_string), - GNUNET_GETOPT_option_string ( - 'f', - "printf", - "FORMAT", - gettext_noop ("write search results according to FORMAT; accepted " - "placeholders are: %a, %f, %j, %l, %m, %n, %s; defaults " - "to `" HELP_DEFAULT_FILE_FORMAT "` when omitted"), - &format_string), - GNUNET_GETOPT_option_string ( - 'i', - "iter-printf", - "FORMAT", - gettext_noop ("when the %a or %j placeholders appear in --printf or " - "--dir-printf, list each metadata property according to " - "FORMAT; accepted placeholders are: %i, %l, %n, %p" - HELP_EXTRACTOR_TEXTADD ", %w; defaults to `" - HELP_DEFAULT_META_FORMAT "` when omitted"), - &meta_format_string), - GNUNET_GETOPT_option_uint ('N', - "results", - "VALUE", - gettext_noop ("automatically terminate search " - "after VALUE results are found"), - &results_limit), - GNUNET_GETOPT_option_flag ( - 'n', - "no-network", - gettext_noop ("only search the local peer (no P2P network search)"), - &local_only), - GNUNET_GETOPT_option_string ( - 'o', - "output", - "FILENAME", - gettext_noop ("create a GNUnet directory with search results at " - "FILENAME (e.g. `gnunet-search --output=commons" - GNUNET_FS_DIRECTORY_EXT " commons`)"), - &output_filename), - GNUNET_GETOPT_option_flag ( - 's', - "silent", - gettext_noop ("silent mode (requires the --output argument)"), - &silent_mode), - GNUNET_GETOPT_option_relative_time ( - 't', - "timeout", - "DELAY", - gettext_noop ("automatically terminate search after DELAY; the value " - "given must be a number followed by a space and a time " - "unit, for example \"500 ms\"; without a unit it defaults " - "to microseconds - 1000000 = 1 second; if 0 or omitted " - "it means to wait for CTRL-C"), - &timeout), - GNUNET_GETOPT_option_increment_uint ( - 'V', - "verbose", - gettext_noop ("be verbose (append \"%a\\n\" to the default --printf and " - "--dir-printf arguments - ignored when these are provided " - "by the user)"), - &verbose), - GNUNET_GETOPT_OPTION_END }; - - if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) - return 12; - - if (GNUNET_SYSERR == - GNUNET_PROGRAM_run (argc, - argv, - "gnunet-search [OPTIONS] KEYWORD1 KEYWORD2 ...", - gettext_noop ("Search for files that have been " - "published on GNUnet\n"), - options, - &run, - NULL)) - ret = 1; - - GNUNET_free_nz ((void *) argv); - return ret; -} - - -/* end of gnunet-search.c */ diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c deleted file mode 100644 index 1ab6ac2b8..000000000 --- a/src/fs/gnunet-service-fs.c +++ /dev/null @@ -1,1378 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009-2014, 2016 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs.c - * @brief gnunet anonymity protocol implementation - * @author Christian Grothoff - */ -#include "platform.h" -#include -#include "gnunet_constants.h" -#include "gnunet_core_service.h" -#include "gnunet_dht_service.h" -#include "gnunet_datastore_service.h" -#include "gnunet_load_lib.h" -#include "gnunet_protocols.h" -#include "gnunet_signatures.h" -#include "gnunet_statistics_service.h" -#include "gnunet_util_lib.h" -#include "gnunet-service-fs_cp.h" -#include "gnunet-service-fs_indexing.h" -#include "gnunet-service-fs_pe.h" -#include "gnunet-service-fs_pr.h" -#include "gnunet-service-fs_push.h" -#include "gnunet-service-fs_put.h" -#include "gnunet-service-fs_cadet.h" -#include "fs.h" -#include "fs_api.h" - -/** - * Size for the hash map for DHT requests from the FS - * service. Should be about the number of concurrent - * DHT requests we plan to make. - */ -#define FS_DHT_HT_SIZE 1024 - - -/** - * How quickly do we age cover traffic? At the given - * time interval, remaining cover traffic counters are - * decremented by 1/16th. - */ -#define COVER_AGE_FREQUENCY GNUNET_TIME_relative_multiply ( \ - GNUNET_TIME_UNIT_SECONDS, 5) - -/** - * Collect an instance number of statistics? May cause excessive IPC. - */ -#define INSANE_STATISTICS GNUNET_NO - - -/** - * Doubly-linked list of requests we are performing - * on behalf of the same client. - */ -struct ClientRequest -{ - /** - * This is a doubly-linked list. - */ - struct ClientRequest *next; - - /** - * This is a doubly-linked list. - */ - struct ClientRequest *prev; - - /** - * Request this entry represents. - */ - struct GSF_PendingRequest *pr; - - /** - * Client list this request belongs to. - */ - struct GSF_LocalClient *lc; - - /** - * Task scheduled to destroy the request. - */ - struct GNUNET_SCHEDULER_Task *kill_task; -}; - - -/** - * Replies to be transmitted to the client. The actual - * response message is allocated after this struct. - */ -struct ClientResponse -{ - /** - * This is a doubly-linked list. - */ - struct ClientResponse *next; - - /** - * This is a doubly-linked list. - */ - struct ClientResponse *prev; - - /** - * Client list entry this response belongs to. - */ - struct GSF_LocalClient *lc; - - /** - * Number of bytes in the response. - */ - size_t msize; -}; - - -/** - * Information we track while handling an index - * start request from a client. - */ -struct IndexStartContext -{ - /** - * This is a doubly linked list. - */ - struct IndexStartContext *next; - - /** - * This is a doubly linked list. - */ - struct IndexStartContext *prev; - - /** - * Name of the indexed file. - */ - char *filename; - - /** - * Context for transmitting confirmation to client. - */ - struct GSF_LocalClient *lc; - - /** - * Context for hashing of the file. - */ - struct GNUNET_CRYPTO_FileHashContext *fhc; - - /** - * Hash of the contents of the file. - */ - struct GNUNET_HashCode file_id; -}; - - -/** - * A local client. - */ -struct GSF_LocalClient -{ - /** - * ID of the client. - */ - struct GNUNET_SERVICE_Client *client; - - /** - * Queue for sending replies. - */ - struct GNUNET_MQ_Handle *mq; - - /** - * Head of list of requests performed on behalf - * of this client right now. - */ - struct ClientRequest *cr_head; - - /** - * Tail of list of requests performed on behalf - * of this client right now. - */ - struct ClientRequest *cr_tail; - - /** - * This is a doubly linked list. - */ - struct IndexStartContext *isc_head; - - /** - * This is a doubly linked list. - */ - struct IndexStartContext *isc_tail; - - /** - * Head of linked list of responses. - */ - struct ClientResponse *res_head; - - /** - * Tail of linked list of responses. - */ - struct ClientResponse *res_tail; -}; - - -/* ****************************** globals ****************************** */ - -/** - * Our connection to the datastore. - */ -struct GNUNET_DATASTORE_Handle *GSF_dsh; - -/** - * Our configuration. - */ -const struct GNUNET_CONFIGURATION_Handle *GSF_cfg; - -/** - * Handle for reporting statistics. - */ -struct GNUNET_STATISTICS_Handle *GSF_stats; - -/** - * Handle for DHT operations. - */ -struct GNUNET_DHT_Handle *GSF_dht; - -/** - * How long do requests typically stay in the routing table? - */ -struct GNUNET_LOAD_Value *GSF_rt_entry_lifetime; - -/** - * Running average of the observed latency to other peers (round trip). - * Initialized to 5s as the initial default. - */ -struct GNUNET_TIME_Relative GSF_avg_latency = { 500 }; - - -/** - * Typical priorities we're seeing from other peers right now. Since - * most priorities will be zero, this value is the weighted average of - * non-zero priorities seen "recently". In order to ensure that new - * values do not dramatically change the ratio, values are first - * "capped" to a reasonable range (+N of the current value) and then - * averaged into the existing value by a ratio of 1:N. Hence - * receiving the largest possible priority can still only raise our - * "current_priorities" by at most 1. - */ -double GSF_current_priorities; - -/** - * Size of the datastore queue we assume for common requests. - */ -unsigned int GSF_datastore_queue_size; - -/** - * How many query messages have we received 'recently' that - * have not yet been claimed as cover traffic? - */ -unsigned int GSF_cover_query_count; - -/** - * How many content messages have we received 'recently' that - * have not yet been claimed as cover traffic? - */ -unsigned int GSF_cover_content_count; - -/** - * Our block context. - */ -struct GNUNET_BLOCK_Context *GSF_block_ctx; - -/** - * Pointer to handle to the core service (points to NULL until we've - * connected to it). - */ -struct GNUNET_CORE_Handle *GSF_core; - -/** - * Are we introducing randomized delays for better anonymity? - */ -int GSF_enable_randomized_delays; - -/** - * Identity of this peer. - */ -struct GNUNET_PeerIdentity GSF_my_id; - -/* ***************************** locals ******************************* */ - -/** - * Configuration for block library. - */ -static struct GNUNET_CONFIGURATION_Handle *block_cfg; - -/** - * Private key of this peer. Used to sign LOC URI requests. - */ -static struct GNUNET_CRYPTO_EddsaPrivateKey pk; - -/** - * ID of our task that we use to age the cover counters. - */ -static struct GNUNET_SCHEDULER_Task *cover_age_task; - -/** - * Datastore 'GET' load tracking. - */ -static struct GNUNET_LOAD_Value *datastore_get_load; - - -/** - * Creates a fresh local client handle. - * - * @param cls NULL - * @param client handle of the client - * @param mq message queue for @a client - * @return handle to local client entry - */ -static void * -client_connect_cb (void *cls, - struct GNUNET_SERVICE_Client *client, - struct GNUNET_MQ_Handle *mq) -{ - struct GSF_LocalClient *pos; - - pos = GNUNET_new (struct GSF_LocalClient); - pos->client = client; - pos->mq = mq; - return pos; -} - - -/** - * Free the given client request. - * - * @param cls the client request to free - */ -static void -client_request_destroy (void *cls) -{ - struct ClientRequest *cr = cls; - struct GSF_LocalClient *lc = cr->lc; - - cr->kill_task = NULL; - GNUNET_CONTAINER_DLL_remove (lc->cr_head, - lc->cr_tail, - cr); - GSF_pending_request_cancel_ (cr->pr, - GNUNET_YES); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# client searches active"), - -1, - GNUNET_NO); - GNUNET_free (cr); -} - - -/** - * Handle a reply to a pending request. Also called if a request - * expires (then with data == NULL). The handler may be called - * many times (depending on the request type), but will not be - * called during or after a call to #GSF_pending_request_cancel() - * and will also not be called anymore after a call signalling - * expiration. - * - * @param cls user-specified closure - * @param eval evaluation of the result - * @param pr handle to the original pending request - * @param reply_anonymity_level anonymity level for the reply, UINT32_MAX for "unknown" - * @param expiration when does @a data expire? - * @param last_transmission when was the last time we've tried to download this block? (FOREVER if unknown) - * @param type type of the block - * @param data response data, NULL on request expiration - * @param data_len number of bytes in @a data - */ -static void -client_response_handler (void *cls, - enum GNUNET_BLOCK_ReplyEvaluationResult eval, - struct GSF_PendingRequest *pr, - uint32_t reply_anonymity_level, - struct GNUNET_TIME_Absolute expiration, - struct GNUNET_TIME_Absolute last_transmission, - enum GNUNET_BLOCK_Type type, - const void *data, - size_t data_len) -{ - struct ClientRequest *cr = cls; - struct GSF_LocalClient *lc; - struct GNUNET_MQ_Envelope *env; - struct ClientPutMessage *pm; - const struct GSF_PendingRequestData *prd; - - if (NULL == data) - { - /* local-only request, with no result, clean up. */ - if (NULL == cr->kill_task) - cr->kill_task = GNUNET_SCHEDULER_add_now (&client_request_destroy, - cr); - return; - } - prd = GSF_pending_request_get_data_ (pr); - GNUNET_break (type != GNUNET_BLOCK_TYPE_ANY); - if ((prd->type != type) && (prd->type != GNUNET_BLOCK_TYPE_ANY)) - { - GNUNET_break (0); - return; - } - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop - ("# replies received for local clients"), 1, - GNUNET_NO); - GNUNET_assert (pr == cr->pr); - lc = cr->lc; - env = GNUNET_MQ_msg_extra (pm, - data_len, - GNUNET_MESSAGE_TYPE_FS_PUT); - pm->type = htonl (type); - pm->expiration = GNUNET_TIME_absolute_hton (expiration); - pm->last_transmission = GNUNET_TIME_absolute_hton (last_transmission); - pm->num_transmissions = htonl (prd->num_transmissions); - pm->respect_offered = htonl (prd->respect_offered); - GNUNET_memcpy (&pm[1], - data, - data_len); - GNUNET_MQ_send (lc->mq, - env); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Queued reply to query `%s' for local client\n", - GNUNET_h2s (&prd->query)); - if (GNUNET_BLOCK_REPLY_OK_LAST != eval) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Evaluation %d - keeping query alive\n", - (int) eval); - return; - } - if (NULL == cr->kill_task) - cr->kill_task = GNUNET_SCHEDULER_add_now (&client_request_destroy, - cr); -} - - -/** - * A client disconnected from us. Tear down the local client - * record. - * - * @param cls unused - * @param client handle of the client - * @param app_ctx the `struct GSF_LocalClient` - */ -static void -client_disconnect_cb (void *cls, - struct GNUNET_SERVICE_Client *client, - void *app_ctx) -{ - struct GSF_LocalClient *lc = app_ctx; - struct IndexStartContext *isc; - struct ClientRequest *cr; - struct ClientResponse *res; - - while (NULL != (cr = lc->cr_head)) - { - if (NULL != cr->kill_task) - GNUNET_SCHEDULER_cancel (cr->kill_task); - client_request_destroy (cr); - } - while (NULL != (res = lc->res_head)) - { - GNUNET_CONTAINER_DLL_remove (lc->res_head, - lc->res_tail, - res); - GNUNET_free (res); - } - while (NULL != (isc = lc->isc_head)) - { - GNUNET_CONTAINER_DLL_remove (lc->isc_head, - lc->isc_tail, - isc); - GNUNET_CRYPTO_hash_file_cancel (isc->fhc); - GNUNET_free (isc); - } - GNUNET_free (lc); -} - - -/** - * Task that periodically ages our cover traffic statistics. - * - * @param cls unused closure - */ -static void -age_cover_counters (void *cls) -{ - GSF_cover_content_count = (GSF_cover_content_count * 15) / 16; - GSF_cover_query_count = (GSF_cover_query_count * 15) / 16; - cover_age_task = - GNUNET_SCHEDULER_add_delayed (COVER_AGE_FREQUENCY, - &age_cover_counters, - NULL); -} - - -/** - * We've just now completed a datastore request. Update our - * datastore load calculations. - * - * @param start time when the datastore request was issued - */ -void -GSF_update_datastore_delay_ (struct GNUNET_TIME_Absolute start) -{ - struct GNUNET_TIME_Relative delay; - - delay = GNUNET_TIME_absolute_get_duration (start); - GNUNET_LOAD_update (datastore_get_load, delay.rel_value_us); -} - - -/** - * Test if the DATABASE (GET) load on this peer is too high - * to even consider processing the query at - * all. - * - * @param priority priority of the request (used as a reference point to compare with the load) - * @return #GNUNET_YES if the load is too high to do anything (load high) - * #GNUNET_NO to process normally (load normal) - * #GNUNET_SYSERR to process for free (load low) - */ -int -GSF_test_get_load_too_high_ (uint32_t priority) -{ - double ld; - - ld = GNUNET_LOAD_get_load (datastore_get_load); - if (ld < 1) - return GNUNET_SYSERR; - if (ld <= priority) - return GNUNET_NO; - return GNUNET_YES; -} - - -/** - * Check P2P "PUT" message. - * - * @param cls closure with the `struct GSF_ConnectedPeer` - * @param put the actual message - * @return #GNUNET_OK to keep the connection open, - * #GNUNET_SYSERR to close it (signal serious error) - */ -static int -check_p2p_put (void *cls, - const struct PutMessage *put) -{ - enum GNUNET_BLOCK_Type type; - - type = ntohl (put->type); - if (GNUNET_BLOCK_TYPE_FS_ONDEMAND == type) - { - GNUNET_break_op (0); - return GNUNET_SYSERR; - } - return GNUNET_OK; -} - - -/** - * We have a new request, consider forwarding it to the given - * peer. - * - * @param cls the `struct GSF_PendingRequest` - * @param peer identity of the peer - * @param cp handle to the connected peer record - * @param ppd peer performance data - */ -static void -consider_request_for_forwarding (void *cls, - const struct GNUNET_PeerIdentity *peer, - struct GSF_ConnectedPeer *cp, - const struct GSF_PeerPerformanceData *ppd) -{ - struct GSF_PendingRequest *pr = cls; - - if (GNUNET_YES != - GSF_pending_request_test_target_ (pr, peer)) - { -#if INSANE_STATISTICS - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# Loopback routes suppressed"), 1, - GNUNET_NO); -#endif - return; - } - GSF_plan_add_ (cp, - pr); -} - - -/** - * Function to be called after we're done processing - * replies from the local lookup. If the result status - * code indicates that there may be more replies, plan - * forwarding the request. - * - * @param cls closure (NULL) - * @param pr the pending request we were processing - * @param result final datastore lookup result - */ -void -GSF_consider_forwarding (void *cls, - struct GSF_PendingRequest *pr, - enum GNUNET_BLOCK_ReplyEvaluationResult result) -{ - if (GNUNET_BLOCK_REPLY_OK_LAST == result) - return; /* we're done... */ - if (GNUNET_YES != - GSF_pending_request_test_active_ (pr)) - return; /* request is not actually active, skip! */ - GSF_iterate_connected_peers_ (&consider_request_for_forwarding, - pr); -} - - -/** - * Check P2P "GET" request. - * - * @param cls closure - * @param gm the actual message - * @return #GNUNET_OK to keep the connection open, - * #GNUNET_SYSERR to close it (signal serious error) - */ -static int -check_p2p_get (void *cls, - const struct GetMessage *gm) -{ - size_t msize; - unsigned int bm; - unsigned int bits; - size_t bfsize; - - msize = ntohs (gm->header.size); - bm = ntohl (gm->hash_bitmap); - bits = 0; - while (bm > 0) - { - if (1 == (bm & 1)) - bits++; - bm >>= 1; - } - if (msize < sizeof(struct GetMessage) + bits * sizeof(struct - GNUNET_PeerIdentity)) - { - GNUNET_break_op (0); - return GNUNET_SYSERR; - } - bfsize = msize - sizeof(struct GetMessage) - bits * sizeof(struct - GNUNET_PeerIdentity); - /* bfsize must be power of 2, check! */ - if (0 != ((bfsize - 1) & bfsize)) - { - GNUNET_break_op (0); - return GNUNET_SYSERR; - } - return GNUNET_OK; -} - - -/** - * We're done with the local lookup, now consider - * P2P processing (depending on request options and - * result status). Also signal that we can now - * receive more request information from the client. - * - * @param cls the client doing the request (`struct GSF_LocalClient`) - * @param pr the pending request we were processing - * @param result final datastore lookup result - */ -static void -start_p2p_processing (void *cls, - struct GSF_PendingRequest *pr, - enum GNUNET_BLOCK_ReplyEvaluationResult result) -{ - struct GSF_LocalClient *lc = cls; - struct GSF_PendingRequestData *prd; - - GNUNET_SERVICE_client_continue (lc->client); - if (GNUNET_BLOCK_REPLY_OK_LAST == result) - return; /* we're done, 'pr' was already destroyed... */ - prd = GSF_pending_request_get_data_ (pr); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Finished database lookup for local request `%s' with result %d\n", - GNUNET_h2s (&prd->query), - result); - if (0 == prd->anonymity_level) - { - switch (prd->type) - { - case GNUNET_BLOCK_TYPE_FS_DBLOCK: - case GNUNET_BLOCK_TYPE_FS_IBLOCK: - /* the above block types MAY be available via 'cadet' */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Considering cadet-based download for block\n"); - GSF_cadet_lookup_ (pr); - break; - - case GNUNET_BLOCK_TYPE_FS_UBLOCK: - /* the above block types are in the DHT */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Considering DHT-based search for block\n"); - GSF_dht_lookup_ (pr); - break; - - default: - GNUNET_break (0); - break; - } - } - GSF_consider_forwarding (NULL, - pr, - result); -} - - -/** - * Check #GNUNET_MESSAGE_TYPE_FS_START_SEARCH-message (search request - * from client). - * - * @param cls identification of the client - * @param sm the actual message - * @return #GNUNET_OK if @a sm is well-formed - */ -static int -check_client_start_search (void *cls, - const struct SearchMessage *sm) -{ - uint16_t msize; - - msize = ntohs (sm->header.size) - sizeof(struct SearchMessage); - if (0 != msize % sizeof(struct GNUNET_HashCode)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - return GNUNET_OK; -} - - -/** - * Handle #GNUNET_MESSAGE_TYPE_FS_START_SEARCH-message (search request - * from client). - * - * Responsible for creating the request entry itself and setting - * up reply callback and cancellation on client disconnect. - * - * @param cls identification of the client - * @param sm the actual message - */ -static void -handle_client_start_search (void *cls, - const struct SearchMessage *sm) -{ - static struct GNUNET_PeerIdentity all_zeros; - struct GSF_LocalClient *lc = cls; - struct ClientRequest *cr; - struct GSF_PendingRequestData *prd; - uint16_t msize; - unsigned int sc; - enum GNUNET_BLOCK_Type type; - enum GSF_PendingRequestOptions options; - - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# client searches received"), - 1, - GNUNET_NO); - msize = ntohs (sm->header.size) - sizeof(struct SearchMessage); - sc = msize / sizeof(struct GNUNET_HashCode); - type = ntohl (sm->type); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Received request for `%s' of type %u from local client\n", - GNUNET_h2s (&sm->query), - (unsigned int) type); - cr = NULL; - /* detect duplicate UBLOCK requests */ - if ((type == GNUNET_BLOCK_TYPE_FS_UBLOCK) || - (type == GNUNET_BLOCK_TYPE_ANY)) - { - cr = lc->cr_head; - while (NULL != cr) - { - prd = GSF_pending_request_get_data_ (cr->pr); - /* only unify with queries that hae not yet started local processing - (SEARCH_MESSAGE_OPTION_CONTINUED was always set) and that have a - matching query and type */ - if ((GNUNET_YES != prd->has_started) && - (0 != memcmp (&prd->query, - &sm->query, - sizeof(struct GNUNET_HashCode))) && - (prd->type == type)) - break; - cr = cr->next; - } - } - if (NULL != cr) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Have existing request, merging content-seen lists.\n"); - GSF_pending_request_update_ (cr->pr, - (const struct GNUNET_HashCode *) &sm[1], - sc); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# client searches updated (merged content seen list)"), - 1, - GNUNET_NO); - } - else - { - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# client searches active"), - 1, - GNUNET_NO); - cr = GNUNET_new (struct ClientRequest); - cr->lc = lc; - GNUNET_CONTAINER_DLL_insert (lc->cr_head, - lc->cr_tail, - cr); - options = GSF_PRO_LOCAL_REQUEST; - if (0 != (SEARCH_MESSAGE_OPTION_LOOPBACK_ONLY & ntohl (sm->options))) - options |= GSF_PRO_LOCAL_ONLY; - cr->pr = GSF_pending_request_create_ (options, type, - &sm->query, - (0 != - memcmp (&sm->target, - &all_zeros, - sizeof(struct - GNUNET_PeerIdentity))) - ? &sm->target : NULL, NULL, - 0 /* bf */, - ntohl (sm->anonymity_level), - 0 /* priority */, - 0 /* ttl */, - 0 /* sender PID */, - 0 /* origin PID */, - (const struct - GNUNET_HashCode *) &sm[1], sc, - &client_response_handler, - cr); - } - if (0 != (SEARCH_MESSAGE_OPTION_CONTINUED & ntohl (sm->options))) - { - GNUNET_SERVICE_client_continue (lc->client); - return; - } - GSF_pending_request_get_data_ (cr->pr)->has_started = GNUNET_YES; - GSF_local_lookup_ (cr->pr, - &start_p2p_processing, - lc); -} - - -/** - * Handle request to sign a LOC URI (from client). - * - * @param cls identification of the client - * @param msg the actual message - */ -static void -handle_client_loc_sign (void *cls, - const struct RequestLocSignatureMessage *msg) -{ - struct GSF_LocalClient *lc = cls; - struct GNUNET_FS_Uri base; - struct GNUNET_FS_Uri *loc; - struct GNUNET_MQ_Envelope *env; - struct ResponseLocSignatureMessage *resp; - - GNUNET_break (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT == - ntohl (msg->purpose)); - base.type = GNUNET_FS_URI_CHK; - base.data.chk.chk = msg->chk; - base.data.chk.file_length = GNUNET_ntohll (msg->file_length); - loc = GNUNET_FS_uri_loc_create (&base, - &pk, - GNUNET_TIME_absolute_ntoh ( - msg->expiration_time)); - env = GNUNET_MQ_msg (resp, - GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGNATURE); - resp->purpose = htonl (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT); - resp->expiration_time = GNUNET_TIME_absolute_hton ( - loc->data.loc.expirationTime); - resp->signature = loc->data.loc.contentSignature; - resp->peer = loc->data.loc.peer; - GNUNET_FS_uri_destroy (loc); - GNUNET_MQ_send (lc->mq, - env); - GNUNET_SERVICE_client_continue (lc->client); -} - - -/** - * Check INDEX_START-message. - * - * @param cls identification of the client - * @param ism the actual message - * @return #GNUNET_OK if @a ism is well-formed - */ -static int -check_client_index_start (void *cls, - const struct IndexStartMessage *ism) -{ - char *fn; - - GNUNET_MQ_check_zero_termination (ism); - if (0 != ism->reserved) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - fn = GNUNET_STRINGS_filename_expand ((const char *) &ism[1]); - if (NULL == fn) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - GNUNET_free (fn); - return GNUNET_OK; -} - - -/** - * We've validated the hash of the file we're about to index. Signal - * success to the client and update our internal data structures. - * - * @param isc the data about the index info entry for the request - */ -static void -signal_index_ok (struct IndexStartContext *isc) -{ - struct GSF_LocalClient *lc = isc->lc; - struct GNUNET_MQ_Envelope *env; - struct GNUNET_MessageHeader *msg; - - GNUNET_FS_add_to_index (isc->filename, - &isc->file_id); - env = GNUNET_MQ_msg (msg, - GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK); - GNUNET_MQ_send (lc->mq, - env); - GNUNET_free (isc->filename); - GNUNET_free (isc); - GNUNET_SERVICE_client_continue (lc->client); -} - - -/** - * Function called once the hash computation over an - * indexed file has completed. - * - * @param cls closure, our publishing context - * @param res resulting hash, NULL on error - */ -static void -hash_for_index_val (void *cls, - const struct GNUNET_HashCode *res) -{ - struct IndexStartContext *isc = cls; - struct GSF_LocalClient *lc = isc->lc; - struct GNUNET_MQ_Envelope *env; - struct GNUNET_MessageHeader *msg; - - GNUNET_CONTAINER_DLL_remove (lc->isc_head, - lc->isc_tail, - isc); - isc->fhc = NULL; - if ((NULL == res) || - (0 != memcmp (res, - &isc->file_id, - sizeof(struct GNUNET_HashCode)))) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ( - "Hash mismatch trying to index file `%s' which does not have hash `%s'\n"), - isc->filename, - GNUNET_h2s (&isc->file_id)); - - const char *emsg = "hash mismatch"; - const size_t msize = strlen (emsg) + 1; - - env = GNUNET_MQ_msg_extra (msg, - msize, - GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED); - memcpy ((char*) &msg[1], emsg, msize); - GNUNET_MQ_send (lc->mq, - env); - GNUNET_SERVICE_client_continue (lc->client); - GNUNET_free (isc); - return; - } - signal_index_ok (isc); -} - - -/** - * Handle INDEX_START-message. - * - * @param cls identification of the client - * @param ism the actual message - */ -static void -handle_client_index_start (void *cls, - const struct IndexStartMessage *ism) -{ - struct GSF_LocalClient *lc = cls; - struct IndexStartContext *isc; - char *fn; - uint64_t dev; - uint64_t ino; - uint64_t mydev; - uint64_t myino; - - fn = GNUNET_STRINGS_filename_expand ((const char *) &ism[1]); - GNUNET_assert (NULL != fn); - dev = GNUNET_ntohll (ism->device); - ino = GNUNET_ntohll (ism->inode); - isc = GNUNET_new (struct IndexStartContext); - isc->filename = fn; - isc->file_id = ism->file_id; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Received START_INDEX message for file `%s'\n", - isc->filename); - isc->lc = lc; - mydev = 0; - myino = 0; - if (((dev != 0) || - (ino != 0)) && - (GNUNET_OK == GNUNET_DISK_file_get_identifiers (fn, - &mydev, - &myino)) && - (dev == mydev) && - (ino == myino)) - { - /* fast validation OK! */ - signal_index_ok (isc); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n", - (unsigned long long) ino, - (unsigned long long) myino, - (unsigned int) dev, - (unsigned int) mydev); - /* slow validation, need to hash full file (again) */ - GNUNET_CONTAINER_DLL_insert (lc->isc_head, - lc->isc_tail, - isc); - isc->fhc = GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, - isc->filename, - HASHING_BLOCKSIZE, - &hash_for_index_val, - isc); - if (NULL == isc->fhc) - hash_for_index_val (isc, - NULL); -} - - -/** - * Handle INDEX_LIST_GET-message. - * - * @param cls closure - * @param message the actual message - */ -static void -handle_client_index_list_get (void *cls, - const struct GNUNET_MessageHeader *message) -{ - struct GSF_LocalClient *lc = cls; - - GNUNET_FS_indexing_send_list (lc->mq); - GNUNET_SERVICE_client_continue (lc->client); -} - - -/** - * Handle UNINDEX-message. - * - * @param cls identification of the client - * @param um the actual message - */ -static void -handle_client_unindex (void *cls, - const struct UnindexMessage *um) -{ - struct GSF_LocalClient *lc = cls; - struct GNUNET_MQ_Envelope *env; - struct GNUNET_MessageHeader *msg; - int found; - - GNUNET_break (0 == um->reserved); - found = GNUNET_FS_indexing_do_unindex (&um->file_id); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Client requested unindexing of file `%s': %s\n", - GNUNET_h2s (&um->file_id), - found ? "found" : "not found"); - env = GNUNET_MQ_msg (msg, - GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK); - GNUNET_MQ_send (lc->mq, - env); - GNUNET_SERVICE_client_continue (lc->client); -} - - -/** - * Task run during shutdown. - * - * @param cls unused - */ -static void -shutdown_task (void *cls) -{ - GSF_cadet_stop_server (); - if (NULL != GSF_core) - { - GNUNET_CORE_disconnect (GSF_core); - GSF_core = NULL; - } - GSF_put_done_ (); - GSF_push_done_ (); - GSF_pending_request_done_ (); - GSF_plan_done (); - GSF_connected_peer_done_ (); - GNUNET_DATASTORE_disconnect (GSF_dsh, - GNUNET_NO); - GSF_dsh = NULL; - GNUNET_DHT_disconnect (GSF_dht); - GSF_dht = NULL; - GNUNET_BLOCK_context_destroy (GSF_block_ctx); - GSF_block_ctx = NULL; - GNUNET_CONFIGURATION_destroy (block_cfg); - block_cfg = NULL; - GNUNET_STATISTICS_destroy (GSF_stats, GNUNET_NO); - GSF_stats = NULL; - if (NULL != cover_age_task) - { - GNUNET_SCHEDULER_cancel (cover_age_task); - cover_age_task = NULL; - } - GNUNET_FS_indexing_done (); - GNUNET_LOAD_value_free (datastore_get_load); - datastore_get_load = NULL; - GNUNET_LOAD_value_free (GSF_rt_entry_lifetime); - GSF_rt_entry_lifetime = NULL; -} - - -/** - * Function called after GNUNET_CORE_connect has succeeded - * (or failed for good). Note that the private key of the - * peer is intentionally not exposed here; if you need it, - * your process should try to read the private key file - * directly (which should work if you are authorized...). - * - * @param cls closure - * @param my_identity ID of this peer, NULL if we failed - */ -static void -peer_init_handler (void *cls, - const struct GNUNET_PeerIdentity *my_identity) -{ - if (0 != GNUNET_memcmp (&GSF_my_id, - my_identity)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Peer identity mismatch, refusing to start!\n"); - GNUNET_SCHEDULER_shutdown (); - } -} - - -/** - * Process fs requests. - * - * @param c configuration to use - */ -static int -main_init (const struct GNUNET_CONFIGURATION_Handle *c) -{ - struct GNUNET_MQ_MessageHandler no_p2p_handlers[] = { - GNUNET_MQ_handler_end () - }; - struct GNUNET_MQ_MessageHandler p2p_handlers[] = { - GNUNET_MQ_hd_var_size (p2p_get, - GNUNET_MESSAGE_TYPE_FS_GET, - struct GetMessage, - NULL), - GNUNET_MQ_hd_var_size (p2p_put, - GNUNET_MESSAGE_TYPE_FS_PUT, - struct PutMessage, - NULL), - GNUNET_MQ_hd_fixed_size (p2p_migration_stop, - GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP, - struct MigrationStopMessage, - NULL), - GNUNET_MQ_handler_end () - }; - int anon_p2p_off; - char *keyfile; - - /* this option is really only for testcases that need to disable - _anonymous_ file-sharing for some reason */ - anon_p2p_off = (GNUNET_YES == - GNUNET_CONFIGURATION_get_value_yesno (GSF_cfg, - "fs", - "DISABLE_ANON_TRANSFER")); - - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_filename (GSF_cfg, - "PEER", - "PRIVATE_KEY", - &keyfile)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ( - "FS service is lacking HOSTKEY configuration setting. Exiting.\n")); - GNUNET_SCHEDULER_shutdown (); - return GNUNET_SYSERR; - } - if (GNUNET_SYSERR == - GNUNET_CRYPTO_eddsa_key_from_file (keyfile, - GNUNET_YES, - &pk)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Failed to setup peer's private key\n"); - GNUNET_SCHEDULER_shutdown (); - GNUNET_free (keyfile); - return GNUNET_SYSERR; - } - GNUNET_free (keyfile); - GNUNET_CRYPTO_eddsa_key_get_public (&pk, - &GSF_my_id.public_key); - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "I am peer %s\n", - GNUNET_i2s (&GSF_my_id)); - GSF_core - = GNUNET_CORE_connect (GSF_cfg, - NULL, - &peer_init_handler, - &GSF_peer_connect_handler, - &GSF_peer_disconnect_handler, - (GNUNET_YES == anon_p2p_off) - ? no_p2p_handlers - : p2p_handlers); - if (NULL == GSF_core) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Failed to connect to `%s' service.\n"), - "core"); - return GNUNET_SYSERR; - } - cover_age_task = - GNUNET_SCHEDULER_add_delayed (COVER_AGE_FREQUENCY, - &age_cover_counters, - NULL); - datastore_get_load = GNUNET_LOAD_value_init (DATASTORE_LOAD_AUTODECLINE); - GSF_cadet_start_server (); - GNUNET_SCHEDULER_add_shutdown (&shutdown_task, - NULL); - return GNUNET_OK; -} - - -/** - * Process fs requests. - * - * @param cls closure - * @param cfg configuration to use - * @param service the initialized service - */ -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_SERVICE_Handle *service) -{ - unsigned long long dqs; - - GSF_cfg = cfg; - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_size (GSF_cfg, - "fs", - "DATASTORE_QUEUE_SIZE", - &dqs)) - { - GNUNET_log_config_missing (GNUNET_ERROR_TYPE_INFO, - "fs", - "DATASTORE_QUEUE_SIZE"); - dqs = 32; - } - GSF_datastore_queue_size = (unsigned int) dqs; - GSF_enable_randomized_delays = - GNUNET_CONFIGURATION_get_value_yesno (cfg, "fs", "DELAY"); - GSF_dsh = GNUNET_DATASTORE_connect (cfg); - if (NULL == GSF_dsh) - { - GNUNET_SCHEDULER_shutdown (); - return; - } - GSF_rt_entry_lifetime = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_FOREVER_REL); - GSF_stats = GNUNET_STATISTICS_create ("fs", cfg); - block_cfg = GNUNET_CONFIGURATION_create (); - GSF_block_ctx = GNUNET_BLOCK_context_create (block_cfg); - GNUNET_assert (NULL != GSF_block_ctx); - GSF_dht = GNUNET_DHT_connect (cfg, FS_DHT_HT_SIZE); - GSF_plan_init (); - GSF_pending_request_init_ (); - GSF_connected_peer_init_ (); - - GSF_push_init_ (); - GSF_put_init_ (); - if ((GNUNET_OK != GNUNET_FS_indexing_init (cfg, - GSF_dsh)) || - (GNUNET_OK != main_init (cfg))) - { - GNUNET_SCHEDULER_shutdown (); - shutdown_task (NULL); - return; - } -} - - -/** - * Define "main" method using service macro. - */ -GNUNET_SERVICE_MAIN - ("fs", - GNUNET_SERVICE_OPTION_NONE, - &run, - &client_connect_cb, - &client_disconnect_cb, - NULL, - GNUNET_MQ_hd_var_size (client_index_start, - GNUNET_MESSAGE_TYPE_FS_INDEX_START, - struct IndexStartMessage, - NULL), - GNUNET_MQ_hd_fixed_size (client_index_list_get, - GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_GET, - struct GNUNET_MessageHeader, - NULL), - GNUNET_MQ_hd_fixed_size (client_unindex, - GNUNET_MESSAGE_TYPE_FS_UNINDEX, - struct UnindexMessage, - NULL), - GNUNET_MQ_hd_var_size (client_start_search, - GNUNET_MESSAGE_TYPE_FS_START_SEARCH, - struct SearchMessage, - NULL), - GNUNET_MQ_hd_fixed_size (client_loc_sign, - GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGN, - struct RequestLocSignatureMessage, - NULL), - GNUNET_MQ_handler_end ()); - - -/* end of gnunet-service-fs.c */ diff --git a/src/fs/gnunet-service-fs.h b/src/fs/gnunet-service-fs.h deleted file mode 100644 index e102a1fba..000000000 --- a/src/fs/gnunet-service-fs.h +++ /dev/null @@ -1,304 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009, 2010 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs.h - * @brief shared data structures of gnunet-service-fs.c - * @author Christian Grothoff - */ -#ifndef GNUNET_SERVICE_FS_H -#define GNUNET_SERVICE_FS_H - -#include "gnunet_util_lib.h" -#include "gnunet_statistics_service.h" -#include "gnunet_core_service.h" -#include "gnunet_block_lib.h" -#include "fs.h" - - -/** - * By which amount do we decrement the TTL for simple forwarding / - * indirection of the query; in milli-seconds. Set somewhat in - * accordance to your network latency (above the time it'll take you - * to send a packet and get a reply). - */ -#define TTL_DECREMENT 5000 - -/** - * At what frequency should our datastore load decrease - * automatically (since if we don't use it, clearly the - * load must be going down). - */ -#define DATASTORE_LOAD_AUTODECLINE GNUNET_TIME_relative_multiply ( \ - GNUNET_TIME_UNIT_MILLISECONDS, 250) - -/** - * Only the (mandatory) query is included. - */ -#define GET_MESSAGE_BIT_QUERY_ONLY 0 - -/** - * The peer identity of a peer waiting for the - * reply is included (used if the response - * should be transmitted to someone other than - * the sender of the GET). - */ -#define GET_MESSAGE_BIT_RETURN_TO 1 - -/** - * The peer identity of a peer that had claimed to have the content - * previously is included (can be used if responder-anonymity is not - * desired; note that the precursor presumably lacked a direct - * connection to the specified peer; still, the receiver is in no way - * required to limit forwarding only to the specified peer, it should - * only prefer it somewhat if possible). - */ -#define GET_MESSAGE_BIT_TRANSMIT_TO 4 - - -GNUNET_NETWORK_STRUCT_BEGIN - -/** - * Message sent between peers asking for FS-content. - */ -struct GetMessage -{ - /** - * Message type will be #GNUNET_MESSAGE_TYPE_FS_GET. - */ - struct GNUNET_MessageHeader header; - - /** - * Type of the query (block type). - */ - uint32_t type GNUNET_PACKED; - - /** - * How important is this request (network byte order) - */ - uint32_t priority GNUNET_PACKED; - - /** - * Relative time to live in MILLISECONDS (network byte order) - */ - int32_t ttl GNUNET_PACKED; - - /** - * These days not used. - */ - uint32_t reserved GNUNET_PACKED; - - /** - * Which of the optional hash codes are present at the end of the - * message? See GET_MESSAGE_BIT_xx constants. For each bit that is - * set, an additional `struct GNUNET_HashCode` with the respective content - * (in order of the bits) will be appended to the end of the GET - * message. - */ - uint32_t hash_bitmap GNUNET_PACKED; - - /** - * Hashcodes of the file(s) we're looking for. - * Details depend on the query type. - */ - struct GNUNET_HashCode query; - - /* this is followed by PeerIdentities as specified in the "hash_bitmap"; - * after that, an optional bloomfilter (with bits set for replies - * that should be suppressed) can be present */ -}; - - -/** - * Message send by a peer that wants to be excluded - * from migration for a while. - */ -struct MigrationStopMessage -{ - /** - * Message type will be - * GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP. - */ - struct GNUNET_MessageHeader header; - - /** - * Always zero. - */ - uint32_t reserved GNUNET_PACKED; - - /** - * How long should the block last? - */ - struct GNUNET_TIME_RelativeNBO duration; -}; -GNUNET_NETWORK_STRUCT_END - -/** - * A connected peer. - */ -struct GSF_ConnectedPeer; - -/** - * An active request. - */ -struct GSF_PendingRequest; - -/** - * A local client. - */ -struct GSF_LocalClient; - -/** - * Information kept per plan per request ('pe' module). - */ -struct GSF_RequestPlan; - -/** - * Bijection between request plans and pending requests. - */ -struct GSF_PendingRequestPlanBijection; - -/** - * Our connection to the datastore. - */ -extern struct GNUNET_DATASTORE_Handle *GSF_dsh; - -/** - * Our configuration. - */ -extern const struct GNUNET_CONFIGURATION_Handle *GSF_cfg; - -/** - * Handle for reporting statistics. - */ -extern struct GNUNET_STATISTICS_Handle *GSF_stats; - -/** - * Pointer to handle to the core service (points to NULL until we've - * connected to it). - */ -extern struct GNUNET_CORE_Handle *GSF_core; - -/** - * Handle for DHT operations. - */ -extern struct GNUNET_DHT_Handle *GSF_dht; - -/** - * How long do requests typically stay in the routing table? - */ -extern struct GNUNET_LOAD_Value *GSF_rt_entry_lifetime; - -/** - * Running average of the observed latency to other peers (round trip). - */ -extern struct GNUNET_TIME_Relative GSF_avg_latency; - -/** - * Handle to ATS service. - */ -extern struct GNUNET_ATS_PerformanceHandle *GSF_ats; - -/** - * Identity of this peer. - */ -extern struct GNUNET_PeerIdentity GSF_my_id; - -/** - * Typical priorities we're seeing from other peers right now. Since - * most priorities will be zero, this value is the weighted average of - * non-zero priorities seen "recently". In order to ensure that new - * values do not dramatically change the ratio, values are first - * "capped" to a reasonable range (+N of the current value) and then - * averaged into the existing value by a ratio of 1:N. Hence - * receiving the largest possible priority can still only raise our - * "current_priorities" by at most 1. - */ -extern double GSF_current_priorities; - -/** - * How many query messages have we received 'recently' that - * have not yet been claimed as cover traffic? - */ -extern unsigned int GSF_cover_query_count; - -/** - * How many content messages have we received 'recently' that - * have not yet been claimed as cover traffic? - */ -extern unsigned int GSF_cover_content_count; - -/** - * Our block context. - */ -extern struct GNUNET_BLOCK_Context *GSF_block_ctx; - -/** - * Are we introducing randomized delays for better anonymity? - */ -extern int GSF_enable_randomized_delays; - -/** - * Size of the datastore queue we assume for common requests. - */ -extern unsigned int GSF_datastore_queue_size; - - -/** - * Function to be called after we're done processing - * replies from the local lookup. If the result status - * code indicates that there may be more replies, plan - * forwarding the request. - * - * @param cls closure (NULL) - * @param pr the pending request we were processing - * @param result final datastore lookup result - */ -void -GSF_consider_forwarding (void *cls, - struct GSF_PendingRequest *pr, - enum GNUNET_BLOCK_ReplyEvaluationResult result); - - -/** - * Test if the DATABASE (GET) load on this peer is too high - * to even consider processing the query at - * all. - * - * @return #GNUNET_YES if the load is too high to do anything (load high) - * #GNUNET_NO to process normally (load normal) - * #GNUNET_SYSERR to process for free (load low) - */ -int -GSF_test_get_load_too_high_ (uint32_t priority); - - -/** - * We've just now completed a datastore request. Update our - * datastore load calculations. - * - * @param start time when the datastore request was issued - */ -void -GSF_update_datastore_delay_ (struct GNUNET_TIME_Absolute start); - - -#endif -/* end of gnunet-service-fs.h */ diff --git a/src/fs/gnunet-service-fs_cadet.h b/src/fs/gnunet-service-fs_cadet.h deleted file mode 100644 index c02021a0d..000000000 --- a/src/fs/gnunet-service-fs_cadet.h +++ /dev/null @@ -1,168 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2012, 2017 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_cadet.h - * @brief non-anonymous file-transfer - * @author Christian Grothoff - */ -#ifndef GNUNET_SERVICE_FS_CADET_H -#define GNUNET_SERVICE_FS_CADET_H - -/** - * Handle for a request that is going out via cadet API. - */ -struct GSF_CadetRequest; - - -/** - * Function called with a reply from the cadet. - * - * @param cls closure - * @param type type of the block, ANY on error - * @param expiration expiration time for the block - * @param data_size number of bytes in @a data, 0 on error - * @param data reply block data, NULL on error - */ -typedef void -(*GSF_CadetReplyProcessor)(void *cls, - enum GNUNET_BLOCK_Type type, - struct GNUNET_TIME_Absolute expiration, - size_t data_size, - const void *data); - - -/** - * Look for a block by directly contacting a particular peer. - * - * @param target peer that should have the block - * @param query hash to query for the block - * @param type desired type for the block - * @param proc function to call with result - * @param proc_cls closure for @a proc - * @return handle to cancel the operation - */ -struct GSF_CadetRequest * -GSF_cadet_query (const struct GNUNET_PeerIdentity *target, - const struct GNUNET_HashCode *query, - enum GNUNET_BLOCK_Type type, - GSF_CadetReplyProcessor proc, - void *proc_cls); - -/** - * Function called on each active cadets to shut them down. - * - * @param cls NULL - * @param key target peer, unused - * @param value the `struct CadetHandle` to destroy - * @return #GNUNET_YES (continue to iterate) - */ -int -GSF_cadet_release_clients (void *cls, - const struct GNUNET_PeerIdentity *key, - void *value); - - -/** - * Cancel an active request; must not be called after 'proc' - * was called. - * - * @param sr request to cancel - */ -void -GSF_cadet_query_cancel (struct GSF_CadetRequest *sr); - - -/** - * Initialize subsystem for non-anonymous file-sharing. - */ -void -GSF_cadet_start_server (void); - - -/** - * Shutdown subsystem for non-anonymous file-sharing. - */ -void -GSF_cadet_stop_server (void); - -/** - * Cadet channel for creating outbound channels. - */ -extern struct GNUNET_CADET_Handle *cadet_handle; - -/** - * Map from peer identities to 'struct CadetHandles' with cadet - * channels to those peers. - */ -extern struct GNUNET_CONTAINER_MultiPeerMap *cadet_map; - - -GNUNET_NETWORK_STRUCT_BEGIN - -/** - * Query from one peer, asking the other for CHK-data. - */ -struct CadetQueryMessage -{ - /** - * Type is GNUNET_MESSAGE_TYPE_FS_CADET_QUERY. - */ - struct GNUNET_MessageHeader header; - - /** - * Block type must be DBLOCK or IBLOCK. - */ - uint32_t type GNUNET_PACKED; - - /** - * Query hash from CHK (hash of encrypted block). - */ - struct GNUNET_HashCode query; -}; - - -/** - * Reply to a CadetQueryMessage. - */ -struct CadetReplyMessage -{ - /** - * Type is GNUNET_MESSAGE_TYPE_FS_CADET_REPLY. - */ - struct GNUNET_MessageHeader header; - - /** - * Block type must be DBLOCK or IBLOCK. - */ - uint32_t type GNUNET_PACKED; - - /** - * Expiration time for the block. - */ - struct GNUNET_TIME_AbsoluteNBO expiration; - - /* followed by the encrypted block */ -}; - -GNUNET_NETWORK_STRUCT_END - - -#endif diff --git a/src/fs/gnunet-service-fs_cadet_client.c b/src/fs/gnunet-service-fs_cadet_client.c deleted file mode 100644 index 398fcd604..000000000 --- a/src/fs/gnunet-service-fs_cadet_client.c +++ /dev/null @@ -1,728 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2012, 2013 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_cadet_client.c - * @brief non-anonymous file-transfer - * @author Christian Grothoff - * - * TODO: - * - PORT is set to old application type, unsure if we should keep - * it that way (fine for now) - */ -#include "platform.h" -#include "gnunet_constants.h" -#include "gnunet_util_lib.h" -#include "gnunet_cadet_service.h" -#include "gnunet_protocols.h" -#include "gnunet_applications.h" -#include "gnunet-service-fs.h" -#include "gnunet-service-fs_indexing.h" -#include "gnunet-service-fs_cadet.h" - - -/** - * After how long do we reset connections without replies? - */ -#define CLIENT_RETRY_TIMEOUT \ - GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30) - - -/** - * Handle for a cadet to another peer. - */ -struct CadetHandle; - - -/** - * Handle for a request that is going out via cadet API. - */ -struct GSF_CadetRequest -{ - /** - * DLL. - */ - struct GSF_CadetRequest *next; - - /** - * DLL. - */ - struct GSF_CadetRequest *prev; - - /** - * Which cadet is this request associated with? - */ - struct CadetHandle *mh; - - /** - * Function to call with the result. - */ - GSF_CadetReplyProcessor proc; - - /** - * Closure for @e proc - */ - void *proc_cls; - - /** - * Query to transmit to the other peer. - */ - struct GNUNET_HashCode query; - - /** - * Desired type for the reply. - */ - enum GNUNET_BLOCK_Type type; - - /** - * Did we transmit this request already? #GNUNET_YES if we are - * in the 'waiting_map', #GNUNET_NO if we are in the 'pending' DLL. - */ - int was_transmitted; -}; - - -/** - * Handle for a cadet to another peer. - */ -struct CadetHandle -{ - /** - * Head of DLL of pending requests on this cadet. - */ - struct GSF_CadetRequest *pending_head; - - /** - * Tail of DLL of pending requests on this cadet. - */ - struct GSF_CadetRequest *pending_tail; - - /** - * Map from query to `struct GSF_CadetRequest`s waiting for - * a reply. - */ - struct GNUNET_CONTAINER_MultiHashMap *waiting_map; - - /** - * Channel to the other peer. - */ - struct GNUNET_CADET_Channel *channel; - - /** - * Which peer does this cadet go to? - */ - struct GNUNET_PeerIdentity target; - - /** - * Task to kill inactive cadets (we keep them around for - * a few seconds to give the application a chance to give - * us another query). - */ - struct GNUNET_SCHEDULER_Task *timeout_task; - - /** - * Task to reset cadets that had errors (asynchronously, - * as we may not be able to do it immediately during a - * callback from the cadet API). - */ - struct GNUNET_SCHEDULER_Task *reset_task; -}; - - -/** - * Cadet channel for creating outbound channels. - */ -struct GNUNET_CADET_Handle *cadet_handle; - -/** - * Map from peer identities to 'struct CadetHandles' with cadet - * channels to those peers. - */ -struct GNUNET_CONTAINER_MultiPeerMap *cadet_map; - - -/* ********************* client-side code ************************* */ - - -/** - * Transmit pending requests via the cadet. - * - * @param cls `struct CadetHandle` to process - */ -static void -transmit_pending (void *cls); - - -/** - * Iterator called on each entry in a waiting map to - * move it back to the pending list. - * - * @param cls the `struct CadetHandle` - * @param key the key of the entry in the map (the query) - * @param value the `struct GSF_CadetRequest` to move to pending - * @return #GNUNET_YES (continue to iterate) - */ -static int -move_to_pending (void *cls, const struct GNUNET_HashCode *key, void *value) -{ - struct CadetHandle *mh = cls; - struct GSF_CadetRequest *sr = value; - - GNUNET_assert ( - GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (mh->waiting_map, key, value)); - GNUNET_CONTAINER_DLL_insert (mh->pending_head, mh->pending_tail, sr); - sr->was_transmitted = GNUNET_NO; - return GNUNET_YES; -} - - -/** - * Functions with this signature are called whenever a complete reply - * is received. - * - * @param cls closure with the `struct CadetHandle` - * @param srm the actual message - * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing - */ -static int -check_reply (void *cls, const struct CadetReplyMessage *srm) -{ - /* We check later... */ - return GNUNET_OK; -} - - -/** - * Task called when it is time to reset an cadet. - * - * @param cls the `struct CadetHandle` to tear down - */ -static void -reset_cadet_task (void *cls); - - -/** - * We had a serious error, tear down and re-create cadet from scratch, - * but do so asynchronously. - * - * @param mh cadet to reset - */ -static void -reset_cadet_async (struct CadetHandle *mh) -{ - if (NULL != mh->reset_task) - GNUNET_SCHEDULER_cancel (mh->reset_task); - mh->reset_task = GNUNET_SCHEDULER_add_now (&reset_cadet_task, mh); -} - - -/** - * Closure for handle_reply(). - */ -struct HandleReplyClosure -{ - /** - * Reply payload. - */ - const void *data; - - /** - * Expiration time for the block. - */ - struct GNUNET_TIME_Absolute expiration; - - /** - * Number of bytes in @e data. - */ - size_t data_size; - - /** - * Type of the block. - */ - enum GNUNET_BLOCK_Type type; - - /** - * Did we have a matching query? - */ - int found; -}; - - -/** - * Iterator called on each entry in a waiting map to - * process a result. - * - * @param cls the `struct HandleReplyClosure` - * @param key the key of the entry in the map (the query) - * @param value the `struct GSF_CadetRequest` to handle result for - * @return #GNUNET_YES (continue to iterate) - */ -static int -process_reply (void *cls, const struct GNUNET_HashCode *key, void *value) -{ - struct HandleReplyClosure *hrc = cls; - struct GSF_CadetRequest *sr = value; - - sr->proc (sr->proc_cls, - hrc->type, - hrc->expiration, - hrc->data_size, - hrc->data); - sr->proc = NULL; - GSF_cadet_query_cancel (sr); - hrc->found = GNUNET_YES; - return GNUNET_YES; -} - - -/** - * Iterator called on each entry in a waiting map to - * call the 'proc' continuation and release associated - * resources. - * - * @param cls the `struct CadetHandle` - * @param key the key of the entry in the map (the query) - * @param value the `struct GSF_CadetRequest` to clean up - * @return #GNUNET_YES (continue to iterate) - */ -static int -free_waiting_entry (void *cls, const struct GNUNET_HashCode *key, void *value) -{ - struct GSF_CadetRequest *sr = value; - - GSF_cadet_query_cancel (sr); - return GNUNET_YES; -} - - -/** - * Functions with this signature are called whenever a complete reply - * is received. - * - * @param cls closure with the `struct CadetHandle` - * @param srm the actual message - */ -static void -handle_reply (void *cls, const struct CadetReplyMessage *srm) -{ - struct CadetHandle *mh = cls; - struct HandleReplyClosure hrc; - uint16_t msize; - enum GNUNET_BLOCK_Type type; - struct GNUNET_HashCode query; - - msize = ntohs (srm->header.size) - sizeof(struct CadetReplyMessage); - type = (enum GNUNET_BLOCK_Type) ntohl (srm->type); - if (GNUNET_YES != - GNUNET_BLOCK_get_key (GSF_block_ctx, type, &srm[1], msize, &query)) - { - GNUNET_break_op (0); - GNUNET_log ( - GNUNET_ERROR_TYPE_WARNING, - "Received bogus reply of type %u with %u bytes via cadet from peer %s\n", - type, - msize, - GNUNET_i2s (&mh->target)); - reset_cadet_async (mh); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Received reply `%s' via cadet from peer %s\n", - GNUNET_h2s (&query), - GNUNET_i2s (&mh->target)); - GNUNET_CADET_receive_done (mh->channel); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# replies received via cadet"), - 1, - GNUNET_NO); - hrc.data = &srm[1]; - hrc.data_size = msize; - hrc.expiration = GNUNET_TIME_absolute_ntoh (srm->expiration); - hrc.type = type; - hrc.found = GNUNET_NO; - GNUNET_CONTAINER_multihashmap_get_multiple (mh->waiting_map, - &query, - &process_reply, - &hrc); - if (GNUNET_NO == hrc.found) - { - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# replies received via cadet dropped"), - 1, - GNUNET_NO); - } -} - - -/** - * Function called by cadet when a client disconnects. - * Cleans up our `struct CadetClient` of that channel. - * - * @param cls our `struct CadetClient` - * @param channel channel of the disconnecting client - */ -static void -disconnect_cb (void *cls, const struct GNUNET_CADET_Channel *channel) -{ - struct CadetHandle *mh = cls; - struct GSF_CadetRequest *sr; - - if (NULL == mh->channel) - return; /* being destroyed elsewhere */ - GNUNET_assert (channel == mh->channel); - mh->channel = NULL; - while (NULL != (sr = mh->pending_head)) - GSF_cadet_query_cancel (sr); - /* first remove `mh` from the `cadet_map`, so that if the - callback from `free_waiting_entry()` happens to re-issue - the request, we don't immediately have it back in the - `waiting_map`. */ - GNUNET_assert (GNUNET_OK == GNUNET_CONTAINER_multipeermap_remove (cadet_map, - &mh->target, - mh)); - GNUNET_CONTAINER_multihashmap_iterate (mh->waiting_map, - &free_waiting_entry, - mh); - if (NULL != mh->timeout_task) - GNUNET_SCHEDULER_cancel (mh->timeout_task); - if (NULL != mh->reset_task) - GNUNET_SCHEDULER_cancel (mh->reset_task); - GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap_size (mh->waiting_map)); - GNUNET_CONTAINER_multihashmap_destroy (mh->waiting_map); - GNUNET_free (mh); -} - - -/** - * Function called whenever an MQ-channel's transmission window size changes. - * - * The first callback in an outgoing channel will be with a non-zero value - * and will mean the channel is connected to the destination. - * - * For an incoming channel it will be called immediately after the - * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value. - * - * @param cls Channel closure. - * @param channel Connection to the other end (henceforth invalid). - * @param window_size New window size. If the is more messages than buffer size - * this value will be negative.. - */ -static void -window_change_cb (void *cls, - const struct GNUNET_CADET_Channel *channel, - int window_size) -{ - /* FIXME: for flow control, implement? */ -#if 0 - /* Something like this instead of the GNUNET_MQ_notify_sent() in - transmit_pending() might be good (once the window change CB works...) */ - if (0 < window_size) /* test needed? */ - transmit_pending (mh); -#endif -} - - -/** - * We had a serious error, tear down and re-create cadet from scratch. - * - * @param mh cadet to reset - */ -static void -reset_cadet (struct CadetHandle *mh) -{ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Resetting cadet channel to %s\n", - GNUNET_i2s (&mh->target)); - if (NULL != mh->channel) - { - GNUNET_CADET_channel_destroy (mh->channel); - mh->channel = NULL; - } - GNUNET_CONTAINER_multihashmap_iterate (mh->waiting_map, &move_to_pending, mh); - { - struct GNUNET_MQ_MessageHandler handlers[] = - { GNUNET_MQ_hd_var_size (reply, - GNUNET_MESSAGE_TYPE_FS_CADET_REPLY, - struct CadetReplyMessage, - mh), - GNUNET_MQ_handler_end () }; - struct GNUNET_HashCode port; - - GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER, - strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER), - &port); - mh->channel = GNUNET_CADET_channel_create (cadet_handle, - mh, - &mh->target, - &port, - &window_change_cb, - &disconnect_cb, - handlers); - } - transmit_pending (mh); -} - - -/** - * Task called when it is time to destroy an inactive cadet channel. - * - * @param cls the `struct CadetHandle` to tear down - */ -static void -cadet_timeout (void *cls) -{ - struct CadetHandle *mh = cls; - struct GNUNET_CADET_Channel *tun; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Timeout on cadet channel to %s\n", - GNUNET_i2s (&mh->target)); - mh->timeout_task = NULL; - tun = mh->channel; - mh->channel = NULL; - if (NULL != tun) - GNUNET_CADET_channel_destroy (tun); -} - - -/** - * Task called when it is time to reset an cadet. - * - * @param cls the `struct CadetHandle` to tear down - */ -static void -reset_cadet_task (void *cls) -{ - struct CadetHandle *mh = cls; - - mh->reset_task = NULL; - reset_cadet (mh); -} - - -/** - * Transmit pending requests via the cadet. - * - * @param cls `struct CadetHandle` to process - */ -static void -transmit_pending (void *cls) -{ - struct CadetHandle *mh = cls; - struct GNUNET_MQ_Handle *mq = GNUNET_CADET_get_mq (mh->channel); - struct GSF_CadetRequest *sr; - struct GNUNET_MQ_Envelope *env; - struct CadetQueryMessage *sqm; - - if ((0 != GNUNET_MQ_get_length (mq)) || (NULL == (sr = mh->pending_head))) - return; - GNUNET_CONTAINER_DLL_remove (mh->pending_head, mh->pending_tail, sr); - GNUNET_assert (GNUNET_OK == GNUNET_CONTAINER_multihashmap_put ( - mh->waiting_map, - &sr->query, - sr, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE)); - sr->was_transmitted = GNUNET_YES; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Sending query for %s via cadet to %s\n", - GNUNET_h2s (&sr->query), - GNUNET_i2s (&mh->target)); - env = GNUNET_MQ_msg (sqm, GNUNET_MESSAGE_TYPE_FS_CADET_QUERY); - GNUNET_MQ_env_set_options (env, - GNUNET_MQ_PREF_GOODPUT - | GNUNET_MQ_PREF_CORK_ALLOWED - | GNUNET_MQ_PREF_OUT_OF_ORDER); - sqm->type = htonl (sr->type); - sqm->query = sr->query; - GNUNET_MQ_notify_sent (env, &transmit_pending, mh); - GNUNET_MQ_send (mq, env); -} - - -/** - * Get (or create) a cadet to talk to the given peer. - * - * @param target peer we want to communicate with - */ -static struct CadetHandle * -get_cadet (const struct GNUNET_PeerIdentity *target) -{ - struct CadetHandle *mh; - - mh = GNUNET_CONTAINER_multipeermap_get (cadet_map, target); - if (NULL != mh) - { - if (NULL != mh->timeout_task) - { - GNUNET_SCHEDULER_cancel (mh->timeout_task); - mh->timeout_task = NULL; - } - return mh; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Creating cadet channel to %s\n", - GNUNET_i2s (target)); - mh = GNUNET_new (struct CadetHandle); - mh->reset_task = - GNUNET_SCHEDULER_add_delayed (CLIENT_RETRY_TIMEOUT, &reset_cadet_task, mh); - mh->waiting_map = GNUNET_CONTAINER_multihashmap_create (16, GNUNET_YES); - mh->target = *target; - GNUNET_assert (GNUNET_OK == - GNUNET_CONTAINER_multipeermap_put ( - cadet_map, - &mh->target, - mh, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - { - struct GNUNET_MQ_MessageHandler handlers[] = - { GNUNET_MQ_hd_var_size (reply, - GNUNET_MESSAGE_TYPE_FS_CADET_REPLY, - struct CadetReplyMessage, - mh), - GNUNET_MQ_handler_end () }; - struct GNUNET_HashCode port; - - GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER, - strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER), - &port); - mh->channel = GNUNET_CADET_channel_create (cadet_handle, - mh, - &mh->target, - &port, - &window_change_cb, - &disconnect_cb, - handlers); - } - return mh; -} - - -/** - * Look for a block by directly contacting a particular peer. - * - * @param target peer that should have the block - * @param query hash to query for the block - * @param type desired type for the block - * @param proc function to call with result - * @param proc_cls closure for @a proc - * @return handle to cancel the operation - */ -struct GSF_CadetRequest * -GSF_cadet_query (const struct GNUNET_PeerIdentity *target, - const struct GNUNET_HashCode *query, - enum GNUNET_BLOCK_Type type, - GSF_CadetReplyProcessor proc, - void *proc_cls) -{ - struct CadetHandle *mh; - struct GSF_CadetRequest *sr; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Preparing to send query for %s via cadet to %s\n", - GNUNET_h2s (query), - GNUNET_i2s (target)); - mh = get_cadet (target); - sr = GNUNET_new (struct GSF_CadetRequest); - sr->mh = mh; - sr->proc = proc; - sr->proc_cls = proc_cls; - sr->type = type; - sr->query = *query; - GNUNET_CONTAINER_DLL_insert (mh->pending_head, mh->pending_tail, sr); - transmit_pending (mh); - return sr; -} - - -/** - * Cancel an active request; must not be called after 'proc' - * was called. - * - * @param sr request to cancel - */ -void -GSF_cadet_query_cancel (struct GSF_CadetRequest *sr) -{ - struct CadetHandle *mh = sr->mh; - GSF_CadetReplyProcessor p; - - p = sr->proc; - sr->proc = NULL; - if (NULL != p) - { - /* signal failure / cancellation to callback */ - p (sr->proc_cls, GNUNET_BLOCK_TYPE_ANY, GNUNET_TIME_UNIT_ZERO_ABS, 0, NULL); - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Cancelled query for %s via cadet to %s\n", - GNUNET_h2s (&sr->query), - GNUNET_i2s (&sr->mh->target)); - if (GNUNET_YES == sr->was_transmitted) - GNUNET_assert ( - GNUNET_OK == - GNUNET_CONTAINER_multihashmap_remove (mh->waiting_map, &sr->query, sr)); - else - GNUNET_CONTAINER_DLL_remove (mh->pending_head, mh->pending_tail, sr); - GNUNET_free (sr); - if ((0 == GNUNET_CONTAINER_multihashmap_size (mh->waiting_map)) && - (NULL == mh->pending_head)) - mh->timeout_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS, - &cadet_timeout, - mh); -} - - -/** - * Function called on each active cadets to shut them down. - * - * @param cls NULL - * @param key target peer, unused - * @param value the `struct CadetHandle` to destroy - * @return #GNUNET_YES (continue to iterate) - */ -int -GSF_cadet_release_clients (void *cls, - const struct GNUNET_PeerIdentity *key, - void *value) -{ - struct CadetHandle *mh = value; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Timeout on cadet channel to %s\n", - GNUNET_i2s (&mh->target)); - if (NULL != mh->channel) - { - struct GNUNET_CADET_Channel *channel = mh->channel; - - mh->channel = NULL; - GNUNET_CADET_channel_destroy (channel); - } - if (NULL != mh->reset_task) - { - GNUNET_SCHEDULER_cancel (mh->reset_task); - mh->reset_task = NULL; - } - return GNUNET_YES; -} - - -/* end of gnunet-service-fs_cadet_client.c */ diff --git a/src/fs/gnunet-service-fs_cadet_server.c b/src/fs/gnunet-service-fs_cadet_server.c deleted file mode 100644 index 8bfe91cf0..000000000 --- a/src/fs/gnunet-service-fs_cadet_server.c +++ /dev/null @@ -1,545 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2012, 2013, 2017 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_cadet_server.c - * @brief non-anonymous file-transfer - * @author Christian Grothoff - * - * TODO: - * - PORT is set to old application type, unsure if we should keep - * it that way (fine for now) - */ -#include "platform.h" -#include "gnunet_constants.h" -#include "gnunet_util_lib.h" -#include "gnunet_cadet_service.h" -#include "gnunet_protocols.h" -#include "gnunet_applications.h" -#include "gnunet-service-fs.h" -#include "gnunet-service-fs_indexing.h" -#include "gnunet-service-fs_cadet.h" - -/** - * After how long do we termiante idle connections? - */ -#define IDLE_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 2) - - -/** - * A message in the queue to be written to the cadet. - */ -struct WriteQueueItem -{ - /** - * Kept in a DLL. - */ - struct WriteQueueItem *next; - - /** - * Kept in a DLL. - */ - struct WriteQueueItem *prev; - - /** - * Number of bytes of payload, allocated at the end of this struct. - */ - size_t msize; -}; - - -/** - * Information we keep around for each active cadeting client. - */ -struct CadetClient -{ - /** - * DLL - */ - struct CadetClient *next; - - /** - * DLL - */ - struct CadetClient *prev; - - /** - * Channel for communication. - */ - struct GNUNET_CADET_Channel *channel; - - /** - * Head of write queue. - */ - struct WriteQueueItem *wqi_head; - - /** - * Tail of write queue. - */ - struct WriteQueueItem *wqi_tail; - - /** - * Current active request to the datastore, if we have one pending. - */ - struct GNUNET_DATASTORE_QueueEntry *qe; - - /** - * Task that is scheduled to asynchronously terminate the connection. - */ - struct GNUNET_SCHEDULER_Task *terminate_task; - - /** - * Task that is scheduled to terminate idle connections. - */ - struct GNUNET_SCHEDULER_Task *timeout_task; - - /** - * Size of the last write that was initiated. - */ - size_t reply_size; -}; - - -/** - * Listen port for incoming requests. - */ -static struct GNUNET_CADET_Port *cadet_port; - -/** - * Head of DLL of cadet clients. - */ -static struct CadetClient *sc_head; - -/** - * Tail of DLL of cadet clients. - */ -static struct CadetClient *sc_tail; - -/** - * Number of active cadet clients in the 'sc_*'-DLL. - */ -static unsigned int sc_count; - -/** - * Maximum allowed number of cadet clients. - */ -static unsigned long long sc_count_max; - - -/** - * Task run to asynchronously terminate the cadet due to timeout. - * - * @param cls the 'struct CadetClient' - */ -static void -timeout_cadet_task (void *cls) -{ - struct CadetClient *sc = cls; - struct GNUNET_CADET_Channel *tun; - - sc->timeout_task = NULL; - tun = sc->channel; - sc->channel = NULL; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Timeout for inactive cadet client %p\n", - sc); - GNUNET_CADET_channel_destroy (tun); -} - - -/** - * Reset the timeout for the cadet client (due to activity). - * - * @param sc client handle to reset timeout for - */ -static void -refresh_timeout_task (struct CadetClient *sc) -{ - if (NULL != sc->timeout_task) - GNUNET_SCHEDULER_cancel (sc->timeout_task); - sc->timeout_task = GNUNET_SCHEDULER_add_delayed (IDLE_TIMEOUT, - &timeout_cadet_task, - sc); -} - - -/** - * Check if we are done with the write queue, and if so tell CADET - * that we are ready to read more. - * - * @param cls where to process the write queue - */ -static void -continue_writing (void *cls) -{ - struct CadetClient *sc = cls; - struct GNUNET_MQ_Handle *mq; - - mq = GNUNET_CADET_get_mq (sc->channel); - if (0 != GNUNET_MQ_get_length (mq)) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Write pending, waiting for it to complete\n"); - return; - } - refresh_timeout_task (sc); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Finished processing cadet request from client %p, ready to receive the next one\n", - sc); - GNUNET_CADET_receive_done (sc->channel); -} - - -/** - * Process a datum that was stored in the datastore. - * - * @param cls closure with the `struct CadetClient` which sent the query - * @param key key for the content - * @param size number of bytes in @a data - * @param data content stored - * @param type type of the content - * @param priority priority of the content - * @param anonymity anonymity-level for the content - * @param replication replication-level for the content - * @param expiration expiration time for the content - * @param uid unique identifier for the datum; - * maybe 0 if no unique identifier is available - */ -static void -handle_datastore_reply (void *cls, - const struct GNUNET_HashCode *key, - size_t size, - const void *data, - enum GNUNET_BLOCK_Type type, - uint32_t priority, - uint32_t anonymity, - uint32_t replication, - struct GNUNET_TIME_Absolute expiration, - uint64_t uid) -{ - struct CadetClient *sc = cls; - size_t msize = size + sizeof(struct CadetReplyMessage); - struct GNUNET_MQ_Envelope *env; - struct CadetReplyMessage *srm; - - sc->qe = NULL; - if (NULL == data) - { - /* no result, this should not really happen, as for - non-anonymous routing only peers that HAVE the - answers should be queried; OTOH, this is not a - hard error as we might have had the answer in the - past and the user might have unindexed it. Hence - we log at level "INFO" for now. */if (NULL == key) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Have no answer and the query was NULL\n"); - } - else - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Have no answer for query `%s'\n", - GNUNET_h2s (key)); - } - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# queries received via CADET not answered"), - 1, - GNUNET_NO); - continue_writing (sc); - return; - } - if (GNUNET_BLOCK_TYPE_FS_ONDEMAND == type) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Performing on-demand encoding for query %s\n", - GNUNET_h2s (key)); - if (GNUNET_OK != - GNUNET_FS_handle_on_demand_block (key, - size, - data, - type, - priority, - anonymity, - replication, - expiration, - uid, - &handle_datastore_reply, - sc)) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "On-demand encoding request failed\n"); - continue_writing (sc); - } - return; - } - if (msize > GNUNET_MAX_MESSAGE_SIZE) - { - GNUNET_break (0); - continue_writing (sc); - return; - } - GNUNET_break (GNUNET_BLOCK_TYPE_ANY != type); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting transmission of %u byte reply of type %d for query `%s' via cadet to %p\n", - (unsigned int) size, - (unsigned int) type, - GNUNET_h2s (key), - sc); - env = GNUNET_MQ_msg_extra (srm, - size, - GNUNET_MESSAGE_TYPE_FS_CADET_REPLY); - srm->type = htonl (type); - srm->expiration = GNUNET_TIME_absolute_hton (expiration); - GNUNET_memcpy (&srm[1], - data, - size); - GNUNET_MQ_notify_sent (env, - &continue_writing, - sc); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# Blocks transferred via cadet"), - 1, - GNUNET_NO); - GNUNET_MQ_send (GNUNET_CADET_get_mq (sc->channel), - env); -} - - -/** - * Functions with this signature are called whenever a - * complete query message is received. - * - * @param cls closure with the `struct CadetClient` - * @param sqm the actual message - */ -static void -handle_request (void *cls, - const struct CadetQueryMessage *sqm) -{ - struct CadetClient *sc = cls; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Received query for `%s' via cadet from client %p\n", - GNUNET_h2s (&sqm->query), - sc); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# queries received via cadet"), - 1, - GNUNET_NO); - refresh_timeout_task (sc); - sc->qe = GNUNET_DATASTORE_get_key (GSF_dsh, - 0 /* next_uid */, - false /* random */, - &sqm->query, - ntohl (sqm->type), - 0 /* priority */, - GSF_datastore_queue_size, - &handle_datastore_reply, - sc); - if (NULL == sc->qe) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Queueing request with datastore failed (queue full?)\n"); - continue_writing (sc); - } -} - - -/** - * Functions of this type are called upon new cadet connection from other peers. - * - * @param cls the closure from GNUNET_CADET_connect - * @param channel the channel representing the cadet - * @param initiator the identity of the peer who wants to establish a cadet - * with us; NULL on binding error - * @return initial channel context (our `struct CadetClient`) - */ -static void * -connect_cb (void *cls, - struct GNUNET_CADET_Channel *channel, - const struct GNUNET_PeerIdentity *initiator) -{ - struct CadetClient *sc; - - GNUNET_assert (NULL != channel); - if (sc_count >= sc_count_max) - { - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# cadet client connections rejected"), - 1, - GNUNET_NO); - GNUNET_CADET_channel_destroy (channel); - return NULL; - } - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# cadet connections active"), - 1, - GNUNET_NO); - sc = GNUNET_new (struct CadetClient); - sc->channel = channel; - GNUNET_CONTAINER_DLL_insert (sc_head, - sc_tail, - sc); - sc_count++; - refresh_timeout_task (sc); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Accepting inbound cadet connection from `%s' as client %p\n", - GNUNET_i2s (initiator), - sc); - return sc; -} - - -/** - * Function called by cadet when a client disconnects. - * Cleans up our `struct CadetClient` of that channel. - * - * @param cls our `struct CadetClient` - * @param channel channel of the disconnecting client - */ -static void -disconnect_cb (void *cls, - const struct GNUNET_CADET_Channel *channel) -{ - struct CadetClient *sc = cls; - struct WriteQueueItem *wqi; - - if (NULL == sc) - return; - sc->channel = NULL; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Terminating cadet connection with client %p\n", - sc); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# cadet connections active"), -1, - GNUNET_NO); - if (NULL != sc->terminate_task) - GNUNET_SCHEDULER_cancel (sc->terminate_task); - if (NULL != sc->timeout_task) - GNUNET_SCHEDULER_cancel (sc->timeout_task); - if (NULL != sc->qe) - GNUNET_DATASTORE_cancel (sc->qe); - while (NULL != (wqi = sc->wqi_head)) - { - GNUNET_CONTAINER_DLL_remove (sc->wqi_head, - sc->wqi_tail, - wqi); - GNUNET_free (wqi); - } - GNUNET_CONTAINER_DLL_remove (sc_head, - sc_tail, - sc); - sc_count--; - GNUNET_free (sc); -} - - -/** - * Function called whenever an MQ-channel's transmission window size changes. - * - * The first callback in an outgoing channel will be with a non-zero value - * and will mean the channel is connected to the destination. - * - * For an incoming channel it will be called immediately after the - * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value. - * - * @param cls Channel closure. - * @param channel Connection to the other end (henceforth invalid). - * @param window_size New window size. If the is more messages than buffer size - * this value will be negative.. - */ -static void -window_change_cb (void *cls, - const struct GNUNET_CADET_Channel *channel, - int window_size) -{ - /* FIXME: could do flow control here... */ -} - - -/** - * Initialize subsystem for non-anonymous file-sharing. - */ -void -GSF_cadet_start_server () -{ - struct GNUNET_MQ_MessageHandler handlers[] = { - GNUNET_MQ_hd_fixed_size (request, - GNUNET_MESSAGE_TYPE_FS_CADET_QUERY, - struct CadetQueryMessage, - NULL), - GNUNET_MQ_handler_end () - }; - struct GNUNET_HashCode port; - - if (GNUNET_YES != - GNUNET_CONFIGURATION_get_value_number (GSF_cfg, - "fs", - "MAX_CADET_CLIENTS", - &sc_count_max)) - return; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Initializing cadet FS server with a limit of %llu connections\n", - sc_count_max); - cadet_map = GNUNET_CONTAINER_multipeermap_create (16, GNUNET_YES); - cadet_handle = GNUNET_CADET_connect (GSF_cfg); - GNUNET_assert (NULL != cadet_handle); - GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER, - strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER), - &port); - cadet_port = GNUNET_CADET_open_port (cadet_handle, - &port, - &connect_cb, - NULL, - &window_change_cb, - &disconnect_cb, - handlers); -} - - -/** - * Shutdown subsystem for non-anonymous file-sharing. - */ -void -GSF_cadet_stop_server () -{ - GNUNET_CONTAINER_multipeermap_iterate (cadet_map, - &GSF_cadet_release_clients, - NULL); - GNUNET_CONTAINER_multipeermap_destroy (cadet_map); - cadet_map = NULL; - if (NULL != cadet_port) - { - GNUNET_CADET_close_port (cadet_port); - cadet_port = NULL; - } - if (NULL != cadet_handle) - { - GNUNET_CADET_disconnect (cadet_handle); - cadet_handle = NULL; - } - GNUNET_assert (NULL == sc_head); - GNUNET_assert (0 == sc_count); -} - - -/* end of gnunet-service-fs_cadet.c */ diff --git a/src/fs/gnunet-service-fs_cp.c b/src/fs/gnunet-service-fs_cp.c deleted file mode 100644 index 74dd42daf..000000000 --- a/src/fs/gnunet-service-fs_cp.c +++ /dev/null @@ -1,1659 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2011, 2016 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/gnunet-service-fs_cp.c - * @brief API to handle 'connected peers' - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_load_lib.h" -#include "gnunet-service-fs.h" -#include "gnunet-service-fs_cp.h" -#include "gnunet-service-fs_pe.h" -#include "gnunet-service-fs_pr.h" -#include "gnunet-service-fs_push.h" -#include "gnunet_peerstore_service.h" - - -/** - * Ratio for moving average delay calculation. The previous - * average goes in with a factor of (n-1) into the calculation. - * Must be > 0. - */ -#define RUNAVG_DELAY_N 16 - -/** - * How often do we flush respect values to disk? - */ -#define RESPECT_FLUSH_FREQ GNUNET_TIME_relative_multiply ( \ - GNUNET_TIME_UNIT_MINUTES, 5) - -/** - * After how long do we discard a reply? - */ -#define REPLY_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, \ - 2) - -/** - * Collect an instance number of statistics? May cause excessive IPC. - */ -#define INSANE_STATISTICS GNUNET_NO - - -/** - * Handle to cancel a transmission request. - */ -struct GSF_PeerTransmitHandle -{ - /** - * Kept in a doubly-linked list. - */ - struct GSF_PeerTransmitHandle *next; - - /** - * Kept in a doubly-linked list. - */ - struct GSF_PeerTransmitHandle *prev; - - /** - * Time when this transmission request was issued. - */ - struct GNUNET_TIME_Absolute transmission_request_start_time; - - /** - * Envelope with the actual message. - */ - struct GNUNET_MQ_Envelope *env; - - /** - * Peer this request targets. - */ - struct GSF_ConnectedPeer *cp; - - /** - * #GNUNET_YES if this is a query, #GNUNET_NO for content. - */ - int is_query; - - /** - * Priority of this request. - */ - uint32_t priority; -}; - - -/** - * Handle for an entry in our delay list. - */ -struct GSF_DelayedHandle -{ - /** - * Kept in a doubly-linked list. - */ - struct GSF_DelayedHandle *next; - - /** - * Kept in a doubly-linked list. - */ - struct GSF_DelayedHandle *prev; - - /** - * Peer this transmission belongs to. - */ - struct GSF_ConnectedPeer *cp; - - /** - * Envelope of the message that was delayed. - */ - struct GNUNET_MQ_Envelope *env; - - /** - * Task for the delay. - */ - struct GNUNET_SCHEDULER_Task *delay_task; - - /** - * Size of the message. - */ - size_t msize; -}; - - -/** - * Information per peer and request. - */ -struct PeerRequest -{ - /** - * Handle to generic request (generic: from peer or local client). - */ - struct GSF_PendingRequest *pr; - - /** - * Which specific peer issued this request? - */ - struct GSF_ConnectedPeer *cp; - - /** - * Task for asynchronous stopping of this request. - */ - struct GNUNET_SCHEDULER_Task *kill_task; -}; - - -/** - * A connected peer. - */ -struct GSF_ConnectedPeer -{ - /** - * Performance data for this peer. - */ - struct GSF_PeerPerformanceData ppd; - - /** - * Time until when we blocked this peer from migrating - * data to us. - */ - struct GNUNET_TIME_Absolute last_migration_block; - - /** - * Task scheduled to revive migration to this peer. - */ - struct GNUNET_SCHEDULER_Task *mig_revive_task; - - /** - * Messages (replies, queries, content migration) we would like to - * send to this peer in the near future. Sorted by priority, head. - */ - struct GSF_PeerTransmitHandle *pth_head; - - /** - * Messages (replies, queries, content migration) we would like to - * send to this peer in the near future. Sorted by priority, tail. - */ - struct GSF_PeerTransmitHandle *pth_tail; - - /** - * Messages (replies, queries, content migration) we would like to - * send to this peer in the near future. Sorted by priority, head. - */ - struct GSF_DelayedHandle *delayed_head; - - /** - * Messages (replies, queries, content migration) we would like to - * send to this peer in the near future. Sorted by priority, tail. - */ - struct GSF_DelayedHandle *delayed_tail; - - /** - * Task scheduled if we need to retry bandwidth reservation later. - */ - struct GNUNET_SCHEDULER_Task *rc_delay_task; - - /** - * Active requests from this neighbour, map of query to `struct PeerRequest`. - */ - struct GNUNET_CONTAINER_MultiHashMap *request_map; - - /** - * Handle for an active request for transmission to this - * peer. - */ - struct GNUNET_MQ_Handle *mq; - - /** - * Increase in traffic preference still to be submitted - * to the core service for this peer. - */ - uint64_t inc_preference; - - /** - * Number of entries in @e delayed_head DLL. - */ - unsigned int delay_queue_size; - - /** - * Respect rating for this peer on disk. - */ - uint32_t disk_respect; - - /** - * Which offset in @e last_p2p_replies will be updated next? - * (we go round-robin). - */ - unsigned int last_p2p_replies_woff; - - /** - * Which offset in @e last_client_replies will be updated next? - * (we go round-robin). - */ - unsigned int last_client_replies_woff; - - /** - * Current offset into @e last_request_times ring buffer. - */ - unsigned int last_request_times_off; - - /** - * Handle to the PEERSTORE iterate request for peer respect value - */ - struct GNUNET_PEERSTORE_IterateContext *respect_iterate_req; -}; - - -/** - * Map from peer identities to `struct GSF_ConnectPeer` entries. - */ -static struct GNUNET_CONTAINER_MultiPeerMap *cp_map; - -/** - * Handle to peerstore service. - */ -static struct GNUNET_PEERSTORE_Handle *peerstore; - -/** - * Task used to flush respect values to disk. - */ -static struct GNUNET_SCHEDULER_Task *fr_task; - - -/** - * Update the latency information kept for the given peer. - * - * @param id peer record to update - * @param latency current latency value - */ -void -GSF_update_peer_latency_ (const struct GNUNET_PeerIdentity *id, - struct GNUNET_TIME_Relative latency) -{ - struct GSF_ConnectedPeer *cp; - - cp = GSF_peer_get_ (id); - if (NULL == cp) - return; /* we're not yet connected at the core level, ignore */ - GNUNET_LOAD_value_set_decline (cp->ppd.transmission_delay, - latency); -} - - -/** - * Return the performance data record for the given peer - * - * @param cp peer to query - * @return performance data record for the peer - */ -struct GSF_PeerPerformanceData * -GSF_get_peer_performance_data_ (struct GSF_ConnectedPeer *cp) -{ - return &cp->ppd; -} - - -/** - * Core is ready to transmit to a peer, get the message. - * - * @param cp which peer to send a message to - */ -static void -peer_transmit (struct GSF_ConnectedPeer *cp); - - -/** - * If ready (bandwidth reserved), try to schedule transmission via - * core for the given handle. - * - * @param pth transmission handle to schedule - */ -static void -schedule_transmission (struct GSF_PeerTransmitHandle *pth) -{ - struct GSF_ConnectedPeer *cp; - struct GNUNET_PeerIdentity target; - - cp = pth->cp; - GNUNET_assert (0 != cp->ppd.pid); - GNUNET_PEER_resolve (cp->ppd.pid, &target); - - peer_transmit (cp); -} - - -/** - * Core is ready to transmit to a peer, get the message. - * - * @param cp which peer to send a message to - */ -static void -peer_transmit (struct GSF_ConnectedPeer *cp) -{ - struct GSF_PeerTransmitHandle *pth = cp->pth_head; - struct GSF_PeerTransmitHandle *pos; - - if (NULL == pth) - return; - GNUNET_CONTAINER_DLL_remove (cp->pth_head, - cp->pth_tail, - pth); - if (GNUNET_YES == pth->is_query) - { - cp->ppd.last_request_times[(cp->last_request_times_off++) - % MAX_QUEUE_PER_PEER] = - GNUNET_TIME_absolute_get (); - GNUNET_assert (0 < cp->ppd.pending_queries--); - } - else if (GNUNET_NO == pth->is_query) - { - GNUNET_assert (0 < cp->ppd.pending_replies--); - } - GNUNET_LOAD_update (cp->ppd.transmission_delay, - GNUNET_TIME_absolute_get_duration - (pth->transmission_request_start_time).rel_value_us); - GNUNET_MQ_send (cp->mq, - pth->env); - GNUNET_free (pth); - if (NULL != (pos = cp->pth_head)) - { - GNUNET_assert (pos != pth); - schedule_transmission (pos); - } -} - - -/** - * Function called by PEERSTORE with peer respect record - * - * @param cls handle to connected peer entry - * @param record peerstore record information - * @param emsg error message, or NULL if no errors - */ -static void -peer_respect_cb (void *cls, - const struct GNUNET_PEERSTORE_Record *record, - const char *emsg) -{ - struct GSF_ConnectedPeer *cp = cls; - - GNUNET_assert (NULL != cp->respect_iterate_req); - if ((NULL != record) && - (sizeof(cp->disk_respect) == record->value_size)) - { - cp->disk_respect = *((uint32_t *) record->value); - cp->ppd.respect += *((uint32_t *) record->value); - } - GSF_push_start_ (cp); - if (NULL != record) - GNUNET_PEERSTORE_iterate_cancel (cp->respect_iterate_req); - cp->respect_iterate_req = NULL; -} - - -/** - * Function called for each pending request whenever a new - * peer connects, giving us a chance to decide about submitting - * the existing request to the new peer. - * - * @param cls the `struct GSF_ConnectedPeer` of the new peer - * @param key query for the request - * @param pr handle to the pending request - * @return #GNUNET_YES to continue to iterate - */ -static int -consider_peer_for_forwarding (void *cls, - const struct GNUNET_HashCode *key, - struct GSF_PendingRequest *pr) -{ - struct GSF_ConnectedPeer *cp = cls; - struct GNUNET_PeerIdentity pid; - - if (GNUNET_YES != - GSF_pending_request_test_active_ (pr)) - return GNUNET_YES; /* request is not actually active, skip! */ - GSF_connected_peer_get_identity_ (cp, &pid); - if (GNUNET_YES != - GSF_pending_request_test_target_ (pr, &pid)) - { - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# Loopback routes suppressed"), - 1, - GNUNET_NO); - return GNUNET_YES; - } - GSF_plan_add_ (cp, pr); - return GNUNET_YES; -} - - -void * -GSF_peer_connect_handler (void *cls, - const struct GNUNET_PeerIdentity *peer, - struct GNUNET_MQ_Handle *mq) -{ - struct GSF_ConnectedPeer *cp; - - if (0 == - GNUNET_memcmp (&GSF_my_id, - peer)) - return NULL; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Connected to peer %s\n", - GNUNET_i2s (peer)); - cp = GNUNET_new (struct GSF_ConnectedPeer); - cp->ppd.pid = GNUNET_PEER_intern (peer); - cp->ppd.peer = peer; - cp->mq = mq; - cp->ppd.transmission_delay = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_ZERO); - - cp->request_map = GNUNET_CONTAINER_multihashmap_create (128, - GNUNET_YES); - GNUNET_break (GNUNET_OK == - GNUNET_CONTAINER_multipeermap_put (cp_map, - GSF_connected_peer_get_identity2_ ( - cp), - cp, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - GNUNET_STATISTICS_set (GSF_stats, - gettext_noop ("# peers connected"), - GNUNET_CONTAINER_multipeermap_size (cp_map), - GNUNET_NO); - cp->respect_iterate_req - = GNUNET_PEERSTORE_iterate (peerstore, - "fs", - peer, - "respect", - &peer_respect_cb, - cp); - GSF_iterate_pending_requests_ (&consider_peer_for_forwarding, - cp); - return cp; -} - - -/** - * It may be time to re-start migrating content to this - * peer. Check, and if so, restart migration. - * - * @param cls the `struct GSF_ConnectedPeer` - */ -static void -revive_migration (void *cls) -{ - struct GSF_ConnectedPeer *cp = cls; - struct GNUNET_TIME_Relative bt; - - cp->mig_revive_task = NULL; - bt = GNUNET_TIME_absolute_get_remaining (cp->ppd.migration_blocked_until); - if (0 != bt.rel_value_us) - { - /* still time left... */ - cp->mig_revive_task = - GNUNET_SCHEDULER_add_delayed (bt, &revive_migration, cp); - return; - } - GSF_push_start_ (cp); -} - - -struct GSF_ConnectedPeer * -GSF_peer_get_ (const struct GNUNET_PeerIdentity *peer) -{ - if (NULL == cp_map) - return NULL; - return GNUNET_CONTAINER_multipeermap_get (cp_map, peer); -} - - -/** - * Handle P2P #GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP message. - * - * @param cls closure, the `struct GSF_ConnectedPeer` - * @param msm the actual message - */ -void -handle_p2p_migration_stop (void *cls, - const struct MigrationStopMessage *msm) -{ - struct GSF_ConnectedPeer *cp = cls; - struct GNUNET_TIME_Relative bt; - - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# migration stop messages received"), - 1, GNUNET_NO); - bt = GNUNET_TIME_relative_ntoh (msm->duration); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - _ ("Migration of content to peer `%s' blocked for %s\n"), - GNUNET_i2s (cp->ppd.peer), - GNUNET_STRINGS_relative_time_to_string (bt, GNUNET_YES)); - cp->ppd.migration_blocked_until = GNUNET_TIME_relative_to_absolute (bt); - if ((NULL == cp->mig_revive_task) && - (NULL == cp->respect_iterate_req)) - { - GSF_push_stop_ (cp); - cp->mig_revive_task = - GNUNET_SCHEDULER_add_delayed (bt, - &revive_migration, cp); - } -} - - -/** - * Free resources associated with the given peer request. - * - * @param peerreq request to free - */ -static void -free_pending_request (struct PeerRequest *peerreq) -{ - struct GSF_ConnectedPeer *cp = peerreq->cp; - struct GSF_PendingRequestData *prd; - - prd = GSF_pending_request_get_data_ (peerreq->pr); - if (NULL != peerreq->kill_task) - { - GNUNET_SCHEDULER_cancel (peerreq->kill_task); - peerreq->kill_task = NULL; - } - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# P2P searches active"), - -1, - GNUNET_NO); - GNUNET_break (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (cp->request_map, - &prd->query, - peerreq)); - GNUNET_free (peerreq); -} - - -/** - * Cancel all requests associated with the peer. - * - * @param cls unused - * @param query hash code of the request - * @param value the `struct GSF_PendingRequest` - * @return #GNUNET_YES (continue to iterate) - */ -static int -cancel_pending_request (void *cls, - const struct GNUNET_HashCode *query, - void *value) -{ - struct PeerRequest *peerreq = value; - struct GSF_PendingRequest *pr = peerreq->pr; - - free_pending_request (peerreq); - GSF_pending_request_cancel_ (pr, - GNUNET_NO); - return GNUNET_OK; -} - - -/** - * Free the given request. - * - * @param cls the request to free - */ -static void -peer_request_destroy (void *cls) -{ - struct PeerRequest *peerreq = cls; - struct GSF_PendingRequest *pr = peerreq->pr; - struct GSF_PendingRequestData *prd; - - peerreq->kill_task = NULL; - prd = GSF_pending_request_get_data_ (pr); - cancel_pending_request (NULL, - &prd->query, - peerreq); -} - - -/** - * The artificial delay is over, transmit the message now. - * - * @param cls the `struct GSF_DelayedHandle` with the message - */ -static void -transmit_delayed_now (void *cls) -{ - struct GSF_DelayedHandle *dh = cls; - struct GSF_ConnectedPeer *cp = dh->cp; - - GNUNET_CONTAINER_DLL_remove (cp->delayed_head, - cp->delayed_tail, - dh); - cp->delay_queue_size--; - GSF_peer_transmit_ (cp, - GNUNET_NO, - UINT32_MAX, - dh->env); - GNUNET_free (dh); -} - - -/** - * Get the randomized delay a response should be subjected to. - * - * @return desired delay - */ -static struct GNUNET_TIME_Relative -get_randomized_delay () -{ - struct GNUNET_TIME_Relative ret; - - ret = - GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, - GNUNET_CRYPTO_random_u32 - (GNUNET_CRYPTO_QUALITY_WEAK, - 2 * GSF_avg_latency.rel_value_us + 1)); -#if INSANE_STATISTICS - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop - ("# artificial delays introduced (ms)"), - ret.rel_value_us / 1000LL, GNUNET_NO); -#endif - return ret; -} - - -/** - * Handle a reply to a pending request. Also called if a request - * expires (then with data == NULL). The handler may be called - * many times (depending on the request type), but will not be - * called during or after a call to GSF_pending_request_cancel - * and will also not be called anymore after a call signalling - * expiration. - * - * @param cls `struct PeerRequest` this is an answer for - * @param eval evaluation of the result - * @param pr handle to the original pending request - * @param reply_anonymity_level anonymity level for the reply, UINT32_MAX for "unknown" - * @param expiration when does @a data expire? - * @param last_transmission when did we last transmit a request for this block - * @param type type of the block - * @param data response data, NULL on request expiration - * @param data_len number of bytes in @a data - */ -static void -handle_p2p_reply (void *cls, - enum GNUNET_BLOCK_ReplyEvaluationResult eval, - struct GSF_PendingRequest *pr, - uint32_t reply_anonymity_level, - struct GNUNET_TIME_Absolute expiration, - struct GNUNET_TIME_Absolute last_transmission, - enum GNUNET_BLOCK_Type type, - const void *data, - size_t data_len) -{ - struct PeerRequest *peerreq = cls; - struct GSF_ConnectedPeer *cp = peerreq->cp; - struct GSF_PendingRequestData *prd; - struct GNUNET_MQ_Envelope *env; - struct PutMessage *pm; - size_t msize; - - GNUNET_assert (data_len + sizeof(struct PutMessage) < - GNUNET_MAX_MESSAGE_SIZE); - GNUNET_assert (peerreq->pr == pr); - prd = GSF_pending_request_get_data_ (pr); - if (NULL == data) - { - free_pending_request (peerreq); - return; - } - GNUNET_break (GNUNET_BLOCK_TYPE_ANY != type); - if ( (prd->type != type) && - (GNUNET_BLOCK_TYPE_ANY != prd->type) ) - { - GNUNET_STATISTICS_update (GSF_stats, - "# replies dropped due to type mismatch", - 1, GNUNET_NO); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Transmitting result for query `%s' to peer\n", - GNUNET_h2s (&prd->query)); - GNUNET_STATISTICS_update (GSF_stats, - "# replies received for other peers", - 1, - GNUNET_NO); - msize = sizeof(struct PutMessage) + data_len; - if (msize >= GNUNET_MAX_MESSAGE_SIZE) - { - GNUNET_break (0); - return; - } - if ( (UINT32_MAX != reply_anonymity_level) && - (reply_anonymity_level > 1) ) - { - if (reply_anonymity_level - 1 > GSF_cover_content_count) - { - GNUNET_STATISTICS_update (GSF_stats, - "# replies dropped due to insufficient cover traffic", - 1, GNUNET_NO); - return; - } - GSF_cover_content_count -= (reply_anonymity_level - 1); - } - - env = GNUNET_MQ_msg_extra (pm, - data_len, - GNUNET_MESSAGE_TYPE_FS_PUT); - pm->type = htonl (type); - pm->expiration = GNUNET_TIME_absolute_hton (expiration); - GNUNET_memcpy (&pm[1], - data, - data_len); - if ((UINT32_MAX != reply_anonymity_level) && - (0 != reply_anonymity_level) && - (GNUNET_YES == GSF_enable_randomized_delays)) - { - struct GSF_DelayedHandle *dh; - - dh = GNUNET_new (struct GSF_DelayedHandle); - dh->cp = cp; - dh->env = env; - dh->msize = msize; - GNUNET_CONTAINER_DLL_insert (cp->delayed_head, - cp->delayed_tail, - dh); - cp->delay_queue_size++; - dh->delay_task = - GNUNET_SCHEDULER_add_delayed (get_randomized_delay (), - &transmit_delayed_now, - dh); - } - else - { - GSF_peer_transmit_ (cp, - GNUNET_NO, - UINT32_MAX, - env); - } - if (GNUNET_BLOCK_REPLY_OK_LAST != eval) - return; - if (NULL == peerreq->kill_task) - { - GNUNET_STATISTICS_update (GSF_stats, - "# P2P searches destroyed due to ultimate reply", - 1, - GNUNET_NO); - peerreq->kill_task = - GNUNET_SCHEDULER_add_now (&peer_request_destroy, - peerreq); - } -} - - -/** - * Increase the peer's respect by a value. - * - * @param cp which peer to change the respect value on - * @param value is the int value by which the - * peer's credit is to be increased or decreased - * @returns the actual change in respect (positive or negative) - */ -static int -change_peer_respect (struct GSF_ConnectedPeer *cp, int value) -{ - if (0 == value) - return 0; - GNUNET_assert (NULL != cp); - if (value > 0) - { - if (cp->ppd.respect + value < cp->ppd.respect) - { - value = UINT32_MAX - cp->ppd.respect; - cp->ppd.respect = UINT32_MAX; - } - else - cp->ppd.respect += value; - } - else - { - if (cp->ppd.respect < -value) - { - value = -cp->ppd.respect; - cp->ppd.respect = 0; - } - else - cp->ppd.respect += value; - } - return value; -} - - -/** - * We've received a request with the specified priority. Bound it - * according to how much we respect the given peer. - * - * @param prio_in requested priority - * @param cp the peer making the request - * @return effective priority - */ -static int32_t -bound_priority (uint32_t prio_in, - struct GSF_ConnectedPeer *cp) -{ -#define N ((double) 128.0) - uint32_t ret; - double rret; - int ld; - - ld = GSF_test_get_load_too_high_ (0); - if (GNUNET_SYSERR == ld) - { -#if INSANE_STATISTICS - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop - ("# requests done for free (low load)"), 1, - GNUNET_NO); -#endif - return 0; /* excess resources */ - } - if (prio_in > INT32_MAX) - prio_in = INT32_MAX; - ret = -change_peer_respect (cp, -(int) prio_in); - if (ret > 0) - { - if (ret > GSF_current_priorities + N) - rret = GSF_current_priorities + N; - else - rret = ret; - GSF_current_priorities = (GSF_current_priorities * (N - 1) + rret) / N; - } - if ((GNUNET_YES == ld) && (ret > 0)) - { - /* try with charging */ - ld = GSF_test_get_load_too_high_ (ret); - } - if (GNUNET_YES == ld) - { - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop - ("# request dropped, priority insufficient"), 1, - GNUNET_NO); - /* undo charge */ - change_peer_respect (cp, (int) ret); - return -1; /* not enough resources */ - } - else - { - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop - ("# requests done for a price (normal load)"), - 1, - GNUNET_NO); - } -#undef N - return ret; -} - - -/** - * The priority level imposes a bound on the maximum - * value for the ttl that can be requested. - * - * @param ttl_in requested ttl - * @param prio given priority - * @return @a ttl_in if @a ttl_in is below the limit, - * otherwise the ttl-limit for the given @a prio - */ -static int32_t -bound_ttl (int32_t ttl_in, - uint32_t prio) -{ - unsigned long long allowed; - - if (ttl_in <= 0) - return ttl_in; - allowed = ((unsigned long long) prio) * TTL_DECREMENT / 1000; - if (ttl_in > allowed) - { - if (allowed >= (1 << 30)) - return 1 << 30; - return allowed; - } - return ttl_in; -} - - -/** - * Closure for #test_exist_cb(). - */ -struct TestExistClosure -{ - /** - * Priority of the incoming request. - */ - int32_t priority; - - /** - * Relative TTL of the incoming request. - */ - int32_t ttl; - - /** - * Type of the incoming request. - */ - enum GNUNET_BLOCK_Type type; - - /** - * Set to #GNUNET_YES if we are done handling the query. - */ - int finished; -}; - - -/** - * Test if the query already exists. If so, merge it, otherwise - * keep `finished` at #GNUNET_NO. - * - * @param cls our `struct TestExistClosure` - * @param hc the key of the query - * @param value the existing `struct PeerRequest`. - * @return #GNUNET_YES to continue to iterate, - * #GNUNET_NO if we successfully merged - */ -static int -test_exist_cb (void *cls, - const struct GNUNET_HashCode *hc, - void *value) -{ - struct TestExistClosure *tec = cls; - struct PeerRequest *peerreq = value; - struct GSF_PendingRequest *pr; - struct GSF_PendingRequestData *prd; - - pr = peerreq->pr; - prd = GSF_pending_request_get_data_ (pr); - if (prd->type != tec->type) - return GNUNET_YES; - if (prd->ttl.abs_value_us >= - GNUNET_TIME_absolute_get ().abs_value_us + tec->ttl * 1000LL) - { - /* existing request has higher TTL, drop new one! */ - prd->priority += tec->priority; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Have existing request with higher TTL, dropping new request.\n"); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop - ("# requests dropped due to higher-TTL request"), - 1, GNUNET_NO); - tec->finished = GNUNET_YES; - return GNUNET_NO; - } - /* existing request has lower TTL, drop old one! */ - tec->priority += prd->priority; - free_pending_request (peerreq); - GSF_pending_request_cancel_ (pr, - GNUNET_YES); - return GNUNET_NO; -} - - -/** - * Handle P2P "QUERY" message. Creates the pending request entry - * and sets up all of the data structures to that we will - * process replies properly. Does not initiate forwarding or - * local database lookups. - * - * @param cls the other peer involved (sender of the message) - * @param gm the GET message - */ -void -handle_p2p_get (void *cls, - const struct GetMessage *gm) -{ - struct GSF_ConnectedPeer *cps = cls; - struct PeerRequest *peerreq; - struct GSF_PendingRequest *pr; - struct GSF_ConnectedPeer *cp; - const struct GNUNET_PeerIdentity *target; - enum GSF_PendingRequestOptions options; - uint16_t msize; - unsigned int bits; - const struct GNUNET_PeerIdentity *opt; - uint32_t bm; - size_t bfsize; - uint32_t ttl_decrement; - struct TestExistClosure tec; - GNUNET_PEER_Id spid; - const struct GSF_PendingRequestData *prd; - - msize = ntohs (gm->header.size); - tec.type = ntohl (gm->type); - bm = ntohl (gm->hash_bitmap); - bits = 0; - while (bm > 0) - { - if (1 == (bm & 1)) - bits++; - bm >>= 1; - } - opt = (const struct GNUNET_PeerIdentity *) &gm[1]; - bfsize = msize - sizeof(struct GetMessage) - bits * sizeof(struct - GNUNET_PeerIdentity); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop - ("# GET requests received (from other peers)"), - 1, - GNUNET_NO); - GSF_cover_query_count++; - bm = ntohl (gm->hash_bitmap); - bits = 0; - if (0 != (bm & GET_MESSAGE_BIT_RETURN_TO)) - cp = GSF_peer_get_ (&opt[bits++]); - else - cp = cps; - if (NULL == cp) - { - if (0 != (bm & GET_MESSAGE_BIT_RETURN_TO)) - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Failed to find RETURN-TO peer `%s' in connection set. Dropping query.\n", - GNUNET_i2s (&opt[bits - 1])); - - else - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Failed to find peer `%s' in connection set. Dropping query.\n", - GNUNET_i2s (cps->ppd.peer)); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop - ( - "# requests dropped due to missing reverse route"), - 1, - GNUNET_NO); - return; - } - unsigned int queue_size = GNUNET_MQ_get_length (cp->mq); - queue_size += cp->ppd.pending_replies + cp->delay_queue_size; - if (queue_size > MAX_QUEUE_PER_PEER) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Peer `%s' has too many replies queued already. Dropping query.\n", - GNUNET_i2s (cps->ppd.peer)); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# requests dropped due to full reply queue"), - 1, - GNUNET_NO); - return; - } - /* note that we can really only check load here since otherwise - * peers could find out that we are overloaded by not being - * disconnected after sending us a malformed query... */ - tec.priority = bound_priority (ntohl (gm->priority), - cps); - if (tec.priority < 0) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Dropping query from `%s', this peer is too busy.\n", - GNUNET_i2s (cps->ppd.peer)); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Received request for `%s' of type %u from peer `%s' with flags %u\n", - GNUNET_h2s (&gm->query), - (unsigned int) tec.type, - GNUNET_i2s (cps->ppd.peer), - (unsigned int) bm); - target = - (0 != - (bm & GET_MESSAGE_BIT_TRANSMIT_TO)) ? (&opt[bits++]) : NULL; - options = GSF_PRO_DEFAULTS; - spid = 0; - if ((GNUNET_LOAD_get_load (cp->ppd.transmission_delay) > 3 * (1 - + tec.priority)) - || (GNUNET_LOAD_get_average (cp->ppd.transmission_delay) > - GNUNET_CONSTANTS_MAX_CORK_DELAY.rel_value_us * 2 - + GNUNET_LOAD_get_average (GSF_rt_entry_lifetime))) - { - /* don't have BW to send to peer, or would likely take longer than we have for it, - * so at best indirect the query */ - tec.priority = 0; - options |= GSF_PRO_FORWARD_ONLY; - spid = GNUNET_PEER_intern (cps->ppd.peer); - GNUNET_assert (0 != spid); - } - tec.ttl = bound_ttl (ntohl (gm->ttl), - tec.priority); - /* decrement ttl (always) */ - ttl_decrement = - 2 * TTL_DECREMENT + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, - TTL_DECREMENT); - if ((tec.ttl < 0) && - (((int32_t) (tec.ttl - ttl_decrement)) > 0)) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Dropping query from `%s' due to TTL underflow (%d - %u).\n", - GNUNET_i2s (cps->ppd.peer), - tec.ttl, - ttl_decrement); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop - ("# requests dropped due TTL underflow"), 1, - GNUNET_NO); - /* integer underflow => drop (should be very rare)! */ - return; - } - tec.ttl -= ttl_decrement; - - /* test if the request already exists */ - tec.finished = GNUNET_NO; - GNUNET_CONTAINER_multihashmap_get_multiple (cp->request_map, - &gm->query, - &test_exist_cb, - &tec); - if (GNUNET_YES == tec.finished) - return; /* merged into existing request, we're done */ - - peerreq = GNUNET_new (struct PeerRequest); - peerreq->cp = cp; - pr = GSF_pending_request_create_ (options, - tec.type, - &gm->query, - target, - (bfsize > 0) - ? (const char *) &opt[bits] - : NULL, - bfsize, - 1 /* anonymity */, - (uint32_t) tec.priority, - tec.ttl, - spid, - GNUNET_PEER_intern (cps->ppd.peer), - NULL, 0, /* replies_seen */ - &handle_p2p_reply, - peerreq); - GNUNET_assert (NULL != pr); - prd = GSF_pending_request_get_data_ (pr); - peerreq->pr = pr; - GNUNET_break (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put (cp->request_map, - &prd->query, - peerreq, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE)); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# P2P query messages received and processed"), - 1, - GNUNET_NO); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# P2P searches active"), - 1, - GNUNET_NO); - GSF_pending_request_get_data_ (pr)->has_started = GNUNET_YES; - GSF_local_lookup_ (pr, - &GSF_consider_forwarding, - NULL); -} - - -/** - * Transmit a message to the given peer as soon as possible. - * If the peer disconnects before the transmission can happen, - * the callback is invoked with a `NULL` @a buffer. - * - * @param cp target peer - * @param is_query is this a query (#GNUNET_YES) or content (#GNUNET_NO) or neither (#GNUNET_SYSERR) - * @param priority how important is this request? - * @param env message to send - */ -void -GSF_peer_transmit_ (struct GSF_ConnectedPeer *cp, - int is_query, - uint32_t priority, - struct GNUNET_MQ_Envelope *env) -{ - struct GSF_PeerTransmitHandle *pth; - struct GSF_PeerTransmitHandle *pos; - struct GSF_PeerTransmitHandle *prev; - - pth = GNUNET_new (struct GSF_PeerTransmitHandle); - pth->transmission_request_start_time = GNUNET_TIME_absolute_get (); - pth->env = env; - pth->is_query = is_query; - pth->priority = priority; - pth->cp = cp; - /* insertion sort (by priority, descending) */ - prev = NULL; - pos = cp->pth_head; - while ((NULL != pos) && (pos->priority > priority)) - { - prev = pos; - pos = pos->next; - } - GNUNET_CONTAINER_DLL_insert_after (cp->pth_head, - cp->pth_tail, - prev, - pth); - if (GNUNET_YES == is_query) - cp->ppd.pending_queries++; - else if (GNUNET_NO == is_query) - cp->ppd.pending_replies++; - schedule_transmission (pth); -} - - -/** - * Report on receiving a reply; update the performance record of the given peer. - * - * @param cp responding peer (will be updated) - * @param request_time time at which the original query was transmitted - * @param request_priority priority of the original request - */ -void -GSF_peer_update_performance_ (struct GSF_ConnectedPeer *cp, - struct GNUNET_TIME_Absolute request_time, - uint32_t request_priority) -{ - struct GNUNET_TIME_Relative delay; - - delay = GNUNET_TIME_absolute_get_duration (request_time); - cp->ppd.avg_reply_delay.rel_value_us = - (cp->ppd.avg_reply_delay.rel_value_us * (RUNAVG_DELAY_N - 1) - + delay.rel_value_us) / RUNAVG_DELAY_N; - cp->ppd.avg_priority = - (cp->ppd.avg_priority * (RUNAVG_DELAY_N - 1) - + request_priority) / RUNAVG_DELAY_N; -} - - -/** - * Report on receiving a reply in response to an initiating client. - * Remember that this peer is good for this client. - * - * @param cp responding peer (will be updated) - * @param initiator_client local client on responsible for query - */ -void -GSF_peer_update_responder_client_ (struct GSF_ConnectedPeer *cp, - struct GSF_LocalClient *initiator_client) -{ - cp->ppd.last_client_replies[cp->last_client_replies_woff++ - % CS2P_SUCCESS_LIST_SIZE] = initiator_client; -} - - -/** - * Report on receiving a reply in response to an initiating peer. - * Remember that this peer is good for this initiating peer. - * - * @param cp responding peer (will be updated) - * @param initiator_peer other peer responsible for query - */ -void -GSF_peer_update_responder_peer_ (struct GSF_ConnectedPeer *cp, - const struct GSF_ConnectedPeer *initiator_peer) -{ - unsigned int woff; - - woff = cp->last_p2p_replies_woff % P2P_SUCCESS_LIST_SIZE; - GNUNET_PEER_change_rc (cp->ppd.last_p2p_replies[woff], -1); - cp->ppd.last_p2p_replies[woff] = initiator_peer->ppd.pid; - GNUNET_PEER_change_rc (initiator_peer->ppd.pid, 1); - cp->last_p2p_replies_woff = (woff + 1) % P2P_SUCCESS_LIST_SIZE; -} - - -/** - * Write peer-respect information to a file - flush the buffer entry! - * - * @param cls unused - * @param key peer identity - * @param value the `struct GSF_ConnectedPeer` to flush - * @return #GNUNET_OK to continue iteration - */ -static int -flush_respect (void *cls, - const struct GNUNET_PeerIdentity *key, - void *value) -{ - struct GSF_ConnectedPeer *cp = value; - struct GNUNET_PeerIdentity pid; - - if (cp->ppd.respect == cp->disk_respect) - return GNUNET_OK; /* unchanged */ - GNUNET_assert (0 != cp->ppd.pid); - GNUNET_PEER_resolve (cp->ppd.pid, &pid); - GNUNET_PEERSTORE_store (peerstore, "fs", &pid, "respect", &cp->ppd.respect, - sizeof(cp->ppd.respect), - GNUNET_TIME_UNIT_FOREVER_ABS, - GNUNET_PEERSTORE_STOREOPTION_REPLACE, - NULL, - NULL); - return GNUNET_OK; -} - - -void -GSF_peer_disconnect_handler (void *cls, - const struct GNUNET_PeerIdentity *peer, - void *internal_cls) -{ - struct GSF_ConnectedPeer *cp = internal_cls; - struct GSF_PeerTransmitHandle *pth; - struct GSF_DelayedHandle *dh; - - if (NULL == cp) - return; /* must have been disconnect from core with - * 'peer' == my_id, ignore */ - flush_respect (NULL, - peer, - cp); - GNUNET_assert (GNUNET_YES == - GNUNET_CONTAINER_multipeermap_remove (cp_map, - peer, - cp)); - GNUNET_STATISTICS_set (GSF_stats, - gettext_noop ("# peers connected"), - GNUNET_CONTAINER_multipeermap_size (cp_map), - GNUNET_NO); - if (NULL != cp->respect_iterate_req) - { - GNUNET_PEERSTORE_iterate_cancel (cp->respect_iterate_req); - cp->respect_iterate_req = NULL; - } - GNUNET_CONTAINER_multihashmap_iterate (cp->request_map, - &cancel_pending_request, - cp); - GNUNET_CONTAINER_multihashmap_destroy (cp->request_map); - cp->request_map = NULL; - GSF_plan_notify_peer_disconnect_ (cp); - GNUNET_LOAD_value_free (cp->ppd.transmission_delay); - GNUNET_PEER_decrement_rcs (cp->ppd.last_p2p_replies, - P2P_SUCCESS_LIST_SIZE); - memset (cp->ppd.last_p2p_replies, - 0, - sizeof(cp->ppd.last_p2p_replies)); - GSF_push_stop_ (cp); - while (NULL != (pth = cp->pth_head)) - { - GNUNET_CONTAINER_DLL_remove (cp->pth_head, - cp->pth_tail, - pth); - if (GNUNET_YES == pth->is_query) - GNUNET_assert (0 < cp->ppd.pending_queries--); - else if (GNUNET_NO == pth->is_query) - GNUNET_assert (0 < cp->ppd.pending_replies--); - GNUNET_free (pth); - } - while (NULL != (dh = cp->delayed_head)) - { - GNUNET_CONTAINER_DLL_remove (cp->delayed_head, - cp->delayed_tail, - dh); - GNUNET_MQ_discard (dh->env); - cp->delay_queue_size--; - GNUNET_SCHEDULER_cancel (dh->delay_task); - GNUNET_free (dh); - } - GNUNET_PEER_change_rc (cp->ppd.pid, -1); - if (NULL != cp->mig_revive_task) - { - GNUNET_SCHEDULER_cancel (cp->mig_revive_task); - cp->mig_revive_task = NULL; - } - GNUNET_break (0 == cp->ppd.pending_queries); - GNUNET_break (0 == cp->ppd.pending_replies); - GNUNET_free (cp); -} - - -/** - * Closure for #call_iterator(). - */ -struct IterationContext -{ - /** - * Function to call on each entry. - */ - GSF_ConnectedPeerIterator it; - - /** - * Closure for @e it. - */ - void *it_cls; -}; - - -/** - * Function that calls the callback for each peer. - * - * @param cls the `struct IterationContext *` - * @param key identity of the peer - * @param value the `struct GSF_ConnectedPeer *` - * @return #GNUNET_YES to continue iteration - */ -static int -call_iterator (void *cls, - const struct GNUNET_PeerIdentity *key, - void *value) -{ - struct IterationContext *ic = cls; - struct GSF_ConnectedPeer *cp = value; - - ic->it (ic->it_cls, - key, cp, - &cp->ppd); - return GNUNET_YES; -} - - -void -GSF_iterate_connected_peers_ (GSF_ConnectedPeerIterator it, - void *it_cls) -{ - struct IterationContext ic; - - ic.it = it; - ic.it_cls = it_cls; - GNUNET_CONTAINER_multipeermap_iterate (cp_map, - &call_iterator, - &ic); -} - - -/** - * Obtain the identity of a connected peer. - * - * @param cp peer to get identity of - * @param id identity to set (written to) - */ -void -GSF_connected_peer_get_identity_ (const struct GSF_ConnectedPeer *cp, - struct GNUNET_PeerIdentity *id) -{ - GNUNET_assert (0 != cp->ppd.pid); - GNUNET_PEER_resolve (cp->ppd.pid, id); -} - - -/** - * Obtain the identity of a connected peer. - * - * @param cp peer to get identity of - * @return reference to peer identity, valid until peer disconnects (!) - */ -const struct GNUNET_PeerIdentity * -GSF_connected_peer_get_identity2_ (const struct GSF_ConnectedPeer *cp) -{ - GNUNET_assert (0 != cp->ppd.pid); - return GNUNET_PEER_resolve2 (cp->ppd.pid); -} - - -/** - * Ask a peer to stop migrating data to us until the given point - * in time. - * - * @param cp peer to ask - * @param block_time until when to block - */ -void -GSF_block_peer_migration_ (struct GSF_ConnectedPeer *cp, - struct GNUNET_TIME_Absolute block_time) -{ - struct GNUNET_MQ_Envelope *env; - struct MigrationStopMessage *msm; - - if (cp->last_migration_block.abs_value_us > block_time.abs_value_us) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Migration already blocked for another %s\n", - GNUNET_STRINGS_relative_time_to_string ( - GNUNET_TIME_absolute_get_remaining - (cp-> - last_migration_block), GNUNET_YES)); - return; /* already blocked */ - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Asking to stop migration for %s\n", - GNUNET_STRINGS_relative_time_to_string ( - GNUNET_TIME_absolute_get_remaining (block_time), - GNUNET_YES)); - cp->last_migration_block = block_time; - env = GNUNET_MQ_msg (msm, - GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP); - msm->reserved = htonl (0); - msm->duration - = GNUNET_TIME_relative_hton (GNUNET_TIME_absolute_get_remaining - (cp->last_migration_block)); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# migration stop messages sent"), - 1, - GNUNET_NO); - GSF_peer_transmit_ (cp, - GNUNET_SYSERR, - UINT32_MAX, - env); -} - - -/** - * Notify core about a preference we have for the given peer - * (to allocate more resources towards it). The change will - * be communicated the next time we reserve bandwidth with - * core (not instantly). - * - * @param cp peer to reserve bandwidth from - * @param pref preference change - */ -void -GSF_connected_peer_change_preference_ (struct GSF_ConnectedPeer *cp, - uint64_t pref) -{ - cp->inc_preference += pref; -} - - -/** - * Call this method periodically to flush respect information to disk. - * - * @param cls closure, not used - */ -static void -cron_flush_respect (void *cls) -{ - fr_task = NULL; - GNUNET_CONTAINER_multipeermap_iterate (cp_map, - &flush_respect, - NULL); - fr_task = GNUNET_SCHEDULER_add_delayed_with_priority (RESPECT_FLUSH_FREQ, - GNUNET_SCHEDULER_PRIORITY_HIGH, - &cron_flush_respect, - NULL); -} - - -/** - * Initialize peer management subsystem. - */ -void -GSF_connected_peer_init_ () -{ - cp_map = GNUNET_CONTAINER_multipeermap_create (128, GNUNET_YES); - peerstore = GNUNET_PEERSTORE_connect (GSF_cfg); - fr_task = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_HIGH, - &cron_flush_respect, NULL); -} - - -/** - * Shutdown peer management subsystem. - */ -void -GSF_connected_peer_done_ () -{ - GNUNET_CONTAINER_multipeermap_iterate (cp_map, - &flush_respect, - NULL); - GNUNET_SCHEDULER_cancel (fr_task); - fr_task = NULL; - GNUNET_CONTAINER_multipeermap_destroy (cp_map); - cp_map = NULL; - GNUNET_PEERSTORE_disconnect (peerstore, - GNUNET_YES); -} - - -/** - * Iterator to remove references to LC entry. - * - * @param cls the `struct GSF_LocalClient *` to look for - * @param key current key code - * @param value value in the hash map (peer entry) - * @return #GNUNET_YES (we should continue to iterate) - */ -static int -clean_local_client (void *cls, - const struct GNUNET_PeerIdentity *key, - void *value) -{ - const struct GSF_LocalClient *lc = cls; - struct GSF_ConnectedPeer *cp = value; - unsigned int i; - - for (i = 0; i < CS2P_SUCCESS_LIST_SIZE; i++) - if (cp->ppd.last_client_replies[i] == lc) - cp->ppd.last_client_replies[i] = NULL; - return GNUNET_YES; -} - - -/** - * Notification that a local client disconnected. Clean up all of our - * references to the given handle. - * - * @param lc handle to the local client (henceforth invalid) - */ -void -GSF_handle_local_client_disconnect_ (const struct GSF_LocalClient *lc) -{ - if (NULL == cp_map) - return; /* already cleaned up */ - GNUNET_CONTAINER_multipeermap_iterate (cp_map, - &clean_local_client, - (void *) lc); -} - - -/* end of gnunet-service-fs_cp.c */ diff --git a/src/fs/gnunet-service-fs_cp.h b/src/fs/gnunet-service-fs_cp.h deleted file mode 100644 index cea82b10c..000000000 --- a/src/fs/gnunet-service-fs_cp.h +++ /dev/null @@ -1,415 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2011 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_cp.h - * @brief API to handle 'connected peers' - * @author Christian Grothoff - */ -#ifndef GNUNET_SERVICE_FS_CP_H -#define GNUNET_SERVICE_FS_CP_H - -#include "fs.h" -#include "gnunet-service-fs.h" - - -/** - * Maximum number of outgoing messages we queue per peer. - * - * Performance measurements for 2 peer setup for 50 MB file - * (using perf_gnunet_service_fs_p2p): - * - * 24: 2-3 MB/s # ~ 24 MB RAM - * 256: 8 MB/s # ~256 MB RAM - * - * Conclusion: 24 should suffice (reasonable - * performance, no excessive memory use). - */ -#define MAX_QUEUE_PER_PEER 24 - -/** - * Length of the P2P success tracker. Note that having a very long - * list can also hurt performance. - */ -#define P2P_SUCCESS_LIST_SIZE 8 - -/** - * Length of the CS-2-P success tracker. Note that - * having a very long list can also hurt performance. - */ -#define CS2P_SUCCESS_LIST_SIZE 8 - - -/** - * Performance data kept for a peer. - */ -struct GSF_PeerPerformanceData -{ - /** - * List of the last clients for which this peer successfully - * answered a query. - */ - struct GSF_LocalClient *last_client_replies[CS2P_SUCCESS_LIST_SIZE]; - - /** - * List of the last PIDs for which - * this peer successfully answered a query; - * We use 0 to indicate no successful reply. - */ - GNUNET_PEER_Id last_p2p_replies[P2P_SUCCESS_LIST_SIZE]; - - /** - * Average delay between sending the peer a request and - * getting a reply (only calculated over the requests for - * which we actually got a reply). Calculated - * as a moving average: new_delay = ((n-1)*last_delay+curr_delay) / n - */ - struct GNUNET_TIME_Relative avg_reply_delay; - - /** - * If we get content we already have from this peer, for how - * long do we block it? Adjusted based on the fraction of - * redundant data we receive, between 1s and 1h. - */ - struct GNUNET_TIME_Relative migration_delay; - - /** - * Point in time until which this peer does not want us to migrate content - * to it. - */ - struct GNUNET_TIME_Absolute migration_blocked_until; - - /** - * Transmission times for the last MAX_QUEUE_PER_PEER - * requests for this peer. Used as a ring buffer, current - * offset is stored in 'last_request_times_off'. If the - * oldest entry is more recent than the 'avg_delay', we should - * not send any more requests right now. - */ - struct GNUNET_TIME_Absolute last_request_times[MAX_QUEUE_PER_PEER]; - - /** - * How long does it typically take for us to transmit a message - * to this peer? (delay between the request being issued and - * the callback being invoked). - */ - struct GNUNET_LOAD_Value *transmission_delay; - - /** - * Average priority of successful replies. Calculated - * as a moving average: new_avg = ((n-1)*last_avg+curr_prio) / n - */ - double avg_priority; - - /** - * The peer's identity (interned version). - */ - GNUNET_PEER_Id pid; - - /** - * The peer's identity (pointer). - */ - const struct GNUNET_PeerIdentity *peer; - - /** - * Respect rating for this peer - */ - uint32_t respect; - - /** - * Number of pending queries (replies are not counted) - */ - unsigned int pending_queries; - - /** - * Number of pending replies (queries are not counted) - */ - unsigned int pending_replies; -}; - - -/** - * Signature of function called on a connected peer. - * - * @param cls closure - * @param peer identity of the peer - * @param cp handle to the connected peer record - * @param perf peer performance data - */ -typedef void -(*GSF_ConnectedPeerIterator) (void *cls, - const struct GNUNET_PeerIdentity *peer, - struct GSF_ConnectedPeer *cp, - const struct GSF_PeerPerformanceData *ppd); - - -/** - * Function called to get a message for transmission. - * - * @param cls closure - * @param buf_size number of bytes available in @a buf - * @param buf where to copy the message, NULL on error (peer disconnect) - * @return number of bytes copied to @a buf, can be 0 (without indicating an error) - */ -typedef size_t -(*GSF_GetMessageCallback) (void *cls, - size_t buf_size, - void *buf); - - -/** - * Signature of function called on a reservation success or failure. - * - * @param cls closure - * @param cp handle to the connected peer record - * @param success #GNUNET_YES on success, #GNUNET_NO on failure - */ -typedef void -(*GSF_PeerReserveCallback) (void *cls, - struct GSF_ConnectedPeer *cp, - int success); - - -/** - * Handle to cancel a transmission request. - */ -struct GSF_PeerTransmitHandle; - - -/** - * A peer connected to us. Setup the connected peer - * records. - * - * @param cls NULL - * @param peer identity of peer that connected - * @param mq message queue for talking to @a peer - * @return internal handle for the peer - */ -void * -GSF_peer_connect_handler (void *cls, - const struct GNUNET_PeerIdentity *peer, - struct GNUNET_MQ_Handle *mq); - - -/** - * Get a handle for a connected peer. - * - * @param peer peer's identity - * @return NULL if this peer is not currently connected - */ -struct GSF_ConnectedPeer * -GSF_peer_get_ (const struct GNUNET_PeerIdentity *peer); - - -/** - * Update the latency information kept for the given peer. - * - * @param id peer record to update - * @param latency current latency value - */ -void -GSF_update_peer_latency_ (const struct GNUNET_PeerIdentity *id, - struct GNUNET_TIME_Relative latency); - - -/** - * Transmit a message to the given peer as soon as possible. - * If the peer disconnects before the transmission can happen, - * the callback is invoked with a 'NULL' buffer. - * - * @param cp target peer - * @param is_query is this a query (#GNUNET_YES) or content (#GNUNET_NO) - * @param priority how important is this request? - * @param env envelope of message to send - */ -void -GSF_peer_transmit_ (struct GSF_ConnectedPeer *cp, - int is_query, - uint32_t priority, - struct GNUNET_MQ_Envelope *env); - - -/** - * Report on receiving a reply; update the performance record of the given peer. - * - * @param cp responding peer (will be updated) - * @param request_time time at which the original query was transmitted - * @param request_priority priority of the original request - */ -void -GSF_peer_update_performance_ (struct GSF_ConnectedPeer *cp, - struct GNUNET_TIME_Absolute request_time, - uint32_t request_priority); - - -/** - * Report on receiving a reply in response to an initiating client. - * Remember that this peer is good for this client. - * - * @param cp responding peer (will be updated) - * @param initiator_client local client on responsible for query - */ -void -GSF_peer_update_responder_client_ (struct GSF_ConnectedPeer *cp, - struct GSF_LocalClient *initiator_client); - - -/** - * Report on receiving a reply in response to an initiating peer. - * Remember that this peer is good for this initiating peer. - * - * @param cp responding peer (will be updated) - * @param initiator_peer other peer responsible for query - */ -void -GSF_peer_update_responder_peer_ (struct GSF_ConnectedPeer *cp, - const struct GSF_ConnectedPeer - *initiator_peer); - - -/** - * Handle P2P #GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP message. - * - * @param cls closure, the `struct GSF_ConnectedPeer` - * @param msm the actual message - */ -void -handle_p2p_migration_stop (void *cls, - const struct MigrationStopMessage *message); - - -/** - * Handle P2P "QUERY" message. - * - * @param cls the `struct GSF_ConnectedPeer` of the other sender - * @param gm the actual message - */ -void -handle_p2p_get (void *cls, - const struct GetMessage *gm); - - -/** - * Return the performance data record for the given peer - * - * @param cp peer to query - * @return performance data record for the peer - */ -struct GSF_PeerPerformanceData * -GSF_get_peer_performance_data_ (struct GSF_ConnectedPeer *cp); - - -/** - * Ask a peer to stop migrating data to us until the given point - * in time. - * - * @param cp peer to ask - * @param block_time until when to block - */ -void -GSF_block_peer_migration_ (struct GSF_ConnectedPeer *cp, - struct GNUNET_TIME_Absolute block_time); - - -/** - * A peer disconnected from us. Tear down the connected peer - * record. - * - * @param cls unused - * @param peer identity of peer that disconnected - * @param internal_cls the corresponding `struct GSF_ConnectedPeer` - */ -void -GSF_peer_disconnect_handler (void *cls, - const struct GNUNET_PeerIdentity *peer, - void *internal_cls); - - -/** - * Notification that a local client disconnected. Clean up all of our - * references to the given handle. - * - * @param lc handle to the local client (henceforth invalid) - */ -void -GSF_handle_local_client_disconnect_ (const struct GSF_LocalClient *lc); - - -/** - * Notify core about a preference we have for the given peer - * (to allocate more resources towards it). The change will - * be communicated the next time we reserve bandwidth with - * core (not instantly). - * - * @param cp peer to reserve bandwidth from - * @param pref preference change - */ -void -GSF_connected_peer_change_preference_ (struct GSF_ConnectedPeer *cp, - uint64_t pref); - - -/** - * Obtain the identity of a connected peer. - * - * @param cp peer to get identity of - * @param id identity to set (written to) - */ -void -GSF_connected_peer_get_identity_ (const struct GSF_ConnectedPeer *cp, - struct GNUNET_PeerIdentity *id); - - -/** - * Obtain the identity of a connected peer. - * - * @param cp peer to get identity of - * @return reference to peer identity, valid until peer disconnects (!) - */ -const struct GNUNET_PeerIdentity * -GSF_connected_peer_get_identity2_ (const struct GSF_ConnectedPeer *cp); - - -/** - * Iterate over all connected peers. - * - * @param it function to call for each peer - * @param it_cls closure for @a it - */ -void -GSF_iterate_connected_peers_ (GSF_ConnectedPeerIterator it, void *it_cls); - - -/** - * Initialize peer management subsystem. - */ -void -GSF_connected_peer_init_ (void); - - -/** - * Shutdown peer management subsystem. - */ -void -GSF_connected_peer_done_ (void); - - -#endif -/* end of gnunet-service-fs_cp.h */ diff --git a/src/fs/gnunet-service-fs_indexing.c b/src/fs/gnunet-service-fs_indexing.c deleted file mode 100644 index 8fb34c067..000000000 --- a/src/fs/gnunet-service-fs_indexing.c +++ /dev/null @@ -1,495 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009, 2010 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_indexing.c - * @brief program that provides indexing functions of the file-sharing service - * @author Christian Grothoff - */ -#include "platform.h" -#include -#include "gnunet_core_service.h" -#include "gnunet_datastore_service.h" -#include "gnunet_protocols.h" -#include "gnunet_signatures.h" -#include "gnunet_util_lib.h" -#include "gnunet-service-fs.h" -#include "gnunet-service-fs_indexing.h" -#include "fs.h" - -/** - * In-memory information about indexed files (also available - * on-disk). - */ -struct IndexInfo -{ - /** - * This is a doubly linked list. - */ - struct IndexInfo *next; - - /** - * This is a doubly linked list. - */ - struct IndexInfo *prev; - - /** - * Name of the indexed file. Memory allocated - * at the end of this struct (do not free). - */ - const char *filename; - - /** - * Context for transmitting confirmation to client, - * NULL if we've done this already. - */ - struct GNUNET_SERVER_TransmitContext *tc; - - /** - * Context for hashing of the file. - */ - struct GNUNET_CRYPTO_FileHashContext *fhc; - - /** - * Hash of the contents of the file. - */ - struct GNUNET_HashCode file_id; -}; - - -/** - * Head of linked list of indexed files. - * FIXME: we don't need both a DLL and a hashmap here! - */ -static struct IndexInfo *indexed_files_head; - -/** - * Tail of linked list of indexed files. - */ -static struct IndexInfo *indexed_files_tail; - -/** - * Maps hash over content of indexed files to the respective 'struct IndexInfo'. - * The filenames are pointers into the indexed_files linked list and - * do not need to be freed. - */ -static struct GNUNET_CONTAINER_MultiHashMap *ifm; - -/** - * Our configuration. - */ -static const struct GNUNET_CONFIGURATION_Handle *cfg; - -/** - * Datastore handle. Created and destroyed by code in - * gnunet-service-fs (this is an alias). - */ -static struct GNUNET_DATASTORE_Handle *dsh; - - -/** - * Write the current index information list to disk. - */ -static void -write_index_list () -{ - struct GNUNET_BIO_WriteHandle *wh; - char *fn; - struct IndexInfo *pos; - - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_filename (cfg, "FS", "INDEXDB", &fn)) - { - GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, - "fs", - "INDEXDB"); - return; - } - wh = GNUNET_BIO_write_open_file (fn); - if (NULL == wh) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, - _ ("Could not open `%s'.\n"), - fn); - GNUNET_free (fn); - return; - } - for (pos = indexed_files_head; NULL != pos; pos = pos->next) - if ((GNUNET_OK != GNUNET_BIO_write (wh, - "fs-indexing-file-id", - &pos->file_id, - sizeof(struct GNUNET_HashCode))) || - (GNUNET_OK != GNUNET_BIO_write_string (wh, - "fs-indexing-filename", - pos->filename))) - break; - if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, - _ ("Error writing `%s'.\n"), - fn); - GNUNET_free (fn); - return; - } - GNUNET_free (fn); -} - - -/** - * Read index information from disk. - */ -static void -read_index_list () -{ - struct GNUNET_BIO_ReadHandle *rh; - char *fn; - struct IndexInfo *pos; - char *fname; - struct GNUNET_HashCode hc; - size_t slen; - char *emsg; - - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_filename (cfg, "FS", "INDEXDB", &fn)) - { - GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, - "fs", - "INDEXDB"); - return; - } - if (GNUNET_NO == GNUNET_DISK_file_test (fn)) - { - /* no index info yet */ - GNUNET_free (fn); - return; - } - rh = GNUNET_BIO_read_open_file (fn); - if (NULL == rh) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, - _ ("Could not open `%s'.\n"), - fn); - GNUNET_free (fn); - return; - } - while ( - (GNUNET_OK == GNUNET_BIO_read (rh, - "Hash of indexed file", - &hc, - sizeof(struct GNUNET_HashCode))) && - (GNUNET_OK == - GNUNET_BIO_read_string (rh, "Name of indexed file", &fname, 1024 * 16)) && - (fname != NULL)) - { - slen = strlen (fname) + 1; - pos = GNUNET_malloc (sizeof(struct IndexInfo) + slen); - pos->file_id = hc; - pos->filename = (const char *) &pos[1]; - GNUNET_memcpy (&pos[1], fname, slen); - if (GNUNET_SYSERR == GNUNET_CONTAINER_multihashmap_put ( - ifm, - &pos->file_id, - pos, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) - { - GNUNET_free (pos); - } - else - { - GNUNET_CONTAINER_DLL_insert (indexed_files_head, indexed_files_tail, pos); - } - GNUNET_free (fname); - } - if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) - GNUNET_free (emsg); - GNUNET_free (fn); -} - - -/** - * Continuation called from datastore's remove - * function. - * - * @param cls unused - * @param success did the deletion work? - * @param min_expiration minimum expiration time required for content to be stored - * @param msg error message - */ -static void -remove_cont (void *cls, - int success, - struct GNUNET_TIME_Absolute min_expiration, - const char *msg) -{ - if (GNUNET_OK != success) - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Failed to delete bogus block: %s\n"), - msg); -} - - -int -GNUNET_FS_handle_on_demand_block (const struct GNUNET_HashCode *key, - uint32_t size, - const void *data, - enum GNUNET_BLOCK_Type type, - uint32_t priority, - uint32_t anonymity, - uint32_t replication, - struct GNUNET_TIME_Absolute expiration, - uint64_t uid, - GNUNET_DATASTORE_DatumProcessor cont, - void *cont_cls) -{ - const struct OnDemandBlock *odb; - struct GNUNET_HashCode nkey; - struct GNUNET_CRYPTO_SymmetricSessionKey skey; - struct GNUNET_CRYPTO_SymmetricInitializationVector iv; - struct GNUNET_HashCode query; - ssize_t nsize; - char ndata[DBLOCK_SIZE]; - char edata[DBLOCK_SIZE]; - const char *fn; - struct GNUNET_DISK_FileHandle *fh; - uint64_t off; - struct IndexInfo *ii; - - if (size != sizeof(struct OnDemandBlock)) - { - GNUNET_break (0); - GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL); - return GNUNET_SYSERR; - } - odb = (const struct OnDemandBlock *) data; - off = GNUNET_ntohll (odb->offset); - ii = GNUNET_CONTAINER_multihashmap_get (ifm, &odb->file_id); - if (NULL == ii) - { - GNUNET_break (0); - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Failed to find index %s\n", - GNUNET_h2s (&odb->file_id)); - return GNUNET_SYSERR; - } - fn = ii->filename; - if ((NULL == fn) || (0 != access (fn, R_OK))) - { - GNUNET_STATISTICS_update ( - GSF_stats, - gettext_noop ("# index blocks removed: original file inaccessible"), - 1, - GNUNET_YES); - GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL); - return GNUNET_SYSERR; - } - if ((NULL == (fh = GNUNET_DISK_file_open (fn, - GNUNET_DISK_OPEN_READ, - GNUNET_DISK_PERM_NONE))) || - (off != GNUNET_DISK_file_seek (fh, off, GNUNET_DISK_SEEK_SET)) || - (-1 == (nsize = GNUNET_DISK_file_read (fh, ndata, sizeof(ndata))))) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ( - "Could not access indexed file `%s' (%s) at offset %llu: %s\n"), - GNUNET_h2s (&odb->file_id), - fn, - (unsigned long long) off, - (fn == NULL) ? _ ("not indexed") : strerror (errno)); - if (fh != NULL) - GNUNET_DISK_file_close (fh); - GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL); - return GNUNET_SYSERR; - } - GNUNET_DISK_file_close (fh); - GNUNET_CRYPTO_hash (ndata, nsize, &nkey); - GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv); - GNUNET_CRYPTO_symmetric_encrypt (ndata, nsize, &skey, &iv, edata); - GNUNET_CRYPTO_hash (edata, nsize, &query); - if (0 != memcmp (&query, key, sizeof(struct GNUNET_HashCode))) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("Indexed file `%s' changed at offset %llu\n"), - fn, - (unsigned long long) off); - GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL); - return GNUNET_SYSERR; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "On-demand encoded block for query `%s'\n", - GNUNET_h2s (key)); - cont (cont_cls, - key, - nsize, - edata, - GNUNET_BLOCK_TYPE_FS_DBLOCK, - priority, - anonymity, - replication, - expiration, - uid); - return GNUNET_OK; -} - - -/** - * Transmit information about indexed files to @a mq. - * - * @param mq message queue to send information to - */ -void -GNUNET_FS_indexing_send_list (struct GNUNET_MQ_Handle *mq) -{ - struct GNUNET_MQ_Envelope *env; - struct IndexInfoMessage *iim; - struct GNUNET_MessageHeader *iem; - size_t slen; - const char *fn; - struct IndexInfo *pos; - - for (pos = indexed_files_head; NULL != pos; pos = pos->next) - { - fn = pos->filename; - slen = strlen (fn) + 1; - if (slen + sizeof(struct IndexInfoMessage) >= GNUNET_MAX_MESSAGE_SIZE) - { - GNUNET_break (0); - break; - } - env = - GNUNET_MQ_msg_extra (iim, slen, GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY); - iim->reserved = 0; - iim->file_id = pos->file_id; - GNUNET_memcpy (&iim[1], fn, slen); - GNUNET_MQ_send (mq, env); - } - env = GNUNET_MQ_msg (iem, GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END); - GNUNET_MQ_send (mq, env); -} - - -/** - * Remove a file from the index. - * - * @param fid identifier of the file to remove - * @return #GNUNET_YES if the @a fid was found - */ -int -GNUNET_FS_indexing_do_unindex (const struct GNUNET_HashCode *fid) -{ - struct IndexInfo *pos; - - for (pos = indexed_files_head; NULL != pos; pos = pos->next) - { - if (0 == memcmp (&pos->file_id, fid, sizeof(struct GNUNET_HashCode))) - { - GNUNET_CONTAINER_DLL_remove (indexed_files_head, indexed_files_tail, pos); - GNUNET_break ( - GNUNET_OK == - GNUNET_CONTAINER_multihashmap_remove (ifm, &pos->file_id, pos)); - GNUNET_free (pos); - write_index_list (); - return GNUNET_YES; - } - } - return GNUNET_NO; -} - - -/** - * Add the given file to the list of indexed files. - * - * @param filename name of the file - * @param file_id hash identifier for @a filename - */ -void -GNUNET_FS_add_to_index (const char *filename, - const struct GNUNET_HashCode *file_id) -{ - struct IndexInfo *ii; - size_t slen; - - ii = GNUNET_CONTAINER_multihashmap_get (ifm, file_id); - if (NULL != ii) - { - GNUNET_log ( - GNUNET_ERROR_TYPE_INFO, - _ ( - "Index request received for file `%s' is already indexed as `%s'. Permitting anyway.\n"), - filename, - ii->filename); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Adding file %s to index as %s\n", - filename, - GNUNET_h2s (file_id)); - slen = strlen (filename) + 1; - ii = GNUNET_malloc (sizeof(struct IndexInfo) + slen); - ii->file_id = *file_id; - ii->filename = (const char *) &ii[1]; - GNUNET_memcpy (&ii[1], filename, slen); - GNUNET_CONTAINER_DLL_insert (indexed_files_head, indexed_files_tail, ii); - GNUNET_assert (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put ( - ifm, - &ii->file_id, - ii, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - write_index_list (); -} - - -/** - * Shutdown the module. - */ -void -GNUNET_FS_indexing_done () -{ - struct IndexInfo *pos; - - while (NULL != (pos = indexed_files_head)) - { - GNUNET_CONTAINER_DLL_remove (indexed_files_head, indexed_files_tail, pos); - if (pos->fhc != NULL) - GNUNET_CRYPTO_hash_file_cancel (pos->fhc); - GNUNET_break ( - GNUNET_OK == - GNUNET_CONTAINER_multihashmap_remove (ifm, &pos->file_id, pos)); - GNUNET_free (pos); - } - GNUNET_CONTAINER_multihashmap_destroy (ifm); - ifm = NULL; - cfg = NULL; -} - - -int -GNUNET_FS_indexing_init (const struct GNUNET_CONFIGURATION_Handle *c, - struct GNUNET_DATASTORE_Handle *d) -{ - cfg = c; - dsh = d; - ifm = GNUNET_CONTAINER_multihashmap_create (128, GNUNET_YES); - read_index_list (); - return GNUNET_OK; -} - - -/* end of gnunet-service-fs_indexing.c */ diff --git a/src/fs/gnunet-service-fs_indexing.h b/src/fs/gnunet-service-fs_indexing.h deleted file mode 100644 index 244a51900..000000000 --- a/src/fs/gnunet-service-fs_indexing.h +++ /dev/null @@ -1,120 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009, 2010, 2011 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_indexing.h - * @brief indexing for the file-sharing service - * @author Christian Grothoff - */ -#ifndef GNUNET_SERVICE_FS_INDEXING_H -#define GNUNET_SERVICE_FS_INDEXING_H - -#include "gnunet_block_lib.h" -#include "gnunet_core_service.h" -#include "gnunet_datastore_service.h" -#include "gnunet_protocols.h" -#include "gnunet_signatures.h" -#include "gnunet_util_lib.h" - - -/** - * We've received an on-demand encoded block from the datastore. - * Attempt to do on-demand encoding and (if successful), call the - * continuation with the resulting block. On error, clean up and ask - * the datastore for more results. - * - * @param key key for the content - * @param size number of bytes in data - * @param data content stored - * @param type type of the content - * @param priority priority of the content - * @param anonymity anonymity-level for the content - * @param replication replication-level for the content - * @param expiration expiration time for the content - * @param uid unique identifier for the datum; - * maybe 0 if no unique identifier is available - * @param cont function to call with the actual block (at most once, on success) - * @param cont_cls closure for @a cont - * @return #GNUNET_OK on success - */ -int -GNUNET_FS_handle_on_demand_block (const struct GNUNET_HashCode *key, - uint32_t size, - const void *data, - enum GNUNET_BLOCK_Type type, - uint32_t priority, - uint32_t anonymity, - uint32_t replication, - struct GNUNET_TIME_Absolute expiration, - uint64_t uid, - GNUNET_DATASTORE_DatumProcessor cont, - void *cont_cls); - - -/** - * Transmit information about indexed files to @a mq. - * - * @param mq message queue to send information to - */ -void -GNUNET_FS_indexing_send_list (struct GNUNET_MQ_Handle *mq); - - -/** - * Remove a file from the index. - * - * @param fid identifier of the file to remove - * @return #GNUNET_YES if the @a fid was found - */ -int -GNUNET_FS_indexing_do_unindex (const struct GNUNET_HashCode *fid); - - -/** - * Add the given file to the list of indexed files. - * - * @param filename name of the file - * @param file_id hash identifier for @a filename - */ -void -GNUNET_FS_add_to_index (const char *filename, - const struct GNUNET_HashCode *file_id); - - -/** - * Initialize the indexing submodule. - * - * @param c configuration to use - * @param d datastore to use - * @return GNUNET_OK on success - */ -int -GNUNET_FS_indexing_init (const struct GNUNET_CONFIGURATION_Handle *c, - struct GNUNET_DATASTORE_Handle *d); - - -/** - * Shutdown the module. - */ -void -GNUNET_FS_indexing_done (void); - - -#endif diff --git a/src/fs/gnunet-service-fs_pe.c b/src/fs/gnunet-service-fs_pe.c deleted file mode 100644 index 60dd0ab70..000000000 --- a/src/fs/gnunet-service-fs_pe.c +++ /dev/null @@ -1,814 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2011 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_pe.c - * @brief API to manage query plan - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet-service-fs.h" -#include "gnunet-service-fs_cp.h" -#include "gnunet-service-fs_pe.h" -#include "gnunet-service-fs_pr.h" - -/** - * Collect an instance number of statistics? May cause excessive IPC. - */ -#define INSANE_STATISTICS GNUNET_NO - -/** - * List of GSF_PendingRequests this request plan - * participates with. - */ -struct PendingRequestList; - -/** - * Transmission plan for a peer. - */ -struct PeerPlan; - - -/** - * M:N binding of plans to pending requests. - * Each pending request can be in a number of plans, - * and each plan can have a number of pending requests. - * Objects of this type indicate a mapping of a plan to - * a particular pending request. - * - * The corresponding head and tail of the "PE" MDLL - * are stored in a `struct GSF_RequestPlan`. (We need - * to be able to lookup all pending requests corresponding - * to a given plan entry.) - * - * Similarly head and tail of the "PR" MDLL are stored - * with the `struct GSF_PendingRequest`. (We need - * to be able to lookup all plan entries corresponding - * to a given pending request.) - */ -struct GSF_PendingRequestPlanBijection -{ - /** - * This is a doubly-linked list. - */ - struct GSF_PendingRequestPlanBijection *next_PR; - - /** - * This is a doubly-linked list. - */ - struct GSF_PendingRequestPlanBijection *prev_PR; - - /** - * This is a doubly-linked list. - */ - struct GSF_PendingRequestPlanBijection *next_PE; - - /** - * This is a doubly-linked list. - */ - struct GSF_PendingRequestPlanBijection *prev_PE; - - /** - * Associated request plan (tells us one of the peers that - * we plan to forward the request to). - */ - struct GSF_RequestPlan *rp; - - /** - * Associated pending request (identifies request details - * and one of the origins of the request). - */ - struct GSF_PendingRequest *pr; -}; - - -/** - * Information we keep per request per peer. This is a doubly-linked - * list (with head and tail in the `struct GSF_PendingRequestData`) - * with one entry in each heap of each `struct PeerPlan`. Each - * entry tracks information relevant for this request and this peer. - */ -struct GSF_RequestPlan -{ - /** - * This is a doubly-linked list. - */ - struct GSF_RequestPlan *next; - - /** - * This is a doubly-linked list. - */ - struct GSF_RequestPlan *prev; - - /** - * Heap node associated with this request and this peer. - */ - struct GNUNET_CONTAINER_HeapNode *hn; - - /** - * The transmission plan for a peer that this request is associated with. - */ - struct PeerPlan *pp; - - /** - * Head of list of associated pending requests. This tells us - * which incoming requests from other peers this plan entry - * corresponds to. - */ - struct GSF_PendingRequestPlanBijection *pe_head; - - /** - * Tail of list of associated pending requests. - */ - struct GSF_PendingRequestPlanBijection *pe_tail; - - /** - * Earliest time we'd be happy to (re)transmit this request. - */ - struct GNUNET_TIME_Absolute earliest_transmission; - - /** - * When was the last time we transmitted this request to this peer? 0 for never. - */ - struct GNUNET_TIME_Absolute last_transmission; - - /** - * Current priority for this request for this target. - */ - uint64_t priority; - - /** - * How often did we transmit this request to this peer? - */ - unsigned int transmission_counter; -}; - - -/** - * Transmission plan for a peer. - */ -struct PeerPlan -{ - /** - * Heap with pending queries (`struct GSF_RequestPlan`), higher weights mean higher priority. - */ - struct GNUNET_CONTAINER_Heap *priority_heap; - - /** - * Heap with pending queries (`struct GSF_RequestPlan`), by transmission time, lowest first. - */ - struct GNUNET_CONTAINER_Heap *delay_heap; - - /** - * Map of queries to plan entries. All entries in the @e priority_heap - * or @e delay_heap should be in the @e plan_map. Note that it is - * possible for the @e plan_map to have multiple entries for the same - * query. - */ - struct GNUNET_CONTAINER_MultiHashMap *plan_map; - - /** - * Peer for which this is the plan. - */ - struct GSF_ConnectedPeer *cp; - - /** - * Current task for executing the plan. - */ - struct GNUNET_SCHEDULER_Task *task; - - /** - * Current message under transmission for the plan. - */ - struct GNUNET_MQ_Envelope *env; -}; - - -/** - * Hash map from peer identities to PeerPlans. - */ -static struct GNUNET_CONTAINER_MultiPeerMap *plans; - -/** - * Sum of all transmission counters (equals total delay for all plan entries). - */ -static unsigned long long total_delay; - -/** - * Number of plan entries. - */ -static unsigned long long plan_count; - - -/** - * Return the query (key in the plan_map) for the given request plan. - * Note that this key may change as there can be multiple pending - * requests for the same key and we just return _one_ of them; this - * particular one might complete while another one might still be - * active, hence the lifetime of the returned hash code is NOT - * necessarily identical to that of the `struct GSF_RequestPlan` - * given. - * - * @param rp a request plan - * @return the associated query - */ -static const struct GNUNET_HashCode * -get_rp_key (struct GSF_RequestPlan *rp) -{ - return &GSF_pending_request_get_data_ (rp->pe_head->pr)->query; -} - - -/** - * Insert the given request plan into the heap with the appropriate weight. - * - * @param pp associated peer's plan - * @param rp request to plan - */ -static void -plan (struct PeerPlan *pp, - struct GSF_RequestPlan *rp) -{ -#define N ((double) 128.0) - /** - * Running average delay we currently impose. - */ - static double avg_delay; - - struct GSF_PendingRequestData *prd; - struct GNUNET_TIME_Relative delay; - - GNUNET_assert (rp->pp == pp); - GNUNET_STATISTICS_set (GSF_stats, - gettext_noop ("# average retransmission delay (ms)"), - total_delay * 1000LL / plan_count, GNUNET_NO); - prd = GSF_pending_request_get_data_ (rp->pe_head->pr); - - if (rp->transmission_counter < 8) - delay = - GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, - rp->transmission_counter); - else if (rp->transmission_counter < 32) - delay = - GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, - 8 - + (1LL << (rp->transmission_counter - 8))); - else - delay = - GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, - 8 + (1LL << 24)); - delay.rel_value_us = - GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, - delay.rel_value_us + 1); - /* Add 0.01 to avg_delay to avoid division-by-zero later */ - avg_delay = (((avg_delay * (N - 1.0)) + delay.rel_value_us) / N) + 0.01; - - /* - * For the priority, we need to consider a few basic rules: - * 1) if we just started requesting (delay is small), we should - * virtually always have a priority of zero. - * 2) for requests with average latency, our priority should match - * the average priority observed on the network - * 3) even the longest-running requests should not be WAY out of - * the observed average (thus we bound by a factor of 2) - * 4) we add +1 to the observed average priority to avoid everyone - * staying put at zero (2 * 0 = 0...). - * - * Using the specific calculation below, we get: - * - * delay = 0 => priority = 0; - * delay = avg delay => priority = running-average-observed-priority; - * delay >> avg_delay => priority = 2 * running-average-observed-priority; - * - * which satisfies all of the rules above. - * - * Note: M_PI_4 = PI/4 = arctan(1) - */rp->priority = - round ((GSF_current_priorities - + 1.0) * atan (delay.rel_value_us / avg_delay)) / M_PI_4; - /* Note: usage of 'round' and 'atan' requires -lm */ - - if (rp->transmission_counter != 0) - delay.rel_value_us += TTL_DECREMENT * 1000; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Considering (re)transmission number %u in %s\n", - (unsigned int) rp->transmission_counter, - GNUNET_STRINGS_relative_time_to_string (delay, - GNUNET_YES)); - rp->earliest_transmission = GNUNET_TIME_relative_to_absolute (delay); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Earliest (re)transmission for `%s' in %us\n", - GNUNET_h2s (&prd->query), - rp->transmission_counter); - GNUNET_assert (rp->hn == NULL); - if (0 == GNUNET_TIME_absolute_get_remaining ( - rp->earliest_transmission).rel_value_us) - rp->hn = GNUNET_CONTAINER_heap_insert (pp->priority_heap, - rp, - rp->priority); - else - rp->hn = - GNUNET_CONTAINER_heap_insert (pp->delay_heap, - rp, - rp->earliest_transmission.abs_value_us); - GNUNET_assert (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_contains_value (pp->plan_map, - get_rp_key (rp), - rp)); -#undef N -} - - -/** - * Get the pending request with the highest TTL from the given plan. - * - * @param rp plan to investigate - * @return pending request with highest TTL - */ -struct GSF_PendingRequest * -get_latest (const struct GSF_RequestPlan *rp) -{ - struct GSF_PendingRequest *ret; - struct GSF_PendingRequestPlanBijection *bi; - const struct GSF_PendingRequestData *rprd; - const struct GSF_PendingRequestData *prd; - - bi = rp->pe_head; - if (NULL == bi) - return NULL; /* should never happen */ - ret = bi->pr; - rprd = GSF_pending_request_get_data_ (ret); - for (bi = bi->next_PE; NULL != bi; bi = bi->next_PE) - { - GNUNET_break (GNUNET_YES == - GSF_pending_request_test_active_ (bi->pr)); - prd = GSF_pending_request_get_data_ (bi->pr); - if (prd->ttl.abs_value_us > rprd->ttl.abs_value_us) - { - ret = bi->pr; - rprd = prd; - } - } - return ret; -} - - -/** - * Figure out when and how to transmit to the given peer. - * - * @param cls the `struct PeerPlan` - */ -static void -schedule_peer_transmission (void *cls) -{ - struct PeerPlan *pp = cls; - struct GSF_RequestPlan *rp; - struct GNUNET_TIME_Relative delay; - - if (NULL != pp->task) - { - pp->task = NULL; - } - else - { - GNUNET_assert (NULL != pp->env); - pp->env = NULL; - } - /* move ready requests to priority queue */ - while ((NULL != (rp = GNUNET_CONTAINER_heap_peek (pp->delay_heap))) && - (0 == GNUNET_TIME_absolute_get_remaining - (rp->earliest_transmission).rel_value_us)) - { - GNUNET_assert (rp == GNUNET_CONTAINER_heap_remove_root (pp->delay_heap)); - rp->hn = GNUNET_CONTAINER_heap_insert (pp->priority_heap, - rp, - rp->priority); - } - if (0 == GNUNET_CONTAINER_heap_get_size (pp->priority_heap)) - { - /* priority heap (still) empty, check for delay... */ - rp = GNUNET_CONTAINER_heap_peek (pp->delay_heap); - if (NULL == rp) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "No active requests for plan %p.\n", - pp); - return; /* both queues empty */ - } - delay = GNUNET_TIME_absolute_get_remaining (rp->earliest_transmission); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Sleeping for %s before retrying requests on plan %p.\n", - GNUNET_STRINGS_relative_time_to_string (delay, - GNUNET_YES), - pp); - GNUNET_STATISTICS_set (GSF_stats, - gettext_noop ("# delay heap timeout (ms)"), - delay.rel_value_us / 1000LL, GNUNET_NO); - - pp->task - = GNUNET_SCHEDULER_add_at (rp->earliest_transmission, - &schedule_peer_transmission, - pp); - return; - } -#if INSANE_STATISTICS - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# query plans executed"), - 1, - GNUNET_NO); -#endif - /* process from priority heap */ - rp = GNUNET_CONTAINER_heap_remove_root (pp->priority_heap); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Executing query plan %p\n", - rp); - GNUNET_assert (NULL != rp); - rp->hn = NULL; - rp->last_transmission = GNUNET_TIME_absolute_get (); - rp->transmission_counter++; - total_delay++; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Executing plan %p executed %u times, planning retransmission\n", - rp, - rp->transmission_counter); - GNUNET_assert (NULL == pp->env); - pp->env = GSF_pending_request_get_message_ (get_latest (rp)); - GNUNET_MQ_notify_sent (pp->env, - &schedule_peer_transmission, - pp); - GSF_peer_transmit_ (pp->cp, - GNUNET_YES, - rp->priority, - pp->env); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# query messages sent to other peers"), - 1, - GNUNET_NO); - plan (pp, - rp); -} - - -/** - * Closure for merge_pr(). - */ -struct MergeContext -{ - /** - * Request we are trying to merge. - */ - struct GSF_PendingRequest *pr; - - /** - * Set to #GNUNET_YES if we succeeded to merge. - */ - int merged; -}; - - -/** - * Iterator that checks if an equivalent request is already - * present for this peer. - * - * @param cls closure - * @param query the query - * @param element request plan stored at the node - * @return #GNUNET_YES if we should continue to iterate, - * #GNUNET_NO if not (merge success) - */ -static int -merge_pr (void *cls, - const struct GNUNET_HashCode *query, - void *element) -{ - struct MergeContext *mpr = cls; - struct GSF_RequestPlan *rp = element; - struct GSF_PendingRequestData *prd; - struct GSF_PendingRequestPlanBijection *bi; - struct GSF_PendingRequest *latest; - - GNUNET_break (GNUNET_YES == - GSF_pending_request_test_active_ (mpr->pr)); - if (GNUNET_OK != - GSF_pending_request_is_compatible_ (mpr->pr, - rp->pe_head->pr)) - return GNUNET_YES; - /* merge new request with existing request plan */ - bi = GNUNET_new (struct GSF_PendingRequestPlanBijection); - bi->rp = rp; - bi->pr = mpr->pr; - prd = GSF_pending_request_get_data_ (mpr->pr); - GNUNET_CONTAINER_MDLL_insert (PR, - prd->pr_head, - prd->pr_tail, - bi); - GNUNET_CONTAINER_MDLL_insert (PE, - rp->pe_head, - rp->pe_tail, - bi); - mpr->merged = GNUNET_YES; -#if INSANE_STATISTICS - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# requests merged"), - 1, - GNUNET_NO); -#endif - latest = get_latest (rp); - if (GSF_pending_request_get_data_ (latest)->ttl.abs_value_us < - prd->ttl.abs_value_us) - { -#if INSANE_STATISTICS - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# requests refreshed"), - 1, - GNUNET_NO); -#endif - rp->transmission_counter = 0; /* reset */ - } - return GNUNET_NO; -} - - -/** - * Create a new query plan entry. - * - * @param cp peer with the entry - * @param pr request with the entry - */ -void -GSF_plan_add_ (struct GSF_ConnectedPeer *cp, - struct GSF_PendingRequest *pr) -{ - const struct GNUNET_PeerIdentity *id; - struct PeerPlan *pp; - struct GSF_PendingRequestData *prd; - struct GSF_RequestPlan *rp; - struct GSF_PendingRequestPlanBijection *bi; - struct MergeContext mpc; - - GNUNET_assert (GNUNET_YES == - GSF_pending_request_test_active_ (pr)); - GNUNET_assert (NULL != cp); - id = GSF_connected_peer_get_identity2_ (cp); - pp = GNUNET_CONTAINER_multipeermap_get (plans, id); - if (NULL == pp) - { - pp = GNUNET_new (struct PeerPlan); - pp->plan_map = GNUNET_CONTAINER_multihashmap_create (128, GNUNET_NO); - pp->priority_heap = - GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MAX); - pp->delay_heap = - GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); - pp->cp = cp; - GNUNET_assert (GNUNET_OK == - GNUNET_CONTAINER_multipeermap_put (plans, - id, - pp, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - pp->task = GNUNET_SCHEDULER_add_now (&schedule_peer_transmission, - pp); - } - mpc.merged = GNUNET_NO; - mpc.pr = pr; - prd = GSF_pending_request_get_data_ (pr); - GNUNET_CONTAINER_multihashmap_get_multiple (pp->plan_map, - &prd->query, - &merge_pr, - &mpc); - if (GNUNET_NO != mpc.merged) - return; - plan_count++; - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# query plan entries"), - 1, - GNUNET_NO); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Planning transmission of query `%s' to peer `%s'\n", - GNUNET_h2s (&prd->query), - GNUNET_i2s (id)); - rp = GNUNET_new (struct GSF_RequestPlan); - bi = GNUNET_new (struct GSF_PendingRequestPlanBijection); - bi->rp = rp; - bi->pr = pr; - GNUNET_CONTAINER_MDLL_insert (PR, - prd->pr_head, - prd->pr_tail, - bi); - GNUNET_CONTAINER_MDLL_insert (PE, - rp->pe_head, - rp->pe_tail, - bi); - rp->pp = pp; - GNUNET_assert (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_put (pp->plan_map, - get_rp_key (rp), - rp, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE)); - plan (pp, - rp); -} - - -/** - * Notify the plan about a peer being no longer available; - * destroy all entries associated with this peer. - * - * @param cp connected peer - */ -void -GSF_plan_notify_peer_disconnect_ (const struct GSF_ConnectedPeer *cp) -{ - const struct GNUNET_PeerIdentity *id; - struct PeerPlan *pp; - struct GSF_RequestPlan *rp; - struct GSF_PendingRequestData *prd; - struct GSF_PendingRequestPlanBijection *bi; - - id = GSF_connected_peer_get_identity2_ (cp); - pp = GNUNET_CONTAINER_multipeermap_get (plans, id); - if (NULL == pp) - return; /* nothing was ever planned for this peer */ - GNUNET_assert (GNUNET_YES == - GNUNET_CONTAINER_multipeermap_remove (plans, id, - pp)); - if (NULL != pp->task) - { - GNUNET_SCHEDULER_cancel (pp->task); - pp->task = NULL; - } - while (NULL != (rp = GNUNET_CONTAINER_heap_remove_root (pp->priority_heap))) - { - GNUNET_break (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (pp->plan_map, - get_rp_key (rp), - rp)); - while (NULL != (bi = rp->pe_head)) - { - GNUNET_CONTAINER_MDLL_remove (PE, - rp->pe_head, - rp->pe_tail, - bi); - prd = GSF_pending_request_get_data_ (bi->pr); - GNUNET_CONTAINER_MDLL_remove (PR, - prd->pr_head, - prd->pr_tail, - bi); - GNUNET_free (bi); - } - plan_count--; - GNUNET_free (rp); - } - GNUNET_CONTAINER_heap_destroy (pp->priority_heap); - while (NULL != (rp = GNUNET_CONTAINER_heap_remove_root (pp->delay_heap))) - { - GNUNET_break (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (pp->plan_map, - get_rp_key (rp), - rp)); - while (NULL != (bi = rp->pe_head)) - { - prd = GSF_pending_request_get_data_ (bi->pr); - GNUNET_CONTAINER_MDLL_remove (PE, - rp->pe_head, - rp->pe_tail, - bi); - GNUNET_CONTAINER_MDLL_remove (PR, - prd->pr_head, - prd->pr_tail, - bi); - GNUNET_free (bi); - } - plan_count--; - GNUNET_free (rp); - } - GNUNET_STATISTICS_set (GSF_stats, - gettext_noop ("# query plan entries"), - plan_count, - GNUNET_NO); - GNUNET_CONTAINER_heap_destroy (pp->delay_heap); - GNUNET_CONTAINER_multihashmap_destroy (pp->plan_map); - GNUNET_free (pp); -} - - -/** - * Get the last transmission attempt time for the request plan list - * referenced by @a pr_head, that was sent to @a sender - * - * @param pr_head request plan reference list to check. - * @param sender the peer that we've sent the request to. - * @param result the timestamp to fill, set to #GNUNET_TIME_UNIT_FOREVER_ABS if never transmitted - * @return #GNUNET_YES if @a result was changed, #GNUNET_NO otherwise. - */ -int -GSF_request_plan_reference_get_last_transmission_ (struct - GSF_PendingRequestPlanBijection - *pr_head, - struct GSF_ConnectedPeer * - sender, - struct GNUNET_TIME_Absolute * - result) -{ - struct GSF_PendingRequestPlanBijection *bi; - - for (bi = pr_head; NULL != bi; bi = bi->next_PR) - { - if (bi->rp->pp->cp == sender) - { - if (0 == bi->rp->last_transmission.abs_value_us) - *result = GNUNET_TIME_UNIT_FOREVER_ABS; - else - *result = bi->rp->last_transmission; - return GNUNET_YES; - } - } - return GNUNET_NO; -} - - -/** - * Notify the plan about a request being done; destroy all entries - * associated with this request. - * - * @param pr request that is done - */ -void -GSF_plan_notify_request_done_ (struct GSF_PendingRequest *pr) -{ - struct GSF_RequestPlan *rp; - struct GSF_PendingRequestData *prd; - struct GSF_PendingRequestPlanBijection *bi; - - prd = GSF_pending_request_get_data_ (pr); - while (NULL != (bi = prd->pr_head)) - { - rp = bi->rp; - GNUNET_CONTAINER_MDLL_remove (PR, - prd->pr_head, - prd->pr_tail, - bi); - GNUNET_CONTAINER_MDLL_remove (PE, - rp->pe_head, - rp->pe_tail, - bi); - GNUNET_assert (bi->pr == pr); - if (NULL == rp->pe_head) - { - GNUNET_CONTAINER_heap_remove_node (rp->hn); - plan_count--; - GNUNET_break (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (rp->pp->plan_map, - &prd->query, - rp)); - GNUNET_free (rp); - } - GNUNET_free (bi); - } - GNUNET_STATISTICS_set (GSF_stats, - gettext_noop ("# query plan entries"), - plan_count, - GNUNET_NO); -} - - -/** - * Initialize plan subsystem. - */ -void -GSF_plan_init () -{ - plans = GNUNET_CONTAINER_multipeermap_create (256, - GNUNET_YES); -} - - -/** - * Shutdown plan subsystem. - */ -void -GSF_plan_done () -{ - GNUNET_assert (0 == GNUNET_CONTAINER_multipeermap_size (plans)); - GNUNET_CONTAINER_multipeermap_destroy (plans); -} - - -/* end of gnunet-service-fs_pe.h */ diff --git a/src/fs/gnunet-service-fs_pe.h b/src/fs/gnunet-service-fs_pe.h deleted file mode 100644 index d532b6b71..000000000 --- a/src/fs/gnunet-service-fs_pe.h +++ /dev/null @@ -1,95 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2011 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_pe.h - * @brief API to manage query plan - * @author Christian Grothoff - */ -#ifndef GNUNET_SERVICE_FS_PE_H -#define GNUNET_SERVICE_FS_PE_H - -#include "gnunet-service-fs.h" - - -/** - * Create a new query plan entry. - * - * @param cp peer with the entry - * @param pr request with the entry - */ -void -GSF_plan_add_ (struct GSF_ConnectedPeer *cp, - struct GSF_PendingRequest *pr); - - -/** - * Notify the plan about a peer being no longer available; - * destroy all entries associated with this peer. - * - * @param cp connected peer - */ -void -GSF_plan_notify_peer_disconnect_ (const struct GSF_ConnectedPeer *cp); - - -/** - * Notify the plan about a request being done; - * destroy all entries associated with this request. - * - * @param pr request that is done - */ -void -GSF_plan_notify_request_done_ (struct GSF_PendingRequest *pr); - -/** - * Get the last transmission attempt time for the request plan list - * referenced by 'rpr_head', that was sent to 'sender' - * - * @param pr_head request plan reference list to check. - * @param sender the peer that we've sent the request to. - * @param result the timestamp to fill. - * @return GNUNET_YES if 'result' was changed, GNUNET_NO otherwise. - */ -int -GSF_request_plan_reference_get_last_transmission_ (struct - GSF_PendingRequestPlanBijection - *pr_head, - struct GSF_ConnectedPeer * - sender, - struct GNUNET_TIME_Absolute * - result); - -/** - * Initialize plan subsystem. - */ -void -GSF_plan_init (void); - - -/** - * Shutdown plan subsystem. - */ -void -GSF_plan_done (void); - - -#endif -/* end of gnunet-service-fs_pe.h */ diff --git a/src/fs/gnunet-service-fs_pr.c b/src/fs/gnunet-service-fs_pr.c deleted file mode 100644 index f192c017d..000000000 --- a/src/fs/gnunet-service-fs_pr.c +++ /dev/null @@ -1,1887 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009-2013 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_pr.c - * @brief API to handle pending requests - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_load_lib.h" -#include "gnunet-service-fs.h" -#include "gnunet-service-fs_cp.h" -#include "gnunet-service-fs_indexing.h" -#include "gnunet-service-fs_pe.h" -#include "gnunet-service-fs_pr.h" -#include "gnunet-service-fs_cadet.h" - - -/** - * Desired replication level for GETs. - */ -#define DHT_GET_REPLICATION 5 - -/** - * Maximum size of the datastore queue for P2P operations. Needs to - * be large enough to queue #MAX_QUEUE_PER_PEER operations for roughly - * the number of active (connected) peers. - */ -#define MAX_DATASTORE_QUEUE (16 * MAX_QUEUE_PER_PEER) - -/** - * Bandwidth value of a 0-priority content (must be fairly high - * compared to query since content is typically significantly larger - * -- and more valuable since it can take many queries to get one - * piece of content). - */ -#define CONTENT_BANDWIDTH_VALUE 800 - -/** - * Hard limit on the number of results we may get from the datastore per query. - */ -#define MAX_RESULTS (100 * 1024) - -/** - * Collect an instance number of statistics? May cause excessive IPC. - */ -#define INSANE_STATISTICS GNUNET_NO - -/** - * If obtaining a block via cadet fails, how often do we retry it before - * giving up for good (and sticking to non-anonymous transfer)? - */ -#define CADET_RETRY_MAX 3 - - -/** - * An active request. - */ -struct GSF_PendingRequest -{ - /** - * Public data for the request. - */ - struct GSF_PendingRequestData public_data; - - /** - * Function to call if we encounter a reply. - */ - GSF_PendingRequestReplyHandler rh; - - /** - * Closure for @e rh - */ - void *rh_cls; - - /** - * Array of hash codes of replies we've already seen. - */ - struct GNUNET_HashCode *replies_seen; - - /** - * Block group for filtering replies we've already seen. - */ - struct GNUNET_BLOCK_Group *bg; - - /** - * Entry for this pending request in the expiration heap, or NULL. - */ - struct GNUNET_CONTAINER_HeapNode *hnode; - - /** - * Datastore queue entry for this request (or NULL for none). - */ - struct GNUNET_DATASTORE_QueueEntry *qe; - - /** - * DHT request handle for this request (or NULL for none). - */ - struct GNUNET_DHT_GetHandle *gh; - - /** - * Cadet request handle for this request (or NULL for none). - */ - struct GSF_CadetRequest *cadet_request; - - /** - * Function to call upon completion of the local get - * request, or NULL for none. - */ - GSF_LocalLookupContinuation llc_cont; - - /** - * Closure for @e llc_cont. - */ - void *llc_cont_cls; - - /** - * Last result from the local datastore lookup evaluation. - */ - enum GNUNET_BLOCK_ReplyEvaluationResult local_result; - - /** - * Identity of the peer that we should use for the 'sender' - * (recipient of the response) when forwarding (0 for none). - */ - GNUNET_PEER_Id sender_pid; - - /** - * Identity of the peer that we should never forward this query - * to since it originated this query (0 for none). - */ - GNUNET_PEER_Id origin_pid; - - /** - * Time we started the last datastore lookup. - */ - struct GNUNET_TIME_Absolute qe_start; - - /** - * Task that warns us if the local datastore lookup takes too long. - */ - struct GNUNET_SCHEDULER_Task *warn_task; - - /** - * Do we have a first UID yet? - */ - bool have_first_uid; - - /** - * Have we seen a NULL result yet? - */ - bool seen_null; - - /** - * Unique ID of the first result from the local datastore; - * used to terminate the loop. - */ - uint64_t first_uid; - - /** - * Result count. - */ - size_t result_count; - - /** - * How often have we retried this request via 'cadet'? - * (used to bound overall retries). - */ - unsigned int cadet_retry_count; - - /** - * Number of valid entries in the 'replies_seen' array. - */ - unsigned int replies_seen_count; - - /** - * Length of the 'replies_seen' array. - */ - unsigned int replies_seen_size; -}; - - -/** - * All pending requests, ordered by the query. Entries - * are of type 'struct GSF_PendingRequest*'. - */ -static struct GNUNET_CONTAINER_MultiHashMap *pr_map; - - -/** - * Datastore 'PUT' load tracking. - */ -static struct GNUNET_LOAD_Value *datastore_put_load; - - -/** - * Are we allowed to migrate content to this peer. - */ -static int active_to_migration; - - -/** - * Heap with the request that will expire next at the top. Contains - * pointers of type "struct PendingRequest*"; these will *also* be - * aliased from the "requests_by_peer" data structures and the - * "requests_by_query" table. Note that requests from our clients - * don't expire and are thus NOT in the "requests_by_expiration" - * (or the "requests_by_peer" tables). - */ -static struct GNUNET_CONTAINER_Heap *requests_by_expiration_heap; - - -/** - * Maximum number of requests (from other peers, overall) that we're - * willing to have pending at any given point in time. Can be changed - * via the configuration file (32k is just the default). - */ -static unsigned long long max_pending_requests = (32 * 1024); - - -/** - * Recalculate our bloom filter for filtering replies. This function - * will create a new bloom filter from scratch, so it should only be - * called if we have no bloomfilter at all (and hence can create a - * fresh one of minimal size without problems) OR if our peer is the - * initiator (in which case we may resize to larger than minimum size). - * - * @param type type of the request - * @param pr request for which the BF is to be recomputed - */ -static void -refresh_bloomfilter (enum GNUNET_BLOCK_Type type, - struct GSF_PendingRequest *pr) -{ - if (NULL != pr->bg) - { - GNUNET_BLOCK_group_destroy (pr->bg); - pr->bg = NULL; - } - if (GNUNET_BLOCK_TYPE_FS_UBLOCK != type) - return; /* no need */ - pr->bg = - GNUNET_BLOCK_group_create (GSF_block_ctx, - type, - NULL, - 0, - "seen-set-size", - pr->replies_seen_count, - NULL); - if (NULL == pr->bg) - return; - GNUNET_break (GNUNET_OK == - GNUNET_BLOCK_group_set_seen (pr->bg, - pr->replies_seen, - pr->replies_seen_count)); -} - - -struct GSF_PendingRequest * -GSF_pending_request_create_ (enum GSF_PendingRequestOptions options, - enum GNUNET_BLOCK_Type type, - const struct GNUNET_HashCode *query, - const struct GNUNET_PeerIdentity *target, - const char *bf_data, - size_t bf_size, - uint32_t anonymity_level, - uint32_t priority, - int32_t ttl, - GNUNET_PEER_Id sender_pid, - GNUNET_PEER_Id origin_pid, - const struct GNUNET_HashCode *replies_seen, - unsigned int replies_seen_count, - GSF_PendingRequestReplyHandler rh, - void *rh_cls) -{ - struct GSF_PendingRequest *pr; - struct GSF_PendingRequest *dpr; - size_t extra; - struct GNUNET_HashCode *eptr; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Creating request handle for `%s' of type %d\n", - GNUNET_h2s (query), - type); -#if INSANE_STATISTICS - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# Pending requests created"), - 1, - GNUNET_NO); -#endif - extra = 0; - if (NULL != target) - extra += sizeof(struct GNUNET_PeerIdentity); - pr = GNUNET_malloc (sizeof(struct GSF_PendingRequest) + extra); - pr->public_data.query = *query; - eptr = (struct GNUNET_HashCode *) &pr[1]; - if (NULL != target) - { - pr->public_data.target = (struct GNUNET_PeerIdentity *) eptr; - GNUNET_memcpy (eptr, target, sizeof(struct GNUNET_PeerIdentity)); - } - pr->public_data.anonymity_level = anonymity_level; - pr->public_data.priority = priority; - pr->public_data.original_priority = priority; - pr->public_data.options = options; - pr->public_data.type = type; - pr->public_data.start_time = GNUNET_TIME_absolute_get (); - pr->sender_pid = sender_pid; - pr->origin_pid = origin_pid; - pr->rh = rh; - pr->rh_cls = rh_cls; - GNUNET_assert ((sender_pid != 0) || (0 == (options & GSF_PRO_FORWARD_ONLY))); - if (ttl >= 0) - pr->public_data.ttl = GNUNET_TIME_relative_to_absolute ( - GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, (uint32_t) ttl)); - else - pr->public_data.ttl = GNUNET_TIME_absolute_subtract ( - pr->public_data.start_time, - GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, - (uint32_t) (-ttl))); - if (replies_seen_count > 0) - { - pr->replies_seen_size = replies_seen_count; - pr->replies_seen = - GNUNET_new_array (pr->replies_seen_size, struct GNUNET_HashCode); - GNUNET_memcpy (pr->replies_seen, - replies_seen, - replies_seen_count * sizeof(struct GNUNET_HashCode)); - pr->replies_seen_count = replies_seen_count; - } - if ((NULL != bf_data) && - (GNUNET_BLOCK_TYPE_FS_UBLOCK == pr->public_data.type)) - { - pr->bg = GNUNET_BLOCK_group_create (GSF_block_ctx, - pr->public_data.type, - bf_data, - bf_size, - "seen-set-size", - 0, - NULL); - } - else if ((replies_seen_count > 0) && - (0 != (options & GSF_PRO_BLOOMFILTER_FULL_REFRESH))) - { - refresh_bloomfilter (pr->public_data.type, pr); - } - GNUNET_CONTAINER_multihashmap_put (pr_map, - &pr->public_data.query, - pr, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); - if (0 == (options & GSF_PRO_REQUEST_NEVER_EXPIRES)) - { - pr->hnode = GNUNET_CONTAINER_heap_insert (requests_by_expiration_heap, - pr, - pr->public_data.ttl.abs_value_us); - /* make sure we don't track too many requests */ - while (GNUNET_CONTAINER_heap_get_size (requests_by_expiration_heap) > - max_pending_requests) - { - dpr = GNUNET_CONTAINER_heap_peek (requests_by_expiration_heap); - GNUNET_assert (NULL != dpr); - if (pr == dpr) - break; /* let the request live briefly... */ - if (NULL != dpr->rh) - dpr->rh (dpr->rh_cls, - GNUNET_BLOCK_REPLY_TYPE_NOT_SUPPORTED, - dpr, - UINT32_MAX, - GNUNET_TIME_UNIT_FOREVER_ABS, - GNUNET_TIME_UNIT_FOREVER_ABS, - GNUNET_BLOCK_TYPE_ANY, - NULL, - 0); - GSF_pending_request_cancel_ (dpr, GNUNET_YES); - } - } - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# Pending requests active"), - 1, - GNUNET_NO); - return pr; -} - - -/** - * Obtain the public data associated with a pending request - * - * @param pr pending request - * @return associated public data - */ -struct GSF_PendingRequestData * -GSF_pending_request_get_data_ (struct GSF_PendingRequest *pr) -{ - return &pr->public_data; -} - - -/** - * Test if two pending requests are compatible (would generate - * the same query modulo filters and should thus be processed - * jointly). - * - * @param pra a pending request - * @param prb another pending request - * @return #GNUNET_OK if the requests are compatible - */ -int -GSF_pending_request_is_compatible_ (struct GSF_PendingRequest *pra, - struct GSF_PendingRequest *prb) -{ - if ((pra->public_data.type != prb->public_data.type) || - (0 != memcmp (&pra->public_data.query, - &prb->public_data.query, - sizeof(struct GNUNET_HashCode)))) - return GNUNET_NO; - return GNUNET_OK; -} - - -void -GSF_pending_request_update_ (struct GSF_PendingRequest *pr, - const struct GNUNET_HashCode *replies_seen, - unsigned int replies_seen_count) -{ - if (replies_seen_count + pr->replies_seen_count < pr->replies_seen_count) - return; /* integer overflow */ - if (0 != (pr->public_data.options & GSF_PRO_BLOOMFILTER_FULL_REFRESH)) - { - /* we're responsible for the BF, full refresh */ - if (replies_seen_count + pr->replies_seen_count > pr->replies_seen_size) - GNUNET_array_grow (pr->replies_seen, - pr->replies_seen_size, - replies_seen_count + pr->replies_seen_count); - GNUNET_memcpy (&pr->replies_seen[pr->replies_seen_count], - replies_seen, - sizeof(struct GNUNET_HashCode) * replies_seen_count); - pr->replies_seen_count += replies_seen_count; - refresh_bloomfilter (pr->public_data.type, pr); - } - else - { - if (NULL == pr->bg) - { - /* we're not the initiator, but the initiator did not give us - * any bloom-filter, so we need to create one on-the-fly */ - refresh_bloomfilter (pr->public_data.type, pr); - } - else - { - GNUNET_break (GNUNET_OK == - GNUNET_BLOCK_group_set_seen (pr->bg, - replies_seen, - pr->replies_seen_count)); - } - } - if (NULL != pr->gh) - GNUNET_DHT_get_filter_known_results (pr->gh, - replies_seen_count, - replies_seen); -} - - -/** - * Generate the message corresponding to the given pending request for - * transmission to other peers. - * - * @param pr request to generate the message for - * @return envelope with the request message - */ -struct GNUNET_MQ_Envelope * -GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr) -{ - struct GNUNET_MQ_Envelope *env; - struct GetMessage *gm; - struct GNUNET_PeerIdentity *ext; - unsigned int k; - uint32_t bm; - uint32_t prio; - size_t bf_size; - struct GNUNET_TIME_Absolute now; - int64_t ttl; - int do_route; - void *bf_data; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Building request message for `%s' of type %d\n", - GNUNET_h2s (&pr->public_data.query), - pr->public_data.type); - k = 0; - bm = 0; - do_route = (0 == (pr->public_data.options & GSF_PRO_FORWARD_ONLY)); - if ((! do_route) && (pr->sender_pid == 0)) - { - GNUNET_break (0); - do_route = GNUNET_YES; - } - if (! do_route) - { - bm |= GET_MESSAGE_BIT_RETURN_TO; - k++; - } - if (NULL != pr->public_data.target) - { - bm |= GET_MESSAGE_BIT_TRANSMIT_TO; - k++; - } - if (GNUNET_OK != - GNUNET_BLOCK_group_serialize (pr->bg, - &bf_data, - &bf_size)) - { - bf_size = 0; - bf_data = NULL; - } - env = GNUNET_MQ_msg_extra (gm, - bf_size + k * sizeof(struct GNUNET_PeerIdentity), - GNUNET_MESSAGE_TYPE_FS_GET); - gm->type = htonl (pr->public_data.type); - if (do_route) - prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, - pr->public_data.priority + 1); - else - prio = 0; - pr->public_data.priority -= prio; - pr->public_data.num_transmissions++; - pr->public_data.respect_offered += prio; - gm->priority = htonl (prio); - now = GNUNET_TIME_absolute_get (); - ttl = (int64_t) (pr->public_data.ttl.abs_value_us - now.abs_value_us); - gm->ttl = htonl (ttl / 1000LL / 1000LL); - gm->reserved = htonl (0); - gm->hash_bitmap = htonl (bm); - gm->query = pr->public_data.query; - ext = (struct GNUNET_PeerIdentity *) &gm[1]; - k = 0; - if (! do_route) - GNUNET_PEER_resolve (pr->sender_pid, &ext[k++]); - if (NULL != pr->public_data.target) - ext[k++] = *pr->public_data.target; - GNUNET_memcpy (&ext[k], bf_data, bf_size); - GNUNET_free (bf_data); - return env; -} - - -/** - * Iterator to free pending requests. - * - * @param cls closure, unused - * @param key current key code - * @param value value in the hash map (pending request) - * @return #GNUNET_YES (we should continue to iterate) - */ -static int -clean_request (void *cls, const struct GNUNET_HashCode *key, void *value) -{ - struct GSF_PendingRequest *pr = value; - GSF_LocalLookupContinuation cont; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Cleaning up pending request for `%s'.\n", - GNUNET_h2s (key)); - if (NULL != pr->cadet_request) - { - pr->cadet_retry_count = CADET_RETRY_MAX; - GSF_cadet_query_cancel (pr->cadet_request); - pr->cadet_request = NULL; - } - if (NULL != (cont = pr->llc_cont)) - { - pr->llc_cont = NULL; - cont (pr->llc_cont_cls, - pr, - pr->local_result); - } - GSF_plan_notify_request_done_ (pr); - GNUNET_free (pr->replies_seen); - GNUNET_BLOCK_group_destroy (pr->bg); - pr->bg = NULL; - GNUNET_PEER_change_rc (pr->sender_pid, -1); - pr->sender_pid = 0; - GNUNET_PEER_change_rc (pr->origin_pid, -1); - pr->origin_pid = 0; - if (NULL != pr->hnode) - { - GNUNET_CONTAINER_heap_remove_node (pr->hnode); - pr->hnode = NULL; - } - if (NULL != pr->qe) - { - GNUNET_DATASTORE_cancel (pr->qe); - pr->qe = NULL; - } - if (NULL != pr->gh) - { - GNUNET_DHT_get_stop (pr->gh); - pr->gh = NULL; - } - if (NULL != pr->warn_task) - { - GNUNET_SCHEDULER_cancel (pr->warn_task); - pr->warn_task = NULL; - } - GNUNET_assert ( - GNUNET_OK == - GNUNET_CONTAINER_multihashmap_remove (pr_map, &pr->public_data.query, pr)); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# Pending requests active"), - -1, - GNUNET_NO); - GNUNET_free (pr); - return GNUNET_YES; -} - - -/** - * Explicitly cancel a pending request. - * - * @param pr request to cancel - * @param full_cleanup fully purge the request - */ -void -GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr, int full_cleanup) -{ - GSF_LocalLookupContinuation cont; - - if (NULL == pr_map) - return; /* already cleaned up! */ - if (GNUNET_NO == full_cleanup) - { - /* make request inactive (we're no longer interested in more results), - * but do NOT remove from our data-structures, we still need it there - * to prevent the request from looping */ - pr->rh = NULL; - if (NULL != pr->cadet_request) - { - pr->cadet_retry_count = CADET_RETRY_MAX; - GSF_cadet_query_cancel (pr->cadet_request); - pr->cadet_request = NULL; - } - if (NULL != (cont = pr->llc_cont)) - { - pr->llc_cont = NULL; - cont (pr->llc_cont_cls, - pr, - pr->local_result); - } - GSF_plan_notify_request_done_ (pr); - if (NULL != pr->qe) - { - GNUNET_DATASTORE_cancel (pr->qe); - pr->qe = NULL; - } - if (NULL != pr->gh) - { - GNUNET_DHT_get_stop (pr->gh); - pr->gh = NULL; - } - if (NULL != pr->warn_task) - { - GNUNET_SCHEDULER_cancel (pr->warn_task); - pr->warn_task = NULL; - } - return; - } - GNUNET_assert (GNUNET_YES == - clean_request (NULL, &pr->public_data.query, pr)); -} - - -void -GSF_iterate_pending_requests_ (GSF_PendingRequestIterator it, void *cls) -{ - GNUNET_CONTAINER_multihashmap_iterate ( - pr_map, - (GNUNET_CONTAINER_MultiHashMapIteratorCallback) it, - cls); -} - - -/** - * Closure for process_reply() function. - */ -struct ProcessReplyClosure -{ - /** - * The data for the reply. - */ - const void *data; - - /** - * Who gave us this reply? NULL for local host (or DHT) - */ - struct GSF_ConnectedPeer *sender; - - /** - * When the reply expires. - */ - struct GNUNET_TIME_Absolute expiration; - - /** - * Size of data. - */ - size_t size; - - /** - * Type of the block. - */ - enum GNUNET_BLOCK_Type type; - - /** - * How much was this reply worth to us? - */ - uint32_t priority; - - /** - * Anonymity requirements for this reply. - */ - uint32_t anonymity_level; - - /** - * Evaluation result (returned). - */ - enum GNUNET_BLOCK_ReplyEvaluationResult eval; - - /** - * Did we find a matching request? - */ - int request_found; -}; - - -/** - * Update the performance data for the sender (if any) since - * the sender successfully answered one of our queries. - * - * @param prq information about the sender - * @param pr request that was satisfied - */ -static void -update_request_performance_data (struct ProcessReplyClosure *prq, - struct GSF_PendingRequest *pr) -{ - if (prq->sender == NULL) - return; - GSF_peer_update_performance_ (prq->sender, - pr->public_data.start_time, - prq->priority); -} - - -/** - * We have received a reply; handle it! - * - * @param cls response (a `struct ProcessReplyClosure`) - * @param key our query - * @param value value in the hash map (info about the query) - * @return #GNUNET_YES (we should continue to iterate) - */ -static enum GNUNET_GenericReturnValue -process_reply (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct ProcessReplyClosure *prq = cls; - struct GSF_PendingRequest *pr = value; - struct GNUNET_HashCode chash; - struct GNUNET_TIME_Absolute last_transmission; - - if (NULL == pr->rh) - return GNUNET_YES; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Matched result (type %u) for query `%s' with pending request\n", - (unsigned int) prq->type, - GNUNET_h2s (key)); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# replies received and matched"), - 1, - GNUNET_NO); - prq->eval = GNUNET_BLOCK_check_reply (GSF_block_ctx, - prq->type, - pr->bg, - key, - NULL, 0, - prq->data, - prq->size); - switch (prq->eval) - { - case GNUNET_BLOCK_REPLY_OK_MORE: - update_request_performance_data (prq, pr); - break; - case GNUNET_BLOCK_REPLY_OK_LAST: - /* short cut: stop processing early, no BF-update, etc. */ - update_request_performance_data (prq, pr); - GNUNET_LOAD_update (GSF_rt_entry_lifetime, - GNUNET_TIME_absolute_get_duration ( - pr->public_data.start_time) - .rel_value_us); - if (GNUNET_YES != - GSF_request_plan_reference_get_last_transmission_ (pr->public_data - .pr_head, - prq->sender, - &last_transmission)) - last_transmission = GNUNET_TIME_UNIT_FOREVER_ABS; - /* pass on to other peers / local clients */ - pr->rh (pr->rh_cls, - prq->eval, - pr, - prq->anonymity_level, - prq->expiration, - last_transmission, - prq->type, - prq->data, - prq->size); - return GNUNET_YES; - case GNUNET_BLOCK_REPLY_OK_DUPLICATE: -#if INSANE_STATISTICS - GNUNET_STATISTICS_update (GSF_stats, - "# duplicate replies discarded (bloomfilter)", - 1, - GNUNET_NO); -#endif - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Duplicate response, discarding.\n"); - return GNUNET_YES; /* duplicate */ - case GNUNET_BLOCK_REPLY_IRRELEVANT: - GNUNET_STATISTICS_update (GSF_stats, - "# irrelevant replies discarded", - 1, - GNUNET_NO); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Irrelevant response, ignoring.\n"); - return GNUNET_YES; - case GNUNET_BLOCK_REPLY_TYPE_NOT_SUPPORTED: - GNUNET_break (0); /* bad installation? */ - return GNUNET_NO; - } - /* update bloomfilter */ - GNUNET_CRYPTO_hash (prq->data, - prq->size, - &chash); - GSF_pending_request_update_ (pr, - &chash, - 1); - if (NULL == prq->sender) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Found result for query `%s' in local datastore\n", - GNUNET_h2s (key)); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# results found locally"), - 1, - GNUNET_NO); - } - else - { - GSF_dht_lookup_ (pr); - } - prq->priority += pr->public_data.original_priority; - pr->public_data.priority = 0; - pr->public_data.original_priority = 0; - pr->public_data.results_found++; - prq->request_found = GNUNET_YES; - /* finally, pass on to other peer / local client */ - if (! GSF_request_plan_reference_get_last_transmission_ (pr->public_data - .pr_head, - prq->sender, - &last_transmission)) - last_transmission = GNUNET_TIME_UNIT_FOREVER_ABS; - pr->rh (pr->rh_cls, - prq->eval, - pr, - prq->anonymity_level, - prq->expiration, - last_transmission, - prq->type, - prq->data, - prq->size); - return GNUNET_YES; -} - - -/** - * Context for put_migration_continuation(). - */ -struct PutMigrationContext -{ - /** - * Start time for the operation. - */ - struct GNUNET_TIME_Absolute start; - - /** - * Request origin. - */ - struct GNUNET_PeerIdentity origin; - - /** - * #GNUNET_YES if we had a matching request for this block, - * #GNUNET_NO if not. - */ - int requested; -}; - - -/** - * Continuation called to notify client about result of the - * operation. - * - * @param cls closure - * @param success #GNUNET_SYSERR on failure - * @param min_expiration minimum expiration time required for content to be stored - * @param msg NULL on success, otherwise an error message - */ -static void -put_migration_continuation (void *cls, - int success, - struct GNUNET_TIME_Absolute min_expiration, - const char *msg) -{ - struct PutMigrationContext *pmc = cls; - struct GSF_ConnectedPeer *cp; - struct GNUNET_TIME_Relative mig_pause; - struct GSF_PeerPerformanceData *ppd; - - if (NULL != datastore_put_load) - { - if (GNUNET_SYSERR != success) - { - GNUNET_LOAD_update (datastore_put_load, - GNUNET_TIME_absolute_get_duration (pmc->start) - .rel_value_us); - } - else - { - /* on queue failure / timeout, increase the put load dramatically */ - GNUNET_LOAD_update (datastore_put_load, - GNUNET_TIME_UNIT_MINUTES.rel_value_us); - } - } - cp = GSF_peer_get_ (&pmc->origin); - if (GNUNET_OK == success) - { - if (NULL != cp) - { - ppd = GSF_get_peer_performance_data_ (cp); - ppd->migration_delay.rel_value_us /= 2; - } - GNUNET_free (pmc); - return; - } - if ((GNUNET_NO == success) && (GNUNET_NO == pmc->requested) && (NULL != cp)) - { - ppd = GSF_get_peer_performance_data_ (cp); - if (min_expiration.abs_value_us > 0) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Asking to stop migration for %s because datastore is full\n", - GNUNET_STRINGS_relative_time_to_string ( - GNUNET_TIME_absolute_get_remaining (min_expiration), - GNUNET_YES)); - GSF_block_peer_migration_ (cp, min_expiration); - } - else - { - ppd->migration_delay = GNUNET_TIME_relative_max (GNUNET_TIME_UNIT_SECONDS, - ppd->migration_delay); - ppd->migration_delay = - GNUNET_TIME_relative_min (GNUNET_TIME_UNIT_HOURS, ppd->migration_delay); - mig_pause.rel_value_us = - GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, - ppd->migration_delay.rel_value_us); - ppd->migration_delay = - GNUNET_TIME_relative_saturating_multiply (ppd->migration_delay, 2); - GNUNET_log ( - GNUNET_ERROR_TYPE_DEBUG, - "Replicated content already exists locally, asking to stop migration for %s\n", - GNUNET_STRINGS_relative_time_to_string (mig_pause, GNUNET_YES)); - GSF_block_peer_migration_ (cp, - GNUNET_TIME_relative_to_absolute (mig_pause)); - } - } - GNUNET_free (pmc); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# Datastore `PUT' failures"), - 1, - GNUNET_NO); -} - - -/** - * Test if the DATABASE (PUT) load on this peer is too high - * to even consider processing the query at - * all. - * - * @param priority the priority of the item - * @return #GNUNET_YES if the load is too high to do anything (load high) - * #GNUNET_NO to process normally (load normal or low) - */ -static int -test_put_load_too_high (uint32_t priority) -{ - double ld; - - if (NULL == datastore_put_load) - return GNUNET_NO; - if (GNUNET_LOAD_get_average (datastore_put_load) < 50) - return GNUNET_NO; /* very fast */ - ld = GNUNET_LOAD_get_load (datastore_put_load); - if (ld < 2.0 * (1 + priority)) - return GNUNET_NO; - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# storage requests dropped due to high load"), - 1, - GNUNET_NO); - return GNUNET_YES; -} - - -/** - * Iterator called on each result obtained for a DHT - * operation that expects a reply - * - * @param cls closure - * @param exp when will this value expire - * @param key key of the result - * @param trunc_peer truncated peer, NULL for none - * @param get_path peers on reply path (or NULL if not recorded) - * @param get_path_length number of entries in @a get_path - * @param put_path peers on the PUT path (or NULL if not recorded) - * @param put_path_length number of entries in @a get_path - * @param type type of the result - * @param size number of bytes in @a data - * @param data pointer to the result data - */ -static void -handle_dht_reply (void *cls, - struct GNUNET_TIME_Absolute exp, - const struct GNUNET_HashCode *key, - const struct GNUNET_PeerIdentity *trunc_peer, - const struct GNUNET_DHT_PathElement *get_path, - unsigned int get_path_length, - const struct GNUNET_DHT_PathElement *put_path, - unsigned int put_path_length, - enum GNUNET_BLOCK_Type type, - size_t size, - const void *data) -{ - struct GSF_PendingRequest *pr = cls; - struct ProcessReplyClosure prq; - struct PutMigrationContext *pmc; - - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# Replies received from DHT"), - 1, - GNUNET_NO); - memset (&prq, 0, sizeof(prq)); - prq.data = data; - prq.expiration = exp; - /* do not allow migrated content to live longer than 1 year */ - prq.expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute ( - GNUNET_TIME_UNIT_YEARS), - prq.expiration); - prq.size = size; - prq.type = type; - process_reply (&prq, - key, - pr); - if ((GNUNET_YES == active_to_migration) && - (GNUNET_NO == test_put_load_too_high (prq.priority))) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Replicating result for query `%s' with priority %u\n", - GNUNET_h2s (key), - prq.priority); - pmc = GNUNET_new (struct PutMigrationContext); - pmc->start = GNUNET_TIME_absolute_get (); - pmc->requested = GNUNET_YES; - if (NULL == GNUNET_DATASTORE_put (GSF_dsh, - 0, - key, - size, - data, - type, - prq.priority, - 1 /* anonymity */, - 0 /* replication */, - exp, - 1 + prq.priority, - MAX_DATASTORE_QUEUE, - &put_migration_continuation, - pmc)) - { - put_migration_continuation (pmc, - GNUNET_SYSERR, - GNUNET_TIME_UNIT_ZERO_ABS, - NULL); - } - } -} - - -/** - * Consider looking up the data in the DHT (anonymity-level permitting). - * - * @param pr the pending request to process - */ -void -GSF_dht_lookup_ (struct GSF_PendingRequest *pr) -{ - const void *xquery; - size_t xquery_size; - struct GNUNET_PeerIdentity pi; - char buf[sizeof(struct GNUNET_HashCode) * 2] GNUNET_ALIGN; - - if (0 != pr->public_data.anonymity_level) - return; - if (NULL != pr->gh) - { - GNUNET_DHT_get_stop (pr->gh); - pr->gh = NULL; - } - xquery = NULL; - xquery_size = 0; - if (0 != (pr->public_data.options & GSF_PRO_FORWARD_ONLY)) - { - GNUNET_assert (0 != pr->sender_pid); - GNUNET_PEER_resolve (pr->sender_pid, &pi); - GNUNET_memcpy (&buf[xquery_size], &pi, sizeof(struct GNUNET_PeerIdentity)); - xquery_size += sizeof(struct GNUNET_PeerIdentity); - } - pr->gh = GNUNET_DHT_get_start (GSF_dht, - pr->public_data.type, - &pr->public_data.query, - DHT_GET_REPLICATION, - GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE, - xquery, - xquery_size, - &handle_dht_reply, - pr); - if ((NULL != pr->gh) && (0 != pr->replies_seen_count)) - GNUNET_DHT_get_filter_known_results (pr->gh, - pr->replies_seen_count, - pr->replies_seen); -} - - -/** - * Function called with a reply from the cadet. - * - * @param cls the pending request struct - * @param type type of the block, ANY on error - * @param expiration expiration time for the block - * @param data_size number of bytes in @a data, 0 on error - * @param data reply block data, NULL on error - */ -static void -cadet_reply_proc (void *cls, - enum GNUNET_BLOCK_Type type, - struct GNUNET_TIME_Absolute expiration, - size_t data_size, - const void *data) -{ - struct GSF_PendingRequest *pr = cls; - struct ProcessReplyClosure prq; - struct GNUNET_HashCode query; - - pr->cadet_request = NULL; - if (GNUNET_OK != - GNUNET_BLOCK_check_block (GSF_block_ctx, - type, - data, - data_size)) - { - GNUNET_break_op (0); - return; - } - if (GNUNET_BLOCK_TYPE_ANY == type) - { - GNUNET_break (NULL == data); - GNUNET_break (0 == data_size); - pr->cadet_retry_count++; - if (pr->cadet_retry_count >= CADET_RETRY_MAX) - return; /* give up on cadet */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Error retrieiving block via cadet\n"); - /* retry -- without delay, as this is non-anonymous - and cadet/cadet connect will take some time anyway */ - pr->cadet_request = GSF_cadet_query (pr->public_data.target, - &pr->public_data.query, - pr->public_data.type, - &cadet_reply_proc, - pr); - return; - } - if (GNUNET_YES != - GNUNET_BLOCK_get_key (GSF_block_ctx, - type, - data, - data_size, - &query)) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Failed to derive key for block of type %d\n", - (int) type); - GNUNET_break_op (0); - return; - } - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# Replies received from CADET"), - 1, - GNUNET_NO); - memset (&prq, 0, sizeof(prq)); - prq.data = data; - prq.expiration = expiration; - /* do not allow migrated content to live longer than 1 year */ - prq.expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute ( - GNUNET_TIME_UNIT_YEARS), - prq.expiration); - prq.size = data_size; - prq.type = type; - process_reply (&prq, - &query, - pr); -} - - -/** - * Consider downloading via cadet (if possible) - * - * @param pr the pending request to process - */ -void -GSF_cadet_lookup_ (struct GSF_PendingRequest *pr) -{ - if (0 != pr->public_data.anonymity_level) - return; - if (0 == pr->public_data.target) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Cannot do cadet-based download, target peer not known\n"); - return; - } - if (NULL != pr->cadet_request) - return; - pr->cadet_request = GSF_cadet_query (pr->public_data.target, - &pr->public_data.query, - pr->public_data.type, - &cadet_reply_proc, - pr); -} - - -/** - * Task that issues a warning if the datastore lookup takes too long. - * - * @param cls the `struct GSF_PendingRequest` - */ -static void -warn_delay_task (void *cls) -{ - struct GSF_PendingRequest *pr = cls; - - GNUNET_log (GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK, - _ ("Datastore lookup already took %s!\n"), - GNUNET_STRINGS_relative_time_to_string ( - GNUNET_TIME_absolute_get_duration (pr->qe_start), - GNUNET_YES)); - pr->warn_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, - &warn_delay_task, - pr); -} - - -/** - * Task that issues a warning if the datastore lookup takes too long. - * - * @param cls the `struct GSF_PendingRequest` - */ -static void -odc_warn_delay_task (void *cls) -{ - struct GSF_PendingRequest *pr = cls; - - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _ ("On-demand lookup already took %s!\n"), - GNUNET_STRINGS_relative_time_to_string ( - GNUNET_TIME_absolute_get_duration (pr->qe_start), - GNUNET_YES)); - pr->warn_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, - &odc_warn_delay_task, - pr); -} - - -/* Call our continuation (if we have any) */ -static void -call_continuation (struct GSF_PendingRequest *pr) -{ - GSF_LocalLookupContinuation cont = pr->llc_cont; - - GNUNET_assert (NULL == pr->qe); - if (NULL != pr->warn_task) - { - GNUNET_SCHEDULER_cancel (pr->warn_task); - pr->warn_task = NULL; - } - if (NULL == cont) - return; /* no continuation */ - pr->llc_cont = NULL; - if (0 != (GSF_PRO_LOCAL_ONLY & pr->public_data.options)) - { - if (GNUNET_BLOCK_REPLY_OK_LAST != pr->local_result) - { - /* Signal that we are done and that there won't be any - additional results to allow client to clean up state. */ - pr->rh (pr->rh_cls, - GNUNET_BLOCK_REPLY_OK_LAST, - pr, - UINT32_MAX, - GNUNET_TIME_UNIT_ZERO_ABS, - GNUNET_TIME_UNIT_FOREVER_ABS, - GNUNET_BLOCK_TYPE_ANY, - NULL, - 0); - } - /* Finally, call our continuation to signal that we are - done with local processing of this request; i.e. to - start reading again from the client. */ - cont (pr->llc_cont_cls, - NULL, - GNUNET_BLOCK_REPLY_OK_LAST); - return; - } - - cont (pr->llc_cont_cls, - pr, - pr->local_result); -} - - -/* Update stats and call continuation */ -static void -no_more_local_results (struct GSF_PendingRequest *pr) -{ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, - "No further local responses available.\n"); -#if INSANE_STATISTICS - if ((GNUNET_BLOCK_TYPE_FS_DBLOCK == pr->public_data.type) || - (GNUNET_BLOCK_TYPE_FS_IBLOCK == pr->public_data.type)) - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# requested DBLOCK or IBLOCK not found"), - 1, - GNUNET_NO); -#endif - call_continuation (pr); -} - - -/* forward declaration */ -static void -process_local_reply (void *cls, - const struct GNUNET_HashCode *key, - size_t size, - const void *data, - enum GNUNET_BLOCK_Type type, - uint32_t priority, - uint32_t anonymity, - uint32_t replication, - struct GNUNET_TIME_Absolute expiration, - uint64_t uid); - - -/* Start a local query */ -static void -start_local_query (struct GSF_PendingRequest *pr, - uint64_t next_uid, - bool random) -{ - pr->qe_start = GNUNET_TIME_absolute_get (); - pr->warn_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, - &warn_delay_task, - pr); - pr->qe = GNUNET_DATASTORE_get_key (GSF_dsh, - next_uid, - random, - &pr->public_data.query, - pr->public_data.type == - GNUNET_BLOCK_TYPE_FS_DBLOCK - ? GNUNET_BLOCK_TYPE_ANY - : pr->public_data.type, - (0 != (GSF_PRO_PRIORITY_UNLIMITED - & pr->public_data.options)) - ? UINT_MAX - : 1 - /* queue priority */, - (0 != (GSF_PRO_PRIORITY_UNLIMITED - & pr->public_data.options)) - ? UINT_MAX - : GSF_datastore_queue_size - /* max queue size */, - &process_local_reply, - pr); - if (NULL != pr->qe) - return; - GNUNET_log ( - GNUNET_ERROR_TYPE_DEBUG, - "ERROR Requesting `%s' of type %d with next_uid %llu from datastore.\n", - GNUNET_h2s (&pr->public_data.query), - pr->public_data.type, - (unsigned long long) next_uid); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# Datastore lookups concluded (error queueing)"), - 1, - GNUNET_NO); - call_continuation (pr); -} - - -/** - * We're processing (local) results for a search request - * from another peer. Pass applicable results to the - * peer and if we are done either clean up (operation - * complete) or forward to other peers (more results possible). - * - * @param cls our closure (`struct GSF_PendingRequest *`) - * @param key key for the content - * @param size number of bytes in @a data - * @param data content stored - * @param type type of the content - * @param priority priority of the content - * @param anonymity anonymity-level for the content - * @param replication replication-level for the content - * @param expiration expiration time for the content - * @param uid unique identifier for the datum; - * maybe 0 if no unique identifier is available - */ -static void -process_local_reply (void *cls, - const struct GNUNET_HashCode *key, - size_t size, - const void *data, - enum GNUNET_BLOCK_Type type, - uint32_t priority, - uint32_t anonymity, - uint32_t replication, - struct GNUNET_TIME_Absolute expiration, - uint64_t uid) -{ - struct GSF_PendingRequest *pr = cls; - struct ProcessReplyClosure prq; - struct GNUNET_HashCode query; - unsigned int old_rf; - - GNUNET_SCHEDULER_cancel (pr->warn_task); - pr->warn_task = NULL; - if (NULL == pr->qe) - goto called_from_on_demand; - pr->qe = NULL; - if ( - (NULL == key) && pr->seen_null && - ! pr->have_first_uid) /* We have hit the end for the 2nd time with no results */ - { - /* No results */ -#if INSANE_STATISTICS - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# Datastore lookups concluded (no results)"), - 1, - GNUNET_NO); -#endif - no_more_local_results (pr); - return; - } - if (((NULL == key) && - pr->seen_null) || /* We have hit the end for the 2nd time OR */ - (pr->seen_null && pr->have_first_uid && - (uid >= pr->first_uid))) /* We have hit the end and past first UID */ - { - /* Seen all results */ - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# Datastore lookups concluded (seen all)"), - 1, - GNUNET_NO); - no_more_local_results (pr); - return; - } - if (NULL == key) - { - GNUNET_assert (! pr->seen_null); - pr->seen_null = true; - start_local_query (pr, 0 /* next_uid */, false /* random */); - return; - } - if (! pr->have_first_uid) - { - pr->first_uid = uid; - pr->have_first_uid = true; - } - pr->result_count++; - if (pr->result_count > MAX_RESULTS) - { - GNUNET_STATISTICS_update ( - GSF_stats, - gettext_noop ("# Datastore lookups aborted (more than MAX_RESULTS)"), - 1, - GNUNET_NO); - no_more_local_results (pr); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Received reply for `%s' of type %d with UID %llu from datastore.\n", - GNUNET_h2s (key), - type, - (unsigned long long) uid); - if (GNUNET_BLOCK_TYPE_FS_ONDEMAND == type) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Found ONDEMAND block, performing on-demand encoding\n"); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# on-demand blocks matched requests"), - 1, - GNUNET_NO); - pr->qe_start = GNUNET_TIME_absolute_get (); - pr->warn_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, - &odc_warn_delay_task, - pr); - if (GNUNET_OK == GNUNET_FS_handle_on_demand_block (key, - size, - data, - type, - priority, - anonymity, - replication, - expiration, - uid, - &process_local_reply, - pr)) - { - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# on-demand lookups performed successfully"), - 1, - GNUNET_NO); - return; /* we're done */ - } - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# on-demand lookups failed"), - 1, - GNUNET_NO); - GNUNET_SCHEDULER_cancel (pr->warn_task); - start_local_query (pr, uid + 1 /* next_uid */, false /* random */); - return; - } -called_from_on_demand: - old_rf = pr->public_data.results_found; - memset (&prq, 0, sizeof(prq)); - prq.data = data; - prq.expiration = expiration; - prq.size = size; - if (GNUNET_OK != - GNUNET_BLOCK_get_key (GSF_block_ctx, - type, - data, - size, - &query)) - { - GNUNET_break (0); - GNUNET_DATASTORE_remove (GSF_dsh, - key, - size, - data, - UINT_MAX, - UINT_MAX, - NULL, - NULL); - start_local_query (pr, uid + 1 /* next_uid */, false /* random */); - return; - } - prq.type = type; - prq.priority = priority; - prq.request_found = GNUNET_NO; - prq.anonymity_level = anonymity; - if ((0 == old_rf) && (0 == pr->public_data.results_found)) - GSF_update_datastore_delay_ (pr->public_data.start_time); - process_reply (&prq, - key, - pr); - pr->local_result = prq.eval; - if (GNUNET_BLOCK_REPLY_OK_LAST == prq.eval) - { - GNUNET_STATISTICS_update ( - GSF_stats, - gettext_noop ("# Datastore lookups concluded (found last result)"), - 1, - GNUNET_NO); - call_continuation (pr); - return; - } - if ((0 == (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options)) && - ((GNUNET_YES == GSF_test_get_load_too_high_ (0)) || - (pr->public_data.results_found > 5 + 2 * pr->public_data.priority))) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Load too high, done with request\n"); - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ( - "# Datastore lookups concluded (load too high)"), - 1, - GNUNET_NO); - call_continuation (pr); - return; - } - start_local_query (pr, uid + 1 /* next_uid */, false /* random */); -} - - -/** - * Is the given target a legitimate peer for forwarding the given request? - * - * @param pr request - * @param target - * @return #GNUNET_YES if this request could be forwarded to the given peer - */ -int -GSF_pending_request_test_target_ (struct GSF_PendingRequest *pr, - const struct GNUNET_PeerIdentity *target) -{ - struct GNUNET_PeerIdentity pi; - - if (0 == pr->origin_pid) - return GNUNET_YES; - GNUNET_PEER_resolve (pr->origin_pid, &pi); - return (0 == memcmp (&pi, target, sizeof(struct GNUNET_PeerIdentity))) - ? GNUNET_NO - : GNUNET_YES; -} - - -/** - * Look up the request in the local datastore. - * - * @param pr the pending request to process - * @param cont function to call at the end - * @param cont_cls closure for @a cont - */ -void -GSF_local_lookup_ (struct GSF_PendingRequest *pr, - GSF_LocalLookupContinuation cont, - void *cont_cls) -{ - GNUNET_assert (NULL == pr->gh); - GNUNET_assert (NULL == pr->cadet_request); - GNUNET_assert (NULL == pr->llc_cont); - pr->llc_cont = cont; - pr->llc_cont_cls = cont_cls; -#if INSANE_STATISTICS - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# Datastore lookups initiated"), - 1, - GNUNET_NO); -#endif - start_local_query (pr, 0 /* next_uid */, true /* random */); -} - - -/** - * Handle P2P "CONTENT" message. Checks that the message is - * well-formed and then checks if there are any pending requests for - * this content and possibly passes it on (to local clients or other - * peers). Does NOT perform migration (content caching at this peer). - * - * @param cls the other peer involved - * @param put the actual message - */ -void -handle_p2p_put (void *cls, - const struct PutMessage *put) -{ - struct GSF_ConnectedPeer *cp = cls; - uint16_t msize; - size_t dsize; - enum GNUNET_BLOCK_Type type; - struct GNUNET_TIME_Absolute expiration; - struct GNUNET_HashCode query; - struct ProcessReplyClosure prq; - struct GNUNET_TIME_Relative block_time; - double putl; - struct PutMigrationContext *pmc; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Received P2P PUT from %s\n", - GNUNET_i2s (GSF_get_peer_performance_data_ (cp)->peer)); - GSF_cover_content_count++; - msize = ntohs (put->header.size); - dsize = msize - sizeof(struct PutMessage); - type = ntohl (put->type); - expiration = GNUNET_TIME_absolute_ntoh (put->expiration); - /* do not allow migrated content to live longer than 1 year */ - expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute ( - GNUNET_TIME_UNIT_YEARS), - expiration); - if (GNUNET_OK != - GNUNET_BLOCK_check_block (GSF_block_ctx, - type, - &put[1], - dsize)) - { - GNUNET_break_op (0); - return; - } - if (GNUNET_OK != - GNUNET_BLOCK_get_key (GSF_block_ctx, - type, - &put[1], - dsize, - &query)) - { - GNUNET_break_op (0); - return; - } - GNUNET_STATISTICS_update (GSF_stats, - gettext_noop ("# GAP PUT messages received"), - 1, - GNUNET_NO); - /* now, lookup 'query' */ - prq.data = (const void *) &put[1]; - prq.sender = cp; - prq.size = dsize; - prq.type = type; - prq.expiration = expiration; - prq.priority = 0; - prq.anonymity_level = UINT32_MAX; - prq.request_found = GNUNET_NO; - GNUNET_CONTAINER_multihashmap_get_multiple (pr_map, - &query, - &process_reply, - &prq); - if (NULL != cp) - { - GSF_connected_peer_change_preference_ (cp, - CONTENT_BANDWIDTH_VALUE - + 1000 * prq.priority); - GSF_get_peer_performance_data_ (cp)->respect += prq.priority; - } - if ((GNUNET_YES == active_to_migration) && (NULL != cp) && - (GNUNET_NO == test_put_load_too_high (prq.priority))) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Replicating result for query `%s' with priority %u\n", - GNUNET_h2s (&query), - prq.priority); - pmc = GNUNET_new (struct PutMigrationContext); - pmc->start = GNUNET_TIME_absolute_get (); - pmc->requested = prq.request_found; - GNUNET_assert (0 != GSF_get_peer_performance_data_ (cp)->pid); - GNUNET_PEER_resolve (GSF_get_peer_performance_data_ (cp)->pid, - &pmc->origin); - if (NULL == GNUNET_DATASTORE_put (GSF_dsh, - 0, - &query, - dsize, - &put[1], - type, - prq.priority, - 1 /* anonymity */, - 0 /* replication */, - expiration, - 1 + prq.priority, - MAX_DATASTORE_QUEUE, - &put_migration_continuation, - pmc)) - { - put_migration_continuation (pmc, - GNUNET_SYSERR, - GNUNET_TIME_UNIT_ZERO_ABS, - NULL); - } - } - else if (NULL != cp) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Choosing not to keep content `%s' (%d/%d)\n", - GNUNET_h2s (&query), - active_to_migration, - test_put_load_too_high (prq.priority)); - } - putl = GNUNET_LOAD_get_load (datastore_put_load); - if ((NULL != cp) && (GNUNET_NO == prq.request_found) && - ((GNUNET_YES != active_to_migration) || - (putl > 2.5 * (1 + prq.priority)))) - { - if (GNUNET_YES != active_to_migration) - putl = 1.0 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 5); - block_time = GNUNET_TIME_relative_multiply ( - GNUNET_TIME_UNIT_MILLISECONDS, - 5000 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, - (unsigned int) (60000 * putl * putl))); - GNUNET_log ( - GNUNET_ERROR_TYPE_DEBUG, - "Asking to stop migration for %s because of load %f and events %d/%d\n", - GNUNET_STRINGS_relative_time_to_string (block_time, GNUNET_YES), - putl, - active_to_migration, - (GNUNET_NO == prq.request_found)); - GSF_block_peer_migration_ (cp, - GNUNET_TIME_relative_to_absolute (block_time)); - } -} - - -/** - * Check if the given request is still active. - * - * @param pr pending request - * @return #GNUNET_YES if the request is still active - */ -int -GSF_pending_request_test_active_ (struct GSF_PendingRequest *pr) -{ - return (NULL != pr->rh) ? GNUNET_YES : GNUNET_NO; -} - - -/** - * Setup the subsystem. - */ -void -GSF_pending_request_init_ () -{ - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_number (GSF_cfg, - "fs", - "MAX_PENDING_REQUESTS", - &max_pending_requests)) - { - GNUNET_log_config_missing (GNUNET_ERROR_TYPE_INFO, - "fs", - "MAX_PENDING_REQUESTS"); - } - active_to_migration = - GNUNET_CONFIGURATION_get_value_yesno (GSF_cfg, "FS", "CONTENT_CACHING"); - datastore_put_load = GNUNET_LOAD_value_init (DATASTORE_LOAD_AUTODECLINE); - pr_map = GNUNET_CONTAINER_multihashmap_create (32 * 1024, GNUNET_YES); - requests_by_expiration_heap = - GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); -} - - -/** - * Shutdown the subsystem. - */ -void -GSF_pending_request_done_ () -{ - GNUNET_CONTAINER_multihashmap_iterate (pr_map, &clean_request, NULL); - GNUNET_CONTAINER_multihashmap_destroy (pr_map); - pr_map = NULL; - GNUNET_CONTAINER_heap_destroy (requests_by_expiration_heap); - requests_by_expiration_heap = NULL; - GNUNET_LOAD_value_free (datastore_put_load); - datastore_put_load = NULL; -} - - -/* end of gnunet-service-fs_pr.c */ diff --git a/src/fs/gnunet-service-fs_pr.h b/src/fs/gnunet-service-fs_pr.h deleted file mode 100644 index 339e409c5..000000000 --- a/src/fs/gnunet-service-fs_pr.h +++ /dev/null @@ -1,422 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2009, 2010, 2011 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_pr.h - * @brief API to handle pending requests - * @author Christian Grothoff - */ -#ifndef GNUNET_SERVICE_FS_PR_H -#define GNUNET_SERVICE_FS_PR_H - -#include "gnunet-service-fs.h" - - -/** - * Options for pending requests (bits to be ORed). - */ -enum GSF_PendingRequestOptions -{ - /** - * No special options (P2P-default). - */ - GSF_PRO_DEFAULTS = 0, - - /** - * Request must only be processed locally. - */ - GSF_PRO_LOCAL_ONLY = 1, - - /** - * Request must only be forwarded (no routing) - */ - GSF_PRO_FORWARD_ONLY = 2, - - /** - * Request persists indefinitely (no expiration). - */ - GSF_PRO_REQUEST_NEVER_EXPIRES = 4, - - /** - * Request is allowed to refresh bloomfilter and change mingle value. - */ - GSF_PRO_BLOOMFILTER_FULL_REFRESH = 8, - - /** - * Request priority is allowed to be exceeded. - */ - GSF_PRO_PRIORITY_UNLIMITED = 16, - - /** - * Option mask for typical local requests. - */ - GSF_PRO_LOCAL_REQUEST = - (GSF_PRO_BLOOMFILTER_FULL_REFRESH | GSF_PRO_PRIORITY_UNLIMITED - | GSF_PRO_REQUEST_NEVER_EXPIRES) -}; - - -/** - * Public data (in the sense of not encapsulated within - * 'gnunet-service-fs_pr', not in the sense of network-wide - * known) associated with each pending request. - */ -struct GSF_PendingRequestData -{ - /** - * Primary query hash for this request. - */ - struct GNUNET_HashCode query; - - /** - * Identity of a peer hosting the content, otherwise NULl. - * Allocated after struct only if needed. Do not free! - */ - const struct GNUNET_PeerIdentity *target; - - /** - * Fields for the plan module to track a DLL with the request. - */ - struct GSF_PendingRequestPlanBijection *pr_head; - - /** - * Fields for the plan module to track a DLL with the request. - */ - struct GSF_PendingRequestPlanBijection *pr_tail; - - /** - * Current TTL for the request. - */ - struct GNUNET_TIME_Absolute ttl; - - /** - * When did we start with the request. - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * Desired anonymity level. - */ - uint32_t anonymity_level; - - /** - * Priority that this request (still) has for us. - */ - uint32_t priority; - - /** - * Priority that this request (originally) had for us. - */ - uint32_t original_priority; - - /** - * Counter for how often this request has been transmitted (estimate, - * because we might have the same request pending for multiple clients, - * and of course because a transmission may have failed at a lower - * layer). - */ - uint32_t num_transmissions; - - /** - * How much respect did we (in total) offer for this request so far (estimate, - * because we might have the same request pending for multiple clients, - * and of course because a transmission may have failed at a lower - * layer). - */ - uint32_t respect_offered; - - /** - * Options for the request. - */ - enum GSF_PendingRequestOptions options; - - /** - * Type of the requested block. - */ - enum GNUNET_BLOCK_Type type; - - /** - * Number of results we have found for this request so far. - */ - unsigned int results_found; - - /** - * Has this request been started yet (local/p2p operations)? Or are - * we still constructing it? - */ - int has_started; -}; - - -/** - * Handle a reply to a pending request. Also called if a request - * expires (then with data == NULL). The handler may be called - * many times (depending on the request type), but will not be - * called during or after a call to GSF_pending_request_cancel - * and will also not be called anymore after a call signalling - * expiration. - * - * @param cls user-specified closure - * @param eval evaluation of the result - * @param pr handle to the original pending request - * @param reply_anonymity_level anonymity level for the reply, UINT32_MAX for "unknown" - * @param expiration when does @a data expire? - * @param last_transmission the last time we've tried to get this block (FOREVER if unknown) - * @param type type of the block - * @param data response data, NULL on request expiration - * @param data_len number of bytes in @a data - */ -typedef void -(*GSF_PendingRequestReplyHandler) ( - void *cls, - enum GNUNET_BLOCK_ReplyEvaluationResult eval, - struct GSF_PendingRequest *pr, - uint32_t reply_anonymity_level, - struct GNUNET_TIME_Absolute expiration, - struct GNUNET_TIME_Absolute - last_transmission, - enum GNUNET_BLOCK_Type type, - const void *data, - size_t data_len); - - -/** - * Create a new pending request. - * - * @param options request options - * @param type type of the block that is being requested - * @param query key for the lookup - * @param target preferred target for the request, NULL for none - * @param bf_data raw data for bloom filter for known replies, can be NULL - * @param bf_size number of bytes in bf_data - * @param anonymity_level desired anonymity level - * @param priority maximum outgoing cumulative request priority to use - * @param ttl current time-to-live for the request - * @param sender_pid peer ID to use for the sender when forwarding, 0 for none; - * reference counter is taken over by this function - * @param origin_pid peer ID of origin of query (do not loop back) - * @param replies_seen hash codes of known local replies - * @param replies_seen_count size of the 'replies_seen' array - * @param rh handle to call when we get a reply - * @param rh_cls closure for rh - * @return handle for the new pending request - */ -struct GSF_PendingRequest * -GSF_pending_request_create_ (enum GSF_PendingRequestOptions options, - enum GNUNET_BLOCK_Type type, - const struct GNUNET_HashCode *query, - const struct GNUNET_PeerIdentity *target, - const char *bf_data, - size_t bf_size, - uint32_t anonymity_level, - uint32_t priority, - int32_t ttl, - GNUNET_PEER_Id sender_pid, - GNUNET_PEER_Id origin_pid, - const struct GNUNET_HashCode *replies_seen, - unsigned int replies_seen_count, - GSF_PendingRequestReplyHandler rh, - void *rh_cls); - - -/** - * Update a given pending request with additional replies - * that have been seen. - * - * @param pr request to update - * @param replies_seen hash codes of replies that we've seen - * @param replies_seen_count size of the @a replies_seen array - */ -void -GSF_pending_request_update_ (struct GSF_PendingRequest *pr, - const struct GNUNET_HashCode *replies_seen, - unsigned int replies_seen_count); - - -/** - * Obtain the public data associated with a pending request - * - * @param pr pending request - * @return associated public data - */ -struct GSF_PendingRequestData * -GSF_pending_request_get_data_ (struct GSF_PendingRequest *pr); - - -/** - * Check if the given request is still active. - * - * @param pr pending request - * @return #GNUNET_YES if the request is still active - */ -int -GSF_pending_request_test_active_ (struct GSF_PendingRequest *pr); - - -/** - * Test if two pending requests are compatible (would generate - * the same query modulo filters and should thus be processed - * jointly). - * - * @param pra a pending request - * @param prb another pending request - * @return #GNUNET_OK if the requests are compatible - */ -int -GSF_pending_request_is_compatible_ (struct GSF_PendingRequest *pra, - struct GSF_PendingRequest *prb); - - -/** - * Generate the message corresponding to the given pending request for - * transmission to other peers. - * - * @param pr request to generate the message for - * @return envelope with the request message - */ -struct GNUNET_MQ_Envelope * -GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr); - - -/** - * Explicitly cancel a pending request. - * - * @param pr request to cancel - * @param full_cleanup fully purge the request - */ -void -GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr, - int full_cleanup); - - -/** - * Signature of function called on each request. - * (Note: 'subtype' of GNUNET_CONTAINER_HashMapIterator). - * - * @param cls closure - * @param key query for the request - * @param pr handle to the pending request - * @return #GNUNET_YES to continue to iterate - */ -typedef int -(*GSF_PendingRequestIterator) (void *cls, - const struct GNUNET_HashCode *key, - struct GSF_PendingRequest *pr); - - -/** - * Iterate over all pending requests. - * - * @param it function to call for each request - * @param cls closure for it - */ -void -GSF_iterate_pending_requests_ (GSF_PendingRequestIterator it, - void *cls); - - -/** - * Handle P2P "CONTENT" message. Checks that the message is - * well-formed and then checks if there are any pending requests for - * this content and possibly passes it on (to local clients or other - * peers). Does NOT perform migration (content caching at this peer). - * - * @param cls the other peer involved (sender) - * @param put the actual message - */ -void -handle_p2p_put (void *cls, - const struct PutMessage *put); - - -/** - * Consider looking up the data in the DHT (anonymity-level permitting). - * - * @param pr the pending request to process - */ -void -GSF_dht_lookup_ (struct GSF_PendingRequest *pr); - - -/** - * Consider downloading via cadet (if possible) - * - * @param pr the pending request to process - */ -void -GSF_cadet_lookup_ (struct GSF_PendingRequest *pr); - - -/** - * Function to be called after we're done processing - * replies from the local lookup. - * - * @param cls closure - * @param pr the pending request we were processing - * @param result final datastore lookup result - */ -typedef void -(*GSF_LocalLookupContinuation) ( - void *cls, - struct GSF_PendingRequest *pr, - enum GNUNET_BLOCK_ReplyEvaluationResult result); - - -/** - * Look up the request in the local datastore. - * - * @param pr the pending request to process - * @param cont function to call at the end - * @param cont_cls closure for @a cont - */ -void -GSF_local_lookup_ (struct GSF_PendingRequest *pr, - GSF_LocalLookupContinuation cont, - void *cont_cls); - - -/** - * Is the given target a legitimate peer for forwarding the given request? - * - * @param pr request - * @param target - * @return #GNUNET_YES if this request could be forwarded to the given peer - */ -int -GSF_pending_request_test_target_ (struct GSF_PendingRequest *pr, - const struct GNUNET_PeerIdentity *target); - - -/** - * Setup the subsystem. - */ -void -GSF_pending_request_init_ (void); - - -/** - * Shutdown the subsystem. - */ -void -GSF_pending_request_done_ (void); - - -#endif -/* end of gnunet-service-fs_pr.h */ diff --git a/src/fs/gnunet-service-fs_push.c b/src/fs/gnunet-service-fs_push.c deleted file mode 100644 index 92dbba8e6..000000000 --- a/src/fs/gnunet-service-fs_push.c +++ /dev/null @@ -1,672 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2011, 2016 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_push.c - * @brief API to push content from our datastore to other peers - * ('anonymous'-content P2P migration) - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet-service-fs.h" -#include "gnunet-service-fs_cp.h" -#include "gnunet-service-fs_indexing.h" -#include "gnunet-service-fs_push.h" - - -/** - * Maximum number of blocks we keep in memory for migration. - */ -#define MAX_MIGRATION_QUEUE 8 - -/** - * Blocks are at most migrated to this number of peers - * plus one, each time they are fetched from the database. - */ -#define MIGRATION_LIST_SIZE 2 - -/** - * How long must content remain valid for us to consider it for migration? - * If content will expire too soon, there is clearly no point in pushing - * it to other peers. This value gives the threshold for migration. Note - * that if this value is increased, the migration testcase may need to be - * adjusted as well (especially the CONTENT_LIFETIME in fs_test_lib.c). - */ -#define MIN_MIGRATION_CONTENT_LIFETIME GNUNET_TIME_relative_multiply ( \ - GNUNET_TIME_UNIT_MINUTES, 30) - - -/** - * Block that is ready for migration to other peers. Actual data is at the end of the block. - */ -struct MigrationReadyBlock -{ - /** - * This is a doubly-linked list. - */ - struct MigrationReadyBlock *next; - - /** - * This is a doubly-linked list. - */ - struct MigrationReadyBlock *prev; - - /** - * Query for the block. - */ - struct GNUNET_HashCode query; - - /** - * When does this block expire? - */ - struct GNUNET_TIME_Absolute expiration; - - /** - * Peers we already forwarded this - * block to. Zero for empty entries. - */ - GNUNET_PEER_Id target_list[MIGRATION_LIST_SIZE]; - - /** - * Size of the block. - */ - size_t size; - - /** - * Number of targets already used. - */ - unsigned int used_targets; - - /** - * Type of the block. - */ - enum GNUNET_BLOCK_Type type; -}; - - -/** - * Information about a peer waiting for migratable data. - */ -struct MigrationReadyPeer -{ - /** - * This is a doubly-linked list. - */ - struct MigrationReadyPeer *next; - - /** - * This is a doubly-linked list. - */ - struct MigrationReadyPeer *prev; - - /** - * Handle to peer. - */ - struct GSF_ConnectedPeer *peer; - - /** - * Envelope of the currently pushed message. - */ - struct GNUNET_MQ_Envelope *env; -}; - - -/** - * Head of linked list of blocks that can be migrated. - */ -static struct MigrationReadyBlock *mig_head; - -/** - * Tail of linked list of blocks that can be migrated. - */ -static struct MigrationReadyBlock *mig_tail; - -/** - * Head of linked list of peers. - */ -static struct MigrationReadyPeer *peer_head; - -/** - * Tail of linked list of peers. - */ -static struct MigrationReadyPeer *peer_tail; - -/** - * Request to datastore for migration (or NULL). - */ -static struct GNUNET_DATASTORE_QueueEntry *mig_qe; - -/** - * ID of task that collects blocks for migration. - */ -static struct GNUNET_SCHEDULER_Task *mig_task; - -/** - * What is the maximum frequency at which we are allowed to - * poll the datastore for migration content? - */ -static struct GNUNET_TIME_Relative min_migration_delay; - -/** - * Size of the doubly-linked list of migration blocks. - */ -static unsigned int mig_size; - -/** - * Is this module enabled? - */ -static int enabled; - -/** - * Did we find anything in the datastore? - */ -static int value_found; - - -/** - * Delete the given migration block. - * - * @param mb block to delete - */ -static void -delete_migration_block (struct MigrationReadyBlock *mb) -{ - GNUNET_CONTAINER_DLL_remove (mig_head, - mig_tail, - mb); - GNUNET_PEER_decrement_rcs (mb->target_list, - MIGRATION_LIST_SIZE); - mig_size--; - GNUNET_free (mb); -} - - -/** - * Find content for migration to this peer. - * - * @param cls A `struct MigrationReadyPeer *` to find content for - */ -static void -find_content (void *cls); - - -/** - * Send the given block to the given peer. - * - * @param mrp target peer - * @param block the block - * @return #GNUNET_YES if the block was deleted (!) - */ -static int -transmit_content (struct MigrationReadyPeer *mrp, - struct MigrationReadyBlock *block) -{ - struct PutMessage *msg; - unsigned int i; - struct GSF_PeerPerformanceData *ppd; - int ret; - - ppd = GSF_get_peer_performance_data_ (mrp->peer); - GNUNET_assert (NULL == mrp->env); - mrp->env = GNUNET_MQ_msg_extra (msg, - block->size, - GNUNET_MESSAGE_TYPE_FS_PUT); - msg->type = htonl (block->type); - msg->expiration = GNUNET_TIME_absolute_hton (block->expiration); - GNUNET_memcpy (&msg[1], - &block[1], - block->size); - for (i = 0; i < MIGRATION_LIST_SIZE; i++) - { - if (block->target_list[i] == 0) - { - block->target_list[i] = ppd->pid; - GNUNET_PEER_change_rc (block->target_list[i], - 1); - break; - } - } - if (MIGRATION_LIST_SIZE == i) - { - delete_migration_block (block); - ret = GNUNET_YES; - } - else - { - ret = GNUNET_NO; - } - GNUNET_MQ_notify_sent (mrp->env, - &find_content, - mrp); - GSF_peer_transmit_ (mrp->peer, - GNUNET_NO, - 0 /* priority */, - mrp->env); - return ret; -} - - -/** - * Count the number of peers this block has - * already been forwarded to. - * - * @param block the block - * @return number of times block was forwarded - */ -static unsigned int -count_targets (struct MigrationReadyBlock *block) -{ - unsigned int i; - - for (i = 0; i < MIGRATION_LIST_SIZE; i++) - if (block->target_list[i] == 0) - return i; - return i; -} - - -/** - * Check if sending this block to this peer would - * be a good idea. - * - * @param mrp target peer - * @param block the block - * @return score (>= 0: feasible, negative: infeasible) - */ -static long -score_content (struct MigrationReadyPeer *mrp, - struct MigrationReadyBlock *block) -{ - unsigned int i; - struct GSF_PeerPerformanceData *ppd; - struct GNUNET_PeerIdentity id; - struct GNUNET_HashCode hc; - uint32_t dist; - - ppd = GSF_get_peer_performance_data_ (mrp->peer); - for (i = 0; i < MIGRATION_LIST_SIZE; i++) - if (block->target_list[i] == ppd->pid) - return -1; - GNUNET_assert (0 != ppd->pid); - GNUNET_PEER_resolve (ppd->pid, - &id); - GNUNET_CRYPTO_hash (&id, - sizeof(struct GNUNET_PeerIdentity), - &hc); - dist = GNUNET_CRYPTO_hash_distance_u32 (&block->query, - &hc); - /* closer distance, higher score: */ - return UINT32_MAX - dist; -} - - -/** - * If the migration task is not currently running, consider - * (re)scheduling it with the appropriate delay. - */ -static void -consider_gathering (void); - - -static void -find_content (void *cls) -{ - struct MigrationReadyPeer *mrp = cls; - struct MigrationReadyBlock *pos; - long score; - long best_score; - struct MigrationReadyBlock *best; - - mrp->env = NULL; - best = NULL; - best_score = -1; - pos = mig_head; - while (NULL != pos) - { - score = score_content (mrp, pos); - if (score > best_score) - { - best_score = score; - best = pos; - } - pos = pos->next; - } - if (NULL == best) - { - if (mig_size < MAX_MIGRATION_QUEUE) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "No content found for pushing, waiting for queue to fill\n"); - return; /* will fill up eventually... */ - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "No suitable content found, purging content from full queue\n"); - /* failed to find migration target AND - * queue is full, purge most-forwarded - * block from queue to make room for more */ - pos = mig_head; - while (NULL != pos) - { - score = count_targets (pos); - if (score >= best_score) - { - best_score = score; - best = pos; - } - pos = pos->next; - } - GNUNET_assert (NULL != best); - delete_migration_block (best); - consider_gathering (); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Preparing to push best content to peer\n"); - transmit_content (mrp, - best); -} - - -/** - * Task that is run periodically to obtain blocks for content - * migration - * - * @param cls unused - */ -static void -gather_migration_blocks (void *cls); - - -/** - * If the migration task is not currently running, consider - * (re)scheduling it with the appropriate delay. - */ -static void -consider_gathering () -{ - struct GNUNET_TIME_Relative delay; - - if (NULL == GSF_dsh) - return; - if (NULL != mig_qe) - return; - if (NULL != mig_task) - return; - if (mig_size >= MAX_MIGRATION_QUEUE) - return; - delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, - mig_size); - delay = GNUNET_TIME_relative_divide (delay, - MAX_MIGRATION_QUEUE); - delay = GNUNET_TIME_relative_max (delay, - min_migration_delay); - if (GNUNET_NO == value_found) - { - /* wait at least 5s if the datastore is empty */ - delay = GNUNET_TIME_relative_max (delay, - GNUNET_TIME_relative_multiply ( - GNUNET_TIME_UNIT_SECONDS, - 5)); - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Scheduling gathering task (queue size: %u)\n", - mig_size); - mig_task = GNUNET_SCHEDULER_add_delayed (delay, - &gather_migration_blocks, - NULL); -} - - -/** - * Process content offered for migration. - * - * @param cls closure - * @param key key for the content - * @param size number of bytes in data - * @param data content stored - * @param type type of the content - * @param priority priority of the content - * @param anonymity anonymity-level for the content - * @param replication replication-level for the content - * @param expiration expiration time for the content - * @param uid unique identifier for the datum; - * maybe 0 if no unique identifier is available - */ -static void -process_migration_content (void *cls, - const struct GNUNET_HashCode *key, - size_t size, - const void *data, - enum GNUNET_BLOCK_Type type, - uint32_t priority, - uint32_t anonymity, - uint32_t replication, - struct GNUNET_TIME_Absolute expiration, - uint64_t uid) -{ - struct MigrationReadyBlock *mb; - struct MigrationReadyPeer *pos; - - mig_qe = NULL; - if (NULL == key) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "No content found for migration...\n"); - consider_gathering (); - return; - } - value_found = GNUNET_YES; - if (GNUNET_TIME_absolute_get_remaining (expiration).rel_value_us < - MIN_MIGRATION_CONTENT_LIFETIME.rel_value_us) - { - /* content will expire soon, don't bother */ - consider_gathering (); - return; - } - if (type == GNUNET_BLOCK_TYPE_FS_ONDEMAND) - { - if (GNUNET_OK != - GNUNET_FS_handle_on_demand_block (key, - size, - data, - type, - priority, - anonymity, - replication, - expiration, - uid, - &process_migration_content, - NULL)) - consider_gathering (); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Retrieved block `%s' of type %u for migration (queue size: %u/%u)\n", - GNUNET_h2s (key), - type, mig_size + 1, - MAX_MIGRATION_QUEUE); - mb = GNUNET_malloc (sizeof(struct MigrationReadyBlock) + size); - mb->query = *key; - mb->expiration = expiration; - mb->size = size; - mb->type = type; - GNUNET_memcpy (&mb[1], data, size); - GNUNET_CONTAINER_DLL_insert_after (mig_head, - mig_tail, - mig_tail, - mb); - mig_size++; - for (pos = peer_head; NULL != pos; pos = pos->next) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Preparing to push best content to peer %s\n", - GNUNET_i2s (GSF_connected_peer_get_identity2_ (pos->peer))); - if ((NULL == pos->env) && - (GNUNET_YES == transmit_content (pos, - mb))) - { - break; /* 'mb' was freed! */ - } - } - consider_gathering (); -} - - -/** - * Task that is run periodically to obtain blocks for content - * migration - * - * @param cls unused - */ -static void -gather_migration_blocks (void *cls) -{ - mig_task = NULL; - if (mig_size >= MAX_MIGRATION_QUEUE) - return; - if (NULL == GSF_dsh) - return; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Asking datastore for content for replication (queue size: %u)\n", - mig_size); - value_found = GNUNET_NO; - mig_qe = GNUNET_DATASTORE_get_for_replication (GSF_dsh, - 0, - UINT_MAX, - &process_migration_content, - NULL); - if (NULL == mig_qe) - consider_gathering (); -} - - -/** - * A peer connected to us. Start pushing content - * to this peer. - * - * @param peer handle for the peer that connected - */ -void -GSF_push_start_ (struct GSF_ConnectedPeer *peer) -{ - struct MigrationReadyPeer *mrp; - - if (GNUNET_YES != enabled) - return; - for (mrp = peer_head; NULL != mrp; mrp = mrp->next) - if (mrp->peer == peer) - break; - if (NULL != mrp) - { - /* same peer added twice, must not happen */ - GNUNET_break (0); - return; - } - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Adding peer %s to list for pushing\n", - GNUNET_i2s (GSF_connected_peer_get_identity2_ (peer))); - - mrp = GNUNET_new (struct MigrationReadyPeer); - mrp->peer = peer; - find_content (mrp); - GNUNET_CONTAINER_DLL_insert (peer_head, - peer_tail, - mrp); -} - - -/** - * A peer disconnected from us. Stop pushing content - * to this peer. - * - * @param peer handle for the peer that disconnected - */ -void -GSF_push_stop_ (struct GSF_ConnectedPeer *peer) -{ - struct MigrationReadyPeer *pos; - - for (pos = peer_head; NULL != pos; pos = pos->next) - if (pos->peer == peer) - break; - if (NULL == pos) - return; - if (NULL != pos->env) - GNUNET_MQ_send_cancel (pos->env); - GNUNET_CONTAINER_DLL_remove (peer_head, - peer_tail, - pos); - GNUNET_free (pos); -} - - -/** - * Setup the module. - */ -void -GSF_push_init_ () -{ - enabled = - GNUNET_CONFIGURATION_get_value_yesno (GSF_cfg, - "FS", - "CONTENT_PUSHING"); - if (GNUNET_YES != enabled) - return; - - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_time (GSF_cfg, - "fs", - "MIN_MIGRATION_DELAY", - &min_migration_delay)) - { - GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING, - "fs", - "MIN_MIGRATION_DELAY", - _ ("time required, content pushing disabled")); - return; - } - consider_gathering (); -} - - -/** - * Shutdown the module. - */ -void -GSF_push_done_ () -{ - if (NULL != mig_task) - { - GNUNET_SCHEDULER_cancel (mig_task); - mig_task = NULL; - } - if (NULL != mig_qe) - { - GNUNET_DATASTORE_cancel (mig_qe); - mig_qe = NULL; - } - while (NULL != mig_head) - delete_migration_block (mig_head); - GNUNET_assert (0 == mig_size); -} - - -/* end of gnunet-service-fs_push.c */ diff --git a/src/fs/gnunet-service-fs_push.h b/src/fs/gnunet-service-fs_push.h deleted file mode 100644 index 2cd621bbb..000000000 --- a/src/fs/gnunet-service-fs_push.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2011 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_push.h - * @brief support for pushing out content - * @author Christian Grothoff - */ -#ifndef GNUNET_SERVICE_FS_PUSH_H -#define GNUNET_SERVICE_FS_PUSH_H - -#include "gnunet-service-fs.h" - - -/** - * Setup the module. - */ -void -GSF_push_init_ (void); - - -/** - * Shutdown the module. - */ -void -GSF_push_done_ (void); - - -/** - * A peer connected to us or we are now again allowed to push content. - * Start pushing content to this peer. - * - * @param peer handle for the peer that connected - */ -void -GSF_push_start_ (struct GSF_ConnectedPeer *peer); - - -/** - * A peer disconnected from us or asked us to stop pushing content for - * a while. Stop pushing content to this peer. - * - * @param peer handle for the peer that disconnected - */ -void -GSF_push_stop_ (struct GSF_ConnectedPeer *peer); - - -#endif diff --git a/src/fs/gnunet-service-fs_put.c b/src/fs/gnunet-service-fs_put.c deleted file mode 100644 index ca2c85724..000000000 --- a/src/fs/gnunet-service-fs_put.c +++ /dev/null @@ -1,290 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2011 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_put.c - * @brief API to PUT zero-anonymity index data from our datastore into the DHT - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet-service-fs.h" -#include "gnunet-service-fs_put.h" - - -/** - * How often do we at most PUT content into the DHT? - */ -#define MAX_DHT_PUT_FREQ GNUNET_TIME_relative_multiply ( \ - GNUNET_TIME_UNIT_SECONDS, 5) - -/** - * How many replicas do we try to create per PUT? - */ -#define DEFAULT_PUT_REPLICATION 5 - - -/** - * Context for each zero-anonymity iterator. - */ -struct PutOperator -{ - /** - * Request to datastore for DHT PUTs (or NULL). - */ - struct GNUNET_DATASTORE_QueueEntry *dht_qe; - - /** - * Type we request from the datastore. - */ - enum GNUNET_BLOCK_Type dht_put_type; - - /** - * Handle to PUT operation. - */ - struct GNUNET_DHT_PutHandle *dht_put; - - /** - * ID of task that collects blocks for DHT PUTs. - */ - struct GNUNET_SCHEDULER_Task *dht_task; - - /** - * How many entries with zero anonymity of our type do we currently - * estimate to have in the database? - */ - uint64_t zero_anonymity_count_estimate; - - /** - * Count of results received from the database. - */ - uint64_t result_count; - - /** - * Next UID to request when iterating the database. - */ - uint64_t next_uid; -}; - - -/** - * ANY-terminated list of our operators (one per type - * of block that we're putting into the DHT). - */ -static struct PutOperator operators[] = { - { NULL, GNUNET_BLOCK_TYPE_FS_UBLOCK, 0, 0, 0 }, - { NULL, GNUNET_BLOCK_TYPE_ANY, 0, 0, 0 } -}; - - -/** - * Task that is run periodically to obtain blocks for DHT PUTs. - * - * @param cls type of blocks to gather - */ -static void -gather_dht_put_blocks (void *cls); - - -/** - * Calculate when to run the next PUT operation and schedule it. - * - * @param po put operator to schedule - */ -static void -schedule_next_put (struct PutOperator *po) -{ - struct GNUNET_TIME_Relative delay; - - if (po->zero_anonymity_count_estimate > 0) - { - delay = - GNUNET_TIME_relative_divide (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY, - po->zero_anonymity_count_estimate); - delay = GNUNET_TIME_relative_min (delay, MAX_DHT_PUT_FREQ); - } - else - { - /* if we have NO zero-anonymity content yet, wait 5 minutes for some to - * (hopefully) appear */ - delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5); - } - po->dht_task = - GNUNET_SCHEDULER_add_delayed (delay, &gather_dht_put_blocks, po); -} - - -/** - * Continuation called after DHT PUT operation has finished. - * - * @param cls type of blocks to gather - */ -static void -delay_dht_put_blocks (void *cls) -{ - struct PutOperator *po = cls; - - po->dht_put = NULL; - schedule_next_put (po); -} - - -/** - * Task that is run periodically to obtain blocks for DHT PUTs. - * - * @param cls type of blocks to gather - */ -static void -delay_dht_put_task (void *cls) -{ - struct PutOperator *po = cls; - - po->dht_task = NULL; - schedule_next_put (po); -} - - -/** - * Store content in DHT. - * - * @param cls closure - * @param key key for the content - * @param size number of bytes in data - * @param data content stored - * @param type type of the content - * @param priority priority of the content - * @param anonymity anonymity-level for the content - * @param replication replication-level for the content - * @param expiration expiration time for the content - * @param uid unique identifier for the datum; - * maybe 0 if no unique identifier is available - */ -static void -process_dht_put_content (void *cls, - const struct GNUNET_HashCode *key, - size_t size, - const void *data, - enum GNUNET_BLOCK_Type type, - uint32_t priority, - uint32_t anonymity, - uint32_t replication, - struct GNUNET_TIME_Absolute expiration, - uint64_t uid) -{ - struct PutOperator *po = cls; - - po->dht_qe = NULL; - if (key == NULL) - { - po->zero_anonymity_count_estimate = po->result_count; - po->result_count = 0; - po->next_uid = 0; - po->dht_task = GNUNET_SCHEDULER_add_now (&delay_dht_put_task, po); - return; - } - po->result_count++; - po->next_uid = uid + 1; - po->zero_anonymity_count_estimate = - GNUNET_MAX (po->result_count, po->zero_anonymity_count_estimate); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Retrieved block `%s' of type %u for DHT PUT\n", GNUNET_h2s (key), - type); - po->dht_put = GNUNET_DHT_put (GSF_dht, - key, - DEFAULT_PUT_REPLICATION, - GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE, - type, - size, - data, - expiration, - &delay_dht_put_blocks, - po); -} - - -static void -gather_dht_put_blocks (void *cls) -{ - struct PutOperator *po = cls; - - po->dht_task = NULL; - po->dht_qe = - GNUNET_DATASTORE_get_zero_anonymity (GSF_dsh, - po->next_uid, - 0, - UINT_MAX, - po->dht_put_type, - &process_dht_put_content, - po); - if (NULL == po->dht_qe) - po->dht_task = GNUNET_SCHEDULER_add_now (&delay_dht_put_task, po); -} - - -/** - * Setup the module. - */ -void -GSF_put_init_ () -{ - unsigned int i; - - i = 0; - while (operators[i].dht_put_type != GNUNET_BLOCK_TYPE_ANY) - { - operators[i].dht_task = - GNUNET_SCHEDULER_add_now (&gather_dht_put_blocks, &operators[i]); - i++; - } -} - - -/** - * Shutdown the module. - */ -void -GSF_put_done_ () -{ - struct PutOperator *po; - unsigned int i; - - i = 0; - while ((po = &operators[i])->dht_put_type != GNUNET_BLOCK_TYPE_ANY) - { - if (NULL != po->dht_task) - { - GNUNET_SCHEDULER_cancel (po->dht_task); - po->dht_task = NULL; - } - if (NULL != po->dht_put) - { - GNUNET_DHT_put_cancel (po->dht_put); - po->dht_put = NULL; - } - if (NULL != po->dht_qe) - { - GNUNET_DATASTORE_cancel (po->dht_qe); - po->dht_qe = NULL; - } - i++; - } -} - - -/* end of gnunet-service-fs_put.c */ diff --git a/src/fs/gnunet-service-fs_put.h b/src/fs/gnunet-service-fs_put.h deleted file mode 100644 index b6c9ba86f..000000000 --- a/src/fs/gnunet-service-fs_put.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2011 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/gnunet-service-fs_put.h - * @brief support for putting content into the DHT - * @author Christian Grothoff - */ -#ifndef GNUNET_SERVICE_FS_PUT_H -#define GNUNET_SERVICE_FS_PUT_H - -#include "gnunet-service-fs.h" - - -/** - * Setup the module. - */ -void -GSF_put_init_ (void); - - -/** - * Shutdown the module. - */ -void -GSF_put_done_ (void); - - -#endif diff --git a/src/fs/gnunet-unindex.c b/src/fs/gnunet-unindex.c deleted file mode 100644 index 326f75a63..000000000 --- a/src/fs/gnunet-unindex.c +++ /dev/null @@ -1,206 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/gnunet-unindex.c - * @brief unindex files published on GNUnet - * @author Christian Grothoff - * @author Krista Bennett - * @author James Blackwell - * @author Igor Wronsky - */ -#include "platform.h" - -#include "gnunet_fs_service.h" - -static int ret; - -static unsigned int verbose; - -static const struct GNUNET_CONFIGURATION_Handle *cfg; - -static struct GNUNET_FS_Handle *ctx; - -static struct GNUNET_FS_UnindexContext *uc; - - -static void -cleanup_task (void *cls) -{ - GNUNET_FS_stop (ctx); - ctx = NULL; -} - - -static void -shutdown_task (void *cls) -{ - struct GNUNET_FS_UnindexContext *u; - - if (uc != NULL) - { - u = uc; - uc = NULL; - GNUNET_FS_unindex_stop (u); - } -} - - -/** - * Called by FS client to give information about the progress of an - * operation. - * - * @param cls closure - * @param info details about the event, specifying the event type - * and various bits about the event - * @return client-context (for the next progress call - * for this operation; should be set to NULL for - * SUSPEND and STOPPED events). The value returned - * will be passed to future callbacks in the respective - * field in the GNUNET_FS_ProgressInfo struct. - */ -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *info) -{ - const char *s; - - switch (info->status) - { - case GNUNET_FS_STATUS_UNINDEX_START: - break; - - case GNUNET_FS_STATUS_UNINDEX_PROGRESS: - if (verbose) - { - s = GNUNET_STRINGS_relative_time_to_string (info->value.unindex.eta, - GNUNET_YES); - fprintf (stdout, - _ ("Unindexing at %llu/%llu (%s remaining)\n"), - (unsigned long long) info->value.unindex.completed, - (unsigned long long) info->value.unindex.size, - s); - } - break; - - case GNUNET_FS_STATUS_UNINDEX_ERROR: - fprintf (stderr, - _ ("Error unindexing: %s.\n"), - info->value.unindex.specifics.error.message); - GNUNET_SCHEDULER_shutdown (); - break; - - case GNUNET_FS_STATUS_UNINDEX_COMPLETED: - fprintf (stdout, "%s", _ ("Unindexing done.\n")); - GNUNET_SCHEDULER_shutdown (); - break; - - case GNUNET_FS_STATUS_UNINDEX_STOPPED: - GNUNET_SCHEDULER_add_now (&cleanup_task, NULL); - break; - - default: - fprintf (stderr, _ ("Unexpected status: %d\n"), info->status); - break; - } - return NULL; -} - - -/** - * Main function that will be run by the scheduler. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param c configuration - */ -static void -run (void *cls, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *c) -{ - /* check arguments */ - if ((args[0] == NULL) || (args[1] != NULL)) - { - printf (_ ("You must specify one and only one filename for unindexing.\n")); - ret = -1; - return; - } - cfg = c; - ctx = GNUNET_FS_start (cfg, - "gnunet-unindex", - &progress_cb, - NULL, - GNUNET_FS_FLAGS_NONE, - GNUNET_FS_OPTIONS_END); - if (NULL == ctx) - { - fprintf (stderr, _ ("Could not initialize `%s' subsystem.\n"), "FS"); - ret = 1; - return; - } - uc = GNUNET_FS_unindex_start (ctx, args[0], NULL); - if (NULL == uc) - { - fprintf (stderr, "%s", _ ("Could not start unindex operation.\n")); - GNUNET_FS_stop (ctx); - return; - } - GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL); -} - - -/** - * The main function to unindex content. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, char *const *argv) -{ - struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_option_verbose (&verbose), - - GNUNET_GETOPT_OPTION_END - }; - - if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) - return 2; - - ret = (GNUNET_OK == - GNUNET_PROGRAM_run ( - argc, - argv, - "gnunet-unindex [OPTIONS] FILENAME", - gettext_noop ( - "Unindex a file that was previously indexed with gnunet-publish."), - options, - &run, - NULL)) - ? ret - : 1; - GNUNET_free_nz ((void *) argv); - return ret; -} - - -/* end of gnunet-unindex.c */ diff --git a/src/fs/meson.build b/src/fs/meson.build deleted file mode 100644 index 4246e4fcf..000000000 --- a/src/fs/meson.build +++ /dev/null @@ -1,141 +0,0 @@ -libgnunetfs_src = ['fs_api.c', - 'fs_directory.c', - 'fs_dirmetascan.c', - 'fs_download.c', - 'fs_file_information.c', - 'fs_getopt.c', - 'fs_list_indexed.c', - 'fs_publish.c', - 'fs_publish_ksk.c', - 'fs_publish_ublock.c', - 'fs_misc.c', - 'fs_namespace.c', - 'fs_search.c', - 'fs_sharetree.c', - 'fs_tree.c', - 'fs_unindex.c', - 'fs_uri.c', - 'meta_data.c'] - -gnunetservicefs_src = ['gnunet-service-fs.c', - 'gnunet-service-fs_cp.c', - 'gnunet-service-fs_indexing.c', - 'gnunet-service-fs_pe.c', - 'gnunet-service-fs_pr.c', - 'gnunet-service-fs_push.c', - 'gnunet-service-fs_put.c', - 'gnunet-service-fs_cadet_client.c', - 'gnunet-service-fs_cadet_server.c'] - -configure_file(input : 'fs.conf.in', - output : 'fs.conf', - configuration : cdata, - install: true, - install_dir: pkgcfgdir) - - -if get_option('monolith') - foreach p : libgnunetfs_src + gnunetservicefs_src - gnunet_src += 'fs/' + p - endforeach - subdir_done() -endif - -libgnunetfs = library('gnunetfs', - libgnunetfs_src, - soversion: '2', - version: '2.1.1', - dependencies: [libgnunetutil_dep, - libgnunetdatastore_dep, - libgnunetstatistics_dep, - unistr_dep], - include_directories: [incdir, configuration_inc], - install: true, - install_dir: get_option('libdir')) -libgnunetfs_dep = declare_dependency(link_with : libgnunetfs) -pkg.generate(libgnunetfs, url: 'https://www.gnunet.org', - description : 'Provides API for GNUnet File-Sharing service') - -shared_module('gnunet_plugin_block_fs', - ['plugin_block_fs.c'], - dependencies: [libgnunetutil_dep, - libgnunetblockgroup_dep], - include_directories: [incdir, configuration_inc], - install:true, - install_dir: get_option('libdir')/'gnunet') - -executable ('gnunet-search', - 'gnunet-search.c', - dependencies: [libgnunetfs_dep, - libgnunetutil_dep], - include_directories: [incdir, configuration_inc], - install: true, - install_dir: get_option('bindir')) -executable ('gnunet-unindex', - 'gnunet-unindex.c', - dependencies: [libgnunetfs_dep, - libgnunetutil_dep], - include_directories: [incdir, configuration_inc], - install: true, - install_dir: get_option('bindir')) -executable ('gnunet-auto-share', - 'gnunet-auto-share.c', - dependencies: [libgnunetfs_dep, - libgnunetutil_dep], - include_directories: [incdir, configuration_inc], - install: true, - install_dir: get_option('bindir')) -executable ('gnunet-directory', - 'gnunet-directory.c', - dependencies: [libgnunetfs_dep, - libgnunetutil_dep], - include_directories: [incdir, configuration_inc], - install: true, - install_dir: get_option('bindir')) -executable ('gnunet-download', - 'gnunet-download.c', - dependencies: [libgnunetfs_dep, - libgnunetutil_dep], - include_directories: [incdir, configuration_inc], - install: true, - install_dir: get_option('bindir')) -executable ('gnunet-fs', - 'gnunet-fs.c', - dependencies: [libgnunetfs_dep, - libgnunetutil_dep], - include_directories: [incdir, configuration_inc], - install: true, - install_dir: get_option('bindir')) -executable ('gnunet-publish', - 'gnunet-publish.c', - dependencies: [libgnunetfs_dep, - libgnunetidentity_dep, - libgnunetutil_dep], - include_directories: [incdir, configuration_inc], - install: true, - install_dir: get_option('bindir')) -executable ('gnunet-service-fs', - gnunetservicefs_src, - dependencies: [libgnunetfs_dep, - libgnunetutil_dep, - libgnunetstatistics_dep, - libgnunetcore_dep, - libgnunetdht_dep, - libgnunetidentity_dep, - m_dep, - libgnunetcadet_dep, - libgnunetpeerstore_dep, - libgnunetdatastore_dep, - libgnunetblock_dep], - include_directories: [incdir, configuration_inc], - install: true, - install_dir: get_option('libdir') / 'gnunet' / 'libexec') -executable ('gnunet-helper-fs-publish', - ['gnunet-helper-fs-publish.c'], - dependencies: [libgnunetfs_dep, - libgnunetutil_dep, - libgnunetblock_dep], - include_directories: [incdir, configuration_inc], - install: true, - install_dir: get_option('libdir') / 'gnunet' / 'libexec') - diff --git a/src/fs/meta_data.c b/src/fs/meta_data.c deleted file mode 100644 index 1e75ecf6c..000000000 --- a/src/fs/meta_data.c +++ /dev/null @@ -1,1229 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2003, 2004, 2005, 2006, 2008, 2009, 2010, 2022 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/meta_data.c - * @brief Storing of meta data - * @author Christian Grothoff - * @author Martin Schanzenbach - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_fs_service.h" - -/** - * Maximum size allowed for meta data written/read from disk. - * File-sharing limits to 64k, so this should be rather generous. - */ -#define MAX_META_DATA (1024 * 1024) - - -#define LOG(kind, ...) GNUNET_log_from (kind, "fs-meta-data", \ - __VA_ARGS__) - - - -/** - * Meta data item. - */ -struct MetaItem -{ - /** - * This is a doubly linked list. - */ - struct MetaItem *next; - - /** - * This is a doubly linked list. - */ - struct MetaItem *prev; - - /** - * Name of the extracting plugin. - */ - char *plugin_name; - - /** - * Mime-type of data. - */ - char *mime_type; - - /** - * The actual meta data. - */ - char *data; - - /** - * Number of bytes in 'data'. - */ - size_t data_size; - - /** - * Type of the meta data. - */ - enum EXTRACTOR_MetaType type; - - /** - * Format of the meta data. - */ - enum EXTRACTOR_MetaFormat format; -}; - -/** - * Meta data to associate with a file, directory or namespace. - */ -struct GNUNET_FS_MetaData -{ - /** - * Head of linked list of the meta data items. - */ - struct MetaItem *items_head; - - /** - * Tail of linked list of the meta data items. - */ - struct MetaItem *items_tail; - - /** - * Complete serialized and compressed buffer of the items. - * NULL if we have not computed that buffer yet. - */ - char *sbuf; - - /** - * Number of bytes in 'sbuf'. 0 if the buffer is stale. - */ - size_t sbuf_size; - - /** - * Number of items in the linked list. - */ - unsigned int item_count; -}; - - -/** - * Create a fresh struct FS_MetaData token. - * - * @return empty meta-data container - */ -struct GNUNET_FS_MetaData * -GNUNET_FS_meta_data_create () -{ - return GNUNET_new (struct GNUNET_FS_MetaData); -} - - -/** - * Free meta data item. - * - * @param mi item to free - */ -static void -meta_item_free (struct MetaItem *mi) -{ - GNUNET_free (mi->plugin_name); - GNUNET_free (mi->mime_type); - GNUNET_free (mi->data); - GNUNET_free (mi); -} - - -/** - * The meta data has changed, invalidate its serialization - * buffer. - * - * @param md meta data that changed - */ -static void -invalidate_sbuf (struct GNUNET_FS_MetaData *md) -{ - if (NULL == md->sbuf) - return; - GNUNET_free (md->sbuf); - md->sbuf = NULL; - md->sbuf_size = 0; -} - - -void -GNUNET_FS_meta_data_destroy (struct GNUNET_FS_MetaData *md) -{ - struct MetaItem *pos; - - if (NULL == md) - return; - while (NULL != (pos = md->items_head)) - { - GNUNET_CONTAINER_DLL_remove (md->items_head, md->items_tail, pos); - meta_item_free (pos); - } - GNUNET_free (md->sbuf); - GNUNET_free (md); -} - - -void -GNUNET_FS_meta_data_clear (struct GNUNET_FS_MetaData *md) -{ - struct MetaItem *mi; - - if (NULL == md) - return; - while (NULL != (mi = md->items_head)) - { - GNUNET_CONTAINER_DLL_remove (md->items_head, md->items_tail, mi); - meta_item_free (mi); - } - GNUNET_free (md->sbuf); - memset (md, 0, sizeof(struct GNUNET_FS_MetaData)); -} - - -int -GNUNET_FS_meta_data_test_equal (const struct GNUNET_FS_MetaData - *md1, - const struct GNUNET_FS_MetaData - *md2) -{ - struct MetaItem *i; - struct MetaItem *j; - int found; - - if (md1 == md2) - return GNUNET_YES; - if (md1->item_count != md2->item_count) - return GNUNET_NO; - for (i = md1->items_head; NULL != i; i = i->next) - { - found = GNUNET_NO; - for (j = md2->items_head; NULL != j; j = j->next) - { - if ((i->type == j->type) && (i->format == j->format) && - (i->data_size == j->data_size) && - (0 == memcmp (i->data, j->data, i->data_size))) - { - found = GNUNET_YES; - break; - } - if (j->data_size < i->data_size) - break; /* elements are sorted by (decreasing) size... */ - } - if (GNUNET_NO == found) - return GNUNET_NO; - } - return GNUNET_YES; -} - - -/** - * Extend metadata. Note that the list of meta data items is - * sorted by size (largest first). - * - * @param md metadata to extend - * @param plugin_name name of the plugin that produced this value; - * special values can be used (e.g. '<zlib>' for zlib being - * used in the main libextractor library and yielding - * meta data). - * @param type libextractor-type describing the meta data - * @param format basic format information about data - * @param data_mime_type mime-type of data (not of the original file); - * can be NULL (if mime-type is not known) - * @param data actual meta-data found - * @param data_size number of bytes in @a data - * @return #GNUNET_OK on success, #GNUNET_SYSERR if this entry already exists - * data_mime_type and plugin_name are not considered for "exists" checks - */ -int -GNUNET_FS_meta_data_insert (struct GNUNET_FS_MetaData *md, - const char *plugin_name, - enum EXTRACTOR_MetaType type, - enum EXTRACTOR_MetaFormat format, - const char *data_mime_type, const char *data, - size_t data_size) -{ - struct MetaItem *pos; - struct MetaItem *mi; - char *p; - - if ((EXTRACTOR_METAFORMAT_UTF8 == format) || - (EXTRACTOR_METAFORMAT_C_STRING == format)) - GNUNET_break ('\0' == data[data_size - 1]); - - for (pos = md->items_head; NULL != pos; pos = pos->next) - { - if (pos->data_size < data_size) - break; /* elements are sorted by size in the list */ - if ((pos->type == type) && (pos->data_size == data_size) && - (0 == memcmp (pos->data, data, data_size))) - { - if ((NULL == pos->mime_type) && (NULL != data_mime_type)) - { - pos->mime_type = GNUNET_strdup (data_mime_type); - invalidate_sbuf (md); - } - if ((EXTRACTOR_METAFORMAT_C_STRING == pos->format) && - (EXTRACTOR_METAFORMAT_UTF8 == format)) - { - pos->format = EXTRACTOR_METAFORMAT_UTF8; - invalidate_sbuf (md); - } - return GNUNET_SYSERR; - } - } - md->item_count++; - mi = GNUNET_new (struct MetaItem); - mi->type = type; - mi->format = format; - mi->data_size = data_size; - if (NULL == pos) - GNUNET_CONTAINER_DLL_insert_tail (md->items_head, - md->items_tail, - mi); - else - GNUNET_CONTAINER_DLL_insert_after (md->items_head, - md->items_tail, - pos->prev, - mi); - mi->mime_type = - (NULL == data_mime_type) ? NULL : GNUNET_strdup (data_mime_type); - mi->plugin_name = (NULL == plugin_name) ? NULL : GNUNET_strdup (plugin_name); - mi->data = GNUNET_malloc (data_size); - GNUNET_memcpy (mi->data, data, data_size); - /* change all dir separators to POSIX style ('/') */ - if ((EXTRACTOR_METATYPE_FILENAME == type) || - (EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME == type)) - { - p = mi->data; - while (('\0' != *p) && (p < mi->data + data_size)) - { - if ('\\' == *p) - *p = '/'; - p++; - } - } - invalidate_sbuf (md); - return GNUNET_OK; -} - - -/** - * Merge given meta data. - * - * @param cls the `struct GNUNET_FS_MetaData` to merge into - * @param plugin_name name of the plugin that produced this value; - * special values can be used (e.g. '<zlib>' for zlib being - * used in the main libextractor library and yielding - * meta data). - * @param type libextractor-type describing the meta data - * @param format basic format information about data - * @param data_mime_type mime-type of data (not of the original file); - * can be NULL (if mime-type is not known) - * @param data actual meta-data found - * @param data_size number of bytes in @a data - * @return 0 (to continue) - */ -static int -merge_helper (void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type, - enum EXTRACTOR_MetaFormat format, const char *data_mime_type, - const char *data, size_t data_size) -{ - struct GNUNET_FS_MetaData *md = cls; - - (void) GNUNET_FS_meta_data_insert (md, plugin_name, type, format, - data_mime_type, data, data_size); - return 0; -} - - -void -GNUNET_FS_meta_data_merge (struct GNUNET_FS_MetaData *md, - const struct GNUNET_FS_MetaData *in) -{ - GNUNET_FS_meta_data_iterate (in, &merge_helper, md); -} - - -int -GNUNET_FS_meta_data_delete (struct GNUNET_FS_MetaData *md, - enum EXTRACTOR_MetaType type, - const char *data, size_t data_size) -{ - struct MetaItem *pos; - - for (pos = md->items_head; NULL != pos; pos = pos->next) - { - if (pos->data_size < data_size) - break; /* items are sorted by (decreasing) size */ - if ((pos->type == type) && - ((NULL == data) || - ((pos->data_size == data_size) && - (0 == memcmp (pos->data, data, data_size))))) - { - GNUNET_CONTAINER_DLL_remove (md->items_head, md->items_tail, pos); - meta_item_free (pos); - md->item_count--; - invalidate_sbuf (md); - return GNUNET_OK; - } - } - return GNUNET_SYSERR; -} - - -void -GNUNET_FS_meta_data_add_publication_date (struct - GNUNET_FS_MetaData *md) -{ - const char *dat; - struct GNUNET_TIME_Absolute t; - - t = GNUNET_TIME_absolute_get (); - GNUNET_FS_meta_data_delete (md, - EXTRACTOR_METATYPE_PUBLICATION_DATE, - NULL, 0); - dat = GNUNET_STRINGS_absolute_time_to_string (t); - GNUNET_FS_meta_data_insert (md, "", - EXTRACTOR_METATYPE_PUBLICATION_DATE, - EXTRACTOR_METAFORMAT_UTF8, "text/plain", - dat, strlen (dat) + 1); -} - - -/** - * Iterate over MD entries. - * - * @param md metadata to inspect - * @param iter function to call on each entry - * @param iter_cls closure for iterator - * @return number of entries - */ -int -GNUNET_FS_meta_data_iterate (const struct GNUNET_FS_MetaData *md, - EXTRACTOR_MetaDataProcessor iter, - void *iter_cls) -{ - struct MetaItem *pos; - - if (NULL == md) - return 0; - if (NULL == iter) - return md->item_count; - for (pos = md->items_head; NULL != pos; pos = pos->next) - if (0 != - iter (iter_cls, pos->plugin_name, pos->type, pos->format, - pos->mime_type, pos->data, pos->data_size)) - return md->item_count; - return md->item_count; -} - - -char * -GNUNET_FS_meta_data_get_by_type (const struct - GNUNET_FS_MetaData *md, - enum EXTRACTOR_MetaType type) -{ - struct MetaItem *pos; - - if (NULL == md) - return NULL; - for (pos = md->items_head; NULL != pos; pos = pos->next) - if ((type == pos->type) && - ((pos->format == EXTRACTOR_METAFORMAT_UTF8) || - (pos->format == EXTRACTOR_METAFORMAT_C_STRING))) - return GNUNET_strdup (pos->data); - return NULL; -} - - -char * -GNUNET_FS_meta_data_get_first_by_types (const struct - GNUNET_FS_MetaData *md, - ...) -{ - char *ret; - va_list args; - int type; - - if (NULL == md) - return NULL; - ret = NULL; - va_start (args, md); - while (1) - { - type = va_arg (args, int); - if (-1 == type) - break; - if (NULL != (ret = GNUNET_FS_meta_data_get_by_type (md, type))) - break; - } - va_end (args); - return ret; -} - - -/** - * Get a thumbnail from the meta-data (if present). - * - * @param md metadata to get the thumbnail from - * @param thumb will be set to the thumbnail data. Must be - * freed by the caller! - * @return number of bytes in thumbnail, 0 if not available - */ -size_t -GNUNET_FS_meta_data_get_thumbnail (const struct GNUNET_FS_MetaData - *md, unsigned char **thumb) -{ - struct MetaItem *pos; - struct MetaItem *match; - - if (NULL == md) - return 0; - match = NULL; - for (pos = md->items_head; NULL != pos; pos = pos->next) - { - if ((NULL != pos->mime_type) && - (0 == strncasecmp ("image/", pos->mime_type, strlen ("image/"))) && - (EXTRACTOR_METAFORMAT_BINARY == pos->format)) - { - if (NULL == match) - match = pos; - else if ((match->type != EXTRACTOR_METATYPE_THUMBNAIL) && - (pos->type == EXTRACTOR_METATYPE_THUMBNAIL)) - match = pos; - } - } - if ((NULL == match) || (0 == match->data_size)) - return 0; - *thumb = GNUNET_malloc (match->data_size); - GNUNET_memcpy (*thumb, match->data, match->data_size); - return match->data_size; -} - - -/** - * Duplicate a `struct GNUNET_FS_MetaData`. - * - * @param md what to duplicate - * @return duplicate meta-data container - */ -struct GNUNET_FS_MetaData * -GNUNET_FS_meta_data_duplicate (const struct GNUNET_FS_MetaData - *md) -{ - struct GNUNET_FS_MetaData *ret; - struct MetaItem *pos; - - if (NULL == md) - return NULL; - ret = GNUNET_FS_meta_data_create (); - for (pos = md->items_tail; NULL != pos; pos = pos->prev) - GNUNET_FS_meta_data_insert (ret, pos->plugin_name, pos->type, - pos->format, pos->mime_type, pos->data, - pos->data_size); - return ret; -} - - -/** - * Flag in 'version' that indicates compressed meta-data. - */ -#define HEADER_COMPRESSED 0x80000000 - - -/** - * Bits in 'version' that give the version number. - */ -#define HEADER_VERSION_MASK 0x7FFFFFFF - - -/** - * Header for serialized meta data. - */ -struct MetaDataHeader -{ - /** - * The version of the MD serialization. The highest bit is used to - * indicate compression. - * - * Version 0 is traditional (pre-0.9) meta data (unsupported) - * Version is 1 for a NULL pointer - * Version 2 is for 0.9.x (and possibly higher) - * Other version numbers are not yet defined. - */ - uint32_t version; - - /** - * How many MD entries are there? - */ - uint32_t entries; - - /** - * Size of the decompressed meta data. - */ - uint32_t size; - - /** - * This is followed by 'entries' values of type 'struct MetaDataEntry' - * and then by 'entry' plugin names, mime-types and data blocks - * as specified in those meta data entries. - */ -}; - - -/** - * Entry of serialized meta data. - */ -struct MetaDataEntry -{ - /** - * Meta data type. Corresponds to an 'enum EXTRACTOR_MetaType' - */ - uint32_t type; - - /** - * Meta data format. Corresponds to an 'enum EXTRACTOR_MetaFormat' - */ - uint32_t format; - - /** - * Number of bytes of meta data. - */ - uint32_t data_size; - - /** - * Number of bytes in the plugin name including 0-terminator. 0 for NULL. - */ - uint32_t plugin_name_len; - - /** - * Number of bytes in the mime type including 0-terminator. 0 for NULL. - */ - uint32_t mime_type_len; -}; - - -/** - * Serialize meta-data to target. - * - * @param md metadata to serialize - * @param target where to write the serialized metadata; - * *target can be NULL, in which case memory is allocated - * @param max maximum number of bytes available in target - * @param opt is it ok to just write SOME of the - * meta-data to match the size constraint, - * possibly discarding some data? - * @return number of bytes written on success, - * #GNUNET_SYSERR on error (typically: not enough - * space) - */ -ssize_t -GNUNET_FS_meta_data_serialize (const struct GNUNET_FS_MetaData - *md, char **target, size_t max, - enum - GNUNET_FS_MetaDataSerializationOptions - opt) -{ - struct GNUNET_FS_MetaData *vmd; - struct MetaItem *pos; - struct MetaDataHeader ihdr; - struct MetaDataHeader *hdr; - struct MetaDataEntry *ent; - char *dst; - unsigned int i; - uint64_t msize; - size_t off; - char *mdata; - char *cdata; - size_t mlen; - size_t plen; - size_t size; - size_t left; - size_t clen; - size_t rlen; - int comp; - - if (max < sizeof(struct MetaDataHeader)) - return GNUNET_SYSERR; /* far too small */ - if (NULL == md) - return 0; - - if (NULL != md->sbuf) - { - /* try to use serialization cache */ - if (md->sbuf_size <= max) - { - if (NULL == *target) - *target = GNUNET_malloc (md->sbuf_size); - GNUNET_memcpy (*target, md->sbuf, md->sbuf_size); - return md->sbuf_size; - } - if (0 == (opt & GNUNET_FS_META_DATA_SERIALIZE_PART)) - return GNUNET_SYSERR; /* can say that this will fail */ - /* need to compute a partial serialization, sbuf useless ... */ - } - dst = NULL; - msize = 0; - for (pos = md->items_tail; NULL != pos; pos = pos->prev) - { - msize += sizeof(struct MetaDataEntry); - msize += pos->data_size; - if (NULL != pos->plugin_name) - msize += strlen (pos->plugin_name) + 1; - if (NULL != pos->mime_type) - msize += strlen (pos->mime_type) + 1; - } - size = (size_t) msize; - if (size != msize) - { - GNUNET_break (0); /* integer overflow */ - return GNUNET_SYSERR; - } - if (size >= GNUNET_MAX_MALLOC_CHECKED) - { - /* too large to be processed */ - return GNUNET_SYSERR; - } - ent = GNUNET_malloc (size); - mdata = (char *) &ent[md->item_count]; - off = size - (md->item_count * sizeof(struct MetaDataEntry)); - i = 0; - for (pos = md->items_head; NULL != pos; pos = pos->next) - { - ent[i].type = htonl ((uint32_t) pos->type); - ent[i].format = htonl ((uint32_t) pos->format); - ent[i].data_size = htonl ((uint32_t) pos->data_size); - if (NULL == pos->plugin_name) - plen = 0; - else - plen = strlen (pos->plugin_name) + 1; - ent[i].plugin_name_len = htonl ((uint32_t) plen); - if (NULL == pos->mime_type) - mlen = 0; - else - mlen = strlen (pos->mime_type) + 1; - ent[i].mime_type_len = htonl ((uint32_t) mlen); - off -= pos->data_size; - if ((EXTRACTOR_METAFORMAT_UTF8 == pos->format) || - (EXTRACTOR_METAFORMAT_C_STRING == pos->format)) - GNUNET_break ('\0' == pos->data[pos->data_size - 1]); - GNUNET_memcpy (&mdata[off], pos->data, pos->data_size); - off -= plen; - if (NULL != pos->plugin_name) - GNUNET_memcpy (&mdata[off], pos->plugin_name, plen); - off -= mlen; - if (NULL != pos->mime_type) - GNUNET_memcpy (&mdata[off], pos->mime_type, mlen); - i++; - } - GNUNET_assert (0 == off); - - clen = 0; - cdata = NULL; - left = size; - i = 0; - for (pos = md->items_head; NULL != pos; pos = pos->next) - { - comp = GNUNET_NO; - if (0 == (opt & GNUNET_FS_META_DATA_SERIALIZE_NO_COMPRESS)) - comp = GNUNET_try_compression ((const char *) &ent[i], - left, - &cdata, - &clen); - - if ((NULL == md->sbuf) && (0 == i)) - { - /* fill 'sbuf'; this "modifies" md, but since this is only - * an internal cache we will cast away the 'const' instead - * of making the API look strange. */ - vmd = (struct GNUNET_FS_MetaData *) md; - hdr = GNUNET_malloc (left + sizeof(struct MetaDataHeader)); - hdr->size = htonl (left); - hdr->entries = htonl (md->item_count); - if (GNUNET_YES == comp) - { - GNUNET_assert (clen < left); - hdr->version = htonl (2 | HEADER_COMPRESSED); - GNUNET_memcpy (&hdr[1], cdata, clen); - vmd->sbuf_size = clen + sizeof(struct MetaDataHeader); - } - else - { - hdr->version = htonl (2); - GNUNET_memcpy (&hdr[1], &ent[0], left); - vmd->sbuf_size = left + sizeof(struct MetaDataHeader); - } - vmd->sbuf = (char *) hdr; - } - - if (((left + sizeof(struct MetaDataHeader)) <= max) || - ((GNUNET_YES == comp) && (clen <= max))) - { - /* success, this now fits! */ - if (GNUNET_YES == comp) - { - if (NULL == dst) - dst = GNUNET_malloc (clen + sizeof(struct MetaDataHeader)); - hdr = (struct MetaDataHeader *) dst; - hdr->version = htonl (2 | HEADER_COMPRESSED); - hdr->size = htonl (left); - hdr->entries = htonl (md->item_count - i); - GNUNET_memcpy (&dst[sizeof(struct MetaDataHeader)], cdata, clen); - GNUNET_free (cdata); - cdata = NULL; - GNUNET_free (ent); - rlen = clen + sizeof(struct MetaDataHeader); - } - else - { - if (NULL == dst) - dst = GNUNET_malloc (left + sizeof(struct MetaDataHeader)); - hdr = (struct MetaDataHeader *) dst; - hdr->version = htonl (2); - hdr->entries = htonl (md->item_count - i); - hdr->size = htonl (left); - GNUNET_memcpy (&dst[sizeof(struct MetaDataHeader)], &ent[i], left); - GNUNET_free (ent); - rlen = left + sizeof(struct MetaDataHeader); - } - if (NULL != *target) - { - if (GNUNET_YES == comp) - GNUNET_memcpy (*target, dst, clen + sizeof(struct MetaDataHeader)); - else - GNUNET_memcpy (*target, dst, left + sizeof(struct MetaDataHeader)); - GNUNET_free (dst); - } - else - { - *target = dst; - } - return rlen; - } - - if (0 == (opt & GNUNET_FS_META_DATA_SERIALIZE_PART)) - { - /* does not fit! */ - GNUNET_free (ent); - if (NULL != cdata) - GNUNET_free (cdata); - cdata = NULL; - return GNUNET_SYSERR; - } - - /* next iteration: ignore the corresponding meta data at the - * end and try again without it */ - left -= sizeof(struct MetaDataEntry); - left -= pos->data_size; - if (NULL != pos->plugin_name) - left -= strlen (pos->plugin_name) + 1; - if (NULL != pos->mime_type) - left -= strlen (pos->mime_type) + 1; - - if (NULL != cdata) - GNUNET_free (cdata); - cdata = NULL; - i++; - } - GNUNET_free (ent); - - /* nothing fit, only write header! */ - ihdr.version = htonl (2); - ihdr.entries = htonl (0); - ihdr.size = htonl (0); - if (NULL == *target) - *target = (char *) GNUNET_new (struct MetaDataHeader); - GNUNET_memcpy (*target, &ihdr, sizeof(struct MetaDataHeader)); - return sizeof(struct MetaDataHeader); -} - - -ssize_t -GNUNET_FS_meta_data_get_serialized_size (const struct - GNUNET_FS_MetaData *md) -{ - ssize_t ret; - char *ptr; - - if (NULL != md->sbuf) - return md->sbuf_size; - ptr = NULL; - ret = - GNUNET_FS_meta_data_serialize (md, &ptr, GNUNET_MAX_MALLOC_CHECKED, - GNUNET_FS_META_DATA_SERIALIZE_FULL); - if (-1 != ret) - GNUNET_free (ptr); - return ret; -} - - -/** - * Deserialize meta-data. Initializes md. - * - * @param input buffer with the serialized metadata - * @param size number of bytes available in input - * @return MD on success, NULL on error (i.e. - * bad format) - */ -struct GNUNET_FS_MetaData * -GNUNET_FS_meta_data_deserialize (const char *input, size_t size) -{ - struct GNUNET_FS_MetaData *md; - struct MetaDataHeader hdr; - struct MetaDataEntry ent; - uint32_t ic; - uint32_t i; - char *data; - const char *cdata; - uint32_t version; - uint32_t dataSize; - int compressed; - size_t left; - uint32_t mlen; - uint32_t plen; - uint32_t dlen; - const char *mdata; - const char *meta_data; - const char *plugin_name; - const char *mime_type; - enum EXTRACTOR_MetaFormat format; - - if (size < sizeof(struct MetaDataHeader)) - return NULL; - GNUNET_memcpy (&hdr, input, sizeof(struct MetaDataHeader)); - version = ntohl (hdr.version) & HEADER_VERSION_MASK; - compressed = (ntohl (hdr.version) & HEADER_COMPRESSED) != 0; - - if (1 == version) - return NULL; /* null pointer */ - if (2 != version) - { - GNUNET_break_op (0); /* unsupported version */ - return NULL; - } - - ic = ntohl (hdr.entries); - dataSize = ntohl (hdr.size); - if (((sizeof(struct MetaDataEntry) * ic) > dataSize) || - ((0 != ic) && - (dataSize / ic < sizeof(struct MetaDataEntry)))) - { - GNUNET_break_op (0); - return NULL; - } - - if (compressed) - { - if (dataSize >= GNUNET_MAX_MALLOC_CHECKED) - { - /* make sure we don't blow our memory limit because of a mal-formed - * message... */ - GNUNET_break_op (0); - return NULL; - } - data = - GNUNET_decompress ((const char *) &input[sizeof(struct MetaDataHeader)], - size - sizeof(struct MetaDataHeader), - dataSize); - if (NULL == data) - { - GNUNET_break_op (0); - return NULL; - } - cdata = data; - } - else - { - data = NULL; - cdata = (const char *) &input[sizeof(struct MetaDataHeader)]; - if (dataSize != size - sizeof(struct MetaDataHeader)) - { - GNUNET_break_op (0); - return NULL; - } - } - - md = GNUNET_FS_meta_data_create (); - left = dataSize - ic * sizeof(struct MetaDataEntry); - mdata = &cdata[ic * sizeof(struct MetaDataEntry)]; - for (i = 0; i < ic; i++) - { - GNUNET_memcpy (&ent, &cdata[i * sizeof(struct MetaDataEntry)], - sizeof(struct MetaDataEntry)); - format = (enum EXTRACTOR_MetaFormat) ntohl (ent.format); - if ((EXTRACTOR_METAFORMAT_UTF8 != format) && - (EXTRACTOR_METAFORMAT_C_STRING != format) && - (EXTRACTOR_METAFORMAT_BINARY != format)) - { - GNUNET_break_op (0); - break; - } - dlen = ntohl (ent.data_size); - plen = ntohl (ent.plugin_name_len); - mlen = ntohl (ent.mime_type_len); - if (dlen > left) - { - GNUNET_break_op (0); - break; - } - left -= dlen; - meta_data = &mdata[left]; - if ((EXTRACTOR_METAFORMAT_UTF8 == format) || - (EXTRACTOR_METAFORMAT_C_STRING == format)) - { - if (0 == dlen) - { - GNUNET_break_op (0); - break; - } - if ('\0' != meta_data[dlen - 1]) - { - GNUNET_break_op (0); - break; - } - } - if (plen > left) - { - GNUNET_break_op (0); - break; - } - left -= plen; - if ((plen > 0) && ('\0' != mdata[left + plen - 1])) - { - GNUNET_break_op (0); - break; - } - if (0 == plen) - plugin_name = NULL; - else - plugin_name = &mdata[left]; - - if (mlen > left) - { - GNUNET_break_op (0); - break; - } - left -= mlen; - if ((mlen > 0) && ('\0' != mdata[left + mlen - 1])) - { - GNUNET_break_op (0); - break; - } - if (0 == mlen) - mime_type = NULL; - else - mime_type = &mdata[left]; - GNUNET_FS_meta_data_insert (md, plugin_name, - (enum EXTRACTOR_MetaType) - ntohl (ent.type), format, mime_type, - meta_data, dlen); - } - GNUNET_free (data); - return md; -} - -/** - * Read a metadata container. - * - * @param h handle to an open file - * @param what describes what is being read (for error message creation) - * @param result the buffer to store a pointer to the (allocated) metadata - * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure - */ -enum GNUNET_GenericReturnValue -GNUNET_FS_read_meta_data (struct GNUNET_BIO_ReadHandle *h, - const char *what, - struct GNUNET_FS_MetaData **result) -{ - uint32_t size; - char *buf; - char *emsg; - struct GNUNET_FS_MetaData *meta; - - if (GNUNET_OK != GNUNET_BIO_read_int32 (h, - _ ("metadata length"), - (int32_t *) &size)) - return GNUNET_SYSERR; - if (0 == size) - { - *result = NULL; - return GNUNET_OK; - } - if (MAX_META_DATA < size) - { - GNUNET_asprintf (&emsg, - _ ( - "Serialized metadata `%s' larger than allowed (%u > %u)\n"), - what, - size, - MAX_META_DATA); - GNUNET_BIO_read_set_error (h, emsg); - GNUNET_free (emsg); - return GNUNET_SYSERR; - } - buf = GNUNET_malloc (size); - if (GNUNET_OK != GNUNET_BIO_read (h, what, buf, size)) - { - GNUNET_free (buf); - return GNUNET_SYSERR; - } - meta = GNUNET_FS_meta_data_deserialize (buf, size); - if (NULL == meta) - { - GNUNET_free (buf); - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Failed to deserialize metadata `%s'"), what); - return GNUNET_SYSERR; - } - GNUNET_free (buf); - *result = meta; - return GNUNET_OK; -} - -/** - * Write a metadata container. - * - * @param h the IO handle to write to - * @param what what is being written (for error message creation) - * @param m metadata to write - * @return #GNUNET_OK on success, #GNUNET_SYSERR on error - */ -enum GNUNET_GenericReturnValue -GNUNET_FS_write_meta_data (struct GNUNET_BIO_WriteHandle *h, - const char *what, - const struct GNUNET_FS_MetaData *m) -{ - ssize_t size; - char *buf; - - if (m == NULL) - return GNUNET_BIO_write_int32 (h, _ ("metadata length"), 0); - buf = NULL; - size = GNUNET_FS_meta_data_serialize (m, - &buf, - MAX_META_DATA, - GNUNET_FS_META_DATA_SERIALIZE_PART); - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Serialized %ld bytes of metadata"), - size); - - if (-1 == size) - { - GNUNET_free (buf); - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _ ("Failed to serialize metadata `%s'"), - what); - return GNUNET_SYSERR; - } - if ((GNUNET_OK != GNUNET_BIO_write_int32 (h, - _ ("metadata length"), - (uint32_t) size)) - || (GNUNET_OK != GNUNET_BIO_write (h, what, buf, size))) - { - GNUNET_free (buf); - return GNUNET_SYSERR; - } - GNUNET_free (buf); - return GNUNET_OK; -} - -/** - * Function used internally to read a metadata container from within a read - * spec. - * - * @param cls ignored, always NULL - * @param h the IO handle to read from - * @param what what is being read (for error message creation) - * @param target where to store the data - * @param target_size ignored - * @return #GNUNET_OK on success, #GNUNET_SYSERR on error - */ -static int -read_spec_handler_meta_data (void *cls, - struct GNUNET_BIO_ReadHandle *h, - const char *what, - void *target, - size_t target_size) -{ - struct GNUNET_FS_MetaData **result = target; - return GNUNET_FS_read_meta_data (h, what, result); -} - -/** - * Create the specification to read a metadata container. - * - * @param what describes what is being read (for error message creation) - * @param result the buffer to store a pointer to the (allocated) metadata - * @return the read spec - */ -struct GNUNET_BIO_ReadSpec -GNUNET_FS_read_spec_meta_data (const char *what, - struct GNUNET_FS_MetaData **result) -{ - struct GNUNET_BIO_ReadSpec rs = { - .rh = &read_spec_handler_meta_data, - .cls = NULL, - .target = result, - .size = 0, - }; - - return rs; -} - -/** - * Function used internally to write a metadata container from within a write - * spec. - * - * @param cls ignored, always NULL - * @param h the IO handle to write to - * @param what what is being written (for error message creation) - * @param source the data to write - * @param source_size ignored - * @return #GNUNET_OK on success, #GNUNET_SYSERR otherwise - */ -static int -write_spec_handler_meta_data (void *cls, - struct GNUNET_BIO_WriteHandle *h, - const char *what, - void *source, - size_t source_size) -{ - const struct GNUNET_FS_MetaData *m = source; - return GNUNET_FS_write_meta_data (h, what, m); -} - - -struct GNUNET_BIO_WriteSpec -GNUNET_FS_write_spec_meta_data (const char *what, - const struct GNUNET_FS_MetaData *m) -{ - struct GNUNET_BIO_WriteSpec ws = { - .wh = &write_spec_handler_meta_data, - .cls = NULL, - .what = what, - .source = (void *) m, - .source_size = 0, - }; - - return ws; -} - - -/* end of meta_data.c */ diff --git a/src/fs/perf_gnunet_service_fs_p2p.c b/src/fs/perf_gnunet_service_fs_p2p.c deleted file mode 100644 index 2c7830f5f..000000000 --- a/src/fs/perf_gnunet_service_fs_p2p.c +++ /dev/null @@ -1,370 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2010, 2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/perf_gnunet_service_fs_p2p.c - * @brief profile P2P routing using simple publish + download operation - * @author Christian Grothoff - */ -#include "platform.h" -#include "fs_test_lib.h" -#include "gnunet_testbed_service.h" - -#define VERBOSE GNUNET_NO - -/** - * File-size we use for testing. - */ -#define FILESIZE (1024 * 1024 * 10) - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30) - -#define NUM_DAEMONS 2 - -#define SEED 42 - -static struct GNUNET_TESTBED_Peer *daemons[NUM_DAEMONS]; - -static int ok; - -static struct GNUNET_TIME_Absolute start_time; - -static const char *progname; - -static struct GNUNET_TIME_Absolute start_time; - - -/** - * Master context for 'stat_run'. - */ -struct StatMaster -{ - struct GNUNET_STATISTICS_Handle *stat; - struct GNUNET_TESTBED_Operation *op; - unsigned int daemon; - unsigned int value; -}; - -struct StatValues -{ - const char *subsystem; - const char *name; -}; - -/** - * Statistics we print out. - */ -static struct StatValues stats[] = { - { "fs", "# queries forwarded" }, - { "fs", "# replies received and matched" }, - { "fs", "# results found locally" }, - { "fs", "# requests forwarded due to high load" }, - { "fs", "# requests done for free (low load)" }, - { "fs", "# requests dropped, priority insufficient" }, - { "fs", "# requests done for a price (normal load)" }, - { "fs", "# requests dropped by datastore (queue length limit)" }, - { "fs", "# P2P searches received" }, - { "fs", "# P2P searches discarded (queue length bound)" }, - { "fs", "# replies received for local clients" }, - { "fs", "# queries retransmitted to same target" }, - { "core", "# bytes decrypted" }, - { "core", "# bytes encrypted" }, - { "core", "# discarded CORE_SEND requests" }, - { "core", "# discarded CORE_SEND request bytes" }, - { "core", "# discarded lower priority CORE_SEND requests" }, - { "core", "# discarded lower priority CORE_SEND request bytes" }, - { "transport", "# bytes received via TCP" }, - { "transport", "# bytes transmitted via TCP" }, - { "datacache", "# bytes stored" }, - { NULL, NULL } -}; - - -/** - * Callback function to process statistic values. - * - * @param cls closure - * @param subsystem name of subsystem that created the statistic - * @param name the name of the datum - * @param value the current value - * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if not - * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration - */ -static int -print_stat (void *cls, const char *subsystem, const char *name, uint64_t value, - int is_persistent) -{ - struct StatMaster *sm = cls; - - fprintf (stderr, - "Peer %2u: %12s/%50s = %12llu\n", - sm->daemon, - subsystem, - name, - (unsigned long long) value); - return GNUNET_OK; -} - - -/** - * Function that gathers stats from all daemons. - */ -static void -stat_run (void *cls, - struct GNUNET_TESTBED_Operation *op, - void *ca_result, - const char *emsg); - - -/** - * Function called when GET operation on stats is done. - */ -static void -get_done (void *cls, int success) -{ - struct StatMaster *sm = cls; - - GNUNET_break (GNUNET_OK == success); - sm->value++; - stat_run (sm, sm->op, sm->stat, NULL); -} - - -/** - * Adapter function called to establish a connection to - * statistics service. - * - * @param cls closure - * @param cfg configuration of the peer to connect to; will be available until - * GNUNET_TESTBED_operation_done() is called on the operation returned - * from GNUNET_TESTBED_service_connect() - * @return service handle to return in 'op_result', NULL on error - */ -static void * -statistics_connect_adapter (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg) -{ - return GNUNET_STATISTICS_create ("", - cfg); -} - - -/** - * Adapter function called to destroy a connection to - * statistics service. - * - * @param cls closure - * @param op_result service handle returned from the connect adapter - */ -static void -statistics_disconnect_adapter (void *cls, - void *op_result) -{ - GNUNET_STATISTICS_destroy (op_result, GNUNET_NO); -} - - -/** - * Function that gathers stats from all daemons. - */ -static void -stat_run (void *cls, - struct GNUNET_TESTBED_Operation *op, - void *ca_result, - const char *emsg) -{ - struct StatMaster *sm = cls; - - if (NULL != emsg) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Failed to statistics service: %s\n", - emsg); - GNUNET_SCHEDULER_shutdown (); - return; - } - sm->stat = ca_result; - - if (stats[sm->value].name != NULL) - { - GNUNET_STATISTICS_get (sm->stat, -#if 0 - NULL, NULL, -#else - stats[sm->value].subsystem, stats[sm->value].name, -#endif - &get_done, &print_stat, - sm); - return; - } - GNUNET_TESTBED_operation_done (sm->op); - sm->value = 0; - sm->daemon++; - if (NUM_DAEMONS == sm->daemon) - { - GNUNET_free (sm); - GNUNET_SCHEDULER_shutdown (); - return; - } - sm->op = - GNUNET_TESTBED_service_connect (NULL, - daemons[sm->daemon], - "statistics", - &stat_run, sm, - &statistics_connect_adapter, - &statistics_disconnect_adapter, - NULL); -} - - -static void -do_report (void *cls) -{ - char *fn = cls; - struct GNUNET_TIME_Relative del; - char *fancy; - struct StatMaster *sm; - - if (NULL != fn) - { - GNUNET_DISK_directory_remove (fn); - GNUNET_free (fn); - } - if (0 == - GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_add (start_time, - TIMEOUT)). - rel_value_us) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Timeout during download, shutting down with error\n"); - ok = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - - del = GNUNET_TIME_absolute_get_duration (start_time); - if (del.rel_value_us == 0) - del.rel_value_us = 1; - fancy = - GNUNET_STRINGS_byte_size_fancy (((unsigned long long) FILESIZE) - * 1000000LL / del.rel_value_us); - fprintf (stdout, - "Download speed was %s/s\n", - fancy); - GNUNET_free (fancy); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Finished download, shutting down\n"); - sm = GNUNET_new (struct StatMaster); - sm->op = - GNUNET_TESTBED_service_connect (NULL, - daemons[sm->daemon], - "statistics", - &stat_run, sm, - &statistics_connect_adapter, - &statistics_disconnect_adapter, - NULL); -} - - -static void -do_download (void *cls, - const struct GNUNET_FS_Uri *uri, - const char *fn) -{ - int anonymity; - - if (NULL == uri) - { - GNUNET_SCHEDULER_shutdown (); - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Timeout during upload attempt, shutting down with error\n"); - ok = 1; - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Downloading %llu bytes\n", - (unsigned long long) FILESIZE); - start_time = GNUNET_TIME_absolute_get (); - if (NULL != strstr (progname, "dht")) - anonymity = 0; - else - anonymity = 1; - start_time = GNUNET_TIME_absolute_get (); - GNUNET_FS_TEST_download (daemons[0], - TIMEOUT, - anonymity, - SEED, - uri, - VERBOSE, - &do_report, - (NULL == fn) ? NULL : GNUNET_strdup (fn)); -} - - -static void -do_publish (void *cls, - struct GNUNET_TESTBED_RunHandle *h, - unsigned int num_peers, - struct GNUNET_TESTBED_Peer **peers, - unsigned int links_succeeded, - unsigned int links_failed) -{ - unsigned int i; - int do_index; - int anonymity; - - GNUNET_assert (NUM_DAEMONS == num_peers); - for (i = 0; i < num_peers; i++) - daemons[i] = peers[i]; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n", - (unsigned long long) FILESIZE); - if (NULL != strstr (progname, "index")) - do_index = GNUNET_YES; - else - do_index = GNUNET_NO; - if (NULL != strstr (progname, "dht")) - anonymity = 0; - else - anonymity = 1; - GNUNET_FS_TEST_publish (daemons[NUM_DAEMONS - 1], TIMEOUT, anonymity, - do_index, FILESIZE, SEED, VERBOSE, &do_download, - NULL); -} - - -int -main (int argc, char *argv[]) -{ - progname = argv[0]; - (void) GNUNET_TESTBED_test_run ("perf-gnunet-service-fs-p2p", - "perf_gnunet_service_fs_p2p.conf", - NUM_DAEMONS, - 0, NULL, NULL, - &do_publish, NULL); - GNUNET_DISK_purge_cfg_dir - ("perf_gnunet_service_fs_p2p.conf", - "GNUNET_TEST_HOME"); - return ok; -} - - -/* end of perf_gnunet_service_fs_p2p.c */ diff --git a/src/fs/perf_gnunet_service_fs_p2p.conf b/src/fs/perf_gnunet_service_fs_p2p.conf deleted file mode 100644 index 00f0f512e..000000000 --- a/src/fs/perf_gnunet_service_fs_p2p.conf +++ /dev/null @@ -1,7 +0,0 @@ -@INLINE@ fs_test_lib_data.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-fs-test-lib/ - -[fs] -GAUGER_HEAP = "2-peer 10 MB P2P download" -# PREFIX = valgrind diff --git a/src/fs/perf_gnunet_service_fs_p2p_respect.c b/src/fs/perf_gnunet_service_fs_p2p_respect.c deleted file mode 100644 index 6b71b1f93..000000000 --- a/src/fs/perf_gnunet_service_fs_p2p_respect.c +++ /dev/null @@ -1,480 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2010, 2011, 2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/perf_gnunet_service_fs_p2p_respect.c - * @brief profile P2P routing respect mechanism. Creates - * a clique of NUM_DAEMONS (at least 3) where two - * peers share (seed) different files and download - * them from each other while all the other peers - * just "leach" those files. Ideally, the seeders - * "learn" that they contribute (to each other), - * and give the other seeder higher priority; - * naturally, this only happens nicely for larger - * files; finally, once the seeders are done, the - * leachers should see fast download rates as well. - * @author Christian Grothoff - * - * Sample output: - * - 10 MB, 3 peers, with delays: - * Download speed of type `seeder 1' was 757 KiB/s - * Download speed of type `seeder 2' was 613 KiB/s - * Download speed of type `leach` was 539 KiB/s - * - * - 10 MB, 3 peers, without delays: - * Download speed of type `seeder 1' was 1784 KiB/s - * Download speed of type `seeder 2' was 1604 KiB/s - * Download speed of type `leach` was 1384 KiB/s - */ -#include "platform.h" -#include "fs_test_lib.h" -#include "gnunet_testbed_service.h" - -#define VERBOSE GNUNET_NO - -/** - * File-size we use for testing. - */ -#define FILESIZE (1024 * 1024 * 1) - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30) - -/** - * Number of daemons in clique, must be at least 3 (!). - */ -#define NUM_DAEMONS 3 - -/** - * Seed for first file on offer. - */ -#define SEED1 42 - -/** - * Seed for second file on offer. - */ -#define SEED2 43 - -static struct GNUNET_TESTBED_Peer *daemons[NUM_DAEMONS]; - -static int ok; - -static struct GNUNET_TIME_Absolute start_time; - -static const char *progname; - -static struct GNUNET_FS_Uri *uri1; - -static struct GNUNET_FS_Uri *uri2; - -static char *fn1; - -static char *fn2; - -/** - * Master context for 'stat_run'. - */ -struct StatMaster -{ - struct GNUNET_STATISTICS_Handle *stat; - struct GNUNET_TESTBED_Operation *op; - unsigned int daemon; - unsigned int value; -}; - -struct StatValues -{ - const char *subsystem; - const char *name; -}; - -/** - * Statistics we print out. - */ -static struct StatValues stats[] = { - { "fs", "# artificial delays introduced (ms)" }, - { "fs", "# queries forwarded" }, - { "fs", "# replies received and matched" }, - { "fs", "# results found locally" }, - { "fs", "# requests forwarded due to high load" }, - { "fs", "# requests done for free (low load)" }, - { "fs", "# requests dropped, priority insufficient" }, - { "fs", "# requests done for a price (normal load)" }, - { "fs", "# requests dropped by datastore (queue length limit)" }, - { "fs", "# P2P searches received" }, - { "fs", "# P2P searches discarded (queue length bound)" }, - { "fs", "# replies received for local clients" }, - { "fs", "# queries retransmitted to same target" }, - { "core", "# bytes decrypted" }, - { "core", "# bytes encrypted" }, - { "core", "# discarded CORE_SEND requests" }, - { "core", "# discarded lower priority CORE_SEND requests" }, - { "transport", "# bytes received via TCP" }, - { "transport", "# bytes transmitted via TCP" }, - { "datacache", "# bytes stored" }, - { NULL, NULL } -}; - - -static void -cleanup () -{ - GNUNET_SCHEDULER_shutdown (); - if (NULL != fn1) - { - GNUNET_DISK_directory_remove (fn1); - GNUNET_free (fn1); - } - if (NULL != fn2) - { - GNUNET_DISK_directory_remove (fn2); - GNUNET_free (fn2); - } -} - - -/** - * Callback function to process statistic values. - * - * @param cls closure - * @param subsystem name of subsystem that created the statistic - * @param name the name of the datum - * @param value the current value - * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not - * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration - */ -static int -print_stat (void *cls, const char *subsystem, const char *name, uint64_t value, - int is_persistent) -{ - struct StatMaster *sm = cls; - - fprintf (stderr, "Peer %2u: %12s/%50s = %12llu\n", sm->daemon, subsystem, - name, (unsigned long long) value); - return GNUNET_OK; -} - - -/** - * Function that gathers stats from all daemons. - */ -static void -stat_run (void *cls, - struct GNUNET_TESTBED_Operation *op, - void *ca_result, - const char *emsg); - - -/** - * Function called when GET operation on stats is done. - */ -static void -get_done (void *cls, int success) -{ - struct StatMaster *sm = cls; - - GNUNET_break (GNUNET_OK == success); - sm->value++; - stat_run (sm, sm->op, sm->stat, NULL); -} - - -/** - * Adapter function called to establish a connection to - * statistics service. - * - * @param cls closure - * @param cfg configuration of the peer to connect to; will be available until - * GNUNET_TESTBED_operation_done() is called on the operation returned - * from GNUNET_TESTBED_service_connect() - * @return service handle to return in 'op_result', NULL on error - */ -static void * -statistics_connect_adapter (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg) -{ - return GNUNET_STATISTICS_create ("", - cfg); -} - - -/** - * Adapter function called to destroy a connection to - * statistics service. - * - * @param cls closure - * @param op_result service handle returned from the connect adapter - */ -static void -statistics_disconnect_adapter (void *cls, - void *op_result) -{ - GNUNET_STATISTICS_destroy (op_result, GNUNET_NO); -} - - -/** - * Function that gathers stats from all daemons. - */ -static void -stat_run (void *cls, - struct GNUNET_TESTBED_Operation *op, - void *ca_result, - const char *emsg) -{ - struct StatMaster *sm = cls; - - sm->stat = ca_result; - GNUNET_assert (NULL != sm->stat); - if (NULL != stats[sm->value].name) - { - GNUNET_STATISTICS_get (sm->stat, -#if 0 - NULL, NULL, -#else - stats[sm->value].subsystem, stats[sm->value].name, -#endif - &get_done, &print_stat, - sm); - return; - } - GNUNET_TESTBED_operation_done (sm->op); - sm->value = 0; - sm->daemon++; - if (NUM_DAEMONS == sm->daemon) - { - GNUNET_free (sm); - cleanup (); - return; - } - sm->op = - GNUNET_TESTBED_service_connect (NULL, - daemons[sm->daemon], - "statistics", - &stat_run, sm, - &statistics_connect_adapter, - &statistics_disconnect_adapter, - NULL); -} - - -static void -do_report (void *cls) -{ - static int download_counter; - const char *type = cls; - struct GNUNET_TIME_Relative del; - char *fancy; - struct StatMaster *sm; - - if (0 == - GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_add (start_time, - TIMEOUT)). - rel_value_us) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Timeout during download for type `%s', shutting down with error\n", - type); - ok = 1; - cleanup (); - return; - } - del = GNUNET_TIME_absolute_get_duration (start_time); - if (del.rel_value_us == 0) - del.rel_value_us = 1; - fancy = - GNUNET_STRINGS_byte_size_fancy (((unsigned long long) FILESIZE) - * 1000000LL / del.rel_value_us); - fprintf (stderr, "Download speed of type `%s' was %s/s\n", type, fancy); - GNUNET_free (fancy); - if (NUM_DAEMONS != ++download_counter) - return; /* more downloads to come */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Finished all downloads, getting statistics\n"); - sm = GNUNET_new (struct StatMaster); - sm->op = - GNUNET_TESTBED_service_connect (NULL, - daemons[sm->daemon], - "statistics", - &stat_run, sm, - &statistics_connect_adapter, - &statistics_disconnect_adapter, - NULL); -} - - -static void -do_downloads (void *cls, const struct GNUNET_FS_Uri *u2, - const char *fn) -{ - int anonymity; - unsigned int i; - - if (NULL == u2) - { - cleanup (); - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Timeout during upload attempt, shutting down with error\n"); - ok = 1; - return; - } - if (NULL != fn) - fn2 = GNUNET_strdup (fn); - uri2 = GNUNET_FS_uri_dup (u2); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Downloading %llu bytes\n", - (unsigned long long) FILESIZE); - start_time = GNUNET_TIME_absolute_get (); - if (NULL != strstr (progname, "dht")) - anonymity = 0; - else - anonymity = 1; - /* (semi) leach-download(s); not true leaches since - * these peers do participate in sharing, they just - * don't have to offer anything *initially*. */ - for (i = 0; i < NUM_DAEMONS - 2; i++) - GNUNET_FS_TEST_download (daemons[i], TIMEOUT, anonymity, - 0 == (i % 2) ? SEED1 : SEED2, - 0 == (i % 2) ? uri1 : uri2, VERBOSE, &do_report, - "leach"); - /* mutual downloads of (primary) sharing peers */ - GNUNET_FS_TEST_download (daemons[NUM_DAEMONS - 2], TIMEOUT, anonymity, SEED1, - uri1, VERBOSE, &do_report, "seeder 2"); - GNUNET_FS_TEST_download (daemons[NUM_DAEMONS - 1], TIMEOUT, anonymity, SEED2, - uri2, VERBOSE, &do_report, "seeder 1"); -} - - -static void -do_publish2 (void *cls, - const struct GNUNET_FS_Uri *u1, - const char *fn) -{ - int do_index; - int anonymity; - - if (NULL == u1) - { - cleanup (); - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Timeout during upload attempt, shutting down with error\n"); - ok = 1; - return; - } - if (NULL != fn) - fn1 = GNUNET_strdup (fn); - uri1 = GNUNET_FS_uri_dup (u1); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n", - (unsigned long long) FILESIZE); - if (NULL != strstr (progname, "index")) - do_index = GNUNET_YES; - else - do_index = GNUNET_NO; - if (NULL != strstr (progname, "dht")) - anonymity = 0; - else - anonymity = 1; - - GNUNET_FS_TEST_publish (daemons[NUM_DAEMONS - 2], TIMEOUT, anonymity, - do_index, FILESIZE, SEED2, VERBOSE, &do_downloads, - NULL); -} - - -static void -do_publish1 (void *cls, - struct GNUNET_TESTBED_Operation *op, - const char *emsg) -{ - unsigned int *coco = cls; - int do_index; - int anonymity; - - GNUNET_TESTBED_operation_done (op); - if (NULL != emsg) - { - cleanup (); - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Error trying to connect: %s\n", emsg); - ok = 1; - return; - } - if (0 != (--(*coco))) - return; /* more connections to be created */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n", - (unsigned long long) FILESIZE); - if (NULL != strstr (progname, "index")) - do_index = GNUNET_YES; - else - do_index = GNUNET_NO; - if (NULL != strstr (progname, "dht")) - anonymity = 0; - else - anonymity = 1; - GNUNET_FS_TEST_publish (daemons[NUM_DAEMONS - 1], TIMEOUT, anonymity, - do_index, FILESIZE, SEED1, VERBOSE, &do_publish2, - NULL); -} - - -static void -do_connect (void *cls, - struct GNUNET_TESTBED_RunHandle *h, - unsigned int num_peers, - struct GNUNET_TESTBED_Peer **peers, - unsigned int links_succeeded, - unsigned int links_failed) -{ - static unsigned int coco; - unsigned int i; - unsigned int j; - - GNUNET_assert (NUM_DAEMONS == num_peers); - for (i = 0; i < num_peers; i++) - daemons[i] = peers[i]; - for (i = 0; i < NUM_DAEMONS; i++) - for (j = i + 1; j < NUM_DAEMONS; j++) - { - coco++; - GNUNET_TESTBED_overlay_connect (NULL, - &do_publish1, - &coco, - peers[i], - peers[j]); - } -} - - -int -main (int argc, char *argv[]) -{ - progname = argv[0]; - (void) GNUNET_TESTBED_test_run ("perf-gnunet-service-fs-p2p-respect", - "perf_gnunet_service_fs_p2p.conf", - NUM_DAEMONS, - 0, NULL, NULL, - &do_connect, NULL); - GNUNET_DISK_purge_cfg_dir ("perf_gnunet_service_fs_p2p.conf", - "GNUNET_TEST_HOME"); - return ok; -} - - -/* end of perf_gnunet_service_fs_p2p_respect.c */ diff --git a/src/fs/plugin_block_fs.c b/src/fs/plugin_block_fs.c deleted file mode 100644 index bbd0ff57b..000000000 --- a/src/fs/plugin_block_fs.c +++ /dev/null @@ -1,337 +0,0 @@ -/* - This file is part of GNUnet - Copyright (C) 2010, 2013, 2021 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/plugin_block_fs.c - * @brief blocks used for file-sharing - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_block_plugin.h" - -#include "gnunet_fs_service.h" -#include "block_fs.h" -#include "gnunet_signatures.h" -#include "gnunet_block_group_lib.h" - - -/** - * Number of bits we set per entry in the bloomfilter. - * Do not change! - */ -#define BLOOMFILTER_K 16 - - -/** - * Create a new block group. - * - * @param ctx block context in which the block group is created - * @param type type of the block for which we are creating the group - * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh - * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh - * @param va variable arguments specific to @a type - * @return block group handle, NULL if block groups are not supported - * by this @a type of block (this is not an error) - */ -static struct GNUNET_BLOCK_Group * -block_plugin_fs_create_group (void *cls, - enum GNUNET_BLOCK_Type type, - const void *raw_data, - size_t raw_data_size, - va_list va) -{ - unsigned int size; - const char *guard; - - switch (type) - { - case GNUNET_BLOCK_TYPE_FS_DBLOCK: - GNUNET_break (NULL == va_arg (va, const char *)); - return NULL; - case GNUNET_BLOCK_TYPE_FS_IBLOCK: - GNUNET_break (NULL == va_arg (va, const char *)); - return NULL; - case GNUNET_BLOCK_TYPE_FS_UBLOCK: - guard = va_arg (va, const char *); - if (0 == strcmp (guard, - "seen-set-size")) - { - size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned - int), - BLOOMFILTER_K); - } - else if (0 == strcmp (guard, - "filter-size")) - { - size = va_arg (va, unsigned int); - } - else - { - /* va-args invalid! bad bug, complain! */ - GNUNET_break (0); - size = 8; - } - if (0 == size) - size = raw_data_size; /* not for us to determine, use what we got! */ - GNUNET_break (NULL == va_arg (va, const char *)); - return GNUNET_BLOCK_GROUP_bf_create (cls, - size, - BLOOMFILTER_K, - type, - raw_data, - raw_data_size); - - default: - GNUNET_break (NULL == va_arg (va, const char *)); - GNUNET_break (0); - return NULL; - } -} - - -/** - * Function called to obtain the key for a block. - * - * @param cls closure - * @param type block type - * @param block block to get the key for - * @param block_size number of bytes in @a block - * @param key set to the key (query) for the given block - * @return #GNUNET_OK on success, #GNUNET_SYSERR if type not supported - * (or if extracting a key from a block of this type does not work) - */ -static enum GNUNET_GenericReturnValue -block_plugin_fs_get_key (void *cls, - enum GNUNET_BLOCK_Type type, - const void *block, - size_t block_size, - struct GNUNET_HashCode *key) -{ - const struct UBlock *ub; - - switch (type) - { - case GNUNET_BLOCK_TYPE_FS_DBLOCK: - case GNUNET_BLOCK_TYPE_FS_IBLOCK: - GNUNET_CRYPTO_hash (block, - block_size, - key); - return GNUNET_OK; - case GNUNET_BLOCK_TYPE_FS_UBLOCK: - if (block_size < sizeof(struct UBlock)) - { - GNUNET_break_op (0); - memset (key, - 0, - sizeof (*key)); - return GNUNET_OK; - } - ub = block; - GNUNET_CRYPTO_hash (&ub->verification_key, - sizeof(ub->verification_key), - key); - return GNUNET_OK; - default: - GNUNET_break (0); - return GNUNET_SYSERR; - } -} - - -/** - * Function called to validate a query. - * - * @param cls closure - * @param type block type - * @param query original query (hash) - * @param xquery extended query data (can be NULL, depending on type) - * @param xquery_size number of bytes in @a xquery - * @return #GNUNET_OK if the query is fine, #GNUNET_NO if not - */ -static enum GNUNET_GenericReturnValue -block_plugin_fs_check_query (void *cls, - enum GNUNET_BLOCK_Type type, - const struct GNUNET_HashCode *query, - const void *xquery, - size_t xquery_size) -{ - switch (type) - { - case GNUNET_BLOCK_TYPE_FS_DBLOCK: - case GNUNET_BLOCK_TYPE_FS_IBLOCK: - case GNUNET_BLOCK_TYPE_FS_UBLOCK: - if (0 != xquery_size) - { - GNUNET_break_op (0); - return GNUNET_NO; - } - return GNUNET_OK; - default: - GNUNET_break (0); - return GNUNET_SYSERR; - } -} - - -/** - * Function called to validate a block for storage. - * - * @param cls closure - * @param type block type - * @param block block data to validate - * @param block_size number of bytes in @a block - * @return #GNUNET_OK if the block is fine, #GNUNET_NO if not - */ -static enum GNUNET_GenericReturnValue -block_plugin_fs_check_block (void *cls, - enum GNUNET_BLOCK_Type type, - const void *block, - size_t block_size) -{ - switch (type) - { - case GNUNET_BLOCK_TYPE_FS_DBLOCK: - case GNUNET_BLOCK_TYPE_FS_IBLOCK: - return GNUNET_OK; - case GNUNET_BLOCK_TYPE_FS_UBLOCK: - { - const struct UBlock *ub; - - if (block_size < sizeof(struct UBlock)) - { - GNUNET_break_op (0); - return GNUNET_NO; - } - ub = block; - if (block_size != - ntohl (ub->purpose.size) - + sizeof (struct GNUNET_CRYPTO_EcdsaSignature)) - { - GNUNET_break_op (0); - return GNUNET_NO; - } - if (GNUNET_OK != - GNUNET_CRYPTO_ecdsa_verify_ (GNUNET_SIGNATURE_PURPOSE_FS_UBLOCK, - &ub->purpose, - &ub->signature, - &ub->verification_key)) - { - GNUNET_break_op (0); - return GNUNET_NO; - } - return GNUNET_OK; - } - default: - GNUNET_break (0); - return GNUNET_SYSERR; - } -} - - -/** - * Function called to validate a reply to a request. Note that it is assumed - * that the reply has already been matched to the key (and signatures checked) - * as it would be done with the GetKeyFunction and the - * BlockEvaluationFunction. - * - * @param cls closure - * @param type block type - * @param group which block group to use for evaluation - * @param query original query (hash) - * @param xquery extrended query data (can be NULL, depending on type) - * @param xquery_size number of bytes in @a xquery - * @param reply_block response to validate - * @param reply_block_size number of bytes in @a reply_block - * @return characterization of result - */ -static enum GNUNET_BLOCK_ReplyEvaluationResult -block_plugin_fs_check_reply (void *cls, - enum GNUNET_BLOCK_Type type, - struct GNUNET_BLOCK_Group *group, - const struct GNUNET_HashCode *query, - const void *xquery, - size_t xquery_size, - const void *reply_block, - size_t reply_block_size) -{ - switch (type) - { - case GNUNET_BLOCK_TYPE_FS_DBLOCK: - case GNUNET_BLOCK_TYPE_FS_IBLOCK: - return GNUNET_BLOCK_REPLY_OK_LAST; - case GNUNET_BLOCK_TYPE_FS_UBLOCK: - { - struct GNUNET_HashCode chash; - - GNUNET_CRYPTO_hash (reply_block, - reply_block_size, - &chash); - if (GNUNET_YES == - GNUNET_BLOCK_GROUP_bf_test_and_set (group, - &chash)) - return GNUNET_BLOCK_REPLY_OK_DUPLICATE; - return GNUNET_BLOCK_REPLY_OK_MORE; - } - default: - GNUNET_break (0); - return GNUNET_BLOCK_REPLY_TYPE_NOT_SUPPORTED; - } -} - - -/** - * Entry point for the plugin. - */ -void * -libgnunet_plugin_block_fs_init (void *cls) -{ - static const enum GNUNET_BLOCK_Type types[] = { - GNUNET_BLOCK_TYPE_FS_DBLOCK, - GNUNET_BLOCK_TYPE_FS_IBLOCK, - GNUNET_BLOCK_TYPE_FS_UBLOCK, - GNUNET_BLOCK_TYPE_ANY /* end of list */ - }; - struct GNUNET_BLOCK_PluginFunctions *api; - - api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions); - api->get_key = &block_plugin_fs_get_key; - api->create_group = &block_plugin_fs_create_group; - api->check_query = &block_plugin_fs_check_query; - api->check_block = &block_plugin_fs_check_block; - api->check_reply = &block_plugin_fs_check_reply; - api->types = types; - return api; -} - - -/** - * Exit point from the plugin. - */ -void * -libgnunet_plugin_block_fs_done (void *cls) -{ - struct GNUNET_BLOCK_PluginFunctions *api = cls; - - GNUNET_free (api); - return NULL; -} - - -/* end of plugin_block_fs.c */ diff --git a/src/fs/test_fs.c b/src/fs/test_fs.c deleted file mode 100644 index 7a57e98b0..000000000 --- a/src/fs/test_fs.c +++ /dev/null @@ -1,262 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004, 2005, 2006, 2008 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_fs.c - * @brief testcase for FS (upload-search-download-unindex) - * @author Christian Grothoff - */ - -#include "platform.h" -#include "gnunet_util.h" -#include "gnunet_fsui_lib.h" - -#define DEBUG_VERBOSE GNUNET_NO - -#define CHECK(a) if (! (a)) { ok = GNUNET_NO; GNUNET_GE_BREAK (NULL, 0); \ - goto FAILURE; } - -static char * -makeName (unsigned int i) -{ - char *fn; - - fn = GNUNET_malloc (strlen ("/tmp/gnunet-basic_fsui_test/BASIC_FSUI_TEST") - + 14); - GNUNET_snprintf (fn, - strlen ("/tmp/gnunet-basic_fsui_test/BASIC_FSUI_TEST") + 14, - "/tmp/gnunet-basic_fsui_test/BASIC_FSUI_TEST%u", i); - GNUNET_disk_directory_create_for_file (NULL, fn); - return fn; -} - - -static enum GNUNET_FSUI_EventType lastEvent; - -static struct GNUNET_MetaData *search_meta; - -static struct GNUNET_ECRS_URI *search_uri; - -static struct GNUNET_FSUI_Context *ctx; - -static void * -eventCallback (void *cls, const GNUNET_FSUI_Event *event) -{ - static char unused; - - switch (event->type) - { - case GNUNET_FSUI_search_resumed: - case GNUNET_FSUI_download_resumed: - case GNUNET_FSUI_upload_resumed: - case GNUNET_FSUI_unindex_resumed: - return &unused; - - case GNUNET_FSUI_search_result: -#if DEBUG_VERBOSE - printf ("Received search result\n"); -#endif - search_uri = GNUNET_ECRS_uri_duplicate (event->data.SearchResult.fi.uri); - search_meta = GNUNET_meta_data_duplicate (event->data.SearchResult.fi.meta); - break; - - case GNUNET_FSUI_upload_completed: -#if DEBUG_VERBOSE - printf ("Upload complete.\n"); -#endif - break; - - case GNUNET_FSUI_download_completed: -#if DEBUG_VERBOSE - printf ("Download complete.\n"); -#endif - break; - - case GNUNET_FSUI_unindex_completed: -#if DEBUG_VERBOSE - printf ("Unindex complete.\n"); -#endif - break; - - default: - break; - } - lastEvent = event->type; - return NULL; -} - - -#define START_DAEMON 1 - -int -main (int argc, char *argv[]) -{ -#if START_DAEMON - struct GNUNET_OS_Process *daemon; -#endif - int ok; - struct GNUNET_ECRS_URI *uri; - char *filename = NULL; - - char *keywords[] = { - "fsui_foo", - "fsui_bar", - }; - char keyword[40]; - char *fn; - int prog; - struct GNUNET_MetaData *meta; - struct GNUNET_ECRS_URI *kuri; - struct GNUNET_GC_Configuration *cfg; - struct GNUNET_FSUI_UploadList *upload = NULL; - struct GNUNET_FSUI_SearchList *search = NULL; - struct GNUNET_FSUI_UnindexList *unindex = NULL; - struct GNUNET_FSUI_DownloadList *download = NULL; - - cfg = GNUNET_GC_create (); - if (-1 == GNUNET_GC_parse_configuration (cfg, "check.conf")) - { - GNUNET_GC_free (cfg); - return -1; - } -#if START_DAEMON - daemon = GNUNET_daemon_start (NULL, cfg, "peer.conf", GNUNET_NO); - GNUNET_GE_ASSERT (NULL, daemon != NULL); - CHECK (GNUNET_OK == - GNUNET_wait_for_daemon_running (NULL, cfg, 60 * GNUNET_CRON_SECONDS)); -#endif - GNUNET_thread_sleep (5 * GNUNET_CRON_SECONDS); /* give apps time to start */ - ok = GNUNET_YES; - - /* ACTUAL TEST CODE */ - ctx = GNUNET_FSUI_start (NULL, cfg, "basic_fsui_test", 32, /* thread pool size */ - GNUNET_NO, /* no resume */ - &eventCallback, NULL); - CHECK (ctx != NULL); - filename = makeName (42); - GNUNET_disk_file_write (NULL, filename, "foo bar test!", - strlen ("foo bar test!"), "600"); - meta = GNUNET_meta_data_create (); - kuri = - GNUNET_ECRS_keyword_command_line_to_uri (NULL, 2, - (const char **) keywords); - /* upload */ - upload = GNUNET_FSUI_upload_start (ctx, filename, - (GNUNET_FSUI_DirectoryScanCallback) - & GNUNET_disk_directory_scan, NULL, 0, /* anonymity */ - 0, /* priority */ - GNUNET_YES, GNUNET_NO, GNUNET_NO, - GNUNET_get_time () + 5 * GNUNET_CRON_HOURS, - meta, kuri, kuri); - CHECK (upload != NULL); - GNUNET_ECRS_uri_destroy (kuri); - GNUNET_meta_data_destroy (meta); - prog = 0; - while (lastEvent != GNUNET_FSUI_upload_completed) - { - prog++; - CHECK (prog < 10000) GNUNET_thread_sleep (50 * GNUNET_CRON_MILLISECONDS); - if (GNUNET_shutdown_test () == GNUNET_YES) - break; - } - - /* search */ - GNUNET_snprintf (keyword, 40, "+%s +%s", keywords[0], keywords[1]); - uri = GNUNET_ECRS_keyword_string_to_uri (NULL, keyword); - search = GNUNET_FSUI_search_start (ctx, 0, uri); - GNUNET_ECRS_uri_destroy (uri); - CHECK (search != NULL); - prog = 0; - while (lastEvent != GNUNET_FSUI_search_result) - { - prog++; - CHECK (prog < 10000); - GNUNET_thread_sleep (50 * GNUNET_CRON_MILLISECONDS); - if (GNUNET_shutdown_test () == GNUNET_YES) - break; - } - GNUNET_FSUI_search_abort (search); - GNUNET_FSUI_search_stop (search); - - /* download */ - fn = makeName (43); - download = - GNUNET_FSUI_download_start (ctx, 0, GNUNET_NO, search_uri, search_meta, - fn, NULL, NULL); - GNUNET_free (fn); - prog = 0; - while (lastEvent != GNUNET_FSUI_download_completed) - { - prog++; - CHECK (prog < 10000); - GNUNET_thread_sleep (50 * GNUNET_CRON_MILLISECONDS); - if (GNUNET_shutdown_test () == GNUNET_YES) - break; - } - GNUNET_FSUI_download_stop (download); - download = NULL; - GNUNET_ECRS_uri_destroy (search_uri); - GNUNET_meta_data_destroy (search_meta); - /* unindex */ - unindex = GNUNET_FSUI_unindex_start (ctx, filename); - prog = 0; - while (lastEvent != GNUNET_FSUI_unindex_completed) - { - prog++; - CHECK (prog < 10000); - GNUNET_thread_sleep (50 * GNUNET_CRON_MILLISECONDS); - if (GNUNET_shutdown_test () == GNUNET_YES) - break; - } - if (lastEvent != GNUNET_FSUI_unindex_completed) - GNUNET_FSUI_unindex_abort (unindex); - GNUNET_FSUI_unindex_stop (unindex); - - - /* END OF TEST CODE */ -FAILURE: - if (ctx != NULL) - GNUNET_FSUI_stop (ctx); - if (filename != NULL) - { - unlink (filename); - GNUNET_free (filename); - } - if (download != NULL) - { - GNUNET_FSUI_download_abort (download); - GNUNET_FSUI_download_stop (download); - } - filename = makeName (43); - /* TODO: verify file 'filename(42)' == file 'filename(43)' */ - unlink (filename); - GNUNET_free (filename); - -#if START_DAEMON - GNUNET_GE_ASSERT (NULL, GNUNET_OK == GNUNET_daemon_stop (NULL, daemon)); - GNUNET_OS_process_destroy (daemon); -#endif - GNUNET_GC_free (cfg); - - return (ok == GNUNET_YES) ? 0 : 1; -} - - -/* end of test_fs.c */ diff --git a/src/fs/test_fs_data.conf b/src/fs/test_fs_data.conf deleted file mode 100644 index 993141064..000000000 --- a/src/fs/test_fs_data.conf +++ /dev/null @@ -1,7 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-data/ - -[fs] -ACTIVEMIGRATION = NO - diff --git a/src/fs/test_fs_defaults.conf b/src/fs/test_fs_defaults.conf deleted file mode 100644 index 6ead78257..000000000 --- a/src/fs/test_fs_defaults.conf +++ /dev/null @@ -1,47 +0,0 @@ -@INLINE@ ../../contrib/conf/gnunet/no_forcestart.conf - -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-lib/ - -[transport] -PLUGINS = tcp - -[nat] -DISABLEV6 = YES -ENABLE_UPNP = NO -BEHIND_NAT = NO -ALLOW_NAT = NO -INTERNAL_ADDRESS = 127.0.0.1 -EXTERNAL_ADDRESS = 127.0.0.1 -USE_LOCALADDR = YES -RETURN_LOCAL_ADDRESSES = YES - -[datastore] -QUOTA = 100 MB - -[transport-tcp] -BINDTO = 127.0.0.1 -PORT = 54368 - -[peerinfo] -NO_IO = YES - -[ats] -WAN_QUOTA_IN = 65536 -WAN_QUOTA_OUT = 65536 - -[fs] -CONTENT_CACHING = YES -CONTENT_PUSHING = YES -DELAY = YES -# PREFIX = valgrind --leak-check=full - -[dhtcache] -QUOTA=65536 -DATABASE=heap - -[cadet] -REFRESH_PATH_TIME = 30 min - -[nse] -WORKBITS = 0 diff --git a/src/fs/test_fs_directory.c b/src/fs/test_fs_directory.c deleted file mode 100644 index f5121a3e7..000000000 --- a/src/fs/test_fs_directory.c +++ /dev/null @@ -1,186 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2005, 2006, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/test_fs_directory.c - * @brief Test for fs_directory.c - * @author Christian Grothoff - */ -#include "platform.h" -#if HAVE_EXTRACTOR_H -#include -#endif -#include "gnunet_util_lib.h" -#include "gnunet_fs_service.h" -#include "fs_api.h" - -#define ABORT() { fprintf (stderr, "Error at %s:%d\n", __FILE__, __LINE__); \ - return 1; } - -struct PCLS -{ - struct GNUNET_FS_Uri **uri; - struct GNUNET_FS_MetaData **md; - unsigned int pos; - unsigned int max; -}; - -static void -processor (void *cls, const char *filename, const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *md, size_t length, - const void *data) -{ - struct PCLS *p = cls; - int i; - - if (NULL == uri) - return; /* ignore directory's meta data */ - for (i = 0; i < p->max; i++) - { - if (GNUNET_FS_meta_data_test_equal (p->md[i], md) && - GNUNET_FS_uri_test_equal (p->uri[i], uri)) - { - p->pos++; - return; - } - } - fprintf (stderr, "Error at %s:%d\n", __FILE__, __LINE__); -} - - -static int -testDirectory (unsigned int i) -{ - struct GNUNET_FS_DirectoryBuilder *db; - char *data; - size_t dlen; - struct GNUNET_FS_Uri **uris; - struct GNUNET_FS_MetaData **mds; - struct GNUNET_FS_MetaData *meta; - struct PCLS cls; - char *emsg; - int p; - int q; - char uri[512]; - char txt[128]; - int ret = 0; - struct GNUNET_TIME_Absolute start; - const char *s; - - cls.max = i; - uris = GNUNET_malloc (sizeof(struct GNUNET_FS_Uri *) * i); - mds = GNUNET_malloc (sizeof(struct GNUNET_FS_MetaData *) * i); - meta = GNUNET_FS_meta_data_create (); - GNUNET_FS_meta_data_insert (meta, "", EXTRACTOR_METATYPE_TITLE, - EXTRACTOR_METAFORMAT_UTF8, "text/plain", - "A title", strlen ("A title") + 1); - GNUNET_FS_meta_data_insert (meta, "", - EXTRACTOR_METATYPE_AUTHOR_NAME, - EXTRACTOR_METAFORMAT_UTF8, "text/plain", - "An author", strlen ("An author") + 1); - for (p = 0; p < i; p++) - { - mds[p] = GNUNET_FS_meta_data_create (); - for (q = 0; q <= p; q++) - { - GNUNET_snprintf (txt, sizeof(txt), "%u -- %u\n", p, q); - GNUNET_FS_meta_data_insert (mds[p], "", -#if HAVE_EXTRACTOR_H && HAVE_LIBEXTRACTOR - q % EXTRACTOR_metatype_get_max (), -#else - q % 128, -#endif - EXTRACTOR_METAFORMAT_UTF8, - "text/plain", txt, strlen (txt) + 1); - } - GNUNET_snprintf (uri, sizeof(uri), - "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.%u", - p); - emsg = NULL; - uris[p] = GNUNET_FS_uri_parse (uri, &emsg); - if (uris[p] == NULL) - { - GNUNET_FS_meta_data_destroy (mds[p]); - while (--p > 0) - { - GNUNET_FS_meta_data_destroy (mds[p]); - GNUNET_FS_uri_destroy (uris[p]); - } - GNUNET_free (mds); - GNUNET_free (uris); - GNUNET_free (emsg); - GNUNET_FS_meta_data_destroy (meta); - ABORT (); /* error in testcase */ - } - GNUNET_assert (emsg == NULL); - } - start = GNUNET_TIME_absolute_get (); - db = GNUNET_FS_directory_builder_create (meta); - for (p = 0; p < i; p++) - GNUNET_FS_directory_builder_add (db, uris[p], mds[p], NULL); - GNUNET_FS_directory_builder_finish (db, &dlen, (void **) &data); - s = GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration - (start), - GNUNET_YES); - fprintf (stdout, - "Creating directory with %u entries and total size %llu took %s\n", - i, (unsigned long long) dlen, s); - if (i < 100) - { - cls.pos = 0; - cls.uri = uris; - cls.md = mds; - GNUNET_FS_directory_list_contents (dlen, data, 0, &processor, &cls); - GNUNET_assert (cls.pos == i); - } - GNUNET_free (data); - GNUNET_FS_meta_data_destroy (meta); - for (p = 0; p < i; p++) - { - GNUNET_FS_meta_data_destroy (mds[p]); - GNUNET_FS_uri_destroy (uris[p]); - } - GNUNET_free (uris); - GNUNET_free (mds); - return ret; -} - - -int -main (int argc, char *argv[]) -{ - int failureCount = 0; - int i; - - GNUNET_log_setup ("test_fs_directory", -#if VERBOSE - "DEBUG", -#else - "WARNING", -#endif - NULL); - for (i = 17; i < 1000; i *= 2) - failureCount += testDirectory (i); - if (failureCount != 0) - return 1; - return 0; -} - - -/* end of test_fs_directory.c */ diff --git a/src/fs/test_fs_download.c b/src/fs/test_fs_download.c deleted file mode 100644 index fc6b32c0f..000000000 --- a/src/fs/test_fs_download.c +++ /dev/null @@ -1,368 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004, 2005, 2006, 2008, 2009, 2011, 2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_fs_download.c - * @brief simple testcase for simple publish + download operation - * @author Christian Grothoff - */ - -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_fs_service.h" -#include "gnunet_testing_lib.h" -#include - -/** - * File-size we use for testing. - */ -#define FILESIZE (1024 * 1024 * 2) - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 120) - -/** - * How long should our test-content live? - */ -#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) - -static unsigned int anonymity_level; - -static int indexed; - -static struct GNUNET_TIME_Absolute start; - -static struct GNUNET_FS_Handle *fs; - -static struct GNUNET_FS_DownloadContext *download; - -static struct GNUNET_FS_PublishContext *publish; - -static struct GNUNET_SCHEDULER_Task *timeout_kill; - -static char *fn; - -static char *fn1; - -static int err; - - -static void -timeout_kill_task (void *cls) -{ - if (NULL != download) - { - GNUNET_FS_download_stop (download, GNUNET_YES); - download = NULL; - } - else if (NULL != publish) - { - GNUNET_FS_publish_stop (publish); - publish = NULL; - } - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Timeout downloading file\n"); - timeout_kill = NULL; - err = 1; -} - - -static void -abort_publish_task (void *cls) -{ - if (NULL != publish) - { - GNUNET_FS_publish_stop (publish); - publish = NULL; - } -} - - -static void -stop_fs_task (void *cls) -{ - GNUNET_FS_stop (fs); - fs = NULL; -} - - -static void -abort_download_task (void *cls) -{ - uint64_t size; - - if (NULL != download) - { - GNUNET_FS_download_stop (download, GNUNET_YES); - download = NULL; - } - GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_size (fn, &size, GNUNET_YES, - GNUNET_NO)); - GNUNET_assert (size == FILESIZE); - GNUNET_DISK_directory_remove (fn); - GNUNET_free (fn); - fn = NULL; - GNUNET_SCHEDULER_cancel (timeout_kill); - timeout_kill = NULL; -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - switch (event->status) - { - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publish is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.publish.completed, - (unsigned long long) event->value.publish.size, - event->value.publish.specifics.progress.depth, - (unsigned long long) event->value.publish.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - break; - - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - fprintf (stdout, - "Publishing complete, %llu kb/s.\n", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024LL)); - GAUGER ("FS", - (GNUNET_YES == indexed) - ? "Publishing speed (indexing)" - : "Publishing speed (insertion)", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024LL), "kb/s"); - fn = GNUNET_DISK_mktemp ("gnunet-download-test-dst"); - start = GNUNET_TIME_absolute_get (); - download = - GNUNET_FS_download_start (fs, - event->value.publish.specifics. - completed.chk_uri, NULL, fn, NULL, 0, - FILESIZE, anonymity_level, - GNUNET_FS_DOWNLOAD_OPTION_NONE, - "download", NULL); - GNUNET_assert (download != NULL); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED: - fprintf (stdout, - "Download complete, %llu kb/s.\n", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024LL)); - GAUGER ("FS", - (GNUNET_YES == indexed) - ? "Local download speed (indexed)" - : "Local download speed (inserted)", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024LL), "kb/s"); - GNUNET_SCHEDULER_add_now (&abort_download_task, NULL); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: - GNUNET_assert (download == event->value.download.dc); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Download is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.download.completed, - (unsigned long long) event->value.download.size, - event->value.download.specifics.progress.depth, - (unsigned long long) event->value.download.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - fprintf (stderr, "Error publishing file: %s\n", - event->value.publish.specifics.error.message); - GNUNET_break (0); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - GNUNET_SCHEDULER_shutdown (); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_ERROR: - fprintf (stderr, "Error downloading file: %s\n", - event->value.download.specifics.error.message); - GNUNET_SCHEDULER_add_now (&abort_download_task, NULL); - GNUNET_SCHEDULER_shutdown (); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: - case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: - break; - - case GNUNET_FS_STATUS_PUBLISH_START: - GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); - GNUNET_assert (NULL == event->value.publish.pctx); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (1 == event->value.publish.anonymity); - break; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - GNUNET_assert (publish == event->value.publish.pc); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (1 == event->value.publish.anonymity); - GNUNET_SCHEDULER_add_now (&stop_fs_task, NULL); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_START: - GNUNET_assert (0 == strcmp ("download", event->value.download.cctx)); - GNUNET_assert (NULL == event->value.download.pctx); - GNUNET_assert (NULL != event->value.download.uri); - GNUNET_assert (0 == strcmp (fn, event->value.download.filename)); - GNUNET_assert (FILESIZE == event->value.download.size); - GNUNET_assert (0 == event->value.download.completed); - GNUNET_assert (1 == event->value.download.anonymity); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_STOPPED: - GNUNET_assert (download == event->value.download.dc); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - default: - printf ("Unexpected event: %d\n", event->status); - break; - } - return NULL; -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_TESTING_Peer *peer) -{ - const char *binary_name = cls; - const char *keywords[] = { - "down_foo", - "down_bar", - }; - char *buf; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *kuri; - struct GNUNET_FS_FileInformation *fi; - size_t i; - struct GNUNET_FS_BlockOptions bo; - - if (GNUNET_YES == - GNUNET_CONFIGURATION_get_value_yesno (cfg, - "download-test", - "USE_STREAM")) - anonymity_level = 0; - else - anonymity_level = 1; - fs = GNUNET_FS_start (cfg, binary_name, &progress_cb, NULL, - GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); - GNUNET_assert (NULL != fs); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - meta = GNUNET_FS_meta_data_create (); - kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); - bo.content_priority = 42; - bo.anonymity_level = anonymity_level; - bo.replication_level = 0; - bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); - - if (GNUNET_YES == - GNUNET_CONFIGURATION_get_value_yesno (cfg, - "download-test", - "USE_INDEX")) - { - fn1 = GNUNET_DISK_mktemp ("gnunet-download-indexed-test"); - (void) GNUNET_DISK_directory_remove (fn1); - GNUNET_assert (GNUNET_OK == - GNUNET_DISK_fn_write (fn1, - buf, - FILESIZE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE)); - GNUNET_free (buf); - fi = GNUNET_FS_file_information_create_from_file (fs, "publish-context", - fn1, - kuri, meta, GNUNET_YES, - &bo); - indexed = GNUNET_YES; - } - else - { - fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context", - FILESIZE, buf, kuri, meta, - GNUNET_NO, &bo); - /* note: buf will be free'd as part of 'fi' now */ - indexed = GNUNET_NO; - } - GNUNET_FS_uri_destroy (kuri); - GNUNET_FS_meta_data_destroy (meta); - GNUNET_assert (NULL != fi); - timeout_kill = - GNUNET_SCHEDULER_add_delayed (TIMEOUT, &timeout_kill_task, NULL); - start = GNUNET_TIME_absolute_get (); - publish = - GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); - GNUNET_assert (publish != NULL); -} - - -int -main (int argc, char *argv[]) -{ - const char *binary_name; - const char *config_name; - - binary_name = "test-fs-download"; - config_name = "test_fs_download_data.conf"; - if (NULL != strstr (argv[0], "indexed")) - { - binary_name = "test-fs-download-indexed"; - config_name = "test_fs_download_indexed.conf"; - } - if (NULL != strstr (argv[0], "cadet")) - { - binary_name = "test-fs-download-cadet"; - config_name = "test_fs_download_cadet.conf"; - } - if (0 != GNUNET_TESTING_peer_run (binary_name, - config_name, - &run, (void *) binary_name)) - return 1; - if (NULL != fn1) - { - unlink (fn1); - GNUNET_free (fn1); - } - return err; -} - - -/* end of test_fs_download.c */ diff --git a/src/fs/test_fs_download_data.conf b/src/fs/test_fs_download_data.conf deleted file mode 100644 index 160dec3ae..000000000 --- a/src/fs/test_fs_download_data.conf +++ /dev/null @@ -1,10 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-download/ - -[download-test] -# set to 'YES' to test non-anonymous download -USE_STREAM = NO - -# set to 'YES' to use indexing -USE_INDEX = NO \ No newline at end of file diff --git a/src/fs/test_fs_download_indexed.conf b/src/fs/test_fs_download_indexed.conf deleted file mode 100644 index 7f1e36935..000000000 --- a/src/fs/test_fs_download_indexed.conf +++ /dev/null @@ -1,10 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-download/ - -[download-test] -# set to 'YES' to test non-anonymous download -USE_STREAM = NO - -# set to 'YES' to use indexing -USE_INDEX = YES diff --git a/src/fs/test_fs_download_persistence.c b/src/fs/test_fs_download_persistence.c deleted file mode 100644 index b66fefd6b..000000000 --- a/src/fs/test_fs_download_persistence.c +++ /dev/null @@ -1,351 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004, 2005, 2006, 2008, 2009, 2010 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_fs_download_persistence.c - * @brief simple testcase for persistence of simple download operation - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testing_lib.h" -#include "gnunet_fs_service.h" - -/** - * File-size we use for testing. - */ -#define FILESIZE (1024 * 1024 * 2) - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) - -/** - * How long should our test-content live? - */ -#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) - - -static struct GNUNET_TIME_Absolute start; - -static const struct GNUNET_CONFIGURATION_Handle *cfg; - -static struct GNUNET_FS_Handle *fs; - -static struct GNUNET_FS_DownloadContext *download; - -static struct GNUNET_FS_PublishContext *publish; - -static struct GNUNET_SCHEDULER_Task *timeout_kill; - -static char *fn; - -static int err; - - -static void -timeout_kill_task (void *cls) -{ - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Timeout downloading file\n"); - if (download != NULL) - { - GNUNET_FS_download_stop (download, GNUNET_YES); - download = NULL; - } - else if (publish != NULL) - { - GNUNET_FS_publish_stop (publish); - publish = NULL; - } - timeout_kill = NULL; - err = 1; -} - - -static void -abort_publish_task (void *cls) -{ - if (publish != NULL) - { - GNUNET_FS_publish_stop (publish); - publish = NULL; - } -} - - -static void -abort_download_task (void *cls) -{ - uint64_t size; - - if (download != NULL) - { - GNUNET_FS_download_stop (download, GNUNET_YES); - download = NULL; - } - GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_size (fn, &size, GNUNET_YES, - GNUNET_NO)); - GNUNET_assert (size == FILESIZE); - GNUNET_DISK_directory_remove (fn); - GNUNET_free (fn); - fn = NULL; - GNUNET_SCHEDULER_cancel (timeout_kill); - timeout_kill = NULL; -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event); - - -static void -restart_fs_task (void *cls) -{ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Restarting FS.\n"); - GNUNET_FS_stop (fs); - fs = GNUNET_FS_start (cfg, "test-fs-download-persistence", &progress_cb, NULL, - GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); -} - - -/** - * Consider scheduling the restart-task. - * Only runs the restart task once per event - * category. - * - * @param ev type of the event to consider - */ -static void -consider_restart (int ev) -{ - static int prev[32]; - static int off; - int i; - - for (i = 0; i < off; i++) - if (prev[i] == ev) - return; - prev[off++] = ev; - GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_URGENT, - &restart_fs_task, NULL); -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - switch (event->status) - { - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publish is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.publish.completed, - (unsigned long long) event->value.publish.size, - event->value.publish.specifics.progress.depth, - (unsigned long long) event->value.publish.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - break; - - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - printf ("Publishing complete, %llu kbps.\n", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024LL)); - fn = GNUNET_DISK_mktemp ("gnunet-download-test-dst"); - start = GNUNET_TIME_absolute_get (); - GNUNET_assert (download == NULL); - GNUNET_FS_download_start (fs, - event->value.publish.specifics.completed.chk_uri, - NULL, fn, NULL, 0, FILESIZE, 1, - GNUNET_FS_DOWNLOAD_OPTION_NONE, "download", NULL); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED: - printf ("Download complete, %llu kbps.\n", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024LL)); - GNUNET_SCHEDULER_add_now (&abort_download_task, NULL); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: - consider_restart (event->status); - GNUNET_assert (download == event->value.download.dc); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Download is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.download.completed, - (unsigned long long) event->value.download.size, - event->value.download.specifics.progress.depth, - (unsigned long long) event->value.download.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - fprintf (stderr, "Error publishing file: %s\n", - event->value.publish.specifics.error.message); - GNUNET_break (0); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_ERROR: - fprintf (stderr, "Error downloading file: %s\n", - event->value.download.specifics.error.message); - GNUNET_SCHEDULER_add_now (&abort_download_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_SUSPEND: - GNUNET_assert (event->value.publish.pc == publish); - publish = NULL; - break; - - case GNUNET_FS_STATUS_PUBLISH_RESUME: - GNUNET_assert (NULL == publish); - publish = event->value.publish.pc; - break; - - case GNUNET_FS_STATUS_DOWNLOAD_SUSPEND: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download suspended.\n"); - GNUNET_assert (event->value.download.dc == download); - download = NULL; - break; - - case GNUNET_FS_STATUS_DOWNLOAD_RESUME: - GNUNET_assert (NULL == download); - download = event->value.download.dc; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download resumed.\n"); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: - consider_restart (event->status); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download active.\n"); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: - consider_restart (event->status); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download inactive.\n"); - break; - - case GNUNET_FS_STATUS_PUBLISH_START: - GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); - GNUNET_assert (NULL == event->value.publish.pctx); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (1 == event->value.publish.anonymity); - break; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - GNUNET_assert (publish == event->value.publish.pc); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (1 == event->value.publish.anonymity); - GNUNET_FS_stop (fs); - fs = NULL; - break; - - case GNUNET_FS_STATUS_DOWNLOAD_START: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download started.\n"); - consider_restart (event->status); - GNUNET_assert (download == NULL); - download = event->value.download.dc; - GNUNET_assert (0 == strcmp ("download", event->value.download.cctx)); - GNUNET_assert (NULL == event->value.download.pctx); - GNUNET_assert (NULL != event->value.download.uri); - GNUNET_assert (0 == strcmp (fn, event->value.download.filename)); - GNUNET_assert (FILESIZE == event->value.download.size); - GNUNET_assert (0 == event->value.download.completed); - GNUNET_assert (1 == event->value.download.anonymity); - break; - - case GNUNET_FS_STATUS_DOWNLOAD_STOPPED: - GNUNET_assert (download == event->value.download.dc); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - download = NULL; - break; - - default: - printf ("Unexpected event: %d\n", event->status); - break; - } - return NULL; -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *c, - struct GNUNET_TESTING_Peer *peer) -{ - const char *keywords[] = { - "down_foo", - "down_bar", - }; - char *buf; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *kuri; - struct GNUNET_FS_FileInformation *fi; - size_t i; - struct GNUNET_FS_BlockOptions bo; - - cfg = c; - fs = GNUNET_FS_start (cfg, "test-fs-download-persistence", &progress_cb, NULL, - GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); - GNUNET_assert (NULL != fs); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - meta = GNUNET_FS_meta_data_create (); - kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); - bo.content_priority = 42; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); - fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context", - FILESIZE, buf, kuri, meta, - GNUNET_NO, &bo); - GNUNET_FS_uri_destroy (kuri); - GNUNET_FS_meta_data_destroy (meta); - GNUNET_assert (NULL != fi); - timeout_kill = - GNUNET_SCHEDULER_add_delayed (TIMEOUT, &timeout_kill_task, NULL); - start = GNUNET_TIME_absolute_get (); - publish = - GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); - GNUNET_assert (publish != NULL); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-download-persistence", - "test_fs_download_data.conf", - &run, NULL)) - return 1; - return err; -} - - -/* end of test_fs_download_persistence.c */ diff --git a/src/fs/test_fs_file_information.c b/src/fs/test_fs_file_information.c deleted file mode 100644 index 15380bfc4..000000000 --- a/src/fs/test_fs_file_information.c +++ /dev/null @@ -1,163 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004, 2005, 2006, 2008, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_fs_file_information.c - * @brief simple testcase for file_information operations - * @author Christian Grothoff - * - * TODO: - * - test that metadata, etc. are all correct (for example, - * there is a known bug with dirname never being set that is - * not detected!) - * - need to iterate over file-information structure - * - other API functions may not yet be tested (such as - * filedata-from-callback) - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_fs_service.h" - - -/** - * File-size we use for testing. - */ -#define FILESIZE (1024 * 1024 * 2) - -/** - * How long should our test-content live? - */ -#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) - - -static int -mycleaner (void *cls, struct GNUNET_FS_FileInformation *fi, uint64_t length, - struct GNUNET_FS_MetaData *meta, struct GNUNET_FS_Uri **uri, - struct GNUNET_FS_BlockOptions *bo, int *do_index, void **client_info) -{ - return GNUNET_OK; -} - - -static void -run (void *cls, char *const *args, const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *cfg) -{ - const char *keywords[] = { - "down_foo", - "down_bar", - }; - char *fn1; - char *fn2; - char *buf; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *kuri; - struct GNUNET_FS_FileInformation *fi1; - struct GNUNET_FS_FileInformation *fi2; - struct GNUNET_FS_FileInformation *fidir; - struct GNUNET_FS_Handle *fs; - size_t i; - struct GNUNET_FS_BlockOptions bo; - - fs = GNUNET_FS_start (cfg, "test-fs-file-information", NULL, NULL, - GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); - fn1 = GNUNET_DISK_mktemp ("gnunet-file_information-test-dst"); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - (void) GNUNET_DISK_directory_remove (fn1); - GNUNET_assert (GNUNET_OK == - GNUNET_DISK_fn_write (fn1, buf, FILESIZE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE)); - GNUNET_free (buf); - - fn2 = GNUNET_DISK_mktemp ("gnunet-file_information-test-dst"); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - (void) GNUNET_DISK_directory_remove (fn2); - GNUNET_assert (GNUNET_OK == - GNUNET_DISK_fn_write (fn2, buf, FILESIZE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE)); - GNUNET_free (buf); - - meta = GNUNET_FS_meta_data_create (); - kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); - bo.content_priority = 42; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); - fi1 = - GNUNET_FS_file_information_create_from_file (fs, - "file_information-context1", - fn1, kuri, meta, GNUNET_YES, - &bo); - GNUNET_assert (fi1 != NULL); - fi2 = - GNUNET_FS_file_information_create_from_file (fs, - "file_information-context2", - fn2, kuri, meta, GNUNET_YES, - &bo); - GNUNET_assert (fi2 != NULL); - fidir = - GNUNET_FS_file_information_create_empty_directory (fs, - "file_information-context-dir", - kuri, meta, &bo, NULL); - GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi1)); - GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi2)); - GNUNET_FS_uri_destroy (kuri); - GNUNET_FS_meta_data_destroy (meta); - GNUNET_assert (NULL != fidir); - /* FIXME: test more of API! */ - GNUNET_FS_file_information_destroy (fidir, &mycleaner, NULL); - GNUNET_DISK_directory_remove (fn1); - GNUNET_DISK_directory_remove (fn2); - GNUNET_free (fn1); - GNUNET_free (fn2); - GNUNET_FS_stop (fs); -} - - -int -main (int argc, char *argv[]) -{ - char *const argvx[] = { - "test-fs-file_information", - "-c", - "test_fs_file_information_data.conf", - NULL - }; - struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_OPTION_END - }; - - GNUNET_log_setup ("test_fs_file_information", - "WARNING", - NULL); - GNUNET_PROGRAM_run ((sizeof(argvx) / sizeof(char *)) - 1, argvx, - "test-fs-file_information", "nohelp", options, &run, - NULL); - return 0; -} - - -/* end of test_fs_file_information.c */ diff --git a/src/fs/test_fs_file_information_data.conf b/src/fs/test_fs_file_information_data.conf deleted file mode 100644 index c8fc0938c..000000000 --- a/src/fs/test_fs_file_information_data.conf +++ /dev/null @@ -1,7 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-file-information/ - -[transport] -PLUGINS = - diff --git a/src/fs/test_fs_getopt.c b/src/fs/test_fs_getopt.c deleted file mode 100644 index 3d0da752b..000000000 --- a/src/fs/test_fs_getopt.c +++ /dev/null @@ -1,37 +0,0 @@ -/* - This file is part of GNUnet - Copyright (C) 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/test_fs_getopt.c - * @brief test for fs_getopt.c - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_fs_service.h" - - -int -main (int argc, char *argv[]) -{ - GNUNET_log_setup ("test_fs_getopt", - "WARNING", - NULL); - fprintf (stderr, "%s", "WARNING: testcase not yet written.\n"); - return 0; /* testcase passed */ -} diff --git a/src/fs/test_fs_list_indexed.c b/src/fs/test_fs_list_indexed.c deleted file mode 100644 index 7e06c47f5..000000000 --- a/src/fs/test_fs_list_indexed.c +++ /dev/null @@ -1,265 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004, 2005, 2006, 2008, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_fs_list_indexed.c - * @brief simple testcase for list_indexed operation (indexing, listing - * indexed) - * @author Christian Grothoff - * - * TODO: - * - actually call list_indexed API! - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testing_lib.h" -#include "gnunet_fs_service.h" - -/** - * File-size we use for testing. - */ -#define FILESIZE (1024 * 1024 * 2) - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) - -/** - * How long should our test-content live? - */ -#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) - - -static struct GNUNET_TIME_Absolute start; - -static struct GNUNET_FS_Handle *fs; - -static struct GNUNET_FS_PublishContext *publish; - -static char *fn1; - -static char *fn2; - -static int err; - - -static void -abort_publish_task (void *cls) -{ - GNUNET_FS_publish_stop (publish); - publish = NULL; - GNUNET_DISK_directory_remove (fn1); - GNUNET_free (fn1); - fn1 = NULL; - GNUNET_DISK_directory_remove (fn2); - GNUNET_free (fn2); - fn2 = NULL; -} - - -static void -list_indexed_task (void *cls) -{ - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - void *ret; - - ret = NULL; - switch (event->status) - { - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - ret = event->value.publish.cctx; - printf ("Publish complete, %llu kbps.\n", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024)); - if (0 == strcmp ("list_indexed-context-dir", event->value.publish.cctx)) - GNUNET_SCHEDULER_add_now (&list_indexed_task, NULL); - - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - ret = event->value.publish.cctx; - GNUNET_assert (publish == event->value.publish.pc); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publish is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.publish.completed, - (unsigned long long) event->value.publish.size, - event->value.publish.specifics.progress.depth, - (unsigned long long) event->value.publish.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - ret = event->value.publish.cctx; - break; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - ret = event->value.publish.cctx; - fprintf (stderr, "Error publishing file: %s\n", - event->value.publish.specifics.error.message); - err = 1; - if (0 == strcmp ("list_indexed-context-dir", event->value.publish.cctx)) - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_START: - ret = event->value.publish.cctx; - if (0 == strcmp ("list_indexed-context1", event->value.publish.cctx)) - { - GNUNET_assert (0 == - strcmp ("list_indexed-context-dir", - event->value.publish.pctx)); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (1 == event->value.publish.anonymity); - } - else if (0 == strcmp ("list_indexed-context2", event->value.publish.cctx)) - { - GNUNET_assert (0 == - strcmp ("list_indexed-context-dir", - event->value.publish.pctx)); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (2 == event->value.publish.anonymity); - } - else if (0 == - strcmp ("list_indexed-context-dir", event->value.publish.cctx)) - { - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (3 == event->value.publish.anonymity); - } - else - GNUNET_assert (0); - break; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - if (0 == strcmp ("list_indexed-context-dir", event->value.publish.cctx)) - { - GNUNET_assert (publish == event->value.publish.pc); - publish = NULL; - } - break; - - default: - printf ("Unexpected event: %d\n", event->status); - break; - } - return ret; -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_TESTING_Peer *peer) -{ - const char *keywords[] = { - "down_foo", - "down_bar", - }; - char *buf; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *kuri; - struct GNUNET_FS_FileInformation *fi1; - struct GNUNET_FS_FileInformation *fi2; - struct GNUNET_FS_FileInformation *fidir; - size_t i; - struct GNUNET_FS_BlockOptions bo; - - fs = GNUNET_FS_start (cfg, "test-fs-list_indexed", &progress_cb, NULL, - GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); - GNUNET_assert (NULL != fs); - fn1 = GNUNET_DISK_mktemp ("gnunet-list_indexed-test-dst"); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - (void) GNUNET_DISK_directory_remove (fn1); - GNUNET_assert (GNUNET_OK == - GNUNET_DISK_fn_write (fn1, buf, FILESIZE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE)); - GNUNET_free (buf); - - fn2 = GNUNET_DISK_mktemp ("gnunet-list_indexed-test-dst"); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - (void) GNUNET_DISK_directory_remove (fn2); - GNUNET_assert (GNUNET_OK == - GNUNET_DISK_fn_write (fn2, buf, FILESIZE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE)); - GNUNET_free (buf); - - meta = GNUNET_FS_meta_data_create (); - kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); - bo.content_priority = 42; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); - fi1 = - GNUNET_FS_file_information_create_from_file (fs, "list_indexed-context1", - fn1, kuri, meta, GNUNET_YES, - &bo); - GNUNET_assert (NULL != fi1); - bo.anonymity_level = 2; - fi2 = - GNUNET_FS_file_information_create_from_file (fs, "list_indexed-context2", - fn2, kuri, meta, GNUNET_YES, - &bo); - GNUNET_assert (NULL != fi2); - bo.anonymity_level = 3; - fidir = - GNUNET_FS_file_information_create_empty_directory (fs, - "list_indexed-context-dir", - kuri, meta, &bo, NULL); - GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi1)); - GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi2)); - GNUNET_FS_uri_destroy (kuri); - GNUNET_FS_meta_data_destroy (meta); - GNUNET_assert (NULL != fidir); - start = GNUNET_TIME_absolute_get (); - publish = - GNUNET_FS_publish_start (fs, fidir, NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); - GNUNET_assert (publish != NULL); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-list-indexed", - "test_fs_list_indexed_data.conf", - &run, NULL)) - return 1; - return 0; -} - - -/* end of test_fs_list_indexed.c */ diff --git a/src/fs/test_fs_list_indexed_data.conf b/src/fs/test_fs_list_indexed_data.conf deleted file mode 100644 index 941809322..000000000 --- a/src/fs/test_fs_list_indexed_data.conf +++ /dev/null @@ -1,10 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-list-indexed/ - -[transport] -PLUGINS = - -[fs] -ACTIVEMIGRATION = NO - diff --git a/src/fs/test_fs_meta_data.c b/src/fs/test_fs_meta_data.c deleted file mode 100644 index 4e7439d7b..000000000 --- a/src/fs/test_fs_meta_data.c +++ /dev/null @@ -1,492 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2003, 2004, 2006, 2009, 2010, 2022 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file util/test_fs_meta_data.c - * @brief Test for fs_meta_data.c - * @author Christian Grothoff - * @author Martin Schanzenbach - */ - - -#include "platform.h" -#include "gnunet_util_lib.h" - -#include "gnunet_fs_service.h" - -#define ABORT(m) { fprintf (stderr, "Error at %s:%d\n", __FILE__, __LINE__); \ - if (m != NULL) GNUNET_FS_meta_data_destroy (m); \ - return 1; } - - -static int -testMeta (int i) -{ - struct GNUNET_FS_MetaData *m; - char val[256]; - char *sval; - int j; - unsigned int size; - - m = GNUNET_FS_meta_data_create (); - if (GNUNET_OK != - GNUNET_FS_meta_data_insert (m, "", EXTRACTOR_METATYPE_TITLE, - EXTRACTOR_METAFORMAT_UTF8, - "text/plain", "TestTitle", - strlen ("TestTitle") + 1)) - ABORT (m); - if (GNUNET_OK != - GNUNET_FS_meta_data_insert (m, "", - EXTRACTOR_METATYPE_AUTHOR_NAME, - EXTRACTOR_METAFORMAT_UTF8, - "text/plain", "TestTitle", - strlen ("TestTitle") + 1)) - ABORT (m); - if (GNUNET_OK == GNUNET_FS_meta_data_insert (m, "", - EXTRACTOR_METATYPE_TITLE, - EXTRACTOR_METAFORMAT_UTF8, - "text/plain", - "TestTitle", strlen ( - "TestTitle") + 1)) /* dup! */ - ABORT (m); - if (GNUNET_OK == GNUNET_FS_meta_data_insert (m, "", - EXTRACTOR_METATYPE_AUTHOR_NAME, - EXTRACTOR_METAFORMAT_UTF8, - "text/plain", - "TestTitle", strlen ( - "TestTitle") + 1)) /* dup! */ - ABORT (m); - if (2 != GNUNET_FS_meta_data_iterate (m, NULL, NULL)) - ABORT (m); - if (GNUNET_OK != - GNUNET_FS_meta_data_delete (m, EXTRACTOR_METATYPE_AUTHOR_NAME, - "TestTitle", strlen ("TestTitle") + 1)) - ABORT (m); - if (GNUNET_OK == GNUNET_FS_meta_data_delete (m, - EXTRACTOR_METATYPE_AUTHOR_NAME, - "TestTitle", strlen ( - "TestTitle") + 1)) /* already gone */ - ABORT (m); - if (1 != GNUNET_FS_meta_data_iterate (m, NULL, NULL)) - ABORT (m); - if (GNUNET_OK != - GNUNET_FS_meta_data_delete (m, EXTRACTOR_METATYPE_TITLE, - "TestTitle", strlen ("TestTitle") + 1)) - ABORT (m); - if (GNUNET_OK == GNUNET_FS_meta_data_delete (m, - EXTRACTOR_METATYPE_TITLE, - "TestTitle", strlen ( - "TestTitle") + 1)) /* already gone */ - ABORT (m); - if (0 != GNUNET_FS_meta_data_iterate (m, NULL, NULL)) - ABORT (m); - for (j = 0; j < i; j++) - { - GNUNET_snprintf (val, sizeof(val), "%s.%d", - "A teststring that should compress well.", j); - if (GNUNET_OK != - GNUNET_FS_meta_data_insert (m, "", - EXTRACTOR_METATYPE_UNKNOWN, - EXTRACTOR_METAFORMAT_UTF8, - "text/plain", val, strlen (val) + 1)) - ABORT (m); - } - if (i != GNUNET_FS_meta_data_iterate (m, NULL, NULL)) - ABORT (m); - - size = GNUNET_FS_meta_data_get_serialized_size (m); - sval = NULL; - if (size != - GNUNET_FS_meta_data_serialize (m, &sval, size, - GNUNET_FS_META_DATA_SERIALIZE_FULL)) - { - GNUNET_free (sval); - ABORT (m); - } - GNUNET_FS_meta_data_destroy (m); - m = GNUNET_FS_meta_data_deserialize (sval, size); - GNUNET_free (sval); - if (m == NULL) - ABORT (m); - for (j = 0; j < i; j++) - { - GNUNET_snprintf (val, - sizeof(val), - "%s.%d", - "A teststring that should compress well.", - j); - if (GNUNET_OK != - GNUNET_FS_meta_data_delete (m, - EXTRACTOR_METATYPE_UNKNOWN, - val, - strlen (val) + 1)) - { - ABORT (m); - } - } - if (0 != GNUNET_FS_meta_data_iterate (m, NULL, NULL)) - ABORT (m); - GNUNET_FS_meta_data_destroy (m); - return 0; -} - - -static int -testMetaMore (int i) -{ - struct GNUNET_FS_MetaData *meta; - int q; - char txt[128]; - char *data; - unsigned long long size; - - meta = GNUNET_FS_meta_data_create (); - for (q = 0; q <= i; q++) - { - GNUNET_snprintf (txt, 128, "%u -- %u\n", i, q); - GNUNET_FS_meta_data_insert (meta, "", - q - % 42 /* EXTRACTOR_metatype_get_max () */, - EXTRACTOR_METAFORMAT_UTF8, "text/plain", - txt, strlen (txt) + 1); - } - size = GNUNET_FS_meta_data_get_serialized_size (meta); - data = GNUNET_malloc (size * 4); - if (size != - GNUNET_FS_meta_data_serialize (meta, &data, size * 4, - GNUNET_FS_META_DATA_SERIALIZE_FULL)) - { - GNUNET_free (data); - ABORT (meta); - } - GNUNET_FS_meta_data_destroy (meta); - GNUNET_free (data); - return 0; -} - - -static int -testMetaLink () -{ - struct GNUNET_FS_MetaData *m; - char *val; - unsigned int size; - - m = GNUNET_FS_meta_data_create (); - if (GNUNET_OK != - GNUNET_FS_meta_data_insert (m, "", - EXTRACTOR_METATYPE_UNKNOWN, - EXTRACTOR_METAFORMAT_UTF8, - "text/plain", "link", - strlen ("link") + 1)) - ABORT (m); - if (GNUNET_OK != - GNUNET_FS_meta_data_insert (m, "", - EXTRACTOR_METATYPE_FILENAME, - EXTRACTOR_METAFORMAT_UTF8, - "text/plain", "lib-link.m4", - strlen ("lib-link.m4") + 1)) - ABORT (m); - val = NULL; - size = - GNUNET_FS_meta_data_serialize (m, &val, (size_t) -1, - GNUNET_FS_META_DATA_SERIALIZE_FULL); - GNUNET_FS_meta_data_destroy (m); - m = GNUNET_FS_meta_data_deserialize (val, size); - GNUNET_free (val); - if (m == NULL) - ABORT (m); - GNUNET_FS_meta_data_destroy (m); - return 0; -} - - -static int -check () -{ - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_MetaData *meta2; - int q; - int i = 100; - char txt[128]; - char *str; - unsigned char *thumb; - - meta = GNUNET_FS_meta_data_create (); - meta2 = GNUNET_FS_meta_data_create (); - for (q = 0; q <= i; q++) - { - GNUNET_snprintf (txt, 128, "%u -- %u\n", i, q); - GNUNET_FS_meta_data_insert (meta, "", - EXTRACTOR_METATYPE_UNKNOWN, - EXTRACTOR_METAFORMAT_UTF8, "text/plain", - "TestTitle", strlen ("TestTitle") + 1); - GNUNET_FS_meta_data_insert (meta2, "", - EXTRACTOR_METATYPE_UNKNOWN, - EXTRACTOR_METAFORMAT_UTF8, "text/plain", - "TestTitle", strlen ("TestTitle") + 1); - } - - // check meta_data_test_equal - if (GNUNET_YES != GNUNET_FS_meta_data_test_equal (meta, meta2)) - { - GNUNET_FS_meta_data_destroy (meta2); - ABORT (meta); - } - - // check meta_data_clear - GNUNET_FS_meta_data_clear (meta2); - if (0 != GNUNET_FS_meta_data_iterate (meta2, NULL, NULL)) - { - GNUNET_FS_meta_data_destroy (meta2); - ABORT (meta); - } - // check equal branch in meta_data_test_equal - if (GNUNET_YES != GNUNET_FS_meta_data_test_equal (meta, meta)) - { - GNUNET_FS_meta_data_destroy (meta2); - ABORT (meta); - } - // check "count" branch in meta_data_test_equal - if (GNUNET_NO != GNUNET_FS_meta_data_test_equal (meta, meta2)) - { - GNUNET_FS_meta_data_destroy (meta2); - ABORT (meta); - } - - // check meta_data_add_publication_date - GNUNET_FS_meta_data_add_publication_date (meta2); - - // check meta_data_merge - GNUNET_FS_meta_data_clear (meta2); - GNUNET_FS_meta_data_merge (meta2, meta); - if (100 == GNUNET_FS_meta_data_iterate (meta2, NULL, NULL)) - { - GNUNET_FS_meta_data_destroy (meta2); - ABORT (meta); - } - - // check meta_data_get_by_type - GNUNET_FS_meta_data_clear (meta2); - if (NULL != - (str = - GNUNET_FS_meta_data_get_by_type (meta2, - EXTRACTOR_METATYPE_UNKNOWN))) - { - GNUNET_FS_meta_data_destroy (meta2); - GNUNET_free (str); - ABORT (meta); - } - - str = - GNUNET_FS_meta_data_get_by_type (meta, EXTRACTOR_METATYPE_UNKNOWN); - GNUNET_assert (NULL != str); - if (str[0] != 'T') - { - GNUNET_FS_meta_data_destroy (meta2); - GNUNET_free (str); - ABORT (meta); - } - GNUNET_free (str); - - // check branch - if (NULL != - (str = - GNUNET_FS_meta_data_get_by_type (meta, - EXTRACTOR_METATYPE_PUBLICATION_DATE))) - { - GNUNET_free (str); - GNUNET_FS_meta_data_destroy (meta2); - ABORT (meta); - } - - // check meta_data_get_first_by_types - str = - GNUNET_FS_meta_data_get_first_by_types (meta, - EXTRACTOR_METATYPE_UNKNOWN, - -1); - GNUNET_assert (NULL != str); - if (str[0] != 'T') - { - GNUNET_FS_meta_data_destroy (meta2); - GNUNET_free (str); - ABORT (meta); - } - GNUNET_free (str); - - // check meta_data_get_thumbnail - if (GNUNET_FS_meta_data_get_thumbnail (meta, &thumb) != 0) - { - GNUNET_free (thumb); - GNUNET_FS_meta_data_destroy (meta2); - ABORT (meta); - } - GNUNET_FS_meta_data_destroy (meta2); - // check meta_data_duplicate - meta2 = GNUNET_FS_meta_data_duplicate (meta); - if (200 == GNUNET_FS_meta_data_iterate (meta2, NULL, NULL)) - { - GNUNET_FS_meta_data_destroy (meta2); - ABORT (meta); - } - GNUNET_FS_meta_data_destroy (meta2); - GNUNET_FS_meta_data_destroy (meta); - return 0; -} - - -static int -test_bigmeta_rw (void) -{ - static char meta[1024 * 1024 * 10]; - struct GNUNET_BIO_WriteHandle *wh; - struct GNUNET_BIO_ReadHandle *rh; - char *filename = GNUNET_DISK_mktemp ("gnunet_bio"); - struct GNUNET_FS_MetaData *mdR = NULL; - - memset (meta, 'b', sizeof (meta)); - meta[sizeof (meta) - 1] = '\0'; - - wh = GNUNET_BIO_write_open_file (filename); - GNUNET_assert (NULL != wh); - if (GNUNET_OK != GNUNET_BIO_write_int32 (wh, - "test-bigmeta-rw-int32", - sizeof (meta))) - { - GNUNET_BIO_write_close (wh, NULL); - return 1; - } - if (GNUNET_OK != GNUNET_BIO_write (wh, - "test-bigmeta-rw-bytes", - meta, - sizeof (meta))) - { - GNUNET_BIO_write_close (wh, NULL); - return 1; - } - GNUNET_assert (GNUNET_OK == GNUNET_BIO_write_close (wh, NULL)); - - rh = GNUNET_BIO_read_open_file (filename); - GNUNET_assert (NULL != rh); - GNUNET_assert (GNUNET_SYSERR == - GNUNET_FS_read_meta_data (rh, - "test-bigmeta-rw-metadata", - &mdR)); - GNUNET_assert (GNUNET_SYSERR == GNUNET_BIO_read_close (rh, NULL)); - - GNUNET_assert (NULL == mdR); - - GNUNET_assert (GNUNET_OK == GNUNET_DISK_directory_remove (filename)); - GNUNET_free (filename); - return 0; -} - -static int -test_fakemeta_rw (void) -{ - struct GNUNET_BIO_WriteHandle *wh; - struct GNUNET_BIO_ReadHandle *rh; - char *filename = GNUNET_DISK_mktemp ("gnunet_bio"); - struct GNUNET_FS_MetaData *mdR = NULL; - - wh = GNUNET_BIO_write_open_file (filename); - GNUNET_assert (NULL != wh); - if (GNUNET_OK != GNUNET_BIO_write_int32 (wh, - "test-fakestring-rw-int32", - 2)) - { - GNUNET_BIO_write_close (wh, NULL); - return 1; - } - GNUNET_assert (GNUNET_OK == GNUNET_BIO_write_close (wh, NULL)); - - rh = GNUNET_BIO_read_open_file (filename); - GNUNET_assert (NULL != rh); - GNUNET_assert (GNUNET_SYSERR == - GNUNET_FS_read_meta_data (rh, - "test-fakestring-rw-metadata", - &mdR)); - GNUNET_assert (GNUNET_SYSERR == GNUNET_BIO_read_close (rh, NULL)); - - GNUNET_assert (NULL == mdR); - - GNUNET_assert (GNUNET_OK == GNUNET_DISK_directory_remove (filename)); - GNUNET_free (filename); - return 0; -} - -static int -test_fakebigmeta_rw (void) -{ - struct GNUNET_BIO_WriteHandle *wh; - struct GNUNET_BIO_ReadHandle *rh; - char *filename = GNUNET_DISK_mktemp ("gnunet_bio"); - struct GNUNET_FS_MetaData *mdR = NULL; - int32_t wNum = 1024 * 1024 * 10; - - wh = GNUNET_BIO_write_open_file (filename); - GNUNET_assert (NULL != wh); - GNUNET_assert (GNUNET_OK == GNUNET_BIO_write_int32 (wh, - "test-fakebigmeta-rw-int32", - wNum)); - GNUNET_assert (GNUNET_OK == GNUNET_BIO_write_close (wh, NULL)); - - rh = GNUNET_BIO_read_open_file (filename); - GNUNET_assert (NULL != rh); - GNUNET_assert (GNUNET_SYSERR == - GNUNET_FS_read_meta_data (rh, - "test-fakebigmeta-rw-metadata", - &mdR)); - GNUNET_assert (GNUNET_SYSERR == GNUNET_BIO_read_close (rh, NULL)); - - GNUNET_assert (NULL == mdR); - - GNUNET_assert (GNUNET_OK == GNUNET_DISK_directory_remove (filename)); - GNUNET_free (filename); - return 0; -} - -int -main (int argc, char *argv[]) -{ - int failureCount = 0; - int i; - - GNUNET_log_setup ("test-fs-meta-data", "WARNING", NULL); - for (i = 0; i < 255; i++) - failureCount += testMeta (i); - for (i = 1; i < 255; i++) - failureCount += testMetaMore (i); - failureCount += testMetaLink (); - failureCount += test_fakebigmeta_rw (); - failureCount += test_fakemeta_rw (); - failureCount += test_bigmeta_rw (); - int ret = check (); - - if (ret == 1) - return 1; - - if (failureCount != 0) - return 1; - return 0; -} - - -/* end of test_container_meta_data.c */ diff --git a/src/fs/test_fs_namespace.c b/src/fs/test_fs_namespace.c deleted file mode 100644 index 85d489598..000000000 --- a/src/fs/test_fs_namespace.c +++ /dev/null @@ -1,320 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2005-2013 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_fs_namespace.c - * @brief Test for fs_namespace.c - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testing_lib.h" -#include "gnunet_fs_service.h" - - -static struct GNUNET_CRYPTO_EcdsaPublicKey nsid; - -static struct GNUNET_FS_Uri *sks_expect_uri; - -static struct GNUNET_FS_Uri *ksk_expect_uri; - -static struct GNUNET_FS_Handle *fs; - -static struct GNUNET_FS_SearchContext *sks_search; - -static struct GNUNET_FS_SearchContext *ksk_search; - -static struct GNUNET_SCHEDULER_Task *kill_task; - -static int update_started; - -static int err; - - -static void -abort_ksk_search_task (void *cls) -{ - if (ksk_search != NULL) - { - GNUNET_FS_search_stop (ksk_search); - ksk_search = NULL; - if (sks_search == NULL) - { - GNUNET_FS_stop (fs); - if (NULL != kill_task) - GNUNET_SCHEDULER_cancel (kill_task); - } - } -} - - -static void -abort_sks_search_task (void *cls) -{ - if (sks_search == NULL) - return; - GNUNET_FS_search_stop (sks_search); - sks_search = NULL; - if (ksk_search == NULL) - { - GNUNET_FS_stop (fs); - if (NULL != kill_task) - GNUNET_SCHEDULER_cancel (kill_task); - } -} - - -static void -do_timeout (void *cls) -{ - err = 1; - fprintf (stderr, "%s", "Operation timed out\n"); - kill_task = NULL; - abort_sks_search_task (NULL); - abort_ksk_search_task (NULL); -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - switch (event->status) - { - case GNUNET_FS_STATUS_SEARCH_RESULT: - if (sks_search == event->value.search.sc) - { - if (! GNUNET_FS_uri_test_equal - (sks_expect_uri, event->value.search.specifics.result.uri)) - { - fprintf (stderr, "%s", "Wrong result for sks search!\n"); - err = 1; - } - /* give system 1ms to initiate update search! */ - GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS, - &abort_sks_search_task, NULL); - } - else if (ksk_search == event->value.search.sc) - { - if (! GNUNET_FS_uri_test_equal - (ksk_expect_uri, event->value.search.specifics.result.uri)) - { - fprintf (stderr, "%s", "Wrong result for ksk search!\n"); - err = 1; - } - GNUNET_SCHEDULER_add_now (&abort_ksk_search_task, NULL); - } - else - { - fprintf (stderr, "%s", "Unexpected search result received!\n"); - GNUNET_break (0); - } - break; - - case GNUNET_FS_STATUS_SEARCH_ERROR: - fprintf (stderr, "Error searching file: %s\n", - event->value.search.specifics.error.message); - if (sks_search == event->value.search.sc) - GNUNET_SCHEDULER_add_now (&abort_sks_search_task, NULL); - else if (ksk_search == event->value.search.sc) - GNUNET_SCHEDULER_add_now (&abort_ksk_search_task, NULL); - else - GNUNET_break (0); - break; - - case GNUNET_FS_STATUS_SEARCH_START: - GNUNET_assert ((NULL == event->value.search.cctx) || - (0 == strcmp ("sks_search", event->value.search.cctx)) || - (0 == strcmp ("ksk_search", event->value.search.cctx))); - if (NULL == event->value.search.cctx) - { - GNUNET_assert (0 == strcmp ("sks_search", event->value.search.pctx)); - update_started = GNUNET_YES; - } - GNUNET_assert (1 == event->value.search.anonymity); - break; - - case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED: - return NULL; - - case GNUNET_FS_STATUS_SEARCH_STOPPED: - return NULL; - - default: - fprintf (stderr, "Unexpected event: %d\n", event->status); - break; - } - return event->value.search.cctx; -} - - -static void -publish_cont (void *cls, const struct GNUNET_FS_Uri *ksk_uri, const char *emsg) -{ - char *msg; - struct GNUNET_FS_Uri *sks_uri; - char sbuf[1024]; - char buf[1024]; - char *ret; - - if (NULL != emsg) - { - fprintf (stderr, "Error publishing: %s\n", emsg); - err = 1; - GNUNET_FS_stop (fs); - return; - } - ret = GNUNET_STRINGS_data_to_string (&nsid, sizeof(nsid), buf, sizeof(buf)); - GNUNET_assert (NULL != ret); - ret[0] = '\0'; - GNUNET_snprintf (sbuf, sizeof(sbuf), "gnunet://fs/sks/%s/this", buf); - sks_uri = GNUNET_FS_uri_parse (sbuf, &msg); - if (NULL == sks_uri) - { - fprintf (stderr, "failed to parse URI `%s': %s\n", sbuf, msg); - err = 1; - GNUNET_FS_stop (fs); - GNUNET_free (msg); - return; - } - ksk_search = - GNUNET_FS_search_start (fs, ksk_uri, 1, GNUNET_FS_SEARCH_OPTION_NONE, - "ksk_search"); - sks_search = - GNUNET_FS_search_start (fs, sks_uri, 1, GNUNET_FS_SEARCH_OPTION_NONE, - "sks_search"); - GNUNET_FS_uri_destroy (sks_uri); -} - - -static void -sks_cont (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg) -{ - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *ksk_uri; - char *msg; - struct GNUNET_FS_BlockOptions bo; - - if (NULL == uri) - { - fprintf (stderr, "Error publishing: %s\n", emsg); - err = 1; - GNUNET_FS_stop (fs); - return; - } - meta = GNUNET_FS_meta_data_create (); - msg = NULL; - ksk_uri = GNUNET_FS_uri_parse ("gnunet://fs/ksk/ns-search", &msg); - GNUNET_assert (NULL == msg); - ksk_expect_uri = GNUNET_FS_uri_dup (uri); - bo.content_priority = 1; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = - GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES); - GNUNET_FS_publish_ksk (fs, ksk_uri, meta, uri, &bo, - GNUNET_FS_PUBLISH_OPTION_NONE, &publish_cont, NULL); - GNUNET_FS_uri_destroy (ksk_uri); - GNUNET_FS_meta_data_destroy (meta); -} - - -static void -adv_cont (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg) -{ - struct GNUNET_FS_MetaData *meta; - struct GNUNET_CRYPTO_EcdsaPrivateKey ns; - struct GNUNET_FS_BlockOptions bo; - - if (NULL != emsg) - { - fprintf (stderr, "Error publishing: %s\n", emsg); - err = 1; - GNUNET_FS_stop (fs); - return; - } - GNUNET_CRYPTO_ecdsa_key_create (&ns); - meta = GNUNET_FS_meta_data_create (); - sks_expect_uri = GNUNET_FS_uri_dup (uri); - bo.content_priority = 1; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = - GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES); - GNUNET_CRYPTO_ecdsa_key_get_public (&ns, - &nsid); - GNUNET_FS_publish_sks (fs, - &ns, "this", "next", meta, uri, - &bo, GNUNET_FS_PUBLISH_OPTION_NONE, &sks_cont, NULL); - GNUNET_FS_meta_data_destroy (meta); -} - - -static void -testNamespace (void) -{ - struct GNUNET_FS_BlockOptions bo; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *ksk_uri; - struct GNUNET_FS_Uri *sks_uri; - - meta = GNUNET_FS_meta_data_create (); - ksk_uri = GNUNET_FS_uri_parse ("gnunet://fs/ksk/testnsa", NULL); - bo.content_priority = 1; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = - GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES); - sks_uri = GNUNET_FS_uri_sks_create (&nsid, "root"); - GNUNET_FS_publish_ksk (fs, - ksk_uri, meta, sks_uri, - &bo, GNUNET_FS_PUBLISH_OPTION_NONE, - &adv_cont, NULL); - GNUNET_FS_uri_destroy (sks_uri); - kill_task = - GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, &do_timeout, - NULL); - GNUNET_FS_uri_destroy (ksk_uri); - GNUNET_FS_meta_data_destroy (meta); -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_TESTING_Peer *peer) -{ - fs = GNUNET_FS_start (cfg, "test-fs-namespace", &progress_cb, NULL, - GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); - testNamespace (); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-namespace", - "test_fs_namespace_data.conf", - &run, NULL)) - return 1; - return err; -} - - -/* end of test_fs_namespace.c */ diff --git a/src/fs/test_fs_namespace_data.conf b/src/fs/test_fs_namespace_data.conf deleted file mode 100644 index 70b954f7d..000000000 --- a/src/fs/test_fs_namespace_data.conf +++ /dev/null @@ -1,7 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-namespace/ - -[transport] -PLUGINS = - diff --git a/src/fs/test_fs_namespace_list_updateable.c b/src/fs/test_fs_namespace_list_updateable.c deleted file mode 100644 index d883b7bea..000000000 --- a/src/fs/test_fs_namespace_list_updateable.c +++ /dev/null @@ -1,175 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2005-2013 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_fs_namespace_list_updateable.c - * @brief Test for fs_namespace_list_updateable.c - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testing_lib.h" -#include "gnunet_fs_service.h" - - -static struct GNUNET_FS_Handle *fs; - -static int err; - -static struct GNUNET_CRYPTO_EcdsaPrivateKey ns; - -static struct GNUNET_FS_MetaData *meta; - -static struct GNUNET_FS_Uri *uri_this; - -static struct GNUNET_FS_Uri *uri_next; - -static struct GNUNET_FS_BlockOptions bo; - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - return NULL; -} - - -static void -do_shutdown () -{ - if (uri_this != NULL) - GNUNET_FS_uri_destroy (uri_this); - if (uri_next != NULL) - GNUNET_FS_uri_destroy (uri_next); - if (meta != NULL) - GNUNET_FS_meta_data_destroy (meta); -} - - -static void -check_next (void *cls, const char *last_id, - const struct GNUNET_FS_Uri *last_uri, - const struct GNUNET_FS_MetaData *last_meta, - const char *next_id) -{ - GNUNET_break (0 == strcmp (last_id, "next")); - GNUNET_break (0 == strcmp (next_id, "future")); - err -= 4; -} - - -static void -check_this_next (void *cls, const char *last_id, - const struct GNUNET_FS_Uri *last_uri, - const struct GNUNET_FS_MetaData *last_meta, - const char *next_id) -{ - GNUNET_break (0 == strcmp (last_id, "this")); - GNUNET_break (0 == strcmp (next_id, "next")); - err -= 2; - err += 4; - GNUNET_FS_namespace_list_updateable (fs, &ns, next_id, &check_next, NULL); -} - - -static void -sks_cont_next (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg) -{ - GNUNET_assert (NULL == emsg); - err += 2; - GNUNET_FS_namespace_list_updateable (fs, &ns, NULL, &check_this_next, NULL); -} - - -static void -check_this (void *cls, const char *last_id, - const struct GNUNET_FS_Uri *last_uri, - const struct GNUNET_FS_MetaData *last_meta, - const char *next_id) -{ - GNUNET_break (0 == strcmp (last_id, "this")); - GNUNET_break (0 == strcmp (next_id, "next")); - err -= 1; -} - - -static void -sks_cont_this (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg) -{ - GNUNET_assert (NULL == emsg); - err = 1; - GNUNET_FS_namespace_list_updateable (fs, &ns, NULL, &check_this, NULL); - GNUNET_FS_publish_sks (fs, - &ns, "next", "future", meta, uri_next, &bo, - GNUNET_FS_PUBLISH_OPTION_NONE, &sks_cont_next, NULL); -} - - -static void -testNamespace (void) -{ - GNUNET_CRYPTO_ecdsa_key_create (&ns); - bo.content_priority = 1; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = - GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES); - meta = GNUNET_FS_meta_data_create (); - - uri_this = - GNUNET_FS_uri_parse - ( - "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.42", - NULL); - uri_next = - GNUNET_FS_uri_parse - ( - "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.43", - NULL); - GNUNET_FS_publish_sks (fs, - &ns, "this", "next", meta, uri_this, &bo, - GNUNET_FS_PUBLISH_OPTION_NONE, &sks_cont_this, NULL); -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_TESTING_Peer *peer) -{ - fs = GNUNET_FS_start (cfg, "test-fs-namespace", &progress_cb, NULL, - GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); - testNamespace (); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-namespace-list-updateable", - "test_fs_namespace_data.conf", - &run, NULL)) - return 1; - do_shutdown (); - return err; -} - - -/* end of test_fs_namespace_list_updateable.c */ diff --git a/src/fs/test_fs_publish.c b/src/fs/test_fs_publish.c deleted file mode 100644 index 0e379bc29..000000000 --- a/src/fs/test_fs_publish.c +++ /dev/null @@ -1,251 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004, 2005, 2006, 2008, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/test_fs_publish.c - * @brief simple testcase for publish operation (indexing, listing - * indexed, directory structure) - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testing_lib.h" -#include "gnunet_fs_service.h" - -/** - * File-size we use for testing. - */ -#define FILESIZE (1024 * 1024 * 2) - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) - -/** - * How long should our test-content live? - */ -#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) - - -static struct GNUNET_TIME_Absolute start; - -static struct GNUNET_FS_Handle *fs; - -static struct GNUNET_FS_PublishContext *publish; - -static char *fn1; - -static char *fn2; - -static int err; - - -static void -abort_publish_task (void *cls) -{ - GNUNET_FS_publish_stop (publish); - publish = NULL; - GNUNET_DISK_directory_remove (fn1); - GNUNET_free (fn1); - fn1 = NULL; - GNUNET_DISK_directory_remove (fn2); - GNUNET_free (fn2); - fn2 = NULL; -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - void *ret; - - ret = NULL; - switch (event->status) - { - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - ret = event->value.publish.cctx; - printf ("Publish complete, %llu kbps.\n", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024)); - if (0 == strcmp ("publish-context-dir", event->value.publish.cctx)) - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - ret = event->value.publish.cctx; - GNUNET_assert (publish == event->value.publish.pc); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publish is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.publish.completed, - (unsigned long long) event->value.publish.size, - event->value.publish.specifics.progress.depth, - (unsigned long long) event->value.publish.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - ret = event->value.publish.cctx; - break; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - ret = event->value.publish.cctx; - fprintf (stderr, "Error publishing file: %s\n", - event->value.publish.specifics.error.message); - err = 1; - if (0 == strcmp ("publish-context-dir", event->value.publish.cctx)) - { - fprintf (stderr, "Scheduling abort task for error on `%s'\n", - (const char *) event->value.publish.cctx); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - } - break; - - case GNUNET_FS_STATUS_PUBLISH_START: - ret = event->value.publish.cctx; - if (0 == strcmp ("publish-context1", event->value.publish.cctx)) - { - GNUNET_assert (0 == - strcmp ("publish-context-dir", event->value.publish.pctx)); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (1 == event->value.publish.anonymity); - } - else if (0 == strcmp ("publish-context2", event->value.publish.cctx)) - { - GNUNET_assert (0 == - strcmp ("publish-context-dir", event->value.publish.pctx)); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (2 == event->value.publish.anonymity); - } - else if (0 == strcmp ("publish-context-dir", event->value.publish.cctx)) - { - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (3 == event->value.publish.anonymity); - } - else - GNUNET_assert (0); - break; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - if (0 == strcmp ("publish-context-dir", event->value.publish.cctx)) - GNUNET_assert (publish == event->value.publish.pc); - break; - - default: - printf ("Unexpected event: %d\n", event->status); - break; - } - return ret; -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_TESTING_Peer *peer) -{ - const char *keywords[] = { - "down_foo", - "down_bar", - }; - char *buf; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *kuri; - struct GNUNET_FS_FileInformation *fi1; - struct GNUNET_FS_FileInformation *fi2; - struct GNUNET_FS_FileInformation *fidir; - size_t i; - struct GNUNET_FS_BlockOptions bo; - - fs = GNUNET_FS_start (cfg, "test-fs-publish", &progress_cb, NULL, - GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); - GNUNET_assert (NULL != fs); - fn1 = GNUNET_DISK_mktemp ("gnunet-publish-test-dst"); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - (void) GNUNET_DISK_directory_remove (fn1); - GNUNET_assert (GNUNET_OK == - GNUNET_DISK_fn_write (fn1, buf, FILESIZE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE)); - GNUNET_free (buf); - - fn2 = GNUNET_DISK_mktemp ("gnunet-publish-test-dst"); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - (void) GNUNET_DISK_directory_remove (fn2); - GNUNET_assert (GNUNET_OK == - GNUNET_DISK_fn_write (fn2, buf, FILESIZE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE)); - GNUNET_free (buf); - - meta = GNUNET_FS_meta_data_create (); - kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); - bo.content_priority = 42; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); - - fi1 = - GNUNET_FS_file_information_create_from_file (fs, "publish-context1", fn1, - kuri, meta, GNUNET_YES, &bo); - - GNUNET_assert (NULL != fi1); - bo.anonymity_level = 2; - fi2 = - GNUNET_FS_file_information_create_from_file (fs, "publish-context2", fn2, - kuri, meta, GNUNET_YES, &bo); - GNUNET_assert (NULL != fi2); - bo.anonymity_level = 3; - fidir = - GNUNET_FS_file_information_create_empty_directory (fs, - "publish-context-dir", - kuri, meta, &bo, NULL); - GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi1)); - GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi2)); - GNUNET_FS_uri_destroy (kuri); - GNUNET_FS_meta_data_destroy (meta); - GNUNET_assert (NULL != fidir); - start = GNUNET_TIME_absolute_get (); - publish = - GNUNET_FS_publish_start (fs, fidir, NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); - GNUNET_assert (publish != NULL); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-publish", - "test_fs_publish_data.conf", - &run, NULL)) - return 1; - return err; -} - - -/* end of test_fs_publish.c */ diff --git a/src/fs/test_fs_publish_data.conf b/src/fs/test_fs_publish_data.conf deleted file mode 100644 index 0930cdfed..000000000 --- a/src/fs/test_fs_publish_data.conf +++ /dev/null @@ -1,10 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-publish/ - -[transport] -PLUGINS = - -[fs] -ACTIVEMIGRATION = NO - diff --git a/src/fs/test_fs_publish_persistence.c b/src/fs/test_fs_publish_persistence.c deleted file mode 100644 index e1563f448..000000000 --- a/src/fs/test_fs_publish_persistence.c +++ /dev/null @@ -1,322 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004, 2005, 2006, 2008, 2009, 2010 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/test_fs_publish_persistence.c - * @brief simple testcase for persistence of simple publish operation - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testing_lib.h" -#include "gnunet_fs_service.h" - - -/** - * File-size we use for testing. - */ -#define FILESIZE (1024 * 1024 * 2) - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) - -/** - * How long should our test-content live? - */ -#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) - - -static struct GNUNET_TIME_Absolute start; - -static struct GNUNET_FS_Handle *fs; - -static const struct GNUNET_CONFIGURATION_Handle *cfg; - -static struct GNUNET_FS_PublishContext *publish; - -static struct GNUNET_FS_PublishContext *publish; - -static char *fn1; - -static char *fn2; - -static int err; - -static struct GNUNET_SCHEDULER_Task *rtask; - - -static void -abort_publish_task (void *cls) -{ - GNUNET_FS_publish_stop (publish); - publish = NULL; - GNUNET_DISK_directory_remove (fn1); - GNUNET_free (fn1); - fn1 = NULL; - GNUNET_DISK_directory_remove (fn2); - GNUNET_free (fn2); - fn2 = NULL; - GNUNET_FS_stop (fs); - fs = NULL; - if (NULL != rtask) - { - GNUNET_SCHEDULER_cancel (rtask); - rtask = NULL; - } -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event); - - -static void -restart_fs_task (void *cls) -{ - rtask = NULL; - GNUNET_FS_stop (fs); - fs = GNUNET_FS_start (cfg, "test-fs-publish-persistence", - &progress_cb, NULL, - GNUNET_FS_FLAGS_PERSISTENCE, - GNUNET_FS_OPTIONS_END); -} - - -/** - * Consider scheduling the restart-task. - * Only runs the restart task once per event - * category. - * - * @param ev type of the event to consider - */ -static void -consider_restart (int ev) -{ - static int prev[32]; - static int off; - int i; - - for (i = 0; i < off; i++) - if (prev[i] == ev) - return; - prev[off++] = ev; - rtask = - GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_URGENT, - &restart_fs_task, NULL); -} - - -static void * -progress_cb (void *cls, - const struct GNUNET_FS_ProgressInfo *event) -{ - void *ret; - - ret = NULL; - switch (event->status) - { - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - ret = event->value.publish.cctx; - printf ("Publish complete, %llu kbps.\n", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024)); - if ((NULL != event->value.publish.cctx) && - (0 == strcmp ("publish-context-dir", event->value.publish.cctx))) - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - ret = event->value.publish.cctx; - return ret; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - consider_restart (event->status); - ret = event->value.publish.cctx; - GNUNET_assert (publish == event->value.publish.pc); -#if VERBOSE - printf ("Publish is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.publish.completed, - (unsigned long long) event->value.publish.size, - event->value.publish.specifics.progress.depth, - (unsigned long long) event->value.publish.specifics. - progress.offset); -#endif - break; - - case GNUNET_FS_STATUS_PUBLISH_SUSPEND: - if (event->value.publish.pc == publish) - publish = NULL; - break; - - case GNUNET_FS_STATUS_PUBLISH_RESUME: - if (NULL == publish) - { - GNUNET_assert (GNUNET_YES == - GNUNET_FS_file_information_is_directory (event-> - value.publish. - fi)); - publish = event->value.publish.pc; - return "publish-context-dir"; - } - break; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - ret = event->value.publish.cctx; - fprintf (stderr, "Error publishing file: %s\n", - event->value.publish.specifics.error.message); - err = 1; - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_START: - consider_restart (event->status); - publish = event->value.publish.pc; - ret = event->value.publish.cctx; - if (0 == strcmp ("publish-context1", event->value.publish.cctx)) - { - GNUNET_assert (0 == - strcmp ("publish-context-dir", event->value.publish.pctx)); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (1 == event->value.publish.anonymity); - } - else if (0 == strcmp ("publish-context2", event->value.publish.cctx)) - { - GNUNET_assert (0 == - strcmp ("publish-context-dir", event->value.publish.pctx)); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (2 == event->value.publish.anonymity); - } - else if (0 == strcmp ("publish-context-dir", event->value.publish.cctx)) - { - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (3 == event->value.publish.anonymity); - } - else - GNUNET_assert (0); - break; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - consider_restart (event->status); - if ((NULL != event->value.publish.cctx) && - (0 == strcmp ("publish-context-dir", event->value.publish.cctx))) - GNUNET_assert (publish == event->value.publish.pc); - break; - - default: - printf ("Unexpected event: %d\n", event->status); - break; - } - return ret; -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *c, - struct GNUNET_TESTING_Peer *peer) -{ - const char *keywords[] = { - "down_foo", - "down_bar", - }; - char *buf; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *kuri; - struct GNUNET_FS_FileInformation *fi1; - struct GNUNET_FS_FileInformation *fi2; - struct GNUNET_FS_FileInformation *fidir; - size_t i; - struct GNUNET_FS_BlockOptions bo; - - cfg = c; - fs = GNUNET_FS_start (cfg, "test-fs-publish-persistence", &progress_cb, NULL, - GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); - GNUNET_assert (NULL != fs); - fn1 = GNUNET_DISK_mktemp ("gnunet-publish-test-dst"); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - (void) GNUNET_DISK_directory_remove (fn1); - GNUNET_assert (GNUNET_OK == - GNUNET_DISK_fn_write (fn1, buf, FILESIZE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE)); - GNUNET_free (buf); - - fn2 = GNUNET_DISK_mktemp ("gnunet-publish-test-dst"); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - (void) GNUNET_DISK_directory_remove (fn2); - GNUNET_assert (GNUNET_OK == - GNUNET_DISK_fn_write (fn2, buf, FILESIZE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE)); - GNUNET_free (buf); - - meta = GNUNET_FS_meta_data_create (); - kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); - bo.content_priority = 42; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); - fi1 = - GNUNET_FS_file_information_create_from_file (fs, "publish-context1", fn1, - kuri, meta, GNUNET_YES, &bo); - GNUNET_assert (NULL != fi1); - bo.anonymity_level = 2; - fi2 = - GNUNET_FS_file_information_create_from_file (fs, "publish-context2", fn2, - kuri, meta, GNUNET_YES, &bo); - GNUNET_assert (NULL != fi2); - bo.anonymity_level = 3; - fidir = - GNUNET_FS_file_information_create_empty_directory (fs, - "publish-context-dir", - kuri, meta, &bo, NULL); - GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi1)); - GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi2)); - GNUNET_FS_uri_destroy (kuri); - GNUNET_FS_meta_data_destroy (meta); - GNUNET_assert (NULL != fidir); - start = GNUNET_TIME_absolute_get (); - GNUNET_FS_publish_start (fs, fidir, NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); - GNUNET_assert (publish != NULL); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-publish-persistence", - "test_fs_publish_data.conf", - &run, NULL)) - return 1; - return err; -} - - -/* end of test_fs_publish_persistence.c */ diff --git a/src/fs/test_fs_search.c b/src/fs/test_fs_search.c deleted file mode 100644 index f9266582e..000000000 --- a/src/fs/test_fs_search.c +++ /dev/null @@ -1,252 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004-2013 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/test_fs_search.c - * @brief simple testcase for simple publish + search operation - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testing_lib.h" -#include "gnunet_fs_service.h" - - -/** - * File-size we use for testing. - */ -#define FILESIZE 1024 - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) - -/** - * How long should our test-content live? - */ -#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) - - -static struct GNUNET_TIME_Absolute start; - -static struct GNUNET_FS_Handle *fs; - -static struct GNUNET_FS_SearchContext *search; - -static struct GNUNET_FS_PublishContext *publish; - -static struct GNUNET_SCHEDULER_Task *timeout_task; - -static int err; - - -static void -abort_publish_task (void *cls) -{ - if (NULL != publish) - { - GNUNET_FS_publish_stop (publish); - publish = NULL; - } - if (NULL != timeout_task) - { - GNUNET_SCHEDULER_cancel (timeout_task); - timeout_task = NULL; - } -} - - -static void -abort_error (void *cls) -{ - fprintf (stderr, - "Timeout\n"); - timeout_task = NULL; - if (NULL != search) - { - GNUNET_FS_search_stop (search); - search = NULL; - } - if (NULL != publish) - { - GNUNET_FS_publish_stop (publish); - publish = NULL; - } - err = 1; -} - - -static void -abort_search_task (void *cls) -{ - if (NULL != search) - { - GNUNET_FS_search_stop (search); - search = NULL; - } -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - const char *keywords[] = { - "down_foo" - }; - struct GNUNET_FS_Uri *kuri; - - switch (event->status) - { - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publish is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.publish.completed, - (unsigned long long) event->value.publish.size, - event->value.publish.specifics.progress.depth, - (unsigned long long) event->value.publish.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - break; - - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - kuri = GNUNET_FS_uri_ksk_create_from_args (1, keywords); - start = GNUNET_TIME_absolute_get (); - search = - GNUNET_FS_search_start (fs, kuri, 1, GNUNET_FS_SEARCH_OPTION_NONE, - "search"); - GNUNET_FS_uri_destroy (kuri); - GNUNET_assert (search != NULL); - break; - - case GNUNET_FS_STATUS_SEARCH_RESULT: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Search complete.\n"); - GNUNET_SCHEDULER_add_now (&abort_search_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - fprintf (stderr, "Error publishing file: %s\n", - event->value.publish.specifics.error.message); - GNUNET_break (0); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - case GNUNET_FS_STATUS_SEARCH_ERROR: - fprintf (stderr, "Error searching file: %s\n", - event->value.search.specifics.error.message); - GNUNET_SCHEDULER_add_now (&abort_search_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_START: - GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); - GNUNET_assert (NULL == event->value.publish.pctx); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (1 == event->value.publish.anonymity); - break; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - GNUNET_assert (publish == event->value.publish.pc); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (1 == event->value.publish.anonymity); - GNUNET_FS_stop (fs); - fs = NULL; - break; - - case GNUNET_FS_STATUS_SEARCH_START: - GNUNET_assert (search == NULL); - GNUNET_assert (0 == strcmp ("search", event->value.search.cctx)); - GNUNET_assert (1 == event->value.search.anonymity); - break; - - case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED: - break; - - case GNUNET_FS_STATUS_SEARCH_STOPPED: - GNUNET_assert (search == event->value.search.sc); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - default: - fprintf (stderr, "Unexpected event: %d\n", event->status); - break; - } - return NULL; -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_TESTING_Peer *peer) -{ - const char *keywords[] = { - "down_foo", - "down_bar" - }; - char *buf; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *kuri; - struct GNUNET_FS_BlockOptions bo; - struct GNUNET_FS_FileInformation *fi; - size_t i; - - fs = GNUNET_FS_start (cfg, "test-fs-search", &progress_cb, NULL, - GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); - GNUNET_assert (NULL != fs); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - meta = GNUNET_FS_meta_data_create (); - kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); - bo.content_priority = 42; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); - fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context", - FILESIZE, buf, kuri, meta, - GNUNET_NO, &bo); - GNUNET_FS_uri_destroy (kuri); - GNUNET_FS_meta_data_destroy (meta); - GNUNET_assert (NULL != fi); - start = GNUNET_TIME_absolute_get (); - publish = - GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); - GNUNET_assert (publish != NULL); - timeout_task = GNUNET_SCHEDULER_add_delayed (LIFETIME, - &abort_error, NULL); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-search", - "test_fs_search_data.conf", - &run, NULL)) - return 1; - return err; -} - - -/* end of test_fs_search.c */ diff --git a/src/fs/test_fs_search_data.conf b/src/fs/test_fs_search_data.conf deleted file mode 100644 index 8b24e328d..000000000 --- a/src/fs/test_fs_search_data.conf +++ /dev/null @@ -1,7 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-search/ - -[transport] -PLUGINS = - diff --git a/src/fs/test_fs_search_persistence.c b/src/fs/test_fs_search_persistence.c deleted file mode 100644 index 4ddd40e73..000000000 --- a/src/fs/test_fs_search_persistence.c +++ /dev/null @@ -1,318 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004, 2005, 2006, 2008, 2009, 2010 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/test_fs_search_persistence.c - * @brief simple testcase for persistence of search operation - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testing_lib.h" -#include "gnunet_fs_service.h" - - -/** - * File-size we use for testing. - */ -#define FILESIZE 1024 - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) - -/** - * How long should our test-content live? - */ -#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) - - -static struct GNUNET_TIME_Absolute start; - -static struct GNUNET_FS_Handle *fs; - -static struct GNUNET_FS_SearchContext *search; - -static struct GNUNET_FS_PublishContext *publish; - -static const struct GNUNET_CONFIGURATION_Handle *cfg; - -static struct GNUNET_SCHEDULER_Task *timeout_task; - -static int err; - - -static void -abort_error (void *cls) -{ - timeout_task = NULL; - fprintf (stderr, - "Timeout\n"); - if (NULL != search) - { - GNUNET_FS_search_stop (search); - search = NULL; - } - if (NULL != publish) - { - GNUNET_FS_publish_stop (publish); - publish = NULL; - } - err = 1; -} - - -static void -abort_publish_task (void *cls) -{ - if (NULL != publish) - { - GNUNET_FS_publish_stop (publish); - publish = NULL; - } - if (NULL != timeout_task) - { - GNUNET_SCHEDULER_cancel (timeout_task); - timeout_task = NULL; - } -} - - -static void -abort_search_task (void *cls) -{ - if (NULL != search) - { - GNUNET_FS_search_stop (search); - search = NULL; - } -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event); - - -static void -restart_fs_task (void *cls) -{ - GNUNET_FS_stop (fs); - fs = GNUNET_FS_start (cfg, "test-fs-search-persistence", &progress_cb, NULL, - GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); -} - - -/** - * Consider scheduling the restart-task. - * Only runs the restart task once per event - * category. - * - * @param ev type of the event to consider - */ -static void -consider_restart (int ev) -{ - static int prev[32]; - static int off; - int i; - - for (i = 0; i < off; i++) - if (prev[i] == ev) - return; - prev[off++] = ev; - GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_URGENT, - &restart_fs_task, NULL); -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - const char *keywords[] = { - "down_foo" - }; - struct GNUNET_FS_Uri *kuri; - - switch (event->status) - { - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publish is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.publish.completed, - (unsigned long long) event->value.publish.size, - event->value.publish.specifics.progress.depth, - (unsigned long long) event->value.publish.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - break; - - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - kuri = GNUNET_FS_uri_ksk_create_from_args (1, keywords); - start = GNUNET_TIME_absolute_get (); - GNUNET_FS_search_start (fs, kuri, 1, GNUNET_FS_SEARCH_OPTION_NONE, - "search"); - GNUNET_FS_uri_destroy (kuri); - GNUNET_assert (search != NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_SUSPEND: - if (event->value.publish.pc == publish) - publish = NULL; - break; - - case GNUNET_FS_STATUS_PUBLISH_RESUME: - if (NULL == publish) - publish = event->value.publish.pc; - break; - - case GNUNET_FS_STATUS_SEARCH_RESULT: - /* FIXME: consider_restart (event->status); cannot be tested with - * search result since we exit here after the first one... */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Search complete.\n"); - GNUNET_SCHEDULER_add_now (&abort_search_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - fprintf (stderr, "Error publishing file: %s\n", - event->value.publish.specifics.error.message); - GNUNET_break (0); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - case GNUNET_FS_STATUS_SEARCH_ERROR: - fprintf (stderr, "Error searching file: %s\n", - event->value.search.specifics.error.message); - GNUNET_SCHEDULER_add_now (&abort_search_task, NULL); - break; - - case GNUNET_FS_STATUS_SEARCH_SUSPEND: - if (event->value.search.sc == search) - search = NULL; - break; - - case GNUNET_FS_STATUS_SEARCH_RESUME: - if (NULL == search) - { - search = event->value.search.sc; - return "search"; - } - break; - - case GNUNET_FS_STATUS_PUBLISH_START: - GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); - GNUNET_assert (NULL == event->value.publish.pctx); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (1 == event->value.publish.anonymity); - break; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - GNUNET_assert (publish == event->value.publish.pc); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (1 == event->value.publish.anonymity); - GNUNET_FS_stop (fs); - fs = NULL; - break; - - case GNUNET_FS_STATUS_SEARCH_START: - consider_restart (event->status); - GNUNET_assert (search == NULL); - search = event->value.search.sc; - GNUNET_assert (0 == strcmp ("search", event->value.search.cctx)); - GNUNET_assert (1 == event->value.search.anonymity); - break; - - case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED: - break; - - case GNUNET_FS_STATUS_SEARCH_STOPPED: - GNUNET_assert (search == event->value.search.sc); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - search = NULL; - break; - - default: - fprintf (stderr, "Unexpected event: %d\n", event->status); - break; - } - return NULL; -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *c, - struct GNUNET_TESTING_Peer *peer) -{ - const char *keywords[] = { - "down_foo", - "down_bar" - }; - char *buf; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *kuri; - struct GNUNET_FS_FileInformation *fi; - size_t i; - struct GNUNET_FS_BlockOptions bo; - - cfg = c; - fs = GNUNET_FS_start (cfg, "test-fs-search-persistence", &progress_cb, NULL, - GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); - GNUNET_assert (NULL != fs); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - meta = GNUNET_FS_meta_data_create (); - kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); - bo.content_priority = 42; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); - fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context", - FILESIZE, buf, kuri, meta, - GNUNET_NO, &bo); - GNUNET_FS_uri_destroy (kuri); - GNUNET_FS_meta_data_destroy (meta); - GNUNET_assert (NULL != fi); - start = GNUNET_TIME_absolute_get (); - publish = - GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); - GNUNET_assert (publish != NULL); - timeout_task = GNUNET_SCHEDULER_add_delayed (LIFETIME, - &abort_error, NULL); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-search-persistence", - "test_fs_search_data.conf", - &run, NULL)) - return 1; - return err; -} - - -/* end of test_fs_search_persistence.c */ diff --git a/src/fs/test_fs_search_probes.c b/src/fs/test_fs_search_probes.c deleted file mode 100644 index 776babaee..000000000 --- a/src/fs/test_fs_search_probes.c +++ /dev/null @@ -1,258 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_fs_search_probes.c - * @brief simple testcase for publish + search operation with probes - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testing_lib.h" -#include "gnunet_fs_service.h" - - -/** - * File-size we use for testing. - */ -#define FILESIZE 1024 - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) - -/** - * How long should our test-content live? - */ -#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) - - -static struct GNUNET_TIME_Absolute start; - -static struct GNUNET_FS_Handle *fs; - -static struct GNUNET_FS_SearchContext *search; - -static struct GNUNET_FS_PublishContext *publish; - -static struct GNUNET_SCHEDULER_Task *timeout_task; - -static int err; - - -static void -abort_error (void *cls) -{ - timeout_task = NULL; - fprintf (stderr, - "Timeout\n"); - if (NULL != search) - { - GNUNET_FS_search_stop (search); - search = NULL; - } - if (NULL != publish) - { - GNUNET_FS_publish_stop (publish); - publish = NULL; - } - err = 1; -} - - -static void -abort_publish_task (void *cls) -{ - if (NULL != publish) - { - GNUNET_FS_publish_stop (publish); - publish = NULL; - } - if (NULL != timeout_task) - { - GNUNET_SCHEDULER_cancel (timeout_task); - timeout_task = NULL; - } -} - - -static void -abort_search_task (void *cls) -{ - if (search != NULL) - GNUNET_FS_search_stop (search); - search = NULL; -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - const char *keywords[] = { - "down_foo" - }; - struct GNUNET_FS_Uri *kuri; - - switch (event->status) - { - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publish is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.publish.completed, - (unsigned long long) event->value.publish.size, - event->value.publish.specifics.progress.depth, - (unsigned long long) event->value.publish.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - break; - - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - kuri = GNUNET_FS_uri_ksk_create_from_args (1, keywords); - start = GNUNET_TIME_absolute_get (); - search = - GNUNET_FS_search_start (fs, kuri, 1, GNUNET_FS_SEARCH_OPTION_NONE, - "search"); - GNUNET_FS_uri_destroy (kuri); - GNUNET_assert (search != NULL); - break; - - case GNUNET_FS_STATUS_SEARCH_RESULT: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Search complete.\n"); - break; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - fprintf (stderr, "Error publishing file: %s\n", - event->value.publish.specifics.error.message); - GNUNET_break (0); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - case GNUNET_FS_STATUS_SEARCH_ERROR: - fprintf (stderr, "Error searching file: %s\n", - event->value.search.specifics.error.message); - GNUNET_SCHEDULER_add_now (&abort_search_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_START: - GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); - GNUNET_assert (NULL == event->value.publish.pctx); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (1 == event->value.publish.anonymity); - break; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - GNUNET_assert (publish == event->value.publish.pc); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (1 == event->value.publish.anonymity); - GNUNET_FS_stop (fs); - fs = NULL; - break; - - case GNUNET_FS_STATUS_SEARCH_UPDATE: - if ((0 < event->value.search.specifics.update.availability_rank) && - (0 < event->value.search.specifics.update.availability_certainty)) - GNUNET_SCHEDULER_add_now (&abort_search_task, NULL); - break; - - case GNUNET_FS_STATUS_SEARCH_START: - GNUNET_assert (search == NULL); - GNUNET_assert (0 == strcmp ("search", event->value.search.cctx)); - GNUNET_assert (1 == event->value.search.anonymity); - break; - - case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED: - break; - - case GNUNET_FS_STATUS_SEARCH_STOPPED: - GNUNET_assert (search == event->value.search.sc); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - default: - fprintf (stderr, - "Unexpected event: %d\n", - event->status); - break; - } - return NULL; -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_TESTING_Peer *peer) -{ - const char *keywords[] = { - "down_foo", - "down_bar" - }; - char *buf; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *kuri; - struct GNUNET_FS_BlockOptions bo; - struct GNUNET_FS_FileInformation *fi; - size_t i; - - fs = GNUNET_FS_start (cfg, "test-fs-search", &progress_cb, NULL, - GNUNET_FS_FLAGS_DO_PROBES, - GNUNET_FS_OPTIONS_END); - GNUNET_assert (NULL != fs); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - meta = GNUNET_FS_meta_data_create (); - kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); - bo.content_priority = 42; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); - fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context", - FILESIZE, buf, kuri, meta, - GNUNET_NO, &bo); - GNUNET_FS_uri_destroy (kuri); - GNUNET_FS_meta_data_destroy (meta); - GNUNET_assert (NULL != fi); - start = GNUNET_TIME_absolute_get (); - publish = - GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); - GNUNET_assert (publish != NULL); - timeout_task = GNUNET_SCHEDULER_add_delayed (LIFETIME, - &abort_error, NULL); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-search-probes", - "test_fs_search_data.conf", - &run, NULL)) - return 1; - return err; -} - - -/* end of test_fs_search_probes.c */ diff --git a/src/fs/test_fs_search_with_and.c b/src/fs/test_fs_search_with_and.c deleted file mode 100644 index 9c20936b6..000000000 --- a/src/fs/test_fs_search_with_and.c +++ /dev/null @@ -1,272 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004-2013 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/test_fs_search_with_and.c - * @brief testcase for publishing multiple files and search with a and operator - * @author Bruno Cabral - 99% based on Christian Grothoff code - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testing_lib.h" -#include "gnunet_fs_service.h" - - -/** - * File-size we use for testing. - */ -#define FILESIZE 1024 - -/** - * Number of files for testing. - */ -#define NUM_FILES 10 - - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) - -/** - * How long should our test-content live? - */ -#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) - - -static struct GNUNET_TIME_Absolute start; - -static struct GNUNET_FS_Handle *fs; - -static struct GNUNET_FS_SearchContext *search; - -static struct GNUNET_FS_PublishContext *publish; - -static struct GNUNET_SCHEDULER_Task *timeout_task; - -static int err; - -static int processed_files; - - -static void -abort_publish_task (void *cls) -{ - if (NULL != publish) - { - GNUNET_FS_publish_stop (publish); - publish = NULL; - } - if (NULL != timeout_task) - { - GNUNET_SCHEDULER_cancel (timeout_task); - timeout_task = NULL; - } -} - - -static void -abort_error (void *cls) -{ - fprintf (stderr, - "Timeout\n"); - timeout_task = NULL; - if (NULL != search) - { - GNUNET_FS_search_stop (search); - search = NULL; - } - if (NULL != publish) - { - GNUNET_FS_publish_stop (publish); - publish = NULL; - } - err = 1; -} - - -static void -abort_search_task (void *cls) -{ - if (NULL != search) - { - GNUNET_FS_search_stop (search); - search = NULL; - } -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - struct GNUNET_FS_Uri *kuri; - - switch (event->status) - { - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publish is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.publish.completed, - (unsigned long long) event->value.publish.size, - event->value.publish.specifics.progress.depth, - (unsigned long long) event->value.publish.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - break; - - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - processed_files++; - if (processed_files == NUM_FILES) - { - char *emsg = NULL; - kuri = GNUNET_FS_uri_ksk_create ("+down_foo +down_bar", &emsg); - GNUNET_assert (kuri != NULL); - - start = GNUNET_TIME_absolute_get (); - search = - GNUNET_FS_search_start (fs, kuri, 1, GNUNET_FS_SEARCH_OPTION_NONE, - "search"); - GNUNET_FS_uri_destroy (kuri); - GNUNET_assert (search != NULL); - } - break; - - case GNUNET_FS_STATUS_SEARCH_RESULT: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Search complete.\n"); - GNUNET_SCHEDULER_add_now (&abort_search_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - fprintf (stderr, "Error publishing file: %s\n", - event->value.publish.specifics.error.message); - GNUNET_break (0); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - case GNUNET_FS_STATUS_SEARCH_ERROR: - fprintf (stderr, "Error searching file: %s\n", - event->value.search.specifics.error.message); - GNUNET_SCHEDULER_add_now (&abort_search_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_START: - GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); - GNUNET_assert (NULL == event->value.publish.pctx); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (1 == event->value.publish.anonymity); - break; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - GNUNET_assert (publish == event->value.publish.pc); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (1 == event->value.publish.anonymity); - GNUNET_FS_stop (fs); - fs = NULL; - break; - - case GNUNET_FS_STATUS_SEARCH_START: - GNUNET_assert (search == NULL); - GNUNET_assert (0 == strcmp ("search", event->value.search.cctx)); - GNUNET_assert (1 == event->value.search.anonymity); - break; - - case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED: - break; - - case GNUNET_FS_STATUS_SEARCH_STOPPED: - GNUNET_assert (search == event->value.search.sc); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - default: - fprintf (stderr, "Unexpected event: %d\n", event->status); - break; - } - return NULL; -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_TESTING_Peer *peer) -{ - const char *keywords[] = { - "down_foo", - "down_bar" - }; - char *buf; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *kuri; - struct GNUNET_FS_BlockOptions bo; - struct GNUNET_FS_FileInformation *fi; - size_t i; - size_t j; - - fs = GNUNET_FS_start (cfg, "test-fs-search", &progress_cb, NULL, - GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); - GNUNET_assert (NULL != fs); - - processed_files = 0; - for (j = 0; j < NUM_FILES; j++) - { - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - meta = GNUNET_FS_meta_data_create (); - kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); - bo.content_priority = 42; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); - fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context", - FILESIZE, buf, kuri, meta, - GNUNET_NO, &bo); - GNUNET_FS_uri_destroy (kuri); - GNUNET_FS_meta_data_destroy (meta); - GNUNET_assert (NULL != fi); - start = GNUNET_TIME_absolute_get (); - publish = - GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); - GNUNET_assert (publish != NULL); - } - - - timeout_task = GNUNET_SCHEDULER_add_delayed (LIFETIME, - &abort_error, NULL); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-search-with-and", - "test_fs_search_data.conf", - &run, NULL)) - return 1; - return err; -} - - -/* end of test_fs_search.c */ diff --git a/src/fs/test_fs_start_stop.c b/src/fs/test_fs_start_stop.c deleted file mode 100644 index 509fbe76a..000000000 --- a/src/fs/test_fs_start_stop.c +++ /dev/null @@ -1,64 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004, 2005, 2006, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/test_fs_start_stop.c - * @brief testcase for fs.c (start-stop only) - * @author Christian Grothoff - */ - -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testing_lib.h" -#include "gnunet_fs_service.h" - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - return NULL; -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_TESTING_Peer *peer) -{ - struct GNUNET_FS_Handle *fs; - - fs = GNUNET_FS_start (cfg, "test-fs-start-stop", &progress_cb, NULL, - GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); - GNUNET_assert (NULL != fs); - GNUNET_FS_stop (fs); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-start-stop", - "test_fs_data.conf", - &run, NULL)) - return 1; - return 0; -} - - -/* end of test_fs_start_stop.c */ diff --git a/src/fs/test_fs_test_lib.c b/src/fs/test_fs_test_lib.c deleted file mode 100644 index 714dd452e..000000000 --- a/src/fs/test_fs_test_lib.c +++ /dev/null @@ -1,181 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2010, 2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_fs_test_lib.c - * @brief test fs test library - * @author Christian Grothoff - */ -#include "platform.h" -#include "fs_test_lib.h" - -#define VERBOSE GNUNET_NO - -/** - * File-size we use for testing. - */ -#define FILESIZE (1024 * 1024 * 2) - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300) - -#define NUM_DAEMONS 2 - -#define SEED 42 - -static struct GNUNET_TESTBED_Peer *the_peers[NUM_DAEMONS]; - -static struct GNUNET_TIME_Absolute start_time; - -static int ret; - - -static void -do_stop (void *cls) -{ - char *fn = cls; - - if (0 == - GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_add (start_time, - TIMEOUT)). - rel_value_us) - { - GNUNET_break (0); - ret = 1; - } - else - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Finished download, shutting down\n"); - } - if (NULL != fn) - { - GNUNET_DISK_directory_remove (fn); - GNUNET_free (fn); - } - GNUNET_SCHEDULER_shutdown (); -} - - -static void -do_download (void *cls, - const struct GNUNET_FS_Uri *uri, - const char *fn) -{ - if (NULL == uri) - { - GNUNET_break (0); - GNUNET_SCHEDULER_shutdown (); - ret = 1; - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Downloading %llu bytes\n", - (unsigned long long) FILESIZE); - start_time = GNUNET_TIME_absolute_get (); - GNUNET_FS_TEST_download (the_peers[0], - TIMEOUT, 1, SEED, - uri, - VERBOSE, - &do_stop, - (NULL == fn) ? NULL : GNUNET_strdup (fn)); -} - - -static void -do_publish (void *cls, - struct GNUNET_TESTBED_Operation *op, - const char *emsg) -{ - GNUNET_TESTBED_operation_done (op); - if (NULL != emsg) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to connect peers: %s\n", emsg); - GNUNET_break (0); - ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n", - (unsigned long long) FILESIZE); - GNUNET_FS_TEST_publish (the_peers[0], TIMEOUT, 1, GNUNET_NO, FILESIZE, SEED, - VERBOSE, &do_download, NULL); -} - - -/** - * Actual main function for the test. - * - * @param cls closure - * @param h the run handle - * @param num_peers number of peers in 'peers' - * @param peers handle to peers run in the testbed - * @param links_succeeded the number of overlay link connection attempts that - * succeeded - * @param links_failed the number of overlay link connection attempts that - * failed - */ -static void -run (void *cls, - struct GNUNET_TESTBED_RunHandle *h, - unsigned int num_peers, - struct GNUNET_TESTBED_Peer **peers, - unsigned int links_succeeded, - unsigned int links_failed) -{ - unsigned int i; - - GNUNET_assert (NUM_DAEMONS == num_peers); - for (i = 0; i < num_peers; i++) - the_peers[i] = peers[i]; - GNUNET_TESTBED_overlay_connect (NULL, - &do_publish, - NULL, - peers[0], - peers[1]); -} - - -/** - * Main function that initializes the testbed. - * - * @param argc ignored - * @param argv ignored - * @return 0 on success - */ -int -main (int argc, char *argv[]) -{ - GNUNET_DISK_purge_cfg_dir ("fs_test_lib_data.conf", - "GNUNET_TEST_HOME"); - (void) GNUNET_TESTBED_test_run ("test_fs_test_lib", - "fs_test_lib_data.conf", - NUM_DAEMONS, - 0, NULL, NULL, - &run, NULL); - GNUNET_DISK_purge_cfg_dir ("fs_test_lib_data.conf", - "GNUNET_TEST_HOME"); - return ret; -} - - -/* end of test_fs_test_lib.c */ diff --git a/src/fs/test_fs_unindex.c b/src/fs/test_fs_unindex.c deleted file mode 100644 index dbc33090d..000000000 --- a/src/fs/test_fs_unindex.c +++ /dev/null @@ -1,237 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004, 2005, 2006, 2008, 2009 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_fs_unindex.c - * @brief simple testcase for simple publish + unindex operation - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_fs_service.h" -#include "gnunet_testing_lib.h" - - -/** - * File-size we use for testing. - */ -#define FILESIZE (1024 * 1024 * 2) - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) - -/** - * How long should our test-content live? - */ -#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) - - -static struct GNUNET_TIME_Absolute start; - -static struct GNUNET_FS_Handle *fs; - -static struct GNUNET_FS_UnindexContext *unindex; - -static struct GNUNET_FS_PublishContext *publish; - -static char *fn; - - -static void -abort_publish_task (void *cls) -{ - GNUNET_FS_publish_stop (publish); - publish = NULL; -} - - -static void -abort_unindex_task (void *cls) -{ - GNUNET_FS_unindex_stop (unindex); - unindex = NULL; - GNUNET_DISK_directory_remove (fn); - GNUNET_free (fn); - fn = NULL; -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - switch (event->status) - { - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publish is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.publish.completed, - (unsigned long long) event->value.publish.size, - event->value.publish.specifics.progress.depth, - (unsigned long long) event->value.publish.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - break; - - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - printf ("Publishing complete, %llu kbps.\n", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024)); - start = GNUNET_TIME_absolute_get (); - unindex = GNUNET_FS_unindex_start (fs, fn, "unindex"); - GNUNET_assert (unindex != NULL); - break; - - case GNUNET_FS_STATUS_UNINDEX_COMPLETED: - printf ("Unindex complete, %llu kbps.\n", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024)); - GNUNET_SCHEDULER_add_now (&abort_unindex_task, NULL); - break; - - case GNUNET_FS_STATUS_UNINDEX_PROGRESS: - GNUNET_assert (unindex == event->value.unindex.uc); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Unindex is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.unindex.completed, - (unsigned long long) event->value.unindex.size, - event->value.unindex.specifics.progress.depth, - (unsigned long long) event->value.unindex.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - fprintf (stderr, "Error publishing file: %s\n", - event->value.publish.specifics.error.message); - GNUNET_break (0); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - case GNUNET_FS_STATUS_UNINDEX_ERROR: - fprintf (stderr, "Error unindexing file: %s\n", - event->value.unindex.specifics.error.message); - GNUNET_SCHEDULER_add_now (&abort_unindex_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_START: - GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); - GNUNET_assert (NULL == event->value.publish.pctx); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (1 == event->value.publish.anonymity); - break; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - GNUNET_assert (publish == event->value.publish.pc); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (1 == event->value.publish.anonymity); - GNUNET_FS_stop (fs); - fs = NULL; - break; - - case GNUNET_FS_STATUS_UNINDEX_START: - GNUNET_assert (unindex == NULL); - GNUNET_assert (0 == strcmp ("unindex", event->value.unindex.cctx)); - GNUNET_assert (0 == strcmp (fn, event->value.unindex.filename)); - GNUNET_assert (FILESIZE == event->value.unindex.size); - GNUNET_assert (0 == event->value.unindex.completed); - break; - - case GNUNET_FS_STATUS_UNINDEX_STOPPED: - GNUNET_assert (unindex == event->value.unindex.uc); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - default: - printf ("Unexpected event: %d\n", event->status); - break; - } - return NULL; -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_TESTING_Peer *peer) -{ - const char *keywords[] = { - "down_foo", - "down_bar", - }; - char *buf; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *kuri; - struct GNUNET_FS_FileInformation *fi; - size_t i; - struct GNUNET_FS_BlockOptions bo; - - fn = GNUNET_DISK_mktemp ("gnunet-unindex-test-dst"); - fs = GNUNET_FS_start (cfg, "test-fs-unindex", &progress_cb, NULL, - GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); - GNUNET_assert (NULL != fs); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - (void) GNUNET_DISK_directory_remove (fn); - GNUNET_assert (GNUNET_OK == - GNUNET_DISK_fn_write (fn, buf, FILESIZE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE)); - GNUNET_free (buf); - meta = GNUNET_FS_meta_data_create (); - kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); - bo.content_priority = 42; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); - fi = GNUNET_FS_file_information_create_from_file (fs, "publish-context", fn, - kuri, meta, GNUNET_YES, - &bo); - GNUNET_FS_uri_destroy (kuri); - GNUNET_FS_meta_data_destroy (meta); - GNUNET_assert (NULL != fi); - start = GNUNET_TIME_absolute_get (); - publish = - GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); - GNUNET_assert (publish != NULL); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-unindex", - "test_fs_unindex_data.conf", - &run, NULL)) - return 1; - return 0; -} - - -/* end of test_fs_unindex.c */ diff --git a/src/fs/test_fs_unindex_data.conf b/src/fs/test_fs_unindex_data.conf deleted file mode 100644 index dde401857..000000000 --- a/src/fs/test_fs_unindex_data.conf +++ /dev/null @@ -1,7 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-unindex/ - -[transport] -PLUGINS = - diff --git a/src/fs/test_fs_unindex_persistence.c b/src/fs/test_fs_unindex_persistence.c deleted file mode 100644 index b81ce64ab..000000000 --- a/src/fs/test_fs_unindex_persistence.c +++ /dev/null @@ -1,307 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2004, 2005, 2006, 2008, 2009, 2010 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_fs_unindex_persistence.c - * @brief simple testcase for simple publish + unindex operation - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_testing_lib.h" -#include "gnunet_fs_service.h" - -/** - * File-size we use for testing. - */ -#define FILESIZE (1024 * 1024 * 2) - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) - -/** - * How long should our test-content live? - */ -#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) - - -static struct GNUNET_TIME_Absolute start; - -static struct GNUNET_FS_Handle *fs; - -static struct GNUNET_FS_UnindexContext *unindex; - -static struct GNUNET_FS_PublishContext *publish; - -static char *fn; - -static const struct GNUNET_CONFIGURATION_Handle *cfg; - - -static void -abort_publish_task (void *cls) -{ - GNUNET_FS_publish_stop (publish); - publish = NULL; -} - - -static void -abort_unindex_task (void *cls) -{ - if (unindex != NULL) - { - GNUNET_FS_unindex_stop (unindex); - unindex = NULL; - } - if (fn != NULL) - { - GNUNET_DISK_directory_remove (fn); - GNUNET_free (fn); - fn = NULL; - } -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event); - - -static void -restart_fs_task (void *cls) -{ - GNUNET_FS_stop (fs); - fs = GNUNET_FS_start (cfg, "test-fs-unindex-persistence", &progress_cb, NULL, - GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); -} - - -/** - * Consider scheduling the restart-task. - * Only runs the restart task once per event - * category. - * - * @param ev type of the event to consider - */ -static void -consider_restart (int ev) -{ - static int prev[32]; - static int off; - int i; - - for (i = 0; i < off; i++) - if (prev[i] == ev) - return; - prev[off++] = ev; - GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_URGENT, - &restart_fs_task, NULL); -} - - -static void * -progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) -{ - switch (event->status) - { - case GNUNET_FS_STATUS_PUBLISH_PROGRESS: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publish is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.publish.completed, - (unsigned long long) event->value.publish.size, - event->value.publish.specifics.progress.depth, - (unsigned long long) event->value.publish.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY: - break; - - case GNUNET_FS_STATUS_PUBLISH_COMPLETED: - printf ("Publishing complete, %llu kbps.\n", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024)); - start = GNUNET_TIME_absolute_get (); - unindex = GNUNET_FS_unindex_start (fs, fn, "unindex"); - GNUNET_assert (unindex != NULL); - break; - - case GNUNET_FS_STATUS_UNINDEX_COMPLETED: - printf ("Unindex complete, %llu kbps.\n", - (unsigned long long) (FILESIZE * 1000000LL - / (1 - + GNUNET_TIME_absolute_get_duration - (start).rel_value_us) / 1024)); - GNUNET_SCHEDULER_add_now (&abort_unindex_task, NULL); - break; - - case GNUNET_FS_STATUS_UNINDEX_PROGRESS: - consider_restart (event->status); - GNUNET_assert (unindex == event->value.unindex.uc); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Unindex is progressing (%llu/%llu at level %u off %llu)...\n", - (unsigned long long) event->value.unindex.completed, - (unsigned long long) event->value.unindex.size, - event->value.unindex.specifics.progress.depth, - (unsigned long long) event->value.unindex.specifics. - progress.offset); - break; - - case GNUNET_FS_STATUS_PUBLISH_SUSPEND: - if (event->value.publish.pc == publish) - publish = NULL; - break; - - case GNUNET_FS_STATUS_PUBLISH_RESUME: - if (NULL == publish) - { - publish = event->value.publish.pc; - return "publish-context"; - } - break; - - case GNUNET_FS_STATUS_UNINDEX_SUSPEND: - GNUNET_assert (event->value.unindex.uc == unindex); - unindex = NULL; - break; - - case GNUNET_FS_STATUS_UNINDEX_RESUME: - GNUNET_assert (NULL == unindex); - unindex = event->value.unindex.uc; - return "unindex"; - - case GNUNET_FS_STATUS_PUBLISH_ERROR: - fprintf (stderr, "Error publishing file: %s\n", - event->value.publish.specifics.error.message); - GNUNET_break (0); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - case GNUNET_FS_STATUS_UNINDEX_ERROR: - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Error unindexing file: %s\n", - event->value.unindex.specifics.error.message); - GNUNET_SCHEDULER_add_now (&abort_unindex_task, NULL); - break; - - case GNUNET_FS_STATUS_PUBLISH_START: - GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); - GNUNET_assert (NULL == event->value.publish.pctx); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (0 == event->value.publish.completed); - GNUNET_assert (1 == event->value.publish.anonymity); - break; - - case GNUNET_FS_STATUS_PUBLISH_STOPPED: - GNUNET_assert (publish == event->value.publish.pc); - GNUNET_assert (FILESIZE == event->value.publish.size); - GNUNET_assert (1 == event->value.publish.anonymity); - GNUNET_FS_stop (fs); - fs = NULL; - break; - - case GNUNET_FS_STATUS_UNINDEX_START: - consider_restart (event->status); - GNUNET_assert (unindex == NULL); - GNUNET_assert (0 == strcmp ("unindex", event->value.unindex.cctx)); - GNUNET_assert (0 == strcmp (fn, event->value.unindex.filename)); - GNUNET_assert (FILESIZE == event->value.unindex.size); - GNUNET_assert (0 == event->value.unindex.completed); - break; - - case GNUNET_FS_STATUS_UNINDEX_STOPPED: - GNUNET_assert (unindex == event->value.unindex.uc); - GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); - break; - - default: - printf ("Unexpected event: %d\n", event->status); - break; - } - return NULL; -} - - -static void -run (void *cls, - const struct GNUNET_CONFIGURATION_Handle *c, - struct GNUNET_TESTING_Peer *peer) -{ - const char *keywords[] = { - "down_foo", - "down_bar", - }; - char *buf; - struct GNUNET_FS_MetaData *meta; - struct GNUNET_FS_Uri *kuri; - struct GNUNET_FS_FileInformation *fi; - size_t i; - struct GNUNET_FS_BlockOptions bo; - - cfg = c; - fn = GNUNET_DISK_mktemp ("gnunet-unindex-test-dst"); - fs = GNUNET_FS_start (cfg, "test-fs-unindex-persistence", &progress_cb, NULL, - GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); - GNUNET_assert (NULL != fs); - buf = GNUNET_malloc (FILESIZE); - for (i = 0; i < FILESIZE; i++) - buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); - (void) GNUNET_DISK_directory_remove (fn); - GNUNET_assert (GNUNET_OK == - GNUNET_DISK_fn_write (fn, buf, FILESIZE, - GNUNET_DISK_PERM_USER_READ - | GNUNET_DISK_PERM_USER_WRITE)); - GNUNET_free (buf); - meta = GNUNET_FS_meta_data_create (); - kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); - bo.content_priority = 42; - bo.anonymity_level = 1; - bo.replication_level = 0; - bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); - fi = GNUNET_FS_file_information_create_from_file (fs, "publish-context", fn, - kuri, meta, GNUNET_YES, - &bo); - GNUNET_FS_uri_destroy (kuri); - GNUNET_FS_meta_data_destroy (meta); - GNUNET_assert (NULL != fi); - start = GNUNET_TIME_absolute_get (); - publish = - GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, - GNUNET_FS_PUBLISH_OPTION_NONE); - GNUNET_assert (publish != NULL); -} - - -int -main (int argc, char *argv[]) -{ - if (0 != GNUNET_TESTING_peer_run ("test-fs-unindex-persistence", - "test_fs_unindex_data.conf", - &run, NULL)) - return 1; - return 0; -} - - -/* end of test_fs_unindex_persistence.c */ diff --git a/src/fs/test_fs_uri.c b/src/fs/test_fs_uri.c deleted file mode 100644 index e0f23097b..000000000 --- a/src/fs/test_fs_uri.c +++ /dev/null @@ -1,340 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2003-2014 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/test_fs_uri.c - * @brief Test for fs_uri.c - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "gnunet_fs_service.h" -#include "fs_api.h" - - -static int -testKeyword () -{ - char *uri; - struct GNUNET_FS_Uri *ret; - char *emsg; - - if (NULL != (ret = GNUNET_FS_uri_parse ("gnunet://fs/ksk/++", &emsg))) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - GNUNET_free (emsg); - ret = GNUNET_FS_uri_parse ("gnunet://fs/ksk/foo+bar", &emsg); - if (NULL == ret) - { - GNUNET_free (emsg); - GNUNET_assert (0); - } - if (! GNUNET_FS_uri_test_ksk (ret)) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - if ((2 != ret->data.ksk.keywordCount) || - (0 != strcmp (" foo", ret->data.ksk.keywords[0])) || - (0 != strcmp (" bar", ret->data.ksk.keywords[1]))) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - - uri = GNUNET_FS_uri_to_string (ret); - if (0 != strcmp (uri, "gnunet://fs/ksk/foo+bar")) - { - GNUNET_free (uri); - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - GNUNET_free (uri); - GNUNET_FS_uri_destroy (ret); - return 0; -} - - -static int -testLocation () -{ - struct GNUNET_FS_Uri *uri; - char *uric; - struct GNUNET_FS_Uri *uri2; - struct GNUNET_FS_Uri *baseURI; - char *emsg = NULL; - struct GNUNET_CRYPTO_EddsaPrivateKey pk; - - baseURI = - GNUNET_FS_uri_parse - ( - "gnunet://fs/chk/4QZP479A9SKGFNMQ2ZBCYE71YV2QMTVGWTVPB6A10ASVCKXDHB05DKPSC7ZF6E9P9W1VE47394EQY7NXA47Q6R35M7P1MJPGP59D1Z8.D54QD1K5XCG5878T6YZ19AM60MQ6FC0YNVK7QY08KK0KM0FJJ3KQWYG112FN5T07KN7J0X35DF6WVBT9B8ZMZ3X2BXJ22X3KFQ6MV2G.15999", - &emsg); - GNUNET_assert (baseURI != NULL); - GNUNET_assert (emsg == NULL); - GNUNET_CRYPTO_eddsa_key_create (&pk); - uri = GNUNET_FS_uri_loc_create (baseURI, - &pk, - GNUNET_TIME_absolute_get ()); - if (NULL == uri) - { - GNUNET_break (0); - GNUNET_FS_uri_destroy (baseURI); - return 1; - } - if (! GNUNET_FS_uri_test_loc (uri)) - { - GNUNET_break (0); - GNUNET_FS_uri_destroy (uri); - GNUNET_FS_uri_destroy (baseURI); - return 1; - } - uri2 = GNUNET_FS_uri_loc_get_uri (uri); - if (! GNUNET_FS_uri_test_equal (baseURI, uri2)) - { - GNUNET_break (0); - GNUNET_FS_uri_destroy (uri); - GNUNET_FS_uri_destroy (uri2); - GNUNET_FS_uri_destroy (baseURI); - return 1; - } - GNUNET_FS_uri_destroy (uri2); - GNUNET_FS_uri_destroy (baseURI); - uric = GNUNET_FS_uri_to_string (uri); -#if 0 - /* not for the faint of heart: */ - printf ("URI: `%s'\n", uric); -#endif - uri2 = GNUNET_FS_uri_parse (uric, &emsg); - GNUNET_free (uric); - if (uri2 == NULL) - { - fprintf (stderr, "URI parsing failed: %s\n", emsg); - GNUNET_break (0); - GNUNET_FS_uri_destroy (uri); - GNUNET_free (emsg); - return 1; - } - GNUNET_assert (NULL == emsg); - if (GNUNET_YES != GNUNET_FS_uri_test_equal (uri, uri2)) - { - GNUNET_break (0); - GNUNET_FS_uri_destroy (uri); - GNUNET_FS_uri_destroy (uri2); - return 1; - } - GNUNET_FS_uri_destroy (uri2); - GNUNET_FS_uri_destroy (uri); - return 0; -} - - -static int -testNamespace (int i) -{ - char *uri; - struct GNUNET_FS_Uri *ret; - char *emsg; - struct GNUNET_CRYPTO_EcdsaPrivateKey ph; - struct GNUNET_CRYPTO_EcdsaPublicKey id; - char buf[1024]; - char ubuf[1024]; - char *sret; - - if (NULL != - (ret = - GNUNET_FS_uri_parse ( - "gnunet://fs/sks/D1KJS9H2A82Q65VKQ0ML3RFU6U1D3VUK", - &emsg))) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - GNUNET_free (emsg); - if (NULL != - (ret = - GNUNET_FS_uri_parse - ( - "gnunet://fs/sks/XQHH4R288W26EBV369F6RCE0PJVJTX2Y74Q2FJPMPGA31HJX2JG/this", - &emsg))) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - GNUNET_free (emsg); - if (NULL != (ret = GNUNET_FS_uri_parse ("gnunet://fs/sks/test", &emsg))) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - GNUNET_free (emsg); - GNUNET_CRYPTO_ecdsa_key_create (&ph); - GNUNET_CRYPTO_ecdsa_key_get_public (&ph, - &id); - sret = GNUNET_STRINGS_data_to_string (&id, sizeof(id), - ubuf, sizeof(ubuf) - 1); - GNUNET_assert (NULL != sret); - sret[0] = '\0'; - GNUNET_snprintf (buf, sizeof(buf), - "gnunet://fs/sks/%s/test", - ubuf); - ret = GNUNET_FS_uri_parse (buf, &emsg); - if (NULL == ret) - { - GNUNET_free (emsg); - GNUNET_assert (0); - } - if (GNUNET_FS_uri_test_ksk (ret)) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - if (! GNUNET_FS_uri_test_sks (ret)) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - - uri = GNUNET_FS_uri_to_string (ret); - if (0 != - strcmp (uri, - buf)) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_free (uri); - GNUNET_assert (0); - } - GNUNET_free (uri); - GNUNET_FS_uri_destroy (ret); - return 0; -} - - -static int -testFile (int i) -{ - char *uri; - struct GNUNET_FS_Uri *ret; - char *emsg; - - if (NULL != - (ret = - GNUNET_FS_uri_parse - ( - "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H00000440000.42", - &emsg))) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - GNUNET_free (emsg); - if (NULL != - (ret = - GNUNET_FS_uri_parse - ( - "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000", - &emsg))) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - GNUNET_free (emsg); - if (NULL != - (ret = - GNUNET_FS_uri_parse - ( - "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.FGH", - &emsg))) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - GNUNET_free (emsg); - ret = - GNUNET_FS_uri_parse - ( - "gnunet://fs/chk/4QZP479A9SKGFNMQ2ZBCYE71YV2QMTVGWTVPB6A10ASVCKXDHB05DKPSC7ZF6E9P9W1VE47394EQY7NXA47Q6R35M7P1MJPGP59D1Z8.D54QD1K5XCG5878T6YZ19AM60MQ6FC0YNVK7QY08KK0KM0FJJ3KQWYG112FN5T07KN7J0X35DF6WVBT9B8ZMZ3X2BXJ22X3KFQ6MV2G.42", - &emsg); - if (ret == NULL) - { - GNUNET_free (emsg); - GNUNET_assert (0); - } - if (GNUNET_FS_uri_test_ksk (ret)) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - if (GNUNET_FS_uri_test_sks (ret)) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - if (GNUNET_ntohll (ret->data.chk.file_length) != 42) - { - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - - uri = GNUNET_FS_uri_to_string (ret); - if (0 != - strcmp (uri, - "gnunet://fs/chk/4QZP479A9SKGFNMQ2ZBCYE71YV2QMTVGWTVPB6A10ASVCKXDHB05DKPSC7ZF6E9P9W1VE47394EQY7NXA47Q6R35M7P1MJPGP59D1Z8.D54QD1K5XCG5878T6YZ19AM60MQ6FC0YNVK7QY08KK0KM0FJJ3KQWYG112FN5T07KN7J0X35DF6WVBT9B8ZMZ3X2BXJ22X3KFQ6MV2G.42")) - { - GNUNET_free (uri); - GNUNET_FS_uri_destroy (ret); - GNUNET_assert (0); - } - GNUNET_free (uri); - GNUNET_FS_uri_destroy (ret); - return 0; -} - - -int -main (int argc, char *argv[]) -{ - int failureCount = 0; - int i; - - GNUNET_log_setup ("test_fs_uri", - "WARNING", - NULL); - failureCount += testKeyword (); - failureCount += testLocation (); - for (i = 0; i < 255; i++) - { - /* fprintf (stderr, "%s", "."); */ - failureCount += testNamespace (i); - failureCount += testFile (i); - } - /* fprintf (stderr, "%s", "\n"); */ - GNUNET_DISK_purge_cfg_dir - ("test_fs_uri.conf", - "GNUNET_TEST_HOME"); - - if (failureCount != 0) - return 1; - return 0; -} - - -/* end of test_fs_uri.c */ diff --git a/src/fs/test_gnunet_fs_idx.py.in b/src/fs/test_gnunet_fs_idx.py.in deleted file mode 100755 index 564dd68f2..000000000 --- a/src/fs/test_gnunet_fs_idx.py.in +++ /dev/null @@ -1,113 +0,0 @@ -#!@PYTHONEXE@ -# This file is part of GNUnet. -# (C) 2010 Christian Grothoff (and other contributing authors) -# -# GNUnet is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, -# or (at your option) any later version. -# -# GNUnet is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# SPDX-License-Identifier: AGPL3.0-or-later -# -# Testcase for file-sharing command-line tools (indexing and unindexing) -import sys -import os -import subprocess -import re -import shutil - -srcdir = "../.." -gnunet_pyexpect_dir = os.path.join(srcdir, "contrib/scripts") -if gnunet_pyexpect_dir not in sys.path: - sys.path.append(gnunet_pyexpect_dir) - -from gnunet_pyexpect import pexpect - -if os.name == 'posix': - download = './gnunet-download' - gnunetarm = 'gnunet-arm' - publish = './gnunet-publish' - unindex = './gnunet-unindex' -elif os.name == 'nt': - download = './gnunet-download.exe' - gnunetarm = 'gnunet-arm.exe' - publish = './gnunet-publish.exe' - unindex = './gnunet-unindex.exe' - -if os.name == "nt": - shutil.rmtree( - os.path.join(os.getenv("TEMP"), "gnunet-test-fs-py-idx"), True - ) -else: - shutil.rmtree("/tmp/gnunet-test-fs-py-idx", True) - -arm = subprocess.Popen([gnunetarm, '-sq', '-c', 'test_gnunet_fs_idx_data.conf']) -arm.communicate() - -try: - pub = pexpect() - - pub.spawn( - None, [ - publish, '-c', 'test_gnunet_fs_idx_data.conf', '-m', - "description:Test archive", '-k', 'tst', - 'test_gnunet_fs_rec_data.tgz' - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT - ) - pub.expect( - "stdout", - re.compile( - r"URI is `gnunet://fs/chk/2ZMHKPV74CB6GB1GFKQRR95BXJQA2SER25FN48GAW7WSBPA0GDEM5Y74V1ZJHM0NA6919TVW376BHTFDRE3RYS0KRY92M1QJVKPHFCR\.49BT3V5C10KA1695JF71FCT8ZZG4JMJSH04BD9CT22R6KEM915A7CEST17RD0QYTHXV5M4HHEGJMEZSFRDB7JAYC0EMJAN2V781E9DG\.17822'\.\r?\n" - ) - ) - - down = pexpect() - down.spawn( - None, [ - download, '-c', 'test_gnunet_fs_idx_data.conf', '-o', - 'test_gnunet_fs_rec_data.tar.gz', - 'gnunet://fs/chk/2ZMHKPV74CB6GB1GFKQRR95BXJQA2SER25FN48GAW7WSBPA0GDEM5Y74V1ZJHM0NA6919TVW376BHTFDRE3RYS0KRY92M1QJVKPHFCR.49BT3V5C10KA1695JF71FCT8ZZG4JMJSH04BD9CT22R6KEM915A7CEST17RD0QYTHXV5M4HHEGJMEZSFRDB7JAYC0EMJAN2V781E9DG.17822' - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT - ) - down.expect( - "stdout", - re.compile( - r"Downloading `test_gnunet_fs_rec_data.tar.gz' done (.*).\r?\n" - ) - ) - os.remove("test_gnunet_fs_rec_data.tar.gz") - - un = pexpect() - un.spawn( - None, [ - unindex, '-c', 'test_gnunet_fs_idx_data.conf', - 'test_gnunet_fs_rec_data.tgz' - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT - ) - un.expect("stdout", re.compile(r'Unindexing done\.\r?\n')) - -finally: - arm = subprocess.Popen([ - gnunetarm, '-eq', '-c', 'test_gnunet_fs_idx_data.conf' - ]) - arm.communicate() - if os.name == "nt": - shutil.rmtree( - os.path.join(os.getenv("TEMP"), "gnunet-test-fs-py-idx"), True - ) - else: - shutil.rmtree("/tmp/gnunet-test-fs-py-idx", True) diff --git a/src/fs/test_gnunet_fs_idx_data.conf b/src/fs/test_gnunet_fs_idx_data.conf deleted file mode 100644 index ba2a872dd..000000000 --- a/src/fs/test_gnunet_fs_idx_data.conf +++ /dev/null @@ -1,7 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-py-idx/ - -[transport] -PLUGINS = - diff --git a/src/fs/test_gnunet_fs_psd.py.in b/src/fs/test_gnunet_fs_psd.py.in deleted file mode 100755 index a25e4eaef..000000000 --- a/src/fs/test_gnunet_fs_psd.py.in +++ /dev/null @@ -1,149 +0,0 @@ -#!@PYTHONEXE@ -# This file is part of GNUnet. -# (C) 2010, 2018 Christian Grothoff (and other contributing authors) -# -# GNUnet is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, -# or (at your option) any later version. -# -# GNUnet is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# SPDX-License-Identifier: AGPL3.0-or-later -# -# Testcase for file-sharing command-line tools (publish, search, download) - -import sys -import os -import subprocess -import re -import shutil -try: - # Python 2.7 - reload -except NameError: - try: - # Python 3.4+: - from importlib import reload - except ImportError: - # Python 3.0 - 3.3 - from imp import reload - -reload(sys) - -# Force encoding to utf-8, as this test otherwise fails -# on some systems (see #5094). In Python 3+ there is no attribute -# sys.setdefaultencoding anymore. -if (3 < sys.version_info[0]): - sys.setdefaultencoding('utf8') - -srcdir = "../.." -gnunet_pyexpect_dir = os.path.join(srcdir, "contrib/scripts") -if gnunet_pyexpect_dir not in sys.path: - sys.path.append(gnunet_pyexpect_dir) - -from gnunet_pyexpect import pexpect - -if os.name == 'posix': - download = './gnunet-download' - gnunetarm = 'gnunet-arm' - publish = './gnunet-publish' - unindex = './gnunet-unindex' - search = './gnunet-search' -elif os.name == 'nt': - download = './gnunet-download.exe' - gnunetarm = 'gnunet-arm.exe' - publish = './gnunet-publish.exe' - unindex = './gnunet-unindex.exe' - search = './gnunet-search.exe' - -if "GNUNET_PREFIX" in os.environ: - pass -else: - print("You need to export GNUNET_PREFIX") - sys.exit(1) - -if os.name == "nt": - shutil.rmtree( - os.path.join(os.getenv("TEMP"), "gnunet-test-fs-py-psd"), True - ) -else: - shutil.rmtree("/tmp/gnunet-test-fs-py-psd", True) - -arm = subprocess.Popen([gnunetarm, '-sq', '-c', 'test_gnunet_fs_psd_data.conf']) -arm.communicate() - -# first, basic publish-search-download run -try: - pub = pexpect() - pub.spawn( - None, [ - publish, '-c', 'test_gnunet_fs_psd_data.conf', '-n', '-m', - "description:Test archive", '-k', 'tst', - 'test_gnunet_fs_rec_data.tgz' - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT - ) - pub.expect( - "stdout", - re.compile(r"Publishing `.+test_gnunet_fs_rec_data.tgz' done\.\r?\n") - ) - pub.expect( - "stdout", - re.compile( - r"URI is `gnunet://fs/chk/2ZMHKPV74CB6GB1GFKQRR95BXJQA2SER25FN48GAW7WSBPA0GDEM5Y74V1ZJHM0NA6919TVW376BHTFDRE3RYS0KRY92M1QJVKPHFCR\.49BT3V5C10KA1695JF71FCT8ZZG4JMJSH04BD9CT22R6KEM915A7CEST17RD0QYTHXV5M4HHEGJMEZSFRDB7JAYC0EMJAN2V781E9DG\.17822'\.\r?\n" - ) - ) - - s = pexpect() - s.spawn( - None, [ - search, '-V', '-t', '1000 ms', '-N', '1', '-c', - 'test_gnunet_fs_psd_data.conf', 'tst' - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT - ) - s.expect( - "stdout", - re.compile( - r'gnunet-download -o "test_gnunet_fs_rec_data.tgz" gnunet://fs/chk/2ZMHKPV74CB6GB1GFKQRR95BXJQA2SER25FN48GAW7WSBPA0GDEM5Y74V1ZJHM0NA6919TVW376BHTFDRE3RYS0KRY92M1QJVKPHFCR\.49BT3V5C10KA1695JF71FCT8ZZG4JMJSH04BD9CT22R6KEM915A7CEST17RD0QYTHXV5M4HHEGJMEZSFRDB7JAYC0EMJAN2V781E9DG\.17822\r?\n' - ) - ) - - down = pexpect() - down.spawn( - None, [ - download, '-c', 'test_gnunet_fs_psd_data.conf', '-o', - 'test_gnunet_fs_rec_data.tar.gz', - 'gnunet://fs/chk/2ZMHKPV74CB6GB1GFKQRR95BXJQA2SER25FN48GAW7WSBPA0GDEM5Y74V1ZJHM0NA6919TVW376BHTFDRE3RYS0KRY92M1QJVKPHFCR.49BT3V5C10KA1695JF71FCT8ZZG4JMJSH04BD9CT22R6KEM915A7CEST17RD0QYTHXV5M4HHEGJMEZSFRDB7JAYC0EMJAN2V781E9DG.17822' - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT - ) - down.expect( - "stdout", - re.compile( - r"Downloading `test_gnunet_fs_rec_data.tar.gz' done (.*).\r?\n" - ) - ) - os.remove("test_gnunet_fs_rec_data.tar.gz") - -finally: - arm = subprocess.Popen([ - gnunetarm, '-eq', '-c', 'test_gnunet_fs_psd_data.conf' - ]) - arm.communicate() - if os.name == "nt": - shutil.rmtree( - os.path.join(os.getenv("TEMP"), "gnunet-test-fs-py-psd"), True - ) - else: - shutil.rmtree("/tmp/gnunet-test-fs-py-psd", True) diff --git a/src/fs/test_gnunet_fs_psd_data.conf b/src/fs/test_gnunet_fs_psd_data.conf deleted file mode 100644 index f6c05ecdd..000000000 --- a/src/fs/test_gnunet_fs_psd_data.conf +++ /dev/null @@ -1,7 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-py-psd/ - -[transport] -PLUGINS = - diff --git a/src/fs/test_gnunet_fs_rec.py.in b/src/fs/test_gnunet_fs_rec.py.in deleted file mode 100755 index f7e84e3da..000000000 --- a/src/fs/test_gnunet_fs_rec.py.in +++ /dev/null @@ -1,171 +0,0 @@ -#!@PYTHONEXE@ -# This file is part of GNUnet. -# (C) 2010 Christian Grothoff (and other contributing authors) -# -# GNUnet is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, -# or (at your option) any later version. -# -# GNUnet is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# SPDX-License-Identifier: AGPL3.0-or-later -# -# Testcase for file-sharing command-line tools (recursive publishing & download) -import sys -import os -import subprocess -import re -import shutil -import tarfile -import filecmp - -srcdir = "../.." -gnunet_pyexpect_dir = os.path.join(srcdir, "contrib/scripts") -if gnunet_pyexpect_dir not in sys.path: - sys.path.append(gnunet_pyexpect_dir) - -from gnunet_pyexpect import pexpect -from pydiffer import dcdiff - -if os.name == 'posix': - download = './gnunet-download' - gnunetarm = 'gnunet-arm' - publish = './gnunet-publish' - unindex = './gnunet-unindex' - search = './gnunet-search' - directory = './gnunet-directory' -elif os.name == 'nt': - download = './gnunet-download.exe' - gnunetarm = 'gnunet-arm.exe' - publish = './gnunet-publish.exe' - unindex = './gnunet-unindex.exe' - search = './gnunet-search.exe' - directory = './gnunet-directory.exe' - -if os.name == "nt": - shutil.rmtree( - os.path.join(os.getenv("TEMP"), "gnunet-test-fs-py-rec"), True - ) -else: - shutil.rmtree("/tmp/gnunet-test-fs-py-rec", True) - -arm = subprocess.Popen([gnunetarm, '-sq', '-c', 'test_gnunet_fs_rec_data.conf']) -arm.communicate() - -tar = tarfile.open('test_gnunet_fs_rec_data.tgz') -tar.extractall() -# first, basic publish-search-download run -try: - pub = pexpect() - pub.spawn( - None, [ - publish, '-c', 'test_gnunet_fs_rec_data.conf', '-k', 'testdir', - 'dir/' - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT - ) - # Can't say much for publishing, except that the last one is the toplevel directory - pub.expect("stdout", re.compile(r"Publishing `.+' done\.\r?\n")) - pub.expect( - "stdout", - re.compile( - r"URI is `gnunet://fs/chk/[A-Z0-9]{103}\.[A-Z0-9]{103}\.\d+'\.\r?\n" - ) - ) - pub.expect("stdout", re.compile(r"Publishing `.+' done\.\r?\n")) - pub.expect( - "stdout", - re.compile( - r"URI is `gnunet://fs/chk/[A-Z0-9]{103}\.[A-Z0-9]{103}\.\d+'\.\r?\n" - ) - ) - pub.expect("stdout", re.compile(r"Publishing `.+' done\.\r?\n")) - pub.expect( - "stdout", - re.compile( - r"URI is `gnunet://fs/chk/[A-Z0-9]{103}\.[A-Z0-9]{103}\.\d+'\.\r?\n" - ) - ) - pub.expect("stdout", re.compile(r"Publishing `.+' done\.\r?\n")) - pub.expect( - "stdout", - re.compile( - r"URI is `gnunet://fs/chk/[A-Z0-9]{103}\.[A-Z0-9]{103}\.\d+'\.\r?\n" - ) - ) - pub.expect("stdout", re.compile(r"Publishing `.+' done\.\r?\n")) - pub.expect( - "stdout", - re.compile( - r"URI is `gnunet://fs/chk/[A-Z0-9]{103}\.[A-Z0-9]{103}\.\d+'\.\r?\n" - ) - ) - pub.expect("stdout", re.compile(r"Publishing `.+' done\.\r?\n")) - pub.expect( - "stdout", - re.compile( - r"URI is `gnunet://fs/chk/[A-Z0-9]{103}\.[A-Z0-9]{103}\.\d+'\.\r?\n" - ) - ) - pub.expect( - "stdout", re.compile(r"Publishing `.+[\\/]dir[\\/]' done\.\r?\n") - ) - m = pub.expect("stdout", re.compile(r".+\r?\n")) - if not m: - sys.exit(3) - output = m.string - url = output[output.find("`") + 1:output.find("'")] - - down = pexpect() - down.spawn( - None, [ - download, '-c', 'test_gnunet_fs_rec_data.conf', '-R', '-o', - 'rdir.gnd', url - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT - ) - down.expect("stdout", re.compile(r"Downloading `rdir.gnd' done (.*).\r?\n")) - - d = pexpect() - d.spawn( - None, [directory, '-c', 'test_gnunet_fs_rec_data.conf', 'rdir/a.gnd'], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT - ) - d.expect("stdout", re.compile(r"Directory `a/' meta data:\r?\n")) - d.expect("stdout", re.compile(r"Directory `a/' contents:\r?\n")) - d.expect("stdout", re.compile(r"COPYING (.*)\r?\n")) - d.expect("stdout", re.compile(r"INSTALL (.*)\r?\n")) - - os.remove("rdir/b.gnd") - os.remove("rdir/a.gnd") - diff = dcdiff('dir', 'rdir') - if len(diff) != 0: - raise Exception( - "Unexpected difference between source directory and downloaded result:\n{}" - .format(diff) - ) - -finally: - arm = subprocess.Popen([ - gnunetarm, '-eq', '-c', 'test_gnunet_fs_rec_data.conf' - ]) - arm.communicate() - if os.name == "nt": - shutil.rmtree( - os.path.join(os.getenv("TEMP"), "gnunet-test-fs-py-rec"), True - ) - else: - shutil.rmtree("/tmp/gnunet-test-fs-py-rec", True) - shutil.rmtree("dir", True) - shutil.rmtree("rdir", True) - shutil.rmtree("rdir.gnd", True) diff --git a/src/fs/test_gnunet_fs_rec_data.conf b/src/fs/test_gnunet_fs_rec_data.conf deleted file mode 100644 index 82ddb8f49..000000000 --- a/src/fs/test_gnunet_fs_rec_data.conf +++ /dev/null @@ -1,7 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-py-rec/ - -[transport] -PLUGINS = - diff --git a/src/fs/test_gnunet_fs_rec_data.tgz b/src/fs/test_gnunet_fs_rec_data.tgz deleted file mode 100644 index 697794306..000000000 Binary files a/src/fs/test_gnunet_fs_rec_data.tgz and /dev/null differ diff --git a/src/fs/test_gnunet_service_fs_migration.c b/src/fs/test_gnunet_service_fs_migration.c deleted file mode 100644 index 38b00f3e8..000000000 --- a/src/fs/test_gnunet_service_fs_migration.c +++ /dev/null @@ -1,223 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2010, 2012, 2015 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_gnunet_service_fs_migration.c - * @brief test content migration between two peers - * @author Christian Grothoff - */ -#include "platform.h" -#include "fs_test_lib.h" -#include "gnunet_testbed_service.h" - -#define VERBOSE GNUNET_NO - -/** - * File-size we use for testing. - */ -#define FILESIZE (2 * 32 * 1024) - -/** - * How long until we give up on transmitting the message? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 120) - -/** - * How long do we give the peers for content migration? - */ -#define MIGRATION_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, \ - 90) - -#define SEED 42 - -static struct GNUNET_TESTBED_Peer *daemons[2]; - -static int ok; - -static struct GNUNET_TIME_Absolute start_time; - -static struct GNUNET_TESTBED_Operation *op; - - -struct DownloadContext -{ - char *fn; - - struct GNUNET_FS_Uri *uri; -}; - - -static void -do_stop (void *cls) -{ - struct GNUNET_TIME_Relative del; - char *fancy; - - GNUNET_SCHEDULER_shutdown (); - if (0 == - GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_add (start_time, - TIMEOUT)). - rel_value_us) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Timeout during download, shutting down with error\n"); - ok = 1; - } - else - { - del = GNUNET_TIME_absolute_get_duration (start_time); - if (del.rel_value_us == 0) - del.rel_value_us = 1; - fancy = - GNUNET_STRINGS_byte_size_fancy (((unsigned long long) FILESIZE) - * 1000000LL / del.rel_value_us); - fprintf (stdout, - "Download speed was %s/s\n", - fancy); - GNUNET_free (fancy); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Finished download, shutting down\n"); - } -} - - -static void -do_download (void *cls, - const char *emsg) -{ - struct DownloadContext *dc = cls; - struct GNUNET_FS_Uri *uri = dc->uri; - - GNUNET_TESTBED_operation_done (op); - op = NULL; - if (NULL != dc->fn) - { - GNUNET_DISK_directory_remove (dc->fn); - GNUNET_free (dc->fn); - } - GNUNET_free (dc); - if (NULL != emsg) - { - GNUNET_SCHEDULER_shutdown (); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Failed to stop source daemon: %s\n", - emsg); - GNUNET_FS_uri_destroy (uri); - ok = 1; - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Downloading %llu bytes\n", - (unsigned long long) FILESIZE); - start_time = GNUNET_TIME_absolute_get (); - GNUNET_FS_TEST_download (daemons[0], - TIMEOUT, - 1, - SEED, - uri, - VERBOSE, - &do_stop, - NULL); - GNUNET_FS_uri_destroy (uri); -} - - -static void -stop_source_peer (void *cls) -{ - struct DownloadContext *dc = cls; - - /* FIXME: We should not interact with testbed when shutting down */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Stopping source peer\n"); - op = GNUNET_TESTBED_peer_stop (NULL, - daemons[1], - &do_download, dc); - GNUNET_assert (NULL != op); -} - - -static void -do_wait (void *cls, - const struct GNUNET_FS_Uri *uri, - const char *fn) -{ - struct DownloadContext *dc; - - if (NULL == uri) - { - GNUNET_SCHEDULER_shutdown (); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Timeout during upload attempt, shutting down with error\n"); - ok = 1; - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Waiting to allow content to migrate\n"); - dc = GNUNET_new (struct DownloadContext); - dc->uri = GNUNET_FS_uri_dup (uri); - if (NULL != fn) - dc->fn = GNUNET_strdup (fn); - (void) GNUNET_SCHEDULER_add_delayed (MIGRATION_DELAY, - &stop_source_peer, - dc); -} - - -static void -do_publish (void *cls, - struct GNUNET_TESTBED_RunHandle *h, - unsigned int num_peers, - struct GNUNET_TESTBED_Peer **peers, - unsigned int links_succeeded, - unsigned int links_failed) -{ - unsigned int i; - - GNUNET_assert (2 == num_peers); - for (i = 0; i < num_peers; i++) - daemons[i] = peers[i]; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publishing %llu bytes\n", - (unsigned long long) FILESIZE); - GNUNET_FS_TEST_publish (daemons[1], TIMEOUT, 1, GNUNET_NO, - FILESIZE, SEED, - VERBOSE, &do_wait, NULL); -} - - -int -main (int argc, - char *argv[]) -{ - (void) GNUNET_TESTBED_test_run ("test-gnunet-service-fs-migration", - "fs_test_lib_data.conf", - 2, - 0, NULL, NULL, - &do_publish, - NULL); - GNUNET_DISK_purge_cfg_dir - ("test_gnunet_service_fs_migration_data.conf", - "GNUNET_TEST_HOME"); - return ok; -} - - -/* end of test_gnunet_service_fs_migration.c */ diff --git a/src/fs/test_gnunet_service_fs_migration_data.conf b/src/fs/test_gnunet_service_fs_migration_data.conf deleted file mode 100644 index fca6c5a29..000000000 --- a/src/fs/test_gnunet_service_fs_migration_data.conf +++ /dev/null @@ -1,10 +0,0 @@ -@INLINE@ test_fs_defaults.conf -[PATHS] -GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-service-fs-migration/ - -[testbed] -OVERLAY_TOPOLOGY = CLIQUE - -[ats] -WAN_QUOTA_IN = 3932160 -WAN_QUOTA_OUT = 3932160 diff --git a/src/fs/test_gnunet_service_fs_p2p.c b/src/fs/test_gnunet_service_fs_p2p.c deleted file mode 100644 index 2d1fbb788..000000000 --- a/src/fs/test_gnunet_service_fs_p2p.c +++ /dev/null @@ -1,167 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2010, 2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ - -/** - * @file fs/test_gnunet_service_fs_p2p.c - * @brief test P2P routing using simple publish + download operation - * @author Christian Grothoff - */ -#include "platform.h" -#include "fs_test_lib.h" - -#define VERBOSE GNUNET_NO - -/** - * File-size we use for testing. - */ -#define FILESIZE (1024 * 1024 * 1) - -/** - * How long until we give up on the download? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300) - -#define NUM_DAEMONS 2 - -#define SEED 42 - -static const char *progname; - -static unsigned int anonymity_level; - -static struct GNUNET_TESTBED_Peer *daemons[NUM_DAEMONS]; - -static int ok; - -static struct GNUNET_TIME_Absolute start_time; - - -static void -do_stop (void *cls) -{ - char *fn = cls; - struct GNUNET_TIME_Relative del; - char *fancy; - - GNUNET_SCHEDULER_shutdown (); - if (0 == - GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_add (start_time, - TIMEOUT)). - rel_value_us) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Timeout during download, shutting down with error\n"); - ok = 1; - } - else - { - del = GNUNET_TIME_absolute_get_duration (start_time); - if (0 == del.rel_value_us) - del.rel_value_us = 1; - fancy = - GNUNET_STRINGS_byte_size_fancy (((unsigned long long) FILESIZE) - * 1000000LL / del.rel_value_us); - fprintf (stdout, - "Download speed was %s/s\n", - fancy); - GNUNET_free (fancy); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Finished download, shutting down\n"); - } - if (NULL != fn) - { - GNUNET_DISK_directory_remove (fn); - GNUNET_free (fn); - } -} - - -static void -do_download (void *cls, const struct GNUNET_FS_Uri *uri, - const char *fn) -{ - if (NULL == uri) - { - GNUNET_SCHEDULER_shutdown (); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Timeout during upload attempt, shutting down with error\n"); - ok = 1; - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Downloading %llu bytes\n", - (unsigned long long) FILESIZE); - start_time = GNUNET_TIME_absolute_get (); - GNUNET_FS_TEST_download (daemons[0], TIMEOUT, - anonymity_level, SEED, uri, - VERBOSE, &do_stop, - (NULL == fn) - ? NULL - : GNUNET_strdup (fn)); -} - - -static void -do_publish (void *cls, - struct GNUNET_TESTBED_RunHandle *h, - unsigned int num_peers, - struct GNUNET_TESTBED_Peer **peers, - unsigned int links_succeeded, - unsigned int links_failed) -{ - unsigned int i; - - if (NULL != strstr (progname, "cadet")) - anonymity_level = 0; - else - anonymity_level = 1; - GNUNET_assert (NUM_DAEMONS == num_peers); - for (i = 0; i < num_peers; i++) - daemons[i] = peers[i]; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n", - (unsigned long long) FILESIZE); - GNUNET_FS_TEST_publish (daemons[1], TIMEOUT, - anonymity_level, GNUNET_NO, - FILESIZE, SEED, - VERBOSE, &do_download, NULL); -} - - -int -main (int argc, char *argv[]) -{ - const char *config; - - progname = argv[0]; - if (NULL != strstr (progname, "cadet")) - config = "test_gnunet_service_fs_p2p_cadet.conf"; - else - config = "fs_test_lib_data.conf"; - (void) GNUNET_TESTBED_test_run ("test-gnunet-service-fs-p2p", - config, - NUM_DAEMONS, - 0, NULL, NULL, - &do_publish, NULL); - GNUNET_DISK_purge_cfg_dir (config, - "GNUNET_TEST_HOME"); - return ok; -} - - -/* end of test_gnunet_service_fs_p2p.c */ diff --git a/src/fs/test_gnunet_service_fs_p2p_cadet.conf b/src/fs/test_gnunet_service_fs_p2p_cadet.conf deleted file mode 100644 index 12e106968..000000000 --- a/src/fs/test_gnunet_service_fs_p2p_cadet.conf +++ /dev/null @@ -1,20 +0,0 @@ -@INLINE@ fs_test_lib_data.conf - -[fs] -# FIXME: this option needs to be set for the -# testcase to truly work; however, as the code -# is not finished, not setting the option should -# allow the test to at least pass for now... -DISABLE_ANON_TRANSFER = YES - -# Do we cache content from other nodes? (may improve anonymity) -CONTENT_CACHING = NO - -# Do we send unsolicited data to other nodes if we have excess bandwidth? -# (may improve anonymity, probably not a good idea if content_caching is NO) -CONTENT_PUSHING = NO - -#PREFIX = valgrind - -[cadet] -#PREFIX = valgrind diff --git a/src/fs/test_plugin_block_fs.c b/src/fs/test_plugin_block_fs.c deleted file mode 100644 index f15d10b17..000000000 --- a/src/fs/test_plugin_block_fs.c +++ /dev/null @@ -1,86 +0,0 @@ -/* - This file is part of GNUnet - Copyright (C) 2010, 2012 GNUnet e.V. - - GNUnet is free software: you can redistribute it and/or modify it - under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, - or (at your option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - SPDX-License-Identifier: AGPL3.0-or-later - */ -/** - * @file fs/test_plugin_block_fs.c - * @brief test for plugin_block_fs.c - * @author Christian Grothoff - */ -#include "platform.h" -#include "gnunet_block_lib.h" - - -static int -test_fs (struct GNUNET_BLOCK_Context *ctx) -{ - struct GNUNET_HashCode key; - char block[4]; - - memset (block, 1, sizeof(block)); - if (GNUNET_OK != - GNUNET_BLOCK_get_key (ctx, - GNUNET_BLOCK_TYPE_FS_DBLOCK, - block, - sizeof(block), - &key)) - return 1; - if (GNUNET_OK != - GNUNET_BLOCK_check_block (ctx, - GNUNET_BLOCK_TYPE_FS_DBLOCK, - block, - sizeof(block))) - return 2; - if (GNUNET_OK != - GNUNET_BLOCK_check_query (ctx, - GNUNET_BLOCK_TYPE_FS_DBLOCK, - &key, - NULL, 0)) - return 4; - GNUNET_log_skip (1, GNUNET_NO); - if (GNUNET_NO != - GNUNET_BLOCK_check_query (ctx, - GNUNET_BLOCK_TYPE_FS_DBLOCK, - &key, - "bogus", 5)) - return 8; - GNUNET_log_skip (0, GNUNET_YES); - return 0; -} - - -int -main (int argc, char *argv[]) -{ - int ret; - struct GNUNET_BLOCK_Context *ctx; - struct GNUNET_CONFIGURATION_Handle *cfg; - - GNUNET_log_setup ("test-block", "WARNING", NULL); - cfg = GNUNET_CONFIGURATION_create (); - ctx = GNUNET_BLOCK_context_create (cfg); - ret = test_fs (ctx); - GNUNET_BLOCK_context_destroy (ctx); - GNUNET_CONFIGURATION_destroy (cfg); - if (ret != 0) - fprintf (stderr, "Tests failed: %d\n", ret); - return ret; -} - - -/* end of test_plugin_block_fs.c */ diff --git a/src/fs/test_pseudonym_data.conf b/src/fs/test_pseudonym_data.conf deleted file mode 100644 index 5827721b8..000000000 --- a/src/fs/test_pseudonym_data.conf +++ /dev/null @@ -1,6 +0,0 @@ -# General settings -[fs] - -[TESTING] -WEAKRANDOM = YES - -- cgit v1.2.3