From ecdc35a09b840720e1b87f2d51aba537976ccf3f Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 30 May 2013 19:10:02 +0000 Subject: adding tool to convert/merge datastores (#2875) --- doc/man/Makefile.am | 1 + doc/man/gnunet-datastore.1 | 41 ++++++ src/datastore/Makefile.am | 9 ++ src/datastore/gnunet-datastore.c | 262 +++++++++++++++++++++++++++++++++ src/include/gnunet_datastore_service.h | 4 +- src/util/gnunet-resolver.c | 3 +- 6 files changed, 317 insertions(+), 3 deletions(-) create mode 100644 doc/man/gnunet-datastore.1 create mode 100644 src/datastore/gnunet-datastore.c diff --git a/doc/man/Makefile.am b/doc/man/Makefile.am index dc68375f5..4f4d736d6 100644 --- a/doc/man/Makefile.am +++ b/doc/man/Makefile.am @@ -7,6 +7,7 @@ man_MANS = \ gnunet-core.1 \ gnunet-directory.1 \ gnunet-dns2gns.1 \ + gnunet-datastore.1 \ gnunet-download.1 \ gnunet-download-manager.1 \ gnunet-ecc.1 \ diff --git a/doc/man/gnunet-datastore.1 b/doc/man/gnunet-datastore.1 new file mode 100644 index 000000000..661b92828 --- /dev/null +++ b/doc/man/gnunet-datastore.1 @@ -0,0 +1,41 @@ +.TH gnunet\-datastore "1" "30 May 2013" "GNUnet" +.SH NAME +gnunet\-datastore \- merge or convert GNUnet datastore databases + +.SH SYNOPSIS +.B gnunet\-datastore +[\fIOPTIONS\fR] +.SH DESCRIPTION +.PP + +gnunet\-datastore can be used to convert or merge GNUnet datastores. This is useful if a datastore is to be migrated between SQL databases, i.e. from sqlite to postgres or vice versa. gnunet\-datastore basically takes two configuration files (which must specify different databases) and reads in all of the data from the datasource (\-s option) and copies it to the destination (\-c option). Note that replication level information is lost in the process at this time. + +.TP +\fB\-c \fIFILENAME\fR, \fB\-\-config=FILENAME\fR +configuration file to use for the destination database +.TP +\fB\-h\fR, \fB\-\-help\fR +print help page +.TP +\fB\-i\fR, \fB\-\-list-indexed\fR +print information about files that are currently indexed by file-sharing +.TP +\fB\-L \fILOGLEVEL\fR, \fB\-\-loglevel=LOGLEVEL\fR +Change the loglevel. Possible values for LOGLEVEL are ERROR, WARNING, INFO and DEBUG. +.TP +\fB\-s \fIFILENAME\fR, \fB\-\-sourcecfg=FILENAME\fR +configuration file to use for the source database +.TP +\fB\-v\fR, \fB\-\-version\fR +print the version number +.TP +\fB\-V\fR, \fB\-\-verbose\fR +be verbose + +.SH NOTES + + +.SH "REPORTING BUGS" +Report bugs by using mantis or by sending electronic mail to +.SH "SEE ALSO" +\fBgnunet\-publish\fP(1) diff --git a/src/datastore/Makefile.am b/src/datastore/Makefile.am index 57639fa37..c2acc2ffe 100644 --- a/src/datastore/Makefile.am +++ b/src/datastore/Makefile.am @@ -32,6 +32,8 @@ libgnunetdatastore_la_LDFLAGS = \ $(GN_LIB_LDFLAGS) $(WINFLAGS) \ -version-info 1:0:0 +bin_PROGRAMS = \ + gnunet-datastore libexec_PROGRAMS = \ gnunet-service-datastore @@ -43,6 +45,13 @@ gnunet_service_datastore_LDADD = \ $(top_builddir)/src/util/libgnunetutil.la \ $(GN_LIBINTL) +gnunet_datastore_SOURCES = \ + gnunet-datastore.c +gnunet_datastore_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) + if HAVE_MYSQL MYSQL_PLUGIN = libgnunet_plugin_datastore_mysql.la if HAVE_BENCHMARKS diff --git a/src/datastore/gnunet-datastore.c b/src/datastore/gnunet-datastore.c new file mode 100644 index 000000000..4388818d7 --- /dev/null +++ b/src/datastore/gnunet-datastore.c @@ -0,0 +1,262 @@ +/* + This file is part of GNUnet. + (C) 2010, 2013 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file datastore/gnunet-datastore.c + * @brief tool to manipulate datastores + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_datastore_service.h" + + +/** + * Name of the second configuration file. + */ +static char *alternative_cfg; + +/** + * Global return value. + */ +static int ret; + +/** + * Our offset on 'get'. + */ +static uint64_t offset; + +/** + * First UID ever returned. + */ +static uint64_t first_uid; + +/** + * Configuration for the source database. + */ +static struct GNUNET_CONFIGURATION_Handle *scfg; + +/** + * Handle for database source. + */ +static struct GNUNET_DATASTORE_Handle *db_src; + +/** + * Handle for database destination. + */ +static struct GNUNET_DATASTORE_Handle *db_dst; + +/** + * Current operation. + */ +static struct GNUNET_DATASTORE_QueueEntry *qe; + + +static void +do_shutdown (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (NULL != qe) + GNUNET_DATASTORE_cancel (qe); + GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO); + GNUNET_DATASTORE_disconnect (db_dst, GNUNET_NO); + GNUNET_CONFIGURATION_destroy (scfg); +} + + +/** + * Perform next GET operation. + */ +static void +do_get (void); + + +/** + * Continuation called to notify client about result of the + * operation. + * + * @param cls closure + * @param success GNUNET_SYSERR on failure (including timeout/queue drop) + * GNUNET_NO if content was already there + * GNUNET_YES (or other positive value) on success + * @param min_expiration minimum expiration time required for 0-priority content to be stored + * by the datacache at this time, zero for unknown, forever if we have no + * space for 0-priority content + * @param msg NULL on success, otherwise an error message + */ +static void +do_finish (void *cls, + int32_t success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + qe = NULL; + if (GNUNET_SYSERR == success) + { + fprintf (stderr, + _("Failed to store item: %s, aborting\n"), + msg); + ret = 1; + GNUNET_SCHEDULER_shutdown (); + return; + } + do_get (); +} + + +/** + * Process a datum that was stored in the datastore. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + */ +static void +do_put (void *cls, + const struct GNUNET_HashCode *key, + size_t size, const void *data, + enum GNUNET_BLOCK_Type type, + uint32_t priority, + uint32_t anonymity, + struct GNUNET_TIME_Absolute + expiration, uint64_t uid) +{ + qe = NULL; + if ( (0 != offset) && + (uid == first_uid) ) + { + GNUNET_SCHEDULER_shutdown (); + return; + } + if (0 == offset) + first_uid = uid; + qe = GNUNET_DATASTORE_put (db_dst, 0, + key, size, data, type, + priority, anonymity, + 0 /* FIXME: replication is lost... */, + expiration, + 0, 1, GNUNET_TIME_UNIT_FOREVER_REL, + &do_finish, NULL); +} + + +/** + * Perform next GET operation. + */ +static void +do_get () +{ + qe = GNUNET_DATASTORE_get_key (db_src, + offset, + NULL, GNUNET_BLOCK_TYPE_ANY, + 0, 1, + GNUNET_TIME_UNIT_FOREVER_REL, + &do_put, NULL); +} + + + +/** + * Main function that will be run by the scheduler. + * + * @param cls closure + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used + * @param cfg configuration -- for destination datastore + */ +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + if (NULL == alternative_cfg) + return; /* nothing to be done */ + if (0 == strcmp (cfgfile, alternative_cfg)) + { + fprintf (stderr, + _("Cannot use the same configuration for source and destination\n")); + ret = 1; + return; + } + scfg = GNUNET_CONFIGURATION_create (); + if (GNUNET_OK != + GNUNET_CONFIGURATION_load (scfg, + alternative_cfg)) + { + GNUNET_CONFIGURATION_destroy (scfg); + ret = 1; + return; + } + db_src = GNUNET_DATASTORE_connect (scfg); + if (NULL == db_src) + { + GNUNET_CONFIGURATION_destroy (scfg); + ret = 1; + return; + } + db_dst = GNUNET_DATASTORE_connect (cfg); + if (NULL == db_dst) + { + GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO); + GNUNET_CONFIGURATION_destroy (scfg); + ret = 1; + return; + } + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, + &do_shutdown, NULL); + do_get (); +} + + +/** + * The main function to manipulate datastores. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, char *const *argv) +{ + static const struct GNUNET_GETOPT_CommandLineOption options[] = { + { 's', "sourcecfg", "FILENAME", + gettext_noop ("specifies the configuration to use to access an alternative datastore; will merge that datastore into our current datastore"), + 1, &GNUNET_GETOPT_set_filename, &alternative_cfg }, + GNUNET_GETOPT_OPTION_END + }; + if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) + return 2; + + if (GNUNET_OK != + GNUNET_PROGRAM_run (argc, argv, "gnunet-datastore", + gettext_noop ("Manipulate GNUnet datastore"), + options, &run, NULL)) + ret = 1; + GNUNET_free ((void*) argv); + return ret; +} + +/* end of gnunet-datastore.c */ diff --git a/src/include/gnunet_datastore_service.h b/src/include/gnunet_datastore_service.h index 721963bdd..9ebf5e48a 100644 --- a/src/include/gnunet_datastore_service.h +++ b/src/include/gnunet_datastore_service.h @@ -153,7 +153,7 @@ GNUNET_DATASTORE_reserve (struct GNUNET_DATASTORE_Handle *h, uint64_t amount, */ struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h, uint32_t rid, - const struct GNUNET_HashCode * key, size_t size, + const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, @@ -300,7 +300,7 @@ typedef void (*GNUNET_DATASTORE_DatumProcessor) (void *cls, */ struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h, uint64_t offset, - const struct GNUNET_HashCode * key, + const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, unsigned int queue_priority, unsigned int max_queue_size, diff --git a/src/util/gnunet-resolver.c b/src/util/gnunet-resolver.c index 6cb6ac564..72ece9710 100644 --- a/src/util/gnunet-resolver.c +++ b/src/util/gnunet-resolver.c @@ -133,8 +133,9 @@ run (void *cls, char *const *args, const char *cfgfile, NULL); } + /** - * The main function to obtain statistics in GNUnet. + * The main function to access GNUnet's DNS resolver. * * @param argc number of arguments from the command line * @param argv command line arguments -- cgit v1.2.3