summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--doc/man/gnunet-datastore.124
-rw-r--r--src/datastore/gnunet-datastore.c452
2 files changed, 364 insertions, 112 deletions
diff --git a/doc/man/gnunet-datastore.1 b/doc/man/gnunet-datastore.1
index 14372a49b..1605006b3 100644
--- a/doc/man/gnunet-datastore.1
+++ b/doc/man/gnunet-datastore.1
@@ -1,6 +1,6 @@
-.TH gnunet\-datastore "1" "30 May 2013" "GNUnet"
+.TH gnunet\-datastore "1" "13 April 2017" "GNUnet"
.SH NAME
-gnunet\-datastore \- merge or convert GNUnet datastore databases
+gnunet\-datastore \- dump or insert (restore) GNUnet datastore databases
.SH SYNOPSIS
.B gnunet\-datastore
@@ -8,26 +8,32 @@ gnunet\-datastore \- merge or convert GNUnet datastore databases
.SH DESCRIPTION
.PP
-gnunet\-datastore can be used to convert or merge GNUnet datastores. This is useful if a datastore is to be migrated between SQL databases, i.e. from sqlite to postgres or vice versa. gnunet\-datastore basically takes two configuration files (which must specify different databases) and reads in all of the data from the datasource (\-s option) and copies it to the destination (\-c option). Note that replication level information is lost in the process at this time.
+gnunet\-datastore can be used to backup and restore or merge GNUnet datastores. This is useful if a datastore is to be migrated between SQL databases, i.e. from sqlite to postgres or vice versa. gnunet\-datastore will dump the entire contents of the database or insert a dump file into the database.
.TP
\fB\-c \fIFILENAME\fR, \fB\-\-config=FILENAME\fR
-configuration file to use for the destination database
+configuration file to use
+.TP
+\fB\-d\fR, \fB\-\-dump\fR
+dump all records to a file
+.TP
+\fB\-f \fIFILENAME\fR, \fB\-\-file=FILENAME\fR
+file to dump to or insert from. Otherwise stdin/stdout are used.
.TP
\fB\-h\fR, \fB\-\-help\fR
print help page
.TP
+\fB\-i\fR, \fB\-\-insert\fR
+insert from dump file
+.TP
\fB\-L \fILOGLEVEL\fR, \fB\-\-loglevel=LOGLEVEL\fR
Change the loglevel. Possible values for LOGLEVEL are ERROR, WARNING, INFO and DEBUG.
.TP
-\fB\-s \fIFILENAME\fR, \fB\-\-sourcecfg=FILENAME\fR
-configuration file to use for the source database
+\fB\-l \fIFILENAME\fR, \fB\-\-logfile=FILENAME\fR
+configure logging to write logs to FILENAME
.TP
\fB\-v\fR, \fB\-\-version\fR
print the version number
-.TP
-\fB\-V\fR, \fB\-\-verbose\fR
-be verbose
.SH NOTES
diff --git a/src/datastore/gnunet-datastore.c b/src/datastore/gnunet-datastore.c
index 9e0ee205e..891343e17 100644
--- a/src/datastore/gnunet-datastore.c
+++ b/src/datastore/gnunet-datastore.c
@@ -23,101 +23,136 @@
* @brief tool to manipulate datastores
* @author Christian Grothoff
*/
+#include <inttypes.h>
#include "platform.h"
#include "gnunet_util_lib.h"
#include "gnunet_datastore_service.h"
+GNUNET_NETWORK_STRUCT_BEGIN
+
+struct DataRecord
+{
+ /**
+ * Number of bytes in the item (NBO).
+ */
+ uint32_t size GNUNET_PACKED;
+
+ /**
+ * Type of the item (NBO) (actually an enum GNUNET_BLOCK_Type)
+ */
+ uint32_t type GNUNET_PACKED;
+
+ /**
+ * Priority of the item (NBO).
+ */
+ uint32_t priority GNUNET_PACKED;
+
+ /**
+ * Desired anonymity level (NBO).
+ */
+ uint32_t anonymity GNUNET_PACKED;
+
+ /**
+ * Desired replication level (NBO).
+ */
+ uint32_t replication GNUNET_PACKED;
+
+ /**
+ * Expiration time (NBO).
+ */
+ struct GNUNET_TIME_AbsoluteNBO expiration;
+
+ /**
+ * Key under which the item can be found.
+ */
+ struct GNUNET_HashCode key;
+
+};
+GNUNET_NETWORK_STRUCT_END
+
/**
- * Name of the second configuration file.
+ * Length of our magic header.
*/
-static char *alternative_cfg;
+static const size_t MAGIC_LEN = 16;
/**
- * Global return value.
+ * Magic header bytes.
*/
-static int ret;
+static const uint8_t MAGIC_BYTES[16] = "GNUNETDATASTORE1";
+
+/**
+ * Dump the database.
+ */
+static int dump;
/**
- * Our offset on 'get'.
+ * Insert into the database.
*/
-static uint64_t offset;
+static int insert;
/**
- * First UID ever returned.
+ * Dump file name.
*/
-static uint64_t first_uid;
+static char *file_name;
/**
- * Configuration for the source database.
+ * Dump file handle.
*/
-static struct GNUNET_CONFIGURATION_Handle *scfg;
+static struct GNUNET_DISK_FileHandle *file_handle;
/**
- * Handle for database source.
+ * Global return value.
*/
-static struct GNUNET_DATASTORE_Handle *db_src;
+static int ret;
/**
- * Handle for database destination.
+ * Handle for datastore.
*/
-static struct GNUNET_DATASTORE_Handle *db_dst;
+static struct GNUNET_DATASTORE_Handle *datastore;
/**
* Current operation.
*/
static struct GNUNET_DATASTORE_QueueEntry *qe;
+/**
+ * Record count.
+ */
+static uint64_t record_count;
+
static void
do_shutdown (void *cls)
{
if (NULL != qe)
GNUNET_DATASTORE_cancel (qe);
- GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO);
- GNUNET_DATASTORE_disconnect (db_dst, GNUNET_NO);
- GNUNET_CONFIGURATION_destroy (scfg);
+ if (NULL != datastore)
+ GNUNET_DATASTORE_disconnect (datastore, GNUNET_NO);
+ if (NULL != file_handle)
+ GNUNET_DISK_file_close (file_handle);
}
/**
- * Perform next GET operation.
+ * Begin dumping the database.
*/
static void
-do_get (void);
+start_dump (void);
/**
- * Continuation called to notify client about result of the
- * operation.
- *
- * @param cls closure
- * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
- * GNUNET_NO if content was already there
- * GNUNET_YES (or other positive value) on success
- * @param min_expiration minimum expiration time required for 0-priority content to be stored
- * by the datacache at this time, zero for unknown, forever if we have no
- * space for 0-priority content
- * @param msg NULL on success, otherwise an error message
+ * Begin inserting into the database.
*/
static void
-do_finish (void *cls,
- int32_t success,
- struct GNUNET_TIME_Absolute min_expiration,
- const char *msg)
-{
- qe = NULL;
- if (GNUNET_SYSERR == success)
- {
- fprintf (stderr,
- _("Failed to store item: %s, aborting\n"),
- msg);
- ret = 1;
- GNUNET_SCHEDULER_shutdown ();
- return;
- }
- do_get ();
-}
+start_insert (void);
+
+
+/**
+ * Perform next GET operation.
+ */
+static void
+do_get (const uint64_t next_uid);
/**
@@ -136,7 +171,7 @@ do_finish (void *cls,
* maybe 0 if no unique identifier is available
*/
static void
-do_put (void *cls,
+get_cb (void *cls,
const struct GNUNET_HashCode *key,
size_t size,
const void *data,
@@ -144,33 +179,63 @@ do_put (void *cls,
uint32_t priority,
uint32_t anonymity,
uint32_t replication,
- struct GNUNET_TIME_Absolute
- expiration,
+ struct GNUNET_TIME_Absolute expiration,
uint64_t uid)
{
qe = NULL;
- if ( (0 != offset) &&
- (uid == first_uid) )
+ if (NULL == key)
{
+ FPRINTF (stderr,
+ _("Dumped %" PRIu64 " records\n"),
+ record_count);
+ GNUNET_DISK_file_close (file_handle);
+ file_handle = NULL;
+ if (insert)
+ start_insert();
+ else
+ {
+ ret = 0;
+ GNUNET_SCHEDULER_shutdown ();
+ }
+ return;
+ }
+
+ struct DataRecord dr;
+ dr.size = htonl ((uint32_t) size);
+ dr.type = htonl (type);
+ dr.priority = htonl (priority);
+ dr.anonymity = htonl (anonymity);
+ dr.replication = htonl (replication);
+ dr.expiration = GNUNET_TIME_absolute_hton (expiration);
+ dr.key = *key;
+
+ ssize_t len;
+ len = GNUNET_DISK_file_write (file_handle, &dr, sizeof (dr));
+ if (sizeof (dr) != len)
+ {
+ FPRINTF (stderr,
+ _("Short write to file: %zd bytes expecting %zd\n"),
+ len,
+ sizeof (dr));
+ ret = 1;
GNUNET_SCHEDULER_shutdown ();
return;
}
- if (0 == offset)
- first_uid = uid;
- qe = GNUNET_DATASTORE_put (db_dst,
- 0,
- key,
- size,
- data,
- type,
- priority,
- anonymity,
- replication,
- expiration,
- 0,
- 1,
- &do_finish,
- NULL);
+
+ len = GNUNET_DISK_file_write (file_handle, data, size);
+ if (size != len)
+ {
+ FPRINTF (stderr,
+ _("Short write to file: %zd bytes expecting %zd\n"),
+ len,
+ size);
+ ret = 1;
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
+
+ record_count++;
+ do_get(uid + 1);
}
@@ -178,64 +243,236 @@ do_put (void *cls,
* Perform next GET operation.
*/
static void
-do_get ()
+do_get (const uint64_t next_uid)
{
- qe = GNUNET_DATASTORE_get_key (db_src,
- 0, false,
- NULL, GNUNET_BLOCK_TYPE_ANY,
- 0, 1,
- &do_put, NULL);
+ GNUNET_assert (NULL == qe);
+ qe = GNUNET_DATASTORE_get_key (datastore,
+ next_uid,
+ false /* random */,
+ NULL /* key */,
+ GNUNET_BLOCK_TYPE_ANY,
+ 0 /* queue_priority */,
+ 1 /* max_queue_size */,
+ &get_cb,
+ NULL /* proc_cls */);
+ if (NULL == qe)
+ {
+ FPRINTF (stderr,
+ _("Error queueing datastore GET operation\n"));
+ ret = 1;
+ GNUNET_SCHEDULER_shutdown ();
+ }
}
+/**
+ * Begin dumping the database.
+ */
+static void
+start_dump ()
+{
+ record_count = 0;
+
+ if (NULL != file_name)
+ {
+ file_handle = GNUNET_DISK_file_open (file_name,
+ GNUNET_DISK_OPEN_WRITE |
+ GNUNET_DISK_OPEN_TRUNCATE |
+ GNUNET_DISK_OPEN_CREATE,
+ GNUNET_DISK_PERM_USER_READ |
+ GNUNET_DISK_PERM_USER_WRITE);
+ if (NULL == file_handle)
+ {
+ FPRINTF (stderr,
+ _("Unable to open dump file: %s\n"),
+ file_name);
+ ret = 1;
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
+ }
+ else
+ {
+ file_handle = GNUNET_DISK_get_handle_from_int_fd (STDOUT_FILENO);
+ }
+ GNUNET_DISK_file_write (file_handle, MAGIC_BYTES, MAGIC_LEN);
+ do_get(0);
+}
+
/**
- * Main function that will be run by the scheduler.
+ * Continuation called to notify client about result of the
+ * operation.
*
* @param cls closure
- * @param args remaining command-line arguments
- * @param cfgfile name of the configuration file used
- * @param cfg configuration -- for destination datastore
+ * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
+ * GNUNET_NO if content was already there
+ * GNUNET_YES (or other positive value) on success
+ * @param min_expiration minimum expiration time required for 0-priority content to be stored
+ * by the datacache at this time, zero for unknown, forever if we have no
+ * space for 0-priority content
+ * @param msg NULL on success, otherwise an error message
*/
static void
-run (void *cls, char *const *args, const char *cfgfile,
- const struct GNUNET_CONFIGURATION_Handle *cfg)
+put_cb (void *cls,
+ int32_t success,
+ struct GNUNET_TIME_Absolute min_expiration,
+ const char *msg)
{
- if (NULL == alternative_cfg)
- return; /* nothing to be done */
- if (0 == strcmp (cfgfile, alternative_cfg))
+ qe = NULL;
+ if (GNUNET_SYSERR == success)
{
- fprintf (stderr,
- _("Cannot use the same configuration for source and destination\n"));
+ FPRINTF (stderr,
+ _("Failed to store item: %s, aborting\n"),
+ msg);
ret = 1;
+ GNUNET_SCHEDULER_shutdown ();
return;
}
- scfg = GNUNET_CONFIGURATION_create ();
- if (GNUNET_OK !=
- GNUNET_CONFIGURATION_load (scfg,
- alternative_cfg))
+
+ struct DataRecord dr;
+ ssize_t len;
+
+ len = GNUNET_DISK_file_read (file_handle, &dr, sizeof (dr));
+ if (0 == len)
+ {
+ FPRINTF (stderr,
+ _("Inserted %" PRIu64 " records\n"),
+ record_count);
+ ret = 0;
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
+ else if (sizeof (dr) != len)
{
- GNUNET_CONFIGURATION_destroy (scfg);
+ FPRINTF (stderr,
+ _("Short read from file: %zd bytes expecting %zd\n"),
+ len,
+ sizeof (dr));
ret = 1;
+ GNUNET_SCHEDULER_shutdown ();
return;
}
- db_src = GNUNET_DATASTORE_connect (scfg);
- if (NULL == db_src)
+
+ const size_t size = ntohl (dr.size);
+ uint8_t data[size];
+ len = GNUNET_DISK_file_read (file_handle, data, size);
+ if (size != len)
{
- GNUNET_CONFIGURATION_destroy (scfg);
+ FPRINTF (stderr,
+ _("Short read from file: %zd bytes expecting %zd\n"),
+ len,
+ size);
ret = 1;
+ GNUNET_SCHEDULER_shutdown ();
return;
}
- db_dst = GNUNET_DATASTORE_connect (cfg);
- if (NULL == db_dst)
+
+ record_count++;
+ qe = GNUNET_DATASTORE_put (datastore,
+ 0,
+ &dr.key,
+ size,
+ data,
+ ntohl (dr.type),
+ ntohl (dr.priority),
+ ntohl (dr.anonymity),
+ ntohl (dr.replication),
+ GNUNET_TIME_absolute_ntoh (dr.expiration),
+ 0,
+ 1,
+ &put_cb,
+ NULL);
+ if (NULL == qe)
{
- GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO);
- GNUNET_CONFIGURATION_destroy (scfg);
+ FPRINTF (stderr,
+ _("Error queueing datastore PUT operation\n"));
ret = 1;
+ GNUNET_SCHEDULER_shutdown ();
+ }
+}
+
+
+/**
+ * Begin inserting into the database.
+ */
+static void
+start_insert ()
+{
+ record_count = 0;
+
+ if (NULL != file_name)
+ {
+ file_handle = GNUNET_DISK_file_open (file_name,
+ GNUNET_DISK_OPEN_READ,
+ GNUNET_DISK_PERM_NONE);
+ if (NULL == file_handle)
+ {
+ FPRINTF (stderr,
+ _("Unable to open dump file: %s\n"),
+ file_name);
+ ret = 1;
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
+ }
+ else
+ {
+ file_handle = GNUNET_DISK_get_handle_from_int_fd (STDIN_FILENO);
+ }
+
+ uint8_t buf[MAGIC_LEN];
+ ssize_t len;
+
+ len = GNUNET_DISK_file_read (file_handle, buf, MAGIC_LEN);
+ if (len != MAGIC_LEN ||
+ 0 != memcmp (buf, MAGIC_BYTES, MAGIC_LEN))
+ {
+ FPRINTF (stderr,
+ _("Input file is not of a supported format\n"));
return;
}
+ put_cb (NULL, GNUNET_YES, GNUNET_TIME_UNIT_ZERO_ABS, NULL);
+}
+
+
+/**
+ * Main function that will be run by the scheduler.
+ *
+ * @param cls closure
+ * @param args remaining command-line arguments
+ * @param cfgfile name of the configuration file used
+ * @param cfg configuration
+ */
+static void
+run (void *cls,
+ char *const *args,
+ const char *cfgfile,
+ const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
- do_get ();
+ datastore = GNUNET_DATASTORE_connect (cfg);
+ if (NULL == datastore)
+ {
+ FPRINTF (stderr,
+ _("Failed connecting to the datastore.\n"));
+ ret = 1;
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
+ if (dump)
+ start_dump();
+ else if (insert)
+ start_insert();
+ else
+ {
+ FPRINTF (stderr,
+ _("Please choose at least one operation: %s, %s\n"),
+ "dump",
+ "insert");
+ ret = 1;
+ GNUNET_SCHEDULER_shutdown ();
+ }
}
@@ -247,14 +484,23 @@ run (void *cls, char *const *args, const char *cfgfile,
* @return 0 ok, 1 on error
*/
int
-main (int argc, char *const *argv)
+main (int argc,
+ char *const *argv)
{
struct GNUNET_GETOPT_CommandLineOption options[] = {
- GNUNET_GETOPT_option_filename ('s',
- "sourcecfg",
+ GNUNET_GETOPT_option_flag ('d',
+ "dump",
+ gettext_noop ("Dump all records from the datastore"),
+ &dump),
+ GNUNET_GETOPT_option_flag ('i',
+ "insert",
+ gettext_noop ("Insert records into the datastore"),
+ &insert),
+ GNUNET_GETOPT_option_filename ('f',
+ "file",
"FILENAME",
- gettext_noop ("specifies the configuration to use to access an alternative datastore; will merge that datastore into our current datastore"),
- &alternative_cfg),
+ gettext_noop ("File to dump or insert"),
+ &file_name),
GNUNET_GETOPT_OPTION_END
};
if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))