aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--doc/man/gnunet-datastore.124
-rw-r--r--src/datastore/gnunet-datastore.c452
2 files changed, 364 insertions, 112 deletions
diff --git a/doc/man/gnunet-datastore.1 b/doc/man/gnunet-datastore.1
index 14372a49b..1605006b3 100644
--- a/doc/man/gnunet-datastore.1
+++ b/doc/man/gnunet-datastore.1
@@ -1,6 +1,6 @@
1.TH gnunet\-datastore "1" "30 May 2013" "GNUnet" 1.TH gnunet\-datastore "1" "13 April 2017" "GNUnet"
2.SH NAME 2.SH NAME
3gnunet\-datastore \- merge or convert GNUnet datastore databases 3gnunet\-datastore \- dump or insert (restore) GNUnet datastore databases
4 4
5.SH SYNOPSIS 5.SH SYNOPSIS
6.B gnunet\-datastore 6.B gnunet\-datastore
@@ -8,26 +8,32 @@ gnunet\-datastore \- merge or convert GNUnet datastore databases
8.SH DESCRIPTION 8.SH DESCRIPTION
9.PP 9.PP
10 10
11gnunet\-datastore can be used to convert or merge GNUnet datastores. This is useful if a datastore is to be migrated between SQL databases, i.e. from sqlite to postgres or vice versa. gnunet\-datastore basically takes two configuration files (which must specify different databases) and reads in all of the data from the datasource (\-s option) and copies it to the destination (\-c option). Note that replication level information is lost in the process at this time. 11gnunet\-datastore can be used to backup and restore or merge GNUnet datastores. This is useful if a datastore is to be migrated between SQL databases, i.e. from sqlite to postgres or vice versa. gnunet\-datastore will dump the entire contents of the database or insert a dump file into the database.
12 12
13.TP 13.TP
14\fB\-c \fIFILENAME\fR, \fB\-\-config=FILENAME\fR 14\fB\-c \fIFILENAME\fR, \fB\-\-config=FILENAME\fR
15configuration file to use for the destination database 15configuration file to use
16.TP
17\fB\-d\fR, \fB\-\-dump\fR
18dump all records to a file
19.TP
20\fB\-f \fIFILENAME\fR, \fB\-\-file=FILENAME\fR
21file to dump to or insert from. Otherwise stdin/stdout are used.
16.TP 22.TP
17\fB\-h\fR, \fB\-\-help\fR 23\fB\-h\fR, \fB\-\-help\fR
18print help page 24print help page
19.TP 25.TP
26\fB\-i\fR, \fB\-\-insert\fR
27insert from dump file
28.TP
20\fB\-L \fILOGLEVEL\fR, \fB\-\-loglevel=LOGLEVEL\fR 29\fB\-L \fILOGLEVEL\fR, \fB\-\-loglevel=LOGLEVEL\fR
21Change the loglevel. Possible values for LOGLEVEL are ERROR, WARNING, INFO and DEBUG. 30Change the loglevel. Possible values for LOGLEVEL are ERROR, WARNING, INFO and DEBUG.
22.TP 31.TP
23\fB\-s \fIFILENAME\fR, \fB\-\-sourcecfg=FILENAME\fR 32\fB\-l \fIFILENAME\fR, \fB\-\-logfile=FILENAME\fR
24configuration file to use for the source database 33configure logging to write logs to FILENAME
25.TP 34.TP
26\fB\-v\fR, \fB\-\-version\fR 35\fB\-v\fR, \fB\-\-version\fR
27print the version number 36print the version number
28.TP
29\fB\-V\fR, \fB\-\-verbose\fR
30be verbose
31 37
32.SH NOTES 38.SH NOTES
33 39
diff --git a/src/datastore/gnunet-datastore.c b/src/datastore/gnunet-datastore.c
index 9e0ee205e..891343e17 100644
--- a/src/datastore/gnunet-datastore.c
+++ b/src/datastore/gnunet-datastore.c
@@ -23,101 +23,136 @@
23 * @brief tool to manipulate datastores 23 * @brief tool to manipulate datastores
24 * @author Christian Grothoff 24 * @author Christian Grothoff
25 */ 25 */
26#include <inttypes.h>
26#include "platform.h" 27#include "platform.h"
27#include "gnunet_util_lib.h" 28#include "gnunet_util_lib.h"
28#include "gnunet_datastore_service.h" 29#include "gnunet_datastore_service.h"
29 30
31GNUNET_NETWORK_STRUCT_BEGIN
32
33struct DataRecord
34{
35 /**
36 * Number of bytes in the item (NBO).
37 */
38 uint32_t size GNUNET_PACKED;
39
40 /**
41 * Type of the item (NBO) (actually an enum GNUNET_BLOCK_Type)
42 */
43 uint32_t type GNUNET_PACKED;
44
45 /**
46 * Priority of the item (NBO).
47 */
48 uint32_t priority GNUNET_PACKED;
49
50 /**
51 * Desired anonymity level (NBO).
52 */
53 uint32_t anonymity GNUNET_PACKED;
54
55 /**
56 * Desired replication level (NBO).
57 */
58 uint32_t replication GNUNET_PACKED;
59
60 /**
61 * Expiration time (NBO).
62 */
63 struct GNUNET_TIME_AbsoluteNBO expiration;
64
65 /**
66 * Key under which the item can be found.
67 */
68 struct GNUNET_HashCode key;
69
70};
71GNUNET_NETWORK_STRUCT_END
72
30 73
31/** 74/**
32 * Name of the second configuration file. 75 * Length of our magic header.
33 */ 76 */
34static char *alternative_cfg; 77static const size_t MAGIC_LEN = 16;
35 78
36/** 79/**
37 * Global return value. 80 * Magic header bytes.
38 */ 81 */
39static int ret; 82static const uint8_t MAGIC_BYTES[16] = "GNUNETDATASTORE1";
83
84/**
85 * Dump the database.
86 */
87static int dump;
40 88
41/** 89/**
42 * Our offset on 'get'. 90 * Insert into the database.
43 */ 91 */
44static uint64_t offset; 92static int insert;
45 93
46/** 94/**
47 * First UID ever returned. 95 * Dump file name.
48 */ 96 */
49static uint64_t first_uid; 97static char *file_name;
50 98
51/** 99/**
52 * Configuration for the source database. 100 * Dump file handle.
53 */ 101 */
54static struct GNUNET_CONFIGURATION_Handle *scfg; 102static struct GNUNET_DISK_FileHandle *file_handle;
55 103
56/** 104/**
57 * Handle for database source. 105 * Global return value.
58 */ 106 */
59static struct GNUNET_DATASTORE_Handle *db_src; 107static int ret;
60 108
61/** 109/**
62 * Handle for database destination. 110 * Handle for datastore.
63 */ 111 */
64static struct GNUNET_DATASTORE_Handle *db_dst; 112static struct GNUNET_DATASTORE_Handle *datastore;
65 113
66/** 114/**
67 * Current operation. 115 * Current operation.
68 */ 116 */
69static struct GNUNET_DATASTORE_QueueEntry *qe; 117static struct GNUNET_DATASTORE_QueueEntry *qe;
70 118
119/**
120 * Record count.
121 */
122static uint64_t record_count;
123
71 124
72static void 125static void
73do_shutdown (void *cls) 126do_shutdown (void *cls)
74{ 127{
75 if (NULL != qe) 128 if (NULL != qe)
76 GNUNET_DATASTORE_cancel (qe); 129 GNUNET_DATASTORE_cancel (qe);
77 GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO); 130 if (NULL != datastore)
78 GNUNET_DATASTORE_disconnect (db_dst, GNUNET_NO); 131 GNUNET_DATASTORE_disconnect (datastore, GNUNET_NO);
79 GNUNET_CONFIGURATION_destroy (scfg); 132 if (NULL != file_handle)
133 GNUNET_DISK_file_close (file_handle);
80} 134}
81 135
82 136
83/** 137/**
84 * Perform next GET operation. 138 * Begin dumping the database.
85 */ 139 */
86static void 140static void
87do_get (void); 141start_dump (void);
88 142
89 143
90/** 144/**
91 * Continuation called to notify client about result of the 145 * Begin inserting into the database.
92 * operation.
93 *
94 * @param cls closure
95 * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
96 * GNUNET_NO if content was already there
97 * GNUNET_YES (or other positive value) on success
98 * @param min_expiration minimum expiration time required for 0-priority content to be stored
99 * by the datacache at this time, zero for unknown, forever if we have no
100 * space for 0-priority content
101 * @param msg NULL on success, otherwise an error message
102 */ 146 */
103static void 147static void
104do_finish (void *cls, 148start_insert (void);
105 int32_t success, 149
106 struct GNUNET_TIME_Absolute min_expiration, 150
107 const char *msg) 151/**
108{ 152 * Perform next GET operation.
109 qe = NULL; 153 */
110 if (GNUNET_SYSERR == success) 154static void
111 { 155do_get (const uint64_t next_uid);
112 fprintf (stderr,
113 _("Failed to store item: %s, aborting\n"),
114 msg);
115 ret = 1;
116 GNUNET_SCHEDULER_shutdown ();
117 return;
118 }
119 do_get ();
120}
121 156
122 157
123/** 158/**
@@ -136,7 +171,7 @@ do_finish (void *cls,
136 * maybe 0 if no unique identifier is available 171 * maybe 0 if no unique identifier is available
137 */ 172 */
138static void 173static void
139do_put (void *cls, 174get_cb (void *cls,
140 const struct GNUNET_HashCode *key, 175 const struct GNUNET_HashCode *key,
141 size_t size, 176 size_t size,
142 const void *data, 177 const void *data,
@@ -144,33 +179,63 @@ do_put (void *cls,
144 uint32_t priority, 179 uint32_t priority,
145 uint32_t anonymity, 180 uint32_t anonymity,
146 uint32_t replication, 181 uint32_t replication,
147 struct GNUNET_TIME_Absolute 182 struct GNUNET_TIME_Absolute expiration,
148 expiration,
149 uint64_t uid) 183 uint64_t uid)
150{ 184{
151 qe = NULL; 185 qe = NULL;
152 if ( (0 != offset) && 186 if (NULL == key)
153 (uid == first_uid) )
154 { 187 {
188 FPRINTF (stderr,
189 _("Dumped %" PRIu64 " records\n"),
190 record_count);
191 GNUNET_DISK_file_close (file_handle);
192 file_handle = NULL;
193 if (insert)
194 start_insert();
195 else
196 {
197 ret = 0;
198 GNUNET_SCHEDULER_shutdown ();
199 }
200 return;
201 }
202
203 struct DataRecord dr;
204 dr.size = htonl ((uint32_t) size);
205 dr.type = htonl (type);
206 dr.priority = htonl (priority);
207 dr.anonymity = htonl (anonymity);
208 dr.replication = htonl (replication);
209 dr.expiration = GNUNET_TIME_absolute_hton (expiration);
210 dr.key = *key;
211
212 ssize_t len;
213 len = GNUNET_DISK_file_write (file_handle, &dr, sizeof (dr));
214 if (sizeof (dr) != len)
215 {
216 FPRINTF (stderr,
217 _("Short write to file: %zd bytes expecting %zd\n"),
218 len,
219 sizeof (dr));
220 ret = 1;
155 GNUNET_SCHEDULER_shutdown (); 221 GNUNET_SCHEDULER_shutdown ();
156 return; 222 return;
157 } 223 }
158 if (0 == offset) 224
159 first_uid = uid; 225 len = GNUNET_DISK_file_write (file_handle, data, size);
160 qe = GNUNET_DATASTORE_put (db_dst, 226 if (size != len)
161 0, 227 {
162 key, 228 FPRINTF (stderr,
163 size, 229 _("Short write to file: %zd bytes expecting %zd\n"),
164 data, 230 len,
165 type, 231 size);
166 priority, 232 ret = 1;
167 anonymity, 233 GNUNET_SCHEDULER_shutdown ();
168 replication, 234 return;
169 expiration, 235 }
170 0, 236
171 1, 237 record_count++;
172 &do_finish, 238 do_get(uid + 1);
173 NULL);
174} 239}
175 240
176 241
@@ -178,64 +243,236 @@ do_put (void *cls,
178 * Perform next GET operation. 243 * Perform next GET operation.
179 */ 244 */
180static void 245static void
181do_get () 246do_get (const uint64_t next_uid)
182{ 247{
183 qe = GNUNET_DATASTORE_get_key (db_src, 248 GNUNET_assert (NULL == qe);
184 0, false, 249 qe = GNUNET_DATASTORE_get_key (datastore,
185 NULL, GNUNET_BLOCK_TYPE_ANY, 250 next_uid,
186 0, 1, 251 false /* random */,
187 &do_put, NULL); 252 NULL /* key */,
253 GNUNET_BLOCK_TYPE_ANY,
254 0 /* queue_priority */,
255 1 /* max_queue_size */,
256 &get_cb,
257 NULL /* proc_cls */);
258 if (NULL == qe)
259 {
260 FPRINTF (stderr,
261 _("Error queueing datastore GET operation\n"));
262 ret = 1;
263 GNUNET_SCHEDULER_shutdown ();
264 }
188} 265}
189 266
190 267
268/**
269 * Begin dumping the database.
270 */
271static void
272start_dump ()
273{
274 record_count = 0;
275
276 if (NULL != file_name)
277 {
278 file_handle = GNUNET_DISK_file_open (file_name,
279 GNUNET_DISK_OPEN_WRITE |
280 GNUNET_DISK_OPEN_TRUNCATE |
281 GNUNET_DISK_OPEN_CREATE,
282 GNUNET_DISK_PERM_USER_READ |
283 GNUNET_DISK_PERM_USER_WRITE);
284 if (NULL == file_handle)
285 {
286 FPRINTF (stderr,
287 _("Unable to open dump file: %s\n"),
288 file_name);
289 ret = 1;
290 GNUNET_SCHEDULER_shutdown ();
291 return;
292 }
293 }
294 else
295 {
296 file_handle = GNUNET_DISK_get_handle_from_int_fd (STDOUT_FILENO);
297 }
298 GNUNET_DISK_file_write (file_handle, MAGIC_BYTES, MAGIC_LEN);
299 do_get(0);
300}
301
191 302
192/** 303/**
193 * Main function that will be run by the scheduler. 304 * Continuation called to notify client about result of the
305 * operation.
194 * 306 *
195 * @param cls closure 307 * @param cls closure
196 * @param args remaining command-line arguments 308 * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
197 * @param cfgfile name of the configuration file used 309 * GNUNET_NO if content was already there
198 * @param cfg configuration -- for destination datastore 310 * GNUNET_YES (or other positive value) on success
311 * @param min_expiration minimum expiration time required for 0-priority content to be stored
312 * by the datacache at this time, zero for unknown, forever if we have no
313 * space for 0-priority content
314 * @param msg NULL on success, otherwise an error message
199 */ 315 */
200static void 316static void
201run (void *cls, char *const *args, const char *cfgfile, 317put_cb (void *cls,
202 const struct GNUNET_CONFIGURATION_Handle *cfg) 318 int32_t success,
319 struct GNUNET_TIME_Absolute min_expiration,
320 const char *msg)
203{ 321{
204 if (NULL == alternative_cfg) 322 qe = NULL;
205 return; /* nothing to be done */ 323 if (GNUNET_SYSERR == success)
206 if (0 == strcmp (cfgfile, alternative_cfg))
207 { 324 {
208 fprintf (stderr, 325 FPRINTF (stderr,
209 _("Cannot use the same configuration for source and destination\n")); 326 _("Failed to store item: %s, aborting\n"),
327 msg);
210 ret = 1; 328 ret = 1;
329 GNUNET_SCHEDULER_shutdown ();
211 return; 330 return;
212 } 331 }
213 scfg = GNUNET_CONFIGURATION_create (); 332
214 if (GNUNET_OK != 333 struct DataRecord dr;
215 GNUNET_CONFIGURATION_load (scfg, 334 ssize_t len;
216 alternative_cfg)) 335
336 len = GNUNET_DISK_file_read (file_handle, &dr, sizeof (dr));
337 if (0 == len)
338 {
339 FPRINTF (stderr,
340 _("Inserted %" PRIu64 " records\n"),
341 record_count);
342 ret = 0;
343 GNUNET_SCHEDULER_shutdown ();
344 return;
345 }
346 else if (sizeof (dr) != len)
217 { 347 {
218 GNUNET_CONFIGURATION_destroy (scfg); 348 FPRINTF (stderr,
349 _("Short read from file: %zd bytes expecting %zd\n"),
350 len,
351 sizeof (dr));
219 ret = 1; 352 ret = 1;
353 GNUNET_SCHEDULER_shutdown ();
220 return; 354 return;
221 } 355 }
222 db_src = GNUNET_DATASTORE_connect (scfg); 356
223 if (NULL == db_src) 357 const size_t size = ntohl (dr.size);
358 uint8_t data[size];
359 len = GNUNET_DISK_file_read (file_handle, data, size);
360 if (size != len)
224 { 361 {
225 GNUNET_CONFIGURATION_destroy (scfg); 362 FPRINTF (stderr,
363 _("Short read from file: %zd bytes expecting %zd\n"),
364 len,
365 size);
226 ret = 1; 366 ret = 1;
367 GNUNET_SCHEDULER_shutdown ();
227 return; 368 return;
228 } 369 }
229 db_dst = GNUNET_DATASTORE_connect (cfg); 370
230 if (NULL == db_dst) 371 record_count++;
372 qe = GNUNET_DATASTORE_put (datastore,
373 0,
374 &dr.key,
375 size,
376 data,
377 ntohl (dr.type),
378 ntohl (dr.priority),
379 ntohl (dr.anonymity),
380 ntohl (dr.replication),
381 GNUNET_TIME_absolute_ntoh (dr.expiration),
382 0,
383 1,
384 &put_cb,
385 NULL);
386 if (NULL == qe)
231 { 387 {
232 GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO); 388 FPRINTF (stderr,
233 GNUNET_CONFIGURATION_destroy (scfg); 389 _("Error queueing datastore PUT operation\n"));
234 ret = 1; 390 ret = 1;
391 GNUNET_SCHEDULER_shutdown ();
392 }
393}
394
395
396/**
397 * Begin inserting into the database.
398 */
399static void
400start_insert ()
401{
402 record_count = 0;
403
404 if (NULL != file_name)
405 {
406 file_handle = GNUNET_DISK_file_open (file_name,
407 GNUNET_DISK_OPEN_READ,
408 GNUNET_DISK_PERM_NONE);
409 if (NULL == file_handle)
410 {
411 FPRINTF (stderr,
412 _("Unable to open dump file: %s\n"),
413 file_name);
414 ret = 1;
415 GNUNET_SCHEDULER_shutdown ();
416 return;
417 }
418 }
419 else
420 {
421 file_handle = GNUNET_DISK_get_handle_from_int_fd (STDIN_FILENO);
422 }
423
424 uint8_t buf[MAGIC_LEN];
425 ssize_t len;
426
427 len = GNUNET_DISK_file_read (file_handle, buf, MAGIC_LEN);
428 if (len != MAGIC_LEN ||
429 0 != memcmp (buf, MAGIC_BYTES, MAGIC_LEN))
430 {
431 FPRINTF (stderr,
432 _("Input file is not of a supported format\n"));
235 return; 433 return;
236 } 434 }
435 put_cb (NULL, GNUNET_YES, GNUNET_TIME_UNIT_ZERO_ABS, NULL);
436}
437
438
439/**
440 * Main function that will be run by the scheduler.
441 *
442 * @param cls closure
443 * @param args remaining command-line arguments
444 * @param cfgfile name of the configuration file used
445 * @param cfg configuration
446 */
447static void
448run (void *cls,
449 char *const *args,
450 const char *cfgfile,
451 const struct GNUNET_CONFIGURATION_Handle *cfg)
452{
237 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL); 453 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
238 do_get (); 454 datastore = GNUNET_DATASTORE_connect (cfg);
455 if (NULL == datastore)
456 {
457 FPRINTF (stderr,
458 _("Failed connecting to the datastore.\n"));
459 ret = 1;
460 GNUNET_SCHEDULER_shutdown ();
461 return;
462 }
463 if (dump)
464 start_dump();
465 else if (insert)
466 start_insert();
467 else
468 {
469 FPRINTF (stderr,
470 _("Please choose at least one operation: %s, %s\n"),
471 "dump",
472 "insert");
473 ret = 1;
474 GNUNET_SCHEDULER_shutdown ();
475 }
239} 476}
240 477
241 478
@@ -247,14 +484,23 @@ run (void *cls, char *const *args, const char *cfgfile,
247 * @return 0 ok, 1 on error 484 * @return 0 ok, 1 on error
248 */ 485 */
249int 486int
250main (int argc, char *const *argv) 487main (int argc,
488 char *const *argv)
251{ 489{
252 struct GNUNET_GETOPT_CommandLineOption options[] = { 490 struct GNUNET_GETOPT_CommandLineOption options[] = {
253 GNUNET_GETOPT_option_filename ('s', 491 GNUNET_GETOPT_option_flag ('d',
254 "sourcecfg", 492 "dump",
493 gettext_noop ("Dump all records from the datastore"),
494 &dump),
495 GNUNET_GETOPT_option_flag ('i',
496 "insert",
497 gettext_noop ("Insert records into the datastore"),
498 &insert),
499 GNUNET_GETOPT_option_filename ('f',
500 "file",
255 "FILENAME", 501 "FILENAME",
256 gettext_noop ("specifies the configuration to use to access an alternative datastore; will merge that datastore into our current datastore"), 502 gettext_noop ("File to dump or insert"),
257 &alternative_cfg), 503 &file_name),
258 GNUNET_GETOPT_OPTION_END 504 GNUNET_GETOPT_OPTION_END
259 }; 505 };
260 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) 506 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))