aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorxrs <xrs@mail36.net>2017-04-19 09:06:09 +0200
committerxrs <xrs@mail36.net>2017-04-19 09:06:09 +0200
commit94d431574c4c4ce98db5382132b34300747ec574 (patch)
tree25d36d3aff3e5ca4d25f2480ff954f924cd0e164 /src
parentacfd78293352b5cdd640023343db3578fdceea94 (diff)
parent50d0ee6161da036cf7e8761f945793f09227fa10 (diff)
downloadgnunet-94d431574c4c4ce98db5382132b34300747ec574.tar.gz
gnunet-94d431574c4c4ce98db5382132b34300747ec574.zip
Merge branch 'master' of ssh://gnunet.org/gnunet
Diffstat (limited to 'src')
-rw-r--r--src/datastore/gnunet-datastore.c452
-rw-r--r--src/datastore/gnunet-service-datastore.c274
-rw-r--r--src/datastore/perf_plugin_datastore.c13
-rw-r--r--src/datastore/plugin_datastore_heap.c372
-rw-r--r--src/datastore/plugin_datastore_mysql.c336
-rw-r--r--src/datastore/plugin_datastore_postgres.c219
-rw-r--r--src/datastore/plugin_datastore_sqlite.c259
-rw-r--r--src/datastore/plugin_datastore_template.c88
-rw-r--r--src/datastore/test_plugin_datastore.c76
-rw-r--r--src/include/gnunet_datastore_plugin.h112
-rw-r--r--src/include/gnunet_json_lib.h11
-rw-r--r--src/json/json_helper.c77
12 files changed, 1329 insertions, 960 deletions
diff --git a/src/datastore/gnunet-datastore.c b/src/datastore/gnunet-datastore.c
index 9e0ee205e..891343e17 100644
--- a/src/datastore/gnunet-datastore.c
+++ b/src/datastore/gnunet-datastore.c
@@ -23,101 +23,136 @@
23 * @brief tool to manipulate datastores 23 * @brief tool to manipulate datastores
24 * @author Christian Grothoff 24 * @author Christian Grothoff
25 */ 25 */
26#include <inttypes.h>
26#include "platform.h" 27#include "platform.h"
27#include "gnunet_util_lib.h" 28#include "gnunet_util_lib.h"
28#include "gnunet_datastore_service.h" 29#include "gnunet_datastore_service.h"
29 30
31GNUNET_NETWORK_STRUCT_BEGIN
32
33struct DataRecord
34{
35 /**
36 * Number of bytes in the item (NBO).
37 */
38 uint32_t size GNUNET_PACKED;
39
40 /**
41 * Type of the item (NBO) (actually an enum GNUNET_BLOCK_Type)
42 */
43 uint32_t type GNUNET_PACKED;
44
45 /**
46 * Priority of the item (NBO).
47 */
48 uint32_t priority GNUNET_PACKED;
49
50 /**
51 * Desired anonymity level (NBO).
52 */
53 uint32_t anonymity GNUNET_PACKED;
54
55 /**
56 * Desired replication level (NBO).
57 */
58 uint32_t replication GNUNET_PACKED;
59
60 /**
61 * Expiration time (NBO).
62 */
63 struct GNUNET_TIME_AbsoluteNBO expiration;
64
65 /**
66 * Key under which the item can be found.
67 */
68 struct GNUNET_HashCode key;
69
70};
71GNUNET_NETWORK_STRUCT_END
72
30 73
31/** 74/**
32 * Name of the second configuration file. 75 * Length of our magic header.
33 */ 76 */
34static char *alternative_cfg; 77static const size_t MAGIC_LEN = 16;
35 78
36/** 79/**
37 * Global return value. 80 * Magic header bytes.
38 */ 81 */
39static int ret; 82static const uint8_t MAGIC_BYTES[16] = "GNUNETDATASTORE1";
83
84/**
85 * Dump the database.
86 */
87static int dump;
40 88
41/** 89/**
42 * Our offset on 'get'. 90 * Insert into the database.
43 */ 91 */
44static uint64_t offset; 92static int insert;
45 93
46/** 94/**
47 * First UID ever returned. 95 * Dump file name.
48 */ 96 */
49static uint64_t first_uid; 97static char *file_name;
50 98
51/** 99/**
52 * Configuration for the source database. 100 * Dump file handle.
53 */ 101 */
54static struct GNUNET_CONFIGURATION_Handle *scfg; 102static struct GNUNET_DISK_FileHandle *file_handle;
55 103
56/** 104/**
57 * Handle for database source. 105 * Global return value.
58 */ 106 */
59static struct GNUNET_DATASTORE_Handle *db_src; 107static int ret;
60 108
61/** 109/**
62 * Handle for database destination. 110 * Handle for datastore.
63 */ 111 */
64static struct GNUNET_DATASTORE_Handle *db_dst; 112static struct GNUNET_DATASTORE_Handle *datastore;
65 113
66/** 114/**
67 * Current operation. 115 * Current operation.
68 */ 116 */
69static struct GNUNET_DATASTORE_QueueEntry *qe; 117static struct GNUNET_DATASTORE_QueueEntry *qe;
70 118
119/**
120 * Record count.
121 */
122static uint64_t record_count;
123
71 124
72static void 125static void
73do_shutdown (void *cls) 126do_shutdown (void *cls)
74{ 127{
75 if (NULL != qe) 128 if (NULL != qe)
76 GNUNET_DATASTORE_cancel (qe); 129 GNUNET_DATASTORE_cancel (qe);
77 GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO); 130 if (NULL != datastore)
78 GNUNET_DATASTORE_disconnect (db_dst, GNUNET_NO); 131 GNUNET_DATASTORE_disconnect (datastore, GNUNET_NO);
79 GNUNET_CONFIGURATION_destroy (scfg); 132 if (NULL != file_handle)
133 GNUNET_DISK_file_close (file_handle);
80} 134}
81 135
82 136
83/** 137/**
84 * Perform next GET operation. 138 * Begin dumping the database.
85 */ 139 */
86static void 140static void
87do_get (void); 141start_dump (void);
88 142
89 143
90/** 144/**
91 * Continuation called to notify client about result of the 145 * Begin inserting into the database.
92 * operation.
93 *
94 * @param cls closure
95 * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
96 * GNUNET_NO if content was already there
97 * GNUNET_YES (or other positive value) on success
98 * @param min_expiration minimum expiration time required for 0-priority content to be stored
99 * by the datacache at this time, zero for unknown, forever if we have no
100 * space for 0-priority content
101 * @param msg NULL on success, otherwise an error message
102 */ 146 */
103static void 147static void
104do_finish (void *cls, 148start_insert (void);
105 int32_t success, 149
106 struct GNUNET_TIME_Absolute min_expiration, 150
107 const char *msg) 151/**
108{ 152 * Perform next GET operation.
109 qe = NULL; 153 */
110 if (GNUNET_SYSERR == success) 154static void
111 { 155do_get (const uint64_t next_uid);
112 fprintf (stderr,
113 _("Failed to store item: %s, aborting\n"),
114 msg);
115 ret = 1;
116 GNUNET_SCHEDULER_shutdown ();
117 return;
118 }
119 do_get ();
120}
121 156
122 157
123/** 158/**
@@ -136,7 +171,7 @@ do_finish (void *cls,
136 * maybe 0 if no unique identifier is available 171 * maybe 0 if no unique identifier is available
137 */ 172 */
138static void 173static void
139do_put (void *cls, 174get_cb (void *cls,
140 const struct GNUNET_HashCode *key, 175 const struct GNUNET_HashCode *key,
141 size_t size, 176 size_t size,
142 const void *data, 177 const void *data,
@@ -144,33 +179,63 @@ do_put (void *cls,
144 uint32_t priority, 179 uint32_t priority,
145 uint32_t anonymity, 180 uint32_t anonymity,
146 uint32_t replication, 181 uint32_t replication,
147 struct GNUNET_TIME_Absolute 182 struct GNUNET_TIME_Absolute expiration,
148 expiration,
149 uint64_t uid) 183 uint64_t uid)
150{ 184{
151 qe = NULL; 185 qe = NULL;
152 if ( (0 != offset) && 186 if (NULL == key)
153 (uid == first_uid) )
154 { 187 {
188 FPRINTF (stderr,
189 _("Dumped %" PRIu64 " records\n"),
190 record_count);
191 GNUNET_DISK_file_close (file_handle);
192 file_handle = NULL;
193 if (insert)
194 start_insert();
195 else
196 {
197 ret = 0;
198 GNUNET_SCHEDULER_shutdown ();
199 }
200 return;
201 }
202
203 struct DataRecord dr;
204 dr.size = htonl ((uint32_t) size);
205 dr.type = htonl (type);
206 dr.priority = htonl (priority);
207 dr.anonymity = htonl (anonymity);
208 dr.replication = htonl (replication);
209 dr.expiration = GNUNET_TIME_absolute_hton (expiration);
210 dr.key = *key;
211
212 ssize_t len;
213 len = GNUNET_DISK_file_write (file_handle, &dr, sizeof (dr));
214 if (sizeof (dr) != len)
215 {
216 FPRINTF (stderr,
217 _("Short write to file: %zd bytes expecting %zd\n"),
218 len,
219 sizeof (dr));
220 ret = 1;
155 GNUNET_SCHEDULER_shutdown (); 221 GNUNET_SCHEDULER_shutdown ();
156 return; 222 return;
157 } 223 }
158 if (0 == offset) 224
159 first_uid = uid; 225 len = GNUNET_DISK_file_write (file_handle, data, size);
160 qe = GNUNET_DATASTORE_put (db_dst, 226 if (size != len)
161 0, 227 {
162 key, 228 FPRINTF (stderr,
163 size, 229 _("Short write to file: %zd bytes expecting %zd\n"),
164 data, 230 len,
165 type, 231 size);
166 priority, 232 ret = 1;
167 anonymity, 233 GNUNET_SCHEDULER_shutdown ();
168 replication, 234 return;
169 expiration, 235 }
170 0, 236
171 1, 237 record_count++;
172 &do_finish, 238 do_get(uid + 1);
173 NULL);
174} 239}
175 240
176 241
@@ -178,64 +243,236 @@ do_put (void *cls,
178 * Perform next GET operation. 243 * Perform next GET operation.
179 */ 244 */
180static void 245static void
181do_get () 246do_get (const uint64_t next_uid)
182{ 247{
183 qe = GNUNET_DATASTORE_get_key (db_src, 248 GNUNET_assert (NULL == qe);
184 0, false, 249 qe = GNUNET_DATASTORE_get_key (datastore,
185 NULL, GNUNET_BLOCK_TYPE_ANY, 250 next_uid,
186 0, 1, 251 false /* random */,
187 &do_put, NULL); 252 NULL /* key */,
253 GNUNET_BLOCK_TYPE_ANY,
254 0 /* queue_priority */,
255 1 /* max_queue_size */,
256 &get_cb,
257 NULL /* proc_cls */);
258 if (NULL == qe)
259 {
260 FPRINTF (stderr,
261 _("Error queueing datastore GET operation\n"));
262 ret = 1;
263 GNUNET_SCHEDULER_shutdown ();
264 }
188} 265}
189 266
190 267
268/**
269 * Begin dumping the database.
270 */
271static void
272start_dump ()
273{
274 record_count = 0;
275
276 if (NULL != file_name)
277 {
278 file_handle = GNUNET_DISK_file_open (file_name,
279 GNUNET_DISK_OPEN_WRITE |
280 GNUNET_DISK_OPEN_TRUNCATE |
281 GNUNET_DISK_OPEN_CREATE,
282 GNUNET_DISK_PERM_USER_READ |
283 GNUNET_DISK_PERM_USER_WRITE);
284 if (NULL == file_handle)
285 {
286 FPRINTF (stderr,
287 _("Unable to open dump file: %s\n"),
288 file_name);
289 ret = 1;
290 GNUNET_SCHEDULER_shutdown ();
291 return;
292 }
293 }
294 else
295 {
296 file_handle = GNUNET_DISK_get_handle_from_int_fd (STDOUT_FILENO);
297 }
298 GNUNET_DISK_file_write (file_handle, MAGIC_BYTES, MAGIC_LEN);
299 do_get(0);
300}
301
191 302
192/** 303/**
193 * Main function that will be run by the scheduler. 304 * Continuation called to notify client about result of the
305 * operation.
194 * 306 *
195 * @param cls closure 307 * @param cls closure
196 * @param args remaining command-line arguments 308 * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
197 * @param cfgfile name of the configuration file used 309 * GNUNET_NO if content was already there
198 * @param cfg configuration -- for destination datastore 310 * GNUNET_YES (or other positive value) on success
311 * @param min_expiration minimum expiration time required for 0-priority content to be stored
312 * by the datacache at this time, zero for unknown, forever if we have no
313 * space for 0-priority content
314 * @param msg NULL on success, otherwise an error message
199 */ 315 */
200static void 316static void
201run (void *cls, char *const *args, const char *cfgfile, 317put_cb (void *cls,
202 const struct GNUNET_CONFIGURATION_Handle *cfg) 318 int32_t success,
319 struct GNUNET_TIME_Absolute min_expiration,
320 const char *msg)
203{ 321{
204 if (NULL == alternative_cfg) 322 qe = NULL;
205 return; /* nothing to be done */ 323 if (GNUNET_SYSERR == success)
206 if (0 == strcmp (cfgfile, alternative_cfg))
207 { 324 {
208 fprintf (stderr, 325 FPRINTF (stderr,
209 _("Cannot use the same configuration for source and destination\n")); 326 _("Failed to store item: %s, aborting\n"),
327 msg);
210 ret = 1; 328 ret = 1;
329 GNUNET_SCHEDULER_shutdown ();
211 return; 330 return;
212 } 331 }
213 scfg = GNUNET_CONFIGURATION_create (); 332
214 if (GNUNET_OK != 333 struct DataRecord dr;
215 GNUNET_CONFIGURATION_load (scfg, 334 ssize_t len;
216 alternative_cfg)) 335
336 len = GNUNET_DISK_file_read (file_handle, &dr, sizeof (dr));
337 if (0 == len)
338 {
339 FPRINTF (stderr,
340 _("Inserted %" PRIu64 " records\n"),
341 record_count);
342 ret = 0;
343 GNUNET_SCHEDULER_shutdown ();
344 return;
345 }
346 else if (sizeof (dr) != len)
217 { 347 {
218 GNUNET_CONFIGURATION_destroy (scfg); 348 FPRINTF (stderr,
349 _("Short read from file: %zd bytes expecting %zd\n"),
350 len,
351 sizeof (dr));
219 ret = 1; 352 ret = 1;
353 GNUNET_SCHEDULER_shutdown ();
220 return; 354 return;
221 } 355 }
222 db_src = GNUNET_DATASTORE_connect (scfg); 356
223 if (NULL == db_src) 357 const size_t size = ntohl (dr.size);
358 uint8_t data[size];
359 len = GNUNET_DISK_file_read (file_handle, data, size);
360 if (size != len)
224 { 361 {
225 GNUNET_CONFIGURATION_destroy (scfg); 362 FPRINTF (stderr,
363 _("Short read from file: %zd bytes expecting %zd\n"),
364 len,
365 size);
226 ret = 1; 366 ret = 1;
367 GNUNET_SCHEDULER_shutdown ();
227 return; 368 return;
228 } 369 }
229 db_dst = GNUNET_DATASTORE_connect (cfg); 370
230 if (NULL == db_dst) 371 record_count++;
372 qe = GNUNET_DATASTORE_put (datastore,
373 0,
374 &dr.key,
375 size,
376 data,
377 ntohl (dr.type),
378 ntohl (dr.priority),
379 ntohl (dr.anonymity),
380 ntohl (dr.replication),
381 GNUNET_TIME_absolute_ntoh (dr.expiration),
382 0,
383 1,
384 &put_cb,
385 NULL);
386 if (NULL == qe)
231 { 387 {
232 GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO); 388 FPRINTF (stderr,
233 GNUNET_CONFIGURATION_destroy (scfg); 389 _("Error queueing datastore PUT operation\n"));
234 ret = 1; 390 ret = 1;
391 GNUNET_SCHEDULER_shutdown ();
392 }
393}
394
395
396/**
397 * Begin inserting into the database.
398 */
399static void
400start_insert ()
401{
402 record_count = 0;
403
404 if (NULL != file_name)
405 {
406 file_handle = GNUNET_DISK_file_open (file_name,
407 GNUNET_DISK_OPEN_READ,
408 GNUNET_DISK_PERM_NONE);
409 if (NULL == file_handle)
410 {
411 FPRINTF (stderr,
412 _("Unable to open dump file: %s\n"),
413 file_name);
414 ret = 1;
415 GNUNET_SCHEDULER_shutdown ();
416 return;
417 }
418 }
419 else
420 {
421 file_handle = GNUNET_DISK_get_handle_from_int_fd (STDIN_FILENO);
422 }
423
424 uint8_t buf[MAGIC_LEN];
425 ssize_t len;
426
427 len = GNUNET_DISK_file_read (file_handle, buf, MAGIC_LEN);
428 if (len != MAGIC_LEN ||
429 0 != memcmp (buf, MAGIC_BYTES, MAGIC_LEN))
430 {
431 FPRINTF (stderr,
432 _("Input file is not of a supported format\n"));
235 return; 433 return;
236 } 434 }
435 put_cb (NULL, GNUNET_YES, GNUNET_TIME_UNIT_ZERO_ABS, NULL);
436}
437
438
439/**
440 * Main function that will be run by the scheduler.
441 *
442 * @param cls closure
443 * @param args remaining command-line arguments
444 * @param cfgfile name of the configuration file used
445 * @param cfg configuration
446 */
447static void
448run (void *cls,
449 char *const *args,
450 const char *cfgfile,
451 const struct GNUNET_CONFIGURATION_Handle *cfg)
452{
237 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL); 453 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
238 do_get (); 454 datastore = GNUNET_DATASTORE_connect (cfg);
455 if (NULL == datastore)
456 {
457 FPRINTF (stderr,
458 _("Failed connecting to the datastore.\n"));
459 ret = 1;
460 GNUNET_SCHEDULER_shutdown ();
461 return;
462 }
463 if (dump)
464 start_dump();
465 else if (insert)
466 start_insert();
467 else
468 {
469 FPRINTF (stderr,
470 _("Please choose at least one operation: %s, %s\n"),
471 "dump",
472 "insert");
473 ret = 1;
474 GNUNET_SCHEDULER_shutdown ();
475 }
239} 476}
240 477
241 478
@@ -247,14 +484,23 @@ run (void *cls, char *const *args, const char *cfgfile,
247 * @return 0 ok, 1 on error 484 * @return 0 ok, 1 on error
248 */ 485 */
249int 486int
250main (int argc, char *const *argv) 487main (int argc,
488 char *const *argv)
251{ 489{
252 struct GNUNET_GETOPT_CommandLineOption options[] = { 490 struct GNUNET_GETOPT_CommandLineOption options[] = {
253 GNUNET_GETOPT_option_filename ('s', 491 GNUNET_GETOPT_option_flag ('d',
254 "sourcecfg", 492 "dump",
493 gettext_noop ("Dump all records from the datastore"),
494 &dump),
495 GNUNET_GETOPT_option_flag ('i',
496 "insert",
497 gettext_noop ("Insert records into the datastore"),
498 &insert),
499 GNUNET_GETOPT_option_filename ('f',
500 "file",
255 "FILENAME", 501 "FILENAME",
256 gettext_noop ("specifies the configuration to use to access an alternative datastore; will merge that datastore into our current datastore"), 502 gettext_noop ("File to dump or insert"),
257 &alternative_cfg), 503 &file_name),
258 GNUNET_GETOPT_OPTION_END 504 GNUNET_GETOPT_OPTION_END
259 }; 505 };
260 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) 506 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c
index 277530843..53ba858e4 100644
--- a/src/datastore/gnunet-service-datastore.c
+++ b/src/datastore/gnunet-service-datastore.c
@@ -733,41 +733,23 @@ check_data (const struct DataMessage *dm)
733 733
734 734
735/** 735/**
736 * Context for a PUT request used to see if the content is
737 * already present.
738 */
739struct PutContext
740{
741 /**
742 * Client to notify on completion.
743 */
744 struct GNUNET_SERVICE_Client *client;
745
746#if ! HAVE_UNALIGNED_64_ACCESS
747 void *reserved;
748#endif
749
750 /* followed by the 'struct DataMessage' */
751};
752
753
754/**
755 * Put continuation. 736 * Put continuation.
756 * 737 *
757 * @param cls closure 738 * @param cls closure
758 * @param key key for the item stored 739 * @param key key for the item stored
759 * @param size size of the item stored 740 * @param size size of the item stored
760 * @param status #GNUNET_OK or #GNUNET_SYSERROR 741 * @param status #GNUNET_OK if inserted, #GNUNET_NO if updated,
742 * or #GNUNET_SYSERROR if error
761 * @param msg error message on error 743 * @param msg error message on error
762 */ 744 */
763static void 745static void
764put_continuation (void *cls, 746put_continuation (void *cls,
765 const struct GNUNET_HashCode *key, 747 const struct GNUNET_HashCode *key,
766 uint32_t size, 748 uint32_t size,
767 int status, 749 int status,
768 const char *msg) 750 const char *msg)
769{ 751{
770 struct PutContext *pc = cls; 752 struct GNUNET_SERVICE_Client *client = cls;
771 753
772 if (GNUNET_OK == status) 754 if (GNUNET_OK == status)
773 { 755 {
@@ -782,10 +764,9 @@ put_continuation (void *cls,
782 size, 764 size,
783 GNUNET_h2s (key)); 765 GNUNET_h2s (key));
784 } 766 }
785 transmit_status (pc->client, 767 transmit_status (client,
786 status, 768 GNUNET_SYSERR == status ? GNUNET_SYSERR : GNUNET_OK,
787 msg); 769 msg);
788 GNUNET_free (pc);
789 if (quota - reserved - cache_size < payload) 770 if (quota - reserved - cache_size < payload)
790 { 771 {
791 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 772 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
@@ -799,125 +780,6 @@ put_continuation (void *cls,
799 780
800 781
801/** 782/**
802 * Actually put the data message.
803 *
804 * @param pc put context
805 */
806static void
807execute_put (struct PutContext *pc)
808{
809 const struct DataMessage *dm;
810
811 dm = (const struct DataMessage *) &pc[1];
812 plugin->api->put (plugin->api->cls,
813 &dm->key,
814 ntohl (dm->size),
815 &dm[1],
816 ntohl (dm->type),
817 ntohl (dm->priority),
818 ntohl (dm->anonymity),
819 ntohl (dm->replication),
820 GNUNET_TIME_absolute_ntoh (dm->expiration),
821 &put_continuation,
822 pc);
823}
824
825
826/**
827 *
828 * @param cls closure
829 * @param status #GNUNET_OK or #GNUNET_SYSERR
830 * @param msg error message on error
831 */
832static void
833check_present_continuation (void *cls,
834 int status,
835 const char *msg)
836{
837 struct GNUNET_SERVICE_Client *client = cls;
838
839 transmit_status (client,
840 GNUNET_NO,
841 NULL);
842}
843
844
845/**
846 * Function that will check if the given datastore entry
847 * matches the put and if none match executes the put.
848 *
849 * @param cls closure, pointer to the client (of type `struct PutContext`).
850 * @param key key for the content
851 * @param size number of bytes in data
852 * @param data content stored
853 * @param type type of the content
854 * @param priority priority of the content
855 * @param anonymity anonymity-level for the content
856 * @param replication replication-level for the content
857 * @param expiration expiration time for the content
858 * @param uid unique identifier for the datum;
859 * maybe 0 if no unique identifier is available
860 * @return #GNUNET_OK usually
861 * #GNUNET_NO to delete the item
862 */
863static int
864check_present (void *cls,
865 const struct GNUNET_HashCode *key,
866 uint32_t size,
867 const void *data,
868 enum GNUNET_BLOCK_Type type,
869 uint32_t priority,
870 uint32_t anonymity,
871 uint32_t replication,
872 struct GNUNET_TIME_Absolute expiration,
873 uint64_t uid)
874{
875 struct PutContext *pc = cls;
876 const struct DataMessage *dm;
877
878 dm = (const struct DataMessage *) &pc[1];
879 if (key == NULL)
880 {
881 execute_put (pc);
882 return GNUNET_OK;
883 }
884 if ( (GNUNET_BLOCK_TYPE_FS_DBLOCK == type) ||
885 (GNUNET_BLOCK_TYPE_FS_IBLOCK == type) ||
886 ( (size == ntohl (dm->size)) &&
887 (0 == memcmp (&dm[1],
888 data,
889 size)) ) )
890 {
891 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
892 "Result already present in datastore\n");
893 if ( (ntohl (dm->priority) > 0) ||
894 (ntohl (dm->replication) > 0) ||
895 (GNUNET_TIME_absolute_ntoh (dm->expiration).abs_value_us >
896 expiration.abs_value_us) )
897 plugin->api->update (plugin->api->cls,
898 uid,
899 ntohl (dm->priority),
900 ntohl (dm->replication),
901 GNUNET_TIME_absolute_ntoh (dm->expiration),
902 &check_present_continuation,
903 pc->client);
904 else
905 {
906 transmit_status (pc->client,
907 GNUNET_NO,
908 NULL);
909 }
910 GNUNET_free (pc);
911 }
912 else
913 {
914 execute_put (pc);
915 }
916 return GNUNET_OK;
917}
918
919
920/**
921 * Verify PUT-message. 783 * Verify PUT-message.
922 * 784 *
923 * @param cls identification of the client 785 * @param cls identification of the client
@@ -950,8 +812,6 @@ handle_put (void *cls,
950 struct GNUNET_SERVICE_Client *client = cls; 812 struct GNUNET_SERVICE_Client *client = cls;
951 int rid; 813 int rid;
952 struct ReservationList *pos; 814 struct ReservationList *pos;
953 struct PutContext *pc;
954 struct GNUNET_HashCode vhash;
955 uint32_t size; 815 uint32_t size;
956 816
957 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 817 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -979,30 +839,20 @@ handle_put (void *cls,
979 GNUNET_NO); 839 GNUNET_NO);
980 } 840 }
981 } 841 }
982 pc = GNUNET_malloc (sizeof (struct PutContext) + size + 842 bool absent = GNUNET_NO == GNUNET_CONTAINER_bloomfilter_test (filter,
983 sizeof (struct DataMessage)); 843 &dm->key);
984 pc->client = client; 844 plugin->api->put (plugin->api->cls,
985 GNUNET_memcpy (&pc[1], 845 &dm->key,
986 dm, 846 absent,
987 size + sizeof (struct DataMessage)); 847 ntohl (dm->size),
988 if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (filter, 848 &dm[1],
989 &dm->key)) 849 ntohl (dm->type),
990 { 850 ntohl (dm->priority),
991 GNUNET_CRYPTO_hash (&dm[1], 851 ntohl (dm->anonymity),
992 size, 852 ntohl (dm->replication),
993 &vhash); 853 GNUNET_TIME_absolute_ntoh (dm->expiration),
994 plugin->api->get_key (plugin->api->cls, 854 &put_continuation,
995 0, 855 client);
996 false,
997 &dm->key,
998 &vhash,
999 ntohl (dm->type),
1000 &check_present,
1001 pc);
1002 GNUNET_SERVICE_client_continue (client);
1003 return;
1004 }
1005 execute_put (pc);
1006 GNUNET_SERVICE_client_continue (client); 856 GNUNET_SERVICE_client_continue (client);
1007} 857}
1008 858
@@ -1030,7 +880,6 @@ handle_get (void *cls,
1030 GNUNET_ntohll (msg->next_uid), 880 GNUNET_ntohll (msg->next_uid),
1031 msg->random, 881 msg->random,
1032 NULL, 882 NULL,
1033 NULL,
1034 ntohl (msg->type), 883 ntohl (msg->type),
1035 &transmit_item, 884 &transmit_item,
1036 client); 885 client);
@@ -1082,7 +931,6 @@ handle_get_key (void *cls,
1082 GNUNET_ntohll (msg->next_uid), 931 GNUNET_ntohll (msg->next_uid),
1083 msg->random, 932 msg->random,
1084 &msg->key, 933 &msg->key,
1085 NULL,
1086 ntohl (msg->type), 934 ntohl (msg->type),
1087 &transmit_item, 935 &transmit_item,
1088 client); 936 client);
@@ -1151,50 +999,46 @@ handle_get_zero_anonymity (void *cls,
1151 999
1152 1000
1153/** 1001/**
1154 * Callback function that will cause the item that is passed 1002 * Remove continuation.
1155 * in to be deleted (by returning #GNUNET_NO).
1156 * 1003 *
1157 * @param cls closure 1004 * @param cls closure
1158 * @param key key for the content 1005 * @param key key for the content
1159 * @param size number of bytes in data 1006 * @param size number of bytes in data
1160 * @param data content stored 1007 * @param status #GNUNET_OK if removed, #GNUNET_NO if not found,
1161 * @param type type of the content 1008 * or #GNUNET_SYSERROR if error
1162 * @param priority priority of the content 1009 * @param msg error message on error
1163 * @param anonymity anonymity-level for the content
1164 * @param replication replication-level for the content
1165 * @param expiration expiration time for the content
1166 * @param uid unique identifier for the datum
1167 * @return #GNUNET_OK to keep the item
1168 * #GNUNET_NO to delete the item
1169 */ 1010 */
1170static int 1011static void
1171remove_callback (void *cls, 1012remove_continuation (void *cls,
1172 const struct GNUNET_HashCode *key, 1013 const struct GNUNET_HashCode *key,
1173 uint32_t size, 1014 uint32_t size,
1174 const void *data, 1015 int status,
1175 enum GNUNET_BLOCK_Type type, 1016 const char *msg)
1176 uint32_t priority,
1177 uint32_t anonymity,
1178 uint32_t replication,
1179 struct GNUNET_TIME_Absolute expiration,
1180 uint64_t uid)
1181{ 1017{
1182 struct GNUNET_SERVICE_Client *client = cls; 1018 struct GNUNET_SERVICE_Client *client = cls;
1183 1019
1184 if (NULL == key) 1020 if (GNUNET_SYSERR == status)
1021 {
1022 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1023 "REMOVE request failed: %s.\n",
1024 msg);
1025 transmit_status (client,
1026 GNUNET_NO,
1027 msg);
1028 return;
1029 }
1030 if (GNUNET_NO == status)
1185 { 1031 {
1186 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1032 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1187 "No further matches for REMOVE request.\n"); 1033 "Content not found for REMOVE request.\n");
1188 transmit_status (client, 1034 transmit_status (client,
1189 GNUNET_NO, 1035 GNUNET_NO,
1190 _("Content not found")); 1036 _("Content not found"));
1191 return GNUNET_OK; /* last item */ 1037 return;
1192 } 1038 }
1193 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1039 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1194 "Item %llu matches REMOVE request for key `%s' and type %u.\n", 1040 "Item matches REMOVE request for key `%s'.\n",
1195 (unsigned long long) uid, 1041 GNUNET_h2s (key));
1196 GNUNET_h2s (key),
1197 type);
1198 GNUNET_STATISTICS_update (stats, 1042 GNUNET_STATISTICS_update (stats,
1199 gettext_noop ("# bytes removed (explicit request)"), 1043 gettext_noop ("# bytes removed (explicit request)"),
1200 size, 1044 size,
@@ -1204,7 +1048,6 @@ remove_callback (void *cls,
1204 transmit_status (client, 1048 transmit_status (client,
1205 GNUNET_OK, 1049 GNUNET_OK,
1206 NULL); 1050 NULL);
1207 return GNUNET_NO;
1208} 1051}
1209 1052
1210 1053
@@ -1240,26 +1083,19 @@ handle_remove (void *cls,
1240 const struct DataMessage *dm) 1083 const struct DataMessage *dm)
1241{ 1084{
1242 struct GNUNET_SERVICE_Client *client = cls; 1085 struct GNUNET_SERVICE_Client *client = cls;
1243 struct GNUNET_HashCode vhash;
1244 1086
1245 GNUNET_STATISTICS_update (stats, 1087 GNUNET_STATISTICS_update (stats,
1246 gettext_noop ("# REMOVE requests received"), 1088 gettext_noop ("# REMOVE requests received"),
1247 1, GNUNET_NO); 1089 1, GNUNET_NO);
1248 GNUNET_CRYPTO_hash (&dm[1],
1249 ntohl (dm->size),
1250 &vhash);
1251 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1090 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1252 "Processing REMOVE request for `%s' of type %u\n", 1091 "Processing REMOVE request for `%s'\n",
1253 GNUNET_h2s (&dm->key), 1092 GNUNET_h2s (&dm->key));
1254 (uint32_t) ntohl (dm->type)); 1093 plugin->api->remove_key (plugin->api->cls,
1255 plugin->api->get_key (plugin->api->cls, 1094 &dm->key,
1256 0, 1095 ntohl (dm->size),
1257 false, 1096 &dm[1],
1258 &dm->key, 1097 &remove_continuation,
1259 &vhash, 1098 client);
1260 (enum GNUNET_BLOCK_Type) ntohl (dm->type),
1261 &remove_callback,
1262 client);
1263 GNUNET_SERVICE_client_continue (client); 1099 GNUNET_SERVICE_client_continue (client);
1264} 1100}
1265 1101
diff --git a/src/datastore/perf_plugin_datastore.c b/src/datastore/perf_plugin_datastore.c
index 2f9502989..d6f44bf9f 100644
--- a/src/datastore/perf_plugin_datastore.c
+++ b/src/datastore/perf_plugin_datastore.c
@@ -181,8 +181,14 @@ do_put (struct CpsRunContext *crc)
181 value[0] = crc->i; 181 value[0] = crc->i;
182 GNUNET_memcpy (&value[4], &i, sizeof (i)); 182 GNUNET_memcpy (&value[4], &i, sizeof (i));
183 prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100); 183 prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100);
184 crc->api->put (crc->api->cls, &key, size, value, 1 + i % 4 /* type */ , 184 crc->api->put (crc->api->cls,
185 prio, i % 4 /* anonymity */ , 185 &key,
186 false /* absent */,
187 size,
188 value,
189 1 + i % 4 /* type */ ,
190 prio,
191 i % 4 /* anonymity */ ,
186 0 /* replication */ , 192 0 /* replication */ ,
187 GNUNET_TIME_relative_to_absolute 193 GNUNET_TIME_relative_to_absolute
188 (GNUNET_TIME_relative_multiply 194 (GNUNET_TIME_relative_multiply
@@ -190,7 +196,8 @@ do_put (struct CpsRunContext *crc)
190 60 * 60 * 60 * 1000 + 196 60 * 60 * 60 * 1000 +
191 GNUNET_CRYPTO_random_u32 197 GNUNET_CRYPTO_random_u32
192 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), 198 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
193 put_continuation, crc); 199 put_continuation,
200 crc);
194 i++; 201 i++;
195} 202}
196 203
diff --git a/src/datastore/plugin_datastore_heap.c b/src/datastore/plugin_datastore_heap.c
index d04c1cf60..9950f7ab2 100644
--- a/src/datastore/plugin_datastore_heap.c
+++ b/src/datastore/plugin_datastore_heap.c
@@ -195,10 +195,89 @@ heap_plugin_estimate_size (void *cls, unsigned long long *estimate)
195 195
196 196
197/** 197/**
198 * Closure for iterator for updating.
199 */
200struct UpdateContext
201{
202 /**
203 * Number of bytes in 'data'.
204 */
205 uint32_t size;
206
207 /**
208 * Pointer to the data.
209 */
210 const void *data;
211
212 /**
213 * Priority of the value.
214 */
215 uint32_t priority;
216
217 /**
218 * Replication level for the value.
219 */
220 uint32_t replication;
221
222 /**
223 * Expiration time for this value.
224 */
225 struct GNUNET_TIME_Absolute expiration;
226
227 /**
228 * True if the value was found and updated.
229 */
230 bool updated;
231};
232
233
234/**
235 * Update the matching value.
236 *
237 * @param cls the 'struct UpdateContext'
238 * @param key unused
239 * @param val the 'struct Value'
240 * @return GNUNET_YES (continue iteration), GNUNET_NO if value was found
241 */
242static int
243update_iterator (void *cls,
244 const struct GNUNET_HashCode *key,
245 void *val)
246{
247 struct UpdateContext *uc = cls;
248 struct Value *value = val;
249
250 if (value->size != uc->size)
251 return GNUNET_YES;
252 if (0 != memcmp (value->data, uc->data, uc->size))
253 return GNUNET_YES;
254 uc->expiration = GNUNET_TIME_absolute_max (value->expiration,
255 uc->expiration);
256 if (value->expiration.abs_value_us != uc->expiration.abs_value_us)
257 {
258 value->expiration = uc->expiration;
259 GNUNET_CONTAINER_heap_update_cost (value->expire_heap,
260 value->expiration.abs_value_us);
261 }
262 /* Saturating adds, don't overflow */
263 if (value->priority > UINT32_MAX - uc->priority)
264 value->priority = UINT32_MAX;
265 else
266 value->priority += uc->priority;
267 if (value->replication > UINT32_MAX - uc->replication)
268 value->replication = UINT32_MAX;
269 else
270 value->replication += uc->replication;
271 uc->updated = true;
272 return GNUNET_NO;
273}
274
275/**
198 * Store an item in the datastore. 276 * Store an item in the datastore.
199 * 277 *
200 * @param cls closure 278 * @param cls closure
201 * @param key key for the item 279 * @param key key for the item
280 * @param absent true if the key was not found in the bloom filter
202 * @param size number of bytes in data 281 * @param size number of bytes in data
203 * @param data content stored 282 * @param data content stored
204 * @param type type of the content 283 * @param type type of the content
@@ -211,19 +290,40 @@ heap_plugin_estimate_size (void *cls, unsigned long long *estimate)
211 */ 290 */
212static void 291static void
213heap_plugin_put (void *cls, 292heap_plugin_put (void *cls,
214 const struct GNUNET_HashCode * key, 293 const struct GNUNET_HashCode *key,
215 uint32_t size, 294 bool absent,
216 const void *data, 295 uint32_t size,
217 enum GNUNET_BLOCK_Type type, 296 const void *data,
218 uint32_t priority, uint32_t anonymity, 297 enum GNUNET_BLOCK_Type type,
219 uint32_t replication, 298 uint32_t priority,
220 struct GNUNET_TIME_Absolute expiration, 299 uint32_t anonymity,
221 PluginPutCont cont, 300 uint32_t replication,
222 void *cont_cls) 301 struct GNUNET_TIME_Absolute expiration,
302 PluginPutCont cont,
303 void *cont_cls)
223{ 304{
224 struct Plugin *plugin = cls; 305 struct Plugin *plugin = cls;
225 struct Value *value; 306 struct Value *value;
226 307
308 if (!absent) {
309 struct UpdateContext uc;
310
311 uc.size = size;
312 uc.data = data;
313 uc.priority = priority;
314 uc.replication = replication;
315 uc.expiration = expiration;
316 uc.updated = false;
317 GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
318 key,
319 &update_iterator,
320 &uc);
321 if (uc.updated)
322 {
323 cont (cont_cls, key, size, GNUNET_NO, NULL);
324 return;
325 }
326 }
227 value = GNUNET_malloc (sizeof (struct Value) + size); 327 value = GNUNET_malloc (sizeof (struct Value) + size);
228 value->key = *key; 328 value->key = *key;
229 value->data = &value[1]; 329 value->data = &value[1];
@@ -333,11 +433,6 @@ struct GetContext
333 struct Value *value; 433 struct Value *value;
334 434
335 /** 435 /**
336 * Requested value hash.
337 */
338 const struct GNUNET_HashCode *vhash;
339
340 /**
341 * Requested type. 436 * Requested type.
342 */ 437 */
343 enum GNUNET_BLOCK_Type type; 438 enum GNUNET_BLOCK_Type type;
@@ -365,17 +460,10 @@ get_iterator (void *cls,
365{ 460{
366 struct GetContext *gc = cls; 461 struct GetContext *gc = cls;
367 struct Value *value = val; 462 struct Value *value = val;
368 struct GNUNET_HashCode vh;
369 463
370 if ( (gc->type != GNUNET_BLOCK_TYPE_ANY) && 464 if ( (gc->type != GNUNET_BLOCK_TYPE_ANY) &&
371 (gc->type != value->type) ) 465 (gc->type != value->type) )
372 return GNUNET_OK; 466 return GNUNET_OK;
373 if (NULL != gc->vhash)
374 {
375 GNUNET_CRYPTO_hash (&value[1], value->size, &vh);
376 if (0 != memcmp (&vh, gc->vhash, sizeof (struct GNUNET_HashCode)))
377 return GNUNET_OK;
378 }
379 if (gc->random) 467 if (gc->random)
380 { 468 {
381 gc->value = value; 469 gc->value = value;
@@ -398,23 +486,20 @@ get_iterator (void *cls,
398 * @param next_uid return the result with lowest uid >= next_uid 486 * @param next_uid return the result with lowest uid >= next_uid
399 * @param random if true, return a random result instead of using next_uid 487 * @param random if true, return a random result instead of using next_uid
400 * @param key maybe NULL (to match all entries) 488 * @param key maybe NULL (to match all entries)
401 * @param vhash hash of the value, maybe NULL (to
402 * match all values that have the right key).
403 * Note that for DBlocks there is no difference
404 * betwen key and vhash, but for other blocks
405 * there may be!
406 * @param type entries of which type are relevant? 489 * @param type entries of which type are relevant?
407 * Use 0 for any type. 490 * Use 0 for any type.
408 * @param proc function to call on each matching value; 491 * @param proc function to call on the matching value;
409 * will be called with NULL if nothing matches 492 * will be called with NULL if nothing matches
410 * @param proc_cls closure for proc 493 * @param proc_cls closure for @a proc
411 */ 494 */
412static void 495static void
413heap_plugin_get_key (void *cls, uint64_t next_uid, bool random, 496heap_plugin_get_key (void *cls,
414 const struct GNUNET_HashCode *key, 497 uint64_t next_uid,
415 const struct GNUNET_HashCode *vhash, 498 bool random,
416 enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, 499 const struct GNUNET_HashCode *key,
417 void *proc_cls) 500 enum GNUNET_BLOCK_Type type,
501 PluginDatumProcessor proc,
502 void *proc_cls)
418{ 503{
419 struct Plugin *plugin = cls; 504 struct Plugin *plugin = cls;
420 struct GetContext gc; 505 struct GetContext gc;
@@ -422,7 +507,6 @@ heap_plugin_get_key (void *cls, uint64_t next_uid, bool random,
422 gc.value = NULL; 507 gc.value = NULL;
423 gc.next_uid = next_uid; 508 gc.next_uid = next_uid;
424 gc.random = random; 509 gc.random = random;
425 gc.vhash = vhash;
426 gc.type = type; 510 gc.type = type;
427 if (NULL == key) 511 if (NULL == key)
428 { 512 {
@@ -442,20 +526,17 @@ heap_plugin_get_key (void *cls, uint64_t next_uid, bool random,
442 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); 526 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
443 return; 527 return;
444 } 528 }
445 if (GNUNET_NO == 529 GNUNET_assert (GNUNET_OK ==
446 proc (proc_cls, 530 proc (proc_cls,
447 &gc.value->key, 531 &gc.value->key,
448 gc.value->size, 532 gc.value->size,
449 &gc.value[1], 533 &gc.value[1],
450 gc.value->type, 534 gc.value->type,
451 gc.value->priority, 535 gc.value->priority,
452 gc.value->anonymity, 536 gc.value->anonymity,
453 gc.value->replication, 537 gc.value->replication,
454 gc.value->expiration, 538 gc.value->expiration,
455 (uint64_t) (intptr_t) gc.value)) 539 (uint64_t) (intptr_t) gc.value));
456 {
457 delete_value (plugin, gc.value);
458 }
459} 540}
460 541
461 542
@@ -499,18 +580,17 @@ heap_plugin_get_replication (void *cls,
499 value->replication); 580 value->replication);
500 value = GNUNET_CONTAINER_heap_walk_get_next (plugin->by_replication); 581 value = GNUNET_CONTAINER_heap_walk_get_next (plugin->by_replication);
501 } 582 }
502 if (GNUNET_NO == 583 GNUNET_assert (GNUNET_OK ==
503 proc (proc_cls, 584 proc (proc_cls,
504 &value->key, 585 &value->key,
505 value->size, 586 value->size,
506 &value[1], 587 &value[1],
507 value->type, 588 value->type,
508 value->priority, 589 value->priority,
509 value->anonymity, 590 value->anonymity,
510 value->replication, 591 value->replication,
511 value->expiration, 592 value->expiration,
512 (uint64_t) (intptr_t) value)) 593 (uint64_t) (intptr_t) value));
513 delete_value (plugin, value);
514} 594}
515 595
516 596
@@ -551,57 +631,6 @@ heap_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
551 631
552 632
553/** 633/**
554 * Update the priority, replication and expiration for a particular
555 * unique ID in the datastore. If the expiration time in value is
556 * different than the time found in the datastore, the higher value
557 * should be kept. The specified priority and replication is added
558 * to the existing value.
559 *
560 * @param cls our `struct Plugin *`
561 * @param uid unique identifier of the datum
562 * @param priority by how much should the priority
563 * change?
564 * @param replication by how much should the replication
565 * change?
566 * @param expire new expiration time should be the
567 * MAX of any existing expiration time and
568 * this value
569 * @param cont continuation called with success or failure status
570 * @param cons_cls continuation closure
571 */
572static void
573heap_plugin_update (void *cls,
574 uint64_t uid,
575 uint32_t priority,
576 uint32_t replication,
577 struct GNUNET_TIME_Absolute expire,
578 PluginUpdateCont cont,
579 void *cont_cls)
580{
581 struct Value *value;
582
583 value = (struct Value*) (intptr_t) uid;
584 GNUNET_assert (NULL != value);
585 if (value->expiration.abs_value_us != expire.abs_value_us)
586 {
587 value->expiration = expire;
588 GNUNET_CONTAINER_heap_update_cost (value->expire_heap,
589 expire.abs_value_us);
590 }
591 /* Saturating adds, don't overflow */
592 if (value->priority > UINT32_MAX - priority)
593 value->priority = UINT32_MAX;
594 else
595 value->priority += priority;
596 if (value->replication > UINT32_MAX - replication)
597 value->replication = UINT32_MAX;
598 else
599 value->replication += replication;
600 cont (cont_cls, GNUNET_OK, NULL);
601}
602
603
604/**
605 * Call the given processor on an item with zero anonymity. 634 * Call the given processor on an item with zero anonymity.
606 * 635 *
607 * @param cls our "struct Plugin*" 636 * @param cls our "struct Plugin*"
@@ -641,18 +670,17 @@ heap_plugin_get_zero_anonymity (void *cls, uint64_t next_uid,
641 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); 670 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
642 return; 671 return;
643 } 672 }
644 if (GNUNET_NO == 673 GNUNET_assert (GNUNET_OK ==
645 proc (proc_cls, 674 proc (proc_cls,
646 &value->key, 675 &value->key,
647 value->size, 676 value->size,
648 &value[1], 677 &value[1],
649 value->type, 678 value->type,
650 value->priority, 679 value->priority,
651 value->anonymity, 680 value->anonymity,
652 value->replication, 681 value->replication,
653 value->expiration, 682 value->expiration,
654 (uint64_t) (intptr_t) value)) 683 (uint64_t) (intptr_t) value));
655 delete_value (plugin, value);
656} 684}
657 685
658 686
@@ -730,6 +758,102 @@ heap_get_keys (void *cls,
730 758
731 759
732/** 760/**
761 * Closure for iterator called during 'remove_key'.
762 */
763struct RemoveContext
764{
765
766 /**
767 * Value found.
768 */
769 struct Value *value;
770
771 /**
772 * Size of data.
773 */
774 uint32_t size;
775
776 /**
777 * Data to remove.
778 */
779 const void *data;
780
781};
782
783
784/**
785 * Obtain the matching value with the lowest uid >= next_uid.
786 *
787 * @param cls the 'struct GetContext'
788 * @param key unused
789 * @param val the 'struct Value'
790 * @return GNUNET_YES (continue iteration), GNUNET_NO if result was found
791 */
792static int
793remove_iterator (void *cls,
794 const struct GNUNET_HashCode *key,
795 void *val)
796{
797 struct RemoveContext *rc = cls;
798 struct Value *value = val;
799
800 if (value->size != rc->size)
801 return GNUNET_YES;
802 if (0 != memcmp (value->data, rc->data, rc->size))
803 return GNUNET_YES;
804 rc->value = value;
805 return GNUNET_NO;
806}
807
808
809/**
810 * Remove a particular key in the datastore.
811 *
812 * @param cls closure
813 * @param key key for the content
814 * @param size number of bytes in data
815 * @param data content stored
816 * @param cont continuation called with success or failure status
817 * @param cont_cls continuation closure for @a cont
818 */
819static void
820heap_plugin_remove_key (void *cls,
821 const struct GNUNET_HashCode *key,
822 uint32_t size,
823 const void *data,
824 PluginRemoveCont cont,
825 void *cont_cls)
826{
827 struct Plugin *plugin = cls;
828 struct RemoveContext rc;
829
830 rc.value = NULL;
831 rc.size = size;
832 rc.data = data;
833 GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
834 key,
835 &remove_iterator,
836 &rc);
837 if (NULL == rc.value)
838 {
839 cont (cont_cls,
840 key,
841 size,
842 GNUNET_NO,
843 NULL);
844 return;
845 }
846 delete_value (plugin,
847 rc.value);
848 cont (cont_cls,
849 key,
850 size,
851 GNUNET_OK,
852 NULL);
853}
854
855
856/**
733 * Entry point for the plugin. 857 * Entry point for the plugin.
734 * 858 *
735 * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*" 859 * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*"
@@ -758,13 +882,13 @@ libgnunet_plugin_datastore_heap_init (void *cls)
758 api->cls = plugin; 882 api->cls = plugin;
759 api->estimate_size = &heap_plugin_estimate_size; 883 api->estimate_size = &heap_plugin_estimate_size;
760 api->put = &heap_plugin_put; 884 api->put = &heap_plugin_put;
761 api->update = &heap_plugin_update;
762 api->get_key = &heap_plugin_get_key; 885 api->get_key = &heap_plugin_get_key;
763 api->get_replication = &heap_plugin_get_replication; 886 api->get_replication = &heap_plugin_get_replication;
764 api->get_expiration = &heap_plugin_get_expiration; 887 api->get_expiration = &heap_plugin_get_expiration;
765 api->get_zero_anonymity = &heap_plugin_get_zero_anonymity; 888 api->get_zero_anonymity = &heap_plugin_get_zero_anonymity;
766 api->drop = &heap_plugin_drop; 889 api->drop = &heap_plugin_drop;
767 api->get_keys = &heap_get_keys; 890 api->get_keys = &heap_get_keys;
891 api->remove_key = &heap_plugin_remove_key;
768 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "heap", 892 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "heap",
769 _("Heap database running\n")); 893 _("Heap database running\n"));
770 return api; 894 return api;
diff --git a/src/datastore/plugin_datastore_mysql.c b/src/datastore/plugin_datastore_mysql.c
index 6f2a76499..708e35860 100644
--- a/src/datastore/plugin_datastore_mysql.c
+++ b/src/datastore/plugin_datastore_mysql.c
@@ -150,6 +150,12 @@ struct Plugin
150#define DELETE_ENTRY_BY_UID "DELETE FROM gn090 WHERE uid=?" 150#define DELETE_ENTRY_BY_UID "DELETE FROM gn090 WHERE uid=?"
151 struct GNUNET_MYSQL_StatementHandle *delete_entry_by_uid; 151 struct GNUNET_MYSQL_StatementHandle *delete_entry_by_uid;
152 152
153#define DELETE_ENTRY_BY_HASH_VALUE "DELETE FROM gn090 "\
154 "WHERE hash = ? AND "\
155 "value = ? "\
156 "LIMIT 1"
157 struct GNUNET_MYSQL_StatementHandle *delete_entry_by_hash_value;
158
153#define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, uid" 159#define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, uid"
154 160
155#define SELECT_ENTRY "SELECT " RESULT_COLUMNS " FROM gn090 "\ 161#define SELECT_ENTRY "SELECT " RESULT_COLUMNS " FROM gn090 "\
@@ -159,22 +165,13 @@ struct Plugin
159 struct GNUNET_MYSQL_StatementHandle *select_entry; 165 struct GNUNET_MYSQL_StatementHandle *select_entry;
160 166
161#define SELECT_ENTRY_BY_HASH "SELECT " RESULT_COLUMNS " FROM gn090 "\ 167#define SELECT_ENTRY_BY_HASH "SELECT " RESULT_COLUMNS " FROM gn090 "\
162 "FORCE INDEX (idx_hash) "\ 168 "FORCE INDEX (idx_hash_type_uid) "\
163 "WHERE hash=? AND "\ 169 "WHERE hash=? AND "\
164 "uid >= ? AND "\ 170 "uid >= ? AND "\
165 "(rvalue >= ? OR 0 = ?) "\ 171 "(rvalue >= ? OR 0 = ?) "\
166 "ORDER BY uid LIMIT 1" 172 "ORDER BY uid LIMIT 1"
167 struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash; 173 struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash;
168 174
169#define SELECT_ENTRY_BY_HASH_AND_VHASH "SELECT " RESULT_COLUMNS " FROM gn090 "\
170 "FORCE INDEX (idx_hash_vhash) "\
171 "WHERE hash = ? AND "\
172 "vhash = ? AND "\
173 "uid >= ? AND "\
174 "(rvalue >= ? OR 0 = ?) "\
175 "ORDER BY uid LIMIT 1"
176 struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_and_vhash;
177
178#define SELECT_ENTRY_BY_HASH_AND_TYPE "SELECT " RESULT_COLUMNS " FROM gn090 "\ 175#define SELECT_ENTRY_BY_HASH_AND_TYPE "SELECT " RESULT_COLUMNS " FROM gn090 "\
179 "FORCE INDEX (idx_hash_type_uid) "\ 176 "FORCE INDEX (idx_hash_type_uid) "\
180 "WHERE hash = ? AND "\ 177 "WHERE hash = ? AND "\
@@ -184,22 +181,11 @@ struct Plugin
184 "ORDER BY uid LIMIT 1" 181 "ORDER BY uid LIMIT 1"
185 struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_and_type; 182 struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_and_type;
186 183
187#define SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT " RESULT_COLUMNS " "\
188 "FROM gn090 "\
189 "FORCE INDEX (idx_hash_vhash) "\
190 "WHERE hash = ? AND "\
191 "vhash = ? AND "\
192 "type = ? AND "\
193 "uid >= ? AND "\
194 "(rvalue >= ? OR 0 = ?) "\
195 "ORDER BY uid LIMIT 1"
196 struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_vhash_and_type;
197
198#define UPDATE_ENTRY "UPDATE gn090 SET "\ 184#define UPDATE_ENTRY "UPDATE gn090 SET "\
199 "prio = prio + ?, "\ 185 "prio = prio + ?, "\
200 "repl = repl + ?, "\ 186 "repl = repl + ?, "\
201 "expire = IF(expire >= ?, expire, ?) "\ 187 "expire = GREATEST(expire, ?) "\
202 "WHERE uid = ?" 188 "WHERE hash = ? AND vhash = ?"
203 struct GNUNET_MYSQL_StatementHandle *update_entry; 189 struct GNUNET_MYSQL_StatementHandle *update_entry;
204 190
205#define DEC_REPL "UPDATE gn090 SET repl=GREATEST (1, repl) - 1 WHERE uid=?" 191#define DEC_REPL "UPDATE gn090 SET repl=GREATEST (1, repl) - 1 WHERE uid=?"
@@ -330,6 +316,7 @@ mysql_plugin_estimate_size (void *cls,
330 * 316 *
331 * @param cls closure 317 * @param cls closure
332 * @param key key for the item 318 * @param key key for the item
319 * @param absent true if the key was not found in the bloom filter
333 * @param size number of bytes in @a data 320 * @param size number of bytes in @a data
334 * @param data content stored 321 * @param data content stored
335 * @param type type of the content 322 * @param type type of the content
@@ -343,6 +330,7 @@ mysql_plugin_estimate_size (void *cls,
343static void 330static void
344mysql_plugin_put (void *cls, 331mysql_plugin_put (void *cls,
345 const struct GNUNET_HashCode *key, 332 const struct GNUNET_HashCode *key,
333 bool absent,
346 uint32_t size, 334 uint32_t size,
347 const void *data, 335 const void *data,
348 enum GNUNET_BLOCK_Type type, 336 enum GNUNET_BLOCK_Type type,
@@ -355,9 +343,54 @@ mysql_plugin_put (void *cls,
355{ 343{
356 struct Plugin *plugin = cls; 344 struct Plugin *plugin = cls;
357 uint64_t lexpiration = expiration.abs_value_us; 345 uint64_t lexpiration = expiration.abs_value_us;
346 struct GNUNET_HashCode vhash;
347
348 GNUNET_CRYPTO_hash (data,
349 size,
350 &vhash);
351 if (!absent)
352 {
353 struct GNUNET_MY_QueryParam params_update[] = {
354 GNUNET_MY_query_param_uint32 (&priority),
355 GNUNET_MY_query_param_uint32 (&replication),
356 GNUNET_MY_query_param_uint64 (&lexpiration),
357 GNUNET_MY_query_param_auto_from_type (key),
358 GNUNET_MY_query_param_auto_from_type (&vhash),
359 GNUNET_MY_query_param_end
360 };
361
362 if (GNUNET_OK !=
363 GNUNET_MY_exec_prepared (plugin->mc,
364 plugin->update_entry,
365 params_update))
366 {
367 cont (cont_cls,
368 key,
369 size,
370 GNUNET_SYSERR,
371 _("MySQL statement run failure"));
372 return;
373 }
374
375 MYSQL_STMT *stmt = GNUNET_MYSQL_statement_get_stmt (plugin->update_entry);
376 my_ulonglong rows = mysql_stmt_affected_rows (stmt);
377
378 GNUNET_break (GNUNET_NO ==
379 GNUNET_MY_extract_result (plugin->update_entry,
380 NULL));
381 if (0 != rows)
382 {
383 cont (cont_cls,
384 key,
385 size,
386 GNUNET_NO,
387 NULL);
388 return;
389 }
390 }
391
358 uint64_t lrvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, 392 uint64_t lrvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
359 UINT64_MAX); 393 UINT64_MAX);
360 struct GNUNET_HashCode vhash;
361 struct GNUNET_MY_QueryParam params_insert[] = { 394 struct GNUNET_MY_QueryParam params_insert[] = {
362 GNUNET_MY_query_param_uint32 (&replication), 395 GNUNET_MY_query_param_uint32 (&replication),
363 GNUNET_MY_query_param_uint32 (&type), 396 GNUNET_MY_query_param_uint32 (&type),
@@ -377,9 +410,6 @@ mysql_plugin_put (void *cls,
377 cont (cont_cls, key, size, GNUNET_SYSERR, _("Data too large")); 410 cont (cont_cls, key, size, GNUNET_SYSERR, _("Data too large"));
378 return; 411 return;
379 } 412 }
380 GNUNET_CRYPTO_hash (data,
381 size,
382 &vhash);
383 413
384 if (GNUNET_OK != 414 if (GNUNET_OK !=
385 GNUNET_MY_exec_prepared (plugin->mc, 415 GNUNET_MY_exec_prepared (plugin->mc,
@@ -412,76 +442,6 @@ mysql_plugin_put (void *cls,
412 442
413 443
414/** 444/**
415 * Update the priority, replication and expiration for a particular
416 * unique ID in the datastore. If the expiration time in value is
417 * different than the time found in the datastore, the higher value
418 * should be kept. The specified priority and replication is added
419 * to the existing value.
420 *
421 * @param cls our "struct Plugin*"
422 * @param uid unique identifier of the datum
423 * @param priority by how much should the priority
424 * change?
425 * @param replication by how much should the replication
426 * change?
427 * @param expire new expiration time should be the
428 * MAX of any existing expiration time and
429 * this value
430 * @param cont continuation called with success or failure status
431 * @param cons_cls continuation closure
432 */
433static void
434mysql_plugin_update (void *cls,
435 uint64_t uid,
436 uint32_t priority,
437 uint32_t replication,
438 struct GNUNET_TIME_Absolute expire,
439 PluginUpdateCont cont,
440 void *cont_cls)
441{
442 struct Plugin *plugin = cls;
443 uint64_t lexpire = expire.abs_value_us;
444 int ret;
445
446 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
447 "Updating value %llu adding %d to priority %d to replication and maxing exp at %s\n",
448 (unsigned long long) uid,
449 priority,
450 replication,
451 GNUNET_STRINGS_absolute_time_to_string (expire));
452
453 struct GNUNET_MY_QueryParam params_update[] = {
454 GNUNET_MY_query_param_uint32 (&priority),
455 GNUNET_MY_query_param_uint32 (&replication),
456 GNUNET_MY_query_param_uint64 (&lexpire),
457 GNUNET_MY_query_param_uint64 (&lexpire),
458 GNUNET_MY_query_param_uint64 (&uid),
459 GNUNET_MY_query_param_end
460 };
461
462 ret = GNUNET_MY_exec_prepared (plugin->mc,
463 plugin->update_entry,
464 params_update);
465
466 if (GNUNET_OK != ret)
467 {
468 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
469 "Failed to update value %llu\n",
470 (unsigned long long) uid);
471 }
472 else
473 {
474 GNUNET_break (GNUNET_NO ==
475 GNUNET_MY_extract_result (plugin->update_entry,
476 NULL));
477 }
478 cont (cont_cls,
479 ret,
480 NULL);
481}
482
483
484/**
485 * Run the given select statement and call 'proc' on the resulting 445 * Run the given select statement and call 'proc' on the resulting
486 * values (which must be in particular positions). 446 * values (which must be in particular positions).
487 * 447 *
@@ -578,11 +538,6 @@ execute_select (struct Plugin *plugin,
578 * @param next_uid return the result with lowest uid >= next_uid 538 * @param next_uid return the result with lowest uid >= next_uid
579 * @param random if true, return a random result instead of using next_uid 539 * @param random if true, return a random result instead of using next_uid
580 * @param key key to match, never NULL 540 * @param key key to match, never NULL
581 * @param vhash hash of the value, maybe NULL (to
582 * match all values that have the right key).
583 * Note that for DBlocks there is no difference
584 * betwen key and vhash, but for other blocks
585 * there may be!
586 * @param type entries of which type are relevant? 541 * @param type entries of which type are relevant?
587 * Use 0 for any type. 542 * Use 0 for any type.
588 * @param proc function to call on the matching value, 543 * @param proc function to call on the matching value,
@@ -594,7 +549,6 @@ mysql_plugin_get_key (void *cls,
594 uint64_t next_uid, 549 uint64_t next_uid,
595 bool random, 550 bool random,
596 const struct GNUNET_HashCode *key, 551 const struct GNUNET_HashCode *key,
597 const struct GNUNET_HashCode *vhash,
598 enum GNUNET_BLOCK_Type type, 552 enum GNUNET_BLOCK_Type type,
599 PluginDatumProcessor proc, 553 PluginDatumProcessor proc,
600 void *proc_cls) 554 void *proc_cls)
@@ -628,79 +582,37 @@ mysql_plugin_get_key (void *cls,
628 } 582 }
629 else if (type != GNUNET_BLOCK_TYPE_ANY) 583 else if (type != GNUNET_BLOCK_TYPE_ANY)
630 { 584 {
631 if (NULL != vhash) 585 struct GNUNET_MY_QueryParam params_select[] = {
632 { 586 GNUNET_MY_query_param_auto_from_type (key),
633 struct GNUNET_MY_QueryParam params_select[] = { 587 GNUNET_MY_query_param_uint32 (&type),
634 GNUNET_MY_query_param_auto_from_type (key), 588 GNUNET_MY_query_param_uint64 (&next_uid),
635 GNUNET_MY_query_param_auto_from_type (vhash), 589 GNUNET_MY_query_param_uint64 (&rvalue),
636 GNUNET_MY_query_param_uint32 (&type), 590 GNUNET_MY_query_param_uint64 (&rvalue),
637 GNUNET_MY_query_param_uint64 (&next_uid), 591 GNUNET_MY_query_param_end
638 GNUNET_MY_query_param_uint64 (&rvalue), 592 };
639 GNUNET_MY_query_param_uint64 (&rvalue), 593
640 GNUNET_MY_query_param_end 594 execute_select (plugin,
641 }; 595 plugin->select_entry_by_hash_and_type,
642 596 proc,
643 execute_select (plugin, 597 proc_cls,
644 plugin->select_entry_by_hash_vhash_and_type, 598 params_select);
645 proc,
646 proc_cls,
647 params_select);
648 }
649 else
650 {
651 struct GNUNET_MY_QueryParam params_select[] = {
652 GNUNET_MY_query_param_auto_from_type (key),
653 GNUNET_MY_query_param_uint32 (&type),
654 GNUNET_MY_query_param_uint64 (&next_uid),
655 GNUNET_MY_query_param_uint64 (&rvalue),
656 GNUNET_MY_query_param_uint64 (&rvalue),
657 GNUNET_MY_query_param_end
658 };
659
660 execute_select (plugin,
661 plugin->select_entry_by_hash_and_type,
662 proc,
663 proc_cls,
664 params_select);
665 }
666 } 599 }
667 else 600 else
668 { 601 {
669 if (NULL != vhash) 602 struct GNUNET_MY_QueryParam params_select[] = {
670 { 603 GNUNET_MY_query_param_auto_from_type (key),
671 struct GNUNET_MY_QueryParam params_select[] = { 604 GNUNET_MY_query_param_uint64 (&next_uid),
672 GNUNET_MY_query_param_auto_from_type (key), 605 GNUNET_MY_query_param_uint64 (&rvalue),
673 GNUNET_MY_query_param_auto_from_type (vhash), 606 GNUNET_MY_query_param_uint64 (&rvalue),
674 GNUNET_MY_query_param_uint64 (&next_uid), 607 GNUNET_MY_query_param_end
675 GNUNET_MY_query_param_uint64 (&rvalue), 608 };
676 GNUNET_MY_query_param_uint64 (&rvalue),
677 GNUNET_MY_query_param_end
678 };
679
680 execute_select (plugin,
681 plugin->select_entry_by_hash_and_vhash,
682 proc,
683 proc_cls,
684 params_select);
685 }
686 else
687 {
688 struct GNUNET_MY_QueryParam params_select[] = {
689 GNUNET_MY_query_param_auto_from_type (key),
690 GNUNET_MY_query_param_uint64 (&next_uid),
691 GNUNET_MY_query_param_uint64 (&rvalue),
692 GNUNET_MY_query_param_uint64 (&rvalue),
693 GNUNET_MY_query_param_end
694 };
695
696 execute_select (plugin,
697 plugin->select_entry_by_hash,
698 proc,
699 proc_cls,
700 params_select);
701 }
702 }
703 609
610 execute_select (plugin,
611 plugin->select_entry_by_hash,
612 proc,
613 proc_cls,
614 params_select);
615 }
704} 616}
705 617
706 618
@@ -1124,6 +1036,69 @@ mysql_plugin_drop (void *cls)
1124 1036
1125 1037
1126/** 1038/**
1039 * Remove a particular key in the datastore.
1040 *
1041 * @param cls closure
1042 * @param key key for the content
1043 * @param size number of bytes in data
1044 * @param data content stored
1045 * @param cont continuation called with success or failure status
1046 * @param cont_cls continuation closure for @a cont
1047 */
1048static void
1049mysql_plugin_remove_key (void *cls,
1050 const struct GNUNET_HashCode *key,
1051 uint32_t size,
1052 const void *data,
1053 PluginRemoveCont cont,
1054 void *cont_cls)
1055{
1056 struct Plugin *plugin = cls;
1057 struct GNUNET_MY_QueryParam params_delete[] = {
1058 GNUNET_MY_query_param_auto_from_type (key),
1059 GNUNET_MY_query_param_fixed_size (data, size),
1060 GNUNET_MY_query_param_end
1061 };
1062
1063 if (GNUNET_OK !=
1064 GNUNET_MY_exec_prepared (plugin->mc,
1065 plugin->delete_entry_by_hash_value,
1066 params_delete))
1067 {
1068 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1069 "Removing key `%s' from gn090 table failed\n",
1070 GNUNET_h2s (key));
1071 cont (cont_cls,
1072 key,
1073 size,
1074 GNUNET_SYSERR,
1075 _("MySQL statement run failure"));
1076 return;
1077 }
1078
1079 MYSQL_STMT *stmt = GNUNET_MYSQL_statement_get_stmt (plugin->delete_entry_by_hash_value);
1080 my_ulonglong rows = mysql_stmt_affected_rows (stmt);
1081
1082 if (0 == rows)
1083 {
1084 cont (cont_cls,
1085 key,
1086 size,
1087 GNUNET_NO,
1088 NULL);
1089 return;
1090 }
1091 plugin->env->duc (plugin->env->cls,
1092 -size);
1093 cont (cont_cls,
1094 key,
1095 size,
1096 GNUNET_OK,
1097 NULL);
1098}
1099
1100
1101/**
1127 * Entry point for the plugin. 1102 * Entry point for the plugin.
1128 * 1103 *
1129 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment *` 1104 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment *`
@@ -1158,24 +1133,20 @@ libgnunet_plugin_datastore_mysql_init (void *cls)
1158 " hash BINARY(64) NOT NULL DEFAULT ''," 1133 " hash BINARY(64) NOT NULL DEFAULT '',"
1159 " vhash BINARY(64) NOT NULL DEFAULT ''," 1134 " vhash BINARY(64) NOT NULL DEFAULT '',"
1160 " value BLOB NOT NULL DEFAULT ''," " uid BIGINT NOT NULL AUTO_INCREMENT," 1135 " value BLOB NOT NULL DEFAULT ''," " uid BIGINT NOT NULL AUTO_INCREMENT,"
1161 " PRIMARY KEY (uid)," " INDEX idx_hash (hash(64))," 1136 " PRIMARY KEY (uid),"
1162 " INDEX idx_hash_uid (hash(64),uid),"
1163 " INDEX idx_hash_vhash (hash(64),vhash(64)),"
1164 " INDEX idx_hash_type_uid (hash(64),type,rvalue)," 1137 " INDEX idx_hash_type_uid (hash(64),type,rvalue),"
1165 " INDEX idx_prio (prio)," " INDEX idx_repl_rvalue (repl,rvalue)," 1138 " INDEX idx_prio (prio),"
1139 " INDEX idx_repl_rvalue (repl,rvalue),"
1166 " INDEX idx_expire (expire)," 1140 " INDEX idx_expire (expire),"
1167 " INDEX idx_anonLevel_type_rvalue (anonLevel,type,rvalue)" 1141 " INDEX idx_anonLevel_type_rvalue (anonLevel,type,rvalue)"
1168 ") ENGINE=InnoDB") || MRUNS ("SET AUTOCOMMIT = 1") || 1142 ") ENGINE=InnoDB") || MRUNS ("SET AUTOCOMMIT = 1") ||
1169 PINIT (plugin->insert_entry, INSERT_ENTRY) || 1143 PINIT (plugin->insert_entry, INSERT_ENTRY) ||
1170 PINIT (plugin->delete_entry_by_uid, DELETE_ENTRY_BY_UID) || 1144 PINIT (plugin->delete_entry_by_uid, DELETE_ENTRY_BY_UID) ||
1145 PINIT (plugin->delete_entry_by_hash_value, DELETE_ENTRY_BY_HASH_VALUE) ||
1171 PINIT (plugin->select_entry, SELECT_ENTRY) || 1146 PINIT (plugin->select_entry, SELECT_ENTRY) ||
1172 PINIT (plugin->select_entry_by_hash, SELECT_ENTRY_BY_HASH) || 1147 PINIT (plugin->select_entry_by_hash, SELECT_ENTRY_BY_HASH) ||
1173 PINIT (plugin->select_entry_by_hash_and_vhash,
1174 SELECT_ENTRY_BY_HASH_AND_VHASH) ||
1175 PINIT (plugin->select_entry_by_hash_and_type, 1148 PINIT (plugin->select_entry_by_hash_and_type,
1176 SELECT_ENTRY_BY_HASH_AND_TYPE) || 1149 SELECT_ENTRY_BY_HASH_AND_TYPE) ||
1177 PINIT (plugin->select_entry_by_hash_vhash_and_type,
1178 SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE) ||
1179 PINIT (plugin->get_size, SELECT_SIZE) || 1150 PINIT (plugin->get_size, SELECT_SIZE) ||
1180 PINIT (plugin->update_entry, UPDATE_ENTRY) || 1151 PINIT (plugin->update_entry, UPDATE_ENTRY) ||
1181 PINIT (plugin->dec_repl, DEC_REPL) || 1152 PINIT (plugin->dec_repl, DEC_REPL) ||
@@ -1184,7 +1155,8 @@ libgnunet_plugin_datastore_mysql_init (void *cls)
1184 PINIT (plugin->select_priority, SELECT_IT_PRIORITY) || 1155 PINIT (plugin->select_priority, SELECT_IT_PRIORITY) ||
1185 PINIT (plugin->max_repl, SELECT_MAX_REPL) || 1156 PINIT (plugin->max_repl, SELECT_MAX_REPL) ||
1186 PINIT (plugin->get_all_keys, GET_ALL_KEYS) || 1157 PINIT (plugin->get_all_keys, GET_ALL_KEYS) ||
1187 PINIT (plugin->select_replication, SELECT_IT_REPLICATION)) 1158 PINIT (plugin->select_replication, SELECT_IT_REPLICATION) ||
1159 false)
1188 { 1160 {
1189 GNUNET_MYSQL_context_destroy (plugin->mc); 1161 GNUNET_MYSQL_context_destroy (plugin->mc);
1190 GNUNET_free (plugin); 1162 GNUNET_free (plugin);
@@ -1197,13 +1169,13 @@ libgnunet_plugin_datastore_mysql_init (void *cls)
1197 api->cls = plugin; 1169 api->cls = plugin;
1198 api->estimate_size = &mysql_plugin_estimate_size; 1170 api->estimate_size = &mysql_plugin_estimate_size;
1199 api->put = &mysql_plugin_put; 1171 api->put = &mysql_plugin_put;
1200 api->update = &mysql_plugin_update;
1201 api->get_key = &mysql_plugin_get_key; 1172 api->get_key = &mysql_plugin_get_key;
1202 api->get_replication = &mysql_plugin_get_replication; 1173 api->get_replication = &mysql_plugin_get_replication;
1203 api->get_expiration = &mysql_plugin_get_expiration; 1174 api->get_expiration = &mysql_plugin_get_expiration;
1204 api->get_zero_anonymity = &mysql_plugin_get_zero_anonymity; 1175 api->get_zero_anonymity = &mysql_plugin_get_zero_anonymity;
1205 api->get_keys = &mysql_plugin_get_keys; 1176 api->get_keys = &mysql_plugin_get_keys;
1206 api->drop = &mysql_plugin_drop; 1177 api->drop = &mysql_plugin_drop;
1178 api->remove_key = &mysql_plugin_remove_key;
1207 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "mysql", 1179 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "mysql",
1208 _("Mysql database running\n")); 1180 _("Mysql database running\n"));
1209 return api; 1181 return api;
diff --git a/src/datastore/plugin_datastore_postgres.c b/src/datastore/plugin_datastore_postgres.c
index 87a7acbdc..b6aeb0be6 100644
--- a/src/datastore/plugin_datastore_postgres.c
+++ b/src/datastore/plugin_datastore_postgres.c
@@ -119,9 +119,6 @@ init_connection (struct Plugin *plugin)
119 "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)")) || 119 "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)")) ||
120 (GNUNET_OK != 120 (GNUNET_OK !=
121 GNUNET_POSTGRES_exec (plugin->dbh, 121 GNUNET_POSTGRES_exec (plugin->dbh,
122 "CREATE INDEX IF NOT EXISTS idx_hash_vhash ON gn090 (hash,vhash)")) ||
123 (GNUNET_OK !=
124 GNUNET_POSTGRES_exec (plugin->dbh,
125 "CREATE INDEX IF NOT EXISTS idx_prio ON gn090 (prio)")) || 122 "CREATE INDEX IF NOT EXISTS idx_prio ON gn090 (prio)")) ||
126 (GNUNET_OK != 123 (GNUNET_OK !=
127 GNUNET_POSTGRES_exec (plugin->dbh, 124 GNUNET_POSTGRES_exec (plugin->dbh,
@@ -183,9 +180,8 @@ init_connection (struct Plugin *plugin)
183 "WHERE oid >= $1::bigint AND " 180 "WHERE oid >= $1::bigint AND "
184 "(rvalue >= $2 OR 0 = $3::smallint) AND " 181 "(rvalue >= $2 OR 0 = $3::smallint) AND "
185 "(hash = $4 OR 0 = $5::smallint) AND " 182 "(hash = $4 OR 0 = $5::smallint) AND "
186 "(vhash = $6 OR 0 = $7::smallint) AND " 183 "(type = $6 OR 0 = $7::smallint) "
187 "(type = $8 OR 0 = $9::smallint) " 184 "ORDER BY oid ASC LIMIT 1", 7)) ||
188 "ORDER BY oid ASC LIMIT 1", 9)) ||
189 (GNUNET_OK != 185 (GNUNET_OK !=
190 GNUNET_POSTGRES_prepare (plugin->dbh, "put", 186 GNUNET_POSTGRES_prepare (plugin->dbh, "put",
191 "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) " 187 "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
@@ -195,8 +191,8 @@ init_connection (struct Plugin *plugin)
195 "UPDATE gn090 " 191 "UPDATE gn090 "
196 "SET prio = prio + $1, " 192 "SET prio = prio + $1, "
197 "repl = repl + $2, " 193 "repl = repl + $2, "
198 "expire = CASE WHEN expire < $3 THEN $3 ELSE expire END " 194 "expire = GREATEST(expire, $3) "
199 "WHERE oid = $4", 4)) || 195 "WHERE hash = $4 AND vhash = $5", 5)) ||
200 (GNUNET_OK != 196 (GNUNET_OK !=
201 GNUNET_POSTGRES_prepare (plugin->dbh, "decrepl", 197 GNUNET_POSTGRES_prepare (plugin->dbh, "decrepl",
202 "UPDATE gn090 SET repl = GREATEST (repl - 1, 0) " 198 "UPDATE gn090 SET repl = GREATEST (repl - 1, 0) "
@@ -223,6 +219,10 @@ init_connection (struct Plugin *plugin)
223 (GNUNET_OK != 219 (GNUNET_OK !=
224 GNUNET_POSTGRES_prepare (plugin->dbh, "delrow", "DELETE FROM gn090 " "WHERE oid=$1", 1)) || 220 GNUNET_POSTGRES_prepare (plugin->dbh, "delrow", "DELETE FROM gn090 " "WHERE oid=$1", 1)) ||
225 (GNUNET_OK != 221 (GNUNET_OK !=
222 GNUNET_POSTGRES_prepare (plugin->dbh, "remove", "DELETE FROM gn090 "
223 "WHERE hash = $1 AND "
224 "value = $2", 2)) ||
225 (GNUNET_OK !=
226 GNUNET_POSTGRES_prepare (plugin->dbh, "get_keys", "SELECT hash FROM gn090", 0))) 226 GNUNET_POSTGRES_prepare (plugin->dbh, "get_keys", "SELECT hash FROM gn090", 0)))
227 { 227 {
228 PQfinish (plugin->dbh); 228 PQfinish (plugin->dbh);
@@ -288,6 +288,7 @@ postgres_plugin_estimate_size (void *cls, unsigned long long *estimate)
288 * 288 *
289 * @param cls closure with the `struct Plugin` 289 * @param cls closure with the `struct Plugin`
290 * @param key key for the item 290 * @param key key for the item
291 * @param absent true if the key was not found in the bloom filter
291 * @param size number of bytes in data 292 * @param size number of bytes in data
292 * @param data content stored 293 * @param data content stored
293 * @param type type of the content 294 * @param type type of the content
@@ -300,23 +301,70 @@ postgres_plugin_estimate_size (void *cls, unsigned long long *estimate)
300 */ 301 */
301static void 302static void
302postgres_plugin_put (void *cls, 303postgres_plugin_put (void *cls,
303 const struct GNUNET_HashCode *key, 304 const struct GNUNET_HashCode *key,
304 uint32_t size, 305 bool absent,
306 uint32_t size,
305 const void *data, 307 const void *data,
306 enum GNUNET_BLOCK_Type type, 308 enum GNUNET_BLOCK_Type type,
307 uint32_t priority, 309 uint32_t priority,
308 uint32_t anonymity, 310 uint32_t anonymity,
309 uint32_t replication, 311 uint32_t replication,
310 struct GNUNET_TIME_Absolute expiration, 312 struct GNUNET_TIME_Absolute expiration,
311 PluginPutCont cont, 313 PluginPutCont cont,
312 void *cont_cls) 314 void *cont_cls)
313{ 315{
314 struct Plugin *plugin = cls; 316 struct Plugin *plugin = cls;
315 uint32_t utype = type;
316 struct GNUNET_HashCode vhash; 317 struct GNUNET_HashCode vhash;
318 PGresult *ret;
319
320 GNUNET_CRYPTO_hash (data,
321 size,
322 &vhash);
323
324 if (!absent)
325 {
326 struct GNUNET_PQ_QueryParam params[] = {
327 GNUNET_PQ_query_param_uint32 (&priority),
328 GNUNET_PQ_query_param_uint32 (&replication),
329 GNUNET_PQ_query_param_absolute_time (&expiration),
330 GNUNET_PQ_query_param_auto_from_type (key),
331 GNUNET_PQ_query_param_auto_from_type (&vhash),
332 GNUNET_PQ_query_param_end
333 };
334 ret = GNUNET_PQ_exec_prepared (plugin->dbh,
335 "update",
336 params);
337 if (GNUNET_OK !=
338 GNUNET_POSTGRES_check_result (plugin->dbh,
339 ret,
340 PGRES_COMMAND_OK,
341 "PQexecPrepared",
342 "update"))
343 {
344 cont (cont_cls,
345 key,
346 size,
347 GNUNET_SYSERR,
348 _("Postgress exec failure"));
349 return;
350 }
351 /* What an awful API, this function really does return a string */
352 bool affected = 0 != strcmp ("0", PQcmdTuples (ret));
353 PQclear (ret);
354 if (affected)
355 {
356 cont (cont_cls,
357 key,
358 size,
359 GNUNET_NO,
360 NULL);
361 return;
362 }
363 }
364
365 uint32_t utype = type;
317 uint64_t rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, 366 uint64_t rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
318 UINT64_MAX); 367 UINT64_MAX);
319 PGresult *ret;
320 struct GNUNET_PQ_QueryParam params[] = { 368 struct GNUNET_PQ_QueryParam params[] = {
321 GNUNET_PQ_query_param_uint32 (&replication), 369 GNUNET_PQ_query_param_uint32 (&replication),
322 GNUNET_PQ_query_param_uint32 (&utype), 370 GNUNET_PQ_query_param_uint32 (&utype),
@@ -330,7 +378,6 @@ postgres_plugin_put (void *cls,
330 GNUNET_PQ_query_param_end 378 GNUNET_PQ_query_param_end
331 }; 379 };
332 380
333 GNUNET_CRYPTO_hash (data, size, &vhash);
334 ret = GNUNET_PQ_exec_prepared (plugin->dbh, 381 ret = GNUNET_PQ_exec_prepared (plugin->dbh,
335 "put", 382 "put",
336 params); 383 params);
@@ -489,11 +536,6 @@ process_result (struct Plugin *plugin,
489 * @param next_uid return the result with lowest uid >= next_uid 536 * @param next_uid return the result with lowest uid >= next_uid
490 * @param random if true, return a random result instead of using next_uid 537 * @param random if true, return a random result instead of using next_uid
491 * @param key maybe NULL (to match all entries) 538 * @param key maybe NULL (to match all entries)
492 * @param vhash hash of the value, maybe NULL (to
493 * match all values that have the right key).
494 * Note that for DBlocks there is no difference
495 * betwen key and vhash, but for other blocks
496 * there may be!
497 * @param type entries of which type are relevant? 539 * @param type entries of which type are relevant?
498 * Use 0 for any type. 540 * Use 0 for any type.
499 * @param proc function to call on the matching value; 541 * @param proc function to call on the matching value;
@@ -505,7 +547,6 @@ postgres_plugin_get_key (void *cls,
505 uint64_t next_uid, 547 uint64_t next_uid,
506 bool random, 548 bool random,
507 const struct GNUNET_HashCode *key, 549 const struct GNUNET_HashCode *key,
508 const struct GNUNET_HashCode *vhash,
509 enum GNUNET_BLOCK_Type type, 550 enum GNUNET_BLOCK_Type type,
510 PluginDatumProcessor proc, 551 PluginDatumProcessor proc,
511 void *proc_cls) 552 void *proc_cls)
@@ -514,7 +555,6 @@ postgres_plugin_get_key (void *cls,
514 uint32_t utype = type; 555 uint32_t utype = type;
515 uint16_t use_rvalue = random; 556 uint16_t use_rvalue = random;
516 uint16_t use_key = NULL != key; 557 uint16_t use_key = NULL != key;
517 uint16_t use_vhash = NULL != vhash;
518 uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type; 558 uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type;
519 uint64_t rvalue; 559 uint64_t rvalue;
520 struct GNUNET_PQ_QueryParam params[] = { 560 struct GNUNET_PQ_QueryParam params[] = {
@@ -523,8 +563,6 @@ postgres_plugin_get_key (void *cls,
523 GNUNET_PQ_query_param_uint16 (&use_rvalue), 563 GNUNET_PQ_query_param_uint16 (&use_rvalue),
524 GNUNET_PQ_query_param_auto_from_type (key), 564 GNUNET_PQ_query_param_auto_from_type (key),
525 GNUNET_PQ_query_param_uint16 (&use_key), 565 GNUNET_PQ_query_param_uint16 (&use_key),
526 GNUNET_PQ_query_param_auto_from_type (vhash),
527 GNUNET_PQ_query_param_uint16 (&use_vhash),
528 GNUNET_PQ_query_param_uint32 (&utype), 566 GNUNET_PQ_query_param_uint32 (&utype),
529 GNUNET_PQ_query_param_uint16 (&use_type), 567 GNUNET_PQ_query_param_uint16 (&use_type),
530 GNUNET_PQ_query_param_end 568 GNUNET_PQ_query_param_end
@@ -750,67 +788,6 @@ postgres_plugin_get_expiration (void *cls,
750 788
751 789
752/** 790/**
753 * Update the priority, replication and expiration for a particular
754 * unique ID in the datastore. If the expiration time in value is
755 * different than the time found in the datastore, the higher value
756 * should be kept. The specified priority and replication is added
757 * to the existing value.
758 *
759 * @param cls our `struct Plugin *`
760 * @param uid unique identifier of the datum
761 * @param priority by how much should the priority
762 * change?
763 * @param replication by how much should the replication
764 * change?
765 * @param expire new expiration time should be the
766 * MAX of any existing expiration time and
767 * this value
768 * @param cont continuation called with success or failure status
769 * @param cons_cls continuation closure
770 */
771static void
772postgres_plugin_update (void *cls,
773 uint64_t uid,
774 uint32_t priority,
775 uint32_t replication,
776 struct GNUNET_TIME_Absolute expire,
777 PluginUpdateCont cont,
778 void *cont_cls)
779{
780 struct Plugin *plugin = cls;
781 uint32_t oid = (uint32_t) uid;
782 struct GNUNET_PQ_QueryParam params[] = {
783 GNUNET_PQ_query_param_uint32 (&priority),
784 GNUNET_PQ_query_param_uint32 (&replication),
785 GNUNET_PQ_query_param_absolute_time (&expire),
786 GNUNET_PQ_query_param_uint32 (&oid),
787 GNUNET_PQ_query_param_end
788 };
789 PGresult *ret;
790
791 ret = GNUNET_PQ_exec_prepared (plugin->dbh,
792 "update",
793 params);
794 if (GNUNET_OK !=
795 GNUNET_POSTGRES_check_result (plugin->dbh,
796 ret,
797 PGRES_COMMAND_OK,
798 "PQexecPrepared",
799 "update"))
800 {
801 cont (cont_cls,
802 GNUNET_SYSERR,
803 NULL);
804 return;
805 }
806 PQclear (ret);
807 cont (cont_cls,
808 GNUNET_OK,
809 NULL);
810}
811
812
813/**
814 * Get all of the keys in the datastore. 791 * Get all of the keys in the datastore.
815 * 792 *
816 * @param cls closure with the `struct Plugin *` 793 * @param cls closure with the `struct Plugin *`
@@ -868,6 +845,74 @@ postgres_plugin_drop (void *cls)
868 845
869 846
870/** 847/**
848 * Remove a particular key in the datastore.
849 *
850 * @param cls closure
851 * @param key key for the content
852 * @param size number of bytes in data
853 * @param data content stored
854 * @param cont continuation called with success or failure status
855 * @param cont_cls continuation closure for @a cont
856 */
857static void
858postgres_plugin_remove_key (void *cls,
859 const struct GNUNET_HashCode *key,
860 uint32_t size,
861 const void *data,
862 PluginRemoveCont cont,
863 void *cont_cls)
864{
865 struct Plugin *plugin = cls;
866 PGresult *ret;
867 struct GNUNET_PQ_QueryParam params[] = {
868 GNUNET_PQ_query_param_auto_from_type (key),
869 GNUNET_PQ_query_param_fixed_size (data, size),
870 GNUNET_PQ_query_param_end
871 };
872 ret = GNUNET_PQ_exec_prepared (plugin->dbh,
873 "remove",
874 params);
875 if (GNUNET_OK !=
876 GNUNET_POSTGRES_check_result (plugin->dbh,
877 ret,
878 PGRES_COMMAND_OK,
879 "PQexecPrepared",
880 "remove"))
881 {
882 cont (cont_cls,
883 key,
884 size,
885 GNUNET_SYSERR,
886 _("Postgress exec failure"));
887 return;
888 }
889 /* What an awful API, this function really does return a string */
890 bool affected = 0 != strcmp ("0", PQcmdTuples (ret));
891 PQclear (ret);
892 if (!affected)
893 {
894 cont (cont_cls,
895 key,
896 size,
897 GNUNET_NO,
898 NULL);
899 return;
900 }
901 plugin->env->duc (plugin->env->cls,
902 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
903 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
904 "datastore-postgres",
905 "Deleted %u bytes from database\n",
906 (unsigned int) size);
907 cont (cont_cls,
908 key,
909 size,
910 GNUNET_OK,
911 NULL);
912}
913
914
915/**
871 * Entry point for the plugin. 916 * Entry point for the plugin.
872 * 917 *
873 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment*` 918 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment*`
@@ -891,13 +936,13 @@ libgnunet_plugin_datastore_postgres_init (void *cls)
891 api->cls = plugin; 936 api->cls = plugin;
892 api->estimate_size = &postgres_plugin_estimate_size; 937 api->estimate_size = &postgres_plugin_estimate_size;
893 api->put = &postgres_plugin_put; 938 api->put = &postgres_plugin_put;
894 api->update = &postgres_plugin_update;
895 api->get_key = &postgres_plugin_get_key; 939 api->get_key = &postgres_plugin_get_key;
896 api->get_replication = &postgres_plugin_get_replication; 940 api->get_replication = &postgres_plugin_get_replication;
897 api->get_expiration = &postgres_plugin_get_expiration; 941 api->get_expiration = &postgres_plugin_get_expiration;
898 api->get_zero_anonymity = &postgres_plugin_get_zero_anonymity; 942 api->get_zero_anonymity = &postgres_plugin_get_zero_anonymity;
899 api->get_keys = &postgres_plugin_get_keys; 943 api->get_keys = &postgres_plugin_get_keys;
900 api->drop = &postgres_plugin_drop; 944 api->drop = &postgres_plugin_drop;
945 api->remove_key = &postgres_plugin_remove_key;
901 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, 946 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
902 "datastore-postgres", 947 "datastore-postgres",
903 _("Postgres database running\n")); 948 _("Postgres database running\n"));
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c
index 1f874e190..bcaf27d99 100644
--- a/src/datastore/plugin_datastore_sqlite.c
+++ b/src/datastore/plugin_datastore_sqlite.c
@@ -88,6 +88,11 @@ struct Plugin
88 sqlite3 *dbh; 88 sqlite3 *dbh;
89 89
90 /** 90 /**
91 * Precompiled SQL for remove_key.
92 */
93 sqlite3_stmt *remove;
94
95 /**
91 * Precompiled SQL for deletion. 96 * Precompiled SQL for deletion.
92 */ 97 */
93 sqlite3_stmt *delRow; 98 sqlite3_stmt *delRow;
@@ -95,7 +100,7 @@ struct Plugin
95 /** 100 /**
96 * Precompiled SQL for update. 101 * Precompiled SQL for update.
97 */ 102 */
98 sqlite3_stmt *updPrio; 103 sqlite3_stmt *update;
99 104
100 /** 105 /**
101 * Get maximum repl value in database. 106 * Get maximum repl value in database.
@@ -185,10 +190,6 @@ create_indices (sqlite3 * dbh)
185 NULL, NULL, NULL)) || 190 NULL, NULL, NULL)) ||
186 (SQLITE_OK != 191 (SQLITE_OK !=
187 sqlite3_exec (dbh, 192 sqlite3_exec (dbh,
188 "CREATE INDEX IF NOT EXISTS idx_hash_vhash ON gn090 (hash,vhash)",
189 NULL, NULL, NULL)) ||
190 (SQLITE_OK !=
191 sqlite3_exec (dbh,
192 "CREATE INDEX IF NOT EXISTS idx_expire_repl ON gn090 (expire ASC,repl DESC)", 193 "CREATE INDEX IF NOT EXISTS idx_expire_repl ON gn090 (expire ASC,repl DESC)",
193 NULL, NULL, NULL)) || 194 NULL, NULL, NULL)) ||
194 (SQLITE_OK != 195 (SQLITE_OK !=
@@ -356,8 +357,8 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
356 "SET prio = prio + ?, " 357 "SET prio = prio + ?, "
357 "repl = repl + ?, " 358 "repl = repl + ?, "
358 "expire = MAX(expire, ?) " 359 "expire = MAX(expire, ?) "
359 "WHERE _ROWID_ = ?", 360 "WHERE hash = ? AND vhash = ?",
360 &plugin->updPrio)) || 361 &plugin->update)) ||
361 (SQLITE_OK != 362 (SQLITE_OK !=
362 sq_prepare (plugin->dbh, 363 sq_prepare (plugin->dbh,
363 "UPDATE gn090 " "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?", 364 "UPDATE gn090 " "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
@@ -415,15 +416,21 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
415 "WHERE _ROWID_ >= ? AND " 416 "WHERE _ROWID_ >= ? AND "
416 "(rvalue >= ? OR 0 = ?) AND " 417 "(rvalue >= ? OR 0 = ?) AND "
417 "(hash = ? OR 0 = ?) AND " 418 "(hash = ? OR 0 = ?) AND "
418 "(vhash = ? OR 0 = ?) AND "
419 "(type = ? OR 0 = ?) " 419 "(type = ? OR 0 = ?) "
420 "ORDER BY _ROWID_ ASC LIMIT 1", 420 "ORDER BY _ROWID_ ASC LIMIT 1",
421 &plugin->get)) || 421 &plugin->get)) ||
422 (SQLITE_OK != 422 (SQLITE_OK !=
423 sq_prepare (plugin->dbh, 423 sq_prepare (plugin->dbh,
424 "DELETE FROM gn090 WHERE _ROWID_ = ?", 424 "DELETE FROM gn090 WHERE _ROWID_ = ?",
425 &plugin->delRow)) 425 &plugin->delRow)) ||
426 ) 426 (SQLITE_OK !=
427 sq_prepare (plugin->dbh,
428 "DELETE FROM gn090 "
429 "WHERE hash = ? AND "
430 "value = ? "
431 "LIMIT 1",
432 &plugin->remove)) ||
433 false)
427 { 434 {
428 LOG_SQLITE (plugin, 435 LOG_SQLITE (plugin,
429 GNUNET_ERROR_TYPE_ERROR, 436 GNUNET_ERROR_TYPE_ERROR,
@@ -448,10 +455,12 @@ database_shutdown (struct Plugin *plugin)
448 sqlite3_stmt *stmt; 455 sqlite3_stmt *stmt;
449#endif 456#endif
450 457
458 if (NULL != plugin->remove)
459 sqlite3_finalize (plugin->remove);
451 if (NULL != plugin->delRow) 460 if (NULL != plugin->delRow)
452 sqlite3_finalize (plugin->delRow); 461 sqlite3_finalize (plugin->delRow);
453 if (NULL != plugin->updPrio) 462 if (NULL != plugin->update)
454 sqlite3_finalize (plugin->updPrio); 463 sqlite3_finalize (plugin->update);
455 if (NULL != plugin->updRepl) 464 if (NULL != plugin->updRepl)
456 sqlite3_finalize (plugin->updRepl); 465 sqlite3_finalize (plugin->updRepl);
457 if (NULL != plugin->selRepl) 466 if (NULL != plugin->selRepl)
@@ -541,6 +550,7 @@ delete_by_rowid (struct Plugin *plugin,
541 * 550 *
542 * @param cls closure 551 * @param cls closure
543 * @param key key for the item 552 * @param key key for the item
553 * @param absent true if the key was not found in the bloom filter
544 * @param size number of bytes in @a data 554 * @param size number of bytes in @a data
545 * @param data content stored 555 * @param data content stored
546 * @param type type of the content 556 * @param type type of the content
@@ -554,6 +564,7 @@ delete_by_rowid (struct Plugin *plugin,
554static void 564static void
555sqlite_plugin_put (void *cls, 565sqlite_plugin_put (void *cls,
556 const struct GNUNET_HashCode *key, 566 const struct GNUNET_HashCode *key,
567 bool absent,
557 uint32_t size, 568 uint32_t size,
558 const void *data, 569 const void *data,
559 enum GNUNET_BLOCK_Type type, 570 enum GNUNET_BLOCK_Type type,
@@ -564,8 +575,63 @@ sqlite_plugin_put (void *cls,
564 PluginPutCont cont, 575 PluginPutCont cont,
565 void *cont_cls) 576 void *cont_cls)
566{ 577{
567 uint64_t rvalue; 578 struct Plugin *plugin = cls;
568 struct GNUNET_HashCode vhash; 579 struct GNUNET_HashCode vhash;
580 char *msg = NULL;
581
582 GNUNET_CRYPTO_hash (data,
583 size,
584 &vhash);
585
586 if (!absent)
587 {
588 struct GNUNET_SQ_QueryParam params[] = {
589 GNUNET_SQ_query_param_uint32 (&priority),
590 GNUNET_SQ_query_param_uint32 (&replication),
591 GNUNET_SQ_query_param_absolute_time (&expiration),
592 GNUNET_SQ_query_param_auto_from_type (key),
593 GNUNET_SQ_query_param_auto_from_type (&vhash),
594 GNUNET_SQ_query_param_end
595 };
596
597 if (GNUNET_OK !=
598 GNUNET_SQ_bind (plugin->update,
599 params))
600 {
601 cont (cont_cls,
602 key,
603 size,
604 GNUNET_SYSERR,
605 _("sqlite bind failure"));
606 return;
607 }
608 if (SQLITE_DONE != sqlite3_step (plugin->update))
609 {
610 LOG_SQLITE_MSG (plugin, &msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
611 "sqlite3_step");
612 cont (cont_cls,
613 key,
614 size,
615 GNUNET_SYSERR,
616 msg);
617 GNUNET_free_non_null (msg);
618 return;
619 }
620 int changes = sqlite3_changes (plugin->dbh);
621 GNUNET_SQ_reset (plugin->dbh,
622 plugin->update);
623 if (0 != changes)
624 {
625 cont (cont_cls,
626 key,
627 size,
628 GNUNET_NO,
629 NULL);
630 return;
631 }
632 }
633
634 uint64_t rvalue;
569 uint32_t type32 = (uint32_t) type; 635 uint32_t type32 = (uint32_t) type;
570 struct GNUNET_SQ_QueryParam params[] = { 636 struct GNUNET_SQ_QueryParam params[] = {
571 GNUNET_SQ_query_param_uint32 (&replication), 637 GNUNET_SQ_query_param_uint32 (&replication),
@@ -579,11 +645,9 @@ sqlite_plugin_put (void *cls,
579 GNUNET_SQ_query_param_fixed_size (data, size), 645 GNUNET_SQ_query_param_fixed_size (data, size),
580 GNUNET_SQ_query_param_end 646 GNUNET_SQ_query_param_end
581 }; 647 };
582 struct Plugin *plugin = cls;
583 int n; 648 int n;
584 int ret; 649 int ret;
585 sqlite3_stmt *stmt; 650 sqlite3_stmt *stmt;
586 char *msg = NULL;
587 651
588 if (size > MAX_ITEM_SIZE) 652 if (size > MAX_ITEM_SIZE)
589 { 653 {
@@ -598,15 +662,13 @@ sqlite_plugin_put (void *cls,
598 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (expiration), 662 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (expiration),
599 GNUNET_YES), 663 GNUNET_YES),
600 GNUNET_STRINGS_absolute_time_to_string (expiration)); 664 GNUNET_STRINGS_absolute_time_to_string (expiration));
601 GNUNET_CRYPTO_hash (data, size, &vhash);
602 stmt = plugin->insertContent; 665 stmt = plugin->insertContent;
603 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX); 666 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
604 if (GNUNET_OK != 667 if (GNUNET_OK !=
605 GNUNET_SQ_bind (stmt, 668 GNUNET_SQ_bind (stmt,
606 params)) 669 params))
607 { 670 {
608 cont (cont_cls, key, size, GNUNET_SYSERR, msg); 671 cont (cont_cls, key, size, GNUNET_SYSERR, NULL);
609 GNUNET_free_non_null(msg);
610 return; 672 return;
611 } 673 }
612 n = sqlite3_step (stmt); 674 n = sqlite3_step (stmt);
@@ -646,79 +708,6 @@ sqlite_plugin_put (void *cls,
646 708
647 709
648/** 710/**
649 * Update the priority, replication and expiration for a particular
650 * unique ID in the datastore. If the expiration time in value is
651 * different than the time found in the datastore, the higher value
652 * should be kept. The specified priority and replication is added
653 * to the existing value.
654 *
655 * @param cls the plugin context (state for this module)
656 * @param uid unique identifier of the datum
657 * @param priority by how much should the priority
658 * change?
659 * @param replication by how much should the replication
660 * change?
661 * @param expire new expiration time should be the
662 * MAX of any existing expiration time and
663 * this value
664 * @param cont continuation called with success or failure status
665 * @param cons_cls closure for @a cont
666 */
667static void
668sqlite_plugin_update (void *cls,
669 uint64_t uid,
670 uint32_t priority,
671 uint32_t replication,
672 struct GNUNET_TIME_Absolute expire,
673 PluginUpdateCont cont,
674 void *cont_cls)
675{
676 struct Plugin *plugin = cls;
677 struct GNUNET_SQ_QueryParam params[] = {
678 GNUNET_SQ_query_param_uint32 (&priority),
679 GNUNET_SQ_query_param_uint32 (&replication),
680 GNUNET_SQ_query_param_absolute_time (&expire),
681 GNUNET_SQ_query_param_uint64 (&uid),
682 GNUNET_SQ_query_param_end
683 };
684 int n;
685 char *msg = NULL;
686
687 if (GNUNET_OK !=
688 GNUNET_SQ_bind (plugin->updPrio,
689 params))
690 {
691 cont (cont_cls, GNUNET_SYSERR, msg);
692 GNUNET_free_non_null(msg);
693 return;
694 }
695 n = sqlite3_step (plugin->updPrio);
696 GNUNET_SQ_reset (plugin->dbh,
697 plugin->updPrio);
698 switch (n)
699 {
700 case SQLITE_DONE:
701 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Block updated\n");
702 cont (cont_cls, GNUNET_OK, NULL);
703 return;
704 case SQLITE_BUSY:
705 LOG_SQLITE_MSG (plugin, &msg,
706 GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
707 "sqlite3_step");
708 cont (cont_cls, GNUNET_NO, msg);
709 GNUNET_free_non_null(msg);
710 return;
711 default:
712 LOG_SQLITE_MSG (plugin, &msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
713 "sqlite3_step");
714 cont (cont_cls, GNUNET_SYSERR, msg);
715 GNUNET_free_non_null(msg);
716 return;
717 }
718}
719
720
721/**
722 * Execute statement that gets a row and call the callback 711 * Execute statement that gets a row and call the callback
723 * with the result. Resets the statement afterwards. 712 * with the result. Resets the statement afterwards.
724 * 713 *
@@ -865,15 +854,10 @@ sqlite_plugin_get_zero_anonymity (void *cls,
865 * @param next_uid return the result with lowest uid >= next_uid 854 * @param next_uid return the result with lowest uid >= next_uid
866 * @param random if true, return a random result instead of using next_uid 855 * @param random if true, return a random result instead of using next_uid
867 * @param key maybe NULL (to match all entries) 856 * @param key maybe NULL (to match all entries)
868 * @param vhash hash of the value, maybe NULL (to
869 * match all values that have the right key).
870 * Note that for DBlocks there is no difference
871 * betwen key and vhash, but for other blocks
872 * there may be!
873 * @param type entries of which type are relevant? 857 * @param type entries of which type are relevant?
874 * Use 0 for any type. 858 * Use 0 for any type.
875 * @param proc function to call on each matching value; 859 * @param proc function to call on the matching value;
876 * will be called once with a NULL value at the end 860 * will be called with NULL if nothing matches
877 * @param proc_cls closure for @a proc 861 * @param proc_cls closure for @a proc
878 */ 862 */
879static void 863static void
@@ -881,7 +865,6 @@ sqlite_plugin_get_key (void *cls,
881 uint64_t next_uid, 865 uint64_t next_uid,
882 bool random, 866 bool random,
883 const struct GNUNET_HashCode *key, 867 const struct GNUNET_HashCode *key,
884 const struct GNUNET_HashCode *vhash,
885 enum GNUNET_BLOCK_Type type, 868 enum GNUNET_BLOCK_Type type,
886 PluginDatumProcessor proc, 869 PluginDatumProcessor proc,
887 void *proc_cls) 870 void *proc_cls)
@@ -892,15 +875,12 @@ sqlite_plugin_get_key (void *cls,
892 uint32_t type32 = (uint32_t) type; 875 uint32_t type32 = (uint32_t) type;
893 uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type; 876 uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type;
894 uint16_t use_key = NULL != key; 877 uint16_t use_key = NULL != key;
895 uint16_t use_vhash = NULL != vhash;
896 struct GNUNET_SQ_QueryParam params[] = { 878 struct GNUNET_SQ_QueryParam params[] = {
897 GNUNET_SQ_query_param_uint64 (&next_uid), 879 GNUNET_SQ_query_param_uint64 (&next_uid),
898 GNUNET_SQ_query_param_uint64 (&rvalue), 880 GNUNET_SQ_query_param_uint64 (&rvalue),
899 GNUNET_SQ_query_param_uint16 (&use_rvalue), 881 GNUNET_SQ_query_param_uint16 (&use_rvalue),
900 GNUNET_SQ_query_param_auto_from_type (key), 882 GNUNET_SQ_query_param_auto_from_type (key),
901 GNUNET_SQ_query_param_uint16 (&use_key), 883 GNUNET_SQ_query_param_uint16 (&use_key),
902 GNUNET_SQ_query_param_auto_from_type (vhash),
903 GNUNET_SQ_query_param_uint16 (&use_vhash),
904 GNUNET_SQ_query_param_uint32 (&type32), 884 GNUNET_SQ_query_param_uint32 (&type32),
905 GNUNET_SQ_query_param_uint16 (&use_type), 885 GNUNET_SQ_query_param_uint16 (&use_type),
906 GNUNET_SQ_query_param_end 886 GNUNET_SQ_query_param_end
@@ -1206,6 +1186,79 @@ sqlite_plugin_drop (void *cls)
1206 1186
1207 1187
1208/** 1188/**
1189 * Remove a particular key in the datastore.
1190 *
1191 * @param cls closure
1192 * @param key key for the content
1193 * @param size number of bytes in data
1194 * @param data content stored
1195 * @param cont continuation called with success or failure status
1196 * @param cont_cls continuation closure for @a cont
1197 */
1198static void
1199sqlite_plugin_remove_key (void *cls,
1200 const struct GNUNET_HashCode *key,
1201 uint32_t size,
1202 const void *data,
1203 PluginRemoveCont cont,
1204 void *cont_cls)
1205{
1206 struct Plugin *plugin = cls;
1207 struct GNUNET_SQ_QueryParam params[] = {
1208 GNUNET_SQ_query_param_auto_from_type (key),
1209 GNUNET_SQ_query_param_fixed_size (data, size),
1210 GNUNET_SQ_query_param_end
1211 };
1212
1213 if (GNUNET_OK !=
1214 GNUNET_SQ_bind (plugin->remove,
1215 params))
1216 {
1217 cont (cont_cls,
1218 key,
1219 size,
1220 GNUNET_SYSERR,
1221 "bind failed");
1222 return;
1223 }
1224 if (SQLITE_DONE != sqlite3_step (plugin->remove))
1225 {
1226 LOG_SQLITE (plugin,
1227 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1228 "sqlite3_step");
1229 GNUNET_SQ_reset (plugin->dbh,
1230 plugin->remove);
1231 cont (cont_cls,
1232 key,
1233 size,
1234 GNUNET_SYSERR,
1235 "sqlite3_step failed");
1236 return;
1237 }
1238 int changes = sqlite3_changes (plugin->dbh);
1239 GNUNET_SQ_reset (plugin->dbh,
1240 plugin->remove);
1241 if (0 == changes)
1242 {
1243 cont (cont_cls,
1244 key,
1245 size,
1246 GNUNET_NO,
1247 NULL);
1248 return;
1249 }
1250 if (NULL != plugin->env->duc)
1251 plugin->env->duc (plugin->env->cls,
1252 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
1253 cont (cont_cls,
1254 key,
1255 size,
1256 GNUNET_OK,
1257 NULL);
1258}
1259
1260
1261/**
1209 * Get an estimate of how much space the database is 1262 * Get an estimate of how much space the database is
1210 * currently using. 1263 * currently using.
1211 * 1264 *
@@ -1300,13 +1353,13 @@ libgnunet_plugin_datastore_sqlite_init (void *cls)
1300 api->cls = &plugin; 1353 api->cls = &plugin;
1301 api->estimate_size = &sqlite_plugin_estimate_size; 1354 api->estimate_size = &sqlite_plugin_estimate_size;
1302 api->put = &sqlite_plugin_put; 1355 api->put = &sqlite_plugin_put;
1303 api->update = &sqlite_plugin_update;
1304 api->get_key = &sqlite_plugin_get_key; 1356 api->get_key = &sqlite_plugin_get_key;
1305 api->get_replication = &sqlite_plugin_get_replication; 1357 api->get_replication = &sqlite_plugin_get_replication;
1306 api->get_expiration = &sqlite_plugin_get_expiration; 1358 api->get_expiration = &sqlite_plugin_get_expiration;
1307 api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity; 1359 api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity;
1308 api->get_keys = &sqlite_plugin_get_keys; 1360 api->get_keys = &sqlite_plugin_get_keys;
1309 api->drop = &sqlite_plugin_drop; 1361 api->drop = &sqlite_plugin_drop;
1362 api->remove_key = &sqlite_plugin_remove_key;
1310 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, 1363 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
1311 "sqlite", 1364 "sqlite",
1312 _("Sqlite database running\n")); 1365 _("Sqlite database running\n"));
diff --git a/src/datastore/plugin_datastore_template.c b/src/datastore/plugin_datastore_template.c
index 8e44f020d..16bda45d4 100644
--- a/src/datastore/plugin_datastore_template.c
+++ b/src/datastore/plugin_datastore_template.c
@@ -62,6 +62,7 @@ template_plugin_estimate_size (void *cls, unsigned long long *estimate)
62 * 62 *
63 * @param cls closure 63 * @param cls closure
64 * @param key key for the item 64 * @param key key for the item
65 * @param absent true if the key was not found in the bloom filter
65 * @param size number of bytes in data 66 * @param size number of bytes in data
66 * @param data content stored 67 * @param data content stored
67 * @param type type of the content 68 * @param type type of the content
@@ -73,11 +74,17 @@ template_plugin_estimate_size (void *cls, unsigned long long *estimate)
73 * @param cont_cls continuation closure 74 * @param cont_cls continuation closure
74 */ 75 */
75static void 76static void
76template_plugin_put (void *cls, const struct GNUNET_HashCode * key, uint32_t size, 77template_plugin_put (void *cls,
77 const void *data, enum GNUNET_BLOCK_Type type, 78 const struct GNUNET_HashCode *key,
78 uint32_t priority, uint32_t anonymity, 79 bool absent,
80 uint32_t size,
81 const void *data,
82 enum GNUNET_BLOCK_Type type,
83 uint32_t priority,
84 uint32_t anonymity,
79 uint32_t replication, 85 uint32_t replication,
80 struct GNUNET_TIME_Absolute expiration, PluginPutCont cont, 86 struct GNUNET_TIME_Absolute expiration,
87 PluginPutCont cont,
81 void *cont_cls) 88 void *cont_cls)
82{ 89{
83 GNUNET_break (0); 90 GNUNET_break (0);
@@ -92,11 +99,6 @@ template_plugin_put (void *cls, const struct GNUNET_HashCode * key, uint32_t siz
92 * @param next_uid return the result with lowest uid >= next_uid 99 * @param next_uid return the result with lowest uid >= next_uid
93 * @param random if true, return a random result instead of using next_uid 100 * @param random if true, return a random result instead of using next_uid
94 * @param key maybe NULL (to match all entries) 101 * @param key maybe NULL (to match all entries)
95 * @param vhash hash of the value, maybe NULL (to
96 * match all values that have the right key).
97 * Note that for DBlocks there is no difference
98 * betwen key and vhash, but for other blocks
99 * there may be!
100 * @param type entries of which type are relevant? 102 * @param type entries of which type are relevant?
101 * Use 0 for any type. 103 * Use 0 for any type.
102 * @param proc function to call on each matching value; 104 * @param proc function to call on each matching value;
@@ -104,10 +106,12 @@ template_plugin_put (void *cls, const struct GNUNET_HashCode * key, uint32_t siz
104 * @param proc_cls closure for proc 106 * @param proc_cls closure for proc
105 */ 107 */
106static void 108static void
107template_plugin_get_key (void *cls, uint64_t next_uid, bool random, 109template_plugin_get_key (void *cls,
108 const struct GNUNET_HashCode * key, 110 uint64_t next_uid,
109 const struct GNUNET_HashCode * vhash, 111 bool random,
110 enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, 112 const struct GNUNET_HashCode *key,
113 enum GNUNET_BLOCK_Type type,
114 PluginDatumProcessor proc,
111 void *proc_cls) 115 void *proc_cls)
112{ 116{
113 GNUNET_break (0); 117 GNUNET_break (0);
@@ -151,39 +155,6 @@ template_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
151 155
152 156
153/** 157/**
154 * Update the priority, replication and expiration for a particular
155 * unique ID in the datastore. If the expiration time in value is
156 * different than the time found in the datastore, the higher value
157 * should be kept. The specified priority and replication is added
158 * to the existing value.
159 *
160 * @param cls our "struct Plugin*"
161 * @param uid unique identifier of the datum
162 * @param priority by how much should the priority
163 * change?
164 * @param replication by how much should the replication
165 * change?
166 * @param expire new expiration time should be the
167 * MAX of any existing expiration time and
168 * this value
169 * @param cont continuation called with success or failure status
170 * @param cons_cls continuation closure
171 */
172static void
173template_plugin_update (void *cls,
174 uint64_t uid,
175 uint32_t priority,
176 uint32_t replication,
177 struct GNUNET_TIME_Absolute expire,
178 PluginUpdateCont cont,
179 void *cont_cls)
180{
181 GNUNET_break (0);
182 cont (cont_cls, GNUNET_SYSERR, "not implemented");
183}
184
185
186/**
187 * Call the given processor on an item with zero anonymity. 158 * Call the given processor on an item with zero anonymity.
188 * 159 *
189 * @param cls our "struct Plugin*" 160 * @param cls our "struct Plugin*"
@@ -230,6 +201,29 @@ template_get_keys (void *cls,
230 201
231 202
232/** 203/**
204 * Remove a particular key in the datastore.
205 *
206 * @param cls closure
207 * @param key key for the content
208 * @param size number of bytes in data
209 * @param data content stored
210 * @param cont continuation called with success or failure status
211 * @param cont_cls continuation closure for @a cont
212 */
213static void
214template_plugin_remove_key (void *cls,
215 const struct GNUNET_HashCode *key,
216 uint32_t size,
217 const void *data,
218 PluginRemoveCont cont,
219 void *cont_cls)
220{
221 GNUNET_break (0);
222 cont (cont_cls, key, size, GNUNET_SYSERR, "not implemented");
223}
224
225
226/**
233 * Entry point for the plugin. 227 * Entry point for the plugin.
234 * 228 *
235 * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*" 229 * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*"
@@ -248,13 +242,13 @@ libgnunet_plugin_datastore_template_init (void *cls)
248 api->cls = plugin; 242 api->cls = plugin;
249 api->estimate_size = &template_plugin_estimate_size; 243 api->estimate_size = &template_plugin_estimate_size;
250 api->put = &template_plugin_put; 244 api->put = &template_plugin_put;
251 api->update = &template_plugin_update;
252 api->get_key = &template_plugin_get_key; 245 api->get_key = &template_plugin_get_key;
253 api->get_replication = &template_plugin_get_replication; 246 api->get_replication = &template_plugin_get_replication;
254 api->get_expiration = &template_plugin_get_expiration; 247 api->get_expiration = &template_plugin_get_expiration;
255 api->get_zero_anonymity = &template_plugin_get_zero_anonymity; 248 api->get_zero_anonymity = &template_plugin_get_zero_anonymity;
256 api->drop = &template_plugin_drop; 249 api->drop = &template_plugin_drop;
257 api->get_keys = &template_get_keys; 250 api->get_keys = &template_get_keys;
251 api->remove_key = &template_plugin_remove_key;
258 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "template", 252 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "template",
259 _("Template database running\n")); 253 _("Template database running\n"));
260 return api; 254 return api;
diff --git a/src/datastore/test_plugin_datastore.c b/src/datastore/test_plugin_datastore.c
index 1867d6755..d460daed7 100644
--- a/src/datastore/test_plugin_datastore.c
+++ b/src/datastore/test_plugin_datastore.c
@@ -49,10 +49,10 @@ enum RunPhase
49 RP_ERROR = 0, 49 RP_ERROR = 0,
50 RP_PUT, 50 RP_PUT,
51 RP_GET, 51 RP_GET,
52 RP_UPDATE,
53 RP_ITER_ZERO, 52 RP_ITER_ZERO,
54 RP_REPL_GET, 53 RP_REPL_GET,
55 RP_EXPI_GET, 54 RP_EXPI_GET,
55 RP_REMOVE,
56 RP_DROP 56 RP_DROP
57}; 57};
58 58
@@ -154,7 +154,7 @@ do_put (struct CpsRunContext *crc)
154 /* most content is 32k */ 154 /* most content is 32k */
155 size = 32 * 1024; 155 size = 32 * 1024;
156 156
157 if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ 157 if (0 != i && GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */
158 size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); 158 size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
159 size = size - (size & 7); /* always multiple of 8 */ 159 size = size - (size & 7); /* always multiple of 8 */
160 160
@@ -168,8 +168,13 @@ do_put (struct CpsRunContext *crc)
168 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 168 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
169 "putting type %u, anon %u under key %s\n", i + 1, i, 169 "putting type %u, anon %u under key %s\n", i + 1, i,
170 GNUNET_h2s (&key)); 170 GNUNET_h2s (&key));
171 crc->api->put (crc->api->cls, &key, size, value, i + 1 /* type */ , 171 crc->api->put (crc->api->cls,
172 prio, i /* anonymity */ , 172 &key,
173 false /* absent */,
174 size,
175 value, i + 1 /* type */ ,
176 prio,
177 i /* anonymity */ ,
173 0 /* replication */ , 178 0 /* replication */ ,
174 GNUNET_TIME_relative_to_absolute 179 GNUNET_TIME_relative_to_absolute
175 (GNUNET_TIME_relative_multiply 180 (GNUNET_TIME_relative_multiply
@@ -177,7 +182,8 @@ do_put (struct CpsRunContext *crc)
177 60 * 60 * 60 * 1000 + 182 60 * 60 * 60 * 1000 +
178 GNUNET_CRYPTO_random_u32 183 GNUNET_CRYPTO_random_u32
179 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), 184 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
180 put_continuation, crc); 185 put_continuation,
186 crc);
181 i++; 187 i++;
182} 188}
183 189
@@ -215,6 +221,25 @@ iterate_one_shot (void *cls,
215} 221}
216 222
217 223
224static void
225remove_continuation (void *cls,
226 const struct GNUNET_HashCode *key,
227 uint32_t size,
228 int status,
229 const char *msg)
230{
231 struct CpsRunContext *crc = cls;
232
233 GNUNET_assert (NULL != key);
234 GNUNET_assert (32768 == size);
235 GNUNET_assert (GNUNET_OK == status);
236 GNUNET_assert (NULL == msg);
237 crc->phase++;
238 GNUNET_SCHEDULER_add_now (&test,
239 crc);
240}
241
242
218/** 243/**
219 * Function called when the service shuts 244 * Function called when the service shuts
220 * down. Unloads our datastore plugin. 245 * down. Unloads our datastore plugin.
@@ -264,19 +289,6 @@ cleaning_task (void *cls)
264 289
265 290
266static void 291static void
267update_continuation (void *cls,
268 int status,
269 const char *msg)
270{
271 struct CpsRunContext *crc = cls;
272
273 GNUNET_assert (GNUNET_OK == status);
274 crc->phase++;
275 GNUNET_SCHEDULER_add_now (&test, crc);
276}
277
278
279static void
280test (void *cls) 292test (void *cls)
281{ 293{
282 struct CpsRunContext *crc = cls; 294 struct CpsRunContext *crc = cls;
@@ -311,21 +323,10 @@ test (void *cls)
311 0, 323 0,
312 false, 324 false,
313 &key, 325 &key,
314 NULL,
315 GNUNET_BLOCK_TYPE_ANY, 326 GNUNET_BLOCK_TYPE_ANY,
316 &iterate_one_shot, 327 &iterate_one_shot,
317 crc); 328 crc);
318 break; 329 break;
319 case RP_UPDATE:
320 crc->api->update (crc->api->cls,
321 guid,
322 1,
323 1,
324 GNUNET_TIME_UNIT_ZERO_ABS,
325 &update_continuation,
326 crc);
327 break;
328
329 case RP_ITER_ZERO: 330 case RP_ITER_ZERO:
330 if (crc->cnt == 1) 331 if (crc->cnt == 1)
331 { 332 {
@@ -342,6 +343,23 @@ test (void *cls)
342 case RP_EXPI_GET: 343 case RP_EXPI_GET:
343 crc->api->get_expiration (crc->api->cls, &iterate_one_shot, crc); 344 crc->api->get_expiration (crc->api->cls, &iterate_one_shot, crc);
344 break; 345 break;
346 case RP_REMOVE:
347 {
348 struct GNUNET_HashCode key;
349 uint32_t size = 32768;
350 char value[size];
351
352 gen_key (0, &key);
353 memset (value, 0, size);
354 value[0] = crc->i;
355 crc->api->remove_key (crc->api->cls,
356 &key,
357 size,
358 value,
359 &remove_continuation,
360 crc);
361 break;
362 }
345 case RP_DROP: 363 case RP_DROP:
346 crc->api->drop (crc->api->cls); 364 crc->api->drop (crc->api->cls);
347 GNUNET_SCHEDULER_add_now (&cleaning_task, crc); 365 GNUNET_SCHEDULER_add_now (&cleaning_task, crc);
diff --git a/src/include/gnunet_datastore_plugin.h b/src/include/gnunet_datastore_plugin.h
index 516ba525c..36a3fbec2 100644
--- a/src/include/gnunet_datastore_plugin.h
+++ b/src/include/gnunet_datastore_plugin.h
@@ -134,7 +134,8 @@ typedef void
134 * @param cls closure 134 * @param cls closure
135 * @param key key for the item stored 135 * @param key key for the item stored
136 * @param size size of the item stored 136 * @param size size of the item stored
137 * @param status #GNUNET_OK or #GNUNET_SYSERROR 137 * @param status #GNUNET_OK if inserted, #GNUNET_NO if updated,
138 * or #GNUNET_SYSERROR if error
138 * @param msg error message on error 139 * @param msg error message on error
139 */ 140 */
140typedef void 141typedef void
@@ -152,6 +153,7 @@ typedef void
152 * 153 *
153 * @param cls closure 154 * @param cls closure
154 * @param key key for the item 155 * @param key key for the item
156 * @param absent true if the key was not found in the bloom filter
155 * @param size number of bytes in @a data 157 * @param size number of bytes in @a data
156 * @param data content stored 158 * @param data content stored
157 * @param type type of the content 159 * @param type type of the content
@@ -165,15 +167,16 @@ typedef void
165typedef void 167typedef void
166(*PluginPut) (void *cls, 168(*PluginPut) (void *cls,
167 const struct GNUNET_HashCode *key, 169 const struct GNUNET_HashCode *key,
168 uint32_t size, 170 bool absent,
169 const void *data, 171 uint32_t size,
170 enum GNUNET_BLOCK_Type type, 172 const void *data,
171 uint32_t priority, 173 enum GNUNET_BLOCK_Type type,
172 uint32_t anonymity, 174 uint32_t priority,
173 uint32_t replication, 175 uint32_t anonymity,
174 struct GNUNET_TIME_Absolute expiration, 176 uint32_t replication,
175 PluginPutCont cont, 177 struct GNUNET_TIME_Absolute expiration,
176 void *cont_cls); 178 PluginPutCont cont,
179 void *cont_cls);
177 180
178 181
179/** 182/**
@@ -209,11 +212,6 @@ typedef void
209 * @param next_uid return the result with lowest uid >= next_uid 212 * @param next_uid return the result with lowest uid >= next_uid
210 * @param random if true, return a random result instead of using next_uid 213 * @param random if true, return a random result instead of using next_uid
211 * @param key maybe NULL (to match all entries) 214 * @param key maybe NULL (to match all entries)
212 * @param vhash hash of the value, maybe NULL (to
213 * match all values that have the right key).
214 * Note that for DBlocks there is no difference
215 * betwen key and vhash, but for other blocks
216 * there may be!
217 * @param type entries of which type are relevant? 215 * @param type entries of which type are relevant?
218 * Use 0 for any type. 216 * Use 0 for any type.
219 * @param proc function to call on the matching value; 217 * @param proc function to call on the matching value;
@@ -225,68 +223,62 @@ typedef void
225 uint64_t next_uid, 223 uint64_t next_uid,
226 bool random, 224 bool random,
227 const struct GNUNET_HashCode *key, 225 const struct GNUNET_HashCode *key,
228 const struct GNUNET_HashCode *vhash,
229 enum GNUNET_BLOCK_Type type, 226 enum GNUNET_BLOCK_Type type,
230 PluginDatumProcessor proc, 227 PluginDatumProcessor proc,
231 void *proc_cls); 228 void *proc_cls);
232 229
233 230
234/** 231/**
235 * Get a random item (additional constraints may apply depending on 232 * Remove continuation.
236 * the specific implementation). Calls @a proc with all values ZERO or
237 * NULL if no item applies, otherwise @a proc is called once and only
238 * once with an item.
239 * 233 *
240 * @param cls closure 234 * @param cls closure
241 * @param proc function to call the value (once only). 235 * @param key key for the content removed
242 * @param proc_cls closure for @a proc 236 * @param size number of bytes removed
237 * @param status #GNUNET_OK if removed, #GNUNET_NO if not found,
238 * or #GNUNET_SYSERROR if error
239 * @param msg error message on error
243 */ 240 */
244typedef void 241typedef void
245(*PluginGetRandom) (void *cls, 242(*PluginRemoveCont) (void *cls,
246 PluginDatumProcessor proc, 243 const struct GNUNET_HashCode *key,
247 void *proc_cls); 244 uint32_t size,
245 int status,
246 const char *msg);
248 247
249 248
250/** 249/**
251 * Update continuation. 250 * Remove a particular key in the datastore.
252 * 251 *
253 * @param cls closure 252 * @param cls closure
254 * @param status #GNUNET_OK or #GNUNET_SYSERR 253 * @param key key for the content
255 * @param msg error message on error 254 * @param size number of bytes in data
255 * @param data content stored
256 * @param cont continuation called with success or failure status
257 * @param cont_cls continuation closure for @a cont
256 */ 258 */
257typedef void 259typedef void
258(*PluginUpdateCont) (void *cls, 260(*PluginRemoveKey) (void *cls,
259 int status, 261 const struct GNUNET_HashCode *key,
260 const char *msg); 262 uint32_t size,
263 const void *data,
264 PluginRemoveCont cont,
265 void *cont_cls);
261 266
262 267
263/** 268/**
264 * Update the priority, replication and expiration for a particular 269 * Get a random item (additional constraints may apply depending on
265 * unique ID in the datastore. If the expiration time in value is 270 * the specific implementation). Calls @a proc with all values ZERO or
266 * different than the time found in the datastore, the higher value 271 * NULL if no item applies, otherwise @a proc is called once and only
267 * should be kept. The specified priority and replication is added 272 * once with an item.
268 * to the existing value.
269 * 273 *
270 * @param cls closure 274 * @param cls closure
271 * @param uid unique identifier of the datum 275 * @param proc function to call the value (once only).
272 * @param priority by how much should the priority 276 * @param proc_cls closure for @a proc
273 * change?
274 * @param replication by how much should the replication
275 * change?
276 * @param expire new expiration time should be the
277 * MAX of any existing expiration time and
278 * this value
279 * @param cont continuation called with success or failure status
280 * @param cons_cls continuation closure
281 */ 277 */
282typedef void 278typedef void
283(*PluginUpdate) (void *cls, 279(*PluginGetRandom) (void *cls,
284 uint64_t uid, 280 PluginDatumProcessor proc,
285 uint32_t priority, 281 void *proc_cls);
286 uint32_t replication,
287 struct GNUNET_TIME_Absolute expire,
288 PluginUpdateCont cont,
289 void *cont_cls);
290 282
291 283
292/** 284/**
@@ -342,16 +334,6 @@ struct GNUNET_DATASTORE_PluginFunctions
342 PluginPut put; 334 PluginPut put;
343 335
344 /** 336 /**
345 * Update the priority for a particular key in the datastore. If
346 * the expiration time in value is different than the time found in
347 * the datastore, the higher value should be kept. For the
348 * anonymity level, the lower value is to be used. The specified
349 * priority should be added to the existing priority, ignoring the
350 * priority in value.
351 */
352 PluginUpdate update;
353
354 /**
355 * Get a particular datum matching a given hash from the datastore. 337 * Get a particular datum matching a given hash from the datastore.
356 */ 338 */
357 PluginGetKey get_key; 339 PluginGetKey get_key;
@@ -388,6 +370,10 @@ struct GNUNET_DATASTORE_PluginFunctions
388 */ 370 */
389 PluginGetKeys get_keys; 371 PluginGetKeys get_keys;
390 372
373 /**
374 * Function to remove an item from the database.
375 */
376 PluginRemoveKey remove_key;
391}; 377};
392 378
393#endif 379#endif
diff --git a/src/include/gnunet_json_lib.h b/src/include/gnunet_json_lib.h
index ce721d8d8..f2682bea7 100644
--- a/src/include/gnunet_json_lib.h
+++ b/src/include/gnunet_json_lib.h
@@ -263,6 +263,17 @@ GNUNET_JSON_spec_absolute_time (const char *name,
263 263
264 264
265/** 265/**
266 * Absolute time in network byte order.
267 *
268 * @param name name of the JSON field
269 * @param[out] at where to store the absolute time found under @a name
270 */
271struct GNUNET_JSON_Specification
272GNUNET_JSON_spec_absolute_time_nbo (const char *name,
273 struct GNUNET_TIME_AbsoluteNBO *at);
274
275
276/**
266 * Relative time. 277 * Relative time.
267 * 278 *
268 * @param name name of the JSON field 279 * @param name name of the JSON field
diff --git a/src/json/json_helper.c b/src/json/json_helper.c
index dca2106aa..194ec5c76 100644
--- a/src/json/json_helper.c
+++ b/src/json/json_helper.c
@@ -610,6 +610,83 @@ GNUNET_JSON_spec_absolute_time (const char *name,
610 610
611 611
612/** 612/**
613 * Parse given JSON object to absolute time.
614 *
615 * @param cls closure, NULL
616 * @param root the json object representing data
617 * @param[out] spec where to write the data
618 * @return #GNUNET_OK upon successful parsing; #GNUNET_SYSERR upon error
619 */
620static int
621parse_abs_time_nbo (void *cls,
622 json_t *root,
623 struct GNUNET_JSON_Specification *spec)
624{
625 struct GNUNET_TIME_AbsoluteNBO *abs = spec->ptr;
626 const char *val;
627 unsigned long long int tval;
628 struct GNUNET_TIME_Absolute a;
629
630 val = json_string_value (root);
631 if (NULL == val)
632 {
633 GNUNET_break_op (0);
634 return GNUNET_SYSERR;
635 }
636 if ( (0 == strcasecmp (val,
637 "/forever/")) ||
638 (0 == strcasecmp (val,
639 "/end of time/")) ||
640 (0 == strcasecmp (val,
641 "/never/")) )
642 {
643 *abs = GNUNET_TIME_absolute_hton (GNUNET_TIME_UNIT_FOREVER_ABS);
644 return GNUNET_OK;
645 }
646 if (1 != sscanf (val,
647 "/Date(%llu)/",
648 &tval))
649 {
650 GNUNET_break_op (0);
651 return GNUNET_SYSERR;
652 }
653 /* Time is in seconds in JSON, but in microseconds in GNUNET_TIME_Absolute */
654 a.abs_value_us = tval * 1000LL * 1000LL;
655 if ( (a.abs_value_us) / 1000LL / 1000LL != tval)
656 {
657 /* Integer overflow */
658 GNUNET_break_op (0);
659 return GNUNET_SYSERR;
660 }
661 *abs = GNUNET_TIME_absolute_hton (a);
662 return GNUNET_OK;
663}
664
665
666/**
667 * Absolute time in network byte order.
668 *
669 * @param name name of the JSON field
670 * @param[out] at where to store the absolute time found under @a name
671 */
672struct GNUNET_JSON_Specification
673GNUNET_JSON_spec_absolute_time_nbo (const char *name,
674 struct GNUNET_TIME_AbsoluteNBO *at)
675{
676 struct GNUNET_JSON_Specification ret = {
677 .parser = &parse_abs_time_nbo,
678 .cleaner = NULL,
679 .cls = NULL,
680 .field = name,
681 .ptr = at,
682 .ptr_size = sizeof (uint64_t),
683 .size_ptr = NULL
684 };
685 return ret;
686}
687
688
689/**
613 * Parse given JSON object to relative time. 690 * Parse given JSON object to relative time.
614 * 691 *
615 * @param cls closure, NULL 692 * @param cls closure, NULL