aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--TODO26
-rw-r--r--src/core/core_api.c2
-rw-r--r--src/datastore/datastore.h10
-rw-r--r--src/datastore/datastore_api.c279
-rw-r--r--src/datastore/gnunet-service-datastore.c242
-rw-r--r--src/datastore/perf_datastore_api.c1
-rw-r--r--src/datastore/perf_plugin_datastore.c78
-rw-r--r--src/datastore/perf_plugin_datastore_data_postgres.conf2
-rw-r--r--src/datastore/plugin_datastore_mysql.c790
-rw-r--r--src/datastore/plugin_datastore_postgres.c525
-rw-r--r--src/datastore/plugin_datastore_sqlite.c918
-rw-r--r--src/datastore/plugin_datastore_template.c110
-rw-r--r--src/datastore/test_datastore_api.c285
-rw-r--r--src/datastore/test_datastore_api_data_sqlite.conf2
-rw-r--r--src/datastore/test_datastore_api_management.c68
-rw-r--r--src/datastore/test_plugin_datastore.c89
-rw-r--r--src/fs/Makefile.am4
-rw-r--r--src/fs/fs_download.c19
-rw-r--r--src/fs/fs_test_lib_data.conf6
-rw-r--r--src/fs/gnunet-pseudonym.c2
-rw-r--r--src/fs/gnunet-service-fs_cp.c7
-rw-r--r--src/fs/gnunet-service-fs_indexing.c2
-rw-r--r--src/fs/gnunet-service-fs_indexing.h2
-rw-r--r--src/fs/gnunet-service-fs_pe.c2
-rw-r--r--src/fs/gnunet-service-fs_pr.c147
-rw-r--r--src/fs/gnunet-service-fs_put.c174
-rw-r--r--src/fs/test_fs_download_data.conf3
-rwxr-xr-xsrc/fs/test_gnunet_fs_idx.py.in2
-rw-r--r--src/fs/test_gnunet_fs_ns_data.conf2
-rw-r--r--src/fs/test_gnunet_service_fs_migration_data.conf2
-rw-r--r--src/include/gnunet_datastore_plugin.h143
-rw-r--r--src/include/gnunet_datastore_service.h109
32 files changed, 1446 insertions, 2607 deletions
diff --git a/TODO b/TODO
index bd94dcfbe..b9bfaf729 100644
--- a/TODO
+++ b/TODO
@@ -1,23 +1,31 @@
10.9.0pre3: [2'11] 10.9.0pre3: [2'11]
2* DATASTORE: 2* NAT/UPNP: [Milan / MW]
3 - postgres support currently not implemented
4* NAT/UPNP: [Milan / Ayush / MW]
5 - [#1609] code clean up 3 - [#1609] code clean up
6 - testing 4 - testing
7 - integration with transport service: 5 - integration with transport service:
8 + test TCP 6 + test TCP
9 + implement UDP, HTTP/HTTPS 7 + implement UDP, HTTP/HTTPS
10* Transport: 8* Transport:
11 - UDP fragmentation 9 - ATS crashes [MW]
12* FS/CORE [CG] 10 - UDP fragmentation [MW]
13 - adjust service to deal with new datastore API (also crashes all over the place still, 11* CORE:
14 likely related). 12 - Core API's peer_change_preference leaks 'irc' and
13 Core API's notify_transmit_ready leaks 'th'!
14* FS [CG]
15 - test*.py fail
15 - download of 100 MB file from 'leach' peer hung due to 16 - download of 100 MB file from 'leach' peer hung due to
16 failure of core-api to call back after a change preference request 17 failure of core-api to call back after a change preference request
17 (structs indicate request was transmitted but reply never received?) 18 (structs indicate request was transmitted but reply never received?)
19 => try again!
20 - test_gnunet_service_fs_p2p:
21 => sometimes DATASTORE get operation fails to queue on target (why?)
22 => do we need to just make the queue larger?
23 - with core queue size of 1, we get notify_transmit_ready
24 from core API returning NULL (why? ok? just have larger queue?)
18 - other runs (-L DEBUG) with downloads using the new 'trust' test show 25 - other runs (-L DEBUG) with downloads using the new 'trust' test show
19 non-deterministic results (for any set of peers) 26 non-deterministic results (for any set of peers)
20* FS: [CG] 27 - implement 'SUPPORT_DELAYS'
28 - consider re-issue GSF_dht_lookup_ after non-DHT reply received
21 - implement multi-peer FS performance tests + gauger them! 29 - implement multi-peer FS performance tests + gauger them!
22 + insert 30 + insert
23 + download 31 + download
@@ -59,6 +67,8 @@
59 => If MiM attacker uses vetoed address, blacklist the specific IP for 67 => If MiM attacker uses vetoed address, blacklist the specific IP for
60 the presumed neighbour! 68 the presumed neighbour!
61 - need to periodically probe latency/transport cost changes & possibly switch transport 69 - need to periodically probe latency/transport cost changes & possibly switch transport
70* DATASTORE: [CG]
71 - check indexes / SQL for performance
62* DV: [Nate?] 72* DV: [Nate?]
63 - proper bandwidth allocation 73 - proper bandwidth allocation
64 - performance tests 74 - performance tests
diff --git a/src/core/core_api.c b/src/core/core_api.c
index 185e09d65..efb00c111 100644
--- a/src/core/core_api.c
+++ b/src/core/core_api.c
@@ -1842,6 +1842,7 @@ change_preference_send_continuation (void *cls,
1842 struct GNUNET_CORE_InformationRequestContext *irc = cls; 1842 struct GNUNET_CORE_InformationRequestContext *irc = cls;
1843 1843
1844 irc->cm = NULL; 1844 irc->cm = NULL;
1845 // FIXME: who frees 'irc'?
1845} 1846}
1846 1847
1847 1848
@@ -1901,6 +1902,7 @@ GNUNET_CORE_peer_change_preference (struct GNUNET_CORE_Handle *h,
1901 irc = GNUNET_malloc (sizeof (struct GNUNET_CORE_InformationRequestContext)); 1902 irc = GNUNET_malloc (sizeof (struct GNUNET_CORE_InformationRequestContext));
1902 irc->h = h; 1903 irc->h = h;
1903 irc->pr = pr; 1904 irc->pr = pr;
1905 // FIXME: who frees 'irc'? (if not cancelled?)
1904 cm = GNUNET_malloc (sizeof (struct ControlMessage) + 1906 cm = GNUNET_malloc (sizeof (struct ControlMessage) +
1905 sizeof (struct RequestInfoMessage)); 1907 sizeof (struct RequestInfoMessage));
1906 cm->cont = &change_preference_send_continuation; 1908 cm->cont = &change_preference_send_continuation;
diff --git a/src/datastore/datastore.h b/src/datastore/datastore.h
index 55ca7c8e5..d66ec0e95 100644
--- a/src/datastore/datastore.h
+++ b/src/datastore/datastore.h
@@ -114,6 +114,11 @@ struct GetMessage
114 uint32_t type GNUNET_PACKED; 114 uint32_t type GNUNET_PACKED;
115 115
116 /** 116 /**
117 * Offset of the result.
118 */
119 uint64_t offset GNUNET_PACKED;
120
121 /**
117 * Desired key (optional). Check the "size" of the 122 * Desired key (optional). Check the "size" of the
118 * header to see if the key is actually present. 123 * header to see if the key is actually present.
119 */ 124 */
@@ -138,6 +143,11 @@ struct GetZeroAnonymityMessage
138 */ 143 */
139 uint32_t type GNUNET_PACKED; 144 uint32_t type GNUNET_PACKED;
140 145
146 /**
147 * Offset of the result.
148 */
149 uint64_t offset GNUNET_PACKED;
150
141}; 151};
142 152
143 153
diff --git a/src/datastore/datastore_api.c b/src/datastore/datastore_api.c
index 2bba2e8ee..99060bd60 100644
--- a/src/datastore/datastore_api.c
+++ b/src/datastore/datastore_api.c
@@ -63,14 +63,14 @@ struct StatusContext
63struct ResultContext 63struct ResultContext
64{ 64{
65 /** 65 /**
66 * Iterator to call with the result. 66 * Function to call with the result.
67 */ 67 */
68 GNUNET_DATASTORE_Iterator iter; 68 GNUNET_DATASTORE_DatumProcessor proc;
69 69
70 /** 70 /**
71 * Closure for iter. 71 * Closure for proc.
72 */ 72 */
73 void *iter_cls; 73 void *proc_cls;
74 74
75}; 75};
76 76
@@ -168,12 +168,6 @@ struct GNUNET_DATASTORE_QueueEntry
168 */ 168 */
169 int was_transmitted; 169 int was_transmitted;
170 170
171 /**
172 * Are we expecting a single message in response to this
173 * request (and, if it is data, no 'END' message)?
174 */
175 int one_shot;
176
177}; 171};
178 172
179/** 173/**
@@ -241,10 +235,9 @@ struct GNUNET_DATASTORE_Handle
241 int in_receive; 235 int in_receive;
242 236
243 /** 237 /**
244 * We should either receive (and ignore) an 'END' message or force a 238 * We should ignore the next message(s) from the service.
245 * disconnect for the next message from the service.
246 */ 239 */
247 unsigned int expect_end_or_disconnect; 240 unsigned int skip_next_messages;
248 241
249}; 242};
250 243
@@ -335,7 +328,7 @@ GNUNET_DATASTORE_disconnect (struct GNUNET_DATASTORE_Handle *h,
335 while (NULL != (qe = h->queue_head)) 328 while (NULL != (qe = h->queue_head))
336 { 329 {
337 GNUNET_assert (NULL != qe->response_proc); 330 GNUNET_assert (NULL != qe->response_proc);
338 qe->response_proc (qe, NULL); 331 qe->response_proc (h, NULL);
339 } 332 }
340 if (GNUNET_YES == drop) 333 if (GNUNET_YES == drop)
341 { 334 {
@@ -378,7 +371,7 @@ timeout_queue_entry (void *cls,
378 GNUNET_NO); 371 GNUNET_NO);
379 qe->task = GNUNET_SCHEDULER_NO_TASK; 372 qe->task = GNUNET_SCHEDULER_NO_TASK;
380 GNUNET_assert (qe->was_transmitted == GNUNET_NO); 373 GNUNET_assert (qe->was_transmitted == GNUNET_NO);
381 qe->response_proc (qe, NULL); 374 qe->response_proc (qe->h, NULL);
382} 375}
383 376
384 377
@@ -394,7 +387,7 @@ timeout_queue_entry (void *cls,
394 * @param timeout timeout for the operation 387 * @param timeout timeout for the operation
395 * @param response_proc function to call with replies (can be NULL) 388 * @param response_proc function to call with replies (can be NULL)
396 * @param qc client context (NOT a closure for response_proc) 389 * @param qc client context (NOT a closure for response_proc)
397 * @return NULL if the queue is full (and this entry was dropped) 390 * @return NULL if the queue is full
398 */ 391 */
399static struct GNUNET_DATASTORE_QueueEntry * 392static struct GNUNET_DATASTORE_QueueEntry *
400make_queue_entry (struct GNUNET_DATASTORE_Handle *h, 393make_queue_entry (struct GNUNET_DATASTORE_Handle *h,
@@ -418,6 +411,14 @@ make_queue_entry (struct GNUNET_DATASTORE_Handle *h,
418 c++; 411 c++;
419 pos = pos->next; 412 pos = pos->next;
420 } 413 }
414 if (c >= max_queue_size)
415 {
416 GNUNET_STATISTICS_update (h->stats,
417 gettext_noop ("# queue overflows"),
418 1,
419 GNUNET_NO);
420 return NULL;
421 }
421 ret = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_QueueEntry) + msize); 422 ret = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_QueueEntry) + msize);
422 ret->h = h; 423 ret->h = h;
423 ret->response_proc = response_proc; 424 ret->response_proc = response_proc;
@@ -451,15 +452,6 @@ make_queue_entry (struct GNUNET_DATASTORE_Handle *h,
451 pos, 452 pos,
452 ret); 453 ret);
453 h->queue_size++; 454 h->queue_size++;
454 if (c > max_queue_size)
455 {
456 GNUNET_STATISTICS_update (h->stats,
457 gettext_noop ("# queue overflows"),
458 1,
459 GNUNET_NO);
460 response_proc (ret, NULL);
461 return NULL;
462 }
463 ret->task = GNUNET_SCHEDULER_add_delayed (timeout, 455 ret->task = GNUNET_SCHEDULER_add_delayed (timeout,
464 &timeout_queue_entry, 456 &timeout_queue_entry,
465 ret); 457 ret);
@@ -469,7 +461,15 @@ make_queue_entry (struct GNUNET_DATASTORE_Handle *h,
469 if (pos->max_queue < h->queue_size) 461 if (pos->max_queue < h->queue_size)
470 { 462 {
471 GNUNET_assert (pos->response_proc != NULL); 463 GNUNET_assert (pos->response_proc != NULL);
472 pos->response_proc (pos, NULL); 464 /* move 'pos' element to head so that it will be
465 killed on 'NULL' call below */
466 GNUNET_CONTAINER_DLL_remove (h->queue_head,
467 h->queue_tail,
468 pos);
469 GNUNET_CONTAINER_DLL_insert (h->queue_head,
470 h->queue_tail,
471 pos);
472 pos->response_proc (h, NULL);
473 break; 473 break;
474 } 474 }
475 pos = pos->next; 475 pos = pos->next;
@@ -550,6 +550,7 @@ do_disconnect (struct GNUNET_DATASTORE_Handle *h)
550 GNUNET_NO); 550 GNUNET_NO);
551#endif 551#endif
552 GNUNET_CLIENT_disconnect (h->client, GNUNET_NO); 552 GNUNET_CLIENT_disconnect (h->client, GNUNET_NO);
553 h->skip_next_messages = 0;
553 h->client = NULL; 554 h->client = NULL;
554 h->reconnect_task = GNUNET_SCHEDULER_add_delayed (h->retry_time, 555 h->reconnect_task = GNUNET_SCHEDULER_add_delayed (h->retry_time,
555 &try_reconnect, 556 &try_reconnect,
@@ -700,6 +701,7 @@ free_queue_entry (struct GNUNET_DATASTORE_QueueEntry *qe)
700 qe->task = GNUNET_SCHEDULER_NO_TASK; 701 qe->task = GNUNET_SCHEDULER_NO_TASK;
701 } 702 }
702 h->queue_size--; 703 h->queue_size--;
704 qe->was_transmitted = GNUNET_SYSERR; /* use-after-free warning */
703 GNUNET_free (qe); 705 GNUNET_free (qe);
704} 706}
705 707
@@ -724,16 +726,22 @@ process_status_message (void *cls,
724 int was_transmitted; 726 int was_transmitted;
725 727
726 h->in_receive = GNUNET_NO; 728 h->in_receive = GNUNET_NO;
729 if (h->skip_next_messages > 0)
730 {
731 h->skip_next_messages--;
732 process_queue (h);
733 return;
734 }
727 if (NULL == (qe = h->queue_head)) 735 if (NULL == (qe = h->queue_head))
728 { 736 {
729 GNUNET_break (0); 737 GNUNET_break (0);
730 do_disconnect (h); 738 do_disconnect (h);
731 return; 739 return;
732 } 740 }
733 was_transmitted = qe->was_transmitted;
734 rc = qe->qc.sc; 741 rc = qe->qc.sc;
735 if (msg == NULL) 742 if (msg == NULL)
736 { 743 {
744 was_transmitted = qe->was_transmitted;
737 free_queue_entry (qe); 745 free_queue_entry (qe);
738 if (NULL == h->client) 746 if (NULL == h->client)
739 return; /* forced disconnect */ 747 return; /* forced disconnect */
@@ -1114,7 +1122,7 @@ GNUNET_DATASTORE_update (struct GNUNET_DATASTORE_Handle *h,
1114struct GNUNET_DATASTORE_QueueEntry * 1122struct GNUNET_DATASTORE_QueueEntry *
1115GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h, 1123GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h,
1116 const GNUNET_HashCode *key, 1124 const GNUNET_HashCode *key,
1117 size_t size, 1125 size_t size,
1118 const void *data, 1126 const void *data,
1119 unsigned int queue_priority, 1127 unsigned int queue_priority,
1120 unsigned int max_queue_size, 1128 unsigned int max_queue_size,
@@ -1186,45 +1194,35 @@ process_result_message (void *cls,
1186 struct GNUNET_DATASTORE_QueueEntry *qe; 1194 struct GNUNET_DATASTORE_QueueEntry *qe;
1187 struct ResultContext rc; 1195 struct ResultContext rc;
1188 const struct DataMessage *dm; 1196 const struct DataMessage *dm;
1189 int was_transmitted;
1190 1197
1191 h->in_receive = GNUNET_NO; 1198 h->in_receive = GNUNET_NO;
1199 if (h->skip_next_messages > 0)
1200 {
1201 h->skip_next_messages--;
1202 process_queue (h);
1203 return;
1204 }
1192 if (msg == NULL) 1205 if (msg == NULL)
1193 { 1206 {
1194 if (NULL != (qe = h->queue_head)) 1207 qe = h->queue_head;
1208 GNUNET_assert (NULL != qe);
1209 if (qe->was_transmitted == GNUNET_YES)
1195 { 1210 {
1196 was_transmitted = qe->was_transmitted;
1197 free_queue_entry (qe);
1198 rc = qe->qc.rc; 1211 rc = qe->qc.rc;
1199 if (was_transmitted == GNUNET_YES) 1212 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1200 { 1213 _("Failed to receive response from database.\n"));
1201 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 1214 do_disconnect (h);
1202 _("Failed to receive response from database.\n"));
1203 do_disconnect (h);
1204 }
1205 else
1206 {
1207#if DEBUG_DATASTORE
1208 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1209 "Request dropped due to finite datastore queue length.\n");
1210#endif
1211 }
1212 if (rc.iter != NULL)
1213 rc.iter (rc.iter_cls,
1214 NULL, 0, NULL, 0, 0, 0,
1215 GNUNET_TIME_UNIT_ZERO_ABS, 0);
1216 } 1215 }
1216 free_queue_entry (qe);
1217 if (rc.proc != NULL)
1218 rc.proc (rc.proc_cls,
1219 NULL, 0, NULL, 0, 0, 0,
1220 GNUNET_TIME_UNIT_ZERO_ABS, 0);
1217 return; 1221 return;
1218 } 1222 }
1219 if (ntohs(msg->type) == GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END) 1223 if (ntohs(msg->type) == GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END)
1220 { 1224 {
1221 GNUNET_break (ntohs(msg->size) == sizeof(struct GNUNET_MessageHeader)); 1225 GNUNET_break (ntohs(msg->size) == sizeof(struct GNUNET_MessageHeader));
1222 if (h->expect_end_or_disconnect > 0)
1223 {
1224 h->expect_end_or_disconnect--;
1225 process_queue (h);
1226 return;
1227 }
1228 qe = h->queue_head; 1226 qe = h->queue_head;
1229 rc = qe->qc.rc; 1227 rc = qe->qc.rc;
1230 GNUNET_assert (GNUNET_YES == qe->was_transmitted); 1228 GNUNET_assert (GNUNET_YES == qe->was_transmitted);
@@ -1234,8 +1232,8 @@ process_result_message (void *cls,
1234 "Received end of result set, new queue size is %u\n", 1232 "Received end of result set, new queue size is %u\n",
1235 h->queue_size); 1233 h->queue_size);
1236#endif 1234#endif
1237 if (rc.iter != NULL) 1235 if (rc.proc != NULL)
1238 rc.iter (rc.iter_cls, 1236 rc.proc (rc.proc_cls,
1239 NULL, 0, NULL, 0, 0, 0, 1237 NULL, 0, NULL, 0, 0, 0,
1240 GNUNET_TIME_UNIT_ZERO_ABS, 0); 1238 GNUNET_TIME_UNIT_ZERO_ABS, 0);
1241 h->retry_time.rel_value = 0; 1239 h->retry_time.rel_value = 0;
@@ -1243,13 +1241,6 @@ process_result_message (void *cls,
1243 process_queue (h); 1241 process_queue (h);
1244 return; 1242 return;
1245 } 1243 }
1246 if (h->expect_end_or_disconnect > 0)
1247 {
1248 /* only 'END' allowed, must reconnect */
1249 h->retry_time = GNUNET_TIME_UNIT_ZERO;
1250 do_disconnect (h);
1251 return;
1252 }
1253 qe = h->queue_head; 1244 qe = h->queue_head;
1254 rc = qe->qc.rc; 1245 rc = qe->qc.rc;
1255 GNUNET_assert (GNUNET_YES == qe->was_transmitted); 1246 GNUNET_assert (GNUNET_YES == qe->was_transmitted);
@@ -1261,40 +1252,16 @@ process_result_message (void *cls,
1261 free_queue_entry (qe); 1252 free_queue_entry (qe);
1262 h->retry_time = GNUNET_TIME_UNIT_ZERO; 1253 h->retry_time = GNUNET_TIME_UNIT_ZERO;
1263 do_disconnect (h); 1254 do_disconnect (h);
1264 if (rc.iter != NULL) 1255 if (rc.proc != NULL)
1265 rc.iter (rc.iter_cls, 1256 rc.proc (rc.proc_cls,
1266 NULL, 0, NULL, 0, 0, 0, 1257 NULL, 0, NULL, 0, 0, 0,
1267 GNUNET_TIME_UNIT_ZERO_ABS, 0); 1258 GNUNET_TIME_UNIT_ZERO_ABS, 0);
1268 return; 1259 return;
1269 } 1260 }
1270 GNUNET_STATISTICS_update (h->stats, 1261 GNUNET_STATISTICS_update (h->stats,
1271 gettext_noop ("# Results received"), 1262 gettext_noop ("# Results received"),
1272 1, 1263 1,
1273 GNUNET_NO); 1264 GNUNET_NO);
1274 if (rc.iter == NULL)
1275 {
1276 h->result_count++;
1277 GNUNET_STATISTICS_update (h->stats,
1278 gettext_noop ("# Excess results received"),
1279 1,
1280 GNUNET_NO);
1281 if (h->result_count > MAX_EXCESS_RESULTS)
1282 {
1283 free_queue_entry (qe);
1284 GNUNET_STATISTICS_update (h->stats,
1285 gettext_noop ("# Forced database connection resets"),
1286 1,
1287 GNUNET_NO);
1288 h->retry_time = GNUNET_TIME_UNIT_ZERO;
1289 do_disconnect (h);
1290 return;
1291 }
1292 if (GNUNET_YES == qe->one_shot)
1293 free_queue_entry (qe);
1294 else
1295 GNUNET_DATASTORE_iterate_get_next (h);
1296 return;
1297 }
1298 dm = (const struct DataMessage*) msg; 1265 dm = (const struct DataMessage*) msg;
1299#if DEBUG_DATASTORE 1266#if DEBUG_DATASTORE
1300 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1267 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1304,10 +1271,9 @@ process_result_message (void *cls,
1304 ntohl(dm->size), 1271 ntohl(dm->size),
1305 GNUNET_h2s(&dm->key)); 1272 GNUNET_h2s(&dm->key));
1306#endif 1273#endif
1307 if (GNUNET_YES == qe->one_shot) 1274 free_queue_entry (qe);
1308 free_queue_entry (qe);
1309 h->retry_time.rel_value = 0; 1275 h->retry_time.rel_value = 0;
1310 rc.iter (rc.iter_cls, 1276 rc.proc (rc.proc_cls,
1311 &dm->key, 1277 &dm->key,
1312 ntohl(dm->size), 1278 ntohl(dm->size),
1313 &dm[1], 1279 &dm[1],
@@ -1331,33 +1297,33 @@ process_result_message (void *cls,
1331 * @param max_queue_size at what queue size should this request be dropped 1297 * @param max_queue_size at what queue size should this request be dropped
1332 * (if other requests of higher priority are in the queue) 1298 * (if other requests of higher priority are in the queue)
1333 * @param timeout how long to wait at most for a response 1299 * @param timeout how long to wait at most for a response
1334 * @param iter function to call on a random value; it 1300 * @param proc function to call on a random value; it
1335 * will be called once with a value (if available) 1301 * will be called once with a value (if available)
1336 * and always once with a value of NULL. 1302 * and always once with a value of NULL.
1337 * @param iter_cls closure for iter 1303 * @param proc_cls closure for proc
1338 * @return NULL if the entry was not queued, otherwise a handle that can be used to 1304 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1339 * cancel; note that even if NULL is returned, the callback will be invoked 1305 * cancel
1340 * (or rather, will already have been invoked)
1341 */ 1306 */
1342struct GNUNET_DATASTORE_QueueEntry * 1307struct GNUNET_DATASTORE_QueueEntry *
1343GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h, 1308GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h,
1344 unsigned int queue_priority, 1309 unsigned int queue_priority,
1345 unsigned int max_queue_size, 1310 unsigned int max_queue_size,
1346 struct GNUNET_TIME_Relative timeout, 1311 struct GNUNET_TIME_Relative timeout,
1347 GNUNET_DATASTORE_Iterator iter, 1312 GNUNET_DATASTORE_DatumProcessor proc,
1348 void *iter_cls) 1313 void *proc_cls)
1349{ 1314{
1350 struct GNUNET_DATASTORE_QueueEntry *qe; 1315 struct GNUNET_DATASTORE_QueueEntry *qe;
1351 struct GNUNET_MessageHeader *m; 1316 struct GNUNET_MessageHeader *m;
1352 union QueueContext qc; 1317 union QueueContext qc;
1353 1318
1319 GNUNET_assert (NULL != proc);
1354#if DEBUG_DATASTORE 1320#if DEBUG_DATASTORE
1355 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1321 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1356 "Asked to get replication entry in %llu ms\n", 1322 "Asked to get replication entry in %llu ms\n",
1357 (unsigned long long) timeout.rel_value); 1323 (unsigned long long) timeout.rel_value);
1358#endif 1324#endif
1359 qc.rc.iter = iter; 1325 qc.rc.proc = proc;
1360 qc.rc.iter_cls = iter_cls; 1326 qc.rc.proc_cls = proc_cls;
1361 qe = make_queue_entry (h, sizeof(struct GNUNET_MessageHeader), 1327 qe = make_queue_entry (h, sizeof(struct GNUNET_MessageHeader),
1362 queue_priority, max_queue_size, timeout, 1328 queue_priority, max_queue_size, timeout,
1363 &process_result_message, &qc); 1329 &process_result_message, &qc);
@@ -1369,7 +1335,6 @@ GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h,
1369#endif 1335#endif
1370 return NULL; 1336 return NULL;
1371 } 1337 }
1372 qe->one_shot = GNUNET_YES;
1373 GNUNET_STATISTICS_update (h->stats, 1338 GNUNET_STATISTICS_update (h->stats,
1374 gettext_noop ("# GET REPLICATION requests executed"), 1339 gettext_noop ("# GET REPLICATION requests executed"),
1375 1, 1340 1,
@@ -1383,43 +1348,50 @@ GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h,
1383 1348
1384 1349
1385/** 1350/**
1386 * Get a zero-anonymity value from the datastore. 1351 * Get a single zero-anonymity value from the datastore.
1387 * 1352 *
1388 * @param h handle to the datastore 1353 * @param h handle to the datastore
1354 * @param offset offset of the result (mod #num-results); set to
1355 * a random 64-bit value initially; then increment by
1356 * one each time; detect that all results have been found by uid
1357 * being again the first uid ever returned.
1389 * @param queue_priority ranking of this request in the priority queue 1358 * @param queue_priority ranking of this request in the priority queue
1390 * @param max_queue_size at what queue size should this request be dropped 1359 * @param max_queue_size at what queue size should this request be dropped
1391 * (if other requests of higher priority are in the queue) 1360 * (if other requests of higher priority are in the queue)
1392 * @param timeout how long to wait at most for a response 1361 * @param timeout how long to wait at most for a response
1393 * @param type allowed type for the operation 1362 * @param type allowed type for the operation (never zero)
1394 * @param iter function to call on a random value; it 1363 * @param proc function to call on a random value; it
1395 * will be called once with a value (if available) 1364 * will be called once with a value (if available)
1396 * and always once with a value of NULL. 1365 * or with NULL if none value exists.
1397 * @param iter_cls closure for iter 1366 * @param proc_cls closure for proc
1398 * @return NULL if the entry was not queued, otherwise a handle that can be used to 1367 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1399 * cancel; note that even if NULL is returned, the callback will be invoked 1368 * cancel
1400 * (or rather, will already have been invoked)
1401 */ 1369 */
1402struct GNUNET_DATASTORE_QueueEntry * 1370struct GNUNET_DATASTORE_QueueEntry *
1403GNUNET_DATASTORE_iterate_zero_anonymity (struct GNUNET_DATASTORE_Handle *h, 1371GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
1404 unsigned int queue_priority, 1372 uint64_t offset,
1405 unsigned int max_queue_size, 1373 unsigned int queue_priority,
1406 struct GNUNET_TIME_Relative timeout, 1374 unsigned int max_queue_size,
1407 enum GNUNET_BLOCK_Type type, 1375 struct GNUNET_TIME_Relative timeout,
1408 GNUNET_DATASTORE_Iterator iter, 1376 enum GNUNET_BLOCK_Type type,
1409 void *iter_cls) 1377 GNUNET_DATASTORE_DatumProcessor proc,
1378 void *proc_cls)
1410{ 1379{
1411 struct GNUNET_DATASTORE_QueueEntry *qe; 1380 struct GNUNET_DATASTORE_QueueEntry *qe;
1412 struct GetZeroAnonymityMessage *m; 1381 struct GetZeroAnonymityMessage *m;
1413 union QueueContext qc; 1382 union QueueContext qc;
1414 1383
1384 GNUNET_assert (NULL != proc);
1415 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY); 1385 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
1416#if DEBUG_DATASTORE 1386#if DEBUG_DATASTORE
1417 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1387 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1418 "Asked to get zero-anonymity entry in %llu ms\n", 1388 "Asked to get %llu-th zero-anonymity entry of type %d in %llu ms\n",
1389 (unsigned long long) offset,
1390 type,
1419 (unsigned long long) timeout.rel_value); 1391 (unsigned long long) timeout.rel_value);
1420#endif 1392#endif
1421 qc.rc.iter = iter; 1393 qc.rc.proc = proc;
1422 qc.rc.iter_cls = iter_cls; 1394 qc.rc.proc_cls = proc_cls;
1423 qe = make_queue_entry (h, sizeof(struct GetZeroAnonymityMessage), 1395 qe = make_queue_entry (h, sizeof(struct GetZeroAnonymityMessage),
1424 queue_priority, max_queue_size, timeout, 1396 queue_priority, max_queue_size, timeout,
1425 &process_result_message, &qc); 1397 &process_result_message, &qc);
@@ -1427,7 +1399,7 @@ GNUNET_DATASTORE_iterate_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
1427 { 1399 {
1428#if DEBUG_DATASTORE 1400#if DEBUG_DATASTORE
1429 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1401 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1430 "Could not create queue entry for zero-anonymity iteration\n"); 1402 "Could not create queue entry for zero-anonymity procation\n");
1431#endif 1403#endif
1432 return NULL; 1404 return NULL;
1433 } 1405 }
@@ -1439,55 +1411,57 @@ GNUNET_DATASTORE_iterate_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
1439 m->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY); 1411 m->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY);
1440 m->header.size = htons(sizeof (struct GetZeroAnonymityMessage)); 1412 m->header.size = htons(sizeof (struct GetZeroAnonymityMessage));
1441 m->type = htonl ((uint32_t) type); 1413 m->type = htonl ((uint32_t) type);
1414 m->offset = GNUNET_htonll (offset);
1442 process_queue (h); 1415 process_queue (h);
1443 return qe; 1416 return qe;
1444} 1417}
1445 1418
1446 1419
1447
1448/** 1420/**
1449 * Iterate over the results for a particular key 1421 * Get a result for a particular key from the datastore. The processor
1450 * in the datastore. The iterator will only be called 1422 * will only be called once.
1451 * once initially; if the first call did contain a
1452 * result, further results can be obtained by calling
1453 * "GNUNET_DATASTORE_iterate_get_next" with the given argument.
1454 * 1423 *
1455 * @param h handle to the datastore 1424 * @param h handle to the datastore
1425 * @param offset offset of the result (mod #num-results); set to
1426 * a random 64-bit value initially; then increment by
1427 * one each time; detect that all results have been found by uid
1428 * being again the first uid ever returned.
1456 * @param key maybe NULL (to match all entries) 1429 * @param key maybe NULL (to match all entries)
1457 * @param type desired type, 0 for any 1430 * @param type desired type, 0 for any
1458 * @param queue_priority ranking of this request in the priority queue 1431 * @param queue_priority ranking of this request in the priority queue
1459 * @param max_queue_size at what queue size should this request be dropped 1432 * @param max_queue_size at what queue size should this request be dropped
1460 * (if other requests of higher priority are in the queue) 1433 * (if other requests of higher priority are in the queue)
1461 * @param timeout how long to wait at most for a response 1434 * @param timeout how long to wait at most for a response
1462 * @param iter function to call on each matching value; 1435 * @param proc function to call on each matching value;
1463 * will be called once with a NULL value at the end 1436 * will be called once with a NULL value at the end
1464 * @param iter_cls closure for iter 1437 * @param proc_cls closure for proc
1465 * @return NULL if the entry was not queued, otherwise a handle that can be used to 1438 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1466 * cancel; note that even if NULL is returned, the callback will be invoked 1439 * cancel
1467 * (or rather, will already have been invoked)
1468 */ 1440 */
1469struct GNUNET_DATASTORE_QueueEntry * 1441struct GNUNET_DATASTORE_QueueEntry *
1470GNUNET_DATASTORE_iterate_key (struct GNUNET_DATASTORE_Handle *h, 1442GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
1471 const GNUNET_HashCode * key, 1443 uint64_t offset,
1472 enum GNUNET_BLOCK_Type type, 1444 const GNUNET_HashCode * key,
1473 unsigned int queue_priority, 1445 enum GNUNET_BLOCK_Type type,
1474 unsigned int max_queue_size, 1446 unsigned int queue_priority,
1475 struct GNUNET_TIME_Relative timeout, 1447 unsigned int max_queue_size,
1476 GNUNET_DATASTORE_Iterator iter, 1448 struct GNUNET_TIME_Relative timeout,
1477 void *iter_cls) 1449 GNUNET_DATASTORE_DatumProcessor proc,
1450 void *proc_cls)
1478{ 1451{
1479 struct GNUNET_DATASTORE_QueueEntry *qe; 1452 struct GNUNET_DATASTORE_QueueEntry *qe;
1480 struct GetMessage *gm; 1453 struct GetMessage *gm;
1481 union QueueContext qc; 1454 union QueueContext qc;
1482 1455
1456 GNUNET_assert (NULL != proc);
1483#if DEBUG_DATASTORE 1457#if DEBUG_DATASTORE
1484 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1458 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1485 "Asked to look for data of type %u under key `%s'\n", 1459 "Asked to look for data of type %u under key `%s'\n",
1486 (unsigned int) type, 1460 (unsigned int) type,
1487 GNUNET_h2s (key)); 1461 GNUNET_h2s (key));
1488#endif 1462#endif
1489 qc.rc.iter = iter; 1463 qc.rc.proc = proc;
1490 qc.rc.iter_cls = iter_cls; 1464 qc.rc.proc_cls = proc_cls;
1491 qe = make_queue_entry (h, sizeof(struct GetMessage), 1465 qe = make_queue_entry (h, sizeof(struct GetMessage),
1492 queue_priority, max_queue_size, timeout, 1466 queue_priority, max_queue_size, timeout,
1493 &process_result_message, &qc); 1467 &process_result_message, &qc);
@@ -1507,6 +1481,7 @@ GNUNET_DATASTORE_iterate_key (struct GNUNET_DATASTORE_Handle *h,
1507 gm = (struct GetMessage*) &qe[1]; 1481 gm = (struct GetMessage*) &qe[1];
1508 gm->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_GET); 1482 gm->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_GET);
1509 gm->type = htonl(type); 1483 gm->type = htonl(type);
1484 gm->offset = GNUNET_htonll (offset);
1510 if (key != NULL) 1485 if (key != NULL)
1511 { 1486 {
1512 gm->header.size = htons(sizeof (struct GetMessage)); 1487 gm->header.size = htons(sizeof (struct GetMessage));
@@ -1522,25 +1497,6 @@ GNUNET_DATASTORE_iterate_key (struct GNUNET_DATASTORE_Handle *h,
1522 1497
1523 1498
1524/** 1499/**
1525 * Function called to trigger obtaining the next result
1526 * from the datastore.
1527 *
1528 * @param h handle to the datastore
1529 */
1530void
1531GNUNET_DATASTORE_iterate_get_next (struct GNUNET_DATASTORE_Handle *h)
1532{
1533 struct GNUNET_DATASTORE_QueueEntry *qe = h->queue_head;
1534
1535 h->in_receive = GNUNET_YES;
1536 GNUNET_CLIENT_receive (h->client,
1537 &process_result_message,
1538 h,
1539 GNUNET_TIME_absolute_get_remaining (qe->timeout));
1540}
1541
1542
1543/**
1544 * Cancel a datastore operation. The final callback from the 1500 * Cancel a datastore operation. The final callback from the
1545 * operation must not have been done yet. 1501 * operation must not have been done yet.
1546 * 1502 *
@@ -1551,6 +1507,7 @@ GNUNET_DATASTORE_cancel (struct GNUNET_DATASTORE_QueueEntry *qe)
1551{ 1507{
1552 struct GNUNET_DATASTORE_Handle *h; 1508 struct GNUNET_DATASTORE_Handle *h;
1553 1509
1510 GNUNET_assert (GNUNET_SYSERR != qe->was_transmitted);
1554 h = qe->h; 1511 h = qe->h;
1555#if DEBUG_DATASTORE 1512#if DEBUG_DATASTORE
1556 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1513 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1562,7 +1519,7 @@ GNUNET_DATASTORE_cancel (struct GNUNET_DATASTORE_QueueEntry *qe)
1562 if (GNUNET_YES == qe->was_transmitted) 1519 if (GNUNET_YES == qe->was_transmitted)
1563 { 1520 {
1564 free_queue_entry (qe); 1521 free_queue_entry (qe);
1565 h->expect_end_or_disconnect++; 1522 h->skip_next_messages++;
1566 return; 1523 return;
1567 } 1524 }
1568 free_queue_entry (qe); 1525 free_queue_entry (qe);
diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c
index 566a227c1..deab62dd0 100644
--- a/src/datastore/gnunet-service-datastore.c
+++ b/src/datastore/gnunet-service-datastore.c
@@ -209,18 +209,6 @@ sync_stats ()
209 209
210 210
211 211
212
213/**
214 * Function called once the transmit operation has
215 * either failed or succeeded.
216 *
217 * @param cls closure
218 * @param status GNUNET_OK on success, GNUNET_SYSERR on error
219 */
220typedef void (*TransmitContinuation)(void *cls,
221 int status);
222
223
224/** 212/**
225 * Context for transmitting replies to clients. 213 * Context for transmitting replies to clients.
226 */ 214 */
@@ -252,22 +240,6 @@ struct TransmitCallbackContext
252 */ 240 */
253 struct GNUNET_SERVER_Client *client; 241 struct GNUNET_SERVER_Client *client;
254 242
255 /**
256 * Function to call once msg has been transmitted
257 * (or at least added to the buffer).
258 */
259 TransmitContinuation tc;
260
261 /**
262 * Closure for tc.
263 */
264 void *tc_cls;
265
266 /**
267 * GNUNET_YES if we are supposed to signal the server
268 * completion of the client's request.
269 */
270 int end;
271}; 243};
272 244
273 245
@@ -330,7 +302,6 @@ delete_expired (void *cls,
330 */ 302 */
331static int 303static int
332expired_processor (void *cls, 304expired_processor (void *cls,
333 void *next_cls,
334 const GNUNET_HashCode * key, 305 const GNUNET_HashCode * key,
335 uint32_t size, 306 uint32_t size,
336 const void *data, 307 const void *data,
@@ -396,7 +367,7 @@ delete_expired (void *cls,
396 const struct GNUNET_SCHEDULER_TaskContext *tc) 367 const struct GNUNET_SCHEDULER_TaskContext *tc)
397{ 368{
398 expired_kill_task = GNUNET_SCHEDULER_NO_TASK; 369 expired_kill_task = GNUNET_SCHEDULER_NO_TASK;
399 plugin->api->expiration_get (plugin->api->cls, 370 plugin->api->get_expiration (plugin->api->cls,
400 &expired_processor, 371 &expired_processor,
401 NULL); 372 NULL);
402} 373}
@@ -424,7 +395,6 @@ delete_expired (void *cls,
424 */ 395 */
425static int 396static int
426quota_processor (void *cls, 397quota_processor (void *cls,
427 void *next_cls,
428 const GNUNET_HashCode * key, 398 const GNUNET_HashCode * key,
429 uint32_t size, 399 uint32_t size,
430 const void *data, 400 const void *data,
@@ -487,7 +457,7 @@ manage_space (unsigned long long need)
487 (last != need) ) 457 (last != need) )
488 { 458 {
489 last = need; 459 last = need;
490 plugin->api->expiration_get (plugin->api->cls, 460 plugin->api->get_expiration (plugin->api->cls,
491 &quota_processor, 461 &quota_processor,
492 &need); 462 &need);
493 } 463 }
@@ -521,14 +491,7 @@ transmit_callback (void *cls,
521 { 491 {
522 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 492 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
523 _("Transmission to client failed!\n")); 493 _("Transmission to client failed!\n"));
524 if (tcc->tc != NULL) 494 GNUNET_SERVER_receive_done (tcc->client, GNUNET_SYSERR);
525 tcc->tc (tcc->tc_cls, GNUNET_SYSERR);
526 if (GNUNET_YES == tcc->end)
527 {
528 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
529 _("Disconnecting client due to transmission failure!\n"));
530 GNUNET_SERVER_receive_done (tcc->client, GNUNET_SYSERR);
531 }
532 GNUNET_SERVER_client_drop (tcc->client); 495 GNUNET_SERVER_client_drop (tcc->client);
533 GNUNET_free (tcc->msg); 496 GNUNET_free (tcc->msg);
534 GNUNET_free (tcc); 497 GNUNET_free (tcc);
@@ -536,23 +499,7 @@ transmit_callback (void *cls,
536 } 499 }
537 GNUNET_assert (size >= msize); 500 GNUNET_assert (size >= msize);
538 memcpy (buf, tcc->msg, msize); 501 memcpy (buf, tcc->msg, msize);
539 if (tcc->tc != NULL) 502 GNUNET_SERVER_receive_done (tcc->client, GNUNET_OK);
540 tcc->tc (tcc->tc_cls, GNUNET_OK);
541 if (GNUNET_YES == tcc->end)
542 {
543#if DEBUG_DATASTORE
544 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
545 "Done processing client request\n");
546#endif
547 GNUNET_SERVER_receive_done (tcc->client, GNUNET_OK);
548 }
549 else
550 {
551#if DEBUG_DATASTORE
552 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
553 "Response transmitted, more pending!\n");
554#endif
555 }
556 GNUNET_SERVER_client_drop (tcc->client); 503 GNUNET_SERVER_client_drop (tcc->client);
557 GNUNET_free (tcc->msg); 504 GNUNET_free (tcc->msg);
558 GNUNET_free (tcc); 505 GNUNET_free (tcc);
@@ -567,16 +514,10 @@ transmit_callback (void *cls,
567 * @param msg message to transmit, will be freed! 514 * @param msg message to transmit, will be freed!
568 * @param tc function to call afterwards 515 * @param tc function to call afterwards
569 * @param tc_cls closure for tc 516 * @param tc_cls closure for tc
570 * @param end is this the last response (and we should
571 * signal the server completion accodingly after
572 * transmitting this message)?
573 */ 517 */
574static void 518static void
575transmit (struct GNUNET_SERVER_Client *client, 519transmit (struct GNUNET_SERVER_Client *client,
576 struct GNUNET_MessageHeader *msg, 520 struct GNUNET_MessageHeader *msg)
577 TransmitContinuation tc,
578 void *tc_cls,
579 int end)
580{ 521{
581 struct TransmitCallbackContext *tcc; 522 struct TransmitCallbackContext *tcc;
582 523
@@ -586,17 +527,13 @@ transmit (struct GNUNET_SERVER_Client *client,
586 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 527 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
587 "Shutdown in progress, aborting transmission.\n"); 528 "Shutdown in progress, aborting transmission.\n");
588#endif 529#endif
530 GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
589 GNUNET_free (msg); 531 GNUNET_free (msg);
590 if (NULL != tc)
591 tc (tc_cls, GNUNET_SYSERR);
592 return; 532 return;
593 } 533 }
594 tcc = GNUNET_malloc (sizeof(struct TransmitCallbackContext)); 534 tcc = GNUNET_malloc (sizeof(struct TransmitCallbackContext));
595 tcc->msg = msg; 535 tcc->msg = msg;
596 tcc->client = client; 536 tcc->client = client;
597 tcc->tc = tc;
598 tcc->tc_cls = tc_cls;
599 tcc->end = end;
600 if (NULL == 537 if (NULL ==
601 (tcc->th = GNUNET_SERVER_notify_transmit_ready (client, 538 (tcc->th = GNUNET_SERVER_notify_transmit_ready (client,
602 ntohs(msg->size), 539 ntohs(msg->size),
@@ -605,14 +542,7 @@ transmit (struct GNUNET_SERVER_Client *client,
605 tcc))) 542 tcc)))
606 { 543 {
607 GNUNET_break (0); 544 GNUNET_break (0);
608 if (GNUNET_YES == end) 545 GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
609 {
610 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
611 _("Forcefully disconnecting client.\n"));
612 GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
613 }
614 if (NULL != tc)
615 tc (tc_cls, GNUNET_SYSERR);
616 GNUNET_free (msg); 546 GNUNET_free (msg);
617 GNUNET_free (tcc); 547 GNUNET_free (tcc);
618 return; 548 return;
@@ -653,33 +583,10 @@ transmit_status (struct GNUNET_SERVER_Client *client,
653 sm->status = htonl(code); 583 sm->status = htonl(code);
654 if (slen > 0) 584 if (slen > 0)
655 memcpy (&sm[1], msg, slen); 585 memcpy (&sm[1], msg, slen);
656 transmit (client, &sm->header, NULL, NULL, GNUNET_YES); 586 transmit (client, &sm->header);
657} 587}
658 588
659 589
660/**
661 * Function called once the transmit operation has
662 * either failed or succeeded.
663 *
664 * @param next_cls closure for calling "next_request" callback
665 * @param status GNUNET_OK on success, GNUNET_SYSERR on error
666 */
667static void
668get_next(void *next_cls,
669 int status)
670{
671 if (status != GNUNET_OK)
672 {
673 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
674 _("Failed to transmit an item to the client; aborting iteration.\n"));
675 if (plugin != NULL)
676 plugin->api->next_request (next_cls, GNUNET_YES);
677 return;
678 }
679 if (next_cls != NULL)
680 plugin->api->next_request (next_cls, GNUNET_NO);
681}
682
683 590
684/** 591/**
685 * Function that will transmit the given datastore entry 592 * Function that will transmit the given datastore entry
@@ -702,7 +609,6 @@ get_next(void *next_cls,
702 */ 609 */
703static int 610static int
704transmit_item (void *cls, 611transmit_item (void *cls,
705 void *next_cls,
706 const GNUNET_HashCode * key, 612 const GNUNET_HashCode * key,
707 uint32_t size, 613 uint32_t size,
708 const void *data, 614 const void *data,
@@ -727,10 +633,11 @@ transmit_item (void *cls,
727 end = GNUNET_malloc (sizeof(struct GNUNET_MessageHeader)); 633 end = GNUNET_malloc (sizeof(struct GNUNET_MessageHeader));
728 end->size = htons(sizeof(struct GNUNET_MessageHeader)); 634 end->size = htons(sizeof(struct GNUNET_MessageHeader));
729 end->type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END); 635 end->type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END);
730 transmit (client, end, NULL, NULL, GNUNET_YES); 636 transmit (client, end);
731 GNUNET_SERVER_client_drop (client); 637 GNUNET_SERVER_client_drop (client);
732 return GNUNET_OK; 638 return GNUNET_OK;
733 } 639 }
640 GNUNET_assert (sizeof (struct DataMessage) + size < GNUNET_SERVER_MAX_MESSAGE_SIZE);
734 dm = GNUNET_malloc (sizeof(struct DataMessage) + size); 641 dm = GNUNET_malloc (sizeof(struct DataMessage) + size);
735 dm->header.size = htons(sizeof(struct DataMessage) + size); 642 dm->header.size = htons(sizeof(struct DataMessage) + size);
736 dm->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_DATA); 643 dm->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
@@ -754,8 +661,7 @@ transmit_item (void *cls,
754 gettext_noop ("# results found"), 661 gettext_noop ("# results found"),
755 1, 662 1,
756 GNUNET_NO); 663 GNUNET_NO);
757 transmit (client, &dm->header, &get_next, next_cls, 664 transmit (client, &dm->header);
758 (next_cls != NULL) ? GNUNET_NO : GNUNET_YES);
759 return GNUNET_OK; 665 return GNUNET_OK;
760} 666}
761 667
@@ -939,11 +845,6 @@ struct PutContext
939 * Client to notify on completion. 845 * Client to notify on completion.
940 */ 846 */
941 struct GNUNET_SERVER_Client *client; 847 struct GNUNET_SERVER_Client *client;
942
943 /**
944 * Did we find the data already in the database?
945 */
946 int is_present;
947 848
948 /* followed by the 'struct DataMessage' */ 849 /* followed by the 'struct DataMessage' */
949}; 850};
@@ -1009,7 +910,6 @@ execute_put (struct GNUNET_SERVER_Client *client,
1009 * matches the put and if none match executes the put. 910 * matches the put and if none match executes the put.
1010 * 911 *
1011 * @param cls closure, pointer to the client (of type 'struct PutContext'). 912 * @param cls closure, pointer to the client (of type 'struct PutContext').
1012 * @param next_cls closure to use to ask for the next item
1013 * @param key key for the content 913 * @param key key for the content
1014 * @param size number of bytes in data 914 * @param size number of bytes in data
1015 * @param data content stored 915 * @param data content stored
@@ -1020,12 +920,11 @@ execute_put (struct GNUNET_SERVER_Client *client,
1020 * @param uid unique identifier for the datum; 920 * @param uid unique identifier for the datum;
1021 * maybe 0 if no unique identifier is available 921 * maybe 0 if no unique identifier is available
1022 * 922 *
1023 * @return GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue, 923 * @return GNUNET_OK usually
1024 * GNUNET_NO to delete the item and continue (if supported) 924 * GNUNET_NO to delete the item
1025 */ 925 */
1026static int 926static int
1027check_present (void *cls, 927check_present (void *cls,
1028 void *next_cls,
1029 const GNUNET_HashCode * key, 928 const GNUNET_HashCode * key,
1030 uint32_t size, 929 uint32_t size,
1031 const void *data, 930 const void *data,
@@ -1041,13 +940,10 @@ check_present (void *cls,
1041 dm = (const struct DataMessage*) &pc[1]; 940 dm = (const struct DataMessage*) &pc[1];
1042 if (key == NULL) 941 if (key == NULL)
1043 { 942 {
1044 if (pc->is_present == GNUNET_YES) 943 execute_put (pc->client, dm);
1045 transmit_status (pc->client, GNUNET_NO, NULL);
1046 else
1047 execute_put (pc->client, dm);
1048 GNUNET_SERVER_client_drop (pc->client); 944 GNUNET_SERVER_client_drop (pc->client);
1049 GNUNET_free (pc); 945 GNUNET_free (pc);
1050 return GNUNET_SYSERR; 946 return GNUNET_OK;
1051 } 947 }
1052 if ( (GNUNET_BLOCK_TYPE_FS_DBLOCK == type) || 948 if ( (GNUNET_BLOCK_TYPE_FS_DBLOCK == type) ||
1053 (GNUNET_BLOCK_TYPE_FS_IBLOCK == type) || 949 (GNUNET_BLOCK_TYPE_FS_IBLOCK == type) ||
@@ -1056,12 +952,19 @@ check_present (void *cls,
1056 data, 952 data,
1057 size)) ) ) 953 size)) ) )
1058 { 954 {
1059 pc->is_present = GNUNET_YES; 955#if DEBUG_MYSQL
1060 plugin->api->next_request (next_cls, GNUNET_YES); 956 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
957 "Result already present in datastore\n");
958#endif
959 transmit_status (pc->client, GNUNET_NO, NULL);
960 GNUNET_SERVER_client_drop (pc->client);
961 GNUNET_free (pc);
1061 } 962 }
1062 else 963 else
1063 { 964 {
1064 plugin->api->next_request (next_cls, GNUNET_NO); 965 execute_put (pc->client, dm);
966 GNUNET_SERVER_client_drop (pc->client);
967 GNUNET_free (pc);
1065 } 968 }
1066 return GNUNET_OK; 969 return GNUNET_OK;
1067} 970}
@@ -1083,6 +986,7 @@ handle_put (void *cls,
1083 int rid; 986 int rid;
1084 struct ReservationList *pos; 987 struct ReservationList *pos;
1085 struct PutContext *pc; 988 struct PutContext *pc;
989 GNUNET_HashCode vhash;
1086 uint32_t size; 990 uint32_t size;
1087 991
1088 if ( (dm == NULL) || 992 if ( (dm == NULL) ||
@@ -1124,16 +1028,18 @@ handle_put (void *cls,
1124 if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (filter, 1028 if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (filter,
1125 &dm->key)) 1029 &dm->key))
1126 { 1030 {
1031 GNUNET_CRYPTO_hash (&dm[1], size, &vhash);
1127 pc = GNUNET_malloc (sizeof (struct PutContext) + size + sizeof (struct DataMessage)); 1032 pc = GNUNET_malloc (sizeof (struct PutContext) + size + sizeof (struct DataMessage));
1128 pc->client = client; 1033 pc->client = client;
1129 GNUNET_SERVER_client_keep (client); 1034 GNUNET_SERVER_client_keep (client);
1130 memcpy (&pc[1], dm, size + sizeof (struct DataMessage)); 1035 memcpy (&pc[1], dm, size + sizeof (struct DataMessage));
1131 plugin->api->get (plugin->api->cls, 1036 plugin->api->get_key (plugin->api->cls,
1132 &dm->key, 1037 0,
1133 NULL, 1038 &dm->key,
1134 ntohl (dm->type), 1039 &vhash,
1135 &check_present, 1040 ntohl (dm->type),
1136 pc); 1041 &check_present,
1042 pc);
1137 return; 1043 return;
1138 } 1044 }
1139 execute_put (client, dm); 1045 execute_put (client, dm);
@@ -1192,16 +1098,17 @@ handle_get (void *cls,
1192 1, 1098 1,
1193 GNUNET_NO); 1099 GNUNET_NO);
1194 transmit_item (client, 1100 transmit_item (client,
1195 NULL, NULL, 0, NULL, 0, 0, 0, 1101 NULL, 0, NULL, 0, 0, 0,
1196 GNUNET_TIME_UNIT_ZERO_ABS, 0); 1102 GNUNET_TIME_UNIT_ZERO_ABS, 0);
1197 return; 1103 return;
1198 } 1104 }
1199 plugin->api->get (plugin->api->cls, 1105 plugin->api->get_key (plugin->api->cls,
1200 ((size == sizeof(struct GetMessage)) ? &msg->key : NULL), 1106 GNUNET_ntohll (msg->offset),
1201 NULL, 1107 ((size == sizeof(struct GetMessage)) ? &msg->key : NULL),
1202 ntohl(msg->type), 1108 NULL,
1203 &transmit_item, 1109 ntohl(msg->type),
1204 client); 1110 &transmit_item,
1111 client);
1205} 1112}
1206 1113
1207 1114
@@ -1265,7 +1172,7 @@ handle_get_replication (void *cls,
1265 1, 1172 1,
1266 GNUNET_NO); 1173 GNUNET_NO);
1267 GNUNET_SERVER_client_keep (client); 1174 GNUNET_SERVER_client_keep (client);
1268 plugin->api->replication_get (plugin->api->cls, 1175 plugin->api->get_replication (plugin->api->cls,
1269 &transmit_item, 1176 &transmit_item,
1270 client); 1177 client);
1271} 1178}
@@ -1303,37 +1210,20 @@ handle_get_zero_anonymity (void *cls,
1303 1, 1210 1,
1304 GNUNET_NO); 1211 GNUNET_NO);
1305 GNUNET_SERVER_client_keep (client); 1212 GNUNET_SERVER_client_keep (client);
1306 plugin->api->iter_zero_anonymity (plugin->api->cls, 1213 plugin->api->get_zero_anonymity (plugin->api->cls,
1307 type, 1214 GNUNET_ntohll (msg->offset),
1308 &transmit_item, 1215 type,
1309 client); 1216 &transmit_item,
1217 client);
1310} 1218}
1311 1219
1312 1220
1313/** 1221/**
1314 * Context for the 'remove_callback'.
1315 */
1316struct RemoveContext
1317{
1318 /**
1319 * Client for whom we're doing the remvoing.
1320 */
1321 struct GNUNET_SERVER_Client *client;
1322
1323 /**
1324 * GNUNET_YES if we managed to remove something.
1325 */
1326 int found;
1327};
1328
1329
1330/**
1331 * Callback function that will cause the item that is passed 1222 * Callback function that will cause the item that is passed
1332 * in to be deleted (by returning GNUNET_NO). 1223 * in to be deleted (by returning GNUNET_NO).
1333 */ 1224 */
1334static int 1225static int
1335remove_callback (void *cls, 1226remove_callback (void *cls,
1336 void *next_cls,
1337 const GNUNET_HashCode * key, 1227 const GNUNET_HashCode * key,
1338 uint32_t size, 1228 uint32_t size,
1339 const void *data, 1229 const void *data,
@@ -1343,7 +1233,7 @@ remove_callback (void *cls,
1343 struct GNUNET_TIME_Absolute 1233 struct GNUNET_TIME_Absolute
1344 expiration, uint64_t uid) 1234 expiration, uint64_t uid)
1345{ 1235{
1346 struct RemoveContext *rc = cls; 1236 struct GNUNET_SERVER_Client *client = cls;
1347 1237
1348 if (key == NULL) 1238 if (key == NULL)
1349 { 1239 {
@@ -1352,15 +1242,10 @@ remove_callback (void *cls,
1352 "No further matches for `%s' request.\n", 1242 "No further matches for `%s' request.\n",
1353 "REMOVE"); 1243 "REMOVE");
1354#endif 1244#endif
1355 if (GNUNET_YES == rc->found) 1245 transmit_status (client, GNUNET_NO, _("Content not found"));
1356 transmit_status (rc->client, GNUNET_OK, NULL); 1246 GNUNET_SERVER_client_drop (client);
1357 else
1358 transmit_status (rc->client, GNUNET_NO, _("Content not found"));
1359 GNUNET_SERVER_client_drop (rc->client);
1360 GNUNET_free (rc);
1361 return GNUNET_OK; /* last item */ 1247 return GNUNET_OK; /* last item */
1362 } 1248 }
1363 rc->found = GNUNET_YES;
1364#if DEBUG_DATASTORE 1249#if DEBUG_DATASTORE
1365 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1250 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1366 "Item %llu matches `%s' request for key `%s' and type %u.\n", 1251 "Item %llu matches `%s' request for key `%s' and type %u.\n",
@@ -1375,7 +1260,8 @@ remove_callback (void *cls,
1375 GNUNET_YES); 1260 GNUNET_YES);
1376 GNUNET_CONTAINER_bloomfilter_remove (filter, 1261 GNUNET_CONTAINER_bloomfilter_remove (filter,
1377 key); 1262 key);
1378 plugin->api->next_request (next_cls, GNUNET_YES); 1263 transmit_status (client, GNUNET_OK, NULL);
1264 GNUNET_SERVER_client_drop (client);
1379 return GNUNET_NO; 1265 return GNUNET_NO;
1380} 1266}
1381 1267
@@ -1394,7 +1280,6 @@ handle_remove (void *cls,
1394{ 1280{
1395 const struct DataMessage *dm = check_data (message); 1281 const struct DataMessage *dm = check_data (message);
1396 GNUNET_HashCode vhash; 1282 GNUNET_HashCode vhash;
1397 struct RemoveContext *rc;
1398 1283
1399 if (dm == NULL) 1284 if (dm == NULL)
1400 { 1285 {
@@ -1413,18 +1298,17 @@ handle_remove (void *cls,
1413 gettext_noop ("# REMOVE requests received"), 1298 gettext_noop ("# REMOVE requests received"),
1414 1, 1299 1,
1415 GNUNET_NO); 1300 GNUNET_NO);
1416 rc = GNUNET_malloc (sizeof(struct RemoveContext));
1417 GNUNET_SERVER_client_keep (client); 1301 GNUNET_SERVER_client_keep (client);
1418 rc->client = client;
1419 GNUNET_CRYPTO_hash (&dm[1], 1302 GNUNET_CRYPTO_hash (&dm[1],
1420 ntohl(dm->size), 1303 ntohl(dm->size),
1421 &vhash); 1304 &vhash);
1422 plugin->api->get (plugin->api->cls, 1305 plugin->api->get_key (plugin->api->cls,
1423 &dm->key, 1306 0,
1424 &vhash, 1307 &dm->key,
1425 (enum GNUNET_BLOCK_Type) ntohl(dm->type), 1308 &vhash,
1426 &remove_callback, 1309 (enum GNUNET_BLOCK_Type) ntohl(dm->type),
1427 rc); 1310 &remove_callback,
1311 client);
1428} 1312}
1429 1313
1430 1314
@@ -1469,7 +1353,7 @@ disk_utilization_change_cb (void *cls,
1469 _("Datastore payload inaccurate (%lld < %lld). Trying to fix.\n"), 1353 _("Datastore payload inaccurate (%lld < %lld). Trying to fix.\n"),
1470 (long long) payload, 1354 (long long) payload,
1471 (long long) -delta); 1355 (long long) -delta);
1472 payload = plugin->api->get_size (plugin->api->cls); 1356 payload = plugin->api->estimate_size (plugin->api->cls);
1473 sync_stats (); 1357 sync_stats ();
1474 return; 1358 return;
1475 } 1359 }
@@ -1518,7 +1402,7 @@ process_stat_done (void *cls,
1518 1402
1519 stat_get = NULL; 1403 stat_get = NULL;
1520 if (stats_worked == GNUNET_NO) 1404 if (stats_worked == GNUNET_NO)
1521 payload = plugin->api->get_size (plugin->api->cls); 1405 payload = plugin->api->estimate_size (plugin->api->cls);
1522} 1406}
1523 1407
1524 1408
@@ -1636,8 +1520,6 @@ cleaning_task (void *cls,
1636 GNUNET_CONNECTION_notify_transmit_ready_cancel (tcc->th); 1520 GNUNET_CONNECTION_notify_transmit_ready_cancel (tcc->th);
1637 GNUNET_SERVER_client_drop (tcc->client); 1521 GNUNET_SERVER_client_drop (tcc->client);
1638 } 1522 }
1639 if (NULL != tcc->tc)
1640 tcc->tc (tcc->tc_cls, GNUNET_SYSERR);
1641 GNUNET_free (tcc->msg); 1523 GNUNET_free (tcc->msg);
1642 GNUNET_free (tcc); 1524 GNUNET_free (tcc);
1643 } 1525 }
diff --git a/src/datastore/perf_datastore_api.c b/src/datastore/perf_datastore_api.c
index 6ea65c68d..2f0eb0de9 100644
--- a/src/datastore/perf_datastore_api.c
+++ b/src/datastore/perf_datastore_api.c
@@ -385,6 +385,7 @@ check ()
385 GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, 385 GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
386 argv, "perf-datastore-api", "nohelp", 386 argv, "perf-datastore-api", "nohelp",
387 options, &run, NULL); 387 options, &run, NULL);
388 sleep (1); /* give datastore chance to process 'DROP' */
388 if (0 != GNUNET_OS_process_kill (proc, SIGTERM)) 389 if (0 != GNUNET_OS_process_kill (proc, SIGTERM))
389 { 390 {
390 GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); 391 GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill");
diff --git a/src/datastore/perf_plugin_datastore.c b/src/datastore/perf_plugin_datastore.c
index 2903a8f28..6befa120c 100644
--- a/src/datastore/perf_plugin_datastore.c
+++ b/src/datastore/perf_plugin_datastore.c
@@ -37,7 +37,7 @@
37 * those take too long to run them in the usual "make check" 37 * those take too long to run them in the usual "make check"
38 * sequence. Hence the value used for shipping is tiny. 38 * sequence. Hence the value used for shipping is tiny.
39 */ 39 */
40#define MAX_SIZE 1024LL * 1024 * 128 40#define MAX_SIZE 1024LL * 1024 * 32
41 41
42#define ITERATIONS 2 42#define ITERATIONS 2
43 43
@@ -81,6 +81,7 @@ struct CpsRunContext
81 enum RunPhase phase; 81 enum RunPhase phase;
82 unsigned int cnt; 82 unsigned int cnt;
83 unsigned int iter; 83 unsigned int iter;
84 uint64_t offset;
84}; 85};
85 86
86 87
@@ -100,7 +101,8 @@ disk_utilization_change_cb (void *cls,
100 101
101 102
102static void 103static void
103putValue (struct GNUNET_DATASTORE_PluginFunctions * api, int i, int k) 104putValue (struct GNUNET_DATASTORE_PluginFunctions * api,
105 int i, int k)
104{ 106{
105 char value[65536]; 107 char value[65536];
106 size_t size; 108 size_t size;
@@ -156,7 +158,6 @@ test (void *cls,
156 158
157static int 159static int
158iterate_zeros (void *cls, 160iterate_zeros (void *cls,
159 void *next_cls,
160 const GNUNET_HashCode * key, 161 const GNUNET_HashCode * key,
161 uint32_t size, 162 uint32_t size,
162 const void *data, 163 const void *data,
@@ -171,7 +172,18 @@ iterate_zeros (void *cls,
171 int i; 172 int i;
172 const char *cdata = data; 173 const char *cdata = data;
173 174
174 if (key == NULL) 175 GNUNET_assert (key != NULL);
176 GNUNET_assert (size >= 8);
177 memcpy (&i, &cdata[4], sizeof (i));
178 hits[i/8] |= (1 << (i % 8));
179
180#if VERBOSE
181 fprintf (stderr, "Found result type=%u, priority=%u, size=%u, expire=%llu\n",
182 type, priority, size,
183 (unsigned long long) expiration.abs_value);
184#endif
185 crc->cnt++;
186 if (crc->cnt == PUT_10 / 4 - 1)
175 { 187 {
176 char buf[256]; 188 char buf[256];
177 unsigned int bc; 189 unsigned int bc;
@@ -192,42 +204,17 @@ iterate_zeros (void *cls,
192 crc->cnt); 204 crc->cnt);
193 GAUGER (category, buf, crc->end.abs_value - crc->start.abs_value, "ms"); 205 GAUGER (category, buf, crc->end.abs_value - crc->start.abs_value, "ms");
194 memset (hits, 0, sizeof (hits)); 206 memset (hits, 0, sizeof (hits));
195 if ( (int) (PUT_10 / 4 - crc->cnt) > 2) 207 crc->phase++;
196 { 208 crc->cnt = 0;
197 fprintf (stderr, 209 crc->start = GNUNET_TIME_absolute_get ();
198 "Got %d items, expected %d\n",
199 (int) crc->cnt, (int) PUT_10 / 4);
200 GNUNET_break (0);
201 crc->phase = RP_ERROR;
202 }
203 else
204 {
205 crc->phase++;
206 crc->cnt = 0;
207 crc->start = GNUNET_TIME_absolute_get ();
208 }
209 GNUNET_SCHEDULER_add_now (&test, crc);
210 return GNUNET_OK;
211 } 210 }
212 GNUNET_assert (size >= 8); 211 GNUNET_SCHEDULER_add_now (&test, crc);
213 memcpy (&i, &cdata[4], sizeof (i));
214 hits[i/8] |= (1 << (i % 8));
215
216#if VERBOSE
217 fprintf (stderr, "Found result type=%u, priority=%u, size=%u, expire=%llu\n",
218 type, priority, size,
219 (unsigned long long) expiration.abs_value);
220#endif
221 crc->cnt++;
222 crc->api->next_request (next_cls,
223 GNUNET_NO);
224 return GNUNET_OK; 212 return GNUNET_OK;
225} 213}
226 214
227 215
228static int 216static int
229expiration_get (void *cls, 217expiration_get (void *cls,
230 void *next_cls,
231 const GNUNET_HashCode * key, 218 const GNUNET_HashCode * key,
232 uint32_t size, 219 uint32_t size,
233 const void *data, 220 const void *data,
@@ -281,7 +268,6 @@ expiration_get (void *cls,
281 268
282static int 269static int
283replication_get (void *cls, 270replication_get (void *cls,
284 void *next_cls,
285 const GNUNET_HashCode * key, 271 const GNUNET_HashCode * key,
286 uint32_t size, 272 uint32_t size,
287 const void *data, 273 const void *data,
@@ -323,6 +309,7 @@ replication_get (void *cls,
323 GAUGER (category, buf, crc->end.abs_value - crc->start.abs_value, "ms"); 309 GAUGER (category, buf, crc->end.abs_value - crc->start.abs_value, "ms");
324 memset (hits, 0, sizeof (hits)); 310 memset (hits, 0, sizeof (hits));
325 crc->phase++; 311 crc->phase++;
312 crc->offset = 0;
326 crc->cnt = 0; 313 crc->cnt = 0;
327 crc->start = GNUNET_TIME_absolute_get (); 314 crc->start = GNUNET_TIME_absolute_get ();
328 } 315 }
@@ -386,7 +373,15 @@ test (void *cls,
386 int j; 373 int j;
387 374
388 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) 375 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
389 crc->phase = RP_ERROR; 376 {
377 GNUNET_break (0);
378 crc->phase = RP_ERROR;
379 }
380#if VERBOSE
381 fprintf (stderr, "In phase %d, iteration %u\n",
382 crc->phase,
383 crc->cnt);
384#endif
390 switch (crc->phase) 385 switch (crc->phase)
391 { 386 {
392 case RP_ERROR: 387 case RP_ERROR:
@@ -419,17 +414,19 @@ test (void *cls,
419 GNUNET_SCHEDULER_add_now (&test, crc); 414 GNUNET_SCHEDULER_add_now (&test, crc);
420 break; 415 break;
421 case RP_REP_GET: 416 case RP_REP_GET:
422 crc->api->replication_get (crc->api->cls, 417 crc->api->get_replication (crc->api->cls,
423 &replication_get, 418 &replication_get,
424 crc); 419 crc);
425 break; 420 break;
426 case RP_ZA_GET: 421 case RP_ZA_GET:
427 crc->api->iter_zero_anonymity (crc->api->cls, 1, 422 crc->api->get_zero_anonymity (crc->api->cls,
428 &iterate_zeros, 423 crc->offset++,
429 crc); 424 1,
425 &iterate_zeros,
426 crc);
430 break; 427 break;
431 case RP_EXP_GET: 428 case RP_EXP_GET:
432 crc->api->expiration_get (crc->api->cls, 429 crc->api->get_expiration (crc->api->cls,
433 &expiration_get, 430 &expiration_get,
434 crc); 431 crc);
435 break; 432 break;
@@ -549,7 +546,6 @@ main (int argc, char *argv[])
549 char *pos; 546 char *pos;
550 char dir_name[128]; 547 char dir_name[128];
551 548
552 if (1) return 0;
553 /* determine name of plugin to use */ 549 /* determine name of plugin to use */
554 plugin_name = argv[0]; 550 plugin_name = argv[0];
555 while (NULL != (pos = strstr(plugin_name, "_"))) 551 while (NULL != (pos = strstr(plugin_name, "_")))
diff --git a/src/datastore/perf_plugin_datastore_data_postgres.conf b/src/datastore/perf_plugin_datastore_data_postgres.conf
index c2a181bc7..b7cf174e9 100644
--- a/src/datastore/perf_plugin_datastore_data_postgres.conf
+++ b/src/datastore/perf_plugin_datastore_data_postgres.conf
@@ -20,7 +20,7 @@ DATABASE = postgres
20# REJECT_FROM = 20# REJECT_FROM =
21# REJECT_FROM6 = 21# REJECT_FROM6 =
22# PREFIX = 22# PREFIX =
23 23# DEBUG = YES
24 24
25[dht] 25[dht]
26AUTOSTART = NO 26AUTOSTART = NO
diff --git a/src/datastore/plugin_datastore_mysql.c b/src/datastore/plugin_datastore_mysql.c
index c3d9212d3..2eefd9b04 100644
--- a/src/datastore/plugin_datastore_mysql.c
+++ b/src/datastore/plugin_datastore_mysql.c
@@ -160,58 +160,6 @@ struct GNUNET_MysqlStatementHandle
160 160
161}; 161};
162 162
163/**
164 * Context for the universal iterator.
165 */
166struct NextRequestClosure;
167
168/**
169 * Type of a function that will prepare
170 * the next iteration.
171 *
172 * @param cls closure
173 * @param nc the next context; NULL for the last
174 * call which gives the callback a chance to
175 * clean up the closure
176 * @return GNUNET_OK on success, GNUNET_NO if there are
177 * no more values, GNUNET_SYSERR on error
178 */
179typedef int (*PrepareFunction)(void *cls,
180 struct NextRequestClosure *nc);
181
182
183struct NextRequestClosure
184{
185 struct Plugin *plugin;
186
187 struct GNUNET_TIME_Absolute now;
188
189 /**
190 * Function to call to prepare the next
191 * iteration.
192 */
193 PrepareFunction prep;
194
195 /**
196 * Closure for prep.
197 */
198 void *prep_cls;
199
200 MYSQL_BIND rbind[7];
201
202 enum GNUNET_BLOCK_Type type;
203
204 PluginIterator dviter;
205
206 void *dviter_cls;
207
208 unsigned int count;
209
210 int end_it;
211
212 int one_shot;
213};
214
215 163
216/** 164/**
217 * Context for all functions in this plugin. 165 * Context for all functions in this plugin.
@@ -244,16 +192,6 @@ struct Plugin
244 char *cnffile; 192 char *cnffile;
245 193
246 /** 194 /**
247 * Closure of the 'next_task' (must be freed if 'next_task' is cancelled).
248 */
249 struct NextRequestClosure *next_task_nc;
250
251 /**
252 * Pending task with scheduler for running the next request.
253 */
254 GNUNET_SCHEDULER_TaskIdentifier next_task;
255
256 /**
257 * Prepared statements. 195 * Prepared statements.
258 */ 196 */
259#define INSERT_ENTRY "INSERT INTO gn090 (repl,type,prio,anonLevel,expire,hash,vhash,value) VALUES (?,?,?,?,?,?,?,?)" 197#define INSERT_ENTRY "INSERT INTO gn090 (repl,type,prio,anonLevel,expire,hash,vhash,value) VALUES (?,?,?,?,?,?,?,?)"
@@ -295,7 +233,7 @@ struct Plugin
295#define SELECT_SIZE "SELECT SUM(BIT_LENGTH(value) DIV 8) FROM gn090" 233#define SELECT_SIZE "SELECT SUM(BIT_LENGTH(value) DIV 8) FROM gn090"
296 struct GNUNET_MysqlStatementHandle *get_size; 234 struct GNUNET_MysqlStatementHandle *get_size;
297 235
298#define SELECT_IT_NON_ANONYMOUS "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 WHERE anonLevel=0 ORDER BY uid DESC LIMIT 1 OFFSET ?" 236#define SELECT_IT_NON_ANONYMOUS "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 WHERE anonLevel=0 AND type=? ORDER BY uid DESC LIMIT 1 OFFSET ?"
299 struct GNUNET_MysqlStatementHandle *zero_iter; 237 struct GNUNET_MysqlStatementHandle *zero_iter;
300 238
301#define SELECT_IT_EXPIRATION "(SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 WHERE expire < ? ORDER BY prio ASC LIMIT 1) "\ 239#define SELECT_IT_EXPIRATION "(SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 WHERE expire < ? ORDER BY prio ASC LIMIT 1) "\
@@ -372,7 +310,6 @@ get_my_cnf_path (const struct GNUNET_CONFIGURATION_Handle *cfg)
372} 310}
373 311
374 312
375
376/** 313/**
377 * Free a prepared statement. 314 * Free a prepared statement.
378 * 315 *
@@ -381,8 +318,7 @@ get_my_cnf_path (const struct GNUNET_CONFIGURATION_Handle *cfg)
381 */ 318 */
382static void 319static void
383prepared_statement_destroy (struct Plugin *plugin, 320prepared_statement_destroy (struct Plugin *plugin,
384 struct GNUNET_MysqlStatementHandle 321 struct GNUNET_MysqlStatementHandle *s)
385 *s)
386{ 322{
387 GNUNET_CONTAINER_DLL_remove (plugin->shead, 323 GNUNET_CONTAINER_DLL_remove (plugin->shead,
388 plugin->stail, 324 plugin->stail,
@@ -397,6 +333,8 @@ prepared_statement_destroy (struct Plugin *plugin,
397/** 333/**
398 * Close database connection and all prepared statements (we got a DB 334 * Close database connection and all prepared statements (we got a DB
399 * disconnect error). 335 * disconnect error).
336 *
337 * @param plugin plugin context
400 */ 338 */
401static int 339static int
402iclose (struct Plugin *plugin) 340iclose (struct Plugin *plugin)
@@ -420,10 +358,11 @@ iclose (struct Plugin *plugin)
420 * Open the connection with the database (and initialize 358 * Open the connection with the database (and initialize
421 * our default options). 359 * our default options).
422 * 360 *
361 * @param plugin plugin context
423 * @return GNUNET_OK on success 362 * @return GNUNET_OK on success
424 */ 363 */
425static int 364static int
426iopen (struct Plugin *ret) 365iopen (struct Plugin *plugin)
427{ 366{
428 char *mysql_dbname; 367 char *mysql_dbname;
429 char *mysql_server; 368 char *mysql_server;
@@ -433,67 +372,67 @@ iopen (struct Plugin *ret)
433 my_bool reconnect; 372 my_bool reconnect;
434 unsigned int timeout; 373 unsigned int timeout;
435 374
436 ret->dbf = mysql_init (NULL); 375 plugin->dbf = mysql_init (NULL);
437 if (ret->dbf == NULL) 376 if (plugin->dbf == NULL)
438 return GNUNET_SYSERR; 377 return GNUNET_SYSERR;
439 if (ret->cnffile != NULL) 378 if (plugin->cnffile != NULL)
440 mysql_options (ret->dbf, MYSQL_READ_DEFAULT_FILE, ret->cnffile); 379 mysql_options (plugin->dbf, MYSQL_READ_DEFAULT_FILE, plugin->cnffile);
441 mysql_options (ret->dbf, MYSQL_READ_DEFAULT_GROUP, "client"); 380 mysql_options (plugin->dbf, MYSQL_READ_DEFAULT_GROUP, "client");
442 reconnect = 0; 381 reconnect = 0;
443 mysql_options (ret->dbf, MYSQL_OPT_RECONNECT, &reconnect); 382 mysql_options (plugin->dbf, MYSQL_OPT_RECONNECT, &reconnect);
444 mysql_options (ret->dbf, 383 mysql_options (plugin->dbf,
445 MYSQL_OPT_CONNECT_TIMEOUT, (const void *) &timeout); 384 MYSQL_OPT_CONNECT_TIMEOUT, (const void *) &timeout);
446 mysql_options(ret->dbf, MYSQL_SET_CHARSET_NAME, "UTF8"); 385 mysql_options(plugin->dbf, MYSQL_SET_CHARSET_NAME, "UTF8");
447 timeout = 60; /* in seconds */ 386 timeout = 60; /* in seconds */
448 mysql_options (ret->dbf, MYSQL_OPT_READ_TIMEOUT, (const void *) &timeout); 387 mysql_options (plugin->dbf, MYSQL_OPT_READ_TIMEOUT, (const void *) &timeout);
449 mysql_options (ret->dbf, MYSQL_OPT_WRITE_TIMEOUT, (const void *) &timeout); 388 mysql_options (plugin->dbf, MYSQL_OPT_WRITE_TIMEOUT, (const void *) &timeout);
450 mysql_dbname = NULL; 389 mysql_dbname = NULL;
451 if (GNUNET_YES == GNUNET_CONFIGURATION_have_value (ret->env->cfg, 390 if (GNUNET_YES == GNUNET_CONFIGURATION_have_value (plugin->env->cfg,
452 "datastore-mysql", "DATABASE")) 391 "datastore-mysql", "DATABASE"))
453 GNUNET_assert (GNUNET_OK == 392 GNUNET_assert (GNUNET_OK ==
454 GNUNET_CONFIGURATION_get_value_string (ret->env->cfg, 393 GNUNET_CONFIGURATION_get_value_string (plugin->env->cfg,
455 "datastore-mysql", "DATABASE", 394 "datastore-mysql", "DATABASE",
456 &mysql_dbname)); 395 &mysql_dbname));
457 else 396 else
458 mysql_dbname = GNUNET_strdup ("gnunet"); 397 mysql_dbname = GNUNET_strdup ("gnunet");
459 mysql_user = NULL; 398 mysql_user = NULL;
460 if (GNUNET_YES == GNUNET_CONFIGURATION_have_value (ret->env->cfg, 399 if (GNUNET_YES == GNUNET_CONFIGURATION_have_value (plugin->env->cfg,
461 "datastore-mysql", "USER")) 400 "datastore-mysql", "USER"))
462 { 401 {
463 GNUNET_assert (GNUNET_OK == 402 GNUNET_assert (GNUNET_OK ==
464 GNUNET_CONFIGURATION_get_value_string (ret->env->cfg, 403 GNUNET_CONFIGURATION_get_value_string (plugin->env->cfg,
465 "datastore-mysql", "USER", 404 "datastore-mysql", "USER",
466 &mysql_user)); 405 &mysql_user));
467 } 406 }
468 mysql_password = NULL; 407 mysql_password = NULL;
469 if (GNUNET_YES == GNUNET_CONFIGURATION_have_value (ret->env->cfg, 408 if (GNUNET_YES == GNUNET_CONFIGURATION_have_value (plugin->env->cfg,
470 "datastore-mysql", "PASSWORD")) 409 "datastore-mysql", "PASSWORD"))
471 { 410 {
472 GNUNET_assert (GNUNET_OK == 411 GNUNET_assert (GNUNET_OK ==
473 GNUNET_CONFIGURATION_get_value_string (ret->env->cfg, 412 GNUNET_CONFIGURATION_get_value_string (plugin->env->cfg,
474 "datastore-mysql", "PASSWORD", 413 "datastore-mysql", "PASSWORD",
475 &mysql_password)); 414 &mysql_password));
476 } 415 }
477 mysql_server = NULL; 416 mysql_server = NULL;
478 if (GNUNET_YES == GNUNET_CONFIGURATION_have_value (ret->env->cfg, 417 if (GNUNET_YES == GNUNET_CONFIGURATION_have_value (plugin->env->cfg,
479 "datastore-mysql", "HOST")) 418 "datastore-mysql", "HOST"))
480 { 419 {
481 GNUNET_assert (GNUNET_OK == 420 GNUNET_assert (GNUNET_OK ==
482 GNUNET_CONFIGURATION_get_value_string (ret->env->cfg, 421 GNUNET_CONFIGURATION_get_value_string (plugin->env->cfg,
483 "datastore-mysql", "HOST", 422 "datastore-mysql", "HOST",
484 &mysql_server)); 423 &mysql_server));
485 } 424 }
486 mysql_port = 0; 425 mysql_port = 0;
487 if (GNUNET_YES == GNUNET_CONFIGURATION_have_value (ret->env->cfg, 426 if (GNUNET_YES == GNUNET_CONFIGURATION_have_value (plugin->env->cfg,
488 "datastore-mysql", "PORT")) 427 "datastore-mysql", "PORT"))
489 { 428 {
490 GNUNET_assert (GNUNET_OK == 429 GNUNET_assert (GNUNET_OK ==
491 GNUNET_CONFIGURATION_get_value_number (ret->env->cfg, "datastore-mysql", 430 GNUNET_CONFIGURATION_get_value_number (plugin->env->cfg, "datastore-mysql",
492 "PORT", &mysql_port)); 431 "PORT", &mysql_port));
493 } 432 }
494 433
495 GNUNET_assert (mysql_dbname != NULL); 434 GNUNET_assert (mysql_dbname != NULL);
496 mysql_real_connect (ret->dbf, 435 mysql_real_connect (plugin->dbf,
497 mysql_server, 436 mysql_server,
498 mysql_user, mysql_password, 437 mysql_user, mysql_password,
499 mysql_dbname, 438 mysql_dbname,
@@ -503,10 +442,10 @@ iopen (struct Plugin *ret)
503 GNUNET_free_non_null (mysql_user); 442 GNUNET_free_non_null (mysql_user);
504 GNUNET_free_non_null (mysql_password); 443 GNUNET_free_non_null (mysql_password);
505 GNUNET_free (mysql_dbname); 444 GNUNET_free (mysql_dbname);
506 if (mysql_error (ret->dbf)[0]) 445 if (mysql_error (plugin->dbf)[0])
507 { 446 {
508 LOG_MYSQL (GNUNET_ERROR_TYPE_ERROR, 447 LOG_MYSQL (GNUNET_ERROR_TYPE_ERROR,
509 "mysql_real_connect", ret); 448 "mysql_real_connect", plugin);
510 return GNUNET_SYSERR; 449 return GNUNET_SYSERR;
511 } 450 }
512 return GNUNET_OK; 451 return GNUNET_OK;
@@ -686,19 +625,6 @@ init_params (struct Plugin *plugin,
686 return GNUNET_OK; 625 return GNUNET_OK;
687} 626}
688 627
689/**
690 * Type of a callback that will be called for each
691 * data set returned from MySQL.
692 *
693 * @param cls user-defined argument
694 * @param num_values number of elements in values
695 * @param values values returned by MySQL
696 * @return GNUNET_OK to continue iterating, GNUNET_SYSERR to abort
697 */
698typedef int (*GNUNET_MysqlDataProcessor) (void *cls,
699 unsigned int num_values,
700 MYSQL_BIND *values);
701
702 628
703/** 629/**
704 * Run a prepared SELECT statement. 630 * Run a prepared SELECT statement.
@@ -708,40 +634,31 @@ typedef int (*GNUNET_MysqlDataProcessor) (void *cls,
708 * @param result_size number of elements in results array 634 * @param result_size number of elements in results array
709 * @param results pointer to already initialized MYSQL_BIND 635 * @param results pointer to already initialized MYSQL_BIND
710 * array (of sufficient size) for passing results 636 * array (of sufficient size) for passing results
711 * @param processor function to call on each result 637 * @param ap pairs and triplets of "MYSQL_TYPE_XXX" keys and their respective
712 * @param processor_cls extra argument to processor
713 * @param ... pairs and triplets of "MYSQL_TYPE_XXX" keys and their respective
714 * values (size + buffer-reference for pointers); terminated 638 * values (size + buffer-reference for pointers); terminated
715 * with "-1" 639 * with "-1"
716 * @return GNUNET_SYSERR on error, otherwise 640 * @return GNUNET_SYSERR on error, otherwise GNUNET_OK or GNUNET_NO (no result)
717 * the number of successfully affected (or queried) rows
718 */ 641 */
719static int 642static int
720prepared_statement_run_select (struct Plugin *plugin, 643prepared_statement_run_select_va (struct Plugin *plugin,
721 struct GNUNET_MysqlStatementHandle *s, 644 struct GNUNET_MysqlStatementHandle *s,
722 unsigned int result_size, 645 unsigned int result_size,
723 MYSQL_BIND *results, 646 MYSQL_BIND *results,
724 GNUNET_MysqlDataProcessor processor, void *processor_cls, 647 va_list ap)
725 ...)
726{ 648{
727 va_list ap;
728 int ret; 649 int ret;
729 unsigned int rsize; 650 unsigned int rsize;
730 int total;
731 651
732 if (GNUNET_OK != prepare_statement (plugin, s)) 652 if (GNUNET_OK != prepare_statement (plugin, s))
733 { 653 {
734 GNUNET_break (0); 654 GNUNET_break (0);
735 return GNUNET_SYSERR; 655 return GNUNET_SYSERR;
736 } 656 }
737 va_start (ap, processor_cls);
738 if (GNUNET_OK != init_params (plugin, s, ap)) 657 if (GNUNET_OK != init_params (plugin, s, ap))
739 { 658 {
740 GNUNET_break (0); 659 GNUNET_break (0);
741 va_end (ap);
742 return GNUNET_SYSERR; 660 return GNUNET_SYSERR;
743 } 661 }
744 va_end (ap);
745 rsize = mysql_stmt_field_count (s->statement); 662 rsize = mysql_stmt_field_count (s->statement);
746 if (rsize > result_size) 663 if (rsize > result_size)
747 { 664 {
@@ -757,29 +674,53 @@ prepared_statement_run_select (struct Plugin *plugin,
757 iclose (plugin); 674 iclose (plugin);
758 return GNUNET_SYSERR; 675 return GNUNET_SYSERR;
759 } 676 }
760 677 ret = mysql_stmt_fetch (s->statement);
761 total = 0; 678 if (ret == MYSQL_NO_DATA)
762 while (1) 679 return GNUNET_NO;
680 if (ret != 0)
763 { 681 {
764 ret = mysql_stmt_fetch (s->statement); 682 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
765 if (ret == MYSQL_NO_DATA) 683 _("`%s' failed at %s:%d with error: %s\n"),
766 break; 684 "mysql_stmt_fetch",
767 if (ret != 0) 685 __FILE__, __LINE__, mysql_stmt_error (s->statement));
768 { 686 iclose (plugin);
769 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 687 return GNUNET_SYSERR;
770 _("`%s' failed at %s:%d with error: %s\n"),
771 "mysql_stmt_fetch",
772 __FILE__, __LINE__, mysql_stmt_error (s->statement));
773 iclose (plugin);
774 return GNUNET_SYSERR;
775 }
776 if (processor != NULL)
777 if (GNUNET_OK != processor (processor_cls, rsize, results))
778 break;
779 total++;
780 } 688 }
781 mysql_stmt_reset (s->statement); 689 mysql_stmt_reset (s->statement);
782 return total; 690 return GNUNET_OK;
691}
692
693
694/**
695 * Run a prepared SELECT statement.
696 *
697 * @param plugin plugin context
698 * @param s statement to run
699 * @param result_size number of elements in results array
700 * @param results pointer to already initialized MYSQL_BIND
701 * array (of sufficient size) for passing results
702 * @param ... pairs and triplets of "MYSQL_TYPE_XXX" keys and their respective
703 * values (size + buffer-reference for pointers); terminated
704 * with "-1"
705 * @return GNUNET_SYSERR on error, otherwise
706 * the number of successfully affected (or queried) rows
707 */
708static int
709prepared_statement_run_select (struct Plugin *plugin,
710 struct GNUNET_MysqlStatementHandle *s,
711 unsigned int result_size,
712 MYSQL_BIND *results,
713 ...)
714{
715 va_list ap;
716 int ret;
717
718 va_start (ap, results);
719 ret = prepared_statement_run_select_va (plugin, s,
720 result_size, results,
721 ap);
722 va_end (ap);
723 return ret;
783} 724}
784 725
785 726
@@ -854,23 +795,6 @@ do_delete_entry (struct Plugin *plugin,
854 795
855 796
856/** 797/**
857 * Function that simply returns GNUNET_OK
858 *
859 * @param cls closure, not used
860 * @param num_values not used
861 * @param values not used
862 * @return GNUNET_OK
863 */
864static int
865return_ok (void *cls,
866 unsigned int num_values,
867 MYSQL_BIND *values)
868{
869 return GNUNET_OK;
870}
871
872
873/**
874 * Get an estimate of how much space the database is 798 * Get an estimate of how much space the database is
875 * currently using. 799 * currently using.
876 * 800 *
@@ -878,7 +802,7 @@ return_ok (void *cls,
878 * @return number of bytes used on disk 802 * @return number of bytes used on disk
879 */ 803 */
880static unsigned long long 804static unsigned long long
881mysql_plugin_get_size (void *cls) 805mysql_plugin_estimate_size (void *cls)
882{ 806{
883 struct Plugin *plugin = cls; 807 struct Plugin *plugin = cls;
884 MYSQL_BIND cbind[1]; 808 MYSQL_BIND cbind[1];
@@ -893,7 +817,6 @@ mysql_plugin_get_size (void *cls)
893 prepared_statement_run_select (plugin, 817 prepared_statement_run_select (plugin,
894 plugin->get_size, 818 plugin->get_size,
895 1, cbind, 819 1, cbind,
896 &return_ok, NULL,
897 -1)) 820 -1))
898 return 0; 821 return 0;
899 return total; 822 return total;
@@ -929,7 +852,6 @@ mysql_plugin_put (void *cls,
929{ 852{
930 struct Plugin *plugin = cls; 853 struct Plugin *plugin = cls;
931 unsigned int irepl = replication; 854 unsigned int irepl = replication;
932 unsigned int itype = type;
933 unsigned int ipriority = priority; 855 unsigned int ipriority = priority;
934 unsigned int ianonymity = anonymity; 856 unsigned int ianonymity = anonymity;
935 unsigned long long lexpiration = expiration.abs_value; 857 unsigned long long lexpiration = expiration.abs_value;
@@ -952,7 +874,7 @@ mysql_plugin_put (void *cls,
952 plugin->insert_entry, 874 plugin->insert_entry,
953 NULL, 875 NULL,
954 MYSQL_TYPE_LONG, &irepl, GNUNET_YES, 876 MYSQL_TYPE_LONG, &irepl, GNUNET_YES,
955 MYSQL_TYPE_LONG, &itype, GNUNET_YES, 877 MYSQL_TYPE_LONG, &type, GNUNET_YES,
956 MYSQL_TYPE_LONG, &ipriority, GNUNET_YES, 878 MYSQL_TYPE_LONG, &ipriority, GNUNET_YES,
957 MYSQL_TYPE_LONG, &ianonymity, GNUNET_YES, 879 MYSQL_TYPE_LONG, &ianonymity, GNUNET_YES,
958 MYSQL_TYPE_LONGLONG, &lexpiration, GNUNET_YES, 880 MYSQL_TYPE_LONGLONG, &lexpiration, GNUNET_YES,
@@ -1034,20 +956,23 @@ mysql_plugin_update (void *cls,
1034} 956}
1035 957
1036 958
1037
1038
1039/** 959/**
1040 * Continuation of "mysql_next_request". 960 * Run the given select statement and call 'proc' on the resulting
961 * values (which must be in particular positions).
1041 * 962 *
1042 * @param next_cls the next context 963 * @param plugin the plugin handle
1043 * @param tc the task context (unused) 964 * @param stmt select statement to run
965 * @param proc function to call on result
966 * @param proc_cls closure for proc
967 * @param ... arguments to initialize stmt
1044 */ 968 */
1045static void 969static void
1046mysql_next_request_cont (void *next_cls, 970execute_select (struct Plugin *plugin,
1047 const struct GNUNET_SCHEDULER_TaskContext *tc) 971 struct GNUNET_MysqlStatementHandle *stmt,
972 PluginDatumProcessor proc, void *proc_cls,
973 ...)
1048{ 974{
1049 struct NextRequestClosure *nrc = next_cls; 975 va_list ap;
1050 struct Plugin *plugin;
1051 int ret; 976 int ret;
1052 unsigned int type; 977 unsigned int type;
1053 unsigned int priority; 978 unsigned int priority;
@@ -1059,19 +984,10 @@ mysql_next_request_cont (void *next_cls,
1059 char value[GNUNET_DATASTORE_MAX_VALUE_SIZE]; 984 char value[GNUNET_DATASTORE_MAX_VALUE_SIZE];
1060 GNUNET_HashCode key; 985 GNUNET_HashCode key;
1061 struct GNUNET_TIME_Absolute expiration; 986 struct GNUNET_TIME_Absolute expiration;
1062 MYSQL_BIND *rbind = nrc->rbind; 987 MYSQL_BIND rbind[7];
1063
1064 plugin = nrc->plugin;
1065 plugin->next_task = GNUNET_SCHEDULER_NO_TASK;
1066 plugin->next_task_nc = NULL;
1067 988
1068 if (GNUNET_YES == nrc->end_it)
1069 goto END_SET;
1070 GNUNET_assert (nrc->plugin->next_task == GNUNET_SCHEDULER_NO_TASK);
1071 nrc->now = GNUNET_TIME_absolute_get ();
1072 hashSize = sizeof (GNUNET_HashCode); 989 hashSize = sizeof (GNUNET_HashCode);
1073 memset (nrc->rbind, 0, sizeof (nrc->rbind)); 990 memset (rbind, 0, sizeof (rbind));
1074 rbind = nrc->rbind;
1075 rbind[0].buffer_type = MYSQL_TYPE_LONG; 991 rbind[0].buffer_type = MYSQL_TYPE_LONG;
1076 rbind[0].buffer = &type; 992 rbind[0].buffer = &type;
1077 rbind[0].is_unsigned = 1; 993 rbind[0].is_unsigned = 1;
@@ -1096,16 +1012,28 @@ mysql_next_request_cont (void *next_cls,
1096 rbind[6].buffer = &uid; 1012 rbind[6].buffer = &uid;
1097 rbind[6].is_unsigned = 1; 1013 rbind[6].is_unsigned = 1;
1098 1014
1099 if (GNUNET_OK != nrc->prep (nrc->prep_cls, 1015 va_start (ap, proc_cls);
1100 nrc)) 1016 ret = prepared_statement_run_select_va (plugin,
1101 goto END_SET; 1017 stmt,
1102 GNUNET_assert (nrc->plugin->next_task == GNUNET_SCHEDULER_NO_TASK); 1018 7, rbind,
1019 ap);
1020 va_end (ap);
1021 if (ret <= 0)
1022 {
1023 proc (proc_cls,
1024 NULL, 0, NULL, 0, 0, 0,
1025 GNUNET_TIME_UNIT_ZERO_ABS, 0);
1026 return;
1027 }
1103 GNUNET_assert (size <= sizeof(value)); 1028 GNUNET_assert (size <= sizeof(value));
1104 if ( (rbind[4].buffer_length != sizeof (GNUNET_HashCode)) || 1029 if ( (rbind[4].buffer_length != sizeof (GNUNET_HashCode)) ||
1105 (hashSize != sizeof (GNUNET_HashCode)) ) 1030 (hashSize != sizeof (GNUNET_HashCode)) )
1106 { 1031 {
1107 GNUNET_break (0); 1032 GNUNET_break (0);
1108 goto END_SET; 1033 proc (proc_cls,
1034 NULL, 0, NULL, 0, 0, 0,
1035 GNUNET_TIME_UNIT_ZERO_ABS, 0);
1036 return;
1109 } 1037 }
1110#if DEBUG_MYSQL 1038#if DEBUG_MYSQL
1111 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1039 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1116,18 +1044,13 @@ mysql_next_request_cont (void *next_cls,
1116 anonymity, 1044 anonymity,
1117 exp); 1045 exp);
1118#endif 1046#endif
1047 GNUNET_assert (size < MAX_DATUM_SIZE);
1119 expiration.abs_value = exp; 1048 expiration.abs_value = exp;
1120 ret = nrc->dviter (nrc->dviter_cls, 1049 ret = proc (proc_cls,
1121 (nrc->one_shot == GNUNET_YES) ? NULL : nrc, 1050 &key,
1122 &key, 1051 size, value,
1123 size, value, 1052 type, priority, anonymity, expiration,
1124 type, priority, anonymity, expiration, 1053 uid);
1125 uid);
1126 if (ret == GNUNET_SYSERR)
1127 {
1128 nrc->end_it = GNUNET_YES;
1129 return;
1130 }
1131 if (ret == GNUNET_NO) 1054 if (ret == GNUNET_NO)
1132 { 1055 {
1133 do_delete_entry (plugin, uid); 1056 do_delete_entry (plugin, uid);
@@ -1135,189 +1058,50 @@ mysql_next_request_cont (void *next_cls,
1135 plugin->env->duc (plugin->env->cls, 1058 plugin->env->duc (plugin->env->cls,
1136 - size); 1059 - size);
1137 } 1060 }
1138 if (nrc->one_shot == GNUNET_YES)
1139 GNUNET_free (nrc);
1140 return;
1141 END_SET:
1142 /* call dviter with "end of set" */
1143 GNUNET_assert (nrc->plugin->next_task == GNUNET_SCHEDULER_NO_TASK);
1144 nrc->dviter (nrc->dviter_cls,
1145 NULL, NULL, 0, NULL, 0, 0, 0,
1146 GNUNET_TIME_UNIT_ZERO_ABS, 0);
1147 GNUNET_assert (nrc->plugin->next_task == GNUNET_SCHEDULER_NO_TASK);
1148 nrc->prep (nrc->prep_cls, NULL);
1149 GNUNET_assert (nrc->plugin->next_task == GNUNET_SCHEDULER_NO_TASK);
1150 GNUNET_free (nrc);
1151} 1061}
1152 1062
1153 1063
1154/**
1155 * Function invoked on behalf of a "PluginIterator"
1156 * asking the database plugin to call the iterator
1157 * with the next item.
1158 *
1159 * @param next_cls whatever argument was given
1160 * to the PluginIterator as "next_cls".
1161 * @param end_it set to GNUNET_YES if we
1162 * should terminate the iteration early
1163 * (iterator should be still called once more
1164 * to signal the end of the iteration).
1165 */
1166static void
1167mysql_plugin_next_request (void *next_cls,
1168 int end_it)
1169{
1170 struct NextRequestClosure *nrc = next_cls;
1171
1172 if (GNUNET_YES == end_it)
1173 nrc->end_it = GNUNET_YES;
1174 nrc->plugin->next_task_nc = nrc;
1175 nrc->plugin->next_task = GNUNET_SCHEDULER_add_now (&mysql_next_request_cont,
1176 nrc);
1177}
1178
1179 1064
1180/** 1065/**
1181 * Context for 'get_statement_prepare'. 1066 * Get one of the results for a particular key in the datastore.
1182 */
1183struct GetContext
1184{
1185 GNUNET_HashCode key;
1186 GNUNET_HashCode vhash;
1187
1188 unsigned int prio;
1189 unsigned int anonymity;
1190 unsigned long long expiration;
1191 unsigned long long vkey;
1192 unsigned long long total;
1193 unsigned int off;
1194 unsigned int count;
1195 int have_vhash;
1196};
1197
1198
1199static int
1200get_statement_prepare (void *cls,
1201 struct NextRequestClosure *nrc)
1202{
1203 struct GetContext *gc = cls;
1204 struct Plugin *plugin;
1205 int ret;
1206 unsigned long hashSize;
1207
1208 if (NULL == nrc)
1209 {
1210 GNUNET_free (gc);
1211 return GNUNET_NO;
1212 }
1213 if (gc->count == gc->total)
1214 return GNUNET_NO;
1215 plugin = nrc->plugin;
1216 hashSize = sizeof (GNUNET_HashCode);
1217 if (++gc->off >= gc->total)
1218 gc->off = 0;
1219#if DEBUG_MYSQL
1220 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1221 "Obtaining result number %d/%lld at offset %u for GET `%s'\n",
1222 gc->count+1,
1223 gc->total,
1224 gc->off,
1225 GNUNET_h2s (&gc->key));
1226#endif
1227 if (nrc->type != 0)
1228 {
1229 if (gc->have_vhash)
1230 {
1231 ret = prepared_statement_run_select (plugin,
1232 plugin->select_entry_by_hash_vhash_and_type,
1233 7, nrc->rbind,
1234 &return_ok, NULL,
1235 MYSQL_TYPE_BLOB, &gc->key, hashSize, &hashSize,
1236 MYSQL_TYPE_BLOB, &gc->vhash, hashSize, &hashSize,
1237 MYSQL_TYPE_LONG, &nrc->type, GNUNET_YES,
1238 MYSQL_TYPE_LONG, &gc->off, GNUNET_YES,
1239 -1);
1240 }
1241 else
1242 {
1243 ret =
1244 prepared_statement_run_select (plugin,
1245 plugin->select_entry_by_hash_and_type,
1246 7, nrc->rbind,
1247 &return_ok, NULL,
1248 MYSQL_TYPE_BLOB, &gc->key, hashSize, &hashSize,
1249 MYSQL_TYPE_LONG, &nrc->type, GNUNET_YES,
1250 MYSQL_TYPE_LONG, &gc->off, GNUNET_YES,
1251 -1);
1252 }
1253 }
1254 else
1255 {
1256 if (gc->have_vhash)
1257 {
1258 ret =
1259 prepared_statement_run_select (plugin,
1260 plugin->select_entry_by_hash_and_vhash,
1261 7, nrc->rbind,
1262 &return_ok, NULL,
1263 MYSQL_TYPE_BLOB, &gc->key, hashSize, &hashSize,
1264 MYSQL_TYPE_BLOB, &gc->vhash, hashSize, &hashSize,
1265 MYSQL_TYPE_LONG, &gc->off, GNUNET_YES,
1266 -1);
1267 }
1268 else
1269 {
1270 ret =
1271 prepared_statement_run_select (plugin,
1272 plugin->select_entry_by_hash,
1273 7, nrc->rbind,
1274 &return_ok, NULL,
1275 MYSQL_TYPE_BLOB, &gc->key, hashSize, &hashSize,
1276 MYSQL_TYPE_LONG, &gc->off, GNUNET_YES,
1277 -1);
1278 }
1279 }
1280 gc->count++;
1281 return ret;
1282}
1283
1284
1285/**
1286 * Iterate over the results for a particular key in the datastore.
1287 * 1067 *
1288 * @param cls closure 1068 * @param cls closure
1289 * @param key maybe NULL (to match all entries) 1069 * @param offset offset of the result (mod #num-results);
1070 * specific ordering does not matter for the offset
1071 * @param key key to match, never NULL
1290 * @param vhash hash of the value, maybe NULL (to 1072 * @param vhash hash of the value, maybe NULL (to
1291 * match all values that have the right key). 1073 * match all values that have the right key).
1292 * Note that for DBlocks there is no difference 1074 * Note that for DBlocks there is no difference
1293 * betwen key and vhash, but for other blocks 1075 * betwen key and vhash, but for other blocks
1294 * there may be! 1076 * there may be!
1295 * @param type entries of which type are relevant? 1077 * @param type entries of which type are relevant?
1296 * Use 0 for any type. 1078 * Use 0 for any type.
1297 * @param iter function to call on each matching value; 1079 * @param proc function to call on each matching value; however,
1298 * will be called once with a NULL value at the end 1080 * after the first call to "proc", the plugin must wait
1299 * @param iter_cls closure for iter 1081 * until "NextRequest" was called before giving the processor
1082 * the next item; finally, the "proc" should be called once
1083 * once with a NULL value at the end ("next_cls" should be NULL
1084 * for that last call)
1085 * @param proc_cls closure for proc
1300 */ 1086 */
1301static void 1087static void
1302mysql_plugin_get (void *cls, 1088mysql_plugin_get_key (void *cls,
1303 const GNUNET_HashCode *key, 1089 uint64_t offset,
1304 const GNUNET_HashCode *vhash, 1090 const GNUNET_HashCode *key,
1305 enum GNUNET_BLOCK_Type type, 1091 const GNUNET_HashCode *vhash,
1306 PluginIterator iter, void *iter_cls) 1092 enum GNUNET_BLOCK_Type type,
1093 PluginDatumProcessor proc, void *proc_cls)
1307{ 1094{
1308 struct Plugin *plugin = cls; 1095 struct Plugin *plugin = cls;
1309 unsigned int itype = type;
1310 int ret; 1096 int ret;
1311 MYSQL_BIND cbind[1]; 1097 MYSQL_BIND cbind[1];
1312 struct GetContext *gc;
1313 struct NextRequestClosure *nrc;
1314 long long total; 1098 long long total;
1315 unsigned long hashSize; 1099 unsigned long hashSize;
1316 unsigned long hashSize2; 1100 unsigned long hashSize2;
1101 unsigned long long off;
1317 1102
1318 GNUNET_assert (key != NULL); 1103 GNUNET_assert (key != NULL);
1319 if (iter == NULL) 1104 GNUNET_assert (NULL != proc);
1320 return;
1321 hashSize = sizeof (GNUNET_HashCode); 1105 hashSize = sizeof (GNUNET_HashCode);
1322 hashSize2 = sizeof (GNUNET_HashCode); 1106 hashSize2 = sizeof (GNUNET_HashCode);
1323 memset (cbind, 0, sizeof (cbind)); 1107 memset (cbind, 0, sizeof (cbind));
@@ -1333,10 +1117,9 @@ mysql_plugin_get (void *cls,
1333 prepared_statement_run_select (plugin, 1117 prepared_statement_run_select (plugin,
1334 plugin->count_entry_by_hash_vhash_and_type, 1118 plugin->count_entry_by_hash_vhash_and_type,
1335 1, cbind, 1119 1, cbind,
1336 &return_ok, NULL,
1337 MYSQL_TYPE_BLOB, key, hashSize, &hashSize, 1120 MYSQL_TYPE_BLOB, key, hashSize, &hashSize,
1338 MYSQL_TYPE_BLOB, vhash, hashSize2, &hashSize2, 1121 MYSQL_TYPE_BLOB, vhash, hashSize2, &hashSize2,
1339 MYSQL_TYPE_LONG, &itype, GNUNET_YES, 1122 MYSQL_TYPE_LONG, &type, GNUNET_YES,
1340 -1); 1123 -1);
1341 } 1124 }
1342 else 1125 else
@@ -1345,9 +1128,8 @@ mysql_plugin_get (void *cls,
1345 prepared_statement_run_select (plugin, 1128 prepared_statement_run_select (plugin,
1346 plugin->count_entry_by_hash_and_type, 1129 plugin->count_entry_by_hash_and_type,
1347 1, cbind, 1130 1, cbind,
1348 &return_ok, NULL,
1349 MYSQL_TYPE_BLOB, key, hashSize, &hashSize, 1131 MYSQL_TYPE_BLOB, key, hashSize, &hashSize,
1350 MYSQL_TYPE_LONG, &itype, GNUNET_YES, 1132 MYSQL_TYPE_LONG, &type, GNUNET_YES,
1351 -1); 1133 -1);
1352 } 1134 }
1353 } 1135 }
@@ -1359,7 +1141,6 @@ mysql_plugin_get (void *cls,
1359 prepared_statement_run_select (plugin, 1141 prepared_statement_run_select (plugin,
1360 plugin->count_entry_by_hash_and_vhash, 1142 plugin->count_entry_by_hash_and_vhash,
1361 1, cbind, 1143 1, cbind,
1362 &return_ok, NULL,
1363 MYSQL_TYPE_BLOB, key, hashSize, &hashSize, 1144 MYSQL_TYPE_BLOB, key, hashSize, &hashSize,
1364 MYSQL_TYPE_BLOB, vhash, hashSize2, &hashSize2, 1145 MYSQL_TYPE_BLOB, vhash, hashSize2, &hashSize2,
1365 -1); 1146 -1);
@@ -1371,79 +1152,81 @@ mysql_plugin_get (void *cls,
1371 prepared_statement_run_select (plugin, 1152 prepared_statement_run_select (plugin,
1372 plugin->count_entry_by_hash, 1153 plugin->count_entry_by_hash,
1373 1, cbind, 1154 1, cbind,
1374 &return_ok, NULL,
1375 MYSQL_TYPE_BLOB, key, hashSize, &hashSize, 1155 MYSQL_TYPE_BLOB, key, hashSize, &hashSize,
1376 -1); 1156 -1);
1377 } 1157 }
1378 } 1158 }
1379 if ((ret != GNUNET_OK) || (0 >= total)) 1159 if ((ret != GNUNET_OK) || (0 >= total))
1380 { 1160 {
1381 iter (iter_cls, 1161 proc (proc_cls,
1382 NULL, NULL, 0, NULL, 0, 0, 0, 1162 NULL, 0, NULL, 0, 0, 0,
1383 GNUNET_TIME_UNIT_ZERO_ABS, 0); 1163 GNUNET_TIME_UNIT_ZERO_ABS, 0);
1384 return; 1164 return;
1385 } 1165 }
1166 offset = offset % total;
1167 off = (unsigned long long) offset;
1386#if DEBUG_MYSQL 1168#if DEBUG_MYSQL
1387 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1169 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1388 "Iterating over %lld results for GET `%s'\n", 1170 "Obtaining %llu/%lld result for GET `%s'\n",
1171 off,
1389 total, 1172 total,
1390 GNUNET_h2s (key)); 1173 GNUNET_h2s (key));
1391#endif 1174#endif
1392 gc = GNUNET_malloc (sizeof (struct GetContext)); 1175
1393 gc->key = *key; 1176 if (type != GNUNET_BLOCK_TYPE_ANY)
1394 if (vhash != NULL)
1395 { 1177 {
1396 gc->have_vhash = GNUNET_YES; 1178 if (NULL != vhash)
1397 gc->vhash = *vhash; 1179 {
1180 execute_select (plugin,
1181 plugin->select_entry_by_hash_vhash_and_type,
1182 proc, proc_cls,
1183 MYSQL_TYPE_BLOB, key, hashSize, &hashSize,
1184 MYSQL_TYPE_BLOB, vhash, hashSize, &hashSize,
1185 MYSQL_TYPE_LONG, &type, GNUNET_YES,
1186 MYSQL_TYPE_LONGLONG, &off, GNUNET_YES,
1187 -1);
1188 }
1189 else
1190 {
1191 execute_select (plugin,
1192 plugin->select_entry_by_hash_and_type,
1193 proc, proc_cls,
1194 MYSQL_TYPE_BLOB, key, hashSize, &hashSize,
1195 MYSQL_TYPE_LONG, &type, GNUNET_YES,
1196 MYSQL_TYPE_LONGLONG, &off, GNUNET_YES,
1197 -1);
1198 }
1199 }
1200 else
1201 {
1202 if (NULL != vhash)
1203 {
1204 execute_select (plugin,
1205 plugin->select_entry_by_hash_and_vhash,
1206 proc, proc_cls,
1207 MYSQL_TYPE_BLOB, key, hashSize, &hashSize,
1208 MYSQL_TYPE_BLOB, vhash, hashSize, &hashSize,
1209 MYSQL_TYPE_LONGLONG, &off, GNUNET_YES,
1210 -1);
1211 }
1212 else
1213 {
1214 execute_select (plugin,
1215 plugin->select_entry_by_hash,
1216 proc, proc_cls,
1217 MYSQL_TYPE_BLOB, key, hashSize, &hashSize,
1218 MYSQL_TYPE_LONGLONG, &off, GNUNET_YES,
1219 -1);
1220 }
1398 } 1221 }
1399 gc->total = total;
1400 gc->off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, total);
1401
1402
1403 nrc = GNUNET_malloc (sizeof (struct NextRequestClosure));
1404 nrc->plugin = plugin;
1405 nrc->type = type;
1406 nrc->dviter = iter;
1407 nrc->dviter_cls = iter_cls;
1408 nrc->prep = &get_statement_prepare;
1409 nrc->prep_cls = gc;
1410 mysql_plugin_next_request (nrc, GNUNET_NO);
1411}
1412
1413
1414/**
1415 * Run the prepared statement to get the next data item ready.
1416 *
1417 * @param cls not used
1418 * @param nrc closure for the next request iterator
1419 * @return GNUNET_OK on success, GNUNET_NO if there is no additional item
1420 */
1421static int
1422iterator_zero_prepare (void *cls,
1423 struct NextRequestClosure *nrc)
1424{
1425 struct Plugin *plugin;
1426 int ret;
1427
1428 if (nrc == NULL)
1429 return GNUNET_NO;
1430 plugin = nrc->plugin;
1431 ret = prepared_statement_run_select (plugin,
1432 plugin->zero_iter,
1433 7, nrc->rbind,
1434 &return_ok, NULL,
1435 MYSQL_TYPE_LONG, &nrc->count, GNUNET_YES,
1436 -1);
1437 nrc->count++;
1438 return ret;
1439} 1222}
1440 1223
1441 1224
1442/** 1225/**
1443 * Select a subset of the items in the datastore and call 1226 * Get a zero-anonymity datum from the datastore.
1444 * the given iterator for each of them.
1445 * 1227 *
1446 * @param cls our "struct Plugin*" 1228 * @param cls our "struct Plugin*"
1229 * @param offset offset of the result
1447 * @param type entries of which type should be considered? 1230 * @param type entries of which type should be considered?
1448 * Use 0 for any type. 1231 * Use 0 for any type.
1449 * @param iter function to call on each matching value; 1232 * @param iter function to call on each matching value;
@@ -1451,47 +1234,27 @@ iterator_zero_prepare (void *cls,
1451 * @param iter_cls closure for iter 1234 * @param iter_cls closure for iter
1452 */ 1235 */
1453static void 1236static void
1454mysql_plugin_iter_zero_anonymity (void *cls, 1237mysql_plugin_get_zero_anonymity (void *cls,
1455 enum GNUNET_BLOCK_Type type, 1238 uint64_t offset,
1456 PluginIterator iter, 1239 enum GNUNET_BLOCK_Type type,
1457 void *iter_cls) 1240 PluginDatumProcessor proc, void *proc_cls)
1458{ 1241{
1459 struct Plugin *plugin = cls; 1242 struct Plugin *plugin = cls;
1460 struct NextRequestClosure *nrc; 1243 unsigned long long off;
1461
1462 nrc = GNUNET_malloc (sizeof (struct NextRequestClosure));
1463 nrc->plugin = plugin;
1464 nrc->type = type;
1465 nrc->dviter = iter;
1466 nrc->dviter_cls = iter_cls;
1467 nrc->prep = &iterator_zero_prepare;
1468 mysql_plugin_next_request (nrc, GNUNET_NO);
1469}
1470 1244
1245 off = (unsigned long long) offset;
1246 execute_select (plugin,
1247 plugin->zero_iter,
1248 proc, proc_cls,
1249 MYSQL_TYPE_LONG, &type, GNUNET_YES,
1250 MYSQL_TYPE_LONGLONG, &off, GNUNET_YES,
1251 -1);
1471 1252
1472/**
1473 * Run the SELECT statement for the replication function.
1474 *
1475 * @param cls the 'struct Plugin'
1476 * @param nrc the context (not used)
1477 */
1478static int
1479replication_prepare (void *cls,
1480 struct NextRequestClosure *nrc)
1481{
1482 struct Plugin *plugin = cls;
1483
1484 return prepared_statement_run_select (plugin,
1485 plugin->select_replication,
1486 7, nrc->rbind,
1487 &return_ok, NULL,
1488 -1);
1489} 1253}
1490 1254
1491 1255
1492
1493/** 1256/**
1494 * Context for 'repl_iter' function. 1257 * Context for 'repl_proc' function.
1495 */ 1258 */
1496struct ReplCtx 1259struct ReplCtx
1497{ 1260{
@@ -1504,22 +1267,21 @@ struct ReplCtx
1504 /** 1267 /**
1505 * Function to call for the result (or the NULL). 1268 * Function to call for the result (or the NULL).
1506 */ 1269 */
1507 PluginIterator iter; 1270 PluginDatumProcessor proc;
1508 1271
1509 /** 1272 /**
1510 * Closure for iter. 1273 * Closure for proc.
1511 */ 1274 */
1512 void *iter_cls; 1275 void *proc_cls;
1513}; 1276};
1514 1277
1515 1278
1516/** 1279/**
1517 * Wrapper for the iterator for 'sqlite_plugin_replication_get'. 1280 * Wrapper for the processor for 'mysql_plugin_get_replication'.
1518 * Decrements the replication counter and calls the original 1281 * Decrements the replication counter and calls the original
1519 * iterator. 1282 * iterator.
1520 * 1283 *
1521 * @param cls closure 1284 * @param cls closure
1522 * @param next_cls closure to pass to the "next" function.
1523 * @param key key for the content 1285 * @param key key for the content
1524 * @param size number of bytes in data 1286 * @param size number of bytes in data
1525 * @param data content stored 1287 * @param data content stored
@@ -1535,8 +1297,7 @@ struct ReplCtx
1535 * GNUNET_NO to delete the item and continue (if supported) 1297 * GNUNET_NO to delete the item and continue (if supported)
1536 */ 1298 */
1537static int 1299static int
1538repl_iter (void *cls, 1300repl_proc (void *cls,
1539 void *next_cls,
1540 const GNUNET_HashCode *key, 1301 const GNUNET_HashCode *key,
1541 uint32_t size, 1302 uint32_t size,
1542 const void *data, 1303 const void *data,
@@ -1552,8 +1313,8 @@ repl_iter (void *cls,
1552 int ret; 1313 int ret;
1553 int iret; 1314 int iret;
1554 1315
1555 ret = rc->iter (rc->iter_cls, 1316 ret = rc->proc (rc->proc_cls,
1556 next_cls, key, 1317 key,
1557 size, data, 1318 size, data,
1558 type, priority, anonymity, expiration, 1319 type, priority, anonymity, expiration,
1559 uid); 1320 uid);
@@ -1561,10 +1322,10 @@ repl_iter (void *cls,
1561 { 1322 {
1562 oid = (unsigned long long) uid; 1323 oid = (unsigned long long) uid;
1563 iret = prepared_statement_run (plugin, 1324 iret = prepared_statement_run (plugin,
1564 plugin->dec_repl, 1325 plugin->dec_repl,
1565 NULL, 1326 NULL,
1566 MYSQL_TYPE_LONGLONG, &oid, GNUNET_YES, 1327 MYSQL_TYPE_LONGLONG, &oid, GNUNET_YES,
1567 -1); 1328 -1);
1568 if (iret == GNUNET_SYSERR) 1329 if (iret == GNUNET_SYSERR)
1569 { 1330 {
1570 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 1331 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
@@ -1577,94 +1338,56 @@ repl_iter (void *cls,
1577 1338
1578 1339
1579/** 1340/**
1580 * Get a random item for replication. Returns a single, not expired, random item 1341 * Get a random item for replication. Returns a single, not expired,
1581 * from those with the highest replication counters. The item's 1342 * random item from those with the highest replication counters. The
1582 * replication counter is decremented by one IF it was positive before. 1343 * item's replication counter is decremented by one IF it was positive
1583 * Call 'iter' with all values ZERO or NULL if the datastore is empty. 1344 * before. Call 'proc' with all values ZERO or NULL if the datastore
1345 * is empty.
1584 * 1346 *
1585 * @param cls closure 1347 * @param cls closure
1586 * @param iter function to call the value (once only). 1348 * @param proc function to call the value (once only).
1587 * @param iter_cls closure for iter 1349 * @param iter_cls closure for proc
1588 */ 1350 */
1589static void 1351static void
1590mysql_plugin_replication_get (void *cls, 1352mysql_plugin_get_replication (void *cls,
1591 PluginIterator iter, void *iter_cls) 1353 PluginDatumProcessor proc, void *proc_cls)
1592{ 1354{
1593 struct Plugin *plugin = cls; 1355 struct Plugin *plugin = cls;
1594 struct NextRequestClosure *nrc;
1595 struct ReplCtx rc; 1356 struct ReplCtx rc;
1596 1357
1597 rc.plugin = plugin; 1358 rc.plugin = plugin;
1598 rc.iter = iter; 1359 rc.proc = proc;
1599 rc.iter_cls = iter_cls; 1360 rc.proc_cls = proc_cls;
1600 nrc = GNUNET_malloc (sizeof (struct NextRequestClosure)); 1361 execute_select (plugin,
1601 nrc->plugin = plugin; 1362 plugin->select_replication,
1602 nrc->now = GNUNET_TIME_absolute_get (); 1363 &repl_proc, &rc,
1603 nrc->prep = &replication_prepare; 1364 -1);
1604 nrc->prep_cls = plugin;
1605 nrc->type = 0;
1606 nrc->dviter = &repl_iter;
1607 nrc->dviter_cls = &rc;
1608 nrc->end_it = GNUNET_NO;
1609 nrc->one_shot = GNUNET_YES;
1610 mysql_next_request_cont (nrc, NULL);
1611}
1612
1613 1365
1614/**
1615 * Run the SELECT statement for the expiration function.
1616 *
1617 * @param cls the 'struct Plugin'
1618 * @param nrc the context (not used)
1619 * @return GNUNET_OK on success, GNUNET_NO if there are
1620 * no more values, GNUNET_SYSERR on error
1621 */
1622static int
1623expiration_prepare (void *cls,
1624 struct NextRequestClosure *nrc)
1625{
1626 struct Plugin *plugin = cls;
1627 long long nt;
1628
1629 if (NULL == nrc)
1630 return GNUNET_NO;
1631 nt = (long long) nrc->now.abs_value;
1632 return prepared_statement_run_select
1633 (plugin,
1634 plugin->select_expiration,
1635 7, nrc->rbind,
1636 &return_ok, NULL,
1637 MYSQL_TYPE_LONGLONG, &nt, GNUNET_YES,
1638 -1);
1639} 1366}
1640 1367
1641 1368
1642/** 1369/**
1643 * Get a random item for expiration. 1370 * Get a random item for expiration.
1644 * Call 'iter' with all values ZERO or NULL if the datastore is empty. 1371 * Call 'proc' with all values ZERO or NULL if the datastore is empty.
1645 * 1372 *
1646 * @param cls closure 1373 * @param cls closure
1647 * @param iter function to call the value (once only). 1374 * @param proc function to call the value (once only).
1648 * @param iter_cls closure for iter 1375 * @param proc_cls closure for proc
1649 */ 1376 */
1650static void 1377static void
1651mysql_plugin_expiration_get (void *cls, 1378mysql_plugin_get_expiration (void *cls,
1652 PluginIterator iter, void *iter_cls) 1379 PluginDatumProcessor proc, void *proc_cls)
1653{ 1380{
1654 struct Plugin *plugin = cls; 1381 struct Plugin *plugin = cls;
1655 struct NextRequestClosure *nrc; 1382 long long nt;
1656 1383
1657 nrc = GNUNET_malloc (sizeof (struct NextRequestClosure)); 1384 nt = (long long) GNUNET_TIME_absolute_get().abs_value;
1658 nrc->plugin = plugin; 1385 execute_select (plugin,
1659 nrc->now = GNUNET_TIME_absolute_get (); 1386 plugin->select_expiration,
1660 nrc->prep = &expiration_prepare; 1387 proc, proc_cls,
1661 nrc->prep_cls = plugin; 1388 MYSQL_TYPE_LONGLONG, &nt, GNUNET_YES,
1662 nrc->type = 0; 1389 -1);
1663 nrc->dviter = iter; 1390
1664 nrc->dviter_cls = iter_cls;
1665 nrc->end_it = GNUNET_NO;
1666 nrc->one_shot = GNUNET_YES;
1667 mysql_next_request_cont (nrc, NULL);
1668} 1391}
1669 1392
1670 1393
@@ -1760,14 +1483,13 @@ libgnunet_plugin_datastore_mysql_init (void *cls)
1760 1483
1761 api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions)); 1484 api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions));
1762 api->cls = plugin; 1485 api->cls = plugin;
1763 api->get_size = &mysql_plugin_get_size; 1486 api->estimate_size = &mysql_plugin_estimate_size;
1764 api->put = &mysql_plugin_put; 1487 api->put = &mysql_plugin_put;
1765 api->next_request = &mysql_plugin_next_request;
1766 api->get = &mysql_plugin_get;
1767 api->replication_get = &mysql_plugin_replication_get;
1768 api->expiration_get = &mysql_plugin_expiration_get;
1769 api->update = &mysql_plugin_update; 1488 api->update = &mysql_plugin_update;
1770 api->iter_zero_anonymity = &mysql_plugin_iter_zero_anonymity; 1489 api->get_key = &mysql_plugin_get_key;
1490 api->get_replication = &mysql_plugin_get_replication;
1491 api->get_expiration = &mysql_plugin_get_expiration;
1492 api->get_zero_anonymity = &mysql_plugin_get_zero_anonymity;
1771 api->drop = &mysql_plugin_drop; 1493 api->drop = &mysql_plugin_drop;
1772 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, 1494 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
1773 "mysql", _("Mysql database running\n")); 1495 "mysql", _("Mysql database running\n"));
@@ -1787,14 +1509,6 @@ libgnunet_plugin_datastore_mysql_done (void *cls)
1787 struct Plugin *plugin = api->cls; 1509 struct Plugin *plugin = api->cls;
1788 1510
1789 iclose (plugin); 1511 iclose (plugin);
1790 if (plugin->next_task != GNUNET_SCHEDULER_NO_TASK)
1791 {
1792 GNUNET_SCHEDULER_cancel (plugin->next_task);
1793 plugin->next_task = GNUNET_SCHEDULER_NO_TASK;
1794 plugin->next_task_nc->prep (plugin->next_task_nc->prep_cls, NULL);
1795 GNUNET_free (plugin->next_task_nc);
1796 plugin->next_task_nc = NULL;
1797 }
1798 GNUNET_free_non_null (plugin->cnffile); 1512 GNUNET_free_non_null (plugin->cnffile);
1799 GNUNET_free (plugin); 1513 GNUNET_free (plugin);
1800 GNUNET_free (api); 1514 GNUNET_free (api);
diff --git a/src/datastore/plugin_datastore_postgres.c b/src/datastore/plugin_datastore_postgres.c
index aea87fdf4..cb077f06a 100644
--- a/src/datastore/plugin_datastore_postgres.c
+++ b/src/datastore/plugin_datastore_postgres.c
@@ -44,103 +44,6 @@
44 44
45 45
46/** 46/**
47 * Closure for 'postgres_next_request_cont'.
48 */
49struct NextRequestClosure
50{
51 /**
52 * Global plugin data.
53 */
54 struct Plugin *plugin;
55
56 /**
57 * Function to call for each matching entry.
58 */
59 PluginIterator iter;
60
61 /**
62 * Closure for 'iter'.
63 */
64 void *iter_cls;
65
66 /**
67 * Parameters for the prepared statement.
68 */
69 const char *paramValues[5];
70
71 /**
72 * Name of the prepared statement to run.
73 */
74 const char *pname;
75
76 /**
77 * Size of values pointed to by paramValues.
78 */
79 int paramLengths[5];
80
81 /**
82 * Number of paramters in paramValues/paramLengths.
83 */
84 int nparams;
85
86 /**
87 * Current time (possible parameter), big-endian.
88 */
89 uint64_t bnow;
90
91 /**
92 * Key (possible parameter)
93 */
94 GNUNET_HashCode key;
95
96 /**
97 * Hash of value (possible parameter)
98 */
99 GNUNET_HashCode vhash;
100
101 /**
102 * Number of entries found so far
103 */
104 unsigned long long count;
105
106 /**
107 * Offset this iteration starts at.
108 */
109 uint64_t off;
110
111 /**
112 * Current offset to use in query, big-endian.
113 */
114 uint64_t blimit_off;
115
116 /**
117 * Current total number of entries found so far, big-endian.
118 */
119 uint64_t bcount;
120
121 /**
122 * Overall number of matching entries.
123 */
124 unsigned long long total;
125
126 /**
127 * Type of block (possible paramter), big-endian.
128 */
129 uint32_t btype;
130
131 /**
132 * Flag set to GNUNET_YES to stop iteration.
133 */
134 int end_it;
135
136 /**
137 * Flag to indicate that there should only be one result.
138 */
139 int one_shot;
140};
141
142
143/**
144 * Context for all functions in this plugin. 47 * Context for all functions in this plugin.
145 */ 48 */
146struct Plugin 49struct Plugin
@@ -155,16 +58,6 @@ struct Plugin
155 */ 58 */
156 PGconn *dbh; 59 PGconn *dbh;
157 60
158 /**
159 * Closure of the 'next_task' (must be freed if 'next_task' is cancelled).
160 */
161 struct NextRequestClosure *next_task_nc;
162
163 /**
164 * Pending task with scheduler for running the next request.
165 */
166 GNUNET_SCHEDULER_TaskIdentifier next_task;
167
168}; 61};
169 62
170 63
@@ -434,7 +327,7 @@ init_connection (struct Plugin *plugin)
434 pq_prepare (plugin, 327 pq_prepare (plugin,
435 "select_non_anonymous", 328 "select_non_anonymous",
436 "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 " 329 "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
437 "WHERE anonLevel = 0 ORDER BY oid DESC LIMIT 1 OFFSET $1", 330 "WHERE anonLevel = 0 AND type = $1 ORDER BY oid DESC LIMIT 1 OFFSET $2",
438 1, 331 1,
439 __LINE__)) || 332 __LINE__)) ||
440 (GNUNET_OK != 333 (GNUNET_OK !=
@@ -482,11 +375,13 @@ static int
482delete_by_rowid (struct Plugin *plugin, 375delete_by_rowid (struct Plugin *plugin,
483 unsigned int rowid) 376 unsigned int rowid)
484{ 377{
485 const char *paramValues[] = { (const char *) &rowid }; 378 uint32_t browid;
486 int paramLengths[] = { sizeof (rowid) }; 379 const char *paramValues[] = { (const char *) &browid };
380 int paramLengths[] = { sizeof (browid) };
487 const int paramFormats[] = { 1 }; 381 const int paramFormats[] = { 1 };
488 PGresult *ret; 382 PGresult *ret;
489 383
384 browid = htonl (rowid);
490 ret = PQexecPrepared (plugin->dbh, 385 ret = PQexecPrepared (plugin->dbh,
491 "delrow", 386 "delrow",
492 1, paramValues, paramLengths, paramFormats, 1); 387 1, paramValues, paramLengths, paramFormats, 1);
@@ -510,7 +405,7 @@ delete_by_rowid (struct Plugin *plugin,
510 * @return number of bytes used on disk 405 * @return number of bytes used on disk
511 */ 406 */
512static unsigned long long 407static unsigned long long
513postgres_plugin_get_size (void *cls) 408postgres_plugin_estimate_size (void *cls)
514{ 409{
515 struct Plugin *plugin = cls; 410 struct Plugin *plugin = cls;
516 unsigned long long total; 411 unsigned long long total;
@@ -619,22 +514,20 @@ postgres_plugin_put (void *cls,
619 514
620 515
621/** 516/**
622 * Function invoked on behalf of a "PluginIterator" 517 * Function invoked to process the result and call
623 * asking the database plugin to call the iterator 518 * the processor.
624 * with the next item.
625 * 519 *
626 * @param next_cls the 'struct NextRequestClosure' 520 * @param plugin global plugin data
627 * @param tc scheduler context 521 * @param proc function to call the value (once only).
522 * @param proc_cls closure for proc
523 * @param res result from exec
628 */ 524 */
629static void 525static void
630postgres_next_request_cont (void *next_cls, 526process_result (struct Plugin *plugin,
631 const struct GNUNET_SCHEDULER_TaskContext *tc) 527 PluginDatumProcessor proc, void *proc_cls,
528 PGresult *res)
632{ 529{
633 struct NextRequestClosure *nrc = next_cls;
634 struct Plugin *plugin = nrc->plugin;
635 const int paramFormats[] = { 1, 1, 1, 1, 1 };
636 int iret; 530 int iret;
637 PGresult *res;
638 enum GNUNET_BLOCK_Type type; 531 enum GNUNET_BLOCK_Type type;
639 uint32_t anonymity; 532 uint32_t anonymity;
640 uint32_t priority; 533 uint32_t priority;
@@ -643,38 +536,11 @@ postgres_next_request_cont (void *next_cls,
643 struct GNUNET_TIME_Absolute expiration_time; 536 struct GNUNET_TIME_Absolute expiration_time;
644 GNUNET_HashCode key; 537 GNUNET_HashCode key;
645 538
646 plugin->next_task = GNUNET_SCHEDULER_NO_TASK;
647 plugin->next_task_nc = NULL;
648 if ( (GNUNET_YES == nrc->end_it) ||
649 (nrc->count == nrc->total) )
650 {
651#if DEBUG_POSTGRES
652 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
653 "datastore-postgres",
654 "Ending iteration (%s)\n",
655 (GNUNET_YES == nrc->end_it) ? "client requested it" : "completed result set");
656#endif
657 nrc->iter (nrc->iter_cls,
658 NULL, NULL, 0, NULL, 0, 0, 0,
659 GNUNET_TIME_UNIT_ZERO_ABS, 0);
660 GNUNET_free (nrc);
661 return;
662 }
663 if (nrc->off == nrc->total)
664 nrc->off = 0;
665 nrc->blimit_off = GNUNET_htonll (nrc->off);
666 nrc->bcount = GNUNET_htonll ((uint64_t) nrc->count);
667 res = PQexecPrepared (plugin->dbh,
668 nrc->pname,
669 nrc->nparams,
670 nrc->paramValues,
671 nrc->paramLengths,
672 paramFormats, 1);
673 if (GNUNET_OK != check_result (plugin, 539 if (GNUNET_OK != check_result (plugin,
674 res, 540 res,
675 PGRES_TUPLES_OK, 541 PGRES_TUPLES_OK,
676 "PQexecPrepared", 542 "PQexecPrepared",
677 nrc->pname, 543 "select",
678 __LINE__)) 544 __LINE__))
679 { 545 {
680#if DEBUG_POSTGRES 546#if DEBUG_POSTGRES
@@ -682,10 +548,9 @@ postgres_next_request_cont (void *next_cls,
682 "datastore-postgres", 548 "datastore-postgres",
683 "Ending iteration (postgres error)\n"); 549 "Ending iteration (postgres error)\n");
684#endif 550#endif
685 nrc->iter (nrc->iter_cls, 551 proc (proc_cls,
686 NULL, NULL, 0, NULL, 0, 0, 0, 552 NULL, 0, NULL, 0, 0, 0,
687 GNUNET_TIME_UNIT_ZERO_ABS, 0); 553 GNUNET_TIME_UNIT_ZERO_ABS, 0);
688 GNUNET_free (nrc);
689 return; 554 return;
690 } 555 }
691 556
@@ -697,11 +562,10 @@ postgres_next_request_cont (void *next_cls,
697 "datastore-postgres", 562 "datastore-postgres",
698 "Ending iteration (no more results)\n"); 563 "Ending iteration (no more results)\n");
699#endif 564#endif
700 nrc->iter (nrc->iter_cls, 565 proc (proc_cls,
701 NULL, NULL, 0, NULL, 0, 0, 0, 566 NULL, 0, NULL, 0, 0, 0,
702 GNUNET_TIME_UNIT_ZERO_ABS, 0); 567 GNUNET_TIME_UNIT_ZERO_ABS, 0);
703 PQclear (res); 568 PQclear (res);
704 GNUNET_free (nrc);
705 return; 569 return;
706 } 570 }
707 if ((1 != PQntuples (res)) || 571 if ((1 != PQntuples (res)) ||
@@ -710,11 +574,10 @@ postgres_next_request_cont (void *next_cls,
710 (sizeof (uint32_t) != PQfsize (res, 6))) 574 (sizeof (uint32_t) != PQfsize (res, 6)))
711 { 575 {
712 GNUNET_break (0); 576 GNUNET_break (0);
713 nrc->iter (nrc->iter_cls, 577 proc (proc_cls,
714 NULL, NULL, 0, NULL, 0, 0, 0, 578 NULL, 0, NULL, 0, 0, 0,
715 GNUNET_TIME_UNIT_ZERO_ABS, 0); 579 GNUNET_TIME_UNIT_ZERO_ABS, 0);
716 PQclear (res); 580 PQclear (res);
717 GNUNET_free (nrc);
718 return; 581 return;
719 } 582 }
720 rowid = ntohl (*(uint32_t *) PQgetvalue (res, 0, 6)); 583 rowid = ntohl (*(uint32_t *) PQgetvalue (res, 0, 6));
@@ -727,10 +590,9 @@ postgres_next_request_cont (void *next_cls,
727 GNUNET_break (0); 590 GNUNET_break (0);
728 PQclear (res); 591 PQclear (res);
729 delete_by_rowid (plugin, rowid); 592 delete_by_rowid (plugin, rowid);
730 nrc->iter (nrc->iter_cls, 593 proc (proc_cls,
731 NULL, NULL, 0, NULL, 0, 0, 0, 594 NULL, 0, NULL, 0, 0, 0,
732 GNUNET_TIME_UNIT_ZERO_ABS, 0); 595 GNUNET_TIME_UNIT_ZERO_ABS, 0);
733 GNUNET_free (nrc);
734 return; 596 return;
735 } 597 }
736 598
@@ -749,33 +611,23 @@ postgres_next_request_cont (void *next_cls,
749 (unsigned int) size, 611 (unsigned int) size,
750 (unsigned int) type); 612 (unsigned int) type);
751#endif 613#endif
752 iret = nrc->iter (nrc->iter_cls, 614 iret = proc (proc_cls,
753 (nrc->one_shot == GNUNET_YES) ? NULL : nrc, 615 &key,
754 &key, 616 size,
755 size, 617 PQgetvalue (res, 0, 5),
756 PQgetvalue (res, 0, 5), 618 (enum GNUNET_BLOCK_Type) type,
757 (enum GNUNET_BLOCK_Type) type, 619 priority,
758 priority, 620 anonymity,
759 anonymity, 621 expiration_time,
760 expiration_time, 622 rowid);
761 rowid);
762 PQclear (res); 623 PQclear (res);
763 if (iret != GNUNET_NO) 624 if (iret == GNUNET_NO)
764 {
765 nrc->count++;
766 nrc->off++;
767 }
768 if (iret == GNUNET_SYSERR)
769 { 625 {
770#if DEBUG_POSTGRES 626#if DEBUG_POSTGRES
771 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, 627 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
772 "datastore-postgres", 628 "Processor asked for item %u to be removed.\n",
773 "Ending iteration (client error)\n"); 629 rowid);
774#endif 630#endif
775 return;
776 }
777 if (iret == GNUNET_NO)
778 {
779 if (GNUNET_OK == delete_by_rowid (plugin, rowid)) 631 if (GNUNET_OK == delete_by_rowid (plugin, rowid))
780 { 632 {
781#if DEBUG_POSTGRES 633#if DEBUG_POSTGRES
@@ -794,34 +646,6 @@ postgres_next_request_cont (void *next_cls,
794#endif 646#endif
795 } 647 }
796 } 648 }
797 if (nrc->one_shot == GNUNET_YES)
798 GNUNET_free (nrc);
799}
800
801
802/**
803 * Function invoked on behalf of a "PluginIterator"
804 * asking the database plugin to call the iterator
805 * with the next item.
806 *
807 * @param next_cls whatever argument was given
808 * to the PluginIterator as "next_cls".
809 * @param end_it set to GNUNET_YES if we
810 * should terminate the iteration early
811 * (iterator should be still called once more
812 * to signal the end of the iteration).
813 */
814static void
815postgres_plugin_next_request (void *next_cls,
816 int end_it)
817{
818 struct NextRequestClosure *nrc = next_cls;
819
820 if (GNUNET_YES == end_it)
821 nrc->end_it = GNUNET_YES;
822 nrc->plugin->next_task_nc = nrc;
823 nrc->plugin->next_task = GNUNET_SCHEDULER_add_now (&postgres_next_request_cont,
824 nrc);
825} 649}
826 650
827 651
@@ -843,62 +667,62 @@ postgres_plugin_next_request (void *next_cls,
843 * @param iter_cls closure for iter 667 * @param iter_cls closure for iter
844 */ 668 */
845static void 669static void
846postgres_plugin_get (void *cls, 670postgres_plugin_get_key (void *cls,
847 const GNUNET_HashCode * key, 671 uint64_t offset,
848 const GNUNET_HashCode * vhash, 672 const GNUNET_HashCode *key,
849 enum GNUNET_BLOCK_Type type, 673 const GNUNET_HashCode *vhash,
850 PluginIterator iter, void *iter_cls) 674 enum GNUNET_BLOCK_Type type,
675 PluginDatumProcessor proc, void *proc_cls)
851{ 676{
852 struct Plugin *plugin = cls; 677 struct Plugin *plugin = cls;
853 struct NextRequestClosure *nrc;
854 const int paramFormats[] = { 1, 1, 1, 1, 1 }; 678 const int paramFormats[] = { 1, 1, 1, 1, 1 };
679 int paramLengths[4];
680 const char *paramValues[4];
681 int nparams;
682 const char *pname;
855 PGresult *ret; 683 PGresult *ret;
684 uint64_t total;
685 uint64_t blimit_off;
686 uint32_t btype;
856 687
857 GNUNET_assert (key != NULL); 688 GNUNET_assert (key != NULL);
858 nrc = GNUNET_malloc (sizeof (struct NextRequestClosure)); 689 paramValues[0] = (const char*) key;
859 nrc->plugin = plugin; 690 paramLengths[0] = sizeof (GNUNET_HashCode);
860 nrc->iter = iter; 691 btype = htonl (type);
861 nrc->iter_cls = iter_cls;
862 nrc->key = *key;
863 if (vhash != NULL)
864 nrc->vhash = *vhash;
865 nrc->paramValues[0] = (const char*) &nrc->key;
866 nrc->paramLengths[0] = sizeof (GNUNET_HashCode);
867 nrc->btype = htonl (type);
868 if (type != 0) 692 if (type != 0)
869 { 693 {
870 if (vhash != NULL) 694 if (vhash != NULL)
871 { 695 {
872 nrc->paramValues[1] = (const char *) &nrc->vhash; 696 paramValues[1] = (const char *) vhash;
873 nrc->paramLengths[1] = sizeof (nrc->vhash); 697 paramLengths[1] = sizeof (GNUNET_HashCode);
874 nrc->paramValues[2] = (const char *) &nrc->btype; 698 paramValues[2] = (const char *) &btype;
875 nrc->paramLengths[2] = sizeof (nrc->btype); 699 paramLengths[2] = sizeof (btype);
876 nrc->paramValues[3] = (const char *) &nrc->blimit_off; 700 paramValues[3] = (const char *) &blimit_off;
877 nrc->paramLengths[3] = sizeof (nrc->blimit_off); 701 paramLengths[3] = sizeof (blimit_off);
878 nrc->nparams = 4; 702 nparams = 4;
879 nrc->pname = "getvt"; 703 pname = "getvt";
880 ret = PQexecParams (plugin->dbh, 704 ret = PQexecParams (plugin->dbh,
881 "SELECT count(*) FROM gn090 WHERE hash=$1 AND vhash=$2 AND type=$3", 705 "SELECT count(*) FROM gn090 WHERE hash=$1 AND vhash=$2 AND type=$3",
882 3, 706 3,
883 NULL, 707 NULL,
884 nrc->paramValues, 708 paramValues,
885 nrc->paramLengths, 709 paramLengths,
886 paramFormats, 1); 710 paramFormats, 1);
887 } 711 }
888 else 712 else
889 { 713 {
890 nrc->paramValues[1] = (const char *) &nrc->btype; 714 paramValues[1] = (const char *) &btype;
891 nrc->paramLengths[1] = sizeof (nrc->btype); 715 paramLengths[1] = sizeof (btype);
892 nrc->paramValues[2] = (const char *) &nrc->blimit_off; 716 paramValues[2] = (const char *) &blimit_off;
893 nrc->paramLengths[2] = sizeof (nrc->blimit_off); 717 paramLengths[2] = sizeof (blimit_off);
894 nrc->nparams = 3; 718 nparams = 3;
895 nrc->pname = "gett"; 719 pname = "gett";
896 ret = PQexecParams (plugin->dbh, 720 ret = PQexecParams (plugin->dbh,
897 "SELECT count(*) FROM gn090 WHERE hash=$1 AND type=$2", 721 "SELECT count(*) FROM gn090 WHERE hash=$1 AND type=$2",
898 2, 722 2,
899 NULL, 723 NULL,
900 nrc->paramValues, 724 paramValues,
901 nrc->paramLengths, 725 paramLengths,
902 paramFormats, 1); 726 paramFormats, 1);
903 } 727 }
904 } 728 }
@@ -906,32 +730,32 @@ postgres_plugin_get (void *cls,
906 { 730 {
907 if (vhash != NULL) 731 if (vhash != NULL)
908 { 732 {
909 nrc->paramValues[1] = (const char *) &nrc->vhash; 733 paramValues[1] = (const char *) vhash;
910 nrc->paramLengths[1] = sizeof (nrc->vhash); 734 paramLengths[1] = sizeof (GNUNET_HashCode);
911 nrc->paramValues[2] = (const char *) &nrc->blimit_off; 735 paramValues[2] = (const char *) &blimit_off;
912 nrc->paramLengths[2] = sizeof (nrc->blimit_off); 736 paramLengths[2] = sizeof (blimit_off);
913 nrc->nparams = 3; 737 nparams = 3;
914 nrc->pname = "getv"; 738 pname = "getv";
915 ret = PQexecParams (plugin->dbh, 739 ret = PQexecParams (plugin->dbh,
916 "SELECT count(*) FROM gn090 WHERE hash=$1 AND vhash=$2", 740 "SELECT count(*) FROM gn090 WHERE hash=$1 AND vhash=$2",
917 2, 741 2,
918 NULL, 742 NULL,
919 nrc->paramValues, 743 paramValues,
920 nrc->paramLengths, 744 paramLengths,
921 paramFormats, 1); 745 paramFormats, 1);
922 } 746 }
923 else 747 else
924 { 748 {
925 nrc->paramValues[1] = (const char *) &nrc->blimit_off; 749 paramValues[1] = (const char *) &blimit_off;
926 nrc->paramLengths[1] = sizeof (nrc->blimit_off); 750 paramLengths[1] = sizeof (blimit_off);
927 nrc->nparams = 2; 751 nparams = 2;
928 nrc->pname = "get"; 752 pname = "get";
929 ret = PQexecParams (plugin->dbh, 753 ret = PQexecParams (plugin->dbh,
930 "SELECT count(*) FROM gn090 WHERE hash=$1", 754 "SELECT count(*) FROM gn090 WHERE hash=$1",
931 1, 755 1,
932 NULL, 756 NULL,
933 nrc->paramValues, 757 paramValues,
934 nrc->paramLengths, 758 paramLengths,
935 paramFormats, 1); 759 paramFormats, 1);
936 } 760 }
937 } 761 }
@@ -939,13 +763,12 @@ postgres_plugin_get (void *cls,
939 ret, 763 ret,
940 PGRES_TUPLES_OK, 764 PGRES_TUPLES_OK,
941 "PQexecParams", 765 "PQexecParams",
942 nrc->pname, 766 pname,
943 __LINE__)) 767 __LINE__))
944 { 768 {
945 iter (iter_cls, 769 proc (proc_cls,
946 NULL, NULL, 0, NULL, 0, 0, 0, 770 NULL, 0, NULL, 0, 0, 0,
947 GNUNET_TIME_UNIT_ZERO_ABS, 0); 771 GNUNET_TIME_UNIT_ZERO_ABS, 0);
948 GNUNET_free (nrc);
949 return; 772 return;
950 } 773 }
951 if ((PQntuples (ret) != 1) || 774 if ((PQntuples (ret) != 1) ||
@@ -954,26 +777,30 @@ postgres_plugin_get (void *cls,
954 { 777 {
955 GNUNET_break (0); 778 GNUNET_break (0);
956 PQclear (ret); 779 PQclear (ret);
957 iter (iter_cls, 780 proc (proc_cls,
958 NULL, NULL, 0, NULL, 0, 0, 0, 781 NULL, 0, NULL, 0, 0, 0,
959 GNUNET_TIME_UNIT_ZERO_ABS, 0); 782 GNUNET_TIME_UNIT_ZERO_ABS, 0);
960 GNUNET_free (nrc);
961 return; 783 return;
962 } 784 }
963 nrc->total = GNUNET_ntohll (*(const unsigned long long *) PQgetvalue (ret, 0, 0)); 785 total = GNUNET_ntohll (*(const unsigned long long *) PQgetvalue (ret, 0, 0));
964 PQclear (ret); 786 PQclear (ret);
965 if (nrc->total == 0) 787 if (total == 0)
966 { 788 {
967 iter (iter_cls, 789 proc (proc_cls,
968 NULL, NULL, 0, NULL, 0, 0, 0, 790 NULL, 0, NULL, 0, 0, 0,
969 GNUNET_TIME_UNIT_ZERO_ABS, 0); 791 GNUNET_TIME_UNIT_ZERO_ABS, 0);
970 GNUNET_free (nrc);
971 return; 792 return;
972 } 793 }
973 nrc->off = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, 794 blimit_off = GNUNET_htonll (offset % total);
974 nrc->total); 795 ret = PQexecPrepared (plugin->dbh,
975 postgres_plugin_next_request (nrc, 796 pname,
976 GNUNET_NO); 797 nparams,
798 paramValues,
799 paramLengths,
800 paramFormats, 1);
801 process_result (plugin,
802 proc, proc_cls,
803 ret);
977} 804}
978 805
979 806
@@ -989,28 +816,33 @@ postgres_plugin_get (void *cls,
989 * @param iter_cls closure for iter 816 * @param iter_cls closure for iter
990 */ 817 */
991static void 818static void
992postgres_plugin_iter_zero_anonymity (void *cls, 819postgres_plugin_get_zero_anonymity (void *cls,
993 enum GNUNET_BLOCK_Type type, 820 uint64_t offset,
994 PluginIterator iter, 821 enum GNUNET_BLOCK_Type type,
995 void *iter_cls) 822 PluginDatumProcessor proc, void *proc_cls)
996{ 823{
997 struct Plugin *plugin = cls; 824 struct Plugin *plugin = cls;
998 struct NextRequestClosure *nrc; 825 uint32_t btype;
826 uint64_t boff;
827 const int paramFormats[] = { 1, 1 };
828 int paramLengths[] = { sizeof (btype), sizeof (boff) };
829 const char *paramValues[] = { (const char*) &btype, (const char*) &boff };
830 PGresult *ret;
999 831
1000 nrc = GNUNET_malloc (sizeof (struct NextRequestClosure)); 832 btype = htonl ((uint32_t) type);
1001 nrc->total = UINT32_MAX; 833 boff = GNUNET_htonll (offset);
1002 nrc->btype = htonl ((uint32_t) type); 834 ret = PQexecPrepared (plugin->dbh,
1003 nrc->plugin = plugin; 835 "select_non_anonymous",
1004 nrc->iter = iter; 836 2,
1005 nrc->iter_cls = iter_cls; 837 paramValues,
1006 nrc->pname = "select_non_anonymous"; 838 paramLengths,
1007 nrc->nparams = 1; 839 paramFormats, 1);
1008 nrc->paramLengths[0] = sizeof (nrc->bcount); 840 process_result (plugin,
1009 nrc->paramValues[0] = (const char*) &nrc->bcount; 841 proc, proc_cls,
1010 postgres_plugin_next_request (nrc, 842 ret);
1011 GNUNET_NO);
1012} 843}
1013 844
845
1014/** 846/**
1015 * Context for 'repl_iter' function. 847 * Context for 'repl_iter' function.
1016 */ 848 */
@@ -1025,12 +857,12 @@ struct ReplCtx
1025 /** 857 /**
1026 * Function to call for the result (or the NULL). 858 * Function to call for the result (or the NULL).
1027 */ 859 */
1028 PluginIterator iter; 860 PluginDatumProcessor proc;
1029 861
1030 /** 862 /**
1031 * Closure for iter. 863 * Closure for proc.
1032 */ 864 */
1033 void *iter_cls; 865 void *proc_cls;
1034}; 866};
1035 867
1036 868
@@ -1056,8 +888,7 @@ struct ReplCtx
1056 * GNUNET_NO to delete the item and continue (if supported) 888 * GNUNET_NO to delete the item and continue (if supported)
1057 */ 889 */
1058static int 890static int
1059repl_iter (void *cls, 891repl_proc (void *cls,
1060 void *next_cls,
1061 const GNUNET_HashCode *key, 892 const GNUNET_HashCode *key,
1062 uint32_t size, 893 uint32_t size,
1063 const void *data, 894 const void *data,
@@ -1073,8 +904,8 @@ repl_iter (void *cls,
1073 PGresult *qret; 904 PGresult *qret;
1074 uint32_t boid; 905 uint32_t boid;
1075 906
1076 ret = rc->iter (rc->iter_cls, 907 ret = rc->proc (rc->proc_cls,
1077 next_cls, key, 908 key,
1078 size, data, 909 size, data,
1079 type, priority, anonymity, expiration, 910 type, priority, anonymity, expiration,
1080 uid); 911 uid);
@@ -1107,32 +938,30 @@ repl_iter (void *cls,
1107 * Get a random item for replication. Returns a single, not expired, random item 938 * Get a random item for replication. Returns a single, not expired, random item
1108 * from those with the highest replication counters. The item's 939 * from those with the highest replication counters. The item's
1109 * replication counter is decremented by one IF it was positive before. 940 * replication counter is decremented by one IF it was positive before.
1110 * Call 'iter' with all values ZERO or NULL if the datastore is empty. 941 * Call 'proc' with all values ZERO or NULL if the datastore is empty.
1111 * 942 *
1112 * @param cls closure 943 * @param cls closure
1113 * @param iter function to call the value (once only). 944 * @param proc function to call the value (once only).
1114 * @param iter_cls closure for iter 945 * @param proc_cls closure for iter
1115 */ 946 */
1116static void 947static void
1117postgres_plugin_replication_get (void *cls, 948postgres_plugin_get_replication (void *cls,
1118 PluginIterator iter, void *iter_cls) 949 PluginDatumProcessor proc, void *proc_cls)
1119{ 950{
1120 struct Plugin *plugin = cls; 951 struct Plugin *plugin = cls;
1121 struct NextRequestClosure *nrc;
1122 struct ReplCtx rc; 952 struct ReplCtx rc;
953 PGresult *ret;
1123 954
1124 rc.plugin = plugin; 955 rc.plugin = plugin;
1125 rc.iter = iter; 956 rc.proc = proc;
1126 rc.iter_cls = iter_cls; 957 rc.proc_cls = proc_cls;
1127 nrc = GNUNET_malloc (sizeof (struct NextRequestClosure)); 958 ret = PQexecPrepared (plugin->dbh,
1128 nrc->one_shot = GNUNET_YES; 959 "select_replication_order",
1129 nrc->total = 1; 960 0,
1130 nrc->plugin = plugin; 961 NULL, NULL, NULL, 1);
1131 nrc->iter = &repl_iter; 962 process_result (plugin,
1132 nrc->iter_cls = &rc; 963 &repl_proc, &rc,
1133 nrc->pname = "select_replication_order"; 964 ret);
1134 nrc->nparams = 0;
1135 postgres_next_request_cont (nrc, NULL);
1136} 965}
1137 966
1138 967
@@ -1141,29 +970,31 @@ postgres_plugin_replication_get (void *cls,
1141 * Call 'iter' with all values ZERO or NULL if the datastore is empty. 970 * Call 'iter' with all values ZERO or NULL if the datastore is empty.
1142 * 971 *
1143 * @param cls closure 972 * @param cls closure
1144 * @param iter function to call the value (once only). 973 * @param proc function to call the value (once only).
1145 * @param iter_cls closure for iter 974 * @param proc_cls closure for iter
1146 */ 975 */
1147static void 976static void
1148postgres_plugin_expiration_get (void *cls, 977postgres_plugin_get_expiration (void *cls,
1149 PluginIterator iter, void *iter_cls) 978 PluginDatumProcessor proc, void *proc_cls)
1150{ 979{
1151 struct Plugin *plugin = cls; 980 struct Plugin *plugin = cls;
1152 struct NextRequestClosure *nrc;
1153 uint64_t btime; 981 uint64_t btime;
982 const int paramFormats[] = { 1 };
983 int paramLengths[] = { sizeof (btime) };
984 const char *paramValues[] = { (const char*) &btime };
985 PGresult *ret;
1154 986
1155 btime = GNUNET_htonll (GNUNET_TIME_absolute_get ().abs_value); 987 btime = GNUNET_htonll (GNUNET_TIME_absolute_get ().abs_value);
1156 nrc = GNUNET_malloc (sizeof (struct NextRequestClosure)); 988 ret = PQexecPrepared (plugin->dbh,
1157 nrc->one_shot = GNUNET_YES; 989 "select_expiration_order",
1158 nrc->total = 1; 990 1,
1159 nrc->plugin = plugin; 991 paramValues,
1160 nrc->iter = iter; 992 paramLengths,
1161 nrc->iter_cls = iter_cls; 993 paramFormats,
1162 nrc->pname = "select_expiration_order"; 994 1);
1163 nrc->nparams = 1; 995 process_result (plugin,
1164 nrc->paramValues[0] = (const char *) &btime; 996 proc, proc_cls,
1165 nrc->paramLengths[0] = sizeof (btime); 997 ret);
1166 postgres_next_request_cont (nrc, NULL);
1167} 998}
1168 999
1169 1000
@@ -1260,14 +1091,13 @@ libgnunet_plugin_datastore_postgres_init (void *cls)
1260 } 1091 }
1261 api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions)); 1092 api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions));
1262 api->cls = plugin; 1093 api->cls = plugin;
1263 api->get_size = &postgres_plugin_get_size; 1094 api->estimate_size = &postgres_plugin_estimate_size;
1264 api->put = &postgres_plugin_put; 1095 api->put = &postgres_plugin_put;
1265 api->next_request = &postgres_plugin_next_request;
1266 api->get = &postgres_plugin_get;
1267 api->replication_get = &postgres_plugin_replication_get;
1268 api->expiration_get = &postgres_plugin_expiration_get;
1269 api->update = &postgres_plugin_update; 1096 api->update = &postgres_plugin_update;
1270 api->iter_zero_anonymity = &postgres_plugin_iter_zero_anonymity; 1097 api->get_key = &postgres_plugin_get_key;
1098 api->get_replication = &postgres_plugin_get_replication;
1099 api->get_expiration = &postgres_plugin_get_expiration;
1100 api->get_zero_anonymity = &postgres_plugin_get_zero_anonymity;
1271 api->drop = &postgres_plugin_drop; 1101 api->drop = &postgres_plugin_drop;
1272 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, 1102 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
1273 "datastore-postgres", 1103 "datastore-postgres",
@@ -1287,13 +1117,6 @@ libgnunet_plugin_datastore_postgres_done (void *cls)
1287 struct GNUNET_DATASTORE_PluginFunctions *api = cls; 1117 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1288 struct Plugin *plugin = api->cls; 1118 struct Plugin *plugin = api->cls;
1289 1119
1290 if (plugin->next_task != GNUNET_SCHEDULER_NO_TASK)
1291 {
1292 GNUNET_SCHEDULER_cancel (plugin->next_task);
1293 plugin->next_task = GNUNET_SCHEDULER_NO_TASK;
1294 GNUNET_free (plugin->next_task_nc);
1295 plugin->next_task_nc = NULL;
1296 }
1297 PQfinish (plugin->dbh); 1120 PQfinish (plugin->dbh);
1298 GNUNET_free (plugin); 1121 GNUNET_free (plugin);
1299 GNUNET_free (api); 1122 GNUNET_free (api);
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c
index 6e77ec364..3710b7eb7 100644
--- a/src/datastore/plugin_datastore_sqlite.c
+++ b/src/datastore/plugin_datastore_sqlite.c
@@ -108,19 +108,14 @@ struct Plugin
108 sqlite3_stmt *selExpi; 108 sqlite3_stmt *selExpi;
109 109
110 /** 110 /**
111 * Precompiled SQL for insertion. 111 * Precompiled SQL for expiration selection.
112 */
113 sqlite3_stmt *insertContent;
114
115 /**
116 * Closure of the 'next_task' (must be freed if 'next_task' is cancelled).
117 */ 112 */
118 struct NextContext *next_task_nc; 113 sqlite3_stmt *selZeroAnon;
119 114
120 /** 115 /**
121 * Pending task with scheduler for running the next request. 116 * Precompiled SQL for insertion.
122 */ 117 */
123 GNUNET_SCHEDULER_TaskIdentifier next_task; 118 sqlite3_stmt *insertContent;
124 119
125 /** 120 /**
126 * Should the database be dropped on shutdown? 121 * Should the database be dropped on shutdown?
@@ -326,6 +321,11 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
326 " WHERE NOT EXISTS (SELECT 1 FROM gn090 WHERE expire < ?1 LIMIT 1) OR expire < ?1 " 321 " WHERE NOT EXISTS (SELECT 1 FROM gn090 WHERE expire < ?1 LIMIT 1) OR expire < ?1 "
327 " ORDER BY prio ASC LIMIT 1", 322 " ORDER BY prio ASC LIMIT 1",
328 &plugin->selExpi) != SQLITE_OK) || 323 &plugin->selExpi) != SQLITE_OK) ||
324 (sq_prepare (plugin->dbh,
325 "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn090 "
326 "WHERE (anonLevel = 0 AND type=?1) "
327 "ORDER BY hash DESC LIMIT 1 OFFSET ?2",
328 &plugin->selZeroAnon) != SQLITE_OK) ||
329 (sq_prepare (plugin->dbh, 329 (sq_prepare (plugin->dbh,
330 "INSERT INTO gn090 (repl, type, prio, " 330 "INSERT INTO gn090 (repl, type, prio, "
331 "anonLevel, expire, hash, vhash, value) " 331 "anonLevel, expire, hash, vhash, value) "
@@ -367,6 +367,8 @@ database_shutdown (struct Plugin *plugin)
367 sqlite3_finalize (plugin->selRepl); 367 sqlite3_finalize (plugin->selRepl);
368 if (plugin->selExpi != NULL) 368 if (plugin->selExpi != NULL)
369 sqlite3_finalize (plugin->selExpi); 369 sqlite3_finalize (plugin->selExpi);
370 if (plugin->selZeroAnon != NULL)
371 sqlite3_finalize (plugin->selZeroAnon);
370 if (plugin->insertContent != NULL) 372 if (plugin->insertContent != NULL)
371 sqlite3_finalize (plugin->insertContent); 373 sqlite3_finalize (plugin->insertContent);
372 result = sqlite3_close(plugin->dbh); 374 result = sqlite3_close(plugin->dbh);
@@ -436,247 +438,6 @@ delete_by_rowid (struct Plugin* plugin,
436 438
437 439
438/** 440/**
439 * Context for the universal iterator.
440 */
441struct NextContext;
442
443/**
444 * Type of a function that will prepare
445 * the next iteration.
446 *
447 * @param cls closure
448 * @param nc the next context; NULL for the last
449 * call which gives the callback a chance to
450 * clean up the closure
451 * @return GNUNET_OK on success, GNUNET_NO if there are
452 * no more values, GNUNET_SYSERR on error
453 */
454typedef int (*PrepareFunction)(void *cls,
455 struct NextContext *nc);
456
457
458/**
459 * Context we keep for the "next request" callback.
460 */
461struct NextContext
462{
463 /**
464 * Internal state.
465 */
466 struct Plugin *plugin;
467
468 /**
469 * Function to call on the next value.
470 */
471 PluginIterator iter;
472
473 /**
474 * Closure for iter.
475 */
476 void *iter_cls;
477
478 /**
479 * Function to call to prepare the next
480 * iteration.
481 */
482 PrepareFunction prep;
483
484 /**
485 * Closure for prep.
486 */
487 void *prep_cls;
488
489 /**
490 * Statement that the iterator will get the data
491 * from (updated or set by prep).
492 */
493 sqlite3_stmt *stmt;
494
495 /**
496 * Row ID of the last result.
497 */
498 unsigned long long last_rowid;
499
500 /**
501 * Key of the last result.
502 */
503 GNUNET_HashCode lastKey;
504
505 /**
506 * Priority of the last value visited.
507 */
508 unsigned int lastPriority;
509
510 /**
511 * Number of results processed so far.
512 */
513 unsigned int count;
514
515 /**
516 * Set to GNUNET_YES if we must stop now.
517 */
518 int end_it;
519};
520
521
522/**
523 * Continuation of "sqlite_next_request".
524 *
525 * @param cls the 'struct NextContext*'
526 * @param tc the task context (unused)
527 */
528static void
529sqlite_next_request_cont (void *cls,
530 const struct GNUNET_SCHEDULER_TaskContext *tc)
531{
532 struct NextContext * nc = cls;
533 struct Plugin *plugin;
534 unsigned long long rowid;
535 int ret;
536 unsigned int size;
537 unsigned int hsize;
538 uint32_t anonymity;
539 uint32_t priority;
540 enum GNUNET_BLOCK_Type type;
541 const GNUNET_HashCode *key;
542 struct GNUNET_TIME_Absolute expiration;
543
544 plugin = nc->plugin;
545 plugin->next_task = GNUNET_SCHEDULER_NO_TASK;
546 plugin->next_task_nc = NULL;
547 if ( (GNUNET_YES == nc->end_it) ||
548 (GNUNET_OK != (nc->prep(nc->prep_cls,
549 nc))) )
550 {
551#if DEBUG_SQLITE
552 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
553 "sqlite",
554 "Iteration completes after %u results\n",
555 nc->count);
556#endif
557 END:
558 nc->iter (nc->iter_cls,
559 NULL, NULL, 0, NULL, 0, 0, 0,
560 GNUNET_TIME_UNIT_ZERO_ABS, 0);
561 nc->prep (nc->prep_cls, NULL);
562 GNUNET_free (nc);
563 return;
564 }
565
566 type = sqlite3_column_int (nc->stmt, 0);
567 priority = sqlite3_column_int (nc->stmt, 1);
568 anonymity = sqlite3_column_int (nc->stmt, 2);
569 expiration.abs_value = sqlite3_column_int64 (nc->stmt, 3);
570 hsize = sqlite3_column_bytes (nc->stmt, 4);
571 key = sqlite3_column_blob (nc->stmt, 4);
572 size = sqlite3_column_bytes (nc->stmt, 5);
573 rowid = sqlite3_column_int64 (nc->stmt, 6);
574 if (hsize != sizeof (GNUNET_HashCode))
575 {
576 GNUNET_break (0);
577 if (SQLITE_OK != sqlite3_reset (nc->stmt))
578 LOG_SQLITE (plugin, NULL,
579 GNUNET_ERROR_TYPE_ERROR |
580 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
581 if (GNUNET_OK == delete_by_rowid (plugin, rowid))
582 plugin->env->duc (plugin->env->cls,
583 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
584 goto END;
585 }
586#if DEBUG_SQLITE
587 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
588 "sqlite",
589 "Iterator returns value with type %u/key `%s'/priority %u/expiration %llu (%lld).\n",
590 type,
591 GNUNET_h2s(key),
592 priority,
593 (unsigned long long) GNUNET_TIME_absolute_get_remaining (expiration).rel_value,
594 (long long) expiration.abs_value);
595#endif
596 if (size > MAX_ITEM_SIZE)
597 {
598 GNUNET_break (0);
599 if (SQLITE_OK != sqlite3_reset (nc->stmt))
600 LOG_SQLITE (plugin, NULL,
601 GNUNET_ERROR_TYPE_ERROR |
602 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
603 if (GNUNET_OK == delete_by_rowid (plugin, rowid))
604 plugin->env->duc (plugin->env->cls,
605 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
606 goto END;
607 }
608 {
609 char data[size];
610
611 memcpy (data, sqlite3_column_blob (nc->stmt, 5), size);
612 nc->count++;
613 nc->last_rowid = rowid;
614 nc->lastPriority = priority;
615 nc->lastKey = *key;
616 if (SQLITE_OK != sqlite3_reset (nc->stmt))
617 LOG_SQLITE (plugin, NULL,
618 GNUNET_ERROR_TYPE_ERROR |
619 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
620 ret = nc->iter (nc->iter_cls, nc,
621 &nc->lastKey,
622 size, data,
623 type, priority,
624 anonymity, expiration,
625 rowid);
626 }
627 switch (ret)
628 {
629 case GNUNET_SYSERR:
630 nc->end_it = GNUNET_YES;
631 break;
632 case GNUNET_NO:
633 if (GNUNET_OK == delete_by_rowid (plugin, rowid))
634 {
635#if DEBUG_SQLITE
636 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
637 "sqlite",
638 "Removed entry %llu (%u bytes)\n",
639 (unsigned long long) rowid,
640 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
641#endif
642 plugin->env->duc (plugin->env->cls,
643 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
644 }
645 break;
646 case GNUNET_YES:
647 break;
648 default:
649 GNUNET_break (0);
650 }
651}
652
653
654/**
655 * Function invoked on behalf of a "PluginIterator" asking the
656 * database plugin to call the iterator with the next item.
657 *
658 * @param next_cls whatever argument was given
659 * to the PluginIterator as "next_cls".
660 * @param end_it set to GNUNET_YES if we
661 * should terminate the iteration early
662 * (iterator should be still called once more
663 * to signal the end of the iteration).
664 */
665static void
666sqlite_next_request (void *next_cls,
667 int end_it)
668{
669 struct NextContext * nc= next_cls;
670
671 if (GNUNET_YES == end_it)
672 nc->end_it = GNUNET_YES;
673 nc->plugin->next_task_nc = nc;
674 nc->plugin->next_task = GNUNET_SCHEDULER_add_now (&sqlite_next_request_cont,
675 nc);
676}
677
678
679/**
680 * Store an item in the datastore. 441 * Store an item in the datastore.
681 * 442 *
682 * @param cls closure 443 * @param cls closure
@@ -849,355 +610,147 @@ sqlite_plugin_update (void *cls,
849 610
850 611
851/** 612/**
852 * Internal context for an iteration. 613 * Execute statement that gets a row and call the callback
853 */ 614 * with the result. Resets the statement afterwards.
854struct ZeroIterContext
855{
856 /**
857 * First iterator statement for zero-anonymity iteration.
858 */
859 sqlite3_stmt *stmt_1;
860
861 /**
862 * Second iterator statement for zero-anonymity iteration.
863 */
864 sqlite3_stmt *stmt_2;
865
866 /**
867 * Desired type for blocks returned by this iterator.
868 */
869 enum GNUNET_BLOCK_Type type;
870};
871
872
873/**
874 * Prepare our SQL query to obtain the next record from the database.
875 * 615 *
876 * @param cls our "struct ZeroIterContext" 616 * @param plugin the plugin
877 * @param nc NULL to terminate the iteration, otherwise our context for 617 * @param stmt the statement
878 * getting the next result. 618 * @param proc processor to call
879 * @return GNUNET_OK on success, GNUNET_NO if there are no more results, 619 * @param proc_cls closure for 'proc'
880 * GNUNET_SYSERR on error (or end of iteration)
881 */ 620 */
882static int 621static void
883zero_iter_next_prepare (void *cls, 622execute_get (struct Plugin *plugin,
884 struct NextContext *nc) 623 sqlite3_stmt *stmt,
624 PluginDatumProcessor proc, void *proc_cls)
885{ 625{
886 struct ZeroIterContext *ic = cls; 626 int n;
887 struct Plugin *plugin; 627 struct GNUNET_TIME_Absolute expiration;
628 unsigned long long rowid;
629 unsigned int size;
888 int ret; 630 int ret;
889 631
890 if (nc == NULL) 632 n = sqlite3_step (stmt);
891 { 633 switch (n)
892#if DEBUG_SQLITE
893 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
894 "Asked to clean up iterator state.\n");
895#endif
896 sqlite3_finalize (ic->stmt_1);
897 sqlite3_finalize (ic->stmt_2);
898 return GNUNET_SYSERR;
899 }
900 plugin = nc->plugin;
901
902 /* first try iter 1 */
903#if DEBUG_SQLITE
904 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
905 "Restricting to results larger than the last priority %u and key `%s'\n",
906 nc->lastPriority,
907 GNUNET_h2s (&nc->lastKey));
908#endif
909 if ( (SQLITE_OK != sqlite3_bind_int (ic->stmt_1, 1, nc->lastPriority)) ||
910 (SQLITE_OK != sqlite3_bind_blob (ic->stmt_1, 2,
911 &nc->lastKey,
912 sizeof (GNUNET_HashCode),
913 SQLITE_TRANSIENT)) )
914 { 634 {
915 LOG_SQLITE (plugin, NULL, 635 case SQLITE_ROW:
916 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_bind_XXXX"); 636 size = sqlite3_column_bytes (stmt, 5);
917 if (SQLITE_OK != sqlite3_reset (ic->stmt_1)) 637 rowid = sqlite3_column_int64 (stmt, 6);
638 if (sqlite3_column_bytes (stmt, 4) != sizeof (GNUNET_HashCode))
639 {
640 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
641 "sqlite",
642 _("Invalid data in database. Trying to fix (by deletion).\n"));
643 if (SQLITE_OK != sqlite3_reset (stmt))
644 LOG_SQLITE (plugin, NULL,
645 GNUNET_ERROR_TYPE_ERROR |
646 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
647 if (GNUNET_OK == delete_by_rowid (plugin, rowid))
648 plugin->env->duc (plugin->env->cls,
649 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
650 break;
651 }
652 expiration.abs_value = sqlite3_column_int64 (stmt, 3);
653 ret = proc (proc_cls,
654 sqlite3_column_blob (stmt, 4) /* key */,
655 size,
656 sqlite3_column_blob (stmt, 5) /* data */,
657 sqlite3_column_int (stmt, 0) /* type */,
658 sqlite3_column_int (stmt, 1) /* priority */,
659 sqlite3_column_int (stmt, 2) /* anonymity */,
660 expiration,
661 rowid);
662 if (SQLITE_OK != sqlite3_reset (stmt))
918 LOG_SQLITE (plugin, NULL, 663 LOG_SQLITE (plugin, NULL,
919 GNUNET_ERROR_TYPE_ERROR | 664 GNUNET_ERROR_TYPE_ERROR |
920 GNUNET_ERROR_TYPE_BULK, 665 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
921 "sqlite3_reset"); 666 if ( (GNUNET_NO == ret) &&
922 return GNUNET_SYSERR; 667 (GNUNET_OK == delete_by_rowid (plugin, rowid)) )
923 } 668 plugin->env->duc (plugin->env->cls,
924 if (SQLITE_ROW == (ret = sqlite3_step (ic->stmt_1))) 669 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
925 { 670 return;
926#if DEBUG_SQLITE 671 case SQLITE_DONE:
927 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 672 /* database must be empty */
928 "Result found using iterator 1\n"); 673 if (SQLITE_OK != sqlite3_reset (stmt))
929#endif
930 nc->stmt = ic->stmt_1;
931 return GNUNET_OK;
932 }
933 if (ret != SQLITE_DONE)
934 {
935 LOG_SQLITE (plugin, NULL,
936 GNUNET_ERROR_TYPE_ERROR |
937 GNUNET_ERROR_TYPE_BULK,
938 "sqlite3_step");
939 if (SQLITE_OK != sqlite3_reset (ic->stmt_1))
940 LOG_SQLITE (plugin, NULL, 674 LOG_SQLITE (plugin, NULL,
941 GNUNET_ERROR_TYPE_ERROR | 675 GNUNET_ERROR_TYPE_ERROR |
942 GNUNET_ERROR_TYPE_BULK, 676 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
943 "sqlite3_reset"); 677 break;
944 return GNUNET_SYSERR; 678 case SQLITE_BUSY:
945 } 679 case SQLITE_ERROR:
946 if (SQLITE_OK != sqlite3_reset (ic->stmt_1)) 680 case SQLITE_MISUSE:
947 LOG_SQLITE (plugin, NULL, 681 default:
948 GNUNET_ERROR_TYPE_ERROR |
949 GNUNET_ERROR_TYPE_BULK,
950 "sqlite3_reset");
951
952 /* now try iter 2 */
953 if (SQLITE_OK != sqlite3_bind_int (ic->stmt_2, 1, nc->lastPriority))
954 {
955 LOG_SQLITE (plugin, NULL,
956 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_bind_XXXX");
957 return GNUNET_SYSERR;
958 }
959 if (SQLITE_ROW == (ret = sqlite3_step (ic->stmt_2)))
960 {
961#if DEBUG_SQLITE
962 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
963 "Result found using iterator 2\n");
964#endif
965 nc->stmt = ic->stmt_2;
966 return GNUNET_OK;
967 }
968 if (ret != SQLITE_DONE)
969 {
970 LOG_SQLITE (plugin, NULL, 682 LOG_SQLITE (plugin, NULL,
971 GNUNET_ERROR_TYPE_ERROR | 683 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
972 GNUNET_ERROR_TYPE_BULK,
973 "sqlite3_step"); 684 "sqlite3_step");
974 if (SQLITE_OK != sqlite3_reset (ic->stmt_2)) 685 if (SQLITE_OK != sqlite3_reset (stmt))
975 LOG_SQLITE (plugin, NULL, 686 LOG_SQLITE (plugin, NULL,
976 GNUNET_ERROR_TYPE_ERROR | 687 GNUNET_ERROR_TYPE_ERROR |
977 GNUNET_ERROR_TYPE_BULK, 688 GNUNET_ERROR_TYPE_BULK,
978 "sqlite3_reset"); 689 "sqlite3_reset");
979 return GNUNET_SYSERR; 690 GNUNET_break (0);
691 database_shutdown (plugin);
692 database_setup (plugin->env->cfg,
693 plugin);
694 break;
980 } 695 }
981 if (SQLITE_OK != sqlite3_reset (ic->stmt_2)) 696 if (SQLITE_OK != sqlite3_reset (stmt))
982 LOG_SQLITE (plugin, NULL, 697 LOG_SQLITE (plugin, NULL,
983 GNUNET_ERROR_TYPE_ERROR | 698 GNUNET_ERROR_TYPE_ERROR |
984 GNUNET_ERROR_TYPE_BULK, 699 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
985 "sqlite3_reset"); 700 proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
986#if DEBUG_SQLITE 701 GNUNET_TIME_UNIT_ZERO_ABS, 0);
987 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
988 "No result found using either iterator\n");
989#endif
990 return GNUNET_NO;
991} 702}
992 703
993 704
705
994/** 706/**
995 * Select a subset of the items in the datastore and call 707 * Select a subset of the items in the datastore and call
996 * the given iterator for each of them. 708 * the given processor for the item.
997 * 709 *
998 * @param cls our plugin context 710 * @param cls our plugin context
999 * @param type entries of which type should be considered? 711 * @param type entries of which type should be considered?
1000 * Use 0 for any type. 712 * Use 0 for any type.
1001 * @param iter function to call on each matching value; 713 * @param proc function to call on each matching value;
1002 * will be called once with a NULL value at the end 714 * will be called once with a NULL value at the end
1003 * @param iter_cls closure for iter 715 * @param proc_cls closure for proc
1004 */ 716 */
1005static void 717static void
1006sqlite_plugin_iter_zero_anonymity (void *cls, 718sqlite_plugin_get_zero_anonymity (void *cls,
1007 enum GNUNET_BLOCK_Type type, 719 uint64_t offset,
1008 PluginIterator iter, 720 enum GNUNET_BLOCK_Type type,
1009 void *iter_cls) 721 PluginDatumProcessor proc,
722 void *proc_cls)
1010{ 723{
1011 struct Plugin *plugin = cls; 724 struct Plugin *plugin = cls;
1012 struct GNUNET_TIME_Absolute now; 725 sqlite3_stmt *stmt;
1013 struct NextContext *nc;
1014 struct ZeroIterContext *ic;
1015 sqlite3_stmt *stmt_1;
1016 sqlite3_stmt *stmt_2;
1017 char *q;
1018 726
1019 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY); 727 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
1020 now = GNUNET_TIME_absolute_get (); 728 stmt = plugin->selZeroAnon;
1021 GNUNET_asprintf (&q, 729 if ( (SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) ||
1022 "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn090 " 730 (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, offset)) )
1023 "WHERE (anonLevel = 0 AND expire > %llu AND prio = ?1 AND type=%d AND hash < ?2) "
1024 "ORDER BY hash DESC LIMIT 1",
1025 (unsigned long long) now.abs_value,
1026 type);
1027 if (sq_prepare (plugin->dbh, q, &stmt_1) != SQLITE_OK)
1028 { 731 {
1029 LOG_SQLITE (plugin, NULL, 732 LOG_SQLITE (plugin, NULL,
1030 GNUNET_ERROR_TYPE_ERROR | 733 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1031 GNUNET_ERROR_TYPE_BULK, "sqlite3_prepare_v2"); 734 "sqlite3_bind_XXXX");
1032 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); 735 if (SQLITE_OK != sqlite3_reset (stmt))
1033 GNUNET_free (q); 736 LOG_SQLITE (plugin, NULL,
1034 return; 737 GNUNET_ERROR_TYPE_ERROR |
1035 } 738 GNUNET_ERROR_TYPE_BULK,
1036 GNUNET_free (q); 739 "sqlite3_reset");
1037 GNUNET_asprintf (&q, 740 proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
1038 "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn090 " 741 GNUNET_TIME_UNIT_ZERO_ABS, 0);
1039 "WHERE (anonLevel = 0 AND expire > %llu AND prio < ?1 AND type=%d) "
1040 "ORDER BY prio DESC, hash DESC LIMIT 1",
1041 (unsigned long long) now.abs_value,
1042 type);
1043 if (sq_prepare (plugin->dbh, q, &stmt_2) != SQLITE_OK)
1044 {
1045 LOG_SQLITE (plugin, NULL,
1046 GNUNET_ERROR_TYPE_ERROR |
1047 GNUNET_ERROR_TYPE_BULK, "sqlite3_prepare_v2");
1048 sqlite3_finalize (stmt_1);
1049 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1050 GNUNET_free (q);
1051 return; 742 return;
1052 } 743 }
1053 GNUNET_free (q); 744 execute_get (plugin, stmt, proc, proc_cls);
1054 nc = GNUNET_malloc (sizeof(struct NextContext) +
1055 sizeof(struct ZeroIterContext));
1056 nc->plugin = plugin;
1057 nc->iter = iter;
1058 nc->iter_cls = iter_cls;
1059 nc->stmt = NULL;
1060 ic = (struct ZeroIterContext*) &nc[1];
1061 ic->stmt_1 = stmt_1;
1062 ic->stmt_2 = stmt_2;
1063 ic->type = type;
1064 nc->prep = &zero_iter_next_prepare;
1065 nc->prep_cls = ic;
1066 nc->lastPriority = INT32_MAX;
1067 memset (&nc->lastKey, 255, sizeof (GNUNET_HashCode));
1068 sqlite_next_request (nc, GNUNET_NO);
1069} 745}
1070 746
1071 747
1072/**
1073 * Context for get_next_prepare.
1074 */
1075struct GetNextContext
1076{
1077
1078 /**
1079 * Our prepared statement.
1080 */
1081 sqlite3_stmt *stmt;
1082
1083 /**
1084 * Plugin handle.
1085 */
1086 struct Plugin *plugin;
1087
1088 /**
1089 * Key for the query.
1090 */
1091 GNUNET_HashCode key;
1092
1093 /**
1094 * Vhash for the query.
1095 */
1096 GNUNET_HashCode vhash;
1097
1098 /**
1099 * Expected total number of results.
1100 */
1101 unsigned int total;
1102
1103 /**
1104 * Offset to add for the selected result.
1105 */
1106 unsigned int off;
1107
1108 /**
1109 * Is vhash set?
1110 */
1111 int have_vhash;
1112
1113 /**
1114 * Desired block type.
1115 */
1116 enum GNUNET_BLOCK_Type type;
1117
1118};
1119
1120
1121/**
1122 * Prepare the stmt in 'nc' for the next round of execution, selecting the
1123 * next return value.
1124 *
1125 * @param cls our "struct GetNextContext*"
1126 * @param nc the general context
1127 * @return GNUNET_YES if there are more results,
1128 * GNUNET_NO if there are no more results,
1129 * GNUNET_SYSERR on internal error
1130 */
1131static int
1132get_next_prepare (void *cls,
1133 struct NextContext *nc)
1134{
1135 struct GetNextContext *gnc = cls;
1136 int ret;
1137 int limit_off;
1138 unsigned int sqoff;
1139
1140 if (nc == NULL)
1141 {
1142 sqlite3_finalize (gnc->stmt);
1143 return GNUNET_SYSERR;
1144 }
1145 if (nc->count == gnc->total)
1146 return GNUNET_NO;
1147 if (nc->count + gnc->off == gnc->total)
1148 nc->last_rowid = 0;
1149 if (nc->count == 0)
1150 limit_off = gnc->off;
1151 else
1152 limit_off = 0;
1153 sqlite3_reset (nc->stmt);
1154 sqoff = 1;
1155 ret = sqlite3_bind_blob (nc->stmt,
1156 sqoff++,
1157 &gnc->key,
1158 sizeof (GNUNET_HashCode),
1159 SQLITE_TRANSIENT);
1160 if ((gnc->have_vhash) && (ret == SQLITE_OK))
1161 ret = sqlite3_bind_blob (nc->stmt,
1162 sqoff++,
1163 &gnc->vhash,
1164 sizeof (GNUNET_HashCode), SQLITE_TRANSIENT);
1165 if ((gnc->type != 0) && (ret == SQLITE_OK))
1166 ret = sqlite3_bind_int (nc->stmt, sqoff++, gnc->type);
1167 if (ret == SQLITE_OK)
1168 ret = sqlite3_bind_int64 (nc->stmt, sqoff++, limit_off);
1169 if (ret != SQLITE_OK)
1170 return GNUNET_SYSERR;
1171#if DEBUG_SQLITE
1172 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1173 "sqlite",
1174 "Preparing to GET for key `%s' with type %d at offset %u\n",
1175 GNUNET_h2s (&gnc->key),
1176 gnc->type,
1177 limit_off);
1178#endif
1179 ret = sqlite3_step (nc->stmt);
1180 switch (ret)
1181 {
1182 case SQLITE_ROW:
1183 return GNUNET_OK;
1184 case SQLITE_DONE:
1185 return GNUNET_NO;
1186 default:
1187 LOG_SQLITE (gnc->plugin, NULL,
1188 GNUNET_ERROR_TYPE_ERROR |
1189 GNUNET_ERROR_TYPE_BULK,
1190 "sqlite3_step");
1191 return GNUNET_SYSERR;
1192 }
1193}
1194
1195 748
1196/** 749/**
1197 * Iterate over the results for a particular key 750 * Get results for a particular key in the datastore.
1198 * in the datastore.
1199 * 751 *
1200 * @param cls closure 752 * @param cls closure
753 * @param offset offset (mod count).
1201 * @param key key to match, never NULL 754 * @param key key to match, never NULL
1202 * @param vhash hash of the value, maybe NULL (to 755 * @param vhash hash of the value, maybe NULL (to
1203 * match all values that have the right key). 756 * match all values that have the right key).
@@ -1206,27 +759,27 @@ get_next_prepare (void *cls,
1206 * there may be! 759 * there may be!
1207 * @param type entries of which type are relevant? 760 * @param type entries of which type are relevant?
1208 * Use 0 for any type. 761 * Use 0 for any type.
1209 * @param iter function to call on each matching value; 762 * @param proc function to call on each matching value;
1210 * will be called once with a NULL value at the end 763 * will be called once with a NULL value at the end
1211 * @param iter_cls closure for iter 764 * @param proc_cls closure for proc
1212 */ 765 */
1213static void 766static void
1214sqlite_plugin_get (void *cls, 767sqlite_plugin_get_key (void *cls,
1215 const GNUNET_HashCode *key, 768 uint64_t offset,
1216 const GNUNET_HashCode *vhash, 769 const GNUNET_HashCode *key,
1217 enum GNUNET_BLOCK_Type type, 770 const GNUNET_HashCode *vhash,
1218 PluginIterator iter, void *iter_cls) 771 enum GNUNET_BLOCK_Type type,
772 PluginDatumProcessor proc, void *proc_cls)
1219{ 773{
1220 struct Plugin *plugin = cls; 774 struct Plugin *plugin = cls;
1221 struct GetNextContext *gnc;
1222 struct NextContext *nc;
1223 int ret; 775 int ret;
1224 int total; 776 int total;
777 int limit_off;
778 unsigned int sqoff;
1225 sqlite3_stmt *stmt; 779 sqlite3_stmt *stmt;
1226 char scratch[256]; 780 char scratch[256];
1227 unsigned int sqoff;
1228 781
1229 GNUNET_assert (iter != NULL); 782 GNUNET_assert (proc != NULL);
1230 GNUNET_assert (key != NULL); 783 GNUNET_assert (key != NULL);
1231 GNUNET_snprintf (scratch, sizeof (scratch), 784 GNUNET_snprintf (scratch, sizeof (scratch),
1232 "SELECT count(*) FROM gn090 WHERE hash=?%s%s", 785 "SELECT count(*) FROM gn090 WHERE hash=?%s%s",
@@ -1236,7 +789,7 @@ sqlite_plugin_get (void *cls,
1236 { 789 {
1237 LOG_SQLITE (plugin, NULL, 790 LOG_SQLITE (plugin, NULL,
1238 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite_prepare"); 791 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite_prepare");
1239 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); 792 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1240 return; 793 return;
1241 } 794 }
1242 sqoff = 1; 795 sqoff = 1;
@@ -1253,7 +806,7 @@ sqlite_plugin_get (void *cls,
1253 LOG_SQLITE (plugin, NULL, 806 LOG_SQLITE (plugin, NULL,
1254 GNUNET_ERROR_TYPE_ERROR, "sqlite_bind"); 807 GNUNET_ERROR_TYPE_ERROR, "sqlite_bind");
1255 sqlite3_finalize (stmt); 808 sqlite3_finalize (stmt);
1256 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); 809 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1257 return; 810 return;
1258 } 811 }
1259 ret = sqlite3_step (stmt); 812 ret = sqlite3_step (stmt);
@@ -1263,147 +816,64 @@ sqlite_plugin_get (void *cls,
1263 GNUNET_ERROR_TYPE_ERROR| GNUNET_ERROR_TYPE_BULK, 816 GNUNET_ERROR_TYPE_ERROR| GNUNET_ERROR_TYPE_BULK,
1264 "sqlite_step"); 817 "sqlite_step");
1265 sqlite3_finalize (stmt); 818 sqlite3_finalize (stmt);
1266 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); 819 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1267 return; 820 return;
1268 } 821 }
1269 total = sqlite3_column_int (stmt, 0); 822 total = sqlite3_column_int (stmt, 0);
1270 sqlite3_finalize (stmt); 823 sqlite3_finalize (stmt);
1271 if (0 == total) 824 if (0 == total)
1272 { 825 {
1273 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); 826 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1274 return; 827 return;
1275 } 828 }
829 limit_off = (int) (offset % total);
830 if (limit_off < 0)
831 limit_off += total;
1276 GNUNET_snprintf (scratch, sizeof (scratch), 832 GNUNET_snprintf (scratch, sizeof (scratch),
1277 "SELECT type, prio, anonLevel, expire, hash, value, _ROWID_ " 833 "SELECT type, prio, anonLevel, expire, hash, value, _ROWID_ "
1278 "FROM gn090 WHERE hash=?%s%s " 834 "FROM gn090 WHERE hash=?%s%s "
1279 "ORDER BY _ROWID_ ASC LIMIT 1 OFFSET ?", 835 "ORDER BY _ROWID_ ASC LIMIT 1 OFFSET ?",
1280 vhash == NULL ? "" : " AND vhash=?", 836 vhash == NULL ? "" : " AND vhash=?",
1281 type == 0 ? "" : " AND type=?"); 837 type == 0 ? "" : " AND type=?");
1282
1283 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK) 838 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
1284 { 839 {
1285 LOG_SQLITE (plugin, NULL, 840 LOG_SQLITE (plugin, NULL,
1286 GNUNET_ERROR_TYPE_ERROR | 841 GNUNET_ERROR_TYPE_ERROR |
1287 GNUNET_ERROR_TYPE_BULK, "sqlite_prepare"); 842 GNUNET_ERROR_TYPE_BULK, "sqlite_prepare");
1288 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); 843 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1289 return; 844 return;
1290 } 845 }
1291 nc = GNUNET_malloc (sizeof(struct NextContext) + 846 sqoff = 1;
1292 sizeof(struct GetNextContext)); 847 ret = sqlite3_bind_blob (stmt,
1293 nc->plugin = plugin; 848 sqoff++,
1294 nc->iter = iter; 849 key,
1295 nc->iter_cls = iter_cls; 850 sizeof (GNUNET_HashCode),
1296 nc->stmt = stmt; 851 SQLITE_TRANSIENT);
1297 gnc = (struct GetNextContext*) &nc[1]; 852 if ((vhash != NULL) && (ret == SQLITE_OK))
1298 gnc->total = total; 853 ret = sqlite3_bind_blob (stmt,
1299 gnc->type = type; 854 sqoff++,
1300 gnc->key = *key; 855 vhash,
1301 gnc->plugin = plugin; 856 sizeof (GNUNET_HashCode), SQLITE_TRANSIENT);
1302 gnc->stmt = stmt; /* alias used for freeing at the end! */ 857 if ((type != 0) && (ret == SQLITE_OK))
1303 if (NULL != vhash) 858 ret = sqlite3_bind_int (stmt, sqoff++, type);
1304 { 859 if (ret == SQLITE_OK)
1305 gnc->have_vhash = GNUNET_YES; 860 ret = sqlite3_bind_int64 (stmt, sqoff++, limit_off);
1306 gnc->vhash = *vhash; 861 if (ret != SQLITE_OK)
1307 }
1308 gnc->off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, total);
1309 nc->prep = &get_next_prepare;
1310 nc->prep_cls = gnc;
1311 sqlite_next_request (nc, GNUNET_NO);
1312}
1313
1314
1315/**
1316 * Execute statement that gets a row and call the callback
1317 * with the result. Resets the statement afterwards.
1318 *
1319 * @param plugin the plugin
1320 * @param stmt the statement
1321 * @param iter iterator to call
1322 * @param iter_cls closure for 'iter'
1323 */
1324static void
1325execute_get (struct Plugin *plugin,
1326 sqlite3_stmt *stmt,
1327 PluginIterator iter, void *iter_cls)
1328{
1329 int n;
1330 struct GNUNET_TIME_Absolute expiration;
1331 unsigned long long rowid;
1332 unsigned int size;
1333 int ret;
1334
1335 n = sqlite3_step (stmt);
1336 switch (n)
1337 { 862 {
1338 case SQLITE_ROW:
1339 size = sqlite3_column_bytes (stmt, 5);
1340 rowid = sqlite3_column_int64 (stmt, 6);
1341 if (sqlite3_column_bytes (stmt, 4) != sizeof (GNUNET_HashCode))
1342 {
1343 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
1344 "sqlite",
1345 _("Invalid data in database. Trying to fix (by deletion).\n"));
1346 if (SQLITE_OK != sqlite3_reset (stmt))
1347 LOG_SQLITE (plugin, NULL,
1348 GNUNET_ERROR_TYPE_ERROR |
1349 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
1350 if (GNUNET_OK == delete_by_rowid (plugin, rowid))
1351 plugin->env->duc (plugin->env->cls,
1352 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
1353 break;
1354 }
1355 expiration.abs_value = sqlite3_column_int64 (stmt, 3);
1356 ret = iter (iter_cls,
1357 NULL,
1358 sqlite3_column_blob (stmt, 4) /* key */,
1359 size,
1360 sqlite3_column_blob (stmt, 5) /* data */,
1361 sqlite3_column_int (stmt, 0) /* type */,
1362 sqlite3_column_int (stmt, 1) /* priority */,
1363 sqlite3_column_int (stmt, 2) /* anonymity */,
1364 expiration,
1365 rowid);
1366 if (SQLITE_OK != sqlite3_reset (stmt))
1367 LOG_SQLITE (plugin, NULL,
1368 GNUNET_ERROR_TYPE_ERROR |
1369 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
1370 if ( (GNUNET_NO == ret) &&
1371 (GNUNET_OK == delete_by_rowid (plugin, rowid)) )
1372 plugin->env->duc (plugin->env->cls,
1373 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
1374 return;
1375 case SQLITE_DONE:
1376 /* database must be empty */
1377 if (SQLITE_OK != sqlite3_reset (stmt))
1378 LOG_SQLITE (plugin, NULL,
1379 GNUNET_ERROR_TYPE_ERROR |
1380 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
1381 break;
1382 case SQLITE_BUSY:
1383 case SQLITE_ERROR:
1384 case SQLITE_MISUSE:
1385 default:
1386 LOG_SQLITE (plugin, NULL, 863 LOG_SQLITE (plugin, NULL,
1387 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, 864 GNUNET_ERROR_TYPE_ERROR |
1388 "sqlite3_step"); 865 GNUNET_ERROR_TYPE_BULK, "sqlite_bind");
1389 if (SQLITE_OK != sqlite3_reset (stmt)) 866 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1390 LOG_SQLITE (plugin, NULL, 867 return;
1391 GNUNET_ERROR_TYPE_ERROR |
1392 GNUNET_ERROR_TYPE_BULK,
1393 "sqlite3_reset");
1394 GNUNET_break (0);
1395 database_shutdown (plugin);
1396 database_setup (plugin->env->cfg,
1397 plugin);
1398 break;
1399 } 868 }
1400 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, 869 execute_get (plugin, stmt, proc, proc_cls);
1401 GNUNET_TIME_UNIT_ZERO_ABS, 0); 870 sqlite3_finalize (stmt);
1402} 871}
1403 872
1404 873
874
1405/** 875/**
1406 * Context for 'repl_iter' function. 876 * Context for 'repl_proc' function.
1407 */ 877 */
1408struct ReplCtx 878struct ReplCtx
1409{ 879{
@@ -1416,22 +886,21 @@ struct ReplCtx
1416 /** 886 /**
1417 * Function to call for the result (or the NULL). 887 * Function to call for the result (or the NULL).
1418 */ 888 */
1419 PluginIterator iter; 889 PluginDatumProcessor proc;
1420 890
1421 /** 891 /**
1422 * Closure for iter. 892 * Closure for proc.
1423 */ 893 */
1424 void *iter_cls; 894 void *proc_cls;
1425}; 895};
1426 896
1427 897
1428/** 898/**
1429 * Wrapper for the iterator for 'sqlite_plugin_replication_get'. 899 * Wrapper for the processor for 'sqlite_plugin_replication_get'.
1430 * Decrements the replication counter and calls the original 900 * Decrements the replication counter and calls the original
1431 * iterator. 901 * processor.
1432 * 902 *
1433 * @param cls closure 903 * @param cls closure
1434 * @param next_cls closure to pass to the "next" function.
1435 * @param key key for the content 904 * @param key key for the content
1436 * @param size number of bytes in data 905 * @param size number of bytes in data
1437 * @param data content stored 906 * @param data content stored
@@ -1442,13 +911,11 @@ struct ReplCtx
1442 * @param uid unique identifier for the datum; 911 * @param uid unique identifier for the datum;
1443 * maybe 0 if no unique identifier is available 912 * maybe 0 if no unique identifier is available
1444 * 913 *
1445 * @return GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue 914 * @return GNUNET_OK for normal return,
1446 * (continue on call to "next", of course), 915 * GNUNET_NO to delete the item
1447 * GNUNET_NO to delete the item and continue (if supported)
1448 */ 916 */
1449static int 917static int
1450repl_iter (void *cls, 918repl_proc (void *cls,
1451 void *next_cls,
1452 const GNUNET_HashCode *key, 919 const GNUNET_HashCode *key,
1453 uint32_t size, 920 uint32_t size,
1454 const void *data, 921 const void *data,
@@ -1462,8 +929,8 @@ repl_iter (void *cls,
1462 struct Plugin *plugin = rc->plugin; 929 struct Plugin *plugin = rc->plugin;
1463 int ret; 930 int ret;
1464 931
1465 ret = rc->iter (rc->iter_cls, 932 ret = rc->proc (rc->proc_cls,
1466 next_cls, key, 933 key,
1467 size, data, 934 size, data,
1468 type, priority, anonymity, expiration, 935 type, priority, anonymity, expiration,
1469 uid); 936 uid);
@@ -1494,15 +961,15 @@ repl_iter (void *cls,
1494 * Get a random item for replication. Returns a single random item 961 * Get a random item for replication. Returns a single random item
1495 * from those with the highest replication counters. The item's 962 * from those with the highest replication counters. The item's
1496 * replication counter is decremented by one IF it was positive before. 963 * replication counter is decremented by one IF it was positive before.
1497 * Call 'iter' with all values ZERO or NULL if the datastore is empty. 964 * Call 'proc' with all values ZERO or NULL if the datastore is empty.
1498 * 965 *
1499 * @param cls closure 966 * @param cls closure
1500 * @param iter function to call the value (once only). 967 * @param proc function to call the value (once only).
1501 * @param iter_cls closure for iter 968 * @param proc_cls closure for proc
1502 */ 969 */
1503static void 970static void
1504sqlite_plugin_replication_get (void *cls, 971sqlite_plugin_get_replication (void *cls,
1505 PluginIterator iter, void *iter_cls) 972 PluginDatumProcessor proc, void *proc_cls)
1506{ 973{
1507 struct Plugin *plugin = cls; 974 struct Plugin *plugin = cls;
1508 struct ReplCtx rc; 975 struct ReplCtx rc;
@@ -1513,24 +980,24 @@ sqlite_plugin_replication_get (void *cls,
1513 "Getting random block based on replication order.\n"); 980 "Getting random block based on replication order.\n");
1514#endif 981#endif
1515 rc.plugin = plugin; 982 rc.plugin = plugin;
1516 rc.iter = iter; 983 rc.proc = proc;
1517 rc.iter_cls = iter_cls; 984 rc.proc_cls = proc_cls;
1518 execute_get (plugin, plugin->selRepl, &repl_iter, &rc); 985 execute_get (plugin, plugin->selRepl, &repl_proc, &rc);
1519} 986}
1520 987
1521 988
1522 989
1523/** 990/**
1524 * Get a random item that has expired or has low priority. 991 * Get a random item that has expired or has low priority.
1525 * Call 'iter' with all values ZERO or NULL if the datastore is empty. 992 * Call 'proc' with all values ZERO or NULL if the datastore is empty.
1526 * 993 *
1527 * @param cls closure 994 * @param cls closure
1528 * @param iter function to call the value (once only). 995 * @param proc function to call the value (once only).
1529 * @param iter_cls closure for iter 996 * @param proc_cls closure for proc
1530 */ 997 */
1531static void 998static void
1532sqlite_plugin_expiration_get (void *cls, 999sqlite_plugin_get_expiration (void *cls,
1533 PluginIterator iter, void *iter_cls) 1000 PluginDatumProcessor proc, void *proc_cls)
1534{ 1001{
1535 struct Plugin *plugin = cls; 1002 struct Plugin *plugin = cls;
1536 sqlite3_stmt *stmt; 1003 sqlite3_stmt *stmt;
@@ -1550,11 +1017,11 @@ sqlite_plugin_expiration_get (void *cls,
1550 if (SQLITE_OK != sqlite3_reset (stmt)) 1017 if (SQLITE_OK != sqlite3_reset (stmt))
1551 LOG_SQLITE (plugin, NULL, 1018 LOG_SQLITE (plugin, NULL,
1552 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset"); 1019 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
1553 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, 1020 proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
1554 GNUNET_TIME_UNIT_ZERO_ABS, 0); 1021 GNUNET_TIME_UNIT_ZERO_ABS, 0);
1555 return; 1022 return;
1556 } 1023 }
1557 execute_get (plugin, stmt, iter, iter_cls); 1024 execute_get (plugin, stmt, proc, proc_cls);
1558} 1025}
1559 1026
1560 1027
@@ -1579,7 +1046,7 @@ sqlite_plugin_drop (void *cls)
1579 * @return the size of the database on disk (estimate) 1046 * @return the size of the database on disk (estimate)
1580 */ 1047 */
1581static unsigned long long 1048static unsigned long long
1582sqlite_plugin_get_size (void *cls) 1049sqlite_plugin_estimate_size (void *cls)
1583{ 1050{
1584 struct Plugin *plugin = cls; 1051 struct Plugin *plugin = cls;
1585 sqlite3_stmt *stmt; 1052 sqlite3_stmt *stmt;
@@ -1653,14 +1120,13 @@ libgnunet_plugin_datastore_sqlite_init (void *cls)
1653 } 1120 }
1654 api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions)); 1121 api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions));
1655 api->cls = &plugin; 1122 api->cls = &plugin;
1656 api->get_size = &sqlite_plugin_get_size; 1123 api->estimate_size = &sqlite_plugin_estimate_size;
1657 api->put = &sqlite_plugin_put; 1124 api->put = &sqlite_plugin_put;
1658 api->next_request = &sqlite_next_request;
1659 api->get = &sqlite_plugin_get;
1660 api->replication_get = &sqlite_plugin_replication_get;
1661 api->expiration_get = &sqlite_plugin_expiration_get;
1662 api->update = &sqlite_plugin_update; 1125 api->update = &sqlite_plugin_update;
1663 api->iter_zero_anonymity = &sqlite_plugin_iter_zero_anonymity; 1126 api->get_key = &sqlite_plugin_get_key;
1127 api->get_replication = &sqlite_plugin_get_replication;
1128 api->get_expiration = &sqlite_plugin_get_expiration;
1129 api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity;
1664 api->drop = &sqlite_plugin_drop; 1130 api->drop = &sqlite_plugin_drop;
1665 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, 1131 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
1666 "sqlite", _("Sqlite database running\n")); 1132 "sqlite", _("Sqlite database running\n"));
@@ -1684,27 +1150,9 @@ libgnunet_plugin_datastore_sqlite_done (void *cls)
1684#if DEBUG_SQLITE 1150#if DEBUG_SQLITE
1685 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, 1151 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1686 "sqlite", 1152 "sqlite",
1687 "sqlite plugin is doneing\n"); 1153 "sqlite plugin is done\n");
1688#endif 1154#endif
1689 1155
1690 if (plugin->next_task != GNUNET_SCHEDULER_NO_TASK)
1691 {
1692#if DEBUG_SQLITE
1693 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1694 "sqlite",
1695 "Canceling next task\n");
1696#endif
1697 GNUNET_SCHEDULER_cancel (plugin->next_task);
1698 plugin->next_task = GNUNET_SCHEDULER_NO_TASK;
1699#if DEBUG_SQLITE
1700 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1701 "sqlite",
1702 "Prep'ing next task\n");
1703#endif
1704 plugin->next_task_nc->prep (plugin->next_task_nc->prep_cls, NULL);
1705 GNUNET_free (plugin->next_task_nc);
1706 plugin->next_task_nc = NULL;
1707 }
1708 fn = NULL; 1156 fn = NULL;
1709 if (plugin->drop_on_shutdown) 1157 if (plugin->drop_on_shutdown)
1710 fn = GNUNET_strdup (plugin->fn); 1158 fn = GNUNET_strdup (plugin->fn);
diff --git a/src/datastore/plugin_datastore_template.c b/src/datastore/plugin_datastore_template.c
index 40b191538..6228e8c0c 100644
--- a/src/datastore/plugin_datastore_template.c
+++ b/src/datastore/plugin_datastore_template.c
@@ -1,6 +1,6 @@
1/* 1/*
2 This file is part of GNUnet 2 This file is part of GNUnet
3 (C) 2009 Christian Grothoff (and other contributing authors) 3 (C) 2009, 2011 Christian Grothoff (and other contributing authors)
4 4
5 GNUnet is free software; you can redistribute it and/or modify 5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published 6 it under the terms of the GNU General Public License as published
@@ -47,7 +47,8 @@ struct Plugin
47 * @param cls our "struct Plugin*" 47 * @param cls our "struct Plugin*"
48 * @return number of bytes used on disk 48 * @return number of bytes used on disk
49 */ 49 */
50static unsigned long long template_plugin_get_size (void *cls) 50static unsigned long long
51template_plugin_estimate_size (void *cls)
51{ 52{
52 GNUNET_break (0); 53 GNUNET_break (0);
53 return 0; 54 return 0;
@@ -88,30 +89,11 @@ template_plugin_put (void *cls,
88 89
89 90
90/** 91/**
91 * Function invoked on behalf of a "PluginIterator" 92 * Get one of the results for a particular key in the datastore.
92 * asking the database plugin to call the iterator
93 * with the next item.
94 *
95 * @param next_cls whatever argument was given
96 * to the PluginIterator as "next_cls".
97 * @param end_it set to GNUNET_YES if we
98 * should terminate the iteration early
99 * (iterator should be still called once more
100 * to signal the end of the iteration).
101 */
102static void
103template_plugin_next_request (void *next_cls,
104 int end_it)
105{
106 GNUNET_break (0);
107}
108
109
110/**
111 * Iterate over the results for a particular key
112 * in the datastore.
113 * 93 *
114 * @param cls closure 94 * @param cls closure
95 * @param offset offset of the result (mod #num-results);
96 * specific ordering does not matter for the offset
115 * @param key maybe NULL (to match all entries) 97 * @param key maybe NULL (to match all entries)
116 * @param vhash hash of the value, maybe NULL (to 98 * @param vhash hash of the value, maybe NULL (to
117 * match all values that have the right key). 99 * match all values that have the right key).
@@ -120,16 +102,17 @@ template_plugin_next_request (void *next_cls,
120 * there may be! 102 * there may be!
121 * @param type entries of which type are relevant? 103 * @param type entries of which type are relevant?
122 * Use 0 for any type. 104 * Use 0 for any type.
123 * @param iter function to call on each matching value; 105 * @param proc function to call on each matching value;
124 * will be called once with a NULL value at the end 106 * will be called with NULL if nothing matches
125 * @param iter_cls closure for iter 107 * @param proc_cls closure for proc
126 */ 108 */
127static void 109static void
128template_plugin_get (void *cls, 110template_plugin_get_key (void *cls,
129 const GNUNET_HashCode * key, 111 uint64_t offset,
130 const GNUNET_HashCode * vhash, 112 const GNUNET_HashCode * key,
131 enum GNUNET_BLOCK_Type type, 113 const GNUNET_HashCode * vhash,
132 PluginIterator iter, void *iter_cls) 114 enum GNUNET_BLOCK_Type type,
115 PluginDatumProcessor proc, void *proc_cls)
133{ 116{
134 GNUNET_break (0); 117 GNUNET_break (0);
135} 118}
@@ -137,34 +120,35 @@ template_plugin_get (void *cls,
137 120
138 121
139/** 122/**
140 * Get a random item for replication. Returns a single, not expired, random item 123 * Get a random item for replication. Returns a single, not expired,
141 * from those with the highest replication counters. The item's 124 * random item from those with the highest replication counters. The
142 * replication counter is decremented by one IF it was positive before. 125 * item's replication counter is decremented by one IF it was positive
143 * Call 'iter' with all values ZERO or NULL if the datastore is empty. 126 * before. Call 'proc' with all values ZERO or NULL if the datastore
127 * is empty.
144 * 128 *
145 * @param cls closure 129 * @param cls closure
146 * @param iter function to call the value (once only). 130 * @param proc function to call the value (once only).
147 * @param iter_cls closure for iter 131 * @param proc_cls closure for proc
148 */ 132 */
149static void 133static void
150template_plugin_replication_get (void *cls, 134template_plugin_get_replication (void *cls,
151 PluginIterator iter, void *iter_cls) 135 PluginDatumProcessor proc, void *proc_cls)
152{ 136{
153 GNUNET_break (0); 137 GNUNET_break (0);
154} 138}
155 139
156 140
157/** 141/**
158 * Get a random item for expiration. 142 * Get a random item for expiration. Call 'proc' with all values ZERO
159 * Call 'iter' with all values ZERO or NULL if the datastore is empty. 143 * or NULL if the datastore is empty.
160 * 144 *
161 * @param cls closure 145 * @param cls closure
162 * @param iter function to call the value (once only). 146 * @param proc function to call the value (once only).
163 * @param iter_cls closure for iter 147 * @param proc_cls closure for proc
164 */ 148 */
165static void 149static void
166template_plugin_expiration_get (void *cls, 150template_plugin_get_expiration (void *cls,
167 PluginIterator iter, void *iter_cls) 151 PluginDatumProcessor proc, void *proc_cls)
168{ 152{
169 GNUNET_break (0); 153 GNUNET_break (0);
170} 154}
@@ -196,7 +180,8 @@ template_plugin_expiration_get (void *cls,
196static int 180static int
197template_plugin_update (void *cls, 181template_plugin_update (void *cls,
198 uint64_t uid, 182 uint64_t uid,
199 int delta, struct GNUNET_TIME_Absolute expire, 183 int delta,
184 struct GNUNET_TIME_Absolute expire,
200 char **msg) 185 char **msg)
201{ 186{
202 GNUNET_break (0); 187 GNUNET_break (0);
@@ -206,21 +191,23 @@ template_plugin_update (void *cls,
206 191
207 192
208/** 193/**
209 * Select a subset of the items in the datastore and call 194 * Call the given processor on an item with zero anonymity.
210 * the given iterator for each of them.
211 * 195 *
212 * @param cls our "struct Plugin*" 196 * @param cls our "struct Plugin*"
197 * @param offset offset of the result (mod #num-results);
198 * specific ordering does not matter for the offset
213 * @param type entries of which type should be considered? 199 * @param type entries of which type should be considered?
214 * Use 0 for any type. 200 * Use 0 for any type.
215 * @param iter function to call on each matching value; 201 * @param proc function to call on each matching value;
216 * will be called once with a NULL value at the end 202 * will be called with NULL if no value matches
217 * @param iter_cls closure for iter 203 * @param proc_cls closure for proc
218 */ 204 */
219static void 205static void
220template_plugin_iter_zero_anonymity (void *cls, 206template_plugin_get_zero_anonymity (void *cls,
221 enum GNUNET_BLOCK_Type type, 207 uint64_t offset,
222 PluginIterator iter, 208 enum GNUNET_BLOCK_Type type,
223 void *iter_cls) 209 PluginDatumProcessor proc,
210 void *proc_cls)
224{ 211{
225 GNUNET_break (0); 212 GNUNET_break (0);
226} 213}
@@ -253,14 +240,13 @@ libgnunet_plugin_datastore_template_init (void *cls)
253 plugin->env = env; 240 plugin->env = env;
254 api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions)); 241 api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions));
255 api->cls = plugin; 242 api->cls = plugin;
256 api->get_size = &template_plugin_get_size; 243 api->estimate_size = &template_plugin_estimate_size;
257 api->put = &template_plugin_put; 244 api->put = &template_plugin_put;
258 api->next_request = &template_plugin_next_request;
259 api->get = &template_plugin_get;
260 api->replication_get = &template_plugin_replication_get;
261 api->expiration_get = &template_plugin_expiration_get;
262 api->update = &template_plugin_update; 245 api->update = &template_plugin_update;
263 api->iter_zero_anonymity = &template_plugin_iter_zero_anonymity; 246 api->get_key = &template_plugin_get_key;
247 api->get_replication = &template_plugin_get_replication;
248 api->get_expiration = &template_plugin_get_expiration;
249 api->get_zero_anonymity = &template_plugin_get_zero_anonymity;
264 api->drop = &template_plugin_drop; 250 api->drop = &template_plugin_drop;
265 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, 251 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
266 "template", _("Template database running\n")); 252 "template", _("Template database running\n"));
diff --git a/src/datastore/test_datastore_api.c b/src/datastore/test_datastore_api.c
index 4a4bbc439..deeee7164 100644
--- a/src/datastore/test_datastore_api.c
+++ b/src/datastore/test_datastore_api.c
@@ -102,20 +102,18 @@ get_expiration (int i)
102enum RunPhase 102enum RunPhase
103 { 103 {
104 RP_DONE = 0, 104 RP_DONE = 0,
105 RP_PUT, 105 RP_PUT = 1,
106 RP_GET, 106 RP_GET = 2,
107 RP_DEL, 107 RP_DEL = 3,
108 RP_DO_DEL, 108 RP_DO_DEL = 4,
109 RP_DELVALIDATE, 109 RP_DELVALIDATE = 5,
110 RP_RESERVE, 110 RP_RESERVE = 6,
111 RP_PUT_MULTIPLE, 111 RP_PUT_MULTIPLE = 7,
112 RP_PUT_MULTIPLE_NEXT, 112 RP_PUT_MULTIPLE_NEXT = 8,
113 RP_GET_MULTIPLE, 113 RP_GET_MULTIPLE = 9,
114 RP_GET_MULTIPLE_NEXT, /* 10 */ 114 RP_GET_MULTIPLE_NEXT = 10,
115 RP_GET_MULTIPLE_DONE, 115 RP_UPDATE = 11,
116 RP_UPDATE, 116 RP_UPDATE_VALIDATE = 12,
117 RP_UPDATE_VALIDATE, /* 13 */
118 RP_UPDATE_DONE,
119 RP_ERROR 117 RP_ERROR
120 }; 118 };
121 119
@@ -129,7 +127,9 @@ struct CpsRunContext
129 void *data; 127 void *data;
130 size_t size; 128 size_t size;
131 enum RunPhase phase; 129 enum RunPhase phase;
132 unsigned long long uid; 130 uint64_t uid;
131 uint64_t offset;
132 uint64_t first_uid;
133}; 133};
134 134
135 135
@@ -144,16 +144,15 @@ check_success (void *cls,
144 const char *msg) 144 const char *msg)
145{ 145{
146 struct CpsRunContext *crc = cls; 146 struct CpsRunContext *crc = cls;
147
147 if (GNUNET_OK != success) 148 if (GNUNET_OK != success)
148 { 149 {
149 ok = 42;
150 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 150 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
151 "Operation not successfull: `%s'\n", msg); 151 "Operation %d/%d not successfull: `%s'\n",
152 crc->phase,
153 crc->i,
154 msg);
152 crc->phase = RP_ERROR; 155 crc->phase = RP_ERROR;
153 GNUNET_SCHEDULER_add_continuation (&run_continuation,
154 crc,
155 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
156 return;
157 } 156 }
158 GNUNET_free_non_null (crc->data); 157 GNUNET_free_non_null (crc->data);
159 crc->data = NULL; 158 crc->data = NULL;
@@ -171,7 +170,8 @@ get_reserved (void *cls,
171 struct CpsRunContext *crc = cls; 170 struct CpsRunContext *crc = cls;
172 if (0 >= success) 171 if (0 >= success)
173 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 172 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
174 "%s\n", msg); 173 "Error obtaining reservation: `%s'\n",
174 msg);
175 GNUNET_assert (0 < success); 175 GNUNET_assert (0 < success);
176 crc->rid = success; 176 crc->rid = success;
177 GNUNET_SCHEDULER_add_continuation (&run_continuation, 177 GNUNET_SCHEDULER_add_continuation (&run_continuation,
@@ -188,42 +188,48 @@ check_value (void *cls,
188 enum GNUNET_BLOCK_Type type, 188 enum GNUNET_BLOCK_Type type,
189 uint32_t priority, 189 uint32_t priority,
190 uint32_t anonymity, 190 uint32_t anonymity,
191 struct GNUNET_TIME_Absolute 191 struct GNUNET_TIME_Absolute expiration,
192 expiration, uint64_t uid) 192 uint64_t uid)
193{ 193{
194 static int matched;
195 struct CpsRunContext *crc = cls; 194 struct CpsRunContext *crc = cls;
196 int i; 195 int i;
197 196
198 if (key == NULL)
199 {
200 if (crc->i == 0)
201 {
202 crc->phase = RP_DEL;
203 crc->i = ITERATIONS;
204 }
205 GNUNET_assert (matched == GNUNET_YES);
206 matched = GNUNET_NO;
207 GNUNET_SCHEDULER_add_continuation (&run_continuation,
208 crc,
209 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
210 return;
211 }
212 i = crc->i; 197 i = crc->i;
198#if 0
199 fprintf (stderr,
200 "Check value got `%s' of size %u, type %d, expire %llu\n",
201 GNUNET_h2s (key),
202 (unsigned int) size,
203 type,
204 (unsigned long long) expiration.abs_value);
205 fprintf (stderr,
206 "Check value iteration %d wants size %u, type %d, expire %llu\n",
207 i,
208 (unsigned int) get_size (i),
209 get_type (i),
210 (unsigned long long) get_expiration(i).abs_value);
211#endif
213 GNUNET_assert (size == get_size (i)); 212 GNUNET_assert (size == get_size (i));
214 GNUNET_assert (0 == memcmp (data, get_data(i), size)); 213 GNUNET_assert (0 == memcmp (data, get_data(i), size));
215 GNUNET_assert (type == get_type (i)); 214 GNUNET_assert (type == get_type (i));
216 GNUNET_assert (priority == get_priority (i)); 215 GNUNET_assert (priority == get_priority (i));
217 GNUNET_assert (anonymity == get_anonymity(i)); 216 GNUNET_assert (anonymity == get_anonymity(i));
218 GNUNET_assert (expiration.abs_value == get_expiration(i).abs_value); 217 GNUNET_assert (expiration.abs_value == get_expiration(i).abs_value);
219 matched = GNUNET_YES; 218 crc->offset++;
220 GNUNET_DATASTORE_iterate_get_next (datastore); 219 if (crc->i == 0)
220 {
221 crc->phase = RP_DEL;
222 crc->i = ITERATIONS;
223 }
224 GNUNET_SCHEDULER_add_continuation (&run_continuation,
225 crc,
226 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
221} 227}
222 228
223 229
224static void 230static void
225delete_value (void *cls, 231delete_value (void *cls,
226 const GNUNET_HashCode * key, 232 const GNUNET_HashCode *key,
227 size_t size, 233 size_t size,
228 const void *data, 234 const void *data,
229 enum GNUNET_BLOCK_Type type, 235 enum GNUNET_BLOCK_Type type,
@@ -233,36 +239,23 @@ delete_value (void *cls,
233 expiration, uint64_t uid) 239 expiration, uint64_t uid)
234{ 240{
235 struct CpsRunContext *crc = cls; 241 struct CpsRunContext *crc = cls;
236 if (key == NULL) 242
237 {
238 if (crc->data == NULL)
239 {
240 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
241 "Content %u not found!\n",
242 crc->i);
243 crc->phase = RP_ERROR;
244 }
245 else
246 {
247 crc->phase = RP_DO_DEL;
248 }
249 GNUNET_SCHEDULER_add_continuation (&run_continuation,
250 crc,
251 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
252 return;
253 }
254 GNUNET_assert (crc->data == NULL); 243 GNUNET_assert (crc->data == NULL);
244 GNUNET_assert (NULL != key);
255 crc->size = size; 245 crc->size = size;
256 crc->key = *key; 246 crc->key = *key;
257 crc->data = GNUNET_malloc (size); 247 crc->data = GNUNET_malloc (size);
258 memcpy (crc->data, data, size); 248 memcpy (crc->data, data, size);
259 GNUNET_DATASTORE_iterate_get_next (datastore); 249 crc->phase = RP_DO_DEL;
250 GNUNET_SCHEDULER_add_continuation (&run_continuation,
251 crc,
252 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
260} 253}
261 254
262 255
263static void 256static void
264check_nothing (void *cls, 257check_nothing (void *cls,
265 const GNUNET_HashCode * key, 258 const GNUNET_HashCode *key,
266 size_t size, 259 size_t size,
267 const void *data, 260 const void *data,
268 enum GNUNET_BLOCK_Type type, 261 enum GNUNET_BLOCK_Type type,
@@ -272,11 +265,10 @@ check_nothing (void *cls,
272 expiration, uint64_t uid) 265 expiration, uint64_t uid)
273{ 266{
274 struct CpsRunContext *crc = cls; 267 struct CpsRunContext *crc = cls;
268
275 GNUNET_assert (key == NULL); 269 GNUNET_assert (key == NULL);
276 if (crc->i == 0) 270 if (crc->i == 0)
277 { 271 crc->phase = RP_RESERVE;
278 crc->phase = RP_RESERVE;
279 }
280 GNUNET_SCHEDULER_add_continuation (&run_continuation, 272 GNUNET_SCHEDULER_add_continuation (&run_continuation,
281 crc, 273 crc,
282 GNUNET_SCHEDULER_REASON_PREREQ_DONE); 274 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
@@ -296,47 +288,28 @@ check_multiple (void *cls,
296{ 288{
297 struct CpsRunContext *crc = cls; 289 struct CpsRunContext *crc = cls;
298 290
299 if (key == NULL) 291 GNUNET_assert (key != NULL);
300 {
301 if (crc->phase != RP_GET_MULTIPLE_DONE)
302 {
303 fprintf (stderr,
304 "Wrong phase: %d\n",
305 crc->phase);
306 GNUNET_break (0);
307 crc->phase = RP_ERROR;
308 }
309 else
310 {
311 crc->phase = RP_UPDATE;
312 }
313 GNUNET_SCHEDULER_add_continuation (&run_continuation,
314 crc,
315 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
316 return;
317 }
318 switch (crc->phase) 292 switch (crc->phase)
319 { 293 {
320 case RP_GET_MULTIPLE: 294 case RP_GET_MULTIPLE:
321 crc->phase = RP_GET_MULTIPLE_NEXT; 295 crc->phase = RP_GET_MULTIPLE_NEXT;
296 crc->first_uid = uid;
297 crc->offset++;
322 break; 298 break;
323 case RP_GET_MULTIPLE_NEXT: 299 case RP_GET_MULTIPLE_NEXT:
324 crc->phase = RP_GET_MULTIPLE_DONE; 300 GNUNET_assert (uid != crc->first_uid);
325 break; 301 crc->phase = RP_UPDATE;
326 case RP_GET_MULTIPLE_DONE:
327 /* do not advance further */
328 break; 302 break;
329 default: 303 default:
330 GNUNET_break (0); 304 GNUNET_break (0);
305 crc->phase = RP_ERROR;
331 break; 306 break;
332 } 307 }
333#if VERBOSE
334 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
335 "Test in phase %u\n", crc->phase);
336#endif
337 if (priority == get_priority (42)) 308 if (priority == get_priority (42))
338 crc->uid = uid; 309 crc->uid = uid;
339 GNUNET_DATASTORE_iterate_get_next (datastore); 310 GNUNET_SCHEDULER_add_continuation (&run_continuation,
311 crc,
312 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
340} 313}
341 314
342 315
@@ -353,31 +326,19 @@ check_update (void *cls,
353{ 326{
354 struct CpsRunContext *crc = cls; 327 struct CpsRunContext *crc = cls;
355 328
356 if (key == NULL) 329 GNUNET_assert (key != NULL);
357 {
358 if (crc->phase != RP_UPDATE_DONE)
359 {
360 GNUNET_break (0);
361 crc->phase = RP_ERROR;
362 }
363 else
364 {
365 crc->phase = RP_DONE;
366 }
367 GNUNET_SCHEDULER_add_continuation (&run_continuation,
368 crc,
369 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
370 return;
371 }
372 if ( (anonymity == get_anonymity (42)) && 330 if ( (anonymity == get_anonymity (42)) &&
373 (size == get_size (42)) && 331 (size == get_size (42)) &&
374 (priority == get_priority (42) + 100) ) 332 (priority == get_priority (42) + 100) )
333 crc->phase = RP_DONE;
334 else
375 { 335 {
376 crc->phase = RP_UPDATE_DONE; 336 GNUNET_assert (size == get_size (43));
337 crc->offset++;
377 } 338 }
378 else 339 GNUNET_SCHEDULER_add_continuation (&run_continuation,
379 GNUNET_assert (size == get_size (43)); 340 crc,
380 GNUNET_DATASTORE_iterate_get_next (datastore); 341 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
381} 342}
382 343
383 344
@@ -427,12 +388,13 @@ run_continuation (void *cls,
427 crc->i); 388 crc->i);
428#endif 389#endif
429 GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key); 390 GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
430 GNUNET_DATASTORE_iterate_key (datastore, 391 GNUNET_DATASTORE_get_key (datastore,
431 &crc->key, 392 crc->offset,
432 get_type (crc->i), 393 &crc->key,
433 1, 1, TIMEOUT, 394 get_type (crc->i),
434 &check_value, 395 1, 1, TIMEOUT,
435 crc); 396 &check_value,
397 crc);
436 break; 398 break;
437 case RP_DEL: 399 case RP_DEL:
438 crc->i--; 400 crc->i--;
@@ -444,12 +406,14 @@ run_continuation (void *cls,
444#endif 406#endif
445 crc->data = NULL; 407 crc->data = NULL;
446 GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key); 408 GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
447 GNUNET_DATASTORE_iterate_key (datastore, 409 GNUNET_assert (NULL !=
448 &crc->key, 410 GNUNET_DATASTORE_get_key (datastore,
449 get_type (crc->i), 411 crc->offset,
450 1, 1, TIMEOUT, 412 &crc->key,
451 &delete_value, 413 get_type (crc->i),
452 crc); 414 1, 1, TIMEOUT,
415 &delete_value,
416 crc));
453 break; 417 break;
454 case RP_DO_DEL: 418 case RP_DO_DEL:
455#if VERBOSE 419#if VERBOSE
@@ -467,13 +431,14 @@ run_continuation (void *cls,
467 { 431 {
468 crc->phase = RP_DEL; 432 crc->phase = RP_DEL;
469 } 433 }
470 GNUNET_DATASTORE_remove (datastore, 434 GNUNET_assert (NULL !=
471 &crc->key, 435 GNUNET_DATASTORE_remove (datastore,
472 crc->size, 436 &crc->key,
473 crc->data, 437 crc->size,
474 1, 1, TIMEOUT, 438 crc->data,
475 &check_success, 439 1, 1, TIMEOUT,
476 crc); 440 &check_success,
441 crc));
477 break; 442 break;
478 case RP_DELVALIDATE: 443 case RP_DELVALIDATE:
479 crc->i--; 444 crc->i--;
@@ -484,12 +449,14 @@ run_continuation (void *cls,
484 crc->i); 449 crc->i);
485#endif 450#endif
486 GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key); 451 GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
487 GNUNET_DATASTORE_iterate_key (datastore, 452 GNUNET_assert (NULL !=
488 &crc->key, 453 GNUNET_DATASTORE_get_key (datastore,
489 get_type (crc->i), 454 crc->offset,
490 1, 1, TIMEOUT, 455 &crc->key,
491 &check_nothing, 456 get_type (crc->i),
492 crc); 457 1, 1, TIMEOUT,
458 &check_nothing,
459 crc));
493 break; 460 break;
494 case RP_RESERVE: 461 case RP_RESERVE:
495 crc->phase = RP_PUT_MULTIPLE; 462 crc->phase = RP_PUT_MULTIPLE;
@@ -533,16 +500,24 @@ run_continuation (void *cls,
533 crc); 500 crc);
534 break; 501 break;
535 case RP_GET_MULTIPLE: 502 case RP_GET_MULTIPLE:
536 GNUNET_DATASTORE_iterate_key (datastore, 503 GNUNET_assert (NULL !=
537 &crc->key, 504 GNUNET_DATASTORE_get_key (datastore,
538 get_type (42), 505 crc->offset,
539 1, 1, TIMEOUT, 506 &crc->key,
540 &check_multiple, 507 get_type (42),
541 crc); 508 1, 1, TIMEOUT,
509 &check_multiple,
510 crc));
542 break; 511 break;
543 case RP_GET_MULTIPLE_NEXT: 512 case RP_GET_MULTIPLE_NEXT:
544 case RP_GET_MULTIPLE_DONE: 513 GNUNET_assert (NULL !=
545 GNUNET_assert (0); 514 GNUNET_DATASTORE_get_key (datastore,
515 crc->offset,
516 &crc->key,
517 get_type (42),
518 1, 1, TIMEOUT,
519 &check_multiple,
520 crc));
546 break; 521 break;
547 case RP_UPDATE: 522 case RP_UPDATE:
548 GNUNET_assert (crc->uid > 0); 523 GNUNET_assert (crc->uid > 0);
@@ -556,15 +531,14 @@ run_continuation (void *cls,
556 crc); 531 crc);
557 break; 532 break;
558 case RP_UPDATE_VALIDATE: 533 case RP_UPDATE_VALIDATE:
559 GNUNET_DATASTORE_iterate_key (datastore, 534 GNUNET_assert (NULL !=
560 &crc->key, 535 GNUNET_DATASTORE_get_key (datastore,
561 get_type (42), 536 crc->offset,
562 1, 1, TIMEOUT, 537 &crc->key,
563 &check_update, 538 get_type (42),
564 crc); 539 1, 1, TIMEOUT,
565 break; 540 &check_update,
566 case RP_UPDATE_DONE: 541 crc));
567 GNUNET_assert (0);
568 break; 542 break;
569 case RP_DONE: 543 case RP_DONE:
570#if VERBOSE 544#if VERBOSE
@@ -681,6 +655,7 @@ check ()
681 argv, "test-datastore-api", "nohelp", 655 argv, "test-datastore-api", "nohelp",
682 options, &run, NULL); 656 options, &run, NULL);
683#if START_DATASTORE 657#if START_DATASTORE
658 sleep (1); /* give datastore chance to receive 'DROP' request */
684 if (0 != GNUNET_OS_process_kill (proc, SIGTERM)) 659 if (0 != GNUNET_OS_process_kill (proc, SIGTERM))
685 { 660 {
686 GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); 661 GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill");
diff --git a/src/datastore/test_datastore_api_data_sqlite.conf b/src/datastore/test_datastore_api_data_sqlite.conf
index d7c01fe22..931572025 100644
--- a/src/datastore/test_datastore_api_data_sqlite.conf
+++ b/src/datastore/test_datastore_api_data_sqlite.conf
@@ -29,7 +29,7 @@ DATABASE = sqlite
29# REJECT_FROM = 29# REJECT_FROM =
30# REJECT_FROM6 = 30# REJECT_FROM6 =
31# PREFIX = 31# PREFIX =
32# DEBUG = YES 32#DEBUG = YES
33#PREFIX = valgrind --tool=memcheck --leak-check=yes 33#PREFIX = valgrind --tool=memcheck --leak-check=yes
34#BINARY = /home/grothoff/bin/gnunet-service-datastore 34#BINARY = /home/grothoff/bin/gnunet-service-datastore
35 35
diff --git a/src/datastore/test_datastore_api_management.c b/src/datastore/test_datastore_api_management.c
index 5dfb5cea7..41aa7ae3e 100644
--- a/src/datastore/test_datastore_api_management.c
+++ b/src/datastore/test_datastore_api_management.c
@@ -1,6 +1,6 @@
1/* 1/*
2 This file is part of GNUnet. 2 This file is part of GNUnet.
3 (C) 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other contributing authors) 3 (C) 2004, 2005, 2006, 2007, 2009, 2011 Christian Grothoff (and other contributing authors)
4 4
5 GNUnet is free software; you can redistribute it and/or modify 5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published 6 it under the terms of the GNU General Public License as published
@@ -97,9 +97,9 @@ get_expiration (int i)
97 97
98enum RunPhase 98enum RunPhase
99 { 99 {
100 RP_DONE = 0,
101 RP_PUT, 100 RP_PUT,
102 RP_GET, 101 RP_GET,
102 RP_DONE,
103 RP_GET_FAIL 103 RP_GET_FAIL
104 }; 104 };
105 105
@@ -112,6 +112,7 @@ struct CpsRunContext
112 const struct GNUNET_CONFIGURATION_Handle *cfg; 112 const struct GNUNET_CONFIGURATION_Handle *cfg;
113 void *data; 113 void *data;
114 enum RunPhase phase; 114 enum RunPhase phase;
115 uint64_t offset;
115}; 116};
116 117
117 118
@@ -146,42 +147,26 @@ check_value (void *cls,
146 enum GNUNET_BLOCK_Type type, 147 enum GNUNET_BLOCK_Type type,
147 uint32_t priority, 148 uint32_t priority,
148 uint32_t anonymity, 149 uint32_t anonymity,
149 struct GNUNET_TIME_Absolute 150 struct GNUNET_TIME_Absolute expiration,
150 expiration, uint64_t uid) 151 uint64_t uid)
151{ 152{
152 struct CpsRunContext *crc = cls; 153 struct CpsRunContext *crc = cls;
153 int i; 154 int i;
154 155
155 if (key == NULL)
156 {
157 crc->i--;
158 if (crc->found == GNUNET_YES)
159 {
160 crc->phase = RP_GET;
161 crc->found = GNUNET_NO;
162 }
163 else
164 {
165 fprintf (stderr,
166 "First not found was %u\n", crc->i);
167 crc->phase = RP_GET_FAIL;
168 }
169 if (0 == crc->i)
170 crc->phase = RP_DONE;
171 GNUNET_SCHEDULER_add_continuation (&run_continuation,
172 crc,
173 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
174 return;
175 }
176 i = crc->i; 156 i = crc->i;
177 crc->found = GNUNET_YES;
178 GNUNET_assert (size == get_size (i)); 157 GNUNET_assert (size == get_size (i));
179 GNUNET_assert (0 == memcmp (data, get_data(i), size)); 158 GNUNET_assert (0 == memcmp (data, get_data(i), size));
180 GNUNET_assert (type == get_type (i)); 159 GNUNET_assert (type == get_type (i));
181 GNUNET_assert (priority == get_priority (i)); 160 GNUNET_assert (priority == get_priority (i));
182 GNUNET_assert (anonymity == get_anonymity(i)); 161 GNUNET_assert (anonymity == get_anonymity(i));
183 GNUNET_assert (expiration.abs_value == get_expiration(i).abs_value); 162 GNUNET_assert (expiration.abs_value == get_expiration(i).abs_value);
184 GNUNET_DATASTORE_iterate_get_next (datastore); 163 crc->offset++;
164 crc->i--;
165 if (crc->i == 0)
166 crc->phase = RP_DONE;
167 GNUNET_SCHEDULER_add_continuation (&run_continuation,
168 crc,
169 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
185} 170}
186 171
187 172
@@ -241,7 +226,7 @@ run_continuation (void *cls,
241 { 226 {
242 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 227 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
243 "Sleeping to give datastore time to clean up\n"); 228 "Sleeping to give datastore time to clean up\n");
244 sleep (5); 229 sleep (1);
245 crc->phase = RP_GET; 230 crc->phase = RP_GET;
246 crc->i--; 231 crc->i--;
247 } 232 }
@@ -254,12 +239,13 @@ run_continuation (void *cls,
254 crc->i); 239 crc->i);
255#endif 240#endif
256 GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key); 241 GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
257 GNUNET_DATASTORE_iterate_key (datastore, 242 GNUNET_DATASTORE_get_key (datastore,
258 &crc->key, 243 crc->offset++,
259 get_type (crc->i), 244 &crc->key,
260 1, 1, TIMEOUT, 245 get_type (crc->i),
261 &check_value, 246 1, 1, TIMEOUT,
262 crc); 247 &check_value,
248 crc);
263 break; 249 break;
264 case RP_GET_FAIL: 250 case RP_GET_FAIL:
265#if VERBOSE 251#if VERBOSE
@@ -269,12 +255,13 @@ run_continuation (void *cls,
269 crc->i); 255 crc->i);
270#endif 256#endif
271 GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key); 257 GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
272 GNUNET_DATASTORE_iterate_key (datastore, 258 GNUNET_DATASTORE_get_key (datastore,
273 &crc->key, 259 crc->offset++,
274 get_type (crc->i), 260 &crc->key,
275 1, 1, TIMEOUT, 261 get_type (crc->i),
276 &check_nothing, 262 1, 1, TIMEOUT,
277 crc); 263 &check_nothing,
264 crc);
278 break; 265 break;
279 case RP_DONE: 266 case RP_DONE:
280 GNUNET_assert (0 == crc->i); 267 GNUNET_assert (0 == crc->i);
@@ -372,6 +359,7 @@ check ()
372 GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, 359 GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
373 argv, "test-datastore-api", "nohelp", 360 argv, "test-datastore-api", "nohelp",
374 options, &run, NULL); 361 options, &run, NULL);
362 sleep (1); /* give datastore chance to process 'DROP' request */
375 if (0 != GNUNET_OS_process_kill (proc, SIGTERM)) 363 if (0 != GNUNET_OS_process_kill (proc, SIGTERM))
376 { 364 {
377 GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); 365 GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill");
diff --git a/src/datastore/test_plugin_datastore.c b/src/datastore/test_plugin_datastore.c
index d38e908ac..f0961f51e 100644
--- a/src/datastore/test_plugin_datastore.c
+++ b/src/datastore/test_plugin_datastore.c
@@ -65,6 +65,7 @@ struct CpsRunContext
65 enum RunPhase phase; 65 enum RunPhase phase;
66 unsigned int cnt; 66 unsigned int cnt;
67 unsigned int i; 67 unsigned int i;
68 uint64_t offset;
68}; 69};
69 70
70 71
@@ -120,6 +121,11 @@ put_value (struct GNUNET_DATASTORE_PluginFunctions * api,
120 value[0] = k; 121 value[0] = k;
121 msg = NULL; 122 msg = NULL;
122 prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100); 123 prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100);
124#if VERBOSE
125 fprintf (stderr,
126 "putting type %u, anon %u under key %s\n",
127 i+1, i, GNUNET_h2s (&key));
128#endif
123 if (GNUNET_OK != api->put (api->cls, 129 if (GNUNET_OK != api->put (api->cls,
124 &key, 130 &key,
125 size, 131 size,
@@ -149,9 +155,11 @@ test (void *cls,
149 const struct GNUNET_SCHEDULER_TaskContext *tc); 155 const struct GNUNET_SCHEDULER_TaskContext *tc);
150 156
151 157
158static uint64_t guid;
159
160
152static int 161static int
153iterate_one_shot (void *cls, 162iterate_one_shot (void *cls,
154 void *next_cls,
155 const GNUNET_HashCode * key, 163 const GNUNET_HashCode * key,
156 uint32_t size, 164 uint32_t size,
157 const void *data, 165 const void *data,
@@ -164,57 +172,21 @@ iterate_one_shot (void *cls,
164{ 172{
165 struct CpsRunContext *crc = cls; 173 struct CpsRunContext *crc = cls;
166 174
167 GNUNET_assert (NULL == next_cls);
168 GNUNET_assert (key != NULL); 175 GNUNET_assert (key != NULL);
176 guid = uid;
169 crc->phase++; 177 crc->phase++;
170#if VERBOSE 178#if VERBOSE
171 fprintf (stderr, 179 fprintf (stderr,
172 "Found result type=%u, priority=%u, size=%u, expire=%llu\n", 180 "Found result type=%u, priority=%u, size=%u, expire=%llu, key %s\n",
173 type, priority, size, 181 type, priority, size,
174 (unsigned long long) expiration.abs_value); 182 (unsigned long long) expiration.abs_value,
183 GNUNET_h2s (key));
175#endif 184#endif
176 GNUNET_SCHEDULER_add_now (&test, crc); 185 GNUNET_SCHEDULER_add_now (&test, crc);
177 return GNUNET_OK; 186 return GNUNET_OK;
178} 187}
179 188
180 189
181static uint64_t guid;
182
183static int
184iterate_with_next (void *cls,
185 void *next_cls,
186 const GNUNET_HashCode * key,
187 uint32_t size,
188 const void *data,
189 enum GNUNET_BLOCK_Type type,
190 uint32_t priority,
191 uint32_t anonymity,
192 struct GNUNET_TIME_Absolute
193 expiration,
194 uint64_t uid)
195{
196 struct CpsRunContext *crc = cls;
197
198 if (key == NULL)
199 {
200 crc->phase++;
201 GNUNET_SCHEDULER_add_now (&test, crc);
202 return GNUNET_OK;
203 }
204 guid = uid;
205#if VERBOSE
206 fprintf (stderr,
207 "Found result type=%u, priority=%u, size=%u, expire=%llu\n",
208 type, priority, size,
209 (unsigned long long) expiration.abs_value);
210#endif
211 crc->cnt++;
212 crc->api->next_request (next_cls,
213 GNUNET_NO);
214 return GNUNET_OK;
215}
216
217
218/** 190/**
219 * Function called when the service shuts 191 * Function called when the service shuts
220 * down. Unloads our datastore plugin. 192 * down. Unloads our datastore plugin.
@@ -274,12 +246,19 @@ test (void *cls,
274 246
275 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) 247 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
276 { 248 {
249 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
250 "Test aborted.\n");
277 crc->phase = RP_ERROR; 251 crc->phase = RP_ERROR;
278 ok = 1;
279 } 252 }
253#if VERBOSE
254 fprintf (stderr, "In phase %d, iteration %u\n",
255 crc->phase,
256 crc->cnt);
257#endif
280 switch (crc->phase) 258 switch (crc->phase)
281 { 259 {
282 case RP_ERROR: 260 case RP_ERROR:
261 ok = 1;
283 GNUNET_break (0); 262 GNUNET_break (0);
284 crc->api->drop (crc->api->cls); 263 crc->api->drop (crc->api->cls);
285 GNUNET_SCHEDULER_add_now (&cleaning_task, crc); 264 GNUNET_SCHEDULER_add_now (&cleaning_task, crc);
@@ -289,7 +268,7 @@ test (void *cls,
289 for (j=0;j<PUT_10;j++) 268 for (j=0;j<PUT_10;j++)
290 { 269 {
291 put_value (crc->api, j, crc->i); 270 put_value (crc->api, j, crc->i);
292 cs = crc->api->get_size (crc->api->cls); 271 cs = crc->api->estimate_size (crc->api->cls);
293 GNUNET_assert (os < cs); 272 GNUNET_assert (os < cs);
294 os = cs; 273 os = cs;
295 } 274 }
@@ -305,11 +284,12 @@ test (void *cls,
305 break; 284 break;
306 } 285 }
307 gen_key (5, &key); 286 gen_key (5, &key);
308 crc->api->get (crc->api->cls, 287 crc->api->get_key (crc->api->cls,
309 &key, NULL, 288 crc->offset++,
310 GNUNET_BLOCK_TYPE_ANY, 289 &key, NULL,
311 &iterate_with_next, 290 GNUNET_BLOCK_TYPE_ANY,
312 crc); 291 &iterate_one_shot,
292 crc);
313 break; 293 break;
314 case RP_UPDATE: 294 case RP_UPDATE:
315 GNUNET_assert (GNUNET_OK == 295 GNUNET_assert (GNUNET_OK ==
@@ -329,18 +309,19 @@ test (void *cls,
329 GNUNET_SCHEDULER_add_now (&test, crc); 309 GNUNET_SCHEDULER_add_now (&test, crc);
330 break; 310 break;
331 } 311 }
332 crc->api->iter_zero_anonymity (crc->api->cls, 312 crc->api->get_zero_anonymity (crc->api->cls,
333 1, 313 0,
334 &iterate_with_next, 314 1,
335 crc); 315 &iterate_one_shot,
316 crc);
336 break; 317 break;
337 case RP_REPL_GET: 318 case RP_REPL_GET:
338 crc->api->replication_get (crc->api->cls, 319 crc->api->get_replication (crc->api->cls,
339 &iterate_one_shot, 320 &iterate_one_shot,
340 crc); 321 crc);
341 break; 322 break;
342 case RP_EXPI_GET: 323 case RP_EXPI_GET:
343 crc->api->expiration_get (crc->api->cls, 324 crc->api->get_expiration (crc->api->cls,
344 &iterate_one_shot, 325 &iterate_one_shot,
345 crc); 326 crc);
346 break; 327 break;
diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am
index f980f4206..20aa652ae 100644
--- a/src/fs/Makefile.am
+++ b/src/fs/Makefile.am
@@ -1,4 +1,3 @@
1
2INCLUDES = -I$(top_srcdir)/src/include 1INCLUDES = -I$(top_srcdir)/src/include
3 2
4if MINGW 3if MINGW
@@ -173,8 +172,7 @@ check_SCRIPTS = \
173 test_gnunet_fs_idx.py 172 test_gnunet_fs_idx.py
174endif 173endif
175 174
176#if !DISABLE_TEST_RUN 175if !DISABLE_TEST_RUN
177if 0
178TESTS = \ 176TESTS = \
179 test_fs_directory \ 177 test_fs_directory \
180 test_fs_download \ 178 test_fs_download \
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c
index 8192b8c1f..8eb2b4331 100644
--- a/src/fs/fs_download.c
+++ b/src/fs/fs_download.c
@@ -756,10 +756,12 @@ try_top_down_reconstruction (struct GNUNET_FS_DownloadContext *dc,
756 child_block_size = GNUNET_FS_tree_compute_tree_size (drc->depth); 756 child_block_size = GNUNET_FS_tree_compute_tree_size (drc->depth);
757 GNUNET_assert (0 == (drc->offset - dr->offset) % child_block_size); 757 GNUNET_assert (0 == (drc->offset - dr->offset) % child_block_size);
758 chk_off = (drc->offset - dr->offset) / child_block_size; 758 chk_off = (drc->offset - dr->offset) / child_block_size;
759 GNUNET_assert (drc->state == BRS_INIT); 759 if (drc->state == BRS_INIT)
760 drc->state = BRS_CHK_SET; 760 {
761 drc->chk = chks[chk_off]; 761 drc->state = BRS_CHK_SET;
762 try_top_down_reconstruction (dc, drc); 762 drc->chk = chks[chk_off];
763 try_top_down_reconstruction (dc, drc);
764 }
763 if (drc->state != BRS_DOWNLOAD_UP) 765 if (drc->state != BRS_DOWNLOAD_UP)
764 up_done = GNUNET_NO; /* children not all done */ 766 up_done = GNUNET_NO; /* children not all done */
765 } 767 }
@@ -815,10 +817,11 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
815 dr->depth, 817 dr->depth,
816 GNUNET_h2s (&dr->chk.query)); 818 GNUNET_h2s (&dr->chk.query));
817#endif 819#endif
818 GNUNET_assert (GNUNET_NO == 820 if (GNUNET_NO !=
819 GNUNET_CONTAINER_multihashmap_contains_value (dc->active, 821 GNUNET_CONTAINER_multihashmap_contains_value (dc->active,
820 &dr->chk.query, 822 &dr->chk.query,
821 dr)); 823 dr))
824 return; /* already active */
822 GNUNET_CONTAINER_multihashmap_put (dc->active, 825 GNUNET_CONTAINER_multihashmap_put (dc->active,
823 &dr->chk.query, 826 &dr->chk.query,
824 dr, 827 dr,
diff --git a/src/fs/fs_test_lib_data.conf b/src/fs/fs_test_lib_data.conf
index 68c5166b3..204bb90cf 100644
--- a/src/fs/fs_test_lib_data.conf
+++ b/src/fs/fs_test_lib_data.conf
@@ -43,7 +43,7 @@ HOSTNAME = localhost
43#TOTAL_QUOTA_OUT = 9321 43#TOTAL_QUOTA_OUT = 9321
44TOTAL_QUOTA_IN = 3932160 44TOTAL_QUOTA_IN = 3932160
45TOTAL_QUOTA_OUT = 3932160 45TOTAL_QUOTA_OUT = 3932160
46DEBUG = YES 46#DEBUG = YES
47#PREFIX = valgrind --tool=memcheck --leak-check=yes 47#PREFIX = valgrind --tool=memcheck --leak-check=yes
48#BINARY = /home/grothoff/bin/gnunet-service-core 48#BINARY = /home/grothoff/bin/gnunet-service-core
49 49
@@ -53,8 +53,8 @@ HOSTNAME = localhost
53#OPTIONS = -L DEBUG 53#OPTIONS = -L DEBUG
54CONTENT_CACHING = NO 54CONTENT_CACHING = NO
55CONTENT_PUSHING = NO 55CONTENT_PUSHING = NO
56DEBUG = YES 56# DEBUG = YES
57#PREFIX = valgrind --tool=memcheck --leak-check=yes 57# PREFIX = valgrind --tool=memcheck --leak-check=yes --trace-children=yes
58#BINARY = /home/grothoff/gn9/bin/gnunet-service-fs 58#BINARY = /home/grothoff/gn9/bin/gnunet-service-fs
59#PREFIX = xterm -e gdb -x cmd --args 59#PREFIX = xterm -e gdb -x cmd --args
60 60
diff --git a/src/fs/gnunet-pseudonym.c b/src/fs/gnunet-pseudonym.c
index 769b4239d..68a760867 100644
--- a/src/fs/gnunet-pseudonym.c
+++ b/src/fs/gnunet-pseudonym.c
@@ -341,7 +341,7 @@ main (int argc, char *const *argv)
341 0, &GNUNET_GETOPT_set_one, &no_remote_printing}, 341 0, &GNUNET_GETOPT_set_one, &no_remote_printing},
342 {'r', "replication", "LEVEL", 342 {'r', "replication", "LEVEL",
343 gettext_noop ("set the desired replication LEVEL"), 343 gettext_noop ("set the desired replication LEVEL"),
344 0, &GNUNET_GETOPT_set_uint, &bo.replication_level}, 344 1, &GNUNET_GETOPT_set_uint, &bo.replication_level},
345 {'R', "root", "ID", 345 {'R', "root", "ID",
346 gettext_noop 346 gettext_noop
347 ("specify ID of the root of the namespace"), 347 ("specify ID of the root of the namespace"),
diff --git a/src/fs/gnunet-service-fs_cp.c b/src/fs/gnunet-service-fs_cp.c
index 2522cbe7b..acad54501 100644
--- a/src/fs/gnunet-service-fs_cp.c
+++ b/src/fs/gnunet-service-fs_cp.c
@@ -704,9 +704,9 @@ copy_reply (void *cls,
704 704
705 705
706/** 706/**
707 * Free the given client request. 707 * Free the given request.
708 * 708 *
709 * @param cls the client request to free 709 * @param cls the request to free
710 * @param tc task context 710 * @param tc task context
711 */ 711 */
712static void 712static void
@@ -1182,6 +1182,7 @@ GSF_handle_p2p_query_ (const struct GNUNET_PeerIdentity *other,
1182 NULL, 0, /* replies_seen */ 1182 NULL, 0, /* replies_seen */
1183 &handle_p2p_reply, 1183 &handle_p2p_reply,
1184 peerreq); 1184 peerreq);
1185 GNUNET_assert (NULL != pr);
1185 peerreq->pr = pr; 1186 peerreq->pr = pr;
1186 GNUNET_break (GNUNET_OK == 1187 GNUNET_break (GNUNET_OK ==
1187 GNUNET_CONTAINER_multihashmap_put (cp->request_map, 1188 GNUNET_CONTAINER_multihashmap_put (cp->request_map,
@@ -1427,7 +1428,7 @@ cancel_pending_request (void *cls,
1427 const GNUNET_HashCode *query, 1428 const GNUNET_HashCode *query,
1428 void *value) 1429 void *value)
1429{ 1430{
1430 struct PeerRequest *peerreq = cls; 1431 struct PeerRequest *peerreq = value;
1431 struct GSF_PendingRequest *pr = peerreq->pr; 1432 struct GSF_PendingRequest *pr = peerreq->pr;
1432 1433
1433 GSF_pending_request_cancel_ (pr); 1434 GSF_pending_request_cancel_ (pr);
diff --git a/src/fs/gnunet-service-fs_indexing.c b/src/fs/gnunet-service-fs_indexing.c
index cc99d3962..dc6b82952 100644
--- a/src/fs/gnunet-service-fs_indexing.c
+++ b/src/fs/gnunet-service-fs_indexing.c
@@ -566,7 +566,7 @@ GNUNET_FS_handle_on_demand_block (const GNUNET_HashCode * key,
566 uint32_t anonymity, 566 uint32_t anonymity,
567 struct GNUNET_TIME_Absolute 567 struct GNUNET_TIME_Absolute
568 expiration, uint64_t uid, 568 expiration, uint64_t uid,
569 GNUNET_DATASTORE_Iterator cont, 569 GNUNET_DATASTORE_DatumProcessor cont,
570 void *cont_cls) 570 void *cont_cls)
571{ 571{
572 const struct OnDemandBlock *odb; 572 const struct OnDemandBlock *odb;
diff --git a/src/fs/gnunet-service-fs_indexing.h b/src/fs/gnunet-service-fs_indexing.h
index 6a2c3d4a0..e1154830b 100644
--- a/src/fs/gnunet-service-fs_indexing.h
+++ b/src/fs/gnunet-service-fs_indexing.h
@@ -63,7 +63,7 @@ GNUNET_FS_handle_on_demand_block (const GNUNET_HashCode * key,
63 uint32_t anonymity, 63 uint32_t anonymity,
64 struct GNUNET_TIME_Absolute 64 struct GNUNET_TIME_Absolute
65 expiration, uint64_t uid, 65 expiration, uint64_t uid,
66 GNUNET_DATASTORE_Iterator cont, 66 GNUNET_DATASTORE_DatumProcessor cont,
67 void *cont_cls); 67 void *cont_cls);
68 68
69/** 69/**
diff --git a/src/fs/gnunet-service-fs_pe.c b/src/fs/gnunet-service-fs_pe.c
index 28036150f..4dc9de1b8 100644
--- a/src/fs/gnunet-service-fs_pe.c
+++ b/src/fs/gnunet-service-fs_pe.c
@@ -158,7 +158,7 @@ plan (struct PeerPlan *pp,
158 rp->transmission_counter); 158 rp->transmission_counter);
159#endif 159#endif
160 160
161 161 GNUNET_assert (rp->hn == NULL);
162 if (GNUNET_TIME_absolute_get_remaining (rp->earliest_transmission).rel_value == 0) 162 if (GNUNET_TIME_absolute_get_remaining (rp->earliest_transmission).rel_value == 0)
163 rp->hn = GNUNET_CONTAINER_heap_insert (pp->priority_heap, 163 rp->hn = GNUNET_CONTAINER_heap_insert (pp->priority_heap,
164 rp, 164 rp,
diff --git a/src/fs/gnunet-service-fs_pr.c b/src/fs/gnunet-service-fs_pr.c
index 7406bed0f..c1074e8bf 100644
--- a/src/fs/gnunet-service-fs_pr.c
+++ b/src/fs/gnunet-service-fs_pr.c
@@ -100,6 +100,20 @@ struct GSF_PendingRequest
100 GNUNET_PEER_Id sender_pid; 100 GNUNET_PEER_Id sender_pid;
101 101
102 /** 102 /**
103 * Current offset for querying our local datastore for results.
104 * Starts at a random value, incremented until we get the same
105 * UID again (detected using 'first_uid'), which is then used
106 * to termiante the iteration.
107 */
108 uint64_t local_result_offset;
109
110 /**
111 * Unique ID of the first result from the local datastore;
112 * used to detect wrap-around of the offset.
113 */
114 uint64_t first_uid;
115
116 /**
103 * Number of valid entries in the 'replies_seen' array. 117 * Number of valid entries in the 'replies_seen' array.
104 */ 118 */
105 unsigned int replies_seen_count; 119 unsigned int replies_seen_count;
@@ -113,7 +127,7 @@ struct GSF_PendingRequest
113 * Mingle value we currently use for the bf. 127 * Mingle value we currently use for the bf.
114 */ 128 */
115 uint32_t mingle; 129 uint32_t mingle;
116 130
117}; 131};
118 132
119 133
@@ -273,6 +287,8 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options,
273 type); 287 type);
274#endif 288#endif
275 pr = GNUNET_malloc (sizeof (struct GSF_PendingRequest)); 289 pr = GNUNET_malloc (sizeof (struct GSF_PendingRequest));
290 pr->local_result_offset = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
291 UINT64_MAX);
276 pr->public_data.query = *query; 292 pr->public_data.query = *query;
277 if (GNUNET_BLOCK_TYPE_FS_SBLOCK == type) 293 if (GNUNET_BLOCK_TYPE_FS_SBLOCK == type)
278 { 294 {
@@ -535,7 +551,20 @@ clean_request (void *cls,
535 void *value) 551 void *value)
536{ 552{
537 struct GSF_PendingRequest *pr = value; 553 struct GSF_PendingRequest *pr = value;
538 554 GSF_LocalLookupContinuation cont;
555
556#if DEBUG_FS
557 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
558 "Cleaning up pending request for `%s'.\n",
559 GNUNET_h2s (key));
560#endif
561 if (NULL != (cont = pr->llc_cont))
562 {
563 pr->llc_cont = NULL;
564 cont (pr->llc_cont_cls,
565 pr,
566 pr->local_result);
567 }
539 GSF_plan_notify_request_done_ (pr); 568 GSF_plan_notify_request_done_ (pr);
540 GNUNET_free_non_null (pr->replies_seen); 569 GNUNET_free_non_null (pr->replies_seen);
541 if (NULL != pr->bf) 570 if (NULL != pr->bf)
@@ -560,6 +589,7 @@ clean_request (void *cls,
560void 589void
561GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr) 590GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr)
562{ 591{
592 if (NULL == pr_map) return; /* already cleaned up! */
563 GNUNET_assert (GNUNET_OK == 593 GNUNET_assert (GNUNET_OK ==
564 GNUNET_CONTAINER_multihashmap_remove (pr_map, 594 GNUNET_CONTAINER_multihashmap_remove (pr_map,
565 &pr->public_data.query, 595 &pr->public_data.query,
@@ -1023,13 +1053,22 @@ process_local_reply (void *cls,
1023 GNUNET_HashCode query; 1053 GNUNET_HashCode query;
1024 unsigned int old_rf; 1054 unsigned int old_rf;
1025 1055
1056 pr->qe = NULL;
1057 if (0 == pr->replies_seen_count)
1058 {
1059 pr->first_uid = uid;
1060 }
1061 else
1062 {
1063 if (uid == pr->first_uid)
1064 key = NULL; /* all replies seen! */
1065 }
1026 if (NULL == key) 1066 if (NULL == key)
1027 { 1067 {
1028#if DEBUG_FS 1068#if DEBUG_FS
1029 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1069 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1030 "No further local responses available.\n"); 1070 "No further local responses available.\n");
1031#endif 1071#endif
1032 pr->qe = NULL;
1033 if (NULL != (cont = pr->llc_cont)) 1072 if (NULL != (cont = pr->llc_cont))
1034 { 1073 {
1035 pr->llc_cont = NULL; 1074 pr->llc_cont = NULL;
@@ -1041,9 +1080,10 @@ process_local_reply (void *cls,
1041 } 1080 }
1042#if DEBUG_FS 1081#if DEBUG_FS
1043 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1082 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1044 "New local response to `%s' of type %u.\n", 1083 "Received reply for `%s' of type %d with UID %llu from datastore.\n",
1045 GNUNET_h2s (key), 1084 GNUNET_h2s (key),
1046 type); 1085 type,
1086 (unsigned long long) uid);
1047#endif 1087#endif
1048 if (type == GNUNET_BLOCK_TYPE_FS_ONDEMAND) 1088 if (type == GNUNET_BLOCK_TYPE_FS_ONDEMAND)
1049 { 1089 {
@@ -1061,8 +1101,22 @@ process_local_reply (void *cls,
1061 &process_local_reply, 1101 &process_local_reply,
1062 pr)) 1102 pr))
1063 { 1103 {
1064 if (pr->qe != NULL) 1104 pr->qe = GNUNET_DATASTORE_get_key (GSF_dsh,
1065 GNUNET_DATASTORE_iterate_get_next (GSF_dsh); 1105 pr->local_result_offset - 1,
1106 &pr->public_data.query,
1107 pr->public_data.type == GNUNET_BLOCK_TYPE_FS_DBLOCK
1108 ? GNUNET_BLOCK_TYPE_ANY
1109 : pr->public_data.type,
1110 (0 != (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options))
1111 ? UINT_MAX
1112 : 1 /* queue priority */,
1113 (0 != (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options))
1114 ? UINT_MAX
1115 : 1 /* max queue size */,
1116 GNUNET_TIME_UNIT_FOREVER_REL,
1117 &process_local_reply,
1118 pr);
1119 GNUNET_assert (NULL != pr->qe);
1066 } 1120 }
1067 return; 1121 return;
1068 } 1122 }
@@ -1085,7 +1139,22 @@ process_local_reply (void *cls,
1085 -1, -1, 1139 -1, -1,
1086 GNUNET_TIME_UNIT_FOREVER_REL, 1140 GNUNET_TIME_UNIT_FOREVER_REL,
1087 NULL, NULL); 1141 NULL, NULL);
1088 GNUNET_DATASTORE_iterate_get_next (GSF_dsh); 1142 pr->qe = GNUNET_DATASTORE_get_key (GSF_dsh,
1143 pr->local_result_offset - 1,
1144 &pr->public_data.query,
1145 pr->public_data.type == GNUNET_BLOCK_TYPE_FS_DBLOCK
1146 ? GNUNET_BLOCK_TYPE_ANY
1147 : pr->public_data.type,
1148 (0 != (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options))
1149 ? UINT_MAX
1150 : 1 /* queue priority */,
1151 (0 != (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options))
1152 ? UINT_MAX
1153 : 1 /* max queue size */,
1154 GNUNET_TIME_UNIT_FOREVER_REL,
1155 &process_local_reply,
1156 pr);
1157 GNUNET_assert (NULL != pr->qe);
1089 return; 1158 return;
1090 } 1159 }
1091 prq.type = type; 1160 prq.type = type;
@@ -1097,12 +1166,16 @@ process_local_reply (void *cls,
1097 GSF_update_datastore_delay_ (pr->public_data.start_time); 1166 GSF_update_datastore_delay_ (pr->public_data.start_time);
1098 process_reply (&prq, key, pr); 1167 process_reply (&prq, key, pr);
1099 pr->local_result = prq.eval; 1168 pr->local_result = prq.eval;
1100 if (pr->qe == NULL) 1169 if (prq.eval == GNUNET_BLOCK_EVALUATION_OK_LAST)
1101 { 1170 {
1102#if DEBUG_FS 1171 if (NULL != (cont = pr->llc_cont))
1103 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1172 {
1104 "Request cancelled, not asking datastore for more\n"); 1173 pr->llc_cont = NULL;
1105#endif 1174 cont (pr->llc_cont_cls,
1175 pr,
1176 pr->local_result);
1177 }
1178 return;
1106 } 1179 }
1107 if ( (0 == (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options)) && 1180 if ( (0 == (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options)) &&
1108 ( (GNUNET_YES == GSF_test_get_load_too_high_ (0)) || 1181 ( (GNUNET_YES == GSF_test_get_load_too_high_ (0)) ||
@@ -1116,8 +1189,6 @@ process_local_reply (void *cls,
1116 gettext_noop ("# processing result set cut short due to load"), 1189 gettext_noop ("# processing result set cut short due to load"),
1117 1, 1190 1,
1118 GNUNET_NO); 1191 GNUNET_NO);
1119 GNUNET_DATASTORE_cancel (pr->qe);
1120 pr->qe = NULL;
1121 if (NULL != (cont = pr->llc_cont)) 1192 if (NULL != (cont = pr->llc_cont))
1122 { 1193 {
1123 pr->llc_cont = NULL; 1194 pr->llc_cont = NULL;
@@ -1127,7 +1198,22 @@ process_local_reply (void *cls,
1127 } 1198 }
1128 return; 1199 return;
1129 } 1200 }
1130 GNUNET_DATASTORE_iterate_get_next (GSF_dsh); 1201 pr->qe = GNUNET_DATASTORE_get_key (GSF_dsh,
1202 pr->local_result_offset++,
1203 &pr->public_data.query,
1204 pr->public_data.type == GNUNET_BLOCK_TYPE_FS_DBLOCK
1205 ? GNUNET_BLOCK_TYPE_ANY
1206 : pr->public_data.type,
1207 (0 != (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options))
1208 ? UINT_MAX
1209 : 1 /* queue priority */,
1210 (0 != (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options))
1211 ? UINT_MAX
1212 : 1 /* max queue size */,
1213 GNUNET_TIME_UNIT_FOREVER_REL,
1214 &process_local_reply,
1215 pr);
1216 GNUNET_assert (NULL != pr->qe);
1131} 1217}
1132 1218
1133 1219
@@ -1147,20 +1233,21 @@ GSF_local_lookup_ (struct GSF_PendingRequest *pr,
1147 GNUNET_assert (NULL == pr->llc_cont); 1233 GNUNET_assert (NULL == pr->llc_cont);
1148 pr->llc_cont = cont; 1234 pr->llc_cont = cont;
1149 pr->llc_cont_cls = cont_cls; 1235 pr->llc_cont_cls = cont_cls;
1150 pr->qe = GNUNET_DATASTORE_iterate_key (GSF_dsh, 1236 pr->qe = GNUNET_DATASTORE_get_key (GSF_dsh,
1151 &pr->public_data.query, 1237 pr->local_result_offset++,
1152 pr->public_data.type == GNUNET_BLOCK_TYPE_FS_DBLOCK 1238 &pr->public_data.query,
1153 ? GNUNET_BLOCK_TYPE_ANY 1239 pr->public_data.type == GNUNET_BLOCK_TYPE_FS_DBLOCK
1154 : pr->public_data.type, 1240 ? GNUNET_BLOCK_TYPE_ANY
1155 (0 != (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options)) 1241 : pr->public_data.type,
1156 ? UINT_MAX 1242 (0 != (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options))
1157 : 1 /* queue priority */, 1243 ? UINT_MAX
1158 (0 != (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options)) 1244 : 1 /* queue priority */,
1159 ? UINT_MAX 1245 (0 != (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options))
1160 : 1 /* max queue size */, 1246 ? UINT_MAX
1161 GNUNET_TIME_UNIT_FOREVER_REL, 1247 : 1 /* max queue size */,
1162 &process_local_reply, 1248 GNUNET_TIME_UNIT_FOREVER_REL,
1163 pr); 1249 &process_local_reply,
1250 pr);
1164} 1251}
1165 1252
1166 1253
diff --git a/src/fs/gnunet-service-fs_put.c b/src/fs/gnunet-service-fs_put.c
index 121a90bcd..b15207ce8 100644
--- a/src/fs/gnunet-service-fs_put.c
+++ b/src/fs/gnunet-service-fs_put.c
@@ -35,25 +35,50 @@
35 35
36 36
37/** 37/**
38 * Request to datastore for DHT PUTs (or NULL). 38 * Context for each zero-anonymity iterator.
39 */ 39 */
40static struct GNUNET_DATASTORE_QueueEntry *dht_qe; 40struct PutOperator
41{
41 42
42/** 43 /**
43 * Type we will request for the next DHT PUT round from the datastore. 44 * Request to datastore for DHT PUTs (or NULL).
44 */ 45 */
45static enum GNUNET_BLOCK_Type dht_put_type = GNUNET_BLOCK_TYPE_FS_KBLOCK; 46 struct GNUNET_DATASTORE_QueueEntry *dht_qe;
47
48 /**
49 * Type we request from the datastore.
50 */
51 enum GNUNET_BLOCK_Type dht_put_type;
52
53 /**
54 * ID of task that collects blocks for DHT PUTs.
55 */
56 GNUNET_SCHEDULER_TaskIdentifier dht_task;
57
58 /**
59 * How many entires with zero anonymity of our type do we currently
60 * estimate to have in the database?
61 */
62 uint64_t zero_anonymity_count_estimate;
63
64 /**
65 * Current offset when iterating the database.
66 */
67 uint64_t current_offset;
68};
46 69
47/**
48 * ID of task that collects blocks for DHT PUTs.
49 */
50static GNUNET_SCHEDULER_TaskIdentifier dht_task;
51 70
52/** 71/**
53 * How many entires with zero anonymity do we currently estimate 72 * ANY-terminated list of our operators (one per type
54 * to have in the database? 73 * of block that we're putting into the DHT).
55 */ 74 */
56static unsigned int zero_anonymity_count_estimate; 75static struct PutOperator operators[] =
76 {
77 { NULL, GNUNET_BLOCK_TYPE_FS_KBLOCK, 0, 0, 0 },
78 { NULL, GNUNET_BLOCK_TYPE_FS_SBLOCK, 0, 0, 0 },
79 { NULL, GNUNET_BLOCK_TYPE_FS_NBLOCK, 0, 0, 0 },
80 { NULL, GNUNET_BLOCK_TYPE_ANY, 0, 0, 0 }
81 };
57 82
58 83
59/** 84/**
@@ -67,26 +92,26 @@ gather_dht_put_blocks (void *cls,
67 const struct GNUNET_SCHEDULER_TaskContext *tc); 92 const struct GNUNET_SCHEDULER_TaskContext *tc);
68 93
69 94
70
71/** 95/**
72 * If the DHT PUT gathering task is not currently running, consider 96 * Task that is run periodically to obtain blocks for DHT PUTs.
73 * (re)scheduling it with the appropriate delay. 97 *
98 * @param cls type of blocks to gather
99 * @param tc scheduler context (unused)
74 */ 100 */
75static void 101static void
76consider_dht_put_gathering (void *cls) 102delay_dht_put_blocks (void *cls,
103 const struct GNUNET_SCHEDULER_TaskContext *tc)
77{ 104{
105 struct PutOperator *po = cls;
78 struct GNUNET_TIME_Relative delay; 106 struct GNUNET_TIME_Relative delay;
79 107
80 if (GSF_dsh == NULL) 108 po->dht_task = GNUNET_SCHEDULER_NO_TASK;
81 return; 109 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
82 if (dht_qe != NULL)
83 return; 110 return;
84 if (dht_task != GNUNET_SCHEDULER_NO_TASK) 111 if (po->zero_anonymity_count_estimate > 0)
85 return;
86 if (zero_anonymity_count_estimate > 0)
87 { 112 {
88 delay = GNUNET_TIME_relative_divide (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY, 113 delay = GNUNET_TIME_relative_divide (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
89 zero_anonymity_count_estimate); 114 po->zero_anonymity_count_estimate);
90 delay = GNUNET_TIME_relative_min (delay, 115 delay = GNUNET_TIME_relative_min (delay,
91 MAX_DHT_PUT_FREQ); 116 MAX_DHT_PUT_FREQ);
92 } 117 }
@@ -96,20 +121,9 @@ consider_dht_put_gathering (void *cls)
96 (hopefully) appear */ 121 (hopefully) appear */
97 delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5); 122 delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5);
98 } 123 }
99 dht_task = GNUNET_SCHEDULER_add_delayed (delay, 124 po->dht_task = GNUNET_SCHEDULER_add_delayed (delay,
100 &gather_dht_put_blocks, 125 &gather_dht_put_blocks,
101 cls); 126 po);
102}
103
104
105/**
106 * Function called upon completion of the DHT PUT operation.
107 */
108static void
109dht_put_continuation (void *cls,
110 const struct GNUNET_SCHEDULER_TaskContext *tc)
111{
112 GNUNET_DATASTORE_iterate_get_next (GSF_dsh);
113} 127}
114 128
115 129
@@ -138,31 +152,19 @@ process_dht_put_content (void *cls,
138 struct GNUNET_TIME_Absolute 152 struct GNUNET_TIME_Absolute
139 expiration, uint64_t uid) 153 expiration, uint64_t uid)
140{ 154{
141 static unsigned int counter; 155 struct PutOperator *po = cls;
142 static GNUNET_HashCode last_vhash;
143 static GNUNET_HashCode vhash;
144 156
157 po->dht_qe = NULL;
145 if (key == NULL) 158 if (key == NULL)
146 { 159 {
147 dht_qe = NULL; 160 po->zero_anonymity_count_estimate = po->current_offset - 1;
148 consider_dht_put_gathering (cls); 161 po->current_offset = 0;
162 po->dht_task = GNUNET_SCHEDULER_add_now (&delay_dht_put_blocks,
163 po);
149 return; 164 return;
150 } 165 }
151 /* slightly funky code to estimate the total number of values with zero 166 po->zero_anonymity_count_estimate = GNUNET_MAX (po->current_offset,
152 anonymity from the maximum observed length of a monotonically increasing 167 po->zero_anonymity_count_estimate);
153 sequence of hashes over the contents */
154 GNUNET_CRYPTO_hash (data, size, &vhash);
155 if (GNUNET_CRYPTO_hash_cmp (&vhash, &last_vhash) <= 0)
156 {
157 if (zero_anonymity_count_estimate > 0)
158 zero_anonymity_count_estimate /= 2;
159 counter = 0;
160 }
161 last_vhash = vhash;
162 if (counter < 31)
163 counter++;
164 if (zero_anonymity_count_estimate < (1 << counter))
165 zero_anonymity_count_estimate = (1 << counter);
166#if DEBUG_FS 168#if DEBUG_FS
167 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 169 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
168 "Retrieved block `%s' of type %u for DHT PUT\n", 170 "Retrieved block `%s' of type %u for DHT PUT\n",
@@ -178,8 +180,8 @@ process_dht_put_content (void *cls,
178 data, 180 data,
179 expiration, 181 expiration,
180 GNUNET_TIME_UNIT_FOREVER_REL, 182 GNUNET_TIME_UNIT_FOREVER_REL,
181 &dht_put_continuation, 183 &delay_dht_put_blocks,
182 cls); 184 po);
183} 185}
184 186
185 187
@@ -193,17 +195,20 @@ static void
193gather_dht_put_blocks (void *cls, 195gather_dht_put_blocks (void *cls,
194 const struct GNUNET_SCHEDULER_TaskContext *tc) 196 const struct GNUNET_SCHEDULER_TaskContext *tc)
195{ 197{
196 dht_task = GNUNET_SCHEDULER_NO_TASK; 198 struct PutOperator *po = cls;
197 if (GSF_dsh == NULL) 199
200 po->dht_task = GNUNET_SCHEDULER_NO_TASK;
201 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
198 return; 202 return;
199 if (dht_put_type == GNUNET_BLOCK_TYPE_FS_ONDEMAND) 203 po->dht_qe = GNUNET_DATASTORE_get_zero_anonymity (GSF_dsh,
200 dht_put_type = GNUNET_BLOCK_TYPE_FS_KBLOCK; 204 po->current_offset++,
201 dht_qe = GNUNET_DATASTORE_iterate_zero_anonymity (GSF_dsh,
202 0, UINT_MAX, 205 0, UINT_MAX,
203 GNUNET_TIME_UNIT_FOREVER_REL, 206 GNUNET_TIME_UNIT_FOREVER_REL,
204 dht_put_type++, 207 po->dht_put_type,
205 &process_dht_put_content, NULL); 208 &process_dht_put_content, po);
206 GNUNET_assert (dht_qe != NULL); 209 if (NULL == po->dht_qe)
210 po->dht_task = GNUNET_SCHEDULER_add_now (&delay_dht_put_blocks,
211 po);
207} 212}
208 213
209 214
@@ -213,7 +218,14 @@ gather_dht_put_blocks (void *cls,
213void 218void
214GSF_put_init_ () 219GSF_put_init_ ()
215{ 220{
216 dht_task = GNUNET_SCHEDULER_add_now (&gather_dht_put_blocks, NULL); 221 unsigned int i;
222
223 i = 0;
224 while (operators[i].dht_put_type != GNUNET_BLOCK_TYPE_ANY)
225 {
226 operators[i].dht_task = GNUNET_SCHEDULER_add_now (&gather_dht_put_blocks, &operators[i]);
227 i++;
228 }
217} 229}
218 230
219 231
@@ -223,15 +235,23 @@ GSF_put_init_ ()
223void 235void
224GSF_put_done_ () 236GSF_put_done_ ()
225{ 237{
226 if (GNUNET_SCHEDULER_NO_TASK != dht_task) 238 struct PutOperator *po;
227 { 239 unsigned int i;
228 GNUNET_SCHEDULER_cancel (dht_task); 240
229 dht_task = GNUNET_SCHEDULER_NO_TASK; 241 i = 0;
230 } 242 while ((po = &operators[i])->dht_put_type != GNUNET_BLOCK_TYPE_ANY)
231 if (NULL != dht_qe)
232 { 243 {
233 GNUNET_DATASTORE_cancel (dht_qe); 244 if (GNUNET_SCHEDULER_NO_TASK != po->dht_task)
234 dht_qe = NULL; 245 {
246 GNUNET_SCHEDULER_cancel (po->dht_task);
247 po->dht_task = GNUNET_SCHEDULER_NO_TASK;
248 }
249 if (NULL != po->dht_qe)
250 {
251 GNUNET_DATASTORE_cancel (po->dht_qe);
252 po->dht_qe = NULL;
253 }
254 i++;
235 } 255 }
236} 256}
237 257
diff --git a/src/fs/test_fs_download_data.conf b/src/fs/test_fs_download_data.conf
index 0a7eb311a..6bbae9dc9 100644
--- a/src/fs/test_fs_download_data.conf
+++ b/src/fs/test_fs_download_data.conf
@@ -36,7 +36,8 @@ HOSTNAME = localhost
36[fs] 36[fs]
37PORT = 42471 37PORT = 42471
38HOSTNAME = localhost 38HOSTNAME = localhost
39ACTIVEMIGRATION = NO 39CONTENT_CACHING = NO
40CONTENT_PUSHING = NO
40# DEBUG = YES 41# DEBUG = YES
41#PREFIX = valgrind --tool=memcheck --leak-check=yes 42#PREFIX = valgrind --tool=memcheck --leak-check=yes
42#BINARY = /home/grothoff/bin/gnunet-service-fs 43#BINARY = /home/grothoff/bin/gnunet-service-fs
diff --git a/src/fs/test_gnunet_fs_idx.py.in b/src/fs/test_gnunet_fs_idx.py.in
index 3bb3681c6..c97ffd883 100755
--- a/src/fs/test_gnunet_fs_idx.py.in
+++ b/src/fs/test_gnunet_fs_idx.py.in
@@ -31,7 +31,7 @@ try:
31 pub.expect ("URI is `gnunet://fs/chk/PC0M19QMQC0BPSHR6BGA228PP6INER1D610MGEMOMEM87222FN8HVUO7PQGO0O9HD2GVLHF2N5IDHEQUNK6LKE428FPO96SKQEA486O.PG7K85JGQ6N599MD5HEP3CHEVFPKQD9JB6NPSLVA3T1SKDS66CFI499VS6MGQ88B0QUAVT1282TCRD4GGFVUKDLGI8F0SPIANA3J2LG.35147'.\r") 31 pub.expect ("URI is `gnunet://fs/chk/PC0M19QMQC0BPSHR6BGA228PP6INER1D610MGEMOMEM87222FN8HVUO7PQGO0O9HD2GVLHF2N5IDHEQUNK6LKE428FPO96SKQEA486O.PG7K85JGQ6N599MD5HEP3CHEVFPKQD9JB6NPSLVA3T1SKDS66CFI499VS6MGQ88B0QUAVT1282TCRD4GGFVUKDLGI8F0SPIANA3J2LG.35147'.\r")
32 pub.expect (pexpect.EOF) 32 pub.expect (pexpect.EOF)
33 33
34 down = pexpect.spawn ('gnunet-download -c test_gnunet_fs_idx_data.conf -o \"COPYING\" gnunet://fs/chk/PC0M19QMQC0BPSHR6BGA228PP6INER1D610MGEMOMEM87222FN8HVUO7PQGO0O9HD2GVLHF2N5IDHEQUNK6LKE428FPO96SKQEA486O.PG7K85JGQ6N599MD5HEP3CHEVFPKQD9JB6NPSLVA3T1SKDS66CFI499VS6MGQ88B0QUAVT1282TCRD4GGFVUKDLGI8F0SPIANA3J2LG.35147') 34 down = pexpect.spawn ('gnunet-download -c test_gnunet_fs_idx_data.conf -o COPYING gnunet://fs/chk/PC0M19QMQC0BPSHR6BGA228PP6INER1D610MGEMOMEM87222FN8HVUO7PQGO0O9HD2GVLHF2N5IDHEQUNK6LKE428FPO96SKQEA486O.PG7K85JGQ6N599MD5HEP3CHEVFPKQD9JB6NPSLVA3T1SKDS66CFI499VS6MGQ88B0QUAVT1282TCRD4GGFVUKDLGI8F0SPIANA3J2LG.35147')
35 down.expect (re.compile ("Downloading `COPYING\' done \(.*\).\r")); 35 down.expect (re.compile ("Downloading `COPYING\' done \(.*\).\r"));
36 down.expect (pexpect.EOF); 36 down.expect (pexpect.EOF);
37 os.system ('rm COPYING'); 37 os.system ('rm COPYING');
diff --git a/src/fs/test_gnunet_fs_ns_data.conf b/src/fs/test_gnunet_fs_ns_data.conf
index 65bac0a15..2086cd0fd 100644
--- a/src/fs/test_gnunet_fs_ns_data.conf
+++ b/src/fs/test_gnunet_fs_ns_data.conf
@@ -36,7 +36,7 @@ HOSTNAME = localhost
36[fs] 36[fs]
37PORT = 47471 37PORT = 47471
38HOSTNAME = localhost 38HOSTNAME = localhost
39#DEBUG = YES 39DEBUG = YES
40#PREFIX = valgrind --tool=memcheck --leak-check=yes 40#PREFIX = valgrind --tool=memcheck --leak-check=yes
41#BINARY = /home/grothoff/bin/gnunet-service-fs 41#BINARY = /home/grothoff/bin/gnunet-service-fs
42 42
diff --git a/src/fs/test_gnunet_service_fs_migration_data.conf b/src/fs/test_gnunet_service_fs_migration_data.conf
index a72a98e97..3ab61d76c 100644
--- a/src/fs/test_gnunet_service_fs_migration_data.conf
+++ b/src/fs/test_gnunet_service_fs_migration_data.conf
@@ -53,7 +53,7 @@ HOSTNAME = localhost
53ACTIVEMIGRATION = YES 53ACTIVEMIGRATION = YES
54CONTENT_CACHING = YES 54CONTENT_CACHING = YES
55CONTENT_PUSHING = YES 55CONTENT_PUSHING = YES
56DEBUG = YES 56#DEBUG = YES
57#PREFIX = valgrind --tool=memcheck --leak-check=yes 57#PREFIX = valgrind --tool=memcheck --leak-check=yes
58#PREFIX = xterm -e gdb -x cmd --args 58#PREFIX = xterm -e gdb -x cmd --args
59 59
diff --git a/src/include/gnunet_datastore_plugin.h b/src/include/gnunet_datastore_plugin.h
index a5c548146..4d717996d 100644
--- a/src/include/gnunet_datastore_plugin.h
+++ b/src/include/gnunet_datastore_plugin.h
@@ -78,26 +78,9 @@ struct GNUNET_DATASTORE_PluginEnvironment
78 78
79 79
80/** 80/**
81 * Function invoked on behalf of a "PluginIterator" 81 * An processor over a set of items stored in the datastore.
82 * asking the database plugin to call the iterator
83 * with the next item.
84 *
85 * @param next_cls whatever argument was given
86 * to the PluginIterator as "next_cls".
87 * @param end_it set to GNUNET_YES if we
88 * should terminate the iteration early
89 * (iterator should be still called once more
90 * to signal the end of the iteration).
91 */
92typedef void (*PluginNextRequest)(void *next_cls,
93 int end_it);
94
95
96/**
97 * An iterator over a set of items stored in the datastore.
98 * 82 *
99 * @param cls closure 83 * @param cls closure
100 * @param next_cls closure to pass to the "next" function.
101 * @param key key for the content 84 * @param key key for the content
102 * @param size number of bytes in data 85 * @param size number of bytes in data
103 * @param data content stored 86 * @param data content stored
@@ -105,24 +88,21 @@ typedef void (*PluginNextRequest)(void *next_cls,
105 * @param priority priority of the content 88 * @param priority priority of the content
106 * @param anonymity anonymity-level for the content 89 * @param anonymity anonymity-level for the content
107 * @param expiration expiration time for the content 90 * @param expiration expiration time for the content
108 * @param uid unique identifier for the datum; 91 * @param uid unique identifier for the datum
109 * maybe 0 if no unique identifier is available
110 * 92 *
111 * @return GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue 93 * @return GNUNET_OK to keep the item
112 * (continue on call to "next", of course), 94 * GNUNET_NO to delete the item
113 * GNUNET_NO to delete the item and continue (if supported)
114 */ 95 */
115typedef int (*PluginIterator) (void *cls, 96typedef int (*PluginDatumProcessor) (void *cls,
116 void *next_cls, 97 const GNUNET_HashCode * key,
117 const GNUNET_HashCode * key, 98 uint32_t size,
118 uint32_t size, 99 const void *data,
119 const void *data, 100 enum GNUNET_BLOCK_Type type,
120 enum GNUNET_BLOCK_Type type, 101 uint32_t priority,
121 uint32_t priority, 102 uint32_t anonymity,
122 uint32_t anonymity, 103 struct GNUNET_TIME_Absolute
123 struct GNUNET_TIME_Absolute 104 expiration,
124 expiration, 105 uint64_t uid);
125 uint64_t uid);
126 106
127/** 107/**
128 * Get an estimate of how much space the database is 108 * Get an estimate of how much space the database is
@@ -131,7 +111,7 @@ typedef int (*PluginIterator) (void *cls,
131 * @param cls closure 111 * @param cls closure
132 * @return number of bytes used on disk 112 * @return number of bytes used on disk
133 */ 113 */
134typedef unsigned long long (*PluginGetSize) (void *cls); 114typedef unsigned long long (*PluginEstimateSize) (void *cls);
135 115
136 116
137/** 117/**
@@ -165,10 +145,11 @@ typedef int (*PluginPut) (void *cls,
165 145
166 146
167/** 147/**
168 * Iterate over the results for a particular key 148 * Get one of the results for a particular key in the datastore.
169 * in the datastore.
170 * 149 *
171 * @param cls closure 150 * @param cls closure
151 * @param offset offset of the result (mod #num-results);
152 * specific ordering does not matter for the offset
172 * @param key key to match, never NULL 153 * @param key key to match, never NULL
173 * @param vhash hash of the value, maybe NULL (to 154 * @param vhash hash of the value, maybe NULL (to
174 * match all values that have the right key). 155 * match all values that have the right key).
@@ -177,34 +158,31 @@ typedef int (*PluginPut) (void *cls,
177 * there may be! 158 * there may be!
178 * @param type entries of which type are relevant? 159 * @param type entries of which type are relevant?
179 * Use 0 for any type. 160 * Use 0 for any type.
180 * @param iter function to call on each matching value; however, 161 * @param proc function to call on the matching value;
181 * after the first call to "iter", the plugin must wait 162 * proc should be called with NULL if there is no result
182 * until "NextRequest" was called before giving the iterator 163 * @param proc_cls closure for proc
183 * the next item; finally, the "iter" should be called once
184 * once with a NULL value at the end ("next_cls" should be NULL
185 * for that last call)
186 * @param iter_cls closure for iter
187 */ 164 */
188typedef void (*PluginGet) (void *cls, 165typedef void (*PluginGetKey) (void *cls,
189 const GNUNET_HashCode *key, 166 uint64_t offset,
190 const GNUNET_HashCode *vhash, 167 const GNUNET_HashCode *key,
191 enum GNUNET_BLOCK_Type type, 168 const GNUNET_HashCode *vhash,
192 PluginIterator iter, void *iter_cls); 169 enum GNUNET_BLOCK_Type type,
170 PluginDatumProcessor proc, void *proc_cls);
193 171
194 172
195 173
196/** 174/**
197 * Get a random item (additional constraints may apply depending on 175 * Get a random item (additional constraints may apply depending on
198 * the specific implementation). Calls 'iter' with all values ZERO or 176 * the specific implementation). Calls 'proc' with all values ZERO or
199 * NULL if no item applies, otherwise 'iter' is called once and only 177 * NULL if no item applies, otherwise 'proc' is called once and only
200 * once with an item, with the 'next_cls' argument being NULL. 178 * once with an item, with the 'next_cls' argument being NULL.
201 * 179 *
202 * @param cls closure 180 * @param cls closure
203 * @param iter function to call the value (once only). 181 * @param proc function to call the value (once only).
204 * @param iter_cls closure for iter 182 * @param proc_cls closure for proc
205 */ 183 */
206typedef void (*PluginRandomGet) (void *cls, 184typedef void (*PluginGetRandom) (void *cls,
207 PluginIterator iter, void *iter_cls); 185 PluginDatumProcessor proc, void *proc_cls);
208 186
209 187
210/** 188/**
@@ -238,26 +216,22 @@ typedef int (*PluginUpdate) (void *cls,
238 216
239 217
240/** 218/**
241 * Select a subset of the items in the datastore and call the given 219 * Select a single item from the datastore at the specified offset
242 * iterator for the first item; then allow getting more items by 220 * (among those applicable).
243 * calling the 'next_request' callback with the given 'next_cls'
244 * argument passed to 'iter'.
245 * 221 *
246 * @param cls closure 222 * @param cls closure
223 * @param offset offset of the result (mod #num-results);
224 * specific ordering does not matter for the offset
247 * @param type entries of which type should be considered? 225 * @param type entries of which type should be considered?
248 * Myst not be zero (ANY). 226 * Must not be zero (ANY).
249 * @param iter function to call on each matching value; however, 227 * @param proc function to call on the matching value
250 * after the first call to "iter", the plugin must wait 228 * @param proc_cls closure for proc
251 * until "NextRequest" was called before giving the iterator
252 * the next item; finally, the "iter" should be called once
253 * once with a NULL value at the end ("next_cls" should be NULL
254 * for that last call)
255 * @param iter_cls closure for iter
256 */ 229 */
257typedef void (*PluginSelector) (void *cls, 230typedef void (*PluginGetType) (void *cls,
258 enum GNUNET_BLOCK_Type type, 231 uint64_t offset,
259 PluginIterator iter, 232 enum GNUNET_BLOCK_Type type,
260 void *iter_cls); 233 PluginDatumProcessor proc,
234 void *proc_cls);
261 235
262 236
263/** 237/**
@@ -283,10 +257,10 @@ struct GNUNET_DATASTORE_PluginFunctions
283 void *cls; 257 void *cls;
284 258
285 /** 259 /**
286 * Get the current on-disk size of the SQ store. Estimates are 260 * Calculate the current on-disk size of the SQ store. Estimates
287 * fine, if that's the only thing available. 261 * are fine, if that's the only thing available.
288 */ 262 */
289 PluginGetSize get_size; 263 PluginEstimateSize estimate_size;
290 264
291 /** 265 /**
292 * Function to store an item in the datastore. 266 * Function to store an item in the datastore.
@@ -304,23 +278,14 @@ struct GNUNET_DATASTORE_PluginFunctions
304 PluginUpdate update; 278 PluginUpdate update;
305 279
306 /** 280 /**
307 * Function called by iterators whenever they want the next value; 281 * Get a particular datum matching a given hash from the datastore.
308 * note that unlike all of the other callbacks, this one does get a
309 * the "next_cls" closure which is usually different from the "cls"
310 * member of this struct!
311 */
312 PluginNextRequest next_request;
313
314 /**
315 * Function to iterate over the results for a particular key
316 * in the datastore.
317 */ 282 */
318 PluginGet get; 283 PluginGetKey get_key;
319 284
320 /** 285 /**
321 * Iterate over content with anonymity level zero. 286 * Get datum (of the specified type) with anonymity level zero.
322 */ 287 */
323 PluginSelector iter_zero_anonymity; 288 PluginGetType get_zero_anonymity;
324 289
325 /** 290 /**
326 * Function to get a random item with high replication score from 291 * Function to get a random item with high replication score from
@@ -329,13 +294,13 @@ struct GNUNET_DATASTORE_PluginFunctions
329 * counters. The item's replication counter is decremented by one 294 * counters. The item's replication counter is decremented by one
330 * IF it was positive before. 295 * IF it was positive before.
331 */ 296 */
332 PluginRandomGet replication_get; 297 PluginGetRandom get_replication;
333 298
334 /** 299 /**
335 * Function to get a random expired item or, if none are expired, one 300 * Function to get a random expired item or, if none are expired, one
336 * with a low priority. 301 * with a low priority.
337 */ 302 */
338 PluginRandomGet expiration_get; 303 PluginGetRandom get_expiration;
339 304
340 /** 305 /**
341 * Delete the database. The next operation is 306 * Delete the database. The next operation is
diff --git a/src/include/gnunet_datastore_service.h b/src/include/gnunet_datastore_service.h
index 53d04e517..c563e5cc9 100644
--- a/src/include/gnunet_datastore_service.h
+++ b/src/include/gnunet_datastore_service.h
@@ -262,7 +262,7 @@ GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h,
262 262
263 263
264/** 264/**
265 * An iterator over a set of items stored in the datastore. 265 * Process a datum that was stored in the datastore.
266 * 266 *
267 * @param cls closure 267 * @param cls closure
268 * @param key key for the content 268 * @param key key for the content
@@ -275,87 +275,79 @@ GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h,
275 * @param uid unique identifier for the datum; 275 * @param uid unique identifier for the datum;
276 * maybe 0 if no unique identifier is available 276 * maybe 0 if no unique identifier is available
277 */ 277 */
278typedef void (*GNUNET_DATASTORE_Iterator) (void *cls, 278typedef void (*GNUNET_DATASTORE_DatumProcessor) (void *cls,
279 const GNUNET_HashCode * key, 279 const GNUNET_HashCode * key,
280 size_t size, 280 size_t size,
281 const void *data, 281 const void *data,
282 enum GNUNET_BLOCK_Type type, 282 enum GNUNET_BLOCK_Type type,
283 uint32_t priority, 283 uint32_t priority,
284 uint32_t anonymity, 284 uint32_t anonymity,
285 struct GNUNET_TIME_Absolute 285 struct GNUNET_TIME_Absolute
286 expiration, uint64_t uid); 286 expiration, uint64_t uid);
287 287
288 288
289/** 289/**
290 * Iterate over the results for a particular key 290 * Get a result for a particular key from the datastore. The processor
291 * in the datastore. The iterator will only be called 291 * will only be called once.
292 * once initially; if the first call did contain a
293 * result, further results can be obtained by calling
294 * "GNUNET_DATASTORE_iterate_get_next" with the given argument.
295 * 292 *
296 * @param h handle to the datastore 293 * @param h handle to the datastore
294 * @param offset offset of the result (mod #num-results); set to
295 * a random 64-bit value initially; then increment by
296 * one each time; detect that all results have been found by uid
297 * being again the first uid ever returned.
297 * @param key maybe NULL (to match all entries) 298 * @param key maybe NULL (to match all entries)
298 * @param type desired type, 0 for any 299 * @param type desired type, 0 for any
299 * @param queue_priority ranking of this request in the priority queue 300 * @param queue_priority ranking of this request in the priority queue
300 * @param max_queue_size at what queue size should this request be dropped 301 * @param max_queue_size at what queue size should this request be dropped
301 * (if other requests of higher priority are in the queue) 302 * (if other requests of higher priority are in the queue)
302 * @param timeout how long to wait at most for a response 303 * @param timeout how long to wait at most for a response
303 * @param iter function to call on each matching value; 304 * @param proc function to call on each matching value;
304 * will be called once with a NULL value at the end 305 * will be called once with a NULL value at the end
305 * @param iter_cls closure for iter 306 * @param proc_cls closure for proc
306 * @return NULL if the entry was not queued, otherwise a handle that can be used to 307 * @return NULL if the entry was not queued, otherwise a handle that can be used to
307 * cancel; note that even if NULL is returned, the callback will be invoked 308 * cancel
308 * (or rather, will already have been invoked)
309 */ 309 */
310struct GNUNET_DATASTORE_QueueEntry * 310struct GNUNET_DATASTORE_QueueEntry *
311GNUNET_DATASTORE_iterate_key (struct GNUNET_DATASTORE_Handle *h, 311GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
312 const GNUNET_HashCode * key, 312 uint64_t offset,
313 enum GNUNET_BLOCK_Type type, 313 const GNUNET_HashCode * key,
314 unsigned int queue_priority, 314 enum GNUNET_BLOCK_Type type,
315 unsigned int max_queue_size, 315 unsigned int queue_priority,
316 struct GNUNET_TIME_Relative timeout, 316 unsigned int max_queue_size,
317 GNUNET_DATASTORE_Iterator iter, 317 struct GNUNET_TIME_Relative timeout,
318 void *iter_cls); 318 GNUNET_DATASTORE_DatumProcessor proc,
319 void *proc_cls);
319 320
320 321
321/** 322/**
322 * Get all zero-anonymity values from the datastore. 323 * Get a single zero-anonymity value from the datastore.
323 * 324 *
324 * @param h handle to the datastore 325 * @param h handle to the datastore
326 * @param offset offset of the result (mod #num-results); set to
327 * a random 64-bit value initially; then increment by
328 * one each time; detect that all results have been found by uid
329 * being again the first uid ever returned.
325 * @param queue_priority ranking of this request in the priority queue 330 * @param queue_priority ranking of this request in the priority queue
326 * @param max_queue_size at what queue size should this request be dropped 331 * @param max_queue_size at what queue size should this request be dropped
327 * (if other requests of higher priority are in the queue) 332 * (if other requests of higher priority are in the queue)
328 * @param timeout how long to wait at most for a response 333 * @param timeout how long to wait at most for a response
329 * @param type allowed type for the operation (never zero) 334 * @param type allowed type for the operation (never zero)
330 * @param iter function to call on a random value; it 335 * @param proc function to call on a random value; it
331 * will be called once with a value (if available) 336 * will be called once with a value (if available)
332 * and always once with a value of NULL at the end. 337 * or with NULL if none value exists.
333 * @param iter_cls closure for iter 338 * @param proc_cls closure for proc
334 * @return NULL if the entry was not queued, otherwise a handle that can be used to 339 * @return NULL if the entry was not queued, otherwise a handle that can be used to
335 * cancel; note that even if NULL is returned, the callback will be invoked 340 * cancel
336 * (or rather, will already have been invoked)
337 */ 341 */
338struct GNUNET_DATASTORE_QueueEntry * 342struct GNUNET_DATASTORE_QueueEntry *
339GNUNET_DATASTORE_iterate_zero_anonymity (struct GNUNET_DATASTORE_Handle *h, 343GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
340 unsigned int queue_priority, 344 uint64_t offset,
341 unsigned int max_queue_size, 345 unsigned int queue_priority,
342 struct GNUNET_TIME_Relative timeout, 346 unsigned int max_queue_size,
343 enum GNUNET_BLOCK_Type type, 347 struct GNUNET_TIME_Relative timeout,
344 GNUNET_DATASTORE_Iterator iter, 348 enum GNUNET_BLOCK_Type type,
345 void *iter_cls); 349 GNUNET_DATASTORE_DatumProcessor proc,
346 350 void *proc_cls);
347
348/**
349 * Function called to trigger obtaining the next result
350 * from the datastore. ONLY applies for 'GNUNET_DATASTORE_iterate_*'
351 * calls, not for 'get' calls. FIXME: how much mixing of iterate
352 * calls with other operations can we permit!? Should we pass
353 * the 'QueueEntry' instead of the datastore handle here instead?
354 *
355 * @param h handle to the datastore
356 */
357void
358GNUNET_DATASTORE_iterate_get_next (struct GNUNET_DATASTORE_Handle *h);
359 351
360 352
361/** 353/**
@@ -370,21 +362,20 @@ GNUNET_DATASTORE_iterate_get_next (struct GNUNET_DATASTORE_Handle *h);
370 * @param max_queue_size at what queue size should this request be dropped 362 * @param max_queue_size at what queue size should this request be dropped
371 * (if other requests of higher priority are in the queue) 363 * (if other requests of higher priority are in the queue)
372 * @param timeout how long to wait at most for a response 364 * @param timeout how long to wait at most for a response
373 * @param iter function to call on a random value; it 365 * @param proc function to call on a random value; it
374 * will be called once with a value (if available) 366 * will be called once with a value (if available)
375 * and always once with a value of NULL. 367 * and always once with a value of NULL.
376 * @param iter_cls closure for iter 368 * @param proc_cls closure for proc
377 * @return NULL if the entry was not queued, otherwise a handle that can be used to 369 * @return NULL if the entry was not queued, otherwise a handle that can be used to
378 * cancel; note that even if NULL is returned, the callback will be invoked 370 * cancel
379 * (or rather, will already have been invoked)
380 */ 371 */
381struct GNUNET_DATASTORE_QueueEntry * 372struct GNUNET_DATASTORE_QueueEntry *
382GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h, 373GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h,
383 unsigned int queue_priority, 374 unsigned int queue_priority,
384 unsigned int max_queue_size, 375 unsigned int max_queue_size,
385 struct GNUNET_TIME_Relative timeout, 376 struct GNUNET_TIME_Relative timeout,
386 GNUNET_DATASTORE_Iterator iter, 377 GNUNET_DATASTORE_DatumProcessor proc,
387 void *iter_cls); 378 void *proc_cls);
388 379
389 380
390 381