diff options
author | Christian Grothoff <christian@grothoff.org> | 2019-07-21 11:08:40 +0200 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2019-07-21 11:08:51 +0200 |
commit | dfeef02bf7882cdfb952430cfbdb9793ecd780e2 (patch) | |
tree | 9ed8a31e4774f36d79ab0d606af2769107b7c250 /src/fs | |
parent | 4a85347dcea576d8f297efba24c1aaa77ded92d0 (diff) | |
download | gnunet-dfeef02bf7882cdfb952430cfbdb9793ecd780e2.tar.gz gnunet-dfeef02bf7882cdfb952430cfbdb9793ecd780e2.zip |
fix #5793
Diffstat (limited to 'src/fs')
-rw-r--r-- | src/fs/gnunet-service-fs_pr.c | 781 |
1 files changed, 388 insertions, 393 deletions
diff --git a/src/fs/gnunet-service-fs_pr.c b/src/fs/gnunet-service-fs_pr.c index abbaf94ef..f9702486a 100644 --- a/src/fs/gnunet-service-fs_pr.c +++ b/src/fs/gnunet-service-fs_pr.c | |||
@@ -11,7 +11,7 @@ | |||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | 11 | WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Affero General Public License for more details. | 13 | Affero General Public License for more details. |
14 | 14 | ||
15 | You should have received a copy of the GNU Affero General Public License | 15 | You should have received a copy of the GNU Affero General Public License |
16 | along with this program. If not, see <http://www.gnu.org/licenses/>. | 16 | along with this program. If not, see <http://www.gnu.org/licenses/>. |
17 | 17 | ||
@@ -157,7 +157,7 @@ struct GSF_PendingRequest | |||
157 | /** | 157 | /** |
158 | * Task that warns us if the local datastore lookup takes too long. | 158 | * Task that warns us if the local datastore lookup takes too long. |
159 | */ | 159 | */ |
160 | struct GNUNET_SCHEDULER_Task * warn_task; | 160 | struct GNUNET_SCHEDULER_Task *warn_task; |
161 | 161 | ||
162 | /** | 162 | /** |
163 | * Do we have a first UID yet? | 163 | * Do we have a first UID yet? |
@@ -195,7 +195,6 @@ struct GSF_PendingRequest | |||
195 | * Length of the 'replies_seen' array. | 195 | * Length of the 'replies_seen' array. |
196 | */ | 196 | */ |
197 | unsigned int replies_seen_size; | 197 | unsigned int replies_seen_size; |
198 | |||
199 | }; | 198 | }; |
200 | 199 | ||
201 | 200 | ||
@@ -237,7 +236,6 @@ static struct GNUNET_CONTAINER_Heap *requests_by_expiration_heap; | |||
237 | static unsigned long long max_pending_requests = (32 * 1024); | 236 | static unsigned long long max_pending_requests = (32 * 1024); |
238 | 237 | ||
239 | 238 | ||
240 | |||
241 | /** | 239 | /** |
242 | * Recalculate our bloom filter for filtering replies. This function | 240 | * Recalculate our bloom filter for filtering replies. This function |
243 | * will create a new bloom filter from scratch, so it should only be | 241 | * will create a new bloom filter from scratch, so it should only be |
@@ -249,8 +247,7 @@ static unsigned long long max_pending_requests = (32 * 1024); | |||
249 | * @param pr request for which the BF is to be recomputed | 247 | * @param pr request for which the BF is to be recomputed |
250 | */ | 248 | */ |
251 | static void | 249 | static void |
252 | refresh_bloomfilter (enum GNUNET_BLOCK_Type type, | 250 | refresh_bloomfilter (enum GNUNET_BLOCK_Type type, struct GSF_PendingRequest *pr) |
253 | struct GSF_PendingRequest *pr) | ||
254 | { | 251 | { |
255 | if (NULL != pr->bg) | 252 | if (NULL != pr->bg) |
256 | { | 253 | { |
@@ -259,16 +256,16 @@ refresh_bloomfilter (enum GNUNET_BLOCK_Type type, | |||
259 | } | 256 | } |
260 | if (GNUNET_BLOCK_TYPE_FS_UBLOCK != type) | 257 | if (GNUNET_BLOCK_TYPE_FS_UBLOCK != type) |
261 | return; /* no need */ | 258 | return; /* no need */ |
262 | pr->bg | 259 | pr->bg = |
263 | = GNUNET_BLOCK_group_create (GSF_block_ctx, | 260 | GNUNET_BLOCK_group_create (GSF_block_ctx, |
264 | type, | 261 | type, |
265 | GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, | 262 | GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, |
266 | UINT32_MAX), | 263 | UINT32_MAX), |
267 | NULL, | 264 | NULL, |
268 | 0, | 265 | 0, |
269 | "seen-set-size", | 266 | "seen-set-size", |
270 | pr->replies_seen_count, | 267 | pr->replies_seen_count, |
271 | NULL); | 268 | NULL); |
272 | if (NULL == pr->bg) | 269 | if (NULL == pr->bg) |
273 | return; | 270 | return; |
274 | GNUNET_break (GNUNET_OK == | 271 | GNUNET_break (GNUNET_OK == |
@@ -324,10 +321,12 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options, | |||
324 | 321 | ||
325 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 322 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
326 | "Creating request handle for `%s' of type %d\n", | 323 | "Creating request handle for `%s' of type %d\n", |
327 | GNUNET_h2s (query), type); | 324 | GNUNET_h2s (query), |
325 | type); | ||
328 | #if INSANE_STATISTICS | 326 | #if INSANE_STATISTICS |
329 | GNUNET_STATISTICS_update (GSF_stats, | 327 | GNUNET_STATISTICS_update (GSF_stats, |
330 | gettext_noop ("# Pending requests created"), 1, | 328 | gettext_noop ("# Pending requests created"), |
329 | 1, | ||
331 | GNUNET_NO); | 330 | GNUNET_NO); |
332 | #endif | 331 | #endif |
333 | extra = 0; | 332 | extra = 0; |
@@ -339,9 +338,7 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options, | |||
339 | if (NULL != target) | 338 | if (NULL != target) |
340 | { | 339 | { |
341 | pr->public_data.target = (struct GNUNET_PeerIdentity *) eptr; | 340 | pr->public_data.target = (struct GNUNET_PeerIdentity *) eptr; |
342 | GNUNET_memcpy (eptr, | 341 | GNUNET_memcpy (eptr, target, sizeof (struct GNUNET_PeerIdentity)); |
343 | target, | ||
344 | sizeof (struct GNUNET_PeerIdentity)); | ||
345 | } | 342 | } |
346 | pr->public_data.anonymity_level = anonymity_level; | 343 | pr->public_data.anonymity_level = anonymity_level; |
347 | pr->public_data.priority = priority; | 344 | pr->public_data.priority = priority; |
@@ -355,55 +352,49 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options, | |||
355 | pr->rh_cls = rh_cls; | 352 | pr->rh_cls = rh_cls; |
356 | GNUNET_assert ((sender_pid != 0) || (0 == (options & GSF_PRO_FORWARD_ONLY))); | 353 | GNUNET_assert ((sender_pid != 0) || (0 == (options & GSF_PRO_FORWARD_ONLY))); |
357 | if (ttl >= 0) | 354 | if (ttl >= 0) |
358 | pr->public_data.ttl = | 355 | pr->public_data.ttl = GNUNET_TIME_relative_to_absolute ( |
359 | GNUNET_TIME_relative_to_absolute (GNUNET_TIME_relative_multiply | 356 | GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, (uint32_t) ttl)); |
360 | (GNUNET_TIME_UNIT_SECONDS, | ||
361 | (uint32_t) ttl)); | ||
362 | else | 357 | else |
363 | pr->public_data.ttl = | 358 | pr->public_data.ttl = GNUNET_TIME_absolute_subtract ( |
364 | GNUNET_TIME_absolute_subtract (pr->public_data.start_time, | 359 | pr->public_data.start_time, |
365 | GNUNET_TIME_relative_multiply | 360 | GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, |
366 | (GNUNET_TIME_UNIT_SECONDS, | 361 | (uint32_t) (-ttl))); |
367 | (uint32_t) (-ttl))); | ||
368 | if (replies_seen_count > 0) | 362 | if (replies_seen_count > 0) |
369 | { | 363 | { |
370 | pr->replies_seen_size = replies_seen_count; | 364 | pr->replies_seen_size = replies_seen_count; |
371 | pr->replies_seen = GNUNET_new_array (pr->replies_seen_size, | 365 | pr->replies_seen = |
372 | struct GNUNET_HashCode); | 366 | GNUNET_new_array (pr->replies_seen_size, struct GNUNET_HashCode); |
373 | GNUNET_memcpy (pr->replies_seen, | 367 | GNUNET_memcpy (pr->replies_seen, |
374 | replies_seen, | 368 | replies_seen, |
375 | replies_seen_count * sizeof (struct GNUNET_HashCode)); | 369 | replies_seen_count * sizeof (struct GNUNET_HashCode)); |
376 | pr->replies_seen_count = replies_seen_count; | 370 | pr->replies_seen_count = replies_seen_count; |
377 | } | 371 | } |
378 | if ( (NULL != bf_data) && | 372 | if ((NULL != bf_data) && |
379 | (GNUNET_BLOCK_TYPE_FS_UBLOCK == pr->public_data.type) ) | 373 | (GNUNET_BLOCK_TYPE_FS_UBLOCK == pr->public_data.type)) |
380 | { | 374 | { |
381 | pr->bg | 375 | pr->bg = GNUNET_BLOCK_group_create (GSF_block_ctx, |
382 | = GNUNET_BLOCK_group_create (GSF_block_ctx, | 376 | pr->public_data.type, |
383 | pr->public_data.type, | 377 | mingle, |
384 | mingle, | 378 | bf_data, |
385 | bf_data, | 379 | bf_size, |
386 | bf_size, | 380 | "seen-set-size", |
387 | "seen-set-size", | 381 | 0, |
388 | 0, | 382 | NULL); |
389 | NULL); | ||
390 | } | 383 | } |
391 | else if ((replies_seen_count > 0) && | 384 | else if ((replies_seen_count > 0) && |
392 | (0 != (options & GSF_PRO_BLOOMFILTER_FULL_REFRESH))) | 385 | (0 != (options & GSF_PRO_BLOOMFILTER_FULL_REFRESH))) |
393 | { | 386 | { |
394 | refresh_bloomfilter (pr->public_data.type, | 387 | refresh_bloomfilter (pr->public_data.type, pr); |
395 | pr); | ||
396 | } | 388 | } |
397 | GNUNET_CONTAINER_multihashmap_put (pr_map, | 389 | GNUNET_CONTAINER_multihashmap_put (pr_map, |
398 | &pr->public_data.query, | 390 | &pr->public_data.query, |
399 | pr, | 391 | pr, |
400 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); | 392 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); |
401 | if (0 == (options & GSF_PRO_REQUEST_NEVER_EXPIRES)) | 393 | if (0 == (options & GSF_PRO_REQUEST_NEVER_EXPIRES)) |
402 | { | 394 | { |
403 | pr->hnode = | 395 | pr->hnode = GNUNET_CONTAINER_heap_insert (requests_by_expiration_heap, |
404 | GNUNET_CONTAINER_heap_insert (requests_by_expiration_heap, | 396 | pr, |
405 | pr, | 397 | pr->public_data.ttl.abs_value_us); |
406 | pr->public_data.ttl.abs_value_us); | ||
407 | /* make sure we don't track too many requests */ | 398 | /* make sure we don't track too many requests */ |
408 | while (GNUNET_CONTAINER_heap_get_size (requests_by_expiration_heap) > | 399 | while (GNUNET_CONTAINER_heap_get_size (requests_by_expiration_heap) > |
409 | max_pending_requests) | 400 | max_pending_requests) |
@@ -411,23 +402,23 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options, | |||
411 | dpr = GNUNET_CONTAINER_heap_peek (requests_by_expiration_heap); | 402 | dpr = GNUNET_CONTAINER_heap_peek (requests_by_expiration_heap); |
412 | GNUNET_assert (NULL != dpr); | 403 | GNUNET_assert (NULL != dpr); |
413 | if (pr == dpr) | 404 | if (pr == dpr) |
414 | break; /* let the request live briefly... */ | 405 | break; /* let the request live briefly... */ |
415 | if (NULL != dpr->rh) | 406 | if (NULL != dpr->rh) |
416 | dpr->rh (dpr->rh_cls, | 407 | dpr->rh (dpr->rh_cls, |
417 | GNUNET_BLOCK_EVALUATION_REQUEST_VALID, | 408 | GNUNET_BLOCK_EVALUATION_REQUEST_VALID, |
418 | dpr, | 409 | dpr, |
419 | UINT32_MAX, | 410 | UINT32_MAX, |
420 | GNUNET_TIME_UNIT_FOREVER_ABS, | 411 | GNUNET_TIME_UNIT_FOREVER_ABS, |
421 | GNUNET_TIME_UNIT_FOREVER_ABS, | 412 | GNUNET_TIME_UNIT_FOREVER_ABS, |
422 | GNUNET_BLOCK_TYPE_ANY, | 413 | GNUNET_BLOCK_TYPE_ANY, |
423 | NULL, | 414 | NULL, |
424 | 0); | 415 | 0); |
425 | GSF_pending_request_cancel_ (dpr, | 416 | GSF_pending_request_cancel_ (dpr, GNUNET_YES); |
426 | GNUNET_YES); | ||
427 | } | 417 | } |
428 | } | 418 | } |
429 | GNUNET_STATISTICS_update (GSF_stats, | 419 | GNUNET_STATISTICS_update (GSF_stats, |
430 | gettext_noop ("# Pending requests active"), 1, | 420 | gettext_noop ("# Pending requests active"), |
421 | 1, | ||
431 | GNUNET_NO); | 422 | GNUNET_NO); |
432 | return pr; | 423 | return pr; |
433 | } | 424 | } |
@@ -458,10 +449,10 @@ int | |||
458 | GSF_pending_request_is_compatible_ (struct GSF_PendingRequest *pra, | 449 | GSF_pending_request_is_compatible_ (struct GSF_PendingRequest *pra, |
459 | struct GSF_PendingRequest *prb) | 450 | struct GSF_PendingRequest *prb) |
460 | { | 451 | { |
461 | if ( (pra->public_data.type != prb->public_data.type) || | 452 | if ((pra->public_data.type != prb->public_data.type) || |
462 | (0 != memcmp (&pra->public_data.query, | 453 | (0 != memcmp (&pra->public_data.query, |
463 | &prb->public_data.query, | 454 | &prb->public_data.query, |
464 | sizeof (struct GNUNET_HashCode)))) | 455 | sizeof (struct GNUNET_HashCode)))) |
465 | return GNUNET_NO; | 456 | return GNUNET_NO; |
466 | return GNUNET_OK; | 457 | return GNUNET_OK; |
467 | } | 458 | } |
@@ -477,11 +468,11 @@ GSF_pending_request_is_compatible_ (struct GSF_PendingRequest *pra, | |||
477 | */ | 468 | */ |
478 | void | 469 | void |
479 | GSF_pending_request_update_ (struct GSF_PendingRequest *pr, | 470 | GSF_pending_request_update_ (struct GSF_PendingRequest *pr, |
480 | const struct GNUNET_HashCode * replies_seen, | 471 | const struct GNUNET_HashCode *replies_seen, |
481 | unsigned int replies_seen_count) | 472 | unsigned int replies_seen_count) |
482 | { | 473 | { |
483 | if (replies_seen_count + pr->replies_seen_count < pr->replies_seen_count) | 474 | if (replies_seen_count + pr->replies_seen_count < pr->replies_seen_count) |
484 | return; /* integer overflow */ | 475 | return; /* integer overflow */ |
485 | if (0 != (pr->public_data.options & GSF_PRO_BLOOMFILTER_FULL_REFRESH)) | 476 | if (0 != (pr->public_data.options & GSF_PRO_BLOOMFILTER_FULL_REFRESH)) |
486 | { | 477 | { |
487 | /* we're responsible for the BF, full refresh */ | 478 | /* we're responsible for the BF, full refresh */ |
@@ -493,8 +484,7 @@ GSF_pending_request_update_ (struct GSF_PendingRequest *pr, | |||
493 | replies_seen, | 484 | replies_seen, |
494 | sizeof (struct GNUNET_HashCode) * replies_seen_count); | 485 | sizeof (struct GNUNET_HashCode) * replies_seen_count); |
495 | pr->replies_seen_count += replies_seen_count; | 486 | pr->replies_seen_count += replies_seen_count; |
496 | refresh_bloomfilter (pr->public_data.type, | 487 | refresh_bloomfilter (pr->public_data.type, pr); |
497 | pr); | ||
498 | } | 488 | } |
499 | else | 489 | else |
500 | { | 490 | { |
@@ -502,8 +492,7 @@ GSF_pending_request_update_ (struct GSF_PendingRequest *pr, | |||
502 | { | 492 | { |
503 | /* we're not the initiator, but the initiator did not give us | 493 | /* we're not the initiator, but the initiator did not give us |
504 | * any bloom-filter, so we need to create one on-the-fly */ | 494 | * any bloom-filter, so we need to create one on-the-fly */ |
505 | refresh_bloomfilter (pr->public_data.type, | 495 | refresh_bloomfilter (pr->public_data.type, pr); |
506 | pr); | ||
507 | } | 496 | } |
508 | else | 497 | else |
509 | { | 498 | { |
@@ -515,8 +504,8 @@ GSF_pending_request_update_ (struct GSF_PendingRequest *pr, | |||
515 | } | 504 | } |
516 | if (NULL != pr->gh) | 505 | if (NULL != pr->gh) |
517 | GNUNET_DHT_get_filter_known_results (pr->gh, | 506 | GNUNET_DHT_get_filter_known_results (pr->gh, |
518 | replies_seen_count, | 507 | replies_seen_count, |
519 | replies_seen); | 508 | replies_seen); |
520 | } | 509 | } |
521 | 510 | ||
522 | 511 | ||
@@ -544,18 +533,18 @@ GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr) | |||
544 | uint32_t bf_nonce; | 533 | uint32_t bf_nonce; |
545 | 534 | ||
546 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 535 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
547 | "Building request message for `%s' of type %d\n", | 536 | "Building request message for `%s' of type %d\n", |
548 | GNUNET_h2s (&pr->public_data.query), | 537 | GNUNET_h2s (&pr->public_data.query), |
549 | pr->public_data.type); | 538 | pr->public_data.type); |
550 | k = 0; | 539 | k = 0; |
551 | bm = 0; | 540 | bm = 0; |
552 | do_route = (0 == (pr->public_data.options & GSF_PRO_FORWARD_ONLY)); | 541 | do_route = (0 == (pr->public_data.options & GSF_PRO_FORWARD_ONLY)); |
553 | if ((!do_route) && (pr->sender_pid == 0)) | 542 | if ((! do_route) && (pr->sender_pid == 0)) |
554 | { | 543 | { |
555 | GNUNET_break (0); | 544 | GNUNET_break (0); |
556 | do_route = GNUNET_YES; | 545 | do_route = GNUNET_YES; |
557 | } | 546 | } |
558 | if (!do_route) | 547 | if (! do_route) |
559 | { | 548 | { |
560 | bm |= GET_MESSAGE_BIT_RETURN_TO; | 549 | bm |= GET_MESSAGE_BIT_RETURN_TO; |
561 | k++; | 550 | k++; |
@@ -566,22 +555,18 @@ GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr) | |||
566 | k++; | 555 | k++; |
567 | } | 556 | } |
568 | if (GNUNET_OK != | 557 | if (GNUNET_OK != |
569 | GNUNET_BLOCK_group_serialize (pr->bg, | 558 | GNUNET_BLOCK_group_serialize (pr->bg, &bf_nonce, &bf_data, &bf_size)) |
570 | &bf_nonce, | ||
571 | &bf_data, | ||
572 | &bf_size)) | ||
573 | { | 559 | { |
574 | bf_size = 0; | 560 | bf_size = 0; |
575 | bf_data = NULL; | 561 | bf_data = NULL; |
576 | } | 562 | } |
577 | env = GNUNET_MQ_msg_extra (gm, | 563 | env = GNUNET_MQ_msg_extra (gm, |
578 | bf_size + k * sizeof (struct GNUNET_PeerIdentity), | 564 | bf_size + k * sizeof (struct GNUNET_PeerIdentity), |
579 | GNUNET_MESSAGE_TYPE_FS_GET); | 565 | GNUNET_MESSAGE_TYPE_FS_GET); |
580 | gm->type = htonl (pr->public_data.type); | 566 | gm->type = htonl (pr->public_data.type); |
581 | if (do_route) | 567 | if (do_route) |
582 | prio = | 568 | prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, |
583 | GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, | 569 | pr->public_data.priority + 1); |
584 | pr->public_data.priority + 1); | ||
585 | else | 570 | else |
586 | prio = 0; | 571 | prio = 0; |
587 | pr->public_data.priority -= prio; | 572 | pr->public_data.priority -= prio; |
@@ -597,13 +582,10 @@ GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr) | |||
597 | ext = (struct GNUNET_PeerIdentity *) &gm[1]; | 582 | ext = (struct GNUNET_PeerIdentity *) &gm[1]; |
598 | k = 0; | 583 | k = 0; |
599 | if (! do_route) | 584 | if (! do_route) |
600 | GNUNET_PEER_resolve (pr->sender_pid, | 585 | GNUNET_PEER_resolve (pr->sender_pid, &ext[k++]); |
601 | &ext[k++]); | ||
602 | if (NULL != pr->public_data.target) | 586 | if (NULL != pr->public_data.target) |
603 | ext[k++] = *pr->public_data.target; | 587 | ext[k++] = *pr->public_data.target; |
604 | GNUNET_memcpy (&ext[k], | 588 | GNUNET_memcpy (&ext[k], bf_data, bf_size); |
605 | bf_data, | ||
606 | bf_size); | ||
607 | GNUNET_free_non_null (bf_data); | 589 | GNUNET_free_non_null (bf_data); |
608 | return env; | 590 | return env; |
609 | } | 591 | } |
@@ -618,16 +600,14 @@ GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr) | |||
618 | * @return #GNUNET_YES (we should continue to iterate) | 600 | * @return #GNUNET_YES (we should continue to iterate) |
619 | */ | 601 | */ |
620 | static int | 602 | static int |
621 | clean_request (void *cls, | 603 | clean_request (void *cls, const struct GNUNET_HashCode *key, void *value) |
622 | const struct GNUNET_HashCode *key, | ||
623 | void *value) | ||
624 | { | 604 | { |
625 | struct GSF_PendingRequest *pr = value; | 605 | struct GSF_PendingRequest *pr = value; |
626 | GSF_LocalLookupContinuation cont; | 606 | GSF_LocalLookupContinuation cont; |
627 | 607 | ||
628 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 608 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
629 | "Cleaning up pending request for `%s'.\n", | 609 | "Cleaning up pending request for `%s'.\n", |
630 | GNUNET_h2s (key)); | 610 | GNUNET_h2s (key)); |
631 | if (NULL != pr->cadet_request) | 611 | if (NULL != pr->cadet_request) |
632 | { | 612 | { |
633 | pr->cadet_retry_count = CADET_RETRY_MAX; | 613 | pr->cadet_retry_count = CADET_RETRY_MAX; |
@@ -637,9 +617,7 @@ clean_request (void *cls, | |||
637 | if (NULL != (cont = pr->llc_cont)) | 617 | if (NULL != (cont = pr->llc_cont)) |
638 | { | 618 | { |
639 | pr->llc_cont = NULL; | 619 | pr->llc_cont = NULL; |
640 | cont (pr->llc_cont_cls, | 620 | cont (pr->llc_cont_cls, pr, pr->local_result); |
641 | pr, | ||
642 | pr->local_result); | ||
643 | } | 621 | } |
644 | GSF_plan_notify_request_done_ (pr); | 622 | GSF_plan_notify_request_done_ (pr); |
645 | GNUNET_free_non_null (pr->replies_seen); | 623 | GNUNET_free_non_null (pr->replies_seen); |
@@ -669,10 +647,9 @@ clean_request (void *cls, | |||
669 | GNUNET_SCHEDULER_cancel (pr->warn_task); | 647 | GNUNET_SCHEDULER_cancel (pr->warn_task); |
670 | pr->warn_task = NULL; | 648 | pr->warn_task = NULL; |
671 | } | 649 | } |
672 | GNUNET_assert (GNUNET_OK == | 650 | GNUNET_assert ( |
673 | GNUNET_CONTAINER_multihashmap_remove (pr_map, | 651 | GNUNET_OK == |
674 | &pr->public_data.query, | 652 | GNUNET_CONTAINER_multihashmap_remove (pr_map, &pr->public_data.query, pr)); |
675 | pr)); | ||
676 | GNUNET_STATISTICS_update (GSF_stats, | 653 | GNUNET_STATISTICS_update (GSF_stats, |
677 | gettext_noop ("# Pending requests active"), | 654 | gettext_noop ("# Pending requests active"), |
678 | -1, | 655 | -1, |
@@ -689,13 +666,12 @@ clean_request (void *cls, | |||
689 | * @param full_cleanup fully purge the request | 666 | * @param full_cleanup fully purge the request |
690 | */ | 667 | */ |
691 | void | 668 | void |
692 | GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr, | 669 | GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr, int full_cleanup) |
693 | int full_cleanup) | ||
694 | { | 670 | { |
695 | GSF_LocalLookupContinuation cont; | 671 | GSF_LocalLookupContinuation cont; |
696 | 672 | ||
697 | if (NULL == pr_map) | 673 | if (NULL == pr_map) |
698 | return; /* already cleaned up! */ | 674 | return; /* already cleaned up! */ |
699 | if (GNUNET_NO == full_cleanup) | 675 | if (GNUNET_NO == full_cleanup) |
700 | { | 676 | { |
701 | /* make request inactive (we're no longer interested in more results), | 677 | /* make request inactive (we're no longer interested in more results), |
@@ -711,9 +687,7 @@ GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr, | |||
711 | if (NULL != (cont = pr->llc_cont)) | 687 | if (NULL != (cont = pr->llc_cont)) |
712 | { | 688 | { |
713 | pr->llc_cont = NULL; | 689 | pr->llc_cont = NULL; |
714 | cont (pr->llc_cont_cls, | 690 | cont (pr->llc_cont_cls, pr, pr->local_result); |
715 | pr, | ||
716 | pr->local_result); | ||
717 | } | 691 | } |
718 | GSF_plan_notify_request_done_ (pr); | 692 | GSF_plan_notify_request_done_ (pr); |
719 | if (NULL != pr->qe) | 693 | if (NULL != pr->qe) |
@@ -734,9 +708,7 @@ GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr, | |||
734 | return; | 708 | return; |
735 | } | 709 | } |
736 | GNUNET_assert (GNUNET_YES == | 710 | GNUNET_assert (GNUNET_YES == |
737 | clean_request (NULL, | 711 | clean_request (NULL, &pr->public_data.query, pr)); |
738 | &pr->public_data.query, | ||
739 | pr)); | ||
740 | } | 712 | } |
741 | 713 | ||
742 | 714 | ||
@@ -749,9 +721,10 @@ GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr, | |||
749 | void | 721 | void |
750 | GSF_iterate_pending_requests_ (GSF_PendingRequestIterator it, void *cls) | 722 | GSF_iterate_pending_requests_ (GSF_PendingRequestIterator it, void *cls) |
751 | { | 723 | { |
752 | GNUNET_CONTAINER_multihashmap_iterate (pr_map, | 724 | GNUNET_CONTAINER_multihashmap_iterate ( |
753 | (GNUNET_CONTAINER_HashMapIterator) it, | 725 | pr_map, |
754 | cls); | 726 | (GNUNET_CONTAINER_MulitHashMapIteratorCallback) it, |
727 | cls); | ||
755 | } | 728 | } |
756 | 729 | ||
757 | 730 | ||
@@ -825,7 +798,8 @@ update_request_performance_data (struct ProcessReplyClosure *prq, | |||
825 | { | 798 | { |
826 | if (prq->sender == NULL) | 799 | if (prq->sender == NULL) |
827 | return; | 800 | return; |
828 | GSF_peer_update_performance_ (prq->sender, pr->public_data.start_time, | 801 | GSF_peer_update_performance_ (prq->sender, |
802 | pr->public_data.start_time, | ||
829 | prq->priority); | 803 | prq->priority); |
830 | } | 804 | } |
831 | 805 | ||
@@ -839,9 +813,7 @@ update_request_performance_data (struct ProcessReplyClosure *prq, | |||
839 | * @return #GNUNET_YES (we should continue to iterate) | 813 | * @return #GNUNET_YES (we should continue to iterate) |
840 | */ | 814 | */ |
841 | static int | 815 | static int |
842 | process_reply (void *cls, | 816 | process_reply (void *cls, const struct GNUNET_HashCode *key, void *value) |
843 | const struct GNUNET_HashCode *key, | ||
844 | void *value) | ||
845 | { | 817 | { |
846 | struct ProcessReplyClosure *prq = cls; | 818 | struct ProcessReplyClosure *prq = cls; |
847 | struct GSF_PendingRequest *pr = value; | 819 | struct GSF_PendingRequest *pr = value; |
@@ -855,18 +827,18 @@ process_reply (void *cls, | |||
855 | (unsigned int) prq->type, | 827 | (unsigned int) prq->type, |
856 | GNUNET_h2s (key)); | 828 | GNUNET_h2s (key)); |
857 | GNUNET_STATISTICS_update (GSF_stats, | 829 | GNUNET_STATISTICS_update (GSF_stats, |
858 | gettext_noop ("# replies received and matched"), 1, | 830 | gettext_noop ("# replies received and matched"), |
831 | 1, | ||
859 | GNUNET_NO); | 832 | GNUNET_NO); |
860 | prq->eval = | 833 | prq->eval = GNUNET_BLOCK_evaluate (GSF_block_ctx, |
861 | GNUNET_BLOCK_evaluate (GSF_block_ctx, | 834 | prq->type, |
862 | prq->type, | 835 | pr->bg, |
863 | pr->bg, | 836 | prq->eo, |
864 | prq->eo, | 837 | key, |
865 | key, | 838 | NULL, |
866 | NULL, | 839 | 0, |
867 | 0, | 840 | prq->data, |
868 | prq->data, | 841 | prq->size); |
869 | prq->size); | ||
870 | switch (prq->eval) | 842 | switch (prq->eval) |
871 | { | 843 | { |
872 | case GNUNET_BLOCK_EVALUATION_OK_MORE: | 844 | case GNUNET_BLOCK_EVALUATION_OK_MORE: |
@@ -876,37 +848,46 @@ process_reply (void *cls, | |||
876 | /* short cut: stop processing early, no BF-update, etc. */ | 848 | /* short cut: stop processing early, no BF-update, etc. */ |
877 | update_request_performance_data (prq, pr); | 849 | update_request_performance_data (prq, pr); |
878 | GNUNET_LOAD_update (GSF_rt_entry_lifetime, | 850 | GNUNET_LOAD_update (GSF_rt_entry_lifetime, |
879 | GNUNET_TIME_absolute_get_duration (pr-> | 851 | GNUNET_TIME_absolute_get_duration ( |
880 | public_data.start_time).rel_value_us); | 852 | pr->public_data.start_time) |
853 | .rel_value_us); | ||
881 | if (GNUNET_YES != | 854 | if (GNUNET_YES != |
882 | GSF_request_plan_reference_get_last_transmission_ (pr->public_data.pr_head, | 855 | GSF_request_plan_reference_get_last_transmission_ (pr->public_data |
883 | prq->sender, | 856 | .pr_head, |
884 | &last_transmission)) | 857 | prq->sender, |
885 | last_transmission.abs_value_us = GNUNET_TIME_UNIT_FOREVER_ABS.abs_value_us; | 858 | &last_transmission)) |
859 | last_transmission.abs_value_us = | ||
860 | GNUNET_TIME_UNIT_FOREVER_ABS.abs_value_us; | ||
886 | /* pass on to other peers / local clients */ | 861 | /* pass on to other peers / local clients */ |
887 | pr->rh (pr->rh_cls, prq->eval, pr, prq->anonymity_level, prq->expiration, | 862 | pr->rh (pr->rh_cls, |
888 | last_transmission, prq->type, prq->data, prq->size); | 863 | prq->eval, |
864 | pr, | ||
865 | prq->anonymity_level, | ||
866 | prq->expiration, | ||
867 | last_transmission, | ||
868 | prq->type, | ||
869 | prq->data, | ||
870 | prq->size); | ||
889 | return GNUNET_YES; | 871 | return GNUNET_YES; |
890 | case GNUNET_BLOCK_EVALUATION_OK_DUPLICATE: | 872 | case GNUNET_BLOCK_EVALUATION_OK_DUPLICATE: |
891 | #if INSANE_STATISTICS | 873 | #if INSANE_STATISTICS |
892 | GNUNET_STATISTICS_update (GSF_stats, | 874 | GNUNET_STATISTICS_update (GSF_stats, |
893 | gettext_noop | 875 | gettext_noop ( |
894 | ("# duplicate replies discarded (bloomfilter)"), | 876 | "# duplicate replies discarded (bloomfilter)"), |
895 | 1, GNUNET_NO); | 877 | 1, |
878 | GNUNET_NO); | ||
896 | #endif | 879 | #endif |
897 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 880 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Duplicate response, discarding.\n"); |
898 | "Duplicate response, discarding.\n"); | 881 | return GNUNET_YES; /* duplicate */ |
899 | return GNUNET_YES; /* duplicate */ | ||
900 | case GNUNET_BLOCK_EVALUATION_RESULT_IRRELEVANT: | 882 | case GNUNET_BLOCK_EVALUATION_RESULT_IRRELEVANT: |
901 | GNUNET_STATISTICS_update (GSF_stats, | 883 | GNUNET_STATISTICS_update (GSF_stats, |
902 | gettext_noop | 884 | gettext_noop ("# irrelevant replies discarded"), |
903 | ("# irrelevant replies discarded"), | 885 | 1, |
904 | 1, GNUNET_NO); | 886 | GNUNET_NO); |
905 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 887 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Irrelevant response, ignoring.\n"); |
906 | "Irrelevant response, ignoring.\n"); | ||
907 | return GNUNET_YES; | 888 | return GNUNET_YES; |
908 | case GNUNET_BLOCK_EVALUATION_RESULT_INVALID: | 889 | case GNUNET_BLOCK_EVALUATION_RESULT_INVALID: |
909 | return GNUNET_YES; /* wrong namespace */ | 890 | return GNUNET_YES; /* wrong namespace */ |
910 | case GNUNET_BLOCK_EVALUATION_REQUEST_VALID: | 891 | case GNUNET_BLOCK_EVALUATION_REQUEST_VALID: |
911 | GNUNET_break (0); | 892 | GNUNET_break (0); |
912 | return GNUNET_YES; | 893 | return GNUNET_YES; |
@@ -915,17 +896,13 @@ process_reply (void *cls, | |||
915 | return GNUNET_YES; | 896 | return GNUNET_YES; |
916 | case GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED: | 897 | case GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED: |
917 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | 898 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, |
918 | _("Unsupported block type %u\n"), | 899 | _ ("Unsupported block type %u\n"), |
919 | prq->type); | 900 | prq->type); |
920 | return GNUNET_NO; | 901 | return GNUNET_NO; |
921 | } | 902 | } |
922 | /* update bloomfilter */ | 903 | /* update bloomfilter */ |
923 | GNUNET_CRYPTO_hash (prq->data, | 904 | GNUNET_CRYPTO_hash (prq->data, prq->size, &chash); |
924 | prq->size, | 905 | GSF_pending_request_update_ (pr, &chash, 1); |
925 | &chash); | ||
926 | GSF_pending_request_update_ (pr, | ||
927 | &chash, | ||
928 | 1); | ||
929 | if (NULL == prq->sender) | 906 | if (NULL == prq->sender) |
930 | { | 907 | { |
931 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 908 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
@@ -946,14 +923,15 @@ process_reply (void *cls, | |||
946 | pr->public_data.results_found++; | 923 | pr->public_data.results_found++; |
947 | prq->request_found = GNUNET_YES; | 924 | prq->request_found = GNUNET_YES; |
948 | /* finally, pass on to other peer / local client */ | 925 | /* finally, pass on to other peer / local client */ |
949 | if (! GSF_request_plan_reference_get_last_transmission_ (pr->public_data.pr_head, | 926 | if (! GSF_request_plan_reference_get_last_transmission_ (pr->public_data |
950 | prq->sender, | 927 | .pr_head, |
951 | &last_transmission)) | 928 | prq->sender, |
929 | &last_transmission)) | ||
952 | last_transmission.abs_value_us = GNUNET_TIME_UNIT_FOREVER_ABS.abs_value_us; | 930 | last_transmission.abs_value_us = GNUNET_TIME_UNIT_FOREVER_ABS.abs_value_us; |
953 | pr->rh (pr->rh_cls, | 931 | pr->rh (pr->rh_cls, |
954 | prq->eval, | 932 | prq->eval, |
955 | pr, | 933 | pr, |
956 | prq->anonymity_level, | 934 | prq->anonymity_level, |
957 | prq->expiration, | 935 | prq->expiration, |
958 | last_transmission, | 936 | last_transmission, |
959 | prq->type, | 937 | prq->type, |
@@ -997,9 +975,10 @@ struct PutMigrationContext | |||
997 | * @param msg NULL on success, otherwise an error message | 975 | * @param msg NULL on success, otherwise an error message |
998 | */ | 976 | */ |
999 | static void | 977 | static void |
1000 | put_migration_continuation (void *cls, int success, | 978 | put_migration_continuation (void *cls, |
1001 | struct GNUNET_TIME_Absolute min_expiration, | 979 | int success, |
1002 | const char *msg) | 980 | struct GNUNET_TIME_Absolute min_expiration, |
981 | const char *msg) | ||
1003 | { | 982 | { |
1004 | struct PutMigrationContext *pmc = cls; | 983 | struct PutMigrationContext *pmc = cls; |
1005 | struct GSF_ConnectedPeer *cp; | 984 | struct GSF_ConnectedPeer *cp; |
@@ -1011,13 +990,14 @@ put_migration_continuation (void *cls, int success, | |||
1011 | if (GNUNET_SYSERR != success) | 990 | if (GNUNET_SYSERR != success) |
1012 | { | 991 | { |
1013 | GNUNET_LOAD_update (datastore_put_load, | 992 | GNUNET_LOAD_update (datastore_put_load, |
1014 | GNUNET_TIME_absolute_get_duration (pmc->start).rel_value_us); | 993 | GNUNET_TIME_absolute_get_duration (pmc->start) |
994 | .rel_value_us); | ||
1015 | } | 995 | } |
1016 | else | 996 | else |
1017 | { | 997 | { |
1018 | /* on queue failure / timeout, increase the put load dramatically */ | 998 | /* on queue failure / timeout, increase the put load dramatically */ |
1019 | GNUNET_LOAD_update (datastore_put_load, | 999 | GNUNET_LOAD_update (datastore_put_load, |
1020 | GNUNET_TIME_UNIT_MINUTES.rel_value_us); | 1000 | GNUNET_TIME_UNIT_MINUTES.rel_value_us); |
1021 | } | 1001 | } |
1022 | } | 1002 | } |
1023 | cp = GSF_peer_get_ (&pmc->origin); | 1003 | cp = GSF_peer_get_ (&pmc->origin); |
@@ -1031,38 +1011,41 @@ put_migration_continuation (void *cls, int success, | |||
1031 | GNUNET_free (pmc); | 1011 | GNUNET_free (pmc); |
1032 | return; | 1012 | return; |
1033 | } | 1013 | } |
1034 | if ( (GNUNET_NO == success) && | 1014 | if ((GNUNET_NO == success) && (GNUNET_NO == pmc->requested) && (NULL != cp)) |
1035 | (GNUNET_NO == pmc->requested) && | ||
1036 | (NULL != cp) ) | ||
1037 | { | 1015 | { |
1038 | ppd = GSF_get_peer_performance_data_ (cp); | 1016 | ppd = GSF_get_peer_performance_data_ (cp); |
1039 | if (min_expiration.abs_value_us > 0) | 1017 | if (min_expiration.abs_value_us > 0) |
1040 | { | 1018 | { |
1041 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1019 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1042 | "Asking to stop migration for %s because datastore is full\n", | 1020 | "Asking to stop migration for %s because datastore is full\n", |
1043 | GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (min_expiration), GNUNET_YES)); | 1021 | GNUNET_STRINGS_relative_time_to_string ( |
1022 | GNUNET_TIME_absolute_get_remaining (min_expiration), | ||
1023 | GNUNET_YES)); | ||
1044 | GSF_block_peer_migration_ (cp, min_expiration); | 1024 | GSF_block_peer_migration_ (cp, min_expiration); |
1045 | } | 1025 | } |
1046 | else | 1026 | else |
1047 | { | 1027 | { |
1048 | ppd->migration_delay = GNUNET_TIME_relative_max (GNUNET_TIME_UNIT_SECONDS, | 1028 | ppd->migration_delay = GNUNET_TIME_relative_max (GNUNET_TIME_UNIT_SECONDS, |
1049 | ppd->migration_delay); | 1029 | ppd->migration_delay); |
1050 | ppd->migration_delay = GNUNET_TIME_relative_min (GNUNET_TIME_UNIT_HOURS, | 1030 | ppd->migration_delay = |
1051 | ppd->migration_delay); | 1031 | GNUNET_TIME_relative_min (GNUNET_TIME_UNIT_HOURS, ppd->migration_delay); |
1052 | mig_pause.rel_value_us = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, | 1032 | mig_pause.rel_value_us = |
1053 | ppd->migration_delay.rel_value_us); | 1033 | GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, |
1054 | ppd->migration_delay = GNUNET_TIME_relative_saturating_multiply (ppd->migration_delay, 2); | 1034 | ppd->migration_delay.rel_value_us); |
1055 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1035 | ppd->migration_delay = |
1056 | "Replicated content already exists locally, asking to stop migration for %s\n", | 1036 | GNUNET_TIME_relative_saturating_multiply (ppd->migration_delay, 2); |
1057 | GNUNET_STRINGS_relative_time_to_string (mig_pause, | 1037 | GNUNET_log ( |
1058 | GNUNET_YES)); | 1038 | GNUNET_ERROR_TYPE_DEBUG, |
1039 | "Replicated content already exists locally, asking to stop migration for %s\n", | ||
1040 | GNUNET_STRINGS_relative_time_to_string (mig_pause, GNUNET_YES)); | ||
1059 | GSF_block_peer_migration_ (cp, | 1041 | GSF_block_peer_migration_ (cp, |
1060 | GNUNET_TIME_relative_to_absolute (mig_pause)); | 1042 | GNUNET_TIME_relative_to_absolute (mig_pause)); |
1061 | } | 1043 | } |
1062 | } | 1044 | } |
1063 | GNUNET_free (pmc); | 1045 | GNUNET_free (pmc); |
1064 | GNUNET_STATISTICS_update (GSF_stats, | 1046 | GNUNET_STATISTICS_update (GSF_stats, |
1065 | gettext_noop ("# Datastore `PUT' failures"), 1, | 1047 | gettext_noop ("# Datastore `PUT' failures"), |
1048 | 1, | ||
1066 | GNUNET_NO); | 1049 | GNUNET_NO); |
1067 | } | 1050 | } |
1068 | 1051 | ||
@@ -1084,13 +1067,14 @@ test_put_load_too_high (uint32_t priority) | |||
1084 | if (NULL == datastore_put_load) | 1067 | if (NULL == datastore_put_load) |
1085 | return GNUNET_NO; | 1068 | return GNUNET_NO; |
1086 | if (GNUNET_LOAD_get_average (datastore_put_load) < 50) | 1069 | if (GNUNET_LOAD_get_average (datastore_put_load) < 50) |
1087 | return GNUNET_NO; /* very fast */ | 1070 | return GNUNET_NO; /* very fast */ |
1088 | ld = GNUNET_LOAD_get_load (datastore_put_load); | 1071 | ld = GNUNET_LOAD_get_load (datastore_put_load); |
1089 | if (ld < 2.0 * (1 + priority)) | 1072 | if (ld < 2.0 * (1 + priority)) |
1090 | return GNUNET_NO; | 1073 | return GNUNET_NO; |
1091 | GNUNET_STATISTICS_update (GSF_stats, | 1074 | GNUNET_STATISTICS_update (GSF_stats, |
1092 | gettext_noop | 1075 | gettext_noop ( |
1093 | ("# storage requests dropped due to high load"), 1, | 1076 | "# storage requests dropped due to high load"), |
1077 | 1, | ||
1094 | GNUNET_NO); | 1078 | GNUNET_NO); |
1095 | return GNUNET_YES; | 1079 | return GNUNET_YES; |
1096 | } | 1080 | } |
@@ -1128,14 +1112,16 @@ handle_dht_reply (void *cls, | |||
1128 | struct PutMigrationContext *pmc; | 1112 | struct PutMigrationContext *pmc; |
1129 | 1113 | ||
1130 | GNUNET_STATISTICS_update (GSF_stats, | 1114 | GNUNET_STATISTICS_update (GSF_stats, |
1131 | gettext_noop ("# Replies received from DHT"), 1, | 1115 | gettext_noop ("# Replies received from DHT"), |
1116 | 1, | ||
1132 | GNUNET_NO); | 1117 | GNUNET_NO); |
1133 | memset (&prq, 0, sizeof (prq)); | 1118 | memset (&prq, 0, sizeof (prq)); |
1134 | prq.data = data; | 1119 | prq.data = data; |
1135 | prq.expiration = exp; | 1120 | prq.expiration = exp; |
1136 | /* do not allow migrated content to live longer than 1 year */ | 1121 | /* do not allow migrated content to live longer than 1 year */ |
1137 | prq.expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_YEARS), | 1122 | prq.expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute ( |
1138 | prq.expiration); | 1123 | GNUNET_TIME_UNIT_YEARS), |
1124 | prq.expiration); | ||
1139 | prq.size = size; | 1125 | prq.size = size; |
1140 | prq.type = type; | 1126 | prq.type = type; |
1141 | prq.eo = GNUNET_BLOCK_EO_NONE; | 1127 | prq.eo = GNUNET_BLOCK_EO_NONE; |
@@ -1145,16 +1131,25 @@ handle_dht_reply (void *cls, | |||
1145 | { | 1131 | { |
1146 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1132 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1147 | "Replicating result for query `%s' with priority %u\n", | 1133 | "Replicating result for query `%s' with priority %u\n", |
1148 | GNUNET_h2s (key), prq.priority); | 1134 | GNUNET_h2s (key), |
1135 | prq.priority); | ||
1149 | pmc = GNUNET_new (struct PutMigrationContext); | 1136 | pmc = GNUNET_new (struct PutMigrationContext); |
1150 | pmc->start = GNUNET_TIME_absolute_get (); | 1137 | pmc->start = GNUNET_TIME_absolute_get (); |
1151 | pmc->requested = GNUNET_YES; | 1138 | pmc->requested = GNUNET_YES; |
1152 | if (NULL == | 1139 | if (NULL == GNUNET_DATASTORE_put (GSF_dsh, |
1153 | GNUNET_DATASTORE_put (GSF_dsh, 0, key, size, data, type, prq.priority, | 1140 | 0, |
1154 | 1 /* anonymity */ , | 1141 | key, |
1155 | 0 /* replication */ , | 1142 | size, |
1156 | exp, 1 + prq.priority, MAX_DATASTORE_QUEUE, | 1143 | data, |
1157 | &put_migration_continuation, pmc)) | 1144 | type, |
1145 | prq.priority, | ||
1146 | 1 /* anonymity */, | ||
1147 | 0 /* replication */, | ||
1148 | exp, | ||
1149 | 1 + prq.priority, | ||
1150 | MAX_DATASTORE_QUEUE, | ||
1151 | &put_migration_continuation, | ||
1152 | pmc)) | ||
1158 | { | 1153 | { |
1159 | put_migration_continuation (pmc, | 1154 | put_migration_continuation (pmc, |
1160 | GNUNET_SYSERR, | 1155 | GNUNET_SYSERR, |
@@ -1194,17 +1189,19 @@ GSF_dht_lookup_ (struct GSF_PendingRequest *pr) | |||
1194 | GNUNET_memcpy (&buf[xquery_size], &pi, sizeof (struct GNUNET_PeerIdentity)); | 1189 | GNUNET_memcpy (&buf[xquery_size], &pi, sizeof (struct GNUNET_PeerIdentity)); |
1195 | xquery_size += sizeof (struct GNUNET_PeerIdentity); | 1190 | xquery_size += sizeof (struct GNUNET_PeerIdentity); |
1196 | } | 1191 | } |
1197 | pr->gh = | 1192 | pr->gh = GNUNET_DHT_get_start (GSF_dht, |
1198 | GNUNET_DHT_get_start (GSF_dht, | 1193 | pr->public_data.type, |
1199 | pr->public_data.type, &pr->public_data.query, | 1194 | &pr->public_data.query, |
1200 | DHT_GET_REPLICATION, | 1195 | DHT_GET_REPLICATION, |
1201 | GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE, | 1196 | GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE, |
1202 | xquery, xquery_size, &handle_dht_reply, pr); | 1197 | xquery, |
1203 | if ( (NULL != pr->gh) && | 1198 | xquery_size, |
1204 | (0 != pr->replies_seen_count) ) | 1199 | &handle_dht_reply, |
1200 | pr); | ||
1201 | if ((NULL != pr->gh) && (0 != pr->replies_seen_count)) | ||
1205 | GNUNET_DHT_get_filter_known_results (pr->gh, | 1202 | GNUNET_DHT_get_filter_known_results (pr->gh, |
1206 | pr->replies_seen_count, | 1203 | pr->replies_seen_count, |
1207 | pr->replies_seen); | 1204 | pr->replies_seen); |
1208 | } | 1205 | } |
1209 | 1206 | ||
1210 | 1207 | ||
@@ -1219,10 +1216,10 @@ GSF_dht_lookup_ (struct GSF_PendingRequest *pr) | |||
1219 | */ | 1216 | */ |
1220 | static void | 1217 | static void |
1221 | cadet_reply_proc (void *cls, | 1218 | cadet_reply_proc (void *cls, |
1222 | enum GNUNET_BLOCK_Type type, | 1219 | enum GNUNET_BLOCK_Type type, |
1223 | struct GNUNET_TIME_Absolute expiration, | 1220 | struct GNUNET_TIME_Absolute expiration, |
1224 | size_t data_size, | 1221 | size_t data_size, |
1225 | const void *data) | 1222 | const void *data) |
1226 | { | 1223 | { |
1227 | struct GSF_PendingRequest *pr = cls; | 1224 | struct GSF_PendingRequest *pr = cls; |
1228 | struct ProcessReplyClosure prq; | 1225 | struct ProcessReplyClosure prq; |
@@ -1236,8 +1233,7 @@ cadet_reply_proc (void *cls, | |||
1236 | pr->cadet_retry_count++; | 1233 | pr->cadet_retry_count++; |
1237 | if (pr->cadet_retry_count >= CADET_RETRY_MAX) | 1234 | if (pr->cadet_retry_count >= CADET_RETRY_MAX) |
1238 | return; /* give up on cadet */ | 1235 | return; /* give up on cadet */ |
1239 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1236 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Error retrieiving block via cadet\n"); |
1240 | "Error retrieiving block via cadet\n"); | ||
1241 | /* retry -- without delay, as this is non-anonymous | 1237 | /* retry -- without delay, as this is non-anonymous |
1242 | and cadet/cadet connect will take some time anyway */ | 1238 | and cadet/cadet connect will take some time anyway */ |
1243 | pr->cadet_request = GSF_cadet_query (pr->public_data.target, | 1239 | pr->cadet_request = GSF_cadet_query (pr->public_data.target, |
@@ -1248,25 +1244,25 @@ cadet_reply_proc (void *cls, | |||
1248 | return; | 1244 | return; |
1249 | } | 1245 | } |
1250 | if (GNUNET_YES != | 1246 | if (GNUNET_YES != |
1251 | GNUNET_BLOCK_get_key (GSF_block_ctx, | 1247 | GNUNET_BLOCK_get_key (GSF_block_ctx, type, data, data_size, &query)) |
1252 | type, | ||
1253 | data, data_size, &query)) | ||
1254 | { | 1248 | { |
1255 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1249 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1256 | "Failed to derive key for block of type %d\n", | 1250 | "Failed to derive key for block of type %d\n", |
1257 | (int) type); | 1251 | (int) type); |
1258 | GNUNET_break_op (0); | 1252 | GNUNET_break_op (0); |
1259 | return; | 1253 | return; |
1260 | } | 1254 | } |
1261 | GNUNET_STATISTICS_update (GSF_stats, | 1255 | GNUNET_STATISTICS_update (GSF_stats, |
1262 | gettext_noop ("# Replies received from CADET"), 1, | 1256 | gettext_noop ("# Replies received from CADET"), |
1257 | 1, | ||
1263 | GNUNET_NO); | 1258 | GNUNET_NO); |
1264 | memset (&prq, 0, sizeof (prq)); | 1259 | memset (&prq, 0, sizeof (prq)); |
1265 | prq.data = data; | 1260 | prq.data = data; |
1266 | prq.expiration = expiration; | 1261 | prq.expiration = expiration; |
1267 | /* do not allow migrated content to live longer than 1 year */ | 1262 | /* do not allow migrated content to live longer than 1 year */ |
1268 | prq.expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_YEARS), | 1263 | prq.expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute ( |
1269 | prq.expiration); | 1264 | GNUNET_TIME_UNIT_YEARS), |
1265 | prq.expiration); | ||
1270 | prq.size = data_size; | 1266 | prq.size = data_size; |
1271 | prq.type = type; | 1267 | prq.type = type; |
1272 | prq.eo = GNUNET_BLOCK_EO_NONE; | 1268 | prq.eo = GNUNET_BLOCK_EO_NONE; |
@@ -1287,7 +1283,7 @@ GSF_cadet_lookup_ (struct GSF_PendingRequest *pr) | |||
1287 | if (0 == pr->public_data.target) | 1283 | if (0 == pr->public_data.target) |
1288 | { | 1284 | { |
1289 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1285 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1290 | "Cannot do cadet-based download, target peer not known\n"); | 1286 | "Cannot do cadet-based download, target peer not known\n"); |
1291 | return; | 1287 | return; |
1292 | } | 1288 | } |
1293 | if (NULL != pr->cadet_request) | 1289 | if (NULL != pr->cadet_request) |
@@ -1311,13 +1307,13 @@ warn_delay_task (void *cls) | |||
1311 | struct GSF_PendingRequest *pr = cls; | 1307 | struct GSF_PendingRequest *pr = cls; |
1312 | 1308 | ||
1313 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK, | 1309 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK, |
1314 | _("Datastore lookup already took %s!\n"), | 1310 | _ ("Datastore lookup already took %s!\n"), |
1315 | GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (pr->qe_start), | 1311 | GNUNET_STRINGS_relative_time_to_string ( |
1316 | GNUNET_YES)); | 1312 | GNUNET_TIME_absolute_get_duration (pr->qe_start), |
1317 | pr->warn_task = | 1313 | GNUNET_YES)); |
1318 | GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, | 1314 | pr->warn_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, |
1319 | &warn_delay_task, | 1315 | &warn_delay_task, |
1320 | pr); | 1316 | pr); |
1321 | } | 1317 | } |
1322 | 1318 | ||
1323 | 1319 | ||
@@ -1332,11 +1328,13 @@ odc_warn_delay_task (void *cls) | |||
1332 | struct GSF_PendingRequest *pr = cls; | 1328 | struct GSF_PendingRequest *pr = cls; |
1333 | 1329 | ||
1334 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | 1330 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, |
1335 | _("On-demand lookup already took %s!\n"), | 1331 | _ ("On-demand lookup already took %s!\n"), |
1336 | GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (pr->qe_start), GNUNET_YES)); | 1332 | GNUNET_STRINGS_relative_time_to_string ( |
1337 | pr->warn_task = | 1333 | GNUNET_TIME_absolute_get_duration (pr->qe_start), |
1338 | GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, | 1334 | GNUNET_YES)); |
1339 | &odc_warn_delay_task, pr); | 1335 | pr->warn_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, |
1336 | &odc_warn_delay_task, | ||
1337 | pr); | ||
1340 | } | 1338 | } |
1341 | 1339 | ||
1342 | 1340 | ||
@@ -1353,7 +1351,7 @@ call_continuation (struct GSF_PendingRequest *pr) | |||
1353 | pr->warn_task = NULL; | 1351 | pr->warn_task = NULL; |
1354 | } | 1352 | } |
1355 | if (NULL == cont) | 1353 | if (NULL == cont) |
1356 | return; /* no continuation */ | 1354 | return; /* no continuation */ |
1357 | pr->llc_cont = NULL; | 1355 | pr->llc_cont = NULL; |
1358 | if (0 != (GSF_PRO_LOCAL_ONLY & pr->public_data.options)) | 1356 | if (0 != (GSF_PRO_LOCAL_ONLY & pr->public_data.options)) |
1359 | { | 1357 | { |
@@ -1362,14 +1360,14 @@ call_continuation (struct GSF_PendingRequest *pr) | |||
1362 | /* Signal that we are done and that there won't be any | 1360 | /* Signal that we are done and that there won't be any |
1363 | additional results to allow client to clean up state. */ | 1361 | additional results to allow client to clean up state. */ |
1364 | pr->rh (pr->rh_cls, | 1362 | pr->rh (pr->rh_cls, |
1365 | GNUNET_BLOCK_EVALUATION_OK_LAST, | 1363 | GNUNET_BLOCK_EVALUATION_OK_LAST, |
1366 | pr, | 1364 | pr, |
1367 | UINT32_MAX, | 1365 | UINT32_MAX, |
1368 | GNUNET_TIME_UNIT_ZERO_ABS, | 1366 | GNUNET_TIME_UNIT_ZERO_ABS, |
1369 | GNUNET_TIME_UNIT_FOREVER_ABS, | 1367 | GNUNET_TIME_UNIT_FOREVER_ABS, |
1370 | GNUNET_BLOCK_TYPE_ANY, | 1368 | GNUNET_BLOCK_TYPE_ANY, |
1371 | NULL, | 1369 | NULL, |
1372 | 0); | 1370 | 0); |
1373 | } | 1371 | } |
1374 | /* Finally, call our continuation to signal that we are | 1372 | /* Finally, call our continuation to signal that we are |
1375 | done with local processing of this request; i.e. to | 1373 | done with local processing of this request; i.e. to |
@@ -1389,10 +1387,11 @@ no_more_local_results (struct GSF_PendingRequest *pr) | |||
1389 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, | 1387 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, |
1390 | "No further local responses available.\n"); | 1388 | "No further local responses available.\n"); |
1391 | #if INSANE_STATISTICS | 1389 | #if INSANE_STATISTICS |
1392 | if ( (GNUNET_BLOCK_TYPE_FS_DBLOCK == pr->public_data.type) || | 1390 | if ((GNUNET_BLOCK_TYPE_FS_DBLOCK == pr->public_data.type) || |
1393 | (GNUNET_BLOCK_TYPE_FS_IBLOCK == pr->public_data.type) ) | 1391 | (GNUNET_BLOCK_TYPE_FS_IBLOCK == pr->public_data.type)) |
1394 | GNUNET_STATISTICS_update (GSF_stats, | 1392 | GNUNET_STATISTICS_update (GSF_stats, |
1395 | gettext_noop ("# requested DBLOCK or IBLOCK not found"), | 1393 | gettext_noop ( |
1394 | "# requested DBLOCK or IBLOCK not found"), | ||
1396 | 1, | 1395 | 1, |
1397 | GNUNET_NO); | 1396 | GNUNET_NO); |
1398 | #endif | 1397 | #endif |
@@ -1421,37 +1420,40 @@ start_local_query (struct GSF_PendingRequest *pr, | |||
1421 | bool random) | 1420 | bool random) |
1422 | { | 1421 | { |
1423 | pr->qe_start = GNUNET_TIME_absolute_get (); | 1422 | pr->qe_start = GNUNET_TIME_absolute_get (); |
1424 | pr->warn_task = | 1423 | pr->warn_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, |
1425 | GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, | 1424 | &warn_delay_task, |
1426 | &warn_delay_task, | 1425 | pr); |
1427 | pr); | 1426 | pr->qe = GNUNET_DATASTORE_get_key (GSF_dsh, |
1428 | pr->qe = | 1427 | next_uid, |
1429 | GNUNET_DATASTORE_get_key (GSF_dsh, | 1428 | random, |
1430 | next_uid, | 1429 | &pr->public_data.query, |
1431 | random, | 1430 | pr->public_data.type == |
1432 | &pr->public_data.query, | 1431 | GNUNET_BLOCK_TYPE_FS_DBLOCK |
1433 | pr->public_data.type == | 1432 | ? GNUNET_BLOCK_TYPE_ANY |
1434 | GNUNET_BLOCK_TYPE_FS_DBLOCK ? | 1433 | : pr->public_data.type, |
1435 | GNUNET_BLOCK_TYPE_ANY : pr->public_data.type, | 1434 | (0 != (GSF_PRO_PRIORITY_UNLIMITED & |
1436 | (0 != | 1435 | pr->public_data.options)) |
1437 | (GSF_PRO_PRIORITY_UNLIMITED & pr-> | 1436 | ? UINT_MAX |
1438 | public_data.options)) ? UINT_MAX : 1 | 1437 | : 1 |
1439 | /* queue priority */ , | 1438 | /* queue priority */, |
1440 | (0 != | 1439 | (0 != (GSF_PRO_PRIORITY_UNLIMITED & |
1441 | (GSF_PRO_PRIORITY_UNLIMITED & pr-> | 1440 | pr->public_data.options)) |
1442 | public_data.options)) ? UINT_MAX : | 1441 | ? UINT_MAX |
1443 | GSF_datastore_queue_size | 1442 | : GSF_datastore_queue_size |
1444 | /* max queue size */ , | 1443 | /* max queue size */, |
1445 | &process_local_reply, pr); | 1444 | &process_local_reply, |
1445 | pr); | ||
1446 | if (NULL != pr->qe) | 1446 | if (NULL != pr->qe) |
1447 | return; | 1447 | return; |
1448 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1448 | GNUNET_log ( |
1449 | "ERROR Requesting `%s' of type %d with next_uid %llu from datastore.\n", | 1449 | GNUNET_ERROR_TYPE_DEBUG, |
1450 | GNUNET_h2s (&pr->public_data.query), | 1450 | "ERROR Requesting `%s' of type %d with next_uid %llu from datastore.\n", |
1451 | pr->public_data.type, | 1451 | GNUNET_h2s (&pr->public_data.query), |
1452 | (unsigned long long) next_uid); | 1452 | pr->public_data.type, |
1453 | (unsigned long long) next_uid); | ||
1453 | GNUNET_STATISTICS_update (GSF_stats, | 1454 | GNUNET_STATISTICS_update (GSF_stats, |
1454 | gettext_noop ("# Datastore lookups concluded (error queueing)"), | 1455 | gettext_noop ( |
1456 | "# Datastore lookups concluded (error queueing)"), | ||
1455 | 1, | 1457 | 1, |
1456 | GNUNET_NO); | 1458 | GNUNET_NO); |
1457 | call_continuation (pr); | 1459 | call_continuation (pr); |
@@ -1498,44 +1500,43 @@ process_local_reply (void *cls, | |||
1498 | if (NULL == pr->qe) | 1500 | if (NULL == pr->qe) |
1499 | goto called_from_on_demand; | 1501 | goto called_from_on_demand; |
1500 | pr->qe = NULL; | 1502 | pr->qe = NULL; |
1501 | if ( (NULL == key) && | 1503 | if ( |
1502 | pr->seen_null && | 1504 | (NULL == key) && pr->seen_null && |
1503 | !pr->have_first_uid) /* We have hit the end for the 2nd time with no results */ | 1505 | ! pr->have_first_uid) /* We have hit the end for the 2nd time with no results */ |
1504 | { | 1506 | { |
1505 | /* No results */ | 1507 | /* No results */ |
1506 | #if INSANE_STATISTICS | 1508 | #if INSANE_STATISTICS |
1507 | GNUNET_STATISTICS_update (GSF_stats, | 1509 | GNUNET_STATISTICS_update (GSF_stats, |
1508 | gettext_noop | 1510 | gettext_noop ( |
1509 | ("# Datastore lookups concluded (no results)"), | 1511 | "# Datastore lookups concluded (no results)"), |
1510 | 1, GNUNET_NO); | 1512 | 1, |
1513 | GNUNET_NO); | ||
1511 | #endif | 1514 | #endif |
1512 | no_more_local_results (pr); | 1515 | no_more_local_results (pr); |
1513 | return; | 1516 | return; |
1514 | } | 1517 | } |
1515 | if ( ( (NULL == key) && | 1518 | if (((NULL == key) && |
1516 | pr->seen_null ) || /* We have hit the end for the 2nd time OR */ | 1519 | pr->seen_null) || /* We have hit the end for the 2nd time OR */ |
1517 | ( pr->seen_null && | 1520 | (pr->seen_null && pr->have_first_uid && |
1518 | pr->have_first_uid && | 1521 | (uid >= pr->first_uid))) /* We have hit the end and past first UID */ |
1519 | (uid >= pr->first_uid) ) ) /* We have hit the end and past first UID */ | ||
1520 | { | 1522 | { |
1521 | /* Seen all results */ | 1523 | /* Seen all results */ |
1522 | GNUNET_STATISTICS_update (GSF_stats, | 1524 | GNUNET_STATISTICS_update (GSF_stats, |
1523 | gettext_noop | 1525 | gettext_noop ( |
1524 | ("# Datastore lookups concluded (seen all)"), | 1526 | "# Datastore lookups concluded (seen all)"), |
1525 | 1, GNUNET_NO); | 1527 | 1, |
1528 | GNUNET_NO); | ||
1526 | no_more_local_results (pr); | 1529 | no_more_local_results (pr); |
1527 | return; | 1530 | return; |
1528 | } | 1531 | } |
1529 | if (NULL == key) | 1532 | if (NULL == key) |
1530 | { | 1533 | { |
1531 | GNUNET_assert (!pr->seen_null); | 1534 | GNUNET_assert (! pr->seen_null); |
1532 | pr->seen_null = true; | 1535 | pr->seen_null = true; |
1533 | start_local_query (pr, | 1536 | start_local_query (pr, 0 /* next_uid */, false /* random */); |
1534 | 0 /* next_uid */, | ||
1535 | false /* random */); | ||
1536 | return; | 1537 | return; |
1537 | } | 1538 | } |
1538 | if (!pr->have_first_uid) | 1539 | if (! pr->have_first_uid) |
1539 | { | 1540 | { |
1540 | pr->first_uid = uid; | 1541 | pr->first_uid = uid; |
1541 | pr->have_first_uid = true; | 1542 | pr->have_first_uid = true; |
@@ -1543,54 +1544,57 @@ process_local_reply (void *cls, | |||
1543 | pr->result_count++; | 1544 | pr->result_count++; |
1544 | if (pr->result_count > MAX_RESULTS) | 1545 | if (pr->result_count > MAX_RESULTS) |
1545 | { | 1546 | { |
1546 | GNUNET_STATISTICS_update (GSF_stats, | 1547 | GNUNET_STATISTICS_update ( |
1547 | gettext_noop | 1548 | GSF_stats, |
1548 | ("# Datastore lookups aborted (more than MAX_RESULTS)"), | 1549 | gettext_noop ("# Datastore lookups aborted (more than MAX_RESULTS)"), |
1549 | 1, GNUNET_NO); | 1550 | 1, |
1551 | GNUNET_NO); | ||
1550 | no_more_local_results (pr); | 1552 | no_more_local_results (pr); |
1551 | return; | 1553 | return; |
1552 | } | 1554 | } |
1553 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1555 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1554 | "Received reply for `%s' of type %d with UID %llu from datastore.\n", | 1556 | "Received reply for `%s' of type %d with UID %llu from datastore.\n", |
1555 | GNUNET_h2s (key), type, (unsigned long long) uid); | 1557 | GNUNET_h2s (key), |
1558 | type, | ||
1559 | (unsigned long long) uid); | ||
1556 | if (GNUNET_BLOCK_TYPE_FS_ONDEMAND == type) | 1560 | if (GNUNET_BLOCK_TYPE_FS_ONDEMAND == type) |
1557 | { | 1561 | { |
1558 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1562 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1559 | "Found ONDEMAND block, performing on-demand encoding\n"); | 1563 | "Found ONDEMAND block, performing on-demand encoding\n"); |
1560 | GNUNET_STATISTICS_update (GSF_stats, | 1564 | GNUNET_STATISTICS_update (GSF_stats, |
1561 | gettext_noop | 1565 | gettext_noop ( |
1562 | ("# on-demand blocks matched requests"), 1, | 1566 | "# on-demand blocks matched requests"), |
1567 | 1, | ||
1563 | GNUNET_NO); | 1568 | GNUNET_NO); |
1564 | pr->qe_start = GNUNET_TIME_absolute_get (); | 1569 | pr->qe_start = GNUNET_TIME_absolute_get (); |
1565 | pr->warn_task = | 1570 | pr->warn_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, |
1566 | GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, | 1571 | &odc_warn_delay_task, |
1567 | &odc_warn_delay_task, pr); | 1572 | pr); |
1568 | if (GNUNET_OK == | 1573 | if (GNUNET_OK == GNUNET_FS_handle_on_demand_block (key, |
1569 | GNUNET_FS_handle_on_demand_block (key, | 1574 | size, |
1570 | size, | 1575 | data, |
1571 | data, | 1576 | type, |
1572 | type, | 1577 | priority, |
1573 | priority, | 1578 | anonymity, |
1574 | anonymity, | 1579 | replication, |
1575 | replication, | 1580 | expiration, |
1576 | expiration, | 1581 | uid, |
1577 | uid, | 1582 | &process_local_reply, |
1578 | &process_local_reply, | 1583 | pr)) |
1579 | pr)) | ||
1580 | { | 1584 | { |
1581 | GNUNET_STATISTICS_update (GSF_stats, | 1585 | GNUNET_STATISTICS_update (GSF_stats, |
1582 | gettext_noop | 1586 | gettext_noop ( |
1583 | ("# on-demand lookups performed successfully"), | 1587 | "# on-demand lookups performed successfully"), |
1584 | 1, GNUNET_NO); | 1588 | 1, |
1585 | return; /* we're done */ | 1589 | GNUNET_NO); |
1590 | return; /* we're done */ | ||
1586 | } | 1591 | } |
1587 | GNUNET_STATISTICS_update (GSF_stats, | 1592 | GNUNET_STATISTICS_update (GSF_stats, |
1588 | gettext_noop ("# on-demand lookups failed"), 1, | 1593 | gettext_noop ("# on-demand lookups failed"), |
1594 | 1, | ||
1589 | GNUNET_NO); | 1595 | GNUNET_NO); |
1590 | GNUNET_SCHEDULER_cancel (pr->warn_task); | 1596 | GNUNET_SCHEDULER_cancel (pr->warn_task); |
1591 | start_local_query (pr, | 1597 | start_local_query (pr, uid + 1 /* next_uid */, false /* random */); |
1592 | uid + 1 /* next_uid */, | ||
1593 | false /* random */); | ||
1594 | return; | 1598 | return; |
1595 | } | 1599 | } |
1596 | called_from_on_demand: | 1600 | called_from_on_demand: |
@@ -1603,11 +1607,15 @@ called_from_on_demand: | |||
1603 | GNUNET_BLOCK_get_key (GSF_block_ctx, type, data, size, &query)) | 1607 | GNUNET_BLOCK_get_key (GSF_block_ctx, type, data, size, &query)) |
1604 | { | 1608 | { |
1605 | GNUNET_break (0); | 1609 | GNUNET_break (0); |
1606 | GNUNET_DATASTORE_remove (GSF_dsh, key, size, data, UINT_MAX, UINT_MAX, | 1610 | GNUNET_DATASTORE_remove (GSF_dsh, |
1607 | NULL, NULL); | 1611 | key, |
1608 | start_local_query (pr, | 1612 | size, |
1609 | uid + 1 /* next_uid */, | 1613 | data, |
1610 | false /* random */); | 1614 | UINT_MAX, |
1615 | UINT_MAX, | ||
1616 | NULL, | ||
1617 | NULL); | ||
1618 | start_local_query (pr, uid + 1 /* next_uid */, false /* random */); | ||
1611 | return; | 1619 | return; |
1612 | } | 1620 | } |
1613 | prq.type = type; | 1621 | prq.type = type; |
@@ -1621,11 +1629,11 @@ called_from_on_demand: | |||
1621 | pr->local_result = prq.eval; | 1629 | pr->local_result = prq.eval; |
1622 | if (GNUNET_BLOCK_EVALUATION_OK_LAST == prq.eval) | 1630 | if (GNUNET_BLOCK_EVALUATION_OK_LAST == prq.eval) |
1623 | { | 1631 | { |
1624 | GNUNET_STATISTICS_update (GSF_stats, | 1632 | GNUNET_STATISTICS_update ( |
1625 | gettext_noop | 1633 | GSF_stats, |
1626 | ("# Datastore lookups concluded (found last result)"), | 1634 | gettext_noop ("# Datastore lookups concluded (found last result)"), |
1627 | 1, | 1635 | 1, |
1628 | GNUNET_NO); | 1636 | GNUNET_NO); |
1629 | call_continuation (pr); | 1637 | call_continuation (pr); |
1630 | return; | 1638 | return; |
1631 | } | 1639 | } |
@@ -1633,18 +1641,16 @@ called_from_on_demand: | |||
1633 | ((GNUNET_YES == GSF_test_get_load_too_high_ (0)) || | 1641 | ((GNUNET_YES == GSF_test_get_load_too_high_ (0)) || |
1634 | (pr->public_data.results_found > 5 + 2 * pr->public_data.priority))) | 1642 | (pr->public_data.results_found > 5 + 2 * pr->public_data.priority))) |
1635 | { | 1643 | { |
1636 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1644 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Load too high, done with request\n"); |
1637 | "Load too high, done with request\n"); | ||
1638 | GNUNET_STATISTICS_update (GSF_stats, | 1645 | GNUNET_STATISTICS_update (GSF_stats, |
1639 | gettext_noop ("# Datastore lookups concluded (load too high)"), | 1646 | gettext_noop ( |
1647 | "# Datastore lookups concluded (load too high)"), | ||
1640 | 1, | 1648 | 1, |
1641 | GNUNET_NO); | 1649 | GNUNET_NO); |
1642 | call_continuation (pr); | 1650 | call_continuation (pr); |
1643 | return; | 1651 | return; |
1644 | } | 1652 | } |
1645 | start_local_query (pr, | 1653 | start_local_query (pr, uid + 1 /* next_uid */, false /* random */); |
1646 | uid + 1 /* next_uid */, | ||
1647 | false /* random */); | ||
1648 | } | 1654 | } |
1649 | 1655 | ||
1650 | 1656 | ||
@@ -1664,10 +1670,9 @@ GSF_pending_request_test_target_ (struct GSF_PendingRequest *pr, | |||
1664 | if (0 == pr->origin_pid) | 1670 | if (0 == pr->origin_pid) |
1665 | return GNUNET_YES; | 1671 | return GNUNET_YES; |
1666 | GNUNET_PEER_resolve (pr->origin_pid, &pi); | 1672 | GNUNET_PEER_resolve (pr->origin_pid, &pi); |
1667 | return (0 == | 1673 | return (0 == memcmp (&pi, target, sizeof (struct GNUNET_PeerIdentity))) |
1668 | memcmp (&pi, target, | 1674 | ? GNUNET_NO |
1669 | sizeof (struct GNUNET_PeerIdentity))) ? GNUNET_NO : | 1675 | : GNUNET_YES; |
1670 | GNUNET_YES; | ||
1671 | } | 1676 | } |
1672 | 1677 | ||
1673 | 1678 | ||
@@ -1690,16 +1695,14 @@ GSF_local_lookup_ (struct GSF_PendingRequest *pr, | |||
1690 | pr->llc_cont_cls = cont_cls; | 1695 | pr->llc_cont_cls = cont_cls; |
1691 | #if INSANE_STATISTICS | 1696 | #if INSANE_STATISTICS |
1692 | GNUNET_STATISTICS_update (GSF_stats, | 1697 | GNUNET_STATISTICS_update (GSF_stats, |
1693 | gettext_noop ("# Datastore lookups initiated"), 1, | 1698 | gettext_noop ("# Datastore lookups initiated"), |
1699 | 1, | ||
1694 | GNUNET_NO); | 1700 | GNUNET_NO); |
1695 | #endif | 1701 | #endif |
1696 | start_local_query(pr, | 1702 | start_local_query (pr, 0 /* next_uid */, true /* random */); |
1697 | 0 /* next_uid */, | ||
1698 | true /* random */); | ||
1699 | } | 1703 | } |
1700 | 1704 | ||
1701 | 1705 | ||
1702 | |||
1703 | /** | 1706 | /** |
1704 | * Handle P2P "CONTENT" message. Checks that the message is | 1707 | * Handle P2P "CONTENT" message. Checks that the message is |
1705 | * well-formed and then checks if there are any pending requests for | 1708 | * well-formed and then checks if there are any pending requests for |
@@ -1710,8 +1713,7 @@ GSF_local_lookup_ (struct GSF_PendingRequest *pr, | |||
1710 | * @param put the actual message | 1713 | * @param put the actual message |
1711 | */ | 1714 | */ |
1712 | void | 1715 | void |
1713 | handle_p2p_put (void *cls, | 1716 | handle_p2p_put (void *cls, const struct PutMessage *put) |
1714 | const struct PutMessage *put) | ||
1715 | { | 1717 | { |
1716 | struct GSF_ConnectedPeer *cp = cls; | 1718 | struct GSF_ConnectedPeer *cp = cls; |
1717 | uint16_t msize; | 1719 | uint16_t msize; |
@@ -1733,14 +1735,11 @@ handle_p2p_put (void *cls, | |||
1733 | type = ntohl (put->type); | 1735 | type = ntohl (put->type); |
1734 | expiration = GNUNET_TIME_absolute_ntoh (put->expiration); | 1736 | expiration = GNUNET_TIME_absolute_ntoh (put->expiration); |
1735 | /* do not allow migrated content to live longer than 1 year */ | 1737 | /* do not allow migrated content to live longer than 1 year */ |
1736 | expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_YEARS), | 1738 | expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute ( |
1737 | expiration); | 1739 | GNUNET_TIME_UNIT_YEARS), |
1740 | expiration); | ||
1738 | if (GNUNET_OK != | 1741 | if (GNUNET_OK != |
1739 | GNUNET_BLOCK_get_key (GSF_block_ctx, | 1742 | GNUNET_BLOCK_get_key (GSF_block_ctx, type, &put[1], dsize, &query)) |
1740 | type, | ||
1741 | &put[1], | ||
1742 | dsize, | ||
1743 | &query)) | ||
1744 | { | 1743 | { |
1745 | GNUNET_break_op (0); | 1744 | GNUNET_break_op (0); |
1746 | return; | 1745 | return; |
@@ -1767,11 +1766,10 @@ handle_p2p_put (void *cls, | |||
1767 | { | 1766 | { |
1768 | GSF_connected_peer_change_preference_ (cp, | 1767 | GSF_connected_peer_change_preference_ (cp, |
1769 | CONTENT_BANDWIDTH_VALUE + | 1768 | CONTENT_BANDWIDTH_VALUE + |
1770 | 1000 * prq.priority); | 1769 | 1000 * prq.priority); |
1771 | GSF_get_peer_performance_data_ (cp)->respect += prq.priority; | 1770 | GSF_get_peer_performance_data_ (cp)->respect += prq.priority; |
1772 | } | 1771 | } |
1773 | if ((GNUNET_YES == active_to_migration) && | 1772 | if ((GNUNET_YES == active_to_migration) && (NULL != cp) && |
1774 | (NULL != cp) && | ||
1775 | (GNUNET_NO == test_put_load_too_high (prq.priority))) | 1773 | (GNUNET_NO == test_put_load_too_high (prq.priority))) |
1776 | { | 1774 | { |
1777 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1775 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
@@ -1784,20 +1782,20 @@ handle_p2p_put (void *cls, | |||
1784 | GNUNET_assert (0 != GSF_get_peer_performance_data_ (cp)->pid); | 1782 | GNUNET_assert (0 != GSF_get_peer_performance_data_ (cp)->pid); |
1785 | GNUNET_PEER_resolve (GSF_get_peer_performance_data_ (cp)->pid, | 1783 | GNUNET_PEER_resolve (GSF_get_peer_performance_data_ (cp)->pid, |
1786 | &pmc->origin); | 1784 | &pmc->origin); |
1787 | if (NULL == | 1785 | if (NULL == GNUNET_DATASTORE_put (GSF_dsh, |
1788 | GNUNET_DATASTORE_put (GSF_dsh, | 1786 | 0, |
1789 | 0, | 1787 | &query, |
1790 | &query, | 1788 | dsize, |
1791 | dsize, | 1789 | &put[1], |
1792 | &put[1], | 1790 | type, |
1793 | type, | 1791 | prq.priority, |
1794 | prq.priority, | 1792 | 1 /* anonymity */, |
1795 | 1 /* anonymity */ , | 1793 | 0 /* replication */, |
1796 | 0 /* replication */ , | 1794 | expiration, |
1797 | expiration, 1 + prq.priority, | 1795 | 1 + prq.priority, |
1798 | MAX_DATASTORE_QUEUE, | 1796 | MAX_DATASTORE_QUEUE, |
1799 | &put_migration_continuation, | 1797 | &put_migration_continuation, |
1800 | pmc)) | 1798 | pmc)) |
1801 | { | 1799 | { |
1802 | put_migration_continuation (pmc, | 1800 | put_migration_continuation (pmc, |
1803 | GNUNET_SYSERR, | 1801 | GNUNET_SYSERR, |
@@ -1810,32 +1808,29 @@ handle_p2p_put (void *cls, | |||
1810 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1808 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1811 | "Choosing not to keep content `%s' (%d/%d)\n", | 1809 | "Choosing not to keep content `%s' (%d/%d)\n", |
1812 | GNUNET_h2s (&query), | 1810 | GNUNET_h2s (&query), |
1813 | active_to_migration, | 1811 | active_to_migration, |
1814 | test_put_load_too_high (prq.priority)); | 1812 | test_put_load_too_high (prq.priority)); |
1815 | } | 1813 | } |
1816 | putl = GNUNET_LOAD_get_load (datastore_put_load); | 1814 | putl = GNUNET_LOAD_get_load (datastore_put_load); |
1817 | if ( (NULL != cp) && | 1815 | if ((NULL != cp) && (GNUNET_NO == prq.request_found) && |
1818 | (GNUNET_NO == prq.request_found) && | 1816 | ((GNUNET_YES != active_to_migration) || |
1819 | ( (GNUNET_YES != active_to_migration) || | 1817 | (putl > 2.5 * (1 + prq.priority)))) |
1820 | (putl > 2.5 * (1 + prq.priority)) ) ) | ||
1821 | { | 1818 | { |
1822 | if (GNUNET_YES != active_to_migration) | 1819 | if (GNUNET_YES != active_to_migration) |
1823 | putl = 1.0 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 5); | 1820 | putl = 1.0 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 5); |
1824 | block_time = | 1821 | block_time = GNUNET_TIME_relative_multiply ( |
1825 | GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, | 1822 | GNUNET_TIME_UNIT_MILLISECONDS, |
1826 | 5000 + | 1823 | 5000 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, |
1827 | GNUNET_CRYPTO_random_u32 | 1824 | (unsigned int) (60000 * putl * putl))); |
1828 | (GNUNET_CRYPTO_QUALITY_WEAK, | 1825 | GNUNET_log ( |
1829 | (unsigned int) (60000 * putl * putl))); | 1826 | GNUNET_ERROR_TYPE_DEBUG, |
1830 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1827 | "Asking to stop migration for %s because of load %f and events %d/%d\n", |
1831 | "Asking to stop migration for %s because of load %f and events %d/%d\n", | 1828 | GNUNET_STRINGS_relative_time_to_string (block_time, GNUNET_YES), |
1832 | GNUNET_STRINGS_relative_time_to_string (block_time, | 1829 | putl, |
1833 | GNUNET_YES), | 1830 | active_to_migration, |
1834 | putl, | 1831 | (GNUNET_NO == prq.request_found)); |
1835 | active_to_migration, | ||
1836 | (GNUNET_NO == prq.request_found)); | ||
1837 | GSF_block_peer_migration_ (cp, | 1832 | GSF_block_peer_migration_ (cp, |
1838 | GNUNET_TIME_relative_to_absolute (block_time)); | 1833 | GNUNET_TIME_relative_to_absolute (block_time)); |
1839 | } | 1834 | } |
1840 | } | 1835 | } |
1841 | 1836 | ||
@@ -1860,19 +1855,21 @@ void | |||
1860 | GSF_pending_request_init_ () | 1855 | GSF_pending_request_init_ () |
1861 | { | 1856 | { |
1862 | if (GNUNET_OK != | 1857 | if (GNUNET_OK != |
1863 | GNUNET_CONFIGURATION_get_value_number (GSF_cfg, "fs", | 1858 | GNUNET_CONFIGURATION_get_value_number (GSF_cfg, |
1859 | "fs", | ||
1864 | "MAX_PENDING_REQUESTS", | 1860 | "MAX_PENDING_REQUESTS", |
1865 | &max_pending_requests)) | 1861 | &max_pending_requests)) |
1866 | { | 1862 | { |
1867 | GNUNET_log_config_missing (GNUNET_ERROR_TYPE_INFO, | 1863 | GNUNET_log_config_missing (GNUNET_ERROR_TYPE_INFO, |
1868 | "fs", "MAX_PENDING_REQUESTS"); | 1864 | "fs", |
1865 | "MAX_PENDING_REQUESTS"); | ||
1869 | } | 1866 | } |
1870 | active_to_migration = | 1867 | active_to_migration = |
1871 | GNUNET_CONFIGURATION_get_value_yesno (GSF_cfg, "FS", "CONTENT_CACHING"); | 1868 | GNUNET_CONFIGURATION_get_value_yesno (GSF_cfg, "FS", "CONTENT_CACHING"); |
1872 | datastore_put_load = GNUNET_LOAD_value_init (DATASTORE_LOAD_AUTODECLINE); | 1869 | datastore_put_load = GNUNET_LOAD_value_init (DATASTORE_LOAD_AUTODECLINE); |
1873 | pr_map = GNUNET_CONTAINER_multihashmap_create (32 * 1024, GNUNET_YES); | 1870 | pr_map = GNUNET_CONTAINER_multihashmap_create (32 * 1024, GNUNET_YES); |
1874 | requests_by_expiration_heap = | 1871 | requests_by_expiration_heap = |
1875 | GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); | 1872 | GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); |
1876 | } | 1873 | } |
1877 | 1874 | ||
1878 | 1875 | ||
@@ -1882,9 +1879,7 @@ GSF_pending_request_init_ () | |||
1882 | void | 1879 | void |
1883 | GSF_pending_request_done_ () | 1880 | GSF_pending_request_done_ () |
1884 | { | 1881 | { |
1885 | GNUNET_CONTAINER_multihashmap_iterate (pr_map, | 1882 | GNUNET_CONTAINER_multihashmap_iterate (pr_map, &clean_request, NULL); |
1886 | &clean_request, | ||
1887 | NULL); | ||
1888 | GNUNET_CONTAINER_multihashmap_destroy (pr_map); | 1883 | GNUNET_CONTAINER_multihashmap_destroy (pr_map); |
1889 | pr_map = NULL; | 1884 | pr_map = NULL; |
1890 | GNUNET_CONTAINER_heap_destroy (requests_by_expiration_heap); | 1885 | GNUNET_CONTAINER_heap_destroy (requests_by_expiration_heap); |