diff options
author | Christian Grothoff <christian@grothoff.org> | 2015-03-30 19:46:35 +0000 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2015-03-30 19:46:35 +0000 |
commit | 3905f70a1f4491c789b6c752361bd257e1f6c931 (patch) | |
tree | 344e26685ae04473908b7ce4c3134402f9c6a09f /src | |
parent | bf7619be0f8e5fca7a16c28720ab908134d139bc (diff) | |
download | gnunet-3905f70a1f4491c789b6c752361bd257e1f6c931.tar.gz gnunet-3905f70a1f4491c789b6c752361bd257e1f6c931.zip |
address #3745
Diffstat (limited to 'src')
-rw-r--r-- | src/fs/fs_api.h | 10 | ||||
-rw-r--r-- | src/fs/fs_unindex.c | 142 |
2 files changed, 98 insertions, 54 deletions
diff --git a/src/fs/fs_api.h b/src/fs/fs_api.h index 02f4a70c0..5ff0c4f1d 100644 --- a/src/fs/fs_api.h +++ b/src/fs/fs_api.h | |||
@@ -1473,11 +1473,6 @@ struct GNUNET_FS_UnindexContext | |||
1473 | struct GNUNET_HashCode uquery; | 1473 | struct GNUNET_HashCode uquery; |
1474 | 1474 | ||
1475 | /** | 1475 | /** |
1476 | * First content UID, 0 for none. | ||
1477 | */ | ||
1478 | uint64_t first_uid; | ||
1479 | |||
1480 | /** | ||
1481 | * Error message, NULL on success. | 1476 | * Error message, NULL on success. |
1482 | */ | 1477 | */ |
1483 | char *emsg; | 1478 | char *emsg; |
@@ -1488,6 +1483,11 @@ struct GNUNET_FS_UnindexContext | |||
1488 | struct GNUNET_CRYPTO_FileHashContext *fhc; | 1483 | struct GNUNET_CRYPTO_FileHashContext *fhc; |
1489 | 1484 | ||
1490 | /** | 1485 | /** |
1486 | * Which values have we seen already? | ||
1487 | */ | ||
1488 | struct GNUNET_CONTAINER_MultiHashMap *seen_dh; | ||
1489 | |||
1490 | /** | ||
1491 | * Overall size of the file. | 1491 | * Overall size of the file. |
1492 | */ | 1492 | */ |
1493 | uint64_t file_size; | 1493 | uint64_t file_size; |
diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c index c87c4e4e9..2d6012202 100644 --- a/src/fs/fs_unindex.c +++ b/src/fs/fs_unindex.c | |||
@@ -48,7 +48,11 @@ | |||
48 | * @return number of bytes copied to buf, 0 on error | 48 | * @return number of bytes copied to buf, 0 on error |
49 | */ | 49 | */ |
50 | static size_t | 50 | static size_t |
51 | unindex_reader (void *cls, uint64_t offset, size_t max, void *buf, char **emsg) | 51 | unindex_reader (void *cls, |
52 | uint64_t offset, | ||
53 | size_t max, | ||
54 | void *buf, | ||
55 | char **emsg) | ||
52 | { | 56 | { |
53 | struct GNUNET_FS_UnindexContext *uc = cls; | 57 | struct GNUNET_FS_UnindexContext *uc = cls; |
54 | size_t pt_size; | 58 | size_t pt_size; |
@@ -144,12 +148,15 @@ signal_unindex_error (struct GNUNET_FS_UnindexContext *uc) | |||
144 | * datastore removal operation. | 148 | * datastore removal operation. |
145 | * | 149 | * |
146 | * @param cls closure | 150 | * @param cls closure |
147 | * @param success GNUNET_SYSERR on failure | 151 | * @param success #GNUNET_SYSERR on failure |
148 | * @param min_expiration minimum expiration time required for content to be stored | 152 | * @param min_expiration minimum expiration time required for content to be stored |
149 | * @param msg NULL on success, otherwise an error message | 153 | * @param msg NULL on success, otherwise an error message |
150 | */ | 154 | */ |
151 | static void | 155 | static void |
152 | process_cont (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg) | 156 | process_cont (void *cls, |
157 | int success, | ||
158 | struct GNUNET_TIME_Absolute min_expiration, | ||
159 | const char *msg) | ||
153 | { | 160 | { |
154 | struct GNUNET_FS_UnindexContext *uc = cls; | 161 | struct GNUNET_FS_UnindexContext *uc = cls; |
155 | 162 | ||
@@ -180,9 +187,13 @@ process_cont (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration | |||
180 | * @param block_size size of block (in bytes) | 187 | * @param block_size size of block (in bytes) |
181 | */ | 188 | */ |
182 | static void | 189 | static void |
183 | unindex_process (void *cls, const struct ContentHashKey *chk, uint64_t offset, | 190 | unindex_process (void *cls, |
184 | unsigned int depth, enum GNUNET_BLOCK_Type type, | 191 | const struct ContentHashKey *chk, |
185 | const void *block, uint16_t block_size) | 192 | uint64_t offset, |
193 | unsigned int depth, | ||
194 | enum GNUNET_BLOCK_Type type, | ||
195 | const void *block, | ||
196 | uint16_t block_size) | ||
186 | { | 197 | { |
187 | struct GNUNET_FS_UnindexContext *uc = cls; | 198 | struct GNUNET_FS_UnindexContext *uc = cls; |
188 | uint32_t size; | 199 | uint32_t size; |
@@ -281,6 +292,8 @@ unindex_finish (struct GNUNET_FS_UnindexContext *uc) | |||
281 | uc->fh = NULL; | 292 | uc->fh = NULL; |
282 | GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); | 293 | GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); |
283 | uc->dsh = NULL; | 294 | uc->dsh = NULL; |
295 | GNUNET_CONTAINER_multihashmap_destroy (uc->seen_dh); | ||
296 | uc->seen_dh = NULL; | ||
284 | uc->state = UNINDEX_STATE_FS_NOTIFY; | 297 | uc->state = UNINDEX_STATE_FS_NOTIFY; |
285 | GNUNET_FS_unindex_sync_ (uc); | 298 | GNUNET_FS_unindex_sync_ (uc); |
286 | uc->client = GNUNET_CLIENT_connect ("fs", uc->h->cfg); | 299 | uc->client = GNUNET_CLIENT_connect ("fs", uc->h->cfg); |
@@ -316,9 +329,9 @@ unindex_finish (struct GNUNET_FS_UnindexContext *uc) | |||
316 | * | 329 | * |
317 | * @param cls the 'struct GNUNET_FS_UnindexContext *' | 330 | * @param cls the 'struct GNUNET_FS_UnindexContext *' |
318 | * @param filename which file we are making progress on | 331 | * @param filename which file we are making progress on |
319 | * @param is_directory GNUNET_YES if this is a directory, | 332 | * @param is_directory #GNUNET_YES if this is a directory, |
320 | * GNUNET_NO if this is a file | 333 | * #GNUNET_NO if this is a file |
321 | * GNUNET_SYSERR if it is neither (or unknown) | 334 | * #GNUNET_SYSERR if it is neither (or unknown) |
322 | * @param reason kind of progress we are making | 335 | * @param reason kind of progress we are making |
323 | */ | 336 | */ |
324 | static void | 337 | static void |
@@ -413,6 +426,7 @@ continue_after_remove (void *cls, | |||
413 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | 426 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, |
414 | _("Failed to remove UBlock: %s\n"), | 427 | _("Failed to remove UBlock: %s\n"), |
415 | msg); | 428 | msg); |
429 | GNUNET_CONTAINER_multihashmap_clear (uc->seen_dh); | ||
416 | uc->ksk_offset++; | 430 | uc->ksk_offset++; |
417 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); | 431 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); |
418 | } | 432 | } |
@@ -441,38 +455,46 @@ continue_after_remove (void *cls, | |||
441 | static void | 455 | static void |
442 | process_kblock_for_unindex (void *cls, | 456 | process_kblock_for_unindex (void *cls, |
443 | const struct GNUNET_HashCode *key, | 457 | const struct GNUNET_HashCode *key, |
444 | size_t size, const void *data, | 458 | size_t size, |
459 | const void *data, | ||
445 | enum GNUNET_BLOCK_Type type, | 460 | enum GNUNET_BLOCK_Type type, |
446 | uint32_t priority, | 461 | uint32_t priority, |
447 | uint32_t anonymity, | 462 | uint32_t anonymity, |
448 | struct GNUNET_TIME_Absolute | 463 | struct GNUNET_TIME_Absolute expiration, |
449 | expiration, uint64_t uid) | 464 | uint64_t uid) |
450 | { | 465 | { |
451 | struct GNUNET_FS_UnindexContext *uc = cls; | 466 | struct GNUNET_FS_UnindexContext *uc = cls; |
452 | const struct UBlock *ub; | 467 | const struct UBlock *ub; |
453 | struct GNUNET_FS_Uri *chk_uri; | 468 | struct GNUNET_FS_Uri *chk_uri; |
454 | struct GNUNET_HashCode query; | 469 | struct GNUNET_HashCode query; |
470 | struct GNUNET_HashCode dh; | ||
455 | 471 | ||
456 | uc->dqe = NULL; | 472 | uc->dqe = NULL; |
457 | if (NULL == data) | 473 | if (NULL == data) |
458 | { | 474 | { |
459 | /* no result */ | 475 | /* no result */ |
476 | GNUNET_CONTAINER_multihashmap_clear (uc->seen_dh); | ||
460 | uc->ksk_offset++; | 477 | uc->ksk_offset++; |
461 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); | 478 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); |
462 | return; | 479 | return; |
463 | } | 480 | } |
464 | if (0 == uc->first_uid) | 481 | GNUNET_CRYPTO_hash (data, |
482 | size, | ||
483 | &dh); | ||
484 | if (GNUNET_YES == | ||
485 | GNUNET_CONTAINER_multihashmap_contains (uc->seen_dh, | ||
486 | &dh)) | ||
465 | { | 487 | { |
466 | /* remember UID of first result to detect cycles */ | 488 | GNUNET_CONTAINER_multihashmap_clear (uc->seen_dh); |
467 | uc->first_uid = uid; | ||
468 | } | ||
469 | else if (uid == uc->first_uid) | ||
470 | { | ||
471 | /* no more additional results */ | ||
472 | uc->ksk_offset++; | 489 | uc->ksk_offset++; |
473 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); | 490 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); |
474 | return; | 491 | return; |
475 | } | 492 | } |
493 | GNUNET_assert (GNUNET_OK == | ||
494 | GNUNET_CONTAINER_multihashmap_put (uc->seen_dh, | ||
495 | &dh, | ||
496 | uc, | ||
497 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); | ||
476 | GNUNET_assert (GNUNET_BLOCK_TYPE_FS_UBLOCK == type); | 498 | GNUNET_assert (GNUNET_BLOCK_TYPE_FS_UBLOCK == type); |
477 | if (size < sizeof (struct UBlock)) | 499 | if (size < sizeof (struct UBlock)) |
478 | { | 500 | { |
@@ -483,7 +505,9 @@ process_kblock_for_unindex (void *cls, | |||
483 | GNUNET_CRYPTO_hash (&ub->verification_key, | 505 | GNUNET_CRYPTO_hash (&ub->verification_key, |
484 | sizeof (ub->verification_key), | 506 | sizeof (ub->verification_key), |
485 | &query); | 507 | &query); |
486 | if (0 != memcmp (&query, key, sizeof (struct GNUNET_HashCode))) | 508 | if (0 != memcmp (&query, |
509 | key, | ||
510 | sizeof (struct GNUNET_HashCode))) | ||
487 | { | 511 | { |
488 | /* result does not match our keyword, skip */ | 512 | /* result does not match our keyword, skip */ |
489 | goto get_next; | 513 | goto get_next; |
@@ -494,7 +518,7 @@ process_kblock_for_unindex (void *cls, | |||
494 | const char *keyword; | 518 | const char *keyword; |
495 | 519 | ||
496 | GNUNET_CRYPTO_ecdsa_key_get_public (GNUNET_CRYPTO_ecdsa_key_get_anonymous (), | 520 | GNUNET_CRYPTO_ecdsa_key_get_public (GNUNET_CRYPTO_ecdsa_key_get_anonymous (), |
497 | &anon_pub); | 521 | &anon_pub); |
498 | keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1]; | 522 | keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1]; |
499 | GNUNET_FS_ublock_decrypt_ (&ub[1], size - sizeof (struct UBlock), | 523 | GNUNET_FS_ublock_decrypt_ (&ub[1], size - sizeof (struct UBlock), |
500 | &anon_pub, | 524 | &anon_pub, |
@@ -523,8 +547,11 @@ process_kblock_for_unindex (void *cls, | |||
523 | GNUNET_FS_uri_destroy (chk_uri); | 547 | GNUNET_FS_uri_destroy (chk_uri); |
524 | /* matches! */ | 548 | /* matches! */ |
525 | uc->dqe = GNUNET_DATASTORE_remove (uc->dsh, | 549 | uc->dqe = GNUNET_DATASTORE_remove (uc->dsh, |
526 | key, size, data, | 550 | key, |
527 | 0 /* priority */, 1 /* queue size */, | 551 | size, |
552 | data, | ||
553 | 0 /* priority */, | ||
554 | 1 /* queue size */, | ||
528 | GNUNET_TIME_UNIT_FOREVER_REL, | 555 | GNUNET_TIME_UNIT_FOREVER_REL, |
529 | &continue_after_remove, | 556 | &continue_after_remove, |
530 | uc); | 557 | uc); |
@@ -534,7 +561,8 @@ process_kblock_for_unindex (void *cls, | |||
534 | uc->roff++, | 561 | uc->roff++, |
535 | &uc->uquery, | 562 | &uc->uquery, |
536 | GNUNET_BLOCK_TYPE_FS_UBLOCK, | 563 | GNUNET_BLOCK_TYPE_FS_UBLOCK, |
537 | 0 /* priority */, 1 /* queue size */, | 564 | 0 /* priority */, |
565 | 1 /* queue size */, | ||
538 | GNUNET_TIME_UNIT_FOREVER_REL, | 566 | GNUNET_TIME_UNIT_FOREVER_REL, |
539 | &process_kblock_for_unindex, | 567 | &process_kblock_for_unindex, |
540 | uc); | 568 | uc); |
@@ -571,21 +599,22 @@ GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc) | |||
571 | return; | 599 | return; |
572 | } | 600 | } |
573 | anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous (); | 601 | anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous (); |
574 | GNUNET_CRYPTO_ecdsa_key_get_public (anon, &anon_pub); | 602 | GNUNET_CRYPTO_ecdsa_key_get_public (anon, |
603 | &anon_pub); | ||
575 | keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1]; | 604 | keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1]; |
576 | GNUNET_CRYPTO_ecdsa_public_key_derive (&anon_pub, | 605 | GNUNET_CRYPTO_ecdsa_public_key_derive (&anon_pub, |
577 | keyword, | 606 | keyword, |
578 | "fs-ublock", | 607 | "fs-ublock", |
579 | &dpub); | 608 | &dpub); |
580 | GNUNET_CRYPTO_hash (&dpub, | 609 | GNUNET_CRYPTO_hash (&dpub, |
581 | sizeof (dpub), | 610 | sizeof (dpub), |
582 | &uc->uquery); | 611 | &uc->uquery); |
583 | uc->first_uid = 0; | ||
584 | uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh, | 612 | uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh, |
585 | uc->roff++, | 613 | uc->roff++, |
586 | &uc->uquery, | 614 | &uc->uquery, |
587 | GNUNET_BLOCK_TYPE_FS_UBLOCK, | 615 | GNUNET_BLOCK_TYPE_FS_UBLOCK, |
588 | 0 /* priority */, 1 /* queue size */, | 616 | 0 /* priority */, |
617 | 1 /* queue size */, | ||
589 | GNUNET_TIME_UNIT_FOREVER_REL, | 618 | GNUNET_TIME_UNIT_FOREVER_REL, |
590 | &process_kblock_for_unindex, | 619 | &process_kblock_for_unindex, |
591 | uc); | 620 | uc); |
@@ -600,7 +629,8 @@ GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc) | |||
600 | * @param tc not used | 629 | * @param tc not used |
601 | */ | 630 | */ |
602 | static void | 631 | static void |
603 | unindex_extract_keywords (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) | 632 | unindex_extract_keywords (void *cls, |
633 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
604 | { | 634 | { |
605 | struct GNUNET_FS_UnindexContext *uc = cls; | 635 | struct GNUNET_FS_UnindexContext *uc = cls; |
606 | 636 | ||
@@ -642,8 +672,12 @@ GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc) | |||
642 | return; | 672 | return; |
643 | } | 673 | } |
644 | uc->tc = | 674 | uc->tc = |
645 | GNUNET_FS_tree_encoder_create (uc->h, uc->file_size, uc, &unindex_reader, | 675 | GNUNET_FS_tree_encoder_create (uc->h, |
646 | &unindex_process, &unindex_progress, | 676 | uc->file_size, |
677 | uc, | ||
678 | &unindex_reader, | ||
679 | &unindex_process, | ||
680 | &unindex_progress, | ||
647 | &unindex_extract_keywords); | 681 | &unindex_extract_keywords); |
648 | GNUNET_FS_tree_encoder_next (uc->tc); | 682 | GNUNET_FS_tree_encoder_next (uc->tc); |
649 | } | 683 | } |
@@ -657,7 +691,8 @@ GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc) | |||
657 | * @param file_id computed hash, NULL on error | 691 | * @param file_id computed hash, NULL on error |
658 | */ | 692 | */ |
659 | void | 693 | void |
660 | GNUNET_FS_unindex_process_hash_ (void *cls, const struct GNUNET_HashCode * file_id) | 694 | GNUNET_FS_unindex_process_hash_ (void *cls, |
695 | const struct GNUNET_HashCode *file_id) | ||
661 | { | 696 | { |
662 | struct GNUNET_FS_UnindexContext *uc = cls; | 697 | struct GNUNET_FS_UnindexContext *uc = cls; |
663 | 698 | ||
@@ -761,28 +796,37 @@ GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h, | |||
761 | const char *filename, | 796 | const char *filename, |
762 | void *cctx) | 797 | void *cctx) |
763 | { | 798 | { |
764 | struct GNUNET_FS_UnindexContext *ret; | 799 | struct GNUNET_FS_UnindexContext *uc; |
765 | struct GNUNET_FS_ProgressInfo pi; | 800 | struct GNUNET_FS_ProgressInfo pi; |
766 | uint64_t size; | 801 | uint64_t size; |
767 | 802 | ||
768 | if (GNUNET_OK != GNUNET_DISK_file_size (filename, &size, GNUNET_YES, GNUNET_YES)) | 803 | if (GNUNET_OK != |
804 | GNUNET_DISK_file_size (filename, | ||
805 | &size, | ||
806 | GNUNET_YES, | ||
807 | GNUNET_YES)) | ||
769 | return NULL; | 808 | return NULL; |
770 | ret = GNUNET_new (struct GNUNET_FS_UnindexContext); | 809 | uc = GNUNET_new (struct GNUNET_FS_UnindexContext); |
771 | ret->h = h; | 810 | uc->h = h; |
772 | ret->filename = GNUNET_strdup (filename); | 811 | uc->filename = GNUNET_strdup (filename); |
773 | ret->start_time = GNUNET_TIME_absolute_get (); | 812 | uc->start_time = GNUNET_TIME_absolute_get (); |
774 | ret->file_size = size; | 813 | uc->file_size = size; |
775 | ret->client_info = cctx; | 814 | uc->client_info = cctx; |
776 | GNUNET_FS_unindex_sync_ (ret); | 815 | uc->seen_dh = GNUNET_CONTAINER_multihashmap_create (4, |
816 | GNUNET_NO); | ||
817 | GNUNET_FS_unindex_sync_ (uc); | ||
777 | pi.status = GNUNET_FS_STATUS_UNINDEX_START; | 818 | pi.status = GNUNET_FS_STATUS_UNINDEX_START; |
778 | pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL; | 819 | pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL; |
779 | GNUNET_FS_unindex_make_status_ (&pi, ret, 0); | 820 | GNUNET_FS_unindex_make_status_ (&pi, uc, 0); |
780 | ret->fhc = | 821 | uc->fhc = |
781 | GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, filename, | 822 | GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, |
823 | filename, | ||
782 | HASHING_BLOCKSIZE, | 824 | HASHING_BLOCKSIZE, |
783 | &GNUNET_FS_unindex_process_hash_, ret); | 825 | &GNUNET_FS_unindex_process_hash_, uc); |
784 | ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_unindex_signal_suspend_, ret); | 826 | uc->top = GNUNET_FS_make_top (h, |
785 | return ret; | 827 | &GNUNET_FS_unindex_signal_suspend_, |
828 | uc); | ||
829 | return uc; | ||
786 | } | 830 | } |
787 | 831 | ||
788 | 832 | ||