diff options
Diffstat (limited to 'src/fs')
-rw-r--r-- | src/fs/fs.c | 12 | ||||
-rw-r--r-- | src/fs/fs.h | 38 | ||||
-rw-r--r-- | src/fs/fs_download.c | 333 | ||||
-rw-r--r-- | src/fs/fs_publish.c | 2 | ||||
-rw-r--r-- | src/fs/fs_search.c | 269 | ||||
-rw-r--r-- | src/fs/fs_tree.c | 14 | ||||
-rw-r--r-- | src/fs/fs_tree.h | 2 | ||||
-rw-r--r-- | src/fs/fs_unindex.c | 2 | ||||
-rw-r--r-- | src/fs/gnunet-download.c | 4 | ||||
-rw-r--r-- | src/fs/gnunet-service-fs.c | 88 |
10 files changed, 639 insertions, 125 deletions
diff --git a/src/fs/fs.c b/src/fs/fs.c index 736c3b026..c8ce4d651 100644 --- a/src/fs/fs.c +++ b/src/fs/fs.c | |||
@@ -1815,7 +1815,9 @@ GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc) | |||
1815 | (GNUNET_OK != | 1815 | (GNUNET_OK != |
1816 | GNUNET_BIO_write_int32 (wh, (uint32_t) dc->has_finished)) || | 1816 | GNUNET_BIO_write_int32 (wh, (uint32_t) dc->has_finished)) || |
1817 | (GNUNET_OK != | 1817 | (GNUNET_OK != |
1818 | GNUNET_BIO_write_int32 (wh, num_pending)) ) | 1818 | GNUNET_BIO_write_int32 (wh, num_pending)) || |
1819 | (GNUNET_OK != | ||
1820 | GNUNET_BIO_write_int32 (wh, dc->start_task != GNUNET_SCHEDULER_NO_TASK)) ) | ||
1819 | { | 1821 | { |
1820 | GNUNET_break (0); | 1822 | GNUNET_break (0); |
1821 | goto cleanup; | 1823 | goto cleanup; |
@@ -2583,6 +2585,7 @@ deserialize_download (struct GNUNET_FS_Handle *h, | |||
2583 | uint32_t options; | 2585 | uint32_t options; |
2584 | uint32_t status; | 2586 | uint32_t status; |
2585 | uint32_t num_pending; | 2587 | uint32_t num_pending; |
2588 | int32_t start_pending; | ||
2586 | 2589 | ||
2587 | uris = NULL; | 2590 | uris = NULL; |
2588 | emsg = NULL; | 2591 | emsg = NULL; |
@@ -2621,7 +2624,9 @@ deserialize_download (struct GNUNET_FS_Handle *h, | |||
2621 | (GNUNET_OK != | 2624 | (GNUNET_OK != |
2622 | GNUNET_BIO_read_int32 (rh, &status)) || | 2625 | GNUNET_BIO_read_int32 (rh, &status)) || |
2623 | (GNUNET_OK != | 2626 | (GNUNET_OK != |
2624 | GNUNET_BIO_read_int32 (rh, &num_pending)) ) | 2627 | GNUNET_BIO_read_int32 (rh, &num_pending)) || |
2628 | (GNUNET_OK != | ||
2629 | GNUNET_BIO_read_int32 (rh, &start_pending)) ) | ||
2625 | { | 2630 | { |
2626 | GNUNET_break (0); | 2631 | GNUNET_break (0); |
2627 | goto cleanup; | 2632 | goto cleanup; |
@@ -2692,6 +2697,9 @@ deserialize_download (struct GNUNET_FS_Handle *h, | |||
2692 | signal_download_resume (dc); | 2697 | signal_download_resume (dc); |
2693 | } | 2698 | } |
2694 | GNUNET_free (uris); | 2699 | GNUNET_free (uris); |
2700 | if (start_pending) | ||
2701 | dc->start_task | ||
2702 | = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); | ||
2695 | return; | 2703 | return; |
2696 | cleanup: | 2704 | cleanup: |
2697 | GNUNET_free_non_null (uris); | 2705 | GNUNET_free_non_null (uris); |
diff --git a/src/fs/fs.h b/src/fs/fs.h index ccd949c59..6e6d0dd8e 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h | |||
@@ -1,4 +1,3 @@ | |||
1 | |||
2 | /* | 1 | /* |
3 | This file is part of GNUnet. | 2 | This file is part of GNUnet. |
4 | (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) | 3 | (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) |
@@ -915,6 +914,18 @@ GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, | |||
915 | 914 | ||
916 | 915 | ||
917 | /** | 916 | /** |
917 | * Task that creates the initial (top-level) download | ||
918 | * request for the file. | ||
919 | * | ||
920 | * @param cls the 'struct GNUNET_FS_DownloadContext' | ||
921 | * @param tc scheduler context | ||
922 | */ | ||
923 | void | ||
924 | GNUNET_FS_download_start_task_ (void *cls, | ||
925 | const struct GNUNET_SCHEDULER_TaskContext *tc); | ||
926 | |||
927 | |||
928 | /** | ||
918 | * Fill in all of the generic fields for | 929 | * Fill in all of the generic fields for |
919 | * an unindex event and call the callback. | 930 | * an unindex event and call the callback. |
920 | * | 931 | * |
@@ -1601,8 +1612,20 @@ struct GNUNET_FS_SearchContext | |||
1601 | * when the search is being stopped (if not | 1612 | * when the search is being stopped (if not |
1602 | * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some | 1613 | * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some |
1603 | * artificial delay when trying to reconnect to the FS service. | 1614 | * artificial delay when trying to reconnect to the FS service. |
1604 | */ | 1615 | o */ |
1605 | GNUNET_SCHEDULER_TaskIdentifier task; | 1616 | GNUNET_SCHEDULER_TaskIdentifier task; |
1617 | |||
1618 | /** | ||
1619 | * How many of the entries in the search request | ||
1620 | * map have been passed to the service so far? | ||
1621 | */ | ||
1622 | unsigned int search_request_map_offset; | ||
1623 | |||
1624 | /** | ||
1625 | * How many of the keywords in the KSK | ||
1626 | * map have been passed to the service so far? | ||
1627 | */ | ||
1628 | unsigned int keyword_offset; | ||
1606 | 1629 | ||
1607 | /** | 1630 | /** |
1608 | * Anonymity level for the search. | 1631 | * Anonymity level for the search. |
@@ -1792,6 +1815,11 @@ struct GNUNET_FS_DownloadContext | |||
1792 | GNUNET_SCHEDULER_TaskIdentifier task; | 1815 | GNUNET_SCHEDULER_TaskIdentifier task; |
1793 | 1816 | ||
1794 | /** | 1817 | /** |
1818 | * Task used to start the download. | ||
1819 | */ | ||
1820 | GNUNET_SCHEDULER_TaskIdentifier start_task; | ||
1821 | |||
1822 | /** | ||
1795 | * What was the size of the file on disk that we're downloading | 1823 | * What was the size of the file on disk that we're downloading |
1796 | * before we started? Used to detect if there is a point in | 1824 | * before we started? Used to detect if there is a point in |
1797 | * checking an existing block on disk for matching the desired | 1825 | * checking an existing block on disk for matching the desired |
@@ -1849,6 +1877,12 @@ struct GNUNET_FS_DownloadContext | |||
1849 | * data from the meta data yet? | 1877 | * data from the meta data yet? |
1850 | */ | 1878 | */ |
1851 | int tried_full_data; | 1879 | int tried_full_data; |
1880 | |||
1881 | /** | ||
1882 | * Have we tried to reconstruct an IBLOCK from disk | ||
1883 | * and failed (and should hence not try again?) | ||
1884 | */ | ||
1885 | int reconstruct_failed; | ||
1852 | }; | 1886 | }; |
1853 | 1887 | ||
1854 | 1888 | ||
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index ebe9b5cac..6943f10b5 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c | |||
@@ -24,7 +24,6 @@ | |||
24 | * | 24 | * |
25 | * TODO: | 25 | * TODO: |
26 | * - different priority for scheduling probe downloads? | 26 | * - different priority for scheduling probe downloads? |
27 | * - check if iblocks can be computed from existing blocks (can wait, hard) | ||
28 | */ | 27 | */ |
29 | #include "platform.h" | 28 | #include "platform.h" |
30 | #include "gnunet_constants.h" | 29 | #include "gnunet_constants.h" |
@@ -32,7 +31,7 @@ | |||
32 | #include "fs.h" | 31 | #include "fs.h" |
33 | #include "fs_tree.h" | 32 | #include "fs_tree.h" |
34 | 33 | ||
35 | #define DEBUG_DOWNLOAD GNUNET_YES | 34 | #define DEBUG_DOWNLOAD GNUNET_NO |
36 | 35 | ||
37 | /** | 36 | /** |
38 | * Determine if the given download (options and meta data) should cause | 37 | * Determine if the given download (options and meta data) should cause |
@@ -415,6 +414,223 @@ match_full_data (void *cls, | |||
415 | } | 414 | } |
416 | 415 | ||
417 | 416 | ||
417 | |||
418 | /** | ||
419 | * Closure for 'reconstruct_cont' and 'reconstruct_cb'. | ||
420 | */ | ||
421 | struct ReconstructContext | ||
422 | { | ||
423 | /** | ||
424 | * File handle open for the reconstruction. | ||
425 | */ | ||
426 | struct GNUNET_DISK_FileHandle *fh; | ||
427 | |||
428 | /** | ||
429 | * the download context. | ||
430 | */ | ||
431 | struct GNUNET_FS_DownloadContext *dc; | ||
432 | |||
433 | /** | ||
434 | * Tree encoder used for the reconstruction. | ||
435 | */ | ||
436 | struct GNUNET_FS_TreeEncoder *te; | ||
437 | |||
438 | /** | ||
439 | * CHK of block we are trying to reconstruct. | ||
440 | */ | ||
441 | struct ContentHashKey chk; | ||
442 | |||
443 | /** | ||
444 | * Request that was generated. | ||
445 | */ | ||
446 | struct DownloadRequest *sm; | ||
447 | |||
448 | /** | ||
449 | * Offset of block we are trying to reconstruct. | ||
450 | */ | ||
451 | uint64_t offset; | ||
452 | |||
453 | /** | ||
454 | * Depth of block we are trying to reconstruct. | ||
455 | */ | ||
456 | unsigned int depth; | ||
457 | |||
458 | }; | ||
459 | |||
460 | |||
461 | /** | ||
462 | * Continuation after a possible attempt to reconstruct | ||
463 | * the current IBlock from the existing file. | ||
464 | * | ||
465 | * @param cls the 'struct ReconstructContext' | ||
466 | * @param tc scheduler context | ||
467 | */ | ||
468 | static void | ||
469 | reconstruct_cont (void *cls, | ||
470 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
471 | { | ||
472 | struct ReconstructContext *rcc = cls; | ||
473 | |||
474 | if (rcc->te != NULL) | ||
475 | GNUNET_FS_tree_encoder_finish (rcc->te, NULL, NULL); | ||
476 | rcc->dc->reconstruct_failed = GNUNET_YES; | ||
477 | if (rcc->fh != NULL) | ||
478 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (rcc->fh)); | ||
479 | if ( (rcc->dc->th == NULL) && | ||
480 | (rcc->dc->client != NULL) ) | ||
481 | { | ||
482 | #if DEBUG_DOWNLOAD | ||
483 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
484 | "Asking for transmission to FS service\n"); | ||
485 | #endif | ||
486 | rcc->dc->th = GNUNET_CLIENT_notify_transmit_ready (rcc->dc->client, | ||
487 | sizeof (struct SearchMessage), | ||
488 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | ||
489 | GNUNET_NO, | ||
490 | &transmit_download_request, | ||
491 | rcc->dc); | ||
492 | } | ||
493 | else | ||
494 | { | ||
495 | #if DEBUG_DOWNLOAD | ||
496 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
497 | "Transmission request not issued (%p %p)\n", | ||
498 | rcc->dc->th, | ||
499 | rcc->dc->client); | ||
500 | #endif | ||
501 | } | ||
502 | GNUNET_free (rcc); | ||
503 | } | ||
504 | |||
505 | |||
506 | static void | ||
507 | get_next_block (void *cls, | ||
508 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
509 | { | ||
510 | struct ReconstructContext *rcc = cls; | ||
511 | GNUNET_FS_tree_encoder_next (rcc->te); | ||
512 | } | ||
513 | |||
514 | |||
515 | /** | ||
516 | * Function called asking for the current (encoded) | ||
517 | * block to be processed. After processing the | ||
518 | * client should either call "GNUNET_FS_tree_encode_next" | ||
519 | * or (on error) "GNUNET_FS_tree_encode_finish". | ||
520 | * | ||
521 | * This function checks if the content on disk matches | ||
522 | * the expected content based on the URI. | ||
523 | * | ||
524 | * @param cls closure | ||
525 | * @param query the query for the block (key for lookup in the datastore) | ||
526 | * @param offset offset of the block | ||
527 | * @param type type of the block (IBLOCK or DBLOCK) | ||
528 | * @param block the (encrypted) block | ||
529 | * @param block_size size of block (in bytes) | ||
530 | */ | ||
531 | static void | ||
532 | reconstruct_cb (void *cls, | ||
533 | const GNUNET_HashCode *query, | ||
534 | uint64_t offset, | ||
535 | unsigned int depth, | ||
536 | enum GNUNET_BLOCK_Type type, | ||
537 | const void *block, | ||
538 | uint16_t block_size) | ||
539 | { | ||
540 | struct ReconstructContext *rcc = cls; | ||
541 | struct ProcessResultClosure prc; | ||
542 | struct GNUNET_FS_TreeEncoder *te; | ||
543 | uint64_t off; | ||
544 | uint64_t boff; | ||
545 | uint64_t roff; | ||
546 | unsigned int i; | ||
547 | |||
548 | roff = offset / DBLOCK_SIZE; | ||
549 | for (i=rcc->dc->treedepth;i>depth;i--) | ||
550 | roff /= CHK_PER_INODE; | ||
551 | boff = roff * DBLOCK_SIZE; | ||
552 | for (i=rcc->dc->treedepth;i>depth;i--) | ||
553 | boff *= CHK_PER_INODE; | ||
554 | /* convert reading offset into IBLOCKs on-disk offset */ | ||
555 | off = compute_disk_offset (GNUNET_FS_uri_chk_get_file_size (rcc->dc->uri), | ||
556 | boff, | ||
557 | depth, | ||
558 | rcc->dc->treedepth); | ||
559 | if ( (off == rcc->offset) && | ||
560 | (depth == rcc->depth) && | ||
561 | (0 == memcmp (query, | ||
562 | &rcc->chk.query, | ||
563 | sizeof (GNUNET_HashCode))) ) | ||
564 | { | ||
565 | /* already got it! */ | ||
566 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
567 | _("Block reconstruction at offset %llu and depth %u successful\n"), | ||
568 | (unsigned long long) offset, | ||
569 | depth); | ||
570 | prc.dc = rcc->dc; | ||
571 | prc.data = block; | ||
572 | prc.size = block_size; | ||
573 | prc.type = type; | ||
574 | prc.query = rcc->chk.query; | ||
575 | prc.do_store = GNUNET_NO; | ||
576 | process_result_with_request (&prc, | ||
577 | &rcc->chk.key, | ||
578 | rcc->sm); | ||
579 | te = rcc->te; | ||
580 | rcc->te = NULL; | ||
581 | GNUNET_FS_tree_encoder_finish (te, NULL, NULL); | ||
582 | GNUNET_free (rcc); | ||
583 | return; | ||
584 | } | ||
585 | GNUNET_SCHEDULER_add_continuation (&get_next_block, | ||
586 | rcc, | ||
587 | GNUNET_SCHEDULER_REASON_PREREQ_DONE); | ||
588 | } | ||
589 | |||
590 | |||
591 | /** | ||
592 | * Function called by the tree encoder to obtain | ||
593 | * a block of plaintext data (for the lowest level | ||
594 | * of the tree). | ||
595 | * | ||
596 | * @param cls our 'struct ReconstructContext' | ||
597 | * @param offset identifies which block to get | ||
598 | * @param max (maximum) number of bytes to get; returning | ||
599 | * fewer will also cause errors | ||
600 | * @param buf where to copy the plaintext buffer | ||
601 | * @param emsg location to store an error message (on error) | ||
602 | * @return number of bytes copied to buf, 0 on error | ||
603 | */ | ||
604 | static size_t | ||
605 | fh_reader (void *cls, | ||
606 | uint64_t offset, | ||
607 | size_t max, | ||
608 | void *buf, | ||
609 | char **emsg) | ||
610 | { | ||
611 | struct ReconstructContext *rcc = cls; | ||
612 | struct GNUNET_DISK_FileHandle *fh = rcc->fh; | ||
613 | ssize_t ret; | ||
614 | |||
615 | *emsg = NULL; | ||
616 | if (offset != | ||
617 | GNUNET_DISK_file_seek (fh, | ||
618 | offset, | ||
619 | GNUNET_DISK_SEEK_SET)) | ||
620 | { | ||
621 | *emsg = GNUNET_strdup (strerror (errno)); | ||
622 | return 0; | ||
623 | } | ||
624 | ret = GNUNET_DISK_file_read (fh, buf, max); | ||
625 | if (ret < 0) | ||
626 | { | ||
627 | *emsg = GNUNET_strdup (strerror (errno)); | ||
628 | return 0; | ||
629 | } | ||
630 | return ret; | ||
631 | } | ||
632 | |||
633 | |||
418 | /** | 634 | /** |
419 | * Schedule the download of the specified block in the tree. | 635 | * Schedule the download of the specified block in the tree. |
420 | * | 636 | * |
@@ -440,8 +656,9 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc, | |||
440 | GNUNET_HashCode key; | 656 | GNUNET_HashCode key; |
441 | struct MatchDataContext mdc; | 657 | struct MatchDataContext mdc; |
442 | struct GNUNET_DISK_FileHandle *fh; | 658 | struct GNUNET_DISK_FileHandle *fh; |
659 | struct ReconstructContext *rcc; | ||
443 | 660 | ||
444 | total = GNUNET_ntohll (dc->uri->data.chk.file_length); | 661 | total = GNUNET_FS_uri_chk_get_file_size (dc->uri); |
445 | len = GNUNET_FS_tree_calculate_block_size (total, | 662 | len = GNUNET_FS_tree_calculate_block_size (total, |
446 | dc->treedepth, | 663 | dc->treedepth, |
447 | offset, | 664 | offset, |
@@ -485,12 +702,15 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc, | |||
485 | GNUNET_h2s (&chk->query)); | 702 | GNUNET_h2s (&chk->query)); |
486 | #endif | 703 | #endif |
487 | fh = NULL; | 704 | fh = NULL; |
488 | if ( (dc->old_file_size > off) && | 705 | if ( ( (dc->old_file_size > off) || |
706 | ( (depth < dc->treedepth) && | ||
707 | (dc->reconstruct_failed == GNUNET_NO) ) ) && | ||
489 | (dc->filename != NULL) ) | 708 | (dc->filename != NULL) ) |
490 | fh = GNUNET_DISK_file_open (dc->filename, | 709 | fh = GNUNET_DISK_file_open (dc->filename, |
491 | GNUNET_DISK_OPEN_READ, | 710 | GNUNET_DISK_OPEN_READ, |
492 | GNUNET_DISK_PERM_NONE); | 711 | GNUNET_DISK_PERM_NONE); |
493 | if ( (fh != NULL) && | 712 | if ( (fh != NULL) && |
713 | (dc->old_file_size > off) && | ||
494 | (off == | 714 | (off == |
495 | GNUNET_DISK_file_seek (fh, | 715 | GNUNET_DISK_file_seek (fh, |
496 | off, | 716 | off, |
@@ -517,46 +737,31 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc, | |||
517 | return; | 737 | return; |
518 | } | 738 | } |
519 | } | 739 | } |
520 | if (fh != NULL) | 740 | rcc = GNUNET_malloc (sizeof (struct ReconstructContext)); |
521 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh)); | 741 | rcc->fh = fh; |
522 | if (depth < dc->treedepth) | 742 | rcc->dc = dc; |
523 | { | 743 | rcc->sm = sm; |
524 | // FIXME: try if we could | 744 | rcc->chk = *chk; |
525 | // reconstitute this IBLOCK | 745 | rcc->offset = off; |
526 | // from the existing blocks on disk (can wait) | 746 | rcc->depth = depth; |
527 | // (read block(s), encode, compare with | 747 | if ( (depth < dc->treedepth) && |
528 | // query; if matches, simply return) | 748 | (dc->reconstruct_failed == GNUNET_NO) && |
529 | } | 749 | (fh != NULL) ) |
530 | |||
531 | if ( (dc->th == NULL) && | ||
532 | (dc->client != NULL) ) | ||
533 | { | ||
534 | #if DEBUG_DOWNLOAD | ||
535 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
536 | "Asking for transmission to FS service\n"); | ||
537 | #endif | ||
538 | dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client, | ||
539 | sizeof (struct SearchMessage), | ||
540 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | ||
541 | GNUNET_NO, | ||
542 | &transmit_download_request, | ||
543 | dc); | ||
544 | } | ||
545 | else | ||
546 | { | 750 | { |
547 | #if DEBUG_DOWNLOAD | 751 | rcc->te = GNUNET_FS_tree_encoder_create (dc->h, |
548 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 752 | dc->old_file_size, |
549 | "Transmission request not issued (%p %p)\n", | 753 | rcc, |
550 | dc->th, | 754 | fh_reader, |
551 | dc->client); | 755 | &reconstruct_cb, |
552 | #endif | 756 | NULL, |
553 | 757 | &reconstruct_cont); | |
758 | GNUNET_FS_tree_encoder_next (rcc->te); | ||
759 | return; | ||
554 | } | 760 | } |
555 | 761 | reconstruct_cont (rcc, NULL); | |
556 | } | 762 | } |
557 | 763 | ||
558 | 764 | ||
559 | |||
560 | /** | 765 | /** |
561 | * Suggest a filename based on given metadata. | 766 | * Suggest a filename based on given metadata. |
562 | * | 767 | * |
@@ -1624,6 +1829,29 @@ deactivate_fs_download (void *cls) | |||
1624 | 1829 | ||
1625 | 1830 | ||
1626 | /** | 1831 | /** |
1832 | * Task that creates the initial (top-level) download | ||
1833 | * request for the file. | ||
1834 | * | ||
1835 | * @param cls the 'struct GNUNET_FS_DownloadContext' | ||
1836 | * @param tc scheduler context | ||
1837 | */ | ||
1838 | void | ||
1839 | GNUNET_FS_download_start_task_ (void *cls, | ||
1840 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
1841 | { | ||
1842 | struct GNUNET_FS_DownloadContext *dc = cls; | ||
1843 | |||
1844 | dc->start_task = GNUNET_SCHEDULER_NO_TASK; | ||
1845 | schedule_block_download (dc, | ||
1846 | (dc->uri->type == chk) | ||
1847 | ? &dc->uri->data.chk.chk | ||
1848 | : &dc->uri->data.loc.fi.chk, | ||
1849 | 0, | ||
1850 | 1 /* 0 == CHK, 1 == top */); | ||
1851 | } | ||
1852 | |||
1853 | |||
1854 | /** | ||
1627 | * Create SUSPEND event for the given download operation | 1855 | * Create SUSPEND event for the given download operation |
1628 | * and then clean up our state (without stop signal). | 1856 | * and then clean up our state (without stop signal). |
1629 | * | 1857 | * |
@@ -1634,7 +1862,12 @@ GNUNET_FS_download_signal_suspend_ (void *cls) | |||
1634 | { | 1862 | { |
1635 | struct GNUNET_FS_DownloadContext *dc = cls; | 1863 | struct GNUNET_FS_DownloadContext *dc = cls; |
1636 | struct GNUNET_FS_ProgressInfo pi; | 1864 | struct GNUNET_FS_ProgressInfo pi; |
1637 | 1865 | ||
1866 | if (dc->start_task != GNUNET_SCHEDULER_NO_TASK) | ||
1867 | { | ||
1868 | GNUNET_SCHEDULER_cancel (dc->start_task); | ||
1869 | dc->start_task = GNUNET_SCHEDULER_NO_TASK; | ||
1870 | } | ||
1638 | if (dc->top != NULL) | 1871 | if (dc->top != NULL) |
1639 | GNUNET_FS_end_top (dc->h, dc->top); | 1872 | GNUNET_FS_end_top (dc->h, dc->top); |
1640 | while (NULL != dc->child_head) | 1873 | while (NULL != dc->child_head) |
@@ -1785,12 +2018,7 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, | |||
1785 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; | 2018 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; |
1786 | pi.value.download.specifics.start.meta = meta; | 2019 | pi.value.download.specifics.start.meta = meta; |
1787 | GNUNET_FS_download_make_status_ (&pi, dc); | 2020 | GNUNET_FS_download_make_status_ (&pi, dc); |
1788 | schedule_block_download (dc, | 2021 | dc->start_task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); |
1789 | (dc->uri->type == chk) | ||
1790 | ? &dc->uri->data.chk.chk | ||
1791 | : &dc->uri->data.loc.fi.chk, | ||
1792 | 0, | ||
1793 | 1 /* 0 == CHK, 1 == top */); | ||
1794 | GNUNET_FS_download_sync_ (dc); | 2022 | GNUNET_FS_download_sync_ (dc); |
1795 | GNUNET_FS_download_start_downloading_ (dc); | 2023 | GNUNET_FS_download_start_downloading_ (dc); |
1796 | return dc; | 2024 | return dc; |
@@ -1913,16 +2141,12 @@ GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h, | |||
1913 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; | 2141 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; |
1914 | pi.value.download.specifics.start.meta = dc->meta; | 2142 | pi.value.download.specifics.start.meta = dc->meta; |
1915 | GNUNET_FS_download_make_status_ (&pi, dc); | 2143 | GNUNET_FS_download_make_status_ (&pi, dc); |
1916 | schedule_block_download (dc, | 2144 | dc->start_task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); |
1917 | &dc->uri->data.chk.chk, | ||
1918 | 0, | ||
1919 | 1 /* 0 == CHK, 1 == top */); | ||
1920 | GNUNET_FS_download_sync_ (dc); | 2145 | GNUNET_FS_download_sync_ (dc); |
1921 | GNUNET_FS_download_start_downloading_ (dc); | 2146 | GNUNET_FS_download_start_downloading_ (dc); |
1922 | return dc; | 2147 | return dc; |
1923 | } | 2148 | } |
1924 | 2149 | ||
1925 | |||
1926 | /** | 2150 | /** |
1927 | * Start the downloading process (by entering the queue). | 2151 | * Start the downloading process (by entering the queue). |
1928 | * | 2152 | * |
@@ -1931,6 +2155,8 @@ GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h, | |||
1931 | void | 2155 | void |
1932 | GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc) | 2156 | GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc) |
1933 | { | 2157 | { |
2158 | if (dc->completed == dc->length) | ||
2159 | return; | ||
1934 | GNUNET_assert (dc->job_queue == NULL); | 2160 | GNUNET_assert (dc->job_queue == NULL); |
1935 | dc->job_queue = GNUNET_FS_queue_ (dc->h, | 2161 | dc->job_queue = GNUNET_FS_queue_ (dc->h, |
1936 | &activate_fs_download, | 2162 | &activate_fs_download, |
@@ -1955,6 +2181,11 @@ GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, | |||
1955 | 2181 | ||
1956 | if (dc->top != NULL) | 2182 | if (dc->top != NULL) |
1957 | GNUNET_FS_end_top (dc->h, dc->top); | 2183 | GNUNET_FS_end_top (dc->h, dc->top); |
2184 | if (dc->start_task != GNUNET_SCHEDULER_NO_TASK) | ||
2185 | { | ||
2186 | GNUNET_SCHEDULER_cancel (dc->start_task); | ||
2187 | dc->start_task = GNUNET_SCHEDULER_NO_TASK; | ||
2188 | } | ||
1958 | if (dc->search != NULL) | 2189 | if (dc->search != NULL) |
1959 | { | 2190 | { |
1960 | dc->search->download = NULL; | 2191 | dc->search->download = NULL; |
diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c index 5c1cd0ea2..752677f13 100644 --- a/src/fs/fs_publish.c +++ b/src/fs/fs_publish.c | |||
@@ -509,6 +509,7 @@ encode_cont (void *cls, | |||
509 | * @param cls closure | 509 | * @param cls closure |
510 | * @param query the query for the block (key for lookup in the datastore) | 510 | * @param query the query for the block (key for lookup in the datastore) |
511 | * @param offset offset of the block in the file | 511 | * @param offset offset of the block in the file |
512 | * @param depth depth of the block in the file | ||
512 | * @param type type of the block (IBLOCK or DBLOCK) | 513 | * @param type type of the block (IBLOCK or DBLOCK) |
513 | * @param block the (encrypted) block | 514 | * @param block the (encrypted) block |
514 | * @param block_size size of block (in bytes) | 515 | * @param block_size size of block (in bytes) |
@@ -517,6 +518,7 @@ static void | |||
517 | block_proc (void *cls, | 518 | block_proc (void *cls, |
518 | const GNUNET_HashCode *query, | 519 | const GNUNET_HashCode *query, |
519 | uint64_t offset, | 520 | uint64_t offset, |
521 | unsigned int depth, | ||
520 | enum GNUNET_BLOCK_Type type, | 522 | enum GNUNET_BLOCK_Type type, |
521 | const void *block, | 523 | const void *block, |
522 | uint16_t block_size) | 524 | uint16_t block_size) |
diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c index 1ffd681aa..109fc3272 100644 --- a/src/fs/fs_search.c +++ b/src/fs/fs_search.c | |||
@@ -22,10 +22,6 @@ | |||
22 | * @file fs/fs_search.c | 22 | * @file fs/fs_search.c |
23 | * @brief Helper functions for searching. | 23 | * @brief Helper functions for searching. |
24 | * @author Christian Grothoff | 24 | * @author Christian Grothoff |
25 | * | ||
26 | * TODO: | ||
27 | * - add support for pushing "already seen" information | ||
28 | * to FS service for bloomfilter (can wait) | ||
29 | */ | 25 | */ |
30 | 26 | ||
31 | #include "platform.h" | 27 | #include "platform.h" |
@@ -913,6 +909,110 @@ receive_results (void *cls, | |||
913 | 909 | ||
914 | 910 | ||
915 | /** | 911 | /** |
912 | * Schedule the transmission of the (next) search request | ||
913 | * to the service. | ||
914 | * | ||
915 | * @param sc context for the search | ||
916 | */ | ||
917 | static void | ||
918 | schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc); | ||
919 | |||
920 | |||
921 | /** | ||
922 | * Closure for 'build_result_set'. | ||
923 | */ | ||
924 | struct MessageBuilderContext | ||
925 | { | ||
926 | /** | ||
927 | * How many entries can we store to xoff. | ||
928 | */ | ||
929 | unsigned int put_cnt; | ||
930 | |||
931 | /** | ||
932 | * How many entries should we skip. | ||
933 | */ | ||
934 | unsigned int skip_cnt; | ||
935 | |||
936 | /** | ||
937 | * Where to store the keys. | ||
938 | */ | ||
939 | GNUNET_HashCode *xoff; | ||
940 | |||
941 | /** | ||
942 | * Search context we are iterating for. | ||
943 | */ | ||
944 | struct GNUNET_FS_SearchContext *sc; | ||
945 | |||
946 | /** | ||
947 | * URI the search result must match, NULL for any | ||
948 | */ | ||
949 | struct GNUNET_FS_Uri *uri; | ||
950 | }; | ||
951 | |||
952 | |||
953 | /** | ||
954 | * Iterating over the known results, pick those | ||
955 | * matching the given result range and store | ||
956 | * their keys at 'xoff'. | ||
957 | * | ||
958 | * @param cls the 'struct MessageBuilderContext' | ||
959 | * @param key key for a result | ||
960 | * @param value the search result | ||
961 | * @return GNUNET_OK to continue iterating | ||
962 | */ | ||
963 | static int | ||
964 | build_result_set (void *cls, | ||
965 | const GNUNET_HashCode *key, | ||
966 | void *value) | ||
967 | { | ||
968 | struct MessageBuilderContext *mbc = cls; | ||
969 | struct GNUNET_FS_SearchResult *sr = value; | ||
970 | |||
971 | if ( (mbc->uri != NULL) && | ||
972 | (GNUNET_YES != GNUNET_FS_uri_test_equal (mbc->uri, | ||
973 | sr->uri)) ) | ||
974 | return GNUNET_OK; | ||
975 | if (mbc->skip_cnt > 0) | ||
976 | { | ||
977 | mbc->skip_cnt--; | ||
978 | return GNUNET_OK; | ||
979 | } | ||
980 | if (mbc->put_cnt == 0) | ||
981 | return GNUNET_SYSERR; | ||
982 | mbc->sc->search_request_map_offset++; | ||
983 | mbc->xoff[--mbc->put_cnt] = *key; | ||
984 | return GNUNET_OK; | ||
985 | } | ||
986 | |||
987 | |||
988 | /** | ||
989 | * Iterating over the known results, count those | ||
990 | * matching the given result range and increment | ||
991 | * put count for each. | ||
992 | * | ||
993 | * @param cls the 'struct MessageBuilderContext' | ||
994 | * @param key key for a result | ||
995 | * @param value the search result | ||
996 | * @return GNUNET_OK to continue iterating | ||
997 | */ | ||
998 | static int | ||
999 | find_result_set (void *cls, | ||
1000 | const GNUNET_HashCode *key, | ||
1001 | void *value) | ||
1002 | { | ||
1003 | struct MessageBuilderContext *mbc = cls; | ||
1004 | struct GNUNET_FS_SearchResult *sr = value; | ||
1005 | |||
1006 | if ( (mbc->uri != NULL) && | ||
1007 | (GNUNET_YES != GNUNET_FS_uri_test_equal (mbc->uri, | ||
1008 | sr->uri)) ) | ||
1009 | return GNUNET_OK; | ||
1010 | mbc->put_cnt++; | ||
1011 | return GNUNET_OK; | ||
1012 | } | ||
1013 | |||
1014 | |||
1015 | /** | ||
916 | * We're ready to transmit the search request to the | 1016 | * We're ready to transmit the search request to the |
917 | * file-sharing service. Do it. | 1017 | * file-sharing service. Do it. |
918 | * | 1018 | * |
@@ -927,38 +1027,66 @@ transmit_search_request (void *cls, | |||
927 | void *buf) | 1027 | void *buf) |
928 | { | 1028 | { |
929 | struct GNUNET_FS_SearchContext *sc = cls; | 1029 | struct GNUNET_FS_SearchContext *sc = cls; |
1030 | struct MessageBuilderContext mbc; | ||
930 | size_t msize; | 1031 | size_t msize; |
931 | struct SearchMessage *sm; | 1032 | struct SearchMessage *sm; |
932 | unsigned int i; | ||
933 | const char *identifier; | 1033 | const char *identifier; |
934 | GNUNET_HashCode key; | 1034 | GNUNET_HashCode key; |
935 | GNUNET_HashCode idh; | 1035 | GNUNET_HashCode idh; |
1036 | unsigned int sqms; | ||
936 | 1037 | ||
937 | if (NULL == buf) | 1038 | if (NULL == buf) |
938 | { | 1039 | { |
939 | try_reconnect (sc); | 1040 | try_reconnect (sc); |
940 | return 0; | 1041 | return 0; |
941 | } | 1042 | } |
1043 | mbc.sc = sc; | ||
1044 | mbc.skip_cnt = sc->search_request_map_offset; | ||
1045 | sm = buf; | ||
1046 | sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH); | ||
1047 | mbc.xoff = (GNUNET_HashCode* ) &sm[1]; | ||
942 | if (GNUNET_FS_uri_test_ksk (sc->uri)) | 1048 | if (GNUNET_FS_uri_test_ksk (sc->uri)) |
943 | { | 1049 | { |
944 | msize = sizeof (struct SearchMessage) * sc->uri->data.ksk.keywordCount; | 1050 | msize = sizeof (struct SearchMessage); |
945 | GNUNET_assert (size >= msize); | 1051 | GNUNET_assert (size >= msize); |
946 | sm = buf; | 1052 | mbc.uri = NULL; |
947 | memset (sm, 0, msize); | 1053 | mbc.put_cnt = 0; |
948 | for (i=0;i<sc->uri->data.ksk.keywordCount;i++) | 1054 | GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, |
1055 | &find_result_set, | ||
1056 | &mbc); | ||
1057 | sqms = mbc.put_cnt; | ||
1058 | mbc.put_cnt = (size - msize) / sizeof (GNUNET_HashCode); | ||
1059 | mbc.put_cnt = GNUNET_MIN (mbc.put_cnt, | ||
1060 | sqms - mbc.skip_cnt); | ||
1061 | if (sc->search_request_map_offset < sqms) | ||
1062 | GNUNET_assert (mbc.put_cnt > 0); | ||
1063 | |||
1064 | sm->header.size = htons (msize); | ||
1065 | if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY)) | ||
1066 | sm->options = htonl (1); | ||
1067 | else | ||
1068 | sm->options = htonl (0); | ||
1069 | sm->type = htonl (GNUNET_BLOCK_TYPE_ANY); | ||
1070 | sm->anonymity_level = htonl (sc->anonymity); | ||
1071 | sm->query = sc->requests[sc->keyword_offset].query; | ||
1072 | msize += sizeof (GNUNET_HashCode) * mbc.put_cnt; | ||
1073 | GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, | ||
1074 | &build_result_set, | ||
1075 | &mbc); | ||
1076 | sm->header.size = htons (msize); | ||
1077 | if (sqms != sc->search_request_map_offset) | ||
949 | { | 1078 | { |
950 | sm[i].header.size = htons (sizeof (struct SearchMessage)); | 1079 | /* more requesting to be done... */ |
951 | sm[i].header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH); | 1080 | schedule_transmit_search_request (sc); |
952 | if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY)) | 1081 | return msize; |
953 | sm[i].options = htonl (1); | 1082 | } |
954 | else | 1083 | sc->keyword_offset++; |
955 | sm[i].options = htonl (0); | 1084 | if (sc->uri->data.ksk.keywordCount != |
956 | sm[i].type = htonl (GNUNET_BLOCK_TYPE_ANY); | 1085 | sc->keyword_offset) |
957 | sm[i].anonymity_level = htonl (sc->anonymity); | 1086 | { |
958 | sm[i].query = sc->requests[i].query; | 1087 | /* more requesting to be done... */ |
959 | /* FIXME: should transmit hash codes of all already-known results here! | 1088 | schedule_transmit_search_request (sc); |
960 | (and if they do not fit, add another message with the same | 1089 | return msize; |
961 | header and additional already-seen results!) */ | ||
962 | } | 1090 | } |
963 | } | 1091 | } |
964 | else | 1092 | else |
@@ -966,10 +1094,6 @@ transmit_search_request (void *cls, | |||
966 | GNUNET_assert (GNUNET_FS_uri_test_sks (sc->uri)); | 1094 | GNUNET_assert (GNUNET_FS_uri_test_sks (sc->uri)); |
967 | msize = sizeof (struct SearchMessage); | 1095 | msize = sizeof (struct SearchMessage); |
968 | GNUNET_assert (size >= msize); | 1096 | GNUNET_assert (size >= msize); |
969 | sm = buf; | ||
970 | memset (sm, 0, msize); | ||
971 | sm->header.size = htons (sizeof (struct SearchMessage)); | ||
972 | sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH); | ||
973 | if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY)) | 1097 | if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY)) |
974 | sm->options = htonl (1); | 1098 | sm->options = htonl (1); |
975 | else | 1099 | else |
@@ -987,10 +1111,26 @@ transmit_search_request (void *cls, | |||
987 | GNUNET_CRYPTO_hash_xor (&idh, | 1111 | GNUNET_CRYPTO_hash_xor (&idh, |
988 | &sm->target, | 1112 | &sm->target, |
989 | &sm->query); | 1113 | &sm->query); |
990 | /* FIXME: should transmit hash codes of all already-known results here! | 1114 | mbc.skip_cnt = sc->search_request_map_offset; |
991 | (and if they do not fit, add another message with the same | 1115 | mbc.put_cnt = (size - msize) / sizeof (GNUNET_HashCode); |
992 | header and additional already-seen results!) */ | 1116 | sqms = GNUNET_CONTAINER_multihashmap_size (sc->master_result_map); |
993 | } | 1117 | mbc.put_cnt = GNUNET_MIN (mbc.put_cnt, |
1118 | sqms - mbc.skip_cnt); | ||
1119 | mbc.uri = NULL; | ||
1120 | if (sc->search_request_map_offset < sqms) | ||
1121 | GNUNET_assert (mbc.put_cnt > 0); | ||
1122 | msize += sizeof (GNUNET_HashCode) * mbc.put_cnt; | ||
1123 | GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, | ||
1124 | &build_result_set, | ||
1125 | &mbc); | ||
1126 | sm->header.size = htons (msize); | ||
1127 | if (sqms != sc->search_request_map_offset) | ||
1128 | { | ||
1129 | /* more requesting to be done... */ | ||
1130 | schedule_transmit_search_request (sc); | ||
1131 | return msize; | ||
1132 | } | ||
1133 | } | ||
994 | GNUNET_CLIENT_receive (sc->client, | 1134 | GNUNET_CLIENT_receive (sc->client, |
995 | &receive_results, | 1135 | &receive_results, |
996 | sc, | 1136 | sc, |
@@ -1000,6 +1140,34 @@ transmit_search_request (void *cls, | |||
1000 | 1140 | ||
1001 | 1141 | ||
1002 | /** | 1142 | /** |
1143 | * Schedule the transmission of the (next) search request | ||
1144 | * to the service. | ||
1145 | * | ||
1146 | * @param sc context for the search | ||
1147 | */ | ||
1148 | static void | ||
1149 | schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc) | ||
1150 | { | ||
1151 | size_t size; | ||
1152 | unsigned int sqms; | ||
1153 | unsigned int fit; | ||
1154 | |||
1155 | size = sizeof (struct SearchMessage); | ||
1156 | sqms = GNUNET_CONTAINER_multihashmap_size (sc->master_result_map) - sc->search_request_map_offset; | ||
1157 | fit = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - size) / sizeof (GNUNET_HashCode); | ||
1158 | fit = GNUNET_MIN (fit, sqms); | ||
1159 | size += sizeof (GNUNET_HashCode) * fit; | ||
1160 | GNUNET_CLIENT_notify_transmit_ready (sc->client, | ||
1161 | size, | ||
1162 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | ||
1163 | GNUNET_NO, | ||
1164 | &transmit_search_request, | ||
1165 | sc); | ||
1166 | |||
1167 | } | ||
1168 | |||
1169 | |||
1170 | /** | ||
1003 | * Reconnect to the FS service and transmit | 1171 | * Reconnect to the FS service and transmit |
1004 | * our queries NOW. | 1172 | * our queries NOW. |
1005 | * | 1173 | * |
@@ -1012,8 +1180,7 @@ do_reconnect (void *cls, | |||
1012 | { | 1180 | { |
1013 | struct GNUNET_FS_SearchContext *sc = cls; | 1181 | struct GNUNET_FS_SearchContext *sc = cls; |
1014 | struct GNUNET_CLIENT_Connection *client; | 1182 | struct GNUNET_CLIENT_Connection *client; |
1015 | size_t size; | 1183 | |
1016 | |||
1017 | sc->task = GNUNET_SCHEDULER_NO_TASK; | 1184 | sc->task = GNUNET_SCHEDULER_NO_TASK; |
1018 | client = GNUNET_CLIENT_connect ("fs", | 1185 | client = GNUNET_CLIENT_connect ("fs", |
1019 | sc->h->cfg); | 1186 | sc->h->cfg); |
@@ -1023,16 +1190,9 @@ do_reconnect (void *cls, | |||
1023 | return; | 1190 | return; |
1024 | } | 1191 | } |
1025 | sc->client = client; | 1192 | sc->client = client; |
1026 | if (GNUNET_FS_uri_test_ksk (sc->uri)) | 1193 | sc->search_request_map_offset = 0; |
1027 | size = sizeof (struct SearchMessage) * sc->uri->data.ksk.keywordCount; | 1194 | sc->keyword_offset = 0; |
1028 | else | 1195 | schedule_transmit_search_request (sc); |
1029 | size = sizeof (struct SearchMessage); | ||
1030 | GNUNET_CLIENT_notify_transmit_ready (client, | ||
1031 | size, | ||
1032 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | ||
1033 | GNUNET_NO, | ||
1034 | &transmit_search_request, | ||
1035 | sc); | ||
1036 | } | 1196 | } |
1037 | 1197 | ||
1038 | 1198 | ||
@@ -1124,33 +1284,19 @@ GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc) | |||
1124 | GNUNET_HashCode hc; | 1284 | GNUNET_HashCode hc; |
1125 | struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pub; | 1285 | struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pub; |
1126 | struct GNUNET_CRYPTO_RsaPrivateKey *pk; | 1286 | struct GNUNET_CRYPTO_RsaPrivateKey *pk; |
1127 | size_t size; | ||
1128 | 1287 | ||
1129 | GNUNET_assert (NULL == sc->client); | 1288 | GNUNET_assert (NULL == sc->client); |
1130 | if (GNUNET_FS_uri_test_ksk (sc->uri)) | 1289 | if (GNUNET_FS_uri_test_ksk (sc->uri)) |
1131 | { | 1290 | { |
1132 | size = sizeof (struct SearchMessage) * sc->uri->data.ksk.keywordCount; | ||
1133 | } | ||
1134 | else | ||
1135 | { | ||
1136 | GNUNET_assert (GNUNET_FS_uri_test_sks (sc->uri)); | ||
1137 | size = sizeof (struct SearchMessage); | ||
1138 | } | ||
1139 | if (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE) | ||
1140 | { | ||
1141 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | ||
1142 | _("Too many keywords specified for a single search.")); | ||
1143 | return GNUNET_SYSERR; | ||
1144 | } | ||
1145 | if (GNUNET_FS_uri_test_ksk (sc->uri)) | ||
1146 | { | ||
1147 | GNUNET_assert (0 != sc->uri->data.ksk.keywordCount); | 1291 | GNUNET_assert (0 != sc->uri->data.ksk.keywordCount); |
1148 | sc->requests = GNUNET_malloc (sizeof (struct SearchRequestEntry) * | 1292 | sc->requests = GNUNET_malloc (sizeof (struct SearchRequestEntry) * |
1149 | sc->uri->data.ksk.keywordCount); | 1293 | sc->uri->data.ksk.keywordCount); |
1150 | for (i=0;i<sc->uri->data.ksk.keywordCount;i++) | 1294 | for (i=0;i<sc->uri->data.ksk.keywordCount;i++) |
1151 | { | 1295 | { |
1152 | keyword = &sc->uri->data.ksk.keywords[i][1]; | 1296 | keyword = &sc->uri->data.ksk.keywords[i][1]; |
1153 | GNUNET_CRYPTO_hash (keyword, strlen (keyword), &hc); | 1297 | GNUNET_CRYPTO_hash (keyword, |
1298 | strlen (keyword), | ||
1299 | &hc); | ||
1154 | pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&hc); | 1300 | pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&hc); |
1155 | GNUNET_assert (pk != NULL); | 1301 | GNUNET_assert (pk != NULL); |
1156 | GNUNET_CRYPTO_rsa_key_get_public (pk, &pub); | 1302 | GNUNET_CRYPTO_rsa_key_get_public (pk, &pub); |
@@ -1171,12 +1317,7 @@ GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc) | |||
1171 | sc->h->cfg); | 1317 | sc->h->cfg); |
1172 | if (NULL == sc->client) | 1318 | if (NULL == sc->client) |
1173 | return GNUNET_SYSERR; | 1319 | return GNUNET_SYSERR; |
1174 | GNUNET_CLIENT_notify_transmit_ready (sc->client, | 1320 | schedule_transmit_search_request (sc); |
1175 | size, | ||
1176 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | ||
1177 | GNUNET_NO, | ||
1178 | &transmit_search_request, | ||
1179 | sc); | ||
1180 | return GNUNET_OK; | 1321 | return GNUNET_OK; |
1181 | } | 1322 | } |
1182 | 1323 | ||
diff --git a/src/fs/fs_tree.c b/src/fs/fs_tree.c index b38a9c382..211714b86 100644 --- a/src/fs/fs_tree.c +++ b/src/fs/fs_tree.c | |||
@@ -109,6 +109,11 @@ struct GNUNET_FS_TreeEncoder | |||
109 | */ | 109 | */ |
110 | struct ContentHashKey *chk_tree; | 110 | struct ContentHashKey *chk_tree; |
111 | 111 | ||
112 | /** | ||
113 | * Are we currently in 'GNUNET_FS_tree_encoder_next'? | ||
114 | * Flag used to prevent recursion. | ||
115 | */ | ||
116 | int in_next; | ||
112 | }; | 117 | }; |
113 | 118 | ||
114 | 119 | ||
@@ -307,7 +312,8 @@ compute_chk_offset (unsigned int height, | |||
307 | * | 312 | * |
308 | * @param te tree encoder to use | 313 | * @param te tree encoder to use |
309 | */ | 314 | */ |
310 | void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | 315 | void |
316 | GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | ||
311 | { | 317 | { |
312 | struct ContentHashKey *mychk; | 318 | struct ContentHashKey *mychk; |
313 | const void *pt_block; | 319 | const void *pt_block; |
@@ -318,6 +324,8 @@ void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
318 | struct GNUNET_CRYPTO_AesInitializationVector iv; | 324 | struct GNUNET_CRYPTO_AesInitializationVector iv; |
319 | unsigned int off; | 325 | unsigned int off; |
320 | 326 | ||
327 | GNUNET_assert (GNUNET_NO == te->in_next); | ||
328 | te->in_next = GNUNET_YES; | ||
321 | if (te->current_depth == te->chk_tree_depth) | 329 | if (te->current_depth == te->chk_tree_depth) |
322 | { | 330 | { |
323 | pt_size = GNUNET_MIN(DBLOCK_SIZE, | 331 | pt_size = GNUNET_MIN(DBLOCK_SIZE, |
@@ -332,6 +340,7 @@ void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
332 | GNUNET_SCHEDULER_add_continuation (te->cont, | 340 | GNUNET_SCHEDULER_add_continuation (te->cont, |
333 | te->cls, | 341 | te->cls, |
334 | GNUNET_SCHEDULER_REASON_TIMEOUT); | 342 | GNUNET_SCHEDULER_REASON_TIMEOUT); |
343 | te->in_next = GNUNET_NO; | ||
335 | return; | 344 | return; |
336 | } | 345 | } |
337 | pt_block = iob; | 346 | pt_block = iob; |
@@ -352,6 +361,7 @@ void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
352 | GNUNET_SCHEDULER_add_continuation (te->cont, | 361 | GNUNET_SCHEDULER_add_continuation (te->cont, |
353 | te->cls, | 362 | te->cls, |
354 | GNUNET_SCHEDULER_REASON_PREREQ_DONE); | 363 | GNUNET_SCHEDULER_REASON_PREREQ_DONE); |
364 | te->in_next = GNUNET_NO; | ||
355 | return; | 365 | return; |
356 | } | 366 | } |
357 | off = compute_chk_offset (te->chk_tree_depth - te->current_depth, | 367 | off = compute_chk_offset (te->chk_tree_depth - te->current_depth, |
@@ -382,6 +392,7 @@ void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
382 | te->proc (te->cls, | 392 | te->proc (te->cls, |
383 | &mychk->query, | 393 | &mychk->query, |
384 | te->publish_offset, | 394 | te->publish_offset, |
395 | te->current_depth, | ||
385 | (te->current_depth == te->chk_tree_depth) | 396 | (te->current_depth == te->chk_tree_depth) |
386 | ? GNUNET_BLOCK_TYPE_FS_DBLOCK | 397 | ? GNUNET_BLOCK_TYPE_FS_DBLOCK |
387 | : GNUNET_BLOCK_TYPE_FS_IBLOCK, | 398 | : GNUNET_BLOCK_TYPE_FS_IBLOCK, |
@@ -408,6 +419,7 @@ void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
408 | else | 419 | else |
409 | te->current_depth = te->chk_tree_depth; | 420 | te->current_depth = te->chk_tree_depth; |
410 | } | 421 | } |
422 | te->in_next = GNUNET_NO; | ||
411 | } | 423 | } |
412 | 424 | ||
413 | 425 | ||
diff --git a/src/fs/fs_tree.h b/src/fs/fs_tree.h index 2cc627899..ebeb47e63 100644 --- a/src/fs/fs_tree.h +++ b/src/fs/fs_tree.h | |||
@@ -60,6 +60,7 @@ struct GNUNET_FS_TreeEncoder; | |||
60 | * @param cls closure | 60 | * @param cls closure |
61 | * @param query the query for the block (key for lookup in the datastore) | 61 | * @param query the query for the block (key for lookup in the datastore) |
62 | * @param offset offset of the block | 62 | * @param offset offset of the block |
63 | * @param depth depth of the block | ||
63 | * @param type type of the block (IBLOCK or DBLOCK) | 64 | * @param type type of the block (IBLOCK or DBLOCK) |
64 | * @param block the (encrypted) block | 65 | * @param block the (encrypted) block |
65 | * @param block_size size of block (in bytes) | 66 | * @param block_size size of block (in bytes) |
@@ -67,6 +68,7 @@ struct GNUNET_FS_TreeEncoder; | |||
67 | typedef void (*GNUNET_FS_TreeBlockProcessor)(void *cls, | 68 | typedef void (*GNUNET_FS_TreeBlockProcessor)(void *cls, |
68 | const GNUNET_HashCode *query, | 69 | const GNUNET_HashCode *query, |
69 | uint64_t offset, | 70 | uint64_t offset, |
71 | unsigned int depth, | ||
70 | enum GNUNET_BLOCK_Type type, | 72 | enum GNUNET_BLOCK_Type type, |
71 | const void *block, | 73 | const void *block, |
72 | uint16_t block_size); | 74 | uint16_t block_size); |
diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c index b3bcd48bf..aa73bc01d 100644 --- a/src/fs/fs_unindex.c +++ b/src/fs/fs_unindex.c | |||
@@ -190,6 +190,7 @@ process_cont (void *cls, | |||
190 | * @param cls closure | 190 | * @param cls closure |
191 | * @param query the query for the block (key for lookup in the datastore) | 191 | * @param query the query for the block (key for lookup in the datastore) |
192 | * @param offset offset of the block | 192 | * @param offset offset of the block |
193 | * @param depth depth of the block | ||
193 | * @param type type of the block (IBLOCK or DBLOCK) | 194 | * @param type type of the block (IBLOCK or DBLOCK) |
194 | * @param block the (encrypted) block | 195 | * @param block the (encrypted) block |
195 | * @param block_size size of block (in bytes) | 196 | * @param block_size size of block (in bytes) |
@@ -198,6 +199,7 @@ static void | |||
198 | unindex_process (void *cls, | 199 | unindex_process (void *cls, |
199 | const GNUNET_HashCode *query, | 200 | const GNUNET_HashCode *query, |
200 | uint64_t offset, | 201 | uint64_t offset, |
202 | unsigned int depth, | ||
201 | enum GNUNET_BLOCK_Type type, | 203 | enum GNUNET_BLOCK_Type type, |
202 | const void *block, | 204 | const void *block, |
203 | uint16_t block_size) | 205 | uint16_t block_size) |
diff --git a/src/fs/gnunet-download.c b/src/fs/gnunet-download.c index 99ebb1435..5301edb05 100644 --- a/src/fs/gnunet-download.c +++ b/src/fs/gnunet-download.c | |||
@@ -184,7 +184,7 @@ run (void *cls, | |||
184 | (! GNUNET_FS_uri_test_loc (uri)) ) | 184 | (! GNUNET_FS_uri_test_loc (uri)) ) |
185 | { | 185 | { |
186 | fprintf (stderr, | 186 | fprintf (stderr, |
187 | "Only CHK or LOC URIs supported.\n"); | 187 | _("Only CHK or LOC URIs supported.\n")); |
188 | ret = 1; | 188 | ret = 1; |
189 | GNUNET_FS_uri_destroy (uri); | 189 | GNUNET_FS_uri_destroy (uri); |
190 | return; | 190 | return; |
@@ -192,7 +192,7 @@ run (void *cls, | |||
192 | if (NULL == filename) | 192 | if (NULL == filename) |
193 | { | 193 | { |
194 | fprintf (stderr, | 194 | fprintf (stderr, |
195 | "Target filename must be specified.\n"); | 195 | _("Target filename must be specified.\n")); |
196 | ret = 1; | 196 | ret = 1; |
197 | GNUNET_FS_uri_destroy (uri); | 197 | GNUNET_FS_uri_destroy (uri); |
198 | return; | 198 | return; |
diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c index f02e85bba..08b01fbbf 100644 --- a/src/fs/gnunet-service-fs.c +++ b/src/fs/gnunet-service-fs.c | |||
@@ -85,6 +85,13 @@ | |||
85 | #define TRUST_FLUSH_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5) | 85 | #define TRUST_FLUSH_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5) |
86 | 86 | ||
87 | /** | 87 | /** |
88 | * How quickly do we age cover traffic? At the given | ||
89 | * time interval, remaining cover traffic counters are | ||
90 | * decremented by 1/16th. | ||
91 | */ | ||
92 | #define COVER_AGE_FREQUENCY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5) | ||
93 | |||
94 | /** | ||
88 | * How often do we at most PUT content into the DHT? | 95 | * How often do we at most PUT content into the DHT? |
89 | */ | 96 | */ |
90 | #define MAX_DHT_PUT_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5) | 97 | #define MAX_DHT_PUT_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5) |
@@ -495,9 +502,6 @@ struct UsedTargetEntry | |||
495 | }; | 502 | }; |
496 | 503 | ||
497 | 504 | ||
498 | |||
499 | |||
500 | |||
501 | /** | 505 | /** |
502 | * Doubly-linked list of messages we are performing | 506 | * Doubly-linked list of messages we are performing |
503 | * due to a pending request. | 507 | * due to a pending request. |
@@ -929,6 +933,34 @@ static struct GNUNET_LOAD_Value *datastore_put_load; | |||
929 | static struct GNUNET_LOAD_Value *rt_entry_lifetime; | 933 | static struct GNUNET_LOAD_Value *rt_entry_lifetime; |
930 | 934 | ||
931 | /** | 935 | /** |
936 | * How many query messages have we received 'recently' that | ||
937 | * have not yet been claimed as cover traffic? | ||
938 | */ | ||
939 | static unsigned int cover_query_count; | ||
940 | |||
941 | /** | ||
942 | * How many content messages have we received 'recently' that | ||
943 | * have not yet been claimed as cover traffic? | ||
944 | */ | ||
945 | static unsigned int cover_content_count; | ||
946 | |||
947 | /** | ||
948 | * ID of our task that we use to age the cover counters. | ||
949 | */ | ||
950 | static GNUNET_SCHEDULER_TaskIdentifier cover_age_task; | ||
951 | |||
952 | static void | ||
953 | age_cover_counters (void *cls, | ||
954 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
955 | { | ||
956 | cover_content_count = (cover_content_count * 15) / 16; | ||
957 | cover_query_count = (cover_query_count * 15) / 16; | ||
958 | cover_age_task = GNUNET_SCHEDULER_add_delayed (COVER_AGE_FREQUENCY, | ||
959 | &age_cover_counters, | ||
960 | NULL); | ||
961 | } | ||
962 | |||
963 | /** | ||
932 | * We've just now completed a datastore request. Update our | 964 | * We've just now completed a datastore request. Update our |
933 | * datastore load calculations. | 965 | * datastore load calculations. |
934 | * | 966 | * |
@@ -2084,6 +2116,8 @@ shutdown_task (void *cls, | |||
2084 | cfg = NULL; | 2116 | cfg = NULL; |
2085 | GNUNET_free_non_null (trustDirectory); | 2117 | GNUNET_free_non_null (trustDirectory); |
2086 | trustDirectory = NULL; | 2118 | trustDirectory = NULL; |
2119 | GNUNET_SCHEDULER_cancel (cover_age_task); | ||
2120 | cover_age_task = GNUNET_SCHEDULER_NO_TASK; | ||
2087 | } | 2121 | } |
2088 | 2122 | ||
2089 | 2123 | ||
@@ -3015,6 +3049,26 @@ forward_request_task (void *cls, | |||
3015 | &process_dht_reply, | 3049 | &process_dht_reply, |
3016 | pr); | 3050 | pr); |
3017 | } | 3051 | } |
3052 | |||
3053 | if ( (pr->anonymity_level > 1) && | ||
3054 | (cover_query_count < pr->anonymity_level - 1) ) | ||
3055 | { | ||
3056 | delay = get_processing_delay (); | ||
3057 | #if DEBUG_FS | ||
3058 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
3059 | "Not enough cover traffic to forward query `%s', will try again in %llu ms!\n", | ||
3060 | GNUNET_h2s (&pr->query), | ||
3061 | delay.rel_value); | ||
3062 | #endif | ||
3063 | pr->task = GNUNET_SCHEDULER_add_delayed (delay, | ||
3064 | &forward_request_task, | ||
3065 | pr); | ||
3066 | return; | ||
3067 | } | ||
3068 | /* consume cover traffic */ | ||
3069 | if (pr->anonymity_level > 1) | ||
3070 | cover_query_count -= pr->anonymity_level - 1; | ||
3071 | |||
3018 | /* (1) select target */ | 3072 | /* (1) select target */ |
3019 | psc.pr = pr; | 3073 | psc.pr = pr; |
3020 | psc.target_score = -DBL_MAX; | 3074 | psc.target_score = -DBL_MAX; |
@@ -3221,6 +3275,11 @@ struct ProcessReplyClosure | |||
3221 | uint32_t priority; | 3275 | uint32_t priority; |
3222 | 3276 | ||
3223 | /** | 3277 | /** |
3278 | * Anonymity requirements for this reply. | ||
3279 | */ | ||
3280 | uint32_t anonymity_level; | ||
3281 | |||
3282 | /** | ||
3224 | * Evaluation result (returned). | 3283 | * Evaluation result (returned). |
3225 | */ | 3284 | */ |
3226 | enum GNUNET_BLOCK_EvaluationResult eval; | 3285 | enum GNUNET_BLOCK_EvaluationResult eval; |
@@ -3264,6 +3323,22 @@ struct GNUNET_TIME_Relative art_delay; | |||
3264 | size_t msize; | 3323 | size_t msize; |
3265 | unsigned int i; | 3324 | unsigned int i; |
3266 | 3325 | ||
3326 | if (NULL == pr->client_request_list) | ||
3327 | { | ||
3328 | /* reply will go over the network, check for cover traffic */ | ||
3329 | if ( (prq->anonymity_level > 1) && | ||
3330 | (cover_content_count < prq->anonymity_level - 1) ) | ||
3331 | { | ||
3332 | /* insufficient cover traffic, skip */ | ||
3333 | GNUNET_STATISTICS_update (stats, | ||
3334 | gettext_noop ("# replies suppressed due to lack of cover traffic"), | ||
3335 | 1, | ||
3336 | GNUNET_NO); | ||
3337 | return GNUNET_YES; | ||
3338 | } | ||
3339 | if (prq->anonymity_level > 1) | ||
3340 | cover_content_count -= prq->anonymity_level - 1; | ||
3341 | } | ||
3267 | #if DEBUG_FS | 3342 | #if DEBUG_FS |
3268 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 3343 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
3269 | "Matched result (type %u) for query `%s' with pending request\n", | 3344 | "Matched result (type %u) for query `%s' with pending request\n", |
@@ -3603,6 +3678,7 @@ handle_p2p_put (void *cls, | |||
3603 | GNUNET_break_op (0); | 3678 | GNUNET_break_op (0); |
3604 | return GNUNET_SYSERR; | 3679 | return GNUNET_SYSERR; |
3605 | } | 3680 | } |
3681 | cover_content_count++; | ||
3606 | #if DEBUG_FS | 3682 | #if DEBUG_FS |
3607 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 3683 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
3608 | "Received result for query `%s' from peer `%4s'\n", | 3684 | "Received result for query `%s' from peer `%4s'\n", |
@@ -3624,6 +3700,7 @@ handle_p2p_put (void *cls, | |||
3624 | prq.type = type; | 3700 | prq.type = type; |
3625 | prq.expiration = expiration; | 3701 | prq.expiration = expiration; |
3626 | prq.priority = 0; | 3702 | prq.priority = 0; |
3703 | prq.anonymity_level = 1; | ||
3627 | prq.finished = GNUNET_NO; | 3704 | prq.finished = GNUNET_NO; |
3628 | prq.request_found = GNUNET_NO; | 3705 | prq.request_found = GNUNET_NO; |
3629 | GNUNET_CONTAINER_multihashmap_get_multiple (query_request_map, | 3706 | GNUNET_CONTAINER_multihashmap_get_multiple (query_request_map, |
@@ -3905,6 +3982,7 @@ process_local_reply (void *cls, | |||
3905 | prq.priority = priority; | 3982 | prq.priority = priority; |
3906 | prq.finished = GNUNET_NO; | 3983 | prq.finished = GNUNET_NO; |
3907 | prq.request_found = GNUNET_NO; | 3984 | prq.request_found = GNUNET_NO; |
3985 | prq.anonymity_level = anonymity; | ||
3908 | if ( (old_rf == 0) && | 3986 | if ( (old_rf == 0) && |
3909 | (pr->results_found == 0) ) | 3987 | (pr->results_found == 0) ) |
3910 | update_datastore_delays (pr->start_time); | 3988 | update_datastore_delays (pr->start_time); |
@@ -4100,6 +4178,7 @@ handle_p2p_get (void *cls, | |||
4100 | GNUNET_break_op (0); | 4178 | GNUNET_break_op (0); |
4101 | return GNUNET_SYSERR; | 4179 | return GNUNET_SYSERR; |
4102 | } | 4180 | } |
4181 | cover_query_count++; | ||
4103 | bm = ntohl (gm->hash_bitmap); | 4182 | bm = ntohl (gm->hash_bitmap); |
4104 | bits = 0; | 4183 | bits = 0; |
4105 | cps = GNUNET_CONTAINER_multihashmap_get (connected_peers, | 4184 | cps = GNUNET_CONTAINER_multihashmap_get (connected_peers, |
@@ -4625,6 +4704,9 @@ main_init (struct GNUNET_SERVER_Handle *server, | |||
4625 | 4704 | ||
4626 | 4705 | ||
4627 | GNUNET_SERVER_add_handlers (server, handlers); | 4706 | GNUNET_SERVER_add_handlers (server, handlers); |
4707 | cover_age_task = GNUNET_SCHEDULER_add_delayed (COVER_AGE_FREQUENCY, | ||
4708 | &age_cover_counters, | ||
4709 | NULL); | ||
4628 | GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, | 4710 | GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, |
4629 | &shutdown_task, | 4711 | &shutdown_task, |
4630 | NULL); | 4712 | NULL); |