diff options
-rw-r--r-- | TODO | 117 | ||||
-rw-r--r-- | src/core/gnunet-service-core.c | 8 | ||||
-rw-r--r-- | src/fs/fs.c | 12 | ||||
-rw-r--r-- | src/fs/fs.h | 38 | ||||
-rw-r--r-- | src/fs/fs_download.c | 333 | ||||
-rw-r--r-- | src/fs/fs_publish.c | 2 | ||||
-rw-r--r-- | src/fs/fs_search.c | 269 | ||||
-rw-r--r-- | src/fs/fs_tree.c | 14 | ||||
-rw-r--r-- | src/fs/fs_tree.h | 2 | ||||
-rw-r--r-- | src/fs/fs_unindex.c | 2 | ||||
-rw-r--r-- | src/fs/gnunet-download.c | 4 | ||||
-rw-r--r-- | src/fs/gnunet-service-fs.c | 88 |
12 files changed, 682 insertions, 207 deletions
@@ -1,37 +1,17 @@ | |||
1 | 0.9.0pre2: | 1 | 0.9.0pre2: |
2 | * BIG CORE REFACTORING: | 2 | * BIG CORE REFACTORING: |
3 | - fix transport service API (ATS!) | 3 | - fix transport plugin API (ATS!) [mw] |
4 | - fix transport plugin API (ATS!) | 4 | - fix DV [nate] |
5 | - actually transmit ATS data through core API | ||
6 | - fix FS 'latency' ATS function | ||
7 | - fix DV | ||
8 | * Integration test: | 5 | * Integration test: |
9 | - test bootstrapping via hostlist + transport/core/dht connect | 6 | - test bootstrapping via hostlist + transport/core/dht connect |
10 | |||
11 | 0.9.0pre3: | ||
12 | * Determine RC bugs and fix those (release should have no known real bugs) | ||
13 | * ARM: [CG/Safey] | ||
14 | - start_forwarding should not use 'listen_info->service_addr' for connecting since that one can be 0.0.0.0; | ||
15 | if it is, we need to use (IPv4 & IPv6) loopback [LRN] | ||
16 | - better tracking of which config changes actually need to cause process restarts by ARM. | ||
17 | - handle gnunet-arm -k in combination with auto-start magic (what is the right thing here?) | ||
18 | - discover dependencies between services | ||
19 | * CORE: | 7 | * CORE: |
20 | - provide 'reliable' P2P transmission API and queueing infrastructure | ||
21 | - Jun 27 11:51:54 core-7670 ERROR Assertion failed at gnunet-service-core.c:3616. | 8 | - Jun 27 11:51:54 core-7670 ERROR Assertion failed at gnunet-service-core.c:3616. |
22 | (transport notified us that we connected to ourselves!!!; can we still reproduce this?) | 9 | (transport notified us that we connected to ourselves!!!; can we still reproduce this?) |
23 | * DHT: [Nate] | 10 | => Also, we may want to issue a 'connect to ourselves' automatically on start of |
24 | - track paths content travels (PUT, reply-to-get) in messages, | 11 | core_api AND allow messages to self [CG/phillip] |
25 | pass to client (client API & protocol already support this!) | 12 | |
26 | * DATASTORE: | 13 | 0.9.0pre3: |
27 | - GNUNET_DATASTORE_cancel method not tested | 14 | * TRANSPORT: [MW] |
28 | * TESTING: [Nate] | ||
29 | - test basic peer re-configure | ||
30 | * TOPOLOGY: [Nate] | ||
31 | - needs more testing (especially F2F topology) & transport blacklisting | ||
32 | * TRANSPORT-TCP [MW]: | ||
33 | - should use hash map to look up sessions | ||
34 | * TRANSPORT: | ||
35 | - [#1585, #1583, #1582] transport-level disconnect (i.e. TCP) does not cause core-level | 15 | - [#1585, #1583, #1582] transport-level disconnect (i.e. TCP) does not cause core-level |
36 | disconnect in a timely fashion (waits for connection timeout); | 16 | disconnect in a timely fashion (waits for connection timeout); |
37 | need to figure a way to make it near-instant in those cases | 17 | need to figure a way to make it near-instant in those cases |
@@ -41,73 +21,44 @@ | |||
41 | likely good enough until we get ATS going; still should be tested... | 21 | likely good enough until we get ATS going; still should be tested... |
42 | => "peers connected (transport)" now instantly goes to ZERO (core statistic), | 22 | => "peers connected (transport)" now instantly goes to ZERO (core statistic), |
43 | but "established sessions" stays up... | 23 | but "established sessions" stays up... |
44 | * NAT/UPNP: [Milan] | 24 | * NAT/UPNP: [Milan / Ayush / MW] |
45 | - finalize API design | ||
46 | - [#1609] code clean up | 25 | - [#1609] code clean up |
47 | - testing | 26 | - testing |
48 | - integration with transport service | 27 | - integration with transport service: |
49 | - also do UPnP-based (external) IP detection | 28 | + test TCP |
50 | (Note: build library always, build UPnP service when dependencies like libxml2 are available) | 29 | + implement UDP, HTTP/HTTPS |
30 | * DHT: [Nate] | ||
31 | - track paths content travels (PUT, reply-to-get) in messages, | ||
32 | pass to client (client API & protocol already support this!) | ||
51 | * FS: [CG] | 33 | * FS: [CG] |
52 | - library: | ||
53 | + reconstruct IBLOCKS from DBLOCKS if possible (during download; see FIXME in fs_download) | ||
54 | + add support for pushing "already seen" search results to FS service for bloomfilter | ||
55 | + use different 'priority' for probe downloads vs. normal downloads | ||
56 | - implement multi-peer FS performance tests | 34 | - implement multi-peer FS performance tests |
57 | + insert | 35 | + insert |
58 | + download | 36 | + download |
59 | + search | 37 | + search |
60 | + unindex | 38 | * GNUNET-GTK: [CG] |
61 | - implement anonymity level > 1 | 39 | - figure out where in the GUI we should show active upload operations and allow aborts |
62 | - re-implement gnunet-auto-share | 40 | - handle events: |
63 | * GNUNET-GTK: | ||
64 | - finish publish dialog details: | ||
65 | + normalize keywords (edit subdialog) | ||
66 | - implement download by URI dialog; figure out where to display those downloads! | ||
67 | - figure out where in the GUI we should show active uploads/unindex operations and allow aborts | ||
68 | - implement unindex operation (use dialog with all indexed files for selection) | ||
69 | - events: | ||
70 | + search error | 41 | + search error |
71 | + publish error | 42 | + publish error |
72 | + unindex error | 43 | * Determine RC bugs and fix those (release should have no known real bugs) |
73 | * MONKEY: [Safey] | ||
74 | - better crash management (attach debugging support, capture and analyze | ||
75 | debug output, detect random vs. deterministic crashes) | ||
76 | - '-m EMAIL' option for alternative e-mail TO | ||
77 | - '-f FILENAME' option to write report to file instead of e-mail (for testing!) | ||
78 | 44 | ||
79 | 0.9.0: | 45 | 0.9.0: |
80 | * Determine RC bugs and fix those (release should have no known real bugs) | 46 | * new webpage: |
81 | * UTIL: | ||
82 | - only connect() sockets that are ready (select()) [Nils] | ||
83 | [On W32, we need to select after calling socket before doing connect etc.] | ||
84 | * new webpage: [BL] | ||
85 | - convert documentation pages to books | ||
86 | - update books (especially for developers) | 47 | - update books (especially for developers) |
87 | - make a NICE download page and figure out how to enable developers to publish TGZs nicely | 48 | - make a NICE download page and figure out how to enable developers to publish TGZs nicely |
88 | - port "contact" page | 49 | - port "contact" page |
89 | - add content type for "todo" items | ||
90 | * TBENCH: [MW] | ||
91 | - good to have for transport/DV evaluation! | ||
92 | * TRACEKIT: [MW] | ||
93 | - good to have for DHT evaluation! | ||
94 | * DHT: [Nate] | ||
95 | - performance tests | ||
96 | * BLOCK: | ||
97 | - more testing (KBlock, SBlock, NBlock) | ||
98 | * FS migration: | 50 | * FS migration: |
99 | - exclude content that will "soon" expire from migration? | 51 | - exclude content that will "soon" expire from migration |
100 | - exclude content with zero-anonymity from gap migration? | ||
101 | - make migration data rate & datastore IO-rate configurable | 52 | - make migration data rate & datastore IO-rate configurable |
102 | - exclude certain peers as targets (based on hash values) in each | 53 | - exclude certain peers as targets (based on hash values) in each |
103 | iteration => same peer can only be picked every n-th iteration | 54 | iteration => same peer can only be picked every n-th iteration |
104 | for the same content => fewer duplicate sending! | 55 | for the same content => fewer duplicate sending! |
105 | 56 | * big code review | |
57 | * Determine RC bugs and fix those (release should have no known real bugs) | ||
106 | 58 | ||
107 | 0.9.1: | 59 | 0.9.1: |
108 | * TRANSPORT: [MW] | 60 | * TRANSPORT: [MW] |
109 | - WiFi transport backend [DB] | 61 | - WiFi transport backend [DB] |
110 | - implement gnunet-transport (transport configurator / tester) | ||
111 | - Implement method of learning our external addresses from | 62 | - Implement method of learning our external addresses from |
112 | other peers; need some kind of threshold-based | 63 | other peers; need some kind of threshold-based |
113 | scheme, limiting both the total number of addresses that we accept | 64 | scheme, limiting both the total number of addresses that we accept |
@@ -124,8 +75,7 @@ | |||
124 | => If MiM attacker uses vetoed address, blacklist the specific IP for | 75 | => If MiM attacker uses vetoed address, blacklist the specific IP for |
125 | the presumed neighbour! | 76 | the presumed neighbour! |
126 | - need to periodically probe latency/transport cost changes & possibly switch transport | 77 | - need to periodically probe latency/transport cost changes & possibly switch transport |
127 | - should use hash map to look up Neighbours (service AND plugins!) | 78 | * DV: [Nate?] |
128 | * DV: [Nate] | ||
129 | - proper bandwidth allocation | 79 | - proper bandwidth allocation |
130 | - performance tests | 80 | - performance tests |
131 | * PEERINFO: | 81 | * PEERINFO: |
@@ -133,11 +83,12 @@ | |||
133 | (theoretically reduces overhead; bounds message queue size) | 83 | (theoretically reduces overhead; bounds message queue size) |
134 | - merge multiple iteration requests over "all" peers in the queue | 84 | - merge multiple iteration requests over "all" peers in the queue |
135 | (theoretically reduces overhead; bounds messgae queue size) | 85 | (theoretically reduces overhead; bounds messgae queue size) |
136 | * STATISTICS: [CG] | 86 | * STATISTICS: |
137 | - should use BIO instead of mmap | 87 | - should use BIO instead of mmap |
138 | * FS: [CG] | 88 | * FS: [CG] |
139 | - Remove KBlocks in gnunet-unindex (see discussion with Kenneth Almquist on gnunet-devs in 9/2009) | 89 | - Remove KBlocks in gnunet-unindex (see discussion with Kenneth Almquist on gnunet-devs in 9/2009) |
140 | - use different queue prioritization for probe-downloads vs. normal downloads | 90 | - use different queue prioritization for probe-downloads vs. normal downloads |
91 | - re-implement gnunet-auto-share | ||
141 | * UTIL: [CG] | 92 | * UTIL: [CG] |
142 | - allow limiting UNIX socket access by UID/GID | 93 | - allow limiting UNIX socket access by UID/GID |
143 | * GNUNET-GTK: [CG] | 94 | * GNUNET-GTK: [CG] |
@@ -148,6 +99,19 @@ | |||
148 | - right-clicking on NS list in search dialog should open menu that allows | 99 | - right-clicking on NS list in search dialog should open menu that allows |
149 | * viewing full meta data | 100 | * viewing full meta data |
150 | * deletion of namespace info | 101 | * deletion of namespace info |
102 | - implement unindex operation (use dialog with all indexed files for selection) | ||
103 | - finish publish dialog details: | ||
104 | + normalize keywords (edit subdialog) | ||
105 | - implement download by URI dialog; figure out where to display those downloads! | ||
106 | * ARM: [CG/Safey] | ||
107 | - better tracking of which config changes actually need to cause process restarts by ARM. | ||
108 | - handle gnunet-arm -k in combination with auto-start magic (what is the right thing here?) | ||
109 | - discover dependencies between services | ||
110 | * MONKEY: [Safey] | ||
111 | - better crash management (attach debugging support, capture and analyze | ||
112 | debug output, detect random vs. deterministic crashes) | ||
113 | - '-m EMAIL' option for alternative e-mail TO | ||
114 | - '-f FILENAME' option to write report to file instead of e-mail (for testing!) | ||
151 | 115 | ||
152 | 0.9.2: | 116 | 0.9.2: |
153 | * PEERINFO: [NN] | 117 | * PEERINFO: [NN] |
@@ -167,7 +131,7 @@ | |||
167 | - testcase would be nice | 131 | - testcase would be nice |
168 | - generic block support for DHT | 132 | - generic block support for DHT |
169 | * STATISTICS: | 133 | * STATISTICS: |
170 | - test notification-based statistics API [LT] | 134 | - test notification-based statistics API |
171 | - implement statistics GUI (=> start from gnunet-gtk by button!) | 135 | - implement statistics GUI (=> start from gnunet-gtk by button!) |
172 | * PEERINFO: [NN] | 136 | * PEERINFO: [NN] |
173 | - move peerinfo to new GUI (=> start from gnunet-gtk by button!) | 137 | - move peerinfo to new GUI (=> start from gnunet-gtk by button!) |
@@ -182,10 +146,7 @@ | |||
182 | - improved batching | 146 | - improved batching |
183 | - resource limit integration with ATS | 147 | - resource limit integration with ATS |
184 | * VPN [PT] | 148 | * VPN [PT] |
185 | - DNS hijacking | ||
186 | - DNS exit | ||
187 | - TCP entry/exit | 149 | - TCP entry/exit |
188 | - UDP entry/exit | ||
189 | - internal services | 150 | - internal services |
190 | - integration with DHT routing | 151 | - integration with DHT routing |
191 | - optimized routes (beyond DHT/DV) | 152 | - optimized routes (beyond DHT/DV) |
diff --git a/src/core/gnunet-service-core.c b/src/core/gnunet-service-core.c index 6053ec082..ac9a015d8 100644 --- a/src/core/gnunet-service-core.c +++ b/src/core/gnunet-service-core.c | |||
@@ -1015,7 +1015,7 @@ handle_peer_status_change (struct Neighbour *n) | |||
1015 | GNUNET_i2s (&n->peer)); | 1015 | GNUNET_i2s (&n->peer)); |
1016 | #endif | 1016 | #endif |
1017 | size = sizeof (struct PeerStatusNotifyMessage) + | 1017 | size = sizeof (struct PeerStatusNotifyMessage) + |
1018 | (n->ats_count+1) * sizeof (struct GNUNET_TRANSPORT_ATS_Information); | 1018 | n->ats_count * sizeof (struct GNUNET_TRANSPORT_ATS_Information); |
1019 | if (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE) | 1019 | if (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE) |
1020 | { | 1020 | { |
1021 | GNUNET_break (0); | 1021 | GNUNET_break (0); |
@@ -1024,7 +1024,7 @@ handle_peer_status_change (struct Neighbour *n) | |||
1024 | n->ats_count, | 1024 | n->ats_count, |
1025 | 0); | 1025 | 0); |
1026 | size = sizeof (struct PeerStatusNotifyMessage) + | 1026 | size = sizeof (struct PeerStatusNotifyMessage) + |
1027 | (n->ats_count+1) * sizeof (struct GNUNET_TRANSPORT_ATS_Information); | 1027 | n->ats_count * sizeof (struct GNUNET_TRANSPORT_ATS_Information); |
1028 | } | 1028 | } |
1029 | psnm = (struct PeerStatusNotifyMessage*) buf; | 1029 | psnm = (struct PeerStatusNotifyMessage*) buf; |
1030 | psnm->header.size = htons (size); | 1030 | psnm->header.size = htons (size); |
@@ -1428,7 +1428,7 @@ handle_client_iterate_peers (void *cls, | |||
1428 | n->ats_count, | 1428 | n->ats_count, |
1429 | 0); | 1429 | 0); |
1430 | size = sizeof (struct PeerStatusNotifyMessage) + | 1430 | size = sizeof (struct PeerStatusNotifyMessage) + |
1431 | (n->ats_count) * sizeof (struct GNUNET_TRANSPORT_ATS_Information); | 1431 | n->ats_count * sizeof (struct GNUNET_TRANSPORT_ATS_Information); |
1432 | } | 1432 | } |
1433 | cnm = (struct ConnectNotifyMessage*) buf; | 1433 | cnm = (struct ConnectNotifyMessage*) buf; |
1434 | cnm->header.size = htons (size); | 1434 | cnm->header.size = htons (size); |
@@ -3535,7 +3535,7 @@ handle_pong (struct Neighbour *n, | |||
3535 | n->ats_count, | 3535 | n->ats_count, |
3536 | 0); | 3536 | 0); |
3537 | size = sizeof (struct PeerStatusNotifyMessage) + | 3537 | size = sizeof (struct PeerStatusNotifyMessage) + |
3538 | (n->ats_count) * sizeof (struct GNUNET_TRANSPORT_ATS_Information); | 3538 | n->ats_count * sizeof (struct GNUNET_TRANSPORT_ATS_Information); |
3539 | } | 3539 | } |
3540 | cnm = (struct ConnectNotifyMessage*) buf; | 3540 | cnm = (struct ConnectNotifyMessage*) buf; |
3541 | cnm->header.size = htons (size); | 3541 | cnm->header.size = htons (size); |
diff --git a/src/fs/fs.c b/src/fs/fs.c index 736c3b026..c8ce4d651 100644 --- a/src/fs/fs.c +++ b/src/fs/fs.c | |||
@@ -1815,7 +1815,9 @@ GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc) | |||
1815 | (GNUNET_OK != | 1815 | (GNUNET_OK != |
1816 | GNUNET_BIO_write_int32 (wh, (uint32_t) dc->has_finished)) || | 1816 | GNUNET_BIO_write_int32 (wh, (uint32_t) dc->has_finished)) || |
1817 | (GNUNET_OK != | 1817 | (GNUNET_OK != |
1818 | GNUNET_BIO_write_int32 (wh, num_pending)) ) | 1818 | GNUNET_BIO_write_int32 (wh, num_pending)) || |
1819 | (GNUNET_OK != | ||
1820 | GNUNET_BIO_write_int32 (wh, dc->start_task != GNUNET_SCHEDULER_NO_TASK)) ) | ||
1819 | { | 1821 | { |
1820 | GNUNET_break (0); | 1822 | GNUNET_break (0); |
1821 | goto cleanup; | 1823 | goto cleanup; |
@@ -2583,6 +2585,7 @@ deserialize_download (struct GNUNET_FS_Handle *h, | |||
2583 | uint32_t options; | 2585 | uint32_t options; |
2584 | uint32_t status; | 2586 | uint32_t status; |
2585 | uint32_t num_pending; | 2587 | uint32_t num_pending; |
2588 | int32_t start_pending; | ||
2586 | 2589 | ||
2587 | uris = NULL; | 2590 | uris = NULL; |
2588 | emsg = NULL; | 2591 | emsg = NULL; |
@@ -2621,7 +2624,9 @@ deserialize_download (struct GNUNET_FS_Handle *h, | |||
2621 | (GNUNET_OK != | 2624 | (GNUNET_OK != |
2622 | GNUNET_BIO_read_int32 (rh, &status)) || | 2625 | GNUNET_BIO_read_int32 (rh, &status)) || |
2623 | (GNUNET_OK != | 2626 | (GNUNET_OK != |
2624 | GNUNET_BIO_read_int32 (rh, &num_pending)) ) | 2627 | GNUNET_BIO_read_int32 (rh, &num_pending)) || |
2628 | (GNUNET_OK != | ||
2629 | GNUNET_BIO_read_int32 (rh, &start_pending)) ) | ||
2625 | { | 2630 | { |
2626 | GNUNET_break (0); | 2631 | GNUNET_break (0); |
2627 | goto cleanup; | 2632 | goto cleanup; |
@@ -2692,6 +2697,9 @@ deserialize_download (struct GNUNET_FS_Handle *h, | |||
2692 | signal_download_resume (dc); | 2697 | signal_download_resume (dc); |
2693 | } | 2698 | } |
2694 | GNUNET_free (uris); | 2699 | GNUNET_free (uris); |
2700 | if (start_pending) | ||
2701 | dc->start_task | ||
2702 | = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); | ||
2695 | return; | 2703 | return; |
2696 | cleanup: | 2704 | cleanup: |
2697 | GNUNET_free_non_null (uris); | 2705 | GNUNET_free_non_null (uris); |
diff --git a/src/fs/fs.h b/src/fs/fs.h index ccd949c59..6e6d0dd8e 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h | |||
@@ -1,4 +1,3 @@ | |||
1 | |||
2 | /* | 1 | /* |
3 | This file is part of GNUnet. | 2 | This file is part of GNUnet. |
4 | (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) | 3 | (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) |
@@ -915,6 +914,18 @@ GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, | |||
915 | 914 | ||
916 | 915 | ||
917 | /** | 916 | /** |
917 | * Task that creates the initial (top-level) download | ||
918 | * request for the file. | ||
919 | * | ||
920 | * @param cls the 'struct GNUNET_FS_DownloadContext' | ||
921 | * @param tc scheduler context | ||
922 | */ | ||
923 | void | ||
924 | GNUNET_FS_download_start_task_ (void *cls, | ||
925 | const struct GNUNET_SCHEDULER_TaskContext *tc); | ||
926 | |||
927 | |||
928 | /** | ||
918 | * Fill in all of the generic fields for | 929 | * Fill in all of the generic fields for |
919 | * an unindex event and call the callback. | 930 | * an unindex event and call the callback. |
920 | * | 931 | * |
@@ -1601,8 +1612,20 @@ struct GNUNET_FS_SearchContext | |||
1601 | * when the search is being stopped (if not | 1612 | * when the search is being stopped (if not |
1602 | * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some | 1613 | * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some |
1603 | * artificial delay when trying to reconnect to the FS service. | 1614 | * artificial delay when trying to reconnect to the FS service. |
1604 | */ | 1615 | o */ |
1605 | GNUNET_SCHEDULER_TaskIdentifier task; | 1616 | GNUNET_SCHEDULER_TaskIdentifier task; |
1617 | |||
1618 | /** | ||
1619 | * How many of the entries in the search request | ||
1620 | * map have been passed to the service so far? | ||
1621 | */ | ||
1622 | unsigned int search_request_map_offset; | ||
1623 | |||
1624 | /** | ||
1625 | * How many of the keywords in the KSK | ||
1626 | * map have been passed to the service so far? | ||
1627 | */ | ||
1628 | unsigned int keyword_offset; | ||
1606 | 1629 | ||
1607 | /** | 1630 | /** |
1608 | * Anonymity level for the search. | 1631 | * Anonymity level for the search. |
@@ -1792,6 +1815,11 @@ struct GNUNET_FS_DownloadContext | |||
1792 | GNUNET_SCHEDULER_TaskIdentifier task; | 1815 | GNUNET_SCHEDULER_TaskIdentifier task; |
1793 | 1816 | ||
1794 | /** | 1817 | /** |
1818 | * Task used to start the download. | ||
1819 | */ | ||
1820 | GNUNET_SCHEDULER_TaskIdentifier start_task; | ||
1821 | |||
1822 | /** | ||
1795 | * What was the size of the file on disk that we're downloading | 1823 | * What was the size of the file on disk that we're downloading |
1796 | * before we started? Used to detect if there is a point in | 1824 | * before we started? Used to detect if there is a point in |
1797 | * checking an existing block on disk for matching the desired | 1825 | * checking an existing block on disk for matching the desired |
@@ -1849,6 +1877,12 @@ struct GNUNET_FS_DownloadContext | |||
1849 | * data from the meta data yet? | 1877 | * data from the meta data yet? |
1850 | */ | 1878 | */ |
1851 | int tried_full_data; | 1879 | int tried_full_data; |
1880 | |||
1881 | /** | ||
1882 | * Have we tried to reconstruct an IBLOCK from disk | ||
1883 | * and failed (and should hence not try again?) | ||
1884 | */ | ||
1885 | int reconstruct_failed; | ||
1852 | }; | 1886 | }; |
1853 | 1887 | ||
1854 | 1888 | ||
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index ebe9b5cac..6943f10b5 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c | |||
@@ -24,7 +24,6 @@ | |||
24 | * | 24 | * |
25 | * TODO: | 25 | * TODO: |
26 | * - different priority for scheduling probe downloads? | 26 | * - different priority for scheduling probe downloads? |
27 | * - check if iblocks can be computed from existing blocks (can wait, hard) | ||
28 | */ | 27 | */ |
29 | #include "platform.h" | 28 | #include "platform.h" |
30 | #include "gnunet_constants.h" | 29 | #include "gnunet_constants.h" |
@@ -32,7 +31,7 @@ | |||
32 | #include "fs.h" | 31 | #include "fs.h" |
33 | #include "fs_tree.h" | 32 | #include "fs_tree.h" |
34 | 33 | ||
35 | #define DEBUG_DOWNLOAD GNUNET_YES | 34 | #define DEBUG_DOWNLOAD GNUNET_NO |
36 | 35 | ||
37 | /** | 36 | /** |
38 | * Determine if the given download (options and meta data) should cause | 37 | * Determine if the given download (options and meta data) should cause |
@@ -415,6 +414,223 @@ match_full_data (void *cls, | |||
415 | } | 414 | } |
416 | 415 | ||
417 | 416 | ||
417 | |||
418 | /** | ||
419 | * Closure for 'reconstruct_cont' and 'reconstruct_cb'. | ||
420 | */ | ||
421 | struct ReconstructContext | ||
422 | { | ||
423 | /** | ||
424 | * File handle open for the reconstruction. | ||
425 | */ | ||
426 | struct GNUNET_DISK_FileHandle *fh; | ||
427 | |||
428 | /** | ||
429 | * the download context. | ||
430 | */ | ||
431 | struct GNUNET_FS_DownloadContext *dc; | ||
432 | |||
433 | /** | ||
434 | * Tree encoder used for the reconstruction. | ||
435 | */ | ||
436 | struct GNUNET_FS_TreeEncoder *te; | ||
437 | |||
438 | /** | ||
439 | * CHK of block we are trying to reconstruct. | ||
440 | */ | ||
441 | struct ContentHashKey chk; | ||
442 | |||
443 | /** | ||
444 | * Request that was generated. | ||
445 | */ | ||
446 | struct DownloadRequest *sm; | ||
447 | |||
448 | /** | ||
449 | * Offset of block we are trying to reconstruct. | ||
450 | */ | ||
451 | uint64_t offset; | ||
452 | |||
453 | /** | ||
454 | * Depth of block we are trying to reconstruct. | ||
455 | */ | ||
456 | unsigned int depth; | ||
457 | |||
458 | }; | ||
459 | |||
460 | |||
461 | /** | ||
462 | * Continuation after a possible attempt to reconstruct | ||
463 | * the current IBlock from the existing file. | ||
464 | * | ||
465 | * @param cls the 'struct ReconstructContext' | ||
466 | * @param tc scheduler context | ||
467 | */ | ||
468 | static void | ||
469 | reconstruct_cont (void *cls, | ||
470 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
471 | { | ||
472 | struct ReconstructContext *rcc = cls; | ||
473 | |||
474 | if (rcc->te != NULL) | ||
475 | GNUNET_FS_tree_encoder_finish (rcc->te, NULL, NULL); | ||
476 | rcc->dc->reconstruct_failed = GNUNET_YES; | ||
477 | if (rcc->fh != NULL) | ||
478 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (rcc->fh)); | ||
479 | if ( (rcc->dc->th == NULL) && | ||
480 | (rcc->dc->client != NULL) ) | ||
481 | { | ||
482 | #if DEBUG_DOWNLOAD | ||
483 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
484 | "Asking for transmission to FS service\n"); | ||
485 | #endif | ||
486 | rcc->dc->th = GNUNET_CLIENT_notify_transmit_ready (rcc->dc->client, | ||
487 | sizeof (struct SearchMessage), | ||
488 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | ||
489 | GNUNET_NO, | ||
490 | &transmit_download_request, | ||
491 | rcc->dc); | ||
492 | } | ||
493 | else | ||
494 | { | ||
495 | #if DEBUG_DOWNLOAD | ||
496 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
497 | "Transmission request not issued (%p %p)\n", | ||
498 | rcc->dc->th, | ||
499 | rcc->dc->client); | ||
500 | #endif | ||
501 | } | ||
502 | GNUNET_free (rcc); | ||
503 | } | ||
504 | |||
505 | |||
506 | static void | ||
507 | get_next_block (void *cls, | ||
508 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
509 | { | ||
510 | struct ReconstructContext *rcc = cls; | ||
511 | GNUNET_FS_tree_encoder_next (rcc->te); | ||
512 | } | ||
513 | |||
514 | |||
515 | /** | ||
516 | * Function called asking for the current (encoded) | ||
517 | * block to be processed. After processing the | ||
518 | * client should either call "GNUNET_FS_tree_encode_next" | ||
519 | * or (on error) "GNUNET_FS_tree_encode_finish". | ||
520 | * | ||
521 | * This function checks if the content on disk matches | ||
522 | * the expected content based on the URI. | ||
523 | * | ||
524 | * @param cls closure | ||
525 | * @param query the query for the block (key for lookup in the datastore) | ||
526 | * @param offset offset of the block | ||
527 | * @param type type of the block (IBLOCK or DBLOCK) | ||
528 | * @param block the (encrypted) block | ||
529 | * @param block_size size of block (in bytes) | ||
530 | */ | ||
531 | static void | ||
532 | reconstruct_cb (void *cls, | ||
533 | const GNUNET_HashCode *query, | ||
534 | uint64_t offset, | ||
535 | unsigned int depth, | ||
536 | enum GNUNET_BLOCK_Type type, | ||
537 | const void *block, | ||
538 | uint16_t block_size) | ||
539 | { | ||
540 | struct ReconstructContext *rcc = cls; | ||
541 | struct ProcessResultClosure prc; | ||
542 | struct GNUNET_FS_TreeEncoder *te; | ||
543 | uint64_t off; | ||
544 | uint64_t boff; | ||
545 | uint64_t roff; | ||
546 | unsigned int i; | ||
547 | |||
548 | roff = offset / DBLOCK_SIZE; | ||
549 | for (i=rcc->dc->treedepth;i>depth;i--) | ||
550 | roff /= CHK_PER_INODE; | ||
551 | boff = roff * DBLOCK_SIZE; | ||
552 | for (i=rcc->dc->treedepth;i>depth;i--) | ||
553 | boff *= CHK_PER_INODE; | ||
554 | /* convert reading offset into IBLOCKs on-disk offset */ | ||
555 | off = compute_disk_offset (GNUNET_FS_uri_chk_get_file_size (rcc->dc->uri), | ||
556 | boff, | ||
557 | depth, | ||
558 | rcc->dc->treedepth); | ||
559 | if ( (off == rcc->offset) && | ||
560 | (depth == rcc->depth) && | ||
561 | (0 == memcmp (query, | ||
562 | &rcc->chk.query, | ||
563 | sizeof (GNUNET_HashCode))) ) | ||
564 | { | ||
565 | /* already got it! */ | ||
566 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
567 | _("Block reconstruction at offset %llu and depth %u successful\n"), | ||
568 | (unsigned long long) offset, | ||
569 | depth); | ||
570 | prc.dc = rcc->dc; | ||
571 | prc.data = block; | ||
572 | prc.size = block_size; | ||
573 | prc.type = type; | ||
574 | prc.query = rcc->chk.query; | ||
575 | prc.do_store = GNUNET_NO; | ||
576 | process_result_with_request (&prc, | ||
577 | &rcc->chk.key, | ||
578 | rcc->sm); | ||
579 | te = rcc->te; | ||
580 | rcc->te = NULL; | ||
581 | GNUNET_FS_tree_encoder_finish (te, NULL, NULL); | ||
582 | GNUNET_free (rcc); | ||
583 | return; | ||
584 | } | ||
585 | GNUNET_SCHEDULER_add_continuation (&get_next_block, | ||
586 | rcc, | ||
587 | GNUNET_SCHEDULER_REASON_PREREQ_DONE); | ||
588 | } | ||
589 | |||
590 | |||
591 | /** | ||
592 | * Function called by the tree encoder to obtain | ||
593 | * a block of plaintext data (for the lowest level | ||
594 | * of the tree). | ||
595 | * | ||
596 | * @param cls our 'struct ReconstructContext' | ||
597 | * @param offset identifies which block to get | ||
598 | * @param max (maximum) number of bytes to get; returning | ||
599 | * fewer will also cause errors | ||
600 | * @param buf where to copy the plaintext buffer | ||
601 | * @param emsg location to store an error message (on error) | ||
602 | * @return number of bytes copied to buf, 0 on error | ||
603 | */ | ||
604 | static size_t | ||
605 | fh_reader (void *cls, | ||
606 | uint64_t offset, | ||
607 | size_t max, | ||
608 | void *buf, | ||
609 | char **emsg) | ||
610 | { | ||
611 | struct ReconstructContext *rcc = cls; | ||
612 | struct GNUNET_DISK_FileHandle *fh = rcc->fh; | ||
613 | ssize_t ret; | ||
614 | |||
615 | *emsg = NULL; | ||
616 | if (offset != | ||
617 | GNUNET_DISK_file_seek (fh, | ||
618 | offset, | ||
619 | GNUNET_DISK_SEEK_SET)) | ||
620 | { | ||
621 | *emsg = GNUNET_strdup (strerror (errno)); | ||
622 | return 0; | ||
623 | } | ||
624 | ret = GNUNET_DISK_file_read (fh, buf, max); | ||
625 | if (ret < 0) | ||
626 | { | ||
627 | *emsg = GNUNET_strdup (strerror (errno)); | ||
628 | return 0; | ||
629 | } | ||
630 | return ret; | ||
631 | } | ||
632 | |||
633 | |||
418 | /** | 634 | /** |
419 | * Schedule the download of the specified block in the tree. | 635 | * Schedule the download of the specified block in the tree. |
420 | * | 636 | * |
@@ -440,8 +656,9 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc, | |||
440 | GNUNET_HashCode key; | 656 | GNUNET_HashCode key; |
441 | struct MatchDataContext mdc; | 657 | struct MatchDataContext mdc; |
442 | struct GNUNET_DISK_FileHandle *fh; | 658 | struct GNUNET_DISK_FileHandle *fh; |
659 | struct ReconstructContext *rcc; | ||
443 | 660 | ||
444 | total = GNUNET_ntohll (dc->uri->data.chk.file_length); | 661 | total = GNUNET_FS_uri_chk_get_file_size (dc->uri); |
445 | len = GNUNET_FS_tree_calculate_block_size (total, | 662 | len = GNUNET_FS_tree_calculate_block_size (total, |
446 | dc->treedepth, | 663 | dc->treedepth, |
447 | offset, | 664 | offset, |
@@ -485,12 +702,15 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc, | |||
485 | GNUNET_h2s (&chk->query)); | 702 | GNUNET_h2s (&chk->query)); |
486 | #endif | 703 | #endif |
487 | fh = NULL; | 704 | fh = NULL; |
488 | if ( (dc->old_file_size > off) && | 705 | if ( ( (dc->old_file_size > off) || |
706 | ( (depth < dc->treedepth) && | ||
707 | (dc->reconstruct_failed == GNUNET_NO) ) ) && | ||
489 | (dc->filename != NULL) ) | 708 | (dc->filename != NULL) ) |
490 | fh = GNUNET_DISK_file_open (dc->filename, | 709 | fh = GNUNET_DISK_file_open (dc->filename, |
491 | GNUNET_DISK_OPEN_READ, | 710 | GNUNET_DISK_OPEN_READ, |
492 | GNUNET_DISK_PERM_NONE); | 711 | GNUNET_DISK_PERM_NONE); |
493 | if ( (fh != NULL) && | 712 | if ( (fh != NULL) && |
713 | (dc->old_file_size > off) && | ||
494 | (off == | 714 | (off == |
495 | GNUNET_DISK_file_seek (fh, | 715 | GNUNET_DISK_file_seek (fh, |
496 | off, | 716 | off, |
@@ -517,46 +737,31 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc, | |||
517 | return; | 737 | return; |
518 | } | 738 | } |
519 | } | 739 | } |
520 | if (fh != NULL) | 740 | rcc = GNUNET_malloc (sizeof (struct ReconstructContext)); |
521 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh)); | 741 | rcc->fh = fh; |
522 | if (depth < dc->treedepth) | 742 | rcc->dc = dc; |
523 | { | 743 | rcc->sm = sm; |
524 | // FIXME: try if we could | 744 | rcc->chk = *chk; |
525 | // reconstitute this IBLOCK | 745 | rcc->offset = off; |
526 | // from the existing blocks on disk (can wait) | 746 | rcc->depth = depth; |
527 | // (read block(s), encode, compare with | 747 | if ( (depth < dc->treedepth) && |
528 | // query; if matches, simply return) | 748 | (dc->reconstruct_failed == GNUNET_NO) && |
529 | } | 749 | (fh != NULL) ) |
530 | |||
531 | if ( (dc->th == NULL) && | ||
532 | (dc->client != NULL) ) | ||
533 | { | ||
534 | #if DEBUG_DOWNLOAD | ||
535 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
536 | "Asking for transmission to FS service\n"); | ||
537 | #endif | ||
538 | dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client, | ||
539 | sizeof (struct SearchMessage), | ||
540 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | ||
541 | GNUNET_NO, | ||
542 | &transmit_download_request, | ||
543 | dc); | ||
544 | } | ||
545 | else | ||
546 | { | 750 | { |
547 | #if DEBUG_DOWNLOAD | 751 | rcc->te = GNUNET_FS_tree_encoder_create (dc->h, |
548 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 752 | dc->old_file_size, |
549 | "Transmission request not issued (%p %p)\n", | 753 | rcc, |
550 | dc->th, | 754 | fh_reader, |
551 | dc->client); | 755 | &reconstruct_cb, |
552 | #endif | 756 | NULL, |
553 | 757 | &reconstruct_cont); | |
758 | GNUNET_FS_tree_encoder_next (rcc->te); | ||
759 | return; | ||
554 | } | 760 | } |
555 | 761 | reconstruct_cont (rcc, NULL); | |
556 | } | 762 | } |
557 | 763 | ||
558 | 764 | ||
559 | |||
560 | /** | 765 | /** |
561 | * Suggest a filename based on given metadata. | 766 | * Suggest a filename based on given metadata. |
562 | * | 767 | * |
@@ -1624,6 +1829,29 @@ deactivate_fs_download (void *cls) | |||
1624 | 1829 | ||
1625 | 1830 | ||
1626 | /** | 1831 | /** |
1832 | * Task that creates the initial (top-level) download | ||
1833 | * request for the file. | ||
1834 | * | ||
1835 | * @param cls the 'struct GNUNET_FS_DownloadContext' | ||
1836 | * @param tc scheduler context | ||
1837 | */ | ||
1838 | void | ||
1839 | GNUNET_FS_download_start_task_ (void *cls, | ||
1840 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
1841 | { | ||
1842 | struct GNUNET_FS_DownloadContext *dc = cls; | ||
1843 | |||
1844 | dc->start_task = GNUNET_SCHEDULER_NO_TASK; | ||
1845 | schedule_block_download (dc, | ||
1846 | (dc->uri->type == chk) | ||
1847 | ? &dc->uri->data.chk.chk | ||
1848 | : &dc->uri->data.loc.fi.chk, | ||
1849 | 0, | ||
1850 | 1 /* 0 == CHK, 1 == top */); | ||
1851 | } | ||
1852 | |||
1853 | |||
1854 | /** | ||
1627 | * Create SUSPEND event for the given download operation | 1855 | * Create SUSPEND event for the given download operation |
1628 | * and then clean up our state (without stop signal). | 1856 | * and then clean up our state (without stop signal). |
1629 | * | 1857 | * |
@@ -1634,7 +1862,12 @@ GNUNET_FS_download_signal_suspend_ (void *cls) | |||
1634 | { | 1862 | { |
1635 | struct GNUNET_FS_DownloadContext *dc = cls; | 1863 | struct GNUNET_FS_DownloadContext *dc = cls; |
1636 | struct GNUNET_FS_ProgressInfo pi; | 1864 | struct GNUNET_FS_ProgressInfo pi; |
1637 | 1865 | ||
1866 | if (dc->start_task != GNUNET_SCHEDULER_NO_TASK) | ||
1867 | { | ||
1868 | GNUNET_SCHEDULER_cancel (dc->start_task); | ||
1869 | dc->start_task = GNUNET_SCHEDULER_NO_TASK; | ||
1870 | } | ||
1638 | if (dc->top != NULL) | 1871 | if (dc->top != NULL) |
1639 | GNUNET_FS_end_top (dc->h, dc->top); | 1872 | GNUNET_FS_end_top (dc->h, dc->top); |
1640 | while (NULL != dc->child_head) | 1873 | while (NULL != dc->child_head) |
@@ -1785,12 +2018,7 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, | |||
1785 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; | 2018 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; |
1786 | pi.value.download.specifics.start.meta = meta; | 2019 | pi.value.download.specifics.start.meta = meta; |
1787 | GNUNET_FS_download_make_status_ (&pi, dc); | 2020 | GNUNET_FS_download_make_status_ (&pi, dc); |
1788 | schedule_block_download (dc, | 2021 | dc->start_task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); |
1789 | (dc->uri->type == chk) | ||
1790 | ? &dc->uri->data.chk.chk | ||
1791 | : &dc->uri->data.loc.fi.chk, | ||
1792 | 0, | ||
1793 | 1 /* 0 == CHK, 1 == top */); | ||
1794 | GNUNET_FS_download_sync_ (dc); | 2022 | GNUNET_FS_download_sync_ (dc); |
1795 | GNUNET_FS_download_start_downloading_ (dc); | 2023 | GNUNET_FS_download_start_downloading_ (dc); |
1796 | return dc; | 2024 | return dc; |
@@ -1913,16 +2141,12 @@ GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h, | |||
1913 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; | 2141 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; |
1914 | pi.value.download.specifics.start.meta = dc->meta; | 2142 | pi.value.download.specifics.start.meta = dc->meta; |
1915 | GNUNET_FS_download_make_status_ (&pi, dc); | 2143 | GNUNET_FS_download_make_status_ (&pi, dc); |
1916 | schedule_block_download (dc, | 2144 | dc->start_task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); |
1917 | &dc->uri->data.chk.chk, | ||
1918 | 0, | ||
1919 | 1 /* 0 == CHK, 1 == top */); | ||
1920 | GNUNET_FS_download_sync_ (dc); | 2145 | GNUNET_FS_download_sync_ (dc); |
1921 | GNUNET_FS_download_start_downloading_ (dc); | 2146 | GNUNET_FS_download_start_downloading_ (dc); |
1922 | return dc; | 2147 | return dc; |
1923 | } | 2148 | } |
1924 | 2149 | ||
1925 | |||
1926 | /** | 2150 | /** |
1927 | * Start the downloading process (by entering the queue). | 2151 | * Start the downloading process (by entering the queue). |
1928 | * | 2152 | * |
@@ -1931,6 +2155,8 @@ GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h, | |||
1931 | void | 2155 | void |
1932 | GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc) | 2156 | GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc) |
1933 | { | 2157 | { |
2158 | if (dc->completed == dc->length) | ||
2159 | return; | ||
1934 | GNUNET_assert (dc->job_queue == NULL); | 2160 | GNUNET_assert (dc->job_queue == NULL); |
1935 | dc->job_queue = GNUNET_FS_queue_ (dc->h, | 2161 | dc->job_queue = GNUNET_FS_queue_ (dc->h, |
1936 | &activate_fs_download, | 2162 | &activate_fs_download, |
@@ -1955,6 +2181,11 @@ GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, | |||
1955 | 2181 | ||
1956 | if (dc->top != NULL) | 2182 | if (dc->top != NULL) |
1957 | GNUNET_FS_end_top (dc->h, dc->top); | 2183 | GNUNET_FS_end_top (dc->h, dc->top); |
2184 | if (dc->start_task != GNUNET_SCHEDULER_NO_TASK) | ||
2185 | { | ||
2186 | GNUNET_SCHEDULER_cancel (dc->start_task); | ||
2187 | dc->start_task = GNUNET_SCHEDULER_NO_TASK; | ||
2188 | } | ||
1958 | if (dc->search != NULL) | 2189 | if (dc->search != NULL) |
1959 | { | 2190 | { |
1960 | dc->search->download = NULL; | 2191 | dc->search->download = NULL; |
diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c index 5c1cd0ea2..752677f13 100644 --- a/src/fs/fs_publish.c +++ b/src/fs/fs_publish.c | |||
@@ -509,6 +509,7 @@ encode_cont (void *cls, | |||
509 | * @param cls closure | 509 | * @param cls closure |
510 | * @param query the query for the block (key for lookup in the datastore) | 510 | * @param query the query for the block (key for lookup in the datastore) |
511 | * @param offset offset of the block in the file | 511 | * @param offset offset of the block in the file |
512 | * @param depth depth of the block in the file | ||
512 | * @param type type of the block (IBLOCK or DBLOCK) | 513 | * @param type type of the block (IBLOCK or DBLOCK) |
513 | * @param block the (encrypted) block | 514 | * @param block the (encrypted) block |
514 | * @param block_size size of block (in bytes) | 515 | * @param block_size size of block (in bytes) |
@@ -517,6 +518,7 @@ static void | |||
517 | block_proc (void *cls, | 518 | block_proc (void *cls, |
518 | const GNUNET_HashCode *query, | 519 | const GNUNET_HashCode *query, |
519 | uint64_t offset, | 520 | uint64_t offset, |
521 | unsigned int depth, | ||
520 | enum GNUNET_BLOCK_Type type, | 522 | enum GNUNET_BLOCK_Type type, |
521 | const void *block, | 523 | const void *block, |
522 | uint16_t block_size) | 524 | uint16_t block_size) |
diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c index 1ffd681aa..109fc3272 100644 --- a/src/fs/fs_search.c +++ b/src/fs/fs_search.c | |||
@@ -22,10 +22,6 @@ | |||
22 | * @file fs/fs_search.c | 22 | * @file fs/fs_search.c |
23 | * @brief Helper functions for searching. | 23 | * @brief Helper functions for searching. |
24 | * @author Christian Grothoff | 24 | * @author Christian Grothoff |
25 | * | ||
26 | * TODO: | ||
27 | * - add support for pushing "already seen" information | ||
28 | * to FS service for bloomfilter (can wait) | ||
29 | */ | 25 | */ |
30 | 26 | ||
31 | #include "platform.h" | 27 | #include "platform.h" |
@@ -913,6 +909,110 @@ receive_results (void *cls, | |||
913 | 909 | ||
914 | 910 | ||
915 | /** | 911 | /** |
912 | * Schedule the transmission of the (next) search request | ||
913 | * to the service. | ||
914 | * | ||
915 | * @param sc context for the search | ||
916 | */ | ||
917 | static void | ||
918 | schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc); | ||
919 | |||
920 | |||
921 | /** | ||
922 | * Closure for 'build_result_set'. | ||
923 | */ | ||
924 | struct MessageBuilderContext | ||
925 | { | ||
926 | /** | ||
927 | * How many entries can we store to xoff. | ||
928 | */ | ||
929 | unsigned int put_cnt; | ||
930 | |||
931 | /** | ||
932 | * How many entries should we skip. | ||
933 | */ | ||
934 | unsigned int skip_cnt; | ||
935 | |||
936 | /** | ||
937 | * Where to store the keys. | ||
938 | */ | ||
939 | GNUNET_HashCode *xoff; | ||
940 | |||
941 | /** | ||
942 | * Search context we are iterating for. | ||
943 | */ | ||
944 | struct GNUNET_FS_SearchContext *sc; | ||
945 | |||
946 | /** | ||
947 | * URI the search result must match, NULL for any | ||
948 | */ | ||
949 | struct GNUNET_FS_Uri *uri; | ||
950 | }; | ||
951 | |||
952 | |||
953 | /** | ||
954 | * Iterating over the known results, pick those | ||
955 | * matching the given result range and store | ||
956 | * their keys at 'xoff'. | ||
957 | * | ||
958 | * @param cls the 'struct MessageBuilderContext' | ||
959 | * @param key key for a result | ||
960 | * @param value the search result | ||
961 | * @return GNUNET_OK to continue iterating | ||
962 | */ | ||
963 | static int | ||
964 | build_result_set (void *cls, | ||
965 | const GNUNET_HashCode *key, | ||
966 | void *value) | ||
967 | { | ||
968 | struct MessageBuilderContext *mbc = cls; | ||
969 | struct GNUNET_FS_SearchResult *sr = value; | ||
970 | |||
971 | if ( (mbc->uri != NULL) && | ||
972 | (GNUNET_YES != GNUNET_FS_uri_test_equal (mbc->uri, | ||
973 | sr->uri)) ) | ||
974 | return GNUNET_OK; | ||
975 | if (mbc->skip_cnt > 0) | ||
976 | { | ||
977 | mbc->skip_cnt--; | ||
978 | return GNUNET_OK; | ||
979 | } | ||
980 | if (mbc->put_cnt == 0) | ||
981 | return GNUNET_SYSERR; | ||
982 | mbc->sc->search_request_map_offset++; | ||
983 | mbc->xoff[--mbc->put_cnt] = *key; | ||
984 | return GNUNET_OK; | ||
985 | } | ||
986 | |||
987 | |||
988 | /** | ||
989 | * Iterating over the known results, count those | ||
990 | * matching the given result range and increment | ||
991 | * put count for each. | ||
992 | * | ||
993 | * @param cls the 'struct MessageBuilderContext' | ||
994 | * @param key key for a result | ||
995 | * @param value the search result | ||
996 | * @return GNUNET_OK to continue iterating | ||
997 | */ | ||
998 | static int | ||
999 | find_result_set (void *cls, | ||
1000 | const GNUNET_HashCode *key, | ||
1001 | void *value) | ||
1002 | { | ||
1003 | struct MessageBuilderContext *mbc = cls; | ||
1004 | struct GNUNET_FS_SearchResult *sr = value; | ||
1005 | |||
1006 | if ( (mbc->uri != NULL) && | ||
1007 | (GNUNET_YES != GNUNET_FS_uri_test_equal (mbc->uri, | ||
1008 | sr->uri)) ) | ||
1009 | return GNUNET_OK; | ||
1010 | mbc->put_cnt++; | ||
1011 | return GNUNET_OK; | ||
1012 | } | ||
1013 | |||
1014 | |||
1015 | /** | ||
916 | * We're ready to transmit the search request to the | 1016 | * We're ready to transmit the search request to the |
917 | * file-sharing service. Do it. | 1017 | * file-sharing service. Do it. |
918 | * | 1018 | * |
@@ -927,38 +1027,66 @@ transmit_search_request (void *cls, | |||
927 | void *buf) | 1027 | void *buf) |
928 | { | 1028 | { |
929 | struct GNUNET_FS_SearchContext *sc = cls; | 1029 | struct GNUNET_FS_SearchContext *sc = cls; |
1030 | struct MessageBuilderContext mbc; | ||
930 | size_t msize; | 1031 | size_t msize; |
931 | struct SearchMessage *sm; | 1032 | struct SearchMessage *sm; |
932 | unsigned int i; | ||
933 | const char *identifier; | 1033 | const char *identifier; |
934 | GNUNET_HashCode key; | 1034 | GNUNET_HashCode key; |
935 | GNUNET_HashCode idh; | 1035 | GNUNET_HashCode idh; |
1036 | unsigned int sqms; | ||
936 | 1037 | ||
937 | if (NULL == buf) | 1038 | if (NULL == buf) |
938 | { | 1039 | { |
939 | try_reconnect (sc); | 1040 | try_reconnect (sc); |
940 | return 0; | 1041 | return 0; |
941 | } | 1042 | } |
1043 | mbc.sc = sc; | ||
1044 | mbc.skip_cnt = sc->search_request_map_offset; | ||
1045 | sm = buf; | ||
1046 | sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH); | ||
1047 | mbc.xoff = (GNUNET_HashCode* ) &sm[1]; | ||
942 | if (GNUNET_FS_uri_test_ksk (sc->uri)) | 1048 | if (GNUNET_FS_uri_test_ksk (sc->uri)) |
943 | { | 1049 | { |
944 | msize = sizeof (struct SearchMessage) * sc->uri->data.ksk.keywordCount; | 1050 | msize = sizeof (struct SearchMessage); |
945 | GNUNET_assert (size >= msize); | 1051 | GNUNET_assert (size >= msize); |
946 | sm = buf; | 1052 | mbc.uri = NULL; |
947 | memset (sm, 0, msize); | 1053 | mbc.put_cnt = 0; |
948 | for (i=0;i<sc->uri->data.ksk.keywordCount;i++) | 1054 | GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, |
1055 | &find_result_set, | ||
1056 | &mbc); | ||
1057 | sqms = mbc.put_cnt; | ||
1058 | mbc.put_cnt = (size - msize) / sizeof (GNUNET_HashCode); | ||
1059 | mbc.put_cnt = GNUNET_MIN (mbc.put_cnt, | ||
1060 | sqms - mbc.skip_cnt); | ||
1061 | if (sc->search_request_map_offset < sqms) | ||
1062 | GNUNET_assert (mbc.put_cnt > 0); | ||
1063 | |||
1064 | sm->header.size = htons (msize); | ||
1065 | if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY)) | ||
1066 | sm->options = htonl (1); | ||
1067 | else | ||
1068 | sm->options = htonl (0); | ||
1069 | sm->type = htonl (GNUNET_BLOCK_TYPE_ANY); | ||
1070 | sm->anonymity_level = htonl (sc->anonymity); | ||
1071 | sm->query = sc->requests[sc->keyword_offset].query; | ||
1072 | msize += sizeof (GNUNET_HashCode) * mbc.put_cnt; | ||
1073 | GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, | ||
1074 | &build_result_set, | ||
1075 | &mbc); | ||
1076 | sm->header.size = htons (msize); | ||
1077 | if (sqms != sc->search_request_map_offset) | ||
949 | { | 1078 | { |
950 | sm[i].header.size = htons (sizeof (struct SearchMessage)); | 1079 | /* more requesting to be done... */ |
951 | sm[i].header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH); | 1080 | schedule_transmit_search_request (sc); |
952 | if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY)) | 1081 | return msize; |
953 | sm[i].options = htonl (1); | 1082 | } |
954 | else | 1083 | sc->keyword_offset++; |
955 | sm[i].options = htonl (0); | 1084 | if (sc->uri->data.ksk.keywordCount != |
956 | sm[i].type = htonl (GNUNET_BLOCK_TYPE_ANY); | 1085 | sc->keyword_offset) |
957 | sm[i].anonymity_level = htonl (sc->anonymity); | 1086 | { |
958 | sm[i].query = sc->requests[i].query; | 1087 | /* more requesting to be done... */ |
959 | /* FIXME: should transmit hash codes of all already-known results here! | 1088 | schedule_transmit_search_request (sc); |
960 | (and if they do not fit, add another message with the same | 1089 | return msize; |
961 | header and additional already-seen results!) */ | ||
962 | } | 1090 | } |
963 | } | 1091 | } |
964 | else | 1092 | else |
@@ -966,10 +1094,6 @@ transmit_search_request (void *cls, | |||
966 | GNUNET_assert (GNUNET_FS_uri_test_sks (sc->uri)); | 1094 | GNUNET_assert (GNUNET_FS_uri_test_sks (sc->uri)); |
967 | msize = sizeof (struct SearchMessage); | 1095 | msize = sizeof (struct SearchMessage); |
968 | GNUNET_assert (size >= msize); | 1096 | GNUNET_assert (size >= msize); |
969 | sm = buf; | ||
970 | memset (sm, 0, msize); | ||
971 | sm->header.size = htons (sizeof (struct SearchMessage)); | ||
972 | sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH); | ||
973 | if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY)) | 1097 | if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY)) |
974 | sm->options = htonl (1); | 1098 | sm->options = htonl (1); |
975 | else | 1099 | else |
@@ -987,10 +1111,26 @@ transmit_search_request (void *cls, | |||
987 | GNUNET_CRYPTO_hash_xor (&idh, | 1111 | GNUNET_CRYPTO_hash_xor (&idh, |
988 | &sm->target, | 1112 | &sm->target, |
989 | &sm->query); | 1113 | &sm->query); |
990 | /* FIXME: should transmit hash codes of all already-known results here! | 1114 | mbc.skip_cnt = sc->search_request_map_offset; |
991 | (and if they do not fit, add another message with the same | 1115 | mbc.put_cnt = (size - msize) / sizeof (GNUNET_HashCode); |
992 | header and additional already-seen results!) */ | 1116 | sqms = GNUNET_CONTAINER_multihashmap_size (sc->master_result_map); |
993 | } | 1117 | mbc.put_cnt = GNUNET_MIN (mbc.put_cnt, |
1118 | sqms - mbc.skip_cnt); | ||
1119 | mbc.uri = NULL; | ||
1120 | if (sc->search_request_map_offset < sqms) | ||
1121 | GNUNET_assert (mbc.put_cnt > 0); | ||
1122 | msize += sizeof (GNUNET_HashCode) * mbc.put_cnt; | ||
1123 | GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, | ||
1124 | &build_result_set, | ||
1125 | &mbc); | ||
1126 | sm->header.size = htons (msize); | ||
1127 | if (sqms != sc->search_request_map_offset) | ||
1128 | { | ||
1129 | /* more requesting to be done... */ | ||
1130 | schedule_transmit_search_request (sc); | ||
1131 | return msize; | ||
1132 | } | ||
1133 | } | ||
994 | GNUNET_CLIENT_receive (sc->client, | 1134 | GNUNET_CLIENT_receive (sc->client, |
995 | &receive_results, | 1135 | &receive_results, |
996 | sc, | 1136 | sc, |
@@ -1000,6 +1140,34 @@ transmit_search_request (void *cls, | |||
1000 | 1140 | ||
1001 | 1141 | ||
1002 | /** | 1142 | /** |
1143 | * Schedule the transmission of the (next) search request | ||
1144 | * to the service. | ||
1145 | * | ||
1146 | * @param sc context for the search | ||
1147 | */ | ||
1148 | static void | ||
1149 | schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc) | ||
1150 | { | ||
1151 | size_t size; | ||
1152 | unsigned int sqms; | ||
1153 | unsigned int fit; | ||
1154 | |||
1155 | size = sizeof (struct SearchMessage); | ||
1156 | sqms = GNUNET_CONTAINER_multihashmap_size (sc->master_result_map) - sc->search_request_map_offset; | ||
1157 | fit = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - size) / sizeof (GNUNET_HashCode); | ||
1158 | fit = GNUNET_MIN (fit, sqms); | ||
1159 | size += sizeof (GNUNET_HashCode) * fit; | ||
1160 | GNUNET_CLIENT_notify_transmit_ready (sc->client, | ||
1161 | size, | ||
1162 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | ||
1163 | GNUNET_NO, | ||
1164 | &transmit_search_request, | ||
1165 | sc); | ||
1166 | |||
1167 | } | ||
1168 | |||
1169 | |||
1170 | /** | ||
1003 | * Reconnect to the FS service and transmit | 1171 | * Reconnect to the FS service and transmit |
1004 | * our queries NOW. | 1172 | * our queries NOW. |
1005 | * | 1173 | * |
@@ -1012,8 +1180,7 @@ do_reconnect (void *cls, | |||
1012 | { | 1180 | { |
1013 | struct GNUNET_FS_SearchContext *sc = cls; | 1181 | struct GNUNET_FS_SearchContext *sc = cls; |
1014 | struct GNUNET_CLIENT_Connection *client; | 1182 | struct GNUNET_CLIENT_Connection *client; |
1015 | size_t size; | 1183 | |
1016 | |||
1017 | sc->task = GNUNET_SCHEDULER_NO_TASK; | 1184 | sc->task = GNUNET_SCHEDULER_NO_TASK; |
1018 | client = GNUNET_CLIENT_connect ("fs", | 1185 | client = GNUNET_CLIENT_connect ("fs", |
1019 | sc->h->cfg); | 1186 | sc->h->cfg); |
@@ -1023,16 +1190,9 @@ do_reconnect (void *cls, | |||
1023 | return; | 1190 | return; |
1024 | } | 1191 | } |
1025 | sc->client = client; | 1192 | sc->client = client; |
1026 | if (GNUNET_FS_uri_test_ksk (sc->uri)) | 1193 | sc->search_request_map_offset = 0; |
1027 | size = sizeof (struct SearchMessage) * sc->uri->data.ksk.keywordCount; | 1194 | sc->keyword_offset = 0; |
1028 | else | 1195 | schedule_transmit_search_request (sc); |
1029 | size = sizeof (struct SearchMessage); | ||
1030 | GNUNET_CLIENT_notify_transmit_ready (client, | ||
1031 | size, | ||
1032 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | ||
1033 | GNUNET_NO, | ||
1034 | &transmit_search_request, | ||
1035 | sc); | ||
1036 | } | 1196 | } |
1037 | 1197 | ||
1038 | 1198 | ||
@@ -1124,33 +1284,19 @@ GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc) | |||
1124 | GNUNET_HashCode hc; | 1284 | GNUNET_HashCode hc; |
1125 | struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pub; | 1285 | struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pub; |
1126 | struct GNUNET_CRYPTO_RsaPrivateKey *pk; | 1286 | struct GNUNET_CRYPTO_RsaPrivateKey *pk; |
1127 | size_t size; | ||
1128 | 1287 | ||
1129 | GNUNET_assert (NULL == sc->client); | 1288 | GNUNET_assert (NULL == sc->client); |
1130 | if (GNUNET_FS_uri_test_ksk (sc->uri)) | 1289 | if (GNUNET_FS_uri_test_ksk (sc->uri)) |
1131 | { | 1290 | { |
1132 | size = sizeof (struct SearchMessage) * sc->uri->data.ksk.keywordCount; | ||
1133 | } | ||
1134 | else | ||
1135 | { | ||
1136 | GNUNET_assert (GNUNET_FS_uri_test_sks (sc->uri)); | ||
1137 | size = sizeof (struct SearchMessage); | ||
1138 | } | ||
1139 | if (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE) | ||
1140 | { | ||
1141 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | ||
1142 | _("Too many keywords specified for a single search.")); | ||
1143 | return GNUNET_SYSERR; | ||
1144 | } | ||
1145 | if (GNUNET_FS_uri_test_ksk (sc->uri)) | ||
1146 | { | ||
1147 | GNUNET_assert (0 != sc->uri->data.ksk.keywordCount); | 1291 | GNUNET_assert (0 != sc->uri->data.ksk.keywordCount); |
1148 | sc->requests = GNUNET_malloc (sizeof (struct SearchRequestEntry) * | 1292 | sc->requests = GNUNET_malloc (sizeof (struct SearchRequestEntry) * |
1149 | sc->uri->data.ksk.keywordCount); | 1293 | sc->uri->data.ksk.keywordCount); |
1150 | for (i=0;i<sc->uri->data.ksk.keywordCount;i++) | 1294 | for (i=0;i<sc->uri->data.ksk.keywordCount;i++) |
1151 | { | 1295 | { |
1152 | keyword = &sc->uri->data.ksk.keywords[i][1]; | 1296 | keyword = &sc->uri->data.ksk.keywords[i][1]; |
1153 | GNUNET_CRYPTO_hash (keyword, strlen (keyword), &hc); | 1297 | GNUNET_CRYPTO_hash (keyword, |
1298 | strlen (keyword), | ||
1299 | &hc); | ||
1154 | pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&hc); | 1300 | pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&hc); |
1155 | GNUNET_assert (pk != NULL); | 1301 | GNUNET_assert (pk != NULL); |
1156 | GNUNET_CRYPTO_rsa_key_get_public (pk, &pub); | 1302 | GNUNET_CRYPTO_rsa_key_get_public (pk, &pub); |
@@ -1171,12 +1317,7 @@ GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc) | |||
1171 | sc->h->cfg); | 1317 | sc->h->cfg); |
1172 | if (NULL == sc->client) | 1318 | if (NULL == sc->client) |
1173 | return GNUNET_SYSERR; | 1319 | return GNUNET_SYSERR; |
1174 | GNUNET_CLIENT_notify_transmit_ready (sc->client, | 1320 | schedule_transmit_search_request (sc); |
1175 | size, | ||
1176 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | ||
1177 | GNUNET_NO, | ||
1178 | &transmit_search_request, | ||
1179 | sc); | ||
1180 | return GNUNET_OK; | 1321 | return GNUNET_OK; |
1181 | } | 1322 | } |
1182 | 1323 | ||
diff --git a/src/fs/fs_tree.c b/src/fs/fs_tree.c index b38a9c382..211714b86 100644 --- a/src/fs/fs_tree.c +++ b/src/fs/fs_tree.c | |||
@@ -109,6 +109,11 @@ struct GNUNET_FS_TreeEncoder | |||
109 | */ | 109 | */ |
110 | struct ContentHashKey *chk_tree; | 110 | struct ContentHashKey *chk_tree; |
111 | 111 | ||
112 | /** | ||
113 | * Are we currently in 'GNUNET_FS_tree_encoder_next'? | ||
114 | * Flag used to prevent recursion. | ||
115 | */ | ||
116 | int in_next; | ||
112 | }; | 117 | }; |
113 | 118 | ||
114 | 119 | ||
@@ -307,7 +312,8 @@ compute_chk_offset (unsigned int height, | |||
307 | * | 312 | * |
308 | * @param te tree encoder to use | 313 | * @param te tree encoder to use |
309 | */ | 314 | */ |
310 | void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | 315 | void |
316 | GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | ||
311 | { | 317 | { |
312 | struct ContentHashKey *mychk; | 318 | struct ContentHashKey *mychk; |
313 | const void *pt_block; | 319 | const void *pt_block; |
@@ -318,6 +324,8 @@ void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
318 | struct GNUNET_CRYPTO_AesInitializationVector iv; | 324 | struct GNUNET_CRYPTO_AesInitializationVector iv; |
319 | unsigned int off; | 325 | unsigned int off; |
320 | 326 | ||
327 | GNUNET_assert (GNUNET_NO == te->in_next); | ||
328 | te->in_next = GNUNET_YES; | ||
321 | if (te->current_depth == te->chk_tree_depth) | 329 | if (te->current_depth == te->chk_tree_depth) |
322 | { | 330 | { |
323 | pt_size = GNUNET_MIN(DBLOCK_SIZE, | 331 | pt_size = GNUNET_MIN(DBLOCK_SIZE, |
@@ -332,6 +340,7 @@ void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
332 | GNUNET_SCHEDULER_add_continuation (te->cont, | 340 | GNUNET_SCHEDULER_add_continuation (te->cont, |
333 | te->cls, | 341 | te->cls, |
334 | GNUNET_SCHEDULER_REASON_TIMEOUT); | 342 | GNUNET_SCHEDULER_REASON_TIMEOUT); |
343 | te->in_next = GNUNET_NO; | ||
335 | return; | 344 | return; |
336 | } | 345 | } |
337 | pt_block = iob; | 346 | pt_block = iob; |
@@ -352,6 +361,7 @@ void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
352 | GNUNET_SCHEDULER_add_continuation (te->cont, | 361 | GNUNET_SCHEDULER_add_continuation (te->cont, |
353 | te->cls, | 362 | te->cls, |
354 | GNUNET_SCHEDULER_REASON_PREREQ_DONE); | 363 | GNUNET_SCHEDULER_REASON_PREREQ_DONE); |
364 | te->in_next = GNUNET_NO; | ||
355 | return; | 365 | return; |
356 | } | 366 | } |
357 | off = compute_chk_offset (te->chk_tree_depth - te->current_depth, | 367 | off = compute_chk_offset (te->chk_tree_depth - te->current_depth, |
@@ -382,6 +392,7 @@ void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
382 | te->proc (te->cls, | 392 | te->proc (te->cls, |
383 | &mychk->query, | 393 | &mychk->query, |
384 | te->publish_offset, | 394 | te->publish_offset, |
395 | te->current_depth, | ||
385 | (te->current_depth == te->chk_tree_depth) | 396 | (te->current_depth == te->chk_tree_depth) |
386 | ? GNUNET_BLOCK_TYPE_FS_DBLOCK | 397 | ? GNUNET_BLOCK_TYPE_FS_DBLOCK |
387 | : GNUNET_BLOCK_TYPE_FS_IBLOCK, | 398 | : GNUNET_BLOCK_TYPE_FS_IBLOCK, |
@@ -408,6 +419,7 @@ void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
408 | else | 419 | else |
409 | te->current_depth = te->chk_tree_depth; | 420 | te->current_depth = te->chk_tree_depth; |
410 | } | 421 | } |
422 | te->in_next = GNUNET_NO; | ||
411 | } | 423 | } |
412 | 424 | ||
413 | 425 | ||
diff --git a/src/fs/fs_tree.h b/src/fs/fs_tree.h index 2cc627899..ebeb47e63 100644 --- a/src/fs/fs_tree.h +++ b/src/fs/fs_tree.h | |||
@@ -60,6 +60,7 @@ struct GNUNET_FS_TreeEncoder; | |||
60 | * @param cls closure | 60 | * @param cls closure |
61 | * @param query the query for the block (key for lookup in the datastore) | 61 | * @param query the query for the block (key for lookup in the datastore) |
62 | * @param offset offset of the block | 62 | * @param offset offset of the block |
63 | * @param depth depth of the block | ||
63 | * @param type type of the block (IBLOCK or DBLOCK) | 64 | * @param type type of the block (IBLOCK or DBLOCK) |
64 | * @param block the (encrypted) block | 65 | * @param block the (encrypted) block |
65 | * @param block_size size of block (in bytes) | 66 | * @param block_size size of block (in bytes) |
@@ -67,6 +68,7 @@ struct GNUNET_FS_TreeEncoder; | |||
67 | typedef void (*GNUNET_FS_TreeBlockProcessor)(void *cls, | 68 | typedef void (*GNUNET_FS_TreeBlockProcessor)(void *cls, |
68 | const GNUNET_HashCode *query, | 69 | const GNUNET_HashCode *query, |
69 | uint64_t offset, | 70 | uint64_t offset, |
71 | unsigned int depth, | ||
70 | enum GNUNET_BLOCK_Type type, | 72 | enum GNUNET_BLOCK_Type type, |
71 | const void *block, | 73 | const void *block, |
72 | uint16_t block_size); | 74 | uint16_t block_size); |
diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c index b3bcd48bf..aa73bc01d 100644 --- a/src/fs/fs_unindex.c +++ b/src/fs/fs_unindex.c | |||
@@ -190,6 +190,7 @@ process_cont (void *cls, | |||
190 | * @param cls closure | 190 | * @param cls closure |
191 | * @param query the query for the block (key for lookup in the datastore) | 191 | * @param query the query for the block (key for lookup in the datastore) |
192 | * @param offset offset of the block | 192 | * @param offset offset of the block |
193 | * @param depth depth of the block | ||
193 | * @param type type of the block (IBLOCK or DBLOCK) | 194 | * @param type type of the block (IBLOCK or DBLOCK) |
194 | * @param block the (encrypted) block | 195 | * @param block the (encrypted) block |
195 | * @param block_size size of block (in bytes) | 196 | * @param block_size size of block (in bytes) |
@@ -198,6 +199,7 @@ static void | |||
198 | unindex_process (void *cls, | 199 | unindex_process (void *cls, |
199 | const GNUNET_HashCode *query, | 200 | const GNUNET_HashCode *query, |
200 | uint64_t offset, | 201 | uint64_t offset, |
202 | unsigned int depth, | ||
201 | enum GNUNET_BLOCK_Type type, | 203 | enum GNUNET_BLOCK_Type type, |
202 | const void *block, | 204 | const void *block, |
203 | uint16_t block_size) | 205 | uint16_t block_size) |
diff --git a/src/fs/gnunet-download.c b/src/fs/gnunet-download.c index 99ebb1435..5301edb05 100644 --- a/src/fs/gnunet-download.c +++ b/src/fs/gnunet-download.c | |||
@@ -184,7 +184,7 @@ run (void *cls, | |||
184 | (! GNUNET_FS_uri_test_loc (uri)) ) | 184 | (! GNUNET_FS_uri_test_loc (uri)) ) |
185 | { | 185 | { |
186 | fprintf (stderr, | 186 | fprintf (stderr, |
187 | "Only CHK or LOC URIs supported.\n"); | 187 | _("Only CHK or LOC URIs supported.\n")); |
188 | ret = 1; | 188 | ret = 1; |
189 | GNUNET_FS_uri_destroy (uri); | 189 | GNUNET_FS_uri_destroy (uri); |
190 | return; | 190 | return; |
@@ -192,7 +192,7 @@ run (void *cls, | |||
192 | if (NULL == filename) | 192 | if (NULL == filename) |
193 | { | 193 | { |
194 | fprintf (stderr, | 194 | fprintf (stderr, |
195 | "Target filename must be specified.\n"); | 195 | _("Target filename must be specified.\n")); |
196 | ret = 1; | 196 | ret = 1; |
197 | GNUNET_FS_uri_destroy (uri); | 197 | GNUNET_FS_uri_destroy (uri); |
198 | return; | 198 | return; |
diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c index f02e85bba..08b01fbbf 100644 --- a/src/fs/gnunet-service-fs.c +++ b/src/fs/gnunet-service-fs.c | |||
@@ -85,6 +85,13 @@ | |||
85 | #define TRUST_FLUSH_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5) | 85 | #define TRUST_FLUSH_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5) |
86 | 86 | ||
87 | /** | 87 | /** |
88 | * How quickly do we age cover traffic? At the given | ||
89 | * time interval, remaining cover traffic counters are | ||
90 | * decremented by 1/16th. | ||
91 | */ | ||
92 | #define COVER_AGE_FREQUENCY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5) | ||
93 | |||
94 | /** | ||
88 | * How often do we at most PUT content into the DHT? | 95 | * How often do we at most PUT content into the DHT? |
89 | */ | 96 | */ |
90 | #define MAX_DHT_PUT_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5) | 97 | #define MAX_DHT_PUT_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5) |
@@ -495,9 +502,6 @@ struct UsedTargetEntry | |||
495 | }; | 502 | }; |
496 | 503 | ||
497 | 504 | ||
498 | |||
499 | |||
500 | |||
501 | /** | 505 | /** |
502 | * Doubly-linked list of messages we are performing | 506 | * Doubly-linked list of messages we are performing |
503 | * due to a pending request. | 507 | * due to a pending request. |
@@ -929,6 +933,34 @@ static struct GNUNET_LOAD_Value *datastore_put_load; | |||
929 | static struct GNUNET_LOAD_Value *rt_entry_lifetime; | 933 | static struct GNUNET_LOAD_Value *rt_entry_lifetime; |
930 | 934 | ||
931 | /** | 935 | /** |
936 | * How many query messages have we received 'recently' that | ||
937 | * have not yet been claimed as cover traffic? | ||
938 | */ | ||
939 | static unsigned int cover_query_count; | ||
940 | |||
941 | /** | ||
942 | * How many content messages have we received 'recently' that | ||
943 | * have not yet been claimed as cover traffic? | ||
944 | */ | ||
945 | static unsigned int cover_content_count; | ||
946 | |||
947 | /** | ||
948 | * ID of our task that we use to age the cover counters. | ||
949 | */ | ||
950 | static GNUNET_SCHEDULER_TaskIdentifier cover_age_task; | ||
951 | |||
952 | static void | ||
953 | age_cover_counters (void *cls, | ||
954 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
955 | { | ||
956 | cover_content_count = (cover_content_count * 15) / 16; | ||
957 | cover_query_count = (cover_query_count * 15) / 16; | ||
958 | cover_age_task = GNUNET_SCHEDULER_add_delayed (COVER_AGE_FREQUENCY, | ||
959 | &age_cover_counters, | ||
960 | NULL); | ||
961 | } | ||
962 | |||
963 | /** | ||
932 | * We've just now completed a datastore request. Update our | 964 | * We've just now completed a datastore request. Update our |
933 | * datastore load calculations. | 965 | * datastore load calculations. |
934 | * | 966 | * |
@@ -2084,6 +2116,8 @@ shutdown_task (void *cls, | |||
2084 | cfg = NULL; | 2116 | cfg = NULL; |
2085 | GNUNET_free_non_null (trustDirectory); | 2117 | GNUNET_free_non_null (trustDirectory); |
2086 | trustDirectory = NULL; | 2118 | trustDirectory = NULL; |
2119 | GNUNET_SCHEDULER_cancel (cover_age_task); | ||
2120 | cover_age_task = GNUNET_SCHEDULER_NO_TASK; | ||
2087 | } | 2121 | } |
2088 | 2122 | ||
2089 | 2123 | ||
@@ -3015,6 +3049,26 @@ forward_request_task (void *cls, | |||
3015 | &process_dht_reply, | 3049 | &process_dht_reply, |
3016 | pr); | 3050 | pr); |
3017 | } | 3051 | } |
3052 | |||
3053 | if ( (pr->anonymity_level > 1) && | ||
3054 | (cover_query_count < pr->anonymity_level - 1) ) | ||
3055 | { | ||
3056 | delay = get_processing_delay (); | ||
3057 | #if DEBUG_FS | ||
3058 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
3059 | "Not enough cover traffic to forward query `%s', will try again in %llu ms!\n", | ||
3060 | GNUNET_h2s (&pr->query), | ||
3061 | delay.rel_value); | ||
3062 | #endif | ||
3063 | pr->task = GNUNET_SCHEDULER_add_delayed (delay, | ||
3064 | &forward_request_task, | ||
3065 | pr); | ||
3066 | return; | ||
3067 | } | ||
3068 | /* consume cover traffic */ | ||
3069 | if (pr->anonymity_level > 1) | ||
3070 | cover_query_count -= pr->anonymity_level - 1; | ||
3071 | |||
3018 | /* (1) select target */ | 3072 | /* (1) select target */ |
3019 | psc.pr = pr; | 3073 | psc.pr = pr; |
3020 | psc.target_score = -DBL_MAX; | 3074 | psc.target_score = -DBL_MAX; |
@@ -3221,6 +3275,11 @@ struct ProcessReplyClosure | |||
3221 | uint32_t priority; | 3275 | uint32_t priority; |
3222 | 3276 | ||
3223 | /** | 3277 | /** |
3278 | * Anonymity requirements for this reply. | ||
3279 | */ | ||
3280 | uint32_t anonymity_level; | ||
3281 | |||
3282 | /** | ||
3224 | * Evaluation result (returned). | 3283 | * Evaluation result (returned). |
3225 | */ | 3284 | */ |
3226 | enum GNUNET_BLOCK_EvaluationResult eval; | 3285 | enum GNUNET_BLOCK_EvaluationResult eval; |
@@ -3264,6 +3323,22 @@ struct GNUNET_TIME_Relative art_delay; | |||
3264 | size_t msize; | 3323 | size_t msize; |
3265 | unsigned int i; | 3324 | unsigned int i; |
3266 | 3325 | ||
3326 | if (NULL == pr->client_request_list) | ||
3327 | { | ||
3328 | /* reply will go over the network, check for cover traffic */ | ||
3329 | if ( (prq->anonymity_level > 1) && | ||
3330 | (cover_content_count < prq->anonymity_level - 1) ) | ||
3331 | { | ||
3332 | /* insufficient cover traffic, skip */ | ||
3333 | GNUNET_STATISTICS_update (stats, | ||
3334 | gettext_noop ("# replies suppressed due to lack of cover traffic"), | ||
3335 | 1, | ||
3336 | GNUNET_NO); | ||
3337 | return GNUNET_YES; | ||
3338 | } | ||
3339 | if (prq->anonymity_level > 1) | ||
3340 | cover_content_count -= prq->anonymity_level - 1; | ||
3341 | } | ||
3267 | #if DEBUG_FS | 3342 | #if DEBUG_FS |
3268 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 3343 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
3269 | "Matched result (type %u) for query `%s' with pending request\n", | 3344 | "Matched result (type %u) for query `%s' with pending request\n", |
@@ -3603,6 +3678,7 @@ handle_p2p_put (void *cls, | |||
3603 | GNUNET_break_op (0); | 3678 | GNUNET_break_op (0); |
3604 | return GNUNET_SYSERR; | 3679 | return GNUNET_SYSERR; |
3605 | } | 3680 | } |
3681 | cover_content_count++; | ||
3606 | #if DEBUG_FS | 3682 | #if DEBUG_FS |
3607 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 3683 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
3608 | "Received result for query `%s' from peer `%4s'\n", | 3684 | "Received result for query `%s' from peer `%4s'\n", |
@@ -3624,6 +3700,7 @@ handle_p2p_put (void *cls, | |||
3624 | prq.type = type; | 3700 | prq.type = type; |
3625 | prq.expiration = expiration; | 3701 | prq.expiration = expiration; |
3626 | prq.priority = 0; | 3702 | prq.priority = 0; |
3703 | prq.anonymity_level = 1; | ||
3627 | prq.finished = GNUNET_NO; | 3704 | prq.finished = GNUNET_NO; |
3628 | prq.request_found = GNUNET_NO; | 3705 | prq.request_found = GNUNET_NO; |
3629 | GNUNET_CONTAINER_multihashmap_get_multiple (query_request_map, | 3706 | GNUNET_CONTAINER_multihashmap_get_multiple (query_request_map, |
@@ -3905,6 +3982,7 @@ process_local_reply (void *cls, | |||
3905 | prq.priority = priority; | 3982 | prq.priority = priority; |
3906 | prq.finished = GNUNET_NO; | 3983 | prq.finished = GNUNET_NO; |
3907 | prq.request_found = GNUNET_NO; | 3984 | prq.request_found = GNUNET_NO; |
3985 | prq.anonymity_level = anonymity; | ||
3908 | if ( (old_rf == 0) && | 3986 | if ( (old_rf == 0) && |
3909 | (pr->results_found == 0) ) | 3987 | (pr->results_found == 0) ) |
3910 | update_datastore_delays (pr->start_time); | 3988 | update_datastore_delays (pr->start_time); |
@@ -4100,6 +4178,7 @@ handle_p2p_get (void *cls, | |||
4100 | GNUNET_break_op (0); | 4178 | GNUNET_break_op (0); |
4101 | return GNUNET_SYSERR; | 4179 | return GNUNET_SYSERR; |
4102 | } | 4180 | } |
4181 | cover_query_count++; | ||
4103 | bm = ntohl (gm->hash_bitmap); | 4182 | bm = ntohl (gm->hash_bitmap); |
4104 | bits = 0; | 4183 | bits = 0; |
4105 | cps = GNUNET_CONTAINER_multihashmap_get (connected_peers, | 4184 | cps = GNUNET_CONTAINER_multihashmap_get (connected_peers, |
@@ -4625,6 +4704,9 @@ main_init (struct GNUNET_SERVER_Handle *server, | |||
4625 | 4704 | ||
4626 | 4705 | ||
4627 | GNUNET_SERVER_add_handlers (server, handlers); | 4706 | GNUNET_SERVER_add_handlers (server, handlers); |
4707 | cover_age_task = GNUNET_SCHEDULER_add_delayed (COVER_AGE_FREQUENCY, | ||
4708 | &age_cover_counters, | ||
4709 | NULL); | ||
4628 | GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, | 4710 | GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, |
4629 | &shutdown_task, | 4711 | &shutdown_task, |
4630 | NULL); | 4712 | NULL); |