diff options
Diffstat (limited to 'src/fs/gnunet-service-fs.c')
-rw-r--r-- | src/fs/gnunet-service-fs.c | 182 |
1 files changed, 151 insertions, 31 deletions
diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c index 93123c8d8..2f9af1ad1 100644 --- a/src/fs/gnunet-service-fs.c +++ b/src/fs/gnunet-service-fs.c | |||
@@ -24,14 +24,11 @@ | |||
24 | * @author Christian Grothoff | 24 | * @author Christian Grothoff |
25 | * | 25 | * |
26 | * TODO: | 26 | * TODO: |
27 | * - forward_request_task (P2P forwarding!) | ||
28 | * - track stats for hot-path routing | 27 | * - track stats for hot-path routing |
29 | * - implement hot-path routing decision procedure | 28 | * - implement hot-path routing decision procedure |
30 | * - detect duplicate requests (P2P and CS) | ||
31 | * - implement: bound_priority, test_load_too_high, validate_skblock | 29 | * - implement: bound_priority, test_load_too_high, validate_skblock |
32 | * - add content migration support (store locally) | 30 | * - add content migration support (store locally) |
33 | * - statistics | 31 | * - statistics |
34 | * | ||
35 | */ | 32 | */ |
36 | #include "platform.h" | 33 | #include "platform.h" |
37 | #include <float.h> | 34 | #include <float.h> |
@@ -527,8 +524,7 @@ struct PendingRequest | |||
527 | uint32_t remaining_priority; | 524 | uint32_t remaining_priority; |
528 | 525 | ||
529 | /** | 526 | /** |
530 | * Number to mingle hashes for bloom-filter | 527 | * Number to mingle hashes for bloom-filter tests with. |
531 | * tests with. | ||
532 | */ | 528 | */ |
533 | int32_t mingle; | 529 | int32_t mingle; |
534 | 530 | ||
@@ -655,9 +651,9 @@ destroy_pending_request (struct PendingRequest *pr) | |||
655 | pr->hnode); | 651 | pr->hnode); |
656 | pr->hnode = NULL; | 652 | pr->hnode = NULL; |
657 | } | 653 | } |
658 | /* might have already been removed from map | 654 | /* might have already been removed from map in 'process_reply' (if |
659 | in 'process_reply' if there was a unique | 655 | there was a unique reply) or never inserted if it was a |
660 | reply; hence ignore the return value here */ | 656 | duplicate; hence ignore the return value here */ |
661 | (void) GNUNET_CONTAINER_multihashmap_remove (query_request_map, | 657 | (void) GNUNET_CONTAINER_multihashmap_remove (query_request_map, |
662 | &pr->query, | 658 | &pr->query, |
663 | pr); | 659 | pr); |
@@ -1127,7 +1123,6 @@ transmit_query_continuation (void *cls, | |||
1127 | } | 1123 | } |
1128 | 1124 | ||
1129 | 1125 | ||
1130 | #if 0 | ||
1131 | /** | 1126 | /** |
1132 | * How many bytes should a bloomfilter be if we have already seen | 1127 | * How many bytes should a bloomfilter be if we have already seen |
1133 | * entry_count responses? Note that BLOOMFILTER_K gives us the number | 1128 | * entry_count responses? Note that BLOOMFILTER_K gives us the number |
@@ -1193,7 +1188,6 @@ refresh_bloomfilter (unsigned int count, | |||
1193 | } | 1188 | } |
1194 | return bf; | 1189 | return bf; |
1195 | } | 1190 | } |
1196 | #endif | ||
1197 | 1191 | ||
1198 | 1192 | ||
1199 | /** | 1193 | /** |
@@ -1244,6 +1238,7 @@ target_reservation_cb (void *cls, | |||
1244 | size_t msize; | 1238 | size_t msize; |
1245 | unsigned int k; | 1239 | unsigned int k; |
1246 | int no_route; | 1240 | int no_route; |
1241 | uint32_t bm; | ||
1247 | 1242 | ||
1248 | pr->irc = NULL; | 1243 | pr->irc = NULL; |
1249 | GNUNET_assert (peer != NULL); | 1244 | GNUNET_assert (peer != NULL); |
@@ -1267,12 +1262,22 @@ target_reservation_cb (void *cls, | |||
1267 | 1262 | ||
1268 | /* build message and insert message into priority queue */ | 1263 | /* build message and insert message into priority queue */ |
1269 | k = 0; | 1264 | k = 0; |
1265 | bm = 0; | ||
1266 | if (GNUNET_YES == no_route) | ||
1267 | { | ||
1268 | bm |= GET_MESSAGE_BIT_RETURN_TO; | ||
1269 | k++; | ||
1270 | } | ||
1270 | if (pr->namespace != NULL) | 1271 | if (pr->namespace != NULL) |
1271 | k++; | 1272 | { |
1273 | bm |= GET_MESSAGE_BIT_SKS_NAMESPACE; | ||
1274 | k++; | ||
1275 | } | ||
1272 | if (pr->target_pid != 0) | 1276 | if (pr->target_pid != 0) |
1273 | k++; | 1277 | { |
1274 | if (GNUNET_YES == no_route) | 1278 | bm |= GET_MESSAGE_BIT_TRANSMIT_TO; |
1275 | k++; | 1279 | k++; |
1280 | } | ||
1276 | msize = sizeof (struct GetMessage) + pr->bf_size + k * sizeof(GNUNET_HashCode); | 1281 | msize = sizeof (struct GetMessage) + pr->bf_size + k * sizeof(GNUNET_HashCode); |
1277 | GNUNET_assert (msize < GNUNET_SERVER_MAX_MESSAGE_SIZE); | 1282 | GNUNET_assert (msize < GNUNET_SERVER_MAX_MESSAGE_SIZE); |
1278 | pm = GNUNET_malloc (sizeof (struct PendingMessage) + msize); | 1283 | pm = GNUNET_malloc (sizeof (struct PendingMessage) + msize); |
@@ -1284,17 +1289,17 @@ target_reservation_cb (void *cls, | |||
1284 | pr->remaining_priority /= 2; | 1289 | pr->remaining_priority /= 2; |
1285 | gm->priority = htonl (pr->remaining_priority); | 1290 | gm->priority = htonl (pr->remaining_priority); |
1286 | gm->ttl = htonl (pr->ttl); | 1291 | gm->ttl = htonl (pr->ttl); |
1287 | gm->filter_mutator = htonl(pr->mingle); // FIXME: bad endianess conversion? | 1292 | gm->filter_mutator = htonl(pr->mingle); |
1288 | gm->hash_bitmap = htonl (42); // FIXME! | 1293 | gm->hash_bitmap = htonl (bm); |
1289 | gm->query = pr->query; | 1294 | gm->query = pr->query; |
1290 | ext = (GNUNET_HashCode*) &gm[1]; | 1295 | ext = (GNUNET_HashCode*) &gm[1]; |
1291 | k = 0; | 1296 | k = 0; |
1297 | if (GNUNET_YES == no_route) | ||
1298 | GNUNET_PEER_resolve (pr->pht_entry->cp->pid, (struct GNUNET_PeerIdentity*) &ext[k++]); | ||
1292 | if (pr->namespace != NULL) | 1299 | if (pr->namespace != NULL) |
1293 | memcpy (&ext[k++], pr->namespace, sizeof (GNUNET_HashCode)); | 1300 | memcpy (&ext[k++], pr->namespace, sizeof (GNUNET_HashCode)); |
1294 | if (pr->target_pid != 0) | 1301 | if (pr->target_pid != 0) |
1295 | GNUNET_PEER_resolve (pr->target_pid, (struct GNUNET_PeerIdentity*) &ext[k++]); | 1302 | GNUNET_PEER_resolve (pr->target_pid, (struct GNUNET_PeerIdentity*) &ext[k++]); |
1296 | if (GNUNET_YES == no_route) | ||
1297 | GNUNET_PEER_resolve (pr->pht_entry->cp->pid, (struct GNUNET_PeerIdentity*) &ext[k++]); | ||
1298 | bfdata = (char *) &ext[k]; | 1303 | bfdata = (char *) &ext[k]; |
1299 | if (pr->bf != NULL) | 1304 | if (pr->bf != NULL) |
1300 | GNUNET_CONTAINER_bloomfilter_get_raw_data (pr->bf, | 1305 | GNUNET_CONTAINER_bloomfilter_get_raw_data (pr->bf, |
@@ -1709,9 +1714,15 @@ process_reply (void *cls, | |||
1709 | GNUNET_array_grow (pr->replies_seen, | 1714 | GNUNET_array_grow (pr->replies_seen, |
1710 | pr->replies_seen_size, | 1715 | pr->replies_seen_size, |
1711 | pr->replies_seen_size * 2 + 4); | 1716 | pr->replies_seen_size * 2 + 4); |
1712 | // FIXME: recalculate BF! | 1717 | if (pr->bf != NULL) |
1718 | GNUNET_CONTAINER_bloomfilter_free (pr->bf); | ||
1719 | pr->bf = refresh_bloomfilter (pr->replies_seen_off, | ||
1720 | &pr->mingle, | ||
1721 | &pr->bf_size, | ||
1722 | pr->replies_seen); | ||
1713 | } | 1723 | } |
1714 | pr->replies_seen[pr->replies_seen_off++] = chash; | 1724 | pr->replies_seen[pr->replies_seen_off++] = chash; |
1725 | |||
1715 | } | 1726 | } |
1716 | break; | 1727 | break; |
1717 | case GNUNET_DATASTORE_BLOCKTYPE_SKBLOCK: | 1728 | case GNUNET_DATASTORE_BLOCKTYPE_SKBLOCK: |
@@ -2048,6 +2059,53 @@ bound_priority (uint32_t prio_in, | |||
2048 | 2059 | ||
2049 | 2060 | ||
2050 | /** | 2061 | /** |
2062 | * Closure for 'check_duplicate_request'. | ||
2063 | */ | ||
2064 | struct CheckDuplicateRequestClosure | ||
2065 | { | ||
2066 | /** | ||
2067 | * The new request we should check if it already exists. | ||
2068 | */ | ||
2069 | const struct PendingRequest *pr; | ||
2070 | |||
2071 | /** | ||
2072 | * Existing request found by the checker, NULL if none. | ||
2073 | */ | ||
2074 | struct PendingRequest *have; | ||
2075 | }; | ||
2076 | |||
2077 | |||
2078 | /** | ||
2079 | * Iterator over entries in the 'query_request_map' that | ||
2080 | * tries to see if we have the same request pending from | ||
2081 | * the same peer already. | ||
2082 | * | ||
2083 | * @param cls closure (our 'struct CheckDuplicateRequestClosure') | ||
2084 | * @param key current key code (query, ignored, must match) | ||
2085 | * @param value value in the hash map (a 'struct PendingRequest' | ||
2086 | * that already exists) | ||
2087 | * @return GNUNET_YES if we should continue to | ||
2088 | * iterate (no match yet) | ||
2089 | * GNUNET_NO if not (match found). | ||
2090 | */ | ||
2091 | static int | ||
2092 | check_duplicate_request (void *cls, | ||
2093 | const GNUNET_HashCode * key, | ||
2094 | void *value) | ||
2095 | { | ||
2096 | struct CheckDuplicateRequestClosure *cdc = cls; | ||
2097 | struct PendingRequest *have = value; | ||
2098 | |||
2099 | if (cdc->pr->target_pid == have->target_pid) | ||
2100 | { | ||
2101 | cdc->have = have; | ||
2102 | return GNUNET_NO; | ||
2103 | } | ||
2104 | return GNUNET_YES; | ||
2105 | } | ||
2106 | |||
2107 | |||
2108 | /** | ||
2051 | * Handle P2P "GET" request. | 2109 | * Handle P2P "GET" request. |
2052 | * | 2110 | * |
2053 | * @param cls closure, always NULL | 2111 | * @param cls closure, always NULL |
@@ -2070,6 +2128,7 @@ handle_p2p_get (void *cls, | |||
2070 | struct PeerRequestEntry *pre; | 2128 | struct PeerRequestEntry *pre; |
2071 | struct ConnectedPeer *cp; | 2129 | struct ConnectedPeer *cp; |
2072 | struct ConnectedPeer *cps; | 2130 | struct ConnectedPeer *cps; |
2131 | struct CheckDuplicateRequestClosure cdc; | ||
2073 | struct GNUNET_TIME_Relative timeout; | 2132 | struct GNUNET_TIME_Relative timeout; |
2074 | uint16_t msize; | 2133 | uint16_t msize; |
2075 | const struct GetMessage *gm; | 2134 | const struct GetMessage *gm; |
@@ -2103,7 +2162,6 @@ handle_p2p_get (void *cls, | |||
2103 | } | 2162 | } |
2104 | opt = (const GNUNET_HashCode*) &gm[1]; | 2163 | opt = (const GNUNET_HashCode*) &gm[1]; |
2105 | bfsize = msize - sizeof (struct GetMessage) + bits * sizeof (GNUNET_HashCode); | 2164 | bfsize = msize - sizeof (struct GetMessage) + bits * sizeof (GNUNET_HashCode); |
2106 | |||
2107 | bm = ntohl (gm->hash_bitmap); | 2165 | bm = ntohl (gm->hash_bitmap); |
2108 | if ( (0 != (bm & GET_MESSAGE_BIT_SKS_NAMESPACE)) && | 2166 | if ( (0 != (bm & GET_MESSAGE_BIT_SKS_NAMESPACE)) && |
2109 | (ntohl (gm->type) == GNUNET_DATASTORE_BLOCKTYPE_SBLOCK) ) | 2167 | (ntohl (gm->type) == GNUNET_DATASTORE_BLOCKTYPE_SBLOCK) ) |
@@ -2150,7 +2208,7 @@ handle_p2p_get (void *cls, | |||
2150 | if ((bm & GET_MESSAGE_BIT_SKS_NAMESPACE)) | 2208 | if ((bm & GET_MESSAGE_BIT_SKS_NAMESPACE)) |
2151 | pr->namespace = (GNUNET_HashCode*) &pr[1]; | 2209 | pr->namespace = (GNUNET_HashCode*) &pr[1]; |
2152 | pr->type = ntohl (gm->type); | 2210 | pr->type = ntohl (gm->type); |
2153 | pr->mingle = gm->filter_mutator; | 2211 | pr->mingle = ntohl (gm->filter_mutator); |
2154 | if (0 != (bm & GET_MESSAGE_BIT_SKS_NAMESPACE)) | 2212 | if (0 != (bm & GET_MESSAGE_BIT_SKS_NAMESPACE)) |
2155 | memcpy (&pr[1], &opt[bits++], sizeof (GNUNET_HashCode)); | 2213 | memcpy (&pr[1], &opt[bits++], sizeof (GNUNET_HashCode)); |
2156 | else if (pr->type == GNUNET_DATASTORE_BLOCKTYPE_SBLOCK) | 2214 | else if (pr->type == GNUNET_DATASTORE_BLOCKTYPE_SBLOCK) |
@@ -2194,8 +2252,35 @@ handle_p2p_get (void *cls, | |||
2194 | pr->bf_size = bfsize; | 2252 | pr->bf_size = bfsize; |
2195 | } | 2253 | } |
2196 | 2254 | ||
2197 | /* FIXME: check somewhere if request already exists, and if so, | 2255 | cdc.have = NULL; |
2198 | recycle old state... */ | 2256 | cdc.pr = pr; |
2257 | GNUNET_CONTAINER_multihashmap_get_multiple (query_request_map, | ||
2258 | &gm->query, | ||
2259 | &check_duplicate_request, | ||
2260 | &cdc); | ||
2261 | if (cdc.have != NULL) | ||
2262 | { | ||
2263 | if (cdc.have->start_time.value + cdc.have->ttl >= | ||
2264 | pr->start_time.value + pr->ttl) | ||
2265 | { | ||
2266 | /* existing request has higher TTL, drop new one! */ | ||
2267 | cdc.have->priority += pr->priority; | ||
2268 | destroy_pending_request (pr); | ||
2269 | return GNUNET_OK; | ||
2270 | } | ||
2271 | else | ||
2272 | { | ||
2273 | /* existing request has lower TTL, drop old one! */ | ||
2274 | pr->priority += cdc.have->priority; | ||
2275 | /* Possible optimization: if we have applicable pending | ||
2276 | replies in 'cdc.have', we might want to move those over | ||
2277 | (this is a really rare special-case, so it is not clear | ||
2278 | that this would be worth it) */ | ||
2279 | destroy_pending_request (cdc.have); | ||
2280 | /* keep processing 'pr'! */ | ||
2281 | } | ||
2282 | } | ||
2283 | |||
2199 | pre = GNUNET_malloc (sizeof (struct PeerRequestEntry)); | 2284 | pre = GNUNET_malloc (sizeof (struct PeerRequestEntry)); |
2200 | pre->cp = cp; | 2285 | pre->cp = cp; |
2201 | pre->req = pr; | 2286 | pre->req = pr; |
@@ -2206,7 +2291,7 @@ handle_p2p_get (void *cls, | |||
2206 | 2291 | ||
2207 | pr->hnode = GNUNET_CONTAINER_heap_insert (requests_by_expiration_heap, | 2292 | pr->hnode = GNUNET_CONTAINER_heap_insert (requests_by_expiration_heap, |
2208 | pr, | 2293 | pr, |
2209 | GNUNET_TIME_absolute_get().value + pr->ttl); | 2294 | pr->start_time.value + pr->ttl); |
2210 | 2295 | ||
2211 | 2296 | ||
2212 | /* calculate change in traffic preference */ | 2297 | /* calculate change in traffic preference */ |
@@ -2306,8 +2391,41 @@ handle_start_search (void *cls, | |||
2306 | GNUNET_h2s (&sm->query), | 2391 | GNUNET_h2s (&sm->query), |
2307 | (unsigned int) type); | 2392 | (unsigned int) type); |
2308 | #endif | 2393 | #endif |
2309 | /* FIXME: detect duplicate request; if duplicate, simply update (merge) | 2394 | |
2310 | 'pr->replies_seen'! */ | 2395 | /* detect duplicate KBLOCK requests */ |
2396 | if (type == GNUNET_DATASTORE_BLOCKTYPE_KBLOCK) | ||
2397 | { | ||
2398 | crl = cl->rl_head; | ||
2399 | while ( (crl != NULL) && | ||
2400 | ( (0 != memcmp (&crl->req->query, | ||
2401 | &sm->query, | ||
2402 | sizeof (GNUNET_HashCode))) || | ||
2403 | (crl->req->type != type) ) ) | ||
2404 | crl = crl->next; | ||
2405 | if (crl != NULL) | ||
2406 | { | ||
2407 | pr = crl->req; | ||
2408 | /* Duplicate request (used to send long list of | ||
2409 | known/blocked results); merge 'pr->replies_seen' | ||
2410 | and update bloom filter */ | ||
2411 | GNUNET_array_grow (pr->replies_seen, | ||
2412 | pr->replies_seen_size, | ||
2413 | pr->replies_seen_off + sc); | ||
2414 | memcpy (&pr->replies_seen[pr->replies_seen_off], | ||
2415 | &sm[1], | ||
2416 | sc * sizeof (GNUNET_HashCode)); | ||
2417 | pr->replies_seen_off += sc; | ||
2418 | if (pr->bf != NULL) | ||
2419 | GNUNET_CONTAINER_bloomfilter_free (pr->bf); | ||
2420 | pr->bf = refresh_bloomfilter (pr->replies_seen_off, | ||
2421 | &pr->mingle, | ||
2422 | &pr->bf_size, | ||
2423 | pr->replies_seen); | ||
2424 | GNUNET_SERVER_receive_done (client, | ||
2425 | GNUNET_OK); | ||
2426 | return; | ||
2427 | } | ||
2428 | } | ||
2311 | pr = GNUNET_malloc (sizeof (struct PendingRequest) + | 2429 | pr = GNUNET_malloc (sizeof (struct PendingRequest) + |
2312 | ((type == GNUNET_DATASTORE_BLOCKTYPE_SBLOCK)?sizeof(GNUNET_HashCode):0)); | 2430 | ((type == GNUNET_DATASTORE_BLOCKTYPE_SBLOCK)?sizeof(GNUNET_HashCode):0)); |
2313 | crl = GNUNET_malloc (sizeof (struct ClientRequestList)); | 2431 | crl = GNUNET_malloc (sizeof (struct ClientRequestList)); |
@@ -2326,10 +2444,12 @@ handle_start_search (void *cls, | |||
2326 | &sm[1], | 2444 | &sm[1], |
2327 | sc * sizeof (GNUNET_HashCode)); | 2445 | sc * sizeof (GNUNET_HashCode)); |
2328 | pr->replies_seen_off = sc; | 2446 | pr->replies_seen_off = sc; |
2329 | pr->anonymity_level = ntohl (sm->anonymity_level); | 2447 | pr->anonymity_level = ntohl (sm->anonymity_level); |
2330 | pr->mingle = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, | 2448 | pr->bf = refresh_bloomfilter (pr->replies_seen_off, |
2331 | (uint32_t) -1); | 2449 | &pr->mingle, |
2332 | pr->query = sm->query; | 2450 | &pr->bf_size, |
2451 | pr->replies_seen); | ||
2452 | pr->query = sm->query; | ||
2333 | switch (type) | 2453 | switch (type) |
2334 | { | 2454 | { |
2335 | case GNUNET_DATASTORE_BLOCKTYPE_DBLOCK: | 2455 | case GNUNET_DATASTORE_BLOCKTYPE_DBLOCK: |