diff options
author | Christian Grothoff <christian@grothoff.org> | 2010-07-16 19:11:35 +0000 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2010-07-16 19:11:35 +0000 |
commit | 2eba86ef5f0496f32b99b2bcb668a3f7d0acffbc (patch) | |
tree | 88da3cf0c65948ccb6c96d6094d2504b930af42c | |
parent | 3519fc71001af482dcfeb34209d6c42cd126dbec (diff) | |
download | gnunet-2eba86ef5f0496f32b99b2bcb668a3f7d0acffbc.tar.gz gnunet-2eba86ef5f0496f32b99b2bcb668a3f7d0acffbc.zip |
inline downloads
-rw-r--r-- | TODO | 4 | ||||
-rw-r--r-- | src/fs/fs.h | 5 | ||||
-rw-r--r-- | src/fs/fs_download.c | 260 | ||||
-rw-r--r-- | src/include/gnunet_crypto_lib.h | 6 | ||||
-rw-r--r-- | src/util/crypto_rsa.c | 6 |
5 files changed, 213 insertions, 68 deletions
@@ -17,10 +17,6 @@ | |||
17 | - implement testcases | 17 | - implement testcases |
18 | - implement performance tests | 18 | - implement performance tests |
19 | * FS: [CG] | 19 | * FS: [CG] |
20 | - library: | ||
21 | + utilize in-line files in meta data always (including in search results or | ||
22 | when download is triggered manually and for probes); currently the data is | ||
23 | only used when users do a general 'recursive' download | ||
24 | - service: | 20 | - service: |
25 | + trust: do not charge when "idle" / load considerations (migration, routing) | 21 | + trust: do not charge when "idle" / load considerations (migration, routing) |
26 | + artificial delays | 22 | + artificial delays |
diff --git a/src/fs/fs.h b/src/fs/fs.h index 8478f4845..41831e8a8 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h | |||
@@ -1832,6 +1832,11 @@ struct GNUNET_FS_DownloadContext | |||
1832 | */ | 1832 | */ |
1833 | int has_finished; | 1833 | int has_finished; |
1834 | 1834 | ||
1835 | /** | ||
1836 | * Have we tried (and failed) to find matching full | ||
1837 | * data from the meta data yet? | ||
1838 | */ | ||
1839 | int tried_full_data; | ||
1835 | }; | 1840 | }; |
1836 | 1841 | ||
1837 | struct GNUNET_FS_Namespace | 1842 | struct GNUNET_FS_Namespace |
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index f1897e65f..80758ebc7 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c | |||
@@ -249,6 +249,171 @@ process_result_with_request (void *cls, | |||
249 | void *value); | 249 | void *value); |
250 | 250 | ||
251 | 251 | ||
252 | /** | ||
253 | * We've found a matching block without downloading it. | ||
254 | * Encrypt it and pass it to our "receive" function as | ||
255 | * if we had received it from the network. | ||
256 | * | ||
257 | * @param dc download in question | ||
258 | * @param chk request this relates to | ||
259 | * @param sm request details | ||
260 | * @param block plaintext data matching request | ||
261 | * @param len number of bytes in block | ||
262 | * @param depth depth of the block | ||
263 | * @param do_store should we still store the block on disk? | ||
264 | * @return GNUNET_OK on success | ||
265 | */ | ||
266 | static int | ||
267 | encrypt_existing_match (struct GNUNET_FS_DownloadContext *dc, | ||
268 | const struct ContentHashKey *chk, | ||
269 | struct DownloadRequest *sm, | ||
270 | const char * block, | ||
271 | size_t len, | ||
272 | int depth, | ||
273 | int do_store) | ||
274 | { | ||
275 | struct ProcessResultClosure prc; | ||
276 | char enc[len]; | ||
277 | struct GNUNET_CRYPTO_AesSessionKey sk; | ||
278 | struct GNUNET_CRYPTO_AesInitializationVector iv; | ||
279 | GNUNET_HashCode query; | ||
280 | |||
281 | GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv); | ||
282 | if (-1 == GNUNET_CRYPTO_aes_encrypt (block, len, | ||
283 | &sk, | ||
284 | &iv, | ||
285 | enc)) | ||
286 | { | ||
287 | GNUNET_break (0); | ||
288 | return GNUNET_SYSERR; | ||
289 | } | ||
290 | GNUNET_CRYPTO_hash (enc, len, &query); | ||
291 | if (0 != memcmp (&query, | ||
292 | &chk->query, | ||
293 | sizeof (GNUNET_HashCode))) | ||
294 | { | ||
295 | GNUNET_break_op (0); | ||
296 | return GNUNET_SYSERR; | ||
297 | } | ||
298 | #if DEBUG_DOWNLOAD | ||
299 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
300 | "Matching block already present, no need for download!\n"); | ||
301 | #endif | ||
302 | /* already got it! */ | ||
303 | prc.dc = dc; | ||
304 | prc.data = enc; | ||
305 | prc.size = len; | ||
306 | prc.type = (dc->treedepth == depth) | ||
307 | ? GNUNET_BLOCK_TYPE_DBLOCK | ||
308 | : GNUNET_BLOCK_TYPE_IBLOCK; | ||
309 | prc.query = chk->query; | ||
310 | prc.do_store = do_store; | ||
311 | process_result_with_request (&prc, | ||
312 | &chk->key, | ||
313 | sm); | ||
314 | return GNUNET_OK; | ||
315 | } | ||
316 | |||
317 | |||
318 | /** | ||
319 | * Closure for match_full_data. | ||
320 | */ | ||
321 | struct MatchDataContext | ||
322 | { | ||
323 | /** | ||
324 | * CHK we are looking for. | ||
325 | */ | ||
326 | const struct ContentHashKey *chk; | ||
327 | |||
328 | /** | ||
329 | * Download we're processing. | ||
330 | */ | ||
331 | struct GNUNET_FS_DownloadContext *dc; | ||
332 | |||
333 | /** | ||
334 | * Request details. | ||
335 | */ | ||
336 | struct DownloadRequest *sm; | ||
337 | |||
338 | /** | ||
339 | * Overall offset in the file. | ||
340 | */ | ||
341 | uint64_t offset; | ||
342 | |||
343 | /** | ||
344 | * Desired length of the block. | ||
345 | */ | ||
346 | size_t len; | ||
347 | |||
348 | /** | ||
349 | * Flag set to GNUNET_YES on success. | ||
350 | */ | ||
351 | int done; | ||
352 | }; | ||
353 | |||
354 | /** | ||
355 | * Type of a function that libextractor calls for each | ||
356 | * meta data item found. | ||
357 | * | ||
358 | * @param cls closure (user-defined) | ||
359 | * @param plugin_name name of the plugin that produced this value; | ||
360 | * special values can be used (i.e. '<zlib>' for zlib being | ||
361 | * used in the main libextractor library and yielding | ||
362 | * meta data). | ||
363 | * @param type libextractor-type describing the meta data | ||
364 | * @param format basic format information about data | ||
365 | * @param data_mime_type mime-type of data (not of the original file); | ||
366 | * can be NULL (if mime-type is not known) | ||
367 | * @param data actual meta-data found | ||
368 | * @param data_len number of bytes in data | ||
369 | * @return 0 to continue extracting, 1 to abort | ||
370 | */ | ||
371 | static int | ||
372 | match_full_data (void *cls, | ||
373 | const char *plugin_name, | ||
374 | enum EXTRACTOR_MetaType type, | ||
375 | enum EXTRACTOR_MetaFormat format, | ||
376 | const char *data_mime_type, | ||
377 | const char *data, | ||
378 | size_t data_len) | ||
379 | { | ||
380 | struct MatchDataContext *mdc = cls; | ||
381 | GNUNET_HashCode key; | ||
382 | |||
383 | if (type == EXTRACTOR_METATYPE_GNUNET_FULL_DATA) | ||
384 | { | ||
385 | if ( (mdc->offset > data_len) || | ||
386 | (mdc->offset + mdc->len > data_len) ) | ||
387 | return 1; | ||
388 | GNUNET_CRYPTO_hash (&data[mdc->offset], | ||
389 | mdc->len, | ||
390 | &key); | ||
391 | if (0 != memcmp (&key, | ||
392 | &mdc->chk->key, | ||
393 | sizeof (GNUNET_HashCode))) | ||
394 | { | ||
395 | GNUNET_break_op (0); | ||
396 | return 1; | ||
397 | } | ||
398 | /* match found! */ | ||
399 | if (GNUNET_OK != | ||
400 | encrypt_existing_match (mdc->dc, | ||
401 | mdc->chk, | ||
402 | mdc->sm, | ||
403 | &data[mdc->offset], | ||
404 | mdc->len, | ||
405 | 0, | ||
406 | GNUNET_YES)) | ||
407 | { | ||
408 | GNUNET_break_op (0); | ||
409 | return 1; | ||
410 | } | ||
411 | mdc->done = GNUNET_YES; | ||
412 | return 1; | ||
413 | } | ||
414 | return 0; | ||
415 | } | ||
416 | |||
252 | 417 | ||
253 | /** | 418 | /** |
254 | * Schedule the download of the specified block in the tree. | 419 | * Schedule the download of the specified block in the tree. |
@@ -273,25 +438,18 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc, | |||
273 | size_t len; | 438 | size_t len; |
274 | char block[DBLOCK_SIZE]; | 439 | char block[DBLOCK_SIZE]; |
275 | GNUNET_HashCode key; | 440 | GNUNET_HashCode key; |
276 | struct ProcessResultClosure prc; | 441 | struct MatchDataContext mdc; |
277 | struct GNUNET_DISK_FileHandle *fh; | 442 | struct GNUNET_DISK_FileHandle *fh; |
278 | 443 | ||
279 | #if DEBUG_DOWNLOAD | ||
280 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
281 | "Scheduling download at offset %llu and depth %u for `%s'\n", | ||
282 | (unsigned long long) offset, | ||
283 | depth, | ||
284 | GNUNET_h2s (&chk->query)); | ||
285 | #endif | ||
286 | total = GNUNET_ntohll (dc->uri->data.chk.file_length); | 444 | total = GNUNET_ntohll (dc->uri->data.chk.file_length); |
287 | off = compute_disk_offset (total, | ||
288 | offset, | ||
289 | depth, | ||
290 | dc->treedepth); | ||
291 | len = GNUNET_FS_tree_calculate_block_size (total, | 445 | len = GNUNET_FS_tree_calculate_block_size (total, |
292 | dc->treedepth, | 446 | dc->treedepth, |
293 | offset, | 447 | offset, |
294 | depth); | 448 | depth); |
449 | off = compute_disk_offset (total, | ||
450 | offset, | ||
451 | depth, | ||
452 | dc->treedepth); | ||
295 | sm = GNUNET_malloc (sizeof (struct DownloadRequest)); | 453 | sm = GNUNET_malloc (sizeof (struct DownloadRequest)); |
296 | sm->chk = *chk; | 454 | sm->chk = *chk; |
297 | sm->offset = offset; | 455 | sm->offset = offset; |
@@ -303,6 +461,29 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc, | |||
303 | &chk->query, | 461 | &chk->query, |
304 | sm, | 462 | sm, |
305 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); | 463 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); |
464 | if ( (dc->tried_full_data == GNUNET_NO) && | ||
465 | (depth == 0) ) | ||
466 | { | ||
467 | mdc.dc = dc; | ||
468 | mdc.sm = sm; | ||
469 | mdc.chk = chk; | ||
470 | mdc.offset = offset; | ||
471 | mdc.len = len; | ||
472 | mdc.done = GNUNET_NO; | ||
473 | GNUNET_CONTAINER_meta_data_iterate (dc->meta, | ||
474 | &match_full_data, | ||
475 | &mdc); | ||
476 | if (mdc.done == GNUNET_YES) | ||
477 | return; | ||
478 | dc->tried_full_data = GNUNET_YES; | ||
479 | } | ||
480 | #if DEBUG_DOWNLOAD | ||
481 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
482 | "Scheduling download at offset %llu and depth %u for `%s'\n", | ||
483 | (unsigned long long) offset, | ||
484 | depth, | ||
485 | GNUNET_h2s (&chk->query)); | ||
486 | #endif | ||
306 | fh = NULL; | 487 | fh = NULL; |
307 | if ( (dc->old_file_size > off) && | 488 | if ( (dc->old_file_size > off) && |
308 | (dc->filename != NULL) ) | 489 | (dc->filename != NULL) ) |
@@ -320,55 +501,22 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc, | |||
320 | len)) ) | 501 | len)) ) |
321 | { | 502 | { |
322 | GNUNET_CRYPTO_hash (block, len, &key); | 503 | GNUNET_CRYPTO_hash (block, len, &key); |
323 | if (0 == memcmp (&key, | 504 | if ( (0 == memcmp (&key, |
324 | &chk->key, | 505 | &chk->key, |
325 | sizeof (GNUNET_HashCode))) | 506 | sizeof (GNUNET_HashCode))) && |
507 | (GNUNET_OK == | ||
508 | encrypt_existing_match (dc, | ||
509 | chk, | ||
510 | sm, | ||
511 | block, | ||
512 | len, | ||
513 | depth, | ||
514 | GNUNET_NO)) ) | ||
326 | { | 515 | { |
327 | char enc[len]; | ||
328 | struct GNUNET_CRYPTO_AesSessionKey sk; | ||
329 | struct GNUNET_CRYPTO_AesInitializationVector iv; | ||
330 | GNUNET_HashCode query; | ||
331 | |||
332 | GNUNET_CRYPTO_hash_to_aes_key (&key, &sk, &iv); | ||
333 | if (-1 == GNUNET_CRYPTO_aes_encrypt (block, len, | ||
334 | &sk, | ||
335 | &iv, | ||
336 | enc)) | ||
337 | { | ||
338 | GNUNET_break (0); | ||
339 | goto do_download; | ||
340 | } | ||
341 | GNUNET_CRYPTO_hash (enc, len, &query); | ||
342 | if (0 == memcmp (&query, | ||
343 | &chk->query, | ||
344 | sizeof (GNUNET_HashCode))) | ||
345 | { | ||
346 | #if DEBUG_DOWNLOAD | ||
347 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
348 | "Matching block already present, no need for download!\n"); | ||
349 | #endif | ||
350 | /* already got it! */ | ||
351 | prc.dc = dc; | ||
352 | prc.data = enc; | ||
353 | prc.size = len; | ||
354 | prc.type = (dc->treedepth == depth) | ||
355 | ? GNUNET_BLOCK_TYPE_DBLOCK | ||
356 | : GNUNET_BLOCK_TYPE_IBLOCK; | ||
357 | prc.query = chk->query; | ||
358 | prc.do_store = GNUNET_NO; /* useless */ | ||
359 | process_result_with_request (&prc, | ||
360 | &key, | ||
361 | sm); | ||
362 | } | ||
363 | else | ||
364 | { | ||
365 | GNUNET_break_op (0); | ||
366 | } | ||
367 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh)); | 516 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh)); |
368 | return; | 517 | return; |
369 | } | 518 | } |
370 | } | 519 | } |
371 | do_download: | ||
372 | if (fh != NULL) | 520 | if (fh != NULL) |
373 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh)); | 521 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh)); |
374 | if (depth < dc->treedepth) | 522 | if (depth < dc->treedepth) |
diff --git a/src/include/gnunet_crypto_lib.h b/src/include/gnunet_crypto_lib.h index 479e0e42d..0b250311a 100644 --- a/src/include/gnunet_crypto_lib.h +++ b/src/include/gnunet_crypto_lib.h | |||
@@ -545,10 +545,8 @@ struct GNUNET_CRYPTO_RsaPrivateKey *GNUNET_CRYPTO_rsa_key_create (void); | |||
545 | * files does not exist, create a new key and write it to the | 545 | * files does not exist, create a new key and write it to the |
546 | * file. Caller must free return value. Note that this function | 546 | * file. Caller must free return value. Note that this function |
547 | * can not guarantee that another process might not be trying | 547 | * can not guarantee that another process might not be trying |
548 | * the same operation on the same file at the same time. The | 548 | * the same operation on the same file at the same time. |
549 | * caller must somehow know that the file either already exists | 549 | * If the contents of the file |
550 | * with a valid key OR be sure that no other process is calling | ||
551 | * this function at the same time. If the contents of the file | ||
552 | * are invalid the old file is deleted and a fresh key is | 550 | * are invalid the old file is deleted and a fresh key is |
553 | * created. | 551 | * created. |
554 | * | 552 | * |
diff --git a/src/util/crypto_rsa.c b/src/util/crypto_rsa.c index 582006530..c7b2c16ee 100644 --- a/src/util/crypto_rsa.c +++ b/src/util/crypto_rsa.c | |||
@@ -546,10 +546,8 @@ rsa_decode_key (const struct RsaPrivateKeyBinaryEncoded *encoding) | |||
546 | * files does not exist, create a new key and write it to the | 546 | * files does not exist, create a new key and write it to the |
547 | * file. Caller must free return value. Note that this function | 547 | * file. Caller must free return value. Note that this function |
548 | * can not guarantee that another process might not be trying | 548 | * can not guarantee that another process might not be trying |
549 | * the same operation on the same file at the same time. The | 549 | * the same operation on the same file at the same time. |
550 | * caller must somehow know that the file either already exists | 550 | * If the contents of the file |
551 | * with a valid key OR be sure that no other process is calling | ||
552 | * this function at the same time. If the contents of the file | ||
553 | * are invalid the old file is deleted and a fresh key is | 551 | * are invalid the old file is deleted and a fresh key is |
554 | * created. | 552 | * created. |
555 | * | 553 | * |