diff options
Diffstat (limited to 'src/fs/fs_unindex.c')
-rw-r--r-- | src/fs/fs_unindex.c | 321 |
1 files changed, 314 insertions, 7 deletions
diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c index ff1996a2f..07b9cccd3 100644 --- a/src/fs/fs_unindex.c +++ b/src/fs/fs_unindex.c | |||
@@ -30,6 +30,7 @@ | |||
30 | #include "gnunet_protocols.h" | 30 | #include "gnunet_protocols.h" |
31 | #include "fs_api.h" | 31 | #include "fs_api.h" |
32 | #include "fs_tree.h" | 32 | #include "fs_tree.h" |
33 | #include "block_fs.h" | ||
33 | 34 | ||
34 | 35 | ||
35 | /** | 36 | /** |
@@ -203,6 +204,7 @@ unindex_process (void *cls, const struct ContentHashKey *chk, uint64_t offset, | |||
203 | "Sending REMOVE request to DATASTORE service\n"); | 204 | "Sending REMOVE request to DATASTORE service\n"); |
204 | GNUNET_DATASTORE_remove (uc->dsh, &chk->query, size, data, -2, 1, | 205 | GNUNET_DATASTORE_remove (uc->dsh, &chk->query, size, data, -2, 1, |
205 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, &process_cont, uc); | 206 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, &process_cont, uc); |
207 | uc->chk = *chk; | ||
206 | } | 208 | } |
207 | 209 | ||
208 | 210 | ||
@@ -258,16 +260,15 @@ process_fs_response (void *cls, const struct GNUNET_MessageHeader *msg) | |||
258 | 260 | ||
259 | 261 | ||
260 | /** | 262 | /** |
261 | * Function called when the tree encoder has | 263 | * Function called when we are done with removing KBlocks. |
262 | * processed all blocks. Clean up. | 264 | * Disconnect from datastore and notify FS service about |
265 | * the unindex event. | ||
263 | * | 266 | * |
264 | * @param cls our unindexing context | 267 | * @param uc our unindexing context |
265 | * @param tc not used | ||
266 | */ | 268 | */ |
267 | static void | 269 | static void |
268 | unindex_finish (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) | 270 | unindex_finish (struct GNUNET_FS_UnindexContext *uc) |
269 | { | 271 | { |
270 | struct GNUNET_FS_UnindexContext *uc = cls; | ||
271 | char *emsg; | 272 | char *emsg; |
272 | struct GNUNET_FS_Uri *uri; | 273 | struct GNUNET_FS_Uri *uri; |
273 | struct UnindexMessage req; | 274 | struct UnindexMessage req; |
@@ -310,6 +311,281 @@ unindex_finish (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) | |||
310 | } | 311 | } |
311 | 312 | ||
312 | 313 | ||
314 | |||
315 | /** | ||
316 | * Function called by the directory scanner as we extract keywords | ||
317 | * that we will need to remove KBlocks. | ||
318 | * | ||
319 | * @param cls the 'struct GNUNET_FS_UnindexContext *' | ||
320 | * @param filename which file we are making progress on | ||
321 | * @param is_directory GNUNET_YES if this is a directory, | ||
322 | * GNUNET_NO if this is a file | ||
323 | * GNUNET_SYSERR if it is neither (or unknown) | ||
324 | * @param reason kind of progress we are making | ||
325 | */ | ||
326 | static void | ||
327 | unindex_directory_scan_cb (void *cls, | ||
328 | const char *filename, | ||
329 | int is_directory, | ||
330 | enum GNUNET_FS_DirScannerProgressUpdateReason reason) | ||
331 | { | ||
332 | struct GNUNET_FS_UnindexContext *uc = cls; | ||
333 | static struct GNUNET_FS_ShareTreeItem * directory_scan_result; | ||
334 | |||
335 | switch (reason) | ||
336 | { | ||
337 | case GNUNET_FS_DIRSCANNER_FINISHED: | ||
338 | directory_scan_result = GNUNET_FS_directory_scan_get_result (uc->dscan); | ||
339 | uc->dscan = NULL; | ||
340 | if (NULL != directory_scan_result->ksk_uri) | ||
341 | { | ||
342 | uc->ksk_uri = GNUNET_FS_uri_dup (directory_scan_result->ksk_uri); | ||
343 | uc->state = UNINDEX_STATE_DS_REMOVE_KBLOCKS; | ||
344 | uc->emsg = GNUNET_strdup (_("Failed to connect to `datastore' service.")); | ||
345 | GNUNET_FS_unindex_sync_ (uc); | ||
346 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); | ||
347 | } | ||
348 | else | ||
349 | { | ||
350 | unindex_finish (uc); | ||
351 | } | ||
352 | GNUNET_FS_share_tree_free (directory_scan_result); | ||
353 | break; | ||
354 | case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR: | ||
355 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | ||
356 | _("Internal error scanning `%s'.\n"), | ||
357 | uc->filename); | ||
358 | break; | ||
359 | default: | ||
360 | break; | ||
361 | } | ||
362 | |||
363 | } | ||
364 | |||
365 | |||
366 | /** | ||
367 | * If necessary, connect to the datastore and remove the KBlocks. | ||
368 | * | ||
369 | * @param uc context for the unindex operation. | ||
370 | */ | ||
371 | void | ||
372 | GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc) | ||
373 | { | ||
374 | char *ex; | ||
375 | |||
376 | if (GNUNET_OK != | ||
377 | GNUNET_CONFIGURATION_get_value_string (uc->h->cfg, "FS", "EXTRACTORS", &ex)) | ||
378 | ex = NULL; | ||
379 | uc->dscan = GNUNET_FS_directory_scan_start (uc->filename, | ||
380 | GNUNET_NO, ex, | ||
381 | &unindex_directory_scan_cb, | ||
382 | uc); | ||
383 | GNUNET_free_non_null (ex); | ||
384 | } | ||
385 | |||
386 | |||
387 | /** | ||
388 | * Continuation called to notify client about result of the remove | ||
389 | * operation for the KBlock. | ||
390 | * | ||
391 | * @param cls the 'struct GNUNET_FS_UnindexContext *' | ||
392 | * @param success GNUNET_SYSERR on failure (including timeout/queue drop) | ||
393 | * GNUNET_NO if content was already there | ||
394 | * GNUNET_YES (or other positive value) on success | ||
395 | * @param min_expiration minimum expiration time required for 0-priority content to be stored | ||
396 | * by the datacache at this time, zero for unknown, forever if we have no | ||
397 | * space for 0-priority content | ||
398 | * @param msg NULL on success, otherwise an error message | ||
399 | */ | ||
400 | static void | ||
401 | continue_after_remove (void *cls, | ||
402 | int32_t success, | ||
403 | struct GNUNET_TIME_Absolute min_expiration, | ||
404 | const char *msg) | ||
405 | { | ||
406 | struct GNUNET_FS_UnindexContext *uc = cls; | ||
407 | |||
408 | if (success != GNUNET_YES) | ||
409 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | ||
410 | _("Failed to remove KBlock: %s\n"), | ||
411 | msg); | ||
412 | uc->ksk_offset++; | ||
413 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); | ||
414 | } | ||
415 | |||
416 | |||
417 | /** | ||
418 | * Function called from datastore with result from us looking for | ||
419 | * a KBlock. There are four cases: | ||
420 | * 1) no result, means we move on to the next keyword | ||
421 | * 2) UID is the same as the first UID, means we move on to next keyword | ||
422 | * 3) KBlock for a different CHK, means we keep looking for more | ||
423 | * 4) KBlock is for our CHK, means we remove the block and then move | ||
424 | * on to the next keyword | ||
425 | * | ||
426 | * @param cls the 'struct GNUNET_FS_UnindexContext *' | ||
427 | * @param key key for the content | ||
428 | * @param size number of bytes in data | ||
429 | * @param data content stored | ||
430 | * @param type type of the content | ||
431 | * @param priority priority of the content | ||
432 | * @param anonymity anonymity-level for the content | ||
433 | * @param expiration expiration time for the content | ||
434 | * @param uid unique identifier for the datum; | ||
435 | * maybe 0 if no unique identifier is available | ||
436 | */ | ||
437 | static void | ||
438 | process_kblock_for_unindex (void *cls, | ||
439 | const GNUNET_HashCode * key, | ||
440 | size_t size, const void *data, | ||
441 | enum GNUNET_BLOCK_Type type, | ||
442 | uint32_t priority, | ||
443 | uint32_t anonymity, | ||
444 | struct GNUNET_TIME_Absolute | ||
445 | expiration, uint64_t uid) | ||
446 | { | ||
447 | struct GNUNET_FS_UnindexContext *uc = cls; | ||
448 | const struct KBlock *kb; | ||
449 | const char *uris; | ||
450 | struct GNUNET_FS_Uri *chk_uri; | ||
451 | |||
452 | uc->dqe = NULL; | ||
453 | if (NULL == data) | ||
454 | { | ||
455 | /* no result */ | ||
456 | uc->ksk_offset++; | ||
457 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); | ||
458 | return; | ||
459 | } | ||
460 | if (0 == uc->first_uid) | ||
461 | { | ||
462 | /* remember UID of first result to detect cycles */ | ||
463 | uc->first_uid = uid; | ||
464 | } | ||
465 | else if (uid == uc->first_uid) | ||
466 | { | ||
467 | /* no more additional results */ | ||
468 | uc->ksk_offset++; | ||
469 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); | ||
470 | return; | ||
471 | } | ||
472 | GNUNET_assert (GNUNET_BLOCK_TYPE_FS_KBLOCK == type); | ||
473 | if (size < sizeof (struct KBlock)) | ||
474 | { | ||
475 | GNUNET_break (0); | ||
476 | goto get_next; | ||
477 | } | ||
478 | kb = data; | ||
479 | uris = (const char*) &kb[1]; | ||
480 | if (NULL == memchr (uris, 0, size - sizeof (struct KBlock))) | ||
481 | { | ||
482 | GNUNET_break (0); | ||
483 | goto get_next; | ||
484 | } | ||
485 | chk_uri = GNUNET_FS_uri_parse (uris, NULL); | ||
486 | if (NULL == chk_uri) | ||
487 | { | ||
488 | GNUNET_break (0); | ||
489 | goto get_next; | ||
490 | } | ||
491 | if (0 != memcmp (&uc->chk, | ||
492 | &chk_uri->data.chk.chk, | ||
493 | sizeof (struct ContentHashKey))) | ||
494 | { | ||
495 | /* different CHK, ignore */ | ||
496 | GNUNET_FS_uri_destroy (chk_uri); | ||
497 | goto get_next; | ||
498 | } | ||
499 | /* matches! */ | ||
500 | uc->dqe = GNUNET_DATASTORE_remove (uc->dsh, | ||
501 | key, size, data, | ||
502 | 0 /* priority */, 1 /* queue size */, | ||
503 | GNUNET_TIME_UNIT_FOREVER_REL, | ||
504 | &continue_after_remove, | ||
505 | uc); | ||
506 | return; | ||
507 | get_next: | ||
508 | uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh, | ||
509 | uc->roff++, | ||
510 | &uc->key, | ||
511 | GNUNET_BLOCK_TYPE_FS_KBLOCK, | ||
512 | 0 /* priority */, 1 /* queue size */, | ||
513 | GNUNET_TIME_UNIT_FOREVER_REL, | ||
514 | &process_kblock_for_unindex, | ||
515 | uc); | ||
516 | } | ||
517 | |||
518 | |||
519 | /** | ||
520 | * If necessary, connect to the datastore and remove the KBlocks. | ||
521 | * | ||
522 | * @param uc context for the unindex operation. | ||
523 | */ | ||
524 | void | ||
525 | GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc) | ||
526 | { | ||
527 | const char *keyword; | ||
528 | GNUNET_HashCode hc; | ||
529 | struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pub; | ||
530 | struct GNUNET_CRYPTO_RsaPrivateKey *pk; | ||
531 | |||
532 | if (NULL != uc->dsh) | ||
533 | uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg); | ||
534 | if (NULL == uc->dsh) | ||
535 | { | ||
536 | uc->state = UNINDEX_STATE_ERROR; | ||
537 | uc->emsg = GNUNET_strdup (_("Failed to connect to `datastore' service.")); | ||
538 | GNUNET_FS_unindex_sync_ (uc); | ||
539 | signal_unindex_error (uc); | ||
540 | return; | ||
541 | } | ||
542 | if ( (NULL == uc->ksk_uri) || | ||
543 | (uc->ksk_offset >= uc->ksk_uri->data.ksk.keywordCount) ) | ||
544 | { | ||
545 | unindex_finish (uc); | ||
546 | return; | ||
547 | } | ||
548 | /* FIXME: code duplication with fs_search.c here... */ | ||
549 | keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1]; | ||
550 | GNUNET_CRYPTO_hash (keyword, strlen (keyword), &hc); | ||
551 | pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&hc); | ||
552 | GNUNET_assert (pk != NULL); | ||
553 | GNUNET_CRYPTO_rsa_key_get_public (pk, &pub); | ||
554 | GNUNET_CRYPTO_rsa_key_free (pk); | ||
555 | GNUNET_CRYPTO_hash (&pub, | ||
556 | sizeof (struct | ||
557 | GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), | ||
558 | &uc->key); | ||
559 | uc->first_uid = 0; | ||
560 | uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh, | ||
561 | uc->roff++, | ||
562 | &uc->key, | ||
563 | GNUNET_BLOCK_TYPE_FS_KBLOCK, | ||
564 | 0 /* priority */, 1 /* queue size */, | ||
565 | GNUNET_TIME_UNIT_FOREVER_REL, | ||
566 | &process_kblock_for_unindex, | ||
567 | uc); | ||
568 | } | ||
569 | |||
570 | |||
571 | /** | ||
572 | * Function called when the tree encoder has | ||
573 | * processed all blocks. Clean up. | ||
574 | * | ||
575 | * @param cls our unindexing context | ||
576 | * @param tc not used | ||
577 | */ | ||
578 | static void | ||
579 | unindex_extract_keywords (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
580 | { | ||
581 | struct GNUNET_FS_UnindexContext *uc = cls; | ||
582 | |||
583 | uc->state = UNINDEX_STATE_EXTRACT_KEYWORDS; | ||
584 | GNUNET_FS_unindex_sync_ (uc); | ||
585 | GNUNET_FS_unindex_do_extract_keywords_ (uc); | ||
586 | } | ||
587 | |||
588 | |||
313 | /** | 589 | /** |
314 | * Connect to the datastore and remove the blocks. | 590 | * Connect to the datastore and remove the blocks. |
315 | * | 591 | * |
@@ -343,7 +619,7 @@ GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc) | |||
343 | uc->tc = | 619 | uc->tc = |
344 | GNUNET_FS_tree_encoder_create (uc->h, uc->file_size, uc, &unindex_reader, | 620 | GNUNET_FS_tree_encoder_create (uc->h, uc->file_size, uc, &unindex_reader, |
345 | &unindex_process, &unindex_progress, | 621 | &unindex_process, &unindex_progress, |
346 | &unindex_finish); | 622 | &unindex_extract_keywords); |
347 | GNUNET_FS_tree_encoder_next (uc->tc); | 623 | GNUNET_FS_tree_encoder_next (uc->tc); |
348 | } | 624 | } |
349 | 625 | ||
@@ -393,11 +669,27 @@ GNUNET_FS_unindex_signal_suspend_ (void *cls) | |||
393 | struct GNUNET_FS_UnindexContext *uc = cls; | 669 | struct GNUNET_FS_UnindexContext *uc = cls; |
394 | struct GNUNET_FS_ProgressInfo pi; | 670 | struct GNUNET_FS_ProgressInfo pi; |
395 | 671 | ||
672 | /* FIXME: lots of duplication with unindex_stop here! */ | ||
673 | if (uc->dscan != NULL) | ||
674 | { | ||
675 | GNUNET_FS_directory_scan_abort (uc->dscan); | ||
676 | uc->dscan = NULL; | ||
677 | } | ||
678 | if (NULL != uc->dqe) | ||
679 | { | ||
680 | GNUNET_DATASTORE_cancel (uc->dqe); | ||
681 | uc->dqe = NULL; | ||
682 | } | ||
396 | if (uc->fhc != NULL) | 683 | if (uc->fhc != NULL) |
397 | { | 684 | { |
398 | GNUNET_CRYPTO_hash_file_cancel (uc->fhc); | 685 | GNUNET_CRYPTO_hash_file_cancel (uc->fhc); |
399 | uc->fhc = NULL; | 686 | uc->fhc = NULL; |
400 | } | 687 | } |
688 | if (NULL != uc->ksk_uri) | ||
689 | { | ||
690 | GNUNET_FS_uri_destroy (uc->ksk_uri); | ||
691 | uc->ksk_uri = NULL; | ||
692 | } | ||
401 | if (uc->client != NULL) | 693 | if (uc->client != NULL) |
402 | { | 694 | { |
403 | GNUNET_CLIENT_disconnect (uc->client, GNUNET_NO); | 695 | GNUNET_CLIENT_disconnect (uc->client, GNUNET_NO); |
@@ -478,6 +770,16 @@ GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc) | |||
478 | { | 770 | { |
479 | struct GNUNET_FS_ProgressInfo pi; | 771 | struct GNUNET_FS_ProgressInfo pi; |
480 | 772 | ||
773 | if (uc->dscan != NULL) | ||
774 | { | ||
775 | GNUNET_FS_directory_scan_abort (uc->dscan); | ||
776 | uc->dscan = NULL; | ||
777 | } | ||
778 | if (NULL != uc->dqe) | ||
779 | { | ||
780 | GNUNET_DATASTORE_cancel (uc->dqe); | ||
781 | uc->dqe = NULL; | ||
782 | } | ||
481 | if (uc->fhc != NULL) | 783 | if (uc->fhc != NULL) |
482 | { | 784 | { |
483 | GNUNET_CRYPTO_hash_file_cancel (uc->fhc); | 785 | GNUNET_CRYPTO_hash_file_cancel (uc->fhc); |
@@ -493,6 +795,11 @@ GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc) | |||
493 | GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); | 795 | GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); |
494 | uc->dsh = NULL; | 796 | uc->dsh = NULL; |
495 | } | 797 | } |
798 | if (NULL != uc->ksk_uri) | ||
799 | { | ||
800 | GNUNET_FS_uri_destroy (uc->ksk_uri); | ||
801 | uc->ksk_uri = NULL; | ||
802 | } | ||
496 | if (NULL != uc->tc) | 803 | if (NULL != uc->tc) |
497 | { | 804 | { |
498 | GNUNET_FS_tree_encoder_finish (uc->tc, NULL, NULL); | 805 | GNUNET_FS_tree_encoder_finish (uc->tc, NULL, NULL); |