diff options
author | Christian Grothoff <christian@grothoff.org> | 2011-01-20 14:20:30 +0000 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2011-01-20 14:20:30 +0000 |
commit | c9432075d84d3c50171367e7d648a3a09f431994 (patch) | |
tree | 816f97d3601f6f255354267225c3b43b076fdac7 /src | |
parent | 81dfe20c8ee82d202244766f2914683a63d078ca (diff) | |
download | gnunet-c9432075d84d3c50171367e7d648a3a09f431994.tar.gz gnunet-c9432075d84d3c50171367e7d648a3a09f431994.zip |
major rewrite of fs_download, hopefully fixing 1641
Diffstat (limited to 'src')
-rw-r--r-- | src/fs/Makefile.am | 1 | ||||
-rw-r--r-- | src/fs/fs.c | 188 | ||||
-rw-r--r-- | src/fs/fs.h | 216 | ||||
-rw-r--r-- | src/fs/fs_download.c | 1842 | ||||
-rw-r--r-- | src/fs/fs_misc.c | 165 | ||||
-rw-r--r-- | src/fs/fs_publish.c | 27 | ||||
-rw-r--r-- | src/fs/fs_tree.c | 250 | ||||
-rw-r--r-- | src/fs/fs_tree.h | 80 | ||||
-rw-r--r-- | src/fs/fs_unindex.c | 10 | ||||
-rw-r--r-- | src/fs/test_fs_download.c | 2 | ||||
-rw-r--r-- | src/fs/test_fs_download_data.conf | 2 | ||||
-rw-r--r-- | src/fs/test_fs_download_persistence.c | 14 |
12 files changed, 1663 insertions, 1134 deletions
diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am index c0a0420b7..2f67ce36a 100644 --- a/src/fs/Makefile.am +++ b/src/fs/Makefile.am | |||
@@ -23,6 +23,7 @@ libgnunetfs_la_SOURCES = \ | |||
23 | fs_getopt.c \ | 23 | fs_getopt.c \ |
24 | fs_list_indexed.c \ | 24 | fs_list_indexed.c \ |
25 | fs_publish.c \ | 25 | fs_publish.c \ |
26 | fs_misc.c \ | ||
26 | fs_namespace.c \ | 27 | fs_namespace.c \ |
27 | fs_search.c \ | 28 | fs_search.c \ |
28 | fs_tree.c fs_tree.h \ | 29 | fs_tree.c fs_tree.h \ |
diff --git a/src/fs/fs.c b/src/fs/fs.c index c8ce4d651..d748e89ab 100644 --- a/src/fs/fs.c +++ b/src/fs/fs.c | |||
@@ -1639,49 +1639,102 @@ GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc) | |||
1639 | 1639 | ||
1640 | 1640 | ||
1641 | /** | 1641 | /** |
1642 | * Serialize an active or pending download request. | 1642 | * Serialize a download request. |
1643 | * | 1643 | * |
1644 | * @param cls the 'struct GNUNET_BIO_WriteHandle*' | 1644 | * @param wh the 'struct GNUNET_BIO_WriteHandle*' |
1645 | * @param key unused, can be NULL | 1645 | * @param dr the 'struct DownloadRequest' |
1646 | * @param value the 'struct DownloadRequest' | ||
1647 | * @return GNUNET_YES on success, GNUNET_NO on error | 1646 | * @return GNUNET_YES on success, GNUNET_NO on error |
1648 | */ | 1647 | */ |
1649 | static int | 1648 | static int |
1650 | write_download_request (void *cls, | 1649 | write_download_request (struct GNUNET_BIO_WriteHandle *wh, |
1651 | const GNUNET_HashCode *key, | 1650 | struct DownloadRequest *dr) |
1652 | void *value) | 1651 | { |
1653 | { | 1652 | unsigned int i; |
1654 | struct GNUNET_BIO_WriteHandle *wh = cls; | ||
1655 | struct DownloadRequest *dr = value; | ||
1656 | 1653 | ||
1657 | if ( (GNUNET_OK != | 1654 | if ( (GNUNET_OK != |
1658 | GNUNET_BIO_write (wh, &dr->chk, sizeof (struct ContentHashKey))) || | 1655 | GNUNET_BIO_write_int32 (wh, dr->state)) || |
1659 | (GNUNET_OK != | 1656 | (GNUNET_OK != |
1660 | GNUNET_BIO_write_int64 (wh, dr->offset)) || | 1657 | GNUNET_BIO_write_int64 (wh, dr->offset)) || |
1661 | (GNUNET_OK != | 1658 | (GNUNET_OK != |
1662 | GNUNET_BIO_write_int32 (wh, dr->depth)) ) | 1659 | GNUNET_BIO_write_int32 (wh, dr->num_children)) || |
1660 | (GNUNET_OK != | ||
1661 | GNUNET_BIO_write_int32 (wh, dr->depth)) ) | ||
1663 | return GNUNET_NO; | 1662 | return GNUNET_NO; |
1663 | if ( (dr->state == BRS_CHK_SET) && | ||
1664 | (GNUNET_OK != | ||
1665 | GNUNET_BIO_write (wh, &dr->chk, sizeof (struct ContentHashKey))) ) | ||
1666 | return GNUNET_NO; | ||
1667 | for (i=0;i<dr->num_children;i++) | ||
1668 | if (GNUNET_NO == | ||
1669 | write_download_request (wh, dr->children[i])) | ||
1670 | return GNUNET_NO; | ||
1664 | return GNUNET_YES; | 1671 | return GNUNET_YES; |
1665 | } | 1672 | } |
1666 | 1673 | ||
1667 | 1674 | ||
1668 | /** | 1675 | /** |
1669 | * Count active download requests. | 1676 | * Read a download request tree. |
1670 | * | 1677 | * |
1671 | * @param cls the 'uint32_t*' counter | 1678 | * @param rh stream to read from |
1672 | * @param key unused, can be NULL | 1679 | * @return value the 'struct DownloadRequest', NULL on error |
1673 | * @param value the 'struct DownloadRequest' | ||
1674 | * @return GNUNET_YES (continue iteration) | ||
1675 | */ | 1680 | */ |
1676 | static int | 1681 | static struct DownloadRequest * |
1677 | count_download_requests (void *cls, | 1682 | read_download_request (struct GNUNET_BIO_ReadHandle *rh) |
1678 | const GNUNET_HashCode *key, | 1683 | { |
1679 | void *value) | 1684 | struct DownloadRequest *dr; |
1680 | { | 1685 | unsigned int i; |
1681 | uint32_t *counter = cls; | 1686 | |
1682 | 1687 | dr = GNUNET_malloc (sizeof (struct DownloadRequest)); | |
1683 | (*counter)++; | 1688 | |
1684 | return GNUNET_YES; | 1689 | if ( (GNUNET_OK != |
1690 | GNUNET_BIO_read_int32 (rh, &dr->state)) || | ||
1691 | (GNUNET_OK != | ||
1692 | GNUNET_BIO_read_int64 (rh, &dr->offset)) || | ||
1693 | (GNUNET_OK != | ||
1694 | GNUNET_BIO_read_int32 (rh, &dr->num_children)) || | ||
1695 | (dr->num_children > CHK_PER_INODE) || | ||
1696 | (GNUNET_OK != | ||
1697 | GNUNET_BIO_read_int32 (rh, &dr->depth)) || | ||
1698 | ( (dr->depth == 0) && (dr->num_children > 0) ) || | ||
1699 | ( (dr->depth > 0) && (dr->num_children == 0) ) ) | ||
1700 | { | ||
1701 | GNUNET_break (0); | ||
1702 | dr->num_children = 0; | ||
1703 | goto cleanup; | ||
1704 | } | ||
1705 | if (dr->num_children > 0) | ||
1706 | dr->children = GNUNET_malloc (dr->num_children * | ||
1707 | sizeof (struct ContentHashKey)); | ||
1708 | switch (dr->state) | ||
1709 | { | ||
1710 | case BRS_INIT: | ||
1711 | case BRS_RECONSTRUCT_DOWN: | ||
1712 | case BRS_RECONSTRUCT_META_UP: | ||
1713 | case BRS_RECONSTRUCT_UP: | ||
1714 | break; | ||
1715 | case BRS_CHK_SET: | ||
1716 | if (GNUNET_OK != | ||
1717 | GNUNET_BIO_read (rh, "chk", &dr->chk, sizeof (struct ContentHashKey))) | ||
1718 | goto cleanup; | ||
1719 | break; | ||
1720 | case BRS_DOWNLOAD_DOWN: | ||
1721 | case BRS_DOWNLOAD_UP: | ||
1722 | case BRS_ERROR: | ||
1723 | break; | ||
1724 | default: | ||
1725 | GNUNET_break (0); | ||
1726 | goto cleanup; | ||
1727 | } | ||
1728 | for (i=0;i<dr->num_children;i++) | ||
1729 | { | ||
1730 | if (NULL == (dr->children[i] = read_download_request (rh))) | ||
1731 | goto cleanup; | ||
1732 | dr->children[i]->parent = dr; | ||
1733 | } | ||
1734 | return dr; | ||
1735 | cleanup: | ||
1736 | GNUNET_FS_free_download_request_ (dr); | ||
1737 | return NULL; | ||
1685 | } | 1738 | } |
1686 | 1739 | ||
1687 | 1740 | ||
@@ -1739,7 +1792,6 @@ GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc) | |||
1739 | char *uris; | 1792 | char *uris; |
1740 | char *fn; | 1793 | char *fn; |
1741 | char *dir; | 1794 | char *dir; |
1742 | uint32_t num_pending; | ||
1743 | 1795 | ||
1744 | if (NULL == dc->serialization) | 1796 | if (NULL == dc->serialization) |
1745 | { | 1797 | { |
@@ -1780,14 +1832,6 @@ GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc) | |||
1780 | GNUNET_assert ( (GNUNET_YES == GNUNET_FS_uri_test_chk (dc->uri)) || | 1832 | GNUNET_assert ( (GNUNET_YES == GNUNET_FS_uri_test_chk (dc->uri)) || |
1781 | (GNUNET_YES == GNUNET_FS_uri_test_loc (dc->uri)) ); | 1833 | (GNUNET_YES == GNUNET_FS_uri_test_loc (dc->uri)) ); |
1782 | uris = GNUNET_FS_uri_to_string (dc->uri); | 1834 | uris = GNUNET_FS_uri_to_string (dc->uri); |
1783 | num_pending = 0; | ||
1784 | if (dc->emsg == NULL) | ||
1785 | (void) GNUNET_CONTAINER_multihashmap_iterate (dc->active, | ||
1786 | &count_download_requests, | ||
1787 | &num_pending); | ||
1788 | GNUNET_assert ( (dc->length == dc->completed) || | ||
1789 | (dc->emsg != NULL) || | ||
1790 | (num_pending > 0) ); | ||
1791 | if ( (GNUNET_OK != | 1835 | if ( (GNUNET_OK != |
1792 | GNUNET_BIO_write_string (wh, uris)) || | 1836 | GNUNET_BIO_write_string (wh, uris)) || |
1793 | (GNUNET_OK != | 1837 | (GNUNET_OK != |
@@ -1813,22 +1857,20 @@ GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc) | |||
1813 | (GNUNET_OK != | 1857 | (GNUNET_OK != |
1814 | GNUNET_BIO_write_int32 (wh, (uint32_t) dc->options)) || | 1858 | GNUNET_BIO_write_int32 (wh, (uint32_t) dc->options)) || |
1815 | (GNUNET_OK != | 1859 | (GNUNET_OK != |
1816 | GNUNET_BIO_write_int32 (wh, (uint32_t) dc->has_finished)) || | 1860 | GNUNET_BIO_write_int32 (wh, (uint32_t) dc->has_finished)) ) |
1817 | (GNUNET_OK != | ||
1818 | GNUNET_BIO_write_int32 (wh, num_pending)) || | ||
1819 | (GNUNET_OK != | ||
1820 | GNUNET_BIO_write_int32 (wh, dc->start_task != GNUNET_SCHEDULER_NO_TASK)) ) | ||
1821 | { | 1861 | { |
1822 | GNUNET_break (0); | 1862 | GNUNET_break (0); |
1823 | goto cleanup; | 1863 | goto cleanup; |
1824 | } | 1864 | } |
1825 | if (GNUNET_SYSERR == | 1865 | if (NULL == dc->emsg) |
1826 | GNUNET_CONTAINER_multihashmap_iterate (dc->active, | ||
1827 | &write_download_request, | ||
1828 | wh)) | ||
1829 | { | 1866 | { |
1830 | GNUNET_break (0); | 1867 | GNUNET_assert (dc->top_request != NULL); |
1831 | goto cleanup; | 1868 | if (GNUNET_YES != |
1869 | write_download_request (wh, dc->top_request)) | ||
1870 | { | ||
1871 | GNUNET_break (0); | ||
1872 | goto cleanup; | ||
1873 | } | ||
1832 | } | 1874 | } |
1833 | GNUNET_free_non_null (uris); | 1875 | GNUNET_free_non_null (uris); |
1834 | uris = NULL; | 1876 | uris = NULL; |
@@ -2348,7 +2390,7 @@ signal_download_resume (struct GNUNET_FS_DownloadContext *dc) | |||
2348 | signal_download_resume (dcc); | 2390 | signal_download_resume (dcc); |
2349 | dcc = dcc->next; | 2391 | dcc = dcc->next; |
2350 | } | 2392 | } |
2351 | if (dc->pending != NULL) | 2393 | if (dc->pending_head != NULL) |
2352 | GNUNET_FS_download_start_downloading_ (dc); | 2394 | GNUNET_FS_download_start_downloading_ (dc); |
2353 | } | 2395 | } |
2354 | 2396 | ||
@@ -2537,7 +2579,7 @@ static void | |||
2537 | free_download_context (struct GNUNET_FS_DownloadContext *dc) | 2579 | free_download_context (struct GNUNET_FS_DownloadContext *dc) |
2538 | { | 2580 | { |
2539 | struct GNUNET_FS_DownloadContext *dcc; | 2581 | struct GNUNET_FS_DownloadContext *dcc; |
2540 | struct DownloadRequest *dr; | 2582 | |
2541 | if (dc->meta != NULL) | 2583 | if (dc->meta != NULL) |
2542 | GNUNET_CONTAINER_meta_data_destroy (dc->meta); | 2584 | GNUNET_CONTAINER_meta_data_destroy (dc->meta); |
2543 | if (dc->uri != NULL) | 2585 | if (dc->uri != NULL) |
@@ -2552,11 +2594,7 @@ free_download_context (struct GNUNET_FS_DownloadContext *dc) | |||
2552 | dcc); | 2594 | dcc); |
2553 | free_download_context (dcc); | 2595 | free_download_context (dcc); |
2554 | } | 2596 | } |
2555 | while (NULL != (dr = dc->pending)) | 2597 | GNUNET_FS_free_download_request_ (dc->top_request); |
2556 | { | ||
2557 | dc->pending = dr->next; | ||
2558 | GNUNET_free (dr); | ||
2559 | } | ||
2560 | GNUNET_free (dc); | 2598 | GNUNET_free (dc); |
2561 | } | 2599 | } |
2562 | 2600 | ||
@@ -2584,8 +2622,6 @@ deserialize_download (struct GNUNET_FS_Handle *h, | |||
2584 | char *dn; | 2622 | char *dn; |
2585 | uint32_t options; | 2623 | uint32_t options; |
2586 | uint32_t status; | 2624 | uint32_t status; |
2587 | uint32_t num_pending; | ||
2588 | int32_t start_pending; | ||
2589 | 2625 | ||
2590 | uris = NULL; | 2626 | uris = NULL; |
2591 | emsg = NULL; | 2627 | emsg = NULL; |
@@ -2622,51 +2658,27 @@ deserialize_download (struct GNUNET_FS_Handle *h, | |||
2622 | (GNUNET_OK != | 2658 | (GNUNET_OK != |
2623 | GNUNET_BIO_read_int32 (rh, &options)) || | 2659 | GNUNET_BIO_read_int32 (rh, &options)) || |
2624 | (GNUNET_OK != | 2660 | (GNUNET_OK != |
2625 | GNUNET_BIO_read_int32 (rh, &status)) || | 2661 | GNUNET_BIO_read_int32 (rh, &status)) ) |
2626 | (GNUNET_OK != | ||
2627 | GNUNET_BIO_read_int32 (rh, &num_pending)) || | ||
2628 | (GNUNET_OK != | ||
2629 | GNUNET_BIO_read_int32 (rh, &start_pending)) ) | ||
2630 | { | 2662 | { |
2631 | GNUNET_break (0); | 2663 | GNUNET_break (0); |
2632 | goto cleanup; | 2664 | goto cleanup; |
2633 | } | 2665 | } |
2634 | dc->options = (enum GNUNET_FS_DownloadOptions) options; | 2666 | dc->options = (enum GNUNET_FS_DownloadOptions) options; |
2635 | dc->active = GNUNET_CONTAINER_multihashmap_create (16); | 2667 | dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (dc->length / DBLOCK_SIZE)); |
2636 | dc->has_finished = (int) status; | 2668 | dc->has_finished = (int) status; |
2637 | dc->treedepth = GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size (dc->uri)); | 2669 | dc->treedepth = GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size (dc->uri)); |
2638 | if (GNUNET_FS_uri_test_loc (dc->uri)) | 2670 | if (GNUNET_FS_uri_test_loc (dc->uri)) |
2639 | GNUNET_assert (GNUNET_OK == | 2671 | GNUNET_assert (GNUNET_OK == |
2640 | GNUNET_FS_uri_loc_get_peer_identity (dc->uri, | 2672 | GNUNET_FS_uri_loc_get_peer_identity (dc->uri, |
2641 | &dc->target)); | 2673 | &dc->target)); |
2642 | if ( (dc->length > dc->completed) && | 2674 | if (dc->emsg == NULL) |
2643 | (num_pending == 0) ) | ||
2644 | { | ||
2645 | GNUNET_break (0); | ||
2646 | goto cleanup; | ||
2647 | } | ||
2648 | while (0 < num_pending--) | ||
2649 | { | 2675 | { |
2650 | dr = GNUNET_malloc (sizeof (struct DownloadRequest)); | 2676 | dc->top_request = read_download_request (rh); |
2651 | if ( (GNUNET_OK != | 2677 | if (dc->top_request == NULL) |
2652 | GNUNET_BIO_read (rh, "chk", &dr->chk, sizeof (struct ContentHashKey))) || | ||
2653 | (GNUNET_OK != | ||
2654 | GNUNET_BIO_read_int64 (rh, &dr->offset)) || | ||
2655 | (GNUNET_OK != | ||
2656 | GNUNET_BIO_read_int32 (rh, &dr->depth)) ) | ||
2657 | { | 2678 | { |
2658 | GNUNET_break (0); | 2679 | GNUNET_break (0); |
2659 | goto cleanup; | 2680 | goto cleanup; |
2660 | } | 2681 | } |
2661 | dr->is_pending = GNUNET_YES; | ||
2662 | dr->next = dc->pending; | ||
2663 | dc->pending = dr; | ||
2664 | GNUNET_CONTAINER_multihashmap_put (dc->active, | ||
2665 | &dr->chk.query, | ||
2666 | dr, | ||
2667 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); | ||
2668 | |||
2669 | dr = NULL; | ||
2670 | } | 2682 | } |
2671 | dn = get_download_sync_filename (dc, dc->serialization, ".dir"); | 2683 | dn = get_download_sync_filename (dc, dc->serialization, ".dir"); |
2672 | if (dn != NULL) | 2684 | if (dn != NULL) |
@@ -2697,13 +2709,11 @@ deserialize_download (struct GNUNET_FS_Handle *h, | |||
2697 | signal_download_resume (dc); | 2709 | signal_download_resume (dc); |
2698 | } | 2710 | } |
2699 | GNUNET_free (uris); | 2711 | GNUNET_free (uris); |
2700 | if (start_pending) | 2712 | dc->task |
2701 | dc->start_task | 2713 | = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); |
2702 | = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); | ||
2703 | return; | 2714 | return; |
2704 | cleanup: | 2715 | cleanup: |
2705 | GNUNET_free_non_null (uris); | 2716 | GNUNET_free_non_null (uris); |
2706 | GNUNET_free_non_null (dr); | ||
2707 | GNUNET_free_non_null (emsg); | 2717 | GNUNET_free_non_null (emsg); |
2708 | free_download_context (dc); | 2718 | free_download_context (dc); |
2709 | } | 2719 | } |
diff --git a/src/fs/fs.h b/src/fs/fs.h index 0ba1d08a1..a9aa7ef74 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h | |||
@@ -925,6 +925,7 @@ GNUNET_FS_download_start_task_ (void *cls, | |||
925 | const struct GNUNET_SCHEDULER_TaskContext *tc); | 925 | const struct GNUNET_SCHEDULER_TaskContext *tc); |
926 | 926 | ||
927 | 927 | ||
928 | |||
928 | /** | 929 | /** |
929 | * Fill in all of the generic fields for | 930 | * Fill in all of the generic fields for |
930 | * an unindex event and call the callback. | 931 | * an unindex event and call the callback. |
@@ -1612,7 +1613,7 @@ struct GNUNET_FS_SearchContext | |||
1612 | * when the search is being stopped (if not | 1613 | * when the search is being stopped (if not |
1613 | * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some | 1614 | * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some |
1614 | * artificial delay when trying to reconnect to the FS service. | 1615 | * artificial delay when trying to reconnect to the FS service. |
1615 | o */ | 1616 | */ |
1616 | GNUNET_SCHEDULER_TaskIdentifier task; | 1617 | GNUNET_SCHEDULER_TaskIdentifier task; |
1617 | 1618 | ||
1618 | /** | 1619 | /** |
@@ -1645,37 +1646,131 @@ o */ | |||
1645 | 1646 | ||
1646 | 1647 | ||
1647 | /** | 1648 | /** |
1649 | * FSM for possible states a block can go through. The typical | ||
1650 | * order of progression is linear through the states, alternatives | ||
1651 | * are documented in the comments. | ||
1652 | */ | ||
1653 | enum BlockRequestState | ||
1654 | { | ||
1655 | /** | ||
1656 | * Initial state, block has only been allocated (since it is | ||
1657 | * relevant to the overall download request). | ||
1658 | */ | ||
1659 | BRS_INIT = 0, | ||
1660 | |||
1661 | /** | ||
1662 | * We've checked the block on the path down the tree, and the | ||
1663 | * content on disk did match the desired CHK, but not all | ||
1664 | * the way down, so at the bottom some blocks will still | ||
1665 | * need to be reconstructed). | ||
1666 | */ | ||
1667 | BRS_RECONSTRUCT_DOWN = 1, | ||
1668 | |||
1669 | /** | ||
1670 | * We've calculated the CHK bottom-up based on the meta data. | ||
1671 | * This may work, but if it did we have to write the meta data to | ||
1672 | * disk at the end (and we still need to check against the | ||
1673 | * CHK set on top). | ||
1674 | */ | ||
1675 | BRS_RECONSTRUCT_META_UP = 2, | ||
1676 | |||
1677 | /** | ||
1678 | * We've calculated the CHK bottom-up based on what we have on | ||
1679 | * disk, which may not be what the desired CHK is. If the | ||
1680 | * reconstructed CHKs match whatever comes from above, we're | ||
1681 | * done with the respective subtree. | ||
1682 | */ | ||
1683 | BRS_RECONSTRUCT_UP = 3, | ||
1684 | |||
1685 | /** | ||
1686 | * We've determined the real, desired CHK for this block | ||
1687 | * (full tree reconstruction failed), request is now pending. | ||
1688 | * If the CHK that bubbled up through reconstruction did match | ||
1689 | * the top-level request, the state machine for the subtree | ||
1690 | * would have moved to BRS_DOWNLOAD_UP. | ||
1691 | */ | ||
1692 | BRS_CHK_SET = 4, | ||
1693 | |||
1694 | /** | ||
1695 | * We've successfully downloaded this block, but the children | ||
1696 | * still need to be either downloaded or verified (download | ||
1697 | * request propagates down). If the download fails, the | ||
1698 | * state machine for this block may move to | ||
1699 | * BRS_DOWNLOAD_ERROR instead. | ||
1700 | */ | ||
1701 | BRS_DOWNLOAD_DOWN = 5, | ||
1702 | |||
1703 | /** | ||
1704 | * This block and all of its children have been downloaded | ||
1705 | * successfully (full completion propagates up). | ||
1706 | */ | ||
1707 | BRS_DOWNLOAD_UP = 6, | ||
1708 | |||
1709 | /** | ||
1710 | * We got a block back that matched the query but did not hash to | ||
1711 | * the key (malicious publisher or hash collision); this block | ||
1712 | * can never be downloaded (error propagates up). | ||
1713 | */ | ||
1714 | BRS_ERROR = 7 | ||
1715 | |||
1716 | }; | ||
1717 | |||
1718 | |||
1719 | /** | ||
1648 | * Information about an active download request. | 1720 | * Information about an active download request. |
1649 | */ | 1721 | */ |
1650 | struct DownloadRequest | 1722 | struct DownloadRequest |
1651 | { | 1723 | { |
1652 | /** | 1724 | /** |
1653 | * While pending, we keep all download requests in a linked list. | 1725 | * While pending, we keep all download requests in a doubly-linked list. |
1654 | */ | 1726 | */ |
1655 | struct DownloadRequest *next; | 1727 | struct DownloadRequest *next; |
1656 | 1728 | ||
1657 | /** | 1729 | /** |
1658 | * CHK for the request. | 1730 | * While pending, we keep all download requests in a doubly-linked list. |
1659 | */ | 1731 | */ |
1660 | struct ContentHashKey chk; | 1732 | struct DownloadRequest *prev; |
1733 | |||
1734 | /** | ||
1735 | * Parent in the CHK-tree. | ||
1736 | */ | ||
1737 | struct DownloadRequest *parent; | ||
1738 | |||
1739 | /** | ||
1740 | * Array (!) of child-requests, or NULL for the bottom of the tree. | ||
1741 | */ | ||
1742 | struct DownloadRequest **children; | ||
1661 | 1743 | ||
1662 | /** | 1744 | /** |
1663 | * Offset of the corresponding block. | 1745 | * CHK for the request for this block (set during reconstruction |
1746 | * to what we have on disk, later to what we want to have). | ||
1747 | */ | ||
1748 | struct ContentHashKey chk; | ||
1749 | |||
1750 | /** | ||
1751 | * Offset of the corresponding block. Specifically, first (!) byte of | ||
1752 | * the first DBLOCK in the subtree induced by block represented by | ||
1753 | * this request. | ||
1664 | */ | 1754 | */ |
1665 | uint64_t offset; | 1755 | uint64_t offset; |
1666 | 1756 | ||
1667 | /** | 1757 | /** |
1668 | * Depth of the corresponding block in the tree. | 1758 | * Number of entries in 'children' array. |
1759 | */ | ||
1760 | unsigned int num_children; | ||
1761 | |||
1762 | /** | ||
1763 | * Depth of the corresponding block in the tree. 0==DBLOCKs. | ||
1669 | */ | 1764 | */ |
1670 | unsigned int depth; | 1765 | unsigned int depth; |
1671 | 1766 | ||
1672 | /** | 1767 | /** |
1673 | * Set if this request is currently in the linked list of pending | 1768 | * State in the FSM. |
1674 | * requests. Needed in case we get a response for a request that we | 1769 | */ |
1675 | * have not yet send (i.e. due to two blocks with identical | 1770 | enum BlockRequestState state; |
1676 | * content); in this case, we would need to remove the block from | 1771 | |
1677 | * the pending list (and need a fast way to check if the block is on | 1772 | /** |
1678 | * it). | 1773 | * GNUNET_YES if this entry is in the pending list. |
1679 | */ | 1774 | */ |
1680 | int is_pending; | 1775 | int is_pending; |
1681 | 1776 | ||
@@ -1683,9 +1778,12 @@ struct DownloadRequest | |||
1683 | 1778 | ||
1684 | 1779 | ||
1685 | /** | 1780 | /** |
1686 | * Closure for 'reconstruct_cont' and 'reconstruct_cb'. | 1781 | * (recursively) free download request structure |
1782 | * | ||
1783 | * @param dr request to free | ||
1687 | */ | 1784 | */ |
1688 | struct ReconstructContext; | 1785 | void |
1786 | GNUNET_FS_free_download_request_ (struct DownloadRequest *dr); | ||
1689 | 1787 | ||
1690 | 1788 | ||
1691 | /** | 1789 | /** |
@@ -1732,11 +1830,6 @@ struct GNUNET_FS_DownloadContext | |||
1732 | struct GNUNET_FS_DownloadContext *child_tail; | 1830 | struct GNUNET_FS_DownloadContext *child_tail; |
1733 | 1831 | ||
1734 | /** | 1832 | /** |
1735 | * State for block reconstruction. | ||
1736 | */ | ||
1737 | struct ReconstructContext *rcc; | ||
1738 | |||
1739 | /** | ||
1740 | * Previous download belonging to the same parent. | 1833 | * Previous download belonging to the same parent. |
1741 | */ | 1834 | */ |
1742 | struct GNUNET_FS_DownloadContext *prev; | 1835 | struct GNUNET_FS_DownloadContext *prev; |
@@ -1752,8 +1845,7 @@ struct GNUNET_FS_DownloadContext | |||
1752 | void *client_info; | 1845 | void *client_info; |
1753 | 1846 | ||
1754 | /** | 1847 | /** |
1755 | * URI that identifies the file that | 1848 | * URI that identifies the file that we are downloading. |
1756 | * we are downloading. | ||
1757 | */ | 1849 | */ |
1758 | struct GNUNET_FS_Uri *uri; | 1850 | struct GNUNET_FS_Uri *uri; |
1759 | 1851 | ||
@@ -1787,16 +1879,9 @@ struct GNUNET_FS_DownloadContext | |||
1787 | char *temp_filename; | 1879 | char *temp_filename; |
1788 | 1880 | ||
1789 | /** | 1881 | /** |
1790 | * Map of active requests (those waiting | 1882 | * Our entry in the job queue. |
1791 | * for a response). The key is the hash | ||
1792 | * of the encryped block (aka query). | ||
1793 | */ | ||
1794 | struct GNUNET_CONTAINER_MultiHashMap *active; | ||
1795 | |||
1796 | /** | ||
1797 | * Linked list of pending requests. | ||
1798 | */ | 1883 | */ |
1799 | struct DownloadRequest *pending; | 1884 | struct GNUNET_FS_QueueEntry *job_queue; |
1800 | 1885 | ||
1801 | /** | 1886 | /** |
1802 | * Non-NULL if we are currently having a request for | 1887 | * Non-NULL if we are currently having a request for |
@@ -1805,38 +1890,52 @@ struct GNUNET_FS_DownloadContext | |||
1805 | struct GNUNET_CLIENT_TransmitHandle *th; | 1890 | struct GNUNET_CLIENT_TransmitHandle *th; |
1806 | 1891 | ||
1807 | /** | 1892 | /** |
1808 | * Our entry in the job queue. | 1893 | * Tree encoder used for the reconstruction. |
1809 | */ | 1894 | */ |
1810 | struct GNUNET_FS_QueueEntry *job_queue; | 1895 | struct GNUNET_FS_TreeEncoder *te; |
1811 | 1896 | ||
1812 | /** | 1897 | /** |
1813 | * Identity of the peer having the content, or all-zeros | 1898 | * File handle for reading data from an existing file |
1814 | * if we don't know of such a peer. | 1899 | * (to pass to tree encoder). |
1815 | */ | 1900 | */ |
1816 | struct GNUNET_PeerIdentity target; | 1901 | struct GNUNET_DISK_FileHandle *rfh; |
1817 | 1902 | ||
1818 | /** | 1903 | /** |
1819 | * ID of a task that is using this struct | 1904 | * Map of active requests (those waiting for a response). The key |
1820 | * and that must be cancelled when the download | 1905 | * is the hash of the encryped block (aka query). |
1821 | * is being stopped (if not GNUNET_SCHEDULER_NO_TASK). | ||
1822 | * Used for the task that adds some artificial | ||
1823 | * delay when trying to reconnect to the FS | ||
1824 | * service. | ||
1825 | */ | 1906 | */ |
1826 | GNUNET_SCHEDULER_TaskIdentifier task; | 1907 | struct GNUNET_CONTAINER_MultiHashMap *active; |
1827 | 1908 | ||
1828 | /** | 1909 | /** |
1829 | * Task used to start the download. | 1910 | * Head of linked list of pending requests. |
1830 | */ | 1911 | */ |
1831 | GNUNET_SCHEDULER_TaskIdentifier start_task; | 1912 | struct DownloadRequest *pending_head; |
1832 | 1913 | ||
1833 | /** | 1914 | /** |
1834 | * What was the size of the file on disk that we're downloading | 1915 | * Head of linked list of pending requests. |
1835 | * before we started? Used to detect if there is a point in | ||
1836 | * checking an existing block on disk for matching the desired | ||
1837 | * content. 0 if the file did not exist already. | ||
1838 | */ | 1916 | */ |
1839 | uint64_t old_file_size; | 1917 | struct DownloadRequest *pending_tail; |
1918 | |||
1919 | /** | ||
1920 | * Top-level download request. | ||
1921 | */ | ||
1922 | struct DownloadRequest *top_request; | ||
1923 | |||
1924 | /** | ||
1925 | * Identity of the peer having the content, or all-zeros | ||
1926 | * if we don't know of such a peer. | ||
1927 | */ | ||
1928 | struct GNUNET_PeerIdentity target; | ||
1929 | |||
1930 | /** | ||
1931 | * ID of a task that is using this struct and that must be cancelled | ||
1932 | * when the download is being stopped (if not | ||
1933 | * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some | ||
1934 | * artificial delay when trying to reconnect to the FS service or | ||
1935 | * the task processing incrementally the data on disk, or the | ||
1936 | * task requesting blocks, etc. | ||
1937 | */ | ||
1938 | GNUNET_SCHEDULER_TaskIdentifier task; | ||
1840 | 1939 | ||
1841 | /** | 1940 | /** |
1842 | * What is the first offset that we're interested | 1941 | * What is the first offset that we're interested |
@@ -1857,6 +1956,14 @@ struct GNUNET_FS_DownloadContext | |||
1857 | uint64_t completed; | 1956 | uint64_t completed; |
1858 | 1957 | ||
1859 | /** | 1958 | /** |
1959 | * What was the size of the file on disk that we're downloading | ||
1960 | * before we started? Used to detect if there is a point in | ||
1961 | * checking an existing block on disk for matching the desired | ||
1962 | * content. 0 if the file did not exist already. | ||
1963 | */ | ||
1964 | uint64_t old_file_size; | ||
1965 | |||
1966 | /** | ||
1860 | * Time download was started. | 1967 | * Time download was started. |
1861 | */ | 1968 | */ |
1862 | struct GNUNET_TIME_Absolute start_time; | 1969 | struct GNUNET_TIME_Absolute start_time; |
@@ -1883,17 +1990,6 @@ struct GNUNET_FS_DownloadContext | |||
1883 | */ | 1990 | */ |
1884 | int has_finished; | 1991 | int has_finished; |
1885 | 1992 | ||
1886 | /** | ||
1887 | * Have we tried (and failed) to find matching full | ||
1888 | * data from the meta data yet? | ||
1889 | */ | ||
1890 | int tried_full_data; | ||
1891 | |||
1892 | /** | ||
1893 | * Have we tried to reconstruct an IBLOCK from disk | ||
1894 | * and failed (and should hence not try again?) | ||
1895 | */ | ||
1896 | int reconstruct_failed; | ||
1897 | }; | 1993 | }; |
1898 | 1994 | ||
1899 | 1995 | ||
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index d7f1de283..8d2e1b7a4 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c | |||
@@ -1,6 +1,6 @@ | |||
1 | /* | 1 | /* |
2 | This file is part of GNUnet. | 2 | This file is part of GNUnet. |
3 | (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors) | 3 | (C) 2001-2011 Christian Grothoff (and other contributing authors) |
4 | 4 | ||
5 | GNUnet is free software; you can redistribute it and/or modify | 5 | GNUnet is free software; you can redistribute it and/or modify |
6 | it under the terms of the GNU General Public License as published | 6 | it under the terms of the GNU General Public License as published |
@@ -61,8 +61,7 @@ is_recursive_download (struct GNUNET_FS_DownloadContext *dc) | |||
61 | * | 61 | * |
62 | * @param fsize overall file size | 62 | * @param fsize overall file size |
63 | * @param off offset of the block in the file | 63 | * @param off offset of the block in the file |
64 | * @param depth depth of the block in the tree | 64 | * @param depth depth of the block in the tree, 0 for DBLOCK |
65 | * @param treedepth maximum depth of the tree | ||
66 | * @return off for DBLOCKS (depth == treedepth), | 65 | * @return off for DBLOCKS (depth == treedepth), |
67 | * otherwise an offset past the end | 66 | * otherwise an offset past the end |
68 | * of the file that does not overlap | 67 | * of the file that does not overlap |
@@ -71,23 +70,22 @@ is_recursive_download (struct GNUNET_FS_DownloadContext *dc) | |||
71 | static uint64_t | 70 | static uint64_t |
72 | compute_disk_offset (uint64_t fsize, | 71 | compute_disk_offset (uint64_t fsize, |
73 | uint64_t off, | 72 | uint64_t off, |
74 | unsigned int depth, | 73 | unsigned int depth) |
75 | unsigned int treedepth) | ||
76 | { | 74 | { |
77 | unsigned int i; | 75 | unsigned int i; |
78 | uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */ | 76 | uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */ |
79 | uint64_t loff; /* where do IBlocks for depth "i" start? */ | 77 | uint64_t loff; /* where do IBlocks for depth "i" start? */ |
80 | unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */ | 78 | unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */ |
81 | 79 | ||
82 | if (depth == treedepth) | 80 | if (depth == 0) |
83 | return off; | 81 | return off; |
84 | /* first IBlocks start at the end of file, rounded up | 82 | /* first IBlocks start at the end of file, rounded up |
85 | to full DBLOCK_SIZE */ | 83 | to full DBLOCK_SIZE */ |
86 | loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE; | 84 | loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE; |
87 | lsize = ( (fsize + DBLOCK_SIZE-1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey); | 85 | lsize = ( (fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey); |
88 | GNUNET_assert (0 == (off % DBLOCK_SIZE)); | 86 | GNUNET_assert (0 == (off % DBLOCK_SIZE)); |
89 | ioff = (off / DBLOCK_SIZE); | 87 | ioff = (off / DBLOCK_SIZE); |
90 | for (i=treedepth-1;i>depth;i--) | 88 | for (i=1;i<depth;i++) |
91 | { | 89 | { |
92 | loff += lsize; | 90 | loff += lsize; |
93 | lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE; | 91 | lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE; |
@@ -100,35 +98,32 @@ compute_disk_offset (uint64_t fsize, | |||
100 | 98 | ||
101 | 99 | ||
102 | /** | 100 | /** |
103 | * Given a file of the specified treedepth and a block at the given | 101 | * Given a block at the given offset and depth, calculate the offset |
104 | * offset and depth, calculate the offset for the CHK at the given | 102 | * for the CHK at the given index. |
105 | * index. | ||
106 | * | 103 | * |
107 | * @param offset the offset of the first | 104 | * @param offset the offset of the first |
108 | * DBLOCK in the subtree of the | 105 | * DBLOCK in the subtree of the |
109 | * identified IBLOCK | 106 | * identified IBLOCK |
110 | * @param depth the depth of the IBLOCK in the tree | 107 | * @param depth the depth of the IBLOCK in the tree, 0 for DBLOCK |
111 | * @param treedepth overall depth of the tree | ||
112 | * @param k which CHK in the IBLOCK are we | 108 | * @param k which CHK in the IBLOCK are we |
113 | * talking about | 109 | * talking about |
114 | * @return offset if k=0, otherwise an appropriately | 110 | * @return offset if k=0, otherwise an appropriately |
115 | * larger value (i.e., if depth = treedepth-1, | 111 | * larger value (i.e., if depth = 1, |
116 | * the returned value should be offset+DBLOCK_SIZE) | 112 | * the returned value should be offset+k*DBLOCK_SIZE) |
117 | */ | 113 | */ |
118 | static uint64_t | 114 | static uint64_t |
119 | compute_dblock_offset (uint64_t offset, | 115 | compute_dblock_offset (uint64_t offset, |
120 | unsigned int depth, | 116 | unsigned int depth, |
121 | unsigned int treedepth, | ||
122 | unsigned int k) | 117 | unsigned int k) |
123 | { | 118 | { |
124 | unsigned int i; | 119 | unsigned int i; |
125 | uint64_t lsize; /* what is the size of the sum of all DBlocks | 120 | uint64_t lsize; /* what is the size of the sum of all DBlocks |
126 | that a CHK at depth i corresponds to? */ | 121 | that a CHK at depth i corresponds to? */ |
127 | 122 | ||
128 | if (depth == treedepth) | 123 | if (depth == 0) |
129 | return offset; | 124 | return offset; |
130 | lsize = DBLOCK_SIZE; | 125 | lsize = DBLOCK_SIZE; |
131 | for (i=treedepth-1;i>depth;i--) | 126 | for (i=1;i<depth;i++) |
132 | lsize *= CHK_PER_INODE; | 127 | lsize *= CHK_PER_INODE; |
133 | return offset + k * lsize; | 128 | return offset + k * lsize; |
134 | } | 129 | } |
@@ -177,6 +172,7 @@ GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, | |||
177 | pi); | 172 | pi); |
178 | } | 173 | } |
179 | 174 | ||
175 | |||
180 | /** | 176 | /** |
181 | * We're ready to transmit a search request to the | 177 | * We're ready to transmit a search request to the |
182 | * file-sharing service. Do it. If there is | 178 | * file-sharing service. Do it. If there is |
@@ -255,20 +251,18 @@ process_result_with_request (void *cls, | |||
255 | * | 251 | * |
256 | * @param dc download in question | 252 | * @param dc download in question |
257 | * @param chk request this relates to | 253 | * @param chk request this relates to |
258 | * @param sm request details | 254 | * @param dr request details |
259 | * @param block plaintext data matching request | 255 | * @param block plaintext data matching request |
260 | * @param len number of bytes in block | 256 | * @param len number of bytes in block |
261 | * @param depth depth of the block | ||
262 | * @param do_store should we still store the block on disk? | 257 | * @param do_store should we still store the block on disk? |
263 | * @return GNUNET_OK on success | 258 | * @return GNUNET_OK on success |
264 | */ | 259 | */ |
265 | static int | 260 | static int |
266 | encrypt_existing_match (struct GNUNET_FS_DownloadContext *dc, | 261 | encrypt_existing_match (struct GNUNET_FS_DownloadContext *dc, |
267 | const struct ContentHashKey *chk, | 262 | const struct ContentHashKey *chk, |
268 | struct DownloadRequest *sm, | 263 | struct DownloadRequest *dr, |
269 | const char * block, | 264 | const char * block, |
270 | size_t len, | 265 | size_t len, |
271 | int depth, | ||
272 | int do_store) | 266 | int do_store) |
273 | { | 267 | { |
274 | struct ProcessResultClosure prc; | 268 | struct ProcessResultClosure prc; |
@@ -302,613 +296,19 @@ encrypt_existing_match (struct GNUNET_FS_DownloadContext *dc, | |||
302 | prc.dc = dc; | 296 | prc.dc = dc; |
303 | prc.data = enc; | 297 | prc.data = enc; |
304 | prc.size = len; | 298 | prc.size = len; |
305 | prc.type = (dc->treedepth == depth) | 299 | prc.type = (0 == dr->depth) |
306 | ? GNUNET_BLOCK_TYPE_FS_DBLOCK | 300 | ? GNUNET_BLOCK_TYPE_FS_DBLOCK |
307 | : GNUNET_BLOCK_TYPE_FS_IBLOCK; | 301 | : GNUNET_BLOCK_TYPE_FS_IBLOCK; |
308 | prc.query = chk->query; | 302 | prc.query = chk->query; |
309 | prc.do_store = do_store; | 303 | prc.do_store = do_store; |
310 | process_result_with_request (&prc, | 304 | process_result_with_request (&prc, |
311 | &chk->key, | 305 | &chk->key, |
312 | sm); | 306 | dr); |
313 | return GNUNET_OK; | 307 | return GNUNET_OK; |
314 | } | 308 | } |
315 | 309 | ||
316 | 310 | ||
317 | /** | 311 | /** |
318 | * Closure for match_full_data. | ||
319 | */ | ||
320 | struct MatchDataContext | ||
321 | { | ||
322 | /** | ||
323 | * CHK we are looking for. | ||
324 | */ | ||
325 | const struct ContentHashKey *chk; | ||
326 | |||
327 | /** | ||
328 | * Download we're processing. | ||
329 | */ | ||
330 | struct GNUNET_FS_DownloadContext *dc; | ||
331 | |||
332 | /** | ||
333 | * Request details. | ||
334 | */ | ||
335 | struct DownloadRequest *sm; | ||
336 | |||
337 | /** | ||
338 | * Overall offset in the file. | ||
339 | */ | ||
340 | uint64_t offset; | ||
341 | |||
342 | /** | ||
343 | * Desired length of the block. | ||
344 | */ | ||
345 | size_t len; | ||
346 | |||
347 | /** | ||
348 | * Flag set to GNUNET_YES on success. | ||
349 | */ | ||
350 | int done; | ||
351 | }; | ||
352 | |||
353 | /** | ||
354 | * Type of a function that libextractor calls for each | ||
355 | * meta data item found. | ||
356 | * | ||
357 | * @param cls closure (user-defined) | ||
358 | * @param plugin_name name of the plugin that produced this value; | ||
359 | * special values can be used (i.e. '<zlib>' for zlib being | ||
360 | * used in the main libextractor library and yielding | ||
361 | * meta data). | ||
362 | * @param type libextractor-type describing the meta data | ||
363 | * @param format basic format information about data | ||
364 | * @param data_mime_type mime-type of data (not of the original file); | ||
365 | * can be NULL (if mime-type is not known) | ||
366 | * @param data actual meta-data found | ||
367 | * @param data_len number of bytes in data | ||
368 | * @return 0 to continue extracting, 1 to abort | ||
369 | */ | ||
370 | static int | ||
371 | match_full_data (void *cls, | ||
372 | const char *plugin_name, | ||
373 | enum EXTRACTOR_MetaType type, | ||
374 | enum EXTRACTOR_MetaFormat format, | ||
375 | const char *data_mime_type, | ||
376 | const char *data, | ||
377 | size_t data_len) | ||
378 | { | ||
379 | struct MatchDataContext *mdc = cls; | ||
380 | GNUNET_HashCode key; | ||
381 | |||
382 | if (type == EXTRACTOR_METATYPE_GNUNET_FULL_DATA) | ||
383 | { | ||
384 | if ( (mdc->offset > data_len) || | ||
385 | (mdc->offset + mdc->len > data_len) ) | ||
386 | return 1; | ||
387 | GNUNET_CRYPTO_hash (&data[mdc->offset], | ||
388 | mdc->len, | ||
389 | &key); | ||
390 | if (0 != memcmp (&key, | ||
391 | &mdc->chk->key, | ||
392 | sizeof (GNUNET_HashCode))) | ||
393 | { | ||
394 | GNUNET_break_op (0); | ||
395 | return 1; | ||
396 | } | ||
397 | /* match found! */ | ||
398 | if (GNUNET_OK != | ||
399 | encrypt_existing_match (mdc->dc, | ||
400 | mdc->chk, | ||
401 | mdc->sm, | ||
402 | &data[mdc->offset], | ||
403 | mdc->len, | ||
404 | 0, | ||
405 | GNUNET_YES)) | ||
406 | { | ||
407 | GNUNET_break_op (0); | ||
408 | return 1; | ||
409 | } | ||
410 | mdc->done = GNUNET_YES; | ||
411 | return 1; | ||
412 | } | ||
413 | return 0; | ||
414 | } | ||
415 | |||
416 | |||
417 | |||
418 | /** | ||
419 | * Closure for 'reconstruct_cont' and 'reconstruct_cb'. | ||
420 | */ | ||
421 | struct ReconstructContext | ||
422 | { | ||
423 | /** | ||
424 | * File handle open for the reconstruction. | ||
425 | */ | ||
426 | struct GNUNET_DISK_FileHandle *fh; | ||
427 | |||
428 | /** | ||
429 | * the download context. | ||
430 | */ | ||
431 | struct GNUNET_FS_DownloadContext *dc; | ||
432 | |||
433 | /** | ||
434 | * Tree encoder used for the reconstruction. | ||
435 | */ | ||
436 | struct GNUNET_FS_TreeEncoder *te; | ||
437 | |||
438 | /** | ||
439 | * CHK of block we are trying to reconstruct. | ||
440 | */ | ||
441 | struct ContentHashKey chk; | ||
442 | |||
443 | /** | ||
444 | * Request that was generated. | ||
445 | */ | ||
446 | struct DownloadRequest *sm; | ||
447 | |||
448 | /** | ||
449 | * Helper task. | ||
450 | */ | ||
451 | GNUNET_SCHEDULER_TaskIdentifier task; | ||
452 | |||
453 | /** | ||
454 | * Offset of block we are trying to reconstruct. | ||
455 | */ | ||
456 | uint64_t offset; | ||
457 | |||
458 | /** | ||
459 | * Depth of block we are trying to reconstruct. | ||
460 | */ | ||
461 | unsigned int depth; | ||
462 | |||
463 | }; | ||
464 | |||
465 | |||
466 | /** | ||
467 | * Continuation after a possible attempt to reconstruct | ||
468 | * the current IBlock from the existing file. | ||
469 | * | ||
470 | * @param cls the 'struct ReconstructContext' | ||
471 | * @param tc scheduler context | ||
472 | */ | ||
473 | static void | ||
474 | reconstruct_cont (void *cls, | ||
475 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
476 | { | ||
477 | struct ReconstructContext *rcc = cls; | ||
478 | |||
479 | if (rcc->te != NULL) | ||
480 | { | ||
481 | GNUNET_FS_tree_encoder_finish (rcc->te, NULL, NULL); | ||
482 | } | ||
483 | rcc->dc->reconstruct_failed = GNUNET_YES; | ||
484 | rcc->dc->rcc = NULL; | ||
485 | if (rcc->fh != NULL) | ||
486 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (rcc->fh)); | ||
487 | if ( (rcc->dc->th == NULL) && | ||
488 | (rcc->dc->client != NULL) ) | ||
489 | { | ||
490 | #if DEBUG_DOWNLOAD | ||
491 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
492 | "Asking for transmission to FS service\n"); | ||
493 | #endif | ||
494 | rcc->dc->th = GNUNET_CLIENT_notify_transmit_ready (rcc->dc->client, | ||
495 | sizeof (struct SearchMessage), | ||
496 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | ||
497 | GNUNET_NO, | ||
498 | &transmit_download_request, | ||
499 | rcc->dc); | ||
500 | } | ||
501 | else | ||
502 | { | ||
503 | #if DEBUG_DOWNLOAD | ||
504 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
505 | "Transmission request not issued (%p %p)\n", | ||
506 | rcc->dc->th, | ||
507 | rcc->dc->client); | ||
508 | #endif | ||
509 | } | ||
510 | GNUNET_free (rcc); | ||
511 | } | ||
512 | |||
513 | |||
514 | static void | ||
515 | get_next_block (void *cls, | ||
516 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
517 | { | ||
518 | struct ReconstructContext *rcc = cls; | ||
519 | |||
520 | rcc->task = GNUNET_SCHEDULER_NO_TASK; | ||
521 | GNUNET_FS_tree_encoder_next (rcc->te); | ||
522 | } | ||
523 | |||
524 | |||
525 | /** | ||
526 | * Function called asking for the current (encoded) | ||
527 | * block to be processed. After processing the | ||
528 | * client should either call "GNUNET_FS_tree_encode_next" | ||
529 | * or (on error) "GNUNET_FS_tree_encode_finish". | ||
530 | * | ||
531 | * This function checks if the content on disk matches | ||
532 | * the expected content based on the URI. | ||
533 | * | ||
534 | * @param cls closure | ||
535 | * @param query the query for the block (key for lookup in the datastore) | ||
536 | * @param offset offset of the block | ||
537 | * @param depth depth | ||
538 | * @param type type of the block (IBLOCK or DBLOCK) | ||
539 | * @param block the (encrypted) block | ||
540 | * @param block_size size of block (in bytes) | ||
541 | */ | ||
542 | static void | ||
543 | reconstruct_cb (void *cls, | ||
544 | const GNUNET_HashCode *query, | ||
545 | uint64_t offset, | ||
546 | unsigned int depth, | ||
547 | enum GNUNET_BLOCK_Type type, | ||
548 | const void *block, | ||
549 | uint16_t block_size) | ||
550 | { | ||
551 | struct ReconstructContext *rcc = cls; | ||
552 | struct ProcessResultClosure prc; | ||
553 | struct GNUNET_FS_TreeEncoder *te; | ||
554 | uint64_t off; | ||
555 | uint64_t boff; | ||
556 | uint64_t roff; | ||
557 | unsigned int i; | ||
558 | |||
559 | roff = offset / DBLOCK_SIZE; | ||
560 | for (i=rcc->dc->treedepth;i>depth;i--) | ||
561 | roff /= CHK_PER_INODE; | ||
562 | boff = roff * DBLOCK_SIZE; | ||
563 | for (i=rcc->dc->treedepth;i>depth;i--) | ||
564 | boff *= CHK_PER_INODE; | ||
565 | /* convert reading offset into IBLOCKs on-disk offset */ | ||
566 | off = compute_disk_offset (GNUNET_FS_uri_chk_get_file_size (rcc->dc->uri), | ||
567 | boff, | ||
568 | depth, | ||
569 | rcc->dc->treedepth); | ||
570 | if ( (off == rcc->offset) && | ||
571 | (depth == rcc->depth) && | ||
572 | (0 == memcmp (query, | ||
573 | &rcc->chk.query, | ||
574 | sizeof (GNUNET_HashCode))) ) | ||
575 | { | ||
576 | /* already got it! */ | ||
577 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
578 | _("Block reconstruction at offset %llu and depth %u successful\n"), | ||
579 | (unsigned long long) offset, | ||
580 | depth); | ||
581 | prc.dc = rcc->dc; | ||
582 | prc.data = block; | ||
583 | prc.size = block_size; | ||
584 | prc.type = type; | ||
585 | prc.query = rcc->chk.query; | ||
586 | prc.do_store = GNUNET_NO; | ||
587 | process_result_with_request (&prc, | ||
588 | &rcc->chk.key, | ||
589 | rcc->sm); | ||
590 | te = rcc->te; | ||
591 | rcc->te = NULL; | ||
592 | GNUNET_FS_tree_encoder_finish (te, NULL, NULL); | ||
593 | GNUNET_free (rcc); | ||
594 | return; | ||
595 | } | ||
596 | rcc->task = GNUNET_SCHEDULER_add_now (&get_next_block, | ||
597 | rcc); | ||
598 | } | ||
599 | |||
600 | |||
601 | /** | ||
602 | * Function called by the tree encoder to obtain | ||
603 | * a block of plaintext data (for the lowest level | ||
604 | * of the tree). | ||
605 | * | ||
606 | * @param cls our 'struct ReconstructContext' | ||
607 | * @param offset identifies which block to get | ||
608 | * @param max (maximum) number of bytes to get; returning | ||
609 | * fewer will also cause errors | ||
610 | * @param buf where to copy the plaintext buffer | ||
611 | * @param emsg location to store an error message (on error) | ||
612 | * @return number of bytes copied to buf, 0 on error | ||
613 | */ | ||
614 | static size_t | ||
615 | fh_reader (void *cls, | ||
616 | uint64_t offset, | ||
617 | size_t max, | ||
618 | void *buf, | ||
619 | char **emsg) | ||
620 | { | ||
621 | struct ReconstructContext *rcc = cls; | ||
622 | struct GNUNET_DISK_FileHandle *fh = rcc->fh; | ||
623 | ssize_t ret; | ||
624 | |||
625 | *emsg = NULL; | ||
626 | if (offset != | ||
627 | GNUNET_DISK_file_seek (fh, | ||
628 | offset, | ||
629 | GNUNET_DISK_SEEK_SET)) | ||
630 | { | ||
631 | *emsg = GNUNET_strdup (strerror (errno)); | ||
632 | return 0; | ||
633 | } | ||
634 | ret = GNUNET_DISK_file_read (fh, buf, max); | ||
635 | if (ret < 0) | ||
636 | { | ||
637 | *emsg = GNUNET_strdup (strerror (errno)); | ||
638 | return 0; | ||
639 | } | ||
640 | return ret; | ||
641 | } | ||
642 | |||
643 | |||
644 | /** | ||
645 | * Schedule the download of the specified block in the tree. | ||
646 | * | ||
647 | * @param dc overall download this block belongs to | ||
648 | * @param chk content-hash-key of the block | ||
649 | * @param offset offset of the block in the file | ||
650 | * (for IBlocks, the offset is the lowest | ||
651 | * offset of any DBlock in the subtree under | ||
652 | * the IBlock) | ||
653 | * @param depth depth of the block, 0 is the root of the tree | ||
654 | */ | ||
655 | static void | ||
656 | schedule_block_download (struct GNUNET_FS_DownloadContext *dc, | ||
657 | const struct ContentHashKey *chk, | ||
658 | uint64_t offset, | ||
659 | unsigned int depth) | ||
660 | { | ||
661 | struct DownloadRequest *sm; | ||
662 | uint64_t total; | ||
663 | uint64_t off; | ||
664 | size_t len; | ||
665 | char block[DBLOCK_SIZE]; | ||
666 | GNUNET_HashCode key; | ||
667 | struct MatchDataContext mdc; | ||
668 | struct GNUNET_DISK_FileHandle *fh; | ||
669 | struct ReconstructContext *rcc; | ||
670 | |||
671 | total = GNUNET_FS_uri_chk_get_file_size (dc->uri); | ||
672 | len = GNUNET_FS_tree_calculate_block_size (total, | ||
673 | dc->treedepth, | ||
674 | offset, | ||
675 | depth); | ||
676 | off = compute_disk_offset (total, | ||
677 | offset, | ||
678 | depth, | ||
679 | dc->treedepth); | ||
680 | sm = GNUNET_malloc (sizeof (struct DownloadRequest)); | ||
681 | sm->chk = *chk; | ||
682 | sm->offset = offset; | ||
683 | sm->depth = depth; | ||
684 | sm->is_pending = GNUNET_YES; | ||
685 | sm->next = dc->pending; | ||
686 | dc->pending = sm; | ||
687 | GNUNET_CONTAINER_multihashmap_put (dc->active, | ||
688 | &chk->query, | ||
689 | sm, | ||
690 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); | ||
691 | if ( (dc->tried_full_data == GNUNET_NO) && | ||
692 | (depth == 0) ) | ||
693 | { | ||
694 | mdc.dc = dc; | ||
695 | mdc.sm = sm; | ||
696 | mdc.chk = chk; | ||
697 | mdc.offset = offset; | ||
698 | mdc.len = len; | ||
699 | mdc.done = GNUNET_NO; | ||
700 | GNUNET_CONTAINER_meta_data_iterate (dc->meta, | ||
701 | &match_full_data, | ||
702 | &mdc); | ||
703 | if (mdc.done == GNUNET_YES) | ||
704 | return; | ||
705 | dc->tried_full_data = GNUNET_YES; | ||
706 | } | ||
707 | #if DEBUG_DOWNLOAD | ||
708 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
709 | "Scheduling download at offset %llu and depth %u for `%s'\n", | ||
710 | (unsigned long long) offset, | ||
711 | depth, | ||
712 | GNUNET_h2s (&chk->query)); | ||
713 | #endif | ||
714 | fh = NULL; | ||
715 | if ( ( (dc->old_file_size > off) || | ||
716 | ( (depth < dc->treedepth) && | ||
717 | (dc->reconstruct_failed == GNUNET_NO) ) ) && | ||
718 | (dc->filename != NULL) ) | ||
719 | fh = GNUNET_DISK_file_open (dc->filename, | ||
720 | GNUNET_DISK_OPEN_READ, | ||
721 | GNUNET_DISK_PERM_NONE); | ||
722 | if ( (fh != NULL) && | ||
723 | (dc->old_file_size > off) && | ||
724 | (off == | ||
725 | GNUNET_DISK_file_seek (fh, | ||
726 | off, | ||
727 | GNUNET_DISK_SEEK_SET) ) && | ||
728 | (len == | ||
729 | GNUNET_DISK_file_read (fh, | ||
730 | block, | ||
731 | len)) ) | ||
732 | { | ||
733 | GNUNET_CRYPTO_hash (block, len, &key); | ||
734 | if ( (0 == memcmp (&key, | ||
735 | &chk->key, | ||
736 | sizeof (GNUNET_HashCode))) && | ||
737 | (GNUNET_OK == | ||
738 | encrypt_existing_match (dc, | ||
739 | chk, | ||
740 | sm, | ||
741 | block, | ||
742 | len, | ||
743 | depth, | ||
744 | GNUNET_NO)) ) | ||
745 | { | ||
746 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh)); | ||
747 | return; | ||
748 | } | ||
749 | } | ||
750 | rcc = GNUNET_malloc (sizeof (struct ReconstructContext)); | ||
751 | rcc->fh = fh; | ||
752 | rcc->dc = dc; | ||
753 | rcc->sm = sm; | ||
754 | rcc->chk = *chk; | ||
755 | rcc->offset = off; | ||
756 | rcc->depth = depth; | ||
757 | dc->rcc = rcc; | ||
758 | if ( (depth < dc->treedepth) && | ||
759 | (dc->reconstruct_failed == GNUNET_NO) && | ||
760 | (fh != NULL) ) | ||
761 | { | ||
762 | rcc->te = GNUNET_FS_tree_encoder_create (dc->h, | ||
763 | dc->old_file_size, | ||
764 | rcc, | ||
765 | fh_reader, | ||
766 | &reconstruct_cb, | ||
767 | NULL, | ||
768 | &reconstruct_cont); | ||
769 | GNUNET_FS_tree_encoder_next (rcc->te); | ||
770 | return; | ||
771 | } | ||
772 | reconstruct_cont (rcc, NULL); | ||
773 | } | ||
774 | |||
775 | |||
776 | /** | ||
777 | * Suggest a filename based on given metadata. | ||
778 | * | ||
779 | * @param md given meta data | ||
780 | * @return NULL if meta data is useless for suggesting a filename | ||
781 | */ | ||
782 | char * | ||
783 | GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_CONTAINER_MetaData *md) | ||
784 | { | ||
785 | static const char *mimeMap[][2] = { | ||
786 | {"application/bz2", ".bz2"}, | ||
787 | {"application/gnunet-directory", ".gnd"}, | ||
788 | {"application/java", ".class"}, | ||
789 | {"application/msword", ".doc"}, | ||
790 | {"application/ogg", ".ogg"}, | ||
791 | {"application/pdf", ".pdf"}, | ||
792 | {"application/pgp-keys", ".key"}, | ||
793 | {"application/pgp-signature", ".pgp"}, | ||
794 | {"application/postscript", ".ps"}, | ||
795 | {"application/rar", ".rar"}, | ||
796 | {"application/rtf", ".rtf"}, | ||
797 | {"application/xml", ".xml"}, | ||
798 | {"application/x-debian-package", ".deb"}, | ||
799 | {"application/x-dvi", ".dvi"}, | ||
800 | {"applixation/x-flac", ".flac"}, | ||
801 | {"applixation/x-gzip", ".gz"}, | ||
802 | {"application/x-java-archive", ".jar"}, | ||
803 | {"application/x-java-vm", ".class"}, | ||
804 | {"application/x-python-code", ".pyc"}, | ||
805 | {"application/x-redhat-package-manager", ".rpm"}, | ||
806 | {"application/x-rpm", ".rpm"}, | ||
807 | {"application/x-tar", ".tar"}, | ||
808 | {"application/x-tex-pk", ".pk"}, | ||
809 | {"application/x-texinfo", ".texinfo"}, | ||
810 | {"application/x-xcf", ".xcf"}, | ||
811 | {"application/x-xfig", ".xfig"}, | ||
812 | {"application/zip", ".zip"}, | ||
813 | |||
814 | {"audio/midi", ".midi"}, | ||
815 | {"audio/mpeg", ".mp3"}, | ||
816 | {"audio/real", ".rm"}, | ||
817 | {"audio/x-wav", ".wav"}, | ||
818 | |||
819 | {"image/gif", ".gif"}, | ||
820 | {"image/jpeg", ".jpg"}, | ||
821 | {"image/pcx", ".pcx"}, | ||
822 | {"image/png", ".png"}, | ||
823 | {"image/tiff", ".tiff"}, | ||
824 | {"image/x-ms-bmp", ".bmp"}, | ||
825 | {"image/x-xpixmap", ".xpm"}, | ||
826 | |||
827 | {"text/css", ".css"}, | ||
828 | {"text/html", ".html"}, | ||
829 | {"text/plain", ".txt"}, | ||
830 | {"text/rtf", ".rtf"}, | ||
831 | {"text/x-c++hdr", ".h++"}, | ||
832 | {"text/x-c++src", ".c++"}, | ||
833 | {"text/x-chdr", ".h"}, | ||
834 | {"text/x-csrc", ".c"}, | ||
835 | {"text/x-java", ".java"}, | ||
836 | {"text/x-moc", ".moc"}, | ||
837 | {"text/x-pascal", ".pas"}, | ||
838 | {"text/x-perl", ".pl"}, | ||
839 | {"text/x-python", ".py"}, | ||
840 | {"text/x-tex", ".tex"}, | ||
841 | |||
842 | {"video/avi", ".avi"}, | ||
843 | {"video/mpeg", ".mpeg"}, | ||
844 | {"video/quicktime", ".qt"}, | ||
845 | {"video/real", ".rm"}, | ||
846 | {"video/x-msvideo", ".avi"}, | ||
847 | {NULL, NULL}, | ||
848 | }; | ||
849 | char *ret; | ||
850 | unsigned int i; | ||
851 | char *mime; | ||
852 | char *base; | ||
853 | const char *ext; | ||
854 | |||
855 | ret = GNUNET_CONTAINER_meta_data_get_by_type (md, | ||
856 | EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); | ||
857 | if (ret != NULL) | ||
858 | return ret; | ||
859 | ext = NULL; | ||
860 | mime = GNUNET_CONTAINER_meta_data_get_by_type (md, | ||
861 | EXTRACTOR_METATYPE_MIMETYPE); | ||
862 | if (mime != NULL) | ||
863 | { | ||
864 | i = 0; | ||
865 | while ( (mimeMap[i][0] != NULL) && | ||
866 | (0 != strcmp (mime, mimeMap[i][0]))) | ||
867 | i++; | ||
868 | if (mimeMap[i][1] == NULL) | ||
869 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | | ||
870 | GNUNET_ERROR_TYPE_BULK, | ||
871 | _("Did not find mime type `%s' in extension list.\n"), | ||
872 | mime); | ||
873 | else | ||
874 | ext = mimeMap[i][1]; | ||
875 | GNUNET_free (mime); | ||
876 | } | ||
877 | base = GNUNET_CONTAINER_meta_data_get_first_by_types (md, | ||
878 | EXTRACTOR_METATYPE_TITLE, | ||
879 | EXTRACTOR_METATYPE_BOOK_TITLE, | ||
880 | EXTRACTOR_METATYPE_ORIGINAL_TITLE, | ||
881 | EXTRACTOR_METATYPE_PACKAGE_NAME, | ||
882 | EXTRACTOR_METATYPE_URL, | ||
883 | EXTRACTOR_METATYPE_URI, | ||
884 | EXTRACTOR_METATYPE_DESCRIPTION, | ||
885 | EXTRACTOR_METATYPE_ISRC, | ||
886 | EXTRACTOR_METATYPE_JOURNAL_NAME, | ||
887 | EXTRACTOR_METATYPE_AUTHOR_NAME, | ||
888 | EXTRACTOR_METATYPE_SUBJECT, | ||
889 | EXTRACTOR_METATYPE_ALBUM, | ||
890 | EXTRACTOR_METATYPE_ARTIST, | ||
891 | EXTRACTOR_METATYPE_KEYWORDS, | ||
892 | EXTRACTOR_METATYPE_COMMENT, | ||
893 | EXTRACTOR_METATYPE_UNKNOWN, | ||
894 | -1); | ||
895 | if ( (base == NULL) && | ||
896 | (ext == NULL) ) | ||
897 | return NULL; | ||
898 | if (base == NULL) | ||
899 | return GNUNET_strdup (ext); | ||
900 | if (ext == NULL) | ||
901 | return base; | ||
902 | GNUNET_asprintf (&ret, | ||
903 | "%s%s", | ||
904 | base, | ||
905 | ext); | ||
906 | GNUNET_free (base); | ||
907 | return ret; | ||
908 | } | ||
909 | |||
910 | |||
911 | /** | ||
912 | * We've lost our connection with the FS service. | 312 | * We've lost our connection with the FS service. |
913 | * Re-establish it and re-transmit all of our | 313 | * Re-establish it and re-transmit all of our |
914 | * pending requests. | 314 | * pending requests. |
@@ -1006,9 +406,14 @@ full_recursive_download (struct GNUNET_FS_DownloadContext *dc) | |||
1006 | 406 | ||
1007 | 407 | ||
1008 | /** | 408 | /** |
1009 | * Check if all child-downloads have completed and | 409 | * Check if all child-downloads have completed (or trigger them if |
1010 | * if so, signal completion (and possibly recurse to | 410 | * necessary) and once we're completely done, signal completion (and |
1011 | * parent). | 411 | * possibly recurse to parent). This function MUST be called when the |
412 | * download of a file itself is done or when the download of a file is | ||
413 | * done and then later a direct child download has completed (and | ||
414 | * hence this download may complete itself). | ||
415 | * | ||
416 | * @param dc download to check for completion of children | ||
1012 | */ | 417 | */ |
1013 | static void | 418 | static void |
1014 | check_completed (struct GNUNET_FS_DownloadContext *dc) | 419 | check_completed (struct GNUNET_FS_DownloadContext *dc) |
@@ -1016,6 +421,11 @@ check_completed (struct GNUNET_FS_DownloadContext *dc) | |||
1016 | struct GNUNET_FS_ProgressInfo pi; | 421 | struct GNUNET_FS_ProgressInfo pi; |
1017 | struct GNUNET_FS_DownloadContext *pos; | 422 | struct GNUNET_FS_DownloadContext *pos; |
1018 | 423 | ||
424 | /* first, check if we need to download children */ | ||
425 | if ( (dc->child_head == NULL) && | ||
426 | (is_recursive_download (dc)) ) | ||
427 | full_recursive_download (dc); | ||
428 | /* then, check if children are done already */ | ||
1019 | pos = dc->child_head; | 429 | pos = dc->child_head; |
1020 | while (pos != NULL) | 430 | while (pos != NULL) |
1021 | { | 431 | { |
@@ -1027,16 +437,416 @@ check_completed (struct GNUNET_FS_DownloadContext *dc) | |||
1027 | return; /* not transitively done yet */ | 437 | return; /* not transitively done yet */ |
1028 | pos = pos->next; | 438 | pos = pos->next; |
1029 | } | 439 | } |
440 | /* All of our children are done, so mark this download done */ | ||
1030 | dc->has_finished = GNUNET_YES; | 441 | dc->has_finished = GNUNET_YES; |
1031 | GNUNET_FS_download_sync_ (dc); | 442 | GNUNET_FS_download_sync_ (dc); |
443 | |||
1032 | /* signal completion */ | 444 | /* signal completion */ |
1033 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED; | 445 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED; |
1034 | GNUNET_FS_download_make_status_ (&pi, dc); | 446 | GNUNET_FS_download_make_status_ (&pi, dc); |
447 | |||
448 | /* let parent know */ | ||
1035 | if (dc->parent != NULL) | 449 | if (dc->parent != NULL) |
1036 | check_completed (dc->parent); | 450 | check_completed (dc->parent); |
1037 | } | 451 | } |
1038 | 452 | ||
1039 | 453 | ||
454 | /** | ||
455 | * We got a block of plaintext data (from the meta data). | ||
456 | * Try it for upward reconstruction of the data. On success, | ||
457 | * the top-level block will move to state BRS_DOWNLOAD_UP. | ||
458 | * | ||
459 | * @param dr one of our request entries | ||
460 | * @param data plaintext data, starting from the beginning of the file | ||
461 | * @param data_len number of bytes in data | ||
462 | */ | ||
463 | static void | ||
464 | try_match_block (struct GNUNET_FS_DownloadContext *dc, | ||
465 | struct DownloadRequest *dr, | ||
466 | const char *data, | ||
467 | size_t data_len) | ||
468 | { | ||
469 | struct GNUNET_FS_ProgressInfo pi; | ||
470 | unsigned int i; | ||
471 | char enc[DBLOCK_SIZE]; | ||
472 | struct ContentHashKey chks[CHK_PER_INODE]; | ||
473 | struct ContentHashKey chk; | ||
474 | struct GNUNET_CRYPTO_AesSessionKey sk; | ||
475 | struct GNUNET_CRYPTO_AesInitializationVector iv; | ||
476 | size_t dlen; | ||
477 | struct DownloadRequest *drc; | ||
478 | struct GNUNET_DISK_FileHandle *fh; | ||
479 | int complete; | ||
480 | const char *fn; | ||
481 | |||
482 | if (BRS_DOWNLOAD_UP == dr->state) | ||
483 | return; | ||
484 | if (dr->depth > 0) | ||
485 | { | ||
486 | complete = GNUNET_YES; | ||
487 | for (i=0;i<dr->num_children;i++) | ||
488 | { | ||
489 | drc = dr->children[i]; | ||
490 | try_match_block (dc, | ||
491 | drc, | ||
492 | data, data_len); | ||
493 | if (drc->state != BRS_RECONSTRUCT_META_UP) | ||
494 | complete = GNUNET_NO; | ||
495 | } | ||
496 | if (GNUNET_YES != complete) | ||
497 | return; | ||
498 | data = (const char*) chks; | ||
499 | dlen = dr->num_children * sizeof (struct ContentHashKey); | ||
500 | } | ||
501 | else | ||
502 | { | ||
503 | if (dr->offset > data_len) | ||
504 | return; /* oops */ | ||
505 | dlen = GNUNET_MIN (data_len - dr->offset, | ||
506 | DBLOCK_SIZE); | ||
507 | } | ||
508 | GNUNET_CRYPTO_hash (&data[dr->offset], | ||
509 | dlen, | ||
510 | &chk.key); | ||
511 | GNUNET_CRYPTO_hash_to_aes_key (&chk.key, &sk, &iv); | ||
512 | if (-1 == GNUNET_CRYPTO_aes_encrypt (data, dlen, | ||
513 | &sk, | ||
514 | &iv, | ||
515 | enc)) | ||
516 | { | ||
517 | GNUNET_break (0); | ||
518 | return; | ||
519 | } | ||
520 | GNUNET_CRYPTO_hash (enc, dlen, &chk.query); | ||
521 | switch (dr->state) | ||
522 | { | ||
523 | case BRS_INIT: | ||
524 | dr->chk = chk; | ||
525 | dr->state = BRS_RECONSTRUCT_META_UP; | ||
526 | break; | ||
527 | case BRS_CHK_SET: | ||
528 | if (0 != memcmp (&chk, | ||
529 | &dr->chk, | ||
530 | sizeof (struct ContentHashKey))) | ||
531 | { | ||
532 | /* other peer provided bogus meta data */ | ||
533 | GNUNET_break_op (0); | ||
534 | break; | ||
535 | } | ||
536 | /* write block to disk */ | ||
537 | fn = dc->filename != NULL | ||
538 | ? dc->filename | ||
539 | : dc->temp_filename; | ||
540 | fh = GNUNET_DISK_file_open (fn, | ||
541 | GNUNET_DISK_OPEN_READWRITE | | ||
542 | GNUNET_DISK_OPEN_CREATE | | ||
543 | GNUNET_DISK_OPEN_TRUNCATE, | ||
544 | GNUNET_DISK_PERM_USER_READ | | ||
545 | GNUNET_DISK_PERM_USER_WRITE | | ||
546 | GNUNET_DISK_PERM_GROUP_READ | | ||
547 | GNUNET_DISK_PERM_OTHER_READ); | ||
548 | if (fh == NULL) | ||
549 | { | ||
550 | GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, | ||
551 | "open", | ||
552 | fn); | ||
553 | GNUNET_asprintf (&dc->emsg, | ||
554 | _("Failed to open file `%s' for writing"), | ||
555 | fn); | ||
556 | GNUNET_DISK_file_close (fh); | ||
557 | dr->state = BRS_ERROR; | ||
558 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; | ||
559 | pi.value.download.specifics.error.message = dc->emsg; | ||
560 | GNUNET_FS_download_make_status_ (&pi, dc); | ||
561 | return; | ||
562 | } | ||
563 | if (data_len != | ||
564 | GNUNET_DISK_file_write (fh, | ||
565 | data, | ||
566 | data_len)) | ||
567 | { | ||
568 | GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, | ||
569 | "write", | ||
570 | fn); | ||
571 | GNUNET_asprintf (&dc->emsg, | ||
572 | _("Failed to open file `%s' for writing"), | ||
573 | fn); | ||
574 | GNUNET_DISK_file_close (fh); | ||
575 | dr->state = BRS_ERROR; | ||
576 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; | ||
577 | pi.value.download.specifics.error.message = dc->emsg; | ||
578 | GNUNET_FS_download_make_status_ (&pi, dc); | ||
579 | return; | ||
580 | } | ||
581 | GNUNET_DISK_file_close (fh); | ||
582 | /* signal success */ | ||
583 | dr->state = BRS_DOWNLOAD_UP; | ||
584 | dc->completed = dc->length; | ||
585 | GNUNET_FS_download_sync_ (dc); | ||
586 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS; | ||
587 | pi.value.download.specifics.progress.data = data; | ||
588 | pi.value.download.specifics.progress.offset = 0; | ||
589 | pi.value.download.specifics.progress.data_len = dlen; | ||
590 | pi.value.download.specifics.progress.depth = 0; | ||
591 | GNUNET_FS_download_make_status_ (&pi, dc); | ||
592 | check_completed (dc); | ||
593 | break; | ||
594 | default: | ||
595 | /* how did we get here? */ | ||
596 | GNUNET_break (0); | ||
597 | break; | ||
598 | } | ||
599 | } | ||
600 | |||
601 | |||
602 | /** | ||
603 | * Type of a function that libextractor calls for each | ||
604 | * meta data item found. If we find full data meta data, | ||
605 | * call 'try_match_block' on it. | ||
606 | * | ||
607 | * @param cls our 'struct GNUNET_FS_DownloadContext*' | ||
608 | * @param plugin_name name of the plugin that produced this value; | ||
609 | * special values can be used (i.e. '<zlib>' for zlib being | ||
610 | * used in the main libextractor library and yielding | ||
611 | * meta data). | ||
612 | * @param type libextractor-type describing the meta data | ||
613 | * @param format basic format information about data | ||
614 | * @param data_mime_type mime-type of data (not of the original file); | ||
615 | * can be NULL (if mime-type is not known) | ||
616 | * @param data actual meta-data found | ||
617 | * @param data_len number of bytes in data | ||
618 | * @return 0 to continue extracting, 1 to abort | ||
619 | */ | ||
620 | static int | ||
621 | match_full_data (void *cls, | ||
622 | const char *plugin_name, | ||
623 | enum EXTRACTOR_MetaType type, | ||
624 | enum EXTRACTOR_MetaFormat format, | ||
625 | const char *data_mime_type, | ||
626 | const char *data, | ||
627 | size_t data_len) | ||
628 | { | ||
629 | struct GNUNET_FS_DownloadContext *dc = cls; | ||
630 | |||
631 | if (type != EXTRACTOR_METATYPE_GNUNET_FULL_DATA) | ||
632 | return 0; | ||
633 | if (GNUNET_FS_uri_chk_get_file_size (dc->uri) != data_len) | ||
634 | { | ||
635 | GNUNET_break_op (0); | ||
636 | return 1; /* bogus meta data */ | ||
637 | } | ||
638 | try_match_block (dc, | ||
639 | dc->top_request, | ||
640 | data, | ||
641 | data_len); | ||
642 | return 1; | ||
643 | } | ||
644 | |||
645 | |||
646 | /** | ||
647 | * Set the state of the given download request to | ||
648 | * BRS_DOWNLOAD_UP and propagate it up the tree. | ||
649 | * | ||
650 | * @param dr download request that is done | ||
651 | */ | ||
652 | static void | ||
653 | propagate_up (struct DownloadRequest *dr) | ||
654 | { | ||
655 | unsigned int i; | ||
656 | |||
657 | do | ||
658 | { | ||
659 | dr->state = BRS_DOWNLOAD_UP; | ||
660 | dr = dr->parent; | ||
661 | if (dr == NULL) | ||
662 | break; | ||
663 | for (i=0;i<dr->num_children;i++) | ||
664 | if (dr->children[i]->state != BRS_DOWNLOAD_UP) | ||
665 | break; | ||
666 | } | ||
667 | while (i == dr->num_children); | ||
668 | } | ||
669 | |||
670 | |||
671 | /** | ||
672 | * Try top-down reconstruction. Before, the given request node | ||
673 | * must have the state BRS_CHK_SET. Afterwards, more nodes may | ||
674 | * have that state or advanced to BRS_DOWNLOAD_DOWN or even | ||
675 | * BRS_DOWNLOAD_UP. It is also possible to get BRS_ERROR on the | ||
676 | * top level. | ||
677 | * | ||
678 | * @param dc overall download this block belongs to | ||
679 | * @param dr block to reconstruct | ||
680 | */ | ||
681 | static void | ||
682 | try_top_down_reconstruction (struct GNUNET_FS_DownloadContext *dc, | ||
683 | struct DownloadRequest *dr) | ||
684 | { | ||
685 | uint64_t off; | ||
686 | char block[DBLOCK_SIZE]; | ||
687 | GNUNET_HashCode key; | ||
688 | uint64_t total; | ||
689 | size_t len; | ||
690 | unsigned int i; | ||
691 | unsigned int chk_off; | ||
692 | struct DownloadRequest *drc; | ||
693 | uint64_t child_block_size; | ||
694 | const struct ContentHashKey *chks; | ||
695 | int up_done; | ||
696 | |||
697 | GNUNET_assert (dc->rfh != NULL); | ||
698 | GNUNET_assert (dr->state == BRS_CHK_SET); | ||
699 | total = GNUNET_FS_uri_chk_get_file_size (dc->uri); | ||
700 | GNUNET_assert (dr->depth < dc->treedepth); | ||
701 | len = GNUNET_FS_tree_calculate_block_size (total, | ||
702 | dr->offset, | ||
703 | dr->depth); | ||
704 | GNUNET_assert (len <= DBLOCK_SIZE); | ||
705 | off = compute_disk_offset (total, | ||
706 | dr->offset, | ||
707 | dr->depth); | ||
708 | if (dc->old_file_size < off + len) | ||
709 | return; /* failure */ | ||
710 | if (off != | ||
711 | GNUNET_DISK_file_seek (dc->rfh, | ||
712 | off, | ||
713 | GNUNET_DISK_SEEK_SET) ) | ||
714 | { | ||
715 | GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, | ||
716 | "seek", | ||
717 | dc->filename); | ||
718 | return; /* failure */ | ||
719 | } | ||
720 | if (len != | ||
721 | GNUNET_DISK_file_read (dc->rfh, | ||
722 | block, | ||
723 | len)) | ||
724 | { | ||
725 | GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, | ||
726 | "read", | ||
727 | dc->filename); | ||
728 | return; /* failure */ | ||
729 | } | ||
730 | GNUNET_CRYPTO_hash (block, len, &key); | ||
731 | if (0 != memcmp (&key, | ||
732 | &dr->chk.key, | ||
733 | sizeof (GNUNET_HashCode))) | ||
734 | return; /* mismatch */ | ||
735 | if (GNUNET_OK != | ||
736 | encrypt_existing_match (dc, | ||
737 | chk, | ||
738 | dr, | ||
739 | block, | ||
740 | len, | ||
741 | GNUNET_NO)) | ||
742 | { | ||
743 | /* hash matches but encrypted block does not, really bad */ | ||
744 | dr->state = BRS_ERROR; | ||
745 | /* propagate up */ | ||
746 | while (dr->parent != NULL) | ||
747 | { | ||
748 | dr = dr->parent; | ||
749 | dr->state = BRS_ERROR; | ||
750 | } | ||
751 | return; | ||
752 | } | ||
753 | /* block matches */ | ||
754 | dr->state = BRS_DOWNLOAD_DOWN; | ||
755 | |||
756 | /* set CHKs for children */ | ||
757 | up_done = GNUNET_YES; | ||
758 | chks = (const struct ContentHashKey*) block; | ||
759 | for (i=0;i<dr->num_children;i++) | ||
760 | { | ||
761 | drc = dr->children[i]; | ||
762 | GNUNET_assert (drc->offset >= dr->offset); | ||
763 | child_block_size = GNUNET_FS_tree_compute_tree_size (drc->depth); | ||
764 | GNUNET_assert (0 == (drc->offset - dr->offset) % child_block_size); | ||
765 | chk_off = (drc->offset - dr->offset) / child_block_size; | ||
766 | GNUNET_assert (drc->state == BRS_INIT); | ||
767 | drc->state = BRS_CHK_SET; | ||
768 | drc->chk = chks[chk_off]; | ||
769 | try_top_down_reconstruction (dc, drc); | ||
770 | if (drc->state != BRS_DOWNLOAD_UP) | ||
771 | up_done = GNUNET_NO; /* children not all done */ | ||
772 | } | ||
773 | if (up_done == GNUNET_YES) | ||
774 | propagate_up (dr); /* children all done (or no children...) */ | ||
775 | } | ||
776 | |||
777 | |||
778 | /** | ||
779 | * Schedule the download of the specified block in the tree. | ||
780 | * | ||
781 | * @param dc overall download this block belongs to | ||
782 | * @param chk content-hash-key of the block | ||
783 | * @param offset offset of the block in the file | ||
784 | * (for IBlocks, the offset is the lowest | ||
785 | * offset of any DBlock in the subtree under | ||
786 | * the IBlock) | ||
787 | * @param depth depth of the block, 0 is the root of the tree | ||
788 | */ | ||
789 | static void | ||
790 | schedule_block_download (struct GNUNET_FS_DownloadContext *dc, | ||
791 | struct DownloadRequest *dr) | ||
792 | { | ||
793 | unsigned int i; | ||
794 | |||
795 | switch (dr->state) | ||
796 | { | ||
797 | case BRS_INIT: | ||
798 | GNUNET_assert (0); | ||
799 | break; | ||
800 | case BRS_RECONSTRUCT_DOWN: | ||
801 | GNUNET_assert (0); | ||
802 | break; | ||
803 | case BRS_RECONSTRUCT_META_UP: | ||
804 | GNUNET_assert (0); | ||
805 | break; | ||
806 | case BRS_RECONSTRUCT_UP: | ||
807 | GNUNET_assert (0); | ||
808 | break; | ||
809 | case BRS_CHK_SET: | ||
810 | /* normal case, start download */ | ||
811 | break; | ||
812 | case BRS_DOWNLOAD_DOWN: | ||
813 | for (i=0;i<dr->num_children;i++) | ||
814 | schedule_block_download (dc, dr->children[i]); | ||
815 | return; | ||
816 | case BRS_DOWNLOAD_UP: | ||
817 | /* We're done! */ | ||
818 | return; | ||
819 | case BRS_ERROR: | ||
820 | GNUNET_break (0); | ||
821 | return; | ||
822 | } | ||
823 | #if DEBUG_DOWNLOAD | ||
824 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
825 | "Scheduling download at offset %llu and depth %u for `%s'\n", | ||
826 | (unsigned long long) dr->offset, | ||
827 | dr->depth, | ||
828 | GNUNET_h2s (&dr->chk.query)); | ||
829 | #endif | ||
830 | GNUNET_CONTAINER_DLL_insert (dc->pending_head, | ||
831 | dc->pending_tail, | ||
832 | dr); | ||
833 | dr->is_pending = GNUNET_YES; | ||
834 | GNUNET_CONTAINER_multihashmap_put (dc->active, | ||
835 | &dr->chk.query, | ||
836 | dr, | ||
837 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); | ||
838 | if (dc->client == NULL) | ||
839 | return; /* download not active */ | ||
840 | if (NULL == dc->th) | ||
841 | dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client, | ||
842 | sizeof (struct SearchMessage), | ||
843 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | ||
844 | GNUNET_NO, | ||
845 | &transmit_download_request, | ||
846 | dc); | ||
847 | } | ||
848 | |||
849 | |||
1040 | #define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX | 850 | #define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX |
1041 | 851 | ||
1042 | /** | 852 | /** |
@@ -1178,7 +988,7 @@ trigger_recursive_download (void *cls, | |||
1178 | { | 988 | { |
1179 | if (full_name == NULL) | 989 | if (full_name == NULL) |
1180 | { | 990 | { |
1181 | temp_name = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp"); | 991 | temp_name = GNUNET_DISK_mktemp ("gnunet-download-trd"); |
1182 | real_name = temp_name; | 992 | real_name = temp_name; |
1183 | } | 993 | } |
1184 | else | 994 | else |
@@ -1226,20 +1036,21 @@ trigger_recursive_download (void *cls, | |||
1226 | 1036 | ||
1227 | 1037 | ||
1228 | /** | 1038 | /** |
1229 | * Free entries in the map. | 1039 | * (recursively) free download request structure |
1230 | * | 1040 | * |
1231 | * @param cls unused (NULL) | 1041 | * @param dr request to free |
1232 | * @param key unused | ||
1233 | * @param entry entry of type "struct DownloadRequest" which is freed | ||
1234 | * @return GNUNET_OK | ||
1235 | */ | 1042 | */ |
1236 | static int | 1043 | void |
1237 | free_entry (void *cls, | 1044 | GNUNET_FS_free_download_request_ (struct DownloadRequest *dr) |
1238 | const GNUNET_HashCode *key, | ||
1239 | void *entry) | ||
1240 | { | 1045 | { |
1241 | GNUNET_free (entry); | 1046 | unsigned int i; |
1242 | return GNUNET_OK; | 1047 | |
1048 | if (dr == NULL) | ||
1049 | return; | ||
1050 | for (i=0;i<dr->num_children;i++) | ||
1051 | GNUNET_FS_free_download_request_ (dr->children[i]); | ||
1052 | GNUNET_free_non_null (dr->children); | ||
1053 | GNUNET_free (dr); | ||
1243 | } | 1054 | } |
1244 | 1055 | ||
1245 | 1056 | ||
@@ -1254,15 +1065,14 @@ free_entry (void *cls, | |||
1254 | */ | 1065 | */ |
1255 | static int | 1066 | static int |
1256 | process_result_with_request (void *cls, | 1067 | process_result_with_request (void *cls, |
1257 | const GNUNET_HashCode * key, | 1068 | const GNUNET_HashCode *key, |
1258 | void *value) | 1069 | void *value) |
1259 | { | 1070 | { |
1260 | struct ProcessResultClosure *prc = cls; | 1071 | struct ProcessResultClosure *prc = cls; |
1261 | struct DownloadRequest *sm = value; | 1072 | struct DownloadRequest *dr = value; |
1262 | struct DownloadRequest *ppos; | ||
1263 | struct DownloadRequest *pprev; | ||
1264 | struct GNUNET_DISK_FileHandle *fh; | ||
1265 | struct GNUNET_FS_DownloadContext *dc = prc->dc; | 1073 | struct GNUNET_FS_DownloadContext *dc = prc->dc; |
1074 | struct DownloadRequest *drc; | ||
1075 | struct GNUNET_DISK_FileHandle *fh = NULL; | ||
1266 | struct GNUNET_CRYPTO_AesSessionKey skey; | 1076 | struct GNUNET_CRYPTO_AesSessionKey skey; |
1267 | struct GNUNET_CRYPTO_AesInitializationVector iv; | 1077 | struct GNUNET_CRYPTO_AesInitializationVector iv; |
1268 | char pt[prc->size]; | 1078 | char pt[prc->size]; |
@@ -1273,43 +1083,53 @@ process_result_with_request (void *cls, | |||
1273 | int i; | 1083 | int i; |
1274 | struct ContentHashKey *chk; | 1084 | struct ContentHashKey *chk; |
1275 | 1085 | ||
1276 | fh = NULL; | 1086 | #if DEBUG_DOWNLOAD |
1087 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1088 | "Received block `%s' matching pending request at depth %u and offset %llu/%llu\n", | ||
1089 | GNUNET_h2s (key), | ||
1090 | dr->depth, | ||
1091 | (unsigned long long) dr->offset, | ||
1092 | (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length)); | ||
1093 | |||
1094 | #endif | ||
1277 | bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length), | 1095 | bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length), |
1278 | dc->treedepth, | 1096 | dr->offset, |
1279 | sm->offset, | 1097 | dr->depth); |
1280 | sm->depth); | ||
1281 | if (prc->size != bs) | 1098 | if (prc->size != bs) |
1282 | { | 1099 | { |
1283 | #if DEBUG_DOWNLOAD | 1100 | GNUNET_asprintf (&dc->emsg, |
1284 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1101 | _("Internal error or bogus download URI (expected %u bytes at depth %u and offset %llu/%llu, got %u bytes)\n"), |
1285 | "Internal error or bogus download URI (expected %u bytes, got %u)\n", | 1102 | bs, |
1286 | bs, | 1103 | dr->depth, |
1287 | prc->size); | 1104 | (unsigned long long) dr->offset, |
1288 | #endif | 1105 | (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length), |
1289 | dc->emsg = GNUNET_strdup ("Internal error or bogus download URI"); | 1106 | prc->size); |
1107 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | ||
1108 | "%s", | ||
1109 | dc->emsg); | ||
1110 | while (dr->parent != NULL) | ||
1111 | { | ||
1112 | dr->state = BRS_ERROR; | ||
1113 | dr = dr->parent; | ||
1114 | } | ||
1115 | dr->state = BRS_ERROR; | ||
1290 | goto signal_error; | 1116 | goto signal_error; |
1291 | } | 1117 | } |
1118 | |||
1292 | GNUNET_assert (GNUNET_YES == | 1119 | GNUNET_assert (GNUNET_YES == |
1293 | GNUNET_CONTAINER_multihashmap_remove (dc->active, | 1120 | GNUNET_CONTAINER_multihashmap_remove (dc->active, |
1294 | &prc->query, | 1121 | &prc->query, |
1295 | sm)); | 1122 | dr)); |
1296 | /* if this request is on the pending list, remove it! */ | 1123 | if (GNUNET_YES == dr->is_pending) |
1297 | pprev = NULL; | ||
1298 | ppos = dc->pending; | ||
1299 | while (ppos != NULL) | ||
1300 | { | 1124 | { |
1301 | if (ppos == sm) | 1125 | GNUNET_CONTAINER_DLL_remove (dc->pending_head, |
1302 | { | 1126 | dc->pending_tail, |
1303 | if (pprev == NULL) | 1127 | dr); |
1304 | dc->pending = ppos->next; | 1128 | dr->is_pending = GNUNET_NO; |
1305 | else | ||
1306 | pprev->next = ppos->next; | ||
1307 | break; | ||
1308 | } | ||
1309 | pprev = ppos; | ||
1310 | ppos = ppos->next; | ||
1311 | } | 1129 | } |
1312 | GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv); | 1130 | |
1131 | |||
1132 | GNUNET_CRYPTO_hash_to_aes_key (&dr->chk.key, &skey, &iv); | ||
1313 | if (-1 == GNUNET_CRYPTO_aes_decrypt (prc->data, | 1133 | if (-1 == GNUNET_CRYPTO_aes_decrypt (prc->data, |
1314 | prc->size, | 1134 | prc->size, |
1315 | &skey, | 1135 | &skey, |
@@ -1317,18 +1137,17 @@ process_result_with_request (void *cls, | |||
1317 | pt)) | 1137 | pt)) |
1318 | { | 1138 | { |
1319 | GNUNET_break (0); | 1139 | GNUNET_break (0); |
1320 | dc->emsg = GNUNET_strdup ("internal error decrypting content"); | 1140 | dc->emsg = GNUNET_strdup (_("internal error decrypting content")); |
1321 | goto signal_error; | 1141 | goto signal_error; |
1322 | } | 1142 | } |
1323 | off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length), | 1143 | off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length), |
1324 | sm->offset, | 1144 | dr->offset, |
1325 | sm->depth, | 1145 | dr->depth); |
1326 | dc->treedepth); | ||
1327 | /* save to disk */ | 1146 | /* save to disk */ |
1328 | if ( ( GNUNET_YES == prc->do_store) && | 1147 | if ( ( GNUNET_YES == prc->do_store) && |
1329 | ( (dc->filename != NULL) || | 1148 | ( (dc->filename != NULL) || |
1330 | (is_recursive_download (dc)) ) && | 1149 | (is_recursive_download (dc)) ) && |
1331 | ( (sm->depth == dc->treedepth) || | 1150 | ( (dr->depth == dc->treedepth) || |
1332 | (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) ) | 1151 | (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) ) |
1333 | { | 1152 | { |
1334 | fh = GNUNET_DISK_file_open (dc->filename != NULL | 1153 | fh = GNUNET_DISK_file_open (dc->filename != NULL |
@@ -1339,23 +1158,15 @@ process_result_with_request (void *cls, | |||
1339 | GNUNET_DISK_PERM_USER_READ | | 1158 | GNUNET_DISK_PERM_USER_READ | |
1340 | GNUNET_DISK_PERM_USER_WRITE | | 1159 | GNUNET_DISK_PERM_USER_WRITE | |
1341 | GNUNET_DISK_PERM_GROUP_READ | | 1160 | GNUNET_DISK_PERM_GROUP_READ | |
1342 | GNUNET_DISK_PERM_OTHER_READ); | 1161 | GNUNET_DISK_PERM_OTHER_READ); |
1343 | } | 1162 | if (NULL == fh) |
1344 | if ( (NULL == fh) && | 1163 | { |
1345 | (GNUNET_YES == prc->do_store) && | 1164 | GNUNET_asprintf (&dc->emsg, |
1346 | ( (dc->filename != NULL) || | 1165 | _("Download failed: could not open file `%s': %s\n"), |
1347 | (is_recursive_download (dc)) ) && | 1166 | dc->filename, |
1348 | ( (sm->depth == dc->treedepth) || | 1167 | STRERROR (errno)); |
1349 | (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) ) | 1168 | goto signal_error; |
1350 | { | 1169 | } |
1351 | GNUNET_asprintf (&dc->emsg, | ||
1352 | _("Download failed: could not open file `%s': %s\n"), | ||
1353 | dc->filename, | ||
1354 | STRERROR (errno)); | ||
1355 | goto signal_error; | ||
1356 | } | ||
1357 | if (fh != NULL) | ||
1358 | { | ||
1359 | #if DEBUG_DOWNLOAD | 1170 | #if DEBUG_DOWNLOAD |
1360 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1171 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1361 | "Saving decrypted block to disk at offset %llu\n", | 1172 | "Saving decrypted block to disk at offset %llu\n", |
@@ -1389,22 +1200,24 @@ process_result_with_request (void *cls, | |||
1389 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh)); | 1200 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh)); |
1390 | fh = NULL; | 1201 | fh = NULL; |
1391 | } | 1202 | } |
1392 | if (sm->depth == dc->treedepth) | 1203 | |
1204 | if (dr->depth == 0) | ||
1393 | { | 1205 | { |
1206 | /* DBLOCK, update progress and try recursion if applicable */ | ||
1394 | app = prc->size; | 1207 | app = prc->size; |
1395 | if (sm->offset < dc->offset) | 1208 | if (dr->offset < dc->offset) |
1396 | { | 1209 | { |
1397 | /* starting offset begins in the middle of pt, | 1210 | /* starting offset begins in the middle of pt, |
1398 | do not count first bytes as progress */ | 1211 | do not count first bytes as progress */ |
1399 | GNUNET_assert (app > (dc->offset - sm->offset)); | 1212 | GNUNET_assert (app > (dc->offset - dr->offset)); |
1400 | app -= (dc->offset - sm->offset); | 1213 | app -= (dc->offset - dr->offset); |
1401 | } | 1214 | } |
1402 | if (sm->offset + prc->size > dc->offset + dc->length) | 1215 | if (dr->offset + prc->size > dc->offset + dc->length) |
1403 | { | 1216 | { |
1404 | /* end of block is after relevant range, | 1217 | /* end of block is after relevant range, |
1405 | do not count last bytes as progress */ | 1218 | do not count last bytes as progress */ |
1406 | GNUNET_assert (app > (sm->offset + prc->size) - (dc->offset + dc->length)); | 1219 | GNUNET_assert (app > (dr->offset + prc->size) - (dc->offset + dc->length)); |
1407 | app -= (sm->offset + prc->size) - (dc->offset + dc->length); | 1220 | app -= (dr->offset + prc->size) - (dc->offset + dc->length); |
1408 | } | 1221 | } |
1409 | dc->completed += app; | 1222 | dc->completed += app; |
1410 | 1223 | ||
@@ -1419,15 +1232,18 @@ process_result_with_request (void *cls, | |||
1419 | dc); | 1232 | dc); |
1420 | 1233 | ||
1421 | } | 1234 | } |
1235 | dr->state = BRS_DOWNLOAD_DOWN; | ||
1422 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS; | 1236 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS; |
1423 | pi.value.download.specifics.progress.data = pt; | 1237 | pi.value.download.specifics.progress.data = pt; |
1424 | pi.value.download.specifics.progress.offset = sm->offset; | 1238 | pi.value.download.specifics.progress.offset = dr->offset; |
1425 | pi.value.download.specifics.progress.data_len = prc->size; | 1239 | pi.value.download.specifics.progress.data_len = prc->size; |
1426 | pi.value.download.specifics.progress.depth = sm->depth; | 1240 | pi.value.download.specifics.progress.depth = dr->depth; |
1427 | GNUNET_FS_download_make_status_ (&pi, dc); | 1241 | GNUNET_FS_download_make_status_ (&pi, dc); |
1428 | GNUNET_assert (dc->completed <= dc->length); | 1242 | GNUNET_assert (dc->completed <= dc->length); |
1243 | |||
1429 | if (dc->completed == dc->length) | 1244 | if (dc->completed == dc->length) |
1430 | { | 1245 | { |
1246 | /* download completed, signal */ | ||
1431 | #if DEBUG_DOWNLOAD | 1247 | #if DEBUG_DOWNLOAD |
1432 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1248 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1433 | "Download completed, truncating file to desired length %llu\n", | 1249 | "Download completed, truncating file to desired length %llu\n", |
@@ -1447,46 +1263,64 @@ process_result_with_request (void *cls, | |||
1447 | GNUNET_FS_dequeue_ (dc->job_queue); | 1263 | GNUNET_FS_dequeue_ (dc->job_queue); |
1448 | dc->job_queue = NULL; | 1264 | dc->job_queue = NULL; |
1449 | } | 1265 | } |
1450 | if (is_recursive_download (dc)) | 1266 | GNUNET_assert (dr->depth == 0); |
1451 | full_recursive_download (dc); | 1267 | check_completed (dc); |
1452 | if (dc->child_head == NULL) | ||
1453 | { | ||
1454 | /* signal completion */ | ||
1455 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED; | ||
1456 | GNUNET_FS_download_make_status_ (&pi, dc); | ||
1457 | if (dc->parent != NULL) | ||
1458 | check_completed (dc->parent); | ||
1459 | } | ||
1460 | GNUNET_assert (sm->depth == dc->treedepth); | ||
1461 | } | 1268 | } |
1462 | if (sm->depth == dc->treedepth) | 1269 | if (dr->depth == 0) |
1463 | { | 1270 | { |
1271 | propagate_up (dr); | ||
1272 | /* bottom of the tree, no child downloads possible, just sync */ | ||
1464 | GNUNET_FS_download_sync_ (dc); | 1273 | GNUNET_FS_download_sync_ (dc); |
1465 | GNUNET_free (sm); | ||
1466 | return GNUNET_YES; | 1274 | return GNUNET_YES; |
1467 | } | 1275 | } |
1276 | |||
1468 | #if DEBUG_DOWNLOAD | 1277 | #if DEBUG_DOWNLOAD |
1469 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1278 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1470 | "Triggering downloads of children (this block was at depth %u and offset %llu)\n", | 1279 | "Triggering downloads of children (this block was at depth %u and offset %llu)\n", |
1471 | sm->depth, | 1280 | dr->depth, |
1472 | (unsigned long long) sm->offset); | 1281 | (unsigned long long) dr->offset); |
1473 | #endif | 1282 | #endif |
1474 | GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey))); | 1283 | GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey))); |
1475 | chk = (struct ContentHashKey*) pt; | 1284 | chk = (struct ContentHashKey*) pt; |
1476 | for (i=(prc->size / sizeof(struct ContentHashKey))-1;i>=0;i--) | 1285 | for (i=(prc->size / sizeof(struct ContentHashKey))-1;i>=0;i--) |
1477 | { | 1286 | { |
1478 | off = compute_dblock_offset (sm->offset, | 1287 | off = compute_dblock_offset (dr->offset, |
1479 | sm->depth, | 1288 | dr->depth, |
1480 | dc->treedepth, | ||
1481 | i); | 1289 | i); |
1482 | if ( (off + DBLOCK_SIZE >= dc->offset) && | 1290 | drc = dr->children[i]; |
1483 | (off < dc->offset + dc->length) ) | 1291 | switch (drc->state) |
1484 | schedule_block_download (dc, | 1292 | { |
1485 | &chk[i], | 1293 | case BRS_INIT: |
1486 | off, | 1294 | drc->chk = chk[i]; |
1487 | sm->depth + 1); | 1295 | drc->state = BRS_CHK_SET; |
1488 | } | 1296 | schedule_block_download (dc, drc); |
1489 | GNUNET_free (sm); | 1297 | break; |
1298 | case BRS_RECONSTRUCT_DOWN: | ||
1299 | GNUNET_assert (0); | ||
1300 | break; | ||
1301 | case BRS_RECONSTRUCT_META_UP: | ||
1302 | GNUNET_assert (0); | ||
1303 | break; | ||
1304 | case BRS_RECONSTRUCT_UP: | ||
1305 | GNUNET_assert (0); | ||
1306 | break; | ||
1307 | case BRS_CHK_SET: | ||
1308 | GNUNET_assert (0); | ||
1309 | break; | ||
1310 | case BRS_DOWNLOAD_DOWN: | ||
1311 | GNUNET_assert (0); | ||
1312 | break; | ||
1313 | case BRS_DOWNLOAD_UP: | ||
1314 | GNUNET_assert (0); | ||
1315 | break; | ||
1316 | case BRS_ERROR: | ||
1317 | GNUNET_assert (0); | ||
1318 | break; | ||
1319 | default: | ||
1320 | GNUNET_assert (0); | ||
1321 | break; | ||
1322 | } | ||
1323 | } | ||
1490 | GNUNET_FS_download_sync_ (dc); | 1324 | GNUNET_FS_download_sync_ (dc); |
1491 | return GNUNET_YES; | 1325 | return GNUNET_YES; |
1492 | 1326 | ||
@@ -1503,12 +1337,13 @@ process_result_with_request (void *cls, | |||
1503 | dc->th = NULL; | 1337 | dc->th = NULL; |
1504 | } | 1338 | } |
1505 | GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO); | 1339 | GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO); |
1506 | GNUNET_CONTAINER_multihashmap_iterate (dc->active, | ||
1507 | &free_entry, | ||
1508 | NULL); | ||
1509 | dc->pending = NULL; | ||
1510 | dc->client = NULL; | 1340 | dc->client = NULL; |
1511 | GNUNET_free (sm); | 1341 | GNUNET_FS_free_download_request_ (dc->top_request); |
1342 | dc->top_request = NULL; | ||
1343 | GNUNET_CONTAINER_multihashmap_destroy (dc->active); | ||
1344 | dc->active = NULL; | ||
1345 | dc->pending_head = NULL; | ||
1346 | dc->pending_tail = NULL; | ||
1512 | GNUNET_FS_download_sync_ (dc); | 1347 | GNUNET_FS_download_sync_ (dc); |
1513 | return GNUNET_NO; | 1348 | return GNUNET_NO; |
1514 | } | 1349 | } |
@@ -1608,6 +1443,7 @@ transmit_download_request (void *cls, | |||
1608 | struct GNUNET_FS_DownloadContext *dc = cls; | 1443 | struct GNUNET_FS_DownloadContext *dc = cls; |
1609 | size_t msize; | 1444 | size_t msize; |
1610 | struct SearchMessage *sm; | 1445 | struct SearchMessage *sm; |
1446 | struct DownloadRequest *dr; | ||
1611 | 1447 | ||
1612 | dc->th = NULL; | 1448 | dc->th = NULL; |
1613 | if (NULL == buf) | 1449 | if (NULL == buf) |
@@ -1622,13 +1458,13 @@ transmit_download_request (void *cls, | |||
1622 | GNUNET_assert (size >= sizeof (struct SearchMessage)); | 1458 | GNUNET_assert (size >= sizeof (struct SearchMessage)); |
1623 | msize = 0; | 1459 | msize = 0; |
1624 | sm = buf; | 1460 | sm = buf; |
1625 | while ( (dc->pending != NULL) && | 1461 | while ( (NULL != (dr = dc->pending_head)) && |
1626 | (size >= msize + sizeof (struct SearchMessage)) ) | 1462 | (size >= msize + sizeof (struct SearchMessage)) ) |
1627 | { | 1463 | { |
1628 | #if DEBUG_DOWNLOAD | 1464 | #if DEBUG_DOWNLOAD |
1629 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1465 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1630 | "Transmitting download request for `%s' to `%s'-service\n", | 1466 | "Transmitting download request for `%s' to `%s'-service\n", |
1631 | GNUNET_h2s (&dc->pending->chk.query), | 1467 | GNUNET_h2s (&dr->chk.query), |
1632 | "FS"); | 1468 | "FS"); |
1633 | #endif | 1469 | #endif |
1634 | memset (sm, 0, sizeof (struct SearchMessage)); | 1470 | memset (sm, 0, sizeof (struct SearchMessage)); |
@@ -1638,25 +1474,30 @@ transmit_download_request (void *cls, | |||
1638 | sm->options = htonl (1); | 1474 | sm->options = htonl (1); |
1639 | else | 1475 | else |
1640 | sm->options = htonl (0); | 1476 | sm->options = htonl (0); |
1641 | if (dc->pending->depth == dc->treedepth) | 1477 | if (dr->depth == 0) |
1642 | sm->type = htonl (GNUNET_BLOCK_TYPE_FS_DBLOCK); | 1478 | sm->type = htonl (GNUNET_BLOCK_TYPE_FS_DBLOCK); |
1643 | else | 1479 | else |
1644 | sm->type = htonl (GNUNET_BLOCK_TYPE_FS_IBLOCK); | 1480 | sm->type = htonl (GNUNET_BLOCK_TYPE_FS_IBLOCK); |
1645 | sm->anonymity_level = htonl (dc->anonymity); | 1481 | sm->anonymity_level = htonl (dc->anonymity); |
1646 | sm->target = dc->target.hashPubKey; | 1482 | sm->target = dc->target.hashPubKey; |
1647 | sm->query = dc->pending->chk.query; | 1483 | sm->query = dr->chk.query; |
1648 | dc->pending->is_pending = GNUNET_NO; | 1484 | GNUNET_CONTAINER_DLL_remove (dc->pending_head, |
1649 | dc->pending = dc->pending->next; | 1485 | dc->pending_tail, |
1486 | dr); | ||
1487 | dr->is_pending = GNUNET_NO; | ||
1650 | msize += sizeof (struct SearchMessage); | 1488 | msize += sizeof (struct SearchMessage); |
1651 | sm++; | 1489 | sm++; |
1652 | } | 1490 | } |
1653 | if (dc->pending != NULL) | 1491 | if (dc->pending_head != NULL) |
1654 | dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client, | 1492 | { |
1655 | sizeof (struct SearchMessage), | 1493 | dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client, |
1656 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | 1494 | sizeof (struct SearchMessage), |
1657 | GNUNET_NO, | 1495 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, |
1658 | &transmit_download_request, | 1496 | GNUNET_NO, |
1659 | dc); | 1497 | &transmit_download_request, |
1498 | dc); | ||
1499 | GNUNET_assert (dc->th != NULL); | ||
1500 | } | ||
1660 | return msize; | 1501 | return msize; |
1661 | } | 1502 | } |
1662 | 1503 | ||
@@ -1686,12 +1527,16 @@ do_reconnect (void *cls, | |||
1686 | return; | 1527 | return; |
1687 | } | 1528 | } |
1688 | dc->client = client; | 1529 | dc->client = client; |
1689 | dc->th = GNUNET_CLIENT_notify_transmit_ready (client, | 1530 | if (dc->pending_head != NULL) |
1690 | sizeof (struct SearchMessage), | 1531 | { |
1691 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | 1532 | dc->th = GNUNET_CLIENT_notify_transmit_ready (client, |
1692 | GNUNET_NO, | 1533 | sizeof (struct SearchMessage), |
1693 | &transmit_download_request, | 1534 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, |
1694 | dc); | 1535 | GNUNET_NO, |
1536 | &transmit_download_request, | ||
1537 | dc); | ||
1538 | GNUNET_assert (dc->th != NULL); | ||
1539 | } | ||
1695 | GNUNET_CLIENT_receive (client, | 1540 | GNUNET_CLIENT_receive (client, |
1696 | &receive_results, | 1541 | &receive_results, |
1697 | dc, | 1542 | dc, |
@@ -1700,7 +1545,7 @@ do_reconnect (void *cls, | |||
1700 | 1545 | ||
1701 | 1546 | ||
1702 | /** | 1547 | /** |
1703 | * Add entries that are not yet pending back to the pending list. | 1548 | * Add entries to the pending list. |
1704 | * | 1549 | * |
1705 | * @param cls our download context | 1550 | * @param cls our download context |
1706 | * @param key unused | 1551 | * @param key unused |
@@ -1715,12 +1560,10 @@ retry_entry (void *cls, | |||
1715 | struct GNUNET_FS_DownloadContext *dc = cls; | 1560 | struct GNUNET_FS_DownloadContext *dc = cls; |
1716 | struct DownloadRequest *dr = entry; | 1561 | struct DownloadRequest *dr = entry; |
1717 | 1562 | ||
1718 | if (! dr->is_pending) | 1563 | GNUNET_CONTAINER_DLL_insert (dc->pending_head, |
1719 | { | 1564 | dc->pending_tail, |
1720 | dr->next = dc->pending; | 1565 | dr); |
1721 | dr->is_pending = GNUNET_YES; | 1566 | dr->is_pending = GNUNET_YES; |
1722 | dc->pending = entry; | ||
1723 | } | ||
1724 | return GNUNET_OK; | 1567 | return GNUNET_OK; |
1725 | } | 1568 | } |
1726 | 1569 | ||
@@ -1747,6 +1590,9 @@ try_reconnect (struct GNUNET_FS_DownloadContext *dc) | |||
1747 | GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th); | 1590 | GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th); |
1748 | dc->th = NULL; | 1591 | dc->th = NULL; |
1749 | } | 1592 | } |
1593 | /* full reset of the pending list */ | ||
1594 | dc->pending_head = NULL; | ||
1595 | dc->pending_tail = NULL; | ||
1750 | GNUNET_CONTAINER_multihashmap_iterate (dc->active, | 1596 | GNUNET_CONTAINER_multihashmap_iterate (dc->active, |
1751 | &retry_entry, | 1597 | &retry_entry, |
1752 | dc); | 1598 | dc); |
@@ -1764,7 +1610,6 @@ try_reconnect (struct GNUNET_FS_DownloadContext *dc) | |||
1764 | } | 1610 | } |
1765 | 1611 | ||
1766 | 1612 | ||
1767 | |||
1768 | /** | 1613 | /** |
1769 | * We're allowed to ask the FS service for our blocks. Start the download. | 1614 | * We're allowed to ask the FS service for our blocks. Start the download. |
1770 | * | 1615 | * |
@@ -1799,13 +1644,16 @@ activate_fs_download (void *cls, | |||
1799 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1644 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1800 | "Asking for transmission to FS service\n"); | 1645 | "Asking for transmission to FS service\n"); |
1801 | #endif | 1646 | #endif |
1802 | dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client, | 1647 | if (dc->pending_head != NULL) |
1803 | sizeof (struct SearchMessage), | 1648 | { |
1804 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, | 1649 | dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client, |
1805 | GNUNET_NO, | 1650 | sizeof (struct SearchMessage), |
1806 | &transmit_download_request, | 1651 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, |
1807 | dc); | 1652 | GNUNET_NO, |
1808 | GNUNET_assert (dc->th != NULL); | 1653 | &transmit_download_request, |
1654 | dc); | ||
1655 | GNUNET_assert (dc->th != NULL); | ||
1656 | } | ||
1809 | } | 1657 | } |
1810 | 1658 | ||
1811 | 1659 | ||
@@ -1840,6 +1688,271 @@ deactivate_fs_download (void *cls) | |||
1840 | 1688 | ||
1841 | 1689 | ||
1842 | /** | 1690 | /** |
1691 | * (recursively) Create a download request structure. | ||
1692 | * | ||
1693 | * @param parent parent of the current entry | ||
1694 | * @param depth depth of the current entry, 0 are the DBLOCKs, | ||
1695 | * top level block is 'dc->treedepth - 1' | ||
1696 | * @param dr_offset offset in the original file this block maps to | ||
1697 | * (as in, offset of the first byte of the first DBLOCK | ||
1698 | * in the subtree rooted in the returned download request tree) | ||
1699 | * @param file_start_offset desired starting offset for the download | ||
1700 | * in the original file; requesting tree should not contain | ||
1701 | * DBLOCKs prior to the file_start_offset | ||
1702 | * @param file_length desired number of bytes the user wanted to access | ||
1703 | * (from file_start_offset). Resulting tree should not contain | ||
1704 | * DBLOCKs after file_start_offset + file_length. | ||
1705 | * @return download request tree for the given range of DBLOCKs at | ||
1706 | * the specified depth | ||
1707 | */ | ||
1708 | static struct DownloadRequest * | ||
1709 | create_download_request (struct DownloadRequest *parent, | ||
1710 | unsigned int depth, | ||
1711 | uint64_t dr_offset, | ||
1712 | uint64_t file_start_offset, | ||
1713 | uint64_t desired_length) | ||
1714 | { | ||
1715 | struct DownloadRequest *dr; | ||
1716 | unsigned int i; | ||
1717 | unsigned int head_skip; | ||
1718 | uint64_t child_block_size; | ||
1719 | |||
1720 | dr = GNUNET_malloc (sizeof (struct DownloadRequest)); | ||
1721 | dr->parent = parent; | ||
1722 | dr->depth = depth; | ||
1723 | dr->offset = dr_offset; | ||
1724 | if (depth > 0) | ||
1725 | { | ||
1726 | child_block_size = GNUNET_FS_tree_compute_tree_size (depth - 1); | ||
1727 | |||
1728 | /* calculate how many blocks at this level are not interesting | ||
1729 | from the start (rounded down), either because of the requested | ||
1730 | file offset or because this IBlock is further along */ | ||
1731 | if (dr_offset < file_start_offset) | ||
1732 | head_skip = file_start_offset / child_block_size; | ||
1733 | else | ||
1734 | head_skip = dr_offset / child_block_size; | ||
1735 | |||
1736 | /* calculate index of last block at this level that is interesting (rounded up) */ | ||
1737 | dr->num_children = file_start_offset + desired_length / child_block_size; | ||
1738 | if (dr->num_children * child_block_size < file_start_offset + desired_length) | ||
1739 | dr->num_children++; /* round up */ | ||
1740 | |||
1741 | /* now we can get the total number of children for this block */ | ||
1742 | dr->num_children -= head_skip; | ||
1743 | if (dr->num_children > CHK_PER_INODE) | ||
1744 | dr->num_children = CHK_PER_INODE; /* cap at max */ | ||
1745 | |||
1746 | /* why else would we have gotten here to begin with? (that'd be a bad logic error) */ | ||
1747 | GNUNET_assert (dr->num_children > 0); | ||
1748 | |||
1749 | dr->children = GNUNET_malloc (dr->num_children * | ||
1750 | sizeof (struct DownloadRequest *)); | ||
1751 | for (i=0;i<dr->num_children;i++) | ||
1752 | dr->children[i] = create_download_request (dr, | ||
1753 | depth - 1, | ||
1754 | dr_offset + i * child_block_size, | ||
1755 | file_start_offset, | ||
1756 | desired_length); | ||
1757 | } | ||
1758 | return dr; | ||
1759 | } | ||
1760 | |||
1761 | |||
1762 | /** | ||
1763 | * Continuation after a possible attempt to reconstruct | ||
1764 | * the current IBlock from the existing file. | ||
1765 | * | ||
1766 | * @param cls the 'struct ReconstructContext' | ||
1767 | * @param tc scheduler context | ||
1768 | */ | ||
1769 | static void | ||
1770 | reconstruct_cont (void *cls, | ||
1771 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
1772 | { | ||
1773 | struct GNUNET_FS_DownloadContext *dc = cls; | ||
1774 | |||
1775 | /* clean up state from tree encoder */ | ||
1776 | if (dc->te != NULL) | ||
1777 | { | ||
1778 | GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL); | ||
1779 | dc->te = NULL; | ||
1780 | } | ||
1781 | if (dc->task != GNUNET_SCHEDULER_NO_TASK) | ||
1782 | { | ||
1783 | GNUNET_SCHEDULER_cancel (dc->task); | ||
1784 | dc->task = GNUNET_SCHEDULER_NO_TASK; | ||
1785 | } | ||
1786 | if (dc->rfh != NULL) | ||
1787 | { | ||
1788 | GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (dc->rfh)); | ||
1789 | dc->rfh = NULL; | ||
1790 | } | ||
1791 | /* start "normal" download */ | ||
1792 | schedule_block_download (dc, | ||
1793 | dc->top_request); | ||
1794 | } | ||
1795 | |||
1796 | |||
1797 | /** | ||
1798 | * Task requesting the next block from the tree encoder. | ||
1799 | * | ||
1800 | * @param tc task context | ||
1801 | */ | ||
1802 | static void | ||
1803 | get_next_block (void *cls, | ||
1804 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
1805 | { | ||
1806 | struct GNUNET_FS_DownloadContext *dc = cls; | ||
1807 | |||
1808 | dc->task = GNUNET_SCHEDULER_NO_TASK; | ||
1809 | GNUNET_FS_tree_encoder_next (dc->te); | ||
1810 | } | ||
1811 | |||
1812 | |||
1813 | |||
1814 | /** | ||
1815 | * Function called asking for the current (encoded) | ||
1816 | * block to be processed. After processing the | ||
1817 | * client should either call "GNUNET_FS_tree_encode_next" | ||
1818 | * or (on error) "GNUNET_FS_tree_encode_finish". | ||
1819 | * | ||
1820 | * This function checks if the content on disk matches | ||
1821 | * the expected content based on the URI. | ||
1822 | * | ||
1823 | * @param cls closure | ||
1824 | * @param chk content hash key for the block | ||
1825 | * @param offset offset of the block | ||
1826 | * @param depth depth of the block, 0 for DBLOCK | ||
1827 | * @param type type of the block (IBLOCK or DBLOCK) | ||
1828 | * @param block the (encrypted) block | ||
1829 | * @param block_size size of block (in bytes) | ||
1830 | */ | ||
1831 | static void | ||
1832 | reconstruct_cb (void *cls, | ||
1833 | const struct ContentHashKey *chk, | ||
1834 | uint64_t offset, | ||
1835 | unsigned int depth, | ||
1836 | enum GNUNET_BLOCK_Type type, | ||
1837 | const void *block, | ||
1838 | uint16_t block_size) | ||
1839 | { | ||
1840 | struct GNUNET_FS_DownloadContext *dc = cls; | ||
1841 | struct GNUNET_FS_ProgressInfo pi; | ||
1842 | struct DownloadRequest *dr; | ||
1843 | uint64_t blen; | ||
1844 | unsigned int chld; | ||
1845 | |||
1846 | /* find corresponding request entry */ | ||
1847 | dr = dc->top_request; | ||
1848 | while (dr->depth > depth) | ||
1849 | { | ||
1850 | blen = GNUNET_FS_tree_compute_tree_size (dr->depth); | ||
1851 | chld = (offset - dr->offset) / blen; | ||
1852 | GNUNET_assert (chld < dr->num_children); | ||
1853 | dr = dr->children[chld]; | ||
1854 | } | ||
1855 | switch (dr->state) | ||
1856 | { | ||
1857 | case BRS_INIT: | ||
1858 | break; | ||
1859 | case BRS_RECONSTRUCT_DOWN: | ||
1860 | break; | ||
1861 | case BRS_RECONSTRUCT_META_UP: | ||
1862 | break; | ||
1863 | case BRS_RECONSTRUCT_UP: | ||
1864 | break; | ||
1865 | case BRS_CHK_SET: | ||
1866 | if (0 == memcmp (chk, | ||
1867 | &dr->chk, | ||
1868 | sizeof (struct ContentHashKey))) | ||
1869 | { | ||
1870 | /* block matches, hence tree below matches; | ||
1871 | this request is done! */ | ||
1872 | dr->state = BRS_DOWNLOAD_UP; | ||
1873 | /* calculate how many bytes of payload this block | ||
1874 | corresponds to */ | ||
1875 | blen = GNUNET_FS_tree_compute_tree_size (dr->depth); | ||
1876 | /* how many of those bytes are in the requested range? */ | ||
1877 | blen = GNUNET_MIN (blen, | ||
1878 | dc->length + dc->offset - dr->offset); | ||
1879 | /* signal progress */ | ||
1880 | dc->completed += blen; | ||
1881 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS; | ||
1882 | pi.value.download.specifics.progress.data = NULL; | ||
1883 | pi.value.download.specifics.progress.offset = offset; | ||
1884 | pi.value.download.specifics.progress.data_len = 0; | ||
1885 | pi.value.download.specifics.progress.depth = 0; | ||
1886 | GNUNET_FS_download_make_status_ (&pi, dc); | ||
1887 | } | ||
1888 | else | ||
1889 | { | ||
1890 | } | ||
1891 | break; | ||
1892 | case BRS_DOWNLOAD_DOWN: | ||
1893 | break; | ||
1894 | case BRS_DOWNLOAD_UP: | ||
1895 | break; | ||
1896 | case BRS_ERROR: | ||
1897 | break; | ||
1898 | default: | ||
1899 | GNUNET_assert (0); | ||
1900 | break; | ||
1901 | } | ||
1902 | if ( (dr == dc->top_request) && | ||
1903 | (dr->state == BRS_DOWNLOAD_UP) ) | ||
1904 | { | ||
1905 | check_completed (dc); | ||
1906 | return; | ||
1907 | } | ||
1908 | dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, | ||
1909 | dc); | ||
1910 | } | ||
1911 | |||
1912 | |||
1913 | /** | ||
1914 | * Function called by the tree encoder to obtain a block of plaintext | ||
1915 | * data (for the lowest level of the tree). | ||
1916 | * | ||
1917 | * @param cls our 'struct ReconstructContext' | ||
1918 | * @param offset identifies which block to get | ||
1919 | * @param max (maximum) number of bytes to get; returning | ||
1920 | * fewer will also cause errors | ||
1921 | * @param buf where to copy the plaintext buffer | ||
1922 | * @param emsg location to store an error message (on error) | ||
1923 | * @return number of bytes copied to buf, 0 on error | ||
1924 | */ | ||
1925 | static size_t | ||
1926 | fh_reader (void *cls, | ||
1927 | uint64_t offset, | ||
1928 | size_t max, | ||
1929 | void *buf, | ||
1930 | char **emsg) | ||
1931 | { | ||
1932 | struct GNUNET_FS_DownloadContext *dc = cls; | ||
1933 | struct GNUNET_DISK_FileHandle *fh = dc->rfh; | ||
1934 | ssize_t ret; | ||
1935 | |||
1936 | *emsg = NULL; | ||
1937 | if (offset != | ||
1938 | GNUNET_DISK_file_seek (fh, | ||
1939 | offset, | ||
1940 | GNUNET_DISK_SEEK_SET)) | ||
1941 | { | ||
1942 | *emsg = GNUNET_strdup (strerror (errno)); | ||
1943 | return 0; | ||
1944 | } | ||
1945 | ret = GNUNET_DISK_file_read (fh, buf, max); | ||
1946 | if (ret < 0) | ||
1947 | { | ||
1948 | *emsg = GNUNET_strdup (strerror (errno)); | ||
1949 | return 0; | ||
1950 | } | ||
1951 | return ret; | ||
1952 | } | ||
1953 | |||
1954 | |||
1955 | /** | ||
1843 | * Task that creates the initial (top-level) download | 1956 | * Task that creates the initial (top-level) download |
1844 | * request for the file. | 1957 | * request for the file. |
1845 | * | 1958 | * |
@@ -1854,7 +1967,11 @@ GNUNET_FS_download_start_task_ (void *cls, | |||
1854 | struct GNUNET_FS_ProgressInfo pi; | 1967 | struct GNUNET_FS_ProgressInfo pi; |
1855 | struct GNUNET_DISK_FileHandle *fh; | 1968 | struct GNUNET_DISK_FileHandle *fh; |
1856 | 1969 | ||
1857 | dc->start_task = GNUNET_SCHEDULER_NO_TASK; | 1970 | #if DEBUG_DOWNLOAD |
1971 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1972 | "Start task running...\n"); | ||
1973 | #endif | ||
1974 | dc->task = GNUNET_SCHEDULER_NO_TASK; | ||
1858 | if (dc->length == 0) | 1975 | if (dc->length == 0) |
1859 | { | 1976 | { |
1860 | /* no bytes required! */ | 1977 | /* no bytes required! */ |
@@ -1863,33 +1980,128 @@ GNUNET_FS_download_start_task_ (void *cls, | |||
1863 | fh = GNUNET_DISK_file_open (dc->filename != NULL | 1980 | fh = GNUNET_DISK_file_open (dc->filename != NULL |
1864 | ? dc->filename | 1981 | ? dc->filename |
1865 | : dc->temp_filename, | 1982 | : dc->temp_filename, |
1866 | GNUNET_DISK_OPEN_READWRITE | | 1983 | GNUNET_DISK_OPEN_READWRITE | |
1867 | GNUNET_DISK_OPEN_CREATE, | 1984 | GNUNET_DISK_OPEN_CREATE | |
1985 | ( (0 == GNUNET_FS_uri_chk_get_file_size (dc->uri)) | ||
1986 | ? GNUNET_DISK_OPEN_TRUNCATE : 0), | ||
1868 | GNUNET_DISK_PERM_USER_READ | | 1987 | GNUNET_DISK_PERM_USER_READ | |
1869 | GNUNET_DISK_PERM_USER_WRITE | | 1988 | GNUNET_DISK_PERM_USER_WRITE | |
1870 | GNUNET_DISK_PERM_GROUP_READ | | 1989 | GNUNET_DISK_PERM_GROUP_READ | |
1871 | GNUNET_DISK_PERM_OTHER_READ); | 1990 | GNUNET_DISK_PERM_OTHER_READ); |
1872 | GNUNET_DISK_file_close (fh); | 1991 | GNUNET_DISK_file_close (fh); |
1873 | } | 1992 | } |
1874 | |||
1875 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED; | ||
1876 | GNUNET_FS_download_make_status_ (&pi, dc); | ||
1877 | GNUNET_FS_download_sync_ (dc); | 1993 | GNUNET_FS_download_sync_ (dc); |
1878 | if (dc->parent != NULL) | 1994 | check_completed (dc); |
1879 | check_completed (dc->parent); | ||
1880 | return; | 1995 | return; |
1881 | } | 1996 | } |
1882 | schedule_block_download (dc, | 1997 | if (dc->emsg != NULL) |
1883 | (dc->uri->type == chk) | 1998 | return; |
1884 | ? &dc->uri->data.chk.chk | 1999 | if (dc->top_request == NULL) |
1885 | : &dc->uri->data.loc.fi.chk, | 2000 | { |
1886 | 0, | 2001 | dc->top_request = create_download_request (NULL, dc->treedepth - 1, 0, |
1887 | 1 /* 0 == CHK, 1 == top */); | 2002 | dc->offset, dc->length); |
1888 | GNUNET_FS_download_sync_ (dc); | 2003 | dc->top_request->state = BRS_CHK_SET; |
2004 | dc->top_request->chk = (dc->uri->type == chk) | ||
2005 | ? dc->uri->data.chk.chk | ||
2006 | : dc->uri->data.loc.fi.chk; | ||
2007 | /* signal start */ | ||
2008 | GNUNET_FS_download_sync_ (dc); | ||
2009 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; | ||
2010 | pi.value.download.specifics.start.meta = dc->meta; | ||
2011 | GNUNET_FS_download_make_status_ (&pi, dc); | ||
2012 | } | ||
1889 | GNUNET_FS_download_start_downloading_ (dc); | 2013 | GNUNET_FS_download_start_downloading_ (dc); |
1890 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; | 2014 | /* attempt reconstruction from disk */ |
1891 | pi.value.download.specifics.start.meta = dc->meta; | 2015 | if (GNUNET_YES == GNUNET_DISK_file_test (dc->filename)) |
1892 | GNUNET_FS_download_make_status_ (&pi, dc); | 2016 | dc->rfh = GNUNET_DISK_file_open (dc->filename, |
2017 | GNUNET_DISK_OPEN_READ, | ||
2018 | GNUNET_DISK_PERM_NONE); | ||
2019 | if (dc->top_request->state == BRS_CHK_SET) | ||
2020 | { | ||
2021 | if (dc->rfh != NULL) | ||
2022 | { | ||
2023 | /* first, try top-down */ | ||
2024 | try_top_down_reconstruction (dc, dc->top_request); | ||
2025 | switch (dc->top_request->state) | ||
2026 | { | ||
2027 | case BRS_CHK_SET: | ||
2028 | break; /* normal */ | ||
2029 | case BRS_DOWNLOAD_DOWN: | ||
2030 | break; /* normal, some blocks already down */ | ||
2031 | case BRS_DOWNLOAD_UP: | ||
2032 | /* already done entirely, party! */ | ||
2033 | dc->completed = dc->length; | ||
2034 | GNUNET_FS_download_sync_ (dc); | ||
2035 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS; | ||
2036 | /* slightly ugly: no data provided to callee; maybe mmap the | ||
2037 | file instead? Or is 'data' pure convenience!? */ | ||
2038 | pi.value.download.specifics.progress.data = NULL; | ||
2039 | pi.value.download.specifics.progress.offset = dc->offset; | ||
2040 | pi.value.download.specifics.progress.data_len = dc->length; | ||
2041 | pi.value.download.specifics.progress.depth = 0; | ||
2042 | GNUNET_FS_download_make_status_ (&pi, dc); | ||
2043 | if (dc->rfh != NULL) | ||
2044 | { | ||
2045 | /* avoid hanging on to file handle longer than | ||
2046 | necessary */ | ||
2047 | GNUNET_DISK_file_close (dc->rfh); | ||
2048 | dc->rfh = NULL; | ||
2049 | } | ||
2050 | check_completed (dc); | ||
2051 | return; | ||
2052 | case BRS_ERROR: | ||
2053 | GNUNET_asprintf (&dc->emsg, | ||
2054 | _("Invalid URI")); | ||
2055 | GNUNET_FS_download_sync_ (dc); | ||
2056 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; | ||
2057 | pi.value.download.specifics.error.message = dc->emsg; | ||
2058 | GNUNET_FS_download_make_status_ (&pi, dc); | ||
2059 | return; | ||
2060 | default: | ||
2061 | GNUNET_assert (0); | ||
2062 | break; | ||
2063 | } | ||
2064 | } | ||
2065 | } | ||
2066 | /* attempt reconstruction from meta data */ | ||
2067 | if (GNUNET_FS_uri_chk_get_file_size (dc->uri) <= MAX_INLINE_SIZE) | ||
2068 | { | ||
2069 | GNUNET_CONTAINER_meta_data_iterate (dc->meta, | ||
2070 | &match_full_data, | ||
2071 | dc); | ||
2072 | if (dc->top_request->state == BRS_DOWNLOAD_UP) | ||
2073 | { | ||
2074 | if (dc->rfh != NULL) | ||
2075 | { | ||
2076 | /* avoid hanging on to file handle longer than | ||
2077 | necessary */ | ||
2078 | GNUNET_DISK_file_close (dc->rfh); | ||
2079 | dc->rfh = NULL; | ||
2080 | } | ||
2081 | return; /* finished, status update was already done for us */ | ||
2082 | } | ||
2083 | } | ||
2084 | if (dc->rfh != NULL) | ||
2085 | { | ||
2086 | /* finally, try bottom-up */ | ||
2087 | dc->te = GNUNET_FS_tree_encoder_create (dc->h, | ||
2088 | dc->old_file_size, | ||
2089 | dc, | ||
2090 | &fh_reader, | ||
2091 | &reconstruct_cb, | ||
2092 | NULL, | ||
2093 | &reconstruct_cont); | ||
2094 | dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, | ||
2095 | dc); | ||
2096 | } | ||
2097 | else | ||
2098 | { | ||
2099 | /* simple, top-level download */ | ||
2100 | schedule_block_download (dc, | ||
2101 | dc->top_request); | ||
2102 | } | ||
2103 | if (dc->top_request->state == BRS_DOWNLOAD_UP) | ||
2104 | check_completed (dc); | ||
1893 | } | 2105 | } |
1894 | 2106 | ||
1895 | 2107 | ||
@@ -1923,30 +2135,29 @@ GNUNET_FS_download_signal_suspend_ (void *cls) | |||
1923 | GNUNET_CONTAINER_DLL_remove (dc->parent->child_head, | 2135 | GNUNET_CONTAINER_DLL_remove (dc->parent->child_head, |
1924 | dc->parent->child_tail, | 2136 | dc->parent->child_tail, |
1925 | dc); | 2137 | dc); |
1926 | if (GNUNET_SCHEDULER_NO_TASK != dc->task) | 2138 | if (dc->task != GNUNET_SCHEDULER_NO_TASK) |
1927 | GNUNET_SCHEDULER_cancel (dc->task); | ||
1928 | if (dc->start_task != GNUNET_SCHEDULER_NO_TASK) | ||
1929 | { | 2139 | { |
1930 | GNUNET_SCHEDULER_cancel (dc->start_task); | 2140 | GNUNET_SCHEDULER_cancel (dc->task); |
1931 | dc->start_task = GNUNET_SCHEDULER_NO_TASK; | 2141 | dc->task = GNUNET_SCHEDULER_NO_TASK; |
1932 | } | 2142 | } |
1933 | else | 2143 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND; |
2144 | GNUNET_FS_download_make_status_ (&pi, dc); | ||
2145 | if (dc->te != NULL) | ||
1934 | { | 2146 | { |
1935 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND; | 2147 | GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL); |
1936 | GNUNET_FS_download_make_status_ (&pi, dc); | 2148 | dc->te = NULL; |
1937 | } | 2149 | } |
1938 | if (dc->rcc != NULL) | 2150 | if (dc->rfh != NULL) |
1939 | { | 2151 | { |
1940 | if (dc->rcc->task != GNUNET_SCHEDULER_NO_TASK) | 2152 | GNUNET_DISK_file_close (dc->rfh); |
1941 | GNUNET_SCHEDULER_cancel (dc->rcc->task); | 2153 | dc->rfh = NULL; |
1942 | if (dc->rcc->te != NULL) | 2154 | } |
1943 | GNUNET_FS_tree_encoder_finish (dc->rcc->te, NULL, NULL); | 2155 | GNUNET_FS_free_download_request_ (dc->top_request); |
1944 | dc->rcc = NULL; | 2156 | if (dc->active != NULL) |
2157 | { | ||
2158 | GNUNET_CONTAINER_multihashmap_destroy (dc->active); | ||
2159 | dc->active = NULL; | ||
1945 | } | 2160 | } |
1946 | GNUNET_CONTAINER_multihashmap_iterate (dc->active, | ||
1947 | &free_entry, | ||
1948 | NULL); | ||
1949 | GNUNET_CONTAINER_multihashmap_destroy (dc->active); | ||
1950 | GNUNET_free_non_null (dc->filename); | 2161 | GNUNET_free_non_null (dc->filename); |
1951 | GNUNET_CONTAINER_meta_data_destroy (dc->meta); | 2162 | GNUNET_CONTAINER_meta_data_destroy (dc->meta); |
1952 | GNUNET_FS_uri_destroy (dc->uri); | 2163 | GNUNET_FS_uri_destroy (dc->uri); |
@@ -2067,7 +2278,8 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, | |||
2067 | &GNUNET_FS_download_signal_suspend_, | 2278 | &GNUNET_FS_download_signal_suspend_, |
2068 | dc); | 2279 | dc); |
2069 | } | 2280 | } |
2070 | dc->start_task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); | 2281 | dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, |
2282 | dc); | ||
2071 | return dc; | 2283 | return dc; |
2072 | } | 2284 | } |
2073 | 2285 | ||
@@ -2184,10 +2396,12 @@ GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h, | |||
2184 | "Download tree has depth %u\n", | 2396 | "Download tree has depth %u\n", |
2185 | dc->treedepth); | 2397 | dc->treedepth); |
2186 | #endif | 2398 | #endif |
2187 | dc->start_task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); | 2399 | dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, |
2400 | dc); | ||
2188 | return dc; | 2401 | return dc; |
2189 | } | 2402 | } |
2190 | 2403 | ||
2404 | |||
2191 | /** | 2405 | /** |
2192 | * Start the downloading process (by entering the queue). | 2406 | * Start the downloading process (by entering the queue). |
2193 | * | 2407 | * |
@@ -2222,10 +2436,12 @@ GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, | |||
2222 | 2436 | ||
2223 | if (dc->top != NULL) | 2437 | if (dc->top != NULL) |
2224 | GNUNET_FS_end_top (dc->h, dc->top); | 2438 | GNUNET_FS_end_top (dc->h, dc->top); |
2225 | if (dc->start_task != GNUNET_SCHEDULER_NO_TASK) | 2439 | |
2440 | |||
2441 | if (dc->task != GNUNET_SCHEDULER_NO_TASK) | ||
2226 | { | 2442 | { |
2227 | GNUNET_SCHEDULER_cancel (dc->start_task); | 2443 | GNUNET_SCHEDULER_cancel (dc->task); |
2228 | dc->start_task = GNUNET_SCHEDULER_NO_TASK; | 2444 | dc->task = GNUNET_SCHEDULER_NO_TASK; |
2229 | } | 2445 | } |
2230 | if (dc->search != NULL) | 2446 | if (dc->search != NULL) |
2231 | { | 2447 | { |
@@ -2237,13 +2453,10 @@ GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, | |||
2237 | GNUNET_FS_dequeue_ (dc->job_queue); | 2453 | GNUNET_FS_dequeue_ (dc->job_queue); |
2238 | dc->job_queue = NULL; | 2454 | dc->job_queue = NULL; |
2239 | } | 2455 | } |
2240 | if (dc->rcc != NULL) | 2456 | if (dc->te != NULL) |
2241 | { | 2457 | { |
2242 | if (dc->rcc->task != GNUNET_SCHEDULER_NO_TASK) | 2458 | GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL); |
2243 | GNUNET_SCHEDULER_cancel (dc->rcc->task); | 2459 | dc->te = NULL; |
2244 | if (dc->rcc->te != NULL) | ||
2245 | GNUNET_FS_tree_encoder_finish (dc->rcc->te, NULL, NULL); | ||
2246 | dc->rcc = NULL; | ||
2247 | } | 2460 | } |
2248 | have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO; | 2461 | have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO; |
2249 | while (NULL != dc->child_head) | 2462 | while (NULL != dc->child_head) |
@@ -2268,12 +2481,13 @@ GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, | |||
2268 | dc->serialization); | 2481 | dc->serialization); |
2269 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED; | 2482 | pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED; |
2270 | GNUNET_FS_download_make_status_ (&pi, dc); | 2483 | GNUNET_FS_download_make_status_ (&pi, dc); |
2271 | if (GNUNET_SCHEDULER_NO_TASK != dc->task) | 2484 | GNUNET_FS_free_download_request_ (dc->top_request); |
2272 | GNUNET_SCHEDULER_cancel (dc->task); | 2485 | dc->top_request = NULL; |
2273 | GNUNET_CONTAINER_multihashmap_iterate (dc->active, | 2486 | if (dc->active != NULL) |
2274 | &free_entry, | 2487 | { |
2275 | NULL); | 2488 | GNUNET_CONTAINER_multihashmap_destroy (dc->active); |
2276 | GNUNET_CONTAINER_multihashmap_destroy (dc->active); | 2489 | dc->active = NULL; |
2490 | } | ||
2277 | if (dc->filename != NULL) | 2491 | if (dc->filename != NULL) |
2278 | { | 2492 | { |
2279 | if ( (dc->completed != dc->length) && | 2493 | if ( (dc->completed != dc->length) && |
diff --git a/src/fs/fs_misc.c b/src/fs/fs_misc.c new file mode 100644 index 000000000..e6c4eeea9 --- /dev/null +++ b/src/fs/fs_misc.c | |||
@@ -0,0 +1,165 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet. | ||
3 | (C) 2010, 2011 Christian Grothoff (and other contributing authors) | ||
4 | |||
5 | GNUnet is free software; you can redistribute it and/or modify | ||
6 | it under the terms of the GNU General Public License as published | ||
7 | by the Free Software Foundation; either version 3, or (at your | ||
8 | option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU General Public License | ||
16 | along with GNUnet; see the file COPYING. If not, write to the | ||
17 | Free Software Foundation, Inc., 59 Temple Place - Suite 330, | ||
18 | Boston, MA 02111-1307, USA. | ||
19 | */ | ||
20 | /** | ||
21 | * @file fs/fs_misc.c | ||
22 | * @brief misc. functions related to file-sharing in general | ||
23 | * @author Christian Grothoff | ||
24 | */ | ||
25 | #include "platform.h" | ||
26 | #include "gnunet_constants.h" | ||
27 | #include "gnunet_fs_service.h" | ||
28 | #include "fs.h" | ||
29 | |||
30 | |||
31 | /** | ||
32 | * Suggest a filename based on given metadata. | ||
33 | * | ||
34 | * @param md given meta data | ||
35 | * @return NULL if meta data is useless for suggesting a filename | ||
36 | */ | ||
37 | char * | ||
38 | GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_CONTAINER_MetaData *md) | ||
39 | { | ||
40 | static const char *mimeMap[][2] = { | ||
41 | {"application/bz2", ".bz2"}, | ||
42 | {"application/gnunet-directory", ".gnd"}, | ||
43 | {"application/java", ".class"}, | ||
44 | {"application/msword", ".doc"}, | ||
45 | {"application/ogg", ".ogg"}, | ||
46 | {"application/pdf", ".pdf"}, | ||
47 | {"application/pgp-keys", ".key"}, | ||
48 | {"application/pgp-signature", ".pgp"}, | ||
49 | {"application/postscript", ".ps"}, | ||
50 | {"application/rar", ".rar"}, | ||
51 | {"application/rtf", ".rtf"}, | ||
52 | {"application/xml", ".xml"}, | ||
53 | {"application/x-debian-package", ".deb"}, | ||
54 | {"application/x-dvi", ".dvi"}, | ||
55 | {"applixation/x-flac", ".flac"}, | ||
56 | {"applixation/x-gzip", ".gz"}, | ||
57 | {"application/x-java-archive", ".jar"}, | ||
58 | {"application/x-java-vm", ".class"}, | ||
59 | {"application/x-python-code", ".pyc"}, | ||
60 | {"application/x-redhat-package-manager", ".rpm"}, | ||
61 | {"application/x-rpm", ".rpm"}, | ||
62 | {"application/x-tar", ".tar"}, | ||
63 | {"application/x-tex-pk", ".pk"}, | ||
64 | {"application/x-texinfo", ".texinfo"}, | ||
65 | {"application/x-xcf", ".xcf"}, | ||
66 | {"application/x-xfig", ".xfig"}, | ||
67 | {"application/zip", ".zip"}, | ||
68 | |||
69 | {"audio/midi", ".midi"}, | ||
70 | {"audio/mpeg", ".mp3"}, | ||
71 | {"audio/real", ".rm"}, | ||
72 | {"audio/x-wav", ".wav"}, | ||
73 | |||
74 | {"image/gif", ".gif"}, | ||
75 | {"image/jpeg", ".jpg"}, | ||
76 | {"image/pcx", ".pcx"}, | ||
77 | {"image/png", ".png"}, | ||
78 | {"image/tiff", ".tiff"}, | ||
79 | {"image/x-ms-bmp", ".bmp"}, | ||
80 | {"image/x-xpixmap", ".xpm"}, | ||
81 | |||
82 | {"text/css", ".css"}, | ||
83 | {"text/html", ".html"}, | ||
84 | {"text/plain", ".txt"}, | ||
85 | {"text/rtf", ".rtf"}, | ||
86 | {"text/x-c++hdr", ".h++"}, | ||
87 | {"text/x-c++src", ".c++"}, | ||
88 | {"text/x-chdr", ".h"}, | ||
89 | {"text/x-csrc", ".c"}, | ||
90 | {"text/x-java", ".java"}, | ||
91 | {"text/x-moc", ".moc"}, | ||
92 | {"text/x-pascal", ".pas"}, | ||
93 | {"text/x-perl", ".pl"}, | ||
94 | {"text/x-python", ".py"}, | ||
95 | {"text/x-tex", ".tex"}, | ||
96 | |||
97 | {"video/avi", ".avi"}, | ||
98 | {"video/mpeg", ".mpeg"}, | ||
99 | {"video/quicktime", ".qt"}, | ||
100 | {"video/real", ".rm"}, | ||
101 | {"video/x-msvideo", ".avi"}, | ||
102 | {NULL, NULL}, | ||
103 | }; | ||
104 | char *ret; | ||
105 | unsigned int i; | ||
106 | char *mime; | ||
107 | char *base; | ||
108 | const char *ext; | ||
109 | |||
110 | ret = GNUNET_CONTAINER_meta_data_get_by_type (md, | ||
111 | EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); | ||
112 | if (ret != NULL) | ||
113 | return ret; | ||
114 | ext = NULL; | ||
115 | mime = GNUNET_CONTAINER_meta_data_get_by_type (md, | ||
116 | EXTRACTOR_METATYPE_MIMETYPE); | ||
117 | if (mime != NULL) | ||
118 | { | ||
119 | i = 0; | ||
120 | while ( (mimeMap[i][0] != NULL) && | ||
121 | (0 != strcmp (mime, mimeMap[i][0]))) | ||
122 | i++; | ||
123 | if (mimeMap[i][1] == NULL) | ||
124 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | | ||
125 | GNUNET_ERROR_TYPE_BULK, | ||
126 | _("Did not find mime type `%s' in extension list.\n"), | ||
127 | mime); | ||
128 | else | ||
129 | ext = mimeMap[i][1]; | ||
130 | GNUNET_free (mime); | ||
131 | } | ||
132 | base = GNUNET_CONTAINER_meta_data_get_first_by_types (md, | ||
133 | EXTRACTOR_METATYPE_TITLE, | ||
134 | EXTRACTOR_METATYPE_BOOK_TITLE, | ||
135 | EXTRACTOR_METATYPE_ORIGINAL_TITLE, | ||
136 | EXTRACTOR_METATYPE_PACKAGE_NAME, | ||
137 | EXTRACTOR_METATYPE_URL, | ||
138 | EXTRACTOR_METATYPE_URI, | ||
139 | EXTRACTOR_METATYPE_DESCRIPTION, | ||
140 | EXTRACTOR_METATYPE_ISRC, | ||
141 | EXTRACTOR_METATYPE_JOURNAL_NAME, | ||
142 | EXTRACTOR_METATYPE_AUTHOR_NAME, | ||
143 | EXTRACTOR_METATYPE_SUBJECT, | ||
144 | EXTRACTOR_METATYPE_ALBUM, | ||
145 | EXTRACTOR_METATYPE_ARTIST, | ||
146 | EXTRACTOR_METATYPE_KEYWORDS, | ||
147 | EXTRACTOR_METATYPE_COMMENT, | ||
148 | EXTRACTOR_METATYPE_UNKNOWN, | ||
149 | -1); | ||
150 | if ( (base == NULL) && | ||
151 | (ext == NULL) ) | ||
152 | return NULL; | ||
153 | if (base == NULL) | ||
154 | return GNUNET_strdup (ext); | ||
155 | if (ext == NULL) | ||
156 | return base; | ||
157 | GNUNET_asprintf (&ret, | ||
158 | "%s%s", | ||
159 | base, | ||
160 | ext); | ||
161 | GNUNET_free (base); | ||
162 | return ret; | ||
163 | } | ||
164 | |||
165 | /* end of fs_misc.c */ | ||
diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c index bc75c2a40..6c933a37d 100644 --- a/src/fs/fs_publish.c +++ b/src/fs/fs_publish.c | |||
@@ -457,6 +457,7 @@ encode_cont (void *cls, | |||
457 | struct GNUNET_FS_FileInformation *p; | 457 | struct GNUNET_FS_FileInformation *p; |
458 | struct GNUNET_FS_ProgressInfo pi; | 458 | struct GNUNET_FS_ProgressInfo pi; |
459 | char *emsg; | 459 | char *emsg; |
460 | uint64_t flen; | ||
460 | 461 | ||
461 | p = sc->fi_pos; | 462 | p = sc->fi_pos; |
462 | GNUNET_FS_tree_encoder_finish (p->te, | 463 | GNUNET_FS_tree_encoder_finish (p->te, |
@@ -484,13 +485,13 @@ encode_cont (void *cls, | |||
484 | "Finished with tree encoder\n"); | 485 | "Finished with tree encoder\n"); |
485 | #endif | 486 | #endif |
486 | /* final progress event */ | 487 | /* final progress event */ |
488 | flen = GNUNET_FS_uri_chk_get_file_size (p->chk_uri); | ||
487 | pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS; | 489 | pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS; |
488 | pi.value.publish.specifics.progress.data = NULL; | 490 | pi.value.publish.specifics.progress.data = NULL; |
489 | pi.value.publish.specifics.progress.offset = GNUNET_FS_uri_chk_get_file_size (p->chk_uri); | 491 | pi.value.publish.specifics.progress.offset = flen; |
490 | pi.value.publish.specifics.progress.data_len = 0; | 492 | pi.value.publish.specifics.progress.data_len = 0; |
491 | pi.value.publish.specifics.progress.depth = 0; | 493 | pi.value.publish.specifics.progress.depth = GNUNET_FS_compute_depth (flen); |
492 | p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, | 494 | p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, flen); |
493 | GNUNET_FS_uri_chk_get_file_size (p->chk_uri)); | ||
494 | 495 | ||
495 | /* continue with main */ | 496 | /* continue with main */ |
496 | sc->upload_task | 497 | sc->upload_task |
@@ -507,18 +508,18 @@ encode_cont (void *cls, | |||
507 | * or (on error) "GNUNET_FS_tree_encode_finish". | 508 | * or (on error) "GNUNET_FS_tree_encode_finish". |
508 | * | 509 | * |
509 | * @param cls closure | 510 | * @param cls closure |
510 | * @param query the query for the block (key for lookup in the datastore) | 511 | * @param chk content hash key for the block |
511 | * @param offset offset of the block in the file | 512 | * @param offset offset of the block in the file |
512 | * @param depth depth of the block in the file | 513 | * @param depth depth of the block in the file, 0 for DBLOCK |
513 | * @param type type of the block (IBLOCK or DBLOCK) | 514 | * @param type type of the block (IBLOCK or DBLOCK) |
514 | * @param block the (encrypted) block | 515 | * @param block the (encrypted) block |
515 | * @param block_size size of block (in bytes) | 516 | * @param block_size size of block (in bytes) |
516 | */ | 517 | */ |
517 | static void | 518 | static void |
518 | block_proc (void *cls, | 519 | block_proc (void *cls, |
519 | const GNUNET_HashCode *query, | 520 | const struct ContentHashKey *chk, |
520 | uint64_t offset, | 521 | uint64_t offset, |
521 | unsigned int depth, | 522 | unsigned int depth, |
522 | enum GNUNET_BLOCK_Type type, | 523 | enum GNUNET_BLOCK_Type type, |
523 | const void *block, | 524 | const void *block, |
524 | uint16_t block_size) | 525 | uint16_t block_size) |
@@ -556,7 +557,7 @@ block_proc (void *cls, | |||
556 | #if DEBUG_PUBLISH | 557 | #if DEBUG_PUBLISH |
557 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 558 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
558 | "Indexing block `%s' for offset %llu with index size %u\n", | 559 | "Indexing block `%s' for offset %llu with index size %u\n", |
559 | GNUNET_h2s (query), | 560 | GNUNET_h2s (&chk->query), |
560 | (unsigned long long) offset, | 561 | (unsigned long long) offset, |
561 | sizeof (struct OnDemandBlock)); | 562 | sizeof (struct OnDemandBlock)); |
562 | #endif | 563 | #endif |
@@ -564,7 +565,7 @@ block_proc (void *cls, | |||
564 | odb.file_id = p->data.file.file_id; | 565 | odb.file_id = p->data.file.file_id; |
565 | GNUNET_DATASTORE_put (sc->dsh, | 566 | GNUNET_DATASTORE_put (sc->dsh, |
566 | (p->is_directory) ? 0 : sc->rid, | 567 | (p->is_directory) ? 0 : sc->rid, |
567 | query, | 568 | &chk->query, |
568 | sizeof (struct OnDemandBlock), | 569 | sizeof (struct OnDemandBlock), |
569 | &odb, | 570 | &odb, |
570 | GNUNET_BLOCK_TYPE_FS_ONDEMAND, | 571 | GNUNET_BLOCK_TYPE_FS_ONDEMAND, |
@@ -580,13 +581,13 @@ block_proc (void *cls, | |||
580 | #if DEBUG_PUBLISH | 581 | #if DEBUG_PUBLISH |
581 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 582 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
582 | "Publishing block `%s' for offset %llu with size %u\n", | 583 | "Publishing block `%s' for offset %llu with size %u\n", |
583 | GNUNET_h2s (query), | 584 | GNUNET_h2s (&chk->query), |
584 | (unsigned long long) offset, | 585 | (unsigned long long) offset, |
585 | (unsigned int) block_size); | 586 | (unsigned int) block_size); |
586 | #endif | 587 | #endif |
587 | GNUNET_DATASTORE_put (sc->dsh, | 588 | GNUNET_DATASTORE_put (sc->dsh, |
588 | (p->is_directory) ? 0 : sc->rid, | 589 | (p->is_directory) ? 0 : sc->rid, |
589 | query, | 590 | &chk->query, |
590 | block_size, | 591 | block_size, |
591 | block, | 592 | block, |
592 | type, | 593 | type, |
@@ -608,7 +609,7 @@ block_proc (void *cls, | |||
608 | * @param offset where are we in the file | 609 | * @param offset where are we in the file |
609 | * @param pt_block plaintext of the currently processed block | 610 | * @param pt_block plaintext of the currently processed block |
610 | * @param pt_size size of pt_block | 611 | * @param pt_size size of pt_block |
611 | * @param depth depth of the block in the tree | 612 | * @param depth depth of the block in the tree, 0 for DBLOCK |
612 | */ | 613 | */ |
613 | static void | 614 | static void |
614 | progress_proc (void *cls, | 615 | progress_proc (void *cls, |
diff --git a/src/fs/fs_tree.c b/src/fs/fs_tree.c index 05c9c55cd..e22ce8221 100644 --- a/src/fs/fs_tree.c +++ b/src/fs/fs_tree.c | |||
@@ -1,6 +1,6 @@ | |||
1 | /* | 1 | /* |
2 | This file is part of GNUnet. | 2 | This file is part of GNUnet. |
3 | (C) 2009 Christian Grothoff (and other contributing authors) | 3 | (C) 2009-2011 Christian Grothoff (and other contributing authors) |
4 | 4 | ||
5 | GNUnet is free software; you can redistribute it and/or modify | 5 | GNUnet is free software; you can redistribute it and/or modify |
6 | it under the terms of the GNU General Public License as published | 6 | it under the terms of the GNU General Public License as published |
@@ -75,7 +75,7 @@ struct GNUNET_FS_TreeEncoder | |||
75 | * Set to the URI (upon successful completion) | 75 | * Set to the URI (upon successful completion) |
76 | */ | 76 | */ |
77 | struct GNUNET_FS_Uri *uri; | 77 | struct GNUNET_FS_Uri *uri; |
78 | 78 | ||
79 | /** | 79 | /** |
80 | * Overall file size. | 80 | * Overall file size. |
81 | */ | 81 | */ |
@@ -87,12 +87,12 @@ struct GNUNET_FS_TreeEncoder | |||
87 | uint64_t publish_offset; | 87 | uint64_t publish_offset; |
88 | 88 | ||
89 | /** | 89 | /** |
90 | * How deep are we? | 90 | * How deep are we? Depth 0 is for the DBLOCKs. |
91 | */ | 91 | */ |
92 | unsigned int current_depth; | 92 | unsigned int current_depth; |
93 | 93 | ||
94 | /** | 94 | /** |
95 | * How deep is the tree? | 95 | * How deep is the tree? Always > 0. |
96 | */ | 96 | */ |
97 | unsigned int chk_tree_depth; | 97 | unsigned int chk_tree_depth; |
98 | 98 | ||
@@ -121,7 +121,7 @@ struct GNUNET_FS_TreeEncoder | |||
121 | * Compute the depth of the CHK tree. | 121 | * Compute the depth of the CHK tree. |
122 | * | 122 | * |
123 | * @param flen file length for which to compute the depth | 123 | * @param flen file length for which to compute the depth |
124 | * @return depth of the tree | 124 | * @return depth of the tree, always > 0. A depth of 1 means only a DBLOCK. |
125 | */ | 125 | */ |
126 | unsigned int | 126 | unsigned int |
127 | GNUNET_FS_compute_depth (uint64_t flen) | 127 | GNUNET_FS_compute_depth (uint64_t flen) |
@@ -146,73 +146,53 @@ GNUNET_FS_compute_depth (uint64_t flen) | |||
146 | 146 | ||
147 | 147 | ||
148 | /** | 148 | /** |
149 | * Initialize a tree encoder. This function will call "proc" and | 149 | * Calculate how many bytes of payload a block tree of the given |
150 | * "progress" on each block in the tree. Once all blocks have been | 150 | * depth MAY correspond to at most (this function ignores the fact that |
151 | * processed, "cont" will be scheduled. The "reader" will be called | 151 | * some blocks will only be present partially due to the total file |
152 | * to obtain the (plaintext) blocks for the file. Note that this | 152 | * size cutting some blocks off at the end). |
153 | * function will not actually call "proc". The client must | ||
154 | * call "GNUNET_FS_tree_encoder_next" to trigger encryption (and | ||
155 | * calling of "proc") for the each block. | ||
156 | * | 153 | * |
157 | * @param h the global FS context | 154 | * @param depth depth of the block. depth==0 is a DBLOCK. |
158 | * @param size overall size of the file to encode | 155 | * @return number of bytes of payload a subtree of this depth may correspond to |
159 | * @param cls closure for reader, proc, progress and cont | ||
160 | * @param reader function to call to read plaintext data | ||
161 | * @param proc function to call on each encrypted block | ||
162 | * @param progress function to call with progress information | ||
163 | * @param cont function to call when done | ||
164 | */ | 156 | */ |
165 | struct GNUNET_FS_TreeEncoder * | 157 | uint64_t |
166 | GNUNET_FS_tree_encoder_create (struct GNUNET_FS_Handle *h, | 158 | GNUNET_FS_tree_compute_tree_size (unsigned int depth) |
167 | uint64_t size, | ||
168 | void *cls, | ||
169 | GNUNET_FS_DataReader reader, | ||
170 | GNUNET_FS_TreeBlockProcessor proc, | ||
171 | GNUNET_FS_TreeProgressCallback progress, | ||
172 | GNUNET_SCHEDULER_Task cont) | ||
173 | { | 159 | { |
174 | struct GNUNET_FS_TreeEncoder *te; | 160 | uint64_t rsize; |
175 | 161 | unsigned int i; | |
176 | te = GNUNET_malloc (sizeof (struct GNUNET_FS_TreeEncoder)); | 162 | |
177 | te->h = h; | 163 | rsize = DBLOCK_SIZE; |
178 | te->size = size; | 164 | for (i = 0; i<depth; i++) |
179 | te->cls = cls; | 165 | rsize *= CHK_PER_INODE; |
180 | te->reader = reader; | 166 | return rsize; |
181 | te->proc = proc; | ||
182 | te->progress = progress; | ||
183 | te->cont = cont; | ||
184 | te->chk_tree_depth = GNUNET_FS_compute_depth (size); | ||
185 | te->current_depth = te->chk_tree_depth; | ||
186 | te->chk_tree = GNUNET_malloc (te->chk_tree_depth * | ||
187 | CHK_PER_INODE * | ||
188 | sizeof (struct ContentHashKey)); | ||
189 | return te; | ||
190 | } | 167 | } |
191 | 168 | ||
192 | 169 | ||
193 | /** | 170 | /** |
194 | * Compute the size of the current IBlock. | 171 | * Compute the size of the current IBLOCK. The encoder is |
172 | * triggering the calculation of the size of an IBLOCK at the | ||
173 | * *end* (hence end_offset) of its construction. The IBLOCK | ||
174 | * maybe a full or a partial IBLOCK, and this function is to | ||
175 | * calculate how long it should be. | ||
195 | * | 176 | * |
196 | * @param height height of the IBlock in the tree (aka overall | 177 | * @param depth depth of the IBlock in the tree, 0 would be a DBLOCK, |
197 | * number of tree levels minus depth); 0 == DBlock | 178 | * must be > 0 (this function is for IBLOCKs only!) |
198 | * @param offset current offset in the overall file | 179 | * @param end_offset current offset in the payload (!) of the overall file, |
180 | * must be > 0 (since this function is called at the | ||
181 | * end of a block). | ||
199 | * @return size of the corresponding IBlock | 182 | * @return size of the corresponding IBlock |
200 | */ | 183 | */ |
201 | uint16_t | 184 | static uint16_t |
202 | GNUNET_FS_tree_compute_iblock_size (unsigned int height, | 185 | GNUNET_FS_tree_compute_iblock_size (unsigned int depth, |
203 | uint64_t offset) | 186 | uint64_t end_offset) |
204 | { | 187 | { |
205 | unsigned int ret; | 188 | unsigned int ret; |
206 | unsigned int i; | ||
207 | uint64_t mod; | 189 | uint64_t mod; |
208 | uint64_t bds; | 190 | uint64_t bds; |
209 | 191 | ||
210 | GNUNET_assert (height > 0); | 192 | GNUNET_assert (depth > 0); |
211 | bds = DBLOCK_SIZE; /* number of bytes each CHK at level "i" | 193 | GNUNET_assert (end_offset > 0); |
212 | corresponds to */ | 194 | bds = GNUNET_FS_tree_compute_tree_size (depth); |
213 | for (i=0;i<height;i++) | 195 | mod = end_offset % bds; |
214 | bds *= CHK_PER_INODE; | ||
215 | mod = offset % bds; | ||
216 | if (0 == mod) | 196 | if (0 == mod) |
217 | { | 197 | { |
218 | /* we were triggered at the end of a full block */ | 198 | /* we were triggered at the end of a full block */ |
@@ -232,42 +212,40 @@ GNUNET_FS_tree_compute_iblock_size (unsigned int height, | |||
232 | 212 | ||
233 | /** | 213 | /** |
234 | * Compute how many bytes of data should be stored in | 214 | * Compute how many bytes of data should be stored in |
235 | * the specified node. | 215 | * the specified block. |
236 | * | 216 | * |
237 | * @param fsize overall file size | 217 | * @param fsize overall file size, must be > 0. |
238 | * @param totaldepth depth of the entire tree | 218 | * @param offset offset in the original data corresponding |
239 | * @param offset offset of the node | 219 | * to the beginning of the tree induced by the block; |
240 | * @param depth depth of the node | 220 | * must be <= fsize |
221 | * @param depth depth of the node in the tree, 0 for DBLOCK | ||
241 | * @return number of bytes stored in this node | 222 | * @return number of bytes stored in this node |
242 | */ | 223 | */ |
243 | size_t | 224 | size_t |
244 | GNUNET_FS_tree_calculate_block_size (uint64_t fsize, | 225 | GNUNET_FS_tree_calculate_block_size (uint64_t fsize, |
245 | unsigned int totaldepth, | ||
246 | uint64_t offset, | 226 | uint64_t offset, |
247 | unsigned int depth) | 227 | unsigned int depth) |
248 | { | 228 | { |
249 | unsigned int i; | ||
250 | size_t ret; | 229 | size_t ret; |
251 | uint64_t rsize; | 230 | uint64_t rsize; |
252 | uint64_t epos; | 231 | uint64_t epos; |
253 | unsigned int chks; | 232 | unsigned int chks; |
254 | 233 | ||
234 | GNUNET_assert (fsize > 0); | ||
255 | GNUNET_assert (offset <= fsize); | 235 | GNUNET_assert (offset <= fsize); |
256 | if (depth == totaldepth) | 236 | if (depth == 0) |
257 | { | 237 | { |
258 | ret = DBLOCK_SIZE; | 238 | ret = DBLOCK_SIZE; |
259 | if (offset + ret > fsize) | 239 | if ( (offset + ret > fsize) || |
240 | (offset + ret < offset) ) | ||
260 | ret = (size_t) (fsize - offset); | 241 | ret = (size_t) (fsize - offset); |
261 | return ret; | 242 | return ret; |
262 | } | 243 | } |
263 | /* FIXME: this code should be *equivalent* to the | 244 | |
264 | GNUNET_FS_tree_compute_iblock_size function above! Remove duplication! */ | 245 | rsize = GNUNET_FS_tree_compute_tree_size (depth - 1); |
265 | rsize = DBLOCK_SIZE; | ||
266 | for (i = totaldepth-1; i > depth; i--) | ||
267 | rsize *= CHK_PER_INODE; | ||
268 | epos = offset + rsize * CHK_PER_INODE; | 246 | epos = offset + rsize * CHK_PER_INODE; |
269 | GNUNET_assert (epos > offset); | 247 | if ( (epos < offset) || |
270 | if (epos > fsize) | 248 | (epos > fsize) ) |
271 | epos = fsize; | 249 | epos = fsize; |
272 | /* round up when computing #CHKs in our IBlock */ | 250 | /* round up when computing #CHKs in our IBlock */ |
273 | chks = (epos - offset + rsize - 1) / rsize; | 251 | chks = (epos - offset + rsize - 1) / rsize; |
@@ -277,29 +255,71 @@ GNUNET_FS_tree_calculate_block_size (uint64_t fsize, | |||
277 | 255 | ||
278 | 256 | ||
279 | /** | 257 | /** |
258 | * Initialize a tree encoder. This function will call "proc" and | ||
259 | * "progress" on each block in the tree. Once all blocks have been | ||
260 | * processed, "cont" will be scheduled. The "reader" will be called | ||
261 | * to obtain the (plaintext) blocks for the file. Note that this | ||
262 | * function will not actually call "proc". The client must | ||
263 | * call "GNUNET_FS_tree_encoder_next" to trigger encryption (and | ||
264 | * calling of "proc") for the each block. | ||
265 | * | ||
266 | * @param h the global FS context | ||
267 | * @param size overall size of the file to encode | ||
268 | * @param cls closure for reader, proc, progress and cont | ||
269 | * @param reader function to call to read plaintext data | ||
270 | * @param proc function to call on each encrypted block | ||
271 | * @param progress function to call with progress information | ||
272 | * @param cont function to call when done | ||
273 | */ | ||
274 | struct GNUNET_FS_TreeEncoder * | ||
275 | GNUNET_FS_tree_encoder_create (struct GNUNET_FS_Handle *h, | ||
276 | uint64_t size, | ||
277 | void *cls, | ||
278 | GNUNET_FS_DataReader reader, | ||
279 | GNUNET_FS_TreeBlockProcessor proc, | ||
280 | GNUNET_FS_TreeProgressCallback progress, | ||
281 | GNUNET_SCHEDULER_Task cont) | ||
282 | { | ||
283 | struct GNUNET_FS_TreeEncoder *te; | ||
284 | |||
285 | te = GNUNET_malloc (sizeof (struct GNUNET_FS_TreeEncoder)); | ||
286 | te->h = h; | ||
287 | te->size = size; | ||
288 | te->cls = cls; | ||
289 | te->reader = reader; | ||
290 | te->proc = proc; | ||
291 | te->progress = progress; | ||
292 | te->cont = cont; | ||
293 | te->chk_tree_depth = GNUNET_FS_compute_depth (size); | ||
294 | te->chk_tree = GNUNET_malloc (te->chk_tree_depth * | ||
295 | CHK_PER_INODE * | ||
296 | sizeof (struct ContentHashKey)); | ||
297 | return te; | ||
298 | } | ||
299 | |||
300 | |||
301 | /** | ||
280 | * Compute the offset of the CHK for the | 302 | * Compute the offset of the CHK for the |
281 | * current block in the IBlock above. | 303 | * current block in the IBlock above. |
282 | * | 304 | * |
283 | * @param height height of the IBlock in the tree (aka overall | 305 | * @param depth depth of the IBlock in the tree (aka overall |
284 | * number of tree levels minus depth); 0 == DBlock | 306 | * number of tree levels minus depth); 0 == DBlock |
285 | * @param offset current offset in the overall file | 307 | * @param end_offset current offset in the overall file, |
308 | * at the *beginning* of the block for DBLOCKs (depth==0), | ||
309 | * otherwise at the *end* of the block (exclusive) | ||
286 | * @return (array of CHKs') offset in the above IBlock | 310 | * @return (array of CHKs') offset in the above IBlock |
287 | */ | 311 | */ |
288 | static unsigned int | 312 | static unsigned int |
289 | compute_chk_offset (unsigned int height, | 313 | compute_chk_offset (unsigned int depth, |
290 | uint64_t offset) | 314 | uint64_t end_offset) |
291 | { | 315 | { |
292 | uint64_t bds; | 316 | uint64_t bds; |
293 | unsigned int ret; | 317 | unsigned int ret; |
294 | unsigned int i; | ||
295 | 318 | ||
296 | bds = DBLOCK_SIZE; /* number of bytes each CHK at level "i" | 319 | bds = GNUNET_FS_tree_compute_tree_size (depth); |
297 | corresponds to */ | 320 | if (depth > 0) |
298 | for (i=0;i<height;i++) | 321 | end_offset--; /* round down since for depth > 0 offset is at the END of the block */ |
299 | bds *= CHK_PER_INODE; | 322 | ret = end_offset / bds; |
300 | if (height > 0) | ||
301 | offset--; /* round down since for height > 0 offset is at the END of the block */ | ||
302 | ret = offset / bds; | ||
303 | return ret % CHK_PER_INODE; | 323 | return ret % CHK_PER_INODE; |
304 | } | 324 | } |
305 | 325 | ||
@@ -325,8 +345,26 @@ GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
325 | 345 | ||
326 | GNUNET_assert (GNUNET_NO == te->in_next); | 346 | GNUNET_assert (GNUNET_NO == te->in_next); |
327 | te->in_next = GNUNET_YES; | 347 | te->in_next = GNUNET_YES; |
328 | if (te->current_depth == te->chk_tree_depth) | 348 | if (te->chk_tree_depth == te->current_depth) |
349 | { | ||
350 | off = CHK_PER_INODE * (te->chk_tree_depth - 1); | ||
351 | #if DEBUG_TREE | ||
352 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
353 | "TE done, reading CHK `%s' from %u\n", | ||
354 | GNUNET_h2s (&te->chk_tree[off].query), | ||
355 | off); | ||
356 | #endif | ||
357 | te->uri = GNUNET_malloc (sizeof(struct GNUNET_FS_Uri)); | ||
358 | te->uri->type = chk; | ||
359 | te->uri->data.chk.chk = te->chk_tree[off]; | ||
360 | te->uri->data.chk.file_length = GNUNET_htonll (te->size); | ||
361 | te->in_next = GNUNET_NO; | ||
362 | te->cont (te->cls, NULL); | ||
363 | return; | ||
364 | } | ||
365 | if (0 == te->current_depth) | ||
329 | { | 366 | { |
367 | /* read DBLOCK */ | ||
330 | pt_size = GNUNET_MIN(DBLOCK_SIZE, | 368 | pt_size = GNUNET_MIN(DBLOCK_SIZE, |
331 | te->size - te->publish_offset); | 369 | te->size - te->publish_offset); |
332 | if (pt_size != | 370 | if (pt_size != |
@@ -346,24 +384,12 @@ GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
346 | } | 384 | } |
347 | else | 385 | else |
348 | { | 386 | { |
349 | pt_size = GNUNET_FS_tree_compute_iblock_size (te->chk_tree_depth - te->current_depth, | 387 | pt_size = GNUNET_FS_tree_compute_iblock_size (te->current_depth, |
350 | te->publish_offset); | 388 | te->publish_offset); |
351 | pt_block = &te->chk_tree[te->current_depth * | 389 | pt_block = &te->chk_tree[(te->current_depth - 1) * |
352 | CHK_PER_INODE]; | 390 | CHK_PER_INODE]; |
353 | } | 391 | } |
354 | if (0 == te->current_depth) | 392 | off = compute_chk_offset (te->current_depth, |
355 | { | ||
356 | te->uri = GNUNET_malloc (sizeof(struct GNUNET_FS_Uri)); | ||
357 | te->uri->type = chk; | ||
358 | te->uri->data.chk.chk = te->chk_tree[0]; | ||
359 | te->uri->data.chk.file_length = GNUNET_htonll (te->size); | ||
360 | GNUNET_SCHEDULER_add_continuation (te->cont, | ||
361 | te->cls, | ||
362 | GNUNET_SCHEDULER_REASON_PREREQ_DONE); | ||
363 | te->in_next = GNUNET_NO; | ||
364 | return; | ||
365 | } | ||
366 | off = compute_chk_offset (te->chk_tree_depth - te->current_depth, | ||
367 | te->publish_offset); | 393 | te->publish_offset); |
368 | #if DEBUG_TREE | 394 | #if DEBUG_TREE |
369 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 395 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
@@ -373,7 +399,7 @@ GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
373 | (unsigned int) pt_size, | 399 | (unsigned int) pt_size, |
374 | (unsigned int) off); | 400 | (unsigned int) off); |
375 | #endif | 401 | #endif |
376 | mychk = &te->chk_tree[(te->current_depth-1)*CHK_PER_INODE+off]; | 402 | mychk = &te->chk_tree[te->current_depth*CHK_PER_INODE+off]; |
377 | GNUNET_CRYPTO_hash (pt_block, pt_size, &mychk->key); | 403 | GNUNET_CRYPTO_hash (pt_block, pt_size, &mychk->key); |
378 | GNUNET_CRYPTO_hash_to_aes_key (&mychk->key, &sk, &iv); | 404 | GNUNET_CRYPTO_hash_to_aes_key (&mychk->key, &sk, &iv); |
379 | GNUNET_CRYPTO_aes_encrypt (pt_block, | 405 | GNUNET_CRYPTO_aes_encrypt (pt_block, |
@@ -384,15 +410,16 @@ GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
384 | GNUNET_CRYPTO_hash (enc, pt_size, &mychk->query); | 410 | GNUNET_CRYPTO_hash (enc, pt_size, &mychk->query); |
385 | #if DEBUG_TREE | 411 | #if DEBUG_TREE |
386 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 412 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
387 | "TE calculates query to be `%s'\n", | 413 | "TE calculates query to be `%s', stored at %u\n", |
388 | GNUNET_h2s (&mychk->query)); | 414 | GNUNET_h2s (&mychk->query), |
415 | te->current_depth * CHK_PER_INODE + off); | ||
389 | #endif | 416 | #endif |
390 | if (NULL != te->proc) | 417 | if (NULL != te->proc) |
391 | te->proc (te->cls, | 418 | te->proc (te->cls, |
392 | &mychk->query, | 419 | mychk, |
393 | te->publish_offset, | 420 | te->publish_offset, |
394 | te->current_depth, | 421 | te->current_depth, |
395 | (te->current_depth == te->chk_tree_depth) | 422 | (0 == te->current_depth) |
396 | ? GNUNET_BLOCK_TYPE_FS_DBLOCK | 423 | ? GNUNET_BLOCK_TYPE_FS_DBLOCK |
397 | : GNUNET_BLOCK_TYPE_FS_IBLOCK, | 424 | : GNUNET_BLOCK_TYPE_FS_IBLOCK, |
398 | enc, | 425 | enc, |
@@ -403,20 +430,20 @@ GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
403 | pt_block, | 430 | pt_block, |
404 | pt_size, | 431 | pt_size, |
405 | te->current_depth); | 432 | te->current_depth); |
406 | if (te->current_depth == te->chk_tree_depth) | 433 | if (0 == te->current_depth) |
407 | { | 434 | { |
408 | te->publish_offset += pt_size; | 435 | te->publish_offset += pt_size; |
409 | if ( (te->publish_offset == te->size) || | 436 | if ( (te->publish_offset == te->size) || |
410 | (0 == te->publish_offset % (CHK_PER_INODE * DBLOCK_SIZE) ) ) | 437 | (0 == te->publish_offset % (CHK_PER_INODE * DBLOCK_SIZE) ) ) |
411 | te->current_depth--; | 438 | te->current_depth++; |
412 | } | 439 | } |
413 | else | 440 | else |
414 | { | 441 | { |
415 | if ( (off == CHK_PER_INODE) || | 442 | if ( (off == CHK_PER_INODE) || |
416 | (te->publish_offset == te->size) ) | 443 | (te->publish_offset == te->size) ) |
417 | te->current_depth--; | 444 | te->current_depth++; |
418 | else | 445 | else |
419 | te->current_depth = te->chk_tree_depth; | 446 | te->current_depth = 0; |
420 | } | 447 | } |
421 | te->in_next = GNUNET_NO; | 448 | te->in_next = GNUNET_NO; |
422 | } | 449 | } |
@@ -433,10 +460,11 @@ GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te) | |||
433 | * prior to completion and prior to an internal error, | 460 | * prior to completion and prior to an internal error, |
434 | * both "*uri" and "*emsg" will be set to NULL). | 461 | * both "*uri" and "*emsg" will be set to NULL). |
435 | */ | 462 | */ |
436 | void GNUNET_FS_tree_encoder_finish (struct GNUNET_FS_TreeEncoder * te, | 463 | void GNUNET_FS_tree_encoder_finish (struct GNUNET_FS_TreeEncoder *te, |
437 | struct GNUNET_FS_Uri **uri, | 464 | struct GNUNET_FS_Uri **uri, |
438 | char **emsg) | 465 | char **emsg) |
439 | { | 466 | { |
467 | GNUNET_assert (GNUNET_NO == te->in_next); | ||
440 | if (uri != NULL) | 468 | if (uri != NULL) |
441 | *uri = te->uri; | 469 | *uri = te->uri; |
442 | else | 470 | else |
diff --git a/src/fs/fs_tree.h b/src/fs/fs_tree.h index ebeb47e63..5cd4d0afe 100644 --- a/src/fs/fs_tree.h +++ b/src/fs/fs_tree.h | |||
@@ -21,7 +21,7 @@ | |||
21 | /** | 21 | /** |
22 | * @file fs/fs_tree.h | 22 | * @file fs/fs_tree.h |
23 | * @brief Merkle-tree-ish-CHK file encoding for GNUnet | 23 | * @brief Merkle-tree-ish-CHK file encoding for GNUnet |
24 | * @see http://gnunet.org/encoding.php3 | 24 | * @see https://gnunet.org/encoding |
25 | * @author Krista Bennett | 25 | * @author Krista Bennett |
26 | * @author Christian Grothoff | 26 | * @author Christian Grothoff |
27 | * | 27 | * |
@@ -35,16 +35,46 @@ | |||
35 | #include "fs.h" | 35 | #include "fs.h" |
36 | 36 | ||
37 | /** | 37 | /** |
38 | * Compute the depth of the CHK tree. | 38 | * Compute the depth of the CHK tree. |
39 | * | 39 | * |
40 | * @param flen file length for which to compute the depth | 40 | * @param flen file length for which to compute the depth |
41 | * @return depth of the tree | 41 | * @return depth of the tree, always > 0. A depth of 1 means only a DBLOCK. |
42 | */ | 42 | */ |
43 | unsigned int | 43 | unsigned int |
44 | GNUNET_FS_compute_depth (uint64_t flen); | 44 | GNUNET_FS_compute_depth (uint64_t flen); |
45 | 45 | ||
46 | 46 | ||
47 | /** | 47 | /** |
48 | * Calculate how many bytes of payload a block tree of the given | ||
49 | * depth MAY correspond to at most (this function ignores the fact that | ||
50 | * some blocks will only be present partially due to the total file | ||
51 | * size cutting some blocks off at the end). | ||
52 | * | ||
53 | * @param depth depth of the block. depth==0 is a DBLOCK. | ||
54 | * @return number of bytes of payload a subtree of this depth may correspond to | ||
55 | */ | ||
56 | uint64_t | ||
57 | GNUNET_FS_tree_compute_tree_size (unsigned int depth); | ||
58 | |||
59 | |||
60 | /** | ||
61 | * Compute how many bytes of data should be stored in | ||
62 | * the specified block. | ||
63 | * | ||
64 | * @param fsize overall file size, must be > 0. | ||
65 | * @param offset offset in the original data corresponding | ||
66 | * to the beginning of the tree induced by the block; | ||
67 | * must be < fsize | ||
68 | * @param depth depth of the node in the tree, 0 for DBLOCK | ||
69 | * @return number of bytes stored in this node | ||
70 | */ | ||
71 | size_t | ||
72 | GNUNET_FS_tree_calculate_block_size (uint64_t fsize, | ||
73 | uint64_t offset, | ||
74 | unsigned int depth); | ||
75 | |||
76 | |||
77 | /** | ||
48 | * Context for an ECRS-based file encoder that computes | 78 | * Context for an ECRS-based file encoder that computes |
49 | * the Merkle-ish-CHK tree. | 79 | * the Merkle-ish-CHK tree. |
50 | */ | 80 | */ |
@@ -58,15 +88,15 @@ struct GNUNET_FS_TreeEncoder; | |||
58 | * or (on error) "GNUNET_FS_tree_encode_finish". | 88 | * or (on error) "GNUNET_FS_tree_encode_finish". |
59 | * | 89 | * |
60 | * @param cls closure | 90 | * @param cls closure |
61 | * @param query the query for the block (key for lookup in the datastore) | 91 | * @param chk content hash key for the block |
62 | * @param offset offset of the block | 92 | * @param offset offset of the block |
63 | * @param depth depth of the block | 93 | * @param depth depth of the block, 0 for DBLOCKs |
64 | * @param type type of the block (IBLOCK or DBLOCK) | 94 | * @param type type of the block (IBLOCK or DBLOCK) |
65 | * @param block the (encrypted) block | 95 | * @param block the (encrypted) block |
66 | * @param block_size size of block (in bytes) | 96 | * @param block_size size of block (in bytes) |
67 | */ | 97 | */ |
68 | typedef void (*GNUNET_FS_TreeBlockProcessor)(void *cls, | 98 | typedef void (*GNUNET_FS_TreeBlockProcessor)(void *cls, |
69 | const GNUNET_HashCode *query, | 99 | const struct ContentHashKey *chk, |
70 | uint64_t offset, | 100 | uint64_t offset, |
71 | unsigned int depth, | 101 | unsigned int depth, |
72 | enum GNUNET_BLOCK_Type type, | 102 | enum GNUNET_BLOCK_Type type, |
@@ -82,7 +112,7 @@ typedef void (*GNUNET_FS_TreeBlockProcessor)(void *cls, | |||
82 | * @param offset where are we in the file | 112 | * @param offset where are we in the file |
83 | * @param pt_block plaintext of the currently processed block | 113 | * @param pt_block plaintext of the currently processed block |
84 | * @param pt_size size of pt_block | 114 | * @param pt_size size of pt_block |
85 | * @param depth depth of the block in the tree | 115 | * @param depth depth of the block in the tree, 0 for DBLOCKS |
86 | */ | 116 | */ |
87 | typedef void (*GNUNET_FS_TreeProgressCallback)(void *cls, | 117 | typedef void (*GNUNET_FS_TreeProgressCallback)(void *cls, |
88 | uint64_t offset, | 118 | uint64_t offset, |
@@ -141,41 +171,11 @@ void GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder * te); | |||
141 | * prior to completion and prior to an internal error, | 171 | * prior to completion and prior to an internal error, |
142 | * both "*uri" and "*emsg" will be set to NULL). | 172 | * both "*uri" and "*emsg" will be set to NULL). |
143 | */ | 173 | */ |
144 | void GNUNET_FS_tree_encoder_finish (struct GNUNET_FS_TreeEncoder * te, | 174 | void GNUNET_FS_tree_encoder_finish (struct GNUNET_FS_TreeEncoder *te, |
145 | struct GNUNET_FS_Uri **uri, | 175 | struct GNUNET_FS_Uri **uri, |
146 | char **emsg); | 176 | char **emsg); |
147 | 177 | ||
148 | 178 | ||
149 | /** | ||
150 | * Compute the size of the current IBlock. | ||
151 | * | ||
152 | * @param height height of the IBlock in the tree (aka overall | ||
153 | * number of tree levels minus depth); 0 == DBlock | ||
154 | * @param offset current offset in the overall file | ||
155 | * @return size of the corresponding IBlock | ||
156 | */ | ||
157 | uint16_t | ||
158 | GNUNET_FS_tree_compute_iblock_size (unsigned int height, | ||
159 | uint64_t offset); | ||
160 | |||
161 | |||
162 | /** | ||
163 | * Compute how many bytes of data should be stored in | ||
164 | * the specified node. | ||
165 | * | ||
166 | * @param fsize overall file size | ||
167 | * @param totaldepth depth of the entire tree | ||
168 | * @param offset offset of the node | ||
169 | * @param depth depth of the node | ||
170 | * @return number of bytes stored in this node | ||
171 | */ | ||
172 | size_t | ||
173 | GNUNET_FS_tree_calculate_block_size (uint64_t fsize, | ||
174 | unsigned int totaldepth, | ||
175 | uint64_t offset, | ||
176 | unsigned int depth); | ||
177 | |||
178 | |||
179 | #if 0 | 179 | #if 0 |
180 | /* the functions below will be needed for persistence | 180 | /* the functions below will be needed for persistence |
181 | but are not yet implemented -- FIXME... */ | 181 | but are not yet implemented -- FIXME... */ |
@@ -187,7 +187,7 @@ GNUNET_FS_tree_calculate_block_size (uint64_t fsize, | |||
187 | * @param data set to the resume data | 187 | * @param data set to the resume data |
188 | * @param size set to the size of the resume data | 188 | * @param size set to the size of the resume data |
189 | */ | 189 | */ |
190 | void GNUNET_FS_tree_encoder_resume_get_data (const struct GNUNET_FS_TreeEncoder * te, | 190 | void GNUNET_FS_tree_encoder_resume_get_data (const struct GNUNET_FS_TreeEncoder *te, |
191 | void **data, | 191 | void **data, |
192 | size_t *size); | 192 | size_t *size); |
193 | 193 | ||
@@ -200,7 +200,7 @@ void GNUNET_FS_tree_encoder_resume_get_data (const struct GNUNET_FS_TreeEncoder | |||
200 | * @param data the resume data | 200 | * @param data the resume data |
201 | * @param size the size of the resume data | 201 | * @param size the size of the resume data |
202 | */ | 202 | */ |
203 | void GNUNET_FS_tree_encoder_resume (struct GNUNET_FS_TreeEncoder * te, | 203 | void GNUNET_FS_tree_encoder_resume (struct GNUNET_FS_TreeEncoder *te, |
204 | const void *data, | 204 | const void *data, |
205 | size_t size); | 205 | size_t size); |
206 | #endif | 206 | #endif |
diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c index aa73bc01d..c4360a82d 100644 --- a/src/fs/fs_unindex.c +++ b/src/fs/fs_unindex.c | |||
@@ -114,7 +114,7 @@ GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi, | |||
114 | * @param offset where are we in the file | 114 | * @param offset where are we in the file |
115 | * @param pt_block plaintext of the currently processed block | 115 | * @param pt_block plaintext of the currently processed block |
116 | * @param pt_size size of pt_block | 116 | * @param pt_size size of pt_block |
117 | * @param depth depth of the block in the tree | 117 | * @param depth depth of the block in the tree, 0 for DBLOCK |
118 | */ | 118 | */ |
119 | static void | 119 | static void |
120 | unindex_progress (void *cls, | 120 | unindex_progress (void *cls, |
@@ -188,16 +188,16 @@ process_cont (void *cls, | |||
188 | * or (on error) "GNUNET_FS_tree_encode_finish". | 188 | * or (on error) "GNUNET_FS_tree_encode_finish". |
189 | * | 189 | * |
190 | * @param cls closure | 190 | * @param cls closure |
191 | * @param query the query for the block (key for lookup in the datastore) | 191 | * @param chk content hash key for the block (key for lookup in the datastore) |
192 | * @param offset offset of the block | 192 | * @param offset offset of the block |
193 | * @param depth depth of the block | 193 | * @param depth depth of the block, 0 for DBLOCK |
194 | * @param type type of the block (IBLOCK or DBLOCK) | 194 | * @param type type of the block (IBLOCK or DBLOCK) |
195 | * @param block the (encrypted) block | 195 | * @param block the (encrypted) block |
196 | * @param block_size size of block (in bytes) | 196 | * @param block_size size of block (in bytes) |
197 | */ | 197 | */ |
198 | static void | 198 | static void |
199 | unindex_process (void *cls, | 199 | unindex_process (void *cls, |
200 | const GNUNET_HashCode *query, | 200 | const struct ContentHashKey *chk, |
201 | uint64_t offset, | 201 | uint64_t offset, |
202 | unsigned int depth, | 202 | unsigned int depth, |
203 | enum GNUNET_BLOCK_Type type, | 203 | enum GNUNET_BLOCK_Type type, |
@@ -226,7 +226,7 @@ unindex_process (void *cls, | |||
226 | "Sending REMOVE request to DATASTORE service\n"); | 226 | "Sending REMOVE request to DATASTORE service\n"); |
227 | #endif | 227 | #endif |
228 | GNUNET_DATASTORE_remove (uc->dsh, | 228 | GNUNET_DATASTORE_remove (uc->dsh, |
229 | query, | 229 | &chk->query, |
230 | size, | 230 | size, |
231 | data, | 231 | data, |
232 | -2, 1, | 232 | -2, 1, |
diff --git a/src/fs/test_fs_download.c b/src/fs/test_fs_download.c index 0d13b3639..969e83bf9 100644 --- a/src/fs/test_fs_download.c +++ b/src/fs/test_fs_download.c | |||
@@ -29,7 +29,7 @@ | |||
29 | #include "gnunet_arm_service.h" | 29 | #include "gnunet_arm_service.h" |
30 | #include "gnunet_fs_service.h" | 30 | #include "gnunet_fs_service.h" |
31 | 31 | ||
32 | #define VERBOSE GNUNET_YES | 32 | #define VERBOSE GNUNET_NO |
33 | 33 | ||
34 | #define START_ARM GNUNET_YES | 34 | #define START_ARM GNUNET_YES |
35 | 35 | ||
diff --git a/src/fs/test_fs_download_data.conf b/src/fs/test_fs_download_data.conf index 0bc6cc2b3..a1b202a4d 100644 --- a/src/fs/test_fs_download_data.conf +++ b/src/fs/test_fs_download_data.conf | |||
@@ -37,7 +37,7 @@ HOSTNAME = localhost | |||
37 | PORT = 42471 | 37 | PORT = 42471 |
38 | HOSTNAME = localhost | 38 | HOSTNAME = localhost |
39 | ACTIVEMIGRATION = NO | 39 | ACTIVEMIGRATION = NO |
40 | #DEBUG = YES | 40 | DEBUG = NO |
41 | #PREFIX = valgrind --tool=memcheck --leak-check=yes | 41 | #PREFIX = valgrind --tool=memcheck --leak-check=yes |
42 | #BINARY = /home/grothoff/bin/gnunet-service-fs | 42 | #BINARY = /home/grothoff/bin/gnunet-service-fs |
43 | 43 | ||
diff --git a/src/fs/test_fs_download_persistence.c b/src/fs/test_fs_download_persistence.c index e00d5255d..0ea7a5524 100644 --- a/src/fs/test_fs_download_persistence.c +++ b/src/fs/test_fs_download_persistence.c | |||
@@ -78,6 +78,8 @@ static void | |||
78 | timeout_kill_task (void *cls, | 78 | timeout_kill_task (void *cls, |
79 | const struct GNUNET_SCHEDULER_TaskContext *tc) | 79 | const struct GNUNET_SCHEDULER_TaskContext *tc) |
80 | { | 80 | { |
81 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | ||
82 | "Timeout downloading file\n"); | ||
81 | if (download != NULL) | 83 | if (download != NULL) |
82 | { | 84 | { |
83 | GNUNET_FS_download_stop (download, GNUNET_YES); | 85 | GNUNET_FS_download_stop (download, GNUNET_YES); |
@@ -134,6 +136,8 @@ static void | |||
134 | restart_fs_task (void *cls, | 136 | restart_fs_task (void *cls, |
135 | const struct GNUNET_SCHEDULER_TaskContext *tc) | 137 | const struct GNUNET_SCHEDULER_TaskContext *tc) |
136 | { | 138 | { |
139 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
140 | "Restarting FS.\n"); | ||
137 | GNUNET_FS_stop (fs); | 141 | GNUNET_FS_stop (fs); |
138 | fs = GNUNET_FS_start (cfg, | 142 | fs = GNUNET_FS_start (cfg, |
139 | "test-fs-download-persistence", | 143 | "test-fs-download-persistence", |
@@ -242,18 +246,26 @@ progress_cb (void *cls, | |||
242 | publish = event->value.publish.pc; | 246 | publish = event->value.publish.pc; |
243 | break; | 247 | break; |
244 | case GNUNET_FS_STATUS_DOWNLOAD_SUSPEND: | 248 | case GNUNET_FS_STATUS_DOWNLOAD_SUSPEND: |
249 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
250 | "Download suspended.\n"); | ||
245 | GNUNET_assert (event->value.download.dc == download); | 251 | GNUNET_assert (event->value.download.dc == download); |
246 | download = NULL; | 252 | download = NULL; |
247 | break; | 253 | break; |
248 | case GNUNET_FS_STATUS_DOWNLOAD_RESUME: | 254 | case GNUNET_FS_STATUS_DOWNLOAD_RESUME: |
249 | GNUNET_assert (NULL == download); | 255 | GNUNET_assert (NULL == download); |
250 | download = event->value.download.dc; | 256 | download = event->value.download.dc; |
257 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
258 | "Download resumed.\n"); | ||
251 | break; | 259 | break; |
252 | case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: | 260 | case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: |
253 | consider_restart (event->status); | 261 | consider_restart (event->status); |
262 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
263 | "Download active.\n"); | ||
254 | break; | 264 | break; |
255 | case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: | 265 | case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: |
256 | consider_restart (event->status); | 266 | consider_restart (event->status); |
267 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
268 | "Download inactive.\n"); | ||
257 | break; | 269 | break; |
258 | case GNUNET_FS_STATUS_PUBLISH_START: | 270 | case GNUNET_FS_STATUS_PUBLISH_START: |
259 | GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); | 271 | GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); |
@@ -270,6 +282,8 @@ progress_cb (void *cls, | |||
270 | fs = NULL; | 282 | fs = NULL; |
271 | break; | 283 | break; |
272 | case GNUNET_FS_STATUS_DOWNLOAD_START: | 284 | case GNUNET_FS_STATUS_DOWNLOAD_START: |
285 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
286 | "Download started.\n"); | ||
273 | consider_restart (event->status); | 287 | consider_restart (event->status); |
274 | GNUNET_assert (download == NULL); | 288 | GNUNET_assert (download == NULL); |
275 | download = event->value.download.dc; | 289 | download = event->value.download.dc; |