diff options
author | Christian Grothoff <christian@grothoff.org> | 2010-05-03 13:34:45 +0000 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2010-05-03 13:34:45 +0000 |
commit | 2a40f6fc51bb7ce64a73a21b6399f7764a5d2861 (patch) | |
tree | c5796031c9692bf9232db60951208e05ffaaa9bc | |
parent | 5a7d07767a2cba662a70815c1069821ddc04f465 (diff) | |
download | gnunet-2a40f6fc51bb7ce64a73a21b6399f7764a5d2861.tar.gz gnunet-2a40f6fc51bb7ce64a73a21b6399f7764a5d2861.zip |
search syncing
-rw-r--r-- | ChangeLog | 2 | ||||
-rw-r--r-- | TODO | 7 | ||||
-rw-r--r-- | src/fs/fs.c | 212 | ||||
-rw-r--r-- | src/fs/fs.h | 6 |
4 files changed, 172 insertions, 55 deletions
@@ -1,6 +1,6 @@ | |||
1 | Sun May 2 13:49:10 CEST 2010 | 1 | Sun May 2 13:49:10 CEST 2010 |
2 | Fixed problem with platform-dependence of format for IP addresses | 2 | Fixed problem with platform-dependence of format for IP addresses |
3 | in HELLOs for TCP and UDP transport. | 3 | in HELLOs for TCP and UDP transport. |
4 | Reduced address length field in HELLOs to 16 bit (was 32 bit). | 4 | Reduced address length field in HELLOs to 16 bit (was 32 bit). |
5 | These changes break transport compatibility. | 5 | These changes break transport compatibility. |
6 | 6 | ||
@@ -1,6 +1,8 @@ | |||
1 | 0.9.0pre1: | 1 | 0.9.0pre1: |
2 | * FS: [CG] | 2 | * FS: [CG] |
3 | - serialization and deserialization code (search, download) | 3 | - deserialization code (download) |
4 | - serialization code (download) | ||
5 | - linking of downloads to searches | ||
4 | - generate SUSPEND events (publish, unindex, search, download) | 6 | - generate SUSPEND events (publish, unindex, search, download) |
5 | - actually call 'sync' functions (publish, unindex, search, download) | 7 | - actually call 'sync' functions (publish, unindex, search, download) |
6 | - persistence testing (publish, unindex) | 8 | - persistence testing (publish, unindex) |
@@ -71,6 +73,9 @@ | |||
71 | * FS: [CG] | 73 | * FS: [CG] |
72 | - datastore reservation (publishing) | 74 | - datastore reservation (publishing) |
73 | - location URIs (publish, search, download) | 75 | - location URIs (publish, search, download) |
76 | - utilize in-line files in meta data always (including in search results or | ||
77 | when download is triggered manually and for probes); currently the data is | ||
78 | only used when users do a general 'recursive' download | ||
74 | - non-anonymous FS service (needs DHT) | 79 | - non-anonymous FS service (needs DHT) |
75 | + DHT integration for search | 80 | + DHT integration for search |
76 | + CS-DHT-functions (DHT-put of LOC) | 81 | + CS-DHT-functions (DHT-put of LOC) |
diff --git a/src/fs/fs.c b/src/fs/fs.c index 5b8dc3e90..6556fee18 100644 --- a/src/fs/fs.c +++ b/src/fs/fs.c | |||
@@ -211,8 +211,6 @@ GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh) | |||
211 | } | 211 | } |
212 | 212 | ||
213 | 213 | ||
214 | |||
215 | |||
216 | /** | 214 | /** |
217 | * Closure for "data_reader_file". | 215 | * Closure for "data_reader_file". |
218 | */ | 216 | */ |
@@ -1251,12 +1249,56 @@ GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc) | |||
1251 | * publishing structs should already call "sync" internally, | 1249 | * publishing structs should already call "sync" internally, |
1252 | * so this function is likely not useful for clients. | 1250 | * so this function is likely not useful for clients. |
1253 | * | 1251 | * |
1254 | * @param sc the struct to sync | 1252 | * @param key key for the search result |
1253 | * @param sr the struct to sync | ||
1255 | */ | 1254 | */ |
1256 | void | 1255 | void |
1257 | GNUNET_FS_search_result_sync_ (struct SearchResult *sr) | 1256 | GNUNET_FS_search_result_sync_ (const GNUNET_HashCode *key, |
1257 | struct SearchResult *sr) | ||
1258 | { | 1258 | { |
1259 | /* FIXME */ | 1259 | struct GNUNET_BIO_WriteHandle *wh; |
1260 | char *uris; | ||
1261 | |||
1262 | GNUNET_assert ( (GNUNET_YES == GNUNET_FS_uri_test_chk (sr->uri)) || | ||
1263 | (GNUNET_YES == GNUNET_FS_uri_test_loc (sr->uri)) ); | ||
1264 | uris = NULL; | ||
1265 | if (NULL == sr->serialization) | ||
1266 | sr->serialization = make_serialization_file_name (sr->sc->h, | ||
1267 | "search-results"); | ||
1268 | if (NULL == sr->serialization) | ||
1269 | return; | ||
1270 | wh = get_write_handle (sr->sc->h, "search-results", sr->serialization); | ||
1271 | uris = GNUNET_FS_uri_to_string (sr->uri); | ||
1272 | if ( (GNUNET_OK != | ||
1273 | GNUNET_BIO_write_string (wh, uris)) || | ||
1274 | (GNUNET_OK != | ||
1275 | GNUNET_BIO_write_meta_data (wh, sr->meta)) || | ||
1276 | (GNUNET_OK != | ||
1277 | GNUNET_BIO_write (wh, key, sizeof (GNUNET_HashCode))) || | ||
1278 | (GNUNET_OK != | ||
1279 | GNUNET_BIO_write_int32 (wh, sr->mandatory_missing)) || | ||
1280 | (GNUNET_OK != | ||
1281 | GNUNET_BIO_write_int32 (wh, sr->optional_support)) || | ||
1282 | (GNUNET_OK != | ||
1283 | GNUNET_BIO_write_int32 (wh, sr->availability_success)) || | ||
1284 | (GNUNET_OK != | ||
1285 | GNUNET_BIO_write_int32 (wh, sr->availability_trials)) ) | ||
1286 | goto cleanup; | ||
1287 | if (GNUNET_OK != | ||
1288 | GNUNET_BIO_write_close (wh)) | ||
1289 | { | ||
1290 | wh = NULL; | ||
1291 | goto cleanup; | ||
1292 | } | ||
1293 | GNUNET_free_non_null (uris); | ||
1294 | return; | ||
1295 | cleanup: | ||
1296 | GNUNET_free_non_null (uris); | ||
1297 | if (wh != NULL) | ||
1298 | (void) GNUNET_BIO_write_close (wh); | ||
1299 | GNUNET_FS_remove_sync_file_ (sr->sc->h, "search-results", sr->serialization); | ||
1300 | GNUNET_free (sr->serialization); | ||
1301 | sr->serialization = NULL; | ||
1260 | } | 1302 | } |
1261 | 1303 | ||
1262 | 1304 | ||
@@ -1272,6 +1314,9 @@ void | |||
1272 | GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc) | 1314 | GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc) |
1273 | { | 1315 | { |
1274 | struct GNUNET_BIO_WriteHandle *wh; | 1316 | struct GNUNET_BIO_WriteHandle *wh; |
1317 | struct GNUNET_FS_SearchContext *scc; | ||
1318 | char *uris; | ||
1319 | char in_pause; | ||
1275 | 1320 | ||
1276 | if (NULL == sc->serialization) | 1321 | if (NULL == sc->serialization) |
1277 | sc->serialization = make_serialization_file_name (sc->h, | 1322 | sc->serialization = make_serialization_file_name (sc->h, |
@@ -1279,38 +1324,50 @@ GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc) | |||
1279 | if (NULL == sc->serialization) | 1324 | if (NULL == sc->serialization) |
1280 | return; | 1325 | return; |
1281 | wh = get_write_handle (sc->h, "search", sc->serialization); | 1326 | wh = get_write_handle (sc->h, "search", sc->serialization); |
1282 | #if 0 | 1327 | GNUNET_assert ( (GNUNET_YES == GNUNET_FS_uri_test_ksk (sc->uri)) || |
1328 | (GNUNET_YES == GNUNET_FS_uri_test_sks (sc->uri)) ); | ||
1329 | uris = GNUNET_FS_uri_to_string (sc->uri); | ||
1330 | in_pause = (sc->task != GNUNET_SCHEDULER_NO_TASK) ? 'r' : '\0'; | ||
1283 | if ( (GNUNET_OK != | 1331 | if ( (GNUNET_OK != |
1284 | GNUNET_BIO_write_string (wh, pc->nid)) || | 1332 | GNUNET_BIO_write_string (wh, uris)) || |
1285 | (GNUNET_OK != | 1333 | (GNUNET_OK != |
1286 | GNUNET_BIO_write_string (wh, pc->nuid)) || | 1334 | GNUNET_BIO_write_int64 (wh, sc->start_time.value)) || |
1287 | (GNUNET_OK != | 1335 | (GNUNET_OK != |
1288 | GNUNET_BIO_write_int32 (wh, pc->options)) || | 1336 | GNUNET_BIO_write_string (wh, sc->emsg)) || |
1289 | (GNUNET_OK != | 1337 | (GNUNET_OK != |
1290 | GNUNET_BIO_write_int32 (wh, pc->all_done)) || | 1338 | GNUNET_BIO_write_int32 (wh, (uint32_t) sc->options)) || |
1291 | (GNUNET_OK != | 1339 | (GNUNET_OK != |
1292 | GNUNET_BIO_write_string (wh, pc->fi->serialization)) || | 1340 | GNUNET_BIO_write (wh, &in_pause, sizeof (in_pause))) || |
1293 | (GNUNET_OK != | ||
1294 | GNUNET_BIO_write_string (wh, (pc->fi_pos == NULL) ? NULL : pc->fi_pos->serialization)) || | ||
1295 | (GNUNET_OK != | 1341 | (GNUNET_OK != |
1296 | GNUNET_BIO_write_string (wh, (pc->namespace == NULL) ? NULL : pc->namespace->name)) ) | 1342 | GNUNET_BIO_write_int32 (wh, sc->anonymity)) ) |
1297 | { | 1343 | goto cleanup; |
1298 | (void) GNUNET_BIO_write_close (wh); | 1344 | GNUNET_free (uris); |
1299 | GNUNET_FS_remove_sync_file_ (pc->h, "publish", pc->serialization); | 1345 | uris = NULL; |
1300 | GNUNET_free (pc->serialization); | 1346 | scc = sc->child_head; |
1301 | pc->serialization = NULL; | 1347 | while (NULL != scc) |
1302 | return; | 1348 | { |
1303 | } | 1349 | if (scc->serialization == NULL) |
1304 | #endif | 1350 | break; |
1305 | /* FIXME: do search-specific serialization here! */ | 1351 | if (GNUNET_OK != |
1352 | GNUNET_BIO_write_string (wh, scc->serialization)) | ||
1353 | goto cleanup; | ||
1354 | scc = scc->next; | ||
1355 | } | ||
1356 | GNUNET_BIO_write_string (wh, NULL); | ||
1306 | if (GNUNET_OK != | 1357 | if (GNUNET_OK != |
1307 | GNUNET_BIO_write_close (wh)) | 1358 | GNUNET_BIO_write_close (wh)) |
1308 | { | 1359 | { |
1309 | GNUNET_FS_remove_sync_file_ (sc->h, "search", sc->serialization); | 1360 | wh = NULL; |
1310 | GNUNET_free (sc->serialization); | 1361 | goto cleanup; |
1311 | sc->serialization = NULL; | 1362 | } |
1312 | return; | 1363 | return; |
1313 | } | 1364 | cleanup: |
1365 | if (wh != NULL) | ||
1366 | (void) GNUNET_BIO_write_close (wh); | ||
1367 | GNUNET_free_non_null (uris); | ||
1368 | GNUNET_FS_remove_sync_file_ (sc->h, "search", sc->serialization); | ||
1369 | GNUNET_free (sc->serialization); | ||
1370 | sc->serialization = NULL; | ||
1314 | } | 1371 | } |
1315 | 1372 | ||
1316 | 1373 | ||
@@ -1332,8 +1389,6 @@ deserialize_publish (struct GNUNET_FS_Handle *h) | |||
1332 | } | 1389 | } |
1333 | 1390 | ||
1334 | 1391 | ||
1335 | |||
1336 | |||
1337 | /** | 1392 | /** |
1338 | * Function called with a filename of serialized unindexing operation | 1393 | * Function called with a filename of serialized unindexing operation |
1339 | * to deserialize. | 1394 | * to deserialize. |
@@ -1610,6 +1665,41 @@ free_result (void *cls, | |||
1610 | 1665 | ||
1611 | 1666 | ||
1612 | /** | 1667 | /** |
1668 | * Free memory allocated by the search context and its children | ||
1669 | * | ||
1670 | * @param sc search context to free | ||
1671 | */ | ||
1672 | static void | ||
1673 | free_search_context (struct GNUNET_FS_SearchContext *sc) | ||
1674 | { | ||
1675 | struct GNUNET_FS_SearchContext *scc; | ||
1676 | |||
1677 | while (NULL != (scc = sc->child_head)) | ||
1678 | { | ||
1679 | GNUNET_CONTAINER_DLL_remove (sc->child_head, | ||
1680 | sc->child_tail, | ||
1681 | scc); | ||
1682 | free_search_context (scc); | ||
1683 | } | ||
1684 | GNUNET_free_non_null (sc->emsg); | ||
1685 | if (sc->serialization != NULL) | ||
1686 | GNUNET_FS_remove_sync_file_ (sc->h, "search", sc->serialization); | ||
1687 | /* FIXME: remove 'pbuf' directory with search results as well! */ | ||
1688 | GNUNET_free_non_null (sc->serialization); | ||
1689 | if (sc->uri != NULL) | ||
1690 | GNUNET_FS_uri_destroy (sc->uri); | ||
1691 | if (sc->master_result_map != NULL) | ||
1692 | { | ||
1693 | GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, | ||
1694 | &free_result, | ||
1695 | sc); | ||
1696 | GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map); | ||
1697 | } | ||
1698 | GNUNET_free (sc); | ||
1699 | } | ||
1700 | |||
1701 | |||
1702 | /** | ||
1613 | * Deserialize a search. | 1703 | * Deserialize a search. |
1614 | * | 1704 | * |
1615 | * @param h overall context | 1705 | * @param h overall context |
@@ -1624,6 +1714,8 @@ deserialize_search (struct GNUNET_FS_Handle *h, | |||
1624 | const char *serialization) | 1714 | const char *serialization) |
1625 | { | 1715 | { |
1626 | struct GNUNET_FS_SearchContext *sc; | 1716 | struct GNUNET_FS_SearchContext *sc; |
1717 | struct GNUNET_FS_SearchContext *scc; | ||
1718 | struct GNUNET_BIO_ReadHandle *rhc; | ||
1627 | char pbuf[32]; | 1719 | char pbuf[32]; |
1628 | struct GNUNET_FS_ProgressInfo pi; | 1720 | struct GNUNET_FS_ProgressInfo pi; |
1629 | char *emsg; | 1721 | char *emsg; |
@@ -1634,7 +1726,6 @@ deserialize_search (struct GNUNET_FS_Handle *h, | |||
1634 | char in_pause; | 1726 | char in_pause; |
1635 | 1727 | ||
1636 | uris = NULL; | 1728 | uris = NULL; |
1637 | child_ser = NULL; | ||
1638 | emsg = NULL; | 1729 | emsg = NULL; |
1639 | sc = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchContext)); | 1730 | sc = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchContext)); |
1640 | sc->parent = parent; | 1731 | sc->parent = parent; |
@@ -1648,8 +1739,6 @@ deserialize_search (struct GNUNET_FS_Handle *h, | |||
1648 | (GNUNET_OK != | 1739 | (GNUNET_OK != |
1649 | GNUNET_BIO_read_int64 (rh, &sc->start_time.value)) || | 1740 | GNUNET_BIO_read_int64 (rh, &sc->start_time.value)) || |
1650 | (GNUNET_OK != | 1741 | (GNUNET_OK != |
1651 | GNUNET_BIO_read_string (rh, "child-serialization", &child_ser, 32)) || | ||
1652 | (GNUNET_OK != | ||
1653 | GNUNET_BIO_read_string (rh, "search-emsg", &sc->emsg, 10*1024)) || | 1742 | GNUNET_BIO_read_string (rh, "search-emsg", &sc->emsg, 10*1024)) || |
1654 | (GNUNET_OK != | 1743 | (GNUNET_OK != |
1655 | GNUNET_BIO_read_int32 (rh, &options)) || | 1744 | GNUNET_BIO_read_int32 (rh, &options)) || |
@@ -1679,9 +1768,33 @@ deserialize_search (struct GNUNET_FS_Handle *h, | |||
1679 | GNUNET_FS_search_start_searching_ (sc)) | 1768 | GNUNET_FS_search_start_searching_ (sc)) |
1680 | goto cleanup; | 1769 | goto cleanup; |
1681 | } | 1770 | } |
1682 | if (child_ser != NULL) | 1771 | while (1) |
1683 | { | 1772 | { |
1684 | /* FIXME: deserialize child-search! */ | 1773 | if ( (GNUNET_OK != |
1774 | GNUNET_BIO_read_string (rh, "child-serialization", &child_ser, 32))) | ||
1775 | goto cleanup; | ||
1776 | if (child_ser == NULL) | ||
1777 | break; | ||
1778 | rhc = get_read_handle (h, "search-children", child_ser); | ||
1779 | if (rhc != NULL) | ||
1780 | { | ||
1781 | scc = deserialize_search (h, rhc, sc, child_ser); | ||
1782 | if (scc != NULL) | ||
1783 | GNUNET_CONTAINER_DLL_insert (sc->child_head, | ||
1784 | sc->child_tail, | ||
1785 | scc); | ||
1786 | emsg = NULL; | ||
1787 | if (GNUNET_OK != | ||
1788 | GNUNET_BIO_read_close (rhc, &emsg)) | ||
1789 | { | ||
1790 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | ||
1791 | _("Failed to resume sub-search `%s': %s\n"), | ||
1792 | child_ser, | ||
1793 | emsg); | ||
1794 | GNUNET_free (emsg); | ||
1795 | } | ||
1796 | } | ||
1797 | GNUNET_free (child_ser); | ||
1685 | } | 1798 | } |
1686 | if (parent != NULL) | 1799 | if (parent != NULL) |
1687 | GNUNET_CONTAINER_DLL_insert (parent->child_head, | 1800 | GNUNET_CONTAINER_DLL_insert (parent->child_head, |
@@ -1692,28 +1805,25 @@ deserialize_search (struct GNUNET_FS_Handle *h, | |||
1692 | pi.value.search.specifics.resume.is_paused = ('\0' == in_pause) ? GNUNET_NO : GNUNET_YES; | 1805 | pi.value.search.specifics.resume.is_paused = ('\0' == in_pause) ? GNUNET_NO : GNUNET_YES; |
1693 | sc->client_info = GNUNET_FS_search_make_status_ (&pi, | 1806 | sc->client_info = GNUNET_FS_search_make_status_ (&pi, |
1694 | sc); | 1807 | sc); |
1808 | scc = sc->child_head; | ||
1809 | while (NULL != scc) | ||
1810 | { | ||
1811 | pi.status = GNUNET_FS_STATUS_SEARCH_RESUME; | ||
1812 | pi.value.search.specifics.resume.message = scc->emsg; | ||
1813 | pi.value.search.specifics.resume.is_paused = ('\0' == in_pause) ? GNUNET_NO : GNUNET_YES; | ||
1814 | scc->client_info = GNUNET_FS_search_make_status_ (&pi, | ||
1815 | scc); | ||
1816 | |||
1817 | scc = scc->next; | ||
1818 | } | ||
1695 | GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, | 1819 | GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, |
1696 | &signal_result_resume, | 1820 | &signal_result_resume, |
1697 | sc); | 1821 | sc); |
1822 | GNUNET_free (uris); | ||
1698 | return sc; | 1823 | return sc; |
1699 | cleanup: | 1824 | cleanup: |
1700 | GNUNET_free_non_null (child_ser); | ||
1701 | GNUNET_free_non_null (sc->emsg); | ||
1702 | GNUNET_free_non_null (emsg); | 1825 | GNUNET_free_non_null (emsg); |
1703 | if (sc->serialization != NULL) | 1826 | free_search_context (sc); |
1704 | GNUNET_FS_remove_sync_file_ (h, "search", sc->serialization); | ||
1705 | /* FIXME: remove 'pbuf' directory with search results as well! */ | ||
1706 | GNUNET_free_non_null (sc->serialization); | ||
1707 | if (sc->uri != NULL) | ||
1708 | GNUNET_FS_uri_destroy (sc->uri); | ||
1709 | if (sc->master_result_map != NULL) | ||
1710 | { | ||
1711 | GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, | ||
1712 | &free_result, | ||
1713 | sc); | ||
1714 | GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map); | ||
1715 | } | ||
1716 | GNUNET_free (sc); | ||
1717 | GNUNET_free_non_null (uris); | 1827 | GNUNET_free_non_null (uris); |
1718 | return NULL; | 1828 | return NULL; |
1719 | } | 1829 | } |
diff --git a/src/fs/fs.h b/src/fs/fs.h index 33ee8dde0..338b38fc7 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h | |||
@@ -880,10 +880,12 @@ GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc); | |||
880 | * publishing structs should already call "sync" internally, | 880 | * publishing structs should already call "sync" internally, |
881 | * so this function is likely not useful for clients. | 881 | * so this function is likely not useful for clients. |
882 | * | 882 | * |
883 | * @param sc the struct to sync | 883 | * @param key key for the search result |
884 | * @param sr the struct to sync | ||
884 | */ | 885 | */ |
885 | void | 886 | void |
886 | GNUNET_FS_search_result_sync_ (struct SearchResult *sr); | 887 | GNUNET_FS_search_result_sync_ (const GNUNET_HashCode *key, |
888 | struct SearchResult *sr); | ||
887 | 889 | ||
888 | 890 | ||
889 | /** | 891 | /** |