diff options
author | Christian Grothoff <christian@grothoff.org> | 2012-03-09 15:16:20 +0000 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2012-03-09 15:16:20 +0000 |
commit | 27b9c06bc5b0432f1522353da36502687dcb3cd6 (patch) | |
tree | 7109cd4c1757825ee515d4a007d0285df71dca73 | |
parent | 0fb1248fdd6a348cb1e0b4f56e4a195226adccd3 (diff) | |
download | gnunet-27b9c06bc5b0432f1522353da36502687dcb3cd6.tar.gz gnunet-27b9c06bc5b0432f1522353da36502687dcb3cd6.zip |
implementing removal of KBlocks during unindex operation (#1926)
-rw-r--r-- | src/fs/fs_api.c | 37 | ||||
-rw-r--r-- | src/fs/fs_api.h | 112 | ||||
-rw-r--r-- | src/fs/fs_unindex.c | 321 |
3 files changed, 441 insertions, 29 deletions
diff --git a/src/fs/fs_api.c b/src/fs/fs_api.c index d718acb65..29539a498 100644 --- a/src/fs/fs_api.c +++ b/src/fs/fs_api.c | |||
@@ -1524,6 +1524,7 @@ void | |||
1524 | GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc) | 1524 | GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc) |
1525 | { | 1525 | { |
1526 | struct GNUNET_BIO_WriteHandle *wh; | 1526 | struct GNUNET_BIO_WriteHandle *wh; |
1527 | char *uris; | ||
1527 | 1528 | ||
1528 | if (NULL == uc->serialization) | 1529 | if (NULL == uc->serialization) |
1529 | uc->serialization = | 1530 | uc->serialization = |
@@ -1538,10 +1539,15 @@ GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc) | |||
1538 | GNUNET_break (0); | 1539 | GNUNET_break (0); |
1539 | goto cleanup; | 1540 | goto cleanup; |
1540 | } | 1541 | } |
1542 | uris = GNUNET_FS_uri_to_string (uc->ksk_uri); | ||
1541 | if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uc->filename)) || | 1543 | if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uc->filename)) || |
1542 | (GNUNET_OK != GNUNET_BIO_write_int64 (wh, uc->file_size)) || | 1544 | (GNUNET_OK != GNUNET_BIO_write_int64 (wh, uc->file_size)) || |
1543 | (GNUNET_OK != write_start_time (wh, uc->start_time)) || | 1545 | (GNUNET_OK != write_start_time (wh, uc->start_time)) || |
1544 | (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) uc->state)) || | 1546 | (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) uc->state)) || |
1547 | (GNUNET_OK != | ||
1548 | GNUNET_BIO_write (wh, &uc->chk, sizeof (struct ContentHashKey))) || | ||
1549 | (GNUNET_OK != GNUNET_BIO_write_string (wh, uris)) || | ||
1550 | (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) uc->ksk_offset)) || | ||
1545 | ((uc->state == UNINDEX_STATE_FS_NOTIFY) && | 1551 | ((uc->state == UNINDEX_STATE_FS_NOTIFY) && |
1546 | (GNUNET_OK != | 1552 | (GNUNET_OK != |
1547 | GNUNET_BIO_write (wh, &uc->file_id, sizeof (GNUNET_HashCode)))) || | 1553 | GNUNET_BIO_write (wh, &uc->file_id, sizeof (GNUNET_HashCode)))) || |
@@ -1964,6 +1970,7 @@ deserialize_unindex_file (void *cls, const char *filename) | |||
1964 | struct GNUNET_FS_UnindexContext *uc; | 1970 | struct GNUNET_FS_UnindexContext *uc; |
1965 | struct GNUNET_FS_ProgressInfo pi; | 1971 | struct GNUNET_FS_ProgressInfo pi; |
1966 | char *emsg; | 1972 | char *emsg; |
1973 | char *uris; | ||
1967 | uint32_t state; | 1974 | uint32_t state; |
1968 | 1975 | ||
1969 | uc = GNUNET_malloc (sizeof (struct GNUNET_FS_UnindexContext)); | 1976 | uc = GNUNET_malloc (sizeof (struct GNUNET_FS_UnindexContext)); |
@@ -1979,11 +1986,31 @@ deserialize_unindex_file (void *cls, const char *filename) | |||
1979 | GNUNET_BIO_read_string (rh, "unindex-fn", &uc->filename, 10 * 1024)) || | 1986 | GNUNET_BIO_read_string (rh, "unindex-fn", &uc->filename, 10 * 1024)) || |
1980 | (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &uc->file_size)) || | 1987 | (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &uc->file_size)) || |
1981 | (GNUNET_OK != read_start_time (rh, &uc->start_time)) || | 1988 | (GNUNET_OK != read_start_time (rh, &uc->start_time)) || |
1982 | (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &state))) | 1989 | (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &state)) || |
1990 | (GNUNET_OK != GNUNET_BIO_read (rh, "uri", &uc->chk, sizeof (struct ContentHashKey))) || | ||
1991 | (GNUNET_BIO_read_string (rh, "unindex-kskuri", &uris, 10 * 1024)) || | ||
1992 | (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &uc->ksk_offset)) ) | ||
1983 | { | 1993 | { |
1984 | GNUNET_break (0); | 1994 | GNUNET_break (0); |
1985 | goto cleanup; | 1995 | goto cleanup; |
1986 | } | 1996 | } |
1997 | if (NULL != uris) | ||
1998 | { | ||
1999 | uc->ksk_uri = GNUNET_FS_uri_parse (uris, &emsg); | ||
2000 | GNUNET_free (uris); | ||
2001 | if (NULL == uc->ksk_uri) | ||
2002 | { | ||
2003 | GNUNET_break (0); | ||
2004 | goto cleanup; | ||
2005 | } | ||
2006 | } | ||
2007 | if ( (uc->ksk_offset > 0) && | ||
2008 | ( (NULL == uc->ksk_uri) || | ||
2009 | (uc->ksk_offset > uc->ksk_uri->data.ksk.keywordCount) ) ) | ||
2010 | { | ||
2011 | GNUNET_break (0); | ||
2012 | goto cleanup; | ||
2013 | } | ||
1987 | uc->state = (enum UnindexState) state; | 2014 | uc->state = (enum UnindexState) state; |
1988 | switch (state) | 2015 | switch (state) |
1989 | { | 2016 | { |
@@ -1999,6 +2026,8 @@ deserialize_unindex_file (void *cls, const char *filename) | |||
1999 | } | 2026 | } |
2000 | break; | 2027 | break; |
2001 | case UNINDEX_STATE_DS_REMOVE: | 2028 | case UNINDEX_STATE_DS_REMOVE: |
2029 | case UNINDEX_STATE_EXTRACT_KEYWORDS: | ||
2030 | case UNINDEX_STATE_DS_REMOVE_KBLOCKS: | ||
2002 | break; | 2031 | break; |
2003 | case UNINDEX_STATE_COMPLETE: | 2032 | case UNINDEX_STATE_COMPLETE: |
2004 | break; | 2033 | break; |
@@ -2035,6 +2064,12 @@ deserialize_unindex_file (void *cls, const char *filename) | |||
2035 | case UNINDEX_STATE_DS_REMOVE: | 2064 | case UNINDEX_STATE_DS_REMOVE: |
2036 | GNUNET_FS_unindex_do_remove_ (uc); | 2065 | GNUNET_FS_unindex_do_remove_ (uc); |
2037 | break; | 2066 | break; |
2067 | case UNINDEX_STATE_EXTRACT_KEYWORDS: | ||
2068 | GNUNET_FS_unindex_do_extract_keywords_ (uc); | ||
2069 | break; | ||
2070 | case UNINDEX_STATE_DS_REMOVE_KBLOCKS: | ||
2071 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); | ||
2072 | break; | ||
2038 | case UNINDEX_STATE_COMPLETE: | 2073 | case UNINDEX_STATE_COMPLETE: |
2039 | case UNINDEX_STATE_ERROR: | 2074 | case UNINDEX_STATE_ERROR: |
2040 | /* no need to resume any operation, we were done */ | 2075 | /* no need to resume any operation, we were done */ |
diff --git a/src/fs/fs_api.h b/src/fs/fs_api.h index 1d18e20bc..924911b63 100644 --- a/src/fs/fs_api.h +++ b/src/fs/fs_api.h | |||
@@ -734,6 +734,24 @@ GNUNET_FS_unindex_process_hash_ (void *cls, const GNUNET_HashCode * file_id); | |||
734 | 734 | ||
735 | 735 | ||
736 | /** | 736 | /** |
737 | * Extract the keywords for KBlock removal | ||
738 | * | ||
739 | * @param uc context for the unindex operation. | ||
740 | */ | ||
741 | void | ||
742 | GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc); | ||
743 | |||
744 | |||
745 | /** | ||
746 | * If necessary, connect to the datastore and remove the KBlocks. | ||
747 | * | ||
748 | * @param uc context for the unindex operation. | ||
749 | */ | ||
750 | void | ||
751 | GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc); | ||
752 | |||
753 | |||
754 | /** | ||
737 | * Fill in all of the generic fields for a publish event and call the | 755 | * Fill in all of the generic fields for a publish event and call the |
738 | * callback. | 756 | * callback. |
739 | * | 757 | * |
@@ -1244,32 +1262,42 @@ struct GNUNET_FS_PublishContext | |||
1244 | */ | 1262 | */ |
1245 | enum UnindexState | 1263 | enum UnindexState |
1246 | { | 1264 | { |
1247 | /** | 1265 | /** |
1248 | * We're currently hashing the file. | 1266 | * We're currently hashing the file. |
1249 | */ | 1267 | */ |
1250 | UNINDEX_STATE_HASHING = 0, | 1268 | UNINDEX_STATE_HASHING = 0, |
1251 | 1269 | ||
1252 | /** | 1270 | /** |
1253 | * We're telling the datastore to delete | 1271 | * We're telling the datastore to delete |
1254 | * the respective entries. | 1272 | * the respective DBlocks and IBlocks. |
1255 | */ | 1273 | */ |
1256 | UNINDEX_STATE_DS_REMOVE = 1, | 1274 | UNINDEX_STATE_DS_REMOVE = 1, |
1275 | |||
1276 | /** | ||
1277 | * Find out which keywords apply. | ||
1278 | */ | ||
1279 | UNINDEX_STATE_EXTRACT_KEYWORDS = 2, | ||
1257 | 1280 | ||
1258 | /** | 1281 | /** |
1259 | * We're notifying the FS service about | 1282 | * We're telling the datastore to remove KBlocks. |
1260 | * the unindexing. | 1283 | */ |
1261 | */ | 1284 | UNINDEX_STATE_DS_REMOVE_KBLOCKS = 3, |
1262 | UNINDEX_STATE_FS_NOTIFY = 2, | ||
1263 | |||
1264 | /** | ||
1265 | * We're done. | ||
1266 | */ | ||
1267 | UNINDEX_STATE_COMPLETE = 3, | ||
1268 | 1285 | ||
1269 | /** | 1286 | /** |
1270 | * We've encountered a fatal error. | 1287 | * We're notifying the FS service about |
1271 | */ | 1288 | * the unindexing. |
1272 | UNINDEX_STATE_ERROR = 4 | 1289 | */ |
1290 | UNINDEX_STATE_FS_NOTIFY = 4, | ||
1291 | |||
1292 | /** | ||
1293 | * We're done. | ||
1294 | */ | ||
1295 | UNINDEX_STATE_COMPLETE = 5, | ||
1296 | |||
1297 | /** | ||
1298 | * We've encountered a fatal error. | ||
1299 | */ | ||
1300 | UNINDEX_STATE_ERROR = 6 | ||
1273 | }; | 1301 | }; |
1274 | 1302 | ||
1275 | 1303 | ||
@@ -1280,6 +1308,12 @@ struct GNUNET_FS_UnindexContext | |||
1280 | { | 1308 | { |
1281 | 1309 | ||
1282 | /** | 1310 | /** |
1311 | * The content hash key of the last block we processed, will in the | ||
1312 | * end be set to the CHK from the URI. Used to remove the KBlocks. | ||
1313 | */ | ||
1314 | struct ContentHashKey chk; | ||
1315 | |||
1316 | /** | ||
1283 | * Global FS context. | 1317 | * Global FS context. |
1284 | */ | 1318 | */ |
1285 | struct GNUNET_FS_Handle *h; | 1319 | struct GNUNET_FS_Handle *h; |
@@ -1290,6 +1324,21 @@ struct GNUNET_FS_UnindexContext | |||
1290 | struct TopLevelActivity *top; | 1324 | struct TopLevelActivity *top; |
1291 | 1325 | ||
1292 | /** | 1326 | /** |
1327 | * Directory scanner to find keywords (KBlock removal). | ||
1328 | */ | ||
1329 | struct GNUNET_FS_DirScanner *dscan; | ||
1330 | |||
1331 | /** | ||
1332 | * Keywords found (telling us which KBlocks to remove). | ||
1333 | */ | ||
1334 | struct GNUNET_FS_Uri *ksk_uri; | ||
1335 | |||
1336 | /** | ||
1337 | * Current offset in KSK removal. | ||
1338 | */ | ||
1339 | uint32_t ksk_offset; | ||
1340 | |||
1341 | /** | ||
1293 | * Name of the file that we are unindexing. | 1342 | * Name of the file that we are unindexing. |
1294 | */ | 1343 | */ |
1295 | char *filename; | 1344 | char *filename; |
@@ -1327,6 +1376,22 @@ struct GNUNET_FS_UnindexContext | |||
1327 | struct GNUNET_DISK_FileHandle *fh; | 1376 | struct GNUNET_DISK_FileHandle *fh; |
1328 | 1377 | ||
1329 | /** | 1378 | /** |
1379 | * Handle to datastore 'get_key' operation issued for | ||
1380 | * obtaining KBlocks. | ||
1381 | */ | ||
1382 | struct GNUNET_DATASTORE_QueueEntry *dqe; | ||
1383 | |||
1384 | /** | ||
1385 | * Current query key of 'get_key' operation. | ||
1386 | */ | ||
1387 | GNUNET_HashCode key; | ||
1388 | |||
1389 | /** | ||
1390 | * First content UID, 0 for none. | ||
1391 | */ | ||
1392 | uint64_t first_uid; | ||
1393 | |||
1394 | /** | ||
1330 | * Error message, NULL on success. | 1395 | * Error message, NULL on success. |
1331 | */ | 1396 | */ |
1332 | char *emsg; | 1397 | char *emsg; |
@@ -1342,6 +1407,11 @@ struct GNUNET_FS_UnindexContext | |||
1342 | uint64_t file_size; | 1407 | uint64_t file_size; |
1343 | 1408 | ||
1344 | /** | 1409 | /** |
1410 | * Random offset given to 'GNUNET_DATASTORE_get_key'. | ||
1411 | */ | ||
1412 | uint64_t roff; | ||
1413 | |||
1414 | /** | ||
1345 | * When did we start? | 1415 | * When did we start? |
1346 | */ | 1416 | */ |
1347 | struct GNUNET_TIME_Absolute start_time; | 1417 | struct GNUNET_TIME_Absolute start_time; |
diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c index ff1996a2f..07b9cccd3 100644 --- a/src/fs/fs_unindex.c +++ b/src/fs/fs_unindex.c | |||
@@ -30,6 +30,7 @@ | |||
30 | #include "gnunet_protocols.h" | 30 | #include "gnunet_protocols.h" |
31 | #include "fs_api.h" | 31 | #include "fs_api.h" |
32 | #include "fs_tree.h" | 32 | #include "fs_tree.h" |
33 | #include "block_fs.h" | ||
33 | 34 | ||
34 | 35 | ||
35 | /** | 36 | /** |
@@ -203,6 +204,7 @@ unindex_process (void *cls, const struct ContentHashKey *chk, uint64_t offset, | |||
203 | "Sending REMOVE request to DATASTORE service\n"); | 204 | "Sending REMOVE request to DATASTORE service\n"); |
204 | GNUNET_DATASTORE_remove (uc->dsh, &chk->query, size, data, -2, 1, | 205 | GNUNET_DATASTORE_remove (uc->dsh, &chk->query, size, data, -2, 1, |
205 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, &process_cont, uc); | 206 | GNUNET_CONSTANTS_SERVICE_TIMEOUT, &process_cont, uc); |
207 | uc->chk = *chk; | ||
206 | } | 208 | } |
207 | 209 | ||
208 | 210 | ||
@@ -258,16 +260,15 @@ process_fs_response (void *cls, const struct GNUNET_MessageHeader *msg) | |||
258 | 260 | ||
259 | 261 | ||
260 | /** | 262 | /** |
261 | * Function called when the tree encoder has | 263 | * Function called when we are done with removing KBlocks. |
262 | * processed all blocks. Clean up. | 264 | * Disconnect from datastore and notify FS service about |
265 | * the unindex event. | ||
263 | * | 266 | * |
264 | * @param cls our unindexing context | 267 | * @param uc our unindexing context |
265 | * @param tc not used | ||
266 | */ | 268 | */ |
267 | static void | 269 | static void |
268 | unindex_finish (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) | 270 | unindex_finish (struct GNUNET_FS_UnindexContext *uc) |
269 | { | 271 | { |
270 | struct GNUNET_FS_UnindexContext *uc = cls; | ||
271 | char *emsg; | 272 | char *emsg; |
272 | struct GNUNET_FS_Uri *uri; | 273 | struct GNUNET_FS_Uri *uri; |
273 | struct UnindexMessage req; | 274 | struct UnindexMessage req; |
@@ -310,6 +311,281 @@ unindex_finish (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) | |||
310 | } | 311 | } |
311 | 312 | ||
312 | 313 | ||
314 | |||
315 | /** | ||
316 | * Function called by the directory scanner as we extract keywords | ||
317 | * that we will need to remove KBlocks. | ||
318 | * | ||
319 | * @param cls the 'struct GNUNET_FS_UnindexContext *' | ||
320 | * @param filename which file we are making progress on | ||
321 | * @param is_directory GNUNET_YES if this is a directory, | ||
322 | * GNUNET_NO if this is a file | ||
323 | * GNUNET_SYSERR if it is neither (or unknown) | ||
324 | * @param reason kind of progress we are making | ||
325 | */ | ||
326 | static void | ||
327 | unindex_directory_scan_cb (void *cls, | ||
328 | const char *filename, | ||
329 | int is_directory, | ||
330 | enum GNUNET_FS_DirScannerProgressUpdateReason reason) | ||
331 | { | ||
332 | struct GNUNET_FS_UnindexContext *uc = cls; | ||
333 | static struct GNUNET_FS_ShareTreeItem * directory_scan_result; | ||
334 | |||
335 | switch (reason) | ||
336 | { | ||
337 | case GNUNET_FS_DIRSCANNER_FINISHED: | ||
338 | directory_scan_result = GNUNET_FS_directory_scan_get_result (uc->dscan); | ||
339 | uc->dscan = NULL; | ||
340 | if (NULL != directory_scan_result->ksk_uri) | ||
341 | { | ||
342 | uc->ksk_uri = GNUNET_FS_uri_dup (directory_scan_result->ksk_uri); | ||
343 | uc->state = UNINDEX_STATE_DS_REMOVE_KBLOCKS; | ||
344 | uc->emsg = GNUNET_strdup (_("Failed to connect to `datastore' service.")); | ||
345 | GNUNET_FS_unindex_sync_ (uc); | ||
346 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); | ||
347 | } | ||
348 | else | ||
349 | { | ||
350 | unindex_finish (uc); | ||
351 | } | ||
352 | GNUNET_FS_share_tree_free (directory_scan_result); | ||
353 | break; | ||
354 | case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR: | ||
355 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | ||
356 | _("Internal error scanning `%s'.\n"), | ||
357 | uc->filename); | ||
358 | break; | ||
359 | default: | ||
360 | break; | ||
361 | } | ||
362 | |||
363 | } | ||
364 | |||
365 | |||
366 | /** | ||
367 | * If necessary, connect to the datastore and remove the KBlocks. | ||
368 | * | ||
369 | * @param uc context for the unindex operation. | ||
370 | */ | ||
371 | void | ||
372 | GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc) | ||
373 | { | ||
374 | char *ex; | ||
375 | |||
376 | if (GNUNET_OK != | ||
377 | GNUNET_CONFIGURATION_get_value_string (uc->h->cfg, "FS", "EXTRACTORS", &ex)) | ||
378 | ex = NULL; | ||
379 | uc->dscan = GNUNET_FS_directory_scan_start (uc->filename, | ||
380 | GNUNET_NO, ex, | ||
381 | &unindex_directory_scan_cb, | ||
382 | uc); | ||
383 | GNUNET_free_non_null (ex); | ||
384 | } | ||
385 | |||
386 | |||
387 | /** | ||
388 | * Continuation called to notify client about result of the remove | ||
389 | * operation for the KBlock. | ||
390 | * | ||
391 | * @param cls the 'struct GNUNET_FS_UnindexContext *' | ||
392 | * @param success GNUNET_SYSERR on failure (including timeout/queue drop) | ||
393 | * GNUNET_NO if content was already there | ||
394 | * GNUNET_YES (or other positive value) on success | ||
395 | * @param min_expiration minimum expiration time required for 0-priority content to be stored | ||
396 | * by the datacache at this time, zero for unknown, forever if we have no | ||
397 | * space for 0-priority content | ||
398 | * @param msg NULL on success, otherwise an error message | ||
399 | */ | ||
400 | static void | ||
401 | continue_after_remove (void *cls, | ||
402 | int32_t success, | ||
403 | struct GNUNET_TIME_Absolute min_expiration, | ||
404 | const char *msg) | ||
405 | { | ||
406 | struct GNUNET_FS_UnindexContext *uc = cls; | ||
407 | |||
408 | if (success != GNUNET_YES) | ||
409 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | ||
410 | _("Failed to remove KBlock: %s\n"), | ||
411 | msg); | ||
412 | uc->ksk_offset++; | ||
413 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); | ||
414 | } | ||
415 | |||
416 | |||
417 | /** | ||
418 | * Function called from datastore with result from us looking for | ||
419 | * a KBlock. There are four cases: | ||
420 | * 1) no result, means we move on to the next keyword | ||
421 | * 2) UID is the same as the first UID, means we move on to next keyword | ||
422 | * 3) KBlock for a different CHK, means we keep looking for more | ||
423 | * 4) KBlock is for our CHK, means we remove the block and then move | ||
424 | * on to the next keyword | ||
425 | * | ||
426 | * @param cls the 'struct GNUNET_FS_UnindexContext *' | ||
427 | * @param key key for the content | ||
428 | * @param size number of bytes in data | ||
429 | * @param data content stored | ||
430 | * @param type type of the content | ||
431 | * @param priority priority of the content | ||
432 | * @param anonymity anonymity-level for the content | ||
433 | * @param expiration expiration time for the content | ||
434 | * @param uid unique identifier for the datum; | ||
435 | * maybe 0 if no unique identifier is available | ||
436 | */ | ||
437 | static void | ||
438 | process_kblock_for_unindex (void *cls, | ||
439 | const GNUNET_HashCode * key, | ||
440 | size_t size, const void *data, | ||
441 | enum GNUNET_BLOCK_Type type, | ||
442 | uint32_t priority, | ||
443 | uint32_t anonymity, | ||
444 | struct GNUNET_TIME_Absolute | ||
445 | expiration, uint64_t uid) | ||
446 | { | ||
447 | struct GNUNET_FS_UnindexContext *uc = cls; | ||
448 | const struct KBlock *kb; | ||
449 | const char *uris; | ||
450 | struct GNUNET_FS_Uri *chk_uri; | ||
451 | |||
452 | uc->dqe = NULL; | ||
453 | if (NULL == data) | ||
454 | { | ||
455 | /* no result */ | ||
456 | uc->ksk_offset++; | ||
457 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); | ||
458 | return; | ||
459 | } | ||
460 | if (0 == uc->first_uid) | ||
461 | { | ||
462 | /* remember UID of first result to detect cycles */ | ||
463 | uc->first_uid = uid; | ||
464 | } | ||
465 | else if (uid == uc->first_uid) | ||
466 | { | ||
467 | /* no more additional results */ | ||
468 | uc->ksk_offset++; | ||
469 | GNUNET_FS_unindex_do_remove_kblocks_ (uc); | ||
470 | return; | ||
471 | } | ||
472 | GNUNET_assert (GNUNET_BLOCK_TYPE_FS_KBLOCK == type); | ||
473 | if (size < sizeof (struct KBlock)) | ||
474 | { | ||
475 | GNUNET_break (0); | ||
476 | goto get_next; | ||
477 | } | ||
478 | kb = data; | ||
479 | uris = (const char*) &kb[1]; | ||
480 | if (NULL == memchr (uris, 0, size - sizeof (struct KBlock))) | ||
481 | { | ||
482 | GNUNET_break (0); | ||
483 | goto get_next; | ||
484 | } | ||
485 | chk_uri = GNUNET_FS_uri_parse (uris, NULL); | ||
486 | if (NULL == chk_uri) | ||
487 | { | ||
488 | GNUNET_break (0); | ||
489 | goto get_next; | ||
490 | } | ||
491 | if (0 != memcmp (&uc->chk, | ||
492 | &chk_uri->data.chk.chk, | ||
493 | sizeof (struct ContentHashKey))) | ||
494 | { | ||
495 | /* different CHK, ignore */ | ||
496 | GNUNET_FS_uri_destroy (chk_uri); | ||
497 | goto get_next; | ||
498 | } | ||
499 | /* matches! */ | ||
500 | uc->dqe = GNUNET_DATASTORE_remove (uc->dsh, | ||
501 | key, size, data, | ||
502 | 0 /* priority */, 1 /* queue size */, | ||
503 | GNUNET_TIME_UNIT_FOREVER_REL, | ||
504 | &continue_after_remove, | ||
505 | uc); | ||
506 | return; | ||
507 | get_next: | ||
508 | uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh, | ||
509 | uc->roff++, | ||
510 | &uc->key, | ||
511 | GNUNET_BLOCK_TYPE_FS_KBLOCK, | ||
512 | 0 /* priority */, 1 /* queue size */, | ||
513 | GNUNET_TIME_UNIT_FOREVER_REL, | ||
514 | &process_kblock_for_unindex, | ||
515 | uc); | ||
516 | } | ||
517 | |||
518 | |||
519 | /** | ||
520 | * If necessary, connect to the datastore and remove the KBlocks. | ||
521 | * | ||
522 | * @param uc context for the unindex operation. | ||
523 | */ | ||
524 | void | ||
525 | GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc) | ||
526 | { | ||
527 | const char *keyword; | ||
528 | GNUNET_HashCode hc; | ||
529 | struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pub; | ||
530 | struct GNUNET_CRYPTO_RsaPrivateKey *pk; | ||
531 | |||
532 | if (NULL != uc->dsh) | ||
533 | uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg); | ||
534 | if (NULL == uc->dsh) | ||
535 | { | ||
536 | uc->state = UNINDEX_STATE_ERROR; | ||
537 | uc->emsg = GNUNET_strdup (_("Failed to connect to `datastore' service.")); | ||
538 | GNUNET_FS_unindex_sync_ (uc); | ||
539 | signal_unindex_error (uc); | ||
540 | return; | ||
541 | } | ||
542 | if ( (NULL == uc->ksk_uri) || | ||
543 | (uc->ksk_offset >= uc->ksk_uri->data.ksk.keywordCount) ) | ||
544 | { | ||
545 | unindex_finish (uc); | ||
546 | return; | ||
547 | } | ||
548 | /* FIXME: code duplication with fs_search.c here... */ | ||
549 | keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1]; | ||
550 | GNUNET_CRYPTO_hash (keyword, strlen (keyword), &hc); | ||
551 | pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&hc); | ||
552 | GNUNET_assert (pk != NULL); | ||
553 | GNUNET_CRYPTO_rsa_key_get_public (pk, &pub); | ||
554 | GNUNET_CRYPTO_rsa_key_free (pk); | ||
555 | GNUNET_CRYPTO_hash (&pub, | ||
556 | sizeof (struct | ||
557 | GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), | ||
558 | &uc->key); | ||
559 | uc->first_uid = 0; | ||
560 | uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh, | ||
561 | uc->roff++, | ||
562 | &uc->key, | ||
563 | GNUNET_BLOCK_TYPE_FS_KBLOCK, | ||
564 | 0 /* priority */, 1 /* queue size */, | ||
565 | GNUNET_TIME_UNIT_FOREVER_REL, | ||
566 | &process_kblock_for_unindex, | ||
567 | uc); | ||
568 | } | ||
569 | |||
570 | |||
571 | /** | ||
572 | * Function called when the tree encoder has | ||
573 | * processed all blocks. Clean up. | ||
574 | * | ||
575 | * @param cls our unindexing context | ||
576 | * @param tc not used | ||
577 | */ | ||
578 | static void | ||
579 | unindex_extract_keywords (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
580 | { | ||
581 | struct GNUNET_FS_UnindexContext *uc = cls; | ||
582 | |||
583 | uc->state = UNINDEX_STATE_EXTRACT_KEYWORDS; | ||
584 | GNUNET_FS_unindex_sync_ (uc); | ||
585 | GNUNET_FS_unindex_do_extract_keywords_ (uc); | ||
586 | } | ||
587 | |||
588 | |||
313 | /** | 589 | /** |
314 | * Connect to the datastore and remove the blocks. | 590 | * Connect to the datastore and remove the blocks. |
315 | * | 591 | * |
@@ -343,7 +619,7 @@ GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc) | |||
343 | uc->tc = | 619 | uc->tc = |
344 | GNUNET_FS_tree_encoder_create (uc->h, uc->file_size, uc, &unindex_reader, | 620 | GNUNET_FS_tree_encoder_create (uc->h, uc->file_size, uc, &unindex_reader, |
345 | &unindex_process, &unindex_progress, | 621 | &unindex_process, &unindex_progress, |
346 | &unindex_finish); | 622 | &unindex_extract_keywords); |
347 | GNUNET_FS_tree_encoder_next (uc->tc); | 623 | GNUNET_FS_tree_encoder_next (uc->tc); |
348 | } | 624 | } |
349 | 625 | ||
@@ -393,11 +669,27 @@ GNUNET_FS_unindex_signal_suspend_ (void *cls) | |||
393 | struct GNUNET_FS_UnindexContext *uc = cls; | 669 | struct GNUNET_FS_UnindexContext *uc = cls; |
394 | struct GNUNET_FS_ProgressInfo pi; | 670 | struct GNUNET_FS_ProgressInfo pi; |
395 | 671 | ||
672 | /* FIXME: lots of duplication with unindex_stop here! */ | ||
673 | if (uc->dscan != NULL) | ||
674 | { | ||
675 | GNUNET_FS_directory_scan_abort (uc->dscan); | ||
676 | uc->dscan = NULL; | ||
677 | } | ||
678 | if (NULL != uc->dqe) | ||
679 | { | ||
680 | GNUNET_DATASTORE_cancel (uc->dqe); | ||
681 | uc->dqe = NULL; | ||
682 | } | ||
396 | if (uc->fhc != NULL) | 683 | if (uc->fhc != NULL) |
397 | { | 684 | { |
398 | GNUNET_CRYPTO_hash_file_cancel (uc->fhc); | 685 | GNUNET_CRYPTO_hash_file_cancel (uc->fhc); |
399 | uc->fhc = NULL; | 686 | uc->fhc = NULL; |
400 | } | 687 | } |
688 | if (NULL != uc->ksk_uri) | ||
689 | { | ||
690 | GNUNET_FS_uri_destroy (uc->ksk_uri); | ||
691 | uc->ksk_uri = NULL; | ||
692 | } | ||
401 | if (uc->client != NULL) | 693 | if (uc->client != NULL) |
402 | { | 694 | { |
403 | GNUNET_CLIENT_disconnect (uc->client, GNUNET_NO); | 695 | GNUNET_CLIENT_disconnect (uc->client, GNUNET_NO); |
@@ -478,6 +770,16 @@ GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc) | |||
478 | { | 770 | { |
479 | struct GNUNET_FS_ProgressInfo pi; | 771 | struct GNUNET_FS_ProgressInfo pi; |
480 | 772 | ||
773 | if (uc->dscan != NULL) | ||
774 | { | ||
775 | GNUNET_FS_directory_scan_abort (uc->dscan); | ||
776 | uc->dscan = NULL; | ||
777 | } | ||
778 | if (NULL != uc->dqe) | ||
779 | { | ||
780 | GNUNET_DATASTORE_cancel (uc->dqe); | ||
781 | uc->dqe = NULL; | ||
782 | } | ||
481 | if (uc->fhc != NULL) | 783 | if (uc->fhc != NULL) |
482 | { | 784 | { |
483 | GNUNET_CRYPTO_hash_file_cancel (uc->fhc); | 785 | GNUNET_CRYPTO_hash_file_cancel (uc->fhc); |
@@ -493,6 +795,11 @@ GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc) | |||
493 | GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); | 795 | GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); |
494 | uc->dsh = NULL; | 796 | uc->dsh = NULL; |
495 | } | 797 | } |
798 | if (NULL != uc->ksk_uri) | ||
799 | { | ||
800 | GNUNET_FS_uri_destroy (uc->ksk_uri); | ||
801 | uc->ksk_uri = NULL; | ||
802 | } | ||
496 | if (NULL != uc->tc) | 803 | if (NULL != uc->tc) |
497 | { | 804 | { |
498 | GNUNET_FS_tree_encoder_finish (uc->tc, NULL, NULL); | 805 | GNUNET_FS_tree_encoder_finish (uc->tc, NULL, NULL); |