diff options
author | Christian Grothoff <christian@grothoff.org> | 2009-08-27 11:14:19 +0000 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2009-08-27 11:14:19 +0000 |
commit | 57df0fbf8398720fbd4a9cf34934c212ea00ee54 (patch) | |
tree | a00d5740b2a35e21d4cff044c4ce941995b1bdff /src/fs/fs_directory.c | |
parent | 2f7e69f969a822a38f9e311f79358e0fe8bda9d8 (diff) | |
download | gnunet-57df0fbf8398720fbd4a9cf34934c212ea00ee54.tar.gz gnunet-57df0fbf8398720fbd4a9cf34934c212ea00ee54.zip |
syn
Diffstat (limited to 'src/fs/fs_directory.c')
-rw-r--r-- | src/fs/fs_directory.c | 332 |
1 files changed, 215 insertions, 117 deletions
diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c index 3eb3af50d..e5a9f963f 100644 --- a/src/fs/fs_directory.c +++ b/src/fs/fs_directory.c | |||
@@ -25,7 +25,6 @@ | |||
25 | * | 25 | * |
26 | * TODO: | 26 | * TODO: |
27 | * - add support for embedded file data (use padding room!) | 27 | * - add support for embedded file data (use padding room!) |
28 | * - add directory builder API to gnunet_fs_service | ||
29 | * - modify directory builder API to support incremental | 28 | * - modify directory builder API to support incremental |
30 | * generation of directories (to allow directories that | 29 | * generation of directories (to allow directories that |
31 | * would not fit into memory to be created) | 30 | * would not fit into memory to be created) |
@@ -38,6 +37,9 @@ | |||
38 | #include "gnunet_fs_service.h" | 37 | #include "gnunet_fs_service.h" |
39 | #include "fs.h" | 38 | #include "fs.h" |
40 | 39 | ||
40 | #ifndef EXTRACTOR_GNUNET_FULL_DATA | ||
41 | #define EXTRACTOR_GNUNET_FULL_DATA 137 | ||
42 | #endif | ||
41 | 43 | ||
42 | /** | 44 | /** |
43 | * Does the meta-data claim that this is a directory? | 45 | * Does the meta-data claim that this is a directory? |
@@ -215,6 +217,7 @@ GNUNET_FS_directory_list_contents (size_t size, | |||
215 | return; /* malformed ! */ | 217 | return; /* malformed ! */ |
216 | } | 218 | } |
217 | pos += mdSize; | 219 | pos += mdSize; |
220 | // EXTRACTOR_GNUNET_FULL_DATA | ||
218 | /* FIXME: add support for embedded data */ | 221 | /* FIXME: add support for embedded data */ |
219 | filename = GNUNET_CONTAINER_meta_data_get_by_type (md, | 222 | filename = GNUNET_CONTAINER_meta_data_get_by_type (md, |
220 | EXTRACTOR_FILENAME); | 223 | EXTRACTOR_FILENAME); |
@@ -231,14 +234,150 @@ GNUNET_FS_directory_list_contents (size_t size, | |||
231 | } | 234 | } |
232 | } | 235 | } |
233 | 236 | ||
237 | /** | ||
238 | * Entries in the directory (builder). | ||
239 | */ | ||
240 | struct BuilderEntry | ||
241 | { | ||
242 | /** | ||
243 | * This is a linked list. | ||
244 | */ | ||
245 | struct BuilderEntry *next; | ||
246 | |||
247 | /** | ||
248 | * Length of this entry. | ||
249 | */ | ||
250 | size_t len; | ||
251 | }; | ||
234 | 252 | ||
235 | void | 253 | /** |
236 | GNUNET_FS_directory_create () | 254 | * Internal state of a directory builder. |
255 | */ | ||
256 | struct GNUNET_FS_DirectoryBuilder | ||
237 | { | 257 | { |
258 | /** | ||
259 | * Meta-data for the directory itself. | ||
260 | */ | ||
261 | struct GNUNET_CONTAINER_MetaData *meta; | ||
262 | |||
263 | /** | ||
264 | * Head of linked list of entries. | ||
265 | */ | ||
266 | struct BuilderEntry *head; | ||
267 | |||
268 | /** | ||
269 | * Number of entires in the directory. | ||
270 | */ | ||
271 | unsigned int count; | ||
272 | }; | ||
273 | |||
274 | |||
275 | /** | ||
276 | * Create a directory builder. | ||
277 | * | ||
278 | * @param mdir metadata for the directory | ||
279 | */ | ||
280 | struct GNUNET_FS_DirectoryBuilder * | ||
281 | GNUNET_FS_directory_builder_create (const struct GNUNET_CONTAINER_MetaData *mdir) | ||
282 | { | ||
283 | struct GNUNET_FS_DirectoryBuilder *ret; | ||
284 | |||
285 | ret = GNUNET_malloc(sizeof(struct GNUNET_FS_DirectoryBuilder)); | ||
286 | ret->meta = GNUNET_CONTAINER_meta_data_duplicate (mdir); | ||
287 | GNUNET_FS_meta_data_make_directory (ret->meta); | ||
288 | return ret; | ||
238 | } | 289 | } |
239 | 290 | ||
240 | 291 | ||
241 | #if 0 | 292 | /** |
293 | * Add an entry to a directory. | ||
294 | * | ||
295 | * @param bld directory to extend | ||
296 | * @param uri uri of the entry (must not be a KSK) | ||
297 | * @param md metadata of the entry | ||
298 | * @param data raw data of the entry, can be NULL, otherwise | ||
299 | * data must point to exactly the number of bytes specified | ||
300 | * by the uri which must be of type LOC or CHK | ||
301 | */ | ||
302 | void | ||
303 | GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, | ||
304 | const struct GNUNET_FS_Uri *uri, | ||
305 | const struct GNUNET_CONTAINER_MetaData *md, | ||
306 | const void *data) | ||
307 | { | ||
308 | struct BuilderEntry *e; | ||
309 | uint64_t fsize; | ||
310 | uint32_t big; | ||
311 | size_t mds; | ||
312 | size_t mdxs; | ||
313 | char *uris; | ||
314 | char *ser; | ||
315 | size_t slen; | ||
316 | struct GNUNET_CONTAINER_MetaData *meta; | ||
317 | const struct GNUNET_CONTAINER_MetaData *meta_use; | ||
318 | |||
319 | GNUNET_assert (! GNUNET_FS_uri_ksk_test (uri)); | ||
320 | if (NULL != data) | ||
321 | if (GNUNET_FS_uri_chk_test (uri)) | ||
322 | fsize = GNUNET_FS_uri_chk_get_size (uri); | ||
323 | else | ||
324 | fsize = GNUNET_FS_uri_chk_get_size (GNUNET_FS_uri_loc_get_uri (uri)); | ||
325 | else | ||
326 | fsize = 0; /* not given */ | ||
327 | if (fsize > GNUNET_FS_MAX_INLINE_SIZE) | ||
328 | fsize = 0; /* too large */ | ||
329 | if (memchr (data, fsize, '\0')) // FIXME: check memchr args! | ||
330 | fsize = 0; /* must not have 0's in data! */ | ||
331 | uris = GNUNET_FS_uri_to_string (uri); | ||
332 | slen = strlen (uris) + 1; | ||
333 | mds = | ||
334 | GNUNET_CONTAINER_meta_data_get_serialized_size (md, | ||
335 | GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); | ||
336 | meta_use = md; | ||
337 | meta = NULL; | ||
338 | if (fsize > 0) | ||
339 | { | ||
340 | meta = GNUNET_CONTAINER_meta_data_duplicate (md); | ||
341 | GNUNET_CONTAINER_meta_data_insert (meta, | ||
342 | EXTRACTOR_GNUNET_FULL_DATA, | ||
343 | data); | ||
344 | mdxs = | ||
345 | GNUNET_CONTAINER_meta_data_get_serialized_size (meta, | ||
346 | GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); | ||
347 | if ( (slen + sizeof (uint32_t) + mdxs - 1) / GNUNET_FS_DBLOCK_SIZE == | ||
348 | (slen + sizeof (uint32_t) + mds - 1) / GNUNET_FS_DBLOCK_SIZE) | ||
349 | { | ||
350 | /* adding full data would not cause us to cross | ||
351 | additional blocks, so add it! */ | ||
352 | meta_use = meta; | ||
353 | mds = mdxs; | ||
354 | } | ||
355 | } | ||
356 | |||
357 | if (mds > GNUNET_MAX_MALLOC_CHECKED / 2) | ||
358 | mds = GNUNET_MAX_MALLOC_CHECKED / 2; | ||
359 | e = GNUNET_malloc (sizeof(struct BuilderEntry) + | ||
360 | slen + mds + sizeof (uint32_t)); | ||
361 | ser = (char*) &e[1]; | ||
362 | memcpy (ser, uris, slen); | ||
363 | GNUNET_free (uris); | ||
364 | ret = GNUNET_CONTAINER_meta_data_serialize (meta_use, | ||
365 | &ser[slen + sizeof(uint32_t)], | ||
366 | mds, | ||
367 | GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); | ||
368 | if (NULL != meta) | ||
369 | GNUNET_CONTAINER_meta_data_destroy (meta); | ||
370 | if (ret == -1) | ||
371 | mds = 0; | ||
372 | else | ||
373 | mds = ret; | ||
374 | big = htonl (mds); | ||
375 | memcpy (&ser[slen], &big, sizeof (uint32_t)); | ||
376 | e->len = slen + sizeof (uint32_t) + mds; | ||
377 | e->next = bld->head; | ||
378 | bld->head = e; | ||
379 | bld->count++; | ||
380 | } | ||
242 | 381 | ||
243 | 382 | ||
244 | /** | 383 | /** |
@@ -246,11 +385,11 @@ GNUNET_FS_directory_create () | |||
246 | * data, return the end position of that data | 385 | * data, return the end position of that data |
247 | * after alignment to the GNUNET_FS_DBLOCK_SIZE. | 386 | * after alignment to the GNUNET_FS_DBLOCK_SIZE. |
248 | */ | 387 | */ |
249 | static uint64_t | 388 | static size_t |
250 | do_align (uint64_t start_position, | 389 | do_align (size_t start_position, |
251 | uint64_t end_position) | 390 | size_t end_position) |
252 | { | 391 | { |
253 | uint64_t align; | 392 | size_t align; |
254 | 393 | ||
255 | align = (end_position / GNUNET_FS_DBLOCK_SIZE) * GNUNET_FS_DBLOCK_SIZE; | 394 | align = (end_position / GNUNET_FS_DBLOCK_SIZE) * GNUNET_FS_DBLOCK_SIZE; |
256 | if ((start_position < align) && (end_position > align)) | 395 | if ((start_position < align) && (end_position > align)) |
@@ -269,19 +408,19 @@ do_align (uint64_t start_position, | |||
269 | * @param perm the permutation of the blocks (updated) | 408 | * @param perm the permutation of the blocks (updated) |
270 | */ | 409 | */ |
271 | static void | 410 | static void |
272 | block_align (uint64_t start, | 411 | block_align (size_t start, |
273 | unsigned int count, | 412 | unsigned int count, |
274 | const uint64_t *sizes, | 413 | const size_t *sizes, |
275 | unsigned int *perm) | 414 | unsigned int *perm) |
276 | { | 415 | { |
277 | unsigned int i; | 416 | unsigned int i; |
278 | unsigned int j; | 417 | unsigned int j; |
279 | unsigned int tmp; | 418 | unsigned int tmp; |
280 | unsigned int best; | 419 | unsigned int best; |
281 | int64_t badness; | 420 | ssize_t badness; |
282 | uint64_t cpos; | 421 | size_t cpos; |
283 | uint64_t cend; | 422 | size_t cend; |
284 | int64_t cbad; | 423 | ssize_t cbad; |
285 | unsigned int cval; | 424 | unsigned int cval; |
286 | 425 | ||
287 | cpos = start; | 426 | cpos = start; |
@@ -334,135 +473,94 @@ block_align (uint64_t start, | |||
334 | 473 | ||
335 | 474 | ||
336 | /** | 475 | /** |
337 | * Create a directory. We allow packing more than one variable | 476 | * Finish building the directory. Frees the |
338 | * size entry into one block (and an entry could also span more | 477 | * builder context and returns the directory |
339 | * than one block), but an entry that is smaller than a single | 478 | * in-memory. |
340 | * block will never cross the block boundary. This is done to | ||
341 | * allow processing entries of a directory already even if the | ||
342 | * download is still partial.<p> | ||
343 | * | ||
344 | * The first block begins with the directories MAGIC signature, | ||
345 | * followed by the meta-data about the directory itself.<p> | ||
346 | * | 479 | * |
347 | * After that, the directory consists of block-aligned pairs | 480 | * @param bld directory to finish |
348 | * of URIs (0-terminated strings) and serialized meta-data. | 481 | * @param rsize set to the number of bytes needed |
349 | * | 482 | * @param rdata set to the encoded directory |
350 | * @param data pointer set to the beginning of the directory | ||
351 | * @param len set to number of bytes in data | ||
352 | * @param count number of entries in uris and mds | ||
353 | * @param uris URIs of the files in the directory | ||
354 | * @param mds meta-data for the files (must match | ||
355 | * respective values at same offset in in uris) | ||
356 | * @param mdir meta-data for the directory | ||
357 | * @return GNUNET_OK on success, GNUNET_SYSERR on error | ||
358 | */ | 483 | */ |
359 | int | 484 | void |
360 | GNUNET_FS_directory_create (char **data, | 485 | GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, |
361 | size_t *len, | 486 | size_t *rsize, |
362 | unsigned int count, | 487 | void **rdata) |
363 | const struct GNUNET_FS_Uri **uris, | ||
364 | const struct GNUNET_CONTAINER_MetaData **mds, | ||
365 | const struct GNUNET_CONTAINER_MetaData *mdir) | ||
366 | { | 488 | { |
489 | char *data; | ||
490 | size_t *sizes; | ||
491 | unsigned int *perm; | ||
367 | unsigned int i; | 492 | unsigned int i; |
368 | unsigned int j; | 493 | unsigned int j; |
369 | uint64_t psize; | 494 | struct BuilderEntry *pos; |
370 | uint64_t size; | 495 | struct BuilderEntry **bes; |
371 | uint64_t pos; | 496 | size_t size; |
372 | char **ucs; | 497 | size_t psize; |
373 | int ret; | 498 | size_t off; |
374 | uint64_t *sizes; | 499 | ssize_t ret; |
375 | unsigned int *perm; | 500 | uint32_t big; |
376 | 501 | ||
377 | for (i = 0; i < count; i++) | 502 | size = 8 + sizeof (uint32_t); |
378 | { | 503 | size += GNUNET_meta_data_get_serialized_size (bld->meta, |
379 | if (GNUNET_FS_uri_test_ksk (fis[i].uri)) | 504 | GNUNET_SERIALIZE_FULL); |
380 | { | 505 | if (bld->count > 0) |
381 | GNUNET_break (0); | ||
382 | return GNUNET_SYSERR; /* illegal in directory! */ | ||
383 | } | ||
384 | } | ||
385 | ucs = GNUNET_malloc (sizeof (char *) * count); | ||
386 | size = 8 + sizeof (unsigned int); | ||
387 | size += GNUNET_meta_data_get_serialized_size (meta, GNUNET_SERIALIZE_FULL); | ||
388 | sizes = GNUNET_malloc (count * sizeof (unsigned long long)); | ||
389 | perm = GNUNET_malloc (count * sizeof (int)); | ||
390 | for (i = 0; i < count; i++) | ||
391 | { | 506 | { |
392 | perm[i] = i; | 507 | sizes = GNUNET_malloc (bld->count * sizeof (size_t)); |
393 | ucs[i] = GNUNET_FS_uri_to_string (fis[i].uri); | 508 | perm = GNUNET_malloc (bld->count * sizeof (unsigned int)); |
394 | GNUNET_assert (ucs[i] != NULL); | 509 | bes = GNUNET_malloc (bld->count * sizeof (struct BuilderEntry *)); |
395 | psize = | 510 | pos = bld->head; |
396 | GNUNET_meta_data_get_serialized_size (fis[i].meta, | 511 | for (i = 0; i < bld->count; i++) |
397 | GNUNET_SERIALIZE_FULL); | 512 | { |
398 | if (psize == -1) | 513 | perm[i] = i; |
399 | { | 514 | bes[i] = pos; |
400 | GNUNET_break (0); | 515 | sizes[i] = pos->size; |
401 | GNUNET_free (sizes); | 516 | pos = pos->next; |
402 | GNUNET_free (perm); | 517 | } |
403 | while (i >= 0) | 518 | } |
404 | GNUNET_free (ucs[i--]); | 519 | block_align (size, |
405 | GNUNET_free (ucs); | 520 | bld->count, |
406 | return GNUNET_SYSERR; | 521 | sizes, |
407 | } | 522 | perm); |
408 | sizes[i] = psize + sizeof (unsigned int) + strlen (ucs[i]) + 1; | ||
409 | } | ||
410 | /* permutate entries to minimize alignment cost */ | ||
411 | block_align (size, count, sizes, perm); | ||
412 | 523 | ||
413 | /* compute final size with alignment */ | 524 | /* compute final size with alignment */ |
414 | for (i = 0; i < count; i++) | 525 | for (i = 0; i < bld->count; i++) |
415 | { | 526 | { |
416 | psize = size; | 527 | psize = size; |
417 | size += sizes[perm[i]]; | 528 | size += sizes[perm[i]]; |
418 | size = do_align (psize, size); | 529 | size = do_align (psize, size); |
419 | } | 530 | } |
420 | *len = size; | 531 | *rsize = size; |
421 | *data = GNUNET_malloc (size); | 532 | data = GNUNET_malloc (size); |
422 | memset (*data, 0, size); | 533 | *rdata = data; |
423 | 534 | memcpy (data, GNUNET_DIRECTORY_MAGIC, 8); | |
424 | pos = 8; | 535 | off = 8; |
425 | memcpy (*data, GNUNET_DIRECTORY_MAGIC, 8); | ||
426 | 536 | ||
427 | ret = GNUNET_CONTAINER_meta_data_serialize (meta, | 537 | ret = GNUNET_CONTAINER_meta_data_serialize (meta, |
428 | &(*data)[pos + | 538 | &(*data)[off + |
429 | sizeof (unsigned int)], | 539 | sizeof (uint32_t)], |
430 | size - pos - sizeof (unsigned int), | 540 | size - pos - sizeof (uint32_t), |
431 | GNUNET_SERIALIZE_FULL); | 541 | GNUNET_SERIALIZE_FULL); |
432 | GNUNET_assert (ret != GNUNET_SYSERR); | 542 | GNUNET_assert (ret != -1); |
433 | ret = htonl (ret); | 543 | big = htonl (ret); |
434 | memcpy (&(*data)[pos], &ret, sizeof (unsigned int)); | 544 | memcpy (&(*data)[8], &big, sizeof (uint32_t)); |
435 | pos += ntohl (ret) + sizeof (unsigned int); | 545 | pos += sizeof (uint32_t) + ret; |
436 | |||
437 | for (j = 0; j < count; j++) | 546 | for (j = 0; j < count; j++) |
438 | { | 547 | { |
439 | i = perm[j]; | 548 | i = perm[j]; |
440 | psize = pos; | 549 | psize = pos; |
441 | pos += sizes[i]; | 550 | pos += sizes[i]; |
442 | pos = do_align (psize, pos); | 551 | pos = do_align (psize, pos); |
443 | pos -= sizes[i]; /* go back to beginning */ | 552 | memcpy (&data[pos - sizes[i]], |
444 | memcpy (&(*data)[pos], ucs[i], strlen (ucs[i]) + 1); | 553 | &(bes[i])[1], |
445 | pos += strlen (ucs[i]) + 1; | 554 | sizes[i]); |
446 | GNUNET_free (ucs[i]); | 555 | GNUNET_free (bes[i]); |
447 | ret = GNUNET_CONTAINER_meta_data_serialize (mds[i], | ||
448 | &(*data)[pos + | ||
449 | sizeof (unsigned int)], | ||
450 | size - pos - | ||
451 | sizeof (unsigned int), | ||
452 | GNUNET_SERIALIZE_FULL); | ||
453 | GNUNET_assert (ret != GNUNET_SYSERR); | ||
454 | ret = htonl (ret); | ||
455 | memcpy (&(*data)[pos], &ret, sizeof (unsigned int)); | ||
456 | pos += ntohl (ret) + sizeof (unsigned int); | ||
457 | } | 556 | } |
458 | GNUNET_free (sizes); | 557 | GNUNET_free (sizes); |
459 | GNUNET_free (perm); | 558 | GNUNET_free (perm); |
460 | GNUNET_free (ucs); | 559 | GNUNET_free (bes); |
461 | GNUNET_assert (pos == size); | 560 | GNUNET_assert (pos == size); |
462 | return GNUNET_OK; | 561 | GNUNET_CONTAINER_meta_data_destroy (bld->meta); |
562 | GNUNET_free (bld); | ||
463 | } | 563 | } |
464 | 564 | ||
465 | 565 | ||
466 | #endif | ||
467 | |||
468 | /* end of fs_directory.c */ | 566 | /* end of fs_directory.c */ |