From 57df0fbf8398720fbd4a9cf34934c212ea00ee54 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 27 Aug 2009 11:14:19 +0000 Subject: syn --- src/fs/fs.h | 78 ++- src/fs/fs_directory.c | 332 ++++++++----- src/fs/fs_file_information.c | 3 +- src/fs/fs_publish.c | 853 +++++++++++++++++---------------- src/include/gnunet_constants.h | 7 + src/include/gnunet_container_lib.h | 27 +- src/include/gnunet_crypto_lib.h | 2 +- src/include/gnunet_datastore_service.h | 6 + src/include/gnunet_fs_service.h | 46 ++ src/util/container_meta_data.c | 32 +- src/util/crypto_hash.c | 2 +- 11 files changed, 812 insertions(+), 576 deletions(-) (limited to 'src') diff --git a/src/fs/fs.h b/src/fs/fs.h index 4743a594b..c66249613 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h @@ -26,11 +26,33 @@ #ifndef FS_H #define FS_H +#include "gnunet_datastore_service.h" +#include "gnunet_fs_service.h" + /** * Size of the individual blocks used for file-sharing. */ #define GNUNET_FS_DBLOCK_SIZE (32*1024) + +/** + * Pick a multiple of 2 here to achive 8-byte alignment! + * We also probably want DBlocks to have (roughly) the + * same size as IBlocks. With SHA-512, the optimal + * value is 32768 byte / 128 byte = 256 + * (128 byte = 2 * 512 bits). DO NOT CHANGE! + */ +#define GNUNET_FS_CHK_PER_INODE 256 + + +/** + * Maximum size for a file to be considered for + * inlining in a directory. + */ +#define GNUNET_FS_MAX_INLINE_SIZE 65536 + + + /** * @brief content hash key */ @@ -51,7 +73,7 @@ struct FileIdentifier /** * Total size of the file in bytes. (network byte order (!)) */ - unsigned long long file_length; + uint64_t file_length; /** * Query and key of the top GNUNET_EC_IBlock. @@ -186,6 +208,12 @@ struct GNUNET_FS_FileInformation */ struct GNUNET_FS_Uri *keywords; + /** + * CHK for this file or directory. NULL if + * we have not yet computed it. + */ + struct GNUNET_FS_Uri *chk_uri; + /** * At what time should the content expire? */ @@ -197,6 +225,31 @@ struct GNUNET_FS_FileInformation */ char *serialization; + /** + * In-memory cache of the current CHK tree. + * This struct will contain the CHK values + * from the root to the currently processed + * node in the tree as identified by + * "current_depth" and "publish_offset". + * The "chktree" will be initially NULL, + * then allocated to a sufficient number of + * entries for the size of the file and + * finally freed once the upload is complete. + */ + struct ContentHashKey *chk_tree; + + /** + * Number of entries in "chk_tree". + */ + unsigned int chk_tree_depth; + + /** + * Depth in the CHK-tree at which we are + * currently publishing. 0 is the root + * of the tree. + */ + unsigned int current_depth; + /** * How many bytes of this file or directory have been * published so far? @@ -257,6 +310,12 @@ struct GNUNET_FS_FileInformation */ uint64_t dir_size; + /** + * Pointer to the data for the directory (or NULL if not + * available). + */ + char *dir_data; + } dir; } data; @@ -358,6 +417,23 @@ struct GNUNET_FS_PublishContext * if the upload has completed. */ GNUNET_SCHEDULER_TaskIdentifier upload_task; + + /** + * Current position in the file-tree for the + * upload. + */ + struct GNUNET_FS_FileInformation *fi_pos; + + /** + * Connection to the datastore service. + */ + struct GNUNET_DATASTORE_Handle *dsh; + + /** + * Space reservation ID with datastore service + * for this upload. + */ + int rid; }; diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c index 3eb3af50d..e5a9f963f 100644 --- a/src/fs/fs_directory.c +++ b/src/fs/fs_directory.c @@ -25,7 +25,6 @@ * * TODO: * - add support for embedded file data (use padding room!) - * - add directory builder API to gnunet_fs_service * - modify directory builder API to support incremental * generation of directories (to allow directories that * would not fit into memory to be created) @@ -38,6 +37,9 @@ #include "gnunet_fs_service.h" #include "fs.h" +#ifndef EXTRACTOR_GNUNET_FULL_DATA +#define EXTRACTOR_GNUNET_FULL_DATA 137 +#endif /** * Does the meta-data claim that this is a directory? @@ -215,6 +217,7 @@ GNUNET_FS_directory_list_contents (size_t size, return; /* malformed ! */ } pos += mdSize; + // EXTRACTOR_GNUNET_FULL_DATA /* FIXME: add support for embedded data */ filename = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_FILENAME); @@ -231,14 +234,150 @@ GNUNET_FS_directory_list_contents (size_t size, } } +/** + * Entries in the directory (builder). + */ +struct BuilderEntry +{ + /** + * This is a linked list. + */ + struct BuilderEntry *next; + + /** + * Length of this entry. + */ + size_t len; +}; -void -GNUNET_FS_directory_create () +/** + * Internal state of a directory builder. + */ +struct GNUNET_FS_DirectoryBuilder { + /** + * Meta-data for the directory itself. + */ + struct GNUNET_CONTAINER_MetaData *meta; + + /** + * Head of linked list of entries. + */ + struct BuilderEntry *head; + + /** + * Number of entires in the directory. + */ + unsigned int count; +}; + + +/** + * Create a directory builder. + * + * @param mdir metadata for the directory + */ +struct GNUNET_FS_DirectoryBuilder * +GNUNET_FS_directory_builder_create (const struct GNUNET_CONTAINER_MetaData *mdir) +{ + struct GNUNET_FS_DirectoryBuilder *ret; + + ret = GNUNET_malloc(sizeof(struct GNUNET_FS_DirectoryBuilder)); + ret->meta = GNUNET_CONTAINER_meta_data_duplicate (mdir); + GNUNET_FS_meta_data_make_directory (ret->meta); + return ret; } -#if 0 +/** + * Add an entry to a directory. + * + * @param bld directory to extend + * @param uri uri of the entry (must not be a KSK) + * @param md metadata of the entry + * @param data raw data of the entry, can be NULL, otherwise + * data must point to exactly the number of bytes specified + * by the uri which must be of type LOC or CHK + */ +void +GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *md, + const void *data) +{ + struct BuilderEntry *e; + uint64_t fsize; + uint32_t big; + size_t mds; + size_t mdxs; + char *uris; + char *ser; + size_t slen; + struct GNUNET_CONTAINER_MetaData *meta; + const struct GNUNET_CONTAINER_MetaData *meta_use; + + GNUNET_assert (! GNUNET_FS_uri_ksk_test (uri)); + if (NULL != data) + if (GNUNET_FS_uri_chk_test (uri)) + fsize = GNUNET_FS_uri_chk_get_size (uri); + else + fsize = GNUNET_FS_uri_chk_get_size (GNUNET_FS_uri_loc_get_uri (uri)); + else + fsize = 0; /* not given */ + if (fsize > GNUNET_FS_MAX_INLINE_SIZE) + fsize = 0; /* too large */ + if (memchr (data, fsize, '\0')) // FIXME: check memchr args! + fsize = 0; /* must not have 0's in data! */ + uris = GNUNET_FS_uri_to_string (uri); + slen = strlen (uris) + 1; + mds = + GNUNET_CONTAINER_meta_data_get_serialized_size (md, + GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); + meta_use = md; + meta = NULL; + if (fsize > 0) + { + meta = GNUNET_CONTAINER_meta_data_duplicate (md); + GNUNET_CONTAINER_meta_data_insert (meta, + EXTRACTOR_GNUNET_FULL_DATA, + data); + mdxs = + GNUNET_CONTAINER_meta_data_get_serialized_size (meta, + GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); + if ( (slen + sizeof (uint32_t) + mdxs - 1) / GNUNET_FS_DBLOCK_SIZE == + (slen + sizeof (uint32_t) + mds - 1) / GNUNET_FS_DBLOCK_SIZE) + { + /* adding full data would not cause us to cross + additional blocks, so add it! */ + meta_use = meta; + mds = mdxs; + } + } + + if (mds > GNUNET_MAX_MALLOC_CHECKED / 2) + mds = GNUNET_MAX_MALLOC_CHECKED / 2; + e = GNUNET_malloc (sizeof(struct BuilderEntry) + + slen + mds + sizeof (uint32_t)); + ser = (char*) &e[1]; + memcpy (ser, uris, slen); + GNUNET_free (uris); + ret = GNUNET_CONTAINER_meta_data_serialize (meta_use, + &ser[slen + sizeof(uint32_t)], + mds, + GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); + if (NULL != meta) + GNUNET_CONTAINER_meta_data_destroy (meta); + if (ret == -1) + mds = 0; + else + mds = ret; + big = htonl (mds); + memcpy (&ser[slen], &big, sizeof (uint32_t)); + e->len = slen + sizeof (uint32_t) + mds; + e->next = bld->head; + bld->head = e; + bld->count++; +} /** @@ -246,11 +385,11 @@ GNUNET_FS_directory_create () * data, return the end position of that data * after alignment to the GNUNET_FS_DBLOCK_SIZE. */ -static uint64_t -do_align (uint64_t start_position, - uint64_t end_position) +static size_t +do_align (size_t start_position, + size_t end_position) { - uint64_t align; + size_t align; align = (end_position / GNUNET_FS_DBLOCK_SIZE) * GNUNET_FS_DBLOCK_SIZE; if ((start_position < align) && (end_position > align)) @@ -269,19 +408,19 @@ do_align (uint64_t start_position, * @param perm the permutation of the blocks (updated) */ static void -block_align (uint64_t start, +block_align (size_t start, unsigned int count, - const uint64_t *sizes, + const size_t *sizes, unsigned int *perm) { unsigned int i; unsigned int j; unsigned int tmp; unsigned int best; - int64_t badness; - uint64_t cpos; - uint64_t cend; - int64_t cbad; + ssize_t badness; + size_t cpos; + size_t cend; + ssize_t cbad; unsigned int cval; cpos = start; @@ -334,135 +473,94 @@ block_align (uint64_t start, /** - * Create a directory. We allow packing more than one variable - * size entry into one block (and an entry could also span more - * than one block), but an entry that is smaller than a single - * block will never cross the block boundary. This is done to - * allow processing entries of a directory already even if the - * download is still partial.

- * - * The first block begins with the directories MAGIC signature, - * followed by the meta-data about the directory itself.

+ * Finish building the directory. Frees the + * builder context and returns the directory + * in-memory. * - * After that, the directory consists of block-aligned pairs - * of URIs (0-terminated strings) and serialized meta-data. - * - * @param data pointer set to the beginning of the directory - * @param len set to number of bytes in data - * @param count number of entries in uris and mds - * @param uris URIs of the files in the directory - * @param mds meta-data for the files (must match - * respective values at same offset in in uris) - * @param mdir meta-data for the directory - * @return GNUNET_OK on success, GNUNET_SYSERR on error + * @param bld directory to finish + * @param rsize set to the number of bytes needed + * @param rdata set to the encoded directory */ -int -GNUNET_FS_directory_create (char **data, - size_t *len, - unsigned int count, - const struct GNUNET_FS_Uri **uris, - const struct GNUNET_CONTAINER_MetaData **mds, - const struct GNUNET_CONTAINER_MetaData *mdir) +void +GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, + size_t *rsize, + void **rdata) { + char *data; + size_t *sizes; + unsigned int *perm; unsigned int i; unsigned int j; - uint64_t psize; - uint64_t size; - uint64_t pos; - char **ucs; - int ret; - uint64_t *sizes; - unsigned int *perm; - - for (i = 0; i < count; i++) - { - if (GNUNET_FS_uri_test_ksk (fis[i].uri)) - { - GNUNET_break (0); - return GNUNET_SYSERR; /* illegal in directory! */ - } - } - ucs = GNUNET_malloc (sizeof (char *) * count); - size = 8 + sizeof (unsigned int); - size += GNUNET_meta_data_get_serialized_size (meta, GNUNET_SERIALIZE_FULL); - sizes = GNUNET_malloc (count * sizeof (unsigned long long)); - perm = GNUNET_malloc (count * sizeof (int)); - for (i = 0; i < count; i++) + struct BuilderEntry *pos; + struct BuilderEntry **bes; + size_t size; + size_t psize; + size_t off; + ssize_t ret; + uint32_t big; + + size = 8 + sizeof (uint32_t); + size += GNUNET_meta_data_get_serialized_size (bld->meta, + GNUNET_SERIALIZE_FULL); + if (bld->count > 0) { - perm[i] = i; - ucs[i] = GNUNET_FS_uri_to_string (fis[i].uri); - GNUNET_assert (ucs[i] != NULL); - psize = - GNUNET_meta_data_get_serialized_size (fis[i].meta, - GNUNET_SERIALIZE_FULL); - if (psize == -1) - { - GNUNET_break (0); - GNUNET_free (sizes); - GNUNET_free (perm); - while (i >= 0) - GNUNET_free (ucs[i--]); - GNUNET_free (ucs); - return GNUNET_SYSERR; - } - sizes[i] = psize + sizeof (unsigned int) + strlen (ucs[i]) + 1; - } - /* permutate entries to minimize alignment cost */ - block_align (size, count, sizes, perm); + sizes = GNUNET_malloc (bld->count * sizeof (size_t)); + perm = GNUNET_malloc (bld->count * sizeof (unsigned int)); + bes = GNUNET_malloc (bld->count * sizeof (struct BuilderEntry *)); + pos = bld->head; + for (i = 0; i < bld->count; i++) + { + perm[i] = i; + bes[i] = pos; + sizes[i] = pos->size; + pos = pos->next; + } + } + block_align (size, + bld->count, + sizes, + perm); /* compute final size with alignment */ - for (i = 0; i < count; i++) + for (i = 0; i < bld->count; i++) { psize = size; size += sizes[perm[i]]; size = do_align (psize, size); } - *len = size; - *data = GNUNET_malloc (size); - memset (*data, 0, size); - - pos = 8; - memcpy (*data, GNUNET_DIRECTORY_MAGIC, 8); + *rsize = size; + data = GNUNET_malloc (size); + *rdata = data; + memcpy (data, GNUNET_DIRECTORY_MAGIC, 8); + off = 8; ret = GNUNET_CONTAINER_meta_data_serialize (meta, - &(*data)[pos + - sizeof (unsigned int)], - size - pos - sizeof (unsigned int), + &(*data)[off + + sizeof (uint32_t)], + size - pos - sizeof (uint32_t), GNUNET_SERIALIZE_FULL); - GNUNET_assert (ret != GNUNET_SYSERR); - ret = htonl (ret); - memcpy (&(*data)[pos], &ret, sizeof (unsigned int)); - pos += ntohl (ret) + sizeof (unsigned int); - + GNUNET_assert (ret != -1); + big = htonl (ret); + memcpy (&(*data)[8], &big, sizeof (uint32_t)); + pos += sizeof (uint32_t) + ret; for (j = 0; j < count; j++) { i = perm[j]; psize = pos; pos += sizes[i]; pos = do_align (psize, pos); - pos -= sizes[i]; /* go back to beginning */ - memcpy (&(*data)[pos], ucs[i], strlen (ucs[i]) + 1); - pos += strlen (ucs[i]) + 1; - GNUNET_free (ucs[i]); - ret = GNUNET_CONTAINER_meta_data_serialize (mds[i], - &(*data)[pos + - sizeof (unsigned int)], - size - pos - - sizeof (unsigned int), - GNUNET_SERIALIZE_FULL); - GNUNET_assert (ret != GNUNET_SYSERR); - ret = htonl (ret); - memcpy (&(*data)[pos], &ret, sizeof (unsigned int)); - pos += ntohl (ret) + sizeof (unsigned int); + memcpy (&data[pos - sizes[i]], + &(bes[i])[1], + sizes[i]); + GNUNET_free (bes[i]); } GNUNET_free (sizes); GNUNET_free (perm); - GNUNET_free (ucs); - GNUNET_assert (pos == size); - return GNUNET_OK; + GNUNET_free (bes); + GNUNET_assert (pos == size); + GNUNET_CONTAINER_meta_data_destroy (bld->meta); + GNUNET_free (bld); } -#endif - /* end of fs_directory.c */ diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c index 540b518dc..60bcc2529 100644 --- a/src/fs/fs_file_information.c +++ b/src/fs/fs_file_information.c @@ -804,6 +804,7 @@ GNUNET_FS_file_information_destroy (struct GNUNET_FS_FileInformation *fi, &fi->priority, &fi->expirationTime, &fi->client_info); + GNUNET_free_non_null (fi->data.dir.dir_data); GNUNET_free (fi->data.dir.dirname); } else @@ -821,7 +822,7 @@ GNUNET_FS_file_information_destroy (struct GNUNET_FS_FileInformation *fi, &fi->expirationTime, &fi->client_info); } - + GNUNET_free_non_null (fi->chk_tree); /* clean up serialization */ if (0 != UNLINK (fi->serialization)) GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c index 7696dd940..0078d0d32 100644 --- a/src/fs/fs_publish.c +++ b/src/fs/fs_publish.c @@ -1,6 +1,6 @@ /* This file is part of GNUnet. - (C) 2001, 2002, 2003, 2004, 2005, 2006, 2009 Christian Grothoff (and other contributing authors) + (C) 2009 Christian Grothoff (and other contributing authors) GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -24,15 +24,394 @@ * @see http://gnunet.org/encoding.php3 * @author Krista Bennett * @author Christian Grothoff + * + * TODO: + * - directory creation + * - KBlocks + * - SBlocks + * - indexing support + * - calling of progress function + * - handling of IO errors (emsg) + * - code-sharing with unindex + * - datastore reservation support + * - persistence support */ #include "platform.h" +#include "gnunet_constants.h" #include "gnunet_util_lib.h" #include "gnunet_fs_service.h" #include "fs.h" #define DEBUG_PUBLISH GNUNET_YES +/** + * Main function that performs the upload. + * @param cls "struct GNUNET_FS_PublishContext" identifies the upload + * @param tc task context + */ +static void +do_upload (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc); + + +/** + * Context for "ds_put_cont". + */ +struct PutContCtx +{ + /** + * Publishing context for which the datastore + * PUT request was executed. + */ + struct GNUNET_FS_PublishContext *sc; + + /** + * Specific file with the block. + */ + struct GNUNET_FS_FileInformation *p; + + /** + * Function to run next, if any (can be NULL). + */ + GNUNET_SCHEDULER_Task cont; +}; + +/** + * Function called by the datastore API with + * the result from the PUT request. + * + * @param cls our closure + * @param success GNUNET_OK on success + * @param msg error message (or NULL) + */ +static void +ds_put_cont (void *cls, + int success, + const char *msg) +{ + struct PutContCtx *pcc = cls; + + if (GNUNET_OK != success) + { + // FIXME: call progress CB with error + // FIXME: update pcc->p to indicate abort + GNUNET_FS_file_information_sync (pcc->p); + return; + } + GNUNET_FS_file_information_sync (pcc->p); + if (NULL != pcc->cont) + pcc->sc->upload_task + = GNUNET_SCHEDULER_add_delayed (pcc->sc->h->sched, + GNUNET_NO, + GNUNET_SCHEDULER_PRIORITY_BACKGROUND, + GNUNET_SCHEDULER_NO_TASK, + GNUNET_TIME_UNIT_ZERO, + pcc->cont, + pcc->sc); + GNUNET_free (pcc); +} + + +/** + * We need to publish a specific block. Do it. Then continue with + * the main task. + * + * @param sc overall upload data + * @param p file that the block belongs to (needed for options!) + * @param blk encoded block to publish + * @param blk_size size of the block + * @param blk_type type of the block + * @param cont function to run when done + */ +static void +publish_block (struct GNUNET_FS_PublishContext *sc, + struct GNUNET_FS_FileInformation *p, + const void* blk, + uint16_t blk_size, + uint32_t blk_type, + GNUNET_SCHEDULER_Task cont) +{ + struct GNUNET_HashCode key; + + // FIXME: GNUNET_FS_get_key (blk_type, blk, blk_size, &key); + // (or add "key" as argument to reduce hashing?) + dpc_cls = GNUNET_malloc(sizeof(struct PutContCtx)); + dpc_cls->cont = cont; + dpc_cls->sc = sc; + dpc_cls->p = p; + // FIXME: need to do something to "sc" to mark + // that "sc" can not be freed right now due to this + // pending, scheduled operation for which we don't have + // a task ID! + GNUNET_DATASTORE_put (sc->dsh, + sc->rid, + &key, + blk_size, + blk_type, + p->priority, + p->anonymity, + p->expirationTime, + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + &ds_put_cont, + dpc_cls); +} + + +/** + * We are almost done publishing the structure, + * add SBlocks (if needed). + * + * @param sc overall upload data + */ +static void +publish_sblock (struct GNUNET_FS_PublishContext *sc) +{ + struct GNUNET_FS_FileInformation *p; + p = sc->fi; + + // FIXME: build sblock & call publish_block! + + // FIXME: continuation should + // be releasing the datastore reserve + // (once implemented) +} + + +/** + * We have uploaded a file or directory; now publish + * the KBlocks in the global keyword space so that + * it can be found. Then continue with the + * main task. + * + * @param sc overall upload data + * @param p specific file or directory for which kblocks + * should be created + */ +static void +publish_kblocks (struct GNUNET_FS_PublishContext *sc, + struct GNUNET_FS_FileInformation *p) +{ + // FIXME: build all kblocks + // call publish_kblock on each + // last continuation should then call the main continuation again +} + + +/** + * Compute the depth of the CHK tree. + * + * @param flen file length for which to compute the depth + * @return depth of the tree + */ +static unsigned int +compute_depth (uint64_t flen) +{ + unsigned int treeDepth; + uint64_t fl; + + treeDepth = 1; + fl = GNUNET_FS_DBLOCK_SIZE; + while (fl < flen) + { + treeDepth++; + if (fl * GNUNET_FS_CHK_PER_INODE < fl) + { + /* integer overflow, this is a HUGE file... */ + return treeDepth; + } + fl = fl * GNUNET_FS_CHK_PER_INODE; + } + return treeDepth; +} + + +/** + * Compute the size of the current IBlock. + * + * @param height height of the IBlock in the tree (aka overall + * number of tree levels minus depth); 0 == DBlock + * @param offset current offset in the overall file + * @return size of the corresponding IBlock + */ +static uint16_t +compute_iblock_size (unsigned int height, + uint64_t offset) +{ + unsigned int ret; + unsigned int i; + uint64_t mod; + uint64_t bds; + + GNUNET_assert (height > 0); + bds = GNUNET_FS_DBLOCK_SIZE; /* number of bytes each CHK at level "i" + corresponds to */ + for (i=0;iis_directory) ? p->data.dir.dir_size : p->data.file.file_size; + if (NULL == p->chk_tree) + { + if (p->is_directory) + { + /* FIXME: create function to create directory + and use that API here! */ + GNUNET_FS_directory_create (&p->data.dir.dir_size, + &p->data.dir.dir_data, + p->meta, + &directory_entry_lister, + p->data.dir.entries); + size = p->data.dir.data_size; + } + p->chk_tree_depth = compute_depth (size); + p->chk_tree = GNUNET_malloc (p->chk_tree_depth * + sizeof (struct ContentHashKey) * + GNUNET_FS_CHK_PER_INODE); + p->current_depth = p->chk_tree_depth; + } + if (p->current_depth == p->chk_tree_depth) + { + if (p->is_directory) + { + pt_size = GNUNET_MIN(GNUNET_FS_DBLOCK_SIZE, + p->data.dir.dir_size - p->publish_offset); + pt_block = &p->data.dir.dir_data[p->publish_offset]; + } + else + { + pt_size = GNUNET_MIN(GNUNET_FS_DBLOCK_SIZE, + p->data.file.file_size - p->publish_offset); + p->data.file.reader (p->data.file.reader_cls, + p->publish_offset, + pt_size, + iob, + &emsg); + pt_block = iob; + } + } + else + { + pt_size = compute_iblock_size (p->chk_tree_depth - p->current_depth, + p->publish_offset); + pt_block = &p->chk_tree[p->current_depth * + GNUNET_FS_CHK_PER_INODE]; + } + off = compute_chk_offset (p->chk_tree_depth - p->current_depth, + p->publish_offset); + chk = &p->chk_tree[(p->current_depth-1)*GNUNET_FS_CHK_PER_INODE+off]; + GNUNET_CRYPTO_hash (pt_block, pt_size, &chk->key); + GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv); + GNUNET_CRYPTO_aes_encrypt (pt_block, + pt_size, + &sk, + &iv, + enc); + // NOTE: this call (and progress below) is all that really differs + // between publish/unindex! Parameterize & move this code! + // FIXME: something around here would need to change + // for indexing! + publish_block (sc, p, enc, pt_size, + (p->current_depth == p->chk_tree_depth) + ? GNUNET_DATASTORE_BLOCKTYPE_DBLOCK + : GNUNET_DATASTORE_BLOCKTYPE_IBLOCK, + &do_upload); + // FIXME: should call progress function somewhere here! + GNUNET_CRYPTO_hash (enc, pt_size, &chk->query); + if (p->current_depth == p->chk_tree_depth) + { + p->publish_offset += pt_size; + if ( (p->publish_offset == size) || + (0 == p->publish_offset % (GNUNET_FS_CHK_PER_INODE * GNUNET_FS_DBLOCK_SIZE) ) ) + p->current_depth--; + } + else + { + if ( (off == GNUNET_FS_CHK_PER_INODE) || + (p->publish_offset == size) ) + p->current_depth--; + else + p->current_depth = p->chk_tree_depth; + } + if (0 == p->current_depth) + { + p->chk_uri = GNUNET_malloc (sizeof(struct GNUNET_FS_Uri)); + p->chk_uri.type = chk; + p->chk_uri.data.chk.chk = p->chk_tree[0]; + p->chk_uri.data.chk.file_length = size; + GNUNET_free (p->chk_tree); + p->chk_tree = NULL; + } +} + /** * Main function that performs the upload. @@ -44,11 +423,36 @@ do_upload (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { struct GNUNET_FS_PublishContext *sc = cls; + struct GNUNET_FS_FileInformation *p; sc->upload_task = GNUNET_SCHEDULER_NO_TASK; - - // FIXME: find next block, process, schedule - // transmission to FS service + p = sc->fi_pos; + if (NULL == p) + { + /* upload of entire hierarchy complete, + publish namespace entries */ + publish_sblock (sc); + return; + } + if (NULL != p->chk_uri) + { + /* move on to next file */ + if (NULL != p->next) + sc->fi_pos = p->next; + else + sc->fi_pos = p->dir; + /* upload of "p" complete, publish KBlocks! */ + publish_kblocks (sc, p); + return; + } + if (p->do_index) + { + // FIXME: need to pre-compute hash over + // the entire file and ask FS to prepare + // for indexing! + return; + } + publish_content (sc, p); } @@ -75,8 +479,15 @@ GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h, const char *nuid) { struct GNUNET_FS_PublishContext *ret; + struct GNUNET_FS_FileInformation *p; + struct GNUNET_DATASTORE_Handle *dsh; + dsh = GNUNET_DATASTORE_connect (h->cfg, + h->sched); + if (NULL == dsh) + return NULL; ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext)); + ret->dsh = dsh; ret->h = h; ret->client_ctx = ctx; ret->fi = fi; @@ -90,6 +501,17 @@ GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h, ret->nuid = GNUNET_strdup (nuid); } // FIXME: make upload persistent! + + /* find first leaf, DFS */ + p = ret->fi; + while ( (p->is_directory) && + (NULL != p->data.dir.entries) ) + p = p->data.dir.entries; + ret->fi_pos = p; + + // FIXME: calculate space needed for "fi" + // and reserve as first task (then trigger + // "do_upload" from that continuation)! ret->upload_task = GNUNET_SCHEDULER_add_delayed (h->sched, GNUNET_NO, @@ -120,429 +542,8 @@ GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *sc) GNUNET_FS_namespace_delete (sc->namespace, GNUNET_NO); GNUNET_free_non_null (sc->nid); GNUNET_free_non_null (sc->nuid); + GNUNET_DATASTORE_disconnect (sc->dsh); GNUNET_free (sc); } - -#if 0 - -/** - * Append the given key and query to the iblock[level]. If - * iblock[level] is already full, compute its chk and push it to - * level+1 and clear the level. iblocks is guaranteed to be big - * enough. - */ -static int -pushBlock (struct GNUNET_ClientServerConnection *sock, - const GNUNET_EC_ContentHashKey * chk, - unsigned int level, - GNUNET_DatastoreValue ** iblocks, - unsigned int prio, GNUNET_CronTime expirationTime) -{ - unsigned int size; - unsigned int present; - GNUNET_DatastoreValue *value; - GNUNET_EC_DBlock *db; - GNUNET_EC_ContentHashKey ichk; - - size = ntohl (iblocks[level]->size); - GNUNET_GE_ASSERT (NULL, size > sizeof (GNUNET_DatastoreValue)); - size -= sizeof (GNUNET_DatastoreValue); - GNUNET_GE_ASSERT (NULL, - size - sizeof (GNUNET_EC_DBlock) <= - GNUNET_ECRS_IBLOCK_SIZE); - present = - (size - sizeof (GNUNET_EC_DBlock)) / sizeof (GNUNET_EC_ContentHashKey); - db = (GNUNET_EC_DBlock *) & iblocks[level][1]; - if (present == GNUNET_ECRS_CHK_PER_INODE) - { - GNUNET_EC_file_block_get_key (db, size, &ichk.key); - GNUNET_EC_file_block_get_query (db, size, &ichk.query); - if (GNUNET_OK != pushBlock (sock, - &ichk, level + 1, iblocks, prio, - expirationTime)) - return GNUNET_SYSERR; - GNUNET_EC_file_block_encode (db, size, &ichk.query, &value); - if (value == NULL) - { - GNUNET_GE_BREAK (NULL, 0); - return GNUNET_SYSERR; - } - value->priority = htonl (prio); - value->expiration_time = GNUNET_htonll (expirationTime); - if (GNUNET_OK != GNUNET_FS_insert (sock, value)) - { - GNUNET_free (value); - return GNUNET_SYSERR; - } - GNUNET_free (value); - size = sizeof (GNUNET_EC_DBlock); /* type */ - } - /* append GNUNET_EC_ContentHashKey */ - memcpy (&((char *) db)[size], chk, sizeof (GNUNET_EC_ContentHashKey)); - size += sizeof (GNUNET_EC_ContentHashKey) + sizeof (GNUNET_DatastoreValue); - GNUNET_GE_ASSERT (NULL, size < GNUNET_MAX_BUFFER_SIZE); - iblocks[level]->size = htonl (size); - - return GNUNET_OK; -} - -/** - * Index or insert a file. - * - * @param priority what is the priority for OUR node to - * keep this file available? Use 0 for maximum anonymity and - * minimum reliability... - * @param doIndex GNUNET_YES for index, GNUNET_NO for insertion, - * GNUNET_SYSERR for simulation - * @param uri set to the URI of the uploaded file - * @return GNUNET_SYSERR if the upload failed (i.e. not enough space - * or gnunetd not running) - */ -int -GNUNET_ECRS_file_upload (struct GNUNET_GE_Context *ectx, - struct GNUNET_GC_Configuration *cfg, - const char *filename, - int doIndex, - unsigned int anonymityLevel, - unsigned int priority, - GNUNET_CronTime expirationTime, - GNUNET_ECRS_UploadProgressCallback upcb, - void *upcbClosure, - GNUNET_ECRS_TestTerminate tt, - void *ttClosure, struct GNUNET_ECRS_URI **uri) -{ - unsigned long long filesize; - unsigned long long pos; - unsigned int treedepth; - int fd; - int i; - int ret; - unsigned int size; - GNUNET_DatastoreValue **iblocks; - GNUNET_DatastoreValue *dblock; - GNUNET_EC_DBlock *db; - GNUNET_DatastoreValue *value; - struct GNUNET_ClientServerConnection *sock; - GNUNET_HashCode fileId; - GNUNET_EC_ContentHashKey mchk; - GNUNET_CronTime eta; - GNUNET_CronTime start; - GNUNET_CronTime now; - GNUNET_EC_FileIdentifier fid; -#if DEBUG_UPLOAD - GNUNET_EncName enc; -#endif - - GNUNET_GE_ASSERT (ectx, cfg != NULL); - start = GNUNET_get_time (); - memset (&mchk, 0, sizeof (GNUNET_EC_ContentHashKey)); - if (GNUNET_YES != GNUNET_disk_file_test (ectx, filename)) - { - GNUNET_GE_LOG (ectx, - GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER, - _("`%s' is not a file.\n"), filename); - return GNUNET_SYSERR; - } - if (GNUNET_OK != - GNUNET_disk_file_size (ectx, filename, &filesize, GNUNET_YES)) - { - GNUNET_GE_LOG (ectx, - GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER, - _("Cannot get size of file `%s'"), filename); - - return GNUNET_SYSERR; - } - sock = GNUNET_client_connection_create (ectx, cfg); - if (sock == NULL) - { - GNUNET_GE_LOG (ectx, - GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER, - _("Failed to connect to gnunetd.")); - return GNUNET_SYSERR; - } - eta = 0; - if (upcb != NULL) - upcb (filesize, 0, eta, upcbClosure); - if (doIndex == GNUNET_YES) - { - if (GNUNET_SYSERR == GNUNET_hash_file (ectx, filename, &fileId)) - { - GNUNET_GE_LOG (ectx, - GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER, - _("Cannot hash `%s'.\n"), filename); - - GNUNET_client_connection_destroy (sock); - return GNUNET_SYSERR; - } - if (GNUNET_YES == GNUNET_FS_test_indexed (sock, &fileId)) - { - /* file already indexed; simulate only to get the URI! */ - doIndex = GNUNET_SYSERR; - } - } - if (doIndex == GNUNET_YES) - { - now = GNUNET_get_time (); - eta = now + 2 * (now - start); - /* very rough estimate: GNUNET_hash reads once through the file, - we'll do that once more and write it. But of course - the second read may be cached, and we have the encryption, - so a factor of two is really, really just a rough estimate */ - start = now; - /* reset the counter since the formula later does not - take the time for GNUNET_hash_file into account */ - - switch (GNUNET_FS_prepare_to_index (sock, &fileId, filename)) - { - case GNUNET_SYSERR: - GNUNET_GE_LOG (ectx, - GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER, - _("Initialization for indexing file `%s' failed.\n"), - filename); - GNUNET_client_connection_destroy (sock); - return GNUNET_SYSERR; - case GNUNET_NO: - GNUNET_GE_LOG (ectx, - GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER, - _ - ("Indexing file `%s' failed. Suggestion: try to insert the file.\n"), - filename); - GNUNET_client_connection_destroy (sock); - return GNUNET_SYSERR; - default: - break; - } - } - treedepth = GNUNET_ECRS_compute_depth (filesize); - fd = GNUNET_disk_file_open (ectx, filename, O_RDONLY | O_LARGEFILE); - if (fd == -1) - { - GNUNET_GE_LOG (ectx, - GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER, - _("Cannot open file `%s': `%s'"), filename, - STRERROR (errno)); - - GNUNET_client_connection_destroy (sock); - return GNUNET_SYSERR; - } - - dblock = - GNUNET_malloc (sizeof (GNUNET_DatastoreValue) + GNUNET_ECRS_DBLOCK_SIZE + - sizeof (GNUNET_EC_DBlock)); - dblock->size = - htonl (sizeof (GNUNET_DatastoreValue) + GNUNET_ECRS_DBLOCK_SIZE + - sizeof (GNUNET_EC_DBlock)); - dblock->anonymity_level = htonl (anonymityLevel); - dblock->priority = htonl (priority); - dblock->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA); - dblock->expiration_time = GNUNET_htonll (expirationTime); - db = (GNUNET_EC_DBlock *) & dblock[1]; - db->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA); - iblocks = - GNUNET_malloc (sizeof (GNUNET_DatastoreValue *) * (treedepth + 1)); - for (i = 0; i <= treedepth; i++) - { - iblocks[i] = - GNUNET_malloc (sizeof (GNUNET_DatastoreValue) + - GNUNET_ECRS_IBLOCK_SIZE + sizeof (GNUNET_EC_DBlock)); - iblocks[i]->size = - htonl (sizeof (GNUNET_DatastoreValue) + sizeof (GNUNET_EC_DBlock)); - iblocks[i]->anonymity_level = htonl (anonymityLevel); - iblocks[i]->priority = htonl (priority); - iblocks[i]->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA); - iblocks[i]->expiration_time = GNUNET_htonll (expirationTime); - ((GNUNET_EC_DBlock *) & iblocks[i][1])->type = - htonl (GNUNET_ECRS_BLOCKTYPE_DATA); - } - - pos = 0; - while (pos < filesize) - { - if (upcb != NULL) - upcb (filesize, pos, eta, upcbClosure); - if (tt != NULL) - if (GNUNET_OK != tt (ttClosure)) - goto FAILURE; - size = GNUNET_ECRS_DBLOCK_SIZE; - if (size > filesize - pos) - { - size = filesize - pos; - memset (&db[1], 0, GNUNET_ECRS_DBLOCK_SIZE); - } - GNUNET_GE_ASSERT (ectx, - sizeof (GNUNET_DatastoreValue) + size + - sizeof (GNUNET_EC_DBlock) < GNUNET_MAX_BUFFER_SIZE); - dblock->size = - htonl (sizeof (GNUNET_DatastoreValue) + size + - sizeof (GNUNET_EC_DBlock)); - if (size != READ (fd, &db[1], size)) - { - GNUNET_GE_LOG_STRERROR_FILE (ectx, - GNUNET_GE_ERROR | GNUNET_GE_BULK | - GNUNET_GE_ADMIN | GNUNET_GE_USER, - "READ", filename); - goto FAILURE; - } - if (tt != NULL) - if (GNUNET_OK != tt (ttClosure)) - goto FAILURE; - GNUNET_EC_file_block_get_key (db, size + sizeof (GNUNET_EC_DBlock), - &mchk.key); - GNUNET_EC_file_block_get_query (db, size + sizeof (GNUNET_EC_DBlock), - &mchk.query); -#if DEBUG_UPLOAD - GNUNET_hash_to_enc (&mchk.query, &enc); - fprintf (stderr, - "Query for current block of size %u is `%s'\n", size, - (const char *) &enc); -#endif - if (doIndex == GNUNET_YES) - { - if (GNUNET_SYSERR == GNUNET_FS_index (sock, &fileId, dblock, pos)) - { - GNUNET_GE_LOG (ectx, - GNUNET_GE_ERROR | GNUNET_GE_BULK | - GNUNET_GE_USER, - _ - ("Indexing data of file `%s' failed at position %llu.\n"), - filename, pos); - goto FAILURE; - } - } - else - { - value = NULL; - if (GNUNET_OK != - GNUNET_EC_file_block_encode (db, - size + sizeof (GNUNET_EC_DBlock), - &mchk.query, &value)) - { - GNUNET_GE_BREAK (ectx, 0); - goto FAILURE; - } - GNUNET_GE_ASSERT (ectx, value != NULL); - *value = *dblock; /* copy options! */ - if ((doIndex == GNUNET_NO) && - (GNUNET_OK != (ret = GNUNET_FS_insert (sock, value)))) - { - GNUNET_GE_BREAK (ectx, ret == GNUNET_NO); - GNUNET_free (value); - goto FAILURE; - } - GNUNET_free (value); - } - pos += size; - now = GNUNET_get_time (); - if (pos > 0) - { - eta = (GNUNET_CronTime) (start + - (((double) (now - start) / (double) pos)) - * (double) filesize); - } - if (GNUNET_OK != pushBlock (sock, &mchk, 0, /* dblocks are on level 0 */ - iblocks, priority, expirationTime)) - goto FAILURE; - } - if (tt != NULL) - if (GNUNET_OK != tt (ttClosure)) - goto FAILURE; -#if DEBUG_UPLOAD - GNUNET_GE_LOG (ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER, - "Tree depth is %u, walking up tree.\n", treedepth); -#endif - for (i = 0; i < treedepth; i++) - { - size = ntohl (iblocks[i]->size) - sizeof (GNUNET_DatastoreValue); - GNUNET_GE_ASSERT (ectx, size < GNUNET_MAX_BUFFER_SIZE); - if (size == sizeof (GNUNET_EC_DBlock)) - { -#if DEBUG_UPLOAD - GNUNET_GE_LOG (ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER, - "Level %u is empty\n", i); -#endif - continue; - } - db = (GNUNET_EC_DBlock *) & iblocks[i][1]; - GNUNET_EC_file_block_get_key (db, size, &mchk.key); -#if DEBUG_UPLOAD - GNUNET_GE_LOG (ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER, - "Computing query for %u bytes content.\n", size); -#endif - GNUNET_EC_file_block_get_query (db, size, &mchk.query); -#if DEBUG_UPLOAD - IF_GELOG (ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER, - GNUNET_hash_to_enc (&mchk.query, &enc)); - GNUNET_GE_LOG (ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER, - "Query for current block at level %u is `%s'.\n", i, - &enc); -#endif - if (GNUNET_OK != pushBlock (sock, - &mchk, i + 1, iblocks, priority, - expirationTime)) - { - GNUNET_GE_BREAK (ectx, 0); - goto FAILURE; - } - GNUNET_EC_file_block_encode (db, size, &mchk.query, &value); - if (value == NULL) - { - GNUNET_GE_BREAK (ectx, 0); - goto FAILURE; - } - value->expiration_time = GNUNET_htonll (expirationTime); - value->priority = htonl (priority); - if ((doIndex != GNUNET_SYSERR) && - (GNUNET_SYSERR == GNUNET_FS_insert (sock, value))) - { - GNUNET_GE_BREAK (ectx, 0); - GNUNET_free (value); - goto FAILURE; - } - GNUNET_free (value); - GNUNET_free (iblocks[i]); - iblocks[i] = NULL; - } -#if DEBUG_UPLOAD - IF_GELOG (ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER, - GNUNET_hash_to_enc (&mchk.query, &enc)); - GNUNET_GE_LOG (ectx, GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER, - "Query for top block is %s\n", &enc); -#endif - /* build URI */ - fid.file_length = GNUNET_htonll (filesize); - db = (GNUNET_EC_DBlock *) & iblocks[treedepth][1]; - - fid.chk = *(GNUNET_EC_ContentHashKey *) & (db[1]); - *uri = GNUNET_malloc (sizeof (URI)); - (*uri)->type = chk; - (*uri)->data.fi = fid; - - /* free resources */ - GNUNET_free_non_null (iblocks[treedepth]); - GNUNET_free (iblocks); - GNUNET_free (dblock); - if (upcb != NULL) - upcb (filesize, filesize, eta, upcbClosure); - CLOSE (fd); - GNUNET_client_connection_destroy (sock); - return GNUNET_OK; -FAILURE: - for (i = 0; i <= treedepth; i++) - GNUNET_free_non_null (iblocks[i]); - GNUNET_free (iblocks); - GNUNET_free (dblock); - CLOSE (fd); - GNUNET_client_connection_destroy (sock); - return GNUNET_SYSERR; -} - -#endif - /* end of fs_publish.c */ diff --git a/src/include/gnunet_constants.h b/src/include/gnunet_constants.h index 2c917cc59..bb122eb64 100644 --- a/src/include/gnunet_constants.h +++ b/src/include/gnunet_constants.h @@ -55,6 +55,13 @@ extern "C" */ #define GNUNET_CONSTANTS_EXEC_WAIT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 100) +/** + * After how long do we consider a service irresponsive + * even if we assume that the service commonly does not + * respond instantly (DNS, Database, etc.). + */ +#define GNUNET_CONSTANTS_SERVICE_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 10) + #if 0 /* keep Emacsens' auto-indent happy */ { diff --git a/src/include/gnunet_container_lib.h b/src/include/gnunet_container_lib.h index 291b99fa2..e2a0787c8 100644 --- a/src/include/gnunet_container_lib.h +++ b/src/include/gnunet_container_lib.h @@ -305,8 +305,8 @@ int GNUNET_CONTAINER_meta_data_extract_from_file (struct enum GNUNET_CONTAINER_MetaDataSerializationOptions { - GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL = GNUNET_NO, - GNUNET_CONTAINER_META_DATA_SERIALIZE_PART = GNUNET_YES, + GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL = 0, + GNUNET_CONTAINER_META_DATA_SERIALIZE_PART = 1, GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS = 2 }; @@ -323,10 +323,11 @@ enum GNUNET_CONTAINER_MetaDataSerializationOptions * GNUNET_SYSERR on error (typically: not enough * space) */ -int GNUNET_CONTAINER_meta_data_serialize (const struct - GNUNET_CONTAINER_MetaData *md, - char *target, unsigned int size, - enum +ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct + GNUNET_CONTAINER_MetaData *md, + char *target, + size_t size, + enum GNUNET_CONTAINER_MetaDataSerializationOptions opt); @@ -337,12 +338,12 @@ int GNUNET_CONTAINER_meta_data_serialize (const struct * meta-data to match the size constraint, * possibly discarding some data? */ -unsigned int GNUNET_CONTAINER_meta_data_get_serialized_size (const struct - GNUNET_CONTAINER_MetaData - *md, - enum - GNUNET_CONTAINER_MetaDataSerializationOptions - opt); +ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size (const struct + GNUNET_CONTAINER_MetaData + *md, + enum + GNUNET_CONTAINER_MetaDataSerializationOptions + opt); /** * Deserialize meta-data. Initializes md. @@ -352,7 +353,7 @@ unsigned int GNUNET_CONTAINER_meta_data_get_serialized_size (const struct */ struct GNUNET_CONTAINER_MetaData *GNUNET_CONTAINER_meta_data_deserialize (const char *input, - unsigned int size); + size_t size); /** * Does the meta-data claim that this is a directory? diff --git a/src/include/gnunet_crypto_lib.h b/src/include/gnunet_crypto_lib.h index 361d244e2..168c79b2a 100644 --- a/src/include/gnunet_crypto_lib.h +++ b/src/include/gnunet_crypto_lib.h @@ -394,7 +394,7 @@ void GNUNET_CRYPTO_hash_xor (const GNUNET_HashCode * a, /** * Convert a hashcode into a key. */ -void GNUNET_CRYPTO_hash_to_AES_key (const GNUNET_HashCode * hc, +void GNUNET_CRYPTO_hash_to_aes_key (const GNUNET_HashCode * hc, struct GNUNET_CRYPTO_AesSessionKey *skey, struct GNUNET_CRYPTO_AesInitializationVector diff --git a/src/include/gnunet_datastore_service.h b/src/include/gnunet_datastore_service.h index 28419126f..3805dea97 100644 --- a/src/include/gnunet_datastore_service.h +++ b/src/include/gnunet_datastore_service.h @@ -41,6 +41,12 @@ extern "C" #endif #endif +#define GNUNET_DATASTORE_BLOCKTYPE_ANY 0 +#define GNUNET_DATASTORE_BLOCKTYPE_DBLOCK 1 +#define GNUNET_DATASTORE_BLOCKTYPE_IBLOCK 2 +#define GNUNET_DATASTORE_BLOCKTYPE_KBLOCK 3 +#define GNUNET_DATASTORE_BLOCKTYPE_SBLOCK 4 +#define GNUNET_DATASTORE_BLOCKTYPE_SKBLOCK 5 /** * Handle to the datastore service. diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h index 4664c4cee..3c71d40e7 100644 --- a/src/include/gnunet_fs_service.h +++ b/src/include/gnunet_fs_service.h @@ -2169,6 +2169,52 @@ GNUNET_FS_directory_list_contents (size_t size, void *dep_cls); +/** + * Opaque handle to a directory builder. + */ +struct GNUNET_FS_DirectoryBuilder; + +/** + * Create a directory builder. + * + * @param mdir metadata for the directory + */ +struct GNUNET_FS_DirectoryBuilder * +GNUNET_FS_directory_builder_create (const struct GNUNET_CONTAINER_MetaData *mdir); + + +/** + * Add an entry to a directory. + * + * @param bld directory to extend + * @param uri uri of the entry (must not be a KSK) + * @param md metadata of the entry + * @param data raw data of the entry, can be NULL, otherwise + * data must point to exactly the number of bytes specified + * by the uri + */ +void +GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *md, + const void *data); + + +/** + * Finish building the directory. Frees the + * builder context and returns the directory + * in-memory. + * + * @param bld directory to finish + * @param size set to the number of bytes needed + * @param data set to the encoded directory + */ +void +GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, + size_t *size, + void **data); + + #if 0 /* keep Emacsens' auto-indent happy */ { #endif diff --git a/src/util/container_meta_data.c b/src/util/container_meta_data.c index 6265fc142..d80c5fd2b 100644 --- a/src/util/container_meta_data.c +++ b/src/util/container_meta_data.c @@ -442,9 +442,9 @@ struct MetaDataHeader * GNUNET_SYSERR on error (typically: not enough * space) */ -int +ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData - *md, char *target, unsigned int max, + *md, char *target, size_t max, enum GNUNET_CONTAINER_MetaDataSerializationOptions part) @@ -463,7 +463,7 @@ GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData while (1) { size = sizeof (struct MetaDataHeader); - size += sizeof (unsigned int) * ic; + size += sizeof (uint32_t) * ic; for (i = 0; i < ic; i++) size += 1 + strlen (md->items[i].data); while (size % 8 != 0) @@ -472,8 +472,8 @@ GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData hdr->version = htonl (md == NULL ? 1 : 0); hdr->entries = htonl (ic); for (i = 0; i < ic; i++) - ((unsigned int *) &hdr[1])[i] = - htonl ((unsigned int) md->items[i].type); + ((uint32_t *) &hdr[1])[i] = + htonl ((uint32_t) md->items[i].type); pos = sizeof (struct MetaDataHeader); pos += sizeof (unsigned int) * ic; for (i = 0; i < ic; i++) @@ -533,7 +533,7 @@ GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData * serialized form. The estimate MAY be higher * than what is strictly needed. */ -unsigned int +ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size (const struct GNUNET_CONTAINER_MetaData *md, enum @@ -549,7 +549,7 @@ GNUNET_CONTAINER_meta_data_get_serialized_size (const struct ic = md ? md->itemCount : 0; size = sizeof (struct MetaDataHeader); - size += sizeof (unsigned int) * ic; + size += sizeof (uint32_t) * ic; for (i = 0; i < ic; i++) size += 1 + strlen (md->items[i].data); while (size % 8 != 0) @@ -558,9 +558,9 @@ GNUNET_CONTAINER_meta_data_get_serialized_size (const struct hdr->version = htonl (md == NULL ? 1 : 0); hdr->entries = htonl (ic); for (i = 0; i < ic; i++) - ((unsigned int *) &hdr[1])[i] = htonl ((unsigned int) md->items[i].type); + ((uint32_t *) &hdr[1])[i] = htonl ((uint32_t) md->items[i].type); pos = sizeof (struct MetaDataHeader); - pos += sizeof (unsigned int) * ic; + pos += sizeof (uint32_t) * ic; for (i = 0; i < ic; i++) { len = strlen (md->items[i].data) + 1; @@ -590,7 +590,7 @@ GNUNET_CONTAINER_meta_data_get_serialized_size (const struct * bad format) */ struct GNUNET_CONTAINER_MetaData * -GNUNET_CONTAINER_meta_data_deserialize (const char *input, unsigned int size) +GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size) { struct GNUNET_CONTAINER_MetaData *md; const struct MetaDataHeader *hdr; @@ -599,9 +599,9 @@ GNUNET_CONTAINER_meta_data_deserialize (const char *input, unsigned int size) const char *cdata; uint32_t dataSize; int compressed; - int i; - unsigned int pos; - int len; + uint32_t i; + size_t pos; + size_t len; uint32_t version; if (size < sizeof (struct MetaDataHeader)) @@ -651,7 +651,7 @@ GNUNET_CONTAINER_meta_data_deserialize (const char *input, unsigned int size) } } - if ((sizeof (unsigned int) * ic + ic) > dataSize) + if ((sizeof (uint32_t) * ic + ic) > dataSize) { GNUNET_break (0); goto FAILURE; @@ -665,12 +665,12 @@ GNUNET_CONTAINER_meta_data_deserialize (const char *input, unsigned int size) md = GNUNET_CONTAINER_meta_data_create (); GNUNET_array_grow (md->items, md->itemCount, ic); i = 0; - pos = sizeof (unsigned int) * ic; + pos = sizeof (uint32_t) * ic; while ((pos < dataSize) && (i < ic)) { len = strlen (&cdata[pos]) + 1; md->items[i].type = (EXTRACTOR_KeywordType) - ntohl (MAKE_UNALIGNED (((const unsigned int *) cdata)[i])); + ntohl (MAKE_UNALIGNED (((const uint32_t *) cdata)[i])); md->items[i].data = GNUNET_strdup (&cdata[pos]); pos += len; i++; diff --git a/src/util/crypto_hash.c b/src/util/crypto_hash.c index fc4d24320..a8843b5b7 100644 --- a/src/util/crypto_hash.c +++ b/src/util/crypto_hash.c @@ -709,7 +709,7 @@ GNUNET_CRYPTO_hash_xor (const GNUNET_HashCode * a, * Convert a hashcode into a key. */ void -GNUNET_CRYPTO_hash_to_AES_key (const GNUNET_HashCode * hc, +GNUNET_CRYPTO_hash_to_aes_key (const GNUNET_HashCode * hc, struct GNUNET_CRYPTO_AesSessionKey *skey, struct GNUNET_CRYPTO_AesInitializationVector *iv) -- cgit v1.2.3