summaryrefslogtreecommitdiff
path: root/src/fs/fs_sharetree.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/fs/fs_sharetree.c')
-rw-r--r--src/fs/fs_sharetree.c252
1 files changed, 132 insertions, 120 deletions
diff --git a/src/fs/fs_sharetree.c b/src/fs/fs_sharetree.c
index a2cc37170..dab5d00a0 100644
--- a/src/fs/fs_sharetree.c
+++ b/src/fs/fs_sharetree.c
@@ -34,7 +34,8 @@
34 * Entry for each unique keyword to track how often 34 * Entry for each unique keyword to track how often
35 * it occured. Contains the keyword and the counter. 35 * it occured. Contains the keyword and the counter.
36 */ 36 */
37struct KeywordCounter { 37struct KeywordCounter
38{
38 /** 39 /**
39 * This is a doubly-linked list 40 * This is a doubly-linked list
40 */ 41 */
@@ -60,7 +61,8 @@ struct KeywordCounter {
60/** 61/**
61 * Aggregate information we keep for meta data in each directory. 62 * Aggregate information we keep for meta data in each directory.
62 */ 63 */
63struct MetaCounter { 64struct MetaCounter
65{
64 /** 66 /**
65 * This is a doubly-linked list 67 * This is a doubly-linked list
66 */ 68 */
@@ -113,7 +115,8 @@ struct MetaCounter {
113 * A structure that forms a singly-linked list that serves as a stack 115 * A structure that forms a singly-linked list that serves as a stack
114 * for metadata-processing function. 116 * for metadata-processing function.
115 */ 117 */
116struct TrimContext { 118struct TrimContext
119{
117 /** 120 /**
118 * Map from the hash over the keyword to an 'struct KeywordCounter *' 121 * Map from the hash over the keyword to an 'struct KeywordCounter *'
119 * counter that says how often this keyword was 122 * counter that says how often this keyword was
@@ -149,26 +152,26 @@ struct TrimContext {
149 * @return always GNUNET_OK 152 * @return always GNUNET_OK
150 */ 153 */
151static int 154static int
152add_to_keyword_counter(void *cls, const char *keyword, int is_mandatory) 155add_to_keyword_counter (void *cls, const char *keyword, int is_mandatory)
153{ 156{
154 struct GNUNET_CONTAINER_MultiHashMap *mcm = cls; 157 struct GNUNET_CONTAINER_MultiHashMap *mcm = cls;
155 struct KeywordCounter *cnt; 158 struct KeywordCounter *cnt;
156 struct GNUNET_HashCode hc; 159 struct GNUNET_HashCode hc;
157 size_t klen; 160 size_t klen;
158 161
159 klen = strlen(keyword) + 1; 162 klen = strlen (keyword) + 1;
160 GNUNET_CRYPTO_hash(keyword, klen - 1, &hc); 163 GNUNET_CRYPTO_hash (keyword, klen - 1, &hc);
161 cnt = GNUNET_CONTAINER_multihashmap_get(mcm, &hc); 164 cnt = GNUNET_CONTAINER_multihashmap_get (mcm, &hc);
162 if (cnt == NULL) 165 if (cnt == NULL)
163 { 166 {
164 cnt = GNUNET_malloc(sizeof(struct KeywordCounter) + klen); 167 cnt = GNUNET_malloc (sizeof(struct KeywordCounter) + klen);
165 cnt->value = (const char *)&cnt[1]; 168 cnt->value = (const char *) &cnt[1];
166 GNUNET_memcpy(&cnt[1], keyword, klen); 169 GNUNET_memcpy (&cnt[1], keyword, klen);
167 GNUNET_assert(GNUNET_OK == 170 GNUNET_assert (GNUNET_OK ==
168 GNUNET_CONTAINER_multihashmap_put(mcm, 171 GNUNET_CONTAINER_multihashmap_put (mcm,
169 &hc, cnt, 172 &hc, cnt,
170 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); 173 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
171 } 174 }
172 cnt->count++; 175 cnt->count++;
173 return GNUNET_OK; 176 return GNUNET_OK;
174} 177}
@@ -192,30 +195,32 @@ add_to_keyword_counter(void *cls, const char *keyword, int is_mandatory)
192 * @return 0 to continue extracting / iterating 195 * @return 0 to continue extracting / iterating
193 */ 196 */
194static int 197static int
195add_to_meta_counter(void *cls, const char *plugin_name, 198add_to_meta_counter (void *cls, const char *plugin_name,
196 enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, 199 enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat
197 const char *data_mime_type, const char *data, size_t data_len) 200 format,
201 const char *data_mime_type, const char *data, size_t
202 data_len)
198{ 203{
199 struct GNUNET_CONTAINER_MultiHashMap *map = cls; 204 struct GNUNET_CONTAINER_MultiHashMap *map = cls;
200 struct GNUNET_HashCode key; 205 struct GNUNET_HashCode key;
201 struct MetaCounter *cnt; 206 struct MetaCounter *cnt;
202 207
203 GNUNET_CRYPTO_hash(data, data_len, &key); 208 GNUNET_CRYPTO_hash (data, data_len, &key);
204 cnt = GNUNET_CONTAINER_multihashmap_get(map, &key); 209 cnt = GNUNET_CONTAINER_multihashmap_get (map, &key);
205 if (NULL == cnt) 210 if (NULL == cnt)
206 { 211 {
207 cnt = GNUNET_new(struct MetaCounter); 212 cnt = GNUNET_new (struct MetaCounter);
208 cnt->data = data; 213 cnt->data = data;
209 cnt->data_size = data_len; 214 cnt->data_size = data_len;
210 cnt->plugin_name = plugin_name; 215 cnt->plugin_name = plugin_name;
211 cnt->type = type; 216 cnt->type = type;
212 cnt->format = format; 217 cnt->format = format;
213 cnt->data_mime_type = data_mime_type; 218 cnt->data_mime_type = data_mime_type;
214 GNUNET_assert(GNUNET_OK == 219 GNUNET_assert (GNUNET_OK ==
215 GNUNET_CONTAINER_multihashmap_put(map, 220 GNUNET_CONTAINER_multihashmap_put (map,
216 &key, cnt, 221 &key, cnt,
217 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); 222 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
218 } 223 }
219 cnt->count++; 224 cnt->count++;
220 return 0; 225 return 0;
221} 226}
@@ -230,21 +235,22 @@ add_to_meta_counter(void *cls, const char *plugin_name,
230 * @return always GNUNET_OK 235 * @return always GNUNET_OK
231 */ 236 */
232static int 237static int
233remove_high_frequency_keywords(void *cls, const char *keyword, int is_mandatory) 238remove_high_frequency_keywords (void *cls, const char *keyword, int
239 is_mandatory)
234{ 240{
235 struct TrimContext *tc = cls; 241 struct TrimContext *tc = cls;
236 struct KeywordCounter *counter; 242 struct KeywordCounter *counter;
237 struct GNUNET_HashCode hc; 243 struct GNUNET_HashCode hc;
238 size_t klen; 244 size_t klen;
239 245
240 klen = strlen(keyword) + 1; 246 klen = strlen (keyword) + 1;
241 GNUNET_CRYPTO_hash(keyword, klen - 1, &hc); 247 GNUNET_CRYPTO_hash (keyword, klen - 1, &hc);
242 counter = GNUNET_CONTAINER_multihashmap_get(tc->keywordcounter, &hc); 248 counter = GNUNET_CONTAINER_multihashmap_get (tc->keywordcounter, &hc);
243 GNUNET_assert(NULL != counter); 249 GNUNET_assert (NULL != counter);
244 if (counter->count < tc->move_threshold) 250 if (counter->count < tc->move_threshold)
245 return GNUNET_OK; 251 return GNUNET_OK;
246 GNUNET_FS_uri_ksk_remove_keyword(tc->pos->ksk_uri, 252 GNUNET_FS_uri_ksk_remove_keyword (tc->pos->ksk_uri,
247 counter->value); 253 counter->value);
248 return GNUNET_OK; 254 return GNUNET_OK;
249} 255}
250 256
@@ -259,23 +265,26 @@ remove_high_frequency_keywords(void *cls, const char *keyword, int is_mandatory)
259 * @return GNUNET_YES (always) 265 * @return GNUNET_YES (always)
260 */ 266 */
261static int 267static int
262migrate_and_drop_keywords(void *cls, const struct GNUNET_HashCode * key, void *value) 268migrate_and_drop_keywords (void *cls, const struct GNUNET_HashCode *key,
269 void *value)
263{ 270{
264 struct TrimContext *tc = cls; 271 struct TrimContext *tc = cls;
265 struct KeywordCounter *counter = value; 272 struct KeywordCounter *counter = value;
266 273
267 if (counter->count >= tc->move_threshold) 274 if (counter->count >= tc->move_threshold)
268 { 275 {
269 if (NULL == tc->pos->ksk_uri) 276 if (NULL == tc->pos->ksk_uri)
270 tc->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_args(1, &counter->value); 277 tc->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_args (1,
271 else 278 &counter->value);
272 GNUNET_FS_uri_ksk_add_keyword(tc->pos->ksk_uri, counter->value, GNUNET_NO); 279 else
273 } 280 GNUNET_FS_uri_ksk_add_keyword (tc->pos->ksk_uri, counter->value,
274 GNUNET_assert(GNUNET_YES == 281 GNUNET_NO);
275 GNUNET_CONTAINER_multihashmap_remove(tc->keywordcounter, 282 }
276 key, 283 GNUNET_assert (GNUNET_YES ==
277 counter)); 284 GNUNET_CONTAINER_multihashmap_remove (tc->keywordcounter,
278 GNUNET_free(counter); 285 key,
286 counter));
287 GNUNET_free (counter);
279 return GNUNET_YES; 288 return GNUNET_YES;
280} 289}
281 290
@@ -290,27 +299,28 @@ migrate_and_drop_keywords(void *cls, const struct GNUNET_HashCode * key, void *v
290 * @return GNUNET_YES (always) 299 * @return GNUNET_YES (always)
291 */ 300 */
292static int 301static int
293migrate_and_drop_metadata(void *cls, const struct GNUNET_HashCode * key, void *value) 302migrate_and_drop_metadata (void *cls, const struct GNUNET_HashCode *key,
303 void *value)
294{ 304{
295 struct TrimContext *tc = cls; 305 struct TrimContext *tc = cls;
296 struct MetaCounter *counter = value; 306 struct MetaCounter *counter = value;
297 307
298 if (counter->count >= tc->move_threshold) 308 if (counter->count >= tc->move_threshold)
299 { 309 {
300 if (NULL == tc->pos->meta) 310 if (NULL == tc->pos->meta)
301 tc->pos->meta = GNUNET_CONTAINER_meta_data_create(); 311 tc->pos->meta = GNUNET_CONTAINER_meta_data_create ();
302 GNUNET_CONTAINER_meta_data_insert(tc->pos->meta, 312 GNUNET_CONTAINER_meta_data_insert (tc->pos->meta,
303 counter->plugin_name, 313 counter->plugin_name,
304 counter->type, 314 counter->type,
305 counter->format, 315 counter->format,
306 counter->data_mime_type, counter->data, 316 counter->data_mime_type, counter->data,
307 counter->data_size); 317 counter->data_size);
308 } 318 }
309 GNUNET_assert(GNUNET_YES == 319 GNUNET_assert (GNUNET_YES ==
310 GNUNET_CONTAINER_multihashmap_remove(tc->metacounter, 320 GNUNET_CONTAINER_multihashmap_remove (tc->metacounter,
311 key, 321 key,
312 counter)); 322 counter));
313 GNUNET_free(counter); 323 GNUNET_free (counter);
314 return GNUNET_YES; 324 return GNUNET_YES;
315} 325}
316 326
@@ -323,8 +333,8 @@ migrate_and_drop_metadata(void *cls, const struct GNUNET_HashCode * key, void *v
323 * @param tree tree to trim 333 * @param tree tree to trim
324 */ 334 */
325static void 335static void
326share_tree_trim(struct TrimContext *tc, 336share_tree_trim (struct TrimContext *tc,
327 struct GNUNET_FS_ShareTreeItem *tree) 337 struct GNUNET_FS_ShareTreeItem *tree)
328{ 338{
329 struct GNUNET_FS_ShareTreeItem *pos; 339 struct GNUNET_FS_ShareTreeItem *pos;
330 unsigned int num_children; 340 unsigned int num_children;
@@ -332,64 +342,67 @@ share_tree_trim(struct TrimContext *tc,
332 /* first, trim all children */ 342 /* first, trim all children */
333 num_children = 0; 343 num_children = 0;
334 for (pos = tree->children_head; NULL != pos; pos = pos->next) 344 for (pos = tree->children_head; NULL != pos; pos = pos->next)
335 { 345 {
336 share_tree_trim(tc, pos); 346 share_tree_trim (tc, pos);
337 num_children++; 347 num_children++;
338 } 348 }
339 349
340 /* consider adding filename to directory meta data */ 350 /* consider adding filename to directory meta data */
341 if (tree->is_directory == GNUNET_YES) 351 if (tree->is_directory == GNUNET_YES)
352 {
353 const char *user = getenv ("USER");
354 if ((user == NULL) ||
355 (0 != strncasecmp (user, tree->short_filename, strlen (user))))
342 { 356 {
343 const char *user = getenv("USER"); 357 /* only use filename if it doesn't match $USER */
344 if ((user == NULL) || 358 if (NULL == tree->meta)
345 (0 != strncasecmp(user, tree->short_filename, strlen(user)))) 359 tree->meta = GNUNET_CONTAINER_meta_data_create ();
346 { 360 GNUNET_CONTAINER_meta_data_insert (tree->meta, "<libgnunetfs>",
347 /* only use filename if it doesn't match $USER */ 361 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
348 if (NULL == tree->meta) 362 EXTRACTOR_METAFORMAT_UTF8,
349 tree->meta = GNUNET_CONTAINER_meta_data_create(); 363 "text/plain", tree->short_filename,
350 GNUNET_CONTAINER_meta_data_insert(tree->meta, "<libgnunetfs>", 364 strlen (tree->short_filename) + 1);
351 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
352 EXTRACTOR_METAFORMAT_UTF8,
353 "text/plain", tree->short_filename,
354 strlen(tree->short_filename) + 1);
355 }
356 } 365 }
366 }
357 367
358 if (1 >= num_children) 368 if (1 >= num_children)
359 return; /* nothing to trim */ 369 return; /* nothing to trim */
360 370
361 /* now, count keywords and meta data in children */ 371 /* now, count keywords and meta data in children */
362 for (pos = tree->children_head; NULL != pos; pos = pos->next) 372 for (pos = tree->children_head; NULL != pos; pos = pos->next)
363 { 373 {
364 if (NULL != pos->meta) 374 if (NULL != pos->meta)
365 GNUNET_CONTAINER_meta_data_iterate(pos->meta, &add_to_meta_counter, tc->metacounter); 375 GNUNET_CONTAINER_meta_data_iterate (pos->meta, &add_to_meta_counter,
366 if (NULL != pos->ksk_uri) 376 tc->metacounter);
367 GNUNET_FS_uri_ksk_get_keywords(pos->ksk_uri, &add_to_keyword_counter, tc->keywordcounter); 377 if (NULL != pos->ksk_uri)
368 } 378 GNUNET_FS_uri_ksk_get_keywords (pos->ksk_uri, &add_to_keyword_counter,
379 tc->keywordcounter);
380 }
369 381
370 /* calculate threshold for moving keywords / meta data */ 382 /* calculate threshold for moving keywords / meta data */
371 tc->move_threshold = 1 + (num_children / 2); 383 tc->move_threshold = 1 + (num_children / 2);
372 384
373 /* remove high-frequency keywords from children */ 385 /* remove high-frequency keywords from children */
374 for (pos = tree->children_head; NULL != pos; pos = pos->next) 386 for (pos = tree->children_head; NULL != pos; pos = pos->next)
387 {
388 tc->pos = pos;
389 if (NULL != pos->ksk_uri)
375 { 390 {
376 tc->pos = pos; 391 struct GNUNET_FS_Uri *ksk_uri_copy = GNUNET_FS_uri_dup (pos->ksk_uri);
377 if (NULL != pos->ksk_uri) 392 GNUNET_FS_uri_ksk_get_keywords (ksk_uri_copy,
378 { 393 &remove_high_frequency_keywords, tc);
379 struct GNUNET_FS_Uri *ksk_uri_copy = GNUNET_FS_uri_dup(pos->ksk_uri); 394 GNUNET_FS_uri_destroy (ksk_uri_copy);
380 GNUNET_FS_uri_ksk_get_keywords(ksk_uri_copy, &remove_high_frequency_keywords, tc);
381 GNUNET_FS_uri_destroy(ksk_uri_copy);
382 }
383 } 395 }
396 }
384 397
385 /* add high-frequency meta data and keywords to parent */ 398 /* add high-frequency meta data and keywords to parent */
386 tc->pos = tree; 399 tc->pos = tree;
387 GNUNET_CONTAINER_multihashmap_iterate(tc->keywordcounter, 400 GNUNET_CONTAINER_multihashmap_iterate (tc->keywordcounter,
388 &migrate_and_drop_keywords, 401 &migrate_and_drop_keywords,
389 tc); 402 tc);
390 GNUNET_CONTAINER_multihashmap_iterate(tc->metacounter, 403 GNUNET_CONTAINER_multihashmap_iterate (tc->metacounter,
391 &migrate_and_drop_metadata, 404 &migrate_and_drop_metadata,
392 tc); 405 tc);
393} 406}
394 407
395 408
@@ -400,17 +413,17 @@ share_tree_trim(struct TrimContext *tc,
400 * @param toplevel toplevel directory in the tree, returned by the scanner 413 * @param toplevel toplevel directory in the tree, returned by the scanner
401 */ 414 */
402void 415void
403GNUNET_FS_share_tree_trim(struct GNUNET_FS_ShareTreeItem *toplevel) 416GNUNET_FS_share_tree_trim (struct GNUNET_FS_ShareTreeItem *toplevel)
404{ 417{
405 struct TrimContext tc; 418 struct TrimContext tc;
406 419
407 if (toplevel == NULL) 420 if (toplevel == NULL)
408 return; 421 return;
409 tc.keywordcounter = GNUNET_CONTAINER_multihashmap_create(1024, GNUNET_NO); 422 tc.keywordcounter = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO);
410 tc.metacounter = GNUNET_CONTAINER_multihashmap_create(1024, GNUNET_NO); 423 tc.metacounter = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO);
411 share_tree_trim(&tc, toplevel); 424 share_tree_trim (&tc, toplevel);
412 GNUNET_CONTAINER_multihashmap_destroy(tc.keywordcounter); 425 GNUNET_CONTAINER_multihashmap_destroy (tc.keywordcounter);
413 GNUNET_CONTAINER_multihashmap_destroy(tc.metacounter); 426 GNUNET_CONTAINER_multihashmap_destroy (tc.metacounter);
414} 427}
415 428
416 429
@@ -420,24 +433,23 @@ GNUNET_FS_share_tree_trim(struct GNUNET_FS_ShareTreeItem *toplevel)
420 * @param toplevel toplevel of the tree to be freed 433 * @param toplevel toplevel of the tree to be freed
421 */ 434 */
422void 435void
423GNUNET_FS_share_tree_free(struct GNUNET_FS_ShareTreeItem *toplevel) 436GNUNET_FS_share_tree_free (struct GNUNET_FS_ShareTreeItem *toplevel)
424{ 437{
425 struct GNUNET_FS_ShareTreeItem *pos; 438 struct GNUNET_FS_ShareTreeItem *pos;
426 439
427 while (NULL != (pos = toplevel->children_head)) 440 while (NULL != (pos = toplevel->children_head))
428 GNUNET_FS_share_tree_free(pos); 441 GNUNET_FS_share_tree_free (pos);
429 if (NULL != toplevel->parent) 442 if (NULL != toplevel->parent)
430 GNUNET_CONTAINER_DLL_remove(toplevel->parent->children_head, 443 GNUNET_CONTAINER_DLL_remove (toplevel->parent->children_head,
431 toplevel->parent->children_tail, 444 toplevel->parent->children_tail,
432 toplevel); 445 toplevel);
433 if (NULL != toplevel->meta) 446 if (NULL != toplevel->meta)
434 GNUNET_CONTAINER_meta_data_destroy(toplevel->meta); 447 GNUNET_CONTAINER_meta_data_destroy (toplevel->meta);
435 if (NULL != toplevel->ksk_uri) 448 if (NULL != toplevel->ksk_uri)
436 GNUNET_FS_uri_destroy(toplevel->ksk_uri); 449 GNUNET_FS_uri_destroy (toplevel->ksk_uri);
437 GNUNET_free_non_null(toplevel->filename); 450 GNUNET_free_non_null (toplevel->filename);
438 GNUNET_free_non_null(toplevel->short_filename); 451 GNUNET_free_non_null (toplevel->short_filename);
439 GNUNET_free(toplevel); 452 GNUNET_free (toplevel);
440} 453}
441 454
442/* end fs_sharetree.c */ 455/* end fs_sharetree.c */
443