aboutsummaryrefslogtreecommitdiff
path: root/src/fs/fs_sharetree.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/fs/fs_sharetree.c')
-rw-r--r--src/fs/fs_sharetree.c259
1 files changed, 125 insertions, 134 deletions
diff --git a/src/fs/fs_sharetree.c b/src/fs/fs_sharetree.c
index 282d7796a..a2cc37170 100644
--- a/src/fs/fs_sharetree.c
+++ b/src/fs/fs_sharetree.c
@@ -11,12 +11,12 @@
11 WITHOUT ANY WARRANTY; without even the implied warranty of 11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details. 13 Affero General Public License for more details.
14 14
15 You should have received a copy of the GNU Affero General Public License 15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>. 16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17 17
18 SPDX-License-Identifier: AGPL3.0-or-later 18 SPDX-License-Identifier: AGPL3.0-or-later
19*/ 19 */
20 20
21/** 21/**
22 * @file fs/fs_sharetree.c 22 * @file fs/fs_sharetree.c
@@ -34,9 +34,7 @@
34 * Entry for each unique keyword to track how often 34 * Entry for each unique keyword to track how often
35 * it occured. Contains the keyword and the counter. 35 * it occured. Contains the keyword and the counter.
36 */ 36 */
37struct KeywordCounter 37struct KeywordCounter {
38{
39
40 /** 38 /**
41 * This is a doubly-linked list 39 * This is a doubly-linked list
42 */ 40 */
@@ -56,16 +54,13 @@ struct KeywordCounter
56 * How many files have this keyword? 54 * How many files have this keyword?
57 */ 55 */
58 unsigned int count; 56 unsigned int count;
59
60}; 57};
61 58
62 59
63/** 60/**
64 * Aggregate information we keep for meta data in each directory. 61 * Aggregate information we keep for meta data in each directory.
65 */ 62 */
66struct MetaCounter 63struct MetaCounter {
67{
68
69 /** 64 /**
70 * This is a doubly-linked list 65 * This is a doubly-linked list
71 */ 66 */
@@ -111,7 +106,6 @@ struct MetaCounter
111 * (type and format do not have to match). 106 * (type and format do not have to match).
112 */ 107 */
113 unsigned int count; 108 unsigned int count;
114
115}; 109};
116 110
117 111
@@ -119,9 +113,7 @@ struct MetaCounter
119 * A structure that forms a singly-linked list that serves as a stack 113 * A structure that forms a singly-linked list that serves as a stack
120 * for metadata-processing function. 114 * for metadata-processing function.
121 */ 115 */
122struct TrimContext 116struct TrimContext {
123{
124
125 /** 117 /**
126 * Map from the hash over the keyword to an 'struct KeywordCounter *' 118 * Map from the hash over the keyword to an 'struct KeywordCounter *'
127 * counter that says how often this keyword was 119 * counter that says how often this keyword was
@@ -145,7 +137,6 @@ struct TrimContext
145 * Number of times an item has to be found to be moved to the parent. 137 * Number of times an item has to be found to be moved to the parent.
146 */ 138 */
147 unsigned int move_threshold; 139 unsigned int move_threshold;
148
149}; 140};
150 141
151 142
@@ -158,26 +149,26 @@ struct TrimContext
158 * @return always GNUNET_OK 149 * @return always GNUNET_OK
159 */ 150 */
160static int 151static int
161add_to_keyword_counter (void *cls, const char *keyword, int is_mandatory) 152add_to_keyword_counter(void *cls, const char *keyword, int is_mandatory)
162{ 153{
163 struct GNUNET_CONTAINER_MultiHashMap *mcm = cls; 154 struct GNUNET_CONTAINER_MultiHashMap *mcm = cls;
164 struct KeywordCounter *cnt; 155 struct KeywordCounter *cnt;
165 struct GNUNET_HashCode hc; 156 struct GNUNET_HashCode hc;
166 size_t klen; 157 size_t klen;
167 158
168 klen = strlen (keyword) + 1; 159 klen = strlen(keyword) + 1;
169 GNUNET_CRYPTO_hash (keyword, klen - 1, &hc); 160 GNUNET_CRYPTO_hash(keyword, klen - 1, &hc);
170 cnt = GNUNET_CONTAINER_multihashmap_get (mcm, &hc); 161 cnt = GNUNET_CONTAINER_multihashmap_get(mcm, &hc);
171 if (cnt == NULL) 162 if (cnt == NULL)
172 { 163 {
173 cnt = GNUNET_malloc (sizeof (struct KeywordCounter) + klen); 164 cnt = GNUNET_malloc(sizeof(struct KeywordCounter) + klen);
174 cnt->value = (const char *) &cnt[1]; 165 cnt->value = (const char *)&cnt[1];
175 GNUNET_memcpy (&cnt[1], keyword, klen); 166 GNUNET_memcpy(&cnt[1], keyword, klen);
176 GNUNET_assert (GNUNET_OK == 167 GNUNET_assert(GNUNET_OK ==
177 GNUNET_CONTAINER_multihashmap_put (mcm, 168 GNUNET_CONTAINER_multihashmap_put(mcm,
178 &hc, cnt, 169 &hc, cnt,
179 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); 170 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
180 } 171 }
181 cnt->count++; 172 cnt->count++;
182 return GNUNET_OK; 173 return GNUNET_OK;
183} 174}
@@ -201,30 +192,30 @@ add_to_keyword_counter (void *cls, const char *keyword, int is_mandatory)
201 * @return 0 to continue extracting / iterating 192 * @return 0 to continue extracting / iterating
202 */ 193 */
203static int 194static int
204add_to_meta_counter (void *cls, const char *plugin_name, 195add_to_meta_counter(void *cls, const char *plugin_name,
205 enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, 196 enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format,
206 const char *data_mime_type, const char *data, size_t data_len) 197 const char *data_mime_type, const char *data, size_t data_len)
207{ 198{
208 struct GNUNET_CONTAINER_MultiHashMap *map = cls; 199 struct GNUNET_CONTAINER_MultiHashMap *map = cls;
209 struct GNUNET_HashCode key; 200 struct GNUNET_HashCode key;
210 struct MetaCounter *cnt; 201 struct MetaCounter *cnt;
211 202
212 GNUNET_CRYPTO_hash (data, data_len, &key); 203 GNUNET_CRYPTO_hash(data, data_len, &key);
213 cnt = GNUNET_CONTAINER_multihashmap_get (map, &key); 204 cnt = GNUNET_CONTAINER_multihashmap_get(map, &key);
214 if (NULL == cnt) 205 if (NULL == cnt)
215 { 206 {
216 cnt = GNUNET_new (struct MetaCounter); 207 cnt = GNUNET_new(struct MetaCounter);
217 cnt->data = data; 208 cnt->data = data;
218 cnt->data_size = data_len; 209 cnt->data_size = data_len;
219 cnt->plugin_name = plugin_name; 210 cnt->plugin_name = plugin_name;
220 cnt->type = type; 211 cnt->type = type;
221 cnt->format = format; 212 cnt->format = format;
222 cnt->data_mime_type = data_mime_type; 213 cnt->data_mime_type = data_mime_type;
223 GNUNET_assert (GNUNET_OK == 214 GNUNET_assert(GNUNET_OK ==
224 GNUNET_CONTAINER_multihashmap_put (map, 215 GNUNET_CONTAINER_multihashmap_put(map,
225 &key, cnt, 216 &key, cnt,
226 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); 217 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
227 } 218 }
228 cnt->count++; 219 cnt->count++;
229 return 0; 220 return 0;
230} 221}
@@ -239,21 +230,21 @@ add_to_meta_counter (void *cls, const char *plugin_name,
239 * @return always GNUNET_OK 230 * @return always GNUNET_OK
240 */ 231 */
241static int 232static int
242remove_high_frequency_keywords (void *cls, const char *keyword, int is_mandatory) 233remove_high_frequency_keywords(void *cls, const char *keyword, int is_mandatory)
243{ 234{
244 struct TrimContext *tc = cls; 235 struct TrimContext *tc = cls;
245 struct KeywordCounter *counter; 236 struct KeywordCounter *counter;
246 struct GNUNET_HashCode hc; 237 struct GNUNET_HashCode hc;
247 size_t klen; 238 size_t klen;
248 239
249 klen = strlen (keyword) + 1; 240 klen = strlen(keyword) + 1;
250 GNUNET_CRYPTO_hash (keyword, klen - 1, &hc); 241 GNUNET_CRYPTO_hash(keyword, klen - 1, &hc);
251 counter = GNUNET_CONTAINER_multihashmap_get (tc->keywordcounter, &hc); 242 counter = GNUNET_CONTAINER_multihashmap_get(tc->keywordcounter, &hc);
252 GNUNET_assert (NULL != counter); 243 GNUNET_assert(NULL != counter);
253 if (counter->count < tc->move_threshold) 244 if (counter->count < tc->move_threshold)
254 return GNUNET_OK; 245 return GNUNET_OK;
255 GNUNET_FS_uri_ksk_remove_keyword (tc->pos->ksk_uri, 246 GNUNET_FS_uri_ksk_remove_keyword(tc->pos->ksk_uri,
256 counter->value); 247 counter->value);
257 return GNUNET_OK; 248 return GNUNET_OK;
258} 249}
259 250
@@ -268,23 +259,23 @@ remove_high_frequency_keywords (void *cls, const char *keyword, int is_mandatory
268 * @return GNUNET_YES (always) 259 * @return GNUNET_YES (always)
269 */ 260 */
270static int 261static int
271migrate_and_drop_keywords (void *cls, const struct GNUNET_HashCode * key, void *value) 262migrate_and_drop_keywords(void *cls, const struct GNUNET_HashCode * key, void *value)
272{ 263{
273 struct TrimContext *tc = cls; 264 struct TrimContext *tc = cls;
274 struct KeywordCounter *counter = value; 265 struct KeywordCounter *counter = value;
275 266
276 if (counter->count >= tc->move_threshold) 267 if (counter->count >= tc->move_threshold)
277 { 268 {
278 if (NULL == tc->pos->ksk_uri) 269 if (NULL == tc->pos->ksk_uri)
279 tc->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_args (1, &counter->value); 270 tc->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_args(1, &counter->value);
280 else 271 else
281 GNUNET_FS_uri_ksk_add_keyword (tc->pos->ksk_uri, counter->value, GNUNET_NO); 272 GNUNET_FS_uri_ksk_add_keyword(tc->pos->ksk_uri, counter->value, GNUNET_NO);
282 } 273 }
283 GNUNET_assert (GNUNET_YES == 274 GNUNET_assert(GNUNET_YES ==
284 GNUNET_CONTAINER_multihashmap_remove (tc->keywordcounter, 275 GNUNET_CONTAINER_multihashmap_remove(tc->keywordcounter,
285 key, 276 key,
286 counter)); 277 counter));
287 GNUNET_free (counter); 278 GNUNET_free(counter);
288 return GNUNET_YES; 279 return GNUNET_YES;
289} 280}
290 281
@@ -299,27 +290,27 @@ migrate_and_drop_keywords (void *cls, const struct GNUNET_HashCode * key, void *
299 * @return GNUNET_YES (always) 290 * @return GNUNET_YES (always)
300 */ 291 */
301static int 292static int
302migrate_and_drop_metadata (void *cls, const struct GNUNET_HashCode * key, void *value) 293migrate_and_drop_metadata(void *cls, const struct GNUNET_HashCode * key, void *value)
303{ 294{
304 struct TrimContext *tc = cls; 295 struct TrimContext *tc = cls;
305 struct MetaCounter *counter = value; 296 struct MetaCounter *counter = value;
306 297
307 if (counter->count >= tc->move_threshold) 298 if (counter->count >= tc->move_threshold)
308 { 299 {
309 if (NULL == tc->pos->meta) 300 if (NULL == tc->pos->meta)
310 tc->pos->meta = GNUNET_CONTAINER_meta_data_create (); 301 tc->pos->meta = GNUNET_CONTAINER_meta_data_create();
311 GNUNET_CONTAINER_meta_data_insert (tc->pos->meta, 302 GNUNET_CONTAINER_meta_data_insert(tc->pos->meta,
312 counter->plugin_name, 303 counter->plugin_name,
313 counter->type, 304 counter->type,
314 counter->format, 305 counter->format,
315 counter->data_mime_type, counter->data, 306 counter->data_mime_type, counter->data,
316 counter->data_size); 307 counter->data_size);
317 } 308 }
318 GNUNET_assert (GNUNET_YES == 309 GNUNET_assert(GNUNET_YES ==
319 GNUNET_CONTAINER_multihashmap_remove (tc->metacounter, 310 GNUNET_CONTAINER_multihashmap_remove(tc->metacounter,
320 key, 311 key,
321 counter)); 312 counter));
322 GNUNET_free (counter); 313 GNUNET_free(counter);
323 return GNUNET_YES; 314 return GNUNET_YES;
324} 315}
325 316
@@ -332,8 +323,8 @@ migrate_and_drop_metadata (void *cls, const struct GNUNET_HashCode * key, void *
332 * @param tree tree to trim 323 * @param tree tree to trim
333 */ 324 */
334static void 325static void
335share_tree_trim (struct TrimContext *tc, 326share_tree_trim(struct TrimContext *tc,
336 struct GNUNET_FS_ShareTreeItem *tree) 327 struct GNUNET_FS_ShareTreeItem *tree)
337{ 328{
338 struct GNUNET_FS_ShareTreeItem *pos; 329 struct GNUNET_FS_ShareTreeItem *pos;
339 unsigned int num_children; 330 unsigned int num_children;
@@ -341,64 +332,64 @@ share_tree_trim (struct TrimContext *tc,
341 /* first, trim all children */ 332 /* first, trim all children */
342 num_children = 0; 333 num_children = 0;
343 for (pos = tree->children_head; NULL != pos; pos = pos->next) 334 for (pos = tree->children_head; NULL != pos; pos = pos->next)
344 { 335 {
345 share_tree_trim (tc, pos); 336 share_tree_trim(tc, pos);
346 num_children++; 337 num_children++;
347 } 338 }
348 339
349 /* consider adding filename to directory meta data */ 340 /* consider adding filename to directory meta data */
350 if (tree->is_directory == GNUNET_YES) 341 if (tree->is_directory == GNUNET_YES)
351 {
352 const char *user = getenv ("USER");
353 if ( (user == NULL) ||
354 (0 != strncasecmp (user, tree->short_filename, strlen(user))))
355 { 342 {
356 /* only use filename if it doesn't match $USER */ 343 const char *user = getenv("USER");
357 if (NULL == tree->meta) 344 if ((user == NULL) ||
358 tree->meta = GNUNET_CONTAINER_meta_data_create (); 345 (0 != strncasecmp(user, tree->short_filename, strlen(user))))
359 GNUNET_CONTAINER_meta_data_insert (tree->meta, "<libgnunetfs>", 346 {
360 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, 347 /* only use filename if it doesn't match $USER */
361 EXTRACTOR_METAFORMAT_UTF8, 348 if (NULL == tree->meta)
362 "text/plain", tree->short_filename, 349 tree->meta = GNUNET_CONTAINER_meta_data_create();
363 strlen (tree->short_filename) + 1); 350 GNUNET_CONTAINER_meta_data_insert(tree->meta, "<libgnunetfs>",
351 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
352 EXTRACTOR_METAFORMAT_UTF8,
353 "text/plain", tree->short_filename,
354 strlen(tree->short_filename) + 1);
355 }
364 } 356 }
365 }
366 357
367 if (1 >= num_children) 358 if (1 >= num_children)
368 return; /* nothing to trim */ 359 return; /* nothing to trim */
369 360
370 /* now, count keywords and meta data in children */ 361 /* now, count keywords and meta data in children */
371 for (pos = tree->children_head; NULL != pos; pos = pos->next) 362 for (pos = tree->children_head; NULL != pos; pos = pos->next)
372 { 363 {
373 if (NULL != pos->meta) 364 if (NULL != pos->meta)
374 GNUNET_CONTAINER_meta_data_iterate (pos->meta, &add_to_meta_counter, tc->metacounter); 365 GNUNET_CONTAINER_meta_data_iterate(pos->meta, &add_to_meta_counter, tc->metacounter);
375 if (NULL != pos->ksk_uri) 366 if (NULL != pos->ksk_uri)
376 GNUNET_FS_uri_ksk_get_keywords (pos->ksk_uri, &add_to_keyword_counter, tc->keywordcounter); 367 GNUNET_FS_uri_ksk_get_keywords(pos->ksk_uri, &add_to_keyword_counter, tc->keywordcounter);
377 } 368 }
378 369
379 /* calculate threshold for moving keywords / meta data */ 370 /* calculate threshold for moving keywords / meta data */
380 tc->move_threshold = 1 + (num_children / 2); 371 tc->move_threshold = 1 + (num_children / 2);
381 372
382 /* remove high-frequency keywords from children */ 373 /* remove high-frequency keywords from children */
383 for (pos = tree->children_head; NULL != pos; pos = pos->next) 374 for (pos = tree->children_head; NULL != pos; pos = pos->next)
384 {
385 tc->pos = pos;
386 if (NULL != pos->ksk_uri)
387 { 375 {
388 struct GNUNET_FS_Uri *ksk_uri_copy = GNUNET_FS_uri_dup (pos->ksk_uri); 376 tc->pos = pos;
389 GNUNET_FS_uri_ksk_get_keywords (ksk_uri_copy, &remove_high_frequency_keywords, tc); 377 if (NULL != pos->ksk_uri)
390 GNUNET_FS_uri_destroy (ksk_uri_copy); 378 {
379 struct GNUNET_FS_Uri *ksk_uri_copy = GNUNET_FS_uri_dup(pos->ksk_uri);
380 GNUNET_FS_uri_ksk_get_keywords(ksk_uri_copy, &remove_high_frequency_keywords, tc);
381 GNUNET_FS_uri_destroy(ksk_uri_copy);
382 }
391 } 383 }
392 }
393 384
394 /* add high-frequency meta data and keywords to parent */ 385 /* add high-frequency meta data and keywords to parent */
395 tc->pos = tree; 386 tc->pos = tree;
396 GNUNET_CONTAINER_multihashmap_iterate (tc->keywordcounter, 387 GNUNET_CONTAINER_multihashmap_iterate(tc->keywordcounter,
397 &migrate_and_drop_keywords, 388 &migrate_and_drop_keywords,
398 tc); 389 tc);
399 GNUNET_CONTAINER_multihashmap_iterate (tc->metacounter, 390 GNUNET_CONTAINER_multihashmap_iterate(tc->metacounter,
400 &migrate_and_drop_metadata, 391 &migrate_and_drop_metadata,
401 tc); 392 tc);
402} 393}
403 394
404 395
@@ -409,17 +400,17 @@ share_tree_trim (struct TrimContext *tc,
409 * @param toplevel toplevel directory in the tree, returned by the scanner 400 * @param toplevel toplevel directory in the tree, returned by the scanner
410 */ 401 */
411void 402void
412GNUNET_FS_share_tree_trim (struct GNUNET_FS_ShareTreeItem *toplevel) 403GNUNET_FS_share_tree_trim(struct GNUNET_FS_ShareTreeItem *toplevel)
413{ 404{
414 struct TrimContext tc; 405 struct TrimContext tc;
415 406
416 if (toplevel == NULL) 407 if (toplevel == NULL)
417 return; 408 return;
418 tc.keywordcounter = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO); 409 tc.keywordcounter = GNUNET_CONTAINER_multihashmap_create(1024, GNUNET_NO);
419 tc.metacounter = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO); 410 tc.metacounter = GNUNET_CONTAINER_multihashmap_create(1024, GNUNET_NO);
420 share_tree_trim (&tc, toplevel); 411 share_tree_trim(&tc, toplevel);
421 GNUNET_CONTAINER_multihashmap_destroy (tc.keywordcounter); 412 GNUNET_CONTAINER_multihashmap_destroy(tc.keywordcounter);
422 GNUNET_CONTAINER_multihashmap_destroy (tc.metacounter); 413 GNUNET_CONTAINER_multihashmap_destroy(tc.metacounter);
423} 414}
424 415
425 416
@@ -429,23 +420,23 @@ GNUNET_FS_share_tree_trim (struct GNUNET_FS_ShareTreeItem *toplevel)
429 * @param toplevel toplevel of the tree to be freed 420 * @param toplevel toplevel of the tree to be freed
430 */ 421 */
431void 422void
432GNUNET_FS_share_tree_free (struct GNUNET_FS_ShareTreeItem *toplevel) 423GNUNET_FS_share_tree_free(struct GNUNET_FS_ShareTreeItem *toplevel)
433{ 424{
434 struct GNUNET_FS_ShareTreeItem *pos; 425 struct GNUNET_FS_ShareTreeItem *pos;
435 426
436 while (NULL != (pos = toplevel->children_head)) 427 while (NULL != (pos = toplevel->children_head))
437 GNUNET_FS_share_tree_free (pos); 428 GNUNET_FS_share_tree_free(pos);
438 if (NULL != toplevel->parent) 429 if (NULL != toplevel->parent)
439 GNUNET_CONTAINER_DLL_remove (toplevel->parent->children_head, 430 GNUNET_CONTAINER_DLL_remove(toplevel->parent->children_head,
440 toplevel->parent->children_tail, 431 toplevel->parent->children_tail,
441 toplevel); 432 toplevel);
442 if (NULL != toplevel->meta) 433 if (NULL != toplevel->meta)
443 GNUNET_CONTAINER_meta_data_destroy (toplevel->meta); 434 GNUNET_CONTAINER_meta_data_destroy(toplevel->meta);
444 if (NULL != toplevel->ksk_uri) 435 if (NULL != toplevel->ksk_uri)
445 GNUNET_FS_uri_destroy (toplevel->ksk_uri); 436 GNUNET_FS_uri_destroy(toplevel->ksk_uri);
446 GNUNET_free_non_null (toplevel->filename); 437 GNUNET_free_non_null(toplevel->filename);
447 GNUNET_free_non_null (toplevel->short_filename); 438 GNUNET_free_non_null(toplevel->short_filename);
448 GNUNET_free (toplevel); 439 GNUNET_free(toplevel);
449} 440}
450 441
451/* end fs_sharetree.c */ 442/* end fs_sharetree.c */