aboutsummaryrefslogtreecommitdiff
path: root/src/fs/fs_dirmetascan.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/fs/fs_dirmetascan.c')
-rw-r--r--src/fs/fs_dirmetascan.c503
1 files changed, 0 insertions, 503 deletions
diff --git a/src/fs/fs_dirmetascan.c b/src/fs/fs_dirmetascan.c
deleted file mode 100644
index cb50182f9..000000000
--- a/src/fs/fs_dirmetascan.c
+++ /dev/null
@@ -1,503 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2005-2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file fs/fs_dirmetascan.c
23 * @brief code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem'
24 * from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'.
25 * @author LRN
26 * @author Christian Grothoff
27 */
28#include "platform.h"
29#include "gnunet_fs_service.h"
30#include "gnunet_scheduler_lib.h"
31#include <pthread.h>
32
33
34/**
35 * An opaque structure a pointer to which is returned to the
36 * caller to be used to control the scanner.
37 */
38struct GNUNET_FS_DirScanner
39{
40 /**
41 * Helper process.
42 */
43 struct GNUNET_HELPER_Handle *helper;
44
45 /**
46 * Expanded filename (as given by the scan initiator).
47 * The scanner thread stores a copy here, and frees it when it finishes.
48 */
49 char *filename_expanded;
50
51 /**
52 * Second argument to helper process.
53 */
54 char *ex_arg;
55
56 /**
57 * The function that will be called every time there's a progress
58 * message.
59 */
60 GNUNET_FS_DirScannerProgressCallback progress_callback;
61
62 /**
63 * A closure for progress_callback.
64 */
65 void *progress_callback_cls;
66
67 /**
68 * After the scan is finished, it will contain a pointer to the
69 * top-level directory entry in the directory tree built by the
70 * scanner.
71 */
72 struct GNUNET_FS_ShareTreeItem *toplevel;
73
74 /**
75 * Current position during processing.
76 */
77 struct GNUNET_FS_ShareTreeItem *pos;
78
79 /**
80 * Task scheduled when we are done.
81 */
82 struct GNUNET_SCHEDULER_Task *stop_task;
83
84 /**
85 * Arguments for helper.
86 */
87 char *args[4];
88};
89
90
91/**
92 * Abort the scan. Must not be called from within the progress_callback
93 * function.
94 *
95 * @param ds directory scanner structure
96 */
97void
98GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds)
99{
100 /* terminate helper */
101 if (NULL != ds->helper)
102 GNUNET_HELPER_stop (ds->helper, GNUNET_NO);
103
104 /* free resources */
105 if (NULL != ds->toplevel)
106 GNUNET_FS_share_tree_free (ds->toplevel);
107 if (NULL != ds->stop_task)
108 GNUNET_SCHEDULER_cancel (ds->stop_task);
109 GNUNET_free (ds->ex_arg);
110 GNUNET_free (ds->filename_expanded);
111 GNUNET_free (ds);
112}
113
114
115/**
116 * Obtain the result of the scan after the scan has signalled
117 * completion. Must not be called prior to completion. The 'ds' is
118 * freed as part of this call.
119 *
120 * @param ds directory scanner structure
121 * @return the results of the scan (a directory tree)
122 */
123struct GNUNET_FS_ShareTreeItem *
124GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
125{
126 struct GNUNET_FS_ShareTreeItem *result;
127
128 /* check that we're actually done */
129 GNUNET_assert (NULL == ds->helper);
130 /* preserve result */
131 result = ds->toplevel;
132 ds->toplevel = NULL;
133 GNUNET_FS_directory_scan_abort (ds);
134 return result;
135}
136
137
138/**
139 * Move in the directory from the given position to the next file
140 * in DFS traversal.
141 *
142 * @param pos current position
143 * @return next file, NULL for none
144 */
145static struct GNUNET_FS_ShareTreeItem *
146advance (struct GNUNET_FS_ShareTreeItem *pos)
147{
148 int moved;
149
150 GNUNET_assert (NULL != pos);
151 moved = 0; /* must not terminate, even on file, otherwise "normal" */
152 while ((pos->is_directory == GNUNET_YES) || (0 == moved))
153 {
154 if ((moved != -1) && (NULL != pos->children_head))
155 {
156 pos = pos->children_head;
157 moved = 1; /* can terminate if file */
158 continue;
159 }
160 if (NULL != pos->next)
161 {
162 pos = pos->next;
163 moved = 1; /* can terminate if file */
164 continue;
165 }
166 if (NULL != pos->parent)
167 {
168 pos = pos->parent;
169 moved = -1; /* force move to 'next' or 'parent' */
170 continue;
171 }
172 /* no more options, end of traversal */
173 return NULL;
174 }
175 return pos;
176}
177
178
179/**
180 * Add another child node to the tree.
181 *
182 * @param parent parent of the child, NULL for top level
183 * @param filename name of the file or directory
184 * @param is_directory GNUNET_YES for directories
185 * @return new entry that was just created
186 */
187static struct GNUNET_FS_ShareTreeItem *
188expand_tree (struct GNUNET_FS_ShareTreeItem *parent,
189 const char *filename,
190 int is_directory)
191{
192 struct GNUNET_FS_ShareTreeItem *chld;
193 size_t slen;
194
195 chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
196 chld->parent = parent;
197 chld->filename = GNUNET_strdup (filename);
198 GNUNET_asprintf (&chld->short_filename,
199 "%s%s",
200 GNUNET_STRINGS_get_short_name (filename),
201 is_directory == GNUNET_YES ? "/" : "");
202 /* make sure we do not end with '//' */
203 slen = strlen (chld->short_filename);
204 if ((slen >= 2) && (chld->short_filename[slen - 1] == '/') &&
205 (chld->short_filename[slen - 2] == '/'))
206 chld->short_filename[slen - 1] = '\0';
207 chld->is_directory = is_directory;
208 if (NULL != parent)
209 GNUNET_CONTAINER_DLL_insert (parent->children_head,
210 parent->children_tail,
211 chld);
212 return chld;
213}
214
215
216/**
217 * Task run last to shut everything down.
218 *
219 * @param cls the 'struct GNUNET_FS_DirScanner'
220 */
221static void
222finish_scan (void *cls)
223{
224 struct GNUNET_FS_DirScanner *ds = cls;
225
226 ds->stop_task = NULL;
227 if (NULL != ds->helper)
228 {
229 GNUNET_HELPER_stop (ds->helper, GNUNET_NO);
230 ds->helper = NULL;
231 }
232 ds->progress_callback (ds->progress_callback_cls,
233 NULL,
234 GNUNET_SYSERR,
235 GNUNET_FS_DIRSCANNER_FINISHED);
236}
237
238
239/**
240 * Called every time there is data to read from the scanner.
241 * Calls the scanner progress handler.
242 *
243 * @param cls the closure (directory scanner object)
244 * @param msg message from the helper process
245 * @return #GNUNET_OK on success,
246 * #GNUNET_NO to stop further processing (no error)
247 * #GNUNET_SYSERR to stop further processing with error
248 */
249static int
250process_helper_msgs (void *cls, const struct GNUNET_MessageHeader *msg)
251{
252 struct GNUNET_FS_DirScanner *ds = cls;
253 const char *filename;
254 size_t left;
255
256#if 0
257 fprintf (stderr,
258 "DMS parses %u-byte message of type %u\n",
259 (unsigned int) ntohs (msg->size),
260 (unsigned int) ntohs (msg->type));
261#endif
262 left = ntohs (msg->size) - sizeof(struct GNUNET_MessageHeader);
263 filename = (const char *) &msg[1];
264 switch (ntohs (msg->type))
265 {
266 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE:
267 if (filename[left - 1] != '\0')
268 {
269 GNUNET_break (0);
270 break;
271 }
272 ds->progress_callback (ds->progress_callback_cls,
273 filename,
274 GNUNET_NO,
275 GNUNET_FS_DIRSCANNER_FILE_START);
276 if (NULL == ds->toplevel)
277 {
278 ds->toplevel = expand_tree (ds->pos, filename, GNUNET_NO);
279 }
280 else
281 {
282 GNUNET_assert (NULL != ds->pos);
283 (void) expand_tree (ds->pos, filename, GNUNET_NO);
284 }
285 return GNUNET_OK;
286
287 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY:
288 if (filename[left - 1] != '\0')
289 {
290 GNUNET_break (0);
291 break;
292 }
293 if (0 == strcmp ("..", filename))
294 {
295 if (NULL == ds->pos)
296 {
297 GNUNET_break (0);
298 break;
299 }
300 ds->pos = ds->pos->parent;
301 return GNUNET_OK;
302 }
303 ds->progress_callback (ds->progress_callback_cls,
304 filename,
305 GNUNET_YES,
306 GNUNET_FS_DIRSCANNER_FILE_START);
307 ds->pos = expand_tree (ds->pos, filename, GNUNET_YES);
308 if (NULL == ds->toplevel)
309 ds->toplevel = ds->pos;
310 return GNUNET_OK;
311
312 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR:
313 break;
314
315 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE:
316 if ('\0' != filename[left - 1])
317 break;
318 ds->progress_callback (ds->progress_callback_cls,
319 filename,
320 GNUNET_SYSERR,
321 GNUNET_FS_DIRSCANNER_FILE_IGNORED);
322 return GNUNET_OK;
323
324 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE:
325 if (0 != left)
326 {
327 GNUNET_break (0);
328 break;
329 }
330 if (NULL == ds->toplevel)
331 break;
332 ds->progress_callback (ds->progress_callback_cls,
333 NULL,
334 GNUNET_SYSERR,
335 GNUNET_FS_DIRSCANNER_ALL_COUNTED);
336 ds->pos = ds->toplevel;
337 if (GNUNET_YES == ds->pos->is_directory)
338 ds->pos = advance (ds->pos);
339 return GNUNET_OK;
340
341 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA: {
342 size_t nlen;
343 const char *end;
344
345 if (NULL == ds->pos)
346 {
347 GNUNET_break (0);
348 break;
349 }
350 end = memchr (filename, 0, left);
351 if (NULL == end)
352 {
353 GNUNET_break (0);
354 break;
355 }
356 end++;
357 nlen = end - filename;
358 left -= nlen;
359 if (0 != strcmp (filename, ds->pos->filename))
360 {
361 GNUNET_break (0);
362 break;
363 }
364 ds->progress_callback (ds->progress_callback_cls,
365 filename,
366 GNUNET_YES,
367 GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED);
368 if (0 < left)
369 {
370 ds->pos->meta = GNUNET_CONTAINER_meta_data_deserialize (end, left);
371 if (NULL == ds->pos->meta)
372 {
373 GNUNET_break (0);
374 break;
375 }
376 /* having full filenames is too dangerous; always make sure we clean them up */
377 GNUNET_CONTAINER_meta_data_delete (ds->pos->meta,
378 EXTRACTOR_METATYPE_FILENAME,
379 NULL,
380 0);
381 /* instead, put in our 'safer' original filename */
382 GNUNET_CONTAINER_meta_data_insert (ds->pos->meta,
383 "<libgnunetfs>",
384 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
385 EXTRACTOR_METAFORMAT_UTF8,
386 "text/plain",
387 ds->pos->short_filename,
388 strlen (ds->pos->short_filename)
389 + 1);
390 }
391 ds->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_meta_data (
392 ds->pos->meta);
393 ds->pos = advance (ds->pos);
394 return GNUNET_OK;
395 }
396
397 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED:
398 if (NULL != ds->pos)
399 {
400 GNUNET_break (0);
401 break;
402 }
403 if (0 != left)
404 {
405 GNUNET_break (0);
406 break;
407 }
408 if (NULL == ds->toplevel)
409 break;
410 ds->stop_task = GNUNET_SCHEDULER_add_now (&finish_scan, ds);
411 return GNUNET_OK;
412
413 default:
414 GNUNET_break (0);
415 break;
416 }
417 ds->progress_callback (ds->progress_callback_cls,
418 NULL,
419 GNUNET_SYSERR,
420 GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
421 return GNUNET_OK;
422}
423
424
425/**
426 * Function called if our helper process died.
427 *
428 * @param cls the 'struct GNUNET_FS_DirScanner' callback.
429 */
430static void
431helper_died_cb (void *cls)
432{
433 struct GNUNET_FS_DirScanner *ds = cls;
434
435 ds->helper = NULL;
436 if (NULL != ds->stop_task)
437 return; /* normal death, was finished */
438 ds->progress_callback (ds->progress_callback_cls,
439 NULL,
440 GNUNET_SYSERR,
441 GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
442}
443
444
445/**
446 * Start a directory scanner thread.
447 *
448 * @param filename name of the directory to scan
449 * @param disable_extractor #GNUNET_YES to not run libextractor on files (only
450 * build a tree)
451 * @param ex if not NULL, must be a list of extra plugins for extractor
452 * @param cb the callback to call when there are scanning progress messages
453 * @param cb_cls closure for 'cb'
454 * @return directory scanner object to be used for controlling the scanner
455 */
456struct GNUNET_FS_DirScanner *
457GNUNET_FS_directory_scan_start (const char *filename,
458 int disable_extractor,
459 const char *ex,
460 GNUNET_FS_DirScannerProgressCallback cb,
461 void *cb_cls)
462{
463 struct stat sbuf;
464 char *filename_expanded;
465 struct GNUNET_FS_DirScanner *ds;
466
467 if (0 != stat (filename, &sbuf))
468 return NULL;
469 filename_expanded = GNUNET_STRINGS_filename_expand (filename);
470 if (NULL == filename_expanded)
471 return NULL;
472 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
473 "Starting to scan directory `%s'\n",
474 filename_expanded);
475 ds = GNUNET_new (struct GNUNET_FS_DirScanner);
476 ds->progress_callback = cb;
477 ds->progress_callback_cls = cb_cls;
478 ds->filename_expanded = filename_expanded;
479 if (disable_extractor)
480 ds->ex_arg = GNUNET_strdup ("-");
481 else
482 ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL;
483 ds->args[0] = "gnunet-helper-fs-publish";
484 ds->args[1] = ds->filename_expanded;
485 ds->args[2] = ds->ex_arg;
486 ds->args[3] = NULL;
487 ds->helper = GNUNET_HELPER_start (GNUNET_NO,
488 "gnunet-helper-fs-publish",
489 ds->args,
490 &process_helper_msgs,
491 &helper_died_cb,
492 ds);
493 if (NULL == ds->helper)
494 {
495 GNUNET_free (filename_expanded);
496 GNUNET_free (ds);
497 return NULL;
498 }
499 return ds;
500}
501
502
503/* end of fs_dirmetascan.c */