diff options
author | Christian Grothoff <christian@grothoff.org> | 2012-01-29 17:38:57 +0000 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2012-01-29 17:38:57 +0000 |
commit | 4419d11120f7cdb592e612993ef1e1df2d023e8e (patch) | |
tree | 0f142b69f9f7229eaad1f93ff8a46a7315650060 /src/fs/gnunet-helper-fs-publish.c | |
parent | 415c958a01ab6dac73b0109becbe8657b525af9c (diff) | |
download | gnunet-4419d11120f7cdb592e612993ef1e1df2d023e8e.tar.gz gnunet-4419d11120f7cdb592e612993ef1e1df2d023e8e.zip |
-towards resolving thread-issue
Diffstat (limited to 'src/fs/gnunet-helper-fs-publish.c')
-rw-r--r-- | src/fs/gnunet-helper-fs-publish.c | 363 |
1 files changed, 363 insertions, 0 deletions
diff --git a/src/fs/gnunet-helper-fs-publish.c b/src/fs/gnunet-helper-fs-publish.c new file mode 100644 index 000000000..5edc2ec36 --- /dev/null +++ b/src/fs/gnunet-helper-fs-publish.c | |||
@@ -0,0 +1,363 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet. | ||
3 | (C) 2012 Christian Grothoff (and other contributing authors) | ||
4 | |||
5 | GNUnet is free software; you can redistribute it and/or modify | ||
6 | it under the terms of the GNU General Public License as published | ||
7 | by the Free Software Foundation; either version 3, or (at your | ||
8 | option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU General Public License | ||
16 | along with GNUnet; see the file COPYING. If not, write to the | ||
17 | Free Software Foundation, Inc., 59 Temple Place - Suite 330, | ||
18 | Boston, MA 02111-1307, USA. | ||
19 | */ | ||
20 | |||
21 | /** | ||
22 | * @file src/fs/gnunet-helper-fs-publish.c | ||
23 | * @brief Tool to help extract meta data asynchronously | ||
24 | * @author Christian Grothoff | ||
25 | * | ||
26 | * This program will scan a directory for files with meta data | ||
27 | * and report the results to stdout. | ||
28 | */ | ||
29 | #include "platform.h" | ||
30 | #include "gnunet_fs_service.h" | ||
31 | |||
32 | /** | ||
33 | * List of libextractor plugins to use for extracting. | ||
34 | */ | ||
35 | static struct EXTRACTOR_PluginList *plugins; | ||
36 | |||
37 | |||
38 | #if 0 | ||
39 | /** | ||
40 | * Write 'size' bytes from 'buf' into 'out'. | ||
41 | * | ||
42 | * @param in pipe to write to | ||
43 | * @param buf buffer with data to write | ||
44 | * @param size number of bytes to write | ||
45 | * @return GNUNET_OK on success, GNUNET_SYSERR on error | ||
46 | */ | ||
47 | static int | ||
48 | write_all (const struct GNUNET_DISK_FileHandle *out, | ||
49 | const void *buf, | ||
50 | size_t size) | ||
51 | { | ||
52 | const char *cbuf = buf; | ||
53 | size_t total; | ||
54 | ssize_t wr; | ||
55 | |||
56 | total = 0; | ||
57 | do | ||
58 | { | ||
59 | wr = GNUNET_DISK_file_write (out, | ||
60 | &cbuf[total], | ||
61 | size - total); | ||
62 | if (wr > 0) | ||
63 | total += wr; | ||
64 | } while ( (wr > 0) && (total < size) ); | ||
65 | if (wr <= 0) | ||
66 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
67 | "Failed to write to inter thread communication pipe: %s\n", | ||
68 | strerror (errno)); | ||
69 | return (total == size) ? GNUNET_OK : GNUNET_SYSERR; | ||
70 | } | ||
71 | |||
72 | |||
73 | /** | ||
74 | * Write progress message. | ||
75 | * | ||
76 | * @param ds | ||
77 | * @param filename name of the file to transmit, never NULL | ||
78 | * @param is_directory GNUNET_YES for directory, GNUNET_NO for file, GNUNET_SYSERR for neither | ||
79 | * @param reason reason for the progress call | ||
80 | * @return GNUNET_SYSERR to stop scanning (the pipe was broken somehow) | ||
81 | */ | ||
82 | static int | ||
83 | write_progress (struct GNUNET_FS_DirScanner *ds, | ||
84 | const char *filename, | ||
85 | int is_directory, | ||
86 | enum GNUNET_FS_DirScannerProgressUpdateReason reason) | ||
87 | { | ||
88 | size_t slen; | ||
89 | |||
90 | slen = strlen (filename) + 1; | ||
91 | if ( (GNUNET_OK != | ||
92 | write_all (ds->progress_write, | ||
93 | &reason, | ||
94 | sizeof (reason))) || | ||
95 | (GNUNET_OK != | ||
96 | write_all (ds->progress_write, | ||
97 | &slen, | ||
98 | sizeof (slen))) || | ||
99 | (GNUNET_OK != | ||
100 | write_all (ds->progress_write, | ||
101 | filename, | ||
102 | slen)) || | ||
103 | (GNUNET_OK != | ||
104 | write_all (ds->progress_write, | ||
105 | &is_directory, | ||
106 | sizeof (is_directory))) ) | ||
107 | return GNUNET_SYSERR; | ||
108 | return GNUNET_OK; | ||
109 | } | ||
110 | |||
111 | |||
112 | /** | ||
113 | * Function called to (recursively) add all of the files in the | ||
114 | * directory to the tree. Called by the directory scanner to initiate | ||
115 | * the scan. Does NOT yet add any metadata. | ||
116 | * | ||
117 | * @param ds directory scanner context to use | ||
118 | * @param filename file or directory to scan | ||
119 | * @param dst where to store the resulting share tree item | ||
120 | * @return GNUNET_OK on success, GNUNET_SYSERR on error | ||
121 | */ | ||
122 | static int | ||
123 | preprocess_file (struct GNUNET_FS_DirScanner *ds, | ||
124 | const char *filename, | ||
125 | struct GNUNET_FS_ShareTreeItem **dst); | ||
126 | |||
127 | |||
128 | /** | ||
129 | * Closure for the 'scan_callback' | ||
130 | */ | ||
131 | struct RecursionContext | ||
132 | { | ||
133 | /** | ||
134 | * Global scanner context. | ||
135 | */ | ||
136 | struct GNUNET_FS_DirScanner *ds; | ||
137 | |||
138 | /** | ||
139 | * Parent to add the files to. | ||
140 | */ | ||
141 | struct GNUNET_FS_ShareTreeItem *parent; | ||
142 | |||
143 | /** | ||
144 | * Flag to set to GNUNET_YES on serious errors. | ||
145 | */ | ||
146 | int stop; | ||
147 | }; | ||
148 | |||
149 | |||
150 | /** | ||
151 | * Function called by the directory iterator to (recursively) add all | ||
152 | * of the files in the directory to the tree. Called by the directory | ||
153 | * scanner to initiate the scan. Does NOT yet add any metadata. | ||
154 | * | ||
155 | * @param cls the 'struct RecursionContext' | ||
156 | * @param filename file or directory to scan | ||
157 | * @return GNUNET_OK on success, GNUNET_SYSERR on error | ||
158 | */ | ||
159 | static int | ||
160 | scan_callback (void *cls, | ||
161 | const char *filename) | ||
162 | { | ||
163 | struct RecursionContext *rc = cls; | ||
164 | struct GNUNET_FS_ShareTreeItem *chld; | ||
165 | |||
166 | if (GNUNET_OK != | ||
167 | preprocess_file (rc->ds, | ||
168 | filename, | ||
169 | &chld)) | ||
170 | { | ||
171 | rc->stop = GNUNET_YES; | ||
172 | return GNUNET_SYSERR; | ||
173 | } | ||
174 | chld->parent = rc->parent; | ||
175 | GNUNET_CONTAINER_DLL_insert (rc->parent->children_head, | ||
176 | rc->parent->children_tail, | ||
177 | chld); | ||
178 | return GNUNET_OK; | ||
179 | } | ||
180 | |||
181 | |||
182 | /** | ||
183 | * Function called to (recursively) add all of the files in the | ||
184 | * directory to the tree. Called by the directory scanner to initiate | ||
185 | * the scan. Does NOT yet add any metadata. | ||
186 | * | ||
187 | * @param ds directory scanner context to use | ||
188 | * @param filename file or directory to scan | ||
189 | * @param dst where to store the resulting share tree item | ||
190 | * @return GNUNET_OK on success, GNUNET_SYSERR on error | ||
191 | */ | ||
192 | static int | ||
193 | preprocess_file (struct GNUNET_FS_DirScanner *ds, | ||
194 | const char *filename, | ||
195 | struct GNUNET_FS_ShareTreeItem **dst) | ||
196 | { | ||
197 | struct GNUNET_FS_ShareTreeItem *item; | ||
198 | struct stat sbuf; | ||
199 | |||
200 | if (0 != STAT (filename, &sbuf)) | ||
201 | { | ||
202 | /* If the file doesn't exist (or is not stat-able for any other reason) | ||
203 | skip it (but report it), but do continue. */ | ||
204 | if (GNUNET_OK != | ||
205 | write_progress (ds, filename, GNUNET_SYSERR, | ||
206 | GNUNET_FS_DIRSCANNER_DOES_NOT_EXIST)) | ||
207 | return GNUNET_SYSERR; | ||
208 | return GNUNET_OK; | ||
209 | } | ||
210 | |||
211 | /* Report the progress */ | ||
212 | if (GNUNET_OK != | ||
213 | write_progress (ds, | ||
214 | filename, | ||
215 | S_ISDIR (sbuf.st_mode) ? GNUNET_YES : GNUNET_NO, | ||
216 | GNUNET_FS_DIRSCANNER_FILE_START)) | ||
217 | return GNUNET_SYSERR; | ||
218 | item = GNUNET_malloc (sizeof (struct GNUNET_FS_ShareTreeItem)); | ||
219 | item->meta = GNUNET_CONTAINER_meta_data_create (); | ||
220 | item->filename = GNUNET_strdup (filename); | ||
221 | item->short_filename = GNUNET_strdup (GNUNET_STRINGS_get_short_name (filename)); | ||
222 | item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO; | ||
223 | item->file_size = (uint64_t) sbuf.st_size; | ||
224 | if (item->is_directory) | ||
225 | { | ||
226 | struct RecursionContext rc; | ||
227 | |||
228 | rc.parent = item; | ||
229 | rc.ds = ds; | ||
230 | rc.stop = GNUNET_NO; | ||
231 | GNUNET_DISK_directory_scan (filename, | ||
232 | &scan_callback, | ||
233 | &rc); | ||
234 | if ( (rc.stop == GNUNET_YES) || | ||
235 | (GNUNET_OK != | ||
236 | test_thread_stop (ds)) ) | ||
237 | { | ||
238 | GNUNET_FS_share_tree_free (item); | ||
239 | return GNUNET_SYSERR; | ||
240 | } | ||
241 | } | ||
242 | /* Report the progress */ | ||
243 | if (GNUNET_OK != | ||
244 | write_progress (ds, | ||
245 | filename, | ||
246 | S_ISDIR (sbuf.st_mode) ? GNUNET_YES : GNUNET_NO, | ||
247 | GNUNET_FS_DIRSCANNER_SUBTREE_COUNTED)) | ||
248 | { | ||
249 | GNUNET_FS_share_tree_free (item); | ||
250 | return GNUNET_SYSERR; | ||
251 | } | ||
252 | *dst = item; | ||
253 | return GNUNET_OK; | ||
254 | } | ||
255 | |||
256 | |||
257 | /** | ||
258 | * Extract metadata from files. | ||
259 | * | ||
260 | * @param ds directory scanner context | ||
261 | * @param item entry we are processing | ||
262 | * @return GNUNET_OK on success, GNUNET_SYSERR on fatal errors | ||
263 | */ | ||
264 | static int | ||
265 | extract_files (struct GNUNET_FS_DirScanner *ds, | ||
266 | struct GNUNET_FS_ShareTreeItem *item) | ||
267 | { | ||
268 | if (item->is_directory) | ||
269 | { | ||
270 | /* for directories, we simply only descent, no extraction, no | ||
271 | progress reporting */ | ||
272 | struct GNUNET_FS_ShareTreeItem *pos; | ||
273 | |||
274 | for (pos = item->children_head; NULL != pos; pos = pos->next) | ||
275 | if (GNUNET_OK != | ||
276 | extract_files (ds, pos)) | ||
277 | return GNUNET_SYSERR; | ||
278 | return GNUNET_OK; | ||
279 | } | ||
280 | |||
281 | /* this is the expensive operation, *afterwards* we'll check for aborts */ | ||
282 | fprintf (stderr, "\tCalling extract on `%s'\n", item->filename); | ||
283 | GNUNET_FS_meta_data_extract_from_file (item->meta, | ||
284 | item->filename, | ||
285 | ds->plugins); | ||
286 | fprintf (stderr, "\tExtract `%s' done\n", item->filename); | ||
287 | |||
288 | /* having full filenames is too dangerous; always make sure we clean them up */ | ||
289 | GNUNET_CONTAINER_meta_data_delete (item->meta, | ||
290 | EXTRACTOR_METATYPE_FILENAME, | ||
291 | NULL, 0); | ||
292 | GNUNET_CONTAINER_meta_data_insert (item->meta, "<libgnunetfs>", | ||
293 | EXTRACTOR_METATYPE_FILENAME, | ||
294 | EXTRACTOR_METAFORMAT_UTF8, "text/plain", | ||
295 | item->short_filename, | ||
296 | strlen (item->short_filename) + 1); | ||
297 | /* check for abort */ | ||
298 | if (GNUNET_OK != | ||
299 | test_thread_stop (ds)) | ||
300 | return GNUNET_SYSERR; | ||
301 | |||
302 | /* Report the progress */ | ||
303 | if (GNUNET_OK != | ||
304 | write_progress (ds, | ||
305 | item->filename, | ||
306 | GNUNET_NO, | ||
307 | GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED)) | ||
308 | return GNUNET_SYSERR; | ||
309 | return GNUNET_OK; | ||
310 | } | ||
311 | |||
312 | #endif | ||
313 | |||
314 | |||
315 | int main(int argc, | ||
316 | char **argv) | ||
317 | { | ||
318 | const char *filename_expanded; | ||
319 | const char *ex; | ||
320 | |||
321 | if (argc < 3) | ||
322 | { | ||
323 | FPRINTF (stderr, | ||
324 | "%s", | ||
325 | "gnunet-helper-fs-publish needs at least two arguments\n"); | ||
326 | return 1; | ||
327 | } | ||
328 | filename_expanded = argv[1]; | ||
329 | ex = argv[2]; | ||
330 | if (0 != strcmp (ex, "-")) | ||
331 | { | ||
332 | plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY); | ||
333 | if (NULL != ex) | ||
334 | plugins = EXTRACTOR_plugin_add_config (plugins, ex, | ||
335 | EXTRACTOR_OPTION_DEFAULT_POLICY); | ||
336 | } | ||
337 | |||
338 | #if 0 | ||
339 | if (GNUNET_OK != preprocess_file (filename_expanded, | ||
340 | &toplevel)) | ||
341 | { | ||
342 | (void) write_progress (ds, "", GNUNET_SYSERR, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR); | ||
343 | GNUNET_DISK_pipe_close_end (ds->progress_pipe, GNUNET_DISK_PIPE_END_WRITE); | ||
344 | return 2; | ||
345 | } | ||
346 | if (GNUNET_OK != | ||
347 | write_progress (ds, "", GNUNET_SYSERR, GNUNET_FS_DIRSCANNER_ALL_COUNTED)) | ||
348 | { | ||
349 | return 3; | ||
350 | } | ||
351 | if (GNUNET_OK != | ||
352 | extract_files (ds, ds->toplevel)) | ||
353 | { | ||
354 | (void) write_progress (ds, "", GNUNET_SYSERR, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR); | ||
355 | return 4; | ||
356 | } | ||
357 | (void) write_progress (ds, "", GNUNET_SYSERR, GNUNET_FS_DIRSCANNER_FINISHED); | ||
358 | #endif | ||
359 | if (NULL != plugins) | ||
360 | EXTRACTOR_plugin_remove_all (plugins); | ||
361 | |||
362 | return 0; | ||
363 | } | ||