aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorChristian Grothoff <christian@grothoff.org>2011-11-15 09:38:48 +0000
committerChristian Grothoff <christian@grothoff.org>2011-11-15 09:38:48 +0000
commit0443654040fb277df95aae28a7b2a54a4f0a73bf (patch)
tree5f07e7fcf3409d1cb287ef5181b01f3584198acb /src
parentca5418e74010c0034f7f24ad8a7a2ecb6e941af1 (diff)
downloadgnunet-0443654040fb277df95aae28a7b2a54a4f0a73bf.tar.gz
gnunet-0443654040fb277df95aae28a7b2a54a4f0a73bf.zip
dead code elimination, splitting fs.h into fs.h and fs_api.h
Diffstat (limited to 'src')
-rw-r--r--src/fs/Makefile.am2
-rw-r--r--src/fs/fs.h2001
-rw-r--r--src/fs/fs_api.c (renamed from src/fs/fs.c)4
-rw-r--r--src/fs/fs_api.h1907
-rw-r--r--src/fs/fs_directory.c2
-rw-r--r--src/fs/fs_download.c2
-rw-r--r--src/fs/fs_file_information.c2
-rw-r--r--src/fs/fs_getopt.c2
-rw-r--r--src/fs/fs_list_indexed.c2
-rw-r--r--src/fs/fs_misc.c2
-rw-r--r--src/fs/fs_namespace.c14
-rw-r--r--src/fs/fs_publish.c7
-rw-r--r--src/fs/fs_search.c7
-rw-r--r--src/fs/fs_test_lib.c2
-rw-r--r--src/fs/fs_tree.h2
-rw-r--r--src/fs/fs_unindex.c2
-rw-r--r--src/fs/fs_uri.c2
-rw-r--r--src/fs/gnunet-service-fs.c2
-rw-r--r--src/fs/gnunet-service-fs.h23
-rw-r--r--src/fs/gnunet-service-fs_cp.c8
-rw-r--r--src/fs/gnunet-service-fs_cp.h30
-rw-r--r--src/fs/gnunet-service-fs_pr.c15
-rw-r--r--src/fs/gnunet-service-fs_push.c11
-rw-r--r--src/fs/test_fs_directory.c2
-rw-r--r--src/fs/test_fs_uri.c2
25 files changed, 2019 insertions, 2036 deletions
diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am
index 8cc0f9bf3..b1f15afa3 100644
--- a/src/fs/Makefile.am
+++ b/src/fs/Makefile.am
@@ -20,7 +20,7 @@ lib_LTLIBRARIES = libgnunetfs.la
20noinst_LIBRARIES = libgnunetfstest.a 20noinst_LIBRARIES = libgnunetfstest.a
21 21
22libgnunetfs_la_SOURCES = \ 22libgnunetfs_la_SOURCES = \
23 fs.c fs.h \ 23 fs_api.c fs_api.h fs.h \
24 fs_directory.c \ 24 fs_directory.c \
25 fs_download.c \ 25 fs_download.c \
26 fs_file_information.c \ 26 fs_file_information.c \
diff --git a/src/fs/fs.h b/src/fs/fs.h
index b5fdb1cc7..89a375022 100644
--- a/src/fs/fs.h
+++ b/src/fs/fs.h
@@ -35,201 +35,17 @@
35 35
36 36
37/** 37/**
38 * Maximum number of outgoing messages we queue per peer.
39 *
40 * Performance measurements for 2 peer setup for 50 MB file
41 * (with MAX_DATASTORE_QUEUE = 1 and RETRY_PROBABILITY_INV = 1):
42 *
43 * 2: 1700 kb/s, 1372 kb/s
44 * 8: 2117 kb/s, 1284 kb/s, 1112 kb/s
45 * 16: 3500 kb/s, 3200 kb/s, 3388 kb/s
46 * 32: 3441 kb/s, 3163 kb/s, 3277 kb/s
47 * 128: 1700 kb/s; 2010 kb/s, 3383 kb/s, 1156 kb/s
48 *
49 * Conclusion: 16 seems to be a pretty good value (stable
50 * and high performance, no excessive memory use).
51 */
52#define MAX_QUEUE_PER_PEER 16
53
54/**
55 * Maximum size of the datastore queue for P2P operations.
56 * Needs to be large enough to queue MAX_QUEUE_PER_PEER
57 * operations for roughly the number of active (connected)
58 * peers.
59 */
60#define MAX_DATASTORE_QUEUE (16 * MAX_QUEUE_PER_PEER)
61
62/**
63 * Maximum number of blocks we keep in memory for migration.
64 */
65#define MAX_MIGRATION_QUEUE 8
66
67/**
68 * Blocks are at most migrated to this number of peers
69 * plus one, each time they are fetched from the database.
70 */
71#define MIGRATION_LIST_SIZE 2
72
73/**
74 * To how many peers do we forward each migration block ultimately?
75 * This number must be smaller or equal to MIGRATION_LIST_SIZE. Using
76 * a smaller value allows for variation in available bandwidth (for
77 * migration) between the peers.
78 */
79#define MIGRATION_TARGET_COUNT 2
80
81/**
82 * Ratio for moving average delay calculation. The previous
83 * average goes in with a factor of (n-1) into the calculation.
84 * Must be > 0.
85 */
86#define RUNAVG_DELAY_N 16
87
88/**
89 * Size of the individual blocks used for file-sharing. 38 * Size of the individual blocks used for file-sharing.
90 */ 39 */
91#define DBLOCK_SIZE (32*1024) 40#define DBLOCK_SIZE (32*1024)
92 41
93/** 42/**
94 * Maximum legal size for a kblock. 43 * Blocksize to use when hashing files for indexing (blocksize for IO,
95 */ 44 * not for the DBlocks). Larger blocksizes can be more efficient but
96#define MAX_KBLOCK_SIZE (60 * 1024) 45 * will be more disruptive as far as the scheduler is concerned.
97
98/**
99 * Maximum legal size for an sblock.
100 */
101#define MAX_SBLOCK_SIZE (60 * 1024)
102
103/**
104 * Maximum legal size for an nblock.
105 */
106#define MAX_NBLOCK_SIZE (60 * 1024)
107
108/**
109 * Pick a multiple of 2 here to achive 8-byte alignment!
110 * We also probably want DBlocks to have (roughly) the
111 * same size as IBlocks. With SHA-512, the optimal
112 * value is 32768 byte / 128 byte = 256
113 * (128 byte = 2 * 512 bits). DO NOT CHANGE!
114 */
115#define CHK_PER_INODE 256
116
117
118/**
119 * Maximum size for a file to be considered for
120 * inlining in a directory.
121 */
122#define MAX_INLINE_SIZE 65536
123
124
125/**
126 * Blocksize to use when hashing files
127 * for indexing (blocksize for IO, not for
128 * the DBlocks). Larger blocksizes can
129 * be more efficient but will be more disruptive
130 * as far as the scheduler is concerned.
131 */ 46 */
132#define HASHING_BLOCKSIZE (1024 * 128) 47#define HASHING_BLOCKSIZE (1024 * 128)
133 48
134/**
135 * Number of availability trials we perform per search result.
136 */
137#define AVAILABILITY_TRIALS_MAX 8
138
139/**
140 * Length of the P2P success tracker. Note that
141 * having a very long list can also hurt performance.
142 */
143#define P2P_SUCCESS_LIST_SIZE 8
144
145
146/**
147 * Length of the CS-2-P success tracker. Note that
148 * having a very long list can also hurt performance.
149 */
150#define CS2P_SUCCESS_LIST_SIZE 8
151
152/**
153 * How long are we willing to wait for the datastore to be ready to
154 * process a request for a query without priority?
155 */
156#define BASIC_DATASTORE_REQUEST_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
157
158
159/**
160 * How long are we willing to wait for the core to be ready to
161 * transmit a reply to the target peer (if we can not transmit
162 * until then, we will discard the reply).
163 */
164#define ACCEPTABLE_REPLY_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5)
165
166
167/**
168 * Bandwidth value of an (effectively) 0-priority query.
169 */
170#define QUERY_BANDWIDTH_VALUE 1
171
172/**
173 * Bandwidth value of a 0-priority content (must be
174 * fairly high compared to query since content is
175 * typically significantly larger -- and more valueable
176 * since it can take many queries to get one piece of
177 * content).
178 */
179#define CONTENT_BANDWIDTH_VALUE 800
180
181/**
182 * By which amount do we decrement the TTL for simple forwarding /
183 * indirection of the query; in milli-seconds. Set somewhat in
184 * accordance to your network latency (above the time it'll take you
185 * to send a packet and get a reply).
186 */
187#define TTL_DECREMENT 5000
188
189/**
190 * Until which load do we consider the peer idle and do not
191 * charge at all? (should be larger than GNUNET_IDLE_LOAD_THRESHOLD used
192 * by the rest of the code)!
193 */
194#define IDLE_LOAD_THRESHOLD ((100 + GNUNET_CONSTANTS_IDLE_LOAD_THRESHOLD) / 2)
195
196/**
197 * Name of the directory with top-level searches.
198 */
199#define GNUNET_FS_SYNC_PATH_MASTER_SEARCH "search"
200
201/**
202 * Name of the directory with sub-searches (namespace-updates).
203 */
204#define GNUNET_FS_SYNC_PATH_CHILD_SEARCH "search-child"
205
206/**
207 * Name of the directory with master downloads (not associated
208 * with search or part of another download).
209 */
210#define GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD "download"
211
212/**
213 * Name of the directory with downloads that are part of another
214 * download or a search.
215 */
216#define GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD "download-child"
217
218/**
219 * Name of the directory with publishing operations.
220 */
221#define GNUNET_FS_SYNC_PATH_MASTER_PUBLISH "publish"
222
223/**
224 * Name of the directory with files that are being published
225 */
226#define GNUNET_FS_SYNC_PATH_FILE_INFO "publish-file"
227
228/**
229 * Name of the directory with unindex operations.
230 */
231#define GNUNET_FS_SYNC_PATH_MASTER_UNINDEX "unindex"
232
233 49
234/** 50/**
235 * @brief content hash key 51 * @brief content hash key
@@ -249,1817 +65,6 @@ struct ContentHashKey
249 65
250 66
251/** 67/**
252 * @brief complete information needed
253 * to download a file.
254 */
255struct FileIdentifier
256{
257
258 /**
259 * Total size of the file in bytes. (network byte order (!))
260 */
261 uint64_t file_length;
262
263 /**
264 * Query and key of the top GNUNET_EC_IBlock.
265 */
266 struct ContentHashKey chk;
267
268};
269
270
271/**
272 * Information about a file and its location
273 * (peer claiming to share the file).
274 */
275struct Location
276{
277 /**
278 * Information about the shared file.
279 */
280 struct FileIdentifier fi;
281
282 /**
283 * Identity of the peer sharing the file.
284 */
285 struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded peer;
286
287 /**
288 * Time when this location URI expires.
289 */
290 struct GNUNET_TIME_Absolute expirationTime;
291
292 /**
293 * RSA signature over the GNUNET_EC_FileIdentifier,
294 * GNUNET_hash of the peer and expiration time.
295 */
296 struct GNUNET_CRYPTO_RsaSignature contentSignature;
297
298};
299
300/**
301 * Types of URIs.
302 */
303enum uri_types
304{
305 /**
306 * Content-hash-key (simple file).
307 */
308 chk,
309
310 /**
311 * Signed key space (file in namespace).
312 */
313 sks,
314
315 /**
316 * Keyword search key (query with keywords).
317 */
318 ksk,
319
320 /**
321 * Location (chk with identity of hosting peer).
322 */
323 loc
324};
325
326/**
327 * A Universal Resource Identifier (URI), opaque.
328 */
329struct GNUNET_FS_Uri
330{
331 /**
332 * Type of the URI.
333 */
334 enum uri_types type;
335
336 union
337 {
338 struct
339 {
340 /**
341 * Keywords start with a '+' if they are
342 * mandatory (in which case the '+' is NOT
343 * part of the keyword) and with a
344 * simple space if they are optional
345 * (in which case the space is ALSO not
346 * part of the actual keyword).
347 *
348 * Double-quotes to protect spaces and
349 * %-encoding are NOT used internally
350 * (only in URI-strings).
351 */
352 char **keywords;
353
354 /**
355 * Size of the keywords array.
356 */
357 unsigned int keywordCount;
358 } ksk;
359
360 struct
361 {
362 /**
363 * Hash of the public key for the namespace.
364 */
365 GNUNET_HashCode namespace;
366
367 /**
368 * Human-readable identifier chosen for this
369 * entry in the namespace.
370 */
371 char *identifier;
372 } sks;
373
374 /**
375 * Information needed to retrieve a file (content-hash-key
376 * plus file size).
377 */
378 struct FileIdentifier chk;
379
380 /**
381 * Information needed to retrieve a file including signed
382 * location (identity of a peer) of the content.
383 */
384 struct Location loc;
385 } data;
386
387};
388
389
390/**
391 * Information for a file or directory that is
392 * about to be published.
393 */
394struct GNUNET_FS_FileInformation
395{
396
397 /**
398 * Files in a directory are kept as a linked list.
399 */
400 struct GNUNET_FS_FileInformation *next;
401
402 /**
403 * If this is a file in a directory, "dir" refers to
404 * the directory; otherwise NULL.
405 */
406 struct GNUNET_FS_FileInformation *dir;
407
408 /**
409 * Handle to the master context.
410 */
411 struct GNUNET_FS_Handle *h;
412
413 /**
414 * Pointer kept for the client.
415 */
416 void *client_info;
417
418 /**
419 * Metadata to use for the file.
420 */
421 struct GNUNET_CONTAINER_MetaData *meta;
422
423 /**
424 * Keywords to use for KBlocks.
425 */
426 struct GNUNET_FS_Uri *keywords;
427
428 /**
429 * CHK for this file or directory. NULL if
430 * we have not yet computed it.
431 */
432 struct GNUNET_FS_Uri *chk_uri;
433
434 /**
435 * Block options for the file.
436 */
437 struct GNUNET_FS_BlockOptions bo;
438
439 /**
440 * At what time did we start this upload?
441 */
442 struct GNUNET_TIME_Absolute start_time;
443
444 /**
445 * Under what filename is this struct serialized
446 * (for operational persistence). Should be determined
447 * using 'mktemp'.
448 */
449 char *serialization;
450
451 /**
452 * Encoder being used to publish this file.
453 */
454 struct GNUNET_FS_TreeEncoder *te;
455
456 /**
457 * Error message (non-NULL if this operation failed).
458 */
459 char *emsg;
460
461 /**
462 * Name of the file or directory (must be an absolute path).
463 */
464 char *filename;
465
466 /**
467 * Data describing either the file or the directory.
468 */
469 union
470 {
471
472 /**
473 * Data for a file.
474 */
475 struct
476 {
477
478 /**
479 * Function that can be used to read the data for the file.
480 */
481 GNUNET_FS_DataReader reader;
482
483 /**
484 * Closure for reader.
485 */
486 void *reader_cls;
487
488 /**
489 * If this file is being indexed, this value is set to the hash
490 * over the entire file (when the indexing process is started).
491 * Otherwise this field is not used.
492 */
493 GNUNET_HashCode file_id;
494
495 /**
496 * Size of the file (in bytes).
497 */
498 uint64_t file_size;
499
500 /**
501 * Should the file be indexed or inserted?
502 */
503 int do_index;
504
505 /**
506 * Is "file_id" already valid? Set to GNUNET_YES once the hash
507 * has been calculated.
508 */
509 int have_hash;
510
511 /**
512 * Has the service confirmed our INDEX_START request?
513 * GNUNET_YES if this step has been completed.
514 */
515 int index_start_confirmed;
516
517 } file;
518
519 /**
520 * Data for a directory.
521 */
522 struct
523 {
524
525 /**
526 * Linked list of entries in the directory.
527 */
528 struct GNUNET_FS_FileInformation *entries;
529
530 /**
531 * Size of the directory itself (in bytes); 0 if the
532 * size has not yet been calculated.
533 */
534 size_t dir_size;
535
536 /**
537 * Pointer to the data for the directory (or NULL if not
538 * available).
539 */
540 void *dir_data;
541
542 } dir;
543
544 } data;
545
546 /**
547 * Is this struct for a file or directory?
548 */
549 int is_directory;
550
551 /**
552 * Are we done publishing this file?
553 */
554 int is_published;
555
556};
557
558
559/**
560 * The job is now ready to run and should use the given client
561 * handle to communicate with the FS service.
562 *
563 * @param cls closure
564 * @param client handle to use for FS communication
565 */
566typedef void (*GNUNET_FS_QueueStart) (void *cls,
567 struct GNUNET_CLIENT_Connection * client);
568
569
570/**
571 * The job must now stop to run and should destry the client handle as
572 * soon as possible (ideally prior to returning).
573 */
574typedef void (*GNUNET_FS_QueueStop) (void *cls);
575
576
577/**
578 * Entry in the job queue.
579 */
580struct GNUNET_FS_QueueEntry
581{
582 /**
583 * This is a linked list.
584 */
585 struct GNUNET_FS_QueueEntry *next;
586
587 /**
588 * This is a linked list.
589 */
590 struct GNUNET_FS_QueueEntry *prev;
591
592 /**
593 * Function to call when the job is started.
594 */
595 GNUNET_FS_QueueStart start;
596
597 /**
598 * Function to call when the job needs to stop (or is done / dequeued).
599 */
600 GNUNET_FS_QueueStop stop;
601
602 /**
603 * Closure for start and stop.
604 */
605 void *cls;
606
607 /**
608 * Handle to FS primary context.
609 */
610 struct GNUNET_FS_Handle *h;
611
612 /**
613 * Client handle, or NULL if job is not running.
614 */
615 struct GNUNET_CLIENT_Connection *client;
616
617 /**
618 * Time the job was originally queued.
619 */
620 struct GNUNET_TIME_Absolute queue_time;
621
622 /**
623 * Time the job was started last.
624 */
625 struct GNUNET_TIME_Absolute start_time;
626
627 /**
628 * Total amount of time the job has been running (except for the
629 * current run).
630 */
631 struct GNUNET_TIME_Relative run_time;
632
633 /**
634 * How many blocks do the active downloads have?
635 */
636 unsigned int blocks;
637
638 /**
639 * How often have we (re)started this download?
640 */
641 unsigned int start_times;
642
643};
644
645
646
647
648/**
649 * Information we store for each search result.
650 */
651struct GNUNET_FS_SearchResult
652{
653
654 /**
655 * Search context this result belongs to.
656 */
657 struct GNUNET_FS_SearchContext *sc;
658
659 /**
660 * URI to which this search result refers to.
661 */
662 struct GNUNET_FS_Uri *uri;
663
664 /**
665 * Metadata for the search result.
666 */
667 struct GNUNET_CONTAINER_MetaData *meta;
668
669 /**
670 * Client info for this search result.
671 */
672 void *client_info;
673
674 /**
675 * ID of a job that is currently probing this results' availability
676 * (NULL if we are not currently probing).
677 */
678 struct GNUNET_FS_DownloadContext *probe_ctx;
679
680 /**
681 * ID of an associated download based on this search result (or
682 * NULL for none).
683 */
684 struct GNUNET_FS_DownloadContext *download;
685
686 /**
687 * If this search result triggered an update search, this field
688 * links to the update search.
689 */
690 struct GNUNET_FS_SearchContext *update_search;
691
692 /**
693 * Name under which this search result is stored on disk.
694 */
695 char *serialization;
696
697 /**
698 * Key for the search result
699 */
700 GNUNET_HashCode key;
701
702 /**
703 * ID of the task that will clean up the probe_ctx should it not
704 * complete on time (and that will need to be cancelled if we clean
705 * up the search result before then).
706 */
707 GNUNET_SCHEDULER_TaskIdentifier probe_cancel_task;
708
709 /**
710 * When did the current probe become active?
711 */
712 struct GNUNET_TIME_Absolute probe_active_time;
713
714 /**
715 * How much longer should we run the current probe before giving up?
716 */
717 struct GNUNET_TIME_Relative remaining_probe_time;
718
719 /**
720 * Number of mandatory keywords for which we have NOT yet found the
721 * search result; when this value hits zero, the search result is
722 * given to the callback.
723 */
724 uint32_t mandatory_missing;
725
726 /**
727 * Number of optional keywords under which this result was also
728 * found.
729 */
730 uint32_t optional_support;
731
732 /**
733 * Number of availability tests that have succeeded for this result.
734 */
735 uint32_t availability_success;
736
737 /**
738 * Number of availability trials that we have performed for this
739 * search result.
740 */
741 uint32_t availability_trials;
742
743};
744
745
746/**
747 * Add a job to the queue.
748 *
749 * @param h handle to the overall FS state
750 * @param start function to call to begin the job
751 * @param stop function to call to pause the job, or on dequeue (if the job was running)
752 * @param cls closure for start and stop
753 * @param blocks number of blocks this download has
754 * @return queue handle
755 */
756struct GNUNET_FS_QueueEntry *
757GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, GNUNET_FS_QueueStart start,
758 GNUNET_FS_QueueStop stop, void *cls, unsigned int blocks);
759
760
761/**
762 * Dequeue a job from the queue.
763 * @param qh handle for the job
764 */
765void
766GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh);
767
768
769/**
770 * Function that provides data by reading from a file.
771 *
772 * @param cls closure (points to the file information)
773 * @param offset offset to read from; it is possible
774 * that the caller might need to go backwards
775 * a bit at times
776 * @param max maximum number of bytes that should be
777 * copied to buf; readers are not allowed
778 * to provide less data unless there is an error;
779 * a value of "0" will be used at the end to allow
780 * the reader to clean up its internal state
781 * @param buf where the reader should write the data
782 * @param emsg location for the reader to store an error message
783 * @return number of bytes written, usually "max", 0 on error
784 */
785size_t
786GNUNET_FS_data_reader_file_ (void *cls, uint64_t offset, size_t max, void *buf,
787 char **emsg);
788
789
790/**
791 * Create the closure for the 'GNUNET_FS_data_reader_file_' callback.
792 *
793 * @param filename file to read
794 * @return closure to use
795 */
796void *
797GNUNET_FS_make_file_reader_context_ (const char *filename);
798
799
800
801/**
802 * Function that provides data by copying from a buffer.
803 *
804 * @param cls closure (points to the buffer)
805 * @param offset offset to read from; it is possible
806 * that the caller might need to go backwards
807 * a bit at times
808 * @param max maximum number of bytes that should be
809 * copied to buf; readers are not allowed
810 * to provide less data unless there is an error;
811 * a value of "0" will be used at the end to allow
812 * the reader to clean up its internal state
813 * @param buf where the reader should write the data
814 * @param emsg location for the reader to store an error message
815 * @return number of bytes written, usually "max", 0 on error
816 */
817size_t
818GNUNET_FS_data_reader_copy_ (void *cls, uint64_t offset, size_t max, void *buf,
819 char **emsg);
820
821/**
822 * Notification of FS that a search probe has made progress.
823 * This function is used INSTEAD of the client's event handler
824 * for downloads where the GNUNET_FS_DOWNLOAD_IS_PROBE flag is set.
825 *
826 * @param cls closure, always NULL (!), actual closure
827 * is in the client-context of the info struct
828 * @param info details about the event, specifying the event type
829 * and various bits about the event
830 * @return client-context (for the next progress call
831 * for this operation; should be set to NULL for
832 * SUSPEND and STOPPED events). The value returned
833 * will be passed to future callbacks in the respective
834 * field in the GNUNET_FS_ProgressInfo struct.
835 */
836void *
837GNUNET_FS_search_probe_progress_ (void *cls,
838 const struct GNUNET_FS_ProgressInfo *info);
839
840
841/**
842 * Main function that performs the upload.
843 *
844 * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
845 * @param tc task context
846 */
847void
848GNUNET_FS_publish_main_ (void *cls,
849 const struct GNUNET_SCHEDULER_TaskContext *tc);
850
851
852/**
853 * Function called once the hash of the file
854 * that is being unindexed has been computed.
855 *
856 * @param cls closure, unindex context
857 * @param file_id computed hash, NULL on error
858 */
859void
860GNUNET_FS_unindex_process_hash_ (void *cls, const GNUNET_HashCode * file_id);
861
862
863/**
864 * Fill in all of the generic fields for a publish event and call the
865 * callback.
866 *
867 * @param pi structure to fill in
868 * @param pc overall publishing context
869 * @param p file information for the file being published
870 * @param offset where in the file are we so far
871 * @return value returned from callback
872 */
873void *
874GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
875 struct GNUNET_FS_PublishContext *pc,
876 const struct GNUNET_FS_FileInformation *p,
877 uint64_t offset);
878
879
880/**
881 * Fill in all of the generic fields for a download event and call the
882 * callback.
883 *
884 * @param pi structure to fill in
885 * @param dc overall download context
886 */
887void
888GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
889 struct GNUNET_FS_DownloadContext *dc);
890
891
892/**
893 * Task that creates the initial (top-level) download
894 * request for the file.
895 *
896 * @param cls the 'struct GNUNET_FS_DownloadContext'
897 * @param tc scheduler context
898 */
899void
900GNUNET_FS_download_start_task_ (void *cls,
901 const struct GNUNET_SCHEDULER_TaskContext *tc);
902
903
904
905/**
906 * Fill in all of the generic fields for
907 * an unindex event and call the callback.
908 *
909 * @param pi structure to fill in
910 * @param uc overall unindex context
911 * @param offset where we are in the file (for progress)
912 */
913void
914GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
915 struct GNUNET_FS_UnindexContext *uc,
916 uint64_t offset);
917
918/**
919 * Fill in all of the generic fields for a search event and
920 * call the callback.
921 *
922 * @param pi structure to fill in
923 * @param sc overall search context
924 * @return value returned by the callback
925 */
926void *
927GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
928 struct GNUNET_FS_SearchContext *sc);
929
930
931/**
932 * Connect to the datastore and remove the blocks.
933 *
934 * @param uc context for the unindex operation.
935 */
936void
937GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc);
938
939/**
940 * Build the request and actually initiate the search using the
941 * GNUnet FS service.
942 *
943 * @param sc search context
944 * @return GNUNET_OK on success, GNUNET_SYSERR on error
945 */
946int
947GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc);
948
949/**
950 * Start the downloading process (by entering the queue).
951 *
952 * @param dc our download context
953 */
954void
955GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc);
956
957
958/**
959 * Start download probes for the given search result.
960 *
961 * @param sr the search result
962 */
963void
964GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr);
965
966/**
967 * Remove serialization/deserialization file from disk.
968 *
969 * @param h master context
970 * @param ext component of the path
971 * @param ent entity identifier
972 */
973void
974GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, const char *ext,
975 const char *ent);
976
977
978/**
979 * Remove serialization/deserialization directory from disk.
980 *
981 * @param h master context
982 * @param ext component of the path
983 * @param uni unique name of parent
984 */
985void
986GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, const char *ext,
987 const char *uni);
988
989
990/**
991 * Synchronize this file-information struct with its mirror
992 * on disk. Note that all internal FS-operations that change
993 * file information data should already call "sync" internally,
994 * so this function is likely not useful for clients.
995 *
996 * @param fi the struct to sync
997 */
998void
999GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *f);
1000
1001/**
1002 * Synchronize this publishing struct with its mirror
1003 * on disk. Note that all internal FS-operations that change
1004 * publishing structs should already call "sync" internally,
1005 * so this function is likely not useful for clients.
1006 *
1007 * @param pc the struct to sync
1008 */
1009void
1010GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc);
1011
1012/**
1013 * Synchronize this unindex struct with its mirror
1014 * on disk. Note that all internal FS-operations that change
1015 * publishing structs should already call "sync" internally,
1016 * so this function is likely not useful for clients.
1017 *
1018 * @param uc the struct to sync
1019 */
1020void
1021GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc);
1022
1023/**
1024 * Synchronize this search struct with its mirror
1025 * on disk. Note that all internal FS-operations that change
1026 * publishing structs should already call "sync" internally,
1027 * so this function is likely not useful for clients.
1028 *
1029 * @param sc the struct to sync
1030 */
1031void
1032GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc);
1033
1034/**
1035 * Synchronize this search result with its mirror
1036 * on disk. Note that all internal FS-operations that change
1037 * publishing structs should already call "sync" internally,
1038 * so this function is likely not useful for clients.
1039 *
1040 * @param sr the struct to sync
1041 */
1042void
1043GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr);
1044
1045/**
1046 * Synchronize this download struct with its mirror
1047 * on disk. Note that all internal FS-operations that change
1048 * publishing structs should already call "sync" internally,
1049 * so this function is likely not useful for clients.
1050 *
1051 * @param dc the struct to sync
1052 */
1053void
1054GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc);
1055
1056/**
1057 * Create SUSPEND event for the given publish operation
1058 * and then clean up our state (without stop signal).
1059 *
1060 * @param cls the 'struct GNUNET_FS_PublishContext' to signal for
1061 */
1062void
1063GNUNET_FS_publish_signal_suspend_ (void *cls);
1064
1065/**
1066 * Create SUSPEND event for the given search operation
1067 * and then clean up our state (without stop signal).
1068 *
1069 * @param cls the 'struct GNUNET_FS_SearchContext' to signal for
1070 */
1071void
1072GNUNET_FS_search_signal_suspend_ (void *cls);
1073
1074/**
1075 * Create SUSPEND event for the given download operation
1076 * and then clean up our state (without stop signal).
1077 *
1078 * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for
1079 */
1080void
1081GNUNET_FS_download_signal_suspend_ (void *cls);
1082
1083/**
1084 * Create SUSPEND event for the given unindex operation
1085 * and then clean up our state (without stop signal).
1086 *
1087 * @param cls the 'struct GNUNET_FS_UnindexContext' to signal for
1088 */
1089void
1090GNUNET_FS_unindex_signal_suspend_ (void *cls);
1091
1092/**
1093 * Function signature of the functions that can be called
1094 * to trigger suspend signals and clean-up for top-level
1095 * activities.
1096 *
1097 * @param cls closure
1098 */
1099typedef void (*SuspendSignalFunction) (void *cls);
1100
1101/**
1102 * We track all of the top-level activities of FS
1103 * so that we can signal 'suspend' on shutdown.
1104 */
1105struct TopLevelActivity
1106{
1107 /**
1108 * This is a doubly-linked list.
1109 */
1110 struct TopLevelActivity *next;
1111
1112 /**
1113 * This is a doubly-linked list.
1114 */
1115 struct TopLevelActivity *prev;
1116
1117 /**
1118 * Function to call for suspend-signalling and clean up.
1119 */
1120 SuspendSignalFunction ssf;
1121
1122 /**
1123 * Closure for 'ssf' (some struct GNUNET_FS_XXXHandle*)
1124 */
1125 void *ssf_cls;
1126};
1127
1128
1129/**
1130 * Create a top-level activity entry.
1131 *
1132 * @param h global fs handle
1133 * @param ssf suspend signal function to use
1134 * @param ssf_cls closure for ssf
1135 * @return fresh top-level activity handle
1136 */
1137struct TopLevelActivity *
1138GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf,
1139 void *ssf_cls);
1140
1141
1142/**
1143 * Destroy a top-level activity entry.
1144 *
1145 * @param h global fs handle
1146 * @param top top level activity entry
1147 */
1148void
1149GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top);
1150
1151
1152
1153/**
1154 * Master context for most FS operations.
1155 */
1156struct GNUNET_FS_Handle
1157{
1158 /**
1159 * Configuration to use.
1160 */
1161 const struct GNUNET_CONFIGURATION_Handle *cfg;
1162
1163 /**
1164 * Name of our client.
1165 */
1166 char *client_name;
1167
1168 /**
1169 * Function to call with updates on our progress.
1170 */
1171 GNUNET_FS_ProgressCallback upcb;
1172
1173 /**
1174 * Closure for upcb.
1175 */
1176 void *upcb_cls;
1177
1178 /**
1179 * Head of DLL of top-level activities.
1180 */
1181 struct TopLevelActivity *top_head;
1182
1183 /**
1184 * Tail of DLL of top-level activities.
1185 */
1186 struct TopLevelActivity *top_tail;
1187
1188 /**
1189 * Head of DLL of running jobs.
1190 */
1191 struct GNUNET_FS_QueueEntry *running_head;
1192
1193 /**
1194 * Tail of DLL of running jobs.
1195 */
1196 struct GNUNET_FS_QueueEntry *running_tail;
1197
1198 /**
1199 * Head of DLL of pending jobs.
1200 */
1201 struct GNUNET_FS_QueueEntry *pending_head;
1202
1203 /**
1204 * Tail of DLL of pending jobs.
1205 */
1206 struct GNUNET_FS_QueueEntry *pending_tail;
1207
1208 /**
1209 * Task that processes the jobs in the running and pending queues
1210 * (and moves jobs around as needed).
1211 */
1212 GNUNET_SCHEDULER_TaskIdentifier queue_job;
1213
1214 /**
1215 * Average time we take for a single request to be satisfied.
1216 * FIXME: not yet calcualted properly...
1217 */
1218 struct GNUNET_TIME_Relative avg_block_latency;
1219
1220 /**
1221 * How many actual downloads do we have running right now?
1222 */
1223 unsigned int active_downloads;
1224
1225 /**
1226 * How many blocks do the active downloads have?
1227 */
1228 unsigned int active_blocks;
1229
1230 /**
1231 * General flags.
1232 */
1233 enum GNUNET_FS_Flags flags;
1234
1235 /**
1236 * Maximum number of parallel downloads.
1237 */
1238 unsigned int max_parallel_downloads;
1239
1240 /**
1241 * Maximum number of parallel requests.
1242 */
1243 unsigned int max_parallel_requests;
1244
1245};
1246
1247
1248/**
1249 * Handle for controlling a publication process.
1250 */
1251struct GNUNET_FS_PublishContext
1252{
1253 /**
1254 * Handle to the global fs context.
1255 */
1256 struct GNUNET_FS_Handle *h;
1257
1258 /**
1259 * Our top-level activity entry (if we are top-level, otherwise NULL).
1260 */
1261 struct TopLevelActivity *top;
1262
1263 /**
1264 * File-structure that is being shared.
1265 */
1266 struct GNUNET_FS_FileInformation *fi;
1267
1268 /**
1269 * Namespace that we are publishing in, NULL if we have no namespace.
1270 */
1271 struct GNUNET_FS_Namespace *namespace;
1272
1273 /**
1274 * ID of the content in the namespace, NULL if we have no namespace.
1275 */
1276 char *nid;
1277
1278 /**
1279 * ID for future updates, NULL if we have no namespace or no updates.
1280 */
1281 char *nuid;
1282
1283 /**
1284 * Filename used for serializing information about this operation
1285 * (should be determined using 'mktemp').
1286 */
1287 char *serialization;
1288
1289 /**
1290 * Our own client handle for the FS service; only briefly used when
1291 * we start to index a file, otherwise NULL.
1292 */
1293 struct GNUNET_CLIENT_Connection *client;
1294
1295 /**
1296 * Current position in the file-tree for the upload.
1297 */
1298 struct GNUNET_FS_FileInformation *fi_pos;
1299
1300 /**
1301 * Non-null if we are currently hashing a file.
1302 */
1303 struct GNUNET_CRYPTO_FileHashContext *fhc;
1304
1305 /**
1306 * Connection to the datastore service.
1307 */
1308 struct GNUNET_DATASTORE_Handle *dsh;
1309
1310 /**
1311 * Queue entry for reservation/unreservation.
1312 */
1313 struct GNUNET_DATASTORE_QueueEntry *qre;
1314
1315 /**
1316 * ID of the task performing the upload. NO_TASK if the upload has
1317 * completed.
1318 */
1319 GNUNET_SCHEDULER_TaskIdentifier upload_task;
1320
1321 /**
1322 * Storage space to reserve for the operation.
1323 */
1324 uint64_t reserve_space;
1325
1326 /**
1327 * Overall number of entries to reserve for the
1328 * publish operation.
1329 */
1330 uint32_t reserve_entries;
1331
1332 /**
1333 * Typically GNUNET_NO. Set to GNUNET_YES if "upload_task" is
1334 * GNUNET_SCHEDULER_NO_TASK and we're waiting for a response from
1335 * the datastore service (in which case this struct must not be
1336 * freed until we have that response). If someone tries to stop the
1337 * download for good during this period, "in_network_wait" is set to
1338 * GNUNET_SYSERR which will cause the struct to be destroyed right
1339 * after we have the reply (or timeout) from the datastore service.
1340 */
1341 int in_network_wait;
1342
1343 /**
1344 * Options for publishing.
1345 */
1346 enum GNUNET_FS_PublishOptions options;
1347
1348 /**
1349 * Space reservation ID with datastore service
1350 * for this upload.
1351 */
1352 int rid;
1353
1354 /**
1355 * Set to GNUNET_YES if all processing has completed.
1356 */
1357 int all_done;
1358};
1359
1360
1361/**
1362 * Phases of unindex processing (state machine).
1363 */
1364enum UnindexState
1365{
1366 /**
1367 * We're currently hashing the file.
1368 */
1369 UNINDEX_STATE_HASHING = 0,
1370
1371 /**
1372 * We're telling the datastore to delete
1373 * the respective entries.
1374 */
1375 UNINDEX_STATE_DS_REMOVE = 1,
1376
1377 /**
1378 * We're notifying the FS service about
1379 * the unindexing.
1380 */
1381 UNINDEX_STATE_FS_NOTIFY = 2,
1382
1383 /**
1384 * We're done.
1385 */
1386 UNINDEX_STATE_COMPLETE = 3,
1387
1388 /**
1389 * We've encountered a fatal error.
1390 */
1391 UNINDEX_STATE_ERROR = 4
1392};
1393
1394
1395/**
1396 * Handle for controlling an unindexing operation.
1397 */
1398struct GNUNET_FS_UnindexContext
1399{
1400
1401 /**
1402 * Global FS context.
1403 */
1404 struct GNUNET_FS_Handle *h;
1405
1406 /**
1407 * Our top-level activity entry.
1408 */
1409 struct TopLevelActivity *top;
1410
1411 /**
1412 * Name of the file that we are unindexing.
1413 */
1414 char *filename;
1415
1416 /**
1417 * Short name under which we are serializing the state of this operation.
1418 */
1419 char *serialization;
1420
1421 /**
1422 * Connection to the FS service, only valid during the
1423 * UNINDEX_STATE_FS_NOTIFY phase.
1424 */
1425 struct GNUNET_CLIENT_Connection *client;
1426
1427 /**
1428 * Connection to the datastore service, only valid during the
1429 * UNINDEX_STATE_DS_NOTIFY phase.
1430 */
1431 struct GNUNET_DATASTORE_Handle *dsh;
1432
1433 /**
1434 * Pointer kept for the client.
1435 */
1436 void *client_info;
1437
1438 /**
1439 * Merkle-ish tree encoder context.
1440 */
1441 struct GNUNET_FS_TreeEncoder *tc;
1442
1443 /**
1444 * Handle used to read the file.
1445 */
1446 struct GNUNET_DISK_FileHandle *fh;
1447
1448 /**
1449 * Error message, NULL on success.
1450 */
1451 char *emsg;
1452
1453 /**
1454 * Context for hashing of the file.
1455 */
1456 struct GNUNET_CRYPTO_FileHashContext *fhc;
1457
1458 /**
1459 * Overall size of the file.
1460 */
1461 uint64_t file_size;
1462
1463 /**
1464 * When did we start?
1465 */
1466 struct GNUNET_TIME_Absolute start_time;
1467
1468 /**
1469 * Hash of the file's contents (once computed).
1470 */
1471 GNUNET_HashCode file_id;
1472
1473 /**
1474 * Current operatinonal phase.
1475 */
1476 enum UnindexState state;
1477
1478};
1479
1480
1481/**
1482 * Information we keep for each keyword in
1483 * a keyword search.
1484 */
1485struct SearchRequestEntry
1486{
1487 /**
1488 * Hash of the original keyword, also known as the
1489 * key (for decrypting the KBlock).
1490 */
1491 GNUNET_HashCode key;
1492
1493 /**
1494 * Hash of the public key, also known as the query.
1495 */
1496 GNUNET_HashCode query;
1497
1498 /**
1499 * Map that contains a "struct GNUNET_FS_SearchResult" for each result that
1500 * was found under this keyword. Note that the entries will point
1501 * to the same locations as those in the master result map (in
1502 * "struct GNUNET_FS_SearchContext"), so they should not be freed.
1503 * The key for each entry is the XOR of the key and query in the CHK
1504 * URI (as a unique identifier for the search result).
1505 */
1506 struct GNUNET_CONTAINER_MultiHashMap *results;
1507
1508 /**
1509 * Is this keyword a mandatory keyword
1510 * (started with '+')?
1511 */
1512 int mandatory;
1513
1514};
1515
1516
1517/**
1518 * Handle for controlling a search.
1519 */
1520struct GNUNET_FS_SearchContext
1521{
1522 /**
1523 * Handle to the global FS context.
1524 */
1525 struct GNUNET_FS_Handle *h;
1526
1527 /**
1528 * Our top-level activity entry (if we are top-level, otherwise NULL).
1529 */
1530 struct TopLevelActivity *top;
1531
1532 /**
1533 * List of keywords that we're looking for.
1534 */
1535 struct GNUNET_FS_Uri *uri;
1536
1537 /**
1538 * For update-searches, link to the search result that triggered
1539 * the update search; otherwise NULL.
1540 */
1541 struct GNUNET_FS_SearchResult *psearch_result;
1542
1543 /**
1544 * Connection to the FS service.
1545 */
1546 struct GNUNET_CLIENT_Connection *client;
1547
1548 /**
1549 * Pointer we keep for the client.
1550 */
1551 void *client_info;
1552
1553 /**
1554 * Name of the file on disk we use for persistence.
1555 */
1556 char *serialization;
1557
1558 /**
1559 * Error message (non-NULL if this operation failed).
1560 */
1561 char *emsg;
1562
1563 /**
1564 * Map that contains a "struct GNUNET_FS_SearchResult" for each result that
1565 * was found in the search. The key for each entry is the XOR of
1566 * the key and query in the CHK URI (as a unique identifier for the
1567 * search result).
1568 */
1569 struct GNUNET_CONTAINER_MultiHashMap *master_result_map;
1570
1571 /**
1572 * Per-keyword information for a keyword search. This array will
1573 * have exactly as many entries as there were keywords.
1574 */
1575 struct SearchRequestEntry *requests;
1576
1577 /**
1578 * When did we start?
1579 */
1580 struct GNUNET_TIME_Absolute start_time;
1581
1582 /**
1583 * ID of a task that is using this struct and that must be cancelled
1584 * when the search is being stopped (if not
1585 * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some
1586 * artificial delay when trying to reconnect to the FS service.
1587 */
1588 GNUNET_SCHEDULER_TaskIdentifier task;
1589
1590 /**
1591 * How many of the entries in the search request
1592 * map have been passed to the service so far?
1593 */
1594 unsigned int search_request_map_offset;
1595
1596 /**
1597 * How many of the keywords in the KSK
1598 * map have been passed to the service so far?
1599 */
1600 unsigned int keyword_offset;
1601
1602 /**
1603 * Anonymity level for the search.
1604 */
1605 uint32_t anonymity;
1606
1607 /**
1608 * Number of mandatory keywords in this query.
1609 */
1610 uint32_t mandatory_count;
1611
1612 /**
1613 * Options for the search.
1614 */
1615 enum GNUNET_FS_SearchOptions options;
1616};
1617
1618
1619/**
1620 * FSM for possible states a block can go through. The typical
1621 * order of progression is linear through the states, alternatives
1622 * are documented in the comments.
1623 */
1624enum BlockRequestState
1625{
1626 /**
1627 * Initial state, block has only been allocated (since it is
1628 * relevant to the overall download request).
1629 */
1630 BRS_INIT = 0,
1631
1632 /**
1633 * We've checked the block on the path down the tree, and the
1634 * content on disk did match the desired CHK, but not all
1635 * the way down, so at the bottom some blocks will still
1636 * need to be reconstructed).
1637 */
1638 BRS_RECONSTRUCT_DOWN = 1,
1639
1640 /**
1641 * We've calculated the CHK bottom-up based on the meta data.
1642 * This may work, but if it did we have to write the meta data to
1643 * disk at the end (and we still need to check against the
1644 * CHK set on top).
1645 */
1646 BRS_RECONSTRUCT_META_UP = 2,
1647
1648 /**
1649 * We've calculated the CHK bottom-up based on what we have on
1650 * disk, which may not be what the desired CHK is. If the
1651 * reconstructed CHKs match whatever comes from above, we're
1652 * done with the respective subtree.
1653 */
1654 BRS_RECONSTRUCT_UP = 3,
1655
1656 /**
1657 * We've determined the real, desired CHK for this block
1658 * (full tree reconstruction failed), request is now pending.
1659 * If the CHK that bubbled up through reconstruction did match
1660 * the top-level request, the state machine for the subtree
1661 * would have moved to BRS_DOWNLOAD_UP.
1662 */
1663 BRS_CHK_SET = 4,
1664
1665 /**
1666 * We've successfully downloaded this block, but the children
1667 * still need to be either downloaded or verified (download
1668 * request propagates down). If the download fails, the
1669 * state machine for this block may move to
1670 * BRS_DOWNLOAD_ERROR instead.
1671 */
1672 BRS_DOWNLOAD_DOWN = 5,
1673
1674 /**
1675 * This block and all of its children have been downloaded
1676 * successfully (full completion propagates up).
1677 */
1678 BRS_DOWNLOAD_UP = 6,
1679
1680 /**
1681 * We got a block back that matched the query but did not hash to
1682 * the key (malicious publisher or hash collision); this block
1683 * can never be downloaded (error propagates up).
1684 */
1685 BRS_ERROR = 7
1686};
1687
1688
1689/**
1690 * Information about an active download request.
1691 */
1692struct DownloadRequest
1693{
1694 /**
1695 * While pending, we keep all download requests in a doubly-linked list.
1696 */
1697 struct DownloadRequest *next;
1698
1699 /**
1700 * While pending, we keep all download requests in a doubly-linked list.
1701 */
1702 struct DownloadRequest *prev;
1703
1704 /**
1705 * Parent in the CHK-tree.
1706 */
1707 struct DownloadRequest *parent;
1708
1709 /**
1710 * Array (!) of child-requests, or NULL for the bottom of the tree.
1711 */
1712 struct DownloadRequest **children;
1713
1714 /**
1715 * CHK for the request for this block (set during reconstruction
1716 * to what we have on disk, later to what we want to have).
1717 */
1718 struct ContentHashKey chk;
1719
1720 /**
1721 * Offset of the corresponding block. Specifically, first (!) byte of
1722 * the first DBLOCK in the subtree induced by block represented by
1723 * this request.
1724 */
1725 uint64_t offset;
1726
1727 /**
1728 * Number of entries in 'children' array.
1729 */
1730 unsigned int num_children;
1731
1732 /**
1733 * Depth of the corresponding block in the tree. 0==DBLOCKs.
1734 */
1735 unsigned int depth;
1736
1737 /**
1738 * State in the FSM.
1739 */
1740 enum BlockRequestState state;
1741
1742 /**
1743 * GNUNET_YES if this entry is in the pending list.
1744 */
1745 int is_pending;
1746
1747};
1748
1749
1750/**
1751 * (recursively) free download request structure
1752 *
1753 * @param dr request to free
1754 */
1755void
1756GNUNET_FS_free_download_request_ (struct DownloadRequest *dr);
1757
1758
1759/**
1760 * Context for controlling a download.
1761 */
1762struct GNUNET_FS_DownloadContext
1763{
1764
1765 /**
1766 * Global FS context.
1767 */
1768 struct GNUNET_FS_Handle *h;
1769
1770 /**
1771 * Our top-level activity entry (if we are top-level, otherwise NULL).
1772 */
1773 struct TopLevelActivity *top;
1774
1775 /**
1776 * Connection to the FS service.
1777 */
1778 struct GNUNET_CLIENT_Connection *client;
1779
1780 /**
1781 * Parent download (used when downloading files
1782 * in directories).
1783 */
1784 struct GNUNET_FS_DownloadContext *parent;
1785
1786 /**
1787 * Associated search (used when downloading files
1788 * based on search results), or NULL for none.
1789 */
1790 struct GNUNET_FS_SearchResult *search;
1791
1792 /**
1793 * Head of list of child downloads.
1794 */
1795 struct GNUNET_FS_DownloadContext *child_head;
1796
1797 /**
1798 * Tail of list of child downloads.
1799 */
1800 struct GNUNET_FS_DownloadContext *child_tail;
1801
1802 /**
1803 * Previous download belonging to the same parent.
1804 */
1805 struct GNUNET_FS_DownloadContext *prev;
1806
1807 /**
1808 * Next download belonging to the same parent.
1809 */
1810 struct GNUNET_FS_DownloadContext *next;
1811
1812 /**
1813 * Context kept for the client.
1814 */
1815 void *client_info;
1816
1817 /**
1818 * URI that identifies the file that we are downloading.
1819 */
1820 struct GNUNET_FS_Uri *uri;
1821
1822 /**
1823 * Known meta-data for the file (can be NULL).
1824 */
1825 struct GNUNET_CONTAINER_MetaData *meta;
1826
1827 /**
1828 * Error message, NULL if we're doing OK.
1829 */
1830 char *emsg;
1831
1832 /**
1833 * Random portion of filename we use for syncing state of this
1834 * download.
1835 */
1836 char *serialization;
1837
1838 /**
1839 * Where are we writing the data (name of the
1840 * file, can be NULL!).
1841 */
1842 char *filename;
1843
1844 /**
1845 * Where are we writing the data temporarily (name of the
1846 * file, can be NULL!); used if we do not have a permanent
1847 * name and we are a directory and we do a recursive download.
1848 */
1849 char *temp_filename;
1850
1851 /**
1852 * Our entry in the job queue.
1853 */
1854 struct GNUNET_FS_QueueEntry *job_queue;
1855
1856 /**
1857 * Non-NULL if we are currently having a request for
1858 * transmission pending with the client handle.
1859 */
1860 struct GNUNET_CLIENT_TransmitHandle *th;
1861
1862 /**
1863 * Tree encoder used for the reconstruction.
1864 */
1865 struct GNUNET_FS_TreeEncoder *te;
1866
1867 /**
1868 * File handle for reading data from an existing file
1869 * (to pass to tree encoder).
1870 */
1871 struct GNUNET_DISK_FileHandle *rfh;
1872
1873 /**
1874 * Map of active requests (those waiting for a response). The key
1875 * is the hash of the encryped block (aka query).
1876 */
1877 struct GNUNET_CONTAINER_MultiHashMap *active;
1878
1879 /**
1880 * Head of linked list of pending requests.
1881 */
1882 struct DownloadRequest *pending_head;
1883
1884 /**
1885 * Head of linked list of pending requests.
1886 */
1887 struct DownloadRequest *pending_tail;
1888
1889 /**
1890 * Top-level download request.
1891 */
1892 struct DownloadRequest *top_request;
1893
1894 /**
1895 * Identity of the peer having the content, or all-zeros
1896 * if we don't know of such a peer.
1897 */
1898 struct GNUNET_PeerIdentity target;
1899
1900 /**
1901 * ID of a task that is using this struct and that must be cancelled
1902 * when the download is being stopped (if not
1903 * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some
1904 * artificial delay when trying to reconnect to the FS service or
1905 * the task processing incrementally the data on disk, or the
1906 * task requesting blocks, etc.
1907 */
1908 GNUNET_SCHEDULER_TaskIdentifier task;
1909
1910 /**
1911 * What is the first offset that we're interested
1912 * in?
1913 */
1914 uint64_t offset;
1915
1916 /**
1917 * How many bytes starting from offset are desired?
1918 * This is NOT the overall length of the file!
1919 */
1920 uint64_t length;
1921
1922 /**
1923 * How many bytes have we already received within
1924 * the specified range (DBlocks only).
1925 */
1926 uint64_t completed;
1927
1928 /**
1929 * What was the size of the file on disk that we're downloading
1930 * before we started? Used to detect if there is a point in
1931 * checking an existing block on disk for matching the desired
1932 * content. 0 if the file did not exist already.
1933 */
1934 uint64_t old_file_size;
1935
1936 /**
1937 * Time download was started.
1938 */
1939 struct GNUNET_TIME_Absolute start_time;
1940
1941 /**
1942 * Desired level of anonymity.
1943 */
1944 uint32_t anonymity;
1945
1946 /**
1947 * The depth of the file-tree.
1948 */
1949 unsigned int treedepth;
1950
1951 /**
1952 * Options for the download.
1953 */
1954 enum GNUNET_FS_DownloadOptions options;
1955
1956 /**
1957 * Flag set upon transitive completion (includes child downloads).
1958 * This flag is only set to GNUNET_YES for directories where all
1959 * child-downloads have also completed (and signalled completion).
1960 */
1961 int has_finished;
1962
1963 /**
1964 * Have we started the receive continuation yet?
1965 */
1966 int in_receive;
1967
1968};
1969
1970
1971/**
1972 * Information about an (updateable) node in the
1973 * namespace.
1974 */
1975struct NamespaceUpdateNode
1976{
1977 /**
1978 * Identifier for this node.
1979 */
1980 char *id;
1981
1982 /**
1983 * Identifier of children of this node.
1984 */
1985 char *update;
1986
1987 /**
1988 * Metadata for this entry.
1989 */
1990 struct GNUNET_CONTAINER_MetaData *md;
1991
1992 /**
1993 * URI of this entry in the namespace.
1994 */
1995 struct GNUNET_FS_Uri *uri;
1996
1997 /**
1998 * Namespace update generation ID. Used to ensure
1999 * freshness of the tree_id.
2000 */
2001 unsigned int nug;
2002
2003 /**
2004 * TREE this entry belongs to (if nug is current).
2005 */
2006 unsigned int tree_id;
2007
2008};
2009
2010
2011struct GNUNET_FS_Namespace
2012{
2013
2014 /**
2015 * Handle to the FS service context.
2016 */
2017 struct GNUNET_FS_Handle *h;
2018
2019 /**
2020 * Array with information about nodes in the namespace.
2021 */
2022 struct NamespaceUpdateNode **update_nodes;
2023
2024 /**
2025 * Private key for the namespace.
2026 */
2027 struct GNUNET_CRYPTO_RsaPrivateKey *key;
2028
2029 /**
2030 * Hash map mapping identifiers of update nodes
2031 * to the update nodes (initialized on-demand).
2032 */
2033 struct GNUNET_CONTAINER_MultiHashMap *update_map;
2034
2035 /**
2036 * Name of the file with the private key.
2037 */
2038 char *filename;
2039
2040 /**
2041 * Name of the namespace.
2042 */
2043 char *name;
2044
2045 /**
2046 * Size of the update nodes array.
2047 */
2048 unsigned int update_node_count;
2049
2050 /**
2051 * Reference counter.
2052 */
2053 unsigned int rc;
2054
2055 /**
2056 * Generator for unique nug numbers.
2057 */
2058 unsigned int nug_gen;
2059};
2060
2061
2062/**
2063 * Message sent from a GNUnet (fs) publishing activity to the 68 * Message sent from a GNUnet (fs) publishing activity to the
2064 * gnunet-fs-service to initiate indexing of a file. The service is 69 * gnunet-fs-service to initiate indexing of a file. The service is
2065 * supposed to check if the specified file is available and has the 70 * supposed to check if the specified file is available and has the
diff --git a/src/fs/fs.c b/src/fs/fs_api.c
index 83c0bf271..ce4b3b887 100644
--- a/src/fs/fs.c
+++ b/src/fs/fs_api.c
@@ -19,7 +19,7 @@
19*/ 19*/
20 20
21/** 21/**
22 * @file fs/fs.c 22 * @file fs/fs_api.c
23 * @brief main FS functions (master initialization, serialization, deserialization, shared code) 23 * @brief main FS functions (master initialization, serialization, deserialization, shared code)
24 * @author Christian Grothoff 24 * @author Christian Grothoff
25 */ 25 */
@@ -27,7 +27,7 @@
27#include "platform.h" 27#include "platform.h"
28#include "gnunet_util_lib.h" 28#include "gnunet_util_lib.h"
29#include "gnunet_fs_service.h" 29#include "gnunet_fs_service.h"
30#include "fs.h" 30#include "fs_api.h"
31#include "fs_tree.h" 31#include "fs_tree.h"
32 32
33 33
diff --git a/src/fs/fs_api.h b/src/fs/fs_api.h
new file mode 100644
index 000000000..f358047aa
--- /dev/null
+++ b/src/fs/fs_api.h
@@ -0,0 +1,1907 @@
1/*
2 This file is part of GNUnet.
3 (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors)
4
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
19*/
20
21/**
22 * @file fs/fs_api.h
23 * @brief shared definitions for the FS library
24 * @author Igor Wronsky, Christian Grothoff
25 */
26#ifndef FS_API_H
27#define FS_API_H
28
29#include "gnunet_constants.h"
30#include "gnunet_datastore_service.h"
31#include "gnunet_dht_service.h"
32#include "gnunet_fs_service.h"
33#include "gnunet_block_lib.h"
34#include "block_fs.h"
35#include "fs.h"
36
37/**
38 * Size of the individual blocks used for file-sharing.
39 */
40#define DBLOCK_SIZE (32*1024)
41
42/**
43 * Pick a multiple of 2 here to achive 8-byte alignment! We also
44 * probably want DBlocks to have (roughly) the same size as IBlocks.
45 * With SHA-512, the optimal value is 32768 byte / 128 byte = 256 (128
46 * byte = 2 * 512 bits). DO NOT CHANGE!
47 */
48#define CHK_PER_INODE 256
49
50/**
51 * Maximum size for a file to be considered for inlining in a
52 * directory.
53 */
54#define MAX_INLINE_SIZE 65536
55
56/**
57 * Name of the directory with top-level searches.
58 */
59#define GNUNET_FS_SYNC_PATH_MASTER_SEARCH "search"
60
61/**
62 * Name of the directory with sub-searches (namespace-updates).
63 */
64#define GNUNET_FS_SYNC_PATH_CHILD_SEARCH "search-child"
65
66/**
67 * Name of the directory with master downloads (not associated
68 * with search or part of another download).
69 */
70#define GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD "download"
71
72/**
73 * Name of the directory with downloads that are part of another
74 * download or a search.
75 */
76#define GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD "download-child"
77
78/**
79 * Name of the directory with publishing operations.
80 */
81#define GNUNET_FS_SYNC_PATH_MASTER_PUBLISH "publish"
82
83/**
84 * Name of the directory with files that are being published
85 */
86#define GNUNET_FS_SYNC_PATH_FILE_INFO "publish-file"
87
88/**
89 * Name of the directory with unindex operations.
90 */
91#define GNUNET_FS_SYNC_PATH_MASTER_UNINDEX "unindex"
92
93
94/**
95 * @brief complete information needed
96 * to download a file.
97 */
98struct FileIdentifier
99{
100
101 /**
102 * Total size of the file in bytes. (network byte order (!))
103 */
104 uint64_t file_length;
105
106 /**
107 * Query and key of the top GNUNET_EC_IBlock.
108 */
109 struct ContentHashKey chk;
110
111};
112
113
114/**
115 * Information about a file and its location
116 * (peer claiming to share the file).
117 */
118struct Location
119{
120 /**
121 * Information about the shared file.
122 */
123 struct FileIdentifier fi;
124
125 /**
126 * Identity of the peer sharing the file.
127 */
128 struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded peer;
129
130 /**
131 * Time when this location URI expires.
132 */
133 struct GNUNET_TIME_Absolute expirationTime;
134
135 /**
136 * RSA signature over the GNUNET_EC_FileIdentifier,
137 * GNUNET_hash of the peer and expiration time.
138 */
139 struct GNUNET_CRYPTO_RsaSignature contentSignature;
140
141};
142
143/**
144 * Types of URIs.
145 */
146enum uri_types
147{
148 /**
149 * Content-hash-key (simple file).
150 */
151 chk,
152
153 /**
154 * Signed key space (file in namespace).
155 */
156 sks,
157
158 /**
159 * Keyword search key (query with keywords).
160 */
161 ksk,
162
163 /**
164 * Location (chk with identity of hosting peer).
165 */
166 loc
167};
168
169/**
170 * A Universal Resource Identifier (URI), opaque.
171 */
172struct GNUNET_FS_Uri
173{
174 /**
175 * Type of the URI.
176 */
177 enum uri_types type;
178
179 union
180 {
181 struct
182 {
183 /**
184 * Keywords start with a '+' if they are
185 * mandatory (in which case the '+' is NOT
186 * part of the keyword) and with a
187 * simple space if they are optional
188 * (in which case the space is ALSO not
189 * part of the actual keyword).
190 *
191 * Double-quotes to protect spaces and
192 * %-encoding are NOT used internally
193 * (only in URI-strings).
194 */
195 char **keywords;
196
197 /**
198 * Size of the keywords array.
199 */
200 unsigned int keywordCount;
201 } ksk;
202
203 struct
204 {
205 /**
206 * Hash of the public key for the namespace.
207 */
208 GNUNET_HashCode namespace;
209
210 /**
211 * Human-readable identifier chosen for this
212 * entry in the namespace.
213 */
214 char *identifier;
215 } sks;
216
217 /**
218 * Information needed to retrieve a file (content-hash-key
219 * plus file size).
220 */
221 struct FileIdentifier chk;
222
223 /**
224 * Information needed to retrieve a file including signed
225 * location (identity of a peer) of the content.
226 */
227 struct Location loc;
228 } data;
229
230};
231
232
233/**
234 * Information for a file or directory that is
235 * about to be published.
236 */
237struct GNUNET_FS_FileInformation
238{
239
240 /**
241 * Files in a directory are kept as a linked list.
242 */
243 struct GNUNET_FS_FileInformation *next;
244
245 /**
246 * If this is a file in a directory, "dir" refers to
247 * the directory; otherwise NULL.
248 */
249 struct GNUNET_FS_FileInformation *dir;
250
251 /**
252 * Handle to the master context.
253 */
254 struct GNUNET_FS_Handle *h;
255
256 /**
257 * Pointer kept for the client.
258 */
259 void *client_info;
260
261 /**
262 * Metadata to use for the file.
263 */
264 struct GNUNET_CONTAINER_MetaData *meta;
265
266 /**
267 * Keywords to use for KBlocks.
268 */
269 struct GNUNET_FS_Uri *keywords;
270
271 /**
272 * CHK for this file or directory. NULL if
273 * we have not yet computed it.
274 */
275 struct GNUNET_FS_Uri *chk_uri;
276
277 /**
278 * Block options for the file.
279 */
280 struct GNUNET_FS_BlockOptions bo;
281
282 /**
283 * At what time did we start this upload?
284 */
285 struct GNUNET_TIME_Absolute start_time;
286
287 /**
288 * Under what filename is this struct serialized
289 * (for operational persistence). Should be determined
290 * using 'mktemp'.
291 */
292 char *serialization;
293
294 /**
295 * Encoder being used to publish this file.
296 */
297 struct GNUNET_FS_TreeEncoder *te;
298
299 /**
300 * Error message (non-NULL if this operation failed).
301 */
302 char *emsg;
303
304 /**
305 * Name of the file or directory (must be an absolute path).
306 */
307 char *filename;
308
309 /**
310 * Data describing either the file or the directory.
311 */
312 union
313 {
314
315 /**
316 * Data for a file.
317 */
318 struct
319 {
320
321 /**
322 * Function that can be used to read the data for the file.
323 */
324 GNUNET_FS_DataReader reader;
325
326 /**
327 * Closure for reader.
328 */
329 void *reader_cls;
330
331 /**
332 * If this file is being indexed, this value is set to the hash
333 * over the entire file (when the indexing process is started).
334 * Otherwise this field is not used.
335 */
336 GNUNET_HashCode file_id;
337
338 /**
339 * Size of the file (in bytes).
340 */
341 uint64_t file_size;
342
343 /**
344 * Should the file be indexed or inserted?
345 */
346 int do_index;
347
348 /**
349 * Is "file_id" already valid? Set to GNUNET_YES once the hash
350 * has been calculated.
351 */
352 int have_hash;
353
354 /**
355 * Has the service confirmed our INDEX_START request?
356 * GNUNET_YES if this step has been completed.
357 */
358 int index_start_confirmed;
359
360 } file;
361
362 /**
363 * Data for a directory.
364 */
365 struct
366 {
367
368 /**
369 * Linked list of entries in the directory.
370 */
371 struct GNUNET_FS_FileInformation *entries;
372
373 /**
374 * Size of the directory itself (in bytes); 0 if the
375 * size has not yet been calculated.
376 */
377 size_t dir_size;
378
379 /**
380 * Pointer to the data for the directory (or NULL if not
381 * available).
382 */
383 void *dir_data;
384
385 } dir;
386
387 } data;
388
389 /**
390 * Is this struct for a file or directory?
391 */
392 int is_directory;
393
394 /**
395 * Are we done publishing this file?
396 */
397 int is_published;
398
399};
400
401
402/**
403 * The job is now ready to run and should use the given client
404 * handle to communicate with the FS service.
405 *
406 * @param cls closure
407 * @param client handle to use for FS communication
408 */
409typedef void (*GNUNET_FS_QueueStart) (void *cls,
410 struct GNUNET_CLIENT_Connection * client);
411
412
413/**
414 * The job must now stop to run and should destry the client handle as
415 * soon as possible (ideally prior to returning).
416 */
417typedef void (*GNUNET_FS_QueueStop) (void *cls);
418
419
420/**
421 * Entry in the job queue.
422 */
423struct GNUNET_FS_QueueEntry
424{
425 /**
426 * This is a linked list.
427 */
428 struct GNUNET_FS_QueueEntry *next;
429
430 /**
431 * This is a linked list.
432 */
433 struct GNUNET_FS_QueueEntry *prev;
434
435 /**
436 * Function to call when the job is started.
437 */
438 GNUNET_FS_QueueStart start;
439
440 /**
441 * Function to call when the job needs to stop (or is done / dequeued).
442 */
443 GNUNET_FS_QueueStop stop;
444
445 /**
446 * Closure for start and stop.
447 */
448 void *cls;
449
450 /**
451 * Handle to FS primary context.
452 */
453 struct GNUNET_FS_Handle *h;
454
455 /**
456 * Client handle, or NULL if job is not running.
457 */
458 struct GNUNET_CLIENT_Connection *client;
459
460 /**
461 * Time the job was originally queued.
462 */
463 struct GNUNET_TIME_Absolute queue_time;
464
465 /**
466 * Time the job was started last.
467 */
468 struct GNUNET_TIME_Absolute start_time;
469
470 /**
471 * Total amount of time the job has been running (except for the
472 * current run).
473 */
474 struct GNUNET_TIME_Relative run_time;
475
476 /**
477 * How many blocks do the active downloads have?
478 */
479 unsigned int blocks;
480
481 /**
482 * How often have we (re)started this download?
483 */
484 unsigned int start_times;
485
486};
487
488
489
490
491/**
492 * Information we store for each search result.
493 */
494struct GNUNET_FS_SearchResult
495{
496
497 /**
498 * Search context this result belongs to.
499 */
500 struct GNUNET_FS_SearchContext *sc;
501
502 /**
503 * URI to which this search result refers to.
504 */
505 struct GNUNET_FS_Uri *uri;
506
507 /**
508 * Metadata for the search result.
509 */
510 struct GNUNET_CONTAINER_MetaData *meta;
511
512 /**
513 * Client info for this search result.
514 */
515 void *client_info;
516
517 /**
518 * ID of a job that is currently probing this results' availability
519 * (NULL if we are not currently probing).
520 */
521 struct GNUNET_FS_DownloadContext *probe_ctx;
522
523 /**
524 * ID of an associated download based on this search result (or
525 * NULL for none).
526 */
527 struct GNUNET_FS_DownloadContext *download;
528
529 /**
530 * If this search result triggered an update search, this field
531 * links to the update search.
532 */
533 struct GNUNET_FS_SearchContext *update_search;
534
535 /**
536 * Name under which this search result is stored on disk.
537 */
538 char *serialization;
539
540 /**
541 * Key for the search result
542 */
543 GNUNET_HashCode key;
544
545 /**
546 * ID of the task that will clean up the probe_ctx should it not
547 * complete on time (and that will need to be cancelled if we clean
548 * up the search result before then).
549 */
550 GNUNET_SCHEDULER_TaskIdentifier probe_cancel_task;
551
552 /**
553 * When did the current probe become active?
554 */
555 struct GNUNET_TIME_Absolute probe_active_time;
556
557 /**
558 * How much longer should we run the current probe before giving up?
559 */
560 struct GNUNET_TIME_Relative remaining_probe_time;
561
562 /**
563 * Number of mandatory keywords for which we have NOT yet found the
564 * search result; when this value hits zero, the search result is
565 * given to the callback.
566 */
567 uint32_t mandatory_missing;
568
569 /**
570 * Number of optional keywords under which this result was also
571 * found.
572 */
573 uint32_t optional_support;
574
575 /**
576 * Number of availability tests that have succeeded for this result.
577 */
578 uint32_t availability_success;
579
580 /**
581 * Number of availability trials that we have performed for this
582 * search result.
583 */
584 uint32_t availability_trials;
585
586};
587
588
589/**
590 * Add a job to the queue.
591 *
592 * @param h handle to the overall FS state
593 * @param start function to call to begin the job
594 * @param stop function to call to pause the job, or on dequeue (if the job was running)
595 * @param cls closure for start and stop
596 * @param blocks number of blocks this download has
597 * @return queue handle
598 */
599struct GNUNET_FS_QueueEntry *
600GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, GNUNET_FS_QueueStart start,
601 GNUNET_FS_QueueStop stop, void *cls, unsigned int blocks);
602
603
604/**
605 * Dequeue a job from the queue.
606 * @param qh handle for the job
607 */
608void
609GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh);
610
611
612/**
613 * Function that provides data by reading from a file.
614 *
615 * @param cls closure (points to the file information)
616 * @param offset offset to read from; it is possible
617 * that the caller might need to go backwards
618 * a bit at times
619 * @param max maximum number of bytes that should be
620 * copied to buf; readers are not allowed
621 * to provide less data unless there is an error;
622 * a value of "0" will be used at the end to allow
623 * the reader to clean up its internal state
624 * @param buf where the reader should write the data
625 * @param emsg location for the reader to store an error message
626 * @return number of bytes written, usually "max", 0 on error
627 */
628size_t
629GNUNET_FS_data_reader_file_ (void *cls, uint64_t offset, size_t max, void *buf,
630 char **emsg);
631
632
633/**
634 * Create the closure for the 'GNUNET_FS_data_reader_file_' callback.
635 *
636 * @param filename file to read
637 * @return closure to use
638 */
639void *
640GNUNET_FS_make_file_reader_context_ (const char *filename);
641
642
643
644/**
645 * Function that provides data by copying from a buffer.
646 *
647 * @param cls closure (points to the buffer)
648 * @param offset offset to read from; it is possible
649 * that the caller might need to go backwards
650 * a bit at times
651 * @param max maximum number of bytes that should be
652 * copied to buf; readers are not allowed
653 * to provide less data unless there is an error;
654 * a value of "0" will be used at the end to allow
655 * the reader to clean up its internal state
656 * @param buf where the reader should write the data
657 * @param emsg location for the reader to store an error message
658 * @return number of bytes written, usually "max", 0 on error
659 */
660size_t
661GNUNET_FS_data_reader_copy_ (void *cls, uint64_t offset, size_t max, void *buf,
662 char **emsg);
663
664/**
665 * Notification of FS that a search probe has made progress.
666 * This function is used INSTEAD of the client's event handler
667 * for downloads where the GNUNET_FS_DOWNLOAD_IS_PROBE flag is set.
668 *
669 * @param cls closure, always NULL (!), actual closure
670 * is in the client-context of the info struct
671 * @param info details about the event, specifying the event type
672 * and various bits about the event
673 * @return client-context (for the next progress call
674 * for this operation; should be set to NULL for
675 * SUSPEND and STOPPED events). The value returned
676 * will be passed to future callbacks in the respective
677 * field in the GNUNET_FS_ProgressInfo struct.
678 */
679void *
680GNUNET_FS_search_probe_progress_ (void *cls,
681 const struct GNUNET_FS_ProgressInfo *info);
682
683
684/**
685 * Main function that performs the upload.
686 *
687 * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
688 * @param tc task context
689 */
690void
691GNUNET_FS_publish_main_ (void *cls,
692 const struct GNUNET_SCHEDULER_TaskContext *tc);
693
694
695/**
696 * Function called once the hash of the file
697 * that is being unindexed has been computed.
698 *
699 * @param cls closure, unindex context
700 * @param file_id computed hash, NULL on error
701 */
702void
703GNUNET_FS_unindex_process_hash_ (void *cls, const GNUNET_HashCode * file_id);
704
705
706/**
707 * Fill in all of the generic fields for a publish event and call the
708 * callback.
709 *
710 * @param pi structure to fill in
711 * @param pc overall publishing context
712 * @param p file information for the file being published
713 * @param offset where in the file are we so far
714 * @return value returned from callback
715 */
716void *
717GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
718 struct GNUNET_FS_PublishContext *pc,
719 const struct GNUNET_FS_FileInformation *p,
720 uint64_t offset);
721
722
723/**
724 * Fill in all of the generic fields for a download event and call the
725 * callback.
726 *
727 * @param pi structure to fill in
728 * @param dc overall download context
729 */
730void
731GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
732 struct GNUNET_FS_DownloadContext *dc);
733
734
735/**
736 * Task that creates the initial (top-level) download
737 * request for the file.
738 *
739 * @param cls the 'struct GNUNET_FS_DownloadContext'
740 * @param tc scheduler context
741 */
742void
743GNUNET_FS_download_start_task_ (void *cls,
744 const struct GNUNET_SCHEDULER_TaskContext *tc);
745
746
747
748/**
749 * Fill in all of the generic fields for
750 * an unindex event and call the callback.
751 *
752 * @param pi structure to fill in
753 * @param uc overall unindex context
754 * @param offset where we are in the file (for progress)
755 */
756void
757GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
758 struct GNUNET_FS_UnindexContext *uc,
759 uint64_t offset);
760
761/**
762 * Fill in all of the generic fields for a search event and
763 * call the callback.
764 *
765 * @param pi structure to fill in
766 * @param sc overall search context
767 * @return value returned by the callback
768 */
769void *
770GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
771 struct GNUNET_FS_SearchContext *sc);
772
773
774/**
775 * Connect to the datastore and remove the blocks.
776 *
777 * @param uc context for the unindex operation.
778 */
779void
780GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc);
781
782/**
783 * Build the request and actually initiate the search using the
784 * GNUnet FS service.
785 *
786 * @param sc search context
787 * @return GNUNET_OK on success, GNUNET_SYSERR on error
788 */
789int
790GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc);
791
792/**
793 * Start the downloading process (by entering the queue).
794 *
795 * @param dc our download context
796 */
797void
798GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc);
799
800
801/**
802 * Start download probes for the given search result.
803 *
804 * @param sr the search result
805 */
806void
807GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr);
808
809/**
810 * Remove serialization/deserialization file from disk.
811 *
812 * @param h master context
813 * @param ext component of the path
814 * @param ent entity identifier
815 */
816void
817GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, const char *ext,
818 const char *ent);
819
820
821/**
822 * Remove serialization/deserialization directory from disk.
823 *
824 * @param h master context
825 * @param ext component of the path
826 * @param uni unique name of parent
827 */
828void
829GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, const char *ext,
830 const char *uni);
831
832
833/**
834 * Synchronize this file-information struct with its mirror
835 * on disk. Note that all internal FS-operations that change
836 * file information data should already call "sync" internally,
837 * so this function is likely not useful for clients.
838 *
839 * @param fi the struct to sync
840 */
841void
842GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *f);
843
844/**
845 * Synchronize this publishing struct with its mirror
846 * on disk. Note that all internal FS-operations that change
847 * publishing structs should already call "sync" internally,
848 * so this function is likely not useful for clients.
849 *
850 * @param pc the struct to sync
851 */
852void
853GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc);
854
855/**
856 * Synchronize this unindex struct with its mirror
857 * on disk. Note that all internal FS-operations that change
858 * publishing structs should already call "sync" internally,
859 * so this function is likely not useful for clients.
860 *
861 * @param uc the struct to sync
862 */
863void
864GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc);
865
866/**
867 * Synchronize this search struct with its mirror
868 * on disk. Note that all internal FS-operations that change
869 * publishing structs should already call "sync" internally,
870 * so this function is likely not useful for clients.
871 *
872 * @param sc the struct to sync
873 */
874void
875GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc);
876
877/**
878 * Synchronize this search result with its mirror
879 * on disk. Note that all internal FS-operations that change
880 * publishing structs should already call "sync" internally,
881 * so this function is likely not useful for clients.
882 *
883 * @param sr the struct to sync
884 */
885void
886GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr);
887
888/**
889 * Synchronize this download struct with its mirror
890 * on disk. Note that all internal FS-operations that change
891 * publishing structs should already call "sync" internally,
892 * so this function is likely not useful for clients.
893 *
894 * @param dc the struct to sync
895 */
896void
897GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc);
898
899/**
900 * Create SUSPEND event for the given publish operation
901 * and then clean up our state (without stop signal).
902 *
903 * @param cls the 'struct GNUNET_FS_PublishContext' to signal for
904 */
905void
906GNUNET_FS_publish_signal_suspend_ (void *cls);
907
908/**
909 * Create SUSPEND event for the given search operation
910 * and then clean up our state (without stop signal).
911 *
912 * @param cls the 'struct GNUNET_FS_SearchContext' to signal for
913 */
914void
915GNUNET_FS_search_signal_suspend_ (void *cls);
916
917/**
918 * Create SUSPEND event for the given download operation
919 * and then clean up our state (without stop signal).
920 *
921 * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for
922 */
923void
924GNUNET_FS_download_signal_suspend_ (void *cls);
925
926/**
927 * Create SUSPEND event for the given unindex operation
928 * and then clean up our state (without stop signal).
929 *
930 * @param cls the 'struct GNUNET_FS_UnindexContext' to signal for
931 */
932void
933GNUNET_FS_unindex_signal_suspend_ (void *cls);
934
935/**
936 * Function signature of the functions that can be called
937 * to trigger suspend signals and clean-up for top-level
938 * activities.
939 *
940 * @param cls closure
941 */
942typedef void (*SuspendSignalFunction) (void *cls);
943
944/**
945 * We track all of the top-level activities of FS
946 * so that we can signal 'suspend' on shutdown.
947 */
948struct TopLevelActivity
949{
950 /**
951 * This is a doubly-linked list.
952 */
953 struct TopLevelActivity *next;
954
955 /**
956 * This is a doubly-linked list.
957 */
958 struct TopLevelActivity *prev;
959
960 /**
961 * Function to call for suspend-signalling and clean up.
962 */
963 SuspendSignalFunction ssf;
964
965 /**
966 * Closure for 'ssf' (some struct GNUNET_FS_XXXHandle*)
967 */
968 void *ssf_cls;
969};
970
971
972/**
973 * Create a top-level activity entry.
974 *
975 * @param h global fs handle
976 * @param ssf suspend signal function to use
977 * @param ssf_cls closure for ssf
978 * @return fresh top-level activity handle
979 */
980struct TopLevelActivity *
981GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf,
982 void *ssf_cls);
983
984
985/**
986 * Destroy a top-level activity entry.
987 *
988 * @param h global fs handle
989 * @param top top level activity entry
990 */
991void
992GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top);
993
994
995
996/**
997 * Master context for most FS operations.
998 */
999struct GNUNET_FS_Handle
1000{
1001 /**
1002 * Configuration to use.
1003 */
1004 const struct GNUNET_CONFIGURATION_Handle *cfg;
1005
1006 /**
1007 * Name of our client.
1008 */
1009 char *client_name;
1010
1011 /**
1012 * Function to call with updates on our progress.
1013 */
1014 GNUNET_FS_ProgressCallback upcb;
1015
1016 /**
1017 * Closure for upcb.
1018 */
1019 void *upcb_cls;
1020
1021 /**
1022 * Head of DLL of top-level activities.
1023 */
1024 struct TopLevelActivity *top_head;
1025
1026 /**
1027 * Tail of DLL of top-level activities.
1028 */
1029 struct TopLevelActivity *top_tail;
1030
1031 /**
1032 * Head of DLL of running jobs.
1033 */
1034 struct GNUNET_FS_QueueEntry *running_head;
1035
1036 /**
1037 * Tail of DLL of running jobs.
1038 */
1039 struct GNUNET_FS_QueueEntry *running_tail;
1040
1041 /**
1042 * Head of DLL of pending jobs.
1043 */
1044 struct GNUNET_FS_QueueEntry *pending_head;
1045
1046 /**
1047 * Tail of DLL of pending jobs.
1048 */
1049 struct GNUNET_FS_QueueEntry *pending_tail;
1050
1051 /**
1052 * Task that processes the jobs in the running and pending queues
1053 * (and moves jobs around as needed).
1054 */
1055 GNUNET_SCHEDULER_TaskIdentifier queue_job;
1056
1057 /**
1058 * Average time we take for a single request to be satisfied.
1059 * FIXME: not yet calcualted properly...
1060 */
1061 struct GNUNET_TIME_Relative avg_block_latency;
1062
1063 /**
1064 * How many actual downloads do we have running right now?
1065 */
1066 unsigned int active_downloads;
1067
1068 /**
1069 * How many blocks do the active downloads have?
1070 */
1071 unsigned int active_blocks;
1072
1073 /**
1074 * General flags.
1075 */
1076 enum GNUNET_FS_Flags flags;
1077
1078 /**
1079 * Maximum number of parallel downloads.
1080 */
1081 unsigned int max_parallel_downloads;
1082
1083 /**
1084 * Maximum number of parallel requests.
1085 */
1086 unsigned int max_parallel_requests;
1087
1088};
1089
1090
1091/**
1092 * Handle for controlling a publication process.
1093 */
1094struct GNUNET_FS_PublishContext
1095{
1096 /**
1097 * Handle to the global fs context.
1098 */
1099 struct GNUNET_FS_Handle *h;
1100
1101 /**
1102 * Our top-level activity entry (if we are top-level, otherwise NULL).
1103 */
1104 struct TopLevelActivity *top;
1105
1106 /**
1107 * File-structure that is being shared.
1108 */
1109 struct GNUNET_FS_FileInformation *fi;
1110
1111 /**
1112 * Namespace that we are publishing in, NULL if we have no namespace.
1113 */
1114 struct GNUNET_FS_Namespace *namespace;
1115
1116 /**
1117 * ID of the content in the namespace, NULL if we have no namespace.
1118 */
1119 char *nid;
1120
1121 /**
1122 * ID for future updates, NULL if we have no namespace or no updates.
1123 */
1124 char *nuid;
1125
1126 /**
1127 * Filename used for serializing information about this operation
1128 * (should be determined using 'mktemp').
1129 */
1130 char *serialization;
1131
1132 /**
1133 * Our own client handle for the FS service; only briefly used when
1134 * we start to index a file, otherwise NULL.
1135 */
1136 struct GNUNET_CLIENT_Connection *client;
1137
1138 /**
1139 * Current position in the file-tree for the upload.
1140 */
1141 struct GNUNET_FS_FileInformation *fi_pos;
1142
1143 /**
1144 * Non-null if we are currently hashing a file.
1145 */
1146 struct GNUNET_CRYPTO_FileHashContext *fhc;
1147
1148 /**
1149 * Connection to the datastore service.
1150 */
1151 struct GNUNET_DATASTORE_Handle *dsh;
1152
1153 /**
1154 * Queue entry for reservation/unreservation.
1155 */
1156 struct GNUNET_DATASTORE_QueueEntry *qre;
1157
1158 /**
1159 * ID of the task performing the upload. NO_TASK if the upload has
1160 * completed.
1161 */
1162 GNUNET_SCHEDULER_TaskIdentifier upload_task;
1163
1164 /**
1165 * Storage space to reserve for the operation.
1166 */
1167 uint64_t reserve_space;
1168
1169 /**
1170 * Overall number of entries to reserve for the
1171 * publish operation.
1172 */
1173 uint32_t reserve_entries;
1174
1175 /**
1176 * Typically GNUNET_NO. Set to GNUNET_YES if "upload_task" is
1177 * GNUNET_SCHEDULER_NO_TASK and we're waiting for a response from
1178 * the datastore service (in which case this struct must not be
1179 * freed until we have that response). If someone tries to stop the
1180 * download for good during this period, "in_network_wait" is set to
1181 * GNUNET_SYSERR which will cause the struct to be destroyed right
1182 * after we have the reply (or timeout) from the datastore service.
1183 */
1184 int in_network_wait;
1185
1186 /**
1187 * Options for publishing.
1188 */
1189 enum GNUNET_FS_PublishOptions options;
1190
1191 /**
1192 * Space reservation ID with datastore service
1193 * for this upload.
1194 */
1195 int rid;
1196
1197 /**
1198 * Set to GNUNET_YES if all processing has completed.
1199 */
1200 int all_done;
1201};
1202
1203
1204/**
1205 * Phases of unindex processing (state machine).
1206 */
1207enum UnindexState
1208{
1209 /**
1210 * We're currently hashing the file.
1211 */
1212 UNINDEX_STATE_HASHING = 0,
1213
1214 /**
1215 * We're telling the datastore to delete
1216 * the respective entries.
1217 */
1218 UNINDEX_STATE_DS_REMOVE = 1,
1219
1220 /**
1221 * We're notifying the FS service about
1222 * the unindexing.
1223 */
1224 UNINDEX_STATE_FS_NOTIFY = 2,
1225
1226 /**
1227 * We're done.
1228 */
1229 UNINDEX_STATE_COMPLETE = 3,
1230
1231 /**
1232 * We've encountered a fatal error.
1233 */
1234 UNINDEX_STATE_ERROR = 4
1235};
1236
1237
1238/**
1239 * Handle for controlling an unindexing operation.
1240 */
1241struct GNUNET_FS_UnindexContext
1242{
1243
1244 /**
1245 * Global FS context.
1246 */
1247 struct GNUNET_FS_Handle *h;
1248
1249 /**
1250 * Our top-level activity entry.
1251 */
1252 struct TopLevelActivity *top;
1253
1254 /**
1255 * Name of the file that we are unindexing.
1256 */
1257 char *filename;
1258
1259 /**
1260 * Short name under which we are serializing the state of this operation.
1261 */
1262 char *serialization;
1263
1264 /**
1265 * Connection to the FS service, only valid during the
1266 * UNINDEX_STATE_FS_NOTIFY phase.
1267 */
1268 struct GNUNET_CLIENT_Connection *client;
1269
1270 /**
1271 * Connection to the datastore service, only valid during the
1272 * UNINDEX_STATE_DS_NOTIFY phase.
1273 */
1274 struct GNUNET_DATASTORE_Handle *dsh;
1275
1276 /**
1277 * Pointer kept for the client.
1278 */
1279 void *client_info;
1280
1281 /**
1282 * Merkle-ish tree encoder context.
1283 */
1284 struct GNUNET_FS_TreeEncoder *tc;
1285
1286 /**
1287 * Handle used to read the file.
1288 */
1289 struct GNUNET_DISK_FileHandle *fh;
1290
1291 /**
1292 * Error message, NULL on success.
1293 */
1294 char *emsg;
1295
1296 /**
1297 * Context for hashing of the file.
1298 */
1299 struct GNUNET_CRYPTO_FileHashContext *fhc;
1300
1301 /**
1302 * Overall size of the file.
1303 */
1304 uint64_t file_size;
1305
1306 /**
1307 * When did we start?
1308 */
1309 struct GNUNET_TIME_Absolute start_time;
1310
1311 /**
1312 * Hash of the file's contents (once computed).
1313 */
1314 GNUNET_HashCode file_id;
1315
1316 /**
1317 * Current operatinonal phase.
1318 */
1319 enum UnindexState state;
1320
1321};
1322
1323
1324/**
1325 * Information we keep for each keyword in
1326 * a keyword search.
1327 */
1328struct SearchRequestEntry
1329{
1330 /**
1331 * Hash of the original keyword, also known as the
1332 * key (for decrypting the KBlock).
1333 */
1334 GNUNET_HashCode key;
1335
1336 /**
1337 * Hash of the public key, also known as the query.
1338 */
1339 GNUNET_HashCode query;
1340
1341 /**
1342 * Map that contains a "struct GNUNET_FS_SearchResult" for each result that
1343 * was found under this keyword. Note that the entries will point
1344 * to the same locations as those in the master result map (in
1345 * "struct GNUNET_FS_SearchContext"), so they should not be freed.
1346 * The key for each entry is the XOR of the key and query in the CHK
1347 * URI (as a unique identifier for the search result).
1348 */
1349 struct GNUNET_CONTAINER_MultiHashMap *results;
1350
1351 /**
1352 * Is this keyword a mandatory keyword
1353 * (started with '+')?
1354 */
1355 int mandatory;
1356
1357};
1358
1359
1360/**
1361 * Handle for controlling a search.
1362 */
1363struct GNUNET_FS_SearchContext
1364{
1365 /**
1366 * Handle to the global FS context.
1367 */
1368 struct GNUNET_FS_Handle *h;
1369
1370 /**
1371 * Our top-level activity entry (if we are top-level, otherwise NULL).
1372 */
1373 struct TopLevelActivity *top;
1374
1375 /**
1376 * List of keywords that we're looking for.
1377 */
1378 struct GNUNET_FS_Uri *uri;
1379
1380 /**
1381 * For update-searches, link to the search result that triggered
1382 * the update search; otherwise NULL.
1383 */
1384 struct GNUNET_FS_SearchResult *psearch_result;
1385
1386 /**
1387 * Connection to the FS service.
1388 */
1389 struct GNUNET_CLIENT_Connection *client;
1390
1391 /**
1392 * Pointer we keep for the client.
1393 */
1394 void *client_info;
1395
1396 /**
1397 * Name of the file on disk we use for persistence.
1398 */
1399 char *serialization;
1400
1401 /**
1402 * Error message (non-NULL if this operation failed).
1403 */
1404 char *emsg;
1405
1406 /**
1407 * Map that contains a "struct GNUNET_FS_SearchResult" for each result that
1408 * was found in the search. The key for each entry is the XOR of
1409 * the key and query in the CHK URI (as a unique identifier for the
1410 * search result).
1411 */
1412 struct GNUNET_CONTAINER_MultiHashMap *master_result_map;
1413
1414 /**
1415 * Per-keyword information for a keyword search. This array will
1416 * have exactly as many entries as there were keywords.
1417 */
1418 struct SearchRequestEntry *requests;
1419
1420 /**
1421 * When did we start?
1422 */
1423 struct GNUNET_TIME_Absolute start_time;
1424
1425 /**
1426 * ID of a task that is using this struct and that must be cancelled
1427 * when the search is being stopped (if not
1428 * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some
1429 * artificial delay when trying to reconnect to the FS service.
1430 */
1431 GNUNET_SCHEDULER_TaskIdentifier task;
1432
1433 /**
1434 * How many of the entries in the search request
1435 * map have been passed to the service so far?
1436 */
1437 unsigned int search_request_map_offset;
1438
1439 /**
1440 * How many of the keywords in the KSK
1441 * map have been passed to the service so far?
1442 */
1443 unsigned int keyword_offset;
1444
1445 /**
1446 * Anonymity level for the search.
1447 */
1448 uint32_t anonymity;
1449
1450 /**
1451 * Number of mandatory keywords in this query.
1452 */
1453 uint32_t mandatory_count;
1454
1455 /**
1456 * Options for the search.
1457 */
1458 enum GNUNET_FS_SearchOptions options;
1459};
1460
1461
1462/**
1463 * FSM for possible states a block can go through. The typical
1464 * order of progression is linear through the states, alternatives
1465 * are documented in the comments.
1466 */
1467enum BlockRequestState
1468{
1469 /**
1470 * Initial state, block has only been allocated (since it is
1471 * relevant to the overall download request).
1472 */
1473 BRS_INIT = 0,
1474
1475 /**
1476 * We've checked the block on the path down the tree, and the
1477 * content on disk did match the desired CHK, but not all
1478 * the way down, so at the bottom some blocks will still
1479 * need to be reconstructed).
1480 */
1481 BRS_RECONSTRUCT_DOWN = 1,
1482
1483 /**
1484 * We've calculated the CHK bottom-up based on the meta data.
1485 * This may work, but if it did we have to write the meta data to
1486 * disk at the end (and we still need to check against the
1487 * CHK set on top).
1488 */
1489 BRS_RECONSTRUCT_META_UP = 2,
1490
1491 /**
1492 * We've calculated the CHK bottom-up based on what we have on
1493 * disk, which may not be what the desired CHK is. If the
1494 * reconstructed CHKs match whatever comes from above, we're
1495 * done with the respective subtree.
1496 */
1497 BRS_RECONSTRUCT_UP = 3,
1498
1499 /**
1500 * We've determined the real, desired CHK for this block
1501 * (full tree reconstruction failed), request is now pending.
1502 * If the CHK that bubbled up through reconstruction did match
1503 * the top-level request, the state machine for the subtree
1504 * would have moved to BRS_DOWNLOAD_UP.
1505 */
1506 BRS_CHK_SET = 4,
1507
1508 /**
1509 * We've successfully downloaded this block, but the children
1510 * still need to be either downloaded or verified (download
1511 * request propagates down). If the download fails, the
1512 * state machine for this block may move to
1513 * BRS_DOWNLOAD_ERROR instead.
1514 */
1515 BRS_DOWNLOAD_DOWN = 5,
1516
1517 /**
1518 * This block and all of its children have been downloaded
1519 * successfully (full completion propagates up).
1520 */
1521 BRS_DOWNLOAD_UP = 6,
1522
1523 /**
1524 * We got a block back that matched the query but did not hash to
1525 * the key (malicious publisher or hash collision); this block
1526 * can never be downloaded (error propagates up).
1527 */
1528 BRS_ERROR = 7
1529};
1530
1531
1532/**
1533 * Information about an active download request.
1534 */
1535struct DownloadRequest
1536{
1537 /**
1538 * While pending, we keep all download requests in a doubly-linked list.
1539 */
1540 struct DownloadRequest *next;
1541
1542 /**
1543 * While pending, we keep all download requests in a doubly-linked list.
1544 */
1545 struct DownloadRequest *prev;
1546
1547 /**
1548 * Parent in the CHK-tree.
1549 */
1550 struct DownloadRequest *parent;
1551
1552 /**
1553 * Array (!) of child-requests, or NULL for the bottom of the tree.
1554 */
1555 struct DownloadRequest **children;
1556
1557 /**
1558 * CHK for the request for this block (set during reconstruction
1559 * to what we have on disk, later to what we want to have).
1560 */
1561 struct ContentHashKey chk;
1562
1563 /**
1564 * Offset of the corresponding block. Specifically, first (!) byte of
1565 * the first DBLOCK in the subtree induced by block represented by
1566 * this request.
1567 */
1568 uint64_t offset;
1569
1570 /**
1571 * Number of entries in 'children' array.
1572 */
1573 unsigned int num_children;
1574
1575 /**
1576 * Depth of the corresponding block in the tree. 0==DBLOCKs.
1577 */
1578 unsigned int depth;
1579
1580 /**
1581 * State in the FSM.
1582 */
1583 enum BlockRequestState state;
1584
1585 /**
1586 * GNUNET_YES if this entry is in the pending list.
1587 */
1588 int is_pending;
1589
1590};
1591
1592
1593/**
1594 * (recursively) free download request structure
1595 *
1596 * @param dr request to free
1597 */
1598void
1599GNUNET_FS_free_download_request_ (struct DownloadRequest *dr);
1600
1601
1602/**
1603 * Context for controlling a download.
1604 */
1605struct GNUNET_FS_DownloadContext
1606{
1607
1608 /**
1609 * Global FS context.
1610 */
1611 struct GNUNET_FS_Handle *h;
1612
1613 /**
1614 * Our top-level activity entry (if we are top-level, otherwise NULL).
1615 */
1616 struct TopLevelActivity *top;
1617
1618 /**
1619 * Connection to the FS service.
1620 */
1621 struct GNUNET_CLIENT_Connection *client;
1622
1623 /**
1624 * Parent download (used when downloading files
1625 * in directories).
1626 */
1627 struct GNUNET_FS_DownloadContext *parent;
1628
1629 /**
1630 * Associated search (used when downloading files
1631 * based on search results), or NULL for none.
1632 */
1633 struct GNUNET_FS_SearchResult *search;
1634
1635 /**
1636 * Head of list of child downloads.
1637 */
1638 struct GNUNET_FS_DownloadContext *child_head;
1639
1640 /**
1641 * Tail of list of child downloads.
1642 */
1643 struct GNUNET_FS_DownloadContext *child_tail;
1644
1645 /**
1646 * Previous download belonging to the same parent.
1647 */
1648 struct GNUNET_FS_DownloadContext *prev;
1649
1650 /**
1651 * Next download belonging to the same parent.
1652 */
1653 struct GNUNET_FS_DownloadContext *next;
1654
1655 /**
1656 * Context kept for the client.
1657 */
1658 void *client_info;
1659
1660 /**
1661 * URI that identifies the file that we are downloading.
1662 */
1663 struct GNUNET_FS_Uri *uri;
1664
1665 /**
1666 * Known meta-data for the file (can be NULL).
1667 */
1668 struct GNUNET_CONTAINER_MetaData *meta;
1669
1670 /**
1671 * Error message, NULL if we're doing OK.
1672 */
1673 char *emsg;
1674
1675 /**
1676 * Random portion of filename we use for syncing state of this
1677 * download.
1678 */
1679 char *serialization;
1680
1681 /**
1682 * Where are we writing the data (name of the
1683 * file, can be NULL!).
1684 */
1685 char *filename;
1686
1687 /**
1688 * Where are we writing the data temporarily (name of the
1689 * file, can be NULL!); used if we do not have a permanent
1690 * name and we are a directory and we do a recursive download.
1691 */
1692 char *temp_filename;
1693
1694 /**
1695 * Our entry in the job queue.
1696 */
1697 struct GNUNET_FS_QueueEntry *job_queue;
1698
1699 /**
1700 * Non-NULL if we are currently having a request for
1701 * transmission pending with the client handle.
1702 */
1703 struct GNUNET_CLIENT_TransmitHandle *th;
1704
1705 /**
1706 * Tree encoder used for the reconstruction.
1707 */
1708 struct GNUNET_FS_TreeEncoder *te;
1709
1710 /**
1711 * File handle for reading data from an existing file
1712 * (to pass to tree encoder).
1713 */
1714 struct GNUNET_DISK_FileHandle *rfh;
1715
1716 /**
1717 * Map of active requests (those waiting for a response). The key
1718 * is the hash of the encryped block (aka query).
1719 */
1720 struct GNUNET_CONTAINER_MultiHashMap *active;
1721
1722 /**
1723 * Head of linked list of pending requests.
1724 */
1725 struct DownloadRequest *pending_head;
1726
1727 /**
1728 * Head of linked list of pending requests.
1729 */
1730 struct DownloadRequest *pending_tail;
1731
1732 /**
1733 * Top-level download request.
1734 */
1735 struct DownloadRequest *top_request;
1736
1737 /**
1738 * Identity of the peer having the content, or all-zeros
1739 * if we don't know of such a peer.
1740 */
1741 struct GNUNET_PeerIdentity target;
1742
1743 /**
1744 * ID of a task that is using this struct and that must be cancelled
1745 * when the download is being stopped (if not
1746 * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some
1747 * artificial delay when trying to reconnect to the FS service or
1748 * the task processing incrementally the data on disk, or the
1749 * task requesting blocks, etc.
1750 */
1751 GNUNET_SCHEDULER_TaskIdentifier task;
1752
1753 /**
1754 * What is the first offset that we're interested
1755 * in?
1756 */
1757 uint64_t offset;
1758
1759 /**
1760 * How many bytes starting from offset are desired?
1761 * This is NOT the overall length of the file!
1762 */
1763 uint64_t length;
1764
1765 /**
1766 * How many bytes have we already received within
1767 * the specified range (DBlocks only).
1768 */
1769 uint64_t completed;
1770
1771 /**
1772 * What was the size of the file on disk that we're downloading
1773 * before we started? Used to detect if there is a point in
1774 * checking an existing block on disk for matching the desired
1775 * content. 0 if the file did not exist already.
1776 */
1777 uint64_t old_file_size;
1778
1779 /**
1780 * Time download was started.
1781 */
1782 struct GNUNET_TIME_Absolute start_time;
1783
1784 /**
1785 * Desired level of anonymity.
1786 */
1787 uint32_t anonymity;
1788
1789 /**
1790 * The depth of the file-tree.
1791 */
1792 unsigned int treedepth;
1793
1794 /**
1795 * Options for the download.
1796 */
1797 enum GNUNET_FS_DownloadOptions options;
1798
1799 /**
1800 * Flag set upon transitive completion (includes child downloads).
1801 * This flag is only set to GNUNET_YES for directories where all
1802 * child-downloads have also completed (and signalled completion).
1803 */
1804 int has_finished;
1805
1806 /**
1807 * Have we started the receive continuation yet?
1808 */
1809 int in_receive;
1810
1811};
1812
1813
1814/**
1815 * Information about an (updateable) node in the
1816 * namespace.
1817 */
1818struct NamespaceUpdateNode
1819{
1820 /**
1821 * Identifier for this node.
1822 */
1823 char *id;
1824
1825 /**
1826 * Identifier of children of this node.
1827 */
1828 char *update;
1829
1830 /**
1831 * Metadata for this entry.
1832 */
1833 struct GNUNET_CONTAINER_MetaData *md;
1834
1835 /**
1836 * URI of this entry in the namespace.
1837 */
1838 struct GNUNET_FS_Uri *uri;
1839
1840 /**
1841 * Namespace update generation ID. Used to ensure
1842 * freshness of the tree_id.
1843 */
1844 unsigned int nug;
1845
1846 /**
1847 * TREE this entry belongs to (if nug is current).
1848 */
1849 unsigned int tree_id;
1850
1851};
1852
1853
1854struct GNUNET_FS_Namespace
1855{
1856
1857 /**
1858 * Handle to the FS service context.
1859 */
1860 struct GNUNET_FS_Handle *h;
1861
1862 /**
1863 * Array with information about nodes in the namespace.
1864 */
1865 struct NamespaceUpdateNode **update_nodes;
1866
1867 /**
1868 * Private key for the namespace.
1869 */
1870 struct GNUNET_CRYPTO_RsaPrivateKey *key;
1871
1872 /**
1873 * Hash map mapping identifiers of update nodes
1874 * to the update nodes (initialized on-demand).
1875 */
1876 struct GNUNET_CONTAINER_MultiHashMap *update_map;
1877
1878 /**
1879 * Name of the file with the private key.
1880 */
1881 char *filename;
1882
1883 /**
1884 * Name of the namespace.
1885 */
1886 char *name;
1887
1888 /**
1889 * Size of the update nodes array.
1890 */
1891 unsigned int update_node_count;
1892
1893 /**
1894 * Reference counter.
1895 */
1896 unsigned int rc;
1897
1898 /**
1899 * Generator for unique nug numbers.
1900 */
1901 unsigned int nug_gen;
1902};
1903
1904
1905#endif
1906
1907/* end of fs_api.h */
diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c
index f43fe14a7..d5f73623a 100644
--- a/src/fs/fs_directory.c
+++ b/src/fs/fs_directory.c
@@ -34,7 +34,7 @@
34 */ 34 */
35#include "platform.h" 35#include "platform.h"
36#include "gnunet_fs_service.h" 36#include "gnunet_fs_service.h"
37#include "fs.h" 37#include "fs_api.h"
38 38
39/** 39/**
40 * String that is used to indicate that a file 40 * String that is used to indicate that a file
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c
index af889d804..09785f6dd 100644
--- a/src/fs/fs_download.c
+++ b/src/fs/fs_download.c
@@ -28,7 +28,7 @@
28#include "platform.h" 28#include "platform.h"
29#include "gnunet_constants.h" 29#include "gnunet_constants.h"
30#include "gnunet_fs_service.h" 30#include "gnunet_fs_service.h"
31#include "fs.h" 31#include "fs_api.h"
32#include "fs_tree.h" 32#include "fs_tree.h"
33 33
34#define DEBUG_DOWNLOAD GNUNET_EXTRA_LOGGING 34#define DEBUG_DOWNLOAD GNUNET_EXTRA_LOGGING
diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c
index bb84b948e..b094ed7d5 100644
--- a/src/fs/fs_file_information.c
+++ b/src/fs/fs_file_information.c
@@ -30,7 +30,7 @@
30#include "platform.h" 30#include "platform.h"
31#include <extractor.h> 31#include <extractor.h>
32#include "gnunet_fs_service.h" 32#include "gnunet_fs_service.h"
33#include "fs.h" 33#include "fs_api.h"
34#include "fs_tree.h" 34#include "fs_tree.h"
35 35
36 36
diff --git a/src/fs/fs_getopt.c b/src/fs/fs_getopt.c
index b50d0624b..03747742b 100644
--- a/src/fs/fs_getopt.c
+++ b/src/fs/fs_getopt.c
@@ -25,7 +25,7 @@
25 */ 25 */
26#include "platform.h" 26#include "platform.h"
27#include "gnunet_fs_service.h" 27#include "gnunet_fs_service.h"
28#include "fs.h" 28#include "fs_api.h"
29 29
30/* ******************** command-line option parsing API ******************** */ 30/* ******************** command-line option parsing API ******************** */
31 31
diff --git a/src/fs/fs_list_indexed.c b/src/fs/fs_list_indexed.c
index 85e12db32..0896de347 100644
--- a/src/fs/fs_list_indexed.c
+++ b/src/fs/fs_list_indexed.c
@@ -28,7 +28,7 @@
28#include "gnunet_constants.h" 28#include "gnunet_constants.h"
29#include "gnunet_fs_service.h" 29#include "gnunet_fs_service.h"
30#include "gnunet_protocols.h" 30#include "gnunet_protocols.h"
31#include "fs.h" 31#include "fs_api.h"
32 32
33 33
34/** 34/**
diff --git a/src/fs/fs_misc.c b/src/fs/fs_misc.c
index 6bd4e59f8..d4afc0d55 100644
--- a/src/fs/fs_misc.c
+++ b/src/fs/fs_misc.c
@@ -25,7 +25,7 @@
25#include "platform.h" 25#include "platform.h"
26#include "gnunet_constants.h" 26#include "gnunet_constants.h"
27#include "gnunet_fs_service.h" 27#include "gnunet_fs_service.h"
28#include "fs.h" 28#include "fs_api.h"
29 29
30 30
31/** 31/**
diff --git a/src/fs/fs_namespace.c b/src/fs/fs_namespace.c
index 47ca6bad1..9d754009b 100644
--- a/src/fs/fs_namespace.c
+++ b/src/fs/fs_namespace.c
@@ -28,10 +28,22 @@
28#include "gnunet_signatures.h" 28#include "gnunet_signatures.h"
29#include "gnunet_util_lib.h" 29#include "gnunet_util_lib.h"
30#include "gnunet_fs_service.h" 30#include "gnunet_fs_service.h"
31#include "fs.h" 31#include "fs_api.h"
32 32
33#define DEBUG_NAMESPACE GNUNET_EXTRA_LOGGING 33#define DEBUG_NAMESPACE GNUNET_EXTRA_LOGGING
34 34
35
36/**
37 * Maximum legal size for an sblock.
38 */
39#define MAX_SBLOCK_SIZE (60 * 1024)
40
41/**
42 * Maximum legal size for an nblock.
43 */
44#define MAX_NBLOCK_SIZE (60 * 1024)
45
46
35/** 47/**
36 * Return the name of the directory in which we store 48 * Return the name of the directory in which we store
37 * our local namespaces (or rather, their public keys). 49 * our local namespaces (or rather, their public keys).
diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c
index 4f27abae4..9a834ee86 100644
--- a/src/fs/fs_publish.c
+++ b/src/fs/fs_publish.c
@@ -31,11 +31,16 @@
31#include "gnunet_signatures.h" 31#include "gnunet_signatures.h"
32#include "gnunet_util_lib.h" 32#include "gnunet_util_lib.h"
33#include "gnunet_fs_service.h" 33#include "gnunet_fs_service.h"
34#include "fs.h" 34#include "fs_api.h"
35#include "fs_tree.h" 35#include "fs_tree.h"
36 36
37#define DEBUG_PUBLISH GNUNET_EXTRA_LOGGING 37#define DEBUG_PUBLISH GNUNET_EXTRA_LOGGING
38 38
39/**
40 * Maximum legal size for a kblock.
41 */
42#define MAX_KBLOCK_SIZE (60 * 1024)
43
39 44
40/** 45/**
41 * Fill in all of the generic fields for 46 * Fill in all of the generic fields for
diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c
index 93a787477..b5152cc66 100644
--- a/src/fs/fs_search.c
+++ b/src/fs/fs_search.c
@@ -28,11 +28,16 @@
28#include "gnunet_constants.h" 28#include "gnunet_constants.h"
29#include "gnunet_fs_service.h" 29#include "gnunet_fs_service.h"
30#include "gnunet_protocols.h" 30#include "gnunet_protocols.h"
31#include "fs.h" 31#include "fs_api.h"
32 32
33#define DEBUG_SEARCH GNUNET_EXTRA_LOGGING 33#define DEBUG_SEARCH GNUNET_EXTRA_LOGGING
34 34
35/** 35/**
36 * Number of availability trials we perform per search result.
37 */
38#define AVAILABILITY_TRIALS_MAX 8
39
40/**
36 * Fill in all of the generic fields for a search event and 41 * Fill in all of the generic fields for a search event and
37 * call the callback. 42 * call the callback.
38 * 43 *
diff --git a/src/fs/fs_test_lib.c b/src/fs/fs_test_lib.c
index 1c15c6f60..06ab01f8d 100644
--- a/src/fs/fs_test_lib.c
+++ b/src/fs/fs_test_lib.c
@@ -27,7 +27,7 @@
27 * @author Christian Grothoff 27 * @author Christian Grothoff
28 */ 28 */
29#include "platform.h" 29#include "platform.h"
30#include "fs.h" 30#include "fs_api.h"
31#include "fs_test_lib.h" 31#include "fs_test_lib.h"
32#include "gnunet_testing_lib.h" 32#include "gnunet_testing_lib.h"
33 33
diff --git a/src/fs/fs_tree.h b/src/fs/fs_tree.h
index 56c32a390..5b1c2024f 100644
--- a/src/fs/fs_tree.h
+++ b/src/fs/fs_tree.h
@@ -32,7 +32,7 @@
32#ifndef GNUNET_FS_TREE_H 32#ifndef GNUNET_FS_TREE_H
33#define GNUNET_FS_TREE_H 33#define GNUNET_FS_TREE_H
34 34
35#include "fs.h" 35#include "fs_api.h"
36 36
37/** 37/**
38 * Compute the depth of the CHK tree. 38 * Compute the depth of the CHK tree.
diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c
index 8dab62051..79b7e8c73 100644
--- a/src/fs/fs_unindex.c
+++ b/src/fs/fs_unindex.c
@@ -28,7 +28,7 @@
28#include "gnunet_constants.h" 28#include "gnunet_constants.h"
29#include "gnunet_fs_service.h" 29#include "gnunet_fs_service.h"
30#include "gnunet_protocols.h" 30#include "gnunet_protocols.h"
31#include "fs.h" 31#include "fs_api.h"
32#include "fs_tree.h" 32#include "fs_tree.h"
33 33
34#define DEBUG_UNINDEX GNUNET_EXTRA_LOGGING 34#define DEBUG_UNINDEX GNUNET_EXTRA_LOGGING
diff --git a/src/fs/fs_uri.c b/src/fs/fs_uri.c
index 48fabc1fb..62fd51304 100644
--- a/src/fs/fs_uri.c
+++ b/src/fs/fs_uri.c
@@ -81,7 +81,7 @@
81#include "platform.h" 81#include "platform.h"
82#include "gnunet_fs_service.h" 82#include "gnunet_fs_service.h"
83#include "gnunet_signatures.h" 83#include "gnunet_signatures.h"
84#include "fs.h" 84#include "fs_api.h"
85 85
86 86
87/** 87/**
diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c
index daffce444..48b7bca29 100644
--- a/src/fs/gnunet-service-fs.c
+++ b/src/fs/gnunet-service-fs.c
@@ -25,8 +25,6 @@
25 * 25 *
26 * To use: 26 * To use:
27 * - consider re-issue GSF_dht_lookup_ after non-DHT reply received 27 * - consider re-issue GSF_dht_lookup_ after non-DHT reply received
28 * - implement 'SUPPORT_DELAYS'
29 *
30 */ 28 */
31#include "platform.h" 29#include "platform.h"
32#include <float.h> 30#include <float.h>
diff --git a/src/fs/gnunet-service-fs.h b/src/fs/gnunet-service-fs.h
index e4efbb8f5..7efeedd5f 100644
--- a/src/fs/gnunet-service-fs.h
+++ b/src/fs/gnunet-service-fs.h
@@ -37,26 +37,13 @@
37 37
38#define DEBUG_FS_CLIENT GNUNET_EXTRA_LOGGING 38#define DEBUG_FS_CLIENT GNUNET_EXTRA_LOGGING
39 39
40
41/** 40/**
42 * Should we introduce random latency in processing? Required for proper 41 * By which amount do we decrement the TTL for simple forwarding /
43 * implementation of GAP, but can be disabled for performance evaluation of 42 * indirection of the query; in milli-seconds. Set somewhat in
44 * the basic routing algorithm. 43 * accordance to your network latency (above the time it'll take you
45 * 44 * to send a packet and get a reply).
46 * Note that with delays enabled, performance can be significantly lower
47 * (several orders of magnitude in 2-peer test runs); if you want to
48 * measure throughput of other components, set this to NO. Also, you
49 * might want to consider changing 'RETRY_PROBABILITY_INV' to 1 for
50 * a rather wasteful mode of operation (that might still get the highest
51 * throughput overall).
52 *
53 * Performance measurements (for 50 MB file, 2 peers):
54 *
55 * - Without delays: 3300 kb/s
56 * - With delays: 101 kb/s
57 */ 45 */
58#define SUPPORT_DELAYS GNUNET_NO 46#define TTL_DECREMENT 5000
59
60 47
61/** 48/**
62 * At what frequency should our datastore load decrease 49 * At what frequency should our datastore load decrease
diff --git a/src/fs/gnunet-service-fs_cp.c b/src/fs/gnunet-service-fs_cp.c
index d4c81d509..9d2385378 100644
--- a/src/fs/gnunet-service-fs_cp.c
+++ b/src/fs/gnunet-service-fs_cp.c
@@ -32,6 +32,14 @@
32#include "gnunet-service-fs_pr.h" 32#include "gnunet-service-fs_pr.h"
33#include "gnunet-service-fs_push.h" 33#include "gnunet-service-fs_push.h"
34 34
35
36/**
37 * Ratio for moving average delay calculation. The previous
38 * average goes in with a factor of (n-1) into the calculation.
39 * Must be > 0.
40 */
41#define RUNAVG_DELAY_N 16
42
35/** 43/**
36 * How often do we flush trust values to disk? 44 * How often do we flush trust values to disk?
37 */ 45 */
diff --git a/src/fs/gnunet-service-fs_cp.h b/src/fs/gnunet-service-fs_cp.h
index a2cc5232c..11be4c6f1 100644
--- a/src/fs/gnunet-service-fs_cp.h
+++ b/src/fs/gnunet-service-fs_cp.h
@@ -31,6 +31,36 @@
31 31
32 32
33/** 33/**
34 * Maximum number of outgoing messages we queue per peer.
35 *
36 * Performance measurements for 2 peer setup for 50 MB file
37 * (with MAX_DATASTORE_QUEUE = 1 and RETRY_PROBABILITY_INV = 1):
38 *
39 * 2: 1700 kb/s, 1372 kb/s
40 * 8: 2117 kb/s, 1284 kb/s, 1112 kb/s
41 * 16: 3500 kb/s, 3200 kb/s, 3388 kb/s
42 * 32: 3441 kb/s, 3163 kb/s, 3277 kb/s
43 * 128: 1700 kb/s; 2010 kb/s, 3383 kb/s, 1156 kb/s
44 *
45 * Conclusion: 16 seems to be a pretty good value (stable
46 * and high performance, no excessive memory use).
47 */
48#define MAX_QUEUE_PER_PEER 16
49
50/**
51 * Length of the P2P success tracker. Note that having a very long
52 * list can also hurt performance.
53 */
54#define P2P_SUCCESS_LIST_SIZE 8
55
56/**
57 * Length of the CS-2-P success tracker. Note that
58 * having a very long list can also hurt performance.
59 */
60#define CS2P_SUCCESS_LIST_SIZE 8
61
62
63/**
34 * Performance data kept for a peer. 64 * Performance data kept for a peer.
35 */ 65 */
36struct GSF_PeerPerformanceData 66struct GSF_PeerPerformanceData
diff --git a/src/fs/gnunet-service-fs_pr.c b/src/fs/gnunet-service-fs_pr.c
index ac183a980..3b43b4c5f 100644
--- a/src/fs/gnunet-service-fs_pr.c
+++ b/src/fs/gnunet-service-fs_pr.c
@@ -32,6 +32,21 @@
32#include "gnunet-service-fs_pr.h" 32#include "gnunet-service-fs_pr.h"
33 33
34/** 34/**
35 * Maximum size of the datastore queue for P2P operations. Needs to
36 * be large enough to queue MAX_QUEUE_PER_PEER operations for roughly
37 * the number of active (connected) peers.
38 */
39#define MAX_DATASTORE_QUEUE (16 * MAX_QUEUE_PER_PEER)
40
41/**
42 * Bandwidth value of a 0-priority content (must be fairly high
43 * compared to query since content is typically significantly larger
44 * -- and more valueable since it can take many queries to get one
45 * piece of content).
46 */
47#define CONTENT_BANDWIDTH_VALUE 800
48
49/**
35 * Hard limit on the number of results we may get from the datastore per query. 50 * Hard limit on the number of results we may get from the datastore per query.
36 */ 51 */
37#define MAX_RESULTS (100 * 1024) 52#define MAX_RESULTS (100 * 1024)
diff --git a/src/fs/gnunet-service-fs_push.c b/src/fs/gnunet-service-fs_push.c
index 515446117..4fde823a1 100644
--- a/src/fs/gnunet-service-fs_push.c
+++ b/src/fs/gnunet-service-fs_push.c
@@ -34,6 +34,17 @@
34#define DEBUG_FS_MIGRATION GNUNET_EXTRA_LOGGING 34#define DEBUG_FS_MIGRATION GNUNET_EXTRA_LOGGING
35 35
36/** 36/**
37 * Maximum number of blocks we keep in memory for migration.
38 */
39#define MAX_MIGRATION_QUEUE 8
40
41/**
42 * Blocks are at most migrated to this number of peers
43 * plus one, each time they are fetched from the database.
44 */
45#define MIGRATION_LIST_SIZE 2
46
47/**
37 * How long must content remain valid for us to consider it for migration? 48 * How long must content remain valid for us to consider it for migration?
38 * If content will expire too soon, there is clearly no point in pushing 49 * If content will expire too soon, there is clearly no point in pushing
39 * it to other peers. This value gives the threshold for migration. Note 50 * it to other peers. This value gives the threshold for migration. Note
diff --git a/src/fs/test_fs_directory.c b/src/fs/test_fs_directory.c
index ac581bd6f..81012e764 100644
--- a/src/fs/test_fs_directory.c
+++ b/src/fs/test_fs_directory.c
@@ -28,7 +28,7 @@
28#include <extractor.h> 28#include <extractor.h>
29#include "gnunet_util_lib.h" 29#include "gnunet_util_lib.h"
30#include "gnunet_fs_service.h" 30#include "gnunet_fs_service.h"
31#include "fs.h" 31#include "fs_api.h"
32 32
33#define ABORT() { fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); return 1; } 33#define ABORT() { fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); return 1; }
34 34
diff --git a/src/fs/test_fs_uri.c b/src/fs/test_fs_uri.c
index 693bff1d6..72744351b 100644
--- a/src/fs/test_fs_uri.c
+++ b/src/fs/test_fs_uri.c
@@ -27,7 +27,7 @@
27#include "platform.h" 27#include "platform.h"
28#include "gnunet_util_lib.h" 28#include "gnunet_util_lib.h"
29#include "gnunet_fs_service.h" 29#include "gnunet_fs_service.h"
30#include "fs.h" 30#include "fs_api.h"
31 31
32#define ABORT() { fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); return 1; } 32#define ABORT() { fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); return 1; }
33 33