aboutsummaryrefslogtreecommitdiff
path: root/src/fs
diff options
context:
space:
mode:
authorChristian Grothoff <christian@grothoff.org>2010-01-26 09:53:40 +0000
committerChristian Grothoff <christian@grothoff.org>2010-01-26 09:53:40 +0000
commit68907af3eb39fb03936a226f340d469af10d5014 (patch)
treee6365ebb6acb6539e0d8abce0cce4ebe275bbed1 /src/fs
parenta92360c7ed9fe3e5550a2e4c7832ad9224ee1779 (diff)
downloadgnunet-68907af3eb39fb03936a226f340d469af10d5014.tar.gz
gnunet-68907af3eb39fb03936a226f340d469af10d5014.zip
moving indexing out of main file
Diffstat (limited to 'src/fs')
-rw-r--r--src/fs/Makefile.am3
-rw-r--r--src/fs/gnunet-service-fs.c663
-rw-r--r--src/fs/gnunet-service-fs_indexing.c688
-rw-r--r--src/fs/gnunet-service-fs_indexing.h121
4 files changed, 856 insertions, 619 deletions
diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am
index 95212fe5e..70897c5ef 100644
--- a/src/fs/Makefile.am
+++ b/src/fs/Makefile.am
@@ -84,7 +84,8 @@ gnunet_search_LDADD = \
84 $(GN_LIBINTL) 84 $(GN_LIBINTL)
85 85
86gnunet_service_fs_SOURCES = \ 86gnunet_service_fs_SOURCES = \
87 gnunet-service-fs.c 87 gnunet-service-fs.c \
88 gnunet-service-fs_indexing.c gnunet-service-fs_indexing.h
88gnunet_service_fs_LDADD = \ 89gnunet_service_fs_LDADD = \
89 $(top_builddir)/src/fs/libgnunetfs.la \ 90 $(top_builddir)/src/fs/libgnunetfs.la \
90 $(top_builddir)/src/datastore/libgnunetdatastore.la \ 91 $(top_builddir)/src/datastore/libgnunetdatastore.la \
diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c
index fa83eb889..0339567f3 100644
--- a/src/fs/gnunet-service-fs.c
+++ b/src/fs/gnunet-service-fs.c
@@ -45,42 +45,12 @@
45#include "gnunet_protocols.h" 45#include "gnunet_protocols.h"
46#include "gnunet_signatures.h" 46#include "gnunet_signatures.h"
47#include "gnunet_util_lib.h" 47#include "gnunet_util_lib.h"
48#include "gnunet-service-fs_indexing.h"
48#include "fs.h" 49#include "fs.h"
49 50
50#define DEBUG_FS GNUNET_NO 51#define DEBUG_FS GNUNET_NO
51 52
52 53
53/**
54 * In-memory information about indexed files (also available
55 * on-disk).
56 */
57struct IndexInfo
58{
59
60 /**
61 * This is a linked list.
62 */
63 struct IndexInfo *next;
64
65 /**
66 * Name of the indexed file. Memory allocated
67 * at the end of this struct (do not free).
68 */
69 const char *filename;
70
71 /**
72 * Context for transmitting confirmation to client,
73 * NULL if we've done this already.
74 */
75 struct GNUNET_SERVER_TransmitContext *tc;
76
77 /**
78 * Hash of the contents of the file.
79 */
80 GNUNET_HashCode file_id;
81
82};
83
84 54
85/** 55/**
86 * Signature of a function that is called whenever a datastore 56 * Signature of a function that is called whenever a datastore
@@ -359,20 +329,26 @@ struct ProcessGetContext
359 329
360 330
361/** 331/**
362 * Information we keep for each pending reply. 332 * Information we keep for each pending reply. The
333 * actual message follows at the end of this struct.
363 */ 334 */
364struct PendingReply 335struct PendingMessage
365{ 336{
366 /** 337 /**
367 * This is a linked list. 338 * This is a linked list.
368 */ 339 */
369 struct PendingReply *next; 340 struct PendingMessage *next;
370 341
371 /** 342 /**
372 * Size of the reply; actual reply message follows 343 * Size of the reply; actual reply message follows
373 * at the end of this struct. 344 * at the end of this struct.
374 */ 345 */
375 size_t msize; 346 size_t msize;
347
348 /**
349 * How important is this message for us?
350 */
351 uint32_t priority;
376 352
377}; 353};
378 354
@@ -426,7 +402,7 @@ struct PendingRequest
426 * (typically non-null only if we have a pending transmission 402 * (typically non-null only if we have a pending transmission
427 * request with the client or the respective peer). 403 * request with the client or the respective peer).
428 */ 404 */
429 struct PendingReply *replies_pending; 405 struct PendingMessage *replies_pending;
430 406
431 /** 407 /**
432 * Pending transmission request with the core service for the target 408 * Pending transmission request with the core service for the target
@@ -679,6 +655,18 @@ struct ConnectedPeer
679 struct GNUNET_TIME_Relative avg_delay; 655 struct GNUNET_TIME_Relative avg_delay;
680 656
681 /** 657 /**
658 * Handle for an active request for transmission to this
659 * peer, or NULL.
660 */
661 struct GNUNET_CORE_PeerRequestHandle *prh;
662
663 /**
664 * Messages (replies, queries, content migration) we would like to
665 * send to this peer in the near future. Sorted by priority.
666 */
667 struct PendingMessage *pending_messages;
668
669 /**
682 * Average priority of successful replies. Calculated 670 * Average priority of successful replies. Calculated
683 * as a moving average: new_avg = ((n-1)*last_avg+curr_prio) / n 671 * as a moving average: new_avg = ((n-1)*last_avg+curr_prio) / n
684 */ 672 */
@@ -690,10 +678,9 @@ struct ConnectedPeer
690 GNUNET_PEER_Id pid; 678 GNUNET_PEER_Id pid;
691 679
692 /** 680 /**
693 * Number of requests we have currently pending 681 * Number of requests we have currently pending with this peer (that
694 * with this peer (that is, requests that were 682 * is, requests that were transmitted so recently that we would not
695 * transmitted so recently that we would not retransmit 683 * retransmit them right now).
696 * them right now).
697 */ 684 */
698 unsigned int pending_requests; 685 unsigned int pending_requests;
699 686
@@ -753,18 +740,6 @@ static struct DatastoreRequestQueue *drq_head;
753static struct DatastoreRequestQueue *drq_tail; 740static struct DatastoreRequestQueue *drq_tail;
754 741
755/** 742/**
756 * Linked list of indexed files.
757 */
758static struct IndexInfo *indexed_files;
759
760/**
761 * Maps hash over content of indexed files to the respective filename.
762 * The filenames are pointers into the indexed_files linked list and
763 * do not need to be freed.
764 */
765static struct GNUNET_CONTAINER_MultiHashMap *ifm;
766
767/**
768 * Map of query hash codes to requests. 743 * Map of query hash codes to requests.
769 */ 744 */
770static struct GNUNET_CONTAINER_MultiHashMap *requests_by_query; 745static struct GNUNET_CONTAINER_MultiHashMap *requests_by_query;
@@ -803,403 +778,8 @@ static struct GNUNET_CONTAINER_MultiHashMap *connected_peers;
803static uint64_t max_pending_requests = 32; 778static uint64_t max_pending_requests = 32;
804 779
805 780
806/**
807 * Write the current index information list to disk.
808 */
809static void
810write_index_list ()
811{
812 struct GNUNET_BIO_WriteHandle *wh;
813 char *fn;
814 struct IndexInfo *pos;
815
816 if (GNUNET_OK !=
817 GNUNET_CONFIGURATION_get_value_filename (cfg,
818 "FS",
819 "INDEXDB",
820 &fn))
821 {
822 GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
823 _("Configuration option `%s' in section `%s' missing.\n"),
824 "INDEXDB",
825 "FS");
826 return;
827 }
828 wh = GNUNET_BIO_write_open (fn);
829 if (NULL == wh)
830 {
831 GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
832 _("Could not open `%s'.\n"),
833 fn);
834 GNUNET_free (fn);
835 return;
836 }
837 pos = indexed_files;
838 while (pos != NULL)
839 {
840 if ( (GNUNET_OK !=
841 GNUNET_BIO_write (wh,
842 &pos->file_id,
843 sizeof (GNUNET_HashCode))) ||
844 (GNUNET_OK !=
845 GNUNET_BIO_write_string (wh,
846 pos->filename)) )
847 break;
848 pos = pos->next;
849 }
850 if (GNUNET_OK !=
851 GNUNET_BIO_write_close (wh))
852 {
853 GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
854 _("Error writing `%s'.\n"),
855 fn);
856 GNUNET_free (fn);
857 return;
858 }
859 GNUNET_free (fn);
860}
861
862
863/**
864 * Read index information from disk.
865 */
866static void
867read_index_list ()
868{
869 struct GNUNET_BIO_ReadHandle *rh;
870 char *fn;
871 struct IndexInfo *pos;
872 char *fname;
873 GNUNET_HashCode hc;
874 size_t slen;
875 char *emsg;
876
877 if (GNUNET_OK !=
878 GNUNET_CONFIGURATION_get_value_filename (cfg,
879 "FS",
880 "INDEXDB",
881 &fn))
882 {
883 GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
884 _("Configuration option `%s' in section `%s' missing.\n"),
885 "INDEXDB",
886 "FS");
887 return;
888 }
889 if (GNUNET_NO == GNUNET_DISK_file_test (fn))
890 {
891 /* no index info yet */
892 GNUNET_free (fn);
893 return;
894 }
895 rh = GNUNET_BIO_read_open (fn);
896 if (NULL == rh)
897 {
898 GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
899 _("Could not open `%s'.\n"),
900 fn);
901 GNUNET_free (fn);
902 return;
903 }
904
905 while ( (GNUNET_OK ==
906 GNUNET_BIO_read (rh,
907 "Hash of indexed file",
908 &hc,
909 sizeof (GNUNET_HashCode))) &&
910 (GNUNET_OK ==
911 GNUNET_BIO_read_string (rh,
912 "Name of indexed file",
913 &fname,
914 1024 * 16)) )
915 {
916 slen = strlen (fname) + 1;
917 pos = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
918 pos->file_id = hc;
919 pos->filename = (const char *) &pos[1];
920 memcpy (&pos[1], fname, slen);
921 if (GNUNET_SYSERR ==
922 GNUNET_CONTAINER_multihashmap_put (ifm,
923 &hc,
924 (void*) pos->filename,
925 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
926 {
927 GNUNET_free (pos);
928 }
929 else
930 {
931 pos->next = indexed_files;
932 indexed_files = pos;
933 }
934 GNUNET_free (fname);
935 }
936 if (GNUNET_OK !=
937 GNUNET_BIO_read_close (rh, &emsg))
938 GNUNET_free (emsg);
939 GNUNET_free (fn);
940}
941
942
943/**
944 * We've validated the hash of the file we're about to
945 * index. Signal success to the client and update
946 * our internal data structures.
947 *
948 * @param ii the index info entry for the request
949 */
950static void
951signal_index_ok (struct IndexInfo *ii)
952{
953 if (GNUNET_SYSERR ==
954 GNUNET_CONTAINER_multihashmap_put (ifm,
955 &ii->file_id,
956 (void*) ii->filename,
957 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
958 {
959 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
960 _("Index request received for file `%s' is indexed as `%s'. Permitting anyway.\n"),
961 ii->filename,
962 (const char*) GNUNET_CONTAINER_multihashmap_get (ifm,
963 &ii->file_id));
964 GNUNET_SERVER_transmit_context_append_data (ii->tc,
965 NULL, 0,
966 GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
967 GNUNET_SERVER_transmit_context_run (ii->tc,
968 GNUNET_TIME_UNIT_MINUTES);
969 GNUNET_free (ii);
970 return;
971 }
972 ii->next = indexed_files;
973 indexed_files = ii;
974 write_index_list ();
975 GNUNET_SERVER_transmit_context_append_data (ii->tc,
976 NULL, 0,
977 GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
978 GNUNET_SERVER_transmit_context_run (ii->tc,
979 GNUNET_TIME_UNIT_MINUTES);
980 ii->tc = NULL;
981}
982
983
984/**
985 * Function called once the hash computation over an
986 * indexed file has completed.
987 *
988 * @param cls closure, our publishing context
989 * @param res resulting hash, NULL on error
990 */
991static void
992hash_for_index_val (void *cls,
993 const GNUNET_HashCode *
994 res)
995{
996 struct IndexInfo *ii = cls;
997
998 if ( (res == NULL) ||
999 (0 != memcmp (res,
1000 &ii->file_id,
1001 sizeof(GNUNET_HashCode))) )
1002 {
1003 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1004 _("Hash mismatch trying to index file `%s' which has hash `%s'\n"),
1005 ii->filename,
1006 GNUNET_h2s (res));
1007#if DEBUG_FS
1008 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1009 "Wanted `%s'\n",
1010 GNUNET_h2s (&ii->file_id));
1011#endif
1012 GNUNET_SERVER_transmit_context_append_data (ii->tc,
1013 NULL, 0,
1014 GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED);
1015 GNUNET_SERVER_transmit_context_run (ii->tc,
1016 GNUNET_TIME_UNIT_MINUTES);
1017 GNUNET_free (ii);
1018 return;
1019 }
1020 signal_index_ok (ii);
1021}
1022
1023
1024/**
1025 * Handle INDEX_START-message.
1026 *
1027 * @param cls closure
1028 * @param client identification of the client
1029 * @param message the actual message
1030 */
1031static void
1032handle_index_start (void *cls,
1033 struct GNUNET_SERVER_Client *client,
1034 const struct GNUNET_MessageHeader *message)
1035{
1036 const struct IndexStartMessage *ism;
1037 const char *fn;
1038 uint16_t msize;
1039 struct IndexInfo *ii;
1040 size_t slen;
1041 uint32_t dev;
1042 uint64_t ino;
1043 uint32_t mydev;
1044 uint64_t myino;
1045
1046 msize = ntohs(message->size);
1047 if ( (msize <= sizeof (struct IndexStartMessage)) ||
1048 ( ((const char *)message)[msize-1] != '\0') )
1049 {
1050 GNUNET_break (0);
1051 GNUNET_SERVER_receive_done (client,
1052 GNUNET_SYSERR);
1053 return;
1054 }
1055 ism = (const struct IndexStartMessage*) message;
1056 fn = (const char*) &ism[1];
1057 dev = ntohl (ism->device);
1058 ino = GNUNET_ntohll (ism->inode);
1059 ism = (const struct IndexStartMessage*) message;
1060 slen = strlen (fn) + 1;
1061 ii = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
1062 ii->filename = (const char*) &ii[1];
1063 memcpy (&ii[1], fn, slen);
1064 ii->file_id = ism->file_id;
1065 ii->tc = GNUNET_SERVER_transmit_context_create (client);
1066 if ( ( (dev != 0) ||
1067 (ino != 0) ) &&
1068 (GNUNET_OK == GNUNET_DISK_file_get_identifiers (fn,
1069 &mydev,
1070 &myino)) &&
1071 ( (dev == mydev) &&
1072 (ino == myino) ) )
1073 {
1074 /* fast validation OK! */
1075 signal_index_ok (ii);
1076 return;
1077 }
1078#if DEBUG_FS
1079 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1080 "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n",
1081 (unsigned long long) ino,
1082 (unsigned long long) myino,
1083 (unsigned int) dev,
1084 (unsigned int) mydev);
1085#endif
1086 /* slow validation, need to hash full file (again) */
1087 GNUNET_CRYPTO_hash_file (sched,
1088 GNUNET_SCHEDULER_PRIORITY_IDLE,
1089 fn,
1090 HASHING_BLOCKSIZE,
1091 &hash_for_index_val,
1092 ii);
1093}
1094
1095
1096/**
1097 * Handle INDEX_LIST_GET-message.
1098 *
1099 * @param cls closure
1100 * @param client identification of the client
1101 * @param message the actual message
1102 */
1103static void
1104handle_index_list_get (void *cls,
1105 struct GNUNET_SERVER_Client *client,
1106 const struct GNUNET_MessageHeader *message)
1107{
1108 struct GNUNET_SERVER_TransmitContext *tc;
1109 struct IndexInfoMessage *iim;
1110 char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE];
1111 size_t slen;
1112 const char *fn;
1113 struct IndexInfo *pos;
1114
1115 tc = GNUNET_SERVER_transmit_context_create (client);
1116 iim = (struct IndexInfoMessage*) buf;
1117 pos = indexed_files;
1118 while (NULL != pos)
1119 {
1120 fn = pos->filename;
1121 slen = strlen (fn) + 1;
1122 if (slen + sizeof (struct IndexInfoMessage) >
1123 GNUNET_SERVER_MAX_MESSAGE_SIZE)
1124 {
1125 GNUNET_break (0);
1126 break;
1127 }
1128 iim->header.type = htons (GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY);
1129 iim->header.size = htons (slen + sizeof (struct IndexInfoMessage));
1130 iim->reserved = 0;
1131 iim->file_id = pos->file_id;
1132 memcpy (&iim[1], fn, slen);
1133 GNUNET_SERVER_transmit_context_append_message (tc,
1134 &iim->header);
1135 pos = pos->next;
1136 }
1137 GNUNET_SERVER_transmit_context_append_data (tc,
1138 NULL, 0,
1139 GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END);
1140 GNUNET_SERVER_transmit_context_run (tc,
1141 GNUNET_TIME_UNIT_MINUTES);
1142}
1143 781
1144 782
1145/**
1146 * Handle UNINDEX-message.
1147 *
1148 * @param cls closure
1149 * @param client identification of the client
1150 * @param message the actual message
1151 */
1152static void
1153handle_unindex (void *cls,
1154 struct GNUNET_SERVER_Client *client,
1155 const struct GNUNET_MessageHeader *message)
1156{
1157 const struct UnindexMessage *um;
1158 struct IndexInfo *pos;
1159 struct IndexInfo *prev;
1160 struct IndexInfo *next;
1161 struct GNUNET_SERVER_TransmitContext *tc;
1162 int found;
1163
1164 um = (const struct UnindexMessage*) message;
1165 found = GNUNET_NO;
1166 prev = NULL;
1167 pos = indexed_files;
1168 while (NULL != pos)
1169 {
1170 next = pos->next;
1171 if (0 == memcmp (&pos->file_id,
1172 &um->file_id,
1173 sizeof (GNUNET_HashCode)))
1174 {
1175 if (prev == NULL)
1176 indexed_files = next;
1177 else
1178 prev->next = next;
1179 GNUNET_free (pos);
1180 found = GNUNET_YES;
1181 }
1182 else
1183 {
1184 prev = pos;
1185 }
1186 pos = next;
1187 }
1188#if DEBUG_FS
1189 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1190 "Client requested unindexing of file `%s': %s\n",
1191 GNUNET_h2s (&um->file_id),
1192 found ? "found" : "not found");
1193#endif
1194 if (GNUNET_YES == found)
1195 write_index_list ();
1196 tc = GNUNET_SERVER_transmit_context_create (client);
1197 GNUNET_SERVER_transmit_context_append_data (tc,
1198 NULL, 0,
1199 GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK);
1200 GNUNET_SERVER_transmit_context_run (tc,
1201 GNUNET_TIME_UNIT_MINUTES);
1202}
1203 783
1204 784
1205/** 785/**
@@ -1370,27 +950,6 @@ transmit_local_result (void *cls,
1370 950
1371 951
1372/** 952/**
1373 * Continuation called from datastore's remove
1374 * function.
1375 *
1376 * @param cls unused
1377 * @param success did the deletion work?
1378 * @param msg error message
1379 */
1380static void
1381remove_cont (void *cls,
1382 int success,
1383 const char *msg)
1384{
1385 if (GNUNET_OK != success)
1386 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1387 _("Failed to delete bogus block: %s\n"),
1388 msg);
1389 GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
1390}
1391
1392
1393/**
1394 * Mingle hash with the mingle_number to 953 * Mingle hash with the mingle_number to
1395 * produce different bits. 954 * produce different bits.
1396 */ 955 */
@@ -1409,130 +968,6 @@ mingle_hash (const GNUNET_HashCode * in,
1409 968
1410 969
1411/** 970/**
1412 * We've received an on-demand encoded block
1413 * from the datastore. Attempt to do on-demand
1414 * encoding and (if successful), call the
1415 * continuation with the resulting block. On
1416 * error, clean up and ask the datastore for
1417 * more results.
1418 *
1419 * @param key key for the content
1420 * @param size number of bytes in data
1421 * @param data content stored
1422 * @param type type of the content
1423 * @param priority priority of the content
1424 * @param anonymity anonymity-level for the content
1425 * @param expiration expiration time for the content
1426 * @param uid unique identifier for the datum;
1427 * maybe 0 if no unique identifier is available
1428 * @param cont function to call with the actual block
1429 * @param cont_cls closure for cont
1430 */
1431static void
1432handle_on_demand_block (const GNUNET_HashCode * key,
1433 uint32_t size,
1434 const void *data,
1435 uint32_t type,
1436 uint32_t priority,
1437 uint32_t anonymity,
1438 struct GNUNET_TIME_Absolute
1439 expiration, uint64_t uid,
1440 GNUNET_DATASTORE_Iterator cont,
1441 void *cont_cls)
1442{
1443 const struct OnDemandBlock *odb;
1444 GNUNET_HashCode nkey;
1445 struct GNUNET_CRYPTO_AesSessionKey skey;
1446 struct GNUNET_CRYPTO_AesInitializationVector iv;
1447 GNUNET_HashCode query;
1448 ssize_t nsize;
1449 char ndata[DBLOCK_SIZE];
1450 char edata[DBLOCK_SIZE];
1451 const char *fn;
1452 struct GNUNET_DISK_FileHandle *fh;
1453 uint64_t off;
1454
1455 if (size != sizeof (struct OnDemandBlock))
1456 {
1457 GNUNET_break (0);
1458 GNUNET_DATASTORE_remove (dsh,
1459 key,
1460 size,
1461 data,
1462 &remove_cont,
1463 NULL,
1464 GNUNET_TIME_UNIT_FOREVER_REL);
1465 return;
1466 }
1467 odb = (const struct OnDemandBlock*) data;
1468 off = GNUNET_ntohll (odb->offset);
1469 fn = (const char*) GNUNET_CONTAINER_multihashmap_get (ifm,
1470 &odb->file_id);
1471 fh = NULL;
1472 if ( (NULL == fn) ||
1473 (NULL == (fh = GNUNET_DISK_file_open (fn,
1474 GNUNET_DISK_OPEN_READ,
1475 GNUNET_DISK_PERM_NONE))) ||
1476 (off !=
1477 GNUNET_DISK_file_seek (fh,
1478 off,
1479 GNUNET_DISK_SEEK_SET)) ||
1480 (-1 ==
1481 (nsize = GNUNET_DISK_file_read (fh,
1482 ndata,
1483 sizeof (ndata)))) )
1484 {
1485 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1486 _("Could not access indexed file `%s' at offset %llu: %s\n"),
1487 GNUNET_h2s (&odb->file_id),
1488 (unsigned long long) off,
1489 STRERROR (errno));
1490 if (fh != NULL)
1491 GNUNET_DISK_file_close (fh);
1492 /* FIXME: if this happens often, we need
1493 to remove the OnDemand block from the DS! */
1494 GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
1495 return;
1496 }
1497 GNUNET_DISK_file_close (fh);
1498 GNUNET_CRYPTO_hash (ndata,
1499 nsize,
1500 &nkey);
1501 GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv);
1502 GNUNET_CRYPTO_aes_encrypt (ndata,
1503 nsize,
1504 &skey,
1505 &iv,
1506 edata);
1507 GNUNET_CRYPTO_hash (edata,
1508 nsize,
1509 &query);
1510 if (0 != memcmp (&query,
1511 key,
1512 sizeof (GNUNET_HashCode)))
1513 {
1514 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1515 _("Indexed file `%s' changed at offset %llu\n"),
1516 fn,
1517 (unsigned long long) off);
1518 /* FIXME: if this happens often, we need
1519 to remove the OnDemand block from the DS! */
1520 GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
1521 return;
1522 }
1523 cont (cont_cls,
1524 key,
1525 nsize,
1526 edata,
1527 GNUNET_DATASTORE_BLOCKTYPE_DBLOCK,
1528 priority,
1529 anonymity,
1530 expiration,
1531 uid);
1532}
1533
1534
1535/**
1536 * How many bytes should a bloomfilter be if we have already seen 971 * How many bytes should a bloomfilter be if we have already seen
1537 * entry_count responses? Note that BLOOMFILTER_K gives us the number 972 * entry_count responses? Note that BLOOMFILTER_K gives us the number
1538 * of bits set per entry. Furthermore, we should not re-size the 973 * of bits set per entry. Furthermore, we should not re-size the
@@ -2001,10 +1436,11 @@ process_local_get_result (void *cls,
2001 "Received on-demand block for `%s' from local datastore, fetching data.\n", 1436 "Received on-demand block for `%s' from local datastore, fetching data.\n",
2002 GNUNET_h2s (&lgc->query)); 1437 GNUNET_h2s (&lgc->query));
2003#endif 1438#endif
2004 handle_on_demand_block (key, size, data, type, priority, 1439 GNUNET_FS_handle_on_demand_block (key, size, data, type, priority,
2005 anonymity, expiration, uid, 1440 anonymity, expiration, uid,
2006 &process_local_get_result, 1441 dsh,
2007 lgc); 1442 &process_local_get_result,
1443 lgc);
2008 return; 1444 return;
2009 } 1445 }
2010 if ( (type != lgc->type) && 1446 if ( (type != lgc->type) &&
@@ -2184,11 +1620,11 @@ handle_start_search (void *cls,
2184 * service. 1620 * service.
2185 */ 1621 */
2186static struct GNUNET_SERVER_MessageHandler handlers[] = { 1622static struct GNUNET_SERVER_MessageHandler handlers[] = {
2187 {&handle_index_start, NULL, 1623 {&GNUNET_FS_handle_index_start, NULL,
2188 GNUNET_MESSAGE_TYPE_FS_INDEX_START, 0}, 1624 GNUNET_MESSAGE_TYPE_FS_INDEX_START, 0},
2189 {&handle_index_list_get, NULL, 1625 {&GNUNET_FS_handle_index_list_get, NULL,
2190 GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_GET, sizeof(struct GNUNET_MessageHeader) }, 1626 GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_GET, sizeof(struct GNUNET_MessageHeader) },
2191 {&handle_unindex, NULL, GNUNET_MESSAGE_TYPE_FS_UNINDEX, 1627 {&GNUNET_FS_handle_unindex, NULL, GNUNET_MESSAGE_TYPE_FS_UNINDEX,
2192 sizeof (struct UnindexMessage) }, 1628 sizeof (struct UnindexMessage) },
2193 {&handle_start_search, NULL, GNUNET_MESSAGE_TYPE_FS_START_SEARCH, 1629 {&handle_start_search, NULL, GNUNET_MESSAGE_TYPE_FS_START_SEARCH,
2194 0 }, 1630 0 },
@@ -2205,7 +1641,7 @@ static struct GNUNET_SERVER_MessageHandler handlers[] = {
2205static void 1641static void
2206destroy_pending_request (struct PendingRequest *pr) 1642destroy_pending_request (struct PendingRequest *pr)
2207{ 1643{
2208 struct PendingReply *reply; 1644 struct PendingMessage *reply;
2209 struct ClientList *cl; 1645 struct ClientList *cl;
2210 1646
2211 GNUNET_CONTAINER_multihashmap_remove (requests_by_query, 1647 GNUNET_CONTAINER_multihashmap_remove (requests_by_query,
@@ -2333,8 +1769,6 @@ static void
2333shutdown_task (void *cls, 1769shutdown_task (void *cls,
2334 const struct GNUNET_SCHEDULER_TaskContext *tc) 1770 const struct GNUNET_SCHEDULER_TaskContext *tc)
2335{ 1771{
2336 struct IndexInfo *pos;
2337
2338 if (NULL != core) 1772 if (NULL != core)
2339 { 1773 {
2340 GNUNET_CORE_disconnect (core); 1774 GNUNET_CORE_disconnect (core);
@@ -2362,13 +1796,6 @@ shutdown_task (void *cls,
2362 // (or do we get disconnect notifications?) 1796 // (or do we get disconnect notifications?)
2363 GNUNET_CONTAINER_multihashmap_destroy (connected_peers); 1797 GNUNET_CONTAINER_multihashmap_destroy (connected_peers);
2364 connected_peers = NULL; 1798 connected_peers = NULL;
2365 GNUNET_CONTAINER_multihashmap_destroy (ifm);
2366 ifm = NULL;
2367 while (NULL != (pos = indexed_files))
2368 {
2369 indexed_files = pos->next;
2370 GNUNET_free (pos);
2371 }
2372} 1799}
2373 1800
2374 1801
@@ -2623,10 +2050,10 @@ process_p2p_get_result (void *cls,
2623 } 2050 }
2624 if (type == GNUNET_DATASTORE_BLOCKTYPE_ONDEMAND) 2051 if (type == GNUNET_DATASTORE_BLOCKTYPE_ONDEMAND)
2625 { 2052 {
2626 handle_on_demand_block (key, size, data, type, priority, 2053 GNUNET_FS_handle_on_demand_block (key, size, data, type, priority,
2627 anonymity, expiration, uid, 2054 anonymity, expiration, uid, dsh,
2628 &process_p2p_get_result, 2055 &process_p2p_get_result,
2629 pgc); 2056 pgc);
2630 return; 2057 return;
2631 } 2058 }
2632 /* check for duplicates */ 2059 /* check for duplicates */
@@ -2961,7 +2388,7 @@ transmit_result (void *cls,
2961{ 2388{
2962 struct PendingRequest *pr = cls; 2389 struct PendingRequest *pr = cls;
2963 char *cbuf = buf; 2390 char *cbuf = buf;
2964 struct PendingReply *reply; 2391 struct PendingMessage *reply;
2965 size_t ret; 2392 size_t ret;
2966 2393
2967 ret = 0; 2394 ret = 0;
@@ -2995,7 +2422,7 @@ process_reply (void *cls,
2995 struct ProcessReplyClosure *prq = cls; 2422 struct ProcessReplyClosure *prq = cls;
2996 struct PendingRequest *pr = value; 2423 struct PendingRequest *pr = value;
2997 struct PendingRequest *eer; 2424 struct PendingRequest *eer;
2998 struct PendingReply *reply; 2425 struct PendingMessage *reply;
2999 struct PutMessage *pm; 2426 struct PutMessage *pm;
3000 struct ContentMessage *cm; 2427 struct ContentMessage *cm;
3001 GNUNET_HashCode chash; 2428 GNUNET_HashCode chash;
@@ -3046,7 +2473,7 @@ process_reply (void *cls,
3046 if (pr->client == NULL) 2473 if (pr->client == NULL)
3047 { 2474 {
3048 msize = sizeof (struct ContentMessage) + prq->size; 2475 msize = sizeof (struct ContentMessage) + prq->size;
3049 reply = GNUNET_malloc (msize + sizeof (struct PendingReply)); 2476 reply = GNUNET_malloc (msize + sizeof (struct PendingMessage));
3050 reply->msize = msize; 2477 reply->msize = msize;
3051 cm = (struct ContentMessage*) &reply[1]; 2478 cm = (struct ContentMessage*) &reply[1];
3052 cm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_CONTENT); 2479 cm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_CONTENT);
@@ -3084,7 +2511,7 @@ process_reply (void *cls,
3084 else 2511 else
3085 { 2512 {
3086 msize = sizeof (struct PutMessage) + prq->size; 2513 msize = sizeof (struct PutMessage) + prq->size;
3087 reply = GNUNET_malloc (msize + sizeof (struct PendingReply)); 2514 reply = GNUNET_malloc (msize + sizeof (struct PendingMessage));
3088 reply->msize = msize; 2515 reply->msize = msize;
3089 reply->next = pr->replies_pending; 2516 reply->next = pr->replies_pending;
3090 pm = (struct PutMessage*) &reply[1]; 2517 pm = (struct PutMessage*) &reply[1];
@@ -3319,12 +2746,12 @@ run (void *cls,
3319 sched = s; 2746 sched = s;
3320 cfg = c; 2747 cfg = c;
3321 2748
3322 ifm = GNUNET_CONTAINER_multihashmap_create (128); 2749
3323 requests_by_query = GNUNET_CONTAINER_multihashmap_create (128); // FIXME: get size from config 2750 requests_by_query = GNUNET_CONTAINER_multihashmap_create (128); // FIXME: get size from config
3324 requests_by_peer = GNUNET_CONTAINER_multihashmap_create (128); // FIXME: get size from config 2751 requests_by_peer = GNUNET_CONTAINER_multihashmap_create (128); // FIXME: get size from config
3325 connected_peers = GNUNET_CONTAINER_multihashmap_create (64); 2752 connected_peers = GNUNET_CONTAINER_multihashmap_create (64);
3326 requests_by_expiration = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); 2753 requests_by_expiration = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
3327 read_index_list (); 2754 GNUNET_FS_init_indexing (sched, cfg);
3328 dsh = GNUNET_DATASTORE_connect (cfg, 2755 dsh = GNUNET_DATASTORE_connect (cfg,
3329 sched); 2756 sched);
3330 core = GNUNET_CORE_connect (sched, 2757 core = GNUNET_CORE_connect (sched,
diff --git a/src/fs/gnunet-service-fs_indexing.c b/src/fs/gnunet-service-fs_indexing.c
new file mode 100644
index 000000000..ebd7114d3
--- /dev/null
+++ b/src/fs/gnunet-service-fs_indexing.c
@@ -0,0 +1,688 @@
1/*
2 This file is part of GNUnet.
3 (C) 2009, 2010 Christian Grothoff (and other contributing authors)
4
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
19*/
20
21/**
22 * @file fs/gnunet-service-fs_indexing.c
23 * @brief program that provides indexing functions of the file-sharing service
24 * @author Christian Grothoff
25 *
26 * TODO:
27 * - indexed files/blocks not removed on errors
28 */
29#include "platform.h"
30#include <float.h>
31#include "gnunet_core_service.h"
32#include "gnunet_datastore_service.h"
33#include "gnunet_peer_lib.h"
34#include "gnunet_protocols.h"
35#include "gnunet_signatures.h"
36#include "gnunet_util_lib.h"
37#include "gnunet-service-fs_indexing.h"
38#include "fs.h"
39
40#define DEBUG_FS GNUNET_NO
41
42
43/**
44 * In-memory information about indexed files (also available
45 * on-disk).
46 */
47struct IndexInfo
48{
49
50 /**
51 * This is a linked list.
52 */
53 struct IndexInfo *next;
54
55 /**
56 * Name of the indexed file. Memory allocated
57 * at the end of this struct (do not free).
58 */
59 const char *filename;
60
61 /**
62 * Context for transmitting confirmation to client,
63 * NULL if we've done this already.
64 */
65 struct GNUNET_SERVER_TransmitContext *tc;
66
67 /**
68 * Hash of the contents of the file.
69 */
70 GNUNET_HashCode file_id;
71
72};
73
74
75/**
76 * Linked list of indexed files.
77 */
78static struct IndexInfo *indexed_files;
79
80/**
81 * Maps hash over content of indexed files to the respective filename.
82 * The filenames are pointers into the indexed_files linked list and
83 * do not need to be freed.
84 */
85static struct GNUNET_CONTAINER_MultiHashMap *ifm;
86
87/**
88 * Our scheduler.
89 */
90static struct GNUNET_SCHEDULER_Handle *sched;
91
92/**
93 * Our configuration.
94 */
95const struct GNUNET_CONFIGURATION_Handle *cfg;
96
97
98/**
99 * Write the current index information list to disk.
100 */
101static void
102write_index_list ()
103{
104 struct GNUNET_BIO_WriteHandle *wh;
105 char *fn;
106 struct IndexInfo *pos;
107
108 if (GNUNET_OK !=
109 GNUNET_CONFIGURATION_get_value_filename (cfg,
110 "FS",
111 "INDEXDB",
112 &fn))
113 {
114 GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
115 _("Configuration option `%s' in section `%s' missing.\n"),
116 "INDEXDB",
117 "FS");
118 return;
119 }
120 wh = GNUNET_BIO_write_open (fn);
121 if (NULL == wh)
122 {
123 GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
124 _("Could not open `%s'.\n"),
125 fn);
126 GNUNET_free (fn);
127 return;
128 }
129 pos = indexed_files;
130 while (pos != NULL)
131 {
132 if ( (GNUNET_OK !=
133 GNUNET_BIO_write (wh,
134 &pos->file_id,
135 sizeof (GNUNET_HashCode))) ||
136 (GNUNET_OK !=
137 GNUNET_BIO_write_string (wh,
138 pos->filename)) )
139 break;
140 pos = pos->next;
141 }
142 if (GNUNET_OK !=
143 GNUNET_BIO_write_close (wh))
144 {
145 GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
146 _("Error writing `%s'.\n"),
147 fn);
148 GNUNET_free (fn);
149 return;
150 }
151 GNUNET_free (fn);
152}
153
154
155/**
156 * Read index information from disk.
157 */
158static void
159read_index_list ()
160{
161 struct GNUNET_BIO_ReadHandle *rh;
162 char *fn;
163 struct IndexInfo *pos;
164 char *fname;
165 GNUNET_HashCode hc;
166 size_t slen;
167 char *emsg;
168
169 if (GNUNET_OK !=
170 GNUNET_CONFIGURATION_get_value_filename (cfg,
171 "FS",
172 "INDEXDB",
173 &fn))
174 {
175 GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
176 _("Configuration option `%s' in section `%s' missing.\n"),
177 "INDEXDB",
178 "FS");
179 return;
180 }
181 if (GNUNET_NO == GNUNET_DISK_file_test (fn))
182 {
183 /* no index info yet */
184 GNUNET_free (fn);
185 return;
186 }
187 rh = GNUNET_BIO_read_open (fn);
188 if (NULL == rh)
189 {
190 GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
191 _("Could not open `%s'.\n"),
192 fn);
193 GNUNET_free (fn);
194 return;
195 }
196
197 while ( (GNUNET_OK ==
198 GNUNET_BIO_read (rh,
199 "Hash of indexed file",
200 &hc,
201 sizeof (GNUNET_HashCode))) &&
202 (GNUNET_OK ==
203 GNUNET_BIO_read_string (rh,
204 "Name of indexed file",
205 &fname,
206 1024 * 16)) )
207 {
208 slen = strlen (fname) + 1;
209 pos = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
210 pos->file_id = hc;
211 pos->filename = (const char *) &pos[1];
212 memcpy (&pos[1], fname, slen);
213 if (GNUNET_SYSERR ==
214 GNUNET_CONTAINER_multihashmap_put (ifm,
215 &hc,
216 (void*) pos->filename,
217 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
218 {
219 GNUNET_free (pos);
220 }
221 else
222 {
223 pos->next = indexed_files;
224 indexed_files = pos;
225 }
226 GNUNET_free (fname);
227 }
228 if (GNUNET_OK !=
229 GNUNET_BIO_read_close (rh, &emsg))
230 GNUNET_free (emsg);
231 GNUNET_free (fn);
232}
233
234
235/**
236 * We've validated the hash of the file we're about to index. Signal
237 * success to the client and update our internal data structures.
238 *
239 * @param ii the index info entry for the request
240 */
241static void
242signal_index_ok (struct IndexInfo *ii)
243{
244 if (GNUNET_SYSERR ==
245 GNUNET_CONTAINER_multihashmap_put (ifm,
246 &ii->file_id,
247 (void*) ii->filename,
248 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
249 {
250 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
251 _("Index request received for file `%s' is indexed as `%s'. Permitting anyway.\n"),
252 ii->filename,
253 (const char*) GNUNET_CONTAINER_multihashmap_get (ifm,
254 &ii->file_id));
255 GNUNET_SERVER_transmit_context_append_data (ii->tc,
256 NULL, 0,
257 GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
258 GNUNET_SERVER_transmit_context_run (ii->tc,
259 GNUNET_TIME_UNIT_MINUTES);
260 GNUNET_free (ii);
261 return;
262 }
263 ii->next = indexed_files;
264 indexed_files = ii;
265 write_index_list ();
266 GNUNET_SERVER_transmit_context_append_data (ii->tc,
267 NULL, 0,
268 GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
269 GNUNET_SERVER_transmit_context_run (ii->tc,
270 GNUNET_TIME_UNIT_MINUTES);
271 ii->tc = NULL;
272}
273
274
275/**
276 * Function called once the hash computation over an
277 * indexed file has completed.
278 *
279 * @param cls closure, our publishing context
280 * @param res resulting hash, NULL on error
281 */
282static void
283hash_for_index_val (void *cls,
284 const GNUNET_HashCode *
285 res)
286{
287 struct IndexInfo *ii = cls;
288
289 if ( (res == NULL) ||
290 (0 != memcmp (res,
291 &ii->file_id,
292 sizeof(GNUNET_HashCode))) )
293 {
294 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
295 _("Hash mismatch trying to index file `%s' which has hash `%s'\n"),
296 ii->filename,
297 GNUNET_h2s (res));
298#if DEBUG_FS
299 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
300 "Wanted `%s'\n",
301 GNUNET_h2s (&ii->file_id));
302#endif
303 GNUNET_SERVER_transmit_context_append_data (ii->tc,
304 NULL, 0,
305 GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED);
306 GNUNET_SERVER_transmit_context_run (ii->tc,
307 GNUNET_TIME_UNIT_MINUTES);
308 GNUNET_free (ii);
309 return;
310 }
311 signal_index_ok (ii);
312}
313
314
315/**
316 * Handle INDEX_START-message.
317 *
318 * @param cls closure
319 * @param client identification of the client
320 * @param message the actual message
321 */
322void
323GNUNET_FS_handle_index_start (void *cls,
324 struct GNUNET_SERVER_Client *client,
325 const struct GNUNET_MessageHeader *message)
326{
327 const struct IndexStartMessage *ism;
328 const char *fn;
329 uint16_t msize;
330 struct IndexInfo *ii;
331 size_t slen;
332 uint32_t dev;
333 uint64_t ino;
334 uint32_t mydev;
335 uint64_t myino;
336
337 msize = ntohs(message->size);
338 if ( (msize <= sizeof (struct IndexStartMessage)) ||
339 ( ((const char *)message)[msize-1] != '\0') )
340 {
341 GNUNET_break (0);
342 GNUNET_SERVER_receive_done (client,
343 GNUNET_SYSERR);
344 return;
345 }
346 ism = (const struct IndexStartMessage*) message;
347 fn = (const char*) &ism[1];
348 dev = ntohl (ism->device);
349 ino = GNUNET_ntohll (ism->inode);
350 ism = (const struct IndexStartMessage*) message;
351 slen = strlen (fn) + 1;
352 ii = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
353 ii->filename = (const char*) &ii[1];
354 memcpy (&ii[1], fn, slen);
355 ii->file_id = ism->file_id;
356 ii->tc = GNUNET_SERVER_transmit_context_create (client);
357 if ( ( (dev != 0) ||
358 (ino != 0) ) &&
359 (GNUNET_OK == GNUNET_DISK_file_get_identifiers (fn,
360 &mydev,
361 &myino)) &&
362 ( (dev == mydev) &&
363 (ino == myino) ) )
364 {
365 /* fast validation OK! */
366 signal_index_ok (ii);
367 return;
368 }
369#if DEBUG_FS
370 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
371 "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n",
372 (unsigned long long) ino,
373 (unsigned long long) myino,
374 (unsigned int) dev,
375 (unsigned int) mydev);
376#endif
377 /* slow validation, need to hash full file (again) */
378 GNUNET_CRYPTO_hash_file (sched,
379 GNUNET_SCHEDULER_PRIORITY_IDLE,
380 fn,
381 HASHING_BLOCKSIZE,
382 &hash_for_index_val,
383 ii);
384}
385
386
387/**
388 * Handle INDEX_LIST_GET-message.
389 *
390 * @param cls closure
391 * @param client identification of the client
392 * @param message the actual message
393 */
394void
395GNUNET_FS_handle_index_list_get (void *cls,
396 struct GNUNET_SERVER_Client *client,
397 const struct GNUNET_MessageHeader *message)
398{
399 struct GNUNET_SERVER_TransmitContext *tc;
400 struct IndexInfoMessage *iim;
401 char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE];
402 size_t slen;
403 const char *fn;
404 struct IndexInfo *pos;
405
406 tc = GNUNET_SERVER_transmit_context_create (client);
407 iim = (struct IndexInfoMessage*) buf;
408 pos = indexed_files;
409 while (NULL != pos)
410 {
411 fn = pos->filename;
412 slen = strlen (fn) + 1;
413 if (slen + sizeof (struct IndexInfoMessage) >
414 GNUNET_SERVER_MAX_MESSAGE_SIZE)
415 {
416 GNUNET_break (0);
417 break;
418 }
419 iim->header.type = htons (GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY);
420 iim->header.size = htons (slen + sizeof (struct IndexInfoMessage));
421 iim->reserved = 0;
422 iim->file_id = pos->file_id;
423 memcpy (&iim[1], fn, slen);
424 GNUNET_SERVER_transmit_context_append_message (tc,
425 &iim->header);
426 pos = pos->next;
427 }
428 GNUNET_SERVER_transmit_context_append_data (tc,
429 NULL, 0,
430 GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END);
431 GNUNET_SERVER_transmit_context_run (tc,
432 GNUNET_TIME_UNIT_MINUTES);
433}
434
435
436/**
437 * Handle UNINDEX-message.
438 *
439 * @param cls closure
440 * @param client identification of the client
441 * @param message the actual message
442 */
443void
444GNUNET_FS_handle_unindex (void *cls,
445 struct GNUNET_SERVER_Client *client,
446 const struct GNUNET_MessageHeader *message)
447{
448 const struct UnindexMessage *um;
449 struct IndexInfo *pos;
450 struct IndexInfo *prev;
451 struct IndexInfo *next;
452 struct GNUNET_SERVER_TransmitContext *tc;
453 int found;
454
455 um = (const struct UnindexMessage*) message;
456 found = GNUNET_NO;
457 prev = NULL;
458 pos = indexed_files;
459 while (NULL != pos)
460 {
461 next = pos->next;
462 if (0 == memcmp (&pos->file_id,
463 &um->file_id,
464 sizeof (GNUNET_HashCode)))
465 {
466 if (prev == NULL)
467 indexed_files = next;
468 else
469 prev->next = next;
470 GNUNET_free (pos);
471 found = GNUNET_YES;
472 }
473 else
474 {
475 prev = pos;
476 }
477 pos = next;
478 }
479#if DEBUG_FS
480 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
481 "Client requested unindexing of file `%s': %s\n",
482 GNUNET_h2s (&um->file_id),
483 found ? "found" : "not found");
484#endif
485 if (GNUNET_YES == found)
486 write_index_list ();
487 tc = GNUNET_SERVER_transmit_context_create (client);
488 GNUNET_SERVER_transmit_context_append_data (tc,
489 NULL, 0,
490 GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK);
491 GNUNET_SERVER_transmit_context_run (tc,
492 GNUNET_TIME_UNIT_MINUTES);
493}
494
495
496
497
498/**
499 * Continuation called from datastore's remove
500 * function.
501 *
502 * @param cls unused
503 * @param success did the deletion work?
504 * @param msg error message
505 */
506static void
507remove_cont (void *cls,
508 int success,
509 const char *msg)
510{
511 struct GNUNET_DATASTORE_Handle *dsh = cls;
512
513 if (GNUNET_OK != success)
514 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
515 _("Failed to delete bogus block: %s\n"),
516 msg);
517 GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
518}
519
520
521/**
522 * We've received an on-demand encoded block from the datastore.
523 * Attempt to do on-demand encoding and (if successful), call the
524 * continuation with the resulting block. On error, clean up and ask
525 * the datastore for more results.
526 *
527 * @param key key for the content
528 * @param size number of bytes in data
529 * @param data content stored
530 * @param type type of the content
531 * @param priority priority of the content
532 * @param anonymity anonymity-level for the content
533 * @param expiration expiration time for the content
534 * @param uid unique identifier for the datum;
535 * maybe 0 if no unique identifier is available
536 * @param cont function to call with the actual block
537 * @param cont_cls closure for cont
538 */
539void
540GNUNET_FS_handle_on_demand_block (const GNUNET_HashCode * key,
541 uint32_t size,
542 const void *data,
543 uint32_t type,
544 uint32_t priority,
545 uint32_t anonymity,
546 struct GNUNET_TIME_Absolute
547 expiration, uint64_t uid,
548 struct GNUNET_DATASTORE_Handle *dsh,
549 GNUNET_DATASTORE_Iterator cont,
550 void *cont_cls)
551{
552 const struct OnDemandBlock *odb;
553 GNUNET_HashCode nkey;
554 struct GNUNET_CRYPTO_AesSessionKey skey;
555 struct GNUNET_CRYPTO_AesInitializationVector iv;
556 GNUNET_HashCode query;
557 ssize_t nsize;
558 char ndata[DBLOCK_SIZE];
559 char edata[DBLOCK_SIZE];
560 const char *fn;
561 struct GNUNET_DISK_FileHandle *fh;
562 uint64_t off;
563
564 if (size != sizeof (struct OnDemandBlock))
565 {
566 GNUNET_break (0);
567 GNUNET_DATASTORE_remove (dsh,
568 key,
569 size,
570 data,
571 &remove_cont,
572 dsh,
573 GNUNET_TIME_UNIT_FOREVER_REL);
574 return;
575 }
576 odb = (const struct OnDemandBlock*) data;
577 off = GNUNET_ntohll (odb->offset);
578 fn = (const char*) GNUNET_CONTAINER_multihashmap_get (ifm,
579 &odb->file_id);
580 fh = NULL;
581 if ( (NULL == fn) ||
582 (NULL == (fh = GNUNET_DISK_file_open (fn,
583 GNUNET_DISK_OPEN_READ,
584 GNUNET_DISK_PERM_NONE))) ||
585 (off !=
586 GNUNET_DISK_file_seek (fh,
587 off,
588 GNUNET_DISK_SEEK_SET)) ||
589 (-1 ==
590 (nsize = GNUNET_DISK_file_read (fh,
591 ndata,
592 sizeof (ndata)))) )
593 {
594 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
595 _("Could not access indexed file `%s' at offset %llu: %s\n"),
596 GNUNET_h2s (&odb->file_id),
597 (unsigned long long) off,
598 STRERROR (errno));
599 if (fh != NULL)
600 GNUNET_DISK_file_close (fh);
601 /* FIXME: if this happens often, we need
602 to remove the OnDemand block from the DS! */
603 GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
604 return;
605 }
606 GNUNET_DISK_file_close (fh);
607 GNUNET_CRYPTO_hash (ndata,
608 nsize,
609 &nkey);
610 GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv);
611 GNUNET_CRYPTO_aes_encrypt (ndata,
612 nsize,
613 &skey,
614 &iv,
615 edata);
616 GNUNET_CRYPTO_hash (edata,
617 nsize,
618 &query);
619 if (0 != memcmp (&query,
620 key,
621 sizeof (GNUNET_HashCode)))
622 {
623 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
624 _("Indexed file `%s' changed at offset %llu\n"),
625 fn,
626 (unsigned long long) off);
627 /* FIXME: if this happens often, we need
628 to remove the OnDemand block from the DS! */
629 GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
630 return;
631 }
632 cont (cont_cls,
633 key,
634 nsize,
635 edata,
636 GNUNET_DATASTORE_BLOCKTYPE_DBLOCK,
637 priority,
638 anonymity,
639 expiration,
640 uid);
641}
642
643
644/**
645 * Task run during shutdown.
646 *
647 * @param cls unused
648 * @param tc unused
649 */
650static void
651shutdown_task (void *cls,
652 const struct GNUNET_SCHEDULER_TaskContext *tc)
653{
654 struct IndexInfo *pos;
655
656 GNUNET_CONTAINER_multihashmap_destroy (ifm);
657 ifm = NULL;
658 while (NULL != (pos = indexed_files))
659 {
660 indexed_files = pos->next;
661 GNUNET_free (pos);
662 }
663 sched = NULL;
664 cfg = NULL;
665}
666
667
668/**
669 * Initialize the indexing submodule.
670 *
671 * @param s scheduler to use
672 * @param c configuration to use
673 */
674void
675GNUNET_FS_init_indexing (struct GNUNET_SCHEDULER_Handle *s,
676 const struct GNUNET_CONFIGURATION_Handle *c)
677{
678 sched = s;
679 cfg = c;
680 ifm = GNUNET_CONTAINER_multihashmap_create (128);
681 GNUNET_SCHEDULER_add_delayed (sched,
682 GNUNET_TIME_UNIT_FOREVER_REL,
683 &shutdown_task,
684 NULL);
685 read_index_list ();
686}
687
688/* end of gnunet-service-fs_indexing.c */
diff --git a/src/fs/gnunet-service-fs_indexing.h b/src/fs/gnunet-service-fs_indexing.h
new file mode 100644
index 000000000..dc6427234
--- /dev/null
+++ b/src/fs/gnunet-service-fs_indexing.h
@@ -0,0 +1,121 @@
1/*
2 This file is part of GNUnet.
3 (C) 2009, 2010 Christian Grothoff (and other contributing authors)
4
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
19*/
20
21/**
22 * @file fs/gnunet-service-fs_indexing.h
23 * @brief indexing for the file-sharing service
24 * @author Christian Grothoff
25 */
26#ifndef GNUNET_SERVICE_FS_INDEXING_H
27#define GNUNET_SERVICE_FS_INDEXING_H
28
29#include "gnunet_core_service.h"
30#include "gnunet_datastore_service.h"
31#include "gnunet_peer_lib.h"
32#include "gnunet_protocols.h"
33#include "gnunet_signatures.h"
34#include "gnunet_util_lib.h"
35
36
37/**
38 * We've received an on-demand encoded block from the datastore.
39 * Attempt to do on-demand encoding and (if successful), call the
40 * continuation with the resulting block. On error, clean up and ask
41 * the datastore for more results.
42 *
43 * @param key key for the content
44 * @param size number of bytes in data
45 * @param data content stored
46 * @param type type of the content
47 * @param priority priority of the content
48 * @param anonymity anonymity-level for the content
49 * @param expiration expiration time for the content
50 * @param uid unique identifier for the datum;
51 * maybe 0 if no unique identifier is available
52 * @param dsh connection to the datastore (to ask for more)
53 * @param cont function to call with the actual block
54 * @param cont_cls closure for cont
55 */
56void
57GNUNET_FS_handle_on_demand_block (const GNUNET_HashCode * key,
58 uint32_t size,
59 const void *data,
60 uint32_t type,
61 uint32_t priority,
62 uint32_t anonymity,
63 struct GNUNET_TIME_Absolute
64 expiration, uint64_t uid,
65 struct GNUNET_DATASTORE_Handle *dsh,
66 GNUNET_DATASTORE_Iterator cont,
67 void *cont_cls);
68
69/**
70 * Handle INDEX_START-message.
71 *
72 * @param cls closure
73 * @param client identification of the client
74 * @param message the actual message
75 */
76void
77GNUNET_FS_handle_index_start (void *cls,
78 struct GNUNET_SERVER_Client *client,
79 const struct GNUNET_MessageHeader *message);
80
81
82/**
83 * Handle INDEX_LIST_GET-message.
84 *
85 * @param cls closure
86 * @param client identification of the client
87 * @param message the actual message
88 */
89void
90GNUNET_FS_handle_index_list_get (void *cls,
91 struct GNUNET_SERVER_Client *client,
92 const struct GNUNET_MessageHeader *message);
93
94
95/**
96 * Handle UNINDEX-message.
97 *
98 * @param cls closure
99 * @param client identification of the client
100 * @param message the actual message
101 */
102void
103GNUNET_FS_handle_unindex (void *cls,
104 struct GNUNET_SERVER_Client *client,
105 const struct GNUNET_MessageHeader *message);
106
107
108
109
110/**
111 * Initialize the indexing submodule.
112 *
113 * @param s scheduler to use
114 * @param c configuration to use
115 */
116void
117GNUNET_FS_init_indexing (struct GNUNET_SCHEDULER_Handle *s,
118 const struct GNUNET_CONFIGURATION_Handle *c);
119
120
121#endif
generated by cgit v1.2.3 (git 2.39.1) at 2024-04-28 12:56:37 +0000