diff options
author | Christian Grothoff <christian@grothoff.org> | 2018-04-15 20:57:35 +0200 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2018-04-15 20:57:35 +0200 |
commit | 98c02969dbb2bca8ec4aff79f57b8a3e349ed01a (patch) | |
tree | 11d6887ce71b662965b116380ba9b13dfc7c1c40 /src/namestore | |
parent | 47a9a3c95de2dfadc431d34c1dc079d35d3d6d18 (diff) | |
download | gnunet-98c02969dbb2bca8ec4aff79f57b8a3e349ed01a.tar.gz gnunet-98c02969dbb2bca8ec4aff79f57b8a3e349ed01a.zip |
add support for DB transaction optimization to gnunet-zoneimport
Diffstat (limited to 'src/namestore')
-rw-r--r-- | src/namestore/gnunet-zoneimport.c | 73 |
1 files changed, 63 insertions, 10 deletions
diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c index e1d5b650d..2af63dc3d 100644 --- a/src/namestore/gnunet-zoneimport.c +++ b/src/namestore/gnunet-zoneimport.c | |||
@@ -32,6 +32,28 @@ | |||
32 | 32 | ||
33 | 33 | ||
34 | /** | 34 | /** |
35 | * Maximum number of queries pending at the same time. | ||
36 | */ | ||
37 | #define THRESH 20 | ||
38 | |||
39 | /** | ||
40 | * TIME_THRESH is in usecs. How quickly do we submit fresh queries. | ||
41 | * Used as an additional throttle. | ||
42 | */ | ||
43 | #define TIME_THRESH 10 | ||
44 | |||
45 | /** | ||
46 | * How often do we retry a query before giving up for good? | ||
47 | */ | ||
48 | #define MAX_RETRIES 5 | ||
49 | |||
50 | /** | ||
51 | * After how many lookups should we always sync to disk? | ||
52 | */ | ||
53 | #define TRANSACTION_SYNC_FREQ 100 | ||
54 | |||
55 | |||
56 | /** | ||
35 | * Some zones may include authoritative records for other | 57 | * Some zones may include authoritative records for other |
36 | * zones, such as foo.com.uk or bar.com.fr. As for GNS | 58 | * zones, such as foo.com.uk or bar.com.fr. As for GNS |
37 | * each dot represents a zone cut, we then need to create a | 59 | * each dot represents a zone cut, we then need to create a |
@@ -241,20 +263,14 @@ static struct Zone *zone_head; | |||
241 | static struct Zone *zone_tail; | 263 | static struct Zone *zone_tail; |
242 | 264 | ||
243 | /** | 265 | /** |
244 | * Maximum number of queries pending at the same time. | 266 | * Set to #GNUNET_YES if we are currently in a DB transaction. |
245 | */ | 267 | */ |
246 | #define THRESH 20 | 268 | static int in_transaction; |
247 | 269 | ||
248 | /** | 270 | /** |
249 | * TIME_THRESH is in usecs. How quickly do we submit fresh queries. | 271 | * Flag set if we should use transactions to batch DB operations. |
250 | * Used as an additional throttle. | ||
251 | */ | 272 | */ |
252 | #define TIME_THRESH 10 | 273 | static int use_transactions; |
253 | |||
254 | /** | ||
255 | * How often do we retry a query before giving up for good? | ||
256 | */ | ||
257 | #define MAX_RETRIES 5 | ||
258 | 274 | ||
259 | 275 | ||
260 | /** | 276 | /** |
@@ -818,6 +834,15 @@ process_result (void *cls, | |||
818 | /* convert linked list into array */ | 834 | /* convert linked list into array */ |
819 | for (rec = req->rec_head; NULL != rec; rec =rec->next) | 835 | for (rec = req->rec_head; NULL != rec; rec =rec->next) |
820 | rd[off++] = rec->grd; | 836 | rd[off++] = rec->grd; |
837 | if ( (! in_transaction) && | ||
838 | (GNUNET_YES == use_transactions) ) | ||
839 | { | ||
840 | /* not all plugins support transactions, but if one does, | ||
841 | remember we need to eventually commit... */ | ||
842 | if (GNUNET_OK == | ||
843 | ns->begin_transaction (ns->cls)) | ||
844 | in_transaction = GNUNET_YES; | ||
845 | } | ||
821 | if (GNUNET_OK != | 846 | if (GNUNET_OK != |
822 | ns->store_records (ns->cls, | 847 | ns->store_records (ns->cls, |
823 | &req->zone->key, | 848 | &req->zone->key, |
@@ -883,6 +908,25 @@ submit_req (struct Request *req) | |||
883 | 908 | ||
884 | 909 | ||
885 | /** | 910 | /** |
911 | * If we are currently in a transaction, commit it. | ||
912 | */ | ||
913 | static void | ||
914 | finish_transaction () | ||
915 | { | ||
916 | if (! in_transaction) | ||
917 | return; | ||
918 | if (GNUNET_OK != | ||
919 | ns->commit_transaction (ns->cls)) | ||
920 | { | ||
921 | GNUNET_break (0); | ||
922 | GNUNET_SCHEDULER_shutdown (); | ||
923 | return; | ||
924 | } | ||
925 | in_transaction = GNUNET_NO; | ||
926 | } | ||
927 | |||
928 | |||
929 | /** | ||
886 | * Process as many requests as possible from the queue. | 930 | * Process as many requests as possible from the queue. |
887 | * | 931 | * |
888 | * @param cls NULL | 932 | * @param cls NULL |
@@ -891,6 +935,7 @@ static void | |||
891 | process_queue(void *cls) | 935 | process_queue(void *cls) |
892 | { | 936 | { |
893 | struct Request *req; | 937 | struct Request *req; |
938 | static unsigned int cnt; | ||
894 | 939 | ||
895 | (void) cls; | 940 | (void) cls; |
896 | t = NULL; | 941 | t = NULL; |
@@ -915,12 +960,15 @@ process_queue(void *cls) | |||
915 | "Waiting until %s for next record (`%s') to expire\n", | 960 | "Waiting until %s for next record (`%s') to expire\n", |
916 | GNUNET_STRINGS_absolute_time_to_string (req->expires), | 961 | GNUNET_STRINGS_absolute_time_to_string (req->expires), |
917 | req->hostname); | 962 | req->hostname); |
963 | finish_transaction (); | ||
918 | t = GNUNET_SCHEDULER_add_at (req->expires, | 964 | t = GNUNET_SCHEDULER_add_at (req->expires, |
919 | &process_queue, | 965 | &process_queue, |
920 | NULL); | 966 | NULL); |
921 | } | 967 | } |
922 | else | 968 | else |
923 | { | 969 | { |
970 | if (0 == cnt++ % TRANSACTION_SYNC_FREQ) | ||
971 | finish_transaction (); | ||
924 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 972 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
925 | "Throttling for 1ms\n"); | 973 | "Throttling for 1ms\n"); |
926 | t = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS, | 974 | t = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS, |
@@ -959,6 +1007,7 @@ do_shutdown (void *cls) | |||
959 | GNUNET_SCHEDULER_cancel (t); | 1007 | GNUNET_SCHEDULER_cancel (t); |
960 | t = NULL; | 1008 | t = NULL; |
961 | } | 1009 | } |
1010 | finish_transaction (); | ||
962 | if (NULL != ns) | 1011 | if (NULL != ns) |
963 | { | 1012 | { |
964 | GNUNET_break (NULL == | 1013 | GNUNET_break (NULL == |
@@ -1329,6 +1378,10 @@ main (int argc, | |||
1329 | "IP", | 1378 | "IP", |
1330 | "which DNS server should be used", | 1379 | "which DNS server should be used", |
1331 | &dns_server)), | 1380 | &dns_server)), |
1381 | GNUNET_GETOPT_option_flag ('e', | ||
1382 | "enable-transactions", | ||
1383 | "enable use of transactions to reduce disk IO", | ||
1384 | &use_transactions), | ||
1332 | GNUNET_GETOPT_OPTION_END | 1385 | GNUNET_GETOPT_OPTION_END |
1333 | }; | 1386 | }; |
1334 | 1387 | ||