diff options
author | Christian Grothoff <christian@grothoff.org> | 2018-04-26 16:40:09 +0200 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2018-04-26 16:40:09 +0200 |
commit | 823215c974ccb1ef3cad9eb1082999cd1b910416 (patch) | |
tree | 7984816482be2a6a50786796d7c24d297a707593 /src/namestore | |
parent | 1b21e0b643f37675683b09b9e528721cb7df74e5 (diff) | |
download | gnunet-823215c974ccb1ef3cad9eb1082999cd1b910416.tar.gz gnunet-823215c974ccb1ef3cad9eb1082999cd1b910416.zip |
misc zoneimporter fixes
Diffstat (limited to 'src/namestore')
-rw-r--r-- | src/namestore/gnunet-service-namestore.c | 14 | ||||
-rw-r--r-- | src/namestore/gnunet-zoneimport.c | 172 |
2 files changed, 102 insertions, 84 deletions
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c index 994eae2c8..06a50132b 100644 --- a/src/namestore/gnunet-service-namestore.c +++ b/src/namestore/gnunet-service-namestore.c | |||
@@ -708,11 +708,15 @@ refresh_block (struct NamestoreClient *nc, | |||
708 | &res); | 708 | &res); |
709 | GNUNET_free (nick); | 709 | GNUNET_free (nick); |
710 | } | 710 | } |
711 | 711 | if (0 == res_count) | |
712 | exp_time = (0 == res_count) | 712 | { |
713 | ? GNUNET_TIME_UNIT_ZERO_ABS | 713 | send_store_response (nc, |
714 | : GNUNET_GNSRECORD_record_get_expiration_time (res_count, | 714 | GNUNET_OK, |
715 | res); | 715 | rid); |
716 | return; /* no data, no need to update cache */ | ||
717 | } | ||
718 | exp_time = GNUNET_GNSRECORD_record_get_expiration_time (res_count, | ||
719 | res); | ||
716 | if (cache_keys) | 720 | if (cache_keys) |
717 | block = GNUNET_GNSRECORD_block_create2 (zone_key, | 721 | block = GNUNET_GNSRECORD_block_create2 (zone_key, |
718 | exp_time, | 722 | exp_time, |
diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c index 6622a7676..503262487 100644 --- a/src/namestore/gnunet-zoneimport.c +++ b/src/namestore/gnunet-zoneimport.c | |||
@@ -48,6 +48,16 @@ | |||
48 | #define MAX_RETRIES 5 | 48 | #define MAX_RETRIES 5 |
49 | 49 | ||
50 | /** | 50 | /** |
51 | * How many DNS requests do we at most issue in rapid series? | ||
52 | */ | ||
53 | #define MAX_SERIES 10 | ||
54 | |||
55 | /** | ||
56 | * How long do we wait at least between series of requests? | ||
57 | */ | ||
58 | #define SERIES_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10) | ||
59 | |||
60 | /** | ||
51 | * How many requests do we request from NAMESTORE in one batch | 61 | * How many requests do we request from NAMESTORE in one batch |
52 | * during our initial iteration? | 62 | * during our initial iteration? |
53 | */ | 63 | */ |
@@ -200,11 +210,16 @@ static struct GNUNET_NAMESTORE_Handle *ns; | |||
200 | static struct GNUNET_DNSSTUB_Context *ctx; | 210 | static struct GNUNET_DNSSTUB_Context *ctx; |
201 | 211 | ||
202 | /** | 212 | /** |
203 | * The number of queries that are outstanding | 213 | * The number of DNS queries that are outstanding |
204 | */ | 214 | */ |
205 | static unsigned int pending; | 215 | static unsigned int pending; |
206 | 216 | ||
207 | /** | 217 | /** |
218 | * The number of NAMESTORE record store operations that are outstanding | ||
219 | */ | ||
220 | static unsigned int pending_rs; | ||
221 | |||
222 | /** | ||
208 | * Number of lookups we performed overall. | 223 | * Number of lookups we performed overall. |
209 | */ | 224 | */ |
210 | static unsigned int lookups; | 225 | static unsigned int lookups; |
@@ -692,7 +707,7 @@ process_record (void *cls, | |||
692 | req->hostname)) | 707 | req->hostname)) |
693 | { | 708 | { |
694 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 709 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
695 | "DNS returned record for `%s' of type %u while resolving `%s'\n", | 710 | "DNS returned record from zone `%s' of type %u while resolving `%s'\n", |
696 | rec->name, | 711 | rec->name, |
697 | (unsigned int) rec->type, | 712 | (unsigned int) rec->type, |
698 | req->hostname); | 713 | req->hostname); |
@@ -910,7 +925,11 @@ store_completed_cb (void *cls, | |||
910 | struct Request *req = cls; | 925 | struct Request *req = cls; |
911 | 926 | ||
912 | req->qe = NULL; | 927 | req->qe = NULL; |
913 | pending--; | 928 | pending_rs--; |
929 | if (NULL == t) | ||
930 | t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, | ||
931 | &process_queue, | ||
932 | NULL); | ||
914 | if (GNUNET_SYSERR == success) | 933 | if (GNUNET_SYSERR == success) |
915 | { | 934 | { |
916 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | 935 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, |
@@ -967,6 +986,10 @@ process_result (void *cls, | |||
967 | req_tail, | 986 | req_tail, |
968 | req); | 987 | req); |
969 | pending--; | 988 | pending--; |
989 | if (NULL == t) | ||
990 | t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, | ||
991 | &process_queue, | ||
992 | NULL); | ||
970 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | 993 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, |
971 | "Stub gave up on DNS reply for `%s'\n", | 994 | "Stub gave up on DNS reply for `%s'\n", |
972 | req->hostname); | 995 | req->hostname); |
@@ -981,12 +1004,17 @@ process_result (void *cls, | |||
981 | return; | 1004 | return; |
982 | } | 1005 | } |
983 | if (req->id != dns->id) | 1006 | if (req->id != dns->id) |
1007 | { | ||
1008 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1009 | "DNS ID did not match request, ignoring reply\n"); | ||
984 | return; | 1010 | return; |
1011 | } | ||
985 | GNUNET_CONTAINER_DLL_remove (req_head, | 1012 | GNUNET_CONTAINER_DLL_remove (req_head, |
986 | req_tail, | 1013 | req_tail, |
987 | req); | 1014 | req); |
988 | GNUNET_DNSSTUB_resolve_cancel (req->rs); | 1015 | GNUNET_DNSSTUB_resolve_cancel (req->rs); |
989 | req->rs = NULL; | 1016 | req->rs = NULL; |
1017 | pending--; | ||
990 | p = GNUNET_DNSPARSER_parse ((const char *) dns, | 1018 | p = GNUNET_DNSPARSER_parse ((const char *) dns, |
991 | dns_len); | 1019 | dns_len); |
992 | if (NULL == p) | 1020 | if (NULL == p) |
@@ -998,11 +1026,17 @@ process_result (void *cls, | |||
998 | { | 1026 | { |
999 | failures++; | 1027 | failures++; |
1000 | insert_sorted (req); | 1028 | insert_sorted (req); |
1001 | pending--; | 1029 | if (NULL == t) |
1030 | t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, | ||
1031 | &process_queue, | ||
1032 | NULL); | ||
1002 | return; | 1033 | return; |
1003 | } | 1034 | } |
1004 | insert_sorted (req); | 1035 | insert_sorted (req); |
1005 | pending--; | 1036 | if (NULL == t) |
1037 | t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, | ||
1038 | &process_queue, | ||
1039 | NULL); | ||
1006 | return; | 1040 | return; |
1007 | } | 1041 | } |
1008 | /* import new records */ | 1042 | /* import new records */ |
@@ -1047,6 +1081,7 @@ process_result (void *cls, | |||
1047 | /* convert linked list into array */ | 1081 | /* convert linked list into array */ |
1048 | for (rec = req->rec_head; NULL != rec; rec =rec->next) | 1082 | for (rec = req->rec_head; NULL != rec; rec =rec->next) |
1049 | rd[off++] = rec->grd; | 1083 | rd[off++] = rec->grd; |
1084 | pending_rs++; | ||
1050 | req->qe = GNUNET_NAMESTORE_records_store (ns, | 1085 | req->qe = GNUNET_NAMESTORE_records_store (ns, |
1051 | &req->zone->key, | 1086 | &req->zone->key, |
1052 | get_label (req), | 1087 | get_label (req), |
@@ -1054,69 +1089,13 @@ process_result (void *cls, | |||
1054 | rd, | 1089 | rd, |
1055 | &store_completed_cb, | 1090 | &store_completed_cb, |
1056 | req); | 1091 | req); |
1092 | GNUNET_assert (NULL != req->qe); | ||
1057 | } | 1093 | } |
1058 | insert_sorted (req); | 1094 | insert_sorted (req); |
1059 | } | 1095 | } |
1060 | 1096 | ||
1061 | 1097 | ||
1062 | /** | 1098 | /** |
1063 | * Submit a request to DNS unless we need to slow down because | ||
1064 | * we are at the rate limit. | ||
1065 | * | ||
1066 | * @param req request to submit | ||
1067 | * @return #GNUNET_OK if request was submitted | ||
1068 | * #GNUNET_NO if request was already submitted | ||
1069 | * #GNUNET_SYSERR if we are at the rate limit | ||
1070 | */ | ||
1071 | static int | ||
1072 | submit_req (struct Request *req) | ||
1073 | { | ||
1074 | static struct GNUNET_TIME_Absolute last_request; | ||
1075 | struct GNUNET_TIME_Absolute now; | ||
1076 | void *raw; | ||
1077 | size_t raw_size; | ||
1078 | |||
1079 | if (NULL != req->qe) | ||
1080 | return GNUNET_NO; /* namestore op still pending */ | ||
1081 | if (NULL != req->rs) | ||
1082 | { | ||
1083 | GNUNET_break (0); | ||
1084 | return GNUNET_NO; /* already submitted */ | ||
1085 | } | ||
1086 | now = GNUNET_TIME_absolute_get (); | ||
1087 | if ( (now.abs_value_us - last_request.abs_value_us < TIME_THRESH) || | ||
1088 | (pending >= THRESH) ) | ||
1089 | return GNUNET_SYSERR; | ||
1090 | GNUNET_CONTAINER_DLL_insert (req_head, | ||
1091 | req_tail, | ||
1092 | req); | ||
1093 | GNUNET_assert (NULL == req->rs); | ||
1094 | GNUNET_log (GNUNET_ERROR_TYPE_INFO, | ||
1095 | "Requesting resolution for `%s'\n", | ||
1096 | req->hostname); | ||
1097 | raw = build_dns_query (req, | ||
1098 | &raw_size); | ||
1099 | if (NULL == raw) | ||
1100 | { | ||
1101 | GNUNET_break (0); | ||
1102 | free_request (req); | ||
1103 | return GNUNET_SYSERR; | ||
1104 | } | ||
1105 | req->rs = GNUNET_DNSSTUB_resolve (ctx, | ||
1106 | raw, | ||
1107 | raw_size, | ||
1108 | &process_result, | ||
1109 | req); | ||
1110 | GNUNET_assert (NULL != req->rs); | ||
1111 | req->issue_num++; | ||
1112 | last_request = now; | ||
1113 | lookups++; | ||
1114 | pending++; | ||
1115 | return GNUNET_OK; | ||
1116 | } | ||
1117 | |||
1118 | |||
1119 | /** | ||
1120 | * Process as many requests as possible from the queue. | 1099 | * Process as many requests as possible from the queue. |
1121 | * | 1100 | * |
1122 | * @param cls NULL | 1101 | * @param cls NULL |
@@ -1125,23 +1104,60 @@ static void | |||
1125 | process_queue (void *cls) | 1104 | process_queue (void *cls) |
1126 | { | 1105 | { |
1127 | struct Request *req; | 1106 | struct Request *req; |
1107 | unsigned int series; | ||
1108 | void *raw; | ||
1109 | size_t raw_size; | ||
1128 | 1110 | ||
1129 | (void) cls; | 1111 | (void) cls; |
1112 | series = 0; | ||
1130 | t = NULL; | 1113 | t = NULL; |
1131 | while (1) | 1114 | while (pending + pending_rs < THRESH) |
1132 | { | 1115 | { |
1133 | req = GNUNET_CONTAINER_heap_peek (req_heap); | 1116 | req = GNUNET_CONTAINER_heap_peek (req_heap); |
1134 | if (NULL == req) | 1117 | if (NULL == req) |
1135 | break; | 1118 | break; |
1119 | if (NULL != req->qe) | ||
1120 | return; /* namestore op still pending */ | ||
1121 | if (NULL != req->rs) | ||
1122 | { | ||
1123 | GNUNET_break (0); | ||
1124 | return; /* already submitted */ | ||
1125 | } | ||
1136 | if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0) | 1126 | if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0) |
1137 | break; | 1127 | break; |
1138 | if (GNUNET_OK != submit_req (req)) | ||
1139 | break; | ||
1140 | GNUNET_assert (req == | 1128 | GNUNET_assert (req == |
1141 | GNUNET_CONTAINER_heap_remove_root (req_heap)); | 1129 | GNUNET_CONTAINER_heap_remove_root (req_heap)); |
1142 | req->hn = NULL; | 1130 | req->hn = NULL; |
1131 | GNUNET_CONTAINER_DLL_insert (req_head, | ||
1132 | req_tail, | ||
1133 | req); | ||
1134 | GNUNET_assert (NULL == req->rs); | ||
1135 | GNUNET_log (GNUNET_ERROR_TYPE_INFO, | ||
1136 | "Requesting resolution for `%s'\n", | ||
1137 | req->hostname); | ||
1138 | raw = build_dns_query (req, | ||
1139 | &raw_size); | ||
1140 | if (NULL == raw) | ||
1141 | { | ||
1142 | GNUNET_break (0); | ||
1143 | free_request (req); | ||
1144 | continue; | ||
1145 | } | ||
1146 | req->rs = GNUNET_DNSSTUB_resolve (ctx, | ||
1147 | raw, | ||
1148 | raw_size, | ||
1149 | &process_result, | ||
1150 | req); | ||
1151 | GNUNET_assert (NULL != req->rs); | ||
1152 | req->issue_num++; | ||
1153 | lookups++; | ||
1154 | pending++; | ||
1155 | series++; | ||
1156 | if (series > MAX_SERIES) | ||
1157 | break; | ||
1143 | } | 1158 | } |
1144 | 1159 | if (pending + pending_rs >= THRESH) | |
1160 | return; /* wait for replies */ | ||
1145 | req = GNUNET_CONTAINER_heap_peek (req_heap); | 1161 | req = GNUNET_CONTAINER_heap_peek (req_heap); |
1146 | if (NULL == req) | 1162 | if (NULL == req) |
1147 | return; | 1163 | return; |
@@ -1156,17 +1172,15 @@ process_queue (void *cls) | |||
1156 | t = GNUNET_SCHEDULER_add_at (req->expires, | 1172 | t = GNUNET_SCHEDULER_add_at (req->expires, |
1157 | &process_queue, | 1173 | &process_queue, |
1158 | NULL); | 1174 | NULL); |
1175 | return; | ||
1159 | } | 1176 | } |
1160 | else | 1177 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1161 | { | 1178 | "Throttling\n"); |
1162 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1179 | if (NULL != t) |
1163 | "Throttling for 1ms\n"); | 1180 | GNUNET_SCHEDULER_cancel (t); |
1164 | if (NULL != t) | 1181 | t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, |
1165 | GNUNET_SCHEDULER_cancel (t); | 1182 | &process_queue, |
1166 | t = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS, | 1183 | NULL); |
1167 | &process_queue, | ||
1168 | NULL); | ||
1169 | } | ||
1170 | } | 1184 | } |
1171 | 1185 | ||
1172 | 1186 | ||
@@ -1512,7 +1526,6 @@ iterate_zones (void *cls) | |||
1512 | GNUNET_assert (NULL != zone_tail); | 1526 | GNUNET_assert (NULL != zone_tail); |
1513 | if (zone_tail == last) | 1527 | if (zone_tail == last) |
1514 | { | 1528 | { |
1515 | GNUNET_assert (NULL == t); | ||
1516 | /* Done iterating over relevant zones in NAMESTORE, move | 1529 | /* Done iterating over relevant zones in NAMESTORE, move |
1517 | rest of hash map to work queue as well. */ | 1530 | rest of hash map to work queue as well. */ |
1518 | GNUNET_CONTAINER_multihashmap_iterate (ns_pending, | 1531 | GNUNET_CONTAINER_multihashmap_iterate (ns_pending, |
@@ -1750,12 +1763,13 @@ main (int argc, | |||
1750 | NULL); | 1763 | NULL); |
1751 | GNUNET_free ((void*) argv); | 1764 | GNUNET_free ((void*) argv); |
1752 | fprintf (stderr, | 1765 | fprintf (stderr, |
1753 | "Rejected %u names, did %u lookups, found %u records, %u lookups failed, %u pending on shutdown\n", | 1766 | "Rejected %u names, did %u lookups, found %u records, %u lookups failed, %u/%u pending on shutdown\n", |
1754 | rejects, | 1767 | rejects, |
1755 | lookups, | 1768 | lookups, |
1756 | records, | 1769 | records, |
1757 | failures, | 1770 | failures, |
1758 | pending); | 1771 | pending, |
1772 | pending_rs); | ||
1759 | return 0; | 1773 | return 0; |
1760 | } | 1774 | } |
1761 | 1775 | ||