diff options
author | Christian Grothoff <christian@grothoff.org> | 2019-05-03 16:18:59 +0200 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2019-05-03 16:18:59 +0200 |
commit | 782565417337605572b66758bf13d7123e26f744 (patch) | |
tree | aaf2094c7bdea5fa809f03573cb4292f1895160d /src/json | |
parent | 4444fb80284aa86fe24f3640a0b1e4c841a98f9a (diff) | |
download | gnunet-782565417337605572b66758bf13d7123e26f744.tar.gz gnunet-782565417337605572b66758bf13d7123e26f744.zip |
support compressed JSON uploads
Diffstat (limited to 'src/json')
-rw-r--r-- | src/json/.gitignore | 1 | ||||
-rw-r--r-- | src/json/Makefile.am | 5 | ||||
-rw-r--r-- | src/json/json_mhd.c | 122 | ||||
-rw-r--r-- | src/json/test_json_mhd.c | 47 |
4 files changed, 169 insertions, 6 deletions
diff --git a/src/json/.gitignore b/src/json/.gitignore index 6709c749d..347bffd7b 100644 --- a/src/json/.gitignore +++ b/src/json/.gitignore | |||
@@ -1 +1,2 @@ | |||
1 | test_json | 1 | test_json |
2 | test_json_mhd | ||
diff --git a/src/json/Makefile.am b/src/json/Makefile.am index f030c3016..dfe185d5e 100644 --- a/src/json/Makefile.am +++ b/src/json/Makefile.am | |||
@@ -22,7 +22,9 @@ libgnunetjson_la_LIBADD = \ | |||
22 | $(top_builddir)/src/util/libgnunetutil.la \ | 22 | $(top_builddir)/src/util/libgnunetutil.la \ |
23 | $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ | 23 | $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ |
24 | -ljansson \ | 24 | -ljansson \ |
25 | $(XLIB) | 25 | -lmicrohttpd \ |
26 | $(XLIB) \ | ||
27 | $(Z_LIBS) | ||
26 | 28 | ||
27 | check_PROGRAMS = \ | 29 | check_PROGRAMS = \ |
28 | test_json \ | 30 | test_json \ |
@@ -57,6 +59,7 @@ test_json_mhd_LDADD = \ | |||
57 | $(top_builddir)/src/util/libgnunetutil.la \ | 59 | $(top_builddir)/src/util/libgnunetutil.la \ |
58 | -ljansson \ | 60 | -ljansson \ |
59 | -lmicrohttpd \ | 61 | -lmicrohttpd \ |
62 | $(Z_LIBS) \ | ||
60 | $(LIB_GNURL) | 63 | $(LIB_GNURL) |
61 | test_json_mhd_CPPFLAGS = \ | 64 | test_json_mhd_CPPFLAGS = \ |
62 | $(CPP_GNURL) $(AM_CPPFLAGS) | 65 | $(CPP_GNURL) $(AM_CPPFLAGS) |
diff --git a/src/json/json_mhd.c b/src/json/json_mhd.c index 30b29b88e..b6ab2d116 100644 --- a/src/json/json_mhd.c +++ b/src/json/json_mhd.c | |||
@@ -26,6 +26,7 @@ | |||
26 | */ | 26 | */ |
27 | #include "platform.h" | 27 | #include "platform.h" |
28 | #include "gnunet_json_lib.h" | 28 | #include "gnunet_json_lib.h" |
29 | #include <zlib.h> | ||
29 | 30 | ||
30 | 31 | ||
31 | /** | 32 | /** |
@@ -55,6 +56,11 @@ struct Buffer | |||
55 | * Number of allocated bytes in buffer. | 56 | * Number of allocated bytes in buffer. |
56 | */ | 57 | */ |
57 | size_t alloc; | 58 | size_t alloc; |
59 | |||
60 | /** | ||
61 | * Maximum buffer size allowed. | ||
62 | */ | ||
63 | size_t max; | ||
58 | }; | 64 | }; |
59 | 65 | ||
60 | 66 | ||
@@ -80,7 +86,10 @@ buffer_init (struct Buffer *buf, | |||
80 | if (data_size > alloc_size) | 86 | if (data_size > alloc_size) |
81 | alloc_size = data_size; | 87 | alloc_size = data_size; |
82 | buf->data = GNUNET_malloc (alloc_size); | 88 | buf->data = GNUNET_malloc (alloc_size); |
89 | buf->alloc = alloc_size; | ||
83 | GNUNET_memcpy (buf->data, data, data_size); | 90 | GNUNET_memcpy (buf->data, data, data_size); |
91 | buf->fill = data_size; | ||
92 | buf->max = max_size; | ||
84 | return GNUNET_OK; | 93 | return GNUNET_OK; |
85 | } | 94 | } |
86 | 95 | ||
@@ -138,6 +147,99 @@ buffer_append (struct Buffer *buf, | |||
138 | 147 | ||
139 | 148 | ||
140 | /** | 149 | /** |
150 | * Decompress data in @a buf. | ||
151 | * | ||
152 | * @param buf input data to inflate | ||
153 | * @return result code indicating the status of the operation | ||
154 | */ | ||
155 | static enum GNUNET_JSON_PostResult | ||
156 | inflate_data (struct Buffer *buf) | ||
157 | { | ||
158 | z_stream z; | ||
159 | char *tmp; | ||
160 | size_t tmp_size; | ||
161 | int ret; | ||
162 | |||
163 | memset (&z, 0, sizeof (z)); | ||
164 | z.next_in = (Bytef *) buf->data; | ||
165 | z.avail_in = buf->fill; | ||
166 | tmp_size = GNUNET_MIN (buf->max, buf->fill * 4); | ||
167 | tmp = GNUNET_malloc (tmp_size); | ||
168 | z.next_out = (Bytef *) tmp; | ||
169 | z.avail_out = tmp_size; | ||
170 | ret = inflateInit (&z); | ||
171 | switch (ret) | ||
172 | { | ||
173 | case Z_MEM_ERROR: | ||
174 | GNUNET_break (0); | ||
175 | return GNUNET_JSON_PR_OUT_OF_MEMORY; | ||
176 | case Z_STREAM_ERROR: | ||
177 | GNUNET_break_op (0); | ||
178 | return GNUNET_JSON_PR_JSON_INVALID; | ||
179 | case Z_OK: | ||
180 | break; | ||
181 | } | ||
182 | while (1) | ||
183 | { | ||
184 | ret = inflate (&z, 0); | ||
185 | switch (ret) | ||
186 | { | ||
187 | case Z_MEM_ERROR: | ||
188 | GNUNET_break (0); | ||
189 | GNUNET_break (Z_OK == inflateEnd (&z)); | ||
190 | GNUNET_free (tmp); | ||
191 | return GNUNET_JSON_PR_OUT_OF_MEMORY; | ||
192 | case Z_DATA_ERROR: | ||
193 | GNUNET_break (0); | ||
194 | GNUNET_break (Z_OK == inflateEnd (&z)); | ||
195 | GNUNET_free (tmp); | ||
196 | return GNUNET_JSON_PR_JSON_INVALID; | ||
197 | case Z_NEED_DICT: | ||
198 | GNUNET_break (0); | ||
199 | GNUNET_break (Z_OK == inflateEnd (&z)); | ||
200 | GNUNET_free (tmp); | ||
201 | return GNUNET_JSON_PR_JSON_INVALID; | ||
202 | case Z_OK: | ||
203 | if ((0 < z.avail_out) && (0 == z.avail_in)) | ||
204 | { | ||
205 | /* truncated input stream */ | ||
206 | GNUNET_break (0); | ||
207 | GNUNET_break (Z_OK == inflateEnd (&z)); | ||
208 | GNUNET_free (tmp); | ||
209 | return GNUNET_JSON_PR_JSON_INVALID; | ||
210 | } | ||
211 | if (0 < z.avail_out) | ||
212 | continue; /* just call it again */ | ||
213 | /* output buffer full, can we grow it? */ | ||
214 | if (tmp_size == buf->max) | ||
215 | { | ||
216 | /* already at max */ | ||
217 | GNUNET_break (0); | ||
218 | GNUNET_break (Z_OK == inflateEnd (&z)); | ||
219 | GNUNET_free (tmp); | ||
220 | return GNUNET_JSON_PR_OUT_OF_MEMORY; | ||
221 | } | ||
222 | if (tmp_size * 2 < tmp_size) | ||
223 | tmp_size = buf->max; | ||
224 | else | ||
225 | tmp_size = GNUNET_MIN (buf->max, tmp_size * 2); | ||
226 | tmp = GNUNET_realloc (tmp, tmp_size); | ||
227 | z.next_out = (Bytef *) &tmp[z.total_out]; | ||
228 | continue; | ||
229 | case Z_STREAM_END: | ||
230 | /* decompression successful, make 'tmp' the new 'data' */ | ||
231 | GNUNET_free (buf->data); | ||
232 | buf->data = tmp; | ||
233 | buf->alloc = tmp_size; | ||
234 | buf->fill = z.total_out; | ||
235 | GNUNET_break (Z_OK == inflateEnd (&z)); | ||
236 | return GNUNET_JSON_PR_SUCCESS; /* at least for now */ | ||
237 | } | ||
238 | } /* while (1) */ | ||
239 | } | ||
240 | |||
241 | |||
242 | /** | ||
141 | * Process a POST request containing a JSON object. This function | 243 | * Process a POST request containing a JSON object. This function |
142 | * realizes an MHD POST processor that will (incrementally) process | 244 | * realizes an MHD POST processor that will (incrementally) process |
143 | * JSON data uploaded to the HTTP server. It will store the required | 245 | * JSON data uploaded to the HTTP server. It will store the required |
@@ -161,10 +263,13 @@ GNUNET_JSON_post_parser (size_t buffer_max, | |||
161 | json_t **json) | 263 | json_t **json) |
162 | { | 264 | { |
163 | struct Buffer *r = *con_cls; | 265 | struct Buffer *r = *con_cls; |
266 | const char *ce; | ||
267 | int ret; | ||
164 | 268 | ||
165 | *json = NULL; | 269 | *json = NULL; |
166 | if (NULL == *con_cls) | 270 | if (NULL == *con_cls) |
167 | { | 271 | { |
272 | |||
168 | /* We are seeing a fresh POST request. */ | 273 | /* We are seeing a fresh POST request. */ |
169 | r = GNUNET_new (struct Buffer); | 274 | r = GNUNET_new (struct Buffer); |
170 | if (GNUNET_OK != buffer_init (r, | 275 | if (GNUNET_OK != buffer_init (r, |
@@ -202,12 +307,29 @@ GNUNET_JSON_post_parser (size_t buffer_max, | |||
202 | } | 307 | } |
203 | 308 | ||
204 | /* We have seen the whole request. */ | 309 | /* We have seen the whole request. */ |
310 | ce = MHD_lookup_connection_value (connection, | ||
311 | MHD_HEADER_KIND, | ||
312 | MHD_HTTP_HEADER_CONTENT_ENCODING); | ||
313 | if ((NULL != ce) && (0 == strcasecmp ("deflate", ce))) | ||
314 | { | ||
315 | ret = inflate_data (r); | ||
316 | if (GNUNET_JSON_PR_SUCCESS != ret) | ||
317 | { | ||
318 | buffer_deinit (r); | ||
319 | GNUNET_free (r); | ||
320 | *con_cls = NULL; | ||
321 | return ret; | ||
322 | } | ||
323 | } | ||
205 | 324 | ||
206 | *json = json_loadb (r->data, r->fill, 0, NULL); | 325 | *json = json_loadb (r->data, r->fill, 0, NULL); |
207 | if (NULL == *json) | 326 | if (NULL == *json) |
208 | { | 327 | { |
209 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | 328 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, |
210 | "Failed to parse JSON request body\n"); | 329 | "Failed to parse JSON request body\n"); |
330 | buffer_deinit (r); | ||
331 | GNUNET_free (r); | ||
332 | *con_cls = NULL; | ||
211 | return GNUNET_JSON_PR_JSON_INVALID; | 333 | return GNUNET_JSON_PR_JSON_INVALID; |
212 | } | 334 | } |
213 | buffer_deinit (r); | 335 | buffer_deinit (r); |
diff --git a/src/json/test_json_mhd.c b/src/json/test_json_mhd.c index 665fd140e..13338b32c 100644 --- a/src/json/test_json_mhd.c +++ b/src/json/test_json_mhd.c | |||
@@ -27,6 +27,7 @@ | |||
27 | #include "gnunet_util_lib.h" | 27 | #include "gnunet_util_lib.h" |
28 | #include "gnunet_json_lib.h" | 28 | #include "gnunet_json_lib.h" |
29 | #include "gnunet_curl_lib.h" | 29 | #include "gnunet_curl_lib.h" |
30 | #include <zlib.h> | ||
30 | 31 | ||
31 | #define MAX_SIZE 1024 * 1024 | 32 | #define MAX_SIZE 1024 * 1024 |
32 | 33 | ||
@@ -69,7 +70,7 @@ access_handler_cb (void *cls, | |||
69 | global_ret = 6; | 70 | global_ret = 6; |
70 | } | 71 | } |
71 | json_decref (json); | 72 | json_decref (json); |
72 | resp = MHD_create_response_from_buffer (2, "OK", MHD_RESPMEM_PERSISTENT); | 73 | resp = MHD_create_response_from_buffer (3, "OK\n", MHD_RESPMEM_PERSISTENT); |
73 | ret = MHD_queue_response (connection, MHD_HTTP_OK, resp); | 74 | ret = MHD_queue_response (connection, MHD_HTTP_OK, resp); |
74 | MHD_destroy_response (resp); | 75 | MHD_destroy_response (resp); |
75 | return ret; | 76 | return ret; |
@@ -100,8 +101,12 @@ main (int argc, const char *const argv[]) | |||
100 | uint16_t port; | 101 | uint16_t port; |
101 | CURL *easy; | 102 | CURL *easy; |
102 | char *url; | 103 | char *url; |
104 | char *str; | ||
105 | size_t slen; | ||
103 | long post_data_size; | 106 | long post_data_size; |
104 | void *post_data; | 107 | void *post_data; |
108 | uLongf dlen; | ||
109 | struct curl_slist *json_header; | ||
105 | 110 | ||
106 | GNUNET_log_setup ("test-json-mhd", "WARNING", NULL); | 111 | GNUNET_log_setup ("test-json-mhd", "WARNING", NULL); |
107 | global_ret = 2; | 112 | global_ret = 2; |
@@ -123,28 +128,60 @@ main (int argc, const char *const argv[]) | |||
123 | GNUNET_snprintf (tmp, sizeof (tmp), "%u", i); | 128 | GNUNET_snprintf (tmp, sizeof (tmp), "%u", i); |
124 | json_object_set_new (bigj, tmp, json_string (tmp)); | 129 | json_object_set_new (bigj, tmp, json_string (tmp)); |
125 | } | 130 | } |
126 | post_data = json_dumps (bigj, JSON_INDENT (2)); | 131 | str = json_dumps (bigj, JSON_INDENT (2)); |
127 | post_data_size = strlen (post_data); | 132 | slen = strlen (str); |
128 | 133 | ||
134 | #ifdef compressBound | ||
135 | dlen = compressBound (slen); | ||
136 | #else | ||
137 | dlen = slen + slen / 100 + 20; | ||
138 | /* documentation says 100.1% oldSize + 12 bytes, but we | ||
139 | * should be able to overshoot by more to be safe */ | ||
140 | #endif | ||
141 | post_data = GNUNET_malloc (dlen); | ||
142 | if (Z_OK != | ||
143 | compress2 ((Bytef *) post_data, &dlen, (const Bytef *) str, slen, 9)) | ||
144 | { | ||
145 | GNUNET_break (0); | ||
146 | MHD_stop_daemon (daemon); | ||
147 | GNUNET_free (url); | ||
148 | json_decref (bigj); | ||
149 | GNUNET_free (post_data); | ||
150 | GNUNET_free (str); | ||
151 | return 1; | ||
152 | } | ||
153 | post_data_size = (long) dlen; | ||
129 | port = MHD_get_daemon_info (daemon, MHD_DAEMON_INFO_BIND_PORT)->port; | 154 | port = MHD_get_daemon_info (daemon, MHD_DAEMON_INFO_BIND_PORT)->port; |
130 | easy = curl_easy_init (); | 155 | easy = curl_easy_init (); |
131 | GNUNET_asprintf (&url, "http://localhost:%u/", (unsigned int) port); | 156 | GNUNET_asprintf (&url, "http://localhost:%u/", (unsigned int) port); |
132 | curl_easy_setopt (easy, CURLOPT_VERBOSE, 1); | 157 | curl_easy_setopt (easy, CURLOPT_VERBOSE, 0); |
133 | curl_easy_setopt (easy, CURLOPT_URL, url); | 158 | curl_easy_setopt (easy, CURLOPT_URL, url); |
134 | curl_easy_setopt (easy, CURLOPT_POST, 1); | 159 | curl_easy_setopt (easy, CURLOPT_POST, 1); |
135 | curl_easy_setopt (easy, CURLOPT_POSTFIELDS, post_data); | 160 | curl_easy_setopt (easy, CURLOPT_POSTFIELDS, post_data); |
136 | curl_easy_setopt (easy, CURLOPT_POSTFIELDSIZE, post_data_size); | 161 | curl_easy_setopt (easy, CURLOPT_POSTFIELDSIZE, post_data_size); |
162 | |||
163 | json_header = curl_slist_append (NULL, "Content-Type: application/json"); | ||
164 | json_header = curl_slist_append (json_header, "Content-Encoding: deflate"); | ||
165 | curl_easy_setopt (easy, CURLOPT_HTTPHEADER, json_header); | ||
137 | if (0 != curl_easy_perform (easy)) | 166 | if (0 != curl_easy_perform (easy)) |
138 | { | 167 | { |
139 | GNUNET_break (0); | 168 | GNUNET_break (0); |
140 | MHD_stop_daemon (daemon); | 169 | MHD_stop_daemon (daemon); |
141 | GNUNET_free (url); | 170 | GNUNET_free (url); |
142 | json_decref (bigj); | 171 | json_decref (bigj); |
172 | GNUNET_free (post_data); | ||
173 | GNUNET_free (str); | ||
174 | curl_slist_free_all (json_header); | ||
175 | curl_easy_cleanup (easy); | ||
143 | return 1; | 176 | return 1; |
144 | } | 177 | } |
145 | MHD_stop_daemon (daemon); | 178 | MHD_stop_daemon (daemon); |
146 | GNUNET_free (url); | 179 | GNUNET_free (url); |
147 | json_decref (bigj); | 180 | json_decref (bigj); |
181 | GNUNET_free (post_data); | ||
182 | GNUNET_free (str); | ||
183 | curl_slist_free_all (json_header); | ||
184 | curl_easy_cleanup (easy); | ||
148 | return global_ret; | 185 | return global_ret; |
149 | } | 186 | } |
150 | 187 | ||