aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristian Grothoff <christian@grothoff.org>2019-05-03 16:18:59 +0200
committerChristian Grothoff <christian@grothoff.org>2019-05-03 16:18:59 +0200
commit782565417337605572b66758bf13d7123e26f744 (patch)
treeaaf2094c7bdea5fa809f03573cb4292f1895160d
parent4444fb80284aa86fe24f3640a0b1e4c841a98f9a (diff)
downloadgnunet-782565417337605572b66758bf13d7123e26f744.tar.gz
gnunet-782565417337605572b66758bf13d7123e26f744.zip
support compressed JSON uploads
-rw-r--r--src/json/.gitignore1
-rw-r--r--src/json/Makefile.am5
-rw-r--r--src/json/json_mhd.c122
-rw-r--r--src/json/test_json_mhd.c47
4 files changed, 169 insertions, 6 deletions
diff --git a/src/json/.gitignore b/src/json/.gitignore
index 6709c749d..347bffd7b 100644
--- a/src/json/.gitignore
+++ b/src/json/.gitignore
@@ -1 +1,2 @@
1test_json 1test_json
2test_json_mhd
diff --git a/src/json/Makefile.am b/src/json/Makefile.am
index f030c3016..dfe185d5e 100644
--- a/src/json/Makefile.am
+++ b/src/json/Makefile.am
@@ -22,7 +22,9 @@ libgnunetjson_la_LIBADD = \
22 $(top_builddir)/src/util/libgnunetutil.la \ 22 $(top_builddir)/src/util/libgnunetutil.la \
23 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 23 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
24 -ljansson \ 24 -ljansson \
25 $(XLIB) 25 -lmicrohttpd \
26 $(XLIB) \
27 $(Z_LIBS)
26 28
27check_PROGRAMS = \ 29check_PROGRAMS = \
28 test_json \ 30 test_json \
@@ -57,6 +59,7 @@ test_json_mhd_LDADD = \
57 $(top_builddir)/src/util/libgnunetutil.la \ 59 $(top_builddir)/src/util/libgnunetutil.la \
58 -ljansson \ 60 -ljansson \
59 -lmicrohttpd \ 61 -lmicrohttpd \
62 $(Z_LIBS) \
60 $(LIB_GNURL) 63 $(LIB_GNURL)
61test_json_mhd_CPPFLAGS = \ 64test_json_mhd_CPPFLAGS = \
62 $(CPP_GNURL) $(AM_CPPFLAGS) 65 $(CPP_GNURL) $(AM_CPPFLAGS)
diff --git a/src/json/json_mhd.c b/src/json/json_mhd.c
index 30b29b88e..b6ab2d116 100644
--- a/src/json/json_mhd.c
+++ b/src/json/json_mhd.c
@@ -26,6 +26,7 @@
26 */ 26 */
27#include "platform.h" 27#include "platform.h"
28#include "gnunet_json_lib.h" 28#include "gnunet_json_lib.h"
29#include <zlib.h>
29 30
30 31
31/** 32/**
@@ -55,6 +56,11 @@ struct Buffer
55 * Number of allocated bytes in buffer. 56 * Number of allocated bytes in buffer.
56 */ 57 */
57 size_t alloc; 58 size_t alloc;
59
60 /**
61 * Maximum buffer size allowed.
62 */
63 size_t max;
58}; 64};
59 65
60 66
@@ -80,7 +86,10 @@ buffer_init (struct Buffer *buf,
80 if (data_size > alloc_size) 86 if (data_size > alloc_size)
81 alloc_size = data_size; 87 alloc_size = data_size;
82 buf->data = GNUNET_malloc (alloc_size); 88 buf->data = GNUNET_malloc (alloc_size);
89 buf->alloc = alloc_size;
83 GNUNET_memcpy (buf->data, data, data_size); 90 GNUNET_memcpy (buf->data, data, data_size);
91 buf->fill = data_size;
92 buf->max = max_size;
84 return GNUNET_OK; 93 return GNUNET_OK;
85} 94}
86 95
@@ -138,6 +147,99 @@ buffer_append (struct Buffer *buf,
138 147
139 148
140/** 149/**
150 * Decompress data in @a buf.
151 *
152 * @param buf input data to inflate
153 * @return result code indicating the status of the operation
154 */
155static enum GNUNET_JSON_PostResult
156inflate_data (struct Buffer *buf)
157{
158 z_stream z;
159 char *tmp;
160 size_t tmp_size;
161 int ret;
162
163 memset (&z, 0, sizeof (z));
164 z.next_in = (Bytef *) buf->data;
165 z.avail_in = buf->fill;
166 tmp_size = GNUNET_MIN (buf->max, buf->fill * 4);
167 tmp = GNUNET_malloc (tmp_size);
168 z.next_out = (Bytef *) tmp;
169 z.avail_out = tmp_size;
170 ret = inflateInit (&z);
171 switch (ret)
172 {
173 case Z_MEM_ERROR:
174 GNUNET_break (0);
175 return GNUNET_JSON_PR_OUT_OF_MEMORY;
176 case Z_STREAM_ERROR:
177 GNUNET_break_op (0);
178 return GNUNET_JSON_PR_JSON_INVALID;
179 case Z_OK:
180 break;
181 }
182 while (1)
183 {
184 ret = inflate (&z, 0);
185 switch (ret)
186 {
187 case Z_MEM_ERROR:
188 GNUNET_break (0);
189 GNUNET_break (Z_OK == inflateEnd (&z));
190 GNUNET_free (tmp);
191 return GNUNET_JSON_PR_OUT_OF_MEMORY;
192 case Z_DATA_ERROR:
193 GNUNET_break (0);
194 GNUNET_break (Z_OK == inflateEnd (&z));
195 GNUNET_free (tmp);
196 return GNUNET_JSON_PR_JSON_INVALID;
197 case Z_NEED_DICT:
198 GNUNET_break (0);
199 GNUNET_break (Z_OK == inflateEnd (&z));
200 GNUNET_free (tmp);
201 return GNUNET_JSON_PR_JSON_INVALID;
202 case Z_OK:
203 if ((0 < z.avail_out) && (0 == z.avail_in))
204 {
205 /* truncated input stream */
206 GNUNET_break (0);
207 GNUNET_break (Z_OK == inflateEnd (&z));
208 GNUNET_free (tmp);
209 return GNUNET_JSON_PR_JSON_INVALID;
210 }
211 if (0 < z.avail_out)
212 continue; /* just call it again */
213 /* output buffer full, can we grow it? */
214 if (tmp_size == buf->max)
215 {
216 /* already at max */
217 GNUNET_break (0);
218 GNUNET_break (Z_OK == inflateEnd (&z));
219 GNUNET_free (tmp);
220 return GNUNET_JSON_PR_OUT_OF_MEMORY;
221 }
222 if (tmp_size * 2 < tmp_size)
223 tmp_size = buf->max;
224 else
225 tmp_size = GNUNET_MIN (buf->max, tmp_size * 2);
226 tmp = GNUNET_realloc (tmp, tmp_size);
227 z.next_out = (Bytef *) &tmp[z.total_out];
228 continue;
229 case Z_STREAM_END:
230 /* decompression successful, make 'tmp' the new 'data' */
231 GNUNET_free (buf->data);
232 buf->data = tmp;
233 buf->alloc = tmp_size;
234 buf->fill = z.total_out;
235 GNUNET_break (Z_OK == inflateEnd (&z));
236 return GNUNET_JSON_PR_SUCCESS; /* at least for now */
237 }
238 } /* while (1) */
239}
240
241
242/**
141 * Process a POST request containing a JSON object. This function 243 * Process a POST request containing a JSON object. This function
142 * realizes an MHD POST processor that will (incrementally) process 244 * realizes an MHD POST processor that will (incrementally) process
143 * JSON data uploaded to the HTTP server. It will store the required 245 * JSON data uploaded to the HTTP server. It will store the required
@@ -161,10 +263,13 @@ GNUNET_JSON_post_parser (size_t buffer_max,
161 json_t **json) 263 json_t **json)
162{ 264{
163 struct Buffer *r = *con_cls; 265 struct Buffer *r = *con_cls;
266 const char *ce;
267 int ret;
164 268
165 *json = NULL; 269 *json = NULL;
166 if (NULL == *con_cls) 270 if (NULL == *con_cls)
167 { 271 {
272
168 /* We are seeing a fresh POST request. */ 273 /* We are seeing a fresh POST request. */
169 r = GNUNET_new (struct Buffer); 274 r = GNUNET_new (struct Buffer);
170 if (GNUNET_OK != buffer_init (r, 275 if (GNUNET_OK != buffer_init (r,
@@ -202,12 +307,29 @@ GNUNET_JSON_post_parser (size_t buffer_max,
202 } 307 }
203 308
204 /* We have seen the whole request. */ 309 /* We have seen the whole request. */
310 ce = MHD_lookup_connection_value (connection,
311 MHD_HEADER_KIND,
312 MHD_HTTP_HEADER_CONTENT_ENCODING);
313 if ((NULL != ce) && (0 == strcasecmp ("deflate", ce)))
314 {
315 ret = inflate_data (r);
316 if (GNUNET_JSON_PR_SUCCESS != ret)
317 {
318 buffer_deinit (r);
319 GNUNET_free (r);
320 *con_cls = NULL;
321 return ret;
322 }
323 }
205 324
206 *json = json_loadb (r->data, r->fill, 0, NULL); 325 *json = json_loadb (r->data, r->fill, 0, NULL);
207 if (NULL == *json) 326 if (NULL == *json)
208 { 327 {
209 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 328 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
210 "Failed to parse JSON request body\n"); 329 "Failed to parse JSON request body\n");
330 buffer_deinit (r);
331 GNUNET_free (r);
332 *con_cls = NULL;
211 return GNUNET_JSON_PR_JSON_INVALID; 333 return GNUNET_JSON_PR_JSON_INVALID;
212 } 334 }
213 buffer_deinit (r); 335 buffer_deinit (r);
diff --git a/src/json/test_json_mhd.c b/src/json/test_json_mhd.c
index 665fd140e..13338b32c 100644
--- a/src/json/test_json_mhd.c
+++ b/src/json/test_json_mhd.c
@@ -27,6 +27,7 @@
27#include "gnunet_util_lib.h" 27#include "gnunet_util_lib.h"
28#include "gnunet_json_lib.h" 28#include "gnunet_json_lib.h"
29#include "gnunet_curl_lib.h" 29#include "gnunet_curl_lib.h"
30#include <zlib.h>
30 31
31#define MAX_SIZE 1024 * 1024 32#define MAX_SIZE 1024 * 1024
32 33
@@ -69,7 +70,7 @@ access_handler_cb (void *cls,
69 global_ret = 6; 70 global_ret = 6;
70 } 71 }
71 json_decref (json); 72 json_decref (json);
72 resp = MHD_create_response_from_buffer (2, "OK", MHD_RESPMEM_PERSISTENT); 73 resp = MHD_create_response_from_buffer (3, "OK\n", MHD_RESPMEM_PERSISTENT);
73 ret = MHD_queue_response (connection, MHD_HTTP_OK, resp); 74 ret = MHD_queue_response (connection, MHD_HTTP_OK, resp);
74 MHD_destroy_response (resp); 75 MHD_destroy_response (resp);
75 return ret; 76 return ret;
@@ -100,8 +101,12 @@ main (int argc, const char *const argv[])
100 uint16_t port; 101 uint16_t port;
101 CURL *easy; 102 CURL *easy;
102 char *url; 103 char *url;
104 char *str;
105 size_t slen;
103 long post_data_size; 106 long post_data_size;
104 void *post_data; 107 void *post_data;
108 uLongf dlen;
109 struct curl_slist *json_header;
105 110
106 GNUNET_log_setup ("test-json-mhd", "WARNING", NULL); 111 GNUNET_log_setup ("test-json-mhd", "WARNING", NULL);
107 global_ret = 2; 112 global_ret = 2;
@@ -123,28 +128,60 @@ main (int argc, const char *const argv[])
123 GNUNET_snprintf (tmp, sizeof (tmp), "%u", i); 128 GNUNET_snprintf (tmp, sizeof (tmp), "%u", i);
124 json_object_set_new (bigj, tmp, json_string (tmp)); 129 json_object_set_new (bigj, tmp, json_string (tmp));
125 } 130 }
126 post_data = json_dumps (bigj, JSON_INDENT (2)); 131 str = json_dumps (bigj, JSON_INDENT (2));
127 post_data_size = strlen (post_data); 132 slen = strlen (str);
128 133
134#ifdef compressBound
135 dlen = compressBound (slen);
136#else
137 dlen = slen + slen / 100 + 20;
138 /* documentation says 100.1% oldSize + 12 bytes, but we
139 * should be able to overshoot by more to be safe */
140#endif
141 post_data = GNUNET_malloc (dlen);
142 if (Z_OK !=
143 compress2 ((Bytef *) post_data, &dlen, (const Bytef *) str, slen, 9))
144 {
145 GNUNET_break (0);
146 MHD_stop_daemon (daemon);
147 GNUNET_free (url);
148 json_decref (bigj);
149 GNUNET_free (post_data);
150 GNUNET_free (str);
151 return 1;
152 }
153 post_data_size = (long) dlen;
129 port = MHD_get_daemon_info (daemon, MHD_DAEMON_INFO_BIND_PORT)->port; 154 port = MHD_get_daemon_info (daemon, MHD_DAEMON_INFO_BIND_PORT)->port;
130 easy = curl_easy_init (); 155 easy = curl_easy_init ();
131 GNUNET_asprintf (&url, "http://localhost:%u/", (unsigned int) port); 156 GNUNET_asprintf (&url, "http://localhost:%u/", (unsigned int) port);
132 curl_easy_setopt (easy, CURLOPT_VERBOSE, 1); 157 curl_easy_setopt (easy, CURLOPT_VERBOSE, 0);
133 curl_easy_setopt (easy, CURLOPT_URL, url); 158 curl_easy_setopt (easy, CURLOPT_URL, url);
134 curl_easy_setopt (easy, CURLOPT_POST, 1); 159 curl_easy_setopt (easy, CURLOPT_POST, 1);
135 curl_easy_setopt (easy, CURLOPT_POSTFIELDS, post_data); 160 curl_easy_setopt (easy, CURLOPT_POSTFIELDS, post_data);
136 curl_easy_setopt (easy, CURLOPT_POSTFIELDSIZE, post_data_size); 161 curl_easy_setopt (easy, CURLOPT_POSTFIELDSIZE, post_data_size);
162
163 json_header = curl_slist_append (NULL, "Content-Type: application/json");
164 json_header = curl_slist_append (json_header, "Content-Encoding: deflate");
165 curl_easy_setopt (easy, CURLOPT_HTTPHEADER, json_header);
137 if (0 != curl_easy_perform (easy)) 166 if (0 != curl_easy_perform (easy))
138 { 167 {
139 GNUNET_break (0); 168 GNUNET_break (0);
140 MHD_stop_daemon (daemon); 169 MHD_stop_daemon (daemon);
141 GNUNET_free (url); 170 GNUNET_free (url);
142 json_decref (bigj); 171 json_decref (bigj);
172 GNUNET_free (post_data);
173 GNUNET_free (str);
174 curl_slist_free_all (json_header);
175 curl_easy_cleanup (easy);
143 return 1; 176 return 1;
144 } 177 }
145 MHD_stop_daemon (daemon); 178 MHD_stop_daemon (daemon);
146 GNUNET_free (url); 179 GNUNET_free (url);
147 json_decref (bigj); 180 json_decref (bigj);
181 GNUNET_free (post_data);
182 GNUNET_free (str);
183 curl_slist_free_all (json_header);
184 curl_easy_cleanup (easy);
148 return global_ret; 185 return global_ret;
149} 186}
150 187