Klaus Demo bjoern / 9a7aee0
Fixed a severe memory leak by refactoring request body buffering. Thanks to Qingshan Luo for reporting this issue! PyPI bugfix release follows. Jonas Haag 9 years ago
5 changed file(s) with 53 addition(s) and 21 deletion(s). Raw diff Collapse all Expand all
5858 clean:
5959 @rm -f $(BUILD_DIR)/*
6060
61 ab:
62 ab -c 100 -n 10000 'http://127.0.0.1:8080/a/b/c?k=v&k2=v2#fragment'
61 AB = ab -c 100 -n 10000
62 TEST_URL = "http://127.0.0.1:8080/a/b/c?k=v&k2=v2"
63
64 ab: ab1 ab2 ab3 ab4
65
66 ab1:
67 $(AB) $(TEST_URL)
68 ab2:
69 @echo 'asdfghjkl=asdfghjkl&qwerty=qwertyuiop' > /tmp/bjoern-post.tmp
70 $(AB) -p /tmp/bjoern-post.tmp $(TEST_URL)
71 ab3:
72 $(AB) -k $(TEST_URL)
73 ab4:
74 @echo 'asdfghjkl=asdfghjkl&qwerty=qwertyuiop' > /tmp/bjoern-post.tmp
75 $(AB) -k -p /tmp/bjoern-post.tmp $(TEST_URL)
6376
6477 wget:
65 wget -O - -q -S 'http://127.0.0.1:8080/a/b/c?k=v&k2=v2#fragment'
78 wget -O - -q -S $(TEST_URL)
6679
6780 test:
6881 cd tests && python ~/dev/wsgitest/runner.py
55 static PyObject* wsgi_http_header(Request*, const char*, const size_t);
66 static http_parser_settings parser_settings;
77 static PyObject* wsgi_base_dict = NULL;
8
9 /* Non-public type from cStringIO I abuse in on_body */
10 typedef struct {
11 PyObject_HEAD
12 char *buf;
13 Py_ssize_t pos, string_size;
14 PyObject *pbuf;
15 } Iobject;
816
917 Request* Request_new(int client_fd, const char* client_addr)
1018 {
5058 Py_DECREF(request->iterable);
5159 }
5260 Py_XDECREF(request->iterator);
53 Py_XDECREF(request->body);
5461 if(request->headers)
5562 assert(request->headers->ob_refcnt >= 1);
5663 if(request->status)
171178 }
172179
173180 static int
174 on_body(http_parser* parser, const char* body, const size_t len)
175 {
176 if(!REQUEST->body) {
181 on_body(http_parser* parser, const char* data, const size_t len)
182 {
183 Iobject* body;
184
185 body = (Iobject*)PyDict_GetItem(REQUEST->headers, _wsgi_input);
186 if(body == NULL) {
177187 if(!parser->content_length) {
178188 REQUEST->state.error_code = HTTP_LENGTH_REQUIRED;
179189 return 1;
180190 }
181 REQUEST->body = PycStringIO->NewOutput(parser->content_length);
182 }
183 if(PycStringIO->cwrite(REQUEST->body, body, len) < 0) {
184 REQUEST->state.error_code = HTTP_SERVER_ERROR;
185 return 1;
186 }
191 PyObject* buf = PyString_FromStringAndSize(NULL, parser->content_length);
192 body = (Iobject*)PycStringIO->NewInput(buf);
193 Py_XDECREF(buf);
194 if(body == NULL)
195 return 1;
196 _set_header(_wsgi_input, (PyObject*)body);
197 Py_DECREF(body);
198 }
199 memcpy(body->buf + body->pos, data, len);
200 body->pos += len;
187201 return 0;
188202 }
189203
209223 /* REMOTE_ADDR */
210224 _set_header(_REMOTE_ADDR, REQUEST->client_addr);
211225
212 /* wsgi.input */
213 _set_header_free_value(
214 _wsgi_input,
215 PycStringIO->NewInput(REQUEST->body ? PycStringIO->cgetvalue(REQUEST->body)
216 : _empty_string)
217 );
226 PyObject* body = PyDict_GetItem(REQUEST->headers, _wsgi_input);
227 if(body) {
228 /* We abused the `pos` member for tracking the amount of data copied from
229 * the buffer in on_body, so reset it to zero here. */
230 ((Iobject*)body)->pos = 0;
231 } else {
232 /* Request has no body */
233 _set_header_free_value(_wsgi_input, PycStringIO->NewInput(_empty_string));
234 }
218235
219236 PyDict_Update(REQUEST->headers, wsgi_base_dict);
220237
3737 request_state state;
3838
3939 PyObject* headers;
40 PyObject* body;
4140 PyObject* current_chunk;
4241 Py_ssize_t current_chunk_p;
4342 PyObject* iterable;
22
33 def app(env, start_response):
44 pprint.pprint(env)
5 print(len(env['wsgi.input'].read()))
56 start_response('200 yo', [])
67 return []
78
1515 # Content-{Length, Type}
1616 'GET / HTTP/1.0\r\nContent-Length: 11\r\n'
1717 'Content-Type: text/blah\r\nContent-Fype: bla\r\n'
18 'Content-Tength: bla\r\n\r\nhello world'
18 'Content-Tength: bla\r\n\r\nhello world',
19 # POST memory leak
20 'POST / HTTP/1.0\r\nContent-Length: 1000\r\n\r\n%s' % ('a'*1000)
1921 ]
2022 conn.send(msgs[int(sys.argv[1])])
2123 while 1: