Klaus Demo bjoern / 56c2623
Yay HTTP/1.1 support! (experimental) Please give it a shot and tell me about every improvement idea you got :) Source is now 1100 lines fat, so Code Review(tm) will follow. Jonas Haag 9 years ago
10 changed file(s) with 263 addition(s) and 106 deletion(s). Raw diff Collapse all Expand all
4040 Usage
4141 ~~~~~
4242 ::
43
43
4444 bjoern.run(wsgi_application, host, port)
4545
4646 Alternatively, the mainloop can be run separately::
2727 typedef PyObject* PyKeywordFunc(PyObject* self, PyObject* args, PyObject *kwargs);
2828
2929 typedef enum {
30 HTTP_BAD_REQUEST, HTTP_LENGTH_REQUIRED, HTTP_SERVER_ERROR
30 HTTP_BAD_REQUEST = 1, HTTP_LENGTH_REQUIRED, HTTP_SERVER_ERROR
3131 } http_status;
3232
3333 #ifdef DEBUG
5252 #define DBG_REFCOUNT_REQ(request, obj) \
5353 DBG_REQ(request, #obj "->ob_refcnt: %d", obj->ob_refcnt)
5454
55 #ifdef WITHOUT_ASSERTS
56 #undef assert
57 #define assert(...) do{}while(0)
5558 #endif
59
60 #endif
66 static Request _preallocd[REQUEST_PREALLOC_N];
77 static char _preallocd_used[REQUEST_PREALLOC_N];
88
9 static PyObject* wsgi_http_header(const char*, const size_t);
9 static void request_clean(Request*);
10 static PyObject* wsgi_http_header(Request*, const char*, const size_t);
1011 static http_parser_settings parser_settings;
1112 static PyObject* wsgi_base_dict;
1213
1415 Request* Request_new(int client_fd, const char* client_addr)
1516 {
1617 Request* req = _Request_from_prealloc();
17 if(req == NULL)
18 req = malloc(sizeof(Request));
19 if(req == NULL)
20 return NULL;
21
22 memset(req, 0, sizeof(Request));
23
18 if(req == NULL) req = malloc(sizeof(Request));
19 if(req == NULL) return NULL;
2420 #ifdef DEBUG
2521 static unsigned long req_id = 0;
2622 req->id = req_id++;
2723 #endif
28
2924 req->client_fd = client_fd;
30 req->headers = PyDict_New();
31 PyDict_SetItem(req->headers, _REMOTE_ADDR, PyString_FromString(client_addr));
25 req->client_addr = PyString_FromString(client_addr);
3226 http_parser_init((http_parser*)&req->parser, HTTP_REQUEST);
3327 req->parser.parser.data = req;
34
28 Request_reset(req, false);
3529 return req;
30 }
31
32 void Request_reset(Request* req, bool decref_members)
33 {
34 if(decref_members)
35 request_clean(req);
36 memset(&req->state, 0, sizeof(Request) - (size_t)&((Request*)NULL)->state);
37 req->state.response_length_unknown = true;
3638 }
3739
3840 void Request_parse(Request* request,
4648 return;
4749 }
4850
49 request->state.error = true;
5051 request->state.error_code = HTTP_BAD_REQUEST;
5152 }
5253
5354 void Request_free(Request* req)
5455 {
55 Py_XDECREF(req->iterable);
56 Py_XDECREF(req->body);
57 if(req->headers)
58 assert(req->headers->ob_refcnt >= 1);
59 if(req->status)
60 assert(req->status->ob_refcnt >= 1);
61 Py_XDECREF(req->headers);
62 Py_XDECREF(req->status);
63
56 request_clean(req);
6457 if(req >= _preallocd &&
6558 req <= _preallocd+REQUEST_PREALLOC_N*sizeof(Request)) {
6659 _preallocd_used[req-_preallocd] = false;
8679 #endif
8780 }
8881
82 static void request_clean(Request* req)
83 {
84 Py_XDECREF(req->iterable);
85 Py_XDECREF(req->body);
86 if(req->headers)
87 assert(req->headers->ob_refcnt >= 1);
88 if(req->status)
89 assert(req->status->ob_refcnt >= 1);
90 Py_XDECREF(req->headers);
91 Py_XDECREF(req->status);
92 }
93
8994 Request* _Request_from_prealloc()
9095 {
9196 static int i = 0;
142147 static int
143148 on_message_begin(http_parser* parser)
144149 {
150 REQUEST->headers = PyDict_New();
145151 PARSER->field_start = NULL;
146152 PARSER->field_len = 0;
147153 PARSER->value_start = NULL;
195201 if(PARSER->value_start) {
196202 /* Store previous header and start a new one */
197203 _set_header_free_both(
198 wsgi_http_header(PARSER->field_start, PARSER->field_len),
204 wsgi_http_header(REQUEST, PARSER->field_start, PARSER->field_len),
199205 PyString_FromStringAndSize(PARSER->value_start, PARSER->value_len)
200206 );
201207
230236 {
231237 if(PARSER->field_start) {
232238 _set_header_free_both(
233 wsgi_http_header(PARSER->field_start, PARSER->field_len),
239 wsgi_http_header(REQUEST, PARSER->field_start, PARSER->field_len),
234240 PyString_FromStringAndSize(PARSER->value_start, PARSER->value_len)
235241 );
236242 }
242248 const size_t body_len) {
243249 if(!REQUEST->body) {
244250 if(!parser->content_length) {
245 REQUEST->state.error = true;
246251 REQUEST->state.error_code = HTTP_LENGTH_REQUIRED;
247252 return 1;
248253 }
250255 }
251256
252257 if(PycStringIO->cwrite(REQUEST->body, body_start, body_len) < 0) {
253 REQUEST->state.error = true;
254258 REQUEST->state.error_code = HTTP_SERVER_ERROR;
255259 return 1;
256260 }
288292 parser->http_minor == 1 ? _HTTP_1_1 : _HTTP_1_0
289293 );
290294
295 _set_header(
296 _REMOTE_ADDR,
297 REQUEST->client_addr
298 );
299
291300 PyDict_Update(REQUEST->headers, wsgi_base_dict);
292301
293302 REQUEST->state.parse_finished = true;
308317 }
309318
310319 static PyObject*
311 wsgi_http_header(const char* data, size_t len)
320 wsgi_http_header(Request* request, const char* data, size_t len)
312321 {
313322 if(string_iequal(data, len, "Content-Length")) {
314323 Py_INCREF(_CONTENT_LENGTH);
88
99 typedef struct {
1010 unsigned error_code : 2;
11 unsigned error : 1;
1211 unsigned parse_finished : 1;
1312 unsigned start_response_called : 1;
14 unsigned headers_sent : 1;
13 unsigned wsgi_call_done : 1;
14 unsigned keep_alive : 1;
15 unsigned response_length_unknown : 1;
16 unsigned chunked_response : 1;
1517 } request_state;
1618
1719 typedef struct {
2628 #ifdef DEBUG
2729 unsigned long id;
2830 #endif
29 request_state state;
30 int client_fd;
31 bj_parser parser;
3132 ev_io ev_watcher;
3233
33 bj_parser parser;
34 int client_fd;
35 PyObject* client_addr;
36
37 request_state state;
38
3439 PyObject* headers;
3540 PyObject* body;
36
3741 PyObject* current_chunk;
3842 Py_ssize_t current_chunk_p;
3943 PyObject* iterable;
4145 } Request;
4246
4347 Request* Request_new(int client_fd, const char* client_addr);
48 void Request_reset(Request*, bool decref_members);
4449 void Request_parse(Request*, const char*, const size_t);
4550 void Request_free(Request*);
46
4751
4852 static PyObject
4953 * _REMOTE_ADDR,
9797 DBG_REQ(request, "Accepted client %s:%d on fd %d",
9898 inet_ntoa(sockaddr.sin_addr), ntohs(sockaddr.sin_port), client_fd);
9999
100 ev_io_init(&request->ev_watcher, &ev_io_on_read, client_fd, EV_READ);
100 ev_io_init(&request->ev_watcher, &ev_io_on_read,
101 client_fd, EV_READ);
101102 ev_io_start(mainloop, &request->ev_watcher);
102103 }
103104
106107 {
107108 char read_buf[READ_BUFFER_SIZE];
108109 Request* request = ADDR_FROM_MEMBER(watcher, Request, ev_watcher);
109 ssize_t read_bytes = read(request->client_fd, read_buf, READ_BUFFER_SIZE);
110
111 DBG_REQ(request, "read %zd bytes", read_bytes);
110
111 ssize_t read_bytes = read(
112 request->client_fd,
113 read_buf,
114 READ_BUFFER_SIZE
115 );
112116
113117 GIL_LOCK(0);
114118
115 if(read_bytes == -1) {
119 if(read_bytes <= 0) {
116120 if(errno != EAGAIN && errno != EWOULDBLOCK) {
121 if(read_bytes == 0)
122 DBG_REQ(request, "Client disconnected");
123 else
124 DBG_REQ(request, "Hit errno %d while read()ing", errno);
117125 close(request->client_fd);
118126 Request_free(request);
119127 ev_io_stop(mainloop, &request->ev_watcher);
121129 goto out;
122130 }
123131
132 DBG_REQ(request, "read %zd bytes", read_bytes);
133
124134 Request_parse(request, read_buf, read_bytes);
125135
126 if(request->state.error) {
136 if(request->state.error_code) {
127137 DBG_REQ(request, "Parse error");
128138 set_error(request, request->state.error_code);
129139 }
158168 assert(request->current_chunk);
159169
160170 if(send_chunk(request))
161 goto notfinished;
171 goto out;
162172
163173 if(request->iterable) {
164174 PyObject* next_chunk;
165175 DBG_REQ(request, "Getting next iterable chunk.");
166176 next_chunk = wsgi_iterable_get_next_chunk(request);
167 if(next_chunk == NULL) {
177 if(next_chunk) {
178 if(request->state.chunked_response) {
179 request->current_chunk = wrap_http_chunk_cruft_around(next_chunk);
180 Py_DECREF(next_chunk);
181 } else {
182 request->current_chunk = next_chunk;
183 }
184 assert(request->current_chunk_p == 0);
185 goto out;
186 } else {
168187 if(PyErr_Occurred()) {
169 /* Internal server error. */
170188 PyErr_Print();
171 set_error(request, HTTP_SERVER_ERROR);
172 goto notfinished;
189 /* We can't do anything graceful here because at least one
190 chunk is already sent... just close the connection */
191 DBG_REQ(request, "Exception in iterator, can not recover");
192 ev_io_stop(mainloop, &request->ev_watcher);
193 close(request->client_fd);
194 Request_free(request);
195 goto out;
173196 }
174197 DBG_REQ(request, "Iterable exhausted");
175 goto bye;
198 Py_DECREF(request->iterable);
199 request->iterable = NULL;
200 if(request->state.chunked_response) {
201 /* We have to send a terminating empty chunk + \r\n */
202 DBG_REQ(request, "Sentinel chunk");
203 request->current_chunk = PyString_FromString("0\r\n\r\n");
204 assert(request->current_chunk_p == 0);
205 goto out;
206 }
176207 }
177 request->current_chunk = next_chunk;
178 assert(request->current_chunk_p == 0);
179 goto notfinished;
180 }
181
182 bye:
183 DBG_REQ(request, "Done");
184
185 /* Everything done, bye client! */
208 }
209
186210 ev_io_stop(mainloop, &request->ev_watcher);
187 close(request->client_fd);
188 Request_free(request);
189
190 notfinished:
211 if(request->state.keep_alive) {
212 DBG_REQ(request, "done, keep-alive");
213 Request_reset(request, /* decref members? */ true);
214 ev_io_init(&request->ev_watcher, &ev_io_on_read,
215 request->client_fd, EV_READ);
216 ev_io_start(mainloop, &request->ev_watcher);
217 } else {
218 DBG_REQ(request, "done, close");
219 close(request->client_fd);
220 Request_free(request);
221 }
222
223 out:
191224 GIL_UNLOCK(0);
192225 }
193226
194227 static bool
195228 send_chunk(Request* request)
196229 {
230 ssize_t chunk_length;
197231 ssize_t sent_bytes;
198232
199233 assert(request->current_chunk != NULL);
208242 );
209243
210244 if(sent_bytes == -1) {
245 error:
211246 if(errno == EAGAIN || errno == EWOULDBLOCK) {
212247 /* Try again later */
213248 return 1;
221256 }
222257 }
223258
224 DBG_REQ(request, "Sent %zd bytes from %p", sent_bytes, request->current_chunk);
259 DBG_REQ(request, "Sent %zd/%zd bytes from %p", sent_bytes,
260 PyString_GET_SIZE(request->current_chunk), request->current_chunk);
225261 request->current_chunk_p += sent_bytes;
226262 if(request->current_chunk_p == PyString_GET_SIZE(request->current_chunk)) {
227263 DBG_REQ(request, "Done with string %p", request->current_chunk);
252288 default:
253289 assert(0);
254290 }
291 assert(!request->state.chunked_response);
255292 request->current_chunk = PyString_FromString(msg);
256293
257294 if(request->iterable != NULL) {
00 #include "common.h"
1 #include "server.h"
21 #include "bjoernmodule.h"
32 #include "wsgi.h"
43
54 static PyKeywordFunc start_response;
65 static Py_ssize_t wsgi_getheaders(Request*, PyObject* buf);
6 static inline bool inspect_headers(Request*);
7 static inline bool should_keep_alive(Request*);
78
89 typedef struct {
910 PyObject_HEAD
1112 } StartResponse;
1213 PyTypeObject StartResponse_Type;
1314
15 #define APPEND_HEADER(request, name, value) \
16 do { \
17 PyObject *n = PyString_FromString(name), *v = (value); \
18 PyObject* tpl = PyTuple_Pack(2, n, v); \
19 PyList_Append(request->headers, tpl); \
20 Py_DECREF(tpl); Py_DECREF(n); Py_DECREF(v); \
21 } while(0)
1422
1523 bool
1624 wsgi_call_application(Request* request)
6674 */
6775 PyObject* first_chunk = NULL;
6876
69 if(PyString_Check(retval)) {
77 if(PyList_Check(retval) && PyList_GET_SIZE(retval) == 1 &&
78 PyString_Check(PyList_GET_ITEM(retval, 0)))
79 {
80 /* Optimize the most common case, a single string in a list: */
81 first_chunk = PyList_GET_ITEM(retval, 0);
82 Py_INCREF(first_chunk);
83 Py_DECREF(retval);
84 } else if(PyString_Check(retval)) {
7085 /* According to PEP 333 strings should be handled like any other iterable,
7186 i.e. sending the response item for item. "item for item" means
7287 "char for char" if you have a string. -- I'm not that stupid. */
7388 first_chunk = retval;
74 }
75 else if(PyList_Check(retval) && PyList_GET_SIZE(retval) == 1) {
76 /* Optimize the most common case, a single string in a list: */
77 first_chunk = PyList_GET_ITEM(retval, 0);
78 Py_INCREF(first_chunk);
79 Py_DECREF(retval);
8089 } else {
8190 /* Generic iterable (list of length != 1, generator, ...) */
8291 PyObject* iter = PyObject_GetIter(retval);
99108 );
100109 Py_DECREF(first_chunk);
101110 return false;
111 }
112
113 if(!inspect_headers(request))
114 return NULL;
115
116 if(first_chunk == NULL && request->state.response_length_unknown) {
117 /* No body because the iterator was empty. In case of keep-alive we have
118 to do some extra foo to let the client know there's no body. */
119 request->state.response_length_unknown = false;
120 APPEND_HEADER(request, "Content-Length", PyString_FromString("0"));
121 }
122
123 if(should_keep_alive(request)) {
124 request->state.chunked_response = request->state.response_length_unknown;
125 request->state.keep_alive = true;
126 } else {
127 request->state.keep_alive = false;
102128 }
103129
104130 /* Get the headers and concatenate the first body chunk to them.
116142 }
117143
118144 if(first_chunk == NULL) {
119 /* The iterator returned by the application was empty. No body. */
120145 _PyString_Resize(&buf, length);
121146 goto out;
122147 }
123148
124 assert(first_chunk);
149
150 if(request->state.chunked_response) {
151 DBG_REQ(request, "Chunk chunk is of size %zd", PyString_GET_SIZE(first_chunk));
152 PyObject* new_chunk = wrap_http_chunk_cruft_around(first_chunk);
153 Py_DECREF(first_chunk);
154 assert(PyString_GET_SIZE(new_chunk) >= PyString_GET_SIZE(first_chunk) + 5);
155 first_chunk = new_chunk;
156 }
157
125158 assert(buf);
159 DBG_REQ(request, "Resize chunk is of size %zd", PyString_GET_SIZE(first_chunk));
160 DBG_REQ(request, "Resizing buffer to %zd bytes", length+PyString_GET_SIZE(first_chunk));
126161 _PyString_Resize(&buf, length + PyString_GET_SIZE(first_chunk));
127162 memcpy(PyString_AS_STRING(buf)+length, PyString_AS_STRING(first_chunk),
128163 PyString_GET_SIZE(first_chunk));
130165 Py_DECREF(first_chunk);
131166
132167 out:
133 request->state.headers_sent = true;
168 request->state.wsgi_call_done = true;
134169 request->current_chunk = buf;
135170 request->current_chunk_p = 0;
171 return true;
172 }
173
174 static inline bool
175 inspect_headers(Request* request)
176 {
177 for(Py_ssize_t i=0; i<PyList_GET_SIZE(request->headers); ++i) {
178 PyObject* tuple = PyList_GET_ITEM(request->headers, i);
179 assert(tuple);
180 TYPECHECK(tuple, PyTuple, "headers", false);
181
182 if(PyTuple_GET_SIZE(tuple) != 2) {
183 PyErr_Format(
184 PyExc_TypeError,
185 "headers must be tuples of length 2, not %zd",
186 PyTuple_GET_SIZE(tuple)
187 );
188 return false;
189 }
190
191 PyObject* field = PyTuple_GET_ITEM(tuple, 0);
192 PyObject* value = PyTuple_GET_ITEM(tuple, 1);
193
194 TYPECHECK(field, PyString, "header tuple items", false);
195 TYPECHECK(value, PyString, "header tuple items", false);
196
197 if(!strcasecmp(PyString_AS_STRING(field), "Content-Length"))
198 request->state.response_length_unknown = false;
199 }
136200 return true;
137201 }
138202
147211 const char* s = src; \
148212 while(n--) *bufp++ = *s++; \
149213 } while(0)
150
151 buf_write("HTTP/1.0 ", strlen("HTTP/1.0 "));
214 #define buf_write2(src) buf_write(src, strlen(src))
215
216 buf_write("HTTP/1.1 ", strlen("HTTP/1.1 "));
152217 buf_write(PyString_AS_STRING(request->status),
153218 PyString_GET_SIZE(request->status));
154219
155 PyObject *tuple, *field, *value;
156220 for(Py_ssize_t i=0; i<PyList_GET_SIZE(request->headers); ++i) {
157 tuple = PyList_GET_ITEM(request->headers, i);
158 assert(tuple);
159 TYPECHECK(tuple, PyTuple, "headers", 0);
160
161 if(PyTuple_GET_SIZE(tuple) != 2) {
162 PyErr_Format(
163 PyExc_TypeError,
164 "headers must be tuples of length 2, not %zd",
165 PyTuple_GET_SIZE(tuple)
166 );
167 return 0;
168 }
169
170 field = PyTuple_GET_ITEM(tuple, 0);
171 value = PyTuple_GET_ITEM(tuple, 1);
172
173 TYPECHECK(field, PyString, "header tuple items", 0);
174 TYPECHECK(value, PyString, "header tuple items", 0);
175
176 buf_write("\r\n", strlen("\r\n"));
221 PyObject *tuple = PyList_GET_ITEM(request->headers, i);
222 PyObject *field = PyTuple_GET_ITEM(tuple, 0),
223 *value = PyTuple_GET_ITEM(tuple, 1);
224 buf_write2("\r\n");
177225 buf_write(PyString_AS_STRING(field), PyString_GET_SIZE(field));
178 buf_write(": ", strlen(": "));
226 buf_write2(": ");
179227 buf_write(PyString_AS_STRING(value), PyString_GET_SIZE(value));
180228 }
181 buf_write("\r\n\r\n", strlen("\r\n\r\n"));
182
229 if(request->state.chunked_response)
230 buf_write2("\r\nTransfer-Encoding: chunked");
231 buf_write2("\r\n\r\n");
232
233 assert(bufp - PyString_AS_STRING(buf) > strlen("HTTP/X.X nnn R\r\n\r\n"));
183234 return bufp - PyString_AS_STRING(buf);
184235 }
185236
186237 inline PyObject*
187238 wsgi_iterable_get_next_chunk(Request* request)
188239 {
189 PyObject* next = PyIter_Next(request->iterable);
190 if(next)
240 /* Get the next item out of ``request->iterable``, skipping empty ones. */
241 PyObject* next;
242 while(true) {
243 next = PyIter_Next(request->iterable);
244 if(next == NULL)
245 return NULL;
191246 TYPECHECK(next, PyString, "wsgi iterable items", NULL);
192 return next;
247 DBG_REQ(request, "Next chunk is of size %zd", PyString_GET_SIZE(next));
248 if(PyString_GET_SIZE(next))
249 return next;
250 }
193251 }
194252
195253 static inline void
241299
242300 restore_exception_tuple(exc_info, /* incref items? */ true);
243301
244 if(request->state.headers_sent) {
245 /* Headers already sent. According to PEP 333, we should
246 * let the exception propagate in this case. */
302 if(request->state.wsgi_call_done) {
303 /* Too late to change headers. According to PEP 333, we should let
304 the exception propagate in this case. */
247305 return NULL;
248306 }
249307
280338 0, 0, 0, 0, 0, 0, 0, 0, 0, /* tp_{print,getattr,setattr,compare,...} */
281339 start_response /* tp_call (__call__) */
282340 };
341
342 #define F_KEEP_ALIVE 1<<1
343 #define have_http11(parser) (parser.http_major > 0 && parser.http_minor > 0)
344
345 static inline bool
346 should_keep_alive(Request* request)
347 {
348 if(!(request->parser.parser.flags & F_KEEP_ALIVE)) {
349 /* Only keep-alive if the client requested it explicitly */
350 return false;
351 }
352 if(request->state.response_length_unknown) {
353 /* If the client wants to keep-alive the connection but we don't know
354 the response length, we can use Transfer-Encoding: chunked on HTTP/1.1.
355 On HTTP/1.0 no such thing exists so there's no other option than closing
356 the connection to indicate the response end. */
357 return have_http11(request->parser.parser);
358 } else {
359 /* If the response length is known we can keep-alive for both 1.0 and 1.1 */
360 return true;
361 }
362 }
363
364 PyObject*
365 wrap_http_chunk_cruft_around(PyObject* chunk) {
366 /* Who the hell decided to use decimal representation for Content-Length
367 but hexadecimal representation for chunk lengths btw!?! Fuck W3C */
368 size_t chunklen = PyString_GET_SIZE(chunk);
369 char buf[strlen("ffffffffffffffff") + 2];
370 size_t n = sprintf(buf, "%x\r\n", chunklen);
371 PyObject* new_chunk = PyString_FromStringAndSize(NULL, n + chunklen + 2);
372 char* new_chunk_p = PyString_AS_STRING(new_chunk);
373 memcpy(new_chunk_p, buf, n);
374 new_chunk_p += n;
375 memcpy(new_chunk_p, PyString_AS_STRING(chunk), chunklen);
376 new_chunk_p += chunklen;
377 *new_chunk_p++ = '\r'; *new_chunk_p = '\n';
378 assert(new_chunk_p == PyString_AS_STRING(new_chunk) + n + chunklen + 1);
379 return new_chunk;
380 }
381
22 bool wsgi_call_application(Request*);
33 PyObject* wsgi_iterable_get_next_chunk(Request*);
44 PyTypeObject StartResponse_Type;
5 PyObject* wrap_http_chunk_cruft_around(PyObject* chunk);
33 def app(environ, start_response):
44 start_response('200 OK', [])
55 yield 'Hello world'
6 yield ''
67
78 bjoern.listen(app, '0.0.0.0', 8080)
89 bjoern.run()
1616 def return_404(environ, start_response):
1717 start_response('404 Not Found', (('Content-Type','text/plain'), ))
1818 return "URL %s not found" % environ.get('PATH_INFO', 'UNKNOWN')
19
19
2020 dispatch = {
2121 '/tuple': return_tuple,
2222 '/huge': return_huge_answer,
3737
3838 def app(environ, start_response):
3939 app = chose_test()
40 print 'Testing %s...' % app.__name__
4041 return app(environ, start_response)
4142
4243 import bjoern