1#!/usr/bin/python 2 3# Copyright 2012 Google Inc. All Rights Reserved. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17# Modified by Linus Nielsen Feltzing for inclusion in the libcurl test 18# framework 19# 20import SocketServer 21import argparse 22import re 23import select 24import socket 25import time 26import pprint 27import os 28 29INFO_MESSAGE = ''' 30This is a test server to test the libcurl pipelining functionality. 31It is a modified version if Google's HTTP pipelining test server. More 32information can be found here: 33 34http://dev.chromium.org/developers/design-documents/network-stack/http-pipelining 35 36Source code can be found here: 37 38http://code.google.com/p/http-pipelining-test/ 39''' 40MAX_REQUEST_SIZE = 1024 # bytes 41MIN_POLL_TIME = 0.01 # seconds. Minimum time to poll, in order to prevent 42 # excessive looping because Python refuses to poll for 43 # small timeouts. 44SEND_BUFFER_TIME = 0.5 # seconds 45TIMEOUT = 30 # seconds 46 47 48class Error(Exception): 49 pass 50 51 52class RequestTooLargeError(Error): 53 pass 54 55 56class ServeIndexError(Error): 57 pass 58 59 60class UnexpectedMethodError(Error): 61 pass 62 63 64class RequestParser(object): 65 """Parses an input buffer looking for HTTP GET requests.""" 66 67 global logfile 68 69 LOOKING_FOR_GET = 1 70 READING_HEADERS = 2 71 72 HEADER_RE = re.compile('([^:]+):(.*)\n') 73 REQUEST_RE = re.compile('([^ ]+) ([^ ]+) HTTP/(\d+)\.(\d+)\n') 74 75 def __init__(self): 76 """Initializer.""" 77 self._buffer = "" 78 self._pending_headers = {} 79 self._pending_request = "" 80 self._state = self.LOOKING_FOR_GET 81 self._were_all_requests_http_1_1 = True 82 self._valid_requests = [] 83 84 def ParseAdditionalData(self, data): 85 """Finds HTTP requests in |data|. 86 87 Args: 88 data: (String) Newly received input data from the socket. 89 90 Returns: 91 (List of Tuples) 92 (String) The request path. 93 (Map of String to String) The header name and value. 94 95 Raises: 96 RequestTooLargeError: If the request exceeds MAX_REQUEST_SIZE. 97 UnexpectedMethodError: On a non-GET method. 98 Error: On a programming error. 99 """ 100 logfile = open('log/server.input', 'a') 101 logfile.write(data) 102 logfile.close() 103 self._buffer += data.replace('\r', '') 104 should_continue_parsing = True 105 while should_continue_parsing: 106 if self._state == self.LOOKING_FOR_GET: 107 should_continue_parsing = self._DoLookForGet() 108 elif self._state == self.READING_HEADERS: 109 should_continue_parsing = self._DoReadHeader() 110 else: 111 raise Error('Unexpected state: ' + self._state) 112 if len(self._buffer) > MAX_REQUEST_SIZE: 113 raise RequestTooLargeError( 114 'Request is at least %d bytes' % len(self._buffer)) 115 valid_requests = self._valid_requests 116 self._valid_requests = [] 117 return valid_requests 118 119 @property 120 def were_all_requests_http_1_1(self): 121 return self._were_all_requests_http_1_1 122 123 def _DoLookForGet(self): 124 """Tries to parse an HTTTP request line. 125 126 Returns: 127 (Boolean) True if a request was found. 128 129 Raises: 130 UnexpectedMethodError: On a non-GET method. 131 """ 132 m = self.REQUEST_RE.match(self._buffer) 133 if not m: 134 return False 135 method, path, http_major, http_minor = m.groups() 136 137 if method != 'GET': 138 raise UnexpectedMethodError('Unexpected method: ' + method) 139 if path in ['/', '/index.htm', '/index.html']: 140 raise ServeIndexError() 141 142 if http_major != '1' or http_minor != '1': 143 self._were_all_requests_http_1_1 = False 144 145# print method, path 146 147 self._pending_request = path 148 self._buffer = self._buffer[m.end():] 149 self._state = self.READING_HEADERS 150 return True 151 152 def _DoReadHeader(self): 153 """Tries to parse a HTTP header. 154 155 Returns: 156 (Boolean) True if it found the end of the request or a HTTP header. 157 """ 158 if self._buffer.startswith('\n'): 159 self._buffer = self._buffer[1:] 160 self._state = self.LOOKING_FOR_GET 161 self._valid_requests.append((self._pending_request, 162 self._pending_headers)) 163 self._pending_headers = {} 164 self._pending_request = "" 165 return True 166 167 m = self.HEADER_RE.match(self._buffer) 168 if not m: 169 return False 170 171 header = m.group(1).lower() 172 value = m.group(2).strip().lower() 173 if header not in self._pending_headers: 174 self._pending_headers[header] = value 175 self._buffer = self._buffer[m.end():] 176 return True 177 178 179class ResponseBuilder(object): 180 """Builds HTTP responses for a list of accumulated requests.""" 181 182 def __init__(self): 183 """Initializer.""" 184 self._max_pipeline_depth = 0 185 self._requested_paths = [] 186 self._processed_end = False 187 self._were_all_requests_http_1_1 = True 188 189 def QueueRequests(self, requested_paths, were_all_requests_http_1_1): 190 """Adds requests to the queue of requests. 191 192 Args: 193 requested_paths: (List of Strings) Requested paths. 194 """ 195 self._requested_paths.extend(requested_paths) 196 self._were_all_requests_http_1_1 = were_all_requests_http_1_1 197 198 def Chunkify(self, data, chunksize): 199 """ Divides a string into chunks 200 """ 201 return [hex(chunksize)[2:] + "\r\n" + data[i:i+chunksize] + "\r\n" for i in range(0, len(data), chunksize)] 202 203 def BuildResponses(self): 204 """Converts the queue of requests into responses. 205 206 Returns: 207 (String) Buffer containing all of the responses. 208 """ 209 result = "" 210 self._max_pipeline_depth = max(self._max_pipeline_depth, 211 len(self._requested_paths)) 212 for path, headers in self._requested_paths: 213 if path == '/verifiedserver': 214 body = "WE ROOLZ: {}\r\n".format(os.getpid()); 215 result += self._BuildResponse( 216 '200 OK', ['Server: Apache', 217 'Content-Length: {}'.format(len(body)), 218 'Cache-Control: no-store'], body) 219 220 elif path == '/alphabet.txt': 221 body = 'abcdefghijklmnopqrstuvwxyz' 222 result += self._BuildResponse( 223 '200 OK', ['Server: Apache', 224 'Content-Length: 26', 225 'Cache-Control: no-store'], body) 226 227 elif path == '/reverse.txt': 228 body = 'zyxwvutsrqponmlkjihgfedcba' 229 result += self._BuildResponse( 230 '200 OK', ['Content-Length: 26', 'Cache-Control: no-store'], body) 231 232 elif path == '/chunked.txt': 233 body = ('7\r\nchunked\r\n' 234 '8\r\nencoding\r\n' 235 '2\r\nis\r\n' 236 '3\r\nfun\r\n' 237 '0\r\n\r\n') 238 result += self._BuildResponse( 239 '200 OK', ['Transfer-Encoding: chunked', 'Cache-Control: no-store'], 240 body) 241 242 elif path == '/cached.txt': 243 body = 'azbycxdwevfugthsirjqkplomn' 244 result += self._BuildResponse( 245 '200 OK', ['Content-Length: 26', 'Cache-Control: max-age=60'], body) 246 247 elif path == '/connection_close.txt': 248 body = 'azbycxdwevfugthsirjqkplomn' 249 result += self._BuildResponse( 250 '200 OK', ['Content-Length: 26', 'Cache-Control: max-age=60', 'Connection: close'], body) 251 self._processed_end = True 252 253 elif path == '/1k.txt': 254 str = '0123456789abcdef' 255 body = ''.join([str for num in xrange(64)]) 256 result += self._BuildResponse( 257 '200 OK', ['Server: Apache', 258 'Content-Length: 1024', 259 'Cache-Control: max-age=60'], body) 260 261 elif path == '/10k.txt': 262 str = '0123456789abcdef' 263 body = ''.join([str for num in xrange(640)]) 264 result += self._BuildResponse( 265 '200 OK', ['Server: Apache', 266 'Content-Length: 10240', 267 'Cache-Control: max-age=60'], body) 268 269 elif path == '/100k.txt': 270 str = '0123456789abcdef' 271 body = ''.join([str for num in xrange(6400)]) 272 result += self._BuildResponse( 273 '200 OK', 274 ['Server: Apache', 275 'Content-Length: 102400', 276 'Cache-Control: max-age=60'], 277 body) 278 279 elif path == '/100k_chunked.txt': 280 str = '0123456789abcdef' 281 moo = ''.join([str for num in xrange(6400)]) 282 body = self.Chunkify(moo, 20480) 283 body.append('0\r\n\r\n') 284 body = ''.join(body) 285 286 result += self._BuildResponse( 287 '200 OK', ['Transfer-Encoding: chunked', 'Cache-Control: no-store'], body) 288 289 elif path == '/stats.txt': 290 results = { 291 'max_pipeline_depth': self._max_pipeline_depth, 292 'were_all_requests_http_1_1': int(self._were_all_requests_http_1_1), 293 } 294 body = ','.join(['%s:%s' % (k, v) for k, v in results.items()]) 295 result += self._BuildResponse( 296 '200 OK', 297 ['Content-Length: %s' % len(body), 'Cache-Control: no-store'], body) 298 self._processed_end = True 299 300 else: 301 result += self._BuildResponse('404 Not Found', ['Content-Length: 7'], 'Go away') 302 if self._processed_end: 303 break 304 self._requested_paths = [] 305 return result 306 307 def WriteError(self, status, error): 308 """Returns an HTTP response for the specified error. 309 310 Args: 311 status: (String) Response code and descrtion (e.g. "404 Not Found") 312 313 Returns: 314 (String) Text of HTTP response. 315 """ 316 return self._BuildResponse( 317 status, ['Connection: close', 'Content-Type: text/plain'], error) 318 319 @property 320 def processed_end(self): 321 return self._processed_end 322 323 def _BuildResponse(self, status, headers, body): 324 """Builds an HTTP response. 325 326 Args: 327 status: (String) Response code and descrtion (e.g. "200 OK") 328 headers: (List of Strings) Headers (e.g. "Connection: close") 329 body: (String) Response body. 330 331 Returns: 332 (String) Text of HTTP response. 333 """ 334 return ('HTTP/1.1 %s\r\n' 335 '%s\r\n' 336 '\r\n' 337 '%s' % (status, '\r\n'.join(headers), body)) 338 339 340class PipelineRequestHandler(SocketServer.BaseRequestHandler): 341 """Called on an incoming TCP connection.""" 342 343 def _GetTimeUntilTimeout(self): 344 return self._start_time + TIMEOUT - time.time() 345 346 def _GetTimeUntilNextSend(self): 347 if not self._last_queued_time: 348 return TIMEOUT 349 return self._last_queued_time + SEND_BUFFER_TIME - time.time() 350 351 def handle(self): 352 self._request_parser = RequestParser() 353 self._response_builder = ResponseBuilder() 354 self._last_queued_time = 0 355 self._num_queued = 0 356 self._num_written = 0 357 self._send_buffer = "" 358 self._start_time = time.time() 359 try: 360 while not self._response_builder.processed_end or self._send_buffer: 361 362 time_left = self._GetTimeUntilTimeout() 363 time_until_next_send = self._GetTimeUntilNextSend() 364 max_poll_time = min(time_left, time_until_next_send) + MIN_POLL_TIME 365 366 rlist, wlist, xlist = [], [], [] 367 fileno = self.request.fileno() 368 if max_poll_time > 0: 369 rlist.append(fileno) 370 if self._send_buffer: 371 wlist.append(fileno) 372 rlist, wlist, xlist = select.select(rlist, wlist, xlist, max_poll_time) 373 374 if self._GetTimeUntilTimeout() <= 0: 375 return 376 377 if self._GetTimeUntilNextSend() <= 0: 378 self._send_buffer += self._response_builder.BuildResponses() 379 self._num_written = self._num_queued 380 self._last_queued_time = 0 381 382 if fileno in rlist: 383 self.request.setblocking(False) 384 new_data = self.request.recv(MAX_REQUEST_SIZE) 385 self.request.setblocking(True) 386 if not new_data: 387 return 388 new_requests = self._request_parser.ParseAdditionalData(new_data) 389 self._response_builder.QueueRequests( 390 new_requests, self._request_parser.were_all_requests_http_1_1) 391 self._num_queued += len(new_requests) 392 self._last_queued_time = time.time() 393 elif fileno in wlist: 394 num_bytes_sent = self.request.send(self._send_buffer[0:4096]) 395 self._send_buffer = self._send_buffer[num_bytes_sent:] 396 time.sleep(0.05) 397 398 except RequestTooLargeError as e: 399 self.request.send(self._response_builder.WriteError( 400 '413 Request Entity Too Large', e)) 401 raise 402 except UnexpectedMethodError as e: 403 self.request.send(self._response_builder.WriteError( 404 '405 Method Not Allowed', e)) 405 raise 406 except ServeIndexError: 407 self.request.send(self._response_builder.WriteError( 408 '200 OK', INFO_MESSAGE)) 409 except Exception as e: 410 print e 411 self.request.close() 412 413 414class PipelineServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): 415 pass 416 417 418parser = argparse.ArgumentParser() 419parser.add_argument("--port", action="store", default=0, 420 type=int, help="port to listen on") 421parser.add_argument("--verbose", action="store", default=0, 422 type=int, help="verbose output") 423parser.add_argument("--pidfile", action="store", default=0, 424 help="file name for the PID") 425parser.add_argument("--logfile", action="store", default=0, 426 help="file name for the log") 427parser.add_argument("--srcdir", action="store", default=0, 428 help="test directory") 429parser.add_argument("--id", action="store", default=0, 430 help="server ID") 431parser.add_argument("--ipv4", action="store_true", default=0, 432 help="IPv4 flag") 433args = parser.parse_args() 434 435if args.pidfile: 436 pid = os.getpid() 437 f = open(args.pidfile, 'w') 438 f.write('{}'.format(pid)) 439 f.close() 440 441server = PipelineServer(('0.0.0.0', args.port), PipelineRequestHandler) 442server.allow_reuse_address = True 443server.serve_forever() 444