1/* Licensed to the Apache Software Foundation (ASF) under one or more 2 * contributor license agreements. See the NOTICE file distributed with 3 * this work for additional information regarding copyright ownership. 4 * The ASF licenses this file to You under the Apache License, Version 2.0 5 * (the "License"); you may not use this file except in compliance with 6 * the License. You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#ifdef WIN32 18 19#include "apr.h" 20#include <process.h> 21#include "httpd.h" 22#include "http_main.h" 23#include "http_log.h" 24#include "http_config.h" /* for read_config */ 25#include "http_core.h" /* for get_remote_host */ 26#include "http_connection.h" 27#include "http_vhost.h" /* for ap_update_vhost_given_ip */ 28#include "apr_portable.h" 29#include "apr_thread_proc.h" 30#include "apr_getopt.h" 31#include "apr_strings.h" 32#include "apr_lib.h" 33#include "apr_shm.h" 34#include "apr_thread_mutex.h" 35#include "ap_mpm.h" 36#include "ap_config.h" 37#include "ap_listen.h" 38#include "mpm_default.h" 39#include "mpm_winnt.h" 40#include "mpm_common.h" 41#include <malloc.h> 42#include "apr_atomic.h" 43#include "apr_buckets.h" 44#include "scoreboard.h" 45 46#ifdef __MINGW32__ 47#include <mswsock.h> 48 49#ifndef WSAID_ACCEPTEX 50#define WSAID_ACCEPTEX \ 51 {0xb5367df1, 0xcbac, 0x11cf, {0x95, 0xca, 0x00, 0x80, 0x5f, 0x48, 0xa1, 0x92}} 52typedef BOOL (WINAPI *LPFN_ACCEPTEX)(SOCKET, SOCKET, PVOID, DWORD, DWORD, DWORD, LPDWORD, LPOVERLAPPED); 53#endif /* WSAID_ACCEPTEX */ 54 55#ifndef WSAID_GETACCEPTEXSOCKADDRS 56#define WSAID_GETACCEPTEXSOCKADDRS \ 57 {0xb5367df2, 0xcbac, 0x11cf, {0x95, 0xca, 0x00, 0x80, 0x5f, 0x48, 0xa1, 0x92}} 58typedef VOID (WINAPI *LPFN_GETACCEPTEXSOCKADDRS)(PVOID, DWORD, DWORD, DWORD, 59 struct sockaddr **, LPINT, 60 struct sockaddr **, LPINT); 61#endif /* WSAID_GETACCEPTEXSOCKADDRS */ 62 63#endif /* __MINGW32__ */ 64 65/* 66 * The Windows MPM uses a queue of completion contexts that it passes 67 * between the accept threads and the worker threads. Declare the 68 * functions to access the queue and the structures passed on the 69 * queue in the header file to enable modules to access them 70 * if necessary. The queue resides in the MPM. 71 */ 72#ifdef CONTAINING_RECORD 73#undef CONTAINING_RECORD 74#endif 75#define CONTAINING_RECORD(address, type, field) ((type *)( \ 76 (char *)(address) - \ 77 (char *)(&((type *)0)->field))) 78#if APR_HAVE_IPV6 79#define PADDED_ADDR_SIZE (sizeof(SOCKADDR_IN6)+16) 80#else 81#define PADDED_ADDR_SIZE (sizeof(SOCKADDR_IN)+16) 82#endif 83 84APLOG_USE_MODULE(mpm_winnt); 85 86/* Queue for managing the passing of winnt_conn_ctx_t between 87 * the accept and worker threads. 88 */ 89typedef struct winnt_conn_ctx_t_s { 90 struct winnt_conn_ctx_t_s *next; 91 OVERLAPPED overlapped; 92 apr_socket_t *sock; 93 SOCKET accept_socket; 94 char buff[2*PADDED_ADDR_SIZE]; 95 struct sockaddr *sa_server; 96 int sa_server_len; 97 struct sockaddr *sa_client; 98 int sa_client_len; 99 apr_pool_t *ptrans; 100 apr_bucket_alloc_t *ba; 101 apr_bucket *data; 102#if APR_HAVE_IPV6 103 short socket_family; 104#endif 105} winnt_conn_ctx_t; 106 107typedef enum { 108 IOCP_CONNECTION_ACCEPTED = 1, 109 IOCP_WAIT_FOR_RECEIVE = 2, 110 IOCP_WAIT_FOR_TRANSMITFILE = 3, 111 IOCP_SHUTDOWN = 4 112} io_state_e; 113 114static apr_pool_t *pchild; 115static int shutdown_in_progress = 0; 116static int workers_may_exit = 0; 117static unsigned int g_blocked_threads = 0; 118static HANDLE max_requests_per_child_event; 119 120static apr_thread_mutex_t *child_lock; 121static apr_thread_mutex_t *qlock; 122static winnt_conn_ctx_t *qhead = NULL; 123static winnt_conn_ctx_t *qtail = NULL; 124static apr_uint32_t num_completion_contexts = 0; 125static apr_uint32_t max_num_completion_contexts = 0; 126static HANDLE ThreadDispatchIOCP = NULL; 127static HANDLE qwait_event = NULL; 128 129static void mpm_recycle_completion_context(winnt_conn_ctx_t *context) 130{ 131 /* Recycle the completion context. 132 * - clear the ptrans pool 133 * - put the context on the queue to be consumed by the accept thread 134 * Note: 135 * context->accept_socket may be in a disconnected but reusable 136 * state so -don't- close it. 137 */ 138 if (context) { 139 apr_pool_clear(context->ptrans); 140 context->ba = apr_bucket_alloc_create(context->ptrans); 141 context->next = NULL; 142 ResetEvent(context->overlapped.hEvent); 143 apr_thread_mutex_lock(qlock); 144 if (qtail) { 145 qtail->next = context; 146 } else { 147 qhead = context; 148 SetEvent(qwait_event); 149 } 150 qtail = context; 151 apr_thread_mutex_unlock(qlock); 152 } 153} 154 155static winnt_conn_ctx_t *mpm_get_completion_context(int *timeout) 156{ 157 apr_status_t rv; 158 winnt_conn_ctx_t *context = NULL; 159 160 *timeout = 0; 161 while (1) { 162 /* Grab a context off the queue */ 163 apr_thread_mutex_lock(qlock); 164 if (qhead) { 165 context = qhead; 166 qhead = qhead->next; 167 if (!qhead) 168 qtail = NULL; 169 } else { 170 ResetEvent(qwait_event); 171 } 172 apr_thread_mutex_unlock(qlock); 173 174 if (!context) { 175 /* We failed to grab a context off the queue, consider allocating 176 * a new one out of the child pool. There may be up to 177 * (ap_threads_per_child + num_listeners) contexts in the system 178 * at once. 179 */ 180 if (num_completion_contexts >= max_num_completion_contexts) { 181 /* All workers are busy, need to wait for one */ 182 static int reported = 0; 183 if (!reported) { 184 ap_log_error(APLOG_MARK, APLOG_ERR, 0, ap_server_conf, APLOGNO(00326) 185 "Server ran out of threads to serve " 186 "requests. Consider raising the " 187 "ThreadsPerChild setting"); 188 reported = 1; 189 } 190 191 /* Wait for a worker to free a context. Once per second, give 192 * the caller a chance to check for shutdown. If the wait 193 * succeeds, get the context off the queue. It must be 194 * available, since there's only one consumer. 195 */ 196 rv = WaitForSingleObject(qwait_event, 1000); 197 if (rv == WAIT_OBJECT_0) 198 continue; 199 else { 200 if (rv == WAIT_TIMEOUT) { 201 /* somewhat-normal condition where threads are busy */ 202 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(00327) 203 "mpm_get_completion_context: Failed to get a " 204 "free context within 1 second"); 205 *timeout = 1; 206 } 207 else { 208 /* should be the unexpected, generic WAIT_FAILED */ 209 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_os_error(), 210 ap_server_conf, APLOGNO(00328) 211 "mpm_get_completion_context: " 212 "WaitForSingleObject failed to get free context"); 213 } 214 return NULL; 215 } 216 } else { 217 /* Allocate another context. 218 * Note: Multiple failures in the next two steps will cause 219 * the pchild pool to 'leak' storage. I don't think this 220 * is worth fixing... 221 */ 222 apr_allocator_t *allocator; 223 224 apr_thread_mutex_lock(child_lock); 225 context = (winnt_conn_ctx_t *)apr_pcalloc(pchild, 226 sizeof(winnt_conn_ctx_t)); 227 228 229 context->overlapped.hEvent = CreateEvent(NULL, TRUE, 230 FALSE, NULL); 231 if (context->overlapped.hEvent == NULL) { 232 /* Hopefully this is a temporary condition ... */ 233 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_os_error(), 234 ap_server_conf, APLOGNO(00329) 235 "mpm_get_completion_context: " 236 "CreateEvent failed."); 237 238 apr_thread_mutex_unlock(child_lock); 239 return NULL; 240 } 241 242 /* Create the transaction pool */ 243 apr_allocator_create(&allocator); 244 apr_allocator_max_free_set(allocator, ap_max_mem_free); 245 rv = apr_pool_create_ex(&context->ptrans, pchild, NULL, 246 allocator); 247 if (rv != APR_SUCCESS) { 248 ap_log_error(APLOG_MARK, APLOG_WARNING, rv, ap_server_conf, APLOGNO(00330) 249 "mpm_get_completion_context: Failed " 250 "to create the transaction pool."); 251 CloseHandle(context->overlapped.hEvent); 252 253 apr_thread_mutex_unlock(child_lock); 254 return NULL; 255 } 256 apr_allocator_owner_set(allocator, context->ptrans); 257 apr_pool_tag(context->ptrans, "transaction"); 258 259 context->accept_socket = INVALID_SOCKET; 260 context->ba = apr_bucket_alloc_create(context->ptrans); 261 apr_atomic_inc32(&num_completion_contexts); 262 263 apr_thread_mutex_unlock(child_lock); 264 break; 265 } 266 } else { 267 /* Got a context from the queue */ 268 break; 269 } 270 } 271 272 return context; 273} 274 275 276/* Windows NT/2000 specific code... 277 * Accept processing for on Windows NT uses a producer/consumer queue 278 * model. An accept thread accepts connections off the network then issues 279 * PostQueuedCompletionStatus() to awake a thread blocked on the ThreadDispatch 280 * IOCompletionPort. 281 * 282 * winnt_accept() 283 * One or more accept threads run in this function, each of which accepts 284 * connections off the network and calls PostQueuedCompletionStatus() to 285 * queue an io completion packet to the ThreadDispatch IOCompletionPort. 286 * winnt_get_connection() 287 * Worker threads block on the ThreadDispatch IOCompletionPort awaiting 288 * connections to service. 289 */ 290#define MAX_ACCEPTEX_ERR_COUNT 10 291 292static unsigned int __stdcall winnt_accept(void *lr_) 293{ 294 ap_listen_rec *lr = (ap_listen_rec *)lr_; 295 apr_os_sock_info_t sockinfo; 296 winnt_conn_ctx_t *context = NULL; 297 DWORD BytesRead; 298 SOCKET nlsd; 299 LPFN_ACCEPTEX lpfnAcceptEx = NULL; 300 LPFN_GETACCEPTEXSOCKADDRS lpfnGetAcceptExSockaddrs = NULL; 301 GUID GuidAcceptEx = WSAID_ACCEPTEX; 302 GUID GuidGetAcceptExSockaddrs = WSAID_GETACCEPTEXSOCKADDRS; 303 core_server_config *core_sconf; 304 const char *accf_name; 305 int rv; 306 int accf; 307 int err_count = 0; 308 HANDLE events[3]; 309#if APR_HAVE_IPV6 310 SOCKADDR_STORAGE ss_listen; 311 int namelen = sizeof(ss_listen); 312#endif 313 u_long zero = 0; 314 315 core_sconf = ap_get_core_module_config(ap_server_conf->module_config); 316 accf_name = apr_table_get(core_sconf->accf_map, lr->protocol); 317 318 if (!accf_name) { 319 accf = 0; 320 accf_name = "none"; 321 ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, 322 APLOGNO(02531) "winnt_accept: Listen protocol '%s' has " 323 "no known accept filter. Using 'none' instead", 324 lr->protocol); 325 } 326 else if (strcmp(accf_name, "data") == 0) 327 accf = 2; 328 else if (strcmp(accf_name, "connect") == 0) 329 accf = 1; 330 else if (strcmp(accf_name, "none") == 0) 331 accf = 0; 332 else { 333 accf = 0; 334 accf_name = "none"; 335 ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, APLOGNO(00331) 336 "winnt_accept: unrecognized AcceptFilter '%s', " 337 "only 'data', 'connect' or 'none' are valid. " 338 "Using 'none' instead", accf_name); 339 } 340 341 apr_os_sock_get(&nlsd, lr->sd); 342 343#if APR_HAVE_IPV6 344 if (getsockname(nlsd, (struct sockaddr *)&ss_listen, &namelen) == SOCKET_ERROR) { 345 ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_netos_error(), 346 ap_server_conf, APLOGNO(00332) 347 "winnt_accept: getsockname error on listening socket, " 348 "is IPv6 available?"); 349 return 1; 350 } 351#endif 352 353 if (accf > 0) /* 'data' or 'connect' */ 354 { 355 if (WSAIoctl(nlsd, SIO_GET_EXTENSION_FUNCTION_POINTER, 356 &GuidAcceptEx, sizeof GuidAcceptEx, 357 &lpfnAcceptEx, sizeof lpfnAcceptEx, 358 &BytesRead, NULL, NULL) == SOCKET_ERROR) { 359 ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_netos_error(), 360 ap_server_conf, APLOGNO(02322) 361 "winnt_accept: failed to retrieve AcceptEx, try 'AcceptFilter none'"); 362 return 1; 363 } 364 if (WSAIoctl(nlsd, SIO_GET_EXTENSION_FUNCTION_POINTER, 365 &GuidGetAcceptExSockaddrs, sizeof GuidGetAcceptExSockaddrs, 366 &lpfnGetAcceptExSockaddrs, sizeof lpfnGetAcceptExSockaddrs, 367 &BytesRead, NULL, NULL) == SOCKET_ERROR) { 368 ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_netos_error(), 369 ap_server_conf, APLOGNO(02323) 370 "winnt_accept: failed to retrieve GetAcceptExSockaddrs, try 'AcceptFilter none'"); 371 return 1; 372 } 373 /* first, high priority event is an already accepted connection */ 374 events[1] = exit_event; 375 events[2] = max_requests_per_child_event; 376 } 377 else /* accf == 0, 'none' */ 378 { 379reinit: /* target of data or connect upon too many AcceptEx failures */ 380 381 /* last, low priority event is a not yet accepted connection */ 382 events[0] = exit_event; 383 events[1] = max_requests_per_child_event; 384 events[2] = CreateEvent(NULL, FALSE, FALSE, NULL); 385 386 /* The event needs to be removed from the accepted socket, 387 * if not removed from the listen socket prior to accept(), 388 */ 389 rv = WSAEventSelect(nlsd, events[2], FD_ACCEPT); 390 if (rv) { 391 ap_log_error(APLOG_MARK, APLOG_ERR, 392 apr_get_netos_error(), ap_server_conf, APLOGNO(00333) 393 "WSAEventSelect() failed."); 394 CloseHandle(events[2]); 395 return 1; 396 } 397 } 398 399 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(00334) 400 "Child: Accept thread listening on %pI using AcceptFilter %s", 401 lr->bind_addr, accf_name); 402 403 while (!shutdown_in_progress) { 404 if (!context) { 405 int timeout; 406 407 context = mpm_get_completion_context(&timeout); 408 if (!context) { 409 if (!timeout) { 410 /* Hopefully a temporary condition in the provider? */ 411 ++err_count; 412 if (err_count > MAX_ACCEPTEX_ERR_COUNT) { 413 ap_log_error(APLOG_MARK, APLOG_CRIT, 0, ap_server_conf, APLOGNO(00335) 414 "winnt_accept: Too many failures grabbing a " 415 "connection ctx. Aborting."); 416 break; 417 } 418 } 419 Sleep(100); 420 continue; 421 } 422 } 423 424 if (accf > 0) /* Either 'connect' or 'data' */ 425 { 426 DWORD len; 427 char *buf; 428 429 /* Create and initialize the accept socket */ 430#if APR_HAVE_IPV6 431 if (context->accept_socket == INVALID_SOCKET) { 432 context->accept_socket = socket(ss_listen.ss_family, SOCK_STREAM, 433 IPPROTO_TCP); 434 context->socket_family = ss_listen.ss_family; 435 } 436 else if (context->socket_family != ss_listen.ss_family) { 437 closesocket(context->accept_socket); 438 context->accept_socket = socket(ss_listen.ss_family, SOCK_STREAM, 439 IPPROTO_TCP); 440 context->socket_family = ss_listen.ss_family; 441 } 442#else 443 if (context->accept_socket == INVALID_SOCKET) 444 context->accept_socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); 445#endif 446 447 if (context->accept_socket == INVALID_SOCKET) { 448 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(), 449 ap_server_conf, APLOGNO(00336) 450 "winnt_accept: Failed to allocate an accept socket. " 451 "Temporary resource constraint? Try again."); 452 Sleep(100); 453 continue; 454 } 455 456 if (accf == 2) { /* 'data' */ 457 len = APR_BUCKET_BUFF_SIZE; 458 buf = apr_bucket_alloc(len, context->ba); 459 len -= PADDED_ADDR_SIZE * 2; 460 } 461 else /* (accf == 1) 'connect' */ { 462 len = 0; 463 buf = context->buff; 464 } 465 466 /* AcceptEx on the completion context. The completion context will be 467 * signaled when a connection is accepted. 468 */ 469 if (!lpfnAcceptEx(nlsd, context->accept_socket, buf, len, 470 PADDED_ADDR_SIZE, PADDED_ADDR_SIZE, &BytesRead, 471 &context->overlapped)) { 472 rv = apr_get_netos_error(); 473 if ((rv == APR_FROM_OS_ERROR(WSAECONNRESET)) || 474 (rv == APR_FROM_OS_ERROR(WSAEACCES))) { 475 /* We can get here when: 476 * 1) the client disconnects early 477 * 2) handshake was incomplete 478 */ 479 if (accf == 2) 480 apr_bucket_free(buf); 481 closesocket(context->accept_socket); 482 context->accept_socket = INVALID_SOCKET; 483 continue; 484 } 485 else if ((rv == APR_FROM_OS_ERROR(WSAEINVAL)) || 486 (rv == APR_FROM_OS_ERROR(WSAENOTSOCK))) { 487 /* We can get here when: 488 * 1) TransmitFile does not properly recycle the accept socket (typically 489 * because the client disconnected) 490 * 2) there is VPN or Firewall software installed with 491 * buggy WSAAccept or WSADuplicateSocket implementation 492 * 3) the dynamic address / adapter has changed 493 * Give five chances, then fall back on AcceptFilter 'none' 494 */ 495 if (accf == 2) 496 apr_bucket_free(buf); 497 closesocket(context->accept_socket); 498 context->accept_socket = INVALID_SOCKET; 499 ++err_count; 500 if (err_count > MAX_ACCEPTEX_ERR_COUNT) { 501 ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(00337) 502 "Child: Encountered too many AcceptEx " 503 "faults accepting client connections. " 504 "Possible causes: dynamic address renewal, " 505 "or incompatible VPN or firewall software. "); 506 ap_log_error(APLOG_MARK, APLOG_NOTICE, rv, ap_server_conf, APLOGNO(00338) 507 "winnt_mpm: falling back to " 508 "'AcceptFilter none'."); 509 err_count = 0; 510 accf = 0; 511 } 512 continue; 513 } 514 else if ((rv != APR_FROM_OS_ERROR(ERROR_IO_PENDING)) && 515 (rv != APR_FROM_OS_ERROR(WSA_IO_PENDING))) { 516 if (accf == 2) 517 apr_bucket_free(buf); 518 closesocket(context->accept_socket); 519 context->accept_socket = INVALID_SOCKET; 520 ++err_count; 521 if (err_count > MAX_ACCEPTEX_ERR_COUNT) { 522 ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(00339) 523 "Child: Encountered too many AcceptEx " 524 "faults accepting client connections."); 525 ap_log_error(APLOG_MARK, APLOG_NOTICE, rv, ap_server_conf, APLOGNO(00340) 526 "winnt_mpm: falling back to " 527 "'AcceptFilter none'."); 528 err_count = 0; 529 accf = 0; 530 goto reinit; 531 } 532 continue; 533 } 534 535 err_count = 0; 536 events[0] = context->overlapped.hEvent; 537 538 do { 539 rv = WaitForMultipleObjectsEx(3, events, FALSE, INFINITE, TRUE); 540 } while (rv == WAIT_IO_COMPLETION); 541 542 if (rv == WAIT_OBJECT_0) { 543 if ((context->accept_socket != INVALID_SOCKET) && 544 !GetOverlappedResult((HANDLE)context->accept_socket, 545 &context->overlapped, 546 &BytesRead, FALSE)) { 547 ap_log_error(APLOG_MARK, APLOG_WARNING, 548 apr_get_os_error(), ap_server_conf, APLOGNO(00341) 549 "winnt_accept: Asynchronous AcceptEx failed."); 550 closesocket(context->accept_socket); 551 context->accept_socket = INVALID_SOCKET; 552 } 553 } 554 else { 555 /* exit_event triggered or event handle was closed */ 556 closesocket(context->accept_socket); 557 context->accept_socket = INVALID_SOCKET; 558 if (accf == 2) 559 apr_bucket_free(buf); 560 break; 561 } 562 563 if (context->accept_socket == INVALID_SOCKET) { 564 if (accf == 2) 565 apr_bucket_free(buf); 566 continue; 567 } 568 } 569 err_count = 0; 570 571 /* Potential optimization; consider handing off to the worker */ 572 573 /* Inherit the listen socket settings. Required for 574 * shutdown() to work 575 */ 576 if (setsockopt(context->accept_socket, SOL_SOCKET, 577 SO_UPDATE_ACCEPT_CONTEXT, (char *)&nlsd, 578 sizeof(nlsd))) { 579 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(), 580 ap_server_conf, APLOGNO(00342) 581 "setsockopt(SO_UPDATE_ACCEPT_CONTEXT) failed."); 582 /* Not a failure condition. Keep running. */ 583 } 584 585 /* Get the local & remote address 586 * TODO; error check 587 */ 588 lpfnGetAcceptExSockaddrs(buf, len, PADDED_ADDR_SIZE, PADDED_ADDR_SIZE, 589 &context->sa_server, &context->sa_server_len, 590 &context->sa_client, &context->sa_client_len); 591 592 /* For 'data', craft a bucket for our data result 593 * and pass to worker_main as context->overlapped.Pointer 594 */ 595 if (accf == 2 && BytesRead) 596 { 597 apr_bucket *b; 598 b = apr_bucket_heap_create(buf, APR_BUCKET_BUFF_SIZE, 599 apr_bucket_free, context->ba); 600 /* Adjust the bucket to refer to the actual bytes read */ 601 b->length = BytesRead; 602 context->overlapped.Pointer = b; 603 } 604 else 605 context->overlapped.Pointer = NULL; 606 } 607 else /* (accf = 0) e.g. 'none' */ 608 { 609 /* There is no socket reuse without AcceptEx() */ 610 if (context->accept_socket != INVALID_SOCKET) 611 closesocket(context->accept_socket); 612 613 /* This could be a persistent event per-listener rather than 614 * per-accept. However, the event needs to be removed from 615 * the target socket if not removed from the listen socket 616 * prior to accept(), or the event select is inherited. 617 * and must be removed from the accepted socket. 618 */ 619 620 do { 621 rv = WaitForMultipleObjectsEx(3, events, FALSE, INFINITE, TRUE); 622 } while (rv == WAIT_IO_COMPLETION); 623 624 625 if (rv != WAIT_OBJECT_0 + 2) { 626 /* not FD_ACCEPT; 627 * exit_event triggered or event handle was closed 628 */ 629 break; 630 } 631 632 context->sa_server = (void *) context->buff; 633 context->sa_server_len = sizeof(context->buff) / 2; 634 context->sa_client_len = context->sa_server_len; 635 context->sa_client = (void *) (context->buff 636 + context->sa_server_len); 637 638 context->accept_socket = accept(nlsd, context->sa_server, 639 &context->sa_server_len); 640 641 if (context->accept_socket == INVALID_SOCKET) { 642 643 rv = apr_get_netos_error(); 644 if ( rv == APR_FROM_OS_ERROR(WSAECONNRESET) 645 || rv == APR_FROM_OS_ERROR(WSAEINPROGRESS) 646 || rv == APR_FROM_OS_ERROR(WSAEWOULDBLOCK) ) { 647 ap_log_error(APLOG_MARK, APLOG_DEBUG, 648 rv, ap_server_conf, APLOGNO(00343) 649 "accept() failed, retrying."); 650 continue; 651 } 652 653 /* A more serious error than 'retry', log it */ 654 ap_log_error(APLOG_MARK, APLOG_WARNING, 655 rv, ap_server_conf, APLOGNO(00344) 656 "accept() failed."); 657 658 if ( rv == APR_FROM_OS_ERROR(WSAEMFILE) 659 || rv == APR_FROM_OS_ERROR(WSAENOBUFS) ) { 660 /* Hopefully a temporary condition in the provider? */ 661 Sleep(100); 662 ++err_count; 663 if (err_count > MAX_ACCEPTEX_ERR_COUNT) { 664 ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(00345) 665 "Child: Encountered too many accept() " 666 "resource faults, aborting."); 667 break; 668 } 669 continue; 670 } 671 break; 672 } 673 /* Per MSDN, cancel the inherited association of this socket 674 * to the WSAEventSelect API, and restore the state corresponding 675 * to apr_os_sock_make's default assumptions (really, a flaw within 676 * os_sock_make and os_sock_put that it does not query). 677 */ 678 WSAEventSelect(context->accept_socket, 0, 0); 679 context->overlapped.Pointer = NULL; 680 err_count = 0; 681 682 context->sa_server_len = sizeof(context->buff) / 2; 683 if (getsockname(context->accept_socket, context->sa_server, 684 &context->sa_server_len) == SOCKET_ERROR) { 685 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(), ap_server_conf, APLOGNO(00346) 686 "getsockname failed"); 687 continue; 688 } 689 if ((getpeername(context->accept_socket, context->sa_client, 690 &context->sa_client_len)) == SOCKET_ERROR) { 691 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(), ap_server_conf, APLOGNO(00347) 692 "getpeername failed"); 693 memset(&context->sa_client, '\0', sizeof(context->sa_client)); 694 } 695 } 696 697 sockinfo.os_sock = &context->accept_socket; 698 sockinfo.local = context->sa_server; 699 sockinfo.remote = context->sa_client; 700 sockinfo.family = context->sa_server->sa_family; 701 sockinfo.type = SOCK_STREAM; 702 sockinfo.protocol = IPPROTO_TCP; 703 /* Restore the state corresponding to apr_os_sock_make's default 704 * assumption of timeout -1 (really, a flaw of os_sock_make and 705 * os_sock_put that it does not query to determine ->timeout). 706 * XXX: Upon a fix to APR, these three statements should disappear. 707 */ 708 ioctlsocket(context->accept_socket, FIONBIO, &zero); 709 setsockopt(context->accept_socket, SOL_SOCKET, SO_RCVTIMEO, 710 (char *) &zero, sizeof(zero)); 711 setsockopt(context->accept_socket, SOL_SOCKET, SO_SNDTIMEO, 712 (char *) &zero, sizeof(zero)); 713 apr_os_sock_make(&context->sock, &sockinfo, context->ptrans); 714 715 /* When a connection is received, send an io completion notification 716 * to the ThreadDispatchIOCP. 717 */ 718 PostQueuedCompletionStatus(ThreadDispatchIOCP, BytesRead, 719 IOCP_CONNECTION_ACCEPTED, 720 &context->overlapped); 721 context = NULL; 722 } 723 if (!accf) 724 CloseHandle(events[2]); 725 726 if (!shutdown_in_progress) { 727 /* Yow, hit an irrecoverable error! Tell the child to die. */ 728 SetEvent(exit_event); 729 } 730 731 ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf, APLOGNO(00348) 732 "Child: Accept thread exiting."); 733 return 0; 734} 735 736 737static winnt_conn_ctx_t *winnt_get_connection(winnt_conn_ctx_t *context) 738{ 739 int rc; 740 DWORD BytesRead; 741 LPOVERLAPPED pol; 742#ifdef _WIN64 743 ULONG_PTR CompKey; 744#else 745 DWORD CompKey; 746#endif 747 748 mpm_recycle_completion_context(context); 749 750 apr_atomic_inc32(&g_blocked_threads); 751 while (1) { 752 if (workers_may_exit) { 753 apr_atomic_dec32(&g_blocked_threads); 754 return NULL; 755 } 756 rc = GetQueuedCompletionStatus(ThreadDispatchIOCP, &BytesRead, 757 &CompKey, &pol, INFINITE); 758 if (!rc) { 759 rc = apr_get_os_error(); 760 ap_log_error(APLOG_MARK, APLOG_DEBUG, rc, ap_server_conf, APLOGNO(00349) 761 "Child: GetQueuedComplationStatus returned %d", 762 rc); 763 continue; 764 } 765 766 switch (CompKey) { 767 case IOCP_CONNECTION_ACCEPTED: 768 context = CONTAINING_RECORD(pol, winnt_conn_ctx_t, overlapped); 769 break; 770 case IOCP_SHUTDOWN: 771 apr_atomic_dec32(&g_blocked_threads); 772 return NULL; 773 default: 774 apr_atomic_dec32(&g_blocked_threads); 775 return NULL; 776 } 777 break; 778 } 779 apr_atomic_dec32(&g_blocked_threads); 780 781 return context; 782} 783 784apr_status_t winnt_insert_network_bucket(conn_rec *c, 785 apr_bucket_brigade *bb, 786 apr_socket_t *socket) 787{ 788 apr_bucket *e; 789 winnt_conn_ctx_t *context = ap_get_module_config(c->conn_config, 790 &mpm_winnt_module); 791 if (context == NULL || (e = context->overlapped.Pointer) == NULL) 792 return AP_DECLINED; 793 794 /* seed the brigade with AcceptEx read heap bucket */ 795 APR_BRIGADE_INSERT_HEAD(bb, e); 796 /* also seed the brigade with the client socket. */ 797 e = apr_bucket_socket_create(socket, c->bucket_alloc); 798 APR_BRIGADE_INSERT_TAIL(bb, e); 799 return APR_SUCCESS; 800} 801 802/* 803 * worker_main() 804 * Main entry point for the worker threads. Worker threads block in 805 * win*_get_connection() awaiting a connection to service. 806 */ 807static DWORD __stdcall worker_main(void *thread_num_val) 808{ 809 apr_thread_t *thd; 810 apr_os_thread_t osthd; 811 static int requests_this_child = 0; 812 winnt_conn_ctx_t *context = NULL; 813 int thread_num = (int)thread_num_val; 814 ap_sb_handle_t *sbh; 815 apr_bucket *e; 816 int rc; 817 conn_rec *c; 818 apr_int32_t disconnected; 819 820 osthd = apr_os_thread_current(); 821 822 while (1) { 823 824 ap_update_child_status_from_indexes(0, thread_num, SERVER_READY, NULL); 825 826 /* Grab a connection off the network */ 827 context = winnt_get_connection(context); 828 829 if (!context) { 830 /* Time for the thread to exit */ 831 break; 832 } 833 834 /* Have we hit MaxConnectionsPerChild connections? */ 835 if (ap_max_requests_per_child) { 836 requests_this_child++; 837 if (requests_this_child > ap_max_requests_per_child) { 838 SetEvent(max_requests_per_child_event); 839 } 840 } 841 842 e = context->overlapped.Pointer; 843 844 ap_create_sb_handle(&sbh, context->ptrans, 0, thread_num); 845 c = ap_run_create_connection(context->ptrans, ap_server_conf, 846 context->sock, thread_num, sbh, 847 context->ba); 848 849 if (!c) { 850 /* ap_run_create_connection closes the socket on failure */ 851 context->accept_socket = INVALID_SOCKET; 852 if (e) { 853 apr_bucket_free(e); 854 } 855 continue; 856 } 857 858 thd = NULL; 859 apr_os_thread_put(&thd, &osthd, context->ptrans); 860 c->current_thread = thd; 861 862 /* follow ap_process_connection(c, context->sock) logic 863 * as it left us no chance to reinject our first data bucket. 864 */ 865 ap_update_vhost_given_ip(c); 866 867 rc = ap_run_pre_connection(c, context->sock); 868 if (rc != OK && rc != DONE) { 869 c->aborted = 1; 870 } 871 872 if (e && c->aborted) { 873 apr_bucket_free(e); 874 } 875 else { 876 ap_set_module_config(c->conn_config, &mpm_winnt_module, context); 877 } 878 879 if (!c->aborted) { 880 ap_run_process_connection(c); 881 } 882 883 apr_socket_opt_get(context->sock, APR_SO_DISCONNECTED, &disconnected); 884 885 if (!disconnected) { 886 context->accept_socket = INVALID_SOCKET; 887 if (!c->aborted) { 888 ap_lingering_close(c); 889 } 890 } 891 } 892 893 ap_update_child_status_from_indexes(0, thread_num, SERVER_DEAD, 894 (request_rec *) NULL); 895 896 return 0; 897} 898 899 900static void cleanup_thread(HANDLE *handles, int *thread_cnt, 901 int thread_to_clean) 902{ 903 int i; 904 905 CloseHandle(handles[thread_to_clean]); 906 for (i = thread_to_clean; i < ((*thread_cnt) - 1); i++) 907 handles[i] = handles[i + 1]; 908 (*thread_cnt)--; 909} 910 911 912/* 913 * child_main() 914 * Entry point for the main control thread for the child process. 915 * This thread creates the accept thread, worker threads and 916 * monitors the child process for maintenance and shutdown 917 * events. 918 */ 919static void create_listener_thread(void) 920{ 921 unsigned tid; 922 int num_listeners = 0; 923 /* Start an accept thread per listener 924 * XXX: Why would we have a NULL sd in our listeners? 925 */ 926 ap_listen_rec *lr; 927 928 /* Number of completion_contexts allowed in the system is 929 * (ap_threads_per_child + num_listeners). We need the additional 930 * completion contexts to prevent server hangs when ThreadsPerChild 931 * is configured to something less than or equal to the number 932 * of listeners. This is not a usual case, but people have 933 * encountered it. 934 */ 935 for (lr = ap_listeners; lr ; lr = lr->next) { 936 num_listeners++; 937 } 938 max_num_completion_contexts = ap_threads_per_child + num_listeners; 939 940 /* Now start a thread per listener */ 941 for (lr = ap_listeners; lr; lr = lr->next) { 942 if (lr->sd != NULL) { 943 /* A smaller stack is sufficient. 944 * To convert to CreateThread, the returned handle cannot be 945 * ignored, it must be closed/joined. 946 */ 947 _beginthreadex(NULL, 65536, winnt_accept, 948 (void *) lr, stack_res_flag, &tid); 949 } 950 } 951} 952 953 954void child_main(apr_pool_t *pconf, DWORD parent_pid) 955{ 956 apr_status_t status; 957 apr_hash_t *ht; 958 ap_listen_rec *lr; 959 HANDLE child_events[3]; 960 HANDLE *child_handles; 961 int listener_started = 0; 962 int threads_created = 0; 963 int watch_thread; 964 int time_remains; 965 int cld; 966 DWORD tid; 967 int rv; 968 int i; 969 int num_events; 970 971 apr_pool_create(&pchild, pconf); 972 apr_pool_tag(pchild, "pchild"); 973 974 ap_run_child_init(pchild, ap_server_conf); 975 ht = apr_hash_make(pchild); 976 977 /* Initialize the child_events */ 978 max_requests_per_child_event = CreateEvent(NULL, TRUE, FALSE, NULL); 979 if (!max_requests_per_child_event) { 980 ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(), ap_server_conf, APLOGNO(00350) 981 "Child: Failed to create a max_requests event."); 982 exit(APEXIT_CHILDINIT); 983 } 984 child_events[0] = exit_event; 985 child_events[1] = max_requests_per_child_event; 986 987 if (parent_pid != my_pid) { 988 child_events[2] = OpenProcess(SYNCHRONIZE, FALSE, parent_pid); 989 num_events = 3; 990 } 991 else { 992 /* presumably -DONE_PROCESS */ 993 child_events[2] = NULL; 994 num_events = 2; 995 } 996 997 /* 998 * Wait until we have permission to start accepting connections. 999 * start_mutex is used to ensure that only one child ever 1000 * goes into the listen/accept loop at once. 1001 */ 1002 status = apr_proc_mutex_lock(start_mutex); 1003 if (status != APR_SUCCESS) { 1004 ap_log_error(APLOG_MARK, APLOG_ERR, status, ap_server_conf, APLOGNO(00351) 1005 "Child: Failed to acquire the start_mutex. " 1006 "Process will exit."); 1007 exit(APEXIT_CHILDINIT); 1008 } 1009 ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf, APLOGNO(00352) 1010 "Child: Acquired the start mutex."); 1011 1012 /* 1013 * Create the worker thread dispatch IOCompletionPort 1014 */ 1015 /* Create the worker thread dispatch IOCP */ 1016 ThreadDispatchIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE, 1017 NULL, 0, 0); 1018 apr_thread_mutex_create(&qlock, APR_THREAD_MUTEX_DEFAULT, pchild); 1019 qwait_event = CreateEvent(NULL, TRUE, FALSE, NULL); 1020 if (!qwait_event) { 1021 ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(), 1022 ap_server_conf, APLOGNO(00353) 1023 "Child: Failed to create a qwait event."); 1024 exit(APEXIT_CHILDINIT); 1025 } 1026 1027 /* 1028 * Create the pool of worker threads 1029 */ 1030 ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00354) 1031 "Child: Starting %d worker threads.", ap_threads_per_child); 1032 child_handles = (HANDLE) apr_pcalloc(pchild, ap_threads_per_child 1033 * sizeof(HANDLE)); 1034 apr_thread_mutex_create(&child_lock, APR_THREAD_MUTEX_DEFAULT, pchild); 1035 1036 while (1) { 1037 for (i = 0; i < ap_threads_per_child; i++) { 1038 int *score_idx; 1039 int status = ap_scoreboard_image->servers[0][i].status; 1040 if (status != SERVER_GRACEFUL && status != SERVER_DEAD) { 1041 continue; 1042 } 1043 ap_update_child_status_from_indexes(0, i, SERVER_STARTING, NULL); 1044 1045 child_handles[i] = CreateThread(NULL, ap_thread_stacksize, 1046 worker_main, (void *) i, 1047 stack_res_flag, &tid); 1048 if (child_handles[i] == 0) { 1049 ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(), 1050 ap_server_conf, APLOGNO(00355) 1051 "Child: CreateThread failed. Unable to " 1052 "create all worker threads. Created %d of the %d " 1053 "threads requested with the ThreadsPerChild " 1054 "configuration directive.", 1055 threads_created, ap_threads_per_child); 1056 ap_signal_parent(SIGNAL_PARENT_SHUTDOWN); 1057 goto shutdown; 1058 } 1059 threads_created++; 1060 /* Save the score board index in ht keyed to the thread handle. 1061 * We need this when cleaning up threads down below... 1062 */ 1063 apr_thread_mutex_lock(child_lock); 1064 score_idx = apr_pcalloc(pchild, sizeof(int)); 1065 *score_idx = i; 1066 apr_hash_set(ht, &child_handles[i], sizeof(HANDLE), score_idx); 1067 apr_thread_mutex_unlock(child_lock); 1068 } 1069 /* Start the listener only when workers are available */ 1070 if (!listener_started && threads_created) { 1071 create_listener_thread(); 1072 listener_started = 1; 1073 winnt_mpm_state = AP_MPMQ_RUNNING; 1074 } 1075 if (threads_created == ap_threads_per_child) { 1076 break; 1077 } 1078 /* Check to see if the child has been told to exit */ 1079 if (WaitForSingleObject(exit_event, 0) != WAIT_TIMEOUT) { 1080 break; 1081 } 1082 /* wait for previous generation to clean up an entry in the scoreboard 1083 */ 1084 apr_sleep(1 * APR_USEC_PER_SEC); 1085 } 1086 1087 /* Wait for one of three events: 1088 * exit_event: 1089 * The exit_event is signaled by the parent process to notify 1090 * the child that it is time to exit. 1091 * 1092 * max_requests_per_child_event: 1093 * This event is signaled by the worker threads to indicate that 1094 * the process has handled MaxConnectionsPerChild connections. 1095 * 1096 * TIMEOUT: 1097 * To do periodic maintenance on the server (check for thread exits, 1098 * number of completion contexts, etc.) 1099 * 1100 * XXX: thread exits *aren't* being checked. 1101 * 1102 * XXX: other_child - we need the process handles to the other children 1103 * in order to map them to apr_proc_other_child_read (which is not 1104 * named well, it's more like a_p_o_c_died.) 1105 * 1106 * XXX: however - if we get a_p_o_c handle inheritance working, and 1107 * the parent process creates other children and passes the pipes 1108 * to our worker processes, then we have no business doing such 1109 * things in the child_main loop, but should happen in master_main. 1110 */ 1111 while (1) { 1112#if !APR_HAS_OTHER_CHILD 1113 rv = WaitForMultipleObjects(num_events, (HANDLE *)child_events, FALSE, INFINITE); 1114 cld = rv - WAIT_OBJECT_0; 1115#else 1116 rv = WaitForMultipleObjects(num_events, (HANDLE *)child_events, FALSE, 1000); 1117 cld = rv - WAIT_OBJECT_0; 1118 if (rv == WAIT_TIMEOUT) { 1119 apr_proc_other_child_refresh_all(APR_OC_REASON_RUNNING); 1120 } 1121 else 1122#endif 1123 if (rv == WAIT_FAILED) { 1124 /* Something serious is wrong */ 1125 ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(), 1126 ap_server_conf, APLOGNO(00356) 1127 "Child: WAIT_FAILED -- shutting down server"); 1128 break; 1129 } 1130 else if (cld == 0) { 1131 /* Exit event was signaled */ 1132 ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf, APLOGNO(00357) 1133 "Child: Exit event signaled. Child process is " 1134 "ending."); 1135 break; 1136 } 1137 else if (cld == 2) { 1138 /* The parent is dead. Shutdown the child process. */ 1139 ap_log_error(APLOG_MARK, APLOG_CRIT, 0, ap_server_conf, APLOGNO(02538) 1140 "Child: Parent process exited abruptly. Child process " 1141 "is ending"); 1142 break; 1143 } 1144 else { 1145 /* MaxConnectionsPerChild event set by the worker threads. 1146 * Signal the parent to restart 1147 */ 1148 ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00358) 1149 "Child: Process exiting because it reached " 1150 "MaxConnectionsPerChild. Signaling the parent to " 1151 "restart a new child process."); 1152 ap_signal_parent(SIGNAL_PARENT_RESTART); 1153 break; 1154 } 1155 } 1156 1157 /* 1158 * Time to shutdown the child process 1159 */ 1160 1161 shutdown: 1162 1163 winnt_mpm_state = AP_MPMQ_STOPPING; 1164 1165 /* Close the listening sockets. Note, we must close the listeners 1166 * before closing any accept sockets pending in AcceptEx to prevent 1167 * memory leaks in the kernel. 1168 */ 1169 for (lr = ap_listeners; lr ; lr = lr->next) { 1170 apr_socket_close(lr->sd); 1171 } 1172 1173 /* Shutdown listener threads and pending AcceptEx sockets 1174 * but allow the worker threads to continue consuming from 1175 * the queue of accepted connections. 1176 */ 1177 shutdown_in_progress = 1; 1178 1179 Sleep(1000); 1180 1181 /* Tell the worker threads to exit */ 1182 workers_may_exit = 1; 1183 1184 /* Release the start_mutex to let the new process (in the restart 1185 * scenario) a chance to begin accepting and servicing requests 1186 */ 1187 rv = apr_proc_mutex_unlock(start_mutex); 1188 if (rv == APR_SUCCESS) { 1189 ap_log_error(APLOG_MARK, APLOG_DEBUG, rv, ap_server_conf, APLOGNO(00359) 1190 "Child: Released the start mutex"); 1191 } 1192 else { 1193 ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(00360) 1194 "Child: Failure releasing the start mutex"); 1195 } 1196 1197 /* Shutdown the worker threads 1198 * Post worker threads blocked on the ThreadDispatch IOCompletion port 1199 */ 1200 while (g_blocked_threads > 0) { 1201 ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf, APLOGNO(00361) 1202 "Child: %d threads blocked on the completion port", 1203 g_blocked_threads); 1204 for (i=g_blocked_threads; i > 0; i--) { 1205 PostQueuedCompletionStatus(ThreadDispatchIOCP, 0, 1206 IOCP_SHUTDOWN, NULL); 1207 } 1208 Sleep(1000); 1209 } 1210 /* Empty the accept queue of completion contexts */ 1211 apr_thread_mutex_lock(qlock); 1212 while (qhead) { 1213 CloseHandle(qhead->overlapped.hEvent); 1214 closesocket(qhead->accept_socket); 1215 qhead = qhead->next; 1216 } 1217 apr_thread_mutex_unlock(qlock); 1218 1219 /* Give busy threads a chance to service their connections 1220 * (no more than the global server timeout period which 1221 * we track in msec remaining). 1222 */ 1223 watch_thread = 0; 1224 time_remains = (int)(ap_server_conf->timeout / APR_TIME_C(1000)); 1225 1226 while (threads_created) 1227 { 1228 int nFailsafe = MAXIMUM_WAIT_OBJECTS; 1229 DWORD dwRet; 1230 1231 /* Every time we roll over to wait on the first group 1232 * of MAXIMUM_WAIT_OBJECTS threads, take a breather, 1233 * and infrequently update the error log. 1234 */ 1235 if (watch_thread >= threads_created) { 1236 if ((time_remains -= 100) < 0) 1237 break; 1238 1239 /* Every 30 seconds give an update */ 1240 if ((time_remains % 30000) == 0) { 1241 ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, 1242 ap_server_conf, APLOGNO(00362) 1243 "Child: Waiting %d more seconds " 1244 "for %d worker threads to finish.", 1245 time_remains / 1000, threads_created); 1246 } 1247 /* We'll poll from the top, 10 times per second */ 1248 Sleep(100); 1249 watch_thread = 0; 1250 } 1251 1252 /* Fairness, on each iteration we will pick up with the thread 1253 * after the one we just removed, even if it's a single thread. 1254 * We don't block here. 1255 */ 1256 dwRet = WaitForMultipleObjects(min(threads_created - watch_thread, 1257 MAXIMUM_WAIT_OBJECTS), 1258 child_handles + watch_thread, 0, 0); 1259 1260 if (dwRet == WAIT_FAILED) { 1261 break; 1262 } 1263 if (dwRet == WAIT_TIMEOUT) { 1264 /* none ready */ 1265 watch_thread += MAXIMUM_WAIT_OBJECTS; 1266 continue; 1267 } 1268 else if (dwRet >= WAIT_ABANDONED_0) { 1269 /* We just got the ownership of the object, which 1270 * should happen at most MAXIMUM_WAIT_OBJECTS times. 1271 * It does NOT mean that the object is signaled. 1272 */ 1273 if ((nFailsafe--) < 1) 1274 break; 1275 } 1276 else { 1277 watch_thread += (dwRet - WAIT_OBJECT_0); 1278 if (watch_thread >= threads_created) 1279 break; 1280 cleanup_thread(child_handles, &threads_created, watch_thread); 1281 } 1282 } 1283 1284 /* Kill remaining threads off the hard way */ 1285 if (threads_created) { 1286 ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00363) 1287 "Child: Terminating %d threads that failed to exit.", 1288 threads_created); 1289 } 1290 for (i = 0; i < threads_created; i++) { 1291 int *score_idx; 1292 TerminateThread(child_handles[i], 1); 1293 CloseHandle(child_handles[i]); 1294 /* Reset the scoreboard entry for the thread we just whacked */ 1295 score_idx = apr_hash_get(ht, &child_handles[i], sizeof(HANDLE)); 1296 if (score_idx) { 1297 ap_update_child_status_from_indexes(0, *score_idx, SERVER_DEAD, NULL); 1298 } 1299 } 1300 ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00364) 1301 "Child: All worker threads have exited."); 1302 1303 apr_thread_mutex_destroy(child_lock); 1304 apr_thread_mutex_destroy(qlock); 1305 CloseHandle(qwait_event); 1306 1307 apr_pool_destroy(pchild); 1308 CloseHandle(exit_event); 1309 if (child_events[2] != NULL) { 1310 CloseHandle(child_events[2]); 1311 } 1312} 1313 1314#endif /* def WIN32 */ 1315