1/* Licensed to the Apache Software Foundation (ASF) under one or more
2 * contributor license agreements.  See the NOTICE file distributed with
3 * this work for additional information regarding copyright ownership.
4 * The ASF licenses this file to You under the Apache License, Version 2.0
5 * (the "License"); you may not use this file except in compliance with
6 * the License.  You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17/*
18time-sem.c has the basics of the semaphores we use in http_main.c.  It's
19intended for timing differences between various methods on an
20architecture.  In practice we've found many things affect which semaphore
21to be used:
22
23    - NFS filesystems absolutely suck for fcntl() and flock()
24
25    - uslock absolutely sucks on single-processor IRIX boxes, but
26        absolutely rocks on multi-processor boxes.  The converse
27        is true for fcntl.  sysvsem seems a moderate balance.
28
29    - Under Solaris you can't have too many processes use SEM_UNDO, there
30        might be a tuneable somewhere that increases the limit from 29.
31        We're not sure what the tunable is, so there's a define
32        NO_SEM_UNDO which can be used to simulate us trapping/blocking
33        signals to be able to properly release the semaphore on a clean
34        child death.  You'll also need to define NEED_UNION_SEMUN
35        under solaris.
36
37You'll need to define USE_SHMGET_SCOREBOARD if anonymous shared mmap()
38doesn't work on your system (i.e. linux).
39
40argv[1] is the #children, argv[2] is the #iterations per child
41
42You should run each over many different #children inputs, and choose
43#iter such that the program runs for at least a second or so... or even
44longer depending on your patience.
45
46compile with:
47
48gcc -o time-FCNTL -Wall -O time-sem.c -DUSE_FCNTL_SERIALIZED_ACCEPT
49gcc -o time-FLOCK -Wall -O time-sem.c -DUSE_FLOCK_SERIALIZED_ACCEPT
50gcc -o time-SYSVSEM -Wall -O time-sem.c -DUSE_SYSVSEM_SERIALIZED_ACCEPT
51gcc -o time-SYSVSEM2 -Wall -O time-sem.c -DUSE_SYSVSEM_SERIALIZED_ACCEPT -DNO_SEM_UNDO
52gcc -o time-PTHREAD -Wall -O time-sem.c -DUSE_PTHREAD_SERIALIZED_ACCEPT -lpthread
53gcc -o time-USLOCK -Wall -O time-sem.c -DUSE_USLOCK_SERIALIZED_ACCEPT
54
55not all versions work on all systems.
56*/
57
58#include <errno.h>
59#include <sys/time.h>
60#include <unistd.h>
61#include <stdio.h>
62#include <string.h>
63#include <stdlib.h>
64#include <sys/types.h>
65#include <sys/stat.h>
66#include <fcntl.h>
67#include <sys/wait.h>
68#include <sys/mman.h>
69#include <signal.h>
70
71#if defined(USE_FCNTL_SERIALIZED_ACCEPT)
72
73static struct flock lock_it;
74static struct flock unlock_it;
75
76static int fcntl_fd=-1;
77
78#define accept_mutex_child_init()
79#define accept_mutex_cleanup()
80
81/*
82 * Initialize mutex lock.
83 * Must be safe to call this on a restart.
84 */
85void
86accept_mutex_init(void)
87{
88
89    lock_it.l_whence = SEEK_SET;   /* from current point */
90    lock_it.l_start  = 0;          /* -"- */
91    lock_it.l_len    = 0;          /* until end of file */
92    lock_it.l_type   = F_WRLCK;    /* set exclusive/write lock */
93    lock_it.l_pid    = 0;          /* pid not actually interesting */
94    unlock_it.l_whence = SEEK_SET; /* from current point */
95    unlock_it.l_start  = 0;        /* -"- */
96    unlock_it.l_len    = 0;        /* until end of file */
97    unlock_it.l_type   = F_UNLCK;  /* set exclusive/write lock */
98    unlock_it.l_pid    = 0;        /* pid not actually interesting */
99
100    printf("opening test-lock-thing in current directory\n");
101    fcntl_fd = open("test-lock-thing", O_CREAT | O_WRONLY | O_EXCL, 0644);
102    if (fcntl_fd == -1)
103    {
104        perror ("open");
105        fprintf (stderr, "Cannot open lock file: %s\n", "test-lock-thing");
106        exit (1);
107    }
108    unlink("test-lock-thing");
109}
110
111void accept_mutex_on(void)
112{
113    int ret;
114
115    while ((ret = fcntl(fcntl_fd, F_SETLKW, &lock_it)) < 0 && errno == EINTR)
116        continue;
117
118    if (ret < 0) {
119        perror ("fcntl lock_it");
120        exit(1);
121    }
122}
123
124void accept_mutex_off(void)
125{
126    if (fcntl (fcntl_fd, F_SETLKW, &unlock_it) < 0)
127    {
128        perror ("fcntl unlock_it");
129        exit(1);
130    }
131}
132
133#elif defined(USE_FLOCK_SERIALIZED_ACCEPT)
134
135#include <sys/file.h>
136
137static int flock_fd=-1;
138
139#define FNAME "test-lock-thing"
140
141/*
142 * Initialize mutex lock.
143 * Must be safe to call this on a restart.
144 */
145void accept_mutex_init(void)
146{
147
148    printf("opening " FNAME " in current directory\n");
149    flock_fd = open(FNAME, O_CREAT | O_WRONLY | O_EXCL, 0644);
150    if (flock_fd == -1)
151    {
152        perror ("open");
153        fprintf (stderr, "Cannot open lock file: %s\n", "test-lock-thing");
154        exit (1);
155    }
156}
157
158void accept_mutex_child_init(void)
159{
160    flock_fd = open(FNAME, O_WRONLY, 0600);
161    if (flock_fd == -1) {
162        perror("open");
163        exit(1);
164    }
165}
166
167void accept_mutex_cleanup(void)
168{
169    unlink(FNAME);
170}
171
172void accept_mutex_on(void)
173{
174    int ret;
175
176    while ((ret = flock(flock_fd, LOCK_EX)) < 0 && errno == EINTR)
177        continue;
178
179    if (ret < 0) {
180        perror ("flock(LOCK_EX)");
181        exit(1);
182    }
183}
184
185void accept_mutex_off(void)
186{
187    if (flock (flock_fd, LOCK_UN) < 0)
188    {
189        perror ("flock(LOCK_UN)");
190        exit(1);
191    }
192}
193
194#elif defined (USE_SYSVSEM_SERIALIZED_ACCEPT)
195
196#include <sys/types.h>
197#include <sys/ipc.h>
198#include <sys/sem.h>
199
200static   int sem_id = -1;
201#ifdef NO_SEM_UNDO
202static sigset_t accept_block_mask;
203static sigset_t accept_previous_mask;
204#endif
205
206#define accept_mutex_child_init()
207#define accept_mutex_cleanup()
208
209void accept_mutex_init(void)
210{
211#ifdef NEED_UNION_SEMUN
212    /* believe it or not, you need to define this under solaris */
213    union semun {
214        int val;
215        struct semid_ds *buf;
216        ushort *array;
217    };
218#endif
219
220    union semun ick;
221
222    sem_id = semget(999, 1, IPC_CREAT | 0666);
223    if (sem_id < 0) {
224       perror ("semget");
225       exit (1);
226    }
227    ick.val = 1;
228    if (semctl(sem_id, 0, SETVAL, ick) < 0) {
229       perror ("semctl");
230        exit(1);
231    }
232#ifdef NO_SEM_UNDO
233    sigfillset(&accept_block_mask);
234    sigdelset(&accept_block_mask, SIGHUP);
235    sigdelset(&accept_block_mask, SIGTERM);
236    sigdelset(&accept_block_mask, SIGUSR1);
237#endif
238}
239
240void accept_mutex_on()
241{
242    struct sembuf op;
243
244#ifdef NO_SEM_UNDO
245    if (sigprocmask(SIG_BLOCK, &accept_block_mask, &accept_previous_mask)) {
246        perror("sigprocmask(SIG_BLOCK)");
247        exit (1);
248    }
249    op.sem_flg = 0;
250#else
251    op.sem_flg = SEM_UNDO;
252#endif
253    op.sem_num = 0;
254    op.sem_op  = -1;
255    if (semop(sem_id, &op, 1) < 0) {
256        perror ("accept_mutex_on");
257        exit (1);
258    }
259}
260
261void accept_mutex_off()
262{
263    struct sembuf op;
264
265    op.sem_num = 0;
266    op.sem_op  = 1;
267#ifdef NO_SEM_UNDO
268    op.sem_flg = 0;
269#else
270    op.sem_flg = SEM_UNDO;
271#endif
272    if (semop(sem_id, &op, 1) < 0) {
273        perror ("accept_mutex_off");
274        exit (1);
275    }
276#ifdef NO_SEM_UNDO
277    if (sigprocmask(SIG_SETMASK, &accept_previous_mask, NULL)) {
278        perror("sigprocmask(SIG_SETMASK)");
279        exit (1);
280    }
281#endif
282}
283
284#elif defined (USE_PTHREAD_SERIALIZED_ACCEPT)
285
286/* note: pthread mutexes aren't released on child death, hence the
287 * signal goop ... in a real implementation we'd do special things
288 * during hup, term, usr1.
289 */
290
291#include <pthread.h>
292
293static pthread_mutex_t *mutex;
294static sigset_t accept_block_mask;
295static sigset_t accept_previous_mask;
296
297#define accept_mutex_child_init()
298#define accept_mutex_cleanup()
299
300void accept_mutex_init(void)
301{
302    pthread_mutexattr_t mattr;
303    int fd;
304
305    fd = open ("/dev/zero", O_RDWR);
306    if (fd == -1) {
307        perror ("open(/dev/zero)");
308        exit (1);
309    }
310    mutex = (pthread_mutex_t *)mmap ((caddr_t)0, sizeof (*mutex),
311                    PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
312    if (mutex == (void *)(caddr_t)-1) {
313        perror ("mmap");
314        exit (1);
315    }
316    close (fd);
317    if (pthread_mutexattr_init(&mattr)) {
318        perror ("pthread_mutexattr_init");
319        exit (1);
320    }
321    if (pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) {
322        perror ("pthread_mutexattr_setpshared");
323        exit (1);
324    }
325    if (pthread_mutex_init(mutex, &mattr)) {
326        perror ("pthread_mutex_init");
327        exit (1);
328    }
329    sigfillset(&accept_block_mask);
330    sigdelset(&accept_block_mask, SIGHUP);
331    sigdelset(&accept_block_mask, SIGTERM);
332    sigdelset(&accept_block_mask, SIGUSR1);
333}
334
335void accept_mutex_on()
336{
337    if (sigprocmask(SIG_BLOCK, &accept_block_mask, &accept_previous_mask)) {
338        perror("sigprocmask(SIG_BLOCK)");
339        exit (1);
340    }
341    if (pthread_mutex_lock (mutex)) {
342        perror ("pthread_mutex_lock");
343        exit (1);
344    }
345}
346
347void accept_mutex_off()
348{
349    if (pthread_mutex_unlock (mutex)) {
350        perror ("pthread_mutex_unlock");
351        exit (1);
352    }
353    if (sigprocmask(SIG_SETMASK, &accept_previous_mask, NULL)) {
354        perror("sigprocmask(SIG_SETMASK)");
355        exit (1);
356    }
357}
358
359#elif defined (USE_USLOCK_SERIALIZED_ACCEPT)
360
361#include <ulocks.h>
362
363static usptr_t *us = NULL;
364static ulock_t uslock = NULL;
365
366#define accept_mutex_child_init()
367#define accept_mutex_cleanup()
368
369void accept_mutex_init(void)
370{
371    ptrdiff_t old;
372    /* default is 8 */
373#define CONF_INITUSERS_MAX 15
374    if ((old = usconfig(CONF_INITUSERS, CONF_INITUSERS_MAX)) == -1) {
375        perror("usconfig");
376        exit(-1);
377    }
378    if ((old = usconfig(CONF_LOCKTYPE, US_NODEBUG)) == -1) {
379        perror("usconfig");
380        exit(-1);
381    }
382    if ((old = usconfig(CONF_ARENATYPE, US_SHAREDONLY)) == -1) {
383        perror("usconfig");
384        exit(-1);
385    }
386    if ((us = usinit("/dev/zero")) == NULL) {
387        perror("usinit");
388        exit(-1);
389    }
390    if ((uslock = usnewlock(us)) == NULL) {
391        perror("usnewlock");
392        exit(-1);
393    }
394}
395void accept_mutex_on()
396{
397    switch(ussetlock(uslock)) {
398        case 1:
399            /* got lock */
400            break;
401        case 0:
402            fprintf(stderr, "didn't get lock\n");
403            exit(-1);
404        case -1:
405            perror("ussetlock");
406            exit(-1);
407    }
408}
409void accept_mutex_off()
410{
411    if (usunsetlock(uslock) == -1) {
412        perror("usunsetlock");
413        exit(-1);
414    }
415}
416#endif
417
418
419#ifndef USE_SHMGET_SCOREBOARD
420static void *get_shared_mem(apr_size_t size)
421{
422    void *result;
423
424    /* allocate shared memory for the shared_counter */
425    result = (unsigned long *)mmap ((caddr_t)0, size,
426                    PROT_READ|PROT_WRITE, MAP_ANON|MAP_SHARED, -1, 0);
427    if (result == (void *)(caddr_t)-1) {
428        perror ("mmap");
429        exit (1);
430    }
431    return result;
432}
433#else
434#include <sys/types.h>
435#include <sys/ipc.h>
436#ifdef HAVE_SYS_MUTEX_H
437#include <sys/mutex.h>
438#endif
439#include <sys/shm.h>
440
441static void *get_shared_mem(apr_size_t size)
442{
443    key_t shmkey = IPC_PRIVATE;
444    int shmid = -1;
445    void *result;
446#ifdef MOVEBREAK
447    char *obrk;
448#endif
449
450    if ((shmid = shmget(shmkey, size, IPC_CREAT | SHM_R | SHM_W)) == -1) {
451        perror("shmget");
452        exit(1);
453    }
454
455#ifdef MOVEBREAK
456    /*
457     * Some SysV systems place the shared segment WAY too close
458     * to the dynamic memory break point (sbrk(0)). This severely
459     * limits the use of malloc/sbrk in the program since sbrk will
460     * refuse to move past that point.
461     *
462     * To get around this, we move the break point "way up there",
463     * attach the segment and then move break back down. Ugly
464     */
465    if ((obrk = sbrk(MOVEBREAK)) == (char *) -1) {
466        perror("sbrk");
467    }
468#endif
469
470#define BADSHMAT  ((void *)(-1))
471    if ((result = shmat(shmid, 0, 0)) == BADSHMAT) {
472        perror("shmat");
473    }
474    /*
475     * We must avoid leaving segments in the kernel's
476     * (small) tables.
477     */
478    if (shmctl(shmid, IPC_RMID, NULL) != 0) {
479        perror("shmctl(IPC_RMID)");
480    }
481    if (result == BADSHMAT)  /* now bailout */
482        exit(1);
483
484#ifdef MOVEBREAK
485    if (obrk == (char *) -1)
486        return;  /* nothing else to do */
487    if (sbrk(-(MOVEBREAK)) == (char *) -1) {
488        perror("sbrk 2");
489    }
490#endif
491    return result;
492}
493#endif
494
495#ifdef _POSIX_PRIORITY_SCHEDULING
496/* don't ask */
497#define _P __P
498#include <sched.h>
499#define YIELD  sched_yield()
500#else
501#define YIELD  do { struct timeval zero; zero.tv_sec = zero.tv_usec = 0; select(0,0,0,0,&zero); } while(0)
502#endif
503
504void main (int argc, char **argv)
505{
506    int num_iter;
507    int num_child;
508    int i;
509    struct timeval first;
510    struct timeval last;
511    long ms;
512    int pid;
513    unsigned long *shared_counter;
514
515    if (argc != 3) {
516        fprintf (stderr, "Usage: time-sem num-child num iter\n");
517        exit (1);
518    }
519
520    num_child = atoi (argv[1]);
521    num_iter = atoi (argv[2]);
522
523    /* allocate shared memory for the shared_counter */
524    shared_counter = get_shared_mem(sizeof(*shared_counter));
525
526    /* initialize counter to 0 */
527    *shared_counter = 0;
528
529    accept_mutex_init ();
530
531    /* parent grabs mutex until done spawning children */
532    accept_mutex_on ();
533
534    for (i = 0; i < num_child; ++i) {
535        pid = fork();
536        if (pid == 0) {
537            /* child, do our thing */
538            accept_mutex_child_init();
539            for (i = 0; i < num_iter; ++i) {
540                unsigned long tmp;
541
542                accept_mutex_on ();
543                tmp = *shared_counter;
544                YIELD;
545                *shared_counter = tmp + 1;
546                accept_mutex_off ();
547            }
548            exit (0);
549        } else if (pid == -1) {
550            perror ("fork");
551            exit (1);
552        }
553    }
554
555    /* a quick test to see that nothing is screwed up */
556    if (*shared_counter != 0) {
557        puts ("WTF! shared_counter != 0 before the children have been started!");
558        exit (1);
559    }
560
561    gettimeofday (&first, NULL);
562    /* launch children into action */
563    accept_mutex_off ();
564    for (i = 0; i < num_child; ++i) {
565        if (wait(NULL) == -1) {
566            perror ("wait");
567        }
568    }
569    gettimeofday (&last, NULL);
570
571    if (*shared_counter != num_child * num_iter) {
572        printf ("WTF! shared_counter != num_child * num_iter!\n"
573                "shared_counter = %lu\nnum_child = %d\nnum_iter=%d\n",
574                *shared_counter,
575                num_child, num_iter);
576    }
577
578    last.tv_sec -= first.tv_sec;
579    ms = last.tv_usec - first.tv_usec;
580    if (ms < 0) {
581        --last.tv_sec;
582        ms += 1000000;
583    }
584    last.tv_usec = ms;
585    printf ("%8lu.%06lu\n", last.tv_sec, last.tv_usec);
586
587    accept_mutex_cleanup();
588
589    exit(0);
590}
591
592