28c28
< __FBSDID("$FreeBSD: head/sys/kern/subr_taskqueue.c 180583 2008-07-18 06:12:31Z kmacy $");
---
> __FBSDID("$FreeBSD: head/sys/kern/subr_taskqueue.c 180588 2008-07-18 07:10:33Z kmacy $");
35d34
< #include <sys/ktr.h>
52,53d50
< STAILQ_HEAD(task_head, task);
<
63a61
> int tq_spin;
70,72d67
< #define TQ_FLAGS_SPIN (1 << 3)
< #define TQ_FLAGS_NOWAKEUP (1 << 4)
< #define TQ_FLAGS_RUNNING (1 << 5)
74,81c69,76
< #define TQ_LOCK(tq) \
< do { \
< \
< if (tq->tq_flags & TQ_FLAGS_SPIN) \
< mtx_lock_spin(&tq->tq_mutex); \
< else \
< mtx_lock(&tq->tq_mutex); \
< } while (0)
---
> static __inline void
> TQ_LOCK(struct taskqueue *tq)
> {
> if (tq->tq_spin)
> mtx_lock_spin(&tq->tq_mutex);
> else
> mtx_lock(&tq->tq_mutex);
> }
82a78,85
> static __inline void
> TQ_UNLOCK(struct taskqueue *tq)
> {
> if (tq->tq_spin)
> mtx_unlock_spin(&tq->tq_mutex);
> else
> mtx_unlock(&tq->tq_mutex);
> }
84,93d86
< #define TQ_UNLOCK(tq) \
< do { \
< \
< if (tq->tq_flags & TQ_FLAGS_SPIN) \
< mtx_unlock_spin(&tq->tq_mutex); \
< else \
< mtx_unlock(&tq->tq_mutex); \
< } while (0)
<
<
100c93
< if (tq->tq_flags & TQ_FLAGS_SPIN)
---
> if (tq->tq_spin)
121,123c114
< int spin;
<
<
---
>
127c118
< spin = ((mtxflags & MTX_SPIN) ? TQ_FLAGS_SPIN : 0);
---
>
132c123,124
< queue->tq_flags |= TQ_FLAGS_ACTIVE | spin;
---
> queue->tq_spin = (mtxflags & MTX_SPIN) != 0;
> queue->tq_flags |= TQ_FLAGS_ACTIVE;
211c203
< if (task->ta_pending || (task->ta_flags & TA_REFERENCED)) {
---
> if (task->ta_pending) {
213,218d204
< /*
< * overflow
< */
< if (task->ta_pending == 0)
< task->ta_pending--;
<
243c229
< if ((queue->tq_flags & (TQ_FLAGS_BLOCKED|TQ_FLAGS_RUNNING)) == 0)
---
> if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0)
245c231
< else if (queue->tq_flags & TQ_FLAGS_BLOCKED)
---
> else
314c300
< if (queue->tq_flags & TQ_FLAGS_SPIN) { /* XXX */
---
> if (queue->tq_spin) { /* XXX */
482,616d467
<
< static void
< taskqueue_run_drv(void *arg)
< {
< struct task *task, *tmp;
< struct task_head current;
< int restarts = 0;
< struct taskqueue *queue = (struct taskqueue *) arg;
<
< STAILQ_INIT(&current);
< /*
< * First we move all of the tasks off of the taskqueue's list
< * on to current on the stack to avoided repeated serialization
< */
< mtx_lock_spin(&queue->tq_mutex);
< queue->tq_flags |= TQ_FLAGS_RUNNING;
< restart:
< STAILQ_CONCAT(&current, &queue->tq_queue);
< STAILQ_FOREACH(task, &current, ta_link) {
< /*
< * to let taskqueue_enqueue_fast know that this task
< * has been dequeued but is referenced
< * clear pending so that if pending is later set we know that it
< * it needs to be re-enqueued even if the task doesn't return
< * TA_NO_DEQUEUE
< */
< task->ta_ppending = task->ta_pending;
< task->ta_pending = 0;
< task->ta_flags |= TA_REFERENCED;
< }
< mtx_unlock_spin(&queue->tq_mutex);
< STAILQ_FOREACH(task, &current, ta_link) {
< task->ta_rc = task->ta_drv_func(task->ta_context, task->ta_ppending);
<
< }
< /*
< * We've gotten here so we know that we've run the tasks that were
< * on the taskqueue list on the first pass
< */
< mtx_lock_spin(&queue->tq_mutex);
< STAILQ_FOREACH_SAFE(task, &current, ta_link, tmp) {
< if (task->ta_rc != TA_NO_DEQUEUE && task->ta_pending == 0) {
< STAILQ_REMOVE(&current, task, task, ta_link);
< task->ta_flags &= ~TA_REFERENCED;
< }
< task->ta_ppending = 0;
< task->ta_rc = 0;
< }
< /*
< * restart if there are any tasks in the list
< */
< if (STAILQ_FIRST(&current) || STAILQ_FIRST(&queue->tq_queue)) {
< restarts++;
< goto restart;
< }
< queue->tq_flags &= ~TQ_FLAGS_RUNNING;
< mtx_unlock_spin(&queue->tq_mutex);
< CTR2(KTR_INTR, "queue=%s returning from taskqueue_run_drv after %d restarts", queue->tq_name, restarts);
< }
<
< static void
< taskqueue_drv_schedule(void *context)
< {
< swi_sched(context, 0);
< }
<
< struct taskqueue *
< taskqueue_define_drv(void *arg, const char *name)
< {
< struct taskqueue *tq;
< struct thread *td;
<
< tq = malloc(sizeof(struct taskqueue), M_TASKQUEUE,
< M_NOWAIT | M_ZERO);
< if (!tq) {
< printf("%s: Unable to allocate fast drv task queue!\n",
< __func__);
< return (NULL);
< }
<
< STAILQ_INIT(&tq->tq_queue);
< tq->tq_name = name;
< tq->tq_enqueue = taskqueue_drv_schedule;
< tq->tq_flags = (TQ_FLAGS_ACTIVE | TQ_FLAGS_SPIN | TQ_FLAGS_NOWAKEUP);
< mtx_init(&tq->tq_mutex, name, NULL, MTX_SPIN);
<
< mtx_lock(&taskqueue_queues_mutex);
< STAILQ_INSERT_TAIL(&taskqueue_queues, tq, tq_link);
< mtx_unlock(&taskqueue_queues_mutex);
<
< swi_add(NULL, name, taskqueue_run_drv,
< tq, SWI_NET, INTR_MPSAFE, &tq->tq_context);
< td = intr_handler_thread((struct intr_handler *) tq->tq_context);
< return (tq);
< }
<
< struct intr_handler *
< taskqueue_drv_handler(struct taskqueue *tq)
< {
< return ((struct intr_handler *) tq->tq_context);
< }
<
< struct thread *
< taskqueue_drv_thread(void *context)
< {
< struct taskqueue *tq = (struct taskqueue *) context;
<
< return (intr_handler_thread((struct intr_handler *) tq->tq_context));
< }
<
< /*
< * Caller must make sure that there must not be any new tasks getting queued
< * before calling this.
< */
< void
< taskqueue_free_drv(struct taskqueue *queue)
< {
< struct intr_thread *ithd;
< struct intr_event *ie;
<
< mtx_lock(&taskqueue_queues_mutex);
< STAILQ_REMOVE(&taskqueue_queues, queue, taskqueue, tq_link);
< mtx_unlock(&taskqueue_queues_mutex);
<
< ie = ((struct intr_handler *)(queue->tq_context))->ih_event;
< ithd = ie->ie_thread;
< swi_remove(queue->tq_context);
< intr_event_destroy(ie);
<
< mtx_lock_spin(&queue->tq_mutex);
< taskqueue_run(queue);
< mtx_destroy(&queue->tq_mutex);
< free(queue, M_TASKQUEUE);
< }
<