liquidio/base/lio_request_manager.c

60 	struct lio_instr_queue	*iq;
77 	iq = oct->instr_queue[iq_no];
78 	iq->oct_dev = oct;
93 				   &iq->txtag);
100 	iq->base_addr = lio_dma_alloc(q_size, (vm_paddr_t *)&iq->base_addr_dma);
101 	if (!iq->base_addr) {
107 	iq->max_count = num_descs;
113 	iq->request_list = malloc(sizeof(*iq->request_list) * num_descs,
115 	if (iq->request_list == NULL) {
122 		    iq_no, iq->base_addr, LIO_CAST64(iq->base_addr_dma),
123 		    iq->max_count);
126 	request_buf = iq->request_list;
128 		error = bus_dmamap_create(iq->txtag, 0, &request_buf->map);
135 	iq->txpciq.txpciq64 = txpciq.txpciq64;
136 	iq->fill_cnt = 0;
137 	iq->host_write_index = 0;
138 	iq->octeon_read_index = 0;
139 	iq->flush_index = 0;
140 	iq->last_db_time = 0;
141 	iq->db_timeout = (uint32_t)conf->db_timeout;
142 	atomic_store_rel_int(&iq->instr_pending, 0);
145 	mtx_init(&iq->lock, "Tx_lock", NULL, MTX_DEF);
146 	mtx_init(&iq->post_lock, "iq_post_lock", NULL, MTX_DEF);
147 	mtx_init(&iq->enq_lock, "enq_lock", NULL, MTX_DEF);
149 	mtx_init(&iq->iq_flush_running_lock, "iq_flush_running_lock", NULL,
152 	oct->io_qmask.iq |= BIT_ULL(iq_no);
156 	iq->iqcmd_64B = (conf->instr_type == 64);
164 		lio_dev_err(oct, "check db wq create failed for iq %d\n",
194 	struct lio_instr_queue		*iq = oct->instr_queue[iq_no];
216 	request_buf = iq->request_list;
217 	for (i = 0; i < iq->max_count; i++, request_buf++) {
222 				bus_dmamap_sync(iq->txtag, request_buf->map,
224 				bus_dmamap_unload(iq->txtag,
229 					bus_dmamap_destroy(iq->txtag,
234 				bus_dmamap_unload(iq->txtag, request_buf->map);
235 				bus_dmamap_destroy(iq->txtag, request_buf->map);
241 	if (iq->br != NULL) {
242 		buf_ring_free(iq->br, M_DEVBUF);
243 		iq->br = NULL;
246 	if (iq->request_list != NULL) {
247 		free(iq->request_list, M_DEVBUF);
248 		iq->request_list = NULL;
251 	if (iq->txtag != NULL) {
252 		bus_dma_tag_destroy(iq->txtag);
253 		iq->txtag = NULL;
256 	if (iq->base_addr) {
257 		q_size = iq->max_count * desc_size;
258 		lio_dma_free((uint32_t)q_size, iq->base_addr);
260 		oct->io_qmask.iq &= ~(1ULL << iq_no);
308 			if (!(oct->io_qmask.iq & BIT_ULL(i)))
328 lio_ring_doorbell(struct octeon_device *oct, struct lio_instr_queue *iq)
332 		lio_write_csr32(oct, iq->doorbell_reg, iq->fill_cnt);
335 		iq->fill_cnt = 0;
336 		iq->last_db_time = ticks;
342 __lio_copy_cmd_into_iq(struct lio_instr_queue *iq, uint8_t *cmd)
346 	cmdsize = ((iq->iqcmd_64B) ? 64 : 32);
347 	iqptr = iq->base_addr + (cmdsize * iq->host_write_index);
353 __lio_post_command2(struct lio_instr_queue *iq, uint8_t *cmd)
363 	if (atomic_load_acq_int(&iq->instr_pending) >=
364 	    (int32_t)(iq->max_count - 1)) {
370 	if (atomic_load_acq_int(&iq->instr_pending) >=
371 	    (int32_t)(iq->max_count - 2))
374 	__lio_copy_cmd_into_iq(iq, cmd);
377 	st.index = iq->host_write_index;
378 	iq->host_write_index = lio_incr_index(iq->host_write_index, 1,
379 					      iq->max_count);
380 	iq->fill_cnt++;
388 	atomic_add_int(&iq->instr_pending, 1);
394 __lio_add_to_request_list(struct lio_instr_queue *iq, int idx, void *buf,
398 	iq->request_list[idx].buf = buf;
399 	iq->request_list[idx].reqtype = reqtype;
405 			    struct lio_instr_queue *iq, uint32_t budget)
412 	uint32_t			old = iq->flush_index;
415 	while (old != iq->octeon_read_index) {
416 		reqtype = iq->request_list[old].reqtype;
417 		buf = iq->request_list[old].buf;
425 			lio_free_mbuf(iq, buf);
428 			lio_free_sgmbuf(iq, buf);
468 		iq->request_list[old].buf = NULL;
469 		iq->request_list[old].reqtype = 0;
473 		old = lio_incr_index(old, 1, iq->max_count);
479 	iq->flush_index = old;
486 lio_flush_iq(struct octeon_device *oct, struct lio_instr_queue *iq,
493 	if (!mtx_trylock(&iq->iq_flush_running_lock))
496 	mtx_lock(&iq->lock);
498 	iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq);
502 		if (iq->flush_index == iq->octeon_read_index)
507 				lio_process_iq_request_list(oct, iq,
512 				lio_process_iq_request_list(oct, iq, 0);
515 			atomic_subtract_int(&iq->instr_pending, inst_processed);
516 			iq->stats.instr_processed += inst_processed;
526 	iq->last_db_time = ticks;
528 	mtx_unlock(&iq->lock);
530 	mtx_unlock(&iq->iq_flush_running_lock);
542 	struct lio_instr_queue	*iq;
548 	iq = oct->instr_queue[iq_no];
549 	if (iq == NULL)
552 	if (atomic_load_acq_int(&iq->instr_pending)) {
554 		next_time = iq->last_db_time + lio_ms_to_ticks(iq->db_timeout);
558 		iq->last_db_time = ticks;
561 		lio_flush_iq(oct, iq, 0);
563 		lio_enable_irq(NULL, iq);
566 	if (oct->props.ifp != NULL && iq->br != NULL) {
567 		if (mtx_trylock(&iq->enq_lock)) {
568 			if (!drbr_empty(oct->props.ifp, iq->br))
569 				lio_mq_start_locked(oct->props.ifp, iq);
571 			mtx_unlock(&iq->enq_lock);
599 	struct lio_instr_queue		*iq = oct->instr_queue[iq_no];
605 	mtx_lock(&iq->post_lock);
607 	st = __lio_post_command2(iq, cmd);
610 		__lio_add_to_request_list(iq, st.index, buf, reqtype);
615 			lio_ring_doorbell(oct, iq);
620 	mtx_unlock(&iq->post_lock);