mlx5/mlx5_en/mlx5_en_main.c

44 	struct mlx5e_rq_param rq;
872 		struct mlx5e_rq *rq = &pch->rq;
873 		struct mlx5e_rq_stats *rq_stats = &pch->rq.stats;
876 		rq_stats->sw_lro_queued = rq->lro.lro_queued;
877 		rq_stats->sw_lro_flushed = rq->lro.lro_flushed;
1112     struct mlx5e_rq *rq)
1141 	    &rq->dma_tag)))
1144 	err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
1145 	    &rq->wq_ctrl);
1149 	rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
1151 	err = mlx5e_get_wqe_sz(priv, &rq->wqe_sz, &rq->nsegs);
1155 	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1157 	err = -tcp_lro_init_args(&rq->lro, c->ifp, TCP_LRO_ENTRIES, wq_sz);
1161 	rq->mbuf = malloc(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
1163 		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
1166 		err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map);
1169 				bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1174 		for (j = 0; j < rq->nsegs; j++)
1178 	INIT_WORK(&rq->dim.work, mlx5e_dim_work);
1180 		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1183 		    struct mlx5e_channel_param, rq)->rx_cq.cqc;
1187 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1190 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1193 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1198 	rq->ifp = c->ifp;
1199 	rq->channel = c;
1200 	rq->ix = c->ix;
1203 	mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1205 	    rq->stats.arg);
1209 	free(rq->mbuf, M_MLX5EN);
1210 	tcp_lro_free(&rq->lro);
1212 	mlx5_wq_destroy(&rq->wq_ctrl);
1214 	bus_dma_tag_destroy(rq->dma_tag);
1220 mlx5e_destroy_rq(struct mlx5e_rq *rq)
1226 	sysctl_ctx_free(&rq->stats.ctx);
1229 	tcp_lro_free(&rq->lro);
1231 	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1233 		if (rq->mbuf[i].mbuf != NULL) {
1234 			bus_dmamap_unload(rq->dma_tag, rq->mbuf[i].dma_map);
1235 			m_freem(rq->mbuf[i].mbuf);
1237 		bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1239 	free(rq->mbuf, M_MLX5EN);
1240 	mlx5_wq_destroy(&rq->wq_ctrl);
1241 	bus_dma_tag_destroy(rq->dma_tag);
1245 mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
1247 	struct mlx5e_channel *c = rq->channel;
1258 	    sizeof(u64) * rq->wq_ctrl.buf.npages;
1268 	MLX5_SET(rqc, rqc, cqn, c->rq.cq.mcq.cqn);
1273 	MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
1275 	MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
1277 	mlx5_fill_page_array(&rq->wq_ctrl.buf,
1280 	err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
1288 mlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state)
1290 	struct mlx5e_channel *c = rq->channel;
1306 	MLX5_SET(modify_rq_in, in, rqn, rq->rqn);
1318 mlx5e_disable_rq(struct mlx5e_rq *rq)
1320 	struct mlx5e_channel *c = rq->channel;
1324 	mlx5_core_destroy_rq(mdev, rq->rqn);
1328 mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
1330 	struct mlx5e_channel *c = rq->channel;
1332 	struct mlx5_wq_ll *wq = &rq->wq;
1347     struct mlx5e_rq *rq)
1351 	err = mlx5e_create_rq(c, param, rq);
1355 	err = mlx5e_enable_rq(rq, param);
1359 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1363 	c->rq.enabled = 1;
1368 	mlx5e_disable_rq(rq);
1370 	mlx5e_destroy_rq(rq);
1376 mlx5e_close_rq(struct mlx5e_rq *rq)
1378 	mtx_lock(&rq->mtx);
1379 	rq->enabled = 0;
1380 	callout_stop(&rq->watchdog);
1381 	mtx_unlock(&rq->mtx);
1383 	mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
1387 mlx5e_close_rq_wait(struct mlx5e_rq *rq)
1390 	mlx5e_disable_rq(rq);
1391 	mlx5e_close_cq(&rq->cq);
1392 	cancel_work_sync(&rq->dim.work);
1393 	mlx5e_destroy_rq(rq);
2019 	mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF);
2021 	callout_init_mtx(&c->rq.watchdog, &c->rq.mtx, 0);
2040 	callout_drain(&c->rq.watchdog);
2042 	mtx_destroy(&c->rq.mtx);
2059 	MLX5E_ZERO(&c->rq, mlx5e_rq_zero_start);
2069 	err = mlx5e_open_cq(c->priv, &cparam->rx_cq, &c->rq.cq,
2078 	err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
2083 	c->rq.cq.mcq.comp(&c->rq.cq.mcq);
2091 	mlx5e_close_cq(&c->rq.cq);
2103 	mlx5e_close_rq(&c->rq);
2109 	mlx5e_close_rq_wait(&c->rq);
2298 	mlx5e_build_rq_param(priv, &cparam->rq);
2322 		err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j].rq);
2378 mlx5e_refresh_rq_params(struct mlx5e_priv *priv, struct mlx5e_rq *rq)
2399 		mtx_lock(&rq->mtx);
2400 		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
2401 		mtx_unlock(&rq->mtx);
2404 		cancel_work_sync(&rq->dim.work);
2411 			retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2415 			mtx_lock(&rq->mtx);
2416 			rq->dim.mode = dim_mode;
2417 			rq->dim.state = 0;
2418 			rq->dim.profile_ix = MLX5E_DIM_DEFAULT_PROFILE;
2419 			mtx_unlock(&rq->mtx);
2421 			retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2429 	return (mlx5_core_modify_cq_moderation(priv->mdev, &rq->cq.mcq,
2440 	err = mlx5e_refresh_rq_params(priv, &c->rq);
2559 		MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix].rq.rqn);
2651 		    priv->channel[0].rq.rqn);
3656 	struct mlx5e_rq *rq = &ch->rq;
3659 	mtx_lock(&rq->mtx);
3660 	rq->enabled = 0;
3661 	callout_stop(&rq->watchdog);
3662 	mtx_unlock(&rq->mtx);
3664 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
3666 		mlx5_en_err(rq->ifp,
3670 	while (!mlx5_wq_ll_is_empty(&rq->wq)) {
3672 		rq->cq.mcq.comp(&rq->cq.mcq);
3679 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_ERR, MLX5_RQC_STATE_RST);
3681 		mlx5_en_err(rq->ifp,
3689 	struct mlx5e_rq *rq = &ch->rq;
3692 	rq->wq.wqe_ctr = 0;
3693 	mlx5_wq_ll_update_db_record(&rq->wq);
3694 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3696 		mlx5_en_err(rq->ifp,
3700 	rq->enabled = 1;
3702 	rq->cq.mcq.comp(&rq->cq.mcq);