mlx5/core/cmd.c

119 	struct mlx5_cmd_work_ent *ent;
121 	ent = kzalloc(sizeof(*ent), alloc_flags);
122 	if (!ent)
125 	ent->idx	= -EINVAL;
126 	ent->in		= in;
127 	ent->out	= out;
128 	ent->uout	= uout;
129 	ent->uout_size	= uout_size;
130 	ent->callback	= cbk;
131 	ent->context	= context;
132 	ent->cmd	= cmd;
133 	ent->page_queue = page_queue;
134 	ent->op         = in_to_opcode(in->first.data);
135 	refcount_set(&ent->refcnt, 1);
137 	return ent;
140 static void cmd_free_ent(struct mlx5_cmd_work_ent *ent)
142 	kfree(ent);
159 static int cmd_alloc_index(struct mlx5_cmd *cmd, struct mlx5_cmd_work_ent *ent)
168 		ent->idx = ret;
169 		cmd->ent_arr[ent->idx] = ent;
182 static void cmd_ent_get(struct mlx5_cmd_work_ent *ent)
184 	refcount_inc(&ent->refcnt);
187 static void cmd_ent_put(struct mlx5_cmd_work_ent *ent)
189 	struct mlx5_cmd *cmd = ent->cmd;
193 	if (!refcount_dec_and_test(&ent->refcnt))
196 	if (ent->idx >= 0) {
197 		cmd_free_index(cmd, ent->idx);
198 		up(ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem);
201 	cmd_free_ent(ent);
267 static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)
269 	ent->lay->sig = ~xor8_buf(ent->lay, 0,  sizeof(*ent->lay));
271 		calc_chain_sig(ent->in);
272 		calc_chain_sig(ent->out);
276 static void poll_timeout(struct mlx5_cmd_work_ent *ent)
278 	struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd);
286 		own = READ_ONCE(ent->lay->status_own);
288 			ent->ret = 0;
294 	ent->ret = -ETIMEDOUT;
297 static int verify_signature(struct mlx5_cmd_work_ent *ent)
299 	struct mlx5_cmd_mailbox *next = ent->out->next;
300 	int n = mlx5_calc_cmd_blocks(ent->out);
305 	sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay));
859 			 struct mlx5_cmd_work_ent *ent, int input)
861 	struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;
864 	u16 op = ent->op;
870 	mlx5_core_dbg(dev, "cmd[%d]: start dump\n", ent->idx);
876 				   ent->idx, mlx5_command_str(op), op,
880 			      ent->idx, mlx5_command_str(op), op,
885 			dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset, ent->idx);
886 			offset += sizeof(ent->lay->in);
888 			dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset, ent->idx);
889 			offset += sizeof(ent->lay->out);
892 		dump_buf(ent->lay, sizeof(*ent->lay), 0, offset, ent->idx);
893 		offset += sizeof(*ent->lay);
899 			dump_buf(next->buf, dump_len, 1, offset, ent->idx);
902 			mlx5_core_dbg(dev, "cmd[%d]: command block:\n", ent->idx);
904 				 ent->idx);
913 	mlx5_core_dbg(dev, "cmd[%d]: end dump\n", ent->idx);
922 	struct mlx5_cmd_work_ent *ent = container_of(dwork,
925 	struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev,
931 	if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) {
932 		mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx,
933 			       mlx5_command_str(ent->op), ent->op);
937 	ent->ret = -ETIMEDOUT;
939 		       ent->idx, mlx5_command_str(ent->op), ent->op);
940 	mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
943 	cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */
967 	struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work);
968 	struct mlx5_cmd *cmd = ent->cmd;
969 	bool poll_cmd = ent->polling;
977 	complete(&ent->handling);
982 	if (!ent->page_queue) {
985 				       mlx5_command_str(ent->op), ent->op);
986 			if (ent->callback) {
987 				ent->callback(-EBUSY, ent->context);
988 				mlx5_free_cmd_msg(dev, ent->out);
989 				free_msg(dev, ent->in);
990 				cmd_ent_put(ent);
992 				ent->ret = -EBUSY;
993 				complete(&ent->done);
995 			complete(&ent->slotted);
998 		alloc_ret = cmd_alloc_index(cmd, ent);
1001 			if (ent->callback) {
1002 				ent->callback(-EAGAIN, ent->context);
1003 				mlx5_free_cmd_msg(dev, ent->out);
1004 				free_msg(dev, ent->in);
1005 				cmd_ent_put(ent);
1007 				ent->ret = -EAGAIN;
1008 				complete(&ent->done);
1015 		ent->idx = cmd->vars.max_reg_cmds;
1017 		clear_bit(ent->idx, &cmd->vars.bitmask);
1018 		cmd->ent_arr[ent->idx] = ent;
1022 	complete(&ent->slotted);
1024 	lay = get_inst(cmd, ent->idx);
1025 	ent->lay = lay;
1027 	memcpy(lay->in, ent->in->first.data, sizeof(lay->in));
1028 	if (ent->in->next)
1029 		lay->in_ptr = cpu_to_be64(ent->in->next->dma);
1030 	lay->inlen = cpu_to_be32(ent->in->len);
1031 	if (ent->out->next)
1032 		lay->out_ptr = cpu_to_be64(ent->out->next->dma);
1033 	lay->outlen = cpu_to_be32(ent->out->len);
1035 	lay->token = ent->token;
1037 	set_signature(ent, !cmd->checksum_disabled);
1038 	dump_command(dev, ent, 1);
1039 	ent->ts1 = ktime_get_ns();
1042 	if (ent->callback && schedule_delayed_work(&ent->cb_timeout_work, timeout))
1043 		cmd_ent_get(ent);
1044 	set_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state);
1046 	cmd_ent_get(ent); /* for the _real_ FW event on completion */
1048 	if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) {
1049 		ent->ret = -ENXIO;
1050 		mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
1055 	mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx);
1057 	iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);
1058 	/* if not in polling don't use ent after this point */
1060 		poll_timeout(ent);
1063 		mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, (ent->ret == -ETIMEDOUT));
1127 					  struct mlx5_cmd_work_ent *ent)
1133 	/* Re-wait on the ent->done after executing the recovery flow. If the
1138 	if (wait_for_completion_timeout(&ent->done, timeout)) {
1139 		mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx,
1140 			       mlx5_command_str(ent->op), ent->op);
1144 	mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx,
1145 		       mlx5_command_str(ent->op), ent->op);
1147 	ent->ret = -ETIMEDOUT;
1148 	mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
1151 static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
1157 	if (!wait_for_completion_timeout(&ent->handling, timeout) &&
1158 	    cancel_work_sync(&ent->work)) {
1159 		ent->ret = -ECANCELED;
1163 	wait_for_completion(&ent->slotted);
1165 	if (cmd->mode == CMD_MODE_POLLING || ent->polling)
1166 		wait_for_completion(&ent->done);
1167 	else if (!wait_for_completion_timeout(&ent->done, timeout))
1168 		wait_func_handle_exec_timeout(dev, ent);
1171 	err = ent->ret;
1175 			       mlx5_command_str(ent->op), ent->op);
1178 			       mlx5_command_str(ent->op), ent->op);
1181 			       mlx5_command_str(ent->op), ent->op);
1184 		      err, deliv_status_to_str(ent->status), ent->status);
1210 	struct mlx5_cmd_work_ent *ent;
1219 	ent = cmd_alloc_ent(cmd, in, out, uout, uout_size,
1221 	if (IS_ERR(ent))
1222 		return PTR_ERR(ent);
1224 	/* put for this ent is when consumed, depending on the use case
1226 	 * 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled
1229 	ent->token = token;
1230 	ent->polling = force_polling;
1232 	init_completion(&ent->handling);
1233 	init_completion(&ent->slotted);
1235 		init_completion(&ent->done);
1237 	INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler);
1238 	INIT_WORK(&ent->work, cmd_work_handler);
1240 		cmd_work_handler(&ent->work);
1241 	} else if (!queue_work(cmd->wq, &ent->work)) {
1248 		return 0; /* mlx5_cmd_comp_handler() will put(ent) */
1250 	err = wait_func(dev, ent);
1254 	ds = ent->ts2 - ent->ts1;
1255 	stats = xa_load(&cmd->stats, ent->op);
1264 			   mlx5_command_str(ent->op), ds);
1267 	status = ent->status;
1268 	cmd_ent_put(ent);
1673 	struct mlx5_cmd_work_ent *ent;
1687 			ent = cmd->ent_arr[i];
1691 						&ent->state)) {
1695 						      ent->idx);
1696 					cmd_ent_put(ent);
1701 			if (ent->callback && cancel_delayed_work(&ent->cb_timeout_work))
1702 				cmd_ent_put(ent); /* timeout work was canceled */
1706 			     !opcode_allowed(cmd, ent->op))
1707 				cmd_ent_put(ent);
1709 			ent->ts2 = ktime_get_ns();
1710 			memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out));
1711 			dump_command(dev, ent, 0);
1714 				ent->ret = -ENXIO;
1716 			if (!ent->ret) { /* Command completed by FW */
1718 					ent->ret = verify_signature(ent);
1720 				ent->status = ent->lay->status_own >> 1;
1723 					      ent->ret, deliv_status_to_str(ent->status), ent->status);
1726 			if (ent->callback) {
1727 				ds = ent->ts2 - ent->ts1;
1728 				stats = xa_load(&cmd->stats, ent->op);
1736 				callback = ent->callback;
1737 				context = ent->context;
1738 				err = ent->ret ? : ent->status;
1743 					err = mlx5_copy_from_msg(ent->uout,
1744 								 ent->out,
1745 								 ent->uout_size);
1747 				mlx5_free_cmd_msg(dev, ent->out);
1748 				free_msg(dev, ent->in);
1750 				/* final consumer is done, release ent */
1751 				cmd_ent_put(ent);
1757 				complete(&ent->done);