ufs/core/ufshcd.c

279 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
344 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
347 	struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
356 		header = &hba->lrb[tag].ucd_rsp_ptr->header;
373 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
376 	struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag];
413 static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
421 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
430 	ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
462 	trace_ufshcd_command(cmd->device, str_t, tag, doorbell, hwq_id,
538 void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt)
543 	lrbp = &hba->lrb[tag];
546 			tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000));
548 			tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000));
551 		tag, (u64)lrbp->utrd_dma_addr);
555 	dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
559 	dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
571 		tag, prdt_length,
585 	ufshcd_print_tr(hba, req->tag, *(bool *)priv);
602 	int tag;
604 	for_each_set_bit(tag, &bitmap, hba->nutmrs) {
605 		struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
607 		dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
2283  * @task_tag: Task tag of the command
2930 	int tag = scsi_cmd_to_rq(cmd)->tag;
2982 	lrbp = &hba->lrb[tag];
2984 	lrbp->task_tag = tag;
3003 	ufshcd_send_command(hba, tag, hwq);
3018 			     enum dev_cmd_type cmd_type, u8 lun, int tag)
3021 	lrbp->task_tag = tag;
3029 		struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
3031 	ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
3051  * @task_tag: The tag number of the command to be cleared.
3066 			dev_err(hba->dev, "%s: failed tag=%d. err=%d\n",
3156 		 * this function still owns the @lrbp tag so the code below does
3165 		dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
3183 			 * clear the task tag bit from the outstanding_reqs
3205 			dev_err(hba->dev, "%s: failed to clear tag %d\n",
3244 			  const u32 tag, int timeout)
3253 	ufshcd_send_command(hba, tag, hba->dev_cmd_queue);
3270  * NOTE: Since there is only one available tag for device management commands,
3276 	const u32 tag = hba->reserved_slot;
3277 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
3283 	err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
3287 	return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout);
5398 				"OCS error from controller = %x for tag %d\n",
5484  * @task_tag: the task tag of the request to be completed
5522 	int tag;
5524 	for_each_set_bit(tag, &completed_reqs, hba->nutrs)
5525 		ufshcd_compl_one_cqe(hba, tag, NULL);
5536 	int tag;
5538 	for_each_set_bit(tag, completed_reqs, hba->nutrs) {
5539 		struct scsi_cmnd *cmd = hba->lrb[tag].cmd;
5544 			__clear_bit(tag, completed_reqs);
5602 	int tag;
5604 	for (tag = 0; tag < hba->nutrs; tag++) {
5605 		lrbp = &hba->lrb[tag];
6442 	u32 tag = rq->tag;
6447 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
6451 	*ret = ufshcd_try_to_abort_task(hba, tag);
6452 	dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
6453 		hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
6476 	int tag, ret = 0;
6483 	for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
6484 		ret = ufshcd_clear_tm_cmd(hba, tag);
6857 	int tag;
6862 	for_each_set_bit(tag, &issued, hba->nutmrs) {
6863 		struct request *req = hba->tmf_rqs[tag];
6989 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
6992 	u32 mask = 1 << tag;
6995 	if (!test_bit(tag, &hba->outstanding_tasks))
6999 	ufshcd_utmrl_clear(hba, tag);
7007 	dev_err(hba->dev, "Clearing task management function with tag %d %s\n",
7008 		tag, err < 0 ? "failed" : "succeeded");
7025 	 * blk_mq_alloc_request() is used here only to get a free tag.
7036 	task_tag = req->tag;
7037 	hba->tmf_rqs[req->tag] = req;
7070 	hba->tmf_rqs[req->tag] = NULL;
7138  * Therefore, it "rides" the device management infrastructure: uses its tag and
7141  * Since there is only one available tag for device management commands,
7153 	const u32 tag = hba->reserved_slot;
7154 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7161 	ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
7165 	/* update the task tag in the request upiu */
7166 	req_upiu->header.task_tag = tag;
7186 	ufshcd_issue_dev_cmd(hba, lrbp, tag, QUERY_REQ_TIMEOUT);
7301 	const u32 tag = hba->reserved_slot;
7302 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7313 	ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag);
7317 	/* update the task tag */
7318 	req_upiu->header.task_tag = tag;
7330 	err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT);
7440 	int tag;
7442 	for_each_set_bit(tag, &bitmap, hba->nutrs) {
7443 		lrbp = &hba->lrb[tag];
7451  * @tag: Task tag/index to be aborted
7461 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
7463 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7474 			dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
7475 				__func__, tag);
7482 			dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
7483 				__func__, tag);
7492 				dev_err(hba->dev, "%s: cmd at tag=%d is cleared.\n",
7493 					__func__, tag);
7499 			if (reg & (1 << tag)) {
7505 			dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
7506 				__func__, tag);
7510 				"%s: no response from device. tag = %d, err %d\n",
7511 				__func__, tag, err);
7528 			dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
7529 				__func__, tag, err);
7534 	err = ufshcd_clear_cmd(hba, tag);
7536 		dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
7537 			__func__, tag, err);
7553 	int tag = scsi_cmd_to_rq(cmd)->tag;
7554 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7564 		if (!test_bit(tag, &hba->outstanding_reqs)) {
7567 				"%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
7568 				__func__, tag, hba->outstanding_reqs, reg);
7574 	dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
7585 		ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag);
7589 		ufshcd_print_tr(hba, tag, true);
7591 		ufshcd_print_tr(hba, tag, false);
7595 	if (!is_mcq_enabled(hba) && !(reg & (1 << tag))) {
7598 		"%s: cmd was completed, but without a notifying intr, tag = %d",
7599 		__func__, tag);
7600 		__ufshcd_transfer_req_compl(hba, 1UL << tag);
7635 	err = ufshcd_try_to_abort_task(hba, tag);
7648 	outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs);