sugar/linux
1
0
Fork 0
Code Activity

scsi: ufs: core: Optimize the hot path

Browse Source 
Set .cmd_size in the SCSI host template such that the SCSI core makes
struct scsi_cmnd and struct ufshcd_lrb adjacent. Convert the cmd->lrbp
and lrbp->cmd memory loads into pointer offset calculations. Remove the
data structure members that became superfluous, namely ufshcd_lrb.cmd
and ufs_hba.lrb. Since ufshcd_lrb.cmd is removed, this pointer cannot be
used anymore to test whether or not a command is a SCSI command.
Introduce a new function for this purpose, namely ufshcd_is_scsi_cmd().

Signed-off-by: Bart Van Assche <bvanassche@acm.org>
Link: https://patch.msgid.link/20251031204029.2883185-24-bvanassche@acm.org
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
This commit is contained in:
Bart Van Assche
2025-10-31 13:39:31 -07:00
committed by Martin K. Petersen
parent e5f9cc2af9
commit 22089c2180
5 changed files with 179 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
drivers/ufs/core/ufs-mcq.c
View File

@@ -534,8 +534,8 @@ static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
*/ */
int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag) int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)
{ {
struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
struct scsi_cmnd *cmd = lrbp->cmd; struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct ufs_hw_queue *hwq; struct ufs_hw_queue *hwq;
void __iomem *reg, *opr_sqd_base; void __iomem *reg, *opr_sqd_base;
u32 nexus, id, val; u32 nexus, id, val;
@@ -618,7 +618,8 @@ static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd)
static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba, static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,
struct ufs_hw_queue *hwq, int task_tag) struct ufs_hw_queue *hwq, int task_tag)
{ {
struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct utp_transfer_req_desc *utrd; struct utp_transfer_req_desc *utrd;
__le64 cmd_desc_base_addr; __le64 cmd_desc_base_addr;
bool ret = false; bool ret = false;
@@ -669,7 +670,7 @@ int ufshcd_mcq_abort(struct scsi_cmnd *cmd)
struct Scsi_Host *host = cmd->device->host; struct Scsi_Host *host = cmd->device->host;
struct ufs_hba *hba = shost_priv(host); struct ufs_hba *hba = shost_priv(host);
int tag = scsi_cmd_to_rq(cmd)->tag; int tag = scsi_cmd_to_rq(cmd)->tag;
struct ufshcd_lrb *lrbp = &hba->lrb[tag]; struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct ufs_hw_queue *hwq; struct ufs_hw_queue *hwq;
int err; int err;

18
drivers/ufs/core/ufshcd-crypto.h
View File

@@ -38,10 +38,10 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp,
} }
static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp) struct scsi_cmnd *cmd)
{ {
struct scsi_cmnd *cmd = lrbp->cmd;
const struct bio_crypt_ctx *crypt_ctx = scsi_cmd_to_rq(cmd)->crypt_ctx; const struct bio_crypt_ctx *crypt_ctx = scsi_cmd_to_rq(cmd)->crypt_ctx;
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
if (crypt_ctx && hba->vops && hba->vops->fill_crypto_prdt) if (crypt_ctx && hba->vops && hba->vops->fill_crypto_prdt)
return hba->vops->fill_crypto_prdt(hba, crypt_ctx, return hba->vops->fill_crypto_prdt(hba, crypt_ctx,
@@ -51,17 +51,19 @@ static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
} }
static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba, static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp) struct scsi_cmnd *cmd)
{ {
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT)) if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT))
return; return;
if (!(scsi_cmd_to_rq(lrbp->cmd)->crypt_ctx)) if (!(scsi_cmd_to_rq(cmd)->crypt_ctx))
return; return;
/* Zeroize the PRDT because it can contain cryptographic keys. */ /* Zeroize the PRDT because it can contain cryptographic keys. */
memzero_explicit(lrbp->ucd_prdt_ptr, memzero_explicit(lrbp->ucd_prdt_ptr,
ufshcd_sg_entry_size(hba) * scsi_sg_count(lrbp->cmd)); ufshcd_sg_entry_size(hba) * scsi_sg_count(cmd));
} }
bool ufshcd_crypto_enable(struct ufs_hba *hba); bool ufshcd_crypto_enable(struct ufs_hba *hba);
@@ -82,13 +84,15 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp,
struct request_desc_header *h) { } struct request_desc_header *h) { }
static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp) struct scsi_cmnd *cmd)
{ {
return 0; return 0;
} }
static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba, static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp) { } struct scsi_cmnd *cmd)
{
}
static inline bool ufshcd_crypto_enable(struct ufs_hba *hba) static inline bool ufshcd_crypto_enable(struct ufs_hba *hba)
{ {

41
drivers/ufs/core/ufshcd-priv.h
View File

@@ -77,8 +77,7 @@ bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd);
int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag); int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag);
int ufshcd_mcq_abort(struct scsi_cmnd *cmd); int ufshcd_mcq_abort(struct scsi_cmnd *cmd);
int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag); int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
void ufshcd_release_scsi_cmd(struct ufs_hba *hba, void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd);
struct ufshcd_lrb *lrbp);
#define SD_ASCII_STD true #define SD_ASCII_STD true
#define SD_RAW false #define SD_RAW false
@@ -363,6 +362,44 @@ static inline bool ufs_is_valid_unit_desc_lun(struct ufs_dev_info *dev_info, u8
return lun == UFS_UPIU_RPMB_WLUN || (lun < dev_info->max_lu_supported); return lun == UFS_UPIU_RPMB_WLUN || (lun < dev_info->max_lu_supported);
} }
/*
* Convert a block layer tag into a SCSI command pointer. This function is
* called once per I/O completion path and is also called from error paths.
*/
static inline struct scsi_cmnd *ufshcd_tag_to_cmd(struct ufs_hba *hba, u32 tag)
{
struct blk_mq_tags *tags = hba->host->tag_set.shared_tags;
struct request *rq;
/*
* Handle reserved tags differently because the UFS driver does not
* call blk_mq_alloc_request() for allocating reserved requests.
* Allocating reserved tags with blk_mq_alloc_request() would require
* the following:
* - Allocate an additional request queue from &hba->host->tag_set for
* allocating reserved requests from.
* - For that request queue, allocate a SCSI device.
* - Calling blk_mq_alloc_request(hba->dev_mgmt_queue, REQ_OP_DRV_OUT,
* BLK_MQ_REQ_RESERVED) for allocating a reserved request and
* blk_mq_free_request() for freeing reserved requests.
* - Set the .device pointer for these reserved requests.
* - Submit reserved requests with blk_execute_rq().
* - Modify ufshcd_queuecommand() such that it handles reserved requests
* in another way than SCSI requests.
* - Modify ufshcd_compl_one_cqe() such that it calls scsi_done() for
* device management commands.
* - Modify all callback functions called by blk_mq_tagset_busy_iter()
* calls in the UFS driver and skip device management commands.
*/
rq = tag < UFSHCD_NUM_RESERVED ? tags->static_rqs[tag] :
blk_mq_tag_to_rq(tags, tag);
if (WARN_ON_ONCE(!rq))
return NULL;
return blk_mq_rq_to_pdu(rq);
}
static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q) static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q)
__must_hold(&q->sq_lock) __must_hold(&q->sq_lock)
{ {

229
drivers/ufs/core/ufshcd.c
View File

@@ -28,6 +28,7 @@
#include <scsi/scsi_dbg.h> #include <scsi/scsi_dbg.h>
#include <scsi/scsi_driver.h> #include <scsi/scsi_driver.h>
#include <scsi/scsi_eh.h> #include <scsi/scsi_eh.h>
#include <scsi/scsi_tcq.h>
#include "ufshcd-priv.h" #include "ufshcd-priv.h"
#include <ufs/ufs_quirks.h> #include <ufs/ufs_quirks.h>
#include <ufs/unipro.h> #include <ufs/unipro.h>
@@ -483,7 +484,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd,
u32 hwq_id = 0; u32 hwq_id = 0;
struct request *rq = scsi_cmd_to_rq(cmd); struct request *rq = scsi_cmd_to_rq(cmd);
unsigned int tag = rq->tag; unsigned int tag = rq->tag;
struct ufshcd_lrb *lrbp = &hba->lrb[tag]; struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int transfer_len = -1; int transfer_len = -1;
/* trace UPIU also */ /* trace UPIU also */
@@ -594,14 +595,13 @@ static void ufshcd_print_evt_hist(struct ufs_hba *hba)
ufshcd_vops_dbg_register_dump(hba); ufshcd_vops_dbg_register_dump(hba);
} }
static static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd,
void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt) bool pr_prdt)
{ {
const struct ufshcd_lrb *lrbp; struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
const int tag = lrbp->task_tag;
int prdt_length; int prdt_length;
lrbp = &hba->lrb[tag];
if (hba->monitor.enabled) { if (hba->monitor.enabled) {
dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag, dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag,
div_u64(lrbp->issue_time_stamp_local_clock, 1000)); div_u64(lrbp->issue_time_stamp_local_clock, 1000));
@@ -644,7 +644,7 @@ static bool ufshcd_print_tr_iter(struct request *req, void *priv)
struct Scsi_Host *shost = sdev->host; struct Scsi_Host *shost = sdev->host;
struct ufs_hba *hba = shost_priv(shost); struct ufs_hba *hba = shost_priv(shost);
ufshcd_print_tr(hba, req->tag, *(bool *)priv); ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv);
return true; return true;
} }
@@ -2298,8 +2298,7 @@ static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
struct scsi_cmnd *cmd) struct scsi_cmnd *cmd)
{ {
const struct ufs_hba_monitor *m = &hba->monitor; const struct ufs_hba_monitor *m = &hba->monitor;
struct request *rq = scsi_cmd_to_rq(cmd); struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag];
return m->enabled && return m->enabled &&
(!m->chunk_size || m->chunk_size == cmd->sdb.length) && (!m->chunk_size || m->chunk_size == cmd->sdb.length) &&
@@ -2320,7 +2319,7 @@ static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{ {
struct request *req = scsi_cmd_to_rq(cmd); struct request *req = scsi_cmd_to_rq(cmd);
struct ufshcd_lrb *lrbp = &hba->lrb[req->tag]; const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]);
unsigned long flags; unsigned long flags;
@@ -2350,17 +2349,26 @@ static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
spin_unlock_irqrestore(hba->host->host_lock, flags); spin_unlock_irqrestore(hba->host->host_lock, flags);
} }
/*
* Returns %true for SCSI commands and %false for device management commands.
* Must not be called for SCSI commands that have not yet been started.
*/
static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd)
{
return blk_mq_request_started(scsi_cmd_to_rq(cmd));
}
/** /**
* ufshcd_send_command - Send SCSI or device management commands * ufshcd_send_command - Send SCSI or device management commands
* @hba: per adapter instance * @hba: per adapter instance
* @lrbp: Local reference block of SCSI command * @cmd: SCSI command or device management command pointer
* @hwq: pointer to hardware queue instance * @hwq: pointer to hardware queue instance
*/ */
static inline void ufshcd_send_command(struct ufs_hba *hba, static inline void ufshcd_send_command(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp, struct scsi_cmnd *cmd,
struct ufs_hw_queue *hwq) struct ufs_hw_queue *hwq)
{ {
struct scsi_cmnd *cmd = lrbp->cmd; struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
unsigned long flags; unsigned long flags;
if (hba->monitor.enabled) { if (hba->monitor.enabled) {
@@ -2369,7 +2377,7 @@ static inline void ufshcd_send_command(struct ufs_hba *hba,
lrbp->compl_time_stamp = ktime_set(0, 0); lrbp->compl_time_stamp = ktime_set(0, 0);
lrbp->compl_time_stamp_local_clock = 0; lrbp->compl_time_stamp_local_clock = 0;
} }
if (cmd) { if (ufshcd_is_scsi_cmd(cmd)) {
ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND); ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND);
ufshcd_clk_scaling_start_busy(hba); ufshcd_clk_scaling_start_busy(hba);
if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) if (unlikely(ufshcd_should_inform_monitor(hba, cmd)))
@@ -2389,7 +2397,8 @@ static inline void ufshcd_send_command(struct ufs_hba *hba,
} else { } else {
spin_lock_irqsave(&hba->outstanding_lock, flags); spin_lock_irqsave(&hba->outstanding_lock, flags);
if (hba->vops && hba->vops->setup_xfer_req) if (hba->vops && hba->vops->setup_xfer_req)
hba->vops->setup_xfer_req(hba, lrbp->task_tag, !!cmd); hba->vops->setup_xfer_req(hba, lrbp->task_tag,
ufshcd_is_scsi_cmd(cmd));
__set_bit(lrbp->task_tag, &hba->outstanding_reqs); __set_bit(lrbp->task_tag, &hba->outstanding_reqs);
ufshcd_writel(hba, 1 << lrbp->task_tag, ufshcd_writel(hba, 1 << lrbp->task_tag,
REG_UTP_TRANSFER_REQ_DOOR_BELL); REG_UTP_TRANSFER_REQ_DOOR_BELL);
@@ -2399,11 +2408,12 @@ static inline void ufshcd_send_command(struct ufs_hba *hba,
/** /**
* ufshcd_copy_sense_data - Copy sense data in case of check condition * ufshcd_copy_sense_data - Copy sense data in case of check condition
* @lrbp: pointer to local reference block * @cmd: SCSI command
*/ */
static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd)
{ {
u8 *const sense_buffer = lrbp->cmd->sense_buffer; struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
u8 *const sense_buffer = cmd->sense_buffer;
u16 resp_len; u16 resp_len;
int len; int len;
@@ -2708,13 +2718,13 @@ static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int
/** /**
* ufshcd_map_sg - Map scatter-gather list to prdt * ufshcd_map_sg - Map scatter-gather list to prdt
* @hba: per adapter instance * @hba: per adapter instance
* @lrbp: pointer to local reference block * @cmd: SCSI command
* *
* Return: 0 in case of success, non-zero value in case of failure. * Return: 0 in case of success, non-zero value in case of failure.
*/ */
static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{ {
struct scsi_cmnd *cmd = lrbp->cmd; struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int sg_segments = scsi_dma_map(cmd); int sg_segments = scsi_dma_map(cmd);
if (sg_segments < 0) if (sg_segments < 0)
@@ -2722,7 +2732,7 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd)); ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd));
return ufshcd_crypto_fill_prdt(hba, lrbp); return ufshcd_crypto_fill_prdt(hba, cmd);
} }
/** /**
@@ -2781,13 +2791,13 @@ ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
/** /**
* ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
* for scsi commands * for scsi commands
* @lrbp: local reference block pointer * @cmd: SCSI command
* @upiu_flags: flags * @upiu_flags: flags
*/ */
static static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd,
void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags) u8 upiu_flags)
{ {
struct scsi_cmnd *cmd = lrbp->cmd; struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
unsigned short cdb_len; unsigned short cdb_len;
@@ -2890,22 +2900,25 @@ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
* ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU) * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
* for SCSI Purposes * for SCSI Purposes
* @hba: per adapter instance * @hba: per adapter instance
* @lrbp: pointer to local reference block * @cmd: SCSI command
*/ */
static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{ {
struct request *rq = scsi_cmd_to_rq(lrbp->cmd); struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct request *rq = scsi_cmd_to_rq(cmd);
unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
u8 upiu_flags; u8 upiu_flags;
ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0); ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags,
cmd->sc_data_direction, 0);
if (ioprio_class == IOPRIO_CLASS_RT) if (ioprio_class == IOPRIO_CLASS_RT)
upiu_flags |= UPIU_CMD_FLAGS_CP; upiu_flags |= UPIU_CMD_FLAGS_CP;
ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags);
} }
static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{ {
const int i = scsi_cmd_to_rq(cmd)->tag;
struct utp_transfer_cmd_desc *cmd_descp = struct utp_transfer_cmd_desc *cmd_descp =
(void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba); (void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba);
struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
@@ -2913,6 +2926,7 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba); hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba);
u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset);
u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset);
struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd);
lrb->utr_descriptor_ptr = utrdlp + i; lrb->utr_descriptor_ptr = utrdlp + i;
lrb->utrd_dma_addr = lrb->utrd_dma_addr =
@@ -2925,27 +2939,31 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
} }
static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
struct scsi_cmnd *cmd, u8 lun, int tag) u8 lun, int tag)
{ {
ufshcd_init_lrb(hba, lrbp, tag); struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
ufshcd_init_lrb(hba, cmd);
memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr)); memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr));
lrbp->cmd = cmd;
lrbp->task_tag = tag; lrbp->task_tag = tag;
lrbp->lun = lun; lrbp->lun = lun;
ufshcd_prepare_lrbp_crypto(cmd ? scsi_cmd_to_rq(cmd) : NULL, lrbp); ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ?
scsi_cmd_to_rq(cmd) : NULL, lrbp);
} }
static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
struct scsi_cmnd *cmd, u8 lun, int tag) u8 lun, int tag)
{ {
__ufshcd_setup_cmd(hba, lrbp, cmd, lun, tag); struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
__ufshcd_setup_cmd(hba, cmd, lun, tag);
lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
lrbp->req_abort_skip = false; lrbp->req_abort_skip = false;
ufshcd_comp_scsi_upiu(hba, lrbp); ufshcd_comp_scsi_upiu(hba, cmd);
} }
/** /**
@@ -3016,7 +3034,6 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
{ {
struct ufs_hba *hba = shost_priv(host); struct ufs_hba *hba = shost_priv(host);
int tag = scsi_cmd_to_rq(cmd)->tag; int tag = scsi_cmd_to_rq(cmd)->tag;
struct ufshcd_lrb *lrbp;
int err = 0; int err = 0;
struct ufs_hw_queue *hwq = NULL; struct ufs_hw_queue *hwq = NULL;
@@ -3067,11 +3084,10 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
ufshcd_hold(hba); ufshcd_hold(hba);
lrbp = &hba->lrb[tag]; ufshcd_setup_scsi_cmd(hba, cmd,
ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag);
ufshcd_setup_scsi_cmd(hba, lrbp, cmd, ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); err = ufshcd_map_sg(hba, cmd);
err = ufshcd_map_sg(hba, lrbp);
if (err) { if (err) {
ufshcd_release(hba); ufshcd_release(hba);
goto out; goto out;
@@ -3080,7 +3096,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
if (hba->mcq_enabled) if (hba->mcq_enabled)
hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
ufshcd_send_command(hba, lrbp, hwq); ufshcd_send_command(hba, cmd, hwq);
out: out:
if (ufs_trigger_eh(hba)) { if (ufs_trigger_eh(hba)) {
@@ -3094,10 +3110,12 @@ out:
return err; return err;
} }
static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
enum dev_cmd_type cmd_type, u8 lun, int tag) enum dev_cmd_type cmd_type, u8 lun, int tag)
{ {
__ufshcd_setup_cmd(hba, lrbp, NULL, lun, tag); struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
__ufshcd_setup_cmd(hba, cmd, lun, tag);
lrbp->intr_cmd = true; /* No interrupt aggregation */ lrbp->intr_cmd = true; /* No interrupt aggregation */
hba->dev_cmd.type = cmd_type; hba->dev_cmd.type = cmd_type;
} }
@@ -3105,10 +3123,12 @@ static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
/* /*
* Return: 0 upon success; < 0 upon failure. * Return: 0 upon success; < 0 upon failure.
*/ */
static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag) enum dev_cmd_type cmd_type, int tag)
{ {
ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag); struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag);
return ufshcd_compose_devman_upiu(hba, lrbp); return ufshcd_compose_devman_upiu(hba, lrbp);
} }
@@ -3320,13 +3340,14 @@ static void ufshcd_dev_man_unlock(struct ufs_hba *hba)
* Return: 0 upon success; > 0 in case the UFS device reported an OCS error; * Return: 0 upon success; > 0 in case the UFS device reported an OCS error;
* < 0 if another error occurred. * < 0 if another error occurred.
*/ */
static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
const u32 tag, int timeout) const u32 tag, int timeout)
{ {
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err; int err;
ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
ufshcd_send_command(hba, lrbp, hba->dev_cmd_queue); ufshcd_send_command(hba, cmd, hba->dev_cmd_queue);
err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
@@ -3351,17 +3372,17 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
enum dev_cmd_type cmd_type, int timeout) enum dev_cmd_type cmd_type, int timeout)
{ {
const u32 tag = hba->reserved_slot; const u32 tag = hba->reserved_slot;
struct ufshcd_lrb *lrbp = &hba->lrb[tag]; struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
int err; int err;
/* Protects use of hba->reserved_slot. */ /* Protects use of hba->reserved_slot. */
lockdep_assert_held(&hba->dev_cmd.lock); lockdep_assert_held(&hba->dev_cmd.lock);
err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag);
if (unlikely(err)) if (unlikely(err))
return err; return err;
return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout); return ufshcd_issue_dev_cmd(hba, cmd, tag, timeout);
} }
/** /**
@@ -3991,14 +4012,6 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba)
} }
skip_utmrdl: skip_utmrdl:
/* Allocate memory for local reference block */
hba->lrb = devm_kcalloc(hba->dev,
hba->nutrs, sizeof(struct ufshcd_lrb),
GFP_KERNEL);
if (!hba->lrb) {
dev_err(hba->dev, "LRB Memory allocation failed\n");
goto out;
}
return 0; return 0;
out: out:
return -ENOMEM; return -ENOMEM;
@@ -5411,19 +5424,18 @@ static void ufshcd_sdev_destroy(struct scsi_device *sdev)
/** /**
* ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
* @lrbp: pointer to local reference block of completed command * @cmd: SCSI command
* @scsi_status: SCSI command status * @scsi_status: SCSI command status
* *
* Return: value base on SCSI command status. * Return: value base on SCSI command status.
*/ */
static inline int static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status)
ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
{ {
int result = 0; int result = 0;
switch (scsi_status) { switch (scsi_status) {
case SAM_STAT_CHECK_CONDITION: case SAM_STAT_CHECK_CONDITION:
ufshcd_copy_sense_data(lrbp); ufshcd_copy_sense_data(cmd);
fallthrough; fallthrough;
case SAM_STAT_GOOD: case SAM_STAT_GOOD:
result |= DID_OK << 16 | scsi_status; result |= DID_OK << 16 | scsi_status;
@@ -5431,7 +5443,7 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
case SAM_STAT_TASK_SET_FULL: case SAM_STAT_TASK_SET_FULL:
case SAM_STAT_BUSY: case SAM_STAT_BUSY:
case SAM_STAT_TASK_ABORTED: case SAM_STAT_TASK_ABORTED:
ufshcd_copy_sense_data(lrbp); ufshcd_copy_sense_data(cmd);
result |= scsi_status; result |= scsi_status;
break; break;
default: default:
@@ -5445,15 +5457,16 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
/** /**
* ufshcd_transfer_rsp_status - Get overall status of the response * ufshcd_transfer_rsp_status - Get overall status of the response
* @hba: per adapter instance * @hba: per adapter instance
* @lrbp: pointer to local reference block of completed command * @cmd: SCSI command
* @cqe: pointer to the completion queue entry * @cqe: pointer to the completion queue entry
* *
* Return: result of the command to notify SCSI midlayer. * Return: result of the command to notify SCSI midlayer.
*/ */
static inline int static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba,
ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, struct scsi_cmnd *cmd,
struct cq_entry *cqe) struct cq_entry *cqe)
{ {
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int result = 0; int result = 0;
int scsi_status; int scsi_status;
enum utp_ocs ocs; enum utp_ocs ocs;
@@ -5467,7 +5480,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
* not set either flag. * not set either flag.
*/ */
if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW)) if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW))
scsi_set_resid(lrbp->cmd, resid); scsi_set_resid(cmd, resid);
/* overall command status of utrd */ /* overall command status of utrd */
ocs = ufshcd_get_tr_ocs(lrbp, cqe); ocs = ufshcd_get_tr_ocs(lrbp, cqe);
@@ -5488,7 +5501,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
* to notify the SCSI midlayer of the command status * to notify the SCSI midlayer of the command status
*/ */
scsi_status = lrbp->ucd_rsp_ptr->header.status; scsi_status = lrbp->ucd_rsp_ptr->header.status;
result = ufshcd_scsi_cmd_status(lrbp, scsi_status); result = ufshcd_scsi_cmd_status(cmd, scsi_status);
/* /*
* Currently we are only supporting BKOPs exception * Currently we are only supporting BKOPs exception
@@ -5553,7 +5566,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
(host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) { (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) {
if (cqe) if (cqe)
ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry)); ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry));
ufshcd_print_tr(hba, lrbp->task_tag, true); ufshcd_print_tr(hba, cmd, true);
} }
return result; return result;
} }
@@ -5620,13 +5633,10 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
} }
/* Release the resources allocated for processing a SCSI command. */ /* Release the resources allocated for processing a SCSI command. */
void ufshcd_release_scsi_cmd(struct ufs_hba *hba, void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd)
struct ufshcd_lrb *lrbp)
{ {
struct scsi_cmnd *cmd = lrbp->cmd;
scsi_dma_unmap(cmd); scsi_dma_unmap(cmd);
ufshcd_crypto_clear_prdt(hba, lrbp); ufshcd_crypto_clear_prdt(hba, cmd);
ufshcd_release(hba); ufshcd_release(hba);
ufshcd_clk_scaling_update_busy(hba); ufshcd_clk_scaling_update_busy(hba);
} }
@@ -5640,20 +5650,20 @@ void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag,
struct cq_entry *cqe) struct cq_entry *cqe)
{ {
struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
struct scsi_cmnd *cmd = lrbp->cmd; struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
enum utp_ocs ocs; enum utp_ocs ocs;
if (hba->monitor.enabled) { if (hba->monitor.enabled) {
lrbp->compl_time_stamp = ktime_get(); lrbp->compl_time_stamp = ktime_get();
lrbp->compl_time_stamp_local_clock = local_clock(); lrbp->compl_time_stamp_local_clock = local_clock();
} }
if (cmd) { if (ufshcd_is_scsi_cmd(cmd)) {
if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) if (unlikely(ufshcd_should_inform_monitor(hba, cmd)))
ufshcd_update_monitor(hba, cmd); ufshcd_update_monitor(hba, cmd);
ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP); ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP);
cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe); cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe);
ufshcd_release_scsi_cmd(hba, lrbp); ufshcd_release_scsi_cmd(hba, cmd);
/* Do not touch lrbp after scsi done */ /* Do not touch lrbp after scsi done */
scsi_done(cmd); scsi_done(cmd);
} else { } else {
@@ -5690,7 +5700,7 @@ static void ufshcd_clear_polled(struct ufs_hba *hba,
int tag; int tag;
for_each_set_bit(tag, completed_reqs, hba->nutrs) { for_each_set_bit(tag, completed_reqs, hba->nutrs) {
struct scsi_cmnd *cmd = hba->lrb[tag].cmd; struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag);
if (!cmd) if (!cmd)
continue; continue;
@@ -5741,7 +5751,6 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv)
struct scsi_device *sdev = rq->q->queuedata; struct scsi_device *sdev = rq->q->queuedata;
struct Scsi_Host *shost = sdev->host; struct Scsi_Host *shost = sdev->host;
struct ufs_hba *hba = shost_priv(shost); struct ufs_hba *hba = shost_priv(shost);
struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag];
struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq);
if (!hwq) if (!hwq)
@@ -5756,7 +5765,7 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv)
scoped_guard(spinlock_irqsave, &hwq->cq_lock) { scoped_guard(spinlock_irqsave, &hwq->cq_lock) {
if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) {
set_host_byte(cmd, DID_REQUEUE); set_host_byte(cmd, DID_REQUEUE);
ufshcd_release_scsi_cmd(hba, lrbp); ufshcd_release_scsi_cmd(hba, cmd);
scsi_done(cmd); scsi_done(cmd);
} }
} }
@@ -6641,7 +6650,7 @@ static bool ufshcd_abort_one(struct request *rq, void *priv)
*ret = ufshcd_try_to_abort_task(hba, tag); *ret = ufshcd_try_to_abort_task(hba, tag);
dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1, ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1,
*ret ? "failed" : "succeeded"); *ret ? "failed" : "succeeded");
return *ret == 0; return *ret == 0;
@@ -7371,14 +7380,15 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
enum query_opcode desc_op) enum query_opcode desc_op)
{ {
const u32 tag = hba->reserved_slot; const u32 tag = hba->reserved_slot;
struct ufshcd_lrb *lrbp = &hba->lrb[tag]; struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err = 0; int err = 0;
u8 upiu_flags; u8 upiu_flags;
/* Protects use of hba->reserved_slot. */ /* Protects use of hba->reserved_slot. */
lockdep_assert_held(&hba->dev_cmd.lock); lockdep_assert_held(&hba->dev_cmd.lock);
ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag); ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag);
ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0);
@@ -7403,7 +7413,7 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
* bound to fail since dev_cmd.query and dev_cmd.type were left empty. * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
* read the response directly ignoring all errors. * read the response directly ignoring all errors.
*/ */
ufshcd_issue_dev_cmd(hba, lrbp, tag, dev_cmd_timeout); ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout);
/* just copy the upiu response as it is */ /* just copy the upiu response as it is */
memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
@@ -7518,7 +7528,8 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
enum dma_data_direction dir) enum dma_data_direction dir)
{ {
const u32 tag = hba->reserved_slot; const u32 tag = hba->reserved_slot;
struct ufshcd_lrb *lrbp = &hba->lrb[tag]; struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err = 0; int err = 0;
int result; int result;
u8 upiu_flags; u8 upiu_flags;
@@ -7529,7 +7540,8 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
/* Protects use of hba->reserved_slot. */ /* Protects use of hba->reserved_slot. */
ufshcd_dev_man_lock(hba); ufshcd_dev_man_lock(hba);
ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag); ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN,
tag);
ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs); ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs);
@@ -7546,7 +7558,7 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT); err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT);
if (!err) { if (!err) {
/* Just copy the upiu response as it is */ /* Just copy the upiu response as it is */
@@ -7647,11 +7659,12 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
{ {
struct ufshcd_lrb *lrbp;
int tag; int tag;
for_each_set_bit(tag, &bitmap, hba->nutrs) { for_each_set_bit(tag, &bitmap, hba->nutrs) {
lrbp = &hba->lrb[tag]; struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
lrbp->req_abort_skip = true; lrbp->req_abort_skip = true;
} }
} }
@@ -7659,7 +7672,7 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
/** /**
* ufshcd_try_to_abort_task - abort a specific task * ufshcd_try_to_abort_task - abort a specific task
* @hba: Pointer to adapter instance * @hba: Pointer to adapter instance
* @tag: Task tag/index to be aborted * @tag: Tag of the task to be aborted
* *
* Abort the pending command in device by sending UFS_ABORT_TASK task management * Abort the pending command in device by sending UFS_ABORT_TASK task management
* command, and in host controller by clearing the door-bell register. There can * command, and in host controller by clearing the door-bell register. There can
@@ -7671,7 +7684,8 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
*/ */
int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
{ {
struct ufshcd_lrb *lrbp = &hba->lrb[tag]; struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err; int err;
int poll_cnt; int poll_cnt;
u8 resp = 0xF; u8 resp = 0xF;
@@ -7693,7 +7707,7 @@ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
hba->dev, hba->dev,
"%s: cmd with tag %d not pending in the device.\n", "%s: cmd with tag %d not pending in the device.\n",
__func__, tag); __func__, tag);
if (!ufshcd_cmd_inflight(lrbp->cmd)) { if (!ufshcd_cmd_inflight(cmd)) {
dev_info(hba->dev, dev_info(hba->dev,
"%s: cmd with tag=%d completed.\n", "%s: cmd with tag=%d completed.\n",
__func__, tag); __func__, tag);
@@ -7741,7 +7755,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
struct Scsi_Host *host = cmd->device->host; struct Scsi_Host *host = cmd->device->host;
struct ufs_hba *hba = shost_priv(host); struct ufs_hba *hba = shost_priv(host);
int tag = scsi_cmd_to_rq(cmd)->tag; int tag = scsi_cmd_to_rq(cmd)->tag;
struct ufshcd_lrb *lrbp = &hba->lrb[tag]; struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
unsigned long flags; unsigned long flags;
int err = FAILED; int err = FAILED;
bool outstanding; bool outstanding;
@@ -7776,9 +7790,9 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
ufshcd_print_evt_hist(hba); ufshcd_print_evt_hist(hba);
ufshcd_print_host_state(hba); ufshcd_print_host_state(hba);
ufshcd_print_pwr_info(hba); ufshcd_print_pwr_info(hba);
ufshcd_print_tr(hba, tag, true); ufshcd_print_tr(hba, cmd, true);
} else { } else {
ufshcd_print_tr(hba, tag, false); ufshcd_print_tr(hba, cmd, false);
} }
hba->req_abort_count++; hba->req_abort_count++;
@@ -7822,7 +7836,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
goto release; goto release;
} }
err = ufshcd_try_to_abort_task(hba, tag); err = ufshcd_try_to_abort_task(hba, lrbp->task_tag);
if (err) { if (err) {
dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
@@ -7839,7 +7853,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
spin_unlock_irqrestore(&hba->outstanding_lock, flags); spin_unlock_irqrestore(&hba->outstanding_lock, flags);
if (outstanding) if (outstanding)
ufshcd_release_scsi_cmd(hba, lrbp); ufshcd_release_scsi_cmd(hba, cmd);
err = SUCCESS; err = SUCCESS;
@@ -8919,8 +8933,6 @@ static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs)
utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs; utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs;
dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr, dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr,
hba->utrdl_dma_addr); hba->utrdl_dma_addr);
devm_kfree(hba->dev, hba->lrb);
} }
static int ufshcd_alloc_mcq(struct ufs_hba *hba) static int ufshcd_alloc_mcq(struct ufs_hba *hba)
@@ -9191,6 +9203,7 @@ static const struct scsi_host_template ufshcd_driver_template = {
.name = UFSHCD, .name = UFSHCD,
.proc_name = UFSHCD, .proc_name = UFSHCD,
.map_queues = ufshcd_map_queues, .map_queues = ufshcd_map_queues,
.cmd_size = sizeof(struct ufshcd_lrb),
.init_cmd_priv = ufshcd_init_cmd_priv, .init_cmd_priv = ufshcd_init_cmd_priv,
.queuecommand = ufshcd_queuecommand, .queuecommand = ufshcd_queuecommand,
.nr_reserved_cmds = UFSHCD_NUM_RESERVED, .nr_reserved_cmds = UFSHCD_NUM_RESERVED,

5
include/ufs/ufshcd.h
View File

@@ -161,7 +161,6 @@ struct ufs_pm_lvl_states {
* @ucd_prdt_dma_addr: PRDT dma address for debug * @ucd_prdt_dma_addr: PRDT dma address for debug
* @ucd_rsp_dma_addr: UPIU response dma address for debug * @ucd_rsp_dma_addr: UPIU response dma address for debug
* @ucd_req_dma_addr: UPIU request dma address for debug * @ucd_req_dma_addr: UPIU request dma address for debug
* @cmd: pointer to SCSI command
* @scsi_status: SCSI status of the command * @scsi_status: SCSI status of the command
* @command_type: SCSI, UFS, Query. * @command_type: SCSI, UFS, Query.
* @task_tag: Task tag of the command * @task_tag: Task tag of the command
@@ -186,7 +185,6 @@ struct ufshcd_lrb {
dma_addr_t ucd_rsp_dma_addr; dma_addr_t ucd_rsp_dma_addr;
dma_addr_t ucd_prdt_dma_addr; dma_addr_t ucd_prdt_dma_addr;
struct scsi_cmnd *cmd;
int scsi_status; int scsi_status;
int command_type; int command_type;
@@ -833,7 +831,6 @@ enum ufshcd_mcq_opr {
* @spm_lvl: desired UFS power management level during system PM. * @spm_lvl: desired UFS power management level during system PM.
* @pm_op_in_progress: whether or not a PM operation is in progress. * @pm_op_in_progress: whether or not a PM operation is in progress.
* @ahit: value of Auto-Hibernate Idle Timer register. * @ahit: value of Auto-Hibernate Idle Timer register.
* @lrb: local reference block
* @outstanding_tasks: Bits representing outstanding task requests * @outstanding_tasks: Bits representing outstanding task requests
* @outstanding_lock: Protects @outstanding_reqs. * @outstanding_lock: Protects @outstanding_reqs.
* @outstanding_reqs: Bits representing outstanding transfer requests * @outstanding_reqs: Bits representing outstanding transfer requests
@@ -976,8 +973,6 @@ struct ufs_hba {
/* Auto-Hibernate Idle Timer register value */ /* Auto-Hibernate Idle Timer register value */
u32 ahit; u32 ahit;
struct ufshcd_lrb *lrb;
unsigned long outstanding_tasks; unsigned long outstanding_tasks;
spinlock_t outstanding_lock; spinlock_t outstanding_lock;
unsigned long outstanding_reqs; unsigned long outstanding_reqs;