sugar/linux
1
0
Fork 0
Code Activity

scsi: ufs: core: Optimize the hot path

Browse Source 
Set .cmd_size in the SCSI host template such that the SCSI core makes
struct scsi_cmnd and struct ufshcd_lrb adjacent. Convert the cmd->lrbp
and lrbp->cmd memory loads into pointer offset calculations. Remove the
data structure members that became superfluous, namely ufshcd_lrb.cmd
and ufs_hba.lrb. Since ufshcd_lrb.cmd is removed, this pointer cannot be
used anymore to test whether or not a command is a SCSI command.
Introduce a new function for this purpose, namely ufshcd_is_scsi_cmd().

Signed-off-by: Bart Van Assche <bvanassche@acm.org>
Link: https://patch.msgid.link/20251031204029.2883185-24-bvanassche@acm.org
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
This commit is contained in:
Bart Van Assche
2025-10-31 13:39:31 -07:00
committed by Martin K. Petersen
parent e5f9cc2af9
commit 22089c2180
5 changed files with 179 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
drivers/ufs/core/ufs-mcq.c
View File

@@ -534,8 +534,8 @@ static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
*/
int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)
{
struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
struct scsi_cmnd *cmd = lrbp->cmd;
struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct ufs_hw_queue *hwq;
void __iomem *reg, *opr_sqd_base;
u32 nexus, id, val;
@@ -618,7 +618,8 @@ static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd)
static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,
struct ufs_hw_queue *hwq, int task_tag)
{
struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct utp_transfer_req_desc *utrd;
__le64 cmd_desc_base_addr;
bool ret = false;
@@ -669,7 +670,7 @@ int ufshcd_mcq_abort(struct scsi_cmnd *cmd)
struct Scsi_Host *host = cmd->device->host;
struct ufs_hba *hba = shost_priv(host);
int tag = scsi_cmd_to_rq(cmd)->tag;
struct ufshcd_lrb *lrbp = &hba->lrb[tag];
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct ufs_hw_queue *hwq;
int err;

18
drivers/ufs/core/ufshcd-crypto.h
View File

@@ -38,10 +38,10 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp,
}
static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp)
struct scsi_cmnd *cmd)
{
struct scsi_cmnd *cmd = lrbp->cmd;
const struct bio_crypt_ctx *crypt_ctx = scsi_cmd_to_rq(cmd)->crypt_ctx;
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
if (crypt_ctx && hba->vops && hba->vops->fill_crypto_prdt)
return hba->vops->fill_crypto_prdt(hba, crypt_ctx,
@@ -51,17 +51,19 @@ static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
}
static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp)
struct scsi_cmnd *cmd)
{
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT))
return;
if (!(scsi_cmd_to_rq(lrbp->cmd)->crypt_ctx))
if (!(scsi_cmd_to_rq(cmd)->crypt_ctx))
return;
/* Zeroize the PRDT because it can contain cryptographic keys. */
memzero_explicit(lrbp->ucd_prdt_ptr,
ufshcd_sg_entry_size(hba) * scsi_sg_count(lrbp->cmd));
ufshcd_sg_entry_size(hba) * scsi_sg_count(cmd));
}
bool ufshcd_crypto_enable(struct ufs_hba *hba);
@@ -82,13 +84,15 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp,
struct request_desc_header *h) { }
static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp)
struct scsi_cmnd *cmd)
{
return 0;
}
static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp) { }
struct scsi_cmnd *cmd)
{
}
static inline bool ufshcd_crypto_enable(struct ufs_hba *hba)
{

41
drivers/ufs/core/ufshcd-priv.h
View File

@@ -77,8 +77,7 @@ bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd);
int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag);
int ufshcd_mcq_abort(struct scsi_cmnd *cmd);
int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp);
void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd);
#define SD_ASCII_STD true
#define SD_RAW false
@@ -363,6 +362,44 @@ static inline bool ufs_is_valid_unit_desc_lun(struct ufs_dev_info *dev_info, u8
return lun == UFS_UPIU_RPMB_WLUN || (lun < dev_info->max_lu_supported);
}
/*
* Convert a block layer tag into a SCSI command pointer. This function is
* called once per I/O completion path and is also called from error paths.
*/
static inline struct scsi_cmnd *ufshcd_tag_to_cmd(struct ufs_hba *hba, u32 tag)
{
struct blk_mq_tags *tags = hba->host->tag_set.shared_tags;
struct request *rq;
/*
* Handle reserved tags differently because the UFS driver does not
* call blk_mq_alloc_request() for allocating reserved requests.
* Allocating reserved tags with blk_mq_alloc_request() would require
* the following:
* - Allocate an additional request queue from &hba->host->tag_set for
* allocating reserved requests from.
* - For that request queue, allocate a SCSI device.
* - Calling blk_mq_alloc_request(hba->dev_mgmt_queue, REQ_OP_DRV_OUT,
* BLK_MQ_REQ_RESERVED) for allocating a reserved request and
* blk_mq_free_request() for freeing reserved requests.
* - Set the .device pointer for these reserved requests.
* - Submit reserved requests with blk_execute_rq().
* - Modify ufshcd_queuecommand() such that it handles reserved requests
* in another way than SCSI requests.
* - Modify ufshcd_compl_one_cqe() such that it calls scsi_done() for
* device management commands.
* - Modify all callback functions called by blk_mq_tagset_busy_iter()
* calls in the UFS driver and skip device management commands.
*/
rq = tag < UFSHCD_NUM_RESERVED ? tags->static_rqs[tag] :
blk_mq_tag_to_rq(tags, tag);
if (WARN_ON_ONCE(!rq))
return NULL;
return blk_mq_rq_to_pdu(rq);
}
static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q)
__must_hold(&q->sq_lock)
{

235
drivers/ufs/core/ufshcd.c
View File

@@ -28,6 +28,7 @@
#include <scsi/scsi_dbg.h>
#include <scsi/scsi_driver.h>
#include <scsi/scsi_eh.h>
#include <scsi/scsi_tcq.h>
#include "ufshcd-priv.h"
#include <ufs/ufs_quirks.h>
#include <ufs/unipro.h>
@@ -483,7 +484,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd,
u32 hwq_id = 0;
struct request *rq = scsi_cmd_to_rq(cmd);
unsigned int tag = rq->tag;
struct ufshcd_lrb *lrbp = &hba->lrb[tag];
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int transfer_len = -1;
/* trace UPIU also */
@@ -594,14 +595,13 @@ static void ufshcd_print_evt_hist(struct ufs_hba *hba)
ufshcd_vops_dbg_register_dump(hba);
}
static
void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt)
static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd,
bool pr_prdt)
{
const struct ufshcd_lrb *lrbp;
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
const int tag = lrbp->task_tag;
int prdt_length;
lrbp = &hba->lrb[tag];
if (hba->monitor.enabled) {
dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag,
div_u64(lrbp->issue_time_stamp_local_clock, 1000));
@@ -644,7 +644,7 @@ static bool ufshcd_print_tr_iter(struct request *req, void *priv)
struct Scsi_Host *shost = sdev->host;
struct ufs_hba *hba = shost_priv(shost);
ufshcd_print_tr(hba, req->tag, *(bool *)priv);
ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv);
return true;
}
@@ -2298,8 +2298,7 @@ static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
struct scsi_cmnd *cmd)
{
const struct ufs_hba_monitor *m = &hba->monitor;
struct request *rq = scsi_cmd_to_rq(cmd);
struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag];
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
return m->enabled &&
(!m->chunk_size || m->chunk_size == cmd->sdb.length) &&
@@ -2320,7 +2319,7 @@ static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
struct request *req = scsi_cmd_to_rq(cmd);
struct ufshcd_lrb *lrbp = &hba->lrb[req->tag];
const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]);
unsigned long flags;
@@ -2350,17 +2349,26 @@ static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
spin_unlock_irqrestore(hba->host->host_lock, flags);
}
/*
* Returns %true for SCSI commands and %false for device management commands.
* Must not be called for SCSI commands that have not yet been started.
*/
static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd)
{
return blk_mq_request_started(scsi_cmd_to_rq(cmd));
}
/**
* ufshcd_send_command - Send SCSI or device management commands
* @hba: per adapter instance
* @lrbp: Local reference block of SCSI command
* @cmd: SCSI command or device management command pointer
* @hwq: pointer to hardware queue instance
*/
static inline void ufshcd_send_command(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp,
struct scsi_cmnd *cmd,
struct ufs_hw_queue *hwq)
{
struct scsi_cmnd *cmd = lrbp->cmd;
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
unsigned long flags;
if (hba->monitor.enabled) {
@@ -2369,7 +2377,7 @@ static inline void ufshcd_send_command(struct ufs_hba *hba,
lrbp->compl_time_stamp = ktime_set(0, 0);
lrbp->compl_time_stamp_local_clock = 0;
}
if (cmd) {
if (ufshcd_is_scsi_cmd(cmd)) {
ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND);
ufshcd_clk_scaling_start_busy(hba);
if (unlikely(ufshcd_should_inform_monitor(hba, cmd)))
@@ -2389,7 +2397,8 @@ static inline void ufshcd_send_command(struct ufs_hba *hba,
} else {
spin_lock_irqsave(&hba->outstanding_lock, flags);
if (hba->vops && hba->vops->setup_xfer_req)
hba->vops->setup_xfer_req(hba, lrbp->task_tag, !!cmd);
hba->vops->setup_xfer_req(hba, lrbp->task_tag,
ufshcd_is_scsi_cmd(cmd));
__set_bit(lrbp->task_tag, &hba->outstanding_reqs);
ufshcd_writel(hba, 1 << lrbp->task_tag,
REG_UTP_TRANSFER_REQ_DOOR_BELL);
@@ -2399,11 +2408,12 @@ static inline void ufshcd_send_command(struct ufs_hba *hba,
/**
* ufshcd_copy_sense_data - Copy sense data in case of check condition
* @lrbp: pointer to local reference block
* @cmd: SCSI command
*/
static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd)
{
u8 *const sense_buffer = lrbp->cmd->sense_buffer;
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
u8 *const sense_buffer = cmd->sense_buffer;
u16 resp_len;
int len;
@@ -2708,13 +2718,13 @@ static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int
/**
* ufshcd_map_sg - Map scatter-gather list to prdt
* @hba: per adapter instance
* @lrbp: pointer to local reference block
* @cmd: SCSI command
*
* Return: 0 in case of success, non-zero value in case of failure.
*/
static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
struct scsi_cmnd *cmd = lrbp->cmd;
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int sg_segments = scsi_dma_map(cmd);
if (sg_segments < 0)
@@ -2722,7 +2732,7 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd));
return ufshcd_crypto_fill_prdt(hba, lrbp);
return ufshcd_crypto_fill_prdt(hba, cmd);
}
/**
@@ -2781,13 +2791,13 @@ ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
/**
* ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
* for scsi commands
* @lrbp: local reference block pointer
* @cmd: SCSI command
* @upiu_flags: flags
*/
static
void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd,
u8 upiu_flags)
{
struct scsi_cmnd *cmd = lrbp->cmd;
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
unsigned short cdb_len;
@@ -2890,22 +2900,25 @@ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
* ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
* for SCSI Purposes
* @hba: per adapter instance
* @lrbp: pointer to local reference block
* @cmd: SCSI command
*/
static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
struct request *rq = scsi_cmd_to_rq(lrbp->cmd);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct request *rq = scsi_cmd_to_rq(cmd);
unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
u8 upiu_flags;
ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0);
ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags,
cmd->sc_data_direction, 0);
if (ioprio_class == IOPRIO_CLASS_RT)
upiu_flags |= UPIU_CMD_FLAGS_CP;
ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags);
}
static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
const int i = scsi_cmd_to_rq(cmd)->tag;
struct utp_transfer_cmd_desc *cmd_descp =
(void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba);
struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
@@ -2913,6 +2926,7 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba);
u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset);
u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset);
struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd);
lrb->utr_descriptor_ptr = utrdlp + i;
lrb->utrd_dma_addr =
@@ -2925,27 +2939,31 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
}
static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
struct scsi_cmnd *cmd, u8 lun, int tag)
static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
u8 lun, int tag)
{
ufshcd_init_lrb(hba, lrbp, tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
ufshcd_init_lrb(hba, cmd);
memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr));
lrbp->cmd = cmd;
lrbp->task_tag = tag;
lrbp->lun = lun;
ufshcd_prepare_lrbp_crypto(cmd ? scsi_cmd_to_rq(cmd) : NULL, lrbp);
ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ?
scsi_cmd_to_rq(cmd) : NULL, lrbp);
}
static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
struct scsi_cmnd *cmd, u8 lun, int tag)
static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
u8 lun, int tag)
{
__ufshcd_setup_cmd(hba, lrbp, cmd, lun, tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
__ufshcd_setup_cmd(hba, cmd, lun, tag);
lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
lrbp->req_abort_skip = false;
ufshcd_comp_scsi_upiu(hba, lrbp);
ufshcd_comp_scsi_upiu(hba, cmd);
}
/**
@@ -3016,7 +3034,6 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
{
struct ufs_hba *hba = shost_priv(host);
int tag = scsi_cmd_to_rq(cmd)->tag;
struct ufshcd_lrb *lrbp;
int err = 0;
struct ufs_hw_queue *hwq = NULL;
@@ -3067,11 +3084,10 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
ufshcd_hold(hba);
lrbp = &hba->lrb[tag];
ufshcd_setup_scsi_cmd(hba, cmd,
ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag);
ufshcd_setup_scsi_cmd(hba, lrbp, cmd, ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag);
err = ufshcd_map_sg(hba, lrbp);
err = ufshcd_map_sg(hba, cmd);
if (err) {
ufshcd_release(hba);
goto out;
@@ -3080,7 +3096,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
if (hba->mcq_enabled)
hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
ufshcd_send_command(hba, lrbp, hwq);
ufshcd_send_command(hba, cmd, hwq);
out:
if (ufs_trigger_eh(hba)) {
@@ -3094,10 +3110,12 @@ out:
return err;
}
static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
enum dev_cmd_type cmd_type, u8 lun, int tag)
static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
enum dev_cmd_type cmd_type, u8 lun, int tag)
{
__ufshcd_setup_cmd(hba, lrbp, NULL, lun, tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
__ufshcd_setup_cmd(hba, cmd, lun, tag);
lrbp->intr_cmd = true; /* No interrupt aggregation */
hba->dev_cmd.type = cmd_type;
}
@@ -3105,10 +3123,12 @@ static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
/*
* Return: 0 upon success; < 0 upon failure.
*/
static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
enum dev_cmd_type cmd_type, int tag)
{
ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag);
return ufshcd_compose_devman_upiu(hba, lrbp);
}
@@ -3320,13 +3340,14 @@ static void ufshcd_dev_man_unlock(struct ufs_hba *hba)
* Return: 0 upon success; > 0 in case the UFS device reported an OCS error;
* < 0 if another error occurred.
*/
static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
const u32 tag, int timeout)
static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
const u32 tag, int timeout)
{
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err;
ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
ufshcd_send_command(hba, lrbp, hba->dev_cmd_queue);
ufshcd_send_command(hba, cmd, hba->dev_cmd_queue);
err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
@@ -3351,17 +3372,17 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
enum dev_cmd_type cmd_type, int timeout)
{
const u32 tag = hba->reserved_slot;
struct ufshcd_lrb *lrbp = &hba->lrb[tag];
struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
int err;
/* Protects use of hba->reserved_slot. */
lockdep_assert_held(&hba->dev_cmd.lock);
err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag);
if (unlikely(err))
return err;
return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout);
return ufshcd_issue_dev_cmd(hba, cmd, tag, timeout);
}
/**
@@ -3991,14 +4012,6 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba)
}
skip_utmrdl:
/* Allocate memory for local reference block */
hba->lrb = devm_kcalloc(hba->dev,
hba->nutrs, sizeof(struct ufshcd_lrb),
GFP_KERNEL);
if (!hba->lrb) {
dev_err(hba->dev, "LRB Memory allocation failed\n");
goto out;
}
return 0;
out:
return -ENOMEM;
@@ -5411,19 +5424,18 @@ static void ufshcd_sdev_destroy(struct scsi_device *sdev)
/**
* ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
* @lrbp: pointer to local reference block of completed command
* @cmd: SCSI command
* @scsi_status: SCSI command status
*
* Return: value base on SCSI command status.
*/
static inline int
ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status)
{
int result = 0;
switch (scsi_status) {
case SAM_STAT_CHECK_CONDITION:
ufshcd_copy_sense_data(lrbp);
ufshcd_copy_sense_data(cmd);
fallthrough;
case SAM_STAT_GOOD:
result |= DID_OK << 16 | scsi_status;
@@ -5431,7 +5443,7 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
case SAM_STAT_TASK_SET_FULL:
case SAM_STAT_BUSY:
case SAM_STAT_TASK_ABORTED:
ufshcd_copy_sense_data(lrbp);
ufshcd_copy_sense_data(cmd);
result |= scsi_status;
break;
default:
@@ -5445,15 +5457,16 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
/**
* ufshcd_transfer_rsp_status - Get overall status of the response
* @hba: per adapter instance
* @lrbp: pointer to local reference block of completed command
* @cmd: SCSI command
* @cqe: pointer to the completion queue entry
*
* Return: result of the command to notify SCSI midlayer.
*/
static inline int
ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
struct cq_entry *cqe)
static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba,
struct scsi_cmnd *cmd,
struct cq_entry *cqe)
{
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int result = 0;
int scsi_status;
enum utp_ocs ocs;
@@ -5467,7 +5480,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
* not set either flag.
*/
if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW))
scsi_set_resid(lrbp->cmd, resid);
scsi_set_resid(cmd, resid);
/* overall command status of utrd */
ocs = ufshcd_get_tr_ocs(lrbp, cqe);
@@ -5488,7 +5501,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
* to notify the SCSI midlayer of the command status
*/
scsi_status = lrbp->ucd_rsp_ptr->header.status;
result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
result = ufshcd_scsi_cmd_status(cmd, scsi_status);
/*
* Currently we are only supporting BKOPs exception
@@ -5553,7 +5566,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
(host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) {
if (cqe)
ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry));
ufshcd_print_tr(hba, lrbp->task_tag, true);
ufshcd_print_tr(hba, cmd, true);
}
return result;
}
@@ -5620,13 +5633,10 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
}
/* Release the resources allocated for processing a SCSI command. */
void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
struct ufshcd_lrb *lrbp)
void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
struct scsi_cmnd *cmd = lrbp->cmd;
scsi_dma_unmap(cmd);
ufshcd_crypto_clear_prdt(hba, lrbp);
ufshcd_crypto_clear_prdt(hba, cmd);
ufshcd_release(hba);
ufshcd_clk_scaling_update_busy(hba);
}
@@ -5640,20 +5650,20 @@ void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag,
struct cq_entry *cqe)
{
struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
struct scsi_cmnd *cmd = lrbp->cmd;
struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
enum utp_ocs ocs;
if (hba->monitor.enabled) {
lrbp->compl_time_stamp = ktime_get();
lrbp->compl_time_stamp_local_clock = local_clock();
}
if (cmd) {
if (ufshcd_is_scsi_cmd(cmd)) {
if (unlikely(ufshcd_should_inform_monitor(hba, cmd)))
ufshcd_update_monitor(hba, cmd);
ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP);
cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe);
ufshcd_release_scsi_cmd(hba, lrbp);
cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe);
ufshcd_release_scsi_cmd(hba, cmd);
/* Do not touch lrbp after scsi done */
scsi_done(cmd);
} else {
@@ -5690,7 +5700,7 @@ static void ufshcd_clear_polled(struct ufs_hba *hba,
int tag;
for_each_set_bit(tag, completed_reqs, hba->nutrs) {
struct scsi_cmnd *cmd = hba->lrb[tag].cmd;
struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag);
if (!cmd)
continue;
@@ -5741,7 +5751,6 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv)
struct scsi_device *sdev = rq->q->queuedata;
struct Scsi_Host *shost = sdev->host;
struct ufs_hba *hba = shost_priv(shost);
struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag];
struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq);
if (!hwq)
@@ -5756,7 +5765,7 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv)
scoped_guard(spinlock_irqsave, &hwq->cq_lock) {
if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) {
set_host_byte(cmd, DID_REQUEUE);
ufshcd_release_scsi_cmd(hba, lrbp);
ufshcd_release_scsi_cmd(hba, cmd);
scsi_done(cmd);
}
}
@@ -6641,7 +6650,7 @@ static bool ufshcd_abort_one(struct request *rq, void *priv)
*ret = ufshcd_try_to_abort_task(hba, tag);
dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1,
*ret ? "failed" : "succeeded");
return *ret == 0;
@@ -7371,14 +7380,15 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
enum query_opcode desc_op)
{
const u32 tag = hba->reserved_slot;
struct ufshcd_lrb *lrbp = &hba->lrb[tag];
struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err = 0;
u8 upiu_flags;
/* Protects use of hba->reserved_slot. */
lockdep_assert_held(&hba->dev_cmd.lock);
ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag);
ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0);
@@ -7403,7 +7413,7 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
* bound to fail since dev_cmd.query and dev_cmd.type were left empty.
* read the response directly ignoring all errors.
*/
ufshcd_issue_dev_cmd(hba, lrbp, tag, dev_cmd_timeout);
ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout);
/* just copy the upiu response as it is */
memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
@@ -7518,7 +7528,8 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
enum dma_data_direction dir)
{
const u32 tag = hba->reserved_slot;
struct ufshcd_lrb *lrbp = &hba->lrb[tag];
struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err = 0;
int result;
u8 upiu_flags;
@@ -7529,7 +7540,8 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
/* Protects use of hba->reserved_slot. */
ufshcd_dev_man_lock(hba);
ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag);
ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN,
tag);
ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs);
@@ -7546,7 +7558,7 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT);
err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT);
if (!err) {
/* Just copy the upiu response as it is */
@@ -7647,11 +7659,12 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
{
struct ufshcd_lrb *lrbp;
int tag;
for_each_set_bit(tag, &bitmap, hba->nutrs) {
lrbp = &hba->lrb[tag];
struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
lrbp->req_abort_skip = true;
}
}
@@ -7659,7 +7672,7 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
/**
* ufshcd_try_to_abort_task - abort a specific task
* @hba: Pointer to adapter instance
* @tag: Task tag/index to be aborted
* @tag: Tag of the task to be aborted
*
* Abort the pending command in device by sending UFS_ABORT_TASK task management
* command, and in host controller by clearing the door-bell register. There can
@@ -7671,7 +7684,8 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
*/
int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
{
struct ufshcd_lrb *lrbp = &hba->lrb[tag];
struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err;
int poll_cnt;
u8 resp = 0xF;
@@ -7693,7 +7707,7 @@ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
hba->dev,
"%s: cmd with tag %d not pending in the device.\n",
__func__, tag);
if (!ufshcd_cmd_inflight(lrbp->cmd)) {
if (!ufshcd_cmd_inflight(cmd)) {
dev_info(hba->dev,
"%s: cmd with tag=%d completed.\n",
__func__, tag);
@@ -7741,7 +7755,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
struct Scsi_Host *host = cmd->device->host;
struct ufs_hba *hba = shost_priv(host);
int tag = scsi_cmd_to_rq(cmd)->tag;
struct ufshcd_lrb *lrbp = &hba->lrb[tag];
struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
unsigned long flags;
int err = FAILED;
bool outstanding;
@@ -7776,9 +7790,9 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
ufshcd_print_evt_hist(hba);
ufshcd_print_host_state(hba);
ufshcd_print_pwr_info(hba);
ufshcd_print_tr(hba, tag, true);
ufshcd_print_tr(hba, cmd, true);
} else {
ufshcd_print_tr(hba, tag, false);
ufshcd_print_tr(hba, cmd, false);
}
hba->req_abort_count++;
@@ -7822,7 +7836,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
goto release;
}
err = ufshcd_try_to_abort_task(hba, tag);
err = ufshcd_try_to_abort_task(hba, lrbp->task_tag);
if (err) {
dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
@@ -7839,7 +7853,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
spin_unlock_irqrestore(&hba->outstanding_lock, flags);
if (outstanding)
ufshcd_release_scsi_cmd(hba, lrbp);
ufshcd_release_scsi_cmd(hba, cmd);
err = SUCCESS;
@@ -8919,8 +8933,6 @@ static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs)
utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs;
dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr,
hba->utrdl_dma_addr);
devm_kfree(hba->dev, hba->lrb);
}
static int ufshcd_alloc_mcq(struct ufs_hba *hba)
@@ -9191,6 +9203,7 @@ static const struct scsi_host_template ufshcd_driver_template = {
.name = UFSHCD,
.proc_name = UFSHCD,
.map_queues = ufshcd_map_queues,
.cmd_size = sizeof(struct ufshcd_lrb),
.init_cmd_priv = ufshcd_init_cmd_priv,
.queuecommand = ufshcd_queuecommand,
.nr_reserved_cmds = UFSHCD_NUM_RESERVED,

5
include/ufs/ufshcd.h
View File

@@ -161,7 +161,6 @@ struct ufs_pm_lvl_states {
* @ucd_prdt_dma_addr: PRDT dma address for debug
* @ucd_rsp_dma_addr: UPIU response dma address for debug
* @ucd_req_dma_addr: UPIU request dma address for debug
* @cmd: pointer to SCSI command
* @scsi_status: SCSI status of the command
* @command_type: SCSI, UFS, Query.
* @task_tag: Task tag of the command
@@ -186,7 +185,6 @@ struct ufshcd_lrb {
dma_addr_t ucd_rsp_dma_addr;
dma_addr_t ucd_prdt_dma_addr;
struct scsi_cmnd *cmd;
int scsi_status;
int command_type;
@@ -833,7 +831,6 @@ enum ufshcd_mcq_opr {
* @spm_lvl: desired UFS power management level during system PM.
* @pm_op_in_progress: whether or not a PM operation is in progress.
* @ahit: value of Auto-Hibernate Idle Timer register.
* @lrb: local reference block
* @outstanding_tasks: Bits representing outstanding task requests
* @outstanding_lock: Protects @outstanding_reqs.
* @outstanding_reqs: Bits representing outstanding transfer requests
@@ -976,8 +973,6 @@ struct ufs_hba {
/* Auto-Hibernate Idle Timer register value */
u32 ahit;
struct ufshcd_lrb *lrb;
unsigned long outstanding_tasks;
spinlock_t outstanding_lock;
unsigned long outstanding_reqs;