From d604e1ec246d236deff57ac7e89e073dd911d60b Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Fri, 31 Oct 2025 13:39:09 -0700 Subject: [PATCH 01/28] scsi: core: Support allocating reserved commands Quite some drivers are using management commands internally. These commands typically use the same tag pool as regular SCSI commands. Tags for these management commands are set aside before allocating the block-mq tag bitmap for regular SCSI commands. The block layer already supports this via the reserved tag mechanism. Add a new field 'nr_reserved_cmds' to the SCSI host template to instruct the block layer to set aside a tag space for these management commands by using reserved tags. Exclude reserved commands from .can_queue because .can_queue is visible in sysfs. [ bvanassche: modified patch title and patch description. Left out the following statements: "if (sht->nr_reserved_cmds)" and also "if (sdev->host->nr_reserved_cmds) flags |= BLK_MQ_REQ_RESERVED;". Moved nr_reserved_cmds declarations and statements close to the corresponding can_queue declarations and statements. See also https://lore.kernel.org/linux-scsi/20210503150333.130310-11-hare@suse.de/ ] Signed-off-by: Hannes Reinecke Reviewed-by: John Garry Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-2-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/scsi/hosts.c | 1 + drivers/scsi/scsi_lib.c | 3 ++- include/scsi/scsi_host.h | 21 ++++++++++++++++++++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c index eb224a338fa2..8b7f5fafa9e0 100644 --- a/drivers/scsi/hosts.c +++ b/drivers/scsi/hosts.c @@ -436,6 +436,7 @@ struct Scsi_Host *scsi_host_alloc(const struct scsi_host_template *sht, int priv shost->hostt = sht; shost->this_id = sht->this_id; shost->can_queue = sht->can_queue; + shost->nr_reserved_cmds = sht->nr_reserved_cmds; shost->sg_tablesize = sht->sg_tablesize; shost->sg_prot_tablesize = sht->sg_prot_tablesize; shost->cmd_per_lun = sht->cmd_per_lun; diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index d7e42293b864..d52bbbe5a357 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -2083,7 +2083,8 @@ int scsi_mq_setup_tags(struct Scsi_Host *shost) tag_set->ops = &scsi_mq_ops_no_commit; tag_set->nr_hw_queues = shost->nr_hw_queues ? : 1; tag_set->nr_maps = shost->nr_maps ? : 1; - tag_set->queue_depth = shost->can_queue; + tag_set->queue_depth = shost->can_queue + shost->nr_reserved_cmds; + tag_set->reserved_tags = shost->nr_reserved_cmds; tag_set->cmd_size = cmd_size; tag_set->numa_node = dev_to_node(shost->dma_dev); if (shost->hostt->tag_alloc_policy_rr) diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index f5a243261236..7b8f144ccf7d 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h @@ -375,10 +375,19 @@ struct scsi_host_template { /* * This determines if we will use a non-interrupt driven * or an interrupt driven scheme. It is set to the maximum number - * of simultaneous commands a single hw queue in HBA will accept. + * of simultaneous commands a single hw queue in HBA will accept + * excluding internal commands. */ int can_queue; + /* + * This determines how many commands the HBA will set aside + * for internal commands. This number will be added to + * @can_queue to calculate the maximum number of simultaneous + * commands sent to the host. + */ + int nr_reserved_cmds; + /* * In many instances, especially where disconnect / reconnect are * supported, our host also has an ID on the SCSI bus. If this is @@ -611,7 +620,17 @@ struct Scsi_Host { unsigned short max_cmd_len; int this_id; + + /* + * Number of commands this host can handle at the same time. + * This excludes reserved commands as specified by nr_reserved_cmds. + */ int can_queue; + /* + * Number of reserved commands to allocate, if any. + */ + unsigned int nr_reserved_cmds; + short cmd_per_lun; short unsigned int sg_tablesize; short unsigned int sg_prot_tablesize; From 21008cabc5d9bf2864ff244f4f94abebe3380947 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:10 -0700 Subject: [PATCH 02/28] scsi: core: Move two statements Move two statements that will be needed for pseudo SCSI devices in front of code that won't be needed for pseudo SCSI devices. No functionality has been changed. Reviewed-by: John Garry Reviewed-by: Hannes Reinecke Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-3-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_scan.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index 3c6e089e80c3..de039efef290 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -347,6 +347,8 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget, kref_get(&sdev->host->tagset_refcnt); sdev->request_queue = q; + scsi_sysfs_device_initialize(sdev); + depth = sdev->host->cmd_per_lun ?: 1; /* @@ -363,8 +365,6 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget, scsi_change_queue_depth(sdev, depth); - scsi_sysfs_device_initialize(sdev); - if (shost->hostt->sdev_init) { ret = shost->hostt->sdev_init(sdev); if (ret) { @@ -1068,6 +1068,8 @@ static int scsi_add_lun(struct scsi_device *sdev, unsigned char *inq_result, transport_configure_device(&sdev->sdev_gendev); + sdev->sdev_bflags = *bflags; + /* * No need to freeze the queue as it isn't reachable to anyone else yet. */ @@ -1113,7 +1115,6 @@ static int scsi_add_lun(struct scsi_device *sdev, unsigned char *inq_result, sdev->max_queue_depth = sdev->queue_depth; WARN_ON_ONCE(sdev->max_queue_depth > sdev->budget_map.depth); - sdev->sdev_bflags = *bflags; /* * Ok, the device is now all set up, we can From a47c7bef57858d29b83cd7f101112f52821439f9 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:11 -0700 Subject: [PATCH 03/28] scsi: core: Make the budget map optional Prepare for not allocating a budget map for pseudo SCSI devices by checking whether a budget map has been allocated before using it. Reviewed-by: Hannes Reinecke Cc: John Garry Cc: Ming Lei Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-4-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi.c | 5 +++++ drivers/scsi/scsi_error.c | 3 +++ drivers/scsi/scsi_lib.c | 9 +++++++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c index 9a0f467264b3..589ae28b2c8b 100644 --- a/drivers/scsi/scsi.c +++ b/drivers/scsi/scsi.c @@ -216,6 +216,9 @@ int scsi_device_max_queue_depth(struct scsi_device *sdev) */ int scsi_change_queue_depth(struct scsi_device *sdev, int depth) { + if (!sdev->budget_map.map) + return -EINVAL; + depth = min_t(int, depth, scsi_device_max_queue_depth(sdev)); if (depth > 0) { @@ -255,6 +258,8 @@ EXPORT_SYMBOL(scsi_change_queue_depth); */ int scsi_track_queue_full(struct scsi_device *sdev, int depth) { + if (!sdev->budget_map.map) + return 0; /* * Don't let QUEUE_FULLs on the same diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c index 746ff6a1f309..87636068cd37 100644 --- a/drivers/scsi/scsi_error.c +++ b/drivers/scsi/scsi_error.c @@ -749,6 +749,9 @@ static void scsi_handle_queue_ramp_up(struct scsi_device *sdev) const struct scsi_host_template *sht = sdev->host->hostt; struct scsi_device *tmp_sdev; + if (!sdev->budget_map.map) + return; + if (!sht->track_queue_depth || sdev->queue_depth >= sdev->max_queue_depth) return; diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index d52bbbe5a357..53ff348b3a4c 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -396,7 +396,8 @@ void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd) if (starget->can_queue > 0) atomic_dec(&starget->target_busy); - sbitmap_put(&sdev->budget_map, cmd->budget_token); + if (sdev->budget_map.map) + sbitmap_put(&sdev->budget_map, cmd->budget_token); cmd->budget_token = -1; } @@ -1360,6 +1361,9 @@ static inline int scsi_dev_queue_ready(struct request_queue *q, { int token; + if (!sdev->budget_map.map) + return INT_MAX; + token = sbitmap_get(&sdev->budget_map); if (token < 0) return -1; @@ -1749,7 +1753,8 @@ static void scsi_mq_put_budget(struct request_queue *q, int budget_token) { struct scsi_device *sdev = q->queuedata; - sbitmap_put(&sdev->budget_map, budget_token); + if (sdev->budget_map.map) + sbitmap_put(&sdev->budget_map, budget_token); } /* From d630fbf6fc8ce2fc95de7784de5499387b682dc1 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Fri, 31 Oct 2025 13:39:12 -0700 Subject: [PATCH 04/28] scsi: core: Support allocating a pseudo SCSI device Allocate a pseudo SCSI device if 'nr_reserved_cmds' has been set. Pseudo SCSI devices have the SCSI ID :U64_MAX so they won't clash with any devices the LLD might create. Pseudo SCSI devices are excluded from scanning and will not show up in sysfs. Additionally, pseudo SCSI devices are skipped by shost_for_each_device(). This prevents that the SCSI error handler tries to submit a reset to a non-existent logical unit. Do not allocate a budget map for pseudo SCSI devices since the cmd_per_lun limit does not apply to pseudo SCSI devices. Do not perform queue depth ramp up / ramp down for pseudo SCSI devices. Pseudo SCSI devices will be used to send internal commands to a storage device. [ bvanassche: edited patch description / renamed host_sdev into pseudo_sdev / unexported scsi_get_host_dev() / modified error path in scsi_get_pseudo_dev() / skip pseudo devices in __scsi_iterate_devices() and also when calling sdev_init(), sdev_configure() and sdev_destroy(). See also https://lore.kernel.org/linux-scsi/20211125151048.103910-2-hare@suse.de/ ] Reviewed-by: John Garry Signed-off-by: Hannes Reinecke Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-5-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/scsi/hosts.c | 8 +++++ drivers/scsi/scsi.c | 7 ++-- drivers/scsi/scsi_priv.h | 1 + drivers/scsi/scsi_scan.c | 67 +++++++++++++++++++++++++++++++++++++- drivers/scsi/scsi_sysfs.c | 5 ++- include/scsi/scsi_device.h | 16 +++++++++ include/scsi/scsi_host.h | 6 ++++ 7 files changed, 106 insertions(+), 4 deletions(-) diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c index 8b7f5fafa9e0..ad1476fb5035 100644 --- a/drivers/scsi/hosts.c +++ b/drivers/scsi/hosts.c @@ -307,6 +307,14 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, if (error) goto out_del_dev; + if (shost->nr_reserved_cmds) { + shost->pseudo_sdev = scsi_get_pseudo_sdev(shost); + if (!shost->pseudo_sdev) { + error = -ENOMEM; + goto out_del_dev; + } + } + scsi_proc_host_add(shost); scsi_autopm_put_host(shost); return error; diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c index 589ae28b2c8b..76cdad063f7b 100644 --- a/drivers/scsi/scsi.c +++ b/drivers/scsi/scsi.c @@ -831,8 +831,11 @@ struct scsi_device *__scsi_iterate_devices(struct Scsi_Host *shost, spin_lock_irqsave(shost->host_lock, flags); while (list->next != &shost->__devices) { next = list_entry(list->next, struct scsi_device, siblings); - /* skip devices that we can't get a reference to */ - if (!scsi_device_get(next)) + /* + * Skip pseudo devices and also devices we can't get a + * reference to. + */ + if (!scsi_device_is_pseudo_dev(next) && !scsi_device_get(next)) break; next = NULL; list = list->next; diff --git a/drivers/scsi/scsi_priv.h b/drivers/scsi/scsi_priv.h index 5b2b19f5e8ec..d07ec15d6c00 100644 --- a/drivers/scsi/scsi_priv.h +++ b/drivers/scsi/scsi_priv.h @@ -135,6 +135,7 @@ extern int scsi_complete_async_scans(void); extern int scsi_scan_host_selected(struct Scsi_Host *, unsigned int, unsigned int, u64, enum scsi_scan_mode); extern void scsi_forget_host(struct Scsi_Host *); +struct scsi_device *scsi_get_pseudo_sdev(struct Scsi_Host *); /* scsi_sysctl.c */ #ifdef CONFIG_SYSCTL diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index de039efef290..7acbfcfc2172 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -349,6 +349,9 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget, scsi_sysfs_device_initialize(sdev); + if (scsi_device_is_pseudo_dev(sdev)) + return sdev; + depth = sdev->host->cmd_per_lun ?: 1; /* @@ -1070,6 +1073,9 @@ static int scsi_add_lun(struct scsi_device *sdev, unsigned char *inq_result, sdev->sdev_bflags = *bflags; + if (scsi_device_is_pseudo_dev(sdev)) + return SCSI_SCAN_LUN_PRESENT; + /* * No need to freeze the queue as it isn't reachable to anyone else yet. */ @@ -1213,6 +1219,12 @@ static int scsi_probe_and_add_lun(struct scsi_target *starget, if (!sdev) goto out; + if (scsi_device_is_pseudo_dev(sdev)) { + if (bflagsp) + *bflagsp = BLIST_NOLUN; + return SCSI_SCAN_LUN_PRESENT; + } + result = kmalloc(result_len, GFP_KERNEL); if (!result) goto out_free_sdev; @@ -2084,12 +2096,65 @@ void scsi_forget_host(struct Scsi_Host *shost) restart: spin_lock_irqsave(shost->host_lock, flags); list_for_each_entry(sdev, &shost->__devices, siblings) { - if (sdev->sdev_state == SDEV_DEL) + if (scsi_device_is_pseudo_dev(sdev) || + sdev->sdev_state == SDEV_DEL) continue; spin_unlock_irqrestore(shost->host_lock, flags); __scsi_remove_device(sdev); goto restart; } spin_unlock_irqrestore(shost->host_lock, flags); + + /* + * Remove the pseudo device last since it may be needed during removal + * of other SCSI devices. + */ + if (shost->pseudo_sdev) + __scsi_remove_device(shost->pseudo_sdev); } +/** + * scsi_get_pseudo_sdev() - Attach a pseudo SCSI device to a SCSI host + * @shost: Host that needs a pseudo SCSI device + * + * Lock status: None assumed. + * + * Returns: The scsi_device or NULL + * + * Notes: + * Attach a single scsi_device to the Scsi_Host. The primary aim for this + * device is to serve as a container from which SCSI commands can be + * allocated. Each SCSI command will carry a command tag allocated by the + * block layer. These SCSI commands can be used by the LLDD to send + * internal or passthrough commands without having to manage tag allocation + * inside the LLDD. + */ +struct scsi_device *scsi_get_pseudo_sdev(struct Scsi_Host *shost) +{ + struct scsi_device *sdev = NULL; + struct scsi_target *starget; + + guard(mutex)(&shost->scan_mutex); + + if (!scsi_host_scan_allowed(shost)) + goto out; + + starget = scsi_alloc_target(&shost->shost_gendev, 0, shost->max_id); + if (!starget) + goto out; + + sdev = scsi_alloc_sdev(starget, U64_MAX, NULL); + if (!sdev) { + scsi_target_reap(starget); + goto put_target; + } + + sdev->borken = 0; + +put_target: + /* See also the get_device(dev) call in scsi_alloc_target(). */ + put_device(&starget->dev); + +out: + return sdev; +} diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c index 15ba493d2138..c37992147847 100644 --- a/drivers/scsi/scsi_sysfs.c +++ b/drivers/scsi/scsi_sysfs.c @@ -1406,6 +1406,9 @@ int scsi_sysfs_add_sdev(struct scsi_device *sdev) int error; struct scsi_target *starget = sdev->sdev_target; + if (WARN_ON_ONCE(scsi_device_is_pseudo_dev(sdev))) + return -EINVAL; + error = scsi_target_add(starget); if (error) return error; @@ -1513,7 +1516,7 @@ void __scsi_remove_device(struct scsi_device *sdev) kref_put(&sdev->host->tagset_refcnt, scsi_mq_free_tags); cancel_work_sync(&sdev->requeue_work); - if (sdev->host->hostt->sdev_destroy) + if (!scsi_device_is_pseudo_dev(sdev) && sdev->host->hostt->sdev_destroy) sdev->host->hostt->sdev_destroy(sdev); transport_destroy_device(dev); diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h index 4c106342c4ae..918631088711 100644 --- a/include/scsi/scsi_device.h +++ b/include/scsi/scsi_device.h @@ -589,6 +589,22 @@ static inline unsigned int sdev_id(struct scsi_device *sdev) #define scmd_id(scmd) sdev_id((scmd)->device) #define scmd_channel(scmd) sdev_channel((scmd)->device) +/** + * scsi_device_is_pseudo_dev() - Whether a device is a pseudo SCSI device. + * @sdev: SCSI device to examine + * + * A pseudo SCSI device can be used to allocate SCSI commands but does not show + * up in sysfs. Additionally, the logical unit information in *@sdev is made up. + * + * This function tests the LUN number instead of comparing @sdev with + * @sdev->host->pseudo_sdev because this function may be called before + * @sdev->host->pseudo_sdev has been initialized. + */ +static inline bool scsi_device_is_pseudo_dev(struct scsi_device *sdev) +{ + return sdev->lun == U64_MAX; +} + /* * checks for positions of the SCSI state machine */ diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index 7b8f144ccf7d..4f945a20d198 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h @@ -721,6 +721,12 @@ struct Scsi_Host { /* ldm bits */ struct device shost_gendev, shost_dev; + /* + * A SCSI device structure used for sending internal commands to the + * HBA. There is no corresponding logical unit inside the SCSI device. + */ + struct scsi_device *pseudo_sdev; + /* * Points to the transport data (if any) which is allocated * separately From 11ea1de3fc4ba94127034cb01df63a666c4c9836 Mon Sep 17 00:00:00 2001 From: John Garry Date: Fri, 31 Oct 2025 13:39:13 -0700 Subject: [PATCH 05/28] scsi: core: Introduce .queue_reserved_command() Reserved commands will be used by SCSI LLDs for submitting internal commands. Since the SCSI host, target and device limits do not apply to the reserved command use cases, bypass the SCSI host limit checks for reserved commands. Introduce the .queue_reserved_command() callback for reserved commands. Additionally, do not activate the SCSI error handler if a reserved command fails such that reserved commands can be submitted from inside the SCSI error handler. [ bvanassche: modified patch title and patch description. Renamed .reserved_queuecommand() into .queue_reserved_command(). Changed the second argument of __blk_mq_end_request() from 0 into error code in the completion path if cmd->result != 0. Rewrote the scsi_queue_rq() changes. See also https://lore.kernel.org/linux-scsi/1666693096-180008-5-git-send-email-john.garry@huawei.com/ ] Cc: Hannes Reinecke Signed-off-by: John Garry Signed-off-by: Bart Van Assche Reviewed-by: Hannes Reinecke Link: https://patch.msgid.link/20251031204029.2883185-6-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/scsi/hosts.c | 6 +++++ drivers/scsi/scsi_lib.c | 54 ++++++++++++++++++++++++++++------------ include/scsi/scsi_host.h | 6 +++++ 3 files changed, 50 insertions(+), 16 deletions(-) diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c index ad1476fb5035..e047747d4ecf 100644 --- a/drivers/scsi/hosts.c +++ b/drivers/scsi/hosts.c @@ -231,6 +231,12 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, goto fail; } + if (shost->nr_reserved_cmds && !sht->queue_reserved_command) { + shost_printk(KERN_ERR, shost, + "nr_reserved_cmds set but no method to queue\n"); + goto fail; + } + /* Use min_t(int, ...) in case shost->can_queue exceeds SHRT_MAX */ shost->cmd_per_lun = min_t(int, shost->cmd_per_lun, shost->can_queue); diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 53ff348b3a4c..d4e874bbf2ea 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1534,6 +1534,14 @@ static void scsi_complete(struct request *rq) struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); enum scsi_disposition disposition; + if (blk_mq_is_reserved_rq(rq)) { + /* Only pass-through requests are supported in this code path. */ + WARN_ON_ONCE(!blk_rq_is_passthrough(scsi_cmd_to_rq(cmd))); + scsi_mq_uninit_cmd(cmd); + __blk_mq_end_request(rq, scsi_result_to_blk_status(cmd->result)); + return; + } + INIT_LIST_HEAD(&cmd->eh_entry); atomic_inc(&cmd->device->iodone_cnt); @@ -1823,25 +1831,31 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx, WARN_ON_ONCE(cmd->budget_token < 0); /* - * If the device is not in running state we will reject some or all - * commands. + * Bypass the SCSI device, SCSI target and SCSI host checks for + * reserved commands. */ - if (unlikely(sdev->sdev_state != SDEV_RUNNING)) { - ret = scsi_device_state_check(sdev, req); - if (ret != BLK_STS_OK) - goto out_put_budget; - } + if (!blk_mq_is_reserved_rq(req)) { + /* + * If the device is not in running state we will reject some or + * all commands. + */ + if (unlikely(sdev->sdev_state != SDEV_RUNNING)) { + ret = scsi_device_state_check(sdev, req); + if (ret != BLK_STS_OK) + goto out_put_budget; + } - ret = BLK_STS_RESOURCE; - if (!scsi_target_queue_ready(shost, sdev)) - goto out_put_budget; - if (unlikely(scsi_host_in_recovery(shost))) { - if (cmd->flags & SCMD_FAIL_IF_RECOVERING) - ret = BLK_STS_OFFLINE; - goto out_dec_target_busy; + ret = BLK_STS_RESOURCE; + if (!scsi_target_queue_ready(shost, sdev)) + goto out_put_budget; + if (unlikely(scsi_host_in_recovery(shost))) { + if (cmd->flags & SCMD_FAIL_IF_RECOVERING) + ret = BLK_STS_OFFLINE; + goto out_dec_target_busy; + } + if (!scsi_host_queue_ready(q, shost, sdev, cmd)) + goto out_dec_target_busy; } - if (!scsi_host_queue_ready(q, shost, sdev, cmd)) - goto out_dec_target_busy; /* * Only clear the driver-private command data if the LLD does not supply @@ -1870,6 +1884,14 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx, cmd->submitter = SUBMITTED_BY_BLOCK_LAYER; blk_mq_start_request(req); + if (blk_mq_is_reserved_rq(req)) { + reason = shost->hostt->queue_reserved_command(shost, cmd); + if (reason) { + ret = BLK_STS_RESOURCE; + goto out_put_budget; + } + return BLK_STS_OK; + } reason = scsi_dispatch_cmd(cmd); if (reason) { scsi_set_blocked(cmd, reason); diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index 4f945a20d198..e87cf7eadd26 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h @@ -86,6 +86,12 @@ struct scsi_host_template { */ int (* queuecommand)(struct Scsi_Host *, struct scsi_cmnd *); + /* + * Queue a reserved command (BLK_MQ_REQ_RESERVED). The .queuecommand() + * documentation also applies to the .queue_reserved_command() callback. + */ + int (*queue_reserved_command)(struct Scsi_Host *, struct scsi_cmnd *); + /* * The commit_rqs function is used to trigger a hardware * doorbell after some requests have been queued with From a2ab4e33286de37f3fe8f28f86f5f71d6b0ae3b0 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Fri, 31 Oct 2025 13:39:14 -0700 Subject: [PATCH 06/28] scsi: core: Add scsi_{get,put}_internal_cmd() helpers Add helper functions to allow LLDDs to allocate and free internal commands. [ bvanassche: changed the 'nowait' argument into a 'flags' argument. See also https://lore.kernel.org/linux-scsi/20211125151048.103910-3-hare@suse.de/ ] Reviewed-by: John Garry Signed-off-by: Hannes Reinecke Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-7-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_lib.c | 38 ++++++++++++++++++++++++++++++++++++++ include/scsi/scsi_device.h | 4 ++++ 2 files changed, 42 insertions(+) diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index d4e874bbf2ea..51ad2ad07e43 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -2134,6 +2134,44 @@ void scsi_mq_free_tags(struct kref *kref) complete(&shost->tagset_freed); } +/** + * scsi_get_internal_cmd() - Allocate an internal SCSI command. + * @sdev: SCSI device from which to allocate the command + * @data_direction: Data direction for the allocated command + * @flags: request allocation flags, e.g. BLK_MQ_REQ_RESERVED or + * BLK_MQ_REQ_NOWAIT. + * + * Allocates a SCSI command for internal LLDD use. + */ +struct scsi_cmnd *scsi_get_internal_cmd(struct scsi_device *sdev, + enum dma_data_direction data_direction, + blk_mq_req_flags_t flags) +{ + enum req_op op = data_direction == DMA_TO_DEVICE ? REQ_OP_DRV_OUT : + REQ_OP_DRV_IN; + struct scsi_cmnd *scmd; + struct request *rq; + + rq = scsi_alloc_request(sdev->request_queue, op, flags); + if (IS_ERR(rq)) + return NULL; + scmd = blk_mq_rq_to_pdu(rq); + scmd->device = sdev; + + return scmd; +} +EXPORT_SYMBOL_GPL(scsi_get_internal_cmd); + +/** + * scsi_put_internal_cmd() - Free an internal SCSI command. + * @scmd: SCSI command to be freed + */ +void scsi_put_internal_cmd(struct scsi_cmnd *scmd) +{ + blk_mq_free_request(blk_mq_rq_from_pdu(scmd)); +} +EXPORT_SYMBOL_GPL(scsi_put_internal_cmd); + /** * scsi_device_from_queue - return sdev associated with a request_queue * @q: The request queue to return the sdev from diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h index 918631088711..1e2e599517e9 100644 --- a/include/scsi/scsi_device.h +++ b/include/scsi/scsi_device.h @@ -558,6 +558,10 @@ int scsi_execute_cmd(struct scsi_device *sdev, const unsigned char *cmd, const struct scsi_exec_args *args); void scsi_failures_reset_retries(struct scsi_failures *failures); +struct scsi_cmnd *scsi_get_internal_cmd(struct scsi_device *sdev, + enum dma_data_direction data_direction, + blk_mq_req_flags_t flags); +void scsi_put_internal_cmd(struct scsi_cmnd *scmd); extern void sdev_disable_disk_events(struct scsi_device *sdev); extern void sdev_enable_disk_events(struct scsi_device *sdev); extern int scsi_vpd_lun_id(struct scsi_device *, char *, size_t); From 581ca490353ca5f4bee0d662595af27c06e603d4 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:15 -0700 Subject: [PATCH 07/28] scsi: scsi_debug: Abort SCSI commands via an internal command Add a .queue_reserved_command() implementation and call it from the code path that aborts SCSI commands. This ensures that the code for allocating a pseudo SCSI device and also the code for allocating and processing reserved commands gets triggered while running blktests. Most of the code in this patch is a modified version of code from John Garry. See also https://lore.kernel.org/linux-scsi/75018e17-4dea-4e1b-8c92-7a224a1e13b9@oracle.com/ Reviewed-by: John Garry Suggested-by: John Garry Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-8-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_debug.c | 114 ++++++++++++++++++++++++++++++++++---- 1 file changed, 104 insertions(+), 10 deletions(-) diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index b2ab97be5db3..7291b7a7f1b0 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -6752,20 +6752,59 @@ static bool scsi_debug_stop_cmnd(struct scsi_cmnd *cmnd) return false; } +struct sdebug_abort_cmd { + u32 unique_tag; +}; + +enum sdebug_internal_cmd_type { + SCSI_DEBUG_ABORT_CMD, +}; + +struct sdebug_internal_cmd { + enum sdebug_internal_cmd_type type; + + union { + struct sdebug_abort_cmd abort_cmd; + }; +}; + +union sdebug_priv { + struct sdebug_scsi_cmd cmd; + struct sdebug_internal_cmd internal_cmd; +}; + /* - * Called from scsi_debug_abort() only, which is for timed-out cmd. + * Abort SCSI command @cmnd. Only called from scsi_debug_abort(). Although + * it would be possible to call scsi_debug_stop_cmnd() directly, an internal + * command is allocated and submitted to trigger the reserved command + * infrastructure. */ static bool scsi_debug_abort_cmnd(struct scsi_cmnd *cmnd) { - struct sdebug_scsi_cmd *sdsc = scsi_cmd_priv(cmnd); - unsigned long flags; - bool res; + struct Scsi_Host *shost = cmnd->device->host; + struct request *rq = scsi_cmd_to_rq(cmnd); + u32 unique_tag = blk_mq_unique_tag(rq); + struct sdebug_internal_cmd *internal_cmd; + struct scsi_cmnd *abort_cmd; + struct request *abort_rq; + blk_status_t res; - spin_lock_irqsave(&sdsc->lock, flags); - res = scsi_debug_stop_cmnd(cmnd); - spin_unlock_irqrestore(&sdsc->lock, flags); - - return res; + abort_cmd = scsi_get_internal_cmd(shost->pseudo_sdev, DMA_NONE, + BLK_MQ_REQ_RESERVED); + if (!abort_cmd) + return false; + internal_cmd = scsi_cmd_priv(abort_cmd); + *internal_cmd = (struct sdebug_internal_cmd) { + .type = SCSI_DEBUG_ABORT_CMD, + .abort_cmd = { + .unique_tag = unique_tag, + }, + }; + abort_rq = scsi_cmd_to_rq(abort_cmd); + abort_rq->timeout = secs_to_jiffies(3); + res = blk_execute_rq(abort_rq, true); + scsi_put_internal_cmd(abort_cmd); + return res == BLK_STS_OK; } /* @@ -9220,6 +9259,56 @@ out_handle: return ret; } +/* Process @scp, a request to abort a SCSI command by tag. */ +static void scsi_debug_abort_cmd(struct Scsi_Host *shost, struct scsi_cmnd *scp) +{ + struct sdebug_internal_cmd *internal_cmd = scsi_cmd_priv(scp); + struct sdebug_abort_cmd *abort_cmd = &internal_cmd->abort_cmd; + const u32 unique_tag = abort_cmd->unique_tag; + struct scsi_cmnd *to_be_aborted_scmd = + scsi_host_find_tag(shost, unique_tag); + struct sdebug_scsi_cmd *to_be_aborted_sdsc = + scsi_cmd_priv(to_be_aborted_scmd); + bool res = false; + + if (!to_be_aborted_scmd) { + pr_err("%s: command with tag %#x not found\n", __func__, + unique_tag); + return; + } + + scoped_guard(spinlock_irqsave, &to_be_aborted_sdsc->lock) + res = scsi_debug_stop_cmnd(to_be_aborted_scmd); + + if (res) + pr_info("%s: aborted command with tag %#x\n", + __func__, unique_tag); + else + pr_err("%s: failed to abort command with tag %#x\n", + __func__, unique_tag); + + set_host_byte(scp, res ? DID_OK : DID_ERROR); +} + +static int scsi_debug_process_reserved_command(struct Scsi_Host *shost, + struct scsi_cmnd *scp) +{ + struct sdebug_internal_cmd *internal_cmd = scsi_cmd_priv(scp); + + switch (internal_cmd->type) { + case SCSI_DEBUG_ABORT_CMD: + scsi_debug_abort_cmd(shost, scp); + break; + default: + WARN_ON_ONCE(true); + set_host_byte(scp, DID_ERROR); + break; + } + + scsi_done(scp); + return 0; +} + static int scsi_debug_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scp) { @@ -9420,6 +9509,9 @@ static int sdebug_init_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd) struct sdebug_scsi_cmd *sdsc = scsi_cmd_priv(cmd); struct sdebug_defer *sd_dp = &sdsc->sd_dp; + if (blk_mq_is_reserved_rq(scsi_cmd_to_rq(cmd))) + return 0; + spin_lock_init(&sdsc->lock); hrtimer_setup(&sd_dp->hrt, sdebug_q_cmd_hrt_complete, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); @@ -9439,6 +9531,7 @@ static const struct scsi_host_template sdebug_driver_template = { .sdev_destroy = scsi_debug_sdev_destroy, .ioctl = scsi_debug_ioctl, .queuecommand = scsi_debug_queuecommand, + .queue_reserved_command = scsi_debug_process_reserved_command, .change_queue_depth = sdebug_change_qdepth, .map_queues = sdebug_map_queues, .mq_poll = sdebug_blk_mq_poll, @@ -9448,6 +9541,7 @@ static const struct scsi_host_template sdebug_driver_template = { .eh_bus_reset_handler = scsi_debug_bus_reset, .eh_host_reset_handler = scsi_debug_host_reset, .can_queue = SDEBUG_CANQUEUE, + .nr_reserved_cmds = 1, .this_id = 7, .sg_tablesize = SG_MAX_SEGMENTS, .cmd_per_lun = DEF_CMD_PER_LUN, @@ -9456,7 +9550,7 @@ static const struct scsi_host_template sdebug_driver_template = { .module = THIS_MODULE, .skip_settle_delay = 1, .track_queue_depth = 1, - .cmd_size = sizeof(struct sdebug_scsi_cmd), + .cmd_size = sizeof(union sdebug_priv), .init_cmd_priv = sdebug_init_cmd_priv, .target_alloc = sdebug_target_alloc, .target_destroy = sdebug_target_destroy, From dd4299af9b049c0dcb28699327ebc1386aab4b2a Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:16 -0700 Subject: [PATCH 08/28] scsi: ufs: core: Move an assignment in ufshcd_mcq_process_cqe() Since 'tag' is only used inside the if-statement, move the 'tag' assignment into the if-statement. This patch prepares for introducing a WARN_ON_ONCE() call in ufshcd_mcq_get_tag() if the tag lookup fails. Reviewed-by: Avri Altman Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-9-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufs-mcq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index d04662b57cd1..bae35d0f99c5 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -307,9 +307,10 @@ static void ufshcd_mcq_process_cqe(struct ufs_hba *hba, struct ufs_hw_queue *hwq) { struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq); - int tag = ufshcd_mcq_get_tag(hba, cqe); if (cqe->command_desc_base_addr) { + int tag = ufshcd_mcq_get_tag(hba, cqe); + ufshcd_compl_one_cqe(hba, tag, cqe); /* After processed the cqe, mark it empty (invalid) entry */ cqe->command_desc_base_addr = 0; From 9f8e09230f532c806ed088417f1ad794dd19a10f Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:17 -0700 Subject: [PATCH 09/28] scsi: ufs: core: Change the type of one ufshcd_add_cmd_upiu_trace() argument Change the 'tag' argument into an LRB pointer. This patch prepares for the removal of the hba->lrb[] array. Reviewed-by: Avri Altman Reviewed-by: Peter Wang Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-10-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index fc4d1b6576dc..cdb8147fceb9 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -403,10 +403,11 @@ static void ufshcd_configure_wb(struct ufs_hba *hba) ufshcd_wb_toggle_buf_flush(hba, true); } -static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag, +static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, + struct ufshcd_lrb *lrb, enum ufs_trace_str_t str_t) { - struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr; + struct utp_upiu_req *rq = lrb->ucd_req_ptr; struct utp_upiu_header *header; if (!trace_ufshcd_upiu_enabled()) @@ -415,7 +416,7 @@ static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag, if (str_t == UFS_CMD_SEND) header = &rq->header; else - header = &hba->lrb[tag].ucd_rsp_ptr->header; + header = &lrb->ucd_rsp_ptr->header; trace_ufshcd_upiu(hba, str_t, header, &rq->sc.cdb, UFS_TSF_CDB); @@ -489,7 +490,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, return; /* trace UPIU also */ - ufshcd_add_cmd_upiu_trace(hba, tag, str_t); + ufshcd_add_cmd_upiu_trace(hba, lrbp, str_t); if (!trace_ufshcd_command_enabled()) return; From 60a1f6a8ad334475372a128a6856e893a910302e Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:18 -0700 Subject: [PATCH 10/28] scsi: ufs: core: Only call ufshcd_add_command_trace() for SCSI commands Instead of checking inside ufshcd_add_command_trace() whether 'cmd' points at a SCSI command, let the caller perform that check. This patch prepares for removing the lrbp->cmd pointer. Reviewed-by: Avri Altman Reviewed-by: Peter Wang Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-11-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index cdb8147fceb9..87892a13ef57 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -486,9 +486,6 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, struct request *rq = scsi_cmd_to_rq(cmd); int transfer_len = -1; - if (!cmd) - return; - /* trace UPIU also */ ufshcd_add_cmd_upiu_trace(hba, lrbp, str_t); if (!trace_ufshcd_command_enabled()) @@ -2369,9 +2366,10 @@ void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag, lrbp->compl_time_stamp = ktime_set(0, 0); lrbp->compl_time_stamp_local_clock = 0; } - ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND); - if (lrbp->cmd) + if (lrbp->cmd) { + ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND); ufshcd_clk_scaling_start_busy(hba); + } if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) ufshcd_start_monitor(hba, lrbp); From ffa5d8c15300a3047c9a445dce81515cd3cde86e Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:19 -0700 Subject: [PATCH 11/28] scsi: ufs: core: Change the type of one ufshcd_add_command_trace() argument Change the 'tag' argument into a SCSI command pointer. This patch prepares for the removal of the hba->lrb[] array. Reviewed-by: Avri Altman Reviewed-by: Peter Wang Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-12-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 87892a13ef57..edf56e9e825b 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -473,7 +473,7 @@ static void ufshcd_add_uic_command_trace(struct ufs_hba *hba, ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3)); } -static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, +static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd, enum ufs_trace_str_t str_t) { u64 lba = 0; @@ -481,9 +481,9 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, u32 doorbell = 0; u32 intr; u32 hwq_id = 0; - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; - struct scsi_cmnd *cmd = lrbp->cmd; struct request *rq = scsi_cmd_to_rq(cmd); + unsigned int tag = rq->tag; + struct ufshcd_lrb *lrbp = &hba->lrb[tag]; int transfer_len = -1; /* trace UPIU also */ @@ -501,7 +501,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len); lba = scsi_get_lba(cmd); if (opcode == WRITE_10) - group_id = lrbp->cmd->cmnd[6]; + group_id = cmd->cmnd[6]; } else if (opcode == UNMAP) { /* * The number of Bytes to be unmapped beginning with the lba. @@ -2367,7 +2367,7 @@ void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag, lrbp->compl_time_stamp_local_clock = 0; } if (lrbp->cmd) { - ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND); + ufshcd_add_command_trace(hba, lrbp->cmd, UFS_CMD_SEND); ufshcd_clk_scaling_start_busy(hba); } if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) @@ -5641,7 +5641,7 @@ void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, if (cmd) { if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) ufshcd_update_monitor(hba, lrbp); - ufshcd_add_command_trace(hba, task_tag, UFS_CMD_COMP); + ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP); cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe); ufshcd_release_scsi_cmd(hba, lrbp); /* Do not touch lrbp after scsi done */ From 3e7fff3fee5b7fa1538b2cef023733cf10366a4a Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:20 -0700 Subject: [PATCH 12/28] scsi: ufs: core: Change the type of one ufshcd_send_command() argument Change the 'task_tag' argument into an LRB pointer. This patch prepares for the removal of the hba->lrb[] array. Reviewed-by: Avri Altman Reviewed-by: Peter Wang Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-13-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index edf56e9e825b..f478d5b5230d 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -2350,14 +2350,13 @@ static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb * /** * ufshcd_send_command - Send SCSI or device management commands * @hba: per adapter instance - * @task_tag: Task tag of the command + * @lrbp: Local reference block of SCSI command * @hwq: pointer to hardware queue instance */ -static inline -void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag, - struct ufs_hw_queue *hwq) +static inline void ufshcd_send_command(struct ufs_hba *hba, + struct ufshcd_lrb *lrbp, + struct ufs_hw_queue *hwq) { - struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; unsigned long flags; if (hba->monitor.enabled) { @@ -3066,7 +3065,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) if (hba->mcq_enabled) hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); - ufshcd_send_command(hba, tag, hwq); + ufshcd_send_command(hba, lrbp, hwq); out: if (ufs_trigger_eh(hba)) { @@ -3312,7 +3311,7 @@ static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int err; ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); - ufshcd_send_command(hba, tag, hba->dev_cmd_queue); + ufshcd_send_command(hba, lrbp, hba->dev_cmd_queue); err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, From ae7bf255b10e4c8382faec06f527ba832c18be24 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:21 -0700 Subject: [PATCH 13/28] scsi: ufs: core: Only call ufshcd_should_inform_monitor() for SCSI commands ufshcd_should_inform_monitor() only returns 'true' for SCSI commands. Instead of checking inside ufshcd_should_inform_monitor() whether its second argument represents a SCSI command, only call this function for SCSI commands. This patch prepares for removing the lrbp->cmd member. Reviewed-by: Peter Wang Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-14-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index f478d5b5230d..dbdb8c30ca09 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -2293,12 +2293,13 @@ static inline int ufshcd_monitor_opcode2dir(u8 opcode) return -EINVAL; } +/* Must only be called for SCSI commands. */ static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { const struct ufs_hba_monitor *m = &hba->monitor; - return (m->enabled && lrbp && lrbp->cmd && + return (m->enabled && (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) && ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp)); } @@ -2368,9 +2369,9 @@ static inline void ufshcd_send_command(struct ufs_hba *hba, if (lrbp->cmd) { ufshcd_add_command_trace(hba, lrbp->cmd, UFS_CMD_SEND); ufshcd_clk_scaling_start_busy(hba); + if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) + ufshcd_start_monitor(hba, lrbp); } - if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) - ufshcd_start_monitor(hba, lrbp); if (hba->mcq_enabled) { int utrd_size = sizeof(struct utp_transfer_req_desc); From f59568f4e27af9ba3b0e89909c6fb2cab38700c5 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:22 -0700 Subject: [PATCH 14/28] scsi: ufs: core: Change the monitor function argument types Pass a SCSI command pointer instead of a struct ufshcd_lrb pointer. This patch prepares for combining the SCSI command and ufshcd_lrb data structures into a single data structure. Reviewed-by: Avri Altman Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-15-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 44 +++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index dbdb8c30ca09..df3f651c8d91 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -2295,19 +2295,20 @@ static inline int ufshcd_monitor_opcode2dir(u8 opcode) /* Must only be called for SCSI commands. */ static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) + struct scsi_cmnd *cmd) { const struct ufs_hba_monitor *m = &hba->monitor; + struct request *rq = scsi_cmd_to_rq(cmd); + struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag]; - return (m->enabled && - (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) && - ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp)); + return m->enabled && + (!m->chunk_size || m->chunk_size == cmd->sdb.length) && + ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp); } -static void ufshcd_start_monitor(struct ufs_hba *hba, - const struct ufshcd_lrb *lrbp) +static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd); + int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); unsigned long flags; spin_lock_irqsave(hba->host->host_lock, flags); @@ -2316,14 +2317,15 @@ static void ufshcd_start_monitor(struct ufs_hba *hba, spin_unlock_irqrestore(hba->host->host_lock, flags); } -static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *lrbp) +static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd); + struct request *req = scsi_cmd_to_rq(cmd); + struct ufshcd_lrb *lrbp = &hba->lrb[req->tag]; + int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); unsigned long flags; spin_lock_irqsave(hba->host->host_lock, flags); if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) { - const struct request *req = scsi_cmd_to_rq(lrbp->cmd); struct ufs_hba_monitor *m = &hba->monitor; ktime_t now, inc, lat; @@ -2358,6 +2360,7 @@ static inline void ufshcd_send_command(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, struct ufs_hw_queue *hwq) { + struct scsi_cmnd *cmd = lrbp->cmd; unsigned long flags; if (hba->monitor.enabled) { @@ -2366,11 +2369,11 @@ static inline void ufshcd_send_command(struct ufs_hba *hba, lrbp->compl_time_stamp = ktime_set(0, 0); lrbp->compl_time_stamp_local_clock = 0; } - if (lrbp->cmd) { - ufshcd_add_command_trace(hba, lrbp->cmd, UFS_CMD_SEND); + if (cmd) { + ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND); ufshcd_clk_scaling_start_busy(hba); - if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) - ufshcd_start_monitor(hba, lrbp); + if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) + ufshcd_start_monitor(hba, cmd); } if (hba->mcq_enabled) { @@ -2386,8 +2389,7 @@ static inline void ufshcd_send_command(struct ufs_hba *hba, } else { spin_lock_irqsave(&hba->outstanding_lock, flags); if (hba->vops && hba->vops->setup_xfer_req) - hba->vops->setup_xfer_req(hba, lrbp->task_tag, - !!lrbp->cmd); + hba->vops->setup_xfer_req(hba, lrbp->task_tag, !!cmd); __set_bit(lrbp->task_tag, &hba->outstanding_reqs); ufshcd_writel(hba, 1 << lrbp->task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); @@ -5628,19 +5630,17 @@ void ufshcd_release_scsi_cmd(struct ufs_hba *hba, void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, struct cq_entry *cqe) { - struct ufshcd_lrb *lrbp; - struct scsi_cmnd *cmd; + struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; + struct scsi_cmnd *cmd = lrbp->cmd; enum utp_ocs ocs; - lrbp = &hba->lrb[task_tag]; if (hba->monitor.enabled) { lrbp->compl_time_stamp = ktime_get(); lrbp->compl_time_stamp_local_clock = local_clock(); } - cmd = lrbp->cmd; if (cmd) { - if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) - ufshcd_update_monitor(hba, lrbp); + if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) + ufshcd_update_monitor(hba, cmd); ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP); cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe); ufshcd_release_scsi_cmd(hba, lrbp); From 63a5b959c854c33f280533eedd27bd3489e2b704 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:23 -0700 Subject: [PATCH 15/28] scsi: ufs: core: Rework ufshcd_mcq_compl_pending_transfer() Replace a tag loop with blk_mq_tagset_busy_iter(). This patch prepares for removing the hba->lrb[] array. Reviewed-by: Avri Altman Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-16-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 80 ++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 34 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index df3f651c8d91..ed5def669a33 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -5725,6 +5725,48 @@ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) return completed_reqs != 0; } +static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) +{ + struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); + struct scsi_device *sdev = rq->q->queuedata; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag]; + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); + + if (!hwq) + return true; + + ufshcd_mcq_compl_all_cqes_lock(hba, hwq); + + /* + * For those cmds of which the cqes are not present in the cq, complete + * them explicitly. + */ + scoped_guard(spinlock_irqsave, &hwq->cq_lock) { + if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { + set_host_byte(cmd, DID_REQUEUE); + ufshcd_release_scsi_cmd(hba, lrbp); + scsi_done(cmd); + } + } + + return true; +} + +static bool ufshcd_mcq_compl_one(struct request *rq, void *priv) +{ + struct scsi_device *sdev = rq->q->queuedata; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); + + if (hwq) + ufshcd_mcq_poll_cqe_lock(hba, hwq); + + return true; +} + /** * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is * invoked from the error handler context or ufshcd_host_reset_and_restore() @@ -5739,40 +5781,10 @@ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, bool force_compl) { - struct ufs_hw_queue *hwq; - struct ufshcd_lrb *lrbp; - struct scsi_cmnd *cmd; - unsigned long flags; - int tag; - - for (tag = 0; tag < hba->nutrs; tag++) { - lrbp = &hba->lrb[tag]; - cmd = lrbp->cmd; - if (!ufshcd_cmd_inflight(cmd) || - test_bit(SCMD_STATE_COMPLETE, &cmd->state)) - continue; - - hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); - if (!hwq) - continue; - - if (force_compl) { - ufshcd_mcq_compl_all_cqes_lock(hba, hwq); - /* - * For those cmds of which the cqes are not present - * in the cq, complete them explicitly. - */ - spin_lock_irqsave(&hwq->cq_lock, flags); - if (cmd && !test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { - set_host_byte(cmd, DID_REQUEUE); - ufshcd_release_scsi_cmd(hba, lrbp); - scsi_done(cmd); - } - spin_unlock_irqrestore(&hwq->cq_lock, flags); - } else { - ufshcd_mcq_poll_cqe_lock(hba, hwq); - } - } + blk_mq_tagset_busy_iter(&hba->host->tag_set, + force_compl ? ufshcd_mcq_force_compl_one : + ufshcd_mcq_compl_one, + NULL); } /** From f18fac1e2b729d3bdeb41b946d90b57f3515f438 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:24 -0700 Subject: [PATCH 16/28] scsi: ufs: core: Rework ufshcd_eh_device_reset_handler() Merge the MCQ mode and legacy mode loops into a single loop. This patch prepares for optimizing the hot path by removing the direct hba->lrb[] accesses from ufshcd_eh_device_reset_handler(). Reviewed-by: Avri Altman Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-17-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 84 ++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 46 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index ed5def669a33..f64192c672e2 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -7566,6 +7566,36 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r return err ? : result; } +static bool ufshcd_clear_lu_cmds(struct request *req, void *priv) +{ + struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); + struct scsi_device *sdev = cmd->device; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + const u64 lun = *(u64 *)priv; + const u32 tag = req->tag; + + if (sdev->lun != lun) + return true; + + if (ufshcd_clear_cmd(hba, tag) < 0) { + dev_err(hba->dev, "%s: failed to clear request %d\n", __func__, + tag); + return true; + } + + if (hba->mcq_enabled) { + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, req); + + if (hwq) + ufshcd_mcq_poll_cqe_lock(hba, hwq); + return true; + } + + ufshcd_compl_one_cqe(hba, tag, NULL); + return true; +} + /** * ufshcd_eh_device_reset_handler() - Reset a single logical unit. * @cmd: SCSI command pointer @@ -7574,12 +7604,8 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r */ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) { - unsigned long flags, pending_reqs = 0, not_cleared = 0; struct Scsi_Host *host; struct ufs_hba *hba; - struct ufs_hw_queue *hwq; - struct ufshcd_lrb *lrbp; - u32 pos, not_cleared_mask = 0; int err; u8 resp = 0xF, lun; @@ -7588,50 +7614,16 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp); - if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { - if (!err) - err = resp; - goto out; + if (err) { + } else if (resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { + err = resp; + } else { + /* clear the commands that were pending for corresponding LUN */ + blk_mq_tagset_busy_iter(&hba->host->tag_set, + ufshcd_clear_lu_cmds, + &cmd->device->lun); } - if (hba->mcq_enabled) { - for (pos = 0; pos < hba->nutrs; pos++) { - lrbp = &hba->lrb[pos]; - if (ufshcd_cmd_inflight(lrbp->cmd) && - lrbp->lun == lun) { - ufshcd_clear_cmd(hba, pos); - hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd)); - ufshcd_mcq_poll_cqe_lock(hba, hwq); - } - } - err = 0; - goto out; - } - - /* clear the commands that were pending for corresponding LUN */ - spin_lock_irqsave(&hba->outstanding_lock, flags); - for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) - if (hba->lrb[pos].lun == lun) - __set_bit(pos, &pending_reqs); - hba->outstanding_reqs &= ~pending_reqs; - spin_unlock_irqrestore(&hba->outstanding_lock, flags); - - for_each_set_bit(pos, &pending_reqs, hba->nutrs) { - if (ufshcd_clear_cmd(hba, pos) < 0) { - spin_lock_irqsave(&hba->outstanding_lock, flags); - not_cleared = 1U << pos & - ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); - hba->outstanding_reqs |= not_cleared; - not_cleared_mask |= not_cleared; - spin_unlock_irqrestore(&hba->outstanding_lock, flags); - - dev_err(hba->dev, "%s: failed to clear request %d\n", - __func__, pos); - } - } - __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared_mask); - -out: hba->req_abort_count = 0; ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err); if (!err) { From e8ea985a8314e0170124476e0fa5a5da868a7a40 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:25 -0700 Subject: [PATCH 17/28] scsi: ufs: core: Rework the SCSI host queue depth calculation code Prepare for allocating the SCSI host earlier by making the SCSI host queue depth independent of the queue depth supported by the UFS device. This patch may increase the queue depth of the UFS SCSI host. Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-18-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufs-mcq.c | 25 ++++++------------------- drivers/ufs/core/ufshcd-priv.h | 2 +- drivers/ufs/core/ufshcd.c | 17 +++++++++++++++-- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index bae35d0f99c5..00b043b54419 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -134,17 +134,15 @@ unsigned int ufshcd_mcq_queue_cfg_addr(struct ufs_hba *hba) EXPORT_SYMBOL_GPL(ufshcd_mcq_queue_cfg_addr); /** - * ufshcd_mcq_decide_queue_depth - decide the queue depth + * ufshcd_get_hba_mac - Maximum number of commands supported by the host + * controller. * @hba: per adapter instance * - * Return: queue-depth on success, non-zero on error + * Return: queue depth on success; negative upon error. * - * MAC - Max. Active Command of the Host Controller (HC) - * HC wouldn't send more than this commands to the device. - * Calculates and adjusts the queue depth based on the depth - * supported by the HC and ufs device. + * MAC = Maximum number of Active Commands supported by the Host Controller. */ -int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba) +int ufshcd_get_hba_mac(struct ufs_hba *hba) { int mac; @@ -162,18 +160,7 @@ int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba) mac = hba->vops->get_hba_mac(hba); } if (mac < 0) - goto err; - - WARN_ON_ONCE(!hba->dev_info.bqueuedepth); - /* - * max. value of bqueuedepth = 256, mac is host dependent. - * It is mandatory for UFS device to define bQueueDepth if - * shared queuing architecture is enabled. - */ - return min_t(int, mac, hba->dev_info.bqueuedepth); - -err: - dev_err(hba->dev, "Failed to get mac, err=%d\n", mac); + dev_err(hba->dev, "Failed to get mac, err=%d\n", mac); return mac; } diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h index 1f0d38aa37f9..749c0ab2a4ca 100644 --- a/drivers/ufs/core/ufshcd-priv.h +++ b/drivers/ufs/core/ufshcd-priv.h @@ -67,7 +67,7 @@ void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, struct cq_entry *cqe); int ufshcd_mcq_init(struct ufs_hba *hba); void ufshcd_mcq_disable(struct ufs_hba *hba); -int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba); +int ufshcd_get_hba_mac(struct ufs_hba *hba); int ufshcd_mcq_memory_alloc(struct ufs_hba *hba); struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba, struct request *req); diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index f64192c672e2..93f2853ff5df 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -8466,10 +8466,11 @@ static void ufs_init_rtc(struct ufs_hba *hba, u8 *desc_buf) static int ufs_get_device_desc(struct ufs_hba *hba) { + struct ufs_dev_info *dev_info = &hba->dev_info; + struct Scsi_Host *shost = hba->host; int err; u8 model_index; u8 *desc_buf; - struct ufs_dev_info *dev_info = &hba->dev_info; desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); if (!desc_buf) { @@ -8497,6 +8498,18 @@ static int ufs_get_device_desc(struct ufs_hba *hba) desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1]; dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH]; + /* + * According to the UFS standard, the UFS device queue depth + * (bQueueDepth) must be in the range 1..255 if the shared queueing + * architecture is supported. bQueueDepth is zero if the shared queueing + * architecture is not supported. + */ + if (dev_info->bqueuedepth) + shost->cmd_per_lun = min(hba->nutrs, dev_info->bqueuedepth) - + UFSHCD_NUM_RESERVED; + else + shost->cmd_per_lun = shost->can_queue; + dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP]; dev_info->hid_sup = get_unaligned_be32(desc_buf + @@ -8905,7 +8918,7 @@ static int ufshcd_alloc_mcq(struct ufs_hba *hba) int ret; int old_nutrs = hba->nutrs; - ret = ufshcd_mcq_decide_queue_depth(hba); + ret = ufshcd_get_hba_mac(hba); if (ret < 0) return ret; From f46b9a595fa95c82e3279b687869cac249838e96 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:26 -0700 Subject: [PATCH 18/28] scsi: ufs: core: Allocate the SCSI host earlier Call ufshcd_add_scsi_host() before any UPIU commands are sent to the UFS device. This patch prepares for letting ufshcd_add_scsi_host() allocate memory for both SCSI and UPIU commands. Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-19-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 93f2853ff5df..7935aad4ca13 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -10605,6 +10605,9 @@ static int ufshcd_add_scsi_host(struct ufs_hba *hba) { int err; + WARN_ON_ONCE(!hba->host->can_queue); + WARN_ON_ONCE(!hba->host->cmd_per_lun); + if (is_mcq_supported(hba)) { ufshcd_mcq_enable(hba); err = ufshcd_alloc_mcq(hba); @@ -10765,7 +10768,11 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) ufshcd_host_memory_configure(hba); host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; - host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED; + /* + * Set the queue depth for WLUNs. ufs_get_device_desc() will increase + * host->cmd_per_lun to a larger value. + */ + host->cmd_per_lun = 1; host->max_id = UFSHCD_MAX_ID; host->max_lun = UFS_MAX_LUNS; host->max_channel = UFSHCD_MAX_CHANNEL; @@ -10857,6 +10864,10 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3); } + err = ufshcd_add_scsi_host(hba); + if (err) + goto out_disable; + /* Hold auto suspend until async scan completes */ pm_runtime_get_sync(dev); @@ -10907,10 +10918,6 @@ initialized: if (err) goto out_disable; - err = ufshcd_add_scsi_host(hba); - if (err) - goto out_disable; - async_schedule(ufshcd_async_scan, hba); ufs_sysfs_add_nodes(hba->dev); From 45e636ea129486c4cf7bb342b0d59c186ae2fabb Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:27 -0700 Subject: [PATCH 19/28] scsi: ufs: core: Call ufshcd_init_lrb() later Call ufshcd_init_lrb() from inside ufshcd_setup_dev_cmd() instead of ufshcd_host_memory_configure(). This patch prepares for calling ufshcd_host_memory_configure() before the information is available that is required to call ufshcd_setup_dev_cmd(). Reviewed-by: Avri Altman Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-20-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 53 ++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 7935aad4ca13..f6eecc03282a 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -2904,8 +2904,32 @@ static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); } -static void __ufshcd_setup_cmd(struct ufshcd_lrb *lrbp, struct scsi_cmnd *cmd, u8 lun, int tag) +static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) { + struct utp_transfer_cmd_desc *cmd_descp = + (void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba); + struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; + dma_addr_t cmd_desc_element_addr = + hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba); + u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); + u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); + + lrb->utr_descriptor_ptr = utrdlp + i; + lrb->utrd_dma_addr = + hba->utrdl_dma_addr + i * sizeof(struct utp_transfer_req_desc); + lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu; + lrb->ucd_req_dma_addr = cmd_desc_element_addr; + lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu; + lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; + lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table; + lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; +} + +static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, + struct scsi_cmnd *cmd, u8 lun, int tag) +{ + ufshcd_init_lrb(hba, lrbp, tag); + memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr)); lrbp->cmd = cmd; @@ -2917,7 +2941,7 @@ static void __ufshcd_setup_cmd(struct ufshcd_lrb *lrbp, struct scsi_cmnd *cmd, u static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, struct scsi_cmnd *cmd, u8 lun, int tag) { - __ufshcd_setup_cmd(lrbp, cmd, lun, tag); + __ufshcd_setup_cmd(hba, lrbp, cmd, lun, tag); lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); lrbp->req_abort_skip = false; @@ -2972,27 +2996,6 @@ static void ufshcd_map_queues(struct Scsi_Host *shost) } } -static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) -{ - struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr + - i * ufshcd_get_ucd_size(hba); - struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; - dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr + - i * ufshcd_get_ucd_size(hba); - u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); - u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); - - lrb->utr_descriptor_ptr = utrdlp + i; - lrb->utrd_dma_addr = hba->utrdl_dma_addr + - i * sizeof(struct utp_transfer_req_desc); - lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu; - lrb->ucd_req_dma_addr = cmd_desc_element_addr; - lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu; - lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; - lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table; - lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; -} - /** * ufshcd_queuecommand - main entry point for SCSI requests * @host: SCSI host pointer @@ -3085,7 +3088,7 @@ out: static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, u8 lun, int tag) { - __ufshcd_setup_cmd(lrbp, NULL, lun, tag); + __ufshcd_setup_cmd(hba, lrbp, NULL, lun, tag); lrbp->intr_cmd = true; /* No interrupt aggregation */ hba->dev_cmd.type = cmd_type; } @@ -4048,8 +4051,6 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba) utrdlp[i].response_upiu_length = cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); } - - ufshcd_init_lrb(hba, &hba->lrb[i], i); } } From d3fd0fd7768681eaeb03742c9e8ae846c505f564 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:28 -0700 Subject: [PATCH 20/28] scsi: ufs: core: Use hba->reserved_slot Use hba->reserved_slot instead of open-coding it. This patch prepares for changing the value of hba->reserved_slot. Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-21-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufs-mcq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index 00b043b54419..1de3360a7af1 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -544,7 +544,7 @@ int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag) if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) return -ETIMEDOUT; - if (task_tag != hba->nutrs - UFSHCD_NUM_RESERVED) { + if (task_tag != hba->reserved_slot) { if (!cmd) return -EINVAL; hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); From 1d0af94ffb5d4c9b99d38646521b6d3e4119d044 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:29 -0700 Subject: [PATCH 21/28] scsi: ufs: core: Make the reserved slot a reserved request Instead of letting the SCSI core allocate hba->nutrs - 1 commands, let the SCSI core allocate hba->nutrs commands, set the number of reserved tags to 1 and use the reserved tag for device management commands. This patch changes the 'reserved slot' from hba->nutrs - 1 into 0 because the block layer reserves the smallest tags for reserved commands. Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-22-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index f6eecc03282a..20eae5d9487b 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -2476,7 +2476,7 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1; hba->nutmrs = ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; - hba->reserved_slot = hba->nutrs - 1; + hba->reserved_slot = 0; hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1; @@ -8945,7 +8945,6 @@ static int ufshcd_alloc_mcq(struct ufs_hba *hba) goto err; hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; - hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED; return 0; err: @@ -9184,6 +9183,7 @@ static const struct scsi_host_template ufshcd_driver_template = { .proc_name = UFSHCD, .map_queues = ufshcd_map_queues, .queuecommand = ufshcd_queuecommand, + .nr_reserved_cmds = UFSHCD_NUM_RESERVED, .mq_poll = ufshcd_poll, .sdev_init = ufshcd_sdev_init, .sdev_configure = ufshcd_sdev_configure, From e5f9cc2af9a82700e989ac08be3c85c25046a6f6 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:30 -0700 Subject: [PATCH 22/28] scsi: ufs: core: Do not clear driver-private command data Tell the SCSI core to skip the memset() call that clears driver-private data because __ufshcd_setup_cmd() performs all necessary initialization. Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-23-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 20eae5d9487b..1757aa0237da 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -2996,6 +2996,15 @@ static void ufshcd_map_queues(struct Scsi_Host *shost) } } +/* + * The only purpose of this function is to make the SCSI core skip the memset() + * call for the private command data. + */ +static int ufshcd_init_cmd_priv(struct Scsi_Host *host, struct scsi_cmnd *cmd) +{ + return 0; +} + /** * ufshcd_queuecommand - main entry point for SCSI requests * @host: SCSI host pointer @@ -9182,6 +9191,7 @@ static const struct scsi_host_template ufshcd_driver_template = { .name = UFSHCD, .proc_name = UFSHCD, .map_queues = ufshcd_map_queues, + .init_cmd_priv = ufshcd_init_cmd_priv, .queuecommand = ufshcd_queuecommand, .nr_reserved_cmds = UFSHCD_NUM_RESERVED, .mq_poll = ufshcd_poll, From 22089c218037ca7cd50d4fa20e8b5bd746a9b397 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:31 -0700 Subject: [PATCH 23/28] scsi: ufs: core: Optimize the hot path Set .cmd_size in the SCSI host template such that the SCSI core makes struct scsi_cmnd and struct ufshcd_lrb adjacent. Convert the cmd->lrbp and lrbp->cmd memory loads into pointer offset calculations. Remove the data structure members that became superfluous, namely ufshcd_lrb.cmd and ufs_hba.lrb. Since ufshcd_lrb.cmd is removed, this pointer cannot be used anymore to test whether or not a command is a SCSI command. Introduce a new function for this purpose, namely ufshcd_is_scsi_cmd(). Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-24-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufs-mcq.c | 9 +- drivers/ufs/core/ufshcd-crypto.h | 18 ++- drivers/ufs/core/ufshcd-priv.h | 41 +++++- drivers/ufs/core/ufshcd.c | 235 ++++++++++++++++--------------- include/ufs/ufshcd.h | 5 - 5 files changed, 179 insertions(+), 129 deletions(-) diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index 1de3360a7af1..776ff0896a2a 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -534,8 +534,8 @@ static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq) */ int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag) { - struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; - struct scsi_cmnd *cmd = lrbp->cmd; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); struct ufs_hw_queue *hwq; void __iomem *reg, *opr_sqd_base; u32 nexus, id, val; @@ -618,7 +618,8 @@ static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd) static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba, struct ufs_hw_queue *hwq, int task_tag) { - struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); struct utp_transfer_req_desc *utrd; __le64 cmd_desc_base_addr; bool ret = false; @@ -669,7 +670,7 @@ int ufshcd_mcq_abort(struct scsi_cmnd *cmd) struct Scsi_Host *host = cmd->device->host; struct ufs_hba *hba = shost_priv(host); int tag = scsi_cmd_to_rq(cmd)->tag; - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); struct ufs_hw_queue *hwq; int err; diff --git a/drivers/ufs/core/ufshcd-crypto.h b/drivers/ufs/core/ufshcd-crypto.h index 89bb97c14c15..c148a5194378 100644 --- a/drivers/ufs/core/ufshcd-crypto.h +++ b/drivers/ufs/core/ufshcd-crypto.h @@ -38,10 +38,10 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, } static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) + struct scsi_cmnd *cmd) { - struct scsi_cmnd *cmd = lrbp->cmd; const struct bio_crypt_ctx *crypt_ctx = scsi_cmd_to_rq(cmd)->crypt_ctx; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); if (crypt_ctx && hba->vops && hba->vops->fill_crypto_prdt) return hba->vops->fill_crypto_prdt(hba, crypt_ctx, @@ -51,17 +51,19 @@ static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, } static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) + struct scsi_cmnd *cmd) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT)) return; - if (!(scsi_cmd_to_rq(lrbp->cmd)->crypt_ctx)) + if (!(scsi_cmd_to_rq(cmd)->crypt_ctx)) return; /* Zeroize the PRDT because it can contain cryptographic keys. */ memzero_explicit(lrbp->ucd_prdt_ptr, - ufshcd_sg_entry_size(hba) * scsi_sg_count(lrbp->cmd)); + ufshcd_sg_entry_size(hba) * scsi_sg_count(cmd)); } bool ufshcd_crypto_enable(struct ufs_hba *hba); @@ -82,13 +84,15 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, struct request_desc_header *h) { } static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) + struct scsi_cmnd *cmd) { return 0; } static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) { } + struct scsi_cmnd *cmd) +{ +} static inline bool ufshcd_crypto_enable(struct ufs_hba *hba) { diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h index 749c0ab2a4ca..72d2766b19e3 100644 --- a/drivers/ufs/core/ufshcd-priv.h +++ b/drivers/ufs/core/ufshcd-priv.h @@ -77,8 +77,7 @@ bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd); int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag); int ufshcd_mcq_abort(struct scsi_cmnd *cmd); int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag); -void ufshcd_release_scsi_cmd(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp); +void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd); #define SD_ASCII_STD true #define SD_RAW false @@ -363,6 +362,44 @@ static inline bool ufs_is_valid_unit_desc_lun(struct ufs_dev_info *dev_info, u8 return lun == UFS_UPIU_RPMB_WLUN || (lun < dev_info->max_lu_supported); } +/* + * Convert a block layer tag into a SCSI command pointer. This function is + * called once per I/O completion path and is also called from error paths. + */ +static inline struct scsi_cmnd *ufshcd_tag_to_cmd(struct ufs_hba *hba, u32 tag) +{ + struct blk_mq_tags *tags = hba->host->tag_set.shared_tags; + struct request *rq; + + /* + * Handle reserved tags differently because the UFS driver does not + * call blk_mq_alloc_request() for allocating reserved requests. + * Allocating reserved tags with blk_mq_alloc_request() would require + * the following: + * - Allocate an additional request queue from &hba->host->tag_set for + * allocating reserved requests from. + * - For that request queue, allocate a SCSI device. + * - Calling blk_mq_alloc_request(hba->dev_mgmt_queue, REQ_OP_DRV_OUT, + * BLK_MQ_REQ_RESERVED) for allocating a reserved request and + * blk_mq_free_request() for freeing reserved requests. + * - Set the .device pointer for these reserved requests. + * - Submit reserved requests with blk_execute_rq(). + * - Modify ufshcd_queuecommand() such that it handles reserved requests + * in another way than SCSI requests. + * - Modify ufshcd_compl_one_cqe() such that it calls scsi_done() for + * device management commands. + * - Modify all callback functions called by blk_mq_tagset_busy_iter() + * calls in the UFS driver and skip device management commands. + */ + rq = tag < UFSHCD_NUM_RESERVED ? tags->static_rqs[tag] : + blk_mq_tag_to_rq(tags, tag); + + if (WARN_ON_ONCE(!rq)) + return NULL; + + return blk_mq_rq_to_pdu(rq); +} + static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q) __must_hold(&q->sq_lock) { diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 1757aa0237da..ce657b2506fb 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "ufshcd-priv.h" #include #include @@ -483,7 +484,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd, u32 hwq_id = 0; struct request *rq = scsi_cmd_to_rq(cmd); unsigned int tag = rq->tag; - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); int transfer_len = -1; /* trace UPIU also */ @@ -594,14 +595,13 @@ static void ufshcd_print_evt_hist(struct ufs_hba *hba) ufshcd_vops_dbg_register_dump(hba); } -static -void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt) +static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd, + bool pr_prdt) { - const struct ufshcd_lrb *lrbp; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + const int tag = lrbp->task_tag; int prdt_length; - lrbp = &hba->lrb[tag]; - if (hba->monitor.enabled) { dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000)); @@ -644,7 +644,7 @@ static bool ufshcd_print_tr_iter(struct request *req, void *priv) struct Scsi_Host *shost = sdev->host; struct ufs_hba *hba = shost_priv(shost); - ufshcd_print_tr(hba, req->tag, *(bool *)priv); + ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv); return true; } @@ -2298,8 +2298,7 @@ static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) { const struct ufs_hba_monitor *m = &hba->monitor; - struct request *rq = scsi_cmd_to_rq(cmd); - struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag]; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); return m->enabled && (!m->chunk_size || m->chunk_size == cmd->sdb.length) && @@ -2320,7 +2319,7 @@ static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) { struct request *req = scsi_cmd_to_rq(cmd); - struct ufshcd_lrb *lrbp = &hba->lrb[req->tag]; + const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); unsigned long flags; @@ -2350,17 +2349,26 @@ static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) spin_unlock_irqrestore(hba->host->host_lock, flags); } +/* + * Returns %true for SCSI commands and %false for device management commands. + * Must not be called for SCSI commands that have not yet been started. + */ +static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd) +{ + return blk_mq_request_started(scsi_cmd_to_rq(cmd)); +} + /** * ufshcd_send_command - Send SCSI or device management commands * @hba: per adapter instance - * @lrbp: Local reference block of SCSI command + * @cmd: SCSI command or device management command pointer * @hwq: pointer to hardware queue instance */ static inline void ufshcd_send_command(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp, + struct scsi_cmnd *cmd, struct ufs_hw_queue *hwq) { - struct scsi_cmnd *cmd = lrbp->cmd; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); unsigned long flags; if (hba->monitor.enabled) { @@ -2369,7 +2377,7 @@ static inline void ufshcd_send_command(struct ufs_hba *hba, lrbp->compl_time_stamp = ktime_set(0, 0); lrbp->compl_time_stamp_local_clock = 0; } - if (cmd) { + if (ufshcd_is_scsi_cmd(cmd)) { ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND); ufshcd_clk_scaling_start_busy(hba); if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) @@ -2389,7 +2397,8 @@ static inline void ufshcd_send_command(struct ufs_hba *hba, } else { spin_lock_irqsave(&hba->outstanding_lock, flags); if (hba->vops && hba->vops->setup_xfer_req) - hba->vops->setup_xfer_req(hba, lrbp->task_tag, !!cmd); + hba->vops->setup_xfer_req(hba, lrbp->task_tag, + ufshcd_is_scsi_cmd(cmd)); __set_bit(lrbp->task_tag, &hba->outstanding_reqs); ufshcd_writel(hba, 1 << lrbp->task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); @@ -2399,11 +2408,12 @@ static inline void ufshcd_send_command(struct ufs_hba *hba, /** * ufshcd_copy_sense_data - Copy sense data in case of check condition - * @lrbp: pointer to local reference block + * @cmd: SCSI command */ -static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) +static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd) { - u8 *const sense_buffer = lrbp->cmd->sense_buffer; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + u8 *const sense_buffer = cmd->sense_buffer; u16 resp_len; int len; @@ -2708,13 +2718,13 @@ static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int /** * ufshcd_map_sg - Map scatter-gather list to prdt * @hba: per adapter instance - * @lrbp: pointer to local reference block + * @cmd: SCSI command * * Return: 0 in case of success, non-zero value in case of failure. */ -static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) +static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - struct scsi_cmnd *cmd = lrbp->cmd; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); int sg_segments = scsi_dma_map(cmd); if (sg_segments < 0) @@ -2722,7 +2732,7 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd)); - return ufshcd_crypto_fill_prdt(hba, lrbp); + return ufshcd_crypto_fill_prdt(hba, cmd); } /** @@ -2781,13 +2791,13 @@ ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, /** * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, * for scsi commands - * @lrbp: local reference block pointer + * @cmd: SCSI command * @upiu_flags: flags */ -static -void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags) +static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd, + u8 upiu_flags) { - struct scsi_cmnd *cmd = lrbp->cmd; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; unsigned short cdb_len; @@ -2890,22 +2900,25 @@ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU) * for SCSI Purposes * @hba: per adapter instance - * @lrbp: pointer to local reference block + * @cmd: SCSI command */ -static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) +static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - struct request *rq = scsi_cmd_to_rq(lrbp->cmd); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + struct request *rq = scsi_cmd_to_rq(cmd); unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); u8 upiu_flags; - ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0); + ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, + cmd->sc_data_direction, 0); if (ioprio_class == IOPRIO_CLASS_RT) upiu_flags |= UPIU_CMD_FLAGS_CP; - ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); + ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags); } -static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) +static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd) { + const int i = scsi_cmd_to_rq(cmd)->tag; struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba); struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; @@ -2913,6 +2926,7 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba); u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); + struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd); lrb->utr_descriptor_ptr = utrdlp + i; lrb->utrd_dma_addr = @@ -2925,27 +2939,31 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; } -static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, - struct scsi_cmnd *cmd, u8 lun, int tag) +static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + u8 lun, int tag) { - ufshcd_init_lrb(hba, lrbp, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + + ufshcd_init_lrb(hba, cmd); memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr)); - lrbp->cmd = cmd; lrbp->task_tag = tag; lrbp->lun = lun; - ufshcd_prepare_lrbp_crypto(cmd ? scsi_cmd_to_rq(cmd) : NULL, lrbp); + ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ? + scsi_cmd_to_rq(cmd) : NULL, lrbp); } -static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, - struct scsi_cmnd *cmd, u8 lun, int tag) +static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + u8 lun, int tag) { - __ufshcd_setup_cmd(hba, lrbp, cmd, lun, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + + __ufshcd_setup_cmd(hba, cmd, lun, tag); lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); lrbp->req_abort_skip = false; - ufshcd_comp_scsi_upiu(hba, lrbp); + ufshcd_comp_scsi_upiu(hba, cmd); } /** @@ -3016,7 +3034,6 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) { struct ufs_hba *hba = shost_priv(host); int tag = scsi_cmd_to_rq(cmd)->tag; - struct ufshcd_lrb *lrbp; int err = 0; struct ufs_hw_queue *hwq = NULL; @@ -3067,11 +3084,10 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) ufshcd_hold(hba); - lrbp = &hba->lrb[tag]; + ufshcd_setup_scsi_cmd(hba, cmd, + ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); - ufshcd_setup_scsi_cmd(hba, lrbp, cmd, ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); - - err = ufshcd_map_sg(hba, lrbp); + err = ufshcd_map_sg(hba, cmd); if (err) { ufshcd_release(hba); goto out; @@ -3080,7 +3096,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) if (hba->mcq_enabled) hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); - ufshcd_send_command(hba, lrbp, hwq); + ufshcd_send_command(hba, cmd, hwq); out: if (ufs_trigger_eh(hba)) { @@ -3094,10 +3110,12 @@ out: return err; } -static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, - enum dev_cmd_type cmd_type, u8 lun, int tag) +static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + enum dev_cmd_type cmd_type, u8 lun, int tag) { - __ufshcd_setup_cmd(hba, lrbp, NULL, lun, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + + __ufshcd_setup_cmd(hba, cmd, lun, tag); lrbp->intr_cmd = true; /* No interrupt aggregation */ hba->dev_cmd.type = cmd_type; } @@ -3105,10 +3123,12 @@ static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, /* * Return: 0 upon success; < 0 upon failure. */ -static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag) +static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + enum dev_cmd_type cmd_type, int tag) { - ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + + ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); return ufshcd_compose_devman_upiu(hba, lrbp); } @@ -3320,13 +3340,14 @@ static void ufshcd_dev_man_unlock(struct ufs_hba *hba) * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; * < 0 if another error occurred. */ -static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, - const u32 tag, int timeout) +static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + const u32 tag, int timeout) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); int err; ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); - ufshcd_send_command(hba, lrbp, hba->dev_cmd_queue); + ufshcd_send_command(hba, cmd, hba->dev_cmd_queue); err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, @@ -3351,17 +3372,17 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type, int timeout) { const u32 tag = hba->reserved_slot; - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); int err; /* Protects use of hba->reserved_slot. */ lockdep_assert_held(&hba->dev_cmd.lock); - err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); + err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag); if (unlikely(err)) return err; - return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout); + return ufshcd_issue_dev_cmd(hba, cmd, tag, timeout); } /** @@ -3991,14 +4012,6 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba) } skip_utmrdl: - /* Allocate memory for local reference block */ - hba->lrb = devm_kcalloc(hba->dev, - hba->nutrs, sizeof(struct ufshcd_lrb), - GFP_KERNEL); - if (!hba->lrb) { - dev_err(hba->dev, "LRB Memory allocation failed\n"); - goto out; - } return 0; out: return -ENOMEM; @@ -5411,19 +5424,18 @@ static void ufshcd_sdev_destroy(struct scsi_device *sdev) /** * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status - * @lrbp: pointer to local reference block of completed command + * @cmd: SCSI command * @scsi_status: SCSI command status * * Return: value base on SCSI command status. */ -static inline int -ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) +static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status) { int result = 0; switch (scsi_status) { case SAM_STAT_CHECK_CONDITION: - ufshcd_copy_sense_data(lrbp); + ufshcd_copy_sense_data(cmd); fallthrough; case SAM_STAT_GOOD: result |= DID_OK << 16 | scsi_status; @@ -5431,7 +5443,7 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) case SAM_STAT_TASK_SET_FULL: case SAM_STAT_BUSY: case SAM_STAT_TASK_ABORTED: - ufshcd_copy_sense_data(lrbp); + ufshcd_copy_sense_data(cmd); result |= scsi_status; break; default: @@ -5445,15 +5457,16 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) /** * ufshcd_transfer_rsp_status - Get overall status of the response * @hba: per adapter instance - * @lrbp: pointer to local reference block of completed command + * @cmd: SCSI command * @cqe: pointer to the completion queue entry * * Return: result of the command to notify SCSI midlayer. */ -static inline int -ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, - struct cq_entry *cqe) +static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, + struct scsi_cmnd *cmd, + struct cq_entry *cqe) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); int result = 0; int scsi_status; enum utp_ocs ocs; @@ -5467,7 +5480,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, * not set either flag. */ if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW)) - scsi_set_resid(lrbp->cmd, resid); + scsi_set_resid(cmd, resid); /* overall command status of utrd */ ocs = ufshcd_get_tr_ocs(lrbp, cqe); @@ -5488,7 +5501,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, * to notify the SCSI midlayer of the command status */ scsi_status = lrbp->ucd_rsp_ptr->header.status; - result = ufshcd_scsi_cmd_status(lrbp, scsi_status); + result = ufshcd_scsi_cmd_status(cmd, scsi_status); /* * Currently we are only supporting BKOPs exception @@ -5553,7 +5566,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) { if (cqe) ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry)); - ufshcd_print_tr(hba, lrbp->task_tag, true); + ufshcd_print_tr(hba, cmd, true); } return result; } @@ -5620,13 +5633,10 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) } /* Release the resources allocated for processing a SCSI command. */ -void ufshcd_release_scsi_cmd(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) +void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - struct scsi_cmnd *cmd = lrbp->cmd; - scsi_dma_unmap(cmd); - ufshcd_crypto_clear_prdt(hba, lrbp); + ufshcd_crypto_clear_prdt(hba, cmd); ufshcd_release(hba); ufshcd_clk_scaling_update_busy(hba); } @@ -5640,20 +5650,20 @@ void ufshcd_release_scsi_cmd(struct ufs_hba *hba, void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, struct cq_entry *cqe) { - struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; - struct scsi_cmnd *cmd = lrbp->cmd; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); enum utp_ocs ocs; if (hba->monitor.enabled) { lrbp->compl_time_stamp = ktime_get(); lrbp->compl_time_stamp_local_clock = local_clock(); } - if (cmd) { + if (ufshcd_is_scsi_cmd(cmd)) { if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) ufshcd_update_monitor(hba, cmd); ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP); - cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe); - ufshcd_release_scsi_cmd(hba, lrbp); + cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe); + ufshcd_release_scsi_cmd(hba, cmd); /* Do not touch lrbp after scsi done */ scsi_done(cmd); } else { @@ -5690,7 +5700,7 @@ static void ufshcd_clear_polled(struct ufs_hba *hba, int tag; for_each_set_bit(tag, completed_reqs, hba->nutrs) { - struct scsi_cmnd *cmd = hba->lrb[tag].cmd; + struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag); if (!cmd) continue; @@ -5741,7 +5751,6 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) struct scsi_device *sdev = rq->q->queuedata; struct Scsi_Host *shost = sdev->host; struct ufs_hba *hba = shost_priv(shost); - struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag]; struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); if (!hwq) @@ -5756,7 +5765,7 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) scoped_guard(spinlock_irqsave, &hwq->cq_lock) { if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { set_host_byte(cmd, DID_REQUEUE); - ufshcd_release_scsi_cmd(hba, lrbp); + ufshcd_release_scsi_cmd(hba, cmd); scsi_done(cmd); } } @@ -6641,7 +6650,7 @@ static bool ufshcd_abort_one(struct request *rq, void *priv) *ret = ufshcd_try_to_abort_task(hba, tag); dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, - hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1, + ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1, *ret ? "failed" : "succeeded"); return *ret == 0; @@ -7371,14 +7380,15 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, enum query_opcode desc_op) { const u32 tag = hba->reserved_slot; - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); int err = 0; u8 upiu_flags; /* Protects use of hba->reserved_slot. */ lockdep_assert_held(&hba->dev_cmd.lock); - ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag); + ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); @@ -7403,7 +7413,7 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, * bound to fail since dev_cmd.query and dev_cmd.type were left empty. * read the response directly ignoring all errors. */ - ufshcd_issue_dev_cmd(hba, lrbp, tag, dev_cmd_timeout); + ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout); /* just copy the upiu response as it is */ memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); @@ -7518,7 +7528,8 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r enum dma_data_direction dir) { const u32 tag = hba->reserved_slot; - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); int err = 0; int result; u8 upiu_flags; @@ -7529,7 +7540,8 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r /* Protects use of hba->reserved_slot. */ ufshcd_dev_man_lock(hba); - ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag); + ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, + tag); ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs); @@ -7546,7 +7558,7 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); - err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT); + err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT); if (!err) { /* Just copy the upiu response as it is */ @@ -7647,11 +7659,12 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) { - struct ufshcd_lrb *lrbp; int tag; for_each_set_bit(tag, &bitmap, hba->nutrs) { - lrbp = &hba->lrb[tag]; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + lrbp->req_abort_skip = true; } } @@ -7659,7 +7672,7 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) /** * ufshcd_try_to_abort_task - abort a specific task * @hba: Pointer to adapter instance - * @tag: Task tag/index to be aborted + * @tag: Tag of the task to be aborted * * Abort the pending command in device by sending UFS_ABORT_TASK task management * command, and in host controller by clearing the door-bell register. There can @@ -7671,7 +7684,8 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) */ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) { - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); int err; int poll_cnt; u8 resp = 0xF; @@ -7693,7 +7707,7 @@ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) hba->dev, "%s: cmd with tag %d not pending in the device.\n", __func__, tag); - if (!ufshcd_cmd_inflight(lrbp->cmd)) { + if (!ufshcd_cmd_inflight(cmd)) { dev_info(hba->dev, "%s: cmd with tag=%d completed.\n", __func__, tag); @@ -7741,7 +7755,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) struct Scsi_Host *host = cmd->device->host; struct ufs_hba *hba = shost_priv(host); int tag = scsi_cmd_to_rq(cmd)->tag; - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); unsigned long flags; int err = FAILED; bool outstanding; @@ -7776,9 +7790,9 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); ufshcd_print_pwr_info(hba); - ufshcd_print_tr(hba, tag, true); + ufshcd_print_tr(hba, cmd, true); } else { - ufshcd_print_tr(hba, tag, false); + ufshcd_print_tr(hba, cmd, false); } hba->req_abort_count++; @@ -7822,7 +7836,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release; } - err = ufshcd_try_to_abort_task(hba, tag); + err = ufshcd_try_to_abort_task(hba, lrbp->task_tag); if (err) { dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); @@ -7839,7 +7853,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) spin_unlock_irqrestore(&hba->outstanding_lock, flags); if (outstanding) - ufshcd_release_scsi_cmd(hba, lrbp); + ufshcd_release_scsi_cmd(hba, cmd); err = SUCCESS; @@ -8919,8 +8933,6 @@ static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs) utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs; dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr, hba->utrdl_dma_addr); - - devm_kfree(hba->dev, hba->lrb); } static int ufshcd_alloc_mcq(struct ufs_hba *hba) @@ -9191,6 +9203,7 @@ static const struct scsi_host_template ufshcd_driver_template = { .name = UFSHCD, .proc_name = UFSHCD, .map_queues = ufshcd_map_queues, + .cmd_size = sizeof(struct ufshcd_lrb), .init_cmd_priv = ufshcd_init_cmd_priv, .queuecommand = ufshcd_queuecommand, .nr_reserved_cmds = UFSHCD_NUM_RESERVED, diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 00152e135fc9..fbed47b6c61f 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -161,7 +161,6 @@ struct ufs_pm_lvl_states { * @ucd_prdt_dma_addr: PRDT dma address for debug * @ucd_rsp_dma_addr: UPIU response dma address for debug * @ucd_req_dma_addr: UPIU request dma address for debug - * @cmd: pointer to SCSI command * @scsi_status: SCSI status of the command * @command_type: SCSI, UFS, Query. * @task_tag: Task tag of the command @@ -186,7 +185,6 @@ struct ufshcd_lrb { dma_addr_t ucd_rsp_dma_addr; dma_addr_t ucd_prdt_dma_addr; - struct scsi_cmnd *cmd; int scsi_status; int command_type; @@ -833,7 +831,6 @@ enum ufshcd_mcq_opr { * @spm_lvl: desired UFS power management level during system PM. * @pm_op_in_progress: whether or not a PM operation is in progress. * @ahit: value of Auto-Hibernate Idle Timer register. - * @lrb: local reference block * @outstanding_tasks: Bits representing outstanding task requests * @outstanding_lock: Protects @outstanding_reqs. * @outstanding_reqs: Bits representing outstanding transfer requests @@ -976,8 +973,6 @@ struct ufs_hba { /* Auto-Hibernate Idle Timer register value */ u32 ahit; - struct ufshcd_lrb *lrb; - unsigned long outstanding_tasks; spinlock_t outstanding_lock; unsigned long outstanding_reqs; From 176b93004c3475cfc0c939a4aeec846f6bf2b9a0 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:32 -0700 Subject: [PATCH 24/28] scsi: ufs: core: Pass a SCSI pointer instead of an LRB pointer Pass a pointer to a SCSI command between functions instead of an LRB pointer. This change prepares for removing the ufshcd_lrb task_tag member. Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-25-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index ce657b2506fb..cf2c08baa9ae 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -2822,12 +2822,13 @@ static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd, /** * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request * @hba: UFS hba - * @lrbp: local reference block pointer + * @cmd: SCSI command pointer * @upiu_flags: flags */ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp, u8 upiu_flags) + struct scsi_cmnd *cmd, u8 upiu_flags) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; struct ufs_query *query = &hba->dev_cmd.query; u16 len = be16_to_cpu(query->request.upiu_req.length); @@ -2856,8 +2857,9 @@ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, memcpy(ucd_req_ptr + 1, query->descriptor, len); } -static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) +static inline void ufshcd_prepare_utp_nop_upiu(struct scsi_cmnd *cmd) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); @@ -2872,22 +2874,23 @@ static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU) * for Device Management Purposes * @hba: per adapter instance - * @lrbp: pointer to local reference block + * @cmd: SCSI command pointer * * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) + struct scsi_cmnd *cmd) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); u8 upiu_flags; int ret = 0; ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) - ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags); + ufshcd_prepare_utp_query_req_upiu(hba, cmd, upiu_flags); else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) - ufshcd_prepare_utp_nop_upiu(lrbp); + ufshcd_prepare_utp_nop_upiu(cmd); else ret = -EINVAL; @@ -3126,11 +3129,9 @@ static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, enum dev_cmd_type cmd_type, int tag) { - struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); - ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); - return ufshcd_compose_devman_upiu(hba, lrbp); + return ufshcd_compose_devman_upiu(hba, cmd); } /* From 9a2c9500921d5ebbe96f7531adc73d9205c76485 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:33 -0700 Subject: [PATCH 25/28] scsi: ufs: core: Remove the ufshcd_lrb task_tag member Remove the ufshcd_lrb task_tag member and use scsi_cmd_to_rq(cmd)->tag instead. Use rq->tag instead of lrbp->task_tag. This patch reduces the size of struct ufshcd_lrb. Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-26-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 62 +++++++++++++++++++-------------------- include/ufs/ufshcd.h | 1 - 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index cf2c08baa9ae..3e0fa433579d 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -599,7 +599,7 @@ static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd, bool pr_prdt) { struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); - const int tag = lrbp->task_tag; + const int tag = scsi_cmd_to_rq(cmd)->tag; int prdt_length; if (hba->monitor.enabled) { @@ -2369,6 +2369,7 @@ static inline void ufshcd_send_command(struct ufs_hba *hba, struct ufs_hw_queue *hwq) { struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + const int tag = scsi_cmd_to_rq(cmd)->tag; unsigned long flags; if (hba->monitor.enabled) { @@ -2397,11 +2398,10 @@ static inline void ufshcd_send_command(struct ufs_hba *hba, } else { spin_lock_irqsave(&hba->outstanding_lock, flags); if (hba->vops && hba->vops->setup_xfer_req) - hba->vops->setup_xfer_req(hba, lrbp->task_tag, + hba->vops->setup_xfer_req(hba, tag, ufshcd_is_scsi_cmd(cmd)); - __set_bit(lrbp->task_tag, &hba->outstanding_reqs); - ufshcd_writel(hba, 1 << lrbp->task_tag, - REG_UTP_TRANSFER_REQ_DOOR_BELL); + __set_bit(tag, &hba->outstanding_reqs); + ufshcd_writel(hba, 1 << tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); spin_unlock_irqrestore(&hba->outstanding_lock, flags); } } @@ -2798,6 +2798,7 @@ static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd, u8 upiu_flags) { struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + const int tag = scsi_cmd_to_rq(cmd)->tag; struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; unsigned short cdb_len; @@ -2805,11 +2806,11 @@ static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd, .transaction_code = UPIU_TRANSACTION_COMMAND, .flags = upiu_flags, .lun = lrbp->lun, - .task_tag = lrbp->task_tag, + .task_tag = tag, .command_set_type = UPIU_COMMAND_SET_TYPE_SCSI, }; - WARN_ON_ONCE(ucd_req_ptr->header.task_tag != lrbp->task_tag); + WARN_ON_ONCE(ucd_req_ptr->header.task_tag != tag); ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length); @@ -2830,6 +2831,7 @@ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, { struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; + const int tag = scsi_cmd_to_rq(cmd)->tag; struct ufs_query *query = &hba->dev_cmd.query; u16 len = be16_to_cpu(query->request.upiu_req.length); @@ -2838,7 +2840,7 @@ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, .transaction_code = UPIU_TRANSACTION_QUERY_REQ, .flags = upiu_flags, .lun = lrbp->lun, - .task_tag = lrbp->task_tag, + .task_tag = tag, .query_function = query->request.query_func, /* Data segment length only need for WRITE_DESC */ .data_segment_length = @@ -2861,12 +2863,13 @@ static inline void ufshcd_prepare_utp_nop_upiu(struct scsi_cmnd *cmd) { struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; + const int tag = scsi_cmd_to_rq(cmd)->tag; memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); ucd_req_ptr->header = (struct utp_upiu_header){ .transaction_code = UPIU_TRANSACTION_NOP_OUT, - .task_tag = lrbp->task_tag, + .task_tag = tag, }; } @@ -2951,7 +2954,6 @@ static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr)); - lrbp->task_tag = tag; lrbp->lun = lun; ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ? scsi_cmd_to_rq(cmd) : NULL, lrbp); @@ -3249,6 +3251,8 @@ ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int max_timeout) { + struct scsi_cmnd *cmd = (struct scsi_cmnd *)lrbp - 1; + const int tag = scsi_cmd_to_rq(cmd)->tag; unsigned long time_left = msecs_to_jiffies(max_timeout); unsigned long flags; bool pending; @@ -3265,18 +3269,18 @@ retry: } else { err = -ETIMEDOUT; dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n", - __func__, lrbp->task_tag); + __func__, tag); /* MCQ mode */ if (hba->mcq_enabled) { /* successfully cleared the command, retry if needed */ - if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) + if (ufshcd_clear_cmd(hba, tag) == 0) err = -EAGAIN; return err; } /* SDB mode */ - if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) { + if (ufshcd_clear_cmd(hba, tag) == 0) { /* successfully cleared the command, retry if needed */ err = -EAGAIN; /* @@ -3285,11 +3289,9 @@ retry: * variable. */ spin_lock_irqsave(&hba->outstanding_lock, flags); - pending = test_bit(lrbp->task_tag, - &hba->outstanding_reqs); + pending = test_bit(tag, &hba->outstanding_reqs); if (pending) - __clear_bit(lrbp->task_tag, - &hba->outstanding_reqs); + __clear_bit(tag, &hba->outstanding_reqs); spin_unlock_irqrestore(&hba->outstanding_lock, flags); if (!pending) { @@ -3302,11 +3304,10 @@ retry: } } else { dev_err(hba->dev, "%s: failed to clear tag %d\n", - __func__, lrbp->task_tag); + __func__, tag); spin_lock_irqsave(&hba->outstanding_lock, flags); - pending = test_bit(lrbp->task_tag, - &hba->outstanding_reqs); + pending = test_bit(tag, &hba->outstanding_reqs); spin_unlock_irqrestore(&hba->outstanding_lock, flags); if (!pending) { @@ -5468,6 +5469,7 @@ static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct cq_entry *cqe) { struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + const int tag = scsi_cmd_to_rq(cmd)->tag; int result = 0; int scsi_status; enum utp_ocs ocs; @@ -5539,10 +5541,8 @@ static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, case OCS_ABORTED: case OCS_INVALID_COMMAND_STATUS: result |= DID_REQUEUE << 16; - dev_warn(hba->dev, - "OCS %s from controller for tag %d\n", - (ocs == OCS_ABORTED ? "aborted" : "invalid"), - lrbp->task_tag); + dev_warn(hba->dev, "OCS %s from controller for tag %d\n", + ocs == OCS_ABORTED ? "aborted" : "invalid", tag); break; case OCS_INVALID_CMD_TABLE_ATTR: case OCS_INVALID_PRDT_ATTR: @@ -5555,9 +5555,8 @@ static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, case OCS_GENERAL_CRYPTO_ERROR: default: result |= DID_ERROR << 16; - dev_err(hba->dev, - "OCS error from controller = %x for tag %d\n", - ocs, lrbp->task_tag); + dev_err(hba->dev, "OCS error from controller = %x for tag %d\n", + ocs, tag); ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); break; @@ -7692,8 +7691,8 @@ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) u8 resp = 0xF; for (poll_cnt = 100; poll_cnt; poll_cnt--) { - err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, - UFS_QUERY_TASK, &resp); + err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_QUERY_TASK, + &resp); if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { /* cmd pending in the device */ dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n", @@ -7726,8 +7725,7 @@ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) if (!poll_cnt) return -EBUSY; - err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, - UFS_ABORT_TASK, &resp); + err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_ABORT_TASK, &resp); if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { if (!err) { err = resp; /* service response error */ @@ -7837,7 +7835,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release; } - err = ufshcd_try_to_abort_task(hba, lrbp->task_tag); + err = ufshcd_try_to_abort_task(hba, tag); if (err) { dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index fbed47b6c61f..a92062f65455 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -188,7 +188,6 @@ struct ufshcd_lrb { int scsi_status; int command_type; - int task_tag; u8 lun; /* UPIU LUN id field is only 8-bit wide */ bool intr_cmd; bool req_abort_skip; From 4b6c0d9cca359d1d2f666e14c42482befe5c4829 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:34 -0700 Subject: [PATCH 26/28] scsi: ufs: core: Make blk_mq_tagset_busy_iter() skip reserved requests A later patch will convert hba->reserved_slot into a reserved tag. Make blk_mq_tagset_busy_iter() skip reserved requests such that device management commands are skipped. Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-27-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 3e0fa433579d..f493f180279f 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -644,7 +644,8 @@ static bool ufshcd_print_tr_iter(struct request *req, void *priv) struct Scsi_Host *shost = sdev->host; struct ufs_hba *hba = shost_priv(shost); - ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv); + if (!blk_mq_is_reserved_rq(req)) + ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv); return true; } @@ -5753,7 +5754,7 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) struct ufs_hba *hba = shost_priv(shost); struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); - if (!hwq) + if (blk_mq_is_reserved_rq(rq) || !hwq) return true; ufshcd_mcq_compl_all_cqes_lock(hba, hwq); @@ -5780,7 +5781,7 @@ static bool ufshcd_mcq_compl_one(struct request *rq, void *priv) struct ufs_hba *hba = shost_priv(shost); struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); - if (hwq) + if (!blk_mq_is_reserved_rq(rq) && hwq) ufshcd_mcq_poll_cqe_lock(hba, hwq); return true; @@ -6648,6 +6649,9 @@ static bool ufshcd_abort_one(struct request *rq, void *priv) struct Scsi_Host *shost = sdev->host; struct ufs_hba *hba = shost_priv(shost); + if (blk_mq_is_reserved_rq(rq)) + return true; + *ret = ufshcd_try_to_abort_task(hba, tag); dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1, @@ -7597,7 +7601,7 @@ static bool ufshcd_clear_lu_cmds(struct request *req, void *priv) const u64 lun = *(u64 *)priv; const u32 tag = req->tag; - if (sdev->lun != lun) + if (blk_mq_is_reserved_rq(req) || sdev->lun != lun) return true; if (ufshcd_clear_cmd(hba, tag) < 0) { From a11c015c8a4fff99cf91a5cbc9789cc7a72b3ccf Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:35 -0700 Subject: [PATCH 27/28] scsi: ufs: core: Move code out of ufshcd_wait_for_dev_cmd() The ufshcd_dev_cmd_completion() call is useful for some but not for all ufshcd_wait_for_dev_cmd() callers. Hence, remove the ufshcd_dev_cmd_completion() call from ufshcd_wait_for_dev_cmd() and move it past the ufshcd_issue_dev_cmd() calls where appropriate. This makes it easier to detect timeout errors for UPIU frames submitted through the BSG interface. Reviewed-by: Avri Altman Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-28-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index f493f180279f..2175b41262c8 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -3265,8 +3265,6 @@ retry: if (likely(time_left)) { err = ufshcd_get_tr_ocs(lrbp, NULL); - if (!err) - err = ufshcd_dev_cmd_completion(hba, lrbp); } else { err = -ETIMEDOUT; dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n", @@ -3376,6 +3374,7 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, { const u32 tag = hba->reserved_slot; struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); int err; /* Protects use of hba->reserved_slot. */ @@ -3385,7 +3384,11 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, if (unlikely(err)) return err; - return ufshcd_issue_dev_cmd(hba, cmd, tag, timeout); + err = ufshcd_issue_dev_cmd(hba, cmd, tag, timeout); + if (err) + return err; + + return ufshcd_dev_cmd_completion(hba, lrbp); } /** @@ -7412,12 +7415,9 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); - /* - * ignore the returning value here - ufshcd_check_query_response is - * bound to fail since dev_cmd.query and dev_cmd.type were left empty. - * read the response directly ignoring all errors. - */ - ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout); + err = ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout); + if (err) + return err; /* just copy the upiu response as it is */ memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); @@ -7563,7 +7563,10 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT); + if (err) + return err; + err = ufshcd_dev_cmd_completion(hba, lrbp); if (!err) { /* Just copy the upiu response as it is */ memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); From 08b12cda6c44dc015bcc152613c35ee0ae8f37b9 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 31 Oct 2025 13:39:36 -0700 Subject: [PATCH 28/28] scsi: ufs: core: Switch to scsi_get_internal_cmd() Instead of storing the tag of the reserved command in hba->reserved_slot, use scsi_get_internal_cmd() and scsi_put_internal_cmd() to allocate the tag for the reserved command dynamically. Add ufshcd_queue_reserved_command() for submitting reserved commands. Add support in ufshcd_abort() for device management commands. Use blk_execute_rq() for submitting reserved commands. Remove the code and data structures that became superfluous. This includes ufshcd_wait_for_dev_cmd(), hba->reserved_slot and ufs_dev_cmd.complete. Signed-off-by: Bart Van Assche Link: https://patch.msgid.link/20251031204029.2883185-29-bvanassche@acm.org Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufs-mcq.c | 19 +-- drivers/ufs/core/ufshcd-priv.h | 25 +--- drivers/ufs/core/ufshcd.c | 225 ++++++++++++++++----------------- include/ufs/ufshcd.h | 6 - 4 files changed, 116 insertions(+), 159 deletions(-) diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index 776ff0896a2a..9ab91b4c05b0 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -479,9 +479,6 @@ int ufshcd_mcq_init(struct ufs_hba *hba) mutex_init(&hwq->sq_mutex); } - /* The very first HW queue serves device commands */ - hba->dev_cmd_queue = &hba->uhq[0]; - host->host_tagset = 1; return 0; } @@ -536,6 +533,7 @@ int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag) { struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + struct request *rq = scsi_cmd_to_rq(cmd); struct ufs_hw_queue *hwq; void __iomem *reg, *opr_sqd_base; u32 nexus, id, val; @@ -544,15 +542,12 @@ int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag) if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) return -ETIMEDOUT; - if (task_tag != hba->reserved_slot) { - if (!cmd) - return -EINVAL; - hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); - if (!hwq) - return 0; - } else { - hwq = hba->dev_cmd_queue; - } + if (!cmd) + return -EINVAL; + + hwq = ufshcd_mcq_req_to_hwq(hba, rq); + if (!hwq) + return 0; id = hwq->id; diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h index 72d2766b19e3..2f752a45db87 100644 --- a/drivers/ufs/core/ufshcd-priv.h +++ b/drivers/ufs/core/ufshcd-priv.h @@ -369,30 +369,7 @@ static inline bool ufs_is_valid_unit_desc_lun(struct ufs_dev_info *dev_info, u8 static inline struct scsi_cmnd *ufshcd_tag_to_cmd(struct ufs_hba *hba, u32 tag) { struct blk_mq_tags *tags = hba->host->tag_set.shared_tags; - struct request *rq; - - /* - * Handle reserved tags differently because the UFS driver does not - * call blk_mq_alloc_request() for allocating reserved requests. - * Allocating reserved tags with blk_mq_alloc_request() would require - * the following: - * - Allocate an additional request queue from &hba->host->tag_set for - * allocating reserved requests from. - * - For that request queue, allocate a SCSI device. - * - Calling blk_mq_alloc_request(hba->dev_mgmt_queue, REQ_OP_DRV_OUT, - * BLK_MQ_REQ_RESERVED) for allocating a reserved request and - * blk_mq_free_request() for freeing reserved requests. - * - Set the .device pointer for these reserved requests. - * - Submit reserved requests with blk_execute_rq(). - * - Modify ufshcd_queuecommand() such that it handles reserved requests - * in another way than SCSI requests. - * - Modify ufshcd_compl_one_cqe() such that it calls scsi_done() for - * device management commands. - * - Modify all callback functions called by blk_mq_tagset_busy_iter() - * calls in the UFS driver and skip device management commands. - */ - rq = tag < UFSHCD_NUM_RESERVED ? tags->static_rqs[tag] : - blk_mq_tag_to_rq(tags, tag); + struct request *rq = blk_mq_tag_to_rq(tags, tag); if (WARN_ON_ONCE(!rq)) return NULL; diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 2175b41262c8..ca17165f6f0e 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -2350,13 +2350,10 @@ static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) spin_unlock_irqrestore(hba->host->host_lock, flags); } -/* - * Returns %true for SCSI commands and %false for device management commands. - * Must not be called for SCSI commands that have not yet been started. - */ +/* Returns %true for SCSI commands and %false for device management commands. */ static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd) { - return blk_mq_request_started(scsi_cmd_to_rq(cmd)); + return !blk_mq_is_reserved_rq(scsi_cmd_to_rq(cmd)); } /** @@ -2487,7 +2484,6 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1; hba->nutmrs = ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; - hba->reserved_slot = 0; hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1; @@ -3116,6 +3112,20 @@ out: return err; } +static int ufshcd_queue_reserved_command(struct Scsi_Host *host, + struct scsi_cmnd *cmd) +{ + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + struct request *rq = scsi_cmd_to_rq(cmd); + struct ufs_hba *hba = shost_priv(host); + struct ufs_hw_queue *hwq = + hba->mcq_enabled ? ufshcd_mcq_req_to_hwq(hba, rq) : NULL; + + ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); + ufshcd_send_command(hba, cmd, hwq); + return 0; +} + static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, enum dev_cmd_type cmd_type, u8 lun, int tag) { @@ -3245,84 +3255,6 @@ ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) return err; } -/* - * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; - * < 0 if another error occurred. - */ -static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp, int max_timeout) -{ - struct scsi_cmnd *cmd = (struct scsi_cmnd *)lrbp - 1; - const int tag = scsi_cmd_to_rq(cmd)->tag; - unsigned long time_left = msecs_to_jiffies(max_timeout); - unsigned long flags; - bool pending; - int err; - -retry: - time_left = wait_for_completion_timeout(&hba->dev_cmd.complete, - time_left); - - if (likely(time_left)) { - err = ufshcd_get_tr_ocs(lrbp, NULL); - } else { - err = -ETIMEDOUT; - dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n", - __func__, tag); - - /* MCQ mode */ - if (hba->mcq_enabled) { - /* successfully cleared the command, retry if needed */ - if (ufshcd_clear_cmd(hba, tag) == 0) - err = -EAGAIN; - return err; - } - - /* SDB mode */ - if (ufshcd_clear_cmd(hba, tag) == 0) { - /* successfully cleared the command, retry if needed */ - err = -EAGAIN; - /* - * Since clearing the command succeeded we also need to - * clear the task tag bit from the outstanding_reqs - * variable. - */ - spin_lock_irqsave(&hba->outstanding_lock, flags); - pending = test_bit(tag, &hba->outstanding_reqs); - if (pending) - __clear_bit(tag, &hba->outstanding_reqs); - spin_unlock_irqrestore(&hba->outstanding_lock, flags); - - if (!pending) { - /* - * The completion handler ran while we tried to - * clear the command. - */ - time_left = 1; - goto retry; - } - } else { - dev_err(hba->dev, "%s: failed to clear tag %d\n", - __func__, tag); - - spin_lock_irqsave(&hba->outstanding_lock, flags); - pending = test_bit(tag, &hba->outstanding_reqs); - spin_unlock_irqrestore(&hba->outstanding_lock, flags); - - if (!pending) { - /* - * The completion handler ran while we tried to - * clear the command. - */ - time_left = 1; - goto retry; - } - } - } - - return err; -} - static void ufshcd_dev_man_lock(struct ufs_hba *hba) { ufshcd_hold(hba); @@ -3337,6 +3269,24 @@ static void ufshcd_dev_man_unlock(struct ufs_hba *hba) ufshcd_release(hba); } +static struct scsi_cmnd *ufshcd_get_dev_mgmt_cmd(struct ufs_hba *hba) +{ + /* + * The caller must hold this lock to guarantee that the NOWAIT + * allocation will succeed. + */ + lockdep_assert_held(&hba->dev_cmd.lock); + + return scsi_get_internal_cmd( + hba->host->pseudo_sdev, DMA_TO_DEVICE, + BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT); +} + +static void ufshcd_put_dev_mgmt_cmd(struct scsi_cmnd *cmd) +{ + scsi_put_internal_cmd(cmd); +} + /* * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; * < 0 if another error occurred. @@ -3345,16 +3295,14 @@ static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, const u32 tag, int timeout) { struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); - int err; + struct request *rq = scsi_cmd_to_rq(cmd); + blk_status_t sts; - ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); - ufshcd_send_command(hba, cmd, hba->dev_cmd_queue); - err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); - - ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, - (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); - - return err; + rq->timeout = timeout; + sts = blk_execute_rq(rq, true); + if (sts != BLK_STS_OK) + return blk_status_to_errno(sts); + return lrbp->utr_descriptor_ptr->header.ocs; } /** @@ -3372,23 +3320,31 @@ static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type, int timeout) { - const u32 tag = hba->reserved_slot; - struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); + struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + u32 tag; int err; - /* Protects use of hba->reserved_slot. */ + /* Protects use of hba->dev_cmd. */ lockdep_assert_held(&hba->dev_cmd.lock); + if (WARN_ON_ONCE(!cmd)) + return -ENOMEM; + + tag = scsi_cmd_to_rq(cmd)->tag; + err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag); if (unlikely(err)) - return err; + goto out; err = ufshcd_issue_dev_cmd(hba, cmd, tag, timeout); - if (err) - return err; + if (err == 0) + err = ufshcd_dev_cmd_completion(hba, lrbp); - return ufshcd_dev_cmd_completion(hba, lrbp); +out: + ufshcd_put_dev_mgmt_cmd(cmd); + + return err; } /** @@ -5658,6 +5614,10 @@ void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); enum utp_ocs ocs; + if (WARN_ONCE(!cmd, "cqe->command_desc_base_addr = %#llx\n", + le64_to_cpu(cqe->command_desc_base_addr))) + return; + if (hba->monitor.enabled) { lrbp->compl_time_stamp = ktime_get(); lrbp->compl_time_stamp_local_clock = local_clock(); @@ -5668,15 +5628,21 @@ void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP); cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe); ufshcd_release_scsi_cmd(hba, cmd); - /* Do not touch lrbp after scsi done */ - scsi_done(cmd); } else { if (cqe) { ocs = cqe->overall_status & MASK_OCS; lrbp->utr_descriptor_ptr->header.ocs = ocs; + } else { + ocs = lrbp->utr_descriptor_ptr->header.ocs; } - complete(&hba->dev_cmd.complete); + ufshcd_add_query_upiu_trace( + hba, + ocs == OCS_SUCCESS ? UFS_QUERY_COMP : UFS_QUERY_ERR, + (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); + cmd->result = 0; } + /* Do not touch lrbp after scsi_done() has been called. */ + scsi_done(cmd); } /** @@ -7386,15 +7352,20 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type, enum query_opcode desc_op) { - const u32 tag = hba->reserved_slot; - struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); + struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + u32 tag; int err = 0; u8 upiu_flags; - /* Protects use of hba->reserved_slot. */ + /* Protects use of hba->dev_cmd. */ lockdep_assert_held(&hba->dev_cmd.lock); + if (WARN_ON_ONCE(!cmd)) + return -ENOMEM; + + tag = scsi_cmd_to_rq(cmd)->tag; + ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); @@ -7417,7 +7388,7 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, err = ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout); if (err) - return err; + goto put_dev_mgmt_cmd; /* just copy the upiu response as it is */ memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); @@ -7438,6 +7409,9 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, } } +put_dev_mgmt_cmd: + ufshcd_put_dev_mgmt_cmd(cmd); + return err; } @@ -7531,9 +7505,9 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list, enum dma_data_direction dir) { - const u32 tag = hba->reserved_slot; - struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); - struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + struct scsi_cmnd *cmd; + struct ufshcd_lrb *lrbp; + u32 tag; int err = 0; int result; u8 upiu_flags; @@ -7541,9 +7515,18 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r u16 ehs_len; int ehs = (hba->capabilities & MASK_EHSLUTRD_SUPPORTED) ? 2 : 0; - /* Protects use of hba->reserved_slot. */ ufshcd_dev_man_lock(hba); + cmd = ufshcd_get_dev_mgmt_cmd(hba); + + if (WARN_ON_ONCE(!cmd)) { + err = -ENOMEM; + goto unlock; + } + + lrbp = scsi_cmd_priv(cmd); + tag = scsi_cmd_to_rq(cmd)->tag; + ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag); @@ -7564,7 +7547,7 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT); if (err) - return err; + goto put_dev_mgmt_cmd; err = ufshcd_dev_cmd_completion(hba, lrbp); if (!err) { @@ -7590,6 +7573,10 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r } } +put_dev_mgmt_cmd: + ufshcd_put_dev_mgmt_cmd(cmd); + +unlock: ufshcd_dev_man_unlock(hba); return err ? : result; @@ -7760,7 +7747,8 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) { struct Scsi_Host *host = cmd->device->host; struct ufs_hba *hba = shost_priv(host); - int tag = scsi_cmd_to_rq(cmd)->tag; + struct request *rq = scsi_cmd_to_rq(cmd); + int tag = rq->tag; struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); unsigned long flags; int err = FAILED; @@ -7790,7 +7778,8 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) * to reduce repeated printouts. For other aborted requests only print * basic details. */ - scsi_print_command(cmd); + if (ufshcd_is_scsi_cmd(cmd)) + scsi_print_command(cmd); if (!hba->req_abort_count) { ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag); ufshcd_print_evt_hist(hba); @@ -7842,7 +7831,10 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release; } - err = ufshcd_try_to_abort_task(hba, tag); + if (blk_mq_is_reserved_rq(rq)) + err = ufshcd_clear_cmd(hba, tag); + else + err = ufshcd_try_to_abort_task(hba, tag); if (err) { dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); @@ -9212,6 +9204,7 @@ static const struct scsi_host_template ufshcd_driver_template = { .cmd_size = sizeof(struct ufshcd_lrb), .init_cmd_priv = ufshcd_init_cmd_priv, .queuecommand = ufshcd_queuecommand, + .queue_reserved_command = ufshcd_queue_reserved_command, .nr_reserved_cmds = UFSHCD_NUM_RESERVED, .mq_poll = ufshcd_poll, .sdev_init = ufshcd_sdev_init, @@ -10764,8 +10757,6 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) */ hba->vcc_off_delay_us = 2000; - init_completion(&hba->dev_cmd.complete); - err = ufshcd_hba_init(hba); if (err) goto out_error; diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index a92062f65455..c07ba003a5cb 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -236,13 +236,11 @@ struct ufs_query { * struct ufs_dev_cmd - all assosiated fields with device management commands * @type: device management command type - Query, NOP OUT * @lock: lock to allow one command at a time - * @complete: internal commands completion * @query: Device management query information */ struct ufs_dev_cmd { enum dev_cmd_type type; struct mutex lock; - struct completion complete; struct ufs_query query; }; @@ -838,7 +836,6 @@ enum ufshcd_mcq_opr { * @nutrs: Transfer Request Queue depth supported by controller * @nortt - Max outstanding RTTs supported by controller * @nutmrs: Task Management Queue depth supported by controller - * @reserved_slot: Used to submit device commands. Protected by @dev_cmd.lock. * @ufs_version: UFS Version to which controller complies * @vops: pointer to variant specific operations * @vps: pointer to variant specific parameters @@ -929,7 +926,6 @@ enum ufshcd_mcq_opr { * @res: array of resource info of MCQ registers * @mcq_base: Multi circular queue registers base address * @uhq: array of supported hardware queues - * @dev_cmd_queue: Queue for issuing device management commands * @mcq_opr: MCQ operation and runtime registers * @ufs_rtc_update_work: A work for UFS RTC periodic update * @pm_qos_req: PM QoS request handle @@ -981,7 +977,6 @@ struct ufs_hba { int nortt; u32 mcq_capabilities; int nutmrs; - u32 reserved_slot; u32 ufs_version; const struct ufs_hba_variant_ops *vops; struct ufs_hba_variant_params *vps; @@ -1099,7 +1094,6 @@ struct ufs_hba { bool mcq_esi_enabled; void __iomem *mcq_base; struct ufs_hw_queue *uhq; - struct ufs_hw_queue *dev_cmd_queue; struct ufshcd_mcq_opr_info_t mcq_opr[OPR_MAX]; struct delayed_work ufs_rtc_update_work;