diff options
36 files changed, 1561 insertions, 289 deletions
diff --git a/Documentation/ABI/stable/configfs-nvmet b/Documentation/ABI/stable/configfs-nvmet new file mode 100644 index 000000000000..36b587404ee7 --- /dev/null +++ b/Documentation/ABI/stable/configfs-nvmet @@ -0,0 +1,352 @@ +What: /config/nvmet/ports/N/addr_adrfam +What: /config/nvmet/ports/N/addr_portid +What: /config/nvmet/ports/N/addr_traddr +What: /config/nvmet/ports/N/addr_trsvcid +What: /config/nvmet/ports/N/addr_trtype +What: /config/nvmet/ports/N/addr_treq +Date: June 2016 +KernelVersion: 4.8 +Contact: Christoph Hellwig <hch@lst.de> +Description: + Address attributes for an NVMe-oF target port. + + addr_adrfam: Shows or sets the address family. Accepted + values: "pcie", "ipv4", "ipv6", "ib", "fc", "pci", "loop". + + addr_portid: Shows or sets the port identifier (u16). + + addr_traddr: Shows or sets the transport address string. + + addr_trsvcid: Shows or sets the transport service identifier. + + addr_trtype: Shows or sets the transport type. Accepted + values: "rdma", "fc", "tcp", "pci", "loop". Also + initializes default TSAS values. + + addr_treq: Shows or sets the transport security requirements. + Accepted values: "not specified", "required", + "not required". For TCP with TLS1.3, "not specified" is + rejected. + + All attributes require the port to be disabled before + modification. + +What: /config/nvmet/ports/N/referrals/NAME/addr_adrfam +What: /config/nvmet/ports/N/referrals/NAME/addr_portid +What: /config/nvmet/ports/N/referrals/NAME/addr_traddr +What: /config/nvmet/ports/N/referrals/NAME/addr_trsvcid +What: /config/nvmet/ports/N/referrals/NAME/addr_trtype +What: /config/nvmet/ports/N/referrals/NAME/addr_treq +What: /config/nvmet/ports/N/referrals/NAME/enable +Date: June 2016 +KernelVersion: 4.8 +Contact: Christoph Hellwig <hch@lst.de> +Description: + Address attributes and enable control for a referral entry + under a port. The addr_* attributes have the same semantics + as the corresponding port-level attributes. The enable + attribute shows or sets whether this referral is enabled + (boolean). + +What: /config/nvmet/ports/N/param_inline_data_size +Date: June 2018 +KernelVersion: 4.19 +Contact: Steve Wise <swise@opengridcomputing.com> +Description: + Shows or sets the inline data size for this port. Default + is -1 which lets the transport choose. The port must be + disabled before modification. + +What: /config/nvmet/ports/N/ana_groups/ID/ana_state +Date: June 2018 +KernelVersion: 4.19 +Contact: Christoph Hellwig <hch@lst.de> +Description: + Shows or sets the ANA (Asymmetric Namespace Access) state + for this group on this port. Accepted values: "optimized", + "non-optimized", "inaccessible", "persistent-loss", + "change". Changes trigger an ANA change event. + +What: /config/nvmet/ports/N/param_pi_enable +Date: May 2020 +KernelVersion: 5.8 +Contact: Israel Rukshin <israelr@mellanox.com> +Description: + Shows or sets whether protection information (PI) is + enabled/supported for this port. Accepts boolean value. + Only available when CONFIG_BLK_DEV_INTEGRITY is enabled. + The port must be disabled before modification. + +What: /config/nvmet/ports/N/addr_tsas +Date: August 2023 +KernelVersion: 6.7 +Contact: Hannes Reinecke <hare@suse.de> +Description: + Shows or sets the transport-specific address subtype. For + TCP transport, accepted values: "none", "tls1.3" (requires + CONFIG_NVME_TARGET_TCP_TLS). For RDMA transport, shows the + QP type: "connected" or "datagram". The port must be + disabled before modification. + +What: /config/nvmet/ports/N/param_max_queue_size +Date: January 2024 +KernelVersion: 6.9 +Contact: Max Gurtovoy <mgurtovoy@nvidia.com> +Description: + Shows or sets the maximum queue size for this port. Default + is -1 which lets the transport choose. The port must be + disabled before modification. + +What: /config/nvmet/ports/N/param_mdts +Date: April 2026 +KernelVersion: 7.1 +Contact: Aurelien Aptel <aaptel@nvidia.com> +Description: + Shows or sets the maximum data transfer size for this port. + Default is -1 which lets the transport choose. The port + must be disabled before modification. + +What: /config/nvmet/subsystems/NAME/attr_allow_any_host +Date: June 2016 +KernelVersion: 4.8 +Contact: Christoph Hellwig <hch@lst.de> +Description: + Shows or sets whether any host is allowed to connect. + Accepts boolean value. Cannot be set to 1 if explicit + hosts are linked in the allowed_hosts/ directory. + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/device_path +What: /config/nvmet/subsystems/NAME/namespaces/NSID/device_nguid +What: /config/nvmet/subsystems/NAME/namespaces/NSID/enable +Date: June 2016 +KernelVersion: 4.8 +Contact: Christoph Hellwig <hch@lst.de> +Description: + Namespace attributes added with the initial NVMe target. + + device_path: Shows or sets the backend block device path. + The namespace must be disabled before modification. + + device_nguid: Shows or sets the NGUID (128-bit identifier). + Accepts 32 hex digits with optional "-" or ":" separators. + The namespace must be disabled before modification. + + enable: Shows or sets whether this namespace is enabled + (boolean). + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/device_uuid +Date: June 2017 +KernelVersion: 4.13 +Contact: Johannes Thumshirn <jthumshirn@suse.de> +Description: + Shows or sets the UUID for this namespace. The namespace + must be disabled before modification. + +What: /config/nvmet/subsystems/NAME/attr_version +What: /config/nvmet/subsystems/NAME/attr_serial +Date: July 2017 +KernelVersion: 4.13 +Contact: Johannes Thumshirn <jthumshirn@suse.de> +Description: + attr_version: Shows or sets the NVMe version reported by + this subsystem. Format: "major.minor" or + "major.minor.tertiary". Cannot be changed after the + subsystem has been discovered. + + attr_serial: Shows or sets the serial number. Must be a + 1-20 byte ASCII string (characters 0x20-0x7e). Cannot be + changed after the subsystem has been discovered. + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/ana_grpid +Date: June 2018 +KernelVersion: 4.19 +Contact: Christoph Hellwig <hch@lst.de> +Description: + Shows or sets the ANA (Asymmetric Namespace Access) Group + ID for this namespace. Must be between 1 and 128. Changing + triggers an ANA event notification. + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/buffered_io +Date: June 2018 +KernelVersion: 4.19 +Contact: Chaitanya Kulkarni <chaitanya.kulkarni@wdc.com> +Description: + Shows or sets whether buffered I/O is used for this + namespace. Accepts boolean value. The namespace must be + disabled before modification. + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/p2pmem +Date: October 2018 +KernelVersion: 4.20 +Contact: Logan Gunthorpe <logang@deltatee.com> +Description: + Shows or sets the P2P DMA memory device for this namespace. + Accepts a PCI device BDF, "auto", or "none". The namespace + must be disabled before modification. Only available when + CONFIG_PCI_P2PDMA is enabled. + +What: /config/nvmet/subsystems/NAME/attr_cntlid_min +What: /config/nvmet/subsystems/NAME/attr_cntlid_max +Date: January 2020 +KernelVersion: 5.7 +Contact: Chaitanya Kulkarni <chaitanya.kulkarni@wdc.com> +Description: + attr_cntlid_min: Shows or sets the minimum controller ID + (u16). Must be nonzero and not greater than attr_cntlid_max. + + attr_cntlid_max: Shows or sets the maximum controller ID + (u16). Must be nonzero and not less than attr_cntlid_min. + +What: /config/nvmet/subsystems/NAME/attr_model +Date: January 2020 +KernelVersion: 5.7 +Contact: Mark Ruijter <MRuijter@onestopsystems.com> +Description: + Shows or sets the model number for this subsystem. Must + be a 1-40 byte ASCII string (characters 0x20-0x7e). + Cannot be changed after the subsystem has been discovered. + +What: /config/nvmet/subsystems/NAME/attr_pi_enable +Date: May 2020 +KernelVersion: 5.8 +Contact: Israel Rukshin <israelr@mellanox.com> +Description: + Shows or sets whether protection information (PI) is + enabled/supported for this subsystem. Accepts boolean + value. Only available when CONFIG_BLK_DEV_INTEGRITY is + enabled. + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/revalidate_size +Date: May 2020 +KernelVersion: 5.8 +Contact: Chaitanya Kulkarni <chaitanya.kulkarni@wdc.com> +Description: + Write-only. Writing 1 triggers namespace size revalidation. + If the size has changed, a namespace changed AEN is sent. + The namespace must be enabled. + +What: /config/nvmet/subsystems/NAME/passthru/device_path +What: /config/nvmet/subsystems/NAME/passthru/enable +Date: July 2020 +KernelVersion: 5.9 +Contact: Logan Gunthorpe <logang@deltatee.com> +Description: + Passthrough mode attributes. + + device_path: Shows or sets the NVMe controller character + device path (e.g., /dev/nvme0). Cannot be changed while + the passthrough controller is active. + + enable: Shows or sets whether passthrough mode is enabled + (boolean). + + Only available when CONFIG_NVME_TARGET_PASSTHRU is enabled. + +What: /config/nvmet/subsystems/NAME/passthru/admin_timeout +What: /config/nvmet/subsystems/NAME/passthru/io_timeout +Date: November 2020 +KernelVersion: 5.11 +Contact: Chaitanya Kulkarni <chaitanya.kulkarni@wdc.com> +Description: + admin_timeout: Shows or sets the admin command timeout for + passthrough mode, in jiffies. + + io_timeout: Shows or sets the I/O command timeout for + passthrough mode, in jiffies. + + Only available when CONFIG_NVME_TARGET_PASSTHRU is enabled. + +What: /config/nvmet/subsystems/NAME/passthru/clear_ids +Date: June 2022 +KernelVersion: 5.19 +Contact: Alan Adamson <alan.adamson@oracle.com> +Description: + Shows or sets whether to clear identify data IDs in + passthrough mode. Only available when + CONFIG_NVME_TARGET_PASSTHRU is enabled. + +What: /config/nvmet/subsystems/NAME/attr_qid_max +Date: August 2022 +KernelVersion: 6.1 +Contact: Daniel Wagner <dwagner@suse.de> +Description: + Shows or sets the maximum queue ID (number of I/O queues, + u16). Must be between 1 and 128. Changing this value + forces reconnection of all connected controllers. + +What: /config/nvmet/subsystems/NAME/attr_ieee_oui +Date: November 2022 +KernelVersion: 6.2 +Contact: Aleksandr Miloserdov <a.miloserdov@yadro.com> +Description: + Shows or sets the IEEE OUI for this subsystem. Displayed + in "0x%06x" format. Must be a 24-bit value. Cannot be + changed after the subsystem has been discovered. + +What: /config/nvmet/subsystems/NAME/attr_firmware +Date: November 2022 +KernelVersion: 6.2 +Contact: Aleksandr Miloserdov <a.miloserdov@yadro.com> +Description: + Shows or sets the firmware revision string for this + subsystem. Must be a 1-8 byte ASCII string (characters + 0x20-0x7e). Cannot be changed after the subsystem has + been discovered. + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/resv_enable +Date: November 2024 +KernelVersion: 6.13 +Contact: Guixin Liu <kanie@linux.alibaba.com> +Description: + Shows or sets whether persistent reservation support is + enabled for this namespace. Accepts boolean value. The + namespace must be disabled before modification. + +What: /config/nvmet/subsystems/NAME/attr_vendor_id +What: /config/nvmet/subsystems/NAME/attr_subsys_vendor_id +Date: January 2025 +KernelVersion: 6.14 +Contact: Damien Le Moal <dlemoal@kernel.org> +Description: + attr_vendor_id: Shows or sets the PCI vendor ID reported + by this subsystem. Displayed in "0x%x" format. + + attr_subsys_vendor_id: Shows or sets the PCI subsystem + vendor ID. Displayed in "0x%x" format. + +What: /config/nvmet/hosts/HOSTNQN/dhchap_key +What: /config/nvmet/hosts/HOSTNQN/dhchap_ctrl_key +What: /config/nvmet/hosts/HOSTNQN/dhchap_hash +What: /config/nvmet/hosts/HOSTNQN/dhchap_dhgroup +Date: June 2022 +KernelVersion: 6.0 +Contact: Hannes Reinecke <hare@suse.de> +Description: + DH-HMAC-CHAP authentication attributes. + + dhchap_key: Shows or sets the host secret key. Accepts a + key string in "DHHC-1:" format. + + dhchap_ctrl_key: Shows or sets the controller secret key + for bidirectional authentication. Same format as dhchap_key. + + dhchap_hash: Shows or sets the HMAC hash algorithm. + Accepted values: "hmac(sha256)", "hmac(sha384)", + "hmac(sha512)". + + dhchap_dhgroup: Shows or sets the Diffie-Hellman group for + DH-HMAC-CHAP key exchange. Accepted values: "null", + "ffdhe2048", "ffdhe3072", "ffdhe4096", "ffdhe6144". + Non-null groups require the corresponding KPP crypto + algorithm to be available. + + Only available when CONFIG_NVME_TARGET_AUTH is enabled. + +What: /config/nvmet/discovery_nqn +Date: April 2024 +KernelVersion: 6.9 +Contact: Hannes Reinecke <hare@kernel.org> +Description: + Shows or sets the NQN of the discovery subsystem. The + value must be unique and not duplicate any existing + subsystem name. diff --git a/Documentation/ABI/stable/sysfs-nvme b/Documentation/ABI/stable/sysfs-nvme new file mode 100644 index 000000000000..a2f5d0710db4 --- /dev/null +++ b/Documentation/ABI/stable/sysfs-nvme @@ -0,0 +1,453 @@ +What: /sys/class/nvme/nvmeX/model +What: /sys/class/nvme/nvmeX/serial +What: /sys/class/nvme/nvmeX/firmware_rev +Date: January 2016 +KernelVersion: 4.5 +Contact: Keith Busch <kbusch@kernel.org> +Description: + Shows the model, serial number, or firmware revision string + of the NVMe controller, as reported in the Identify + Controller data structure. + +What: /sys/class/nvme/nvmeX/cntlid +Date: February 2016 +KernelVersion: 4.6 +Contact: Ming Lin <ming.l@ssi.samsung.com> +Description: + Shows the controller identifier assigned by the NVMe + subsystem. + +What: /sys/class/nvme/nvmeX/cntrltype +What: /sys/class/nvme/nvmeX/dctype +Date: February 2022 +KernelVersion: 5.18 +Contact: Martin Belanger <martin.belanger@dell.com> +Description: + cntrltype: Shows the controller type. Possible values: "io", + "discovery", "admin", "reserved". + + dctype: Shows the discovery controller type. Possible values: + "none", "ddc", "cdc", "reserved". + +What: /sys/class/nvme/nvmeX/reset_controller +Date: November 2015 +KernelVersion: 4.5 +Contact: Christoph Hellwig <hch@lst.de> +Description: + Write-only. Writing any value triggers a synchronous + controller reset. + +What: /sys/class/nvme/nvmeX/rescan_controller +Date: April 2016 +KernelVersion: 4.7 +Contact: Keith Busch <kbusch@kernel.org> +Description: + Write-only. Writing any value triggers a namespace rescan + on this controller. + +What: /sys/class/nvme/nvmeX/transport +What: /sys/class/nvme/nvmeX/subsysnqn +What: /sys/class/nvme/nvmeX/address +What: /sys/class/nvme/nvmeX/delete_controller +What: /sys/class/nvme/nvmeX/reconnect_delay +What: /sys/class/nvme/nvmeX/ctrl_loss_tmo +Date: June 2016 +KernelVersion: 4.8 +Contact: Ming Lin <ming.l@ssi.samsung.com> +Description: + Fabrics controller attributes added with NVMe-oF support. + + transport: Shows the transport type string. Possible values: + "pcie", "tcp", "rdma", "fc", "loop". + + subsysnqn: Shows the NVMe Qualified Name (NQN) of the + subsystem this controller belongs to. + + address: Shows the transport-specific address string. Only + available for fabrics controllers. + + delete_controller: Write-only. Triggers deletion of this + fabrics controller. + + reconnect_delay: Shows or sets the reconnect delay in + seconds. Reading returns the delay value, or "off" if + disabled. + + ctrl_loss_tmo: Shows or sets the controller loss timeout in + seconds. Reading returns the timeout value, or "off" if + infinite reconnects are allowed. Writing a negative value + disables the timeout. + +What: /sys/class/nvme/nvmeX/hostnqn +What: /sys/class/nvme/nvmeX/hostid +Date: February 2020 +KernelVersion: 5.7 +Contact: Sagi Grimberg <sagi@grimberg.me> +Description: + hostnqn: Shows the host NQN used by this fabrics controller. + + hostid: Shows the host identifier (UUID format) used by this + fabrics controller. + + Only available for fabrics controllers. + +What: /sys/class/nvme/nvmeX/fast_io_fail_tmo +Date: November 2020 +KernelVersion: 5.11 +Contact: Victor Gladkov <Victor.Gladkov@kioxia.com> +Description: + Shows or sets the fast I/O fail timeout in seconds. Reading + returns the timeout value, or "off" if disabled. Writing a + negative value disables the fast I/O fail. Only available + for fabrics controllers. + +What: /sys/class/nvme/nvmeX/kato +Date: April 2021 +KernelVersion: 5.13 +Contact: Hannes Reinecke <hare@suse.de> +Description: + Shows the Keep Alive Timeout value in milliseconds for + this controller. + +What: /sys/class/nvme/nvmeX/cmb +Date: October 2016 +KernelVersion: 4.9 +Contact: Stephen Bates <sbates@raithlin.com> +Description: + Shows the Controller Memory Buffer (CMB) register values + in format "cmbloc : 0x%08x\ncmbsz : 0x%08x\n". Only + visible when the controller has a CMB (cmbsz != 0). + PCI transport only. + +What: /sys/class/nvme/nvmeX/cmbloc +What: /sys/class/nvme/nvmeX/cmbsz +What: /sys/class/nvme/nvmeX/hmb +Date: July 2021 +KernelVersion: 5.15 +Contact: Keith Busch <kbusch@kernel.org> +Description: + cmbloc: Shows the CMBLOC register value. + + cmbsz: Shows the CMBSZ register value. + + cmbloc and cmbsz are only visible when the controller has + a CMB. PCI transport only. + + hmb: Shows or sets whether the Host Memory Buffer (HMB) is + enabled. Reading returns 1 (enabled) or 0 (disabled). + Writing 1 enables HMB; writing 0 disables it. Only + visible when the controller supports HMB (hmpre != 0). + PCI transport only. + +What: /sys/class/nvme/nvmeX/state +Date: November 2016 +KernelVersion: 4.11 +Contact: Sagi Grimberg <sagi@grimberg.me> +Description: + Shows the current state of the controller. Possible values: + "new", "live", "resetting", "connecting", "deleting", + "deleting (no IO)", "dead". + +What: /sys/class/nvme/nvmeX/numa_node +Date: November 2018 +KernelVersion: 5.0 +Contact: Hannes Reinecke <hare@suse.de> +Description: + Shows the NUMA node the controller is attached to. + +What: /sys/class/nvme/nvmeX/queue_count +What: /sys/class/nvme/nvmeX/sqsize +Date: September 2019 +KernelVersion: 5.4 +Contact: James Smart <jsmart2021@gmail.com> +Description: + queue_count: Shows the total number of queues (admin + I/O) + for this controller. + + sqsize: Shows the submission queue size for this controller. + +What: /sys/class/nvme/nvmeX/dhchap_secret +What: /sys/class/nvme/nvmeX/dhchap_ctrl_secret +Date: June 2022 +KernelVersion: 6.0 +Contact: Hannes Reinecke <hare@suse.de> +Description: + dhchap_secret: Shows or sets the host DH-HMAC-CHAP secret + for this controller. Reading returns "none" if not set. + Writing must use the "DHHC-1:" key format and triggers + re-authentication. + + dhchap_ctrl_secret: Shows or sets the controller + DH-HMAC-CHAP secret for bidirectional authentication. + Same format as dhchap_secret. + + Only available when CONFIG_NVME_HOST_AUTH is enabled and + for fabrics controllers. + +What: /sys/class/nvme/nvmeX/tls_key +Date: August 2023 +KernelVersion: 6.7 +Contact: Hannes Reinecke <hare@suse.de> +Description: + Shows the serial of the currently active TLS PSK as hex. + Returns empty if no TLS key is active. Only available for + TCP controllers with TLS or secure concatenation enabled + (CONFIG_NVME_TCP_TLS). + +What: /sys/class/nvme/nvmeX/tls_configured_key +Date: July 2024 +KernelVersion: 6.12 +Contact: Hannes Reinecke <hare@kernel.org> +Description: + Shows the serial of the configured TLS key. Writing 0 + triggers a PSK reauthentication (REPLACETLSPSK) with + the target. After reauthentication the returned serial + will be the new key. Only available for TCP controllers + with secure concatenation enabled (CONFIG_NVME_TCP_TLS). + +What: /sys/class/nvme/nvmeX/tls_keyring +Date: July 2024 +KernelVersion: 6.12 +Contact: Hannes Reinecke <hare@kernel.org> +Description: + Shows the TLS keyring description. Only available for TCP + controllers with a keyring configured (CONFIG_NVME_TCP_TLS). + +What: /sys/class/nvme/nvmeX/tls_mode +Date: April 2026 +KernelVersion: 7.1 +Contact: Daniel Wagner <wagi@kernel.org> +Description: + Shows the TLS mode: "tls" for direct TLS or "concat" for + secure concatenation. Only available for TCP controllers + with TLS or secure concatenation enabled + (CONFIG_NVME_TCP_TLS). + +What: /sys/class/nvme/nvmeX/passthru_err_log_enabled +Date: January 2024 +KernelVersion: 6.8 +Contact: Alan Adamson <alan.adamson@oracle.com> +Description: + Shows or sets whether admin passthrough error logging is + enabled for this controller. Reading returns "on" or "off". + Writing accepts a boolean value. + +What: /sys/class/nvme/nvmeX/quirks +Date: November 2025 +KernelVersion: 7.0 +Contact: Maurizio Lombardi <mlombard@redhat.com> +Description: + Shows the active quirk names for this controller, one per + line. Shows "none" if no quirks are active. + +What: /sys/class/nvme/nvmeX/admin_timeout +What: /sys/class/nvme/nvmeX/io_timeout +Date: May 2026 +KernelVersion: 7.2 +Contact: Maurizio Lombardi <mlombard@redhat.com> +Description: + admin_timeout: Shows or sets the admin command timeout in + milliseconds. + + io_timeout: Shows or sets the I/O command timeout in + milliseconds. Changes are propagated to all namespace + request queues. + + The value must be nonzero. Only writable after the + controller has been started at least once. + +What: /sys/block/nvmeXnY/uuid +What: /sys/block/nvmeXnY/eui +What: /sys/block/nvmeXnY/nsid +Date: December 2015 +KernelVersion: 4.5 +Contact: Keith Busch <kbusch@kernel.org> +Description: + Namespace identification attributes. + + uuid: Shows the UUID for this namespace. Falls back to + showing the NGUID for backward compatibility. Hidden if + both are all zeros. + + eui: Shows the IEEE Extended Unique Identifier (EUI-64). + Hidden if all zeros. + + nsid: Shows the namespace identifier (NSID). + +What: /sys/block/nvmeXnY/wwid +Date: February 2016 +KernelVersion: 4.6 +Contact: Keith Busch <kbusch@kernel.org> +Description: + Shows the World Wide Identifier for this namespace. The + format depends on available identifiers (in priority + order): "uuid.{UUID}", "eui.{NGUID}", "eui.{EUI64}", or + "nvme.{VID}-{SERIAL}-{MODEL}-{NSID}". + +What: /sys/block/nvmeXnY/nguid +Date: June 2017 +KernelVersion: 4.13 +Contact: Johannes Thumshirn <jthumshirn@suse.de> +Description: + Shows the Namespace Globally Unique Identifier (NGUID). + Hidden if the NGUID is all zeros. + +What: /sys/block/nvmeXcYnZ/ana_grpid +What: /sys/block/nvmeXcYnZ/ana_state +Date: May 2018 +KernelVersion: 4.19 +Contact: Christoph Hellwig <hch@lst.de> +Description: + ana_grpid: Shows the ANA Group ID for this namespace + path device. + + ana_state: Shows the ANA state. Possible values: + "optimized", "non-optimized", "inaccessible", + "persistent-loss", "change". + + Only visible when the controller supports ANA. + Requires CONFIG_NVME_MULTIPATH. + +What: /sys/block/nvmeXcYnZ/queue_depth +Date: June 2024 +KernelVersion: 6.11 +Contact: Thomas Song <tsong@purestorage.com> +Description: + Shows the current active I/O count on this path's + controller. Returns empty if iopolicy is not "queue-depth". + Requires CONFIG_NVME_MULTIPATH. + +What: /sys/block/nvmeXcYnZ/numa_nodes +Date: January 2025 +KernelVersion: 6.15 +Contact: Nilay Shroff <nilay@linux.ibm.com> +Description: + Shows the NUMA node mask for which this path is the + currently selected path. Returns empty if iopolicy is not + "numa". Requires CONFIG_NVME_MULTIPATH. + +What: /sys/block/nvmeXnY/delayed_removal_secs +Date: May 2025 +KernelVersion: 6.16 +Contact: Nilay Shroff <nilay@linux.ibm.com> +Description: + Shows or sets the delayed removal timeout in seconds for + the multipath head device. When nonzero, I/O is queued + instead of failed when all paths are gone, and head removal + is deferred. Only visible on multipath head devices. + Requires CONFIG_NVME_MULTIPATH. + +What: /sys/block/nvmeXnY/csi +What: /sys/block/nvmeXnY/metadata_bytes +What: /sys/block/nvmeXnY/nuse +Date: December 2023 +KernelVersion: 6.8 +Contact: Daniel Wagner <dwagner@suse.de> +Description: + csi: Shows the Command Set Identifier for this namespace. + + metadata_bytes: Shows the metadata size in bytes. + + nuse: Shows the Namespace Utilization (NUSE) value. Reading + triggers an Identify Namespace command to refresh the + value (rate-limited to avoid excessive commands). + +What: /sys/block/nvmeXnY/passthru_err_log_enabled +Date: January 2024 +KernelVersion: 6.8 +Contact: Alan Adamson <alan.adamson@oracle.com> +Description: + Shows or sets whether I/O passthrough error logging is + enabled for this namespace. Reading returns "on" or "off". + Writing accepts a boolean value. + +What: /sys/class/nvme/nvmeX/diag/command_error_count +What: /sys/class/nvme/nvmeX/diag/reset_count +What: /sys/class/nvme/nvmeX/diag/reconnect_count +Date: May 2026 +KernelVersion: 7.2 +Contact: Nilay Shroff <nilay@linux.ibm.com> +Description: + Controller diagnostic counters. + + command_error_count: Admin command error counter. + + reset_count: Controller reset counter. + + reconnect_count: Accumulated reconnect counter. Only + available for fabrics controllers. + + All counters can be reset by writing a value. + +What: /sys/block/nvmeXnY/diag/command_retries_count +What: /sys/block/nvmeXnY/diag/command_error_count +Date: May 2026 +KernelVersion: 7.2 +Contact: Nilay Shroff <nilay@linux.ibm.com> +Description: + Namespace diagnostic counters for non-multipath + configurations (when CONFIG_NVME_MULTIPATH is not + configured). + + command_retries_count: I/O command retry counter. + + command_error_count: I/O command error counter. + + All counters can be reset by writing any value. + +What: /sys/block/nvmeXcYnZ/diag/command_retries_count +What: /sys/block/nvmeXcYnZ/diag/command_error_count +What: /sys/block/nvmeXcYnZ/diag/multipath_failover_count +What: /sys/block/nvmeXnY/diag/io_requeue_no_usable_path_count +What: /sys/block/nvmeXnY/diag/io_fail_no_available_path_count +Date: May 2026 +KernelVersion: 7.2 +Contact: Nilay Shroff <nilay@linux.ibm.com> +Description: + Namespace diagnostic counters for multipath + configurations (when CONFIG_NVME_MULTIPATH is + configured). + + command_retries_count: I/O command retry counter. + + command_error_count: I/O command error counter. + + multipath_failover_count: Multipath failover counter. + + io_requeue_no_usable_path_count: Counter of I/Os + requeued because no usable path was available. + + io_fail_no_available_path_count: Counter of I/Os + failed because no available path existed. + + All counters can be reset by writing any value. + +What: /sys/class/nvme-subsystem/nvme-subsysX/model +What: /sys/class/nvme-subsystem/nvme-subsysX/serial +What: /sys/class/nvme-subsystem/nvme-subsysX/firmware_rev +What: /sys/class/nvme-subsystem/nvme-subsysX/subsysnqn +Date: November 2017 +KernelVersion: 4.15 +Contact: Hannes Reinecke <hare@suse.de> +Description: + Shows the model, serial number, firmware revision, or NQN + of the NVMe subsystem. + +What: /sys/class/nvme-subsystem/nvme-subsysX/iopolicy +Date: February 2019 +KernelVersion: 5.1 +Contact: Hannes Reinecke <hare@suse.de> +Description: + Shows or sets the multipath I/O path selection policy for + this subsystem. Accepted values: "numa", "round-robin", + "queue-depth". Changing the policy clears all current path + selections. Only available when CONFIG_NVME_MULTIPATH is + enabled. + +What: /sys/class/nvme-subsystem/nvme-subsysX/subsystype +Date: September 2021 +KernelVersion: 5.16 +Contact: Hannes Reinecke <hare@suse.de> +Description: + Shows the subsystem type. Possible values: "discovery", + "nvm", "reserved". diff --git a/Documentation/ABI/testing/sysfs-nvme b/Documentation/ABI/testing/sysfs-nvme deleted file mode 100644 index 499d5f843cd4..000000000000 --- a/Documentation/ABI/testing/sysfs-nvme +++ /dev/null @@ -1,13 +0,0 @@ -What: /sys/devices/virtual/nvme-fabrics/ctl/.../tls_configured_key -Date: November 2025 -KernelVersion: 6.19 -Contact: Linux NVMe mailing list <linux-nvme@lists.infradead.org> -Description: - The file is avaliable when using a secure concatanation - connection to a NVMe target. Reading the file will return - the serial of the currently negotiated key. - - Writing 0 to the file will trigger a PSK reauthentication - (REPLACETLSPSK) with the target. After a reauthentication - the value returned by tls_configured_key will be the new - serial. diff --git a/MAINTAINERS b/MAINTAINERS index 716acfc3d7c1..46d98b53729d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -19373,6 +19373,9 @@ L: linux-nvme@lists.infradead.org S: Supported W: http://git.infradead.org/nvme.git T: git git://git.infradead.org/nvme.git +F: Documentation/ABI/stable/sysfs-nvme +F: Documentation/admin-guide/nvme-multipath.rst +F: Documentation/fault-injection/nvme-fault-injection.rst F: Documentation/nvme/ F: drivers/nvme/common/ F: drivers/nvme/host/ @@ -19415,6 +19418,7 @@ L: linux-nvme@lists.infradead.org S: Supported W: http://git.infradead.org/nvme.git T: git git://git.infradead.org/nvme.git +F: Documentation/ABI/stable/configfs-nvmet F: drivers/nvme/target/ NVMEM FRAMEWORK diff --git a/drivers/nvme/common/auth.c b/drivers/nvme/common/auth.c index 77f1d22512f8..e2e0c736540a 100644 --- a/drivers/nvme/common/auth.c +++ b/drivers/nvme/common/auth.c @@ -692,8 +692,7 @@ int nvme_auth_derive_tls_psk(int hmac_id, const u8 *psk, size_t psk_len, const char *psk_digest, u8 **ret_psk) { static const u8 default_salt[NVME_AUTH_MAX_DIGEST_SIZE]; - static const char label[] = "tls13 nvme-tls-psk"; - const size_t label_len = sizeof(label) - 1; + static const char label[18] = "tls13 nvme-tls-psk"; u8 prk[NVME_AUTH_MAX_DIGEST_SIZE]; size_t hash_len, ctx_len; u8 *hmac_data = NULL, *tls_key; @@ -729,7 +728,7 @@ int nvme_auth_derive_tls_psk(int hmac_id, const u8 *psk, size_t psk_len, */ hmac_data = kmalloc(/* output length */ 2 + - /* label */ 1 + label_len + + /* label */ 1 + sizeof(label) + /* context (max) */ 1 + 3 + 1 + strlen(psk_digest) + /* counter */ 1, GFP_KERNEL); @@ -743,10 +742,10 @@ int nvme_auth_derive_tls_psk(int hmac_id, const u8 *psk, size_t psk_len, hmac_data[i++] = hash_len; /* label */ - static_assert(label_len <= 255); - hmac_data[i] = label_len; - memcpy(&hmac_data[i + 1], label, label_len); - i += 1 + label_len; + static_assert(sizeof(label) <= 255); + hmac_data[i] = sizeof(label); + memcpy(&hmac_data[i + 1], label, sizeof(label)); + i += 1 + sizeof(label); /* context */ ctx_len = sprintf(&hmac_data[i + 1], "%02d %s", hmac_id, psk_digest); diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile index 6414ec968f99..67563a69f7dc 100644 --- a/drivers/nvme/host/Makefile +++ b/drivers/nvme/host/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +CONTEXT_ANALYSIS := y ccflags-y += -I$(src) obj-$(CONFIG_NVME_CORE) += nvme-core.o diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index be3b91b43ea5..c63e28c75766 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -47,9 +47,6 @@ #define APPLE_ANS_BOOT_STATUS 0x1300 #define APPLE_ANS_BOOT_STATUS_OK 0xde71ce55 -#define APPLE_ANS_UNKNOWN_CTRL 0x24008 -#define APPLE_ANS_PRP_NULL_CHECK BIT(11) - #define APPLE_ANS_LINEAR_SQ_CTRL 0x24908 #define APPLE_ANS_LINEAR_SQ_EN BIT(0) @@ -151,6 +148,23 @@ struct apple_nvme_queue { bool enabled; }; +static inline bool apple_nvme_queue_enabled(struct apple_nvme_queue *q) +{ + /* Pair with apple_nvme_enable_queue(). */ + return smp_load_acquire(&q->enabled); +} + +static inline void apple_nvme_enable_queue(struct apple_nvme_queue *q) +{ + /* Publish queue initialization before setting q->enabled. */ + smp_store_release(&q->enabled, true); +} + +static inline void apple_nvme_disable_queue(struct apple_nvme_queue *q) +{ + WRITE_ONCE(q->enabled, false); +} + /* * The apple_nvme_iod describes the data in an I/O. * @@ -318,13 +332,15 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, u32 tag = nvme_tag_from_cid(cmd->common.command_id); struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; - tcb->opcode = cmd->common.opcode; + tcb->opcode = 0; tcb->prp1 = cmd->common.dptr.prp1; tcb->prp2 = cmd->common.dptr.prp2; tcb->length = cmd->rw.length; tcb->command_id = tag; - if (nvme_is_write(cmd)) + if (!cmd->common.dptr.prp1) + tcb->dma_flags = 0; + else if (nvme_is_write(cmd)) tcb->dma_flags = APPLE_ANS_TCB_DMA_TO_DEVICE; else tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; @@ -677,7 +693,7 @@ static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force) bool found; DEFINE_IO_COMP_BATCH(iob); - if (!READ_ONCE(q->enabled) && !force) + if (!apple_nvme_queue_enabled(q) && !force) return false; found = apple_nvme_poll_cq(q, &iob); @@ -780,7 +796,7 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, * We should not need to do this, but we're still using this to * ensure we can drain requests on a dying queue. */ - if (unlikely(!READ_ONCE(q->enabled))) + if (unlikely(!apple_nvme_queue_enabled(q))) return BLK_STS_IOERR; if (!nvme_check_ready(&anv->ctrl, req, true)) @@ -863,7 +879,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_quiesce_io_queues(&anv->ctrl); if (!dead) { - if (READ_ONCE(anv->ioq.enabled)) { + if (apple_nvme_queue_enabled(&anv->ioq)) { apple_nvme_remove_sq(anv); apple_nvme_remove_cq(anv); } @@ -887,8 +903,8 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_disable_ctrl(&anv->ctrl, false); } - WRITE_ONCE(anv->ioq.enabled, false); - WRITE_ONCE(anv->adminq.enabled, false); + apple_nvme_disable_queue(&anv->ioq); + apple_nvme_disable_queue(&anv->adminq); mb(); /* ensure that nvme_queue_rq() sees that enabled is cleared */ nvme_quiesce_admin_queue(&anv->ctrl); @@ -1016,8 +1032,7 @@ static void apple_nvme_init_queue(struct apple_nvme_queue *q) memset(q->tcbs, 0, anv->hw->max_queue_depth * sizeof(struct apple_nvmmu_tcb)); memset(q->cqes, 0, depth * sizeof(struct nvme_completion)); - WRITE_ONCE(q->enabled, true); - wmb(); /* ensure the first interrupt sees the initialization */ + apple_nvme_enable_queue(q); } static void apple_nvme_reset_work(struct work_struct *work) @@ -1125,17 +1140,6 @@ static void apple_nvme_reset_work(struct work_struct *work) /* Setup the NVMMU for the maximum admin and IO queue depth */ writel(anv->hw->max_queue_depth - 1, anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS); - - /* - * This is probably a chicken bit: without it all commands - * where any PRP is set to zero (including those that don't use - * that field) fail and the co-processor complains about - * "completed with err BAD_CMD-" or a "NULL_PRP_PTR_ERR" in the - * syslog - */ - writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) & - ~APPLE_ANS_PRP_NULL_CHECK, - anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL); } /* Setup the admin queue */ @@ -1567,10 +1571,8 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) ret = devm_request_irq(anv->dev, anv->irq, apple_nvme_irq, 0, "nvme-apple", anv); - if (ret) { - dev_err_probe(dev, ret, "Failed to request IRQ"); + if (ret) goto put_dev; - } anv->rtk = devm_apple_rtkit_init(dev, anv, NULL, 0, &apple_nvme_rtkit_ops); @@ -1581,7 +1583,8 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) } ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, - NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS); + NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS | + NVME_QUIRK_ADMIN_PAGE_ALIGN); if (ret) { dev_err_probe(dev, ret, "Failed to initialize nvme_ctrl"); goto put_dev; @@ -1636,6 +1639,15 @@ static void apple_nvme_remove(struct platform_device *pdev) nvme_stop_ctrl(&anv->ctrl); nvme_remove_namespaces(&anv->ctrl); apple_nvme_disable(anv, true); + if (anv->ctrl.admin_q && !blk_queue_dying(anv->ctrl.admin_q)) { + /* + * If the controller was reset during removal, it's possible + * user requests may be waiting on a stopped queue. Start the + * queue to flush these to completion. + */ + nvme_unquiesce_admin_queue(&anv->ctrl); + blk_mq_destroy_queue(anv->ctrl.admin_q); + } nvme_uninit_ctrl(&anv->ctrl); if (apple_rtkit_is_running(anv->rtk)) { diff --git a/drivers/nvme/host/auth.c b/drivers/nvme/host/auth.c index 16de4499a8e7..e55920642f2c 100644 --- a/drivers/nvme/host/auth.c +++ b/drivers/nvme/host/auth.c @@ -8,6 +8,7 @@ #include <linux/prandom.h> #include <linux/unaligned.h> #include <crypto/dh.h> +#include <crypto/utils.h> #include "nvme.h" #include "fabrics.h" #include <linux/nvme-auth.h> @@ -361,7 +362,7 @@ static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl, return 0; /* Validate controller response */ - if (memcmp(chap->response, data->rval, data->hl)) { + if (crypto_memneq(chap->response, data->rval, data->hl)) { dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n", __func__, chap->qid, (int)chap->hash_len, data->rval); dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n", diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 453c1f0b2dd0..1322c678f4eb 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -33,6 +33,13 @@ #define NVME_MINORS (1U << MINORBITS) +/* + * Write hints (bio->bi_write_stream) are u8, so FDP placement handles beyond + * U8_MAX can never be selected. Cap the handle count to bound both the RUH + * status buffer and the per-head plids array. + */ +#define NVME_MAX_PLIDS U8_MAX + struct nvme_ns_info { struct nvme_ns_ids ids; u32 nsid; @@ -126,8 +133,8 @@ EXPORT_SYMBOL_GPL(nvme_reset_wq); struct workqueue_struct *nvme_delete_wq; EXPORT_SYMBOL_GPL(nvme_delete_wq); -static LIST_HEAD(nvme_subsystems); DEFINE_MUTEX(nvme_subsystems_lock); +static LIST_HEAD_GUARDED(nvme_subsystems, nvme_subsystems_lock); static DEFINE_IDA(nvme_instance_ida); static dev_t nvme_ctrl_base_chr_devt; @@ -693,6 +700,11 @@ static void nvme_free_ns_head(struct kref *ref) kfree(head); } +void nvme_get_ns_head(struct nvme_ns_head *head) +{ + kref_get(&head->ref); +} + bool nvme_tryget_ns_head(struct nvme_ns_head *head) { return kref_get_unless_zero(&head->ref); @@ -1268,7 +1280,7 @@ u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode) } EXPORT_SYMBOL_NS_GPL(nvme_passthru_start, "NVME_TARGET_PASSTHRU"); -void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, +u32 nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, struct nvme_command *cmd, int status) { if (effects & NVME_CMD_EFFECTS_CSE_MASK) { @@ -1289,7 +1301,7 @@ void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, flush_work(&ctrl->scan_work); } if (ns) - return; + return effects; switch (cmd->common.opcode) { case nvme_admin_set_features: @@ -1310,6 +1322,8 @@ void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, default: break; } + + return effects; } EXPORT_SYMBOL_NS_GPL(nvme_passthru_end, "NVME_TARGET_PASSTHRU"); @@ -1583,8 +1597,12 @@ static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) { struct nvme_ns_id_desc *cur = data + pos; + if (pos + sizeof(*cur) > NVME_IDENTIFY_DATA_SIZE) + break; if (cur->nidl == 0) break; + if (pos + sizeof(*cur) + cur->nidl > NVME_IDENTIFY_DATA_SIZE) + break; len = nvme_process_ns_desc(ctrl, &info->ids, cur, &csi_seen); if (len < 0) @@ -2071,7 +2089,10 @@ static void nvme_set_ctrl_limits(struct nvme_ctrl *ctrl, lim->max_integrity_segments = ctrl->max_integrity_segments; lim->virt_boundary_mask = ctrl->ops->get_virt_boundary(ctrl, is_admin); lim->max_segment_size = UINT_MAX; - lim->dma_alignment = 3; + if (is_admin && (ctrl->quirks & NVME_QUIRK_ADMIN_PAGE_ALIGN)) + lim->dma_alignment = NVME_CTRL_PAGE_SIZE - 1; + else + lim->dma_alignment = 3; } static bool nvme_update_disk_info(struct nvme_ns *ns, struct nvme_id_ns *id, @@ -2342,7 +2363,7 @@ static int nvme_query_fdp_info(struct nvme_ns *ns, struct nvme_ns_info *info) if (!info->runs) return ret; - size = struct_size(ruhs, ruhsd, S8_MAX - 1); + size = struct_size(ruhs, ruhsd, NVME_MAX_PLIDS); ruhs = kzalloc(size, GFP_KERNEL); if (!ruhs) return -ENOMEM; @@ -2357,7 +2378,7 @@ static int nvme_query_fdp_info(struct nvme_ns *ns, struct nvme_ns_info *info) goto free; } - head->nr_plids = le16_to_cpu(ruhs->nruhsd); + head->nr_plids = min(le16_to_cpu(ruhs->nruhsd), NVME_MAX_PLIDS); if (!head->nr_plids) goto free; @@ -2592,11 +2613,15 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info) lim.max_write_streams = ns_lim->max_write_streams; lim.write_stream_granularity = ns_lim->write_stream_granularity; ret = queue_limits_commit_update(ns->head->disk->queue, &lim); + if (ret) + goto unfreeze_head_queue; set_capacity_and_notify(ns->head->disk, get_capacity(ns->disk)); set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info)); nvme_mpath_revalidate_paths(ns->head); + ret = nvme_mpath_revalidate_zones(ns->head); +unfreeze_head_queue: blk_mq_unfreeze_queue(ns->head->disk->queue, memflags); } @@ -3198,6 +3223,7 @@ static void nvme_put_subsystem(struct nvme_subsystem *subsys) } static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn) + __must_hold(&nvme_subsystems_lock) { struct nvme_subsystem *subsys; @@ -3242,6 +3268,7 @@ static inline bool nvme_is_io_ctrl(struct nvme_ctrl *ctrl) static bool nvme_validate_cntlid(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) + __must_hold(&nvme_subsystems_lock) { struct nvme_ctrl *tmp; @@ -3272,6 +3299,7 @@ static bool nvme_validate_cntlid(struct nvme_subsystem *subsys, } static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) + __context_unsafe(/* initialize unpublished/lock-guarded variables */) { struct nvme_subsystem *subsys, *found; int ret; @@ -3843,6 +3871,7 @@ static const struct file_operations nvme_dev_fops = { static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl, unsigned nsid) + __must_hold(&ctrl->subsys->lock) { struct nvme_ns_head *h; @@ -3865,6 +3894,7 @@ static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl, static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys, struct nvme_ns_ids *ids) + __must_hold(&subsys->lock) { bool has_uuid = !uuid_is_null(&ids->uuid); bool has_nguid = memchr_inv(ids->nguid, 0, sizeof(ids->nguid)); @@ -3890,6 +3920,11 @@ static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys, static void nvme_cdev_rel(struct device *dev) { ida_free(&nvme_ns_chr_minor_ida, MINOR(dev->devt)); + if (dev->parent->class == &nvme_class) + nvme_put_ns(container_of(dev, struct nvme_ns, cdev_device)); + else + nvme_put_ns_head(container_of(dev, struct nvme_ns_head, + cdev_device)); } void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device) @@ -3955,10 +3990,12 @@ static void nvme_add_ns_cdev(struct nvme_ns *ns) snprintf(name, sizeof(name), "ng%dn%d", ns->ctrl->instance, ns->head->instance); + nvme_get_ns(ns); /* Undone in nvme_cdev_rel() */ if (nvme_cdev_add(name, &ns->cdev, &ns->cdev_device, &nvme_ns_chr_fops, ns->ctrl->ops->module)) { dev_err(ns->ctrl->device, "Unable to create the %s device\n", name); + nvme_put_ns(ns); return; } set_bit(NVME_NS_CDEV_LIVE, &ns->flags); @@ -3966,6 +4003,7 @@ static void nvme_add_ns_cdev(struct nvme_ns *ns) static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, struct nvme_ns_info *info) + __must_hold(&ctrl->subsys->lock) { struct nvme_ns_head *head; size_t size = sizeof(*head); @@ -5195,7 +5233,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) > PAGE_SIZE); - ctrl->discard_page = alloc_page(GFP_KERNEL); + ctrl->discard_page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!ctrl->discard_page) { ret = -ENOMEM; goto out; diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index ac3d4f400601..fd5abd04e080 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -14,11 +14,11 @@ #include "fabrics.h" #include <linux/nvme-keyring.h> -static LIST_HEAD(nvmf_transports); static DECLARE_RWSEM(nvmf_transports_rwsem); +static LIST_HEAD_GUARDED(nvmf_transports, nvmf_transports_rwsem); -static LIST_HEAD(nvmf_hosts); static DEFINE_MUTEX(nvmf_hosts_mutex); +static LIST_HEAD_GUARDED(nvmf_hosts, nvmf_hosts_mutex); static struct nvmf_host *nvmf_default_host; diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 04363b9c4489..023710e08e0d 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2100,9 +2100,15 @@ __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl, dev_err(ctrl->dev, "FCP Op failed - rspiu dma mapping failed.\n"); ret = -EFAULT; + goto out_unmap; } atomic_set(&op->state, FCPOP_STATE_IDLE); + return 0; + +out_unmap: + fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma, + sizeof(op->cmd_iu), DMA_TO_DEVICE); out_on_error: return ret; } @@ -2318,7 +2324,7 @@ nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize) return 0; delete_queues: - for (; i > 0; i--) + for (--i; i > 0; i--) __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i); return ret; } diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index 664216eece4a..6539d4750098 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -14,45 +14,54 @@ enum { NVME_IOCTL_PARTITION = (1 << 1), }; -static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, - unsigned int flags, bool open_for_write) +static bool nvme_admin_cmd_allowed(struct nvme_ctrl *ctrl, + struct nvme_command *c) { - u32 effects; - - /* - * Do not allow unprivileged passthrough on partitions, as that allows an - * escape from the containment of the partition. - */ - if (flags & NVME_IOCTL_PARTITION) - goto admin; - - /* - * Do not allow unprivileged processes to send vendor specific or fabrics - * commands as we can't be sure about their effects. - */ - if (c->common.opcode >= nvme_cmd_vendor_start || - c->common.opcode == nvme_fabrics_command) - goto admin; - /* * Do not allow unprivileged passthrough of admin commands except * for a subset of identify commands that contain information required * to form proper I/O commands in userspace and do not expose any * potentially sensitive information. */ - if (!ns) { - if (c->common.opcode == nvme_admin_identify) { - switch (c->identify.cns) { - case NVME_ID_CNS_NS: - case NVME_ID_CNS_CS_NS: - case NVME_ID_CNS_NS_CS_INDEP: - case NVME_ID_CNS_CS_CTRL: - case NVME_ID_CNS_CTRL: - return true; - } + switch (c->common.opcode) { + case nvme_admin_identify: + switch (c->identify.cns) { + case NVME_ID_CNS_NS: + case NVME_ID_CNS_CS_NS: + case NVME_ID_CNS_NS_CS_INDEP: + case NVME_ID_CNS_CS_CTRL: + case NVME_ID_CNS_CTRL: + return true; + } + break; + case nvme_admin_set_features: + /* + * Reject Set Features that change controller state the driver + * manages itself; setting them behind the driver's back from + * userspace leaves it unable to react correctly. Keep Alive is + * only armed for fabrics - on other transports it has no + * reserved tag and harms idle power states. + */ + switch (le32_to_cpu(c->features.fid) & 0xff) { + case NVME_FEAT_KATO: + if (ctrl->ops->flags & NVME_F_FABRICS) + break; + fallthrough; + case NVME_FEAT_HOST_BEHAVIOR: + case NVME_FEAT_HOST_MEM_BUF: + case NVME_FEAT_NUM_QUEUES: + case NVME_FEAT_AUTO_PST: + return false; } - goto admin; + break; } + return capable(CAP_SYS_ADMIN); +} + +static bool nvme_ns_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, + bool open_for_write) +{ + u32 effects; /* * Check if the controller provides a Commands Supported and Effects log @@ -61,7 +70,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, */ effects = nvme_command_effects(ns->ctrl, ns, c->common.opcode); if (!(effects & NVME_CMD_EFFECTS_CSUPP)) - goto admin; + return capable(CAP_SYS_ADMIN); /* * Don't allow passthrough for command that have intrusive (or unknown) @@ -70,7 +79,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_UUID_SEL | NVME_CMD_EFFECTS_SCOPE_MASK)) - goto admin; + return capable(CAP_SYS_ADMIN); /* * Only allow I/O commands that transfer data to the controller or that @@ -79,11 +88,34 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, */ if ((nvme_is_write(c) || (effects & NVME_CMD_EFFECTS_LBCC)) && !open_for_write) - goto admin; + return capable(CAP_SYS_ADMIN); return true; -admin: - return capable(CAP_SYS_ADMIN); +} + +static bool nvme_cmd_allowed(struct nvme_ctrl *ctrl, struct nvme_ns *ns, + struct nvme_command *c, unsigned int flags, + bool open_for_write) +{ + /* + * Do not allow unprivileged passthrough on partitions, as that + * allows an escape from the containment of the partition. + */ + if (flags & NVME_IOCTL_PARTITION) + return capable(CAP_SYS_ADMIN); + + /* + * Do not allow unprivileged processes to send vendor specific or + * fabrics commands as we can't be sure about their effects. + */ + if (c->common.opcode >= nvme_cmd_vendor_start || + c->common.opcode == nvme_fabrics_command) + return capable(CAP_SYS_ADMIN); + + if (!ns) + return nvme_admin_cmd_allowed(ctrl, c); + + return nvme_ns_cmd_allowed(ns, c, open_for_write); } /* @@ -202,7 +234,8 @@ out_free_req: return ret; } -static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) +static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio, + unsigned int flags, bool open_for_write) { struct nvme_user_io io; struct nvme_command c; @@ -260,6 +293,9 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) c.rw.lbat = cpu_to_le16(io.apptag); c.rw.lbatm = cpu_to_le16(io.appmask); + if (!nvme_cmd_allowed(ns->ctrl, ns, &c, flags, open_for_write)) + return -EACCES; + return nvme_submit_user_cmd(ns->queue, &c, io.addr, length, metadata, meta_len, NULL, 0, 0); } @@ -307,7 +343,7 @@ static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(cmd.cdw14); c.common.cdw15 = cpu_to_le32(cmd.cdw15); - if (!nvme_cmd_allowed(ns, &c, 0, open_for_write)) + if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write)) return -EACCES; if (cmd.timeout_ms) @@ -354,7 +390,7 @@ static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(cmd.cdw14); c.common.cdw15 = cpu_to_le32(cmd.cdw15); - if (!nvme_cmd_allowed(ns, &c, flags, open_for_write)) + if (!nvme_cmd_allowed(ctrl, ns, &c, flags, open_for_write)) return -EACCES; if (cmd.timeout_ms) @@ -449,6 +485,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, const struct nvme_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe, struct nvme_uring_cmd); struct request_queue *q = ns ? ns->queue : ctrl->admin_q; + bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE; struct nvme_uring_data d; struct nvme_command c; struct iov_iter iter; @@ -479,7 +516,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(READ_ONCE(cmd->cdw14)); c.common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15)); - if (!nvme_cmd_allowed(ns, &c, 0, ioucmd->file->f_mode & FMODE_WRITE)) + if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write)) return -EACCES; d.metadata = READ_ONCE(cmd->metadata); @@ -595,7 +632,7 @@ static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned int cmd, case NVME_IOCTL_SUBMIT_IO32: #endif case NVME_IOCTL_SUBMIT_IO: - return nvme_submit_io(ns, argp); + return nvme_submit_io(ns, argp, flags, open_for_write); case NVME_IOCTL_IO64_CMD_VEC: flags |= NVME_IOCTL_VEC; fallthrough; @@ -692,14 +729,14 @@ int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd, static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd, void __user *argp, struct nvme_ns_head *head, int srcu_idx, bool open_for_write) - __releases(&head->srcu) + __releases_shared(&head->srcu) { struct nvme_ctrl *ctrl = ns->ctrl; int ret; nvme_get_ctrl(ns->ctrl); srcu_read_unlock(&head->srcu, srcu_idx); - ret = nvme_ctrl_ioctl(ns->ctrl, cmd, argp, open_for_write); + ret = nvme_ctrl_ioctl(ctrl, cmd, argp, open_for_write); nvme_put_ctrl(ctrl); return ret; diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 9b9a657fa330..75dbb58286a3 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -288,6 +288,25 @@ void nvme_mpath_revalidate_paths(struct nvme_ns_head *head) kblockd_schedule_work(&head->requeue_work); } +#ifdef CONFIG_BLK_DEV_ZONED +int nvme_mpath_revalidate_zones(struct nvme_ns_head *head) +{ + struct gendisk *disk = head->disk; + int ret; + + if (!disk || !blk_queue_is_zoned(disk->queue) || + !test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) + return 0; + + ret = blk_revalidate_disk_zones(disk); + if (ret) + dev_warn_ratelimited(disk_to_dev(disk), + "failed to revalidate zoned namespace head: %d\n", + ret); + return ret; +} +#endif /* CONFIG_BLK_DEV_ZONED */ + static bool nvme_path_is_disabled(struct nvme_ns *ns) { enum nvme_ctrl_state state = nvme_ctrl_state(ns->ctrl); @@ -306,6 +325,7 @@ static bool nvme_path_is_disabled(struct nvme_ns *ns) } static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) + __must_hold_shared(&head->srcu) { int found_distance = INT_MAX, fallback_distance = INT_MAX, distance; struct nvme_ns *found = NULL, *fallback = NULL, *ns; @@ -348,6 +368,7 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head, struct nvme_ns *ns) + __must_hold_shared(&head->srcu) { ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns, siblings); @@ -357,6 +378,7 @@ static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head, } static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head) + __must_hold_shared(&head->srcu) { struct nvme_ns *ns, *found = NULL; int node = numa_node_id(); @@ -405,6 +427,7 @@ out: } static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head) + __must_hold_shared(&head->srcu) { struct nvme_ns *best_opt = NULL, *best_nonopt = NULL, *ns; unsigned int min_depth_opt = UINT_MAX, min_depth_nonopt = UINT_MAX; @@ -448,6 +471,7 @@ static inline bool nvme_path_is_optimized(struct nvme_ns *ns) } static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head) + __must_hold_shared(&head->srcu) { int node = numa_node_id(); struct nvme_ns *ns; @@ -473,6 +497,7 @@ inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) } static bool nvme_available_path(struct nvme_ns_head *head) + __must_hold_shared(&head->srcu) { struct nvme_ns *ns; @@ -611,28 +636,8 @@ const struct block_device_operations nvme_ns_head_ops = { .pr_ops = &nvme_pr_ops, }; -static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev) -{ - return container_of(cdev, struct nvme_ns_head, cdev); -} - -static int nvme_ns_head_chr_open(struct inode *inode, struct file *file) -{ - if (!nvme_tryget_ns_head(cdev_to_ns_head(inode->i_cdev))) - return -ENXIO; - return 0; -} - -static int nvme_ns_head_chr_release(struct inode *inode, struct file *file) -{ - nvme_put_ns_head(cdev_to_ns_head(inode->i_cdev)); - return 0; -} - static const struct file_operations nvme_ns_head_chr_fops = { .owner = THIS_MODULE, - .open = nvme_ns_head_chr_open, - .release = nvme_ns_head_chr_release, .unlocked_ioctl = nvme_ns_head_chr_ioctl, .compat_ioctl = compat_ptr_ioctl, .uring_cmd = nvme_ns_head_chr_uring_cmd, @@ -647,10 +652,12 @@ static void nvme_add_ns_head_cdev(struct nvme_ns_head *head) snprintf(name, sizeof(name), "ng%dn%d", head->subsys->instance, head->instance); + nvme_get_ns_head(head); /* Undone in nvme_cdev_rel() */ if (nvme_cdev_add(name, &head->cdev, &head->cdev_device, &nvme_ns_head_chr_fops, THIS_MODULE)) { dev_err(disk_to_dev(head->disk), "Unable to create the %s device\n", name); + nvme_put_ns_head(head); return; } set_bit(NVME_NSHEAD_CDEV_LIVE, &head->flags); @@ -692,14 +699,15 @@ static void nvme_remove_head(struct nvme_ns_head *head) { if (test_and_clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { /* - * requeue I/O after NVME_NSHEAD_DISK_LIVE has been cleared - * to allow multipath to fail all I/O. + * Requeue I/O after NVME_NSHEAD_DISK_LIVE has been cleared + * to allow multipath to fail all I/O. First synchronize to + * add any bios to the requeue list. */ + synchronize_srcu(&head->srcu); kblockd_schedule_work(&head->requeue_work); if (test_and_clear_bit(NVME_NSHEAD_CDEV_LIVE, &head->flags)) nvme_cdev_del(&head->cdev, &head->cdev_device); - synchronize_srcu(&head->srcu); del_gendisk(head->disk); } nvme_put_ns_head(head); @@ -728,12 +736,10 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) struct queue_limits lim; mutex_init(&head->lock); - bio_list_init(&head->requeue_list); spin_lock_init(&head->requeue_lock); INIT_WORK(&head->requeue_work, nvme_requeue_work); INIT_WORK(&head->partition_scan_work, nvme_partition_scan_work); INIT_DELAYED_WORK(&head->remove_work, nvme_remove_head_work); - head->delayed_removal_secs = 0; /* * If "multipath_always_on" is enabled, a multipath node is added @@ -777,7 +783,7 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) set_bit(GD_SUPPRESS_PART_SCAN, &head->disk->state); sprintf(head->disk->disk_name, "nvme%dn%d", ctrl->subsys->instance, head->instance); - nvme_tryget_ns_head(head); + nvme_get_ns_head(head); return 0; } @@ -819,12 +825,14 @@ static void nvme_mpath_set_live(struct nvme_ns *ns) mutex_unlock(&head->lock); synchronize_srcu(&head->srcu); + nvme_mpath_revalidate_zones(head); kblockd_schedule_work(&head->requeue_work); } static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *, void *)) + __must_hold(&ctrl->ana_lock) { void *base = ctrl->ana_log_buf; size_t offset = sizeof(struct nvme_ana_rsp_hdr); @@ -1375,10 +1383,6 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid) nvme_mpath_set_live(ns); } -#ifdef CONFIG_BLK_DEV_ZONED - if (blk_queue_is_zoned(ns->queue) && ns->head->disk) - ns->head->disk->nr_zones = ns->disk->nr_zones; -#endif } void nvme_mpath_remove_disk(struct nvme_ns_head *head) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 824651cc898d..75e5d5a8a77c 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -178,6 +178,11 @@ enum nvme_quirks { * Align dma pool segment size to 512 bytes */ NVME_QUIRK_DMAPOOL_ALIGN_512 = (1 << 22), + + /* + * Admin queue DMA buffers must be page aligned + */ + NVME_QUIRK_ADMIN_PAGE_ALIGN = (1 << 23), }; static inline char *nvme_quirk_name(enum nvme_quirks q) @@ -229,6 +234,8 @@ static inline char *nvme_quirk_name(enum nvme_quirks q) return "broken_msi"; case NVME_QUIRK_DMAPOOL_ALIGN_512: return "dmapool_align_512"; + case NVME_QUIRK_ADMIN_PAGE_ALIGN: + return "admin_page_align"; } return "unknown"; @@ -361,7 +368,8 @@ struct nvme_ctrl { wait_queue_head_t state_wq; struct nvme_subsystem *subsys; - struct list_head subsys_entry; + struct list_head subsys_entry + __guarded_by(&nvme_subsystems_lock); struct opal_dev *opal_dev; @@ -493,10 +501,13 @@ struct nvme_subsystem { * a separate refcount. */ struct kref ref; - struct list_head entry; + struct list_head entry + __guarded_by(&nvme_subsystems_lock); struct mutex lock; - struct list_head ctrls; - struct list_head nsheads; + struct list_head ctrls + __guarded_by(&nvme_subsystems_lock); + struct list_head nsheads + __guarded_by(&lock); char subnqn[NVMF_NQN_SIZE]; char serial[20]; char model[40]; @@ -561,20 +572,22 @@ struct nvme_ns_head { u16 nr_plids; u16 *plids; #ifdef CONFIG_NVME_MULTIPATH - struct bio_list requeue_list; + struct bio_list requeue_list + __guarded_by(&requeue_lock); spinlock_t requeue_lock; struct work_struct requeue_work; struct work_struct partition_scan_work; struct mutex lock; unsigned long flags; struct delayed_work remove_work; - unsigned int delayed_removal_secs; + unsigned int delayed_removal_secs + __guarded_by(&subsys->lock); atomic_long_t io_requeue_no_usable_path_count; atomic_long_t io_fail_no_available_path_count; #define NVME_NSHEAD_DISK_LIVE 0 #define NVME_NSHEAD_QUEUE_IF_NO_PATH 1 #define NVME_NSHEAD_CDEV_LIVE 2 - struct nvme_ns __rcu *current_path[]; + struct nvme_ns __rcu_guarded *current_path[]; #endif }; @@ -679,12 +692,12 @@ static inline struct request *nvme_find_rq(struct blk_mq_tags *tags, rq = blk_mq_tag_to_rq(tags, tag); if (unlikely(!rq)) { - pr_err("could not locate request for tag %#x\n", - tag); + pr_err_ratelimited("could not locate request for tag %#x\n", + tag); return NULL; } if (unlikely(nvme_genctr_mask(nvme_req(rq)->genctr) != genctr)) { - dev_err(nvme_req(rq)->ctrl->device, + dev_err_ratelimited(nvme_req(rq)->ctrl->device, "request %#x genctr mismatch (got %#x expected %#x)\n", tag, genctr, nvme_genctr_mask(nvme_req(rq)->genctr)); return NULL; @@ -995,6 +1008,7 @@ int nvme_delete_ctrl(struct nvme_ctrl *ctrl); void nvme_queue_scan(struct nvme_ctrl *ctrl); int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, void *log, size_t size, u64 offset); +void nvme_get_ns_head(struct nvme_ns_head *head); bool nvme_tryget_ns_head(struct nvme_ns_head *head); void nvme_put_ns_head(struct nvme_ns_head *head); int nvme_cdev_add(const char *name, struct cdev *cdev, @@ -1032,16 +1046,20 @@ extern const struct attribute_group *nvme_dev_attr_groups[]; extern const struct block_device_operations nvme_bdev_ops; void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl); -struct nvme_ns *nvme_find_path(struct nvme_ns_head *head); +struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) + __must_hold_shared(&head->srcu); #ifdef CONFIG_NVME_MULTIPATH static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl) { return ctrl->ana_log_buf != NULL; } -void nvme_mpath_unfreeze(struct nvme_subsystem *subsys); -void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys); -void nvme_mpath_start_freeze(struct nvme_subsystem *subsys); +void nvme_mpath_unfreeze(struct nvme_subsystem *subsys) + __must_hold(&subsys->lock); +void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) + __must_hold(&subsys->lock); +void nvme_mpath_start_freeze(struct nvme_subsystem *subsys) + __must_hold(&subsys->lock); void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys); void nvme_failover_req(struct request *req); void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl); @@ -1184,6 +1202,15 @@ static inline bool nvme_mpath_queue_if_no_path(struct nvme_ns_head *head) } #endif /* CONFIG_NVME_MULTIPATH */ +#if defined(CONFIG_NVME_MULTIPATH) && defined(CONFIG_BLK_DEV_ZONED) +int nvme_mpath_revalidate_zones(struct nvme_ns_head *head); +#else +static inline int nvme_mpath_revalidate_zones(struct nvme_ns_head *head) +{ + return 0; +} +#endif + int nvme_ns_get_unique_id(struct nvme_ns *ns, u8 id[16], enum blk_unique_id type); @@ -1290,10 +1317,16 @@ static inline void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl) {}; u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode); -u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode); +u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode) + __cond_acquires(nonzero, &ctrl->subsys->lock) + __cond_acquires(nonzero, &ctrl->scan_lock); + int nvme_execute_rq(struct request *rq, bool at_head); -void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, - struct nvme_command *cmd, int status); +u32 nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, + struct nvme_command *cmd, int status) + __cond_releases(nonzero, &ctrl->scan_lock) + __cond_releases(nonzero, &ctrl->subsys->lock); + struct nvme_ctrl *nvme_ctrl_from_file(struct file *file); struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid); bool nvme_get_ns(struct nvme_ns *ns); diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 69932d640b53..da93b505d239 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -213,6 +213,7 @@ static int quirks_param_set(const char *value, const struct kernel_param *kp) if (nvme_parse_quirk_entry(field, &qlist[i])) { pr_err("nvme: failed to parse quirk string %s\n", value); + err = -EINVAL; goto out_free_qlist; } @@ -366,7 +367,8 @@ struct nvme_queue { struct nvme_dev *dev; struct nvme_descriptor_pools descriptor_pools; spinlock_t sq_lock; - void *sq_cmds; + void *sq_cmds + __guarded_by(&sq_lock); /* only used for poll queues: */ spinlock_t cq_poll_lock ____cacheline_aligned_in_smp; struct nvme_completion *cqes; @@ -375,9 +377,11 @@ struct nvme_queue { u32 __iomem *q_db; u32 q_depth; u16 cq_vector; - u16 sq_tail; - u16 last_sq_tail; u16 cq_head; + u16 sq_tail + __guarded_by(&sq_lock); + u16 last_sq_tail + __guarded_by(&sq_lock); u16 qid; u8 cq_phase; u8 sqes; @@ -716,6 +720,7 @@ static void nvme_pci_map_queues(struct blk_mq_tag_set *set) * Write sq tail if we are asked to, or if the next command would wrap. */ static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq) + __must_hold(&nvmeq->sq_lock) { if (!write_sq) { u16 next_tail = nvmeq->sq_tail + 1; @@ -734,6 +739,7 @@ static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq) static inline void nvme_sq_copy_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) + __must_hold(&nvmeq->sq_lock) { memcpy(nvmeq->sq_cmds + (nvmeq->sq_tail << nvmeq->sqes), absolute_pointer(cmd), sizeof(*cmd)); @@ -1580,13 +1586,18 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, req = nvme_find_rq(nvme_queue_tagset(nvmeq), command_id); if (unlikely(!req)) { - dev_warn(nvmeq->dev->ctrl.device, - "invalid id %d completed on queue %d\n", - command_id, le16_to_cpu(cqe->sq_id)); + dev_warn_ratelimited(nvmeq->dev->ctrl.device, + "invalid id %d completed on queue %d\n", + command_id, le16_to_cpu(cqe->sq_id)); return; } - trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail); + /* + * Tracing only; annotate a lockless snapshot of nvmeq->sq_tail using + * data_race(). This would also help suppress context analysis warning + * while accessing nvmeq->sq_tail without acquiring ->sq_lock. + */ + trace_nvme_sq(req, cqe->sq_head, data_race(nvmeq->sq_tail)); if (!nvme_try_complete_req(req, cqe->status, cqe->result) && !blk_mq_add_to_batch(req, iob, nvme_req(req)->status != NVME_SC_SUCCESS, @@ -2013,6 +2024,7 @@ disable: } static void nvme_free_queue(struct nvme_queue *nvmeq) + __context_unsafe(/* frees queue which is no longer in use */) { dma_free_coherent(nvmeq->dev->dev, CQ_SIZE(nvmeq), (void *)nvmeq->cqes, nvmeq->cq_dma_addr); @@ -2107,6 +2119,7 @@ static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues, static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq, int qid) + __context_unsafe(/* safe to allocate sq_cmds without any protection */) { struct pci_dev *pdev = to_pci_dev(dev->dev); @@ -2181,6 +2194,7 @@ static int queue_request_irq(struct nvme_queue *nvmeq) } static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) + __context_unsafe(/* initialize unpublished/lock-guarded variables */) { struct nvme_dev *dev = nvmeq->dev; @@ -2199,6 +2213,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) * Try getting shutdown_lock while setting up IO queues. */ static int nvme_setup_io_queues_trylock(struct nvme_dev *dev) + __cond_acquires(0, &dev->shutdown_lock) { /* * Give up if the lock is being held by nvme_dev_disable. @@ -2400,6 +2415,7 @@ static int nvme_pci_configure_admin_queue(struct nvme_dev *dev) result = queue_request_irq(nvmeq); if (result) { dev->online_queues--; + nvme_disable_ctrl(&dev->ctrl, false); return result; } @@ -3838,6 +3854,7 @@ out_disable: nvme_dev_remove_admin(dev); nvme_dbbuf_dma_free(dev); nvme_free_queues(dev, 0); + nvme_release_descriptor_pools(dev); out_release_iod_mempool: mempool_destroy(dev->dmavec_mempool); out_dev_unmap: diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 6909e3542794..01743ae01466 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -16,6 +16,7 @@ #include <linux/types.h> #include <linux/list.h> #include <linux/mutex.h> +#include <linux/async.h> #include <linux/scatterlist.h> #include <linux/nvme.h> #include <linux/unaligned.h> @@ -39,11 +40,18 @@ #define NVME_RDMA_METADATA_SGL_SIZE \ (sizeof(struct scatterlist) * NVME_INLINE_METADATA_SG_CNT) +static DEFINE_MUTEX(device_list_mutex); +static LIST_HEAD_GUARDED(device_list, device_list_mutex); + +static DEFINE_MUTEX(nvme_rdma_ctrl_mutex); +static LIST_HEAD_GUARDED(nvme_rdma_ctrl_list, nvme_rdma_ctrl_mutex); + struct nvme_rdma_device { struct ib_device *dev; struct ib_pd *pd; struct kref ref; - struct list_head entry; + struct list_head entry + __guarded_by(&device_list_mutex); unsigned int num_inline_segments; }; @@ -100,6 +108,11 @@ struct nvme_rdma_queue { struct mutex queue_lock; }; +struct nvme_rdma_setup_ctx { + struct nvme_rdma_queue *queue; + int *err; +}; + struct nvme_rdma_ctrl { /* read only in the hot path */ struct nvme_rdma_queue *queues; @@ -112,7 +125,8 @@ struct nvme_rdma_ctrl { struct delayed_work reconnect_work; - struct list_head list; + struct list_head list + __guarded_by(&nvme_rdma_ctrl_mutex); struct blk_mq_tag_set admin_tag_set; struct nvme_rdma_device *device; @@ -132,12 +146,6 @@ static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl) return container_of(ctrl, struct nvme_rdma_ctrl, ctrl); } -static LIST_HEAD(device_list); -static DEFINE_MUTEX(device_list_mutex); - -static LIST_HEAD(nvme_rdma_ctrl_list); -static DEFINE_MUTEX(nvme_rdma_ctrl_mutex); - /* * Disabling this option makes small I/O goes faster, but is fundamentally * unsafe. With it turned off we will have to register a global rkey that @@ -566,16 +574,14 @@ out_put_dev: return ret; } -static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl, - int idx, size_t queue_size) +static int nvme_rdma_alloc_queue(struct nvme_rdma_queue *queue) { - struct nvme_rdma_queue *queue; + struct nvme_rdma_ctrl *ctrl = queue->ctrl; + int idx = nvme_rdma_queue_idx(queue); struct sockaddr *src_addr = NULL; int ret; - queue = &ctrl->queues[idx]; mutex_init(&queue->queue_lock); - queue->ctrl = ctrl; if (idx && ctrl->ctrl.max_integrity_segments) queue->pi_support = true; else @@ -587,8 +593,6 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl, else queue->cmnd_capsule_len = sizeof(struct nvme_command); - queue->queue_size = queue_size; - queue->cm_id = rdma_create_id(&init_net, nvme_rdma_cm_handler, queue, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(queue->cm_id)) { @@ -694,59 +698,68 @@ static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) return ret; } -static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl, - int first, int last) +static void nvme_rdma_setup_queue_async(void *data, async_cookie_t cookie) { - int i, ret = 0; + struct nvme_rdma_setup_ctx *ctx = data; + struct nvme_rdma_queue *queue; + int ret; - for (i = first; i < last; i++) { - ret = nvme_rdma_start_queue(ctrl, i); - if (ret) - goto out_stop_queues; - } + queue = ctx->queue; + ret = nvme_rdma_alloc_queue(queue); + if (ret) + goto out_err; - return 0; + ret = nvme_rdma_start_queue(queue->ctrl, nvme_rdma_queue_idx(queue)); + if (ret) + goto out_err; -out_stop_queues: - for (i--; i >= first; i--) - nvme_rdma_stop_queue(&ctrl->queues[i]); - return ret; + return; +out_err: + WRITE_ONCE(*ctx->err, ret); } -static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl) +static int nvme_rdma_setup_io_queues(struct nvme_rdma_ctrl *ctrl, + unsigned int first, unsigned int last, size_t queue_size) { - struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; - unsigned int nr_io_queues; - int i, ret; - - nr_io_queues = nvmf_nr_io_queues(opts); - ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); - if (ret) - return ret; + ASYNC_DOMAIN_EXCLUSIVE(queue_domain); + struct nvme_rdma_setup_ctx *ctxs; + int nr_queues = last - first; + int err = 0, i, ret; - if (nr_io_queues == 0) { - dev_err(ctrl->ctrl.device, - "unable to set any I/O queues\n"); + ctxs = kmalloc_objs(*ctxs, nr_queues); + if (!ctxs) return -ENOMEM; - } - ctrl->ctrl.queue_count = nr_io_queues + 1; - dev_info(ctrl->ctrl.device, - "creating %d I/O queues.\n", nr_io_queues); + for (i = 0; i < nr_queues; i++) { + struct nvme_rdma_queue *queue = &ctrl->queues[first + i]; - nvmf_set_io_queues(opts, nr_io_queues, ctrl->io_queues); - for (i = 1; i < ctrl->ctrl.queue_count; i++) { - ret = nvme_rdma_alloc_queue(ctrl, i, - ctrl->ctrl.sqsize + 1); - if (ret) - goto out_free_queues; + queue->ctrl = ctrl; + queue->queue_size = queue_size; + + ctxs[i].queue = queue; + ctxs[i].err = &err; + async_schedule_domain(nvme_rdma_setup_queue_async, &ctxs[i], + &queue_domain); } - return 0; + async_synchronize_full_domain(&queue_domain); + kfree(ctxs); + ret = READ_ONCE(err); + if (ret) + goto out_free_queues; + + return 0; out_free_queues: - for (i--; i >= 1; i--) - nvme_rdma_free_queue(&ctrl->queues[i]); + for (i = 0; i < nr_queues; i++) { + struct nvme_rdma_queue *queue = + &ctrl->queues[first + i]; + + if (test_bit(NVME_RDMA_Q_LIVE, &queue->flags)) + nvme_rdma_stop_queue(queue); + if (test_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags)) + nvme_rdma_free_queue(queue); + } return ret; } @@ -783,7 +796,9 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, bool pi_capable = false; int error; - error = nvme_rdma_alloc_queue(ctrl, 0, NVME_AQ_DEPTH); + ctrl->queues[0].ctrl = ctrl; + ctrl->queues[0].queue_size = NVME_AQ_DEPTH; + error = nvme_rdma_alloc_queue(&ctrl->queues[0]); if (error) return error; @@ -863,12 +878,23 @@ out_free_queue: static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new) { + unsigned int nr_io_queues; int ret, nr_queues; - ret = nvme_rdma_alloc_io_queues(ctrl); + nr_io_queues = nvmf_nr_io_queues(ctrl->ctrl.opts); + ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); if (ret) return ret; + if (nr_io_queues == 0) { + dev_err(ctrl->ctrl.device, "unable to set any I/O queues\n"); + return -ENOMEM; + } + + ctrl->ctrl.queue_count = nr_io_queues + 1; + dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues); + nvmf_set_io_queues(ctrl->ctrl.opts, nr_io_queues, ctrl->io_queues); + if (new) { ret = nvme_rdma_alloc_tag_set(&ctrl->ctrl); if (ret) @@ -881,7 +907,9 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new) * queue number might have changed. */ nr_queues = min(ctrl->tag_set.nr_hw_queues + 1, ctrl->ctrl.queue_count); - ret = nvme_rdma_start_io_queues(ctrl, 1, nr_queues); + ret = nvme_rdma_setup_io_queues(ctrl, 1, nr_queues, + ctrl->ctrl.sqsize + 1); + if (ret) goto out_cleanup_tagset; @@ -905,12 +933,15 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new) /* * If the number of queues has increased (reconnect case) - * start all new queues now. + * setup all new queues now. */ - ret = nvme_rdma_start_io_queues(ctrl, nr_queues, - ctrl->tag_set.nr_hw_queues + 1); - if (ret) - goto out_wait_freeze_timed_out; + if (ctrl->tag_set.nr_hw_queues + 1 > nr_queues) { + ret = nvme_rdma_setup_io_queues(ctrl, nr_queues, + ctrl->tag_set.nr_hw_queues + 1, + ctrl->ctrl.sqsize + 1); + if (ret) + goto out_wait_freeze_timed_out; + } return 0; @@ -969,7 +1000,7 @@ static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl) { struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl); - if (list_empty(&ctrl->list)) + if (list_empty_careful(&ctrl->list)) goto free_ctrl; mutex_lock(&nvme_rdma_ctrl_mutex); @@ -2254,7 +2285,10 @@ static struct nvme_rdma_ctrl *nvme_rdma_alloc_ctrl(struct device *dev, if (!ctrl) return ERR_PTR(-ENOMEM); ctrl->ctrl.opts = opts; - INIT_LIST_HEAD(&ctrl->list); + /* + * Safe to init list while allocating ctrl object. + */ + context_unsafe(INIT_LIST_HEAD(&ctrl->list)); if (!(opts->mask & NVMF_OPT_TRSVCID)) { opts->trsvcid = diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c index 75b2d69b5957..abf8edaae371 100644 --- a/drivers/nvme/host/sysfs.c +++ b/drivers/nvme/host/sysfs.c @@ -240,8 +240,10 @@ static ssize_t nuse_show(struct device *dev, struct device_attribute *attr, ret = ns_head_update_nuse(head); else ret = ns_update_nuse(disk->private_data); - if (ret) + if (ret < 0) return ret; + else if (ret > 0) + return -EIO; return sysfs_emit(buf, "%llu\n", head->nuse); } diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index ce03a0ea4ded..5fda9661bdb7 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -80,6 +80,7 @@ struct nvme_tcp_request { struct bio *curr_bio; struct iov_iter iter; + u32 data_recvd; /* send state */ size_t offset; @@ -108,6 +109,7 @@ struct nvme_tcp_queue { struct mutex queue_lock; struct mutex send_mutex; + struct mutex pf_cache_lock; struct llist_head req_list; struct list_head send_list; @@ -149,13 +151,17 @@ struct nvme_tcp_queue { #endif }; +static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); +static LIST_HEAD_GUARDED(nvme_tcp_ctrl_list, nvme_tcp_ctrl_mutex); + struct nvme_tcp_ctrl { /* read only in the hot path */ struct nvme_tcp_queue *queues; struct blk_mq_tag_set tag_set; /* other member variables */ - struct list_head list; + struct list_head list + __guarded_by(&nvme_tcp_ctrl_mutex); struct blk_mq_tag_set admin_tag_set; struct sockaddr_storage addr; struct sockaddr_storage src_addr; @@ -167,8 +173,6 @@ struct nvme_tcp_ctrl { u32 io_queues[HCTX_MAX_TYPES]; }; -static LIST_HEAD(nvme_tcp_ctrl_list); -static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); static struct workqueue_struct *nvme_tcp_wq; static const struct blk_mq_ops nvme_tcp_mq_ops; static const struct blk_mq_ops nvme_tcp_admin_mq_ops; @@ -550,9 +554,11 @@ static int nvme_tcp_init_request(struct blk_mq_tag_set *set, struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; u8 hdgst = nvme_tcp_hdgst_len(queue); + mutex_lock(&queue->pf_cache_lock); req->pdu = page_frag_alloc(&queue->pf_cache, sizeof(struct nvme_tcp_cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); + mutex_unlock(&queue->pf_cache_lock); if (!req->pdu) return -ENOMEM; @@ -612,6 +618,29 @@ static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work); } +/* + * NVMe has no short read: a read that completes successfully must + * have transferred everything it asked for. + */ +static bool nvme_tcp_data_in_short(struct nvme_tcp_queue *queue, + struct request *rq) +{ + struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); + + if (le16_to_cpu(req->status) >> 1) + return false; + if (req_op(rq) != REQ_OP_READ || !req->data_len) + return false; + if (likely(req->data_recvd == req->data_len)) + return false; + + dev_err(queue->ctrl->ctrl.device, + "queue %d tag %#x short data-in: got %u of %u\n", + nvme_tcp_queue_id(queue), rq->tag, + req->data_recvd, req->data_len); + return true; +} + static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, struct nvme_completion *cqe) { @@ -631,6 +660,9 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, if (req->status == cpu_to_le16(NVME_SC_SUCCESS)) req->status = cqe->status; + if (unlikely(nvme_tcp_data_in_short(queue, rq))) + return -EPROTO; + if (!nvme_try_complete_req(rq, req->status, cqe->result)) nvme_complete_rq(rq); queue->nr_cqe++; @@ -641,6 +673,7 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, struct nvme_tcp_data_pdu *pdu) { + struct nvme_tcp_request *req; struct request *rq; rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); @@ -651,7 +684,8 @@ static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, return -ENOENT; } - if (!blk_rq_payload_bytes(rq)) { + req = blk_mq_rq_to_pdu(rq); + if (!blk_rq_payload_bytes(rq) || !req->curr_bio || !req->data_len) { dev_err(queue->ctrl->ctrl.device, "queue %d tag %#x unexpected data\n", nvme_tcp_queue_id(queue), rq->tag); @@ -745,6 +779,13 @@ static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, } req = blk_mq_rq_to_pdu(rq); + if (unlikely(rq_data_dir(rq) != WRITE)) { + dev_err(queue->ctrl->ctrl.device, + "req %d unexpected r2t for a non-write command\n", + rq->tag); + return -EPROTO; + } + if (unlikely(!r2t_length)) { dev_err(queue->ctrl->ctrl.device, "req %d r2t len is %u, probably a bug...\n", @@ -953,6 +994,7 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, *len -= recv_len; *offset += recv_len; queue->data_remaining -= recv_len; + req->data_recvd += recv_len; } if (!queue->data_remaining) { @@ -961,6 +1003,8 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; } else { if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { + if (unlikely(nvme_tcp_data_in_short(queue, rq))) + return -EPROTO; nvme_tcp_end_request(rq, le16_to_cpu(req->status)); queue->nr_cqe++; @@ -1009,6 +1053,9 @@ static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, pdu->command_id); struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); + if (unlikely(nvme_tcp_data_in_short(queue, rq))) + return -EPROTO; + nvme_tcp_end_request(rq, le16_to_cpu(req->status)); queue->nr_cqe++; } @@ -1417,9 +1464,11 @@ static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) struct nvme_tcp_request *async = &ctrl->async_req; u8 hdgst = nvme_tcp_hdgst_len(queue); + mutex_lock(&queue->pf_cache_lock); async->pdu = page_frag_alloc(&queue->pf_cache, sizeof(struct nvme_tcp_cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); + mutex_unlock(&queue->pf_cache_lock); if (!async->pdu) return -ENOMEM; @@ -1461,6 +1510,7 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) kfree(queue->pdu); mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); + mutex_destroy(&queue->pf_cache_lock); #ifdef CONFIG_DEBUG_LOCK_ALLOC lockdep_unregister_key(&queue->nvme_tcp_sk_key); @@ -1788,6 +1838,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, INIT_LIST_HEAD(&queue->send_list); mutex_init(&queue->send_mutex); INIT_WORK(&queue->io_work, nvme_tcp_io_work); + mutex_init(&queue->pf_cache_lock); if (qid > 0) queue->cmnd_capsule_len = nctrl->ioccsz * 16; @@ -1928,6 +1979,7 @@ err_sock: err_destroy_mutex: mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); + mutex_destroy(&queue->pf_cache_lock); return ret; } @@ -2577,7 +2629,7 @@ static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl) { struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); - if (list_empty(&ctrl->list)) + if (list_empty_careful(&ctrl->list)) goto free_ctrl; mutex_lock(&nvme_tcp_ctrl_mutex); @@ -2736,6 +2788,7 @@ static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, req->status = cpu_to_le16(NVME_SC_SUCCESS); req->offset = 0; req->data_sent = 0; + req->data_recvd = 0; req->pdu_len = 0; req->pdu_sent = 0; req->h2cdata_left = 0; @@ -2919,7 +2972,10 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev, if (!ctrl) return ERR_PTR(-ENOMEM); - INIT_LIST_HEAD(&ctrl->list); + /* + * Safe to init list while allocating ctrl object. + */ + context_unsafe(INIT_LIST_HEAD(&ctrl->list)); ctrl->ctrl.opts = opts; ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + opts->nr_poll_queues + 1; @@ -2960,7 +3016,8 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev, } if (opts->mask & NVMF_OPT_HOST_IFACE) { - if (!__dev_get_by_name(&init_net, opts->host_iface)) { + if (!__dev_get_by_name(current->nsproxy->net_ns, + opts->host_iface)) { pr_err("invalid interface passed: %s\n", opts->host_iface); ret = -ENODEV; diff --git a/drivers/nvme/host/zns.c b/drivers/nvme/host/zns.c index 8ed1b6a33454..2a152e87bd76 100644 --- a/drivers/nvme/host/zns.c +++ b/drivers/nvme/host/zns.c @@ -155,7 +155,8 @@ static int nvme_zone_parse_entry(struct nvme_ns *ns, struct blk_zone zone = { }; if ((entry->zt & 0xf) != NVME_ZONE_TYPE_SEQWRITE_REQ) { - dev_err(ns->ctrl->device, "invalid zone type %#x\n", entry->zt); + dev_err(ns->ctrl->device, "invalid zone type %#x at zone %u\n", + entry->zt, idx); return -EINVAL; } @@ -178,7 +179,7 @@ int nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, struct nvme_zone_report *report; struct nvme_command c = { }; int ret, zone_idx = 0; - unsigned int nz, i; + unsigned int max_in_buf, nz, i; size_t buflen; if (ns->head->ids.csi != NVME_CSI_ZNS) @@ -188,6 +189,9 @@ int nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, if (!report) return -ENOMEM; + max_in_buf = (buflen - sizeof(struct nvme_zone_report)) / + sizeof(struct nvme_zone_descriptor); + c.zmr.opcode = nvme_cmd_zone_mgmt_recv; c.zmr.nsid = cpu_to_le32(ns->head->ns_id); c.zmr.numd = cpu_to_le32(nvme_bytes_to_numd(buflen)); @@ -207,7 +211,8 @@ int nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, goto out_free; } - nz = min((unsigned int)le64_to_cpu(report->nr_zones), nr_zones); + nz = min3((unsigned int)le64_to_cpu(report->nr_zones), + nr_zones - zone_idx, max_in_buf); if (!nz) break; diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 01b799e92ae6..7764a3c0195c 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -309,8 +309,10 @@ static void nvmet_execute_get_log_page_rmi(struct nvmet_req *req) } log = kzalloc_obj(*log); - if (!log) + if (!log) { + status = NVME_SC_INTERNAL; goto out; + } log->endgid = req->cmd->get_log_page.lsi; disk = req->ns->bdev->bd_disk; @@ -958,7 +960,7 @@ static void nvmet_execute_identify_nslist(struct nvmet_req *req, bool match_css) nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) { if (ns->nsid <= min_nsid) continue; - if (match_css && req->ns->csi != req->cmd->identify.csi) + if (match_css && ns->csi != req->cmd->identify.csi) continue; list[i++] = cpu_to_le32(ns->nsid); if (i == buf_size / sizeof(__le32)) @@ -1335,7 +1337,7 @@ static u16 nvmet_set_feat_arbitration(struct nvmet_req *req) void nvmet_execute_set_features(struct nvmet_req *req) { - struct nvmet_subsys *subsys = nvmet_req_subsys(req); + struct nvmet_ctrl *ctrl = nvmet_req_ctrl(req); u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); u32 cdw11 = le32_to_cpu(req->cmd->common.cdw11); u16 status = 0; @@ -1357,7 +1359,7 @@ void nvmet_execute_set_features(struct nvmet_req *req) break; } nvmet_set_result(req, - (subsys->max_qid - 1) | ((subsys->max_qid - 1) << 16)); + (ctrl->max_qid - 1) | ((ctrl->max_qid - 1) << 16)); break; case NVME_FEAT_IRQ_COALESCE: status = nvmet_set_feat_irq_coalesce(req); @@ -1494,7 +1496,7 @@ void nvmet_get_feat_async_event(struct nvmet_req *req) void nvmet_execute_get_features(struct nvmet_req *req) { - struct nvmet_subsys *subsys = nvmet_req_subsys(req); + struct nvmet_ctrl *ctrl = nvmet_req_ctrl(req); u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); u16 status = 0; @@ -1534,7 +1536,7 @@ void nvmet_execute_get_features(struct nvmet_req *req) break; case NVME_FEAT_NUM_QUEUES: nvmet_set_result(req, - (subsys->max_qid-1) | ((subsys->max_qid-1) << 16)); + (ctrl->max_qid-1) | ((ctrl->max_qid-1) << 16)); break; case NVME_FEAT_KATO: nvmet_get_feat_kato(req); diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 2b69ffcfc8df..413ee2d16d29 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -312,15 +312,17 @@ static ssize_t nvmet_param_mdts_store(struct config_item *item, const char *page, size_t count) { struct nvmet_port *port = to_nvmet_port(item); - int ret; + int ret, mdts; if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - ret = kstrtoint(page, 0, &port->mdts); - if (ret) { - pr_err("Invalid value '%s' for mdts\n", page); + ret = kstrtoint(page, 0, &mdts); + if (ret || mdts < 0 || mdts > NVMET_MAX_MDTS) { + pr_err("Invalid value '%s' for mdts, should be 0-%d\n", + page, NVMET_MAX_MDTS); return -EINVAL; } + port->mdts = mdts; return count; } diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 4477c4d6b1ee..d74c01c98f19 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -610,12 +610,14 @@ int nvmet_ns_enable(struct nvmet_ns *ns) goto out_dev_put; } - if (percpu_ref_init(&ns->ref, nvmet_destroy_namespace, 0, GFP_KERNEL)) + ret = percpu_ref_init(&ns->ref, nvmet_destroy_namespace, 0, GFP_KERNEL); + if (ret) goto out_pr_exit; nvmet_ns_changed(subsys, ns->nsid); ns->enabled = true; xa_set_mark(&subsys->namespaces, ns->nsid, NVMET_NS_ENABLED); + nvmet_debugfs_ns_setup(ns); ret = 0; out_unlock: mutex_unlock(&subsys->lock); @@ -642,6 +644,7 @@ void nvmet_ns_disable(struct nvmet_ns *ns) ns->enabled = false; xa_clear_mark(&subsys->namespaces, ns->nsid, NVMET_NS_ENABLED); + nvmet_debugfs_ns_free(ns); list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) pci_dev_put(radix_tree_delete(&ctrl->p2p_ns_map, ns->nsid)); @@ -875,7 +878,7 @@ u16 nvmet_check_cqid(struct nvmet_ctrl *ctrl, u16 cqid, bool create) if (!ctrl->cqs) return NVME_SC_INTERNAL | NVME_STATUS_DNR; - if (cqid > ctrl->subsys->max_qid) + if (cqid > ctrl->max_qid) return NVME_SC_QID_INVALID | NVME_STATUS_DNR; if ((create && ctrl->cqs[cqid]) || (!create && !ctrl->cqs[cqid])) @@ -923,7 +926,7 @@ u16 nvmet_check_sqid(struct nvmet_ctrl *ctrl, u16 sqid, if (!ctrl->sqs) return NVME_SC_INTERNAL | NVME_STATUS_DNR; - if (sqid > ctrl->subsys->max_qid) + if (sqid > ctrl->max_qid) return NVME_SC_QID_INVALID | NVME_STATUS_DNR; if ((create && ctrl->sqs[sqid]) || @@ -1652,11 +1655,29 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) if (!ctrl->changed_ns_list) goto out_free_ctrl; - ctrl->sqs = kzalloc_objs(struct nvmet_sq *, subsys->max_qid + 1); + /* + * Discovery controllers may use some arbitrary high value + * in order to cleanup stale discovery sessions + */ + if (nvmet_is_disc_subsys(ctrl->subsys) && !kato) + kato = NVMET_DISC_KATO_MS; + + /* keep-alive timeout in seconds */ + ctrl->kato = DIV_ROUND_UP(kato, 1000); + + ctrl->err_counter = 0; + spin_lock_init(&ctrl->error_lock); + + down_read(&nvmet_config_sem); + mutex_lock(&subsys->lock); + + ctrl->max_qid = subsys->max_qid; + + ctrl->sqs = kzalloc_objs(struct nvmet_sq *, ctrl->max_qid + 1); if (!ctrl->sqs) goto out_free_changed_ns_list; - ctrl->cqs = kzalloc_objs(struct nvmet_cq *, subsys->max_qid + 1); + ctrl->cqs = kzalloc_objs(struct nvmet_cq *, ctrl->max_qid + 1); if (!ctrl->cqs) goto out_free_sqs; @@ -1669,22 +1690,6 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) } ctrl->cntlid = ret; - /* - * Discovery controllers may use some arbitrary high value - * in order to cleanup stale discovery sessions - */ - if (nvmet_is_disc_subsys(ctrl->subsys) && !kato) - kato = NVMET_DISC_KATO_MS; - - /* keep-alive timeout in seconds */ - ctrl->kato = DIV_ROUND_UP(kato, 1000); - - ctrl->err_counter = 0; - spin_lock_init(&ctrl->error_lock); - - nvmet_start_keep_alive_timer(ctrl); - - mutex_lock(&subsys->lock); ret = nvmet_ctrl_init_pr(ctrl); if (ret) goto init_pr_fail; @@ -1692,6 +1697,9 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) nvmet_setup_p2p_ns_map(ctrl, args->p2p_client); nvmet_debugfs_ctrl_setup(ctrl); mutex_unlock(&subsys->lock); + up_read(&nvmet_config_sem); + + nvmet_start_keep_alive_timer(ctrl); if (args->hostid) uuid_copy(&ctrl->hostid, args->hostid); @@ -1721,14 +1729,14 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) return ctrl; init_pr_fail: - mutex_unlock(&subsys->lock); - nvmet_stop_keep_alive_timer(ctrl); ida_free(&cntlid_ida, ctrl->cntlid); out_free_cqs: kfree(ctrl->cqs); out_free_sqs: kfree(ctrl->sqs); out_free_changed_ns_list: + mutex_unlock(&subsys->lock); + up_read(&nvmet_config_sem); kfree(ctrl->changed_ns_list); out_free_ctrl: kfree(ctrl); diff --git a/drivers/nvme/target/debugfs.c b/drivers/nvme/target/debugfs.c index 5dcbd5aa86e1..e85fe1d4c9f8 100644 --- a/drivers/nvme/target/debugfs.c +++ b/drivers/nvme/target/debugfs.c @@ -153,6 +153,109 @@ static int nvmet_ctrl_tls_concat_show(struct seq_file *m, void *p) NVMET_DEBUGFS_ATTR(nvmet_ctrl_tls_concat); #endif +static const char *const nvmet_pr_type_names[] = { + [NVME_PR_WRITE_EXCLUSIVE] = "write_exclusive", + [NVME_PR_EXCLUSIVE_ACCESS] = "exclusive_access", + [NVME_PR_WRITE_EXCLUSIVE_REG_ONLY] = "write_exclusive_reg_only", + [NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY] = "exclusive_access_reg_only", + [NVME_PR_WRITE_EXCLUSIVE_ALL_REGS] = "write_exclusive_all_regs", + [NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS] = "exclusive_access_all_regs", +}; + +static const char *nvmet_pr_type_to_str(enum nvme_pr_type type) +{ + if (type < ARRAY_SIZE(nvmet_pr_type_names) && + nvmet_pr_type_names[type]) + return nvmet_pr_type_names[type]; + return "unknown"; +} + +static const char *const nvmet_pr_notify_names[] = { + [NVME_PR_NOTIFY_BIT_REG_PREEMPTED] = "reg_preempted", + [NVME_PR_NOTIFY_BIT_RESV_RELEASED] = "resv_released", + [NVME_PR_NOTIFY_BIT_RESV_PREEMPTED] = "resv_preempted", +}; + +static void nvmet_pr_notify_mask_to_str(struct seq_file *m, unsigned long mask) +{ + bool sep = false; + int i; + + if (!mask) { + seq_puts(m, "none"); + return; + } + + for (i = 0; i < ARRAY_SIZE(nvmet_pr_notify_names); i++) { + if (!test_bit(i, &mask) || !nvmet_pr_notify_names[i]) + continue; + if (sep) + seq_putc(m, ','); + seq_puts(m, nvmet_pr_notify_names[i]); + sep = true; + } +} + +static int nvmet_ns_pr_show(struct seq_file *m, void *p) +{ + struct nvmet_ns *ns = m->private; + struct nvmet_pr *pr = &ns->pr; + struct nvmet_pr_registrant *holder, *reg; + + seq_printf(m, "enable=%d\n", pr->enable); + if (!pr->enable) + return 0; + + seq_printf(m, "generation=%u\n", atomic_read(&pr->generation)); + seq_puts(m, "notify_mask="); + nvmet_pr_notify_mask_to_str(m, pr->notify_mask); + seq_putc(m, '\n'); + + rcu_read_lock(); + holder = rcu_dereference(pr->holder); + if (holder) { + seq_printf(m, "rtype=%s\n", + nvmet_pr_type_to_str(holder->rtype)); + seq_printf(m, "holder=%pUb,0x%llx\n", + &holder->hostid, holder->rkey); + } else { + seq_puts(m, "rtype=none\n"); + seq_puts(m, "holder=none\n"); + } + + list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { + seq_printf(m, "reg=%pUb,0x%llx\n", + ®->hostid, reg->rkey); + } + rcu_read_unlock(); + + return 0; +} +NVMET_DEBUGFS_ATTR(nvmet_ns_pr); + +void nvmet_debugfs_ns_setup(struct nvmet_ns *ns) +{ + char name[16]; + struct dentry *parent = ns->subsys->debugfs_dir; + + if (!parent) + return; + snprintf(name, sizeof(name), "ns%u", ns->nsid); + ns->debugfs_dir = debugfs_create_dir(name, parent); + if (IS_ERR(ns->debugfs_dir)) { + ns->debugfs_dir = NULL; + return; + } + debugfs_create_file("reservation", 0400, ns->debugfs_dir, ns, + &nvmet_ns_pr_fops); +} + +void nvmet_debugfs_ns_free(struct nvmet_ns *ns) +{ + debugfs_remove_recursive(ns->debugfs_dir); + ns->debugfs_dir = NULL; +} + int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl) { char name[32]; diff --git a/drivers/nvme/target/debugfs.h b/drivers/nvme/target/debugfs.h index cfb8bbf6a297..b559d254fc2a 100644 --- a/drivers/nvme/target/debugfs.h +++ b/drivers/nvme/target/debugfs.h @@ -14,6 +14,8 @@ int nvmet_debugfs_subsys_setup(struct nvmet_subsys *subsys); void nvmet_debugfs_subsys_free(struct nvmet_subsys *subsys); int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl); void nvmet_debugfs_ctrl_free(struct nvmet_ctrl *ctrl); +void nvmet_debugfs_ns_setup(struct nvmet_ns *ns); +void nvmet_debugfs_ns_free(struct nvmet_ns *ns); int __init nvmet_init_debugfs(void); void nvmet_exit_debugfs(void); @@ -30,6 +32,9 @@ static inline int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl) } static inline void nvmet_debugfs_ctrl_free(struct nvmet_ctrl *ctrl) {} +static inline void nvmet_debugfs_ns_setup(struct nvmet_ns *ns) {} +static inline void nvmet_debugfs_ns_free(struct nvmet_ns *ns) {} + static inline int __init nvmet_init_debugfs(void) { return 0; diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c index 45820a12750d..92f8a76f10ff 100644 --- a/drivers/nvme/target/fabrics-cmd-auth.c +++ b/drivers/nvme/target/fabrics-cmd-auth.c @@ -9,6 +9,7 @@ #include <linux/random.h> #include <linux/nvme-auth.h> #include <crypto/kpp.h> +#include <crypto/utils.h> #include "nvmet.h" static void nvmet_auth_expired_work(struct work_struct *work) @@ -30,12 +31,16 @@ void nvmet_auth_sq_init(struct nvmet_sq *sq) sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; } -static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d) +static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d, u32 tl) { struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmf_auth_dhchap_negotiate_data *data = d; int i, hash_id = 0, fallback_hash_id = 0, dhgid, fallback_dhgid; + if (tl < sizeof(*data) + + sizeof(struct nvmf_auth_dhchap_protocol_descriptor)) + return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + pr_debug("%s: ctrl %d qid %d: data sc_d %d napd %d authid %d halen %d dhlen %d\n", __func__, ctrl->cntlid, req->sq->qid, data->sc_c, data->napd, data->auth_protocol[0].dhchap.authid, @@ -71,6 +76,10 @@ static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d) NVME_AUTH_DHCHAP_AUTH_ID) return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + if (data->auth_protocol[0].dhchap.dhlen > NVME_AUTH_DHCHAP_MAX_DH_IDS || + data->auth_protocol[0].dhchap.halen > NVME_AUTH_DHCHAP_MAX_HASH_IDS) + return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + for (i = 0; i < data->auth_protocol[0].dhchap.halen; i++) { u8 host_hmac_id = data->auth_protocol[0].dhchap.idlist[i]; @@ -177,7 +186,7 @@ static u8 nvmet_auth_reply(struct nvmet_req *req, void *d, u32 tl) return NVME_AUTH_DHCHAP_FAILURE_FAILED; } - if (memcmp(data->rval, response, data->hl)) { + if (crypto_memneq(data->rval, response, data->hl)) { pr_info("ctrl %d qid %d host response mismatch\n", ctrl->cntlid, req->sq->qid); pr_debug("ctrl %d qid %d rval %*ph\n", @@ -316,7 +325,7 @@ void nvmet_execute_auth_send(struct nvmet_req *req) } else if (data->auth_id != req->sq->dhchap_step) goto done_failure1; /* Validate negotiation parameters */ - dhchap_status = nvmet_auth_negotiate(req, d); + dhchap_status = nvmet_auth_negotiate(req, d, tl); if (dhchap_status == 0) req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE; @@ -557,7 +566,7 @@ void nvmet_execute_auth_receive(struct nvmet_req *req) return; } - d = kmalloc(al, GFP_KERNEL); + d = kzalloc(al, GFP_KERNEL); if (!d) { status = NVME_SC_INTERNAL; goto done; diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c index 7cadd1c9e44c..42d1d1811671 100644 --- a/drivers/nvme/target/fabrics-cmd.c +++ b/drivers/nvme/target/fabrics-cmd.c @@ -370,7 +370,7 @@ static void nvmet_execute_io_connect(struct nvmet_req *req) goto out; } - if (unlikely(qid > ctrl->subsys->max_qid)) { + if (unlikely(qid > ctrl->max_qid)) { pr_warn("invalid queue id (%d)\n", qid); status = NVME_SC_CONNECT_INVALID_PARAM | NVME_STATUS_DNR; req->cqe->result.u32 = IPO_IATTR_CONNECT_SQE(qid); diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index d161707559ce..1b557775e033 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -566,7 +566,7 @@ out_fail: list_del(&iod->ls_rcv_list); } - kfree(iod); + kfree(tgtport->iod); return -EFAULT; } diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index aaba745e3c21..e362d7913a38 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -128,6 +128,9 @@ struct nvmet_ns { u8 csi; struct nvmet_pr pr; struct xarray pr_per_ctrl_refs; +#ifdef CONFIG_NVME_TARGET_DEBUGFS + struct dentry *debugfs_dir; +#endif }; static inline struct nvmet_ns *to_nvmet_ns(struct config_item *item) @@ -265,6 +268,7 @@ struct nvmet_ctrl { uuid_t hostid; u16 cntlid; + u16 max_qid; u32 kato; struct nvmet_port *port; @@ -753,6 +757,11 @@ static inline struct nvmet_subsys *nvmet_req_subsys(struct nvmet_req *req) return req->sq->ctrl->subsys; } +static inline struct nvmet_ctrl *nvmet_req_ctrl(struct nvmet_req *req) +{ + return req->sq->ctrl; +} + static inline bool nvmet_is_disc_subsys(struct nvmet_subsys *subsys) { return subsys->type != NVME_NQN_NVME; diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c index e27f84e3cf2b..fa6527c537e2 100644 --- a/drivers/nvme/target/passthru.c +++ b/drivers/nvme/target/passthru.c @@ -53,13 +53,22 @@ static u16 nvmet_passthru_override_id_descs(struct nvmet_req *req) for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) { struct nvme_ns_id_desc *cur = data + pos; + if (pos + sizeof(*cur) > NVME_IDENTIFY_DATA_SIZE) + break; + if (cur->nidl == 0) break; + if (cur->nidt == NVME_NIDT_CSI) { + if (pos + sizeof(*cur) + NVME_NIDT_CSI_LEN > + NVME_IDENTIFY_DATA_SIZE) + break; + memcpy(&csi, cur + 1, NVME_NIDT_CSI_LEN); csi_seen = true; break; } + len = sizeof(struct nvme_ns_id_desc) + cur->nidl; } diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c index 4e9db96ebfec..803e85df50e5 100644 --- a/drivers/nvme/target/pci-epf.c +++ b/drivers/nvme/target/pci-epf.c @@ -1339,6 +1339,7 @@ err_unmap_queue: nvmet_pci_epf_mem_unmap(ctrl->nvme_epf, &cq->pci_map); err_internal: status = NVME_SC_INTERNAL | NVME_STATUS_DNR; + nvmet_cq_put(&cq->nvme_cq); err: if (test_and_clear_bit(NVMET_PCI_EPF_Q_IRQ_ENABLED, &cq->flags)) nvmet_pci_epf_remove_irq_vector(ctrl, cq->vector); @@ -1594,6 +1595,7 @@ static void nvmet_pci_epf_exec_iod_work(struct work_struct *work) struct nvmet_pci_epf_iod *iod = container_of(work, struct nvmet_pci_epf_iod, work); struct nvmet_req *req = &iod->req; + bool no_wait; int ret; if (!iod->ctrl->link_up) { @@ -1638,14 +1640,16 @@ static void nvmet_pci_epf_exec_iod_work(struct work_struct *work) } } - req->execute(req); - /* * If we do not have data to transfer after the command execution * finishes, nvmet_pci_epf_queue_response() will complete the command * directly. No need to wait for the completion in this case. */ - if (!iod->data_len || iod->dma_dir != DMA_TO_DEVICE) + no_wait = !iod->data_len || iod->dma_dir != DMA_TO_DEVICE; + + req->execute(req); + + if (no_wait) return; wait_for_completion(&iod->done); @@ -2077,7 +2081,7 @@ static int nvmet_pci_epf_create_ctrl(struct nvmet_pci_epf *nvme_epf, } /* Allocate our queues, up to the maximum number. */ - ctrl->nr_queues = min(ctrl->tctrl->subsys->max_qid + 1, max_nr_queues); + ctrl->nr_queues = min(ctrl->tctrl->max_qid + 1, max_nr_queues); ret = nvmet_pci_epf_alloc_queues(ctrl); if (ret) goto out_put_ctrl; diff --git a/drivers/nvme/target/pr.c b/drivers/nvme/target/pr.c index c71ae46244ff..0948a690a1c0 100644 --- a/drivers/nvme/target/pr.c +++ b/drivers/nvme/target/pr.c @@ -8,7 +8,7 @@ #include <linux/unaligned.h> #include "nvmet.h" -#define NVMET_PR_NOTIFI_MASK_ALL \ +#define NVMET_PR_NOTIFY_MASK_ALL \ (1 << NVME_PR_NOTIFY_BIT_REG_PREEMPTED | \ 1 << NVME_PR_NOTIFY_BIT_RESV_RELEASED | \ 1 << NVME_PR_NOTIFY_BIT_RESV_PREEMPTED) @@ -44,7 +44,7 @@ u16 nvmet_set_feat_resv_notif_mask(struct nvmet_req *req, u32 mask) unsigned long idx; u16 status; - if (mask & ~(NVMET_PR_NOTIFI_MASK_ALL)) { + if (mask & ~(NVMET_PR_NOTIFY_MASK_ALL)) { req->error_loc = offsetof(struct nvme_common_command, cdw11); return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; } @@ -169,7 +169,7 @@ static void nvmet_pr_resv_released(struct nvmet_pr *pr, uuid_t *hostid) nvmet_pr_add_resv_log(ctrl, NVME_PR_LOG_RESERVATION_RELEASED, ns->nsid); nvmet_add_async_event(ctrl, NVME_AER_CSS, - NVME_AEN_RESV_LOG_PAGE_AVALIABLE, + NVME_AEN_RESV_LOG_PAGE_AVAILABLE, NVME_LOG_RESERVATION); } } @@ -188,7 +188,7 @@ static void nvmet_pr_send_event_to_host(struct nvmet_pr *pr, uuid_t *hostid, if (uuid_equal(hostid, &ctrl->hostid)) { nvmet_pr_add_resv_log(ctrl, log_type, ns->nsid); nvmet_add_async_event(ctrl, NVME_AER_CSS, - NVME_AEN_RESV_LOG_PAGE_AVALIABLE, + NVME_AEN_RESV_LOG_PAGE_AVAILABLE, NVME_LOG_RESERVATION); } } @@ -201,7 +201,7 @@ static void nvmet_pr_resv_preempted(struct nvmet_pr *pr, uuid_t *hostid) return; nvmet_pr_send_event_to_host(pr, hostid, - NVME_PR_LOG_RESERVATOIN_PREEMPTED); + NVME_PR_LOG_RESERVATION_PREEMPTED); } static void nvmet_pr_registration_preempted(struct nvmet_pr *pr, @@ -355,9 +355,15 @@ static u16 nvmet_pr_replace(struct nvmet_req *req, u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmet_pr *pr = &req->ns->pr; - struct nvmet_pr_registrant *reg; + struct nvmet_pr_registrant *reg, *new = NULL; u64 nrkey = le64_to_cpu(d->nrkey); + if (ignore_key && nrkey) { + new = kzalloc_obj(*new); + if (!new) + return NVME_SC_INTERNAL; + } + down(&pr->pr_sem); list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { if (uuid_equal(®->hostid, &ctrl->hostid)) { @@ -365,9 +371,26 @@ static u16 nvmet_pr_replace(struct nvmet_req *req, status = nvmet_pr_update_reg_attr(pr, reg, nvmet_pr_update_reg_rkey, &nrkey); - break; + goto free_data; + } + } + + if (ignore_key) { + if (!nrkey) { + status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + goto free_data; } + INIT_LIST_HEAD(&new->entry); + new->rkey = nrkey; + uuid_copy(&new->hostid, &ctrl->hostid); + list_add_tail_rcu(&new->entry, &pr->registrant_list); + status = NVME_SC_SUCCESS; + goto out; } + +free_data: + kfree(new); +out: up(&pr->pr_sem); return status; } diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index ea1185b8267e..de5a88fbb233 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -657,18 +657,25 @@ static void nvmet_rdma_rw_ctx_destroy(struct nvmet_rdma_rsp *rsp) req->sg, req->sg_cnt, nvmet_data_dir(req)); } -static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) +static void nvmet_rdma_free_rsp_resources(struct nvmet_rdma_rsp *rsp) { struct nvmet_rdma_queue *queue = rsp->queue; - atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); - if (rsp->n_rdma) nvmet_rdma_rw_ctx_destroy(rsp); if (rsp->req.sg < rsp->cmd->inline_sg || rsp->req.sg >= rsp->cmd->inline_sg + queue->dev->inline_page_count) nvmet_req_free_sgls(&rsp->req); +} + +static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) +{ + struct nvmet_rdma_queue *queue = rsp->queue; + + atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); + + nvmet_rdma_free_rsp_resources(rsp); if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list))) nvmet_rdma_process_wr_wait_list(queue); @@ -1338,9 +1345,27 @@ err_destroy_cq: goto out; } +static bool nvmet_rdma_reclaim_rsp(struct sbitmap *sb, unsigned int bitnr, + void *data) +{ + struct nvmet_rdma_queue *queue = data; + + nvmet_rdma_free_rsp_resources(&queue->rsps[bitnr]); + + return true; +} + static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue *queue) { ib_drain_qp(queue->qp); + + /* + * Reclaim resources of a response that is still in-flight when the + * queue is being torn down. This happens when the connection was + * forcefully disconnected while an I/O is in flight. + */ + sbitmap_for_each_set(&queue->rsp_tags, nvmet_rdma_reclaim_rsp, queue); + if (queue->cm_id) rdma_destroy_id(queue->cm_id); ib_destroy_qp(queue->qp); diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 75a276d73be3..e4f603b2ace7 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -422,6 +422,19 @@ static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd) if (!len) return 0; + /* + * inline_data_size only bounds the in-capsule (type 0x01) SGL + * descriptor below. A non-inline transport SGL data-block + * descriptor skips that check entirely and would otherwise reach + * sgl_alloc() with an attacker-controlled len of up to 4 GiB, + * pinning that much kernel memory for a command that may never + * complete. Bound every descriptor type here, before allocating + * anything, using the same ceiling this file already applies to + * per-PDU H2C data. + */ + if (len > NVMET_TCP_MAXH2CDATA) + return NVME_SC_SGL_INVALID_DATA | NVME_STATUS_DNR; + if (sgl->type == ((NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET)) { if (!nvme_is_write(cmd->req.cmd)) @@ -433,13 +446,15 @@ static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd) } cmd->req.transfer_len += len; - cmd->req.sg = sgl_alloc(len, GFP_KERNEL, &cmd->req.sg_cnt); + cmd->req.sg = sgl_alloc(len, GFP_KERNEL | __GFP_NOWARN, + &cmd->req.sg_cnt); if (!cmd->req.sg) return NVME_SC_INTERNAL; cmd->cur_sg = cmd->req.sg; if (nvmet_tcp_has_data_in(cmd)) { - cmd->iov = kmalloc_objs(*cmd->iov, cmd->req.sg_cnt); + cmd->iov = kmalloc_objs(*cmd->iov, cmd->req.sg_cnt, + GFP_KERNEL | __GFP_NOWARN); if (!cmd->iov) goto err; } diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c index f00921931eb6..23a17c02abee 100644 --- a/drivers/nvme/target/zns.c +++ b/drivers/nvme/target/zns.c @@ -116,7 +116,7 @@ void nvmet_execute_identify_ns_zns(struct nvmet_req *req) mutex_unlock(&req->ns->subsys->lock); } - if (!bdev_is_zoned(req->ns->bdev)) { + if (!req->ns->bdev || !bdev_is_zoned(req->ns->bdev)) { status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; req->error_loc = offsetof(struct nvme_identify, nsid); goto out; @@ -295,11 +295,18 @@ static void nvmet_bdev_zone_zmgmt_recv_work(struct work_struct *w) } /* - * When partial bit is set nr_zones must indicate the number of zone - * descriptors actually transferred. + * Partial report (PR bit set): the host accepts an incomplete listing, + * so cap Number of Zones to the descriptors that fit in the buffer. + * Full report (PR bit clear): Number of Zones is the match count; fail + * if the buffer cannot hold every matching zone descriptor. */ - if (req->cmd->zmr.pr) + if (req->cmd->zmr.pr) { rz_data.nr_zones = min(rz_data.nr_zones, rz_data.out_nr_zones); + } else if (rz_data.nr_zones > rz_data.out_nr_zones) { + req->error_loc = offsetof(struct nvme_zone_mgmt_recv_cmd, numd); + status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + goto out; + } nr_zones = cpu_to_le64(rz_data.nr_zones); status = nvmet_copy_to_sgl(req, 0, &nr_zones, sizeof(nr_zones)); diff --git a/include/linux/list.h b/include/linux/list.h index 09d979976b3b..19212bfc3f6d 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -34,6 +34,14 @@ struct list_head name = LIST_HEAD_INIT(name) /** + * LIST_HEAD_GUARDED - define a &struct list_head annotated with __guarded_by() + * @name: name of the list_head + * @lock: lock protecting the list + */ +#define LIST_HEAD_GUARDED(name, lock) \ + __guarded_by(&(lock)) LIST_HEAD(name) + +/** * INIT_LIST_HEAD - Initialize a list_head structure * @list: list_head structure to be initialized. * @@ -436,6 +444,7 @@ static inline void list_del_init_careful(struct list_head *entry) * if another CPU could re-list_add() it. */ static inline int list_empty_careful(const struct list_head *head) + __context_unsafe(/* intentional lockless access to @head */) { struct list_head *next = smp_load_acquire(&head->next); return list_is_head(next, head) && (next == READ_ONCE(head->prev)); diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 041f30931a90..91ce434a7e8d 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -2272,14 +2272,14 @@ struct nvme_completion { #define NVME_TERTIARY(ver) ((ver) & 0xff) enum { - NVME_AEN_RESV_LOG_PAGE_AVALIABLE = 0x00, + NVME_AEN_RESV_LOG_PAGE_AVAILABLE = 0x00, }; enum { NVME_PR_LOG_EMPTY_LOG_PAGE = 0x00, NVME_PR_LOG_REGISTRATION_PREEMPTED = 0x01, NVME_PR_LOG_RESERVATION_RELEASED = 0x02, - NVME_PR_LOG_RESERVATOIN_PREEMPTED = 0x03, + NVME_PR_LOG_RESERVATION_PREEMPTED = 0x03, }; enum { |