ceph nvme-of

ceph

server:

dingo-dongwei-1,dingo-dongwei-2,dingo-dongwei-3

client:

ubuntu1,ubuntu2,ubuntu3

conceph

NQN

stands for NVMe Qualified Name.It’s a unique identifier string used in NVMe-over-Fabrics (NVMe-oF).
NQN Format (defined by NVMe spec): nqn.yyyy-mm.domain:unique-name

config server

create pool

1
2
3
ceph osd pool create nvmeof_csi
rbd pool init nvmeof_csi
ceph orch apply nvmeof nvmeof_csi --placement="dingo-dongwei-1,dingo-dongwei-2"

create NVMe subsystem

1
2
3
4
5
6
7
8
9
10
11
# latest ceph
podman pull quay.io/ceph/nvmeof-cli:latest
# ceph version 18.2.5 reef (stable)
podman pull quay.io/ceph/nvmeof-cli:1.2.0

# podman run -it --rm quay.io/ceph/nvmeof-cli:latest --server-address GATEWAY_IP --server-port GATEWAY_PORT 5500 subsystem add --subsystem SUSYSTEM_NQN

podman run -it --rm quay.io/ceph/nvmeof-cli:latest \
  --server-address 10.225.10.50 \
  --server-port 5500 \
  subsystem add --subsystem nqn.2025-11.io.ceph:dongwei-nvme

add listener

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
# get the NVME-oF Gateway name:
ceph orch ps | grep nvme

# Define the IP port for the gateway(tell Ceph where to listen for NVMe-oF connections)
# podman run -it --rm quay.io/ceph/nvmeof-cli:latest --server-address GATEWAY_IP --server-port GATEWAY_PORT 5500 listener add --subsystem SUBSYSTEM_NQN --host-name HOST_NAME --traddr GATEWAY_IP --trsvcid 4420

# version > reef
podman run -it --rm quay.io/ceph/nvmeof-cli:latest \
  --server-address 10.225.10.50 \
  --server-port 5500 \
  listener add \
  --subsystem nqn.2025-11.io.ceph:dongwei-nvme \
  --host-name dingo-dongwei-1 --traddr 10.225.10.50 \
  --trsvcid 4420

# version = reff
podman run -it quay.io/ceph/nvmeof-cli:1.1.0 \
  --server-address 10.225.10.50 \
  --server-port 5500 \
  listener add \
  --subsystem nqn.2025-11.io.ceph:dongwei-nvme \
  --gateway-name client.nvmeof.nvmeof_csi.dingo-dongwei-1.ligmtr \
  --traddr 10.225.10.50 \
  --trsvcid 4420

# list listener
podman run -it --rm quay.io/ceph/nvmeof-cli:latest   --server-address 10.225.10.50   --server-port 5500   listener list --subsystem nqn.2025-11.io.ceph:dongwei-nvme

config client

(optional) init NQN on client

1
2
3
sudo nvme gen-hostnqn | sudo tee /etc/nvme/hostnqn

cat /etc/nvme/hostnqn  # nqn.2014-08.org.nvmexpress:uuid:40bf4b78-4229-4d44-81b0-9bee0d1604fe

(on server) add client host access permissions

Allow the initiator host to connect to the newly-created NVMe subsystem:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
podman run -it quay.io/ceph/nvmeof-cli:latest \
--server-address 10.225.10.50 \
--server-port 5500 \
host add --subsystem nqn.2025-11.io.ceph:dongwei-nvme --host "nqn.2014-08.org.nvmexpress:uuid:40bf4b78-4229-4d44-81b0-9bee0d1604fe"

podman run -it quay.io/ceph/nvmeof-cli:latest \
--server-address 10.225.10.50 \
--server-port 5500 \
host add --subsystem nqn.2025-11.io.ceph:dongwei-nvme --host "nqn.2014-08.org.nvmexpress:uuid:74faf23a-5c3e-43d8-87a8-d2247a0593c7"

podman run -it quay.io/ceph/nvmeof-cli:latest \
--server-address 10.225.10.50 \
--server-port 5500 \
host add --subsystem nqn.2025-11.io.ceph:dongwei-nvme --host "nqn.2014-08.org.nvmexpress:uuid:a8c13096-b83a-4350-a98e-355a77c87f7b"

list client

1
podman run -it --rm quay.io/ceph/nvmeof-cli:latest   --server-address 10.225.10.50   --server-port 5500  host list --subsystem nqn.2025-11.io.ceph:dongwei-nvme

create the RBD image

In NVMe terms:
• A subsystem is like a container (it groups one or more namespaces).
• A namespace is the actual block device (e.g. /dev/nvme0n1) that the client will see.
• Each namespace typically maps to an RBD image in Ceph.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# Create the RBD image
rbd create nvmeof_csi/dongwei-rbd1 --size 10G

# Register it as an NVMe namespace
podman run -it --rm quay.io/ceph/nvmeof-cli:latest \
  --server-address 10.225.10.50 \
  --server-port 5500 \
  namespace add \
  --subsystem nqn.2025-11.io.ceph:dongwei-nvme \
  --rbd-pool nvmeof_csi \
  --rbd-image dongwei-rbd1

# verify
podman run -it --rm quay.io/ceph/nvmeof-cli:latest \
  --server-address 10.225.10.50 \
  --server-port 5500 \
  namespace list --subsystem nqn.2025-11.io.ceph:dongwei-nvme

config client initiator

on client machine

prepare

1
2
3
4
yum install nvme-cli
modprobe nvme-fabrics
# (deprecated) verify connect
# nvme discover -t tcp -a 10.220.69.130 -s 4420

connect

1
nvme connect -t tcp -a 10.220.69.130 -s 4420 -n nqn.2025-11.io.ceph:dongwei-nvme