Hi，大家好，我是编程小6，很荣幸遇见你，我把这些年在开发过程中遇到的问题或想法写出来，今天说一说SPDK 入门,希望能够帮助你!!!。

SPDK installation and Hardware check

• Compile SPDK

  $ cd spdk_repo/spdk
  $ git submodule update --init
  $ ./configure --enable-debug --enable-lto --with-shared  
  $ make

• Verify we have an NVMe device

$ lspci | grep "Non-Volatile"
  00:0e.0 Non-Volatile memory controller: InnoTek Systemberatung GmbH Device 4e56

• Run the hello_world example script

 $ sudo scripts/setup.sh
 $ sudo scripts/gen_nvme.sh --json-with-subsystems > ./build/examples/hello_bdev.json
 $ sudo ./build/examples/hello_bdev --json ./build/examples/hello_bdev.json -b Nvme0n1

Run SPDK as unroot user

Enable VFIO

• Enable VFIO in kernel

CONFIG_VFIO_IOMMU_TYPE1=y
CONFIG_VFIO_VIRQFD=y
CONFIG_VFIO=y
CONFIG_VFIO_NOIOMMU=y
CONFIG_VFIO_PCI=y
CONFIG_VFIO_PCI_VGA=y
CONFIG_VFIO_PCI_MMAP=y
CONFIG_VFIO_PCI_INTX=y
CONFIG_VFIO_PCI_IGD=y
CONFIG_VFIO_MDEV=m
CONFIG_VFIO_MDEV_DEVICE=m

• Since the VFIO module is included in the kernel from kernel 5.4.26, it is necessary to specify it with the kernel parameter of grub

/ etc / default / grub
GRUB_CMDLINE_LINUX = “amd_iommu = on iommu = pt pcie_aspm = off vfio_iommu_type1.allow_unsafe_interrupts = 1 vfio_pci.disable_vga = 1 vfio_pci.disable_idle_d3 = 1”

Run examples

Blobstore

• hello_blob
• cli

sudo scripts/setup.sh
sudo scripts/gen_nvme.sh --json-with-subsystems > ./build/examples/hello_bdev.json
sudo LD_LIBRARY_PATH=./build/lib/:./dpdk/build/lib/ ./build/examples/blobcli -j ./build/examples/hello_bdev.json -b Nvme0n1  -S

NVME perf

Example: Using perf for 4K 100% Random Read workload to a local NVMe SSD for 300 seconds

perf -q 128 -o 4096 -w randread -r 'trtype:PCIe traddr:0000:04:00.0' -t 300

Example: Using perf for 4K 70/30 Random Read/Write mix workload to all local NVMe SSDs for 300 seconds

perf -q 128 -o 4096 -w randrw -M 70 -t 300

How to link spdk to an APP

• Build spdk wiith share library
• Use Cmake as following:

cmake_minimum_required(VERSION 3.16)
project(mockapp C)


include(option.txt)

if (ENABLE_BASELINE)
    add_definitions(-DBASELINE)
endif(ENABLE_EVICTOR_TRACE)

set(CMAKE_C_STANDARD 11)
set(CMAKE_C_STANDARD_REQUIRED ON)
#-Wall -Werror 
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-parameter -Wextra -Wno-missing-declarations")

SET (CMAKE_VERBOSE_MAKEFILE 1)
# set(SPDK_ROOT $ENV{ 
     HOME}/spdk)
# set(DPDK_ROOT /home/yaoxin/spdk/dpdk)
set(SPDK_ROOT $ENV{ 
   HOME}/spdk)
set(DPDK_ROOT $ENV{ 
   HOME}/spdk/dpdk)

# set(ENV{ 
     PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:~/spdk/dpdk/build/lib/pkgconfig/ ~/spdk/build/lib/pkgconfig/")
# # find_package(PkgConfig REQUIRED)
# # pkg_check_modules(dpdk REQUIRED IMPORTED_TARGET libdpdk)
add_subdirectory(ring_jiawei)
add_subdirectory(common)

add_library(swap_backend STATIC
                swap_backend_baseline.c
                swap_backend_blobfs.c
                swap_backend.c swap_backend.h swap.h
                swap_backend_ring-bdev.c swap_backend_ring-bdev.h)




target_include_directories(swap_backend PUBLIC ${ 
   CMAKE_CURRENT_SOURCE_DIR}/ring_jiawei ${ 
   CMAKE_CURRENT_SOURCE_DIR}/common ../spdk/build/include/ ../spdk/dpdk/build/include/)



set(DPDKLIBS
##-Wl,--as-needed
        #-Wl,--whole-archive
# dpdk
#-Wl,--no-as-needed
        rte_bus_pci.so
        rte_bus_vdev.so
        rte_cmdline.so
        rte_compressdev.so
        rte_cryptodev.so
        rte_eal.so
        rte_ethdev.so
        rte_hash.so
        rte_kvargs.so
        rte_mbuf.so
        rte_mempool_ring.so
        rte_mempool.so
        rte_meter.so
        rte_net.so
        rte_pci.so
        rte_power.so
        rte_rcu.so
        rte_reorder.so
        rte_ring.so
        rte_security.so
        rte_telemetry.so
        rte_timer.so
        rte_vhost.so)

set(SPDKLIBS
        -Wl,--no-as-needed
        spdk
        spdk_env_dpdk
)
set(SYSLIBS
        #-Wl,--no-whole-archive
        -Wl,--as-needed
        pthread
        rt
        uuid
        crypto
        m  
        aio
        numa
        dl
        fdt)

link_directories(${ 
   PROJECT_SOURCE_DIR}/ring_jiawei ${ 
   PROJECT_SOURCE_DIR}/common ${ 
   SPDK_ROOT}/build/lib ${ 
   DPDK_ROOT}/build/lib )
target_link_libraries(backend
                        ${ 
   SPDKLIBS}
                        ${ 
   DPDKLIBS}
                        #PkgConfig::dpdk ${ 
     CMAKE_THREAD_LIBS_INIT}
                        ${ 
   SYSLIBS}
                        )




add_executable(app app.c)


target_link_libraries(app backend)

第二种方法

find_package(PkgConfig REQUIRED)

set(ENV{ 
   PKG_CONFIG_PATH} ${ 
   SPDK_ROOT_DIR}/build/lib/pkgconfig)
pkg_check_modules(SPDK_NVME REQUIRED spdk_bdev)
pkg_check_modules(SPDK_ENV_DPDK REQUIRED spdk_env_dpdk)

add_executable(app app.cpp)
target_link_libraries(app PRIVATE common log pthread)

target_link_libraries(app PRIVATE ${ 
   SPDK_NVME_LINK_LIBRARIES} ${ 
   SPDK_ENV_DPDK_LINK_LIBRARIES})
target_include_directories(app PRIVATE ${ 
   SPDK_NVME_INCLUDE_DIRS} ${ 
   SPDK_ENV_DPDK_INCLUDE_DIRS})
target_compile_options(app PRIVATE ${ 
   SPDK_NVME_CFLAGS_OTHER} ${ 
   SPDK_ENV_DPDK_CFLAGS_OTHER})

今天的分享到此就结束了，感谢您的阅读，如果确实帮到您，您可以动动手指转发给其他人。