在鲲鹏916服务器上编译和安装dpdk踩坑

 无法爬楼楼的可以从这里下：

http://static.dpdk.org/rel/

[root@kunpeng82 data1]#  cat  /etc/default/grub |grep GRUB_CMDLINE_LINUX
GRUB_CMDLINE_LINUX="crashkernel=auto"
GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=ttyAMA0,115200 no_timer_check nofb nomodeset vga=normal"
[root@kunpeng82 data1]# mkdir /mnt/huge
[root@kunpeng82 data1]# mount -t hugetlbfs nodev /mnt/huge
[root@kunpeng82 data1]# grep Hugepagesize /proc/meminfo
Hugepagesize:     524288 kB

[root@kunpeng82 devuser]# ls /sys/kernel/mm/hugepages/
hugepages-2048kB  hugepages-524288kB
[root@kunpeng82 devuser]#

[root@kunpeng82 devuser]# ls /sys/kernel/mm/hugepages/hugepages-524288kB/
free_hugepages nr_hugepages nr_hugepages_mempolicy nr_overcommit_hugepages resv_hugepages surplus_hugepages
[root@kunpeng82 devuser]# cat /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
0
[root@kunpeng82 devuser]#

[root@kunpeng82 devuser]# echo 256 > /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
[root@kunpeng82 devuser]# cat /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
256
[root@kunpeng82 devuser]#

[root@kunpeng82 devuser]# cat /proc/meminfo |grep HugePages_
HugePages_Total: 256
HugePages_Free: 256
HugePages_Rsvd: 0
HugePages_Surp: 0
[root@kunpeng82 devuser]#

配置DPDK编译目标

DPDK支持多种平台（x86、x64、ARM），编译的目标由ARCH-MACHINE-EXECENV-TOOLCHAIN给出

ARCH can be: i686, x86_64, ppc_64, arm64
MACHINE can be: native, power8, armv8a
EXECENV can be: linuxapp,bsdapp
TOOLCHAIN can be: gcc, icc

[root@kunpeng82 data1]# lscpu
Architecture:          aarch64
Byte Order:            Little Endian
CPU(s):                64
On-line CPU(s) list:   0-63
Thread(s) per core:    1
Core(s) per socket:    32
Socket(s):             2
NUMA node(s):          4
Model:                 2
BogoMIPS:              100.00
L1d cache:             32K
L1i cache:             48K
L2 cache:              1024K
L3 cache:              16384K
NUMA node0 CPU(s):     0-15
NUMA node1 CPU(s):     16-31
NUMA node2 CPU(s):     32-47
NUMA node3 CPU(s):     48-63
Flags:                 fp asimd evtstrm aes pmull sha1 sha2 crc32 cpuid

export RTE_TARGET=arm64-armv8a-linuxapp-gcc
export RTE_SDK=/data1/dpdk_sdk

make config T=arm64-armv8a-linuxapp-gcc
make -j 64
make install T=arm64-armv8a-linuxapp-gcc  -j 64

[root@kunpeng82 dpdk-19.11]# make
== Build lib
== Build lib/librte_kvargs
  SYMLINK-FILE include/rte_kvargs.h
  CC rte_kvargs.o
  AR librte_kvargs.a
  INSTALL-LIB librte_kvargs.a
== Build lib/librte_eal
== Build lib/librte_eal/common
  SYMLINK-FILE include/rte_branch_prediction.h
  SYMLINK-FILE include/rte_common.h
  SYMLINK-FILE include/rte_compat.h
  SYMLINK-FILE include/rte_function_versioning.h
  SYMLINK-FILE include/rte_debug.h
  SYMLINK-FILE include/rte_eal.h
  SYMLINK-FILE include/rte_eal_interrupts.h
  SYMLINK-FILE include/rte_errno.h
  SYMLINK-FILE include/rte_launch.h
  SYMLINK-FILE include/rte_lcore.h
  SYMLINK-FILE include/rte_log.h
  SYMLINK-FILE include/rte_memory.h
  SYMLINK-FILE include/rte_memzone.h
  SYMLINK-FILE include/rte_per_lcore.h
  SYMLINK-FILE include/rte_random.h
  SYMLINK-FILE include/rte_tailq.h
  SYMLINK-FILE include/rte_interrupts.h
  SYMLINK-FILE include/rte_alarm.h
  SYMLINK-FILE include/rte_string_fns.h
  SYMLINK-FILE include/rte_version.h
  SYMLINK-FILE include/rte_eal_memconfig.h
  SYMLINK-FILE include/rte_hexdump.h
  SYMLINK-FILE include/rte_devargs.h
  SYMLINK-FILE include/rte_bus.h
  SYMLINK-FILE include/rte_dev.h
  SYMLINK-FILE include/rte_class.h
  SYMLINK-FILE include/rte_option.h
  SYMLINK-FILE include/rte_pci_dev_feature_defs.h
  SYMLINK-FILE include/rte_pci_dev_features.h
  SYMLINK-FILE include/rte_malloc.h
  SYMLINK-FILE include/rte_keepalive.h
  SYMLINK-FILE include/rte_time.h
  SYMLINK-FILE include/rte_service.h
  SYMLINK-FILE include/rte_service_component.h
  SYMLINK-FILE include/rte_bitmap.h
  SYMLINK-FILE include/rte_vfio.h
  SYMLINK-FILE include/rte_hypervisor.h
  SYMLINK-FILE include/rte_test.h
  SYMLINK-FILE include/rte_reciprocal.h
  SYMLINK-FILE include/rte_fbarray.h
  SYMLINK-FILE include/rte_uuid.h
  SYMLINK-FILE include/rte_atomic.h
  SYMLINK-FILE include/rte_atomic_32.h
  SYMLINK-FILE include/rte_atomic_64.h
  SYMLINK-FILE include/rte_byteorder.h
  SYMLINK-FILE include/rte_cpuflags.h
  SYMLINK-FILE include/rte_cpuflags_32.h
  SYMLINK-FILE include/rte_cpuflags_64.h
  SYMLINK-FILE include/rte_cycles.h
  SYMLINK-FILE include/rte_cycles_32.h
  SYMLINK-FILE include/rte_cycles_64.h
  SYMLINK-FILE include/rte_io.h
  SYMLINK-FILE include/rte_io_64.h
  SYMLINK-FILE include/rte_mcslock.h
  SYMLINK-FILE include/rte_memcpy.h
  SYMLINK-FILE include/rte_memcpy_32.h
  SYMLINK-FILE include/rte_memcpy_64.h
  SYMLINK-FILE include/rte_pause.h
  SYMLINK-FILE include/rte_pause_32.h
  SYMLINK-FILE include/rte_pause_64.h
  SYMLINK-FILE include/rte_prefetch.h
  SYMLINK-FILE include/rte_prefetch_32.h
  SYMLINK-FILE include/rte_prefetch_64.h
  SYMLINK-FILE include/rte_rwlock.h
  SYMLINK-FILE include/rte_spinlock.h
  SYMLINK-FILE include/rte_ticketlock.h
  SYMLINK-FILE include/rte_vect.h
  SYMLINK-FILE include/generic/rte_atomic.h
  SYMLINK-FILE include/generic/rte_byteorder.h
  SYMLINK-FILE include/generic/rte_cycles.h
  SYMLINK-FILE include/generic/rte_prefetch.h
  SYMLINK-FILE include/generic/rte_memcpy.h
  SYMLINK-FILE include/generic/rte_cpuflags.h
  SYMLINK-FILE include/generic/rte_mcslock.h
  SYMLINK-FILE include/generic/rte_spinlock.h
  SYMLINK-FILE include/generic/rte_rwlock.h
  SYMLINK-FILE include/generic/rte_ticketlock.h
  SYMLINK-FILE include/generic/rte_vect.h
  SYMLINK-FILE include/generic/rte_pause.h
  SYMLINK-FILE include/generic/rte_io.h
== Build lib/librte_eal/linux
== Build lib/librte_eal/linux/eal
  SYMLINK-FILE include/rte_kni_common.h
  SYMLINK-FILE include/rte_os.h
  CC eal.o
  CC eal_cpuflags.o
  CC eal_hugepage_info.o
  CC eal_memory.o
/data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_memory.c:32:18: fatal error: numa.h: No such file or directory
 #include <numa.h>
                  ^
compilation terminated.
make[5]: *** [eal_memory.o] Error 1
make[4]: *** [eal] Error 2
make[3]: *** [linux] Error 2
make[2]: *** [librte_eal] Error 2
make[1]: *** [lib] Error 2
make: *** [all] Error 2
[root@kunpeng82 dpdk-19.11]# 

dpdk编译错误 numa.h

libnuma-dev的安装：yum install numactl libnuma-dev

安装
yum install numactl-devel*

make: *** /lib/modules/4.14.0-115.10.1.el7a.aarch64/build: No such file or directory.  Stop

[root@kunpeng82 dpdk-19.11]# ls /lib/modules/4.14.0-115.10.1.el7a.aarch64/build 
/lib/modules/4.14.0-115.10.1.el7a.aarch64/build

 查看链接状态，执行以上指令，可以看到build链接到了一个不存在的路径

[root@kunpeng82 dpdk-19.11]# rm -rfv  /lib/modules/4.14.0-115.10.1.el7a.aarch64/build 
removed ‘/lib/modules/4.14.0-115.10.1.el7a.aarch64/build’

[root@kunpeng82 dpdk-19.11]# ls /usr/src/kernels/
4.18.0-147.0.3.el7.aarch64  4.18.0-147.0.3.el7.aarch64+debug
[root@kunpeng82 dpdk-19.11]# ls /usr/src/kernels/4.18.0-147.0.3.el7.aarch64
arch   crypto    fs       ipc      lib               mm              samples   sound       usr
block  drivers   include  Kconfig  Makefile          Module.symvers  scripts   System.map  virt
certs  firmware  init     kernel   Makefile.rhelver  net             security  tools       vmlinux.id
[root@kunpeng82 dpdk-19.11]# ln -s /usr/src/kernels/4.18.0-147.0.3.el7.aarch64  /lib/modules/4.14.0-115.10.1.el7a.aarch64/build
[root@kunpeng82 dpdk-19.11]# 

编译DPDK的时候可以不用使用系统环境的编译
export RTE_KERNELDIR=$PWD

[root@kunpeng82 dpdk-19.11]# yum install kernel-devel*
Loaded plugins: fastestmirror
Repository base is listed more than once in the configuration
Repository updates is listed more than once in the configuration
Repository extras is listed more than once in the configuration
Repository centosplus is listed more than once in the configuration
Loading mirror speeds from cached hostfile
Package kernel-devel-4.18.0-147.0.3.el7.aarch64 already installed and latest version
Nothing to do
[root@kunpeng82 dpdk-19.11]# yum install kernel-headers*
Loaded plugins: fastestmirror
Repository base is listed more than once in the configuration
Repository updates is listed more than once in the configuration
Repository extras is listed more than once in the configuration
Repository centosplus is listed more than once in the configuration
Loading mirror speeds from cached hostfile
Package kernel-headers-4.18.0-147.0.3.el7.aarch64 already installed and latest version
Nothing to do
[root@kunpeng82 dpdk-19.11]#

[root@kunpeng82 dpdk-19.11]# modprobe vfio-pci
[root@kunpeng82 dpdk-19.11]# modprobe vfio
[root@kunpeng82 dpdk-19.11]# ls

一个设备都没有检测到
[root@kunpeng82 dpdk-19.11]# ./usertools/dpdk-devbind.py  -s

No 'Network' devices detected
=============================

No 'Baseband' devices detected
==============================

No 'Crypto' devices detected
============================

No 'Eventdev' devices detected
==============================

No 'Mempool' devices detected
=============================

No 'Compress' devices detected
==============================

No 'Misc (rawdev)' devices detected
===================================
[root@kunpeng82 dpdk-19.11]# 

 总结：

    鲲鹏916上网卡不支持dpdk

 安装依赖包（如果已安装，可以忽略）

安装依赖包（如果已安装，可以忽略）

yum install kernel-devel

yum install numactl-devel

配置环境变量：

export RTE_SDK=/home/dpdk-19.11 (以当前环境DPDK源码实际路径为准)

export RTE_TARGET= arm64-armv8a-linuxapp-gcc (以当前环境CPU架构为准)

在当前DPDK源码目录下编译DPDK：

    make install T=$RTE_TARGET -j 48

 =====================================================

[root@localhost dpdk-19.11]# mkdir /mnt/huge

[root@localhost dpdk-19.11]# mount -t hugetlbfs nodev /mnt/huge
[root@localhost dpdk-19.11]#
[root@localhost dpdk-19.11]# ls /sys/kernel/mm/hugepages/
hugepages-2048kB hugepages-524288kB
[root@localhost dpdk-19.11]# grep Hugepagesize /proc/meminfo
Hugepagesize: 524288 kB
[root@localhost dpdk-19.11]

cat /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
echo 256 /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages

[root@localhost dpdk-19.11]# export RTE_TARGET= arm64-armv8a-linuxapp-gcc
-bash: export: `arm64-armv8a-linuxapp-gcc': not a valid identifier
[root@localhost dpdk-19.11]# export RTE_TARGET=arm64-armv8a-linuxapp-gcc
[root@localhost dpdk-19.11]#
yum install -y *gcc* *glibc* *stdc* *gfortran* m4 numactl* automake
[root@localhost dpdk-19.11]# ls /lib/modules/4.14.0-115.el7a.0.1.aarch64/build/
ls: cannot access /lib/modules/4.14.0-115.el7a.0.1.aarch64/build/: No such file or directory
 

[root@localhost data1]# ls
CentOS-7-aarch64-Everything-1810.iso  dpdk-19.11.kunpeng.tgz  lost+found
dpdk-19.11                            hicn-master.zip         Python-3.7.7.tgz
[root@localhost data1]# mkdir -p /mnt/CentOS-7.6
[root@localhost data1]# mount -o loop  CentOS-7-aarch64-Everything-1810.iso  /mnt/CentOS-7.6
mount: /dev/loop0 is write-protected, mounting read-only
[root@localhost data1]# vi /etc/yum.
yum.conf     yum.repos.d/ 
[root@localhost data1]# vi /etc/yum.repos.d/local.repo 
[local]
name=local.repo
baseurl=file:///mnt/CentOS-7.6
enabled=1
gpgcheck=0

备份yum.repos.d下的的其他repo

[root@localhost data1]# yum makecache
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
local                                                               | 3.6 kB  00:00:00     
Metadata Cache Created
[root@localhost data1]# yum list | grep kernel-dev
kernel-devel.aarch64                        4.14.0-115.el7a.0.1        local    
[root@localhost data1]# 

[root@localhost dpdk-19.11]# ls /lib/modules/4.14.0-115.el7a.0.1.aarch64/build/ 
arch    drivers   init     lib             net       sound       virt
block   firmware  ipc      Makefile        samples   System.map  vmlinux.id
certs   fs        Kconfig  mm              scripts   tools
crypto  include   kernel   Module.symvers  security  usr
[root@localhost dpdk-19.11]#