https://www.yuque.com/zzqcn/opensource/oirzxh
https://promisechen.github.io/dpdk/zp/mbuf.html
https://promisechen.github.io/dpdk/zp/mbuf.html
#include <stdio.h> #include <stdlib.h> #include <string.h> #include <stdint.h> #include <errno.h> #include <sys/queue.h> #include <rte_launch.h> #include <rte_eal.h> #include <rte_per_lcore.h> #include <rte_lcore.h> #include <rte_debug.h> #include <rte_mempool.h> #include <rte_mbuf.h> #include <rte_ether.h> #define RAW_MEMPOOL #define MBUF_COUNT (1024-1) #define PRIV_SIZE 16 // ETH_MAX_len = 1518 // ETH_MTU = ETH_MAX_LEN - ETH_HDR_LEN - ETHER_CRC_LEN = 1518 - 14 - 4 = 1500 #define MBUF_DATAROOM_SIZE (RTE_PKTMBUF_HEADROOM + RTE_ETHER_MAX_LEN) #define MBUF_SIZE (sizeof(struct rte_mbuf) + PRIV_SIZE + MBUF_DATAROOM_SIZE) #define CACHE_SIZE 32 struct rte_mempool* mpool; struct rte_mbuf *m = NULL; static void mbuf_dump(struct rte_mbuf* m) { printf("RTE_PKTMBUF_HEADROOM: %u ", RTE_PKTMBUF_HEADROOM); printf("sizeof(mbuf): %lu ", sizeof(struct rte_mbuf)); printf("m: %p ", m); printf("m->refcnt: %u ", m->refcnt); printf("m->buf_addr: %p ", m->buf_addr); printf("m->data_off: %u ", m->data_off); printf("m->buf_len: %u ", m->buf_len); printf("m->pkt_len: %u ", m->pkt_len); printf("m->data_len: %u ", m->data_len); printf("m->nb_segs: %u ", m->nb_segs); printf("m->next: %p ", m->next); printf("m->buf_addr+m->data_off: %p ", (char*)m->buf_addr+m->data_off); printf("rte_pktmbuf_mtod(m): %p ", rte_pktmbuf_mtod(m, char*)); printf("rte_pktmbuf_data_len(m): %u ", rte_pktmbuf_data_len(m)); printf("rte_pktmbuf_pkt_len(m): %u ", rte_pktmbuf_pkt_len(m)); printf("rte_pktmbuf_headroom(m): %u ", rte_pktmbuf_headroom(m)); printf("rte_pktmbuf_tailroom(m): %u ", rte_pktmbuf_tailroom(m)); printf("rte_pktmbuf_data_room_size(mpool): %u ", rte_pktmbuf_data_room_size(m->pool)); printf("rte_pktmbuf_priv_size(mpool): %u ", rte_pktmbuf_priv_size(m->pool)); } static int mpool_init(void) { struct rte_mempool_objsz objsz; uint32_t mbuf_size; #ifdef RAW_MEMPOOL struct rte_pktmbuf_pool_private priv; priv.mbuf_data_room_size = MBUF_DATAROOM_SIZE; priv.mbuf_priv_size = PRIV_SIZE; mpool = rte_mempool_create("test_pool", MBUF_COUNT, MBUF_SIZE, CACHE_SIZE, sizeof(struct rte_pktmbuf_pool_private), rte_pktmbuf_pool_init, &priv, rte_pktmbuf_init, NULL, SOCKET_ID_ANY, MEMPOOL_F_SC_GET); #else mpool = rte_pktmbuf_pool_create("test_pool", MBUF_COUNT, CACHE_SIZE, PRIV_SIZE, MBUF_DATAROOM_SIZE, SOCKET_ID_ANY); #endif if(NULL == mpool) return -1; mbuf_size = rte_mempool_calc_obj_size(MBUF_SIZE, 0, &objsz); printf("mbuf_size: %u ", mbuf_size); printf("elt_size: %u, header_size: %u, trailer_size: %u, total_size: %u ", objsz.elt_size, objsz.header_size, objsz.trailer_size, objsz.total_size); return 0; } int lcore_hello() { struct rte_mbuf *m2; printf("mempool count : %u ", rte_mempool_avail_count(mpool)); unsigned lcore_id = rte_lcore_id(); printf("Starting core %u ", lcore_id); if(1 == lcore_id) { m = rte_pktmbuf_alloc(mpool); mbuf_dump(m); rte_pktmbuf_append(m, 1000); mbuf_dump(m); m2 = rte_pktmbuf_alloc(mpool); rte_pktmbuf_append(m2, 500); mbuf_dump(m2); rte_pktmbuf_chain(m, m2); mbuf_dump(m); printf("mempool count before free : %u ", rte_mempool_avail_count(mpool)); rte_pktmbuf_free(m); printf("mempool count after free : %u ", rte_mempool_avail_count(mpool)); } else { } return 0; } int main(int argc, char **argv) { int ret; unsigned lcore_id; ret = rte_eal_init(argc, argv); if (ret < 0) rte_panic("Cannot init EAL "); mpool_init(); /* call on every slave lcore */ RTE_LCORE_FOREACH_SLAVE(lcore_id) { rte_eal_remote_launch(lcore_hello, NULL, lcore_id); } rte_eal_mp_wait_lcore(); return 0; }
INSTALL-MAP helloworld.map [root@localhost mbuf]# build/app/helloworld -c 0x3 -n 4 EAL: Detected 128 lcore(s) EAL: Detected 4 NUMA nodes EAL: Multi-process socket /var/run/dpdk/rte/mp_socket EAL: Selected IOVA mode 'PA' EAL: No available hugepages reported in hugepages-2048kB EAL: Probing VFIO support... EAL: VFIO support initialized EAL: PCI device 0000:05:00.0 on NUMA socket 0 EAL: probe driver: 19e5:200 net_hinic EAL: using IOMMU type 1 (Type 1) net_hinic: Initializing pf hinic-0000:05:00.0 in primary process net_hinic: Device 0000:05:00.0 hwif attribute: net_hinic: func_idx:0, p2p_idx:0, pciintf_idx:0, vf_in_pf:0, ppf_idx:0, global_vf_id:15, func_type:2 net_hinic: num_aeqs:4, num_ceqs:4, num_irqs:32, dma_attr:2 net_hinic: Get public resource capability: net_hinic: host_id: 0x0, ep_id: 0x0, intr_type: 0x0, max_cos_id: 0x7, er_id: 0x0, port_id: 0x0 net_hinic: host_total_function: 0xf2, host_oq_id_mask_val: 0x8, max_vf: 0x78 net_hinic: pf_num: 0x2, pf_id_start: 0x0, vf_num: 0xf0, vf_id_start: 0x10 net_hinic: Get share resource capability: net_hinic: host_pctxs: 0x0, host_cctxs: 0x0, host_scqs: 0x0, host_srqs: 0x0, host_mpts: 0x0 net_hinic: Get l2nic resource capability: net_hinic: max_sqs: 0x10, max_rqs: 0x10, vf_max_sqs: 0x4, vf_max_rqs: 0x4 net_hinic: Initialize 0000:05:00.0 in primary successfully EAL: PCI device 0000:06:00.0 on NUMA socket 0 EAL: probe driver: 19e5:200 net_hinic EAL: PCI device 0000:7d:00.0 on NUMA socket 0 EAL: probe driver: 19e5:a222 net_hns3 EAL: PCI device 0000:7d:00.1 on NUMA socket 0 EAL: probe driver: 19e5:a221 net_hns3 EAL: PCI device 0000:7d:00.2 on NUMA socket 0 EAL: probe driver: 19e5:a222 net_hns3 EAL: PCI device 0000:7d:00.3 on NUMA socket 0 EAL: probe driver: 19e5:a221 net_hns3 mbuf_size: 2176 elt_size: 1792, header_size: 128, trailer_size: 256, total_size: 2176 mempool count : 1023 Starting core 1 RTE_PKTMBUF_HEADROOM: 128 sizeof(mbuf): 128 m: 0x13f8d3c00 m->refcnt: 1 m->buf_addr: 0x13f8d3c90 m->data_off: 128 m->buf_len: 1646 m->pkt_len: 0 m->data_len: 0 m->nb_segs: 1 m->next: (nil) m->buf_addr+m->data_off: 0x13f8d3d10 rte_pktmbuf_mtod(m): 0x13f8d3d10 rte_pktmbuf_data_len(m): 0 rte_pktmbuf_pkt_len(m): 0 rte_pktmbuf_headroom(m): 128 rte_pktmbuf_tailroom(m): 1518 rte_pktmbuf_data_room_size(mpool): 1646 rte_pktmbuf_priv_size(mpool): 16 RTE_PKTMBUF_HEADROOM: 128 sizeof(mbuf): 128 m: 0x13f8d3c00 m->refcnt: 1 m->buf_addr: 0x13f8d3c90 m->data_off: 128 m->buf_len: 1646 m->pkt_len: 1000 m->data_len: 1000 m->nb_segs: 1 m->next: (nil) m->buf_addr+m->data_off: 0x13f8d3d10 rte_pktmbuf_mtod(m): 0x13f8d3d10 rte_pktmbuf_data_len(m): 1000 rte_pktmbuf_pkt_len(m): 1000 rte_pktmbuf_headroom(m): 128 rte_pktmbuf_tailroom(m): 518 rte_pktmbuf_data_room_size(mpool): 1646 rte_pktmbuf_priv_size(mpool): 16 RTE_PKTMBUF_HEADROOM: 128 sizeof(mbuf): 128 m: 0x13f8d3380 m->refcnt: 1 m->buf_addr: 0x13f8d3410 m->data_off: 128 m->buf_len: 1646 m->pkt_len: 500 m->data_len: 500 m->nb_segs: 1 m->next: (nil) m->buf_addr+m->data_off: 0x13f8d3490 rte_pktmbuf_mtod(m): 0x13f8d3490 rte_pktmbuf_data_len(m): 500 rte_pktmbuf_pkt_len(m): 500 rte_pktmbuf_headroom(m): 128 rte_pktmbuf_tailroom(m): 1018 rte_pktmbuf_data_room_size(mpool): 1646 rte_pktmbuf_priv_size(mpool): 16 RTE_PKTMBUF_HEADROOM: 128 sizeof(mbuf): 128 m: 0x13f8d3c00 m->refcnt: 1 m->buf_addr: 0x13f8d3c90 m->data_off: 128 m->buf_len: 1646 m->pkt_len: 1500 m->data_len: 1000 m->nb_segs: 2 m->next: 0x13f8d3380 m->buf_addr+m->data_off: 0x13f8d3d10 rte_pktmbuf_mtod(m): 0x13f8d3d10 rte_pktmbuf_data_len(m): 1000 rte_pktmbuf_pkt_len(m): 1500 rte_pktmbuf_headroom(m): 128 rte_pktmbuf_tailroom(m): 518 rte_pktmbuf_data_room_size(mpool): 1646 rte_pktmbuf_priv_size(mpool): 16 mempool count before free : 1021 mempool count after free : 1023
rte_pktmbuf_append
rte_pktmbuf_chain
rte_pktmbuf_clone
int lcore_hello() { struct rte_mbuf *m2,*m3; printf("mempool count : %u ", rte_mempool_avail_count(mpool)); unsigned lcore_id = rte_lcore_id(); printf("Starting core %u ", lcore_id); if(1 == lcore_id) { m = rte_pktmbuf_alloc(mpool); rte_pktmbuf_append(m, 1000); mbuf_dump(m); m2 = rte_pktmbuf_clone(m, mpool); mbuf_dump(m2); m3 = rte_pktmbuf_clone(m, mpool); mbuf_dump(m3); printf("mempool count before free: %u ", rte_mempool_avail_count(mpool)); printf("m->refcnt: %u ", m->refcnt); rte_pktmbuf_free(m); printf("mempool count after free: %u ", rte_mempool_avail_count(mpool)); printf("m->refcnt: %u ", m->refcnt); rte_pktmbuf_free(m3); printf("mempool count after free: %u ", rte_mempool_avail_count(mpool)); printf("m->refcnt: %u ", m->refcnt); rte_pktmbuf_free(m2); printf("mempool count after free: %u ", rte_mempool_avail_count(mpool)); printf("m->refcnt: %u ", m->refcnt); rte_pktmbuf_free(m); printf("mempool count after free: %u ", rte_mempool_avail_count(mpool)); printf("m->refcnt: %u ", m->refcnt); } else { } return 0; }
mempool count : 1023 Starting core 1 RTE_PKTMBUF_HEADROOM: 128 sizeof(mbuf): 128 m: 0x13f8d3c00 m->refcnt: 1 m->buf_addr: 0x13f8d3c90 m->data_off: 128 m->buf_len: 1646 m->pkt_len: 1000 m->data_len: 1000 m->nb_segs: 1 m->next: (nil) m->buf_addr+m->data_off: 0x13f8d3d10 rte_pktmbuf_mtod(m): 0x13f8d3d10 rte_pktmbuf_data_len(m): 1000 rte_pktmbuf_pkt_len(m): 1000 rte_pktmbuf_headroom(m): 128 rte_pktmbuf_tailroom(m): 518 rte_pktmbuf_data_room_size(mpool): 1646 rte_pktmbuf_priv_size(mpool): 16 RTE_PKTMBUF_HEADROOM: 128 sizeof(mbuf): 128 m: 0x13f8d3380 m->refcnt: 1 ---不是2 m->buf_addr: 0x13f8d3c90 m->data_off: 128 m->buf_len: 1646 m->pkt_len: 1000 m->data_len: 1000 m->nb_segs: 1 m->next: (nil) m->buf_addr+m->data_off: 0x13f8d3d10 rte_pktmbuf_mtod(m): 0x13f8d3d10 rte_pktmbuf_data_len(m): 1000 rte_pktmbuf_pkt_len(m): 1000 rte_pktmbuf_headroom(m): 128 rte_pktmbuf_tailroom(m): 518 rte_pktmbuf_data_room_size(mpool): 1646 rte_pktmbuf_priv_size(mpool): 16 RTE_PKTMBUF_HEADROOM: 128 sizeof(mbuf): 128 m: 0x13f8d2b00 m->refcnt: 1 --不是3
m->buf_addr: 0x13f8d3c90 m->data_off: 128 m->buf_len: 1646 m->pkt_len: 1000 m->data_len: 1000 m->nb_segs: 1 m->next: (nil) m->buf_addr+m->data_off: 0x13f8d3d10 rte_pktmbuf_mtod(m): 0x13f8d3d10 rte_pktmbuf_data_len(m): 1000 rte_pktmbuf_pkt_len(m): 1000 rte_pktmbuf_headroom(m): 128 rte_pktmbuf_tailroom(m): 518 rte_pktmbuf_data_room_size(mpool): 1646 rte_pktmbuf_priv_size(mpool): 16 mempool count before free: 1020 m->refcnt: 3 mempool count after free: 1020 --没有增加 m->refcnt: 2 mempool count after free: 1021 -----竟然增加了 m->refcnt: 1 mempool count after free: 1023 m->refcnt: 1 --不是0 MEMPOOL: obj=0x13f8d3c00, mempool=0x13fae2c00, cookie=f2eef2eedadd2e55 PANIC in rte_mempool_check_cookies(): MEMPOOL: bad header cookie (put) 7: [/lib64/libc.so.6(+0xdf690) [0xffff9762f690]] 6: [/lib64/libpthread.so.0(+0x7d38) [0xffff976e7d38]] 5: [build/app/helloworld(eal_thread_loop+0x324) [0x598fc8]] 4: [build/app/helloworld(lcore_hello+0xab8) [0x465dd8]] 3: [build/app/helloworld(rte_mempool_check_cookies+0xf4) [0x58b678]] 2: [build/app/helloworld(__rte_panic+0xcc) [0x5a0e1c]] 1: [build/app/helloworld(rte_dump_stack+0x1c) [0x5a0cc8]] Aborted
/* Creates a shallow copy of mbuf */ struct rte_mbuf * rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp) { struct rte_mbuf *mc, *mi, **prev; uint32_t pktlen; uint16_t nseg; mc = rte_pktmbuf_alloc(mp); if (unlikely(mc == NULL)) return NULL; mi = mc; prev = &mi->next; pktlen = md->pkt_len; nseg = 0; do { nseg++; rte_pktmbuf_attach(mi, md); *prev = mi; prev = &mi->next; } while ((md = md->next) != NULL && (mi = rte_pktmbuf_alloc(mp)) != NULL); *prev = NULL; mc->nb_segs = nseg; mc->pkt_len = pktlen; /* Allocation of new indirect segment failed */ if (unlikely(mi == NULL)) { rte_pktmbuf_free(mc); return NULL; } __rte_mbuf_sanity_check(mc, 1); return mc; }
static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m) { RTE_ASSERT(RTE_MBUF_DIRECT(mi) && rte_mbuf_refcnt_read(mi) == 1); if (RTE_MBUF_HAS_EXTBUF(m)) { rte_mbuf_ext_refcnt_update(m->shinfo, 1); mi->ol_flags = m->ol_flags; mi->shinfo = m->shinfo; } else { /* if m is not direct, get the mbuf that embeds the data */ rte_mbuf_refcnt_update(rte_mbuf_from_indirect(m), 1); mi->priv_size = m->priv_size; mi->ol_flags = m->ol_flags | IND_ATTACHED_MBUF; } __rte_pktmbuf_copy_hdr(mi, m); mi->data_off = m->data_off; mi->data_len = m->data_len; mi->buf_iova = m->buf_iova; mi->buf_addr = m->buf_addr; mi->buf_len = m->buf_len; mi->next = NULL; ------next等于null mi->pkt_len = mi->data_len; mi->nb_segs = 1; __rte_mbuf_sanity_check(mi, 1); __rte_mbuf_sanity_check(m, 0); }
double free
MEMPOOL: obj=0x13f8d3c00, mempool=0x13fae2c00, cookie=f2eef2eedadd2e55 PANIC in rte_mempool_check_cookies(): MEMPOOL: bad header cookie (put) 8: [/lib64/libc.so.6(+0xdf690) [0xffffbe55f690]] 7: [/lib64/libpthread.so.0(+0x7d38) [0xffffbe617d38]] 6: [/root/dpdk-19.11/examples/mbuf/build/app/helloworld(eal_thread_loop+0x324) [0x598ec8]] 5: [/root/dpdk-19.11/examples/mbuf/build/app/helloworld(lcore_hello+0x210) [0x466434]] 4: [/root/dpdk-19.11/examples/mbuf/build/app/helloworld() [0x465c2c]] 3: [/root/dpdk-19.11/examples/mbuf/build/app/helloworld(rte_mempool_check_cookies+0xf4) [0x58b578]] 2: [/root/dpdk-19.11/examples/mbuf/build/app/helloworld(__rte_panic+0xcc) [0x5a0d1c]] 1: [/root/dpdk-19.11/examples/mbuf/build/app/helloworld(rte_dump_stack+0x1c) [0x5a0bc8]] Program received signal SIGABRT, Aborted. [Switching to Thread 0xffffbd40d910 (LWP 128933)] 0x0000ffffbe4b5238 in raise () from /lib64/libc.so.6 Missing separate debuginfos, use: debuginfo-install glibc-2.17-324.el7_9.aarch64 libgcc-4.8.5-44.el7.aarch64 numactl-libs-2.0.12-5.el7.aarch64 (gdb) bt #0 0x0000ffffbe4b5238 in raise () from /lib64/libc.so.6 #1 0x0000ffffbe4b68b0 in abort () from /lib64/libc.so.6 #2 0x00000000005a0d24 in __rte_panic (funcname=0xbe5610 <__func__.7557> "rte_mempool_check_cookies", format=0xbe5290 "MEMPOOL: bad header cookie (put) %.0s") at /root/dpdk-19.11/lib/librte_eal/linux/eal/eal_debug.c:62 #3 0x000000000058b578 in rte_mempool_check_cookies (mp=0x13fae2c00, obj_table_const=0xffffbd40cec0, n=0, free=0) at /root/dpdk-19.11/lib/librte_mempool/rte_mempool.c:1036 #4 0x0000000000465c2c in rte_mempool_generic_put (cache=0x13faedd80, n=1, obj_table=0xffffbd40cec0, mp=0x13fae2c00) at /root/dpdk-19.11/arm64-armv8a-linuxapp-gcc/include/rte_mempool.h:1364 #5 rte_mempool_put_bulk (n=1, obj_table=0xffffbd40cec0, mp=0x13fae2c00) at /root/dpdk-19.11/arm64-armv8a-linuxapp-gcc/include/rte_mempool.h:1388 #6 rte_mempool_put (obj=0x13f8d3c00, mp=0x13fae2c00) at /root/dpdk-19.11/arm64-armv8a-linuxapp-gcc/include/rte_mempool.h:1406 #7 rte_mbuf_raw_free (m=0x13f8d3c00) at /root/dpdk-19.11/arm64-armv8a-linuxapp-gcc/include/rte_mbuf.h:579 #8 rte_pktmbuf_free_seg (m=0x13f8d3c00) at /root/dpdk-19.11/arm64-armv8a-linuxapp-gcc/include/rte_mbuf.h:1223 #9 rte_pktmbuf_free (m=0x13f8d3c00) at /root/dpdk-19.11/arm64-armv8a-linuxapp-gcc/include/rte_mbuf.h:1244 #10 0x0000000000466434 in lcore_hello () at /root/dpdk-19.11/examples/mbuf/main.c:121 #11 0x0000000000598ec8 in eal_thread_loop (arg=0x0) at /root/dpdk-19.11/lib/librte_eal/linux/eal/eal_thread.c:153 #12 0x0000ffffbe617d38 in start_thread () from /lib64/libpthread.so.0 #13 0x0000ffffbe55f690 in thread_start () from /lib64/libc.so.6 (gdb)