69 #pragma GCC poison RTE_MBUF_SCATTER_GATHER 70 #pragma GCC poison RTE_MBUF_REFCNT 86 #define PKT_RX_VLAN_PKT (1ULL << 0) 87 #define PKT_RX_RSS_HASH (1ULL << 1) 88 #define PKT_RX_FDIR (1ULL << 2) 89 #define PKT_RX_L4_CKSUM_BAD (1ULL << 3) 90 #define PKT_RX_IP_CKSUM_BAD (1ULL << 4) 91 #define PKT_RX_EIP_CKSUM_BAD (1ULL << 5) 92 #define PKT_RX_OVERSIZE (0ULL << 0) 93 #define PKT_RX_HBUF_OVERFLOW (0ULL << 0) 94 #define PKT_RX_RECIP_ERR (0ULL << 0) 95 #define PKT_RX_MAC_ERR (0ULL << 0) 96 #define PKT_RX_IEEE1588_PTP (1ULL << 9) 97 #define PKT_RX_IEEE1588_TMST (1ULL << 10) 98 #define PKT_RX_FDIR_ID (1ULL << 13) 99 #define PKT_RX_FDIR_FLX (1ULL << 14) 100 #define PKT_RX_QINQ_PKT (1ULL << 15) 108 #define PKT_TX_QINQ_PKT (1ULL << 49) 123 #define PKT_TX_TCP_SEG (1ULL << 50) 125 #define PKT_TX_IEEE1588_TMST (1ULL << 51) 138 #define PKT_TX_L4_NO_CKSUM (0ULL << 52) 139 #define PKT_TX_TCP_CKSUM (1ULL << 52) 140 #define PKT_TX_SCTP_CKSUM (2ULL << 52) 141 #define PKT_TX_UDP_CKSUM (3ULL << 52) 142 #define PKT_TX_L4_MASK (3ULL << 52) 151 #define PKT_TX_IP_CKSUM (1ULL << 54) 159 #define PKT_TX_IPV4 (1ULL << 55) 167 #define PKT_TX_IPV6 (1ULL << 56) 169 #define PKT_TX_VLAN_PKT (1ULL << 57) 179 #define PKT_TX_OUTER_IP_CKSUM (1ULL << 58) 186 #define PKT_TX_OUTER_IPV4 (1ULL << 59) 193 #define PKT_TX_OUTER_IPV6 (1ULL << 60) 195 #define __RESERVED (1ULL << 61) 197 #define IND_ATTACHED_MBUF (1ULL << 62) 200 #define CTRL_MBUF_FLAG (1ULL << 63) 250 #define RTE_PTYPE_UNKNOWN 0x00000000 258 #define RTE_PTYPE_L2_ETHER 0x00000001 265 #define RTE_PTYPE_L2_ETHER_TIMESYNC 0x00000002 272 #define RTE_PTYPE_L2_ETHER_ARP 0x00000003 279 #define RTE_PTYPE_L2_ETHER_LLDP 0x00000004 284 #define RTE_PTYPE_L2_MASK 0x0000000f 294 #define RTE_PTYPE_L3_IPV4 0x00000010 304 #define RTE_PTYPE_L3_IPV4_EXT 0x00000030 314 #define RTE_PTYPE_L3_IPV6 0x00000040 324 #define RTE_PTYPE_L3_IPV4_EXT_UNKNOWN 0x00000090 335 #define RTE_PTYPE_L3_IPV6_EXT 0x000000c0 346 #define RTE_PTYPE_L3_IPV6_EXT_UNKNOWN 0x000000e0 351 #define RTE_PTYPE_L3_MASK 0x000000f0 363 #define RTE_PTYPE_L4_TCP 0x00000100 375 #define RTE_PTYPE_L4_UDP 0x00000200 392 #define RTE_PTYPE_L4_FRAG 0x00000300 404 #define RTE_PTYPE_L4_SCTP 0x00000400 416 #define RTE_PTYPE_L4_ICMP 0x00000500 432 #define RTE_PTYPE_L4_NONFRAG 0x00000600 437 #define RTE_PTYPE_L4_MASK 0x00000f00 448 #define RTE_PTYPE_TUNNEL_IP 0x00001000 459 #define RTE_PTYPE_TUNNEL_GRE 0x00002000 472 #define RTE_PTYPE_TUNNEL_VXLAN 0x00003000 486 #define RTE_PTYPE_TUNNEL_NVGRE 0x00004000 499 #define RTE_PTYPE_TUNNEL_GENEVE 0x00005000 506 #define RTE_PTYPE_TUNNEL_GRENAT 0x00006000 510 #define RTE_PTYPE_TUNNEL_MASK 0x0000f000 518 #define RTE_PTYPE_INNER_L2_ETHER 0x00010000 525 #define RTE_PTYPE_INNER_L2_ETHER_VLAN 0x00020000 529 #define RTE_PTYPE_INNER_L2_MASK 0x000f0000 538 #define RTE_PTYPE_INNER_L3_IPV4 0x00100000 547 #define RTE_PTYPE_INNER_L3_IPV4_EXT 0x00200000 556 #define RTE_PTYPE_INNER_L3_IPV6 0x00300000 565 #define RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN 0x00400000 575 #define RTE_PTYPE_INNER_L3_IPV6_EXT 0x00500000 586 #define RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN 0x00600000 590 #define RTE_PTYPE_INNER_L3_MASK 0x00f00000 602 #define RTE_PTYPE_INNER_L4_TCP 0x01000000 614 #define RTE_PTYPE_INNER_L4_UDP 0x02000000 626 #define RTE_PTYPE_INNER_L4_FRAG 0x03000000 638 #define RTE_PTYPE_INNER_L4_SCTP 0x04000000 650 #define RTE_PTYPE_INNER_L4_ICMP 0x05000000 663 #define RTE_PTYPE_INNER_L4_NONFRAG 0x06000000 667 #define RTE_PTYPE_INNER_L4_MASK 0x0f000000 674 #define RTE_ETH_IS_IPV4_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV4) 681 #define RTE_ETH_IS_IPV6_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV6) 684 #define RTE_ETH_IS_TUNNEL_PKT(ptype) ((ptype) & (RTE_PTYPE_TUNNEL_MASK | \ 685 RTE_PTYPE_INNER_L2_MASK | \ 686 RTE_PTYPE_INNER_L3_MASK | \ 687 RTE_PTYPE_INNER_L4_MASK)) 690 #define RTE_MBUF_PRIV_ALIGN 8 720 #define RTE_MBUF_DEFAULT_DATAROOM 2048 721 #define RTE_MBUF_DEFAULT_BUF_SIZE \ 722 (RTE_MBUF_DEFAULT_DATAROOM + RTE_PKTMBUF_HEADROOM) 764 MARKER rx_descriptor_fields1;
814 MARKER cacheline1 __rte_cache_min_aligned;
916 #define RTE_MBUF_INDIRECT(mb) ((mb)->ol_flags & IND_ATTACHED_MBUF) 921 #define RTE_MBUF_DIRECT(mb) (!RTE_MBUF_INDIRECT(mb)) 934 #ifdef RTE_LIBRTE_MBUF_DEBUG 937 #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h) 940 #define __rte_mbuf_sanity_check_raw(m, is_h) do { \ 942 rte_mbuf_sanity_check(m, is_h); \ 946 #define RTE_MBUF_ASSERT(exp) \ 948 rte_panic("line%d\tassert \"" #exp "\" failed\n", __LINE__); \ 954 #define __rte_mbuf_sanity_check(m, is_h) do { } while (0) 957 #define __rte_mbuf_sanity_check_raw(m, is_h) do { } while (0) 960 #define RTE_MBUF_ASSERT(exp) do { } while (0) 964 #ifdef RTE_MBUF_REFCNT_ATOMIC 973 static inline uint16_t
1001 static inline uint16_t
1024 static inline uint16_t
1034 static inline uint16_t
1052 #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \ 1104 static inline void __attribute__((always_inline))
1105 __rte_mbuf_raw_free(struct
rte_mbuf *m)
1133 void *m,
unsigned i);
1147 #define rte_ctrlmbuf_alloc(mp) rte_pktmbuf_alloc(mp) 1155 #define rte_ctrlmbuf_free(m) rte_pktmbuf_free(m) 1165 #define rte_ctrlmbuf_data(m) ((char *)((m)->buf_addr) + (m)->data_off) 1175 #define rte_ctrlmbuf_len(m) rte_pktmbuf_data_len(m) 1214 void *m,
unsigned i);
1286 static inline uint16_t
1307 static inline uint16_t
1336 m->data_off = (RTE_PKTMBUF_HEADROOM <= m->
buf_len) ?
1337 RTE_PKTMBUF_HEADROOM : m->
buf_len;
1359 if ((m = __rte_mbuf_raw_alloc(mp)) != NULL)
1378 struct rte_mbuf **mbufs,
unsigned count)
1392 switch (count % 4) {
1394 while (idx != count) {
1454 mi->data_off = m->data_off;
1488 mbuf_size =
sizeof(
struct rte_mbuf) + priv_size;
1492 m->
buf_addr = (
char *)m + mbuf_size;
1494 m->
buf_len = (uint16_t)buf_len;
1495 m->data_off =
RTE_MIN(RTE_PKTMBUF_HEADROOM, (uint16_t)m->
buf_len);
1500 static inline struct rte_mbuf* __attribute__((always_inline))
1501 __rte_pktmbuf_prefree_seg(
struct rte_mbuf *m)
1515 __rte_mbuf_raw_free(md);
1531 static inline void __attribute__((always_inline))
1534 if (
likely(NULL != (m = __rte_pktmbuf_prefree_seg(m)))) {
1536 __rte_mbuf_raw_free(m);
1599 }
while ((md = md->
next) != NULL &&
1633 }
while ((m = m->
next) != NULL);
1678 while (m2->
next != NULL)
1697 #define rte_pktmbuf_mtod_offset(m, t, o) \ 1698 ((t)((char *)(m)->buf_addr + (m)->data_off + (o))) 1712 #define rte_pktmbuf_mtod(m, t) rte_pktmbuf_mtod_offset(m, t, 0) 1723 #define rte_pktmbuf_mtophys_offset(m, o) \ 1724 (phys_addr_t)((m)->buf_physaddr + (m)->data_off + (o)) 1733 #define rte_pktmbuf_mtophys(m) rte_pktmbuf_mtophys_offset(m, 0) 1743 #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len) 1753 #define rte_pktmbuf_data_len(m) ((m)->data_len) 1782 return (
char *)m->
buf_addr + m->data_off;
1814 return (
char*) tail;
1841 return (
char *)m->
buf_addr + m->data_off;
1914 cur_tail->
next = tail;
static void rte_pktmbuf_reset(struct rte_mbuf *m)
uint16_t mbuf_data_room_size
static int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
static int16_t rte_atomic16_read(const rte_atomic16_t *v)
static struct rte_mbuf * rte_pktmbuf_alloc(struct rte_mempool *mp)
static void rte_pktmbuf_free_seg(struct rte_mbuf *m)
struct rte_mbuf::@68::@76 sched
#define IND_ATTACHED_MBUF
static uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp)
static void rte_pktmbuf_free(struct rte_mbuf *m)
void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg)
static struct rte_mbuf * rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
static struct rte_mbuf * rte_mbuf_from_indirect(struct rte_mbuf *mi)
static int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len)
static uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
static int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count)
char name[RTE_MEMPOOL_NAMESIZE]
static int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
#define RTE_MBUF_DIRECT(mb)
#define RTE_MBUF_ASSERT(exp)
static int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
static uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
void rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header)
#define __rte_mbuf_sanity_check(m, is_h)
const char * rte_get_tx_ol_flag_name(uint64_t mask)
static uint16_t rte_mbuf_refcnt_read(const struct rte_mbuf *m)
static void rte_atomic16_set(rte_atomic16_t *v, int16_t new_value)
static char * rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
static void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
static int16_t rte_atomic16_add_return(rte_atomic16_t *v, int16_t inc)
static int rte_is_ctrlmbuf(struct rte_mbuf *m)
static void rte_pktmbuf_detach(struct rte_mbuf *m)
static uint16_t rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
static uint16_t rte_pktmbuf_data_room_size(struct rte_mempool *mp)
const char * rte_get_rx_ol_flag_name(uint64_t mask)
static phys_addr_t rte_mbuf_data_dma_addr(const struct rte_mbuf *mb)
void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *m, unsigned i)
struct rte_mempool * pool
static char * rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
static void rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
static int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
static char * rte_mbuf_to_baddr(struct rte_mbuf *md)
static char * rte_pktmbuf_prepend(struct rte_mbuf *m, uint16_t len)
static void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
#define RTE_PTR_SUB(ptr, x)
#define RTE_MBUF_INDIRECT(mb)
static struct rte_mbuf * rte_pktmbuf_lastseg(struct rte_mbuf *m)
struct rte_mbuf::@68::@75 fdir
rte_atomic16_t refcnt_atomic
static void * rte_mempool_get_priv(struct rte_mempool *mp)
void rte_ctrlmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *m, unsigned i)
static void rte_mempool_put(struct rte_mempool *mp, void *obj)
static phys_addr_t rte_mbuf_data_dma_addr_default(const struct rte_mbuf *mb)
struct rte_mempool * rte_pktmbuf_pool_create(const char *name, unsigned n, unsigned cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id)
static phys_addr_t rte_mempool_virt2phy(const struct rte_mempool *mp, const void *elt)
#define __rte_cache_aligned