34 #ifndef _RTE_MEMPOOL_H_ 35 #define _RTE_MEMPOOL_H_ 67 #include <sys/queue.h> 80 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL 81 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL 82 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL 84 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 88 struct rte_mempool_debug_stats {
91 uint64_t get_success_bulk;
92 uint64_t get_success_objs;
93 uint64_t get_fail_bulk;
94 uint64_t get_fail_objs;
98 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 102 struct rte_mempool_cache {
108 void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3];
123 #define RTE_MEMPOOL_NAMESIZE 32 124 #define RTE_MEMPOOL_MZ_PREFIX "MP_" 127 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s" 129 #ifdef RTE_LIBRTE_XEN_DOM0 132 #define RTE_MEMPOOL_OBJ_NAME "%s_" RTE_MEMPOOL_MZ_PREFIX "elt" 136 #define RTE_MEMPOOL_OBJ_NAME RTE_MEMPOOL_MZ_FORMAT 140 #define MEMPOOL_PG_SHIFT_MAX (sizeof(uintptr_t) * CHAR_BIT - 1) 143 #define MEMPOOL_PG_NUM_DEFAULT 1 145 #ifndef RTE_MEMPOOL_ALIGN 146 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE 149 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1) 161 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 173 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 197 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 199 struct rte_mempool_cache local_cache[RTE_MAX_LCORE];
202 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 204 struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
222 #define MEMPOOL_F_NO_SPREAD 0x0001 223 #define MEMPOOL_F_NO_CACHE_ALIGN 0x0002 224 #define MEMPOOL_F_SP_PUT 0x0004 225 #define MEMPOOL_F_SC_GET 0x0008 237 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 238 #define __MEMPOOL_STAT_ADD(mp, name, n) do { \ 239 unsigned __lcore_id = rte_lcore_id(); \ 240 if (__lcore_id < RTE_MAX_LCORE) { \ 241 mp->stats[__lcore_id].name##_objs += n; \ 242 mp->stats[__lcore_id].name##_bulk += 1; \ 246 #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0) 257 #define MEMPOOL_HEADER_SIZE(mp, pgn) (sizeof(*(mp)) + \ 258 RTE_ALIGN_CEIL(((pgn) - RTE_DIM((mp)->elt_pa)) * \ 259 sizeof ((mp)->elt_pa[0]), RTE_CACHE_LINE_SIZE)) 264 #define MEMPOOL_IS_CONTIG(mp) \ 265 ((mp)->pg_num == MEMPOOL_PG_NUM_DEFAULT && \ 266 (mp)->phys_addr == (mp)->elt_pa[0]) 310 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 311 #ifndef __INTEL_COMPILER 312 #pragma GCC diagnostic ignored "-Wcast-qual" 314 static inline void __mempool_check_cookies(
const struct rte_mempool *
mp,
315 void *
const *obj_table_const,
316 unsigned n,
int free)
327 tmp = (
void *) obj_table_const;
328 obj_table = (
void **) tmp;
334 rte_panic(
"MEMPOOL: object is owned by another " 337 hdr = __mempool_get_header(obj);
338 cookie = hdr->cookie;
344 "obj=%p, mempool=%p, cookie=%" PRIx64
"\n",
345 obj, (
const void *) mp, cookie);
346 rte_panic(
"MEMPOOL: bad header cookie (put)\n");
350 else if (free == 1) {
354 "obj=%p, mempool=%p, cookie=%" PRIx64
"\n",
355 obj, (
const void *) mp, cookie);
356 rte_panic(
"MEMPOOL: bad header cookie (get)\n");
360 else if (free == 2) {
365 "obj=%p, mempool=%p, cookie=%" PRIx64
"\n",
366 obj, (
const void *) mp, cookie);
367 rte_panic(
"MEMPOOL: bad header cookie (audit)\n");
370 tlr = __mempool_get_trailer(obj);
371 cookie = tlr->cookie;
375 "obj=%p, mempool=%p, cookie=%" PRIx64
"\n",
376 obj, (
const void *) mp, cookie);
377 rte_panic(
"MEMPOOL: bad trailer cookie\n");
381 #ifndef __INTEL_COMPILER 382 #pragma GCC diagnostic error "-Wcast-qual" 385 #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0) 432 uint32_t elt_num,
size_t elt_sz,
size_t align,
433 const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift,
540 int socket_id,
unsigned flags);
640 int socket_id,
unsigned flags,
void *vaddr,
729 int socket_id,
unsigned flags);
754 static inline void __attribute__((always_inline))
755 __mempool_put_bulk(struct
rte_mempool *mp,
void * const *obj_table,
756 unsigned n,
int is_mp)
758 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 759 struct rte_mempool_cache *cache;
763 uint32_t cache_size = mp->cache_size;
764 uint32_t flushthresh = mp->cache_flushthresh;
768 __MEMPOOL_STAT_ADD(mp, put, n);
770 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 772 if (
unlikely(cache_size == 0 || is_mp == 0 ||
773 lcore_id >= RTE_MAX_LCORE))
777 if (
unlikely(n > RTE_MEMPOOL_CACHE_MAX_SIZE))
780 cache = &mp->local_cache[lcore_id];
781 cache_objs = &cache->objs[cache->len];
791 for (index = 0; index < n; ++index, obj_table++)
792 cache_objs[index] = *obj_table;
796 if (cache->len >= flushthresh) {
798 cache->len - cache_size);
799 cache->len = cache_size;
808 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 811 rte_panic(
"cannot put objects in mempool\n");
815 rte_panic(
"cannot put objects in mempool\n");
836 static inline void __attribute__((always_inline))
840 __mempool_check_cookies(mp, obj_table, n, 0);
841 __mempool_put_bulk(mp, obj_table, n, 1);
858 __mempool_check_cookies(mp, obj_table, n, 0);
859 __mempool_put_bulk(mp, obj_table, n, 0);
876 static inline void __attribute__((always_inline))
880 __mempool_check_cookies(mp, obj_table, n, 0);
892 static inline void __attribute__((always_inline))
906 static inline void __attribute__((always_inline))
924 static inline void __attribute__((always_inline))
944 static inline int __attribute__((always_inline))
945 __mempool_get_bulk(
struct rte_mempool *mp,
void **obj_table,
946 unsigned n,
int is_mc)
949 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 950 struct rte_mempool_cache *cache;
957 if (
unlikely(cache_size == 0 || is_mc == 0 ||
958 n >= cache_size || lcore_id >= RTE_MAX_LCORE))
961 cache = &mp->local_cache[lcore_id];
962 cache_objs = cache->objs;
965 if (cache->len < n) {
967 uint32_t req = n + (cache_size - cache->len);
985 for (index = 0, len = cache->len - 1; index < n; ++index, len--, obj_table++)
986 *obj_table = cache_objs[len];
990 __MEMPOOL_STAT_ADD(mp, get_success, n);
1004 __MEMPOOL_STAT_ADD(mp, get_fail, n);
1006 __MEMPOOL_STAT_ADD(mp, get_success, n);
1029 static inline int __attribute__((always_inline))
1033 ret = __mempool_get_bulk(mp, obj_table, n, 1);
1035 __mempool_check_cookies(mp, obj_table, n, 1);
1058 static inline int __attribute__((always_inline))
1062 ret = __mempool_get_bulk(mp, obj_table, n, 0);
1064 __mempool_check_cookies(mp, obj_table, n, 1);
1090 static inline int __attribute__((always_inline))
1094 ret = __mempool_get_bulk(mp, obj_table, n,
1097 __mempool_check_cookies(mp, obj_table, n, 1);
1117 static inline int __attribute__((always_inline))
1139 static inline int __attribute__((always_inline))
1165 static inline int __attribute__((always_inline))
1202 static inline unsigned 1262 off = (
const char *)elt - (
const char *)mp->
elt_va_start;
1391 const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift);
static int rte_mempool_mc_get(struct rte_mempool *mp, void **obj_p)
static int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
struct rte_mempool * rte_mempool_lookup(const char *name)
#define MEMPOOL_PG_NUM_DEFAULT
#define MEMPOOL_HEADER_SIZE(mp, pgn)
phys_addr_t elt_pa[MEMPOOL_PG_NUM_DEFAULT]
unsigned rte_mempool_count(const struct rte_mempool *mp)
static int rte_mempool_sc_get(struct rte_mempool *mp, void **obj_p)
static int rte_ring_mp_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned n)
struct rte_mempool * rte_dom0_mempool_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags)
#define RTE_MEMPOOL_NAMESIZE
void rte_mempool_list_dump(FILE *f)
static void rte_mempool_sp_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned n)
uint32_t cache_flushthresh
struct rte_mempool * rte_mempool_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags)
#define RTE_LOG(l, t,...)
static int rte_mempool_empty(const struct rte_mempool *mp)
int rte_eal_has_hugepages(void)
static int rte_ring_sp_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned n)
void rte_mempool_dump(FILE *f, const struct rte_mempool *mp)
char name[RTE_MEMPOOL_NAMESIZE]
static int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
uint32_t rte_mempool_obj_iter(void *vaddr, uint32_t elt_num, size_t elt_sz, size_t align, const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, rte_mempool_obj_iter_t obj_iter, void *obj_iter_arg)
void( rte_mempool_obj_ctor_t)(struct rte_mempool *, void *, void *, unsigned)
phys_addr_t rte_mem_virt2phy(const void *virt)
void rte_mempool_audit(const struct rte_mempool *mp)
static int rte_ring_mc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned n)
#define RTE_MEMPOOL_HEADER_COOKIE1
static int rte_ring_sc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned n)
#define RTE_PTR_ADD(ptr, x)
static unsigned rte_mempool_free_count(const struct rte_mempool *mp)
void(* rte_mempool_obj_iter_t)(void *, void *, void *, uint32_t)
static void rte_mempool_mp_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned n)
static int rte_mempool_full(const struct rte_mempool *mp)
static unsigned rte_lcore_id(void)
static void rte_mempool_sp_put(struct rte_mempool *mp, void *obj)
static int rte_mempool_mc_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
static void rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned n)
uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct rte_mempool_objsz *sz)
void rte_log_set_history(int enable)
struct rte_mempool * rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags, void *vaddr, const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift)
unsigned private_data_size
void( rte_mempool_ctor_t)(struct rte_mempool *, void *)
static int rte_mempool_sc_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
void rte_mempool_walk(void(*func)(const struct rte_mempool *, void *arg), void *arg)
#define RTE_PTR_SUB(ptr, x)
static struct rte_mempool * rte_mempool_from_obj(void *obj)
uint32_t pg_num __rte_cache_aligned
#define RTE_MEMPOOL_HEADER_COOKIE2
size_t rte_mempool_xmem_size(uint32_t elt_num, size_t elt_sz, uint32_t pg_shift)
#define RTE_MEMPOOL_TRAILER_COOKIE
static void rte_mempool_mp_put(struct rte_mempool *mp, void *obj)
static void * rte_mempool_get_priv(struct rte_mempool *mp)
static void rte_mempool_put(struct rte_mempool *mp, void *obj)
ssize_t rte_mempool_xmem_usage(void *vaddr, uint32_t elt_num, size_t elt_sz, const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift)
static phys_addr_t rte_mempool_virt2phy(const struct rte_mempool *mp, const void *elt)
#define __rte_cache_aligned