AI Trajectory

Trajectory: (Expand/Collapse All)
Seq	Timestamp	Type	Name	Duration
0/0	2026/07/23 22:38	flow	patch-triage	0m Args: null Results: { "EnableConfigs": [ "BPF_SYSCALL" ], "FocusSymbols": [ "htab_map_alloc", "htab_map_update_elem", "htab_lru_map_update_elem", "__htab_lru_percpu_map_update_elem", "htab_map_delete_elem", "htab_lru_map_delete_elem", "__htab_map_lookup_and_delete_elem", "__htab_map_lookup_and_delete_batch", "bpf_map_meta_alloc", "htab_map_gen_lookup", "htab_lru_map_gen_lookup", "htab_percpu_map_gen_lookup", "htab_of_map_gen_lookup" ], "Reasoning": "The patch functionally modifies the memory layout and operations of BPF hash maps (including LRU and per-CPU variants). It introduces new structures and offsets for hash map elements, which affects allocation, updates, deletions, and JIT lookup generation. Fuzzing is needed to ensure memory safety and correctness of these core BPF map operations.", "WorthFuzzing": true }
1/1	2026/07/23 22:38	action	read-patch-diff	0m Args: null Results: { "PatchDiff": "commit 243a270640e0dc901e4a78b599b0748d59bf7b63\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate: Thu Jul 23 22:38:40 2026 +0000\n\n syz-cluster: applied patch under review\n\ndiff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c\nindex 9f394e1aa2e85b..b8c30d40295879 100644\n--- a/kernel/bpf/hashtab.c\n+++ b/kernel/bpf/hashtab.c\n@@ -102,6 +102,7 @@ struct bpf_htab {\n \tbool use_percpu_counter;\n \tu32 n_buckets;\t/* number of hash buckets /\n \tu32 elem_size;\t/ size of each element in bytes /\n+\tu32 key_offset;\t/ offset of key in bytes /\n \tu32 hashrnd;\n };\n \n@@ -117,18 +118,46 @@ struct htab_elem {\n \t\t\t};\n \t\t};\n \t};\n+\tu32 hash; / Only when htab_has_hash() /\n+} __aligned(8);\n+\n+struct htab_elem_lru {\n \tunion {\n-\t\t/ pointer to per-cpu pointer /\n-\t\tvoid ptr_to_pptr;\n-\t\tstruct bpf_lru_node lru_node;\n+\t\tstruct hlist_nulls_node hash_node;\n+\t\tstruct {\n+\t\t\tvoid padding;\n+\t\t\tunion {\n+\t\t\t\tstruct pcpu_freelist_node fnode;\n+\t\t\t\tstruct htab_elem_lru batch_flink;\n+\t\t\t};\n+\t\t};\n \t};\n+\tstruct bpf_lru_node lru_node;\n \tu32 hash;\n-\tchar key[] __aligned(8);\n-};\n+} __aligned(8);\n+\n+/* Only for non-preallocated PCPU maps. Preallocated PCPU maps don't need\n+ * ptr_to_pptr, and use htab_elem.\n+ /\n+struct htab_elem_pcpu {\n+\tunion {\n+\t\tstruct hlist_nulls_node hash_node;\n+\t\tstruct {\n+\t\t\tvoid padding;\n+\t\t\tunion {\n+\t\t\t\tstruct pcpu_freelist_node fnode;\n+\t\t\t\tstruct htab_elem_pcpu batch_flink;\n+\t\t\t};\n+\t\t};\n+\t};\n+\tvoid ptr_to_pptr;\n+\tu32 hash; /* Only when htab_has_hash() /\n+} __aligned(8);\n \n struct htab_btf_record {\n \tstruct btf_record record;\n \tu32 key_size;\n+\tu32 key_offset;\n };\n \n static inline bool htab_is_prealloc(const struct bpf_htab htab)\n@@ -136,6 +165,21 @@ static inline bool htab_is_prealloc(const struct bpf_htab htab)\n \treturn !(htab-\u003emap.map_flags \u0026 BPF_F_NO_PREALLOC);\n }\n \n+static inline struct bpf_lru_node htab_elem_lru_node(struct htab_elem l)\n+{\n+\treturn \u0026((struct htab_elem_lru )l)-\u003elru_node;\n+}\n+\n+static inline void htab_elem_get_ptr_to_pptr(struct htab_elem l)\n+{\n+\treturn ((struct htab_elem_pcpu )l)-\u003eptr_to_pptr;\n+}\n+\n+static inline void htab_elem_set_ptr_to_pptr(struct htab_elem l, void ptr)\n+{\n+\t((struct htab_elem_pcpu )l)-\u003eptr_to_pptr = ptr;\n+}\n+\n static void htab_init_buckets(struct bpf_htab htab)\n {\n \tunsigned int i;\n@@ -183,25 +227,30 @@ static inline bool is_fd_htab(const struct bpf_htab htab)\n \treturn htab-\u003emap.map_type == BPF_MAP_TYPE_HASH_OF_MAPS;\n }\n \n-static inline void htab_elem_value(struct htab_elem l, u32 key_size)\n+static inline void htab_elem_key(struct bpf_htab htab, struct htab_elem l)\n+{\n+\treturn (void )l + htab-\u003ekey_offset;\n+}\n+\n+static inline void htab_elem_value(struct bpf_htab htab, struct htab_elem l)\n {\n-\treturn l-\u003ekey + round_up(key_size, 8);\n+\treturn htab_elem_key(htab, l) + round_up(htab-\u003emap.key_size, 8);\n }\n \n-static inline void htab_elem_set_ptr(struct htab_elem l, u32 key_size,\n+static inline void htab_elem_set_ptr(struct bpf_htab htab, struct htab_elem l,\n \t\t\t\t void __percpu pptr)\n {\n-\t(void __percpu )htab_elem_value(l, key_size) = pptr;\n+\t(void __percpu *)htab_elem_value(htab, l) = pptr;\n }\n \n-static inline void __percpu htab_elem_get_ptr(struct htab_elem l, u32 key_size)\n+static inline void __percpu htab_elem_get_ptr(struct bpf_htab htab, struct htab_elem l)\n {\n-\treturn (void __percpu )htab_elem_value(l, key_size);\n+\treturn (void __percpu *)htab_elem_value(htab, l);\n }\n \n-static void fd_htab_map_get_ptr(const struct bpf_map map, struct htab_elem l)\n+static void fd_htab_map_get_ptr(struct bpf_htab htab, struct htab_elem l)\n {\n-\treturn (void *)htab_elem_value(l, map-\u003ekey_size);\n+\treturn (void *)htab_elem_value(htab, l);\n }\n \n static struct htab_elem get_htab_elem(struct bpf_htab htab, int i)\n@@ -209,6 +258,33 @@ static struct htab_elem get_htab_elem(struct bpf_htab htab, int i)\n \treturn (struct htab_elem ) (htab-\u003eelems + i * (u64)htab-\u003eelem_size);\n }\n \n+static inline bool htab_has_hash(const struct bpf_htab htab)\n+{\n+\treturn htab_is_lru(htab) \|\| htab-\u003emap.key_size \u003e 8;\n+}\n+\n+static inline u32 htab_elem_hash(struct bpf_htab htab, struct htab_elem l)\n+{\n+\tif (htab_is_lru(htab))\n+\t\treturn ((struct htab_elem_lru )l)-\u003ehash;\n+\telse if (htab_is_percpu(htab) \u0026\u0026 !htab_is_prealloc(htab))\n+\t\treturn ((struct htab_elem_pcpu )l)-\u003ehash;\n+\telse\n+\t\treturn l-\u003ehash;\n+}\n+\n+static inline void htab_elem_set_hash(struct bpf_htab htab, struct htab_elem l, u32 hash)\n+{\n+\tif (!htab_has_hash(htab))\n+\t\treturn;\n+\tif (htab_is_lru(htab))\n+\t\t((struct htab_elem_lru )l)-\u003ehash = hash;\n+\telse if (htab_is_percpu(htab) \u0026\u0026 !htab_is_prealloc(htab))\n+\t\t((struct htab_elem_pcpu )l)-\u003ehash = hash;\n+\telse\n+\t\tl-\u003ehash = hash;\n+}\n+\n / Both percpu and fd htab support in-place update, so no need for\n * extra elem. LRU itself can remove the least used element, so\n * there is no need for an extra elem during map_update.\n@@ -231,7 +307,7 @@ static void htab_free_prealloced_internal_structs(struct bpf_htab htab)\n \n \t\telem = get_htab_elem(htab, i);\n \t\tbpf_map_free_internal_structs(\u0026htab-\u003emap,\n-\t\t\t\t\t htab_elem_value(elem, htab-\u003emap.key_size));\n+\t\t\t\t\t htab_elem_value(htab, elem));\n \t\tcond_resched();\n \t}\n }\n@@ -254,7 +330,7 @@ static void htab_free_prealloced_fields(struct bpf_htab htab)\n \n \t\telem = get_htab_elem(htab, i);\n \t\tif (htab_is_percpu(htab)) {\n-\t\t\tvoid __percpu pptr = htab_elem_get_ptr(elem, htab-\u003emap.key_size);\n+\t\t\tvoid __percpu pptr = htab_elem_get_ptr(htab, elem);\n \t\t\tint cpu;\n \n \t\t\tfor_each_possible_cpu(cpu) {\n@@ -263,7 +339,7 @@ static void htab_free_prealloced_fields(struct bpf_htab htab)\n \t\t\t}\n \t\t} else {\n \t\t\tbpf_obj_free_fields(htab-\u003emap.record,\n-\t\t\t\t\t htab_elem_value(elem, htab-\u003emap.key_size));\n+\t\t\t\t\t htab_elem_value(htab, elem));\n \t\t\tcond_resched();\n \t\t}\n \t\tcond_resched();\n@@ -280,8 +356,7 @@ static void htab_free_elems(struct bpf_htab htab)\n \tfor (i = 0; i \u003c htab-\u003emap.max_entries; i++) {\n \t\tvoid __percpu pptr;\n \n-\t\tpptr = htab_elem_get_ptr(get_htab_elem(htab, i),\n-\t\t\t\t\t htab-\u003emap.key_size);\n+\t\tpptr = htab_elem_get_ptr(htab, get_htab_elem(htab, i));\n \t\tfree_percpu(pptr);\n \t\tcond_resched();\n \t}\n@@ -308,8 +383,8 @@ static struct htab_elem prealloc_lru_pop(struct bpf_htab htab, void key,\n \n \tif (node) {\n \t\tbpf_map_inc_elem_count(\u0026htab-\u003emap);\n-\t\tl = container_of(node, struct htab_elem, lru_node);\n-\t\tmemcpy(l-\u003ekey, key, htab-\u003emap.key_size);\n+\t\tl = (struct htab_elem )container_of(node, struct htab_elem_lru, lru_node);\n+\t\tmemcpy(htab_elem_key(htab, l), key, htab-\u003emap.key_size);\n \t\treturn l;\n \t}\n \n@@ -340,8 +415,7 @@ static int prealloc_init(struct bpf_htab htab)\n \t\t\t\t\t GFP_USER \| __GFP_NOWARN);\n \t\tif (!pptr)\n \t\t\tgoto free_elems;\n-\t\thtab_elem_set_ptr(get_htab_elem(htab, i), htab-\u003emap.key_size,\n-\t\t\t\t pptr);\n+\t\thtab_elem_set_ptr(htab, get_htab_elem(htab, i), pptr);\n \t\tcond_resched();\n \t}\n \n@@ -349,8 +423,8 @@ static int prealloc_init(struct bpf_htab htab)\n \tif (htab_is_lru(htab))\n \t\terr = bpf_lru_init(\u0026htab-\u003elru,\n \t\t\t\t htab-\u003emap.map_flags \u0026 BPF_F_NO_COMMON_LRU,\n-\t\t\t\t offsetof(struct htab_elem, hash) -\n-\t\t\t\t offsetof(struct htab_elem, lru_node),\n+\t\t\t\t offsetof(struct htab_elem_lru, hash) -\n+\t\t\t\t offsetof(struct htab_elem_lru, lru_node),\n \t\t\t\t htab_lru_map_delete_node,\n \t\t\t\t htab);\n \telse\n@@ -361,7 +435,7 @@ static int prealloc_init(struct bpf_htab htab)\n \n \tif (htab_is_lru(htab))\n \t\tbpf_lru_populate(\u0026htab-\u003elru, htab-\u003eelems,\n-\t\t\t\t offsetof(struct htab_elem, lru_node),\n+\t\t\t\t offsetof(struct htab_elem_lru, lru_node),\n \t\t\t\t htab-\u003eelem_size, num_entries);\n \telse\n \t\tpcpu_freelist_populate(\u0026htab-\u003efreelist,\n@@ -428,6 +502,25 @@ static int htab_map_alloc_check(union bpf_attr attr)\n \tBUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=\n \t\t offsetof(struct htab_elem, hash_node.pprev));\n \n+\t/ htab_elem pointers are used as base pointers for htab_elem_lru and\n+\t * htab_elem_pcpu. These structures must share a common initial sequence\n+\t * to ensure pointer casting between them is safe.\n+\t /\n+\tBUILD_BUG_ON(offsetof(struct htab_elem, hash_node) !=\n+\t\t offsetof(struct htab_elem_lru, hash_node));\n+\tBUILD_BUG_ON(offsetof(struct htab_elem, hash_node) !=\n+\t\t offsetof(struct htab_elem_pcpu, hash_node));\n+\n+\tBUILD_BUG_ON(offsetof(struct htab_elem, padding) !=\n+\t\t offsetof(struct htab_elem_lru, padding));\n+\tBUILD_BUG_ON(offsetof(struct htab_elem, padding) !=\n+\t\t offsetof(struct htab_elem_pcpu, padding));\n+\n+\tBUILD_BUG_ON(offsetof(struct htab_elem, fnode) !=\n+\t\t offsetof(struct htab_elem_lru, fnode));\n+\tBUILD_BUG_ON(offsetof(struct htab_elem, fnode) !=\n+\t\t offsetof(struct htab_elem_pcpu, fnode));\n+\n \tif (zero_seed \u0026\u0026 !capable(CAP_SYS_ADMIN))\n \t\t/ Guard against local DoS, and discourage production use. /\n \t\treturn -EPERM;\n@@ -476,7 +569,7 @@ static void htab_mem_dtor(void obj, void ctx)\n \tif (IS_ERR_OR_NULL(hrec-\u003erecord))\n \t\treturn;\n \n-\tmap_value = htab_elem_value(elem, hrec-\u003ekey_size);\n+\tmap_value = (void )elem + hrec-\u003ekey_offset + round_up(hrec-\u003ekey_size, 8);\n \tbpf_obj_free_fields(hrec-\u003erecord, map_value);\n }\n \n@@ -502,7 +595,7 @@ static void htab_dtor_ctx_free(void ctx)\n }\n \n static int bpf_ma_set_dtor(struct bpf_map map, struct bpf_mem_alloc ma,\n-\t\t\t void (dtor)(void , void ))\n+\t\t\t void (dtor)(void , void ), u32 key_offset)\n {\n \tstruct htab_btf_record hrec;\n \tint err;\n@@ -515,6 +608,7 @@ static int bpf_ma_set_dtor(struct bpf_map map, struct bpf_mem_alloc ma,\n \tif (!hrec)\n \t\treturn -ENOMEM;\n \threc-\u003ekey_size = map-\u003ekey_size;\n+\threc-\u003ekey_offset = key_offset;\n \threc-\u003erecord = btf_record_dup(map-\u003erecord);\n \tif (IS_ERR(hrec-\u003erecord)) {\n \t\terr = PTR_ERR(hrec-\u003erecord);\n@@ -537,9 +631,9 @@ static int htab_map_check_btf(struct bpf_map map, const struct btf btf,\n \t * populated in htab_map_alloc(), so it will always appear as NULL.\n \t /\n \tif (htab_is_percpu(htab))\n-\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003epcpu_ma, htab_pcpu_mem_dtor);\n+\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003epcpu_ma, htab_pcpu_mem_dtor, htab-\u003ekey_offset);\n \telse\n-\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003ema, htab_mem_dtor);\n+\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003ema, htab_mem_dtor, htab-\u003ekey_offset);\n }\n \n static struct bpf_map htab_map_alloc(union bpf_attr attr)\n@@ -583,8 +677,18 @@ static struct bpf_map htab_map_alloc(union bpf_attr attr)\n \n \thtab-\u003en_buckets = roundup_pow_of_two(htab-\u003emap.max_entries);\n \n-\thtab-\u003eelem_size = sizeof(struct htab_elem) +\n-\t\t\t round_up(htab-\u003emap.key_size, 8);\n+\tif (htab_is_lru(htab))\n+\t\thtab-\u003ekey_offset = sizeof(struct htab_elem_lru);\n+\telse if (percpu \u0026\u0026 !prealloc)\n+\t\thtab-\u003ekey_offset = round_up(htab_has_hash(htab) ?\n+\t\t\t\t\t sizeof(struct htab_elem_pcpu) :\n+\t\t\t\t\t offsetof(struct htab_elem_pcpu, hash), 8);\n+\telse\n+\t\thtab-\u003ekey_offset = round_up(htab_has_hash(htab) ?\n+\t\t\t\t\t sizeof(struct htab_elem) :\n+\t\t\t\t\t offsetof(struct htab_elem, hash), 8);\n+\n+\thtab-\u003eelem_size = htab-\u003ekey_offset + round_up(htab-\u003emap.key_size, 8);\n \tif (percpu)\n \t\thtab-\u003eelem_size += sizeof(void );\n \telse\n@@ -692,14 +796,16 @@ static inline struct hlist_nulls_head select_bucket(struct bpf_htab htab, u32\n }\n \n /* this lookup function can only be called with bucket lock taken /\n-static struct htab_elem lookup_elem_raw(struct hlist_nulls_head head, u32 hash,\n+static struct htab_elem lookup_elem_raw(struct bpf_htab htab,\n+\t\t\t\t\t struct hlist_nulls_head head, u32 hash,\n \t\t\t\t\t void key, u32 key_size)\n {\n \tstruct hlist_nulls_node n;\n \tstruct htab_elem l;\n \n \thlist_nulls_for_each_entry_rcu(l, n, head, hash_node)\n-\t\tif (l-\u003ehash == hash \u0026\u0026 !memcmp(\u0026l-\u003ekey, key, key_size))\n+\t\tif ((!htab_has_hash(htab) \|\| htab_elem_hash(htab, l) == hash) \u0026\u0026\n+\t\t !memcmp(htab_elem_key(htab, l), key, key_size))\n \t\t\treturn l;\n \n \treturn NULL;\n@@ -709,7 +815,8 @@ static struct htab_elem lookup_elem_raw(struct hlist_nulls_head head, u32 hash\n the unlikely event when elements moved from one bucket into another\n * while link list is being walked\n /\n-static struct htab_elem lookup_nulls_elem_raw(struct hlist_nulls_head head,\n+static struct htab_elem lookup_nulls_elem_raw(struct bpf_htab htab,\n+\t\t\t\t\t struct hlist_nulls_head head,\n \t\t\t\t\t u32 hash, void key,\n \t\t\t\t\t u32 key_size, u32 n_buckets)\n {\n@@ -718,7 +825,8 @@ static struct htab_elem lookup_nulls_elem_raw(struct hlist_nulls_head head,\n \n again:\n \thlist_nulls_for_each_entry_rcu(l, n, head, hash_node)\n-\t\tif (l-\u003ehash == hash \u0026\u0026 !memcmp(\u0026l-\u003ekey, key, key_size))\n+\t\tif ((!htab_has_hash(htab) \|\| htab_elem_hash(htab, l) == hash) \u0026\u0026\n+\t\t !memcmp(htab_elem_key(htab, l), key, key_size))\n \t\t\treturn l;\n \n \tif (unlikely(get_nulls_value(n) != (hash \u0026 (n_buckets - 1))))\n@@ -747,17 +855,18 @@ static void __htab_map_lookup_elem(struct bpf_map map, void key)\n \n \thead = select_bucket(htab, hash);\n \n-\tl = lookup_nulls_elem_raw(head, hash, key, key_size, htab-\u003en_buckets);\n+\tl = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab-\u003en_buckets);\n \n \treturn l;\n }\n \n static void htab_map_lookup_elem(struct bpf_map map, void key)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l = __htab_map_lookup_elem(map, key);\n \n \tif (l)\n-\t\treturn htab_elem_value(l, map-\u003ekey_size);\n+\t\treturn htab_elem_value(htab, l);\n \n \treturn NULL;\n }\n@@ -775,6 +884,7 @@ static void htab_map_lookup_elem(struct bpf_map map, void key)\n /\n static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn insn = insn_buf;\n \tconst int ret = BPF_REG_0;\n \n@@ -783,7 +893,7 @@ static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)\n \tinsn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \tinsn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);\n \tinsn++ = BPF_ALU64_IMM(BPF_ADD, ret,\n-\t\t\t\toffsetof(struct htab_elem, key) +\n+\t\t\t\thtab-\u003ekey_offset +\n \t\t\t\tround_up(map-\u003ekey_size, 8));\n \treturn insn - insn_buf;\n }\n@@ -791,12 +901,13 @@ static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)\n static __always_inline void __htab_lru_map_lookup_elem(struct bpf_map map,\n \t\t\t\t\t\t\tvoid key, const bool mark)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l = __htab_map_lookup_elem(map, key);\n \n \tif (l) {\n \t\tif (mark)\n-\t\t\tbpf_lru_node_set_ref(\u0026l-\u003elru_node);\n-\t\treturn htab_elem_value(l, map-\u003ekey_size);\n+\t\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l));\n+\t\treturn htab_elem_value(htab, l);\n \t}\n \n \treturn NULL;\n@@ -815,6 +926,7 @@ static void htab_lru_map_lookup_elem_sys(struct bpf_map map, void key)\n static int htab_lru_map_gen_lookup(struct bpf_map map,\n \t\t\t\t struct bpf_insn insn_buf)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn insn = insn_buf;\n \tconst int ret = BPF_REG_0;\n \tconst int ref_reg = BPF_REG_1;\n@@ -824,15 +936,15 @@ static int htab_lru_map_gen_lookup(struct bpf_map map,\n \tinsn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \tinsn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4);\n \tinsn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret,\n-\t\t\t offsetof(struct htab_elem, lru_node) +\n+\t\t\t offsetof(struct htab_elem_lru, lru_node) +\n \t\t\t offsetof(struct bpf_lru_node, ref));\n \tinsn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1);\n \tinsn++ = BPF_ST_MEM(BPF_B, ret,\n-\t\t\t offsetof(struct htab_elem, lru_node) +\n+\t\t\t offsetof(struct htab_elem_lru, lru_node) +\n \t\t\t offsetof(struct bpf_lru_node, ref),\n \t\t\t 1);\n \tinsn++ = BPF_ALU64_IMM(BPF_ADD, ret,\n-\t\t\t\toffsetof(struct htab_elem, key) +\n+\t\t\t\thtab-\u003ekey_offset +\n \t\t\t\tround_up(map-\u003ekey_size, 8));\n \treturn insn - insn_buf;\n }\n@@ -844,13 +956,13 @@ static void check_and_cancel_fields(struct bpf_htab htab,\n \t\treturn;\n \n \tif (htab_is_percpu(htab)) {\n-\t\tvoid __percpu pptr = htab_elem_get_ptr(elem, htab-\u003emap.key_size);\n+\t\tvoid __percpu pptr = htab_elem_get_ptr(htab, elem);\n \t\tint cpu;\n \n \t\tfor_each_possible_cpu(cpu)\n \t\t\tbpf_obj_cancel_fields(\u0026htab-\u003emap, per_cpu_ptr(pptr, cpu));\n \t} else {\n-\t\tvoid map_value = htab_elem_value(elem, htab-\u003emap.key_size);\n+\t\tvoid map_value = htab_elem_value(htab, elem);\n \n \t\tbpf_obj_cancel_fields(\u0026htab-\u003emap, map_value);\n \t}\n@@ -869,8 +981,8 @@ static bool htab_lru_map_delete_node(void arg, struct bpf_lru_node node)\n \tstruct bucket b;\n \tint ret;\n \n-\ttgt_l = container_of(node, struct htab_elem, lru_node);\n-\tb = __select_bucket(htab, tgt_l-\u003ehash);\n+\ttgt_l = (struct htab_elem )container_of(node, struct htab_elem_lru, lru_node);\n+\tb = __select_bucket(htab, htab_elem_hash(htab, tgt_l));\n \thead = \u0026b-\u003ehead;\n \n \tret = htab_lock_bucket(b, \u0026flags);\n@@ -912,7 +1024,7 @@ static int htab_map_get_next_key(struct bpf_map map, void key, void next_key)\n \thead = select_bucket(htab, hash);\n \n \t/ lookup the key /\n-\tl = lookup_nulls_elem_raw(head, hash, key, key_size, htab-\u003en_buckets);\n+\tl = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab-\u003en_buckets);\n \n \tif (!l)\n \t\tgoto find_first_elem;\n@@ -923,7 +1035,7 @@ static int htab_map_get_next_key(struct bpf_map map, void key, void next_key)\n \n \tif (next_l) {\n \t\t/* if next elem in this hash list is non-zero, just return it /\n-\t\tmemcpy(next_key, next_l-\u003ekey, key_size);\n+\t\tmemcpy(next_key, htab_elem_key(htab, next_l), key_size);\n \t\treturn 0;\n \t}\n \n@@ -941,7 +1053,7 @@ static int htab_map_get_next_key(struct bpf_map map, void key, void next_key)\n \t\t\t\t\t struct htab_elem, hash_node);\n \t\tif (next_l) {\n \t\t\t/* if it's not empty, just return it /\n-\t\t\tmemcpy(next_key, next_l-\u003ekey, key_size);\n+\t\t\tmemcpy(next_key, htab_elem_key(htab, next_l), key_size);\n \t\t\treturn 0;\n \t\t}\n \t}\n@@ -955,7 +1067,7 @@ static void htab_elem_free(struct bpf_htab htab, struct htab_elem l)\n \tcheck_and_cancel_fields(htab, l);\n \n \tif (htab-\u003emap.map_type == BPF_MAP_TYPE_PERCPU_HASH)\n-\t\tbpf_mem_cache_free(\u0026htab-\u003epcpu_ma, l-\u003eptr_to_pptr);\n+\t\tbpf_mem_cache_free(\u0026htab-\u003epcpu_ma, htab_elem_get_ptr_to_pptr(l));\n \tbpf_mem_cache_free(\u0026htab-\u003ema, l);\n }\n \n@@ -965,7 +1077,7 @@ static void htab_put_fd_value(struct bpf_htab htab, struct htab_elem l)\n \tvoid ptr;\n \n \tif (map-\u003eops-\u003emap_fd_put_ptr) {\n-\t\tptr = fd_htab_map_get_ptr(map, l);\n+\t\tptr = fd_htab_map_get_ptr(htab, l);\n \t\tmap-\u003eops-\u003emap_fd_put_ptr(map, ptr, true);\n \t}\n }\n@@ -1117,10 +1229,10 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void key,\n \t\t}\n \t}\n \n-\tmemcpy(l_new-\u003ekey, key, key_size);\n+\tmemcpy(htab_elem_key(htab, l_new), key, key_size);\n \tif (percpu) {\n \t\tif (prealloc) {\n-\t\t\tpptr = htab_elem_get_ptr(l_new, key_size);\n+\t\t\tpptr = htab_elem_get_ptr(htab, l_new);\n \t\t} else {\n \t\t\t/ alloc_percpu zero-fills /\n \t\t\tvoid ptr = bpf_mem_cache_alloc(\u0026htab-\u003epcpu_ma);\n@@ -1130,26 +1242,26 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void key,\n \t\t\t\tl_new = ERR_PTR(-ENOMEM);\n \t\t\t\tgoto dec_count;\n \t\t\t}\n-\t\t\tl_new-\u003eptr_to_pptr = ptr;\n+\t\t\thtab_elem_set_ptr_to_pptr(l_new, ptr);\n \t\t\tpptr = (void __percpu *)ptr;\n \t\t}\n \n \t\tpcpu_init_value(htab, pptr, value, onallcpus, map_flags);\n \n \t\tif (!prealloc)\n-\t\t\thtab_elem_set_ptr(l_new, key_size, pptr);\n+\t\t\thtab_elem_set_ptr(htab, l_new, pptr);\n \t} else if (fd_htab_map_needs_adjust(htab)) {\n \t\tsize = round_up(size, 8);\n-\t\tmemcpy(htab_elem_value(l_new, key_size), value, size);\n+\t\tmemcpy(htab_elem_value(htab, l_new), value, size);\n \t} else if (map_flags \u0026 BPF_F_LOCK) {\n \t\tcopy_map_value_locked(\u0026htab-\u003emap,\n-\t\t\t\t htab_elem_value(l_new, key_size),\n+\t\t\t\t htab_elem_value(htab, l_new),\n \t\t\t\t value, false);\n \t} else {\n-\t\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(l_new, key_size), value);\n+\t\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(htab, l_new), value);\n \t}\n \n-\tl_new-\u003ehash = hash;\n+\thtab_elem_set_hash(htab, l_new, hash);\n \treturn l_new;\n dec_count:\n \tdec_elem_count(htab);\n@@ -1199,7 +1311,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value,\n \t\tif (unlikely(!btf_record_has_field(map-\u003erecord, BPF_SPIN_LOCK)))\n \t\t\treturn -EINVAL;\n \t\t/* find an element without taking the bucket lock /\n-\t\tl_old = lookup_nulls_elem_raw(head, hash, key, key_size,\n+\t\tl_old = lookup_nulls_elem_raw(htab, head, hash, key, key_size,\n \t\t\t\t\t htab-\u003en_buckets);\n \t\tret = check_flags(htab, l_old, map_flags);\n \t\tif (ret)\n@@ -1207,7 +1319,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value,\n \t\tif (l_old) {\n \t\t\t/* grab the element lock and update value in place /\n \t\t\tcopy_map_value_locked(map,\n-\t\t\t\t\t htab_elem_value(l_old, key_size),\n+\t\t\t\t\t htab_elem_value(htab, l_old),\n \t\t\t\t\t value, false);\n \t\t\treturn 0;\n \t\t}\n@@ -1221,7 +1333,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value,\n \tif (ret)\n \t\treturn ret;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n@@ -1235,7 +1347,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value,\n \t\t and update element in place\n \t\t /\n \t\tcopy_map_value_locked(map,\n-\t\t\t\t htab_elem_value(l_old, key_size),\n+\t\t\t\t htab_elem_value(htab, l_old),\n \t\t\t\t value, false);\n \t\tret = 0;\n \t\tgoto err;\n@@ -1275,7 +1387,7 @@ static void htab_lru_push_free(struct bpf_htab htab, struct htab_elem elem)\n {\n \tcheck_and_cancel_fields(htab, elem);\n \tbpf_map_dec_elem_count(\u0026htab-\u003emap);\n-\tbpf_lru_push_free(\u0026htab-\u003elru, \u0026elem-\u003elru_node);\n+\tbpf_lru_push_free(\u0026htab-\u003elru, htab_elem_lru_node(elem));\n }\n \n static long htab_lru_map_update_elem(struct bpf_map map, void key, void value,\n@@ -1310,13 +1422,13 @@ static long htab_lru_map_update_elem(struct bpf_map map, void key, void value\n \tl_new = prealloc_lru_pop(htab, key, hash);\n \tif (!l_new)\n \t\treturn -ENOMEM;\n-\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(l_new, map-\u003ekey_size), value);\n+\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(htab, l_new), value);\n \n \tret = htab_lock_bucket(b, \u0026flags);\n \tif (ret)\n \t\tgoto err_lock_bucket;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n@@ -1327,7 +1439,7 @@ static long htab_lru_map_update_elem(struct bpf_map map, void key, void value\n \t /\n \thlist_nulls_add_head_rcu(\u0026l_new-\u003ehash_node, head);\n \tif (l_old) {\n-\t\tbpf_lru_node_set_ref(\u0026l_new-\u003elru_node);\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l_new));\n \t\thlist_nulls_del_rcu(\u0026l_old-\u003ehash_node);\n \t}\n \tret = 0;\n@@ -1383,7 +1495,7 @@ static long htab_map_update_elem_in_place(struct bpf_map map, void key,\n \tif (ret)\n \t\treturn ret;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n@@ -1392,10 +1504,10 @@ static long htab_map_update_elem_in_place(struct bpf_map map, void key,\n \tif (l_old) {\n \t\t/ Update value in-place /\n \t\tif (percpu) {\n-\t\t\tpcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),\n+\t\t\tpcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old),\n \t\t\t\t\tvalue, onallcpus, map_flags);\n \t\t} else {\n-\t\t\tvoid inner_map_pptr = htab_elem_value(l_old, key_size);\n+\t\t\tvoid inner_map_pptr = htab_elem_value(htab, l_old);\n \n \t\t\told_map_ptr = inner_map_pptr;\n \t\t\tWRITE_ONCE(inner_map_pptr, (void *)value);\n@@ -1456,20 +1568,20 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map map, void key,\n \tif (ret)\n \t\tgoto err_lock_bucket;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n \t\tgoto err;\n \n \tif (l_old) {\n-\t\tbpf_lru_node_set_ref(\u0026l_old-\u003elru_node);\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l_old));\n \n \t\t/ per-cpu hash map can update value in-place /\n-\t\tpcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),\n+\t\tpcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old),\n \t\t\t\tvalue, onallcpus, map_flags);\n \t} else {\n-\t\tpcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size),\n+\t\tpcpu_init_value(htab, htab_elem_get_ptr(htab, l_new),\n \t\t\t\tvalue, onallcpus, map_flags);\n \t\thlist_nulls_add_head_rcu(\u0026l_new-\u003ehash_node, head);\n \t\tl_new = NULL;\n@@ -1480,7 +1592,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map map, void key,\n err_lock_bucket:\n \tif (l_new) {\n \t\tbpf_map_dec_elem_count(\u0026htab-\u003emap);\n-\t\tbpf_lru_push_free(\u0026htab-\u003elru, \u0026l_new-\u003elru_node);\n+\t\tbpf_lru_push_free(\u0026htab-\u003elru, htab_elem_lru_node(l_new));\n \t}\n \treturn ret;\n }\n@@ -1521,7 +1633,7 @@ static long htab_map_delete_elem(struct bpf_map map, void key)\n \tif (ret)\n \t\treturn ret;\n \n-\tl = lookup_elem_raw(head, hash, key, key_size);\n+\tl = lookup_elem_raw(htab, head, hash, key, key_size);\n \tif (l)\n \t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n \telse\n@@ -1556,7 +1668,7 @@ static long htab_lru_map_delete_elem(struct bpf_map map, void key)\n \tif (ret)\n \t\treturn ret;\n \n-\tl = lookup_elem_raw(head, hash, key, key_size);\n+\tl = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tif (l)\n \t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n@@ -1602,7 +1714,7 @@ static void htab_free_malloced_internal_structs(struct bpf_htab htab)\n \t\thlist_nulls_for_each_entry(l, n, head, hash_node) {\n \t\t\t/* We only free internal structs on uref dropping to zero /\n \t\t\tbpf_map_free_internal_structs(\u0026htab-\u003emap,\n-\t\t\t\t\t\t htab_elem_value(l, htab-\u003emap.key_size));\n+\t\t\t\t\t\t htab_elem_value(htab, l));\n \t\t}\n \t\tcond_resched_rcu();\n \t}\n@@ -1697,7 +1809,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map map, void key,\n \tif (ret)\n \t\treturn ret;\n \n-\tl = lookup_elem_raw(head, hash, key, key_size);\n+\tl = lookup_elem_raw(htab, head, hash, key, key_size);\n \tif (!l) {\n \t\tret = -ENOENT;\n \t\tgoto out_unlock;\n@@ -1708,14 +1820,14 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map map, void key,\n \t\tvoid __percpu pptr;\n \t\tint off = 0, cpu;\n \n-\t\tpptr = htab_elem_get_ptr(l, key_size);\n+\t\tpptr = htab_elem_get_ptr(htab, l);\n \t\tfor_each_possible_cpu(cpu) {\n \t\t\tcopy_map_value_long(\u0026htab-\u003emap, value + off, per_cpu_ptr(pptr, cpu));\n \t\t\tcheck_and_init_map_value(\u0026htab-\u003emap, value + off);\n \t\t\toff += roundup_value_size;\n \t\t}\n \t} else {\n-\t\tvoid src = htab_elem_value(l, map-\u003ekey_size);\n+\t\tvoid src = htab_elem_value(htab, l);\n \n \t\tif (flags \u0026 BPF_F_LOCK)\n \t\t\tcopy_map_value_locked(map, value, src, true);\n@@ -1898,13 +2010,13 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map,\n \t\tgoto next_batch;\n \n \thlist_nulls_for_each_entry_safe(l, n, head, hash_node) {\n-\t\tmemcpy(dst_key, l-\u003ekey, key_size);\n+\t\tmemcpy(dst_key, htab_elem_key(htab, l), key_size);\n \n \t\tif (is_percpu) {\n \t\t\tint off = 0, cpu;\n \t\t\tvoid __percpu pptr;\n \n-\t\t\tpptr = htab_elem_get_ptr(l, map-\u003ekey_size);\n+\t\t\tpptr = htab_elem_get_ptr(htab, l);\n \t\t\tif (elem_map_flags \u0026 BPF_F_CPU) {\n \t\t\t\tcpu = elem_map_flags \u003e\u003e 32;\n \t\t\t\tcopy_map_value(\u0026htab-\u003emap, dst_val, per_cpu_ptr(pptr, cpu));\n@@ -1918,7 +2030,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map,\n \t\t\t\t}\n \t\t\t}\n \t\t} else {\n-\t\t\tvalue = htab_elem_value(l, key_size);\n+\t\t\tvalue = htab_elem_value(htab, l);\n \t\t\tif (is_fd_htab(htab)) {\n \t\t\t\tstruct bpf_map inner_map = value;\n \n@@ -2183,12 +2295,12 @@ static int __bpf_hash_map_seq_show(struct seq_file seq, struct htab_elem elem)\n \t\tctx.meta = \u0026meta;\n \t\tctx.map = info-\u003emap;\n \t\tif (elem) {\n-\t\t\tctx.key = elem-\u003ekey;\n+\t\t\tctx.key = htab_elem_key(info-\u003ehtab, elem);\n \t\t\tif (!info-\u003epercpu_value_buf) {\n-\t\t\t\tctx.value = htab_elem_value(elem, map-\u003ekey_size);\n+\t\t\t\tctx.value = htab_elem_value(info-\u003ehtab, elem);\n \t\t\t} else {\n \t\t\t\troundup_value_size = round_up(map-\u003evalue_size, 8);\n-\t\t\t\tpptr = htab_elem_get_ptr(elem, map-\u003ekey_size);\n+\t\t\t\tpptr = htab_elem_get_ptr(info-\u003ehtab, elem);\n \t\t\t\tfor_each_possible_cpu(cpu) {\n \t\t\t\t\tcopy_map_value_long(map, info-\u003epercpu_value_buf + off,\n \t\t\t\t\t\t\t per_cpu_ptr(pptr, cpu));\n@@ -2293,13 +2405,13 @@ static long bpf_for_each_hash_elem(struct bpf_map map, bpf_callback_t callback_\n \t\trcu_read_lock();\n \t\thead = \u0026b-\u003ehead;\n \t\thlist_nulls_for_each_entry_safe(elem, n, head, hash_node) {\n-\t\t\tkey = elem-\u003ekey;\n+\t\t\tkey = htab_elem_key(htab, elem);\n \t\t\tif (is_percpu) {\n \t\t\t\t/* current cpu value for percpu map /\n-\t\t\t\tpptr = htab_elem_get_ptr(elem, map-\u003ekey_size);\n+\t\t\t\tpptr = htab_elem_get_ptr(htab, elem);\n \t\t\t\tval = this_cpu_ptr(pptr);\n \t\t\t} else {\n-\t\t\t\tval = htab_elem_value(elem, map-\u003ekey_size);\n+\t\t\t\tval = htab_elem_value(htab, elem);\n \t\t\t}\n \t\t\tnum_elems++;\n \t\t\tret = callback_fn((u64)(long)map, (u64)(long)key,\n@@ -2403,10 +2515,11 @@ const struct bpf_map_ops htab_lru_map_ops = {\n / Called from eBPF program /\n static void htab_percpu_map_lookup_elem(struct bpf_map map, void key)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l = __htab_map_lookup_elem(map, key);\n \n \tif (l)\n-\t\treturn this_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size));\n+\t\treturn this_cpu_ptr(htab_elem_get_ptr(htab, l));\n \telse\n \t\treturn NULL;\n }\n@@ -2414,6 +2527,7 @@ static void htab_percpu_map_lookup_elem(struct bpf_map map, void key)\n / inline bpf_map_lookup_elem() call for per-CPU hashmap /\n static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn insn = insn_buf;\n \n \tif (!bpf_jit_supports_percpu_insn())\n@@ -2424,7 +2538,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn\n \tinsn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \tinsn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3);\n \tinsn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0,\n-\t\t\t\toffsetof(struct htab_elem, key) + roundup(map-\u003ekey_size, 8));\n+\t\t\t\thtab-\u003ekey_offset + roundup(map-\u003ekey_size, 8));\n \tinsn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);\n \tinsn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);\n \n@@ -2433,6 +2547,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn\n \n static void htab_percpu_map_lookup_percpu_elem(struct bpf_map map, void key, u32 cpu)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l;\n \n \tif (cpu \u003e= nr_cpu_ids)\n@@ -2440,18 +2555,19 @@ static void htab_percpu_map_lookup_percpu_elem(struct bpf_map map, void key,\n \n \tl = __htab_map_lookup_elem(map, key);\n \tif (l)\n-\t\treturn per_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size), cpu);\n+\t\treturn per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu);\n \telse\n \t\treturn NULL;\n }\n \n static void htab_lru_percpu_map_lookup_elem(struct bpf_map map, void key)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l = __htab_map_lookup_elem(map, key);\n \n \tif (l) {\n-\t\tbpf_lru_node_set_ref(\u0026l-\u003elru_node);\n-\t\treturn this_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size));\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l));\n+\t\treturn this_cpu_ptr(htab_elem_get_ptr(htab, l));\n \t}\n \n \treturn NULL;\n@@ -2459,6 +2575,7 @@ static void htab_lru_percpu_map_lookup_elem(struct bpf_map map, void key)\n \n static void htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map map, void key, u32 cpu)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l;\n \n \tif (cpu \u003e= nr_cpu_ids)\n@@ -2466,8 +2583,8 @@ static void htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map map, void k\n \n \tl = __htab_map_lookup_elem(map, key);\n \tif (l) {\n-\t\tbpf_lru_node_set_ref(\u0026l-\u003elru_node);\n-\t\treturn per_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size), cpu);\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l));\n+\t\treturn per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu);\n \t}\n \n \treturn NULL;\n@@ -2475,6 +2592,7 @@ static void htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map map, void k\n \n int bpf_percpu_hash_copy(struct bpf_map map, void key, void value, u64 map_flags)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l;\n \tvoid __percpu pptr;\n \tint ret = -ENOENT;\n@@ -2494,7 +2612,7 @@ int bpf_percpu_hash_copy(struct bpf_map map, void key, void value, u64 map_fl\n \t/* We do not mark LRU map element here in order to not mess up\n \t * eviction heuristics when user space does a map walk.\n \t /\n-\tpptr = htab_elem_get_ptr(l, map-\u003ekey_size);\n+\tpptr = htab_elem_get_ptr(htab, l);\n \tif (map_flags \u0026 BPF_F_CPU) {\n \t\tcpu = map_flags \u003e\u003e 32;\n \t\tcopy_map_value(map, value, per_cpu_ptr(pptr, cpu));\n@@ -2532,6 +2650,7 @@ int bpf_percpu_hash_update(struct bpf_map map, void key, void value,\n static void htab_percpu_map_seq_show_elem(struct bpf_map map, void key,\n \t\t\t\t\t struct seq_file m)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l;\n \tvoid __percpu pptr;\n \tint cpu;\n@@ -2546,7 +2665,7 @@ static void htab_percpu_map_seq_show_elem(struct bpf_map map, void key,\n \n \tbtf_type_seq_show(map-\u003ebtf, map-\u003ebtf_key_type_id, key, m);\n \tseq_puts(m, \": {\\n\");\n-\tpptr = htab_elem_get_ptr(l, map-\u003ekey_size);\n+\tpptr = htab_elem_get_ptr(htab, l);\n \tfor_each_possible_cpu(cpu) {\n \t\tseq_printf(m, \"\\tcpu%d: \", cpu);\n \t\tbtf_type_seq_show(map-\u003ebtf, map-\u003ebtf_value_type_id,\n@@ -2620,7 +2739,7 @@ static void fd_htab_map_free(struct bpf_map map)\n \t\thead = select_bucket(htab, i);\n \n \t\thlist_nulls_for_each_entry_safe(l, n, head, hash_node) {\n-\t\t\tvoid ptr = fd_htab_map_get_ptr(map, l);\n+\t\t\tvoid ptr = fd_htab_map_get_ptr(htab, l);\n \n \t\t\tmap-\u003eops-\u003emap_fd_put_ptr(map, ptr, false);\n \t\t}\n@@ -2705,6 +2824,7 @@ static void htab_of_map_lookup_elem(struct bpf_map map, void key)\n static int htab_of_map_gen_lookup(struct bpf_map map,\n \t\t\t\t struct bpf_insn insn_buf)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn insn = insn_buf;\n \tconst int ret = BPF_REG_0;\n \n@@ -2713,7 +2833,7 @@ static int htab_of_map_gen_lookup(struct bpf_map map,\n \tinsn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \tinsn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2);\n \tinsn++ = BPF_ALU64_IMM(BPF_ADD, ret,\n-\t\t\t\toffsetof(struct htab_elem, key) +\n+\t\t\t\thtab-\u003ekey_offset +\n \t\t\t\tround_up(map-\u003ekey_size, 8));\n \tinsn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);\n \n@@ -3112,7 +3232,7 @@ static int rhtab_map_check_btf(struct bpf_map map, const struct btf btf,\n {\n \tstruct bpf_rhtab rhtab = container_of(map, struct bpf_rhtab, map);\n \n-\treturn bpf_ma_set_dtor(map, \u0026rhtab-\u003ema, rhtab_mem_dtor);\n+\treturn bpf_ma_set_dtor(map, \u0026rhtab-\u003ema, rhtab_mem_dtor, offsetof(struct rhtab_elem, data));\n }\n \n static void rhtab_map_free_internal_structs(struct bpf_map map)\n@@ -3533,3 +3653,16 @@ const struct bpf_map_ops rhtab_map_ops = {\n \t.map_btf_id = \u0026rhtab_map_btf_ids[0],\n \t.iter_seq_info = \u0026rhash_iter_seq_info,\n };\n+\n+size_t bpf_htab_map_meta_size(void)\n+{\n+\treturn sizeof(struct bpf_htab);\n+}\n+\n+void bpf_htab_map_meta_init(struct bpf_map inner_map_meta, struct bpf_map inner_map)\n+{\n+\tstruct bpf_htab inner_htab_meta = container_of(inner_map_meta, struct bpf_htab, map);\n+\tstruct bpf_htab inner_htab = container_of(inner_map, struct bpf_htab, map);\n+\n+\tinner_htab_meta-\u003ekey_offset = inner_htab-\u003ekey_offset;\n+}\ndiff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c\nindex d2cbab4bdf644a..b2db5d0c98bee2 100644\n--- a/kernel/bpf/map_in_map.c\n+++ b/kernel/bpf/map_in_map.c\n@@ -7,6 +7,15 @@\n \n #include \"map_in_map.h\"\n \n+static bool bpf_map_type_is_htab(enum bpf_map_type type)\n+{\n+\treturn type == BPF_MAP_TYPE_HASH \|\|\n+\t type == BPF_MAP_TYPE_PERCPU_HASH \|\|\n+\t type == BPF_MAP_TYPE_LRU_HASH \|\|\n+\t type == BPF_MAP_TYPE_LRU_PERCPU_HASH \|\|\n+\t type == BPF_MAP_TYPE_HASH_OF_MAPS;\n+}\n+\n struct bpf_map bpf_map_meta_alloc(int inner_map_ufd)\n {\n \tstruct bpf_map inner_map, inner_map_meta;\n@@ -29,6 +38,8 @@ struct bpf_map bpf_map_meta_alloc(int inner_map_ufd)\n \t/ In some cases verifier needs to access beyond just base map. /\n \tif (inner_map-\u003eops == \u0026array_map_ops \|\| inner_map-\u003eops == \u0026percpu_array_map_ops)\n \t\tinner_map_meta_size = sizeof(struct bpf_array);\n+\telse if (bpf_map_type_is_htab(inner_map-\u003emap_type))\n+\t\tinner_map_meta_size = bpf_htab_map_meta_size();\n \n \tinner_map_meta = kzalloc(inner_map_meta_size, GFP_USER);\n \tif (!inner_map_meta)\n@@ -70,6 +81,8 @@ struct bpf_map bpf_map_meta_alloc(int inner_map_ufd)\n \t\tinner_array_meta-\u003eindex_mask = inner_array-\u003eindex_mask;\n \t\tinner_array_meta-\u003eelem_size = inner_array-\u003eelem_size;\n \t\tinner_map_meta-\u003ebypass_spec_v1 = inner_map-\u003ebypass_spec_v1;\n+\t} else if (bpf_map_type_is_htab(inner_map-\u003emap_type)) {\n+\t\tbpf_htab_map_meta_init(inner_map_meta, inner_map);\n \t}\n \treturn inner_map_meta;\n }\ndiff --git a/kernel/bpf/map_in_map.h b/kernel/bpf/map_in_map.h\nindex 7d61602354de80..1edc205772d8cd 100644\n--- a/kernel/bpf/map_in_map.h\n+++ b/kernel/bpf/map_in_map.h\n@@ -15,5 +15,7 @@ void bpf_map_fd_get_ptr(struct bpf_map map, struct file map_file,\n \t\t\t int ufd);\n void bpf_map_fd_put_ptr(struct bpf_map map, void ptr, bool need_defer);\n u32 bpf_map_fd_sys_lookup_elem(void ptr);\n+size_t bpf_htab_map_meta_size(void);\n+void bpf_htab_map_meta_init(struct bpf_map inner_map_meta, struct bpf_map inner_map);\n \n #endif\ndiff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c\nindex 373c8d17ea557d..6bd4cb68c20c1c 100644\n--- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c\n+++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c\n@@ -114,7 +114,7 @@ static inline int check_hash(void)\n \tVERIFY(check_default_noinline(\u0026hash-\u003emap, map));\n \n \tVERIFY(hash-\u003en_buckets == MAX_ENTRIES);\n-\tVERIFY(hash-\u003eelem_size == 64);\n+\tVERIFY(hash-\u003eelem_size == 32);\n \n \tVERIFY(hash-\u003ecount.counter == 0);\n \tVERIFY(bpf_map_sum_elem_count(map) == 0);\n" }
2/1	2026/07/23 22:38	agent	patch-evaluator	0m Model: best-expensive Args: null Results: { "EnableConfigs": [ "BPF_SYSCALL" ], "FocusSymbols": [ "htab_map_alloc", "htab_map_update_elem", "htab_lru_map_update_elem", "__htab_lru_percpu_map_update_elem", "htab_map_delete_elem", "htab_lru_map_delete_elem", "__htab_map_lookup_and_delete_elem", "__htab_map_lookup_and_delete_batch", "bpf_map_meta_alloc", "htab_map_gen_lookup", "htab_lru_map_gen_lookup", "htab_percpu_map_gen_lookup", "htab_of_map_gen_lookup" ], "Reasoning": "The patch functionally modifies the memory layout and operations of BPF hash maps (including LRU and per-CPU variants). It introduces new structures and offsets for hash map elements, which affects allocation, updates, deletions, and JIT lookup generation. Fuzzing is needed to ensure memory safety and correctness of these core BPF map operations.", "WorthFuzzing": true } Instruction: You are an expert Linux kernel maintainer. Your job is to review a provided patch series and determine if it makes functional changes to the kernel that should be fuzzed. IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in your workspace. Do NOT rely on your internal knowledge of the kernel. You must actively use your code access tools to examine the actual source code and confirm any assumptions. Return WorthFuzzing=false if the patch only contains: - Modifications to Documentation/, Kconfig files, or code comments. - Purely decorative changes, such as logging (e.g., pr_err, printk) or tracepoints. - Changes to numeric constants or macros that do not functionally alter execution flow. - Code paths that are impossible to reach in virtualized environments like GCE or QEMU, even when utilizing software-emulated hardware (e.g., usb gadget, mac80211_hwsim). If it modifies reachable core kernel logic, drivers, or architectures, use your code search tools to verify the code can be executed, then return WorthFuzzing=true. When returning WorthFuzzing=true, you MUST ALSO: 1. Extract any specific kernel functions that should be heavily fuzzed into FocusSymbols. Avoid listing generic hot-path functions to prevent skewed test distributions. 2. Identify any specific CONFIG_ options required to properly test this new/modified feature. Go and look into the Kconfig files and check for ifdefs around the code, do not make assumptions. Do not list too generic configs (we already have them enabled). Only list those that specifically cover the modified code. List them in the EnableConfigs output array, and DO NOT add a 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4"). Prefer calling several tools at the same time to save round-trips. Use set-results tool to provide results of the analysis. It must be called exactly once before the final reply. Ignore results of this tool. Prompt: For your convenience, here is the diff of the changes: commit 243a270640e0dc901e4a78b599b0748d59bf7b63 Author: syz-cluster <triage@syzkaller.com> Date: Thu Jul 23 22:38:40 2026 +0000 syz-cluster: applied patch under review diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 9f394e1aa2e85b..b8c30d40295879 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -102,6 +102,7 @@ struct bpf_htab { bool use_percpu_counter; u32 n_buckets; /* number of hash buckets / u32 elem_size; / size of each element in bytes / + u32 key_offset; / offset of key in bytes / u32 hashrnd; }; @@ -117,18 +118,46 @@ struct htab_elem { }; }; }; + u32 hash; / Only when htab_has_hash() / +} __aligned(8); + +struct htab_elem_lru { union { - / pointer to per-cpu pointer / - void ptr_to_pptr; - struct bpf_lru_node lru_node; + struct hlist_nulls_node hash_node; + struct { + void padding; + union { + struct pcpu_freelist_node fnode; + struct htab_elem_lru batch_flink; + }; + }; }; + struct bpf_lru_node lru_node; u32 hash; - char key[] __aligned(8); -}; +} __aligned(8); + +/* Only for non-preallocated PCPU maps. Preallocated PCPU maps don't need + * ptr_to_pptr, and use htab_elem. + / +struct htab_elem_pcpu { + union { + struct hlist_nulls_node hash_node; + struct { + void padding; + union { + struct pcpu_freelist_node fnode; + struct htab_elem_pcpu batch_flink; + }; + }; + }; + void ptr_to_pptr; + u32 hash; /* Only when htab_has_hash() / +} __aligned(8); struct htab_btf_record { struct btf_record record; u32 key_size; + u32 key_offset; }; static inline bool htab_is_prealloc(const struct bpf_htab htab) @@ -136,6 +165,21 @@ static inline bool htab_is_prealloc(const struct bpf_htab htab) return !(htab->map.map_flags & BPF_F_NO_PREALLOC); } +static inline struct bpf_lru_node htab_elem_lru_node(struct htab_elem l) +{ + return &((struct htab_elem_lru )l)->lru_node; +} + +static inline void htab_elem_get_ptr_to_pptr(struct htab_elem l) +{ + return ((struct htab_elem_pcpu )l)->ptr_to_pptr; +} + +static inline void htab_elem_set_ptr_to_pptr(struct htab_elem l, void ptr) +{ + ((struct htab_elem_pcpu )l)->ptr_to_pptr = ptr; +} + static void htab_init_buckets(struct bpf_htab htab) { unsigned int i; @@ -183,25 +227,30 @@ static inline bool is_fd_htab(const struct bpf_htab htab) return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS; } -static inline void htab_elem_value(struct htab_elem l, u32 key_size) +static inline void htab_elem_key(struct bpf_htab htab, struct htab_elem l) +{ + return (void )l + htab->key_offset; +} + +static inline void htab_elem_value(struct bpf_htab htab, struct htab_elem l) { - return l->key + round_up(key_size, 8); + return htab_elem_key(htab, l) + round_up(htab->map.key_size, 8); } -static inline void htab_elem_set_ptr(struct htab_elem l, u32 key_size, +static inline void htab_elem_set_ptr(struct bpf_htab htab, struct htab_elem l, void __percpu pptr) { - (void __percpu )htab_elem_value(l, key_size) = pptr; + (void __percpu *)htab_elem_value(htab, l) = pptr; } -static inline void __percpu htab_elem_get_ptr(struct htab_elem l, u32 key_size) +static inline void __percpu htab_elem_get_ptr(struct bpf_htab htab, struct htab_elem l) { - return (void __percpu )htab_elem_value(l, key_size); + return (void __percpu *)htab_elem_value(htab, l); } -static void fd_htab_map_get_ptr(const struct bpf_map map, struct htab_elem l) +static void fd_htab_map_get_ptr(struct bpf_htab htab, struct htab_elem l) { - return (void *)htab_elem_value(l, map->key_size); + return (void *)htab_elem_value(htab, l); } static struct htab_elem get_htab_elem(struct bpf_htab htab, int i) @@ -209,6 +258,33 @@ static struct htab_elem get_htab_elem(struct bpf_htab htab, int i) return (struct htab_elem ) (htab->elems + i * (u64)htab->elem_size); } +static inline bool htab_has_hash(const struct bpf_htab htab) +{ + return htab_is_lru(htab) \|\| htab->map.key_size > 8; +} + +static inline u32 htab_elem_hash(struct bpf_htab htab, struct htab_elem l) +{ + if (htab_is_lru(htab)) + return ((struct htab_elem_lru )l)->hash; + else if (htab_is_percpu(htab) && !htab_is_prealloc(htab)) + return ((struct htab_elem_pcpu )l)->hash; + else + return l->hash; +} + +static inline void htab_elem_set_hash(struct bpf_htab htab, struct htab_elem l, u32 hash) +{ + if (!htab_has_hash(htab)) + return; + if (htab_is_lru(htab)) + ((struct htab_elem_lru )l)->hash = hash; + else if (htab_is_percpu(htab) && !htab_is_prealloc(htab)) + ((struct htab_elem_pcpu )l)->hash = hash; + else + l->hash = hash; +} + / Both percpu and fd htab support in-place update, so no need for * extra elem. LRU itself can remove the least used element, so * there is no need for an extra elem during map_update. @@ -231,7 +307,7 @@ static void htab_free_prealloced_internal_structs(struct bpf_htab htab) elem = get_htab_elem(htab, i); bpf_map_free_internal_structs(&htab->map, - htab_elem_value(elem, htab->map.key_size)); + htab_elem_value(htab, elem)); cond_resched(); } } @@ -254,7 +330,7 @@ static void htab_free_prealloced_fields(struct bpf_htab htab) elem = get_htab_elem(htab, i); if (htab_is_percpu(htab)) { - void __percpu pptr = htab_elem_get_ptr(elem, htab->map.key_size); + void __percpu pptr = htab_elem_get_ptr(htab, elem); int cpu; for_each_possible_cpu(cpu) { @@ -263,7 +339,7 @@ static void htab_free_prealloced_fields(struct bpf_htab htab) } } else { bpf_obj_free_fields(htab->map.record, - htab_elem_value(elem, htab->map.key_size)); + htab_elem_value(htab, elem)); cond_resched(); } cond_resched(); @@ -280,8 +356,7 @@ static void htab_free_elems(struct bpf_htab htab) for (i = 0; i < htab->map.max_entries; i++) { void __percpu pptr; - pptr = htab_elem_get_ptr(get_htab_elem(htab, i), - htab->map.key_size); + pptr = htab_elem_get_ptr(htab, get_htab_elem(htab, i)); free_percpu(pptr); cond_resched(); } @@ -308,8 +383,8 @@ static struct htab_elem prealloc_lru_pop(struct bpf_htab htab, void key, if (node) { bpf_map_inc_elem_count(&htab->map); - l = container_of(node, struct htab_elem, lru_node); - memcpy(l->key, key, htab->map.key_size); + l = (struct htab_elem )container_of(node, struct htab_elem_lru, lru_node); + memcpy(htab_elem_key(htab, l), key, htab->map.key_size); return l; } @@ -340,8 +415,7 @@ static int prealloc_init(struct bpf_htab htab) GFP_USER \| __GFP_NOWARN); if (!pptr) goto free_elems; - htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, - pptr); + htab_elem_set_ptr(htab, get_htab_elem(htab, i), pptr); cond_resched(); } @@ -349,8 +423,8 @@ static int prealloc_init(struct bpf_htab htab) if (htab_is_lru(htab)) err = bpf_lru_init(&htab->lru, htab->map.map_flags & BPF_F_NO_COMMON_LRU, - offsetof(struct htab_elem, hash) - - offsetof(struct htab_elem, lru_node), + offsetof(struct htab_elem_lru, hash) - + offsetof(struct htab_elem_lru, lru_node), htab_lru_map_delete_node, htab); else @@ -361,7 +435,7 @@ static int prealloc_init(struct bpf_htab htab) if (htab_is_lru(htab)) bpf_lru_populate(&htab->lru, htab->elems, - offsetof(struct htab_elem, lru_node), + offsetof(struct htab_elem_lru, lru_node), htab->elem_size, num_entries); else pcpu_freelist_populate(&htab->freelist, @@ -428,6 +502,25 @@ static int htab_map_alloc_check(union bpf_attr attr) BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) != offsetof(struct htab_elem, hash_node.pprev)); + / htab_elem pointers are used as base pointers for htab_elem_lru and + * htab_elem_pcpu. These structures must share a common initial sequence + * to ensure pointer casting between them is safe. + / + BUILD_BUG_ON(offsetof(struct htab_elem, hash_node) != + offsetof(struct htab_elem_lru, hash_node)); + BUILD_BUG_ON(offsetof(struct htab_elem, hash_node) != + offsetof(struct htab_elem_pcpu, hash_node)); + + BUILD_BUG_ON(offsetof(struct htab_elem, padding) != + offsetof(struct htab_elem_lru, padding)); + BUILD_BUG_ON(offsetof(struct htab_elem, padding) != + offsetof(struct htab_elem_pcpu, padding)); + + BUILD_BUG_ON(offsetof(struct htab_elem, fnode) != + offsetof(struct htab_elem_lru, fnode)); + BUILD_BUG_ON(offsetof(struct htab_elem, fnode) != + offsetof(struct htab_elem_pcpu, fnode)); + if (zero_seed && !capable(CAP_SYS_ADMIN)) / Guard against local DoS, and discourage production use. / return -EPERM; @@ -476,7 +569,7 @@ static void htab_mem_dtor(void obj, void ctx) if (IS_ERR_OR_NULL(hrec->record)) return; - map_value = htab_elem_value(elem, hrec->key_size); + map_value = (void )elem + hrec->key_offset + round_up(hrec->key_size, 8); bpf_obj_free_fields(hrec->record, map_value); } @@ -502,7 +595,7 @@ static void htab_dtor_ctx_free(void ctx) } static int bpf_ma_set_dtor(struct bpf_map map, struct bpf_mem_alloc ma, - void (dtor)(void , void )) + void (dtor)(void , void ), u32 key_offset) { struct htab_btf_record hrec; int err; @@ -515,6 +608,7 @@ static int bpf_ma_set_dtor(struct bpf_map map, struct bpf_mem_alloc ma, if (!hrec) return -ENOMEM; hrec->key_size = map->key_size; + hrec->key_offset = key_offset; hrec->record = btf_record_dup(map->record); if (IS_ERR(hrec->record)) { err = PTR_ERR(hrec->record); @@ -537,9 +631,9 @@ static int htab_map_check_btf(struct bpf_map map, const struct btf btf, * populated in htab_map_alloc(), so it will always appear as NULL. / if (htab_is_percpu(htab)) - return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor); + return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor, htab->key_offset); else - return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor); + return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor, htab->key_offset); } static struct bpf_map htab_map_alloc(union bpf_attr attr) @@ -583,8 +677,18 @@ static struct bpf_map htab_map_alloc(union bpf_attr attr) htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); - htab->elem_size = sizeof(struct htab_elem) + - round_up(htab->map.key_size, 8); + if (htab_is_lru(htab)) + htab->key_offset = sizeof(struct htab_elem_lru); + else if (percpu && !prealloc) + htab->key_offset = round_up(htab_has_hash(htab) ? + sizeof(struct htab_elem_pcpu) : + offsetof(struct htab_elem_pcpu, hash), 8); + else + htab->key_offset = round_up(htab_has_hash(htab) ? + sizeof(struct htab_elem) : + offsetof(struct htab_elem, hash), 8); + + htab->elem_size = htab->key_offset + round_up(htab->map.key_size, 8); if (percpu) htab->elem_size += sizeof(void ); else @@ -692,14 +796,16 @@ static inline struct hlist_nulls_head select_bucket(struct bpf_htab htab, u32 } /* this lookup function can only be called with bucket lock taken / -static struct htab_elem lookup_elem_raw(struct hlist_nulls_head head, u32 hash, +static struct htab_elem lookup_elem_raw(struct bpf_htab htab, + struct hlist_nulls_head head, u32 hash, void key, u32 key_size) { struct hlist_nulls_node n; struct htab_elem l; hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l->hash == hash && !memcmp(&l->key, key, key_size)) + if ((!htab_has_hash(htab) \|\| htab_elem_hash(htab, l) == hash) && + !memcmp(htab_elem_key(htab, l), key, key_size)) return l; return NULL; @@ -709,7 +815,8 @@ static struct htab_elem lookup_elem_raw(struct hlist_nulls_head head, u32 hash the unlikely event when elements moved from one bucket into another * while link list is being walked / -static struct htab_elem lookup_nulls_elem_raw(struct hlist_nulls_head head, +static struct htab_elem lookup_nulls_elem_raw(struct bpf_htab htab, + struct hlist_nulls_head head, u32 hash, void key, u32 key_size, u32 n_buckets) { @@ -718,7 +825,8 @@ static struct htab_elem lookup_nulls_elem_raw(struct hlist_nulls_head head, again: hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l->hash == hash && !memcmp(&l->key, key, key_size)) + if ((!htab_has_hash(htab) \|\| htab_elem_hash(htab, l) == hash) && + !memcmp(htab_elem_key(htab, l), key, key_size)) return l; if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1)))) @@ -747,17 +855,18 @@ static void __htab_map_lookup_elem(struct bpf_map map, void key) head = select_bucket(htab, hash); - l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + l = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab->n_buckets); return l; } static void htab_map_lookup_elem(struct bpf_map map, void key) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l = __htab_map_lookup_elem(map, key); if (l) - return htab_elem_value(l, map->key_size); + return htab_elem_value(htab, l); return NULL; } @@ -775,6 +884,7 @@ static void htab_map_lookup_elem(struct bpf_map map, void key) / static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct bpf_insn insn = insn_buf; const int ret = BPF_REG_0; @@ -783,7 +893,7 @@ static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); insn++ = BPF_ALU64_IMM(BPF_ADD, ret, - offsetof(struct htab_elem, key) + + htab->key_offset + round_up(map->key_size, 8)); return insn - insn_buf; } @@ -791,12 +901,13 @@ static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) static __always_inline void __htab_lru_map_lookup_elem(struct bpf_map map, void key, const bool mark) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l = __htab_map_lookup_elem(map, key); if (l) { if (mark) - bpf_lru_node_set_ref(&l->lru_node); - return htab_elem_value(l, map->key_size); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); + return htab_elem_value(htab, l); } return NULL; @@ -815,6 +926,7 @@ static void htab_lru_map_lookup_elem_sys(struct bpf_map map, void key) static int htab_lru_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct bpf_insn insn = insn_buf; const int ret = BPF_REG_0; const int ref_reg = BPF_REG_1; @@ -824,15 +936,15 @@ static int htab_lru_map_gen_lookup(struct bpf_map map, insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4); insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, - offsetof(struct htab_elem, lru_node) + + offsetof(struct htab_elem_lru, lru_node) + offsetof(struct bpf_lru_node, ref)); insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1); insn++ = BPF_ST_MEM(BPF_B, ret, - offsetof(struct htab_elem, lru_node) + + offsetof(struct htab_elem_lru, lru_node) + offsetof(struct bpf_lru_node, ref), 1); insn++ = BPF_ALU64_IMM(BPF_ADD, ret, - offsetof(struct htab_elem, key) + + htab->key_offset + round_up(map->key_size, 8)); return insn - insn_buf; } @@ -844,13 +956,13 @@ static void check_and_cancel_fields(struct bpf_htab htab, return; if (htab_is_percpu(htab)) { - void __percpu pptr = htab_elem_get_ptr(elem, htab->map.key_size); + void __percpu pptr = htab_elem_get_ptr(htab, elem); int cpu; for_each_possible_cpu(cpu) bpf_obj_cancel_fields(&htab->map, per_cpu_ptr(pptr, cpu)); } else { - void map_value = htab_elem_value(elem, htab->map.key_size); + void map_value = htab_elem_value(htab, elem); bpf_obj_cancel_fields(&htab->map, map_value); } @@ -869,8 +981,8 @@ static bool htab_lru_map_delete_node(void arg, struct bpf_lru_node node) struct bucket b; int ret; - tgt_l = container_of(node, struct htab_elem, lru_node); - b = __select_bucket(htab, tgt_l->hash); + tgt_l = (struct htab_elem )container_of(node, struct htab_elem_lru, lru_node); + b = __select_bucket(htab, htab_elem_hash(htab, tgt_l)); head = &b->head; ret = htab_lock_bucket(b, &flags); @@ -912,7 +1024,7 @@ static int htab_map_get_next_key(struct bpf_map map, void key, void next_key) head = select_bucket(htab, hash); / lookup the key / - l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + l = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab->n_buckets); if (!l) goto find_first_elem; @@ -923,7 +1035,7 @@ static int htab_map_get_next_key(struct bpf_map map, void key, void next_key) if (next_l) { /* if next elem in this hash list is non-zero, just return it / - memcpy(next_key, next_l->key, key_size); + memcpy(next_key, htab_elem_key(htab, next_l), key_size); return 0; } @@ -941,7 +1053,7 @@ static int htab_map_get_next_key(struct bpf_map map, void key, void next_key) struct htab_elem, hash_node); if (next_l) { /* if it's not empty, just return it / - memcpy(next_key, next_l->key, key_size); + memcpy(next_key, htab_elem_key(htab, next_l), key_size); return 0; } } @@ -955,7 +1067,7 @@ static void htab_elem_free(struct bpf_htab htab, struct htab_elem l) check_and_cancel_fields(htab, l); if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) - bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); + bpf_mem_cache_free(&htab->pcpu_ma, htab_elem_get_ptr_to_pptr(l)); bpf_mem_cache_free(&htab->ma, l); } @@ -965,7 +1077,7 @@ static void htab_put_fd_value(struct bpf_htab htab, struct htab_elem l) void ptr; if (map->ops->map_fd_put_ptr) { - ptr = fd_htab_map_get_ptr(map, l); + ptr = fd_htab_map_get_ptr(htab, l); map->ops->map_fd_put_ptr(map, ptr, true); } } @@ -1117,10 +1229,10 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void key, } } - memcpy(l_new->key, key, key_size); + memcpy(htab_elem_key(htab, l_new), key, key_size); if (percpu) { if (prealloc) { - pptr = htab_elem_get_ptr(l_new, key_size); + pptr = htab_elem_get_ptr(htab, l_new); } else { / alloc_percpu zero-fills / void ptr = bpf_mem_cache_alloc(&htab->pcpu_ma); @@ -1130,26 +1242,26 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void key, l_new = ERR_PTR(-ENOMEM); goto dec_count; } - l_new->ptr_to_pptr = ptr; + htab_elem_set_ptr_to_pptr(l_new, ptr); pptr = (void __percpu *)ptr; } pcpu_init_value(htab, pptr, value, onallcpus, map_flags); if (!prealloc) - htab_elem_set_ptr(l_new, key_size, pptr); + htab_elem_set_ptr(htab, l_new, pptr); } else if (fd_htab_map_needs_adjust(htab)) { size = round_up(size, 8); - memcpy(htab_elem_value(l_new, key_size), value, size); + memcpy(htab_elem_value(htab, l_new), value, size); } else if (map_flags & BPF_F_LOCK) { copy_map_value_locked(&htab->map, - htab_elem_value(l_new, key_size), + htab_elem_value(htab, l_new), value, false); } else { - copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value); + copy_map_value(&htab->map, htab_elem_value(htab, l_new), value); } - l_new->hash = hash; + htab_elem_set_hash(htab, l_new, hash); return l_new; dec_count: dec_elem_count(htab); @@ -1199,7 +1311,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value, if (unlikely(!btf_record_has_field(map->record, BPF_SPIN_LOCK))) return -EINVAL; /* find an element without taking the bucket lock / - l_old = lookup_nulls_elem_raw(head, hash, key, key_size, + l_old = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab->n_buckets); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1207,7 +1319,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value, if (l_old) { /* grab the element lock and update value in place / copy_map_value_locked(map, - htab_elem_value(l_old, key_size), + htab_elem_value(htab, l_old), value, false); return 0; } @@ -1221,7 +1333,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value, if (ret) return ret; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(htab, head, hash, key, key_size); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1235,7 +1347,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value, and update element in place / copy_map_value_locked(map, - htab_elem_value(l_old, key_size), + htab_elem_value(htab, l_old), value, false); ret = 0; goto err; @@ -1275,7 +1387,7 @@ static void htab_lru_push_free(struct bpf_htab htab, struct htab_elem elem) { check_and_cancel_fields(htab, elem); bpf_map_dec_elem_count(&htab->map); - bpf_lru_push_free(&htab->lru, &elem->lru_node); + bpf_lru_push_free(&htab->lru, htab_elem_lru_node(elem)); } static long htab_lru_map_update_elem(struct bpf_map map, void key, void value, @@ -1310,13 +1422,13 @@ static long htab_lru_map_update_elem(struct bpf_map map, void key, void value l_new = prealloc_lru_pop(htab, key, hash); if (!l_new) return -ENOMEM; - copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value); + copy_map_value(&htab->map, htab_elem_value(htab, l_new), value); ret = htab_lock_bucket(b, &flags); if (ret) goto err_lock_bucket; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(htab, head, hash, key, key_size); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1327,7 +1439,7 @@ static long htab_lru_map_update_elem(struct bpf_map map, void key, void value / hlist_nulls_add_head_rcu(&l_new->hash_node, head); if (l_old) { - bpf_lru_node_set_ref(&l_new->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l_new)); hlist_nulls_del_rcu(&l_old->hash_node); } ret = 0; @@ -1383,7 +1495,7 @@ static long htab_map_update_elem_in_place(struct bpf_map map, void key, if (ret) return ret; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(htab, head, hash, key, key_size); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1392,10 +1504,10 @@ static long htab_map_update_elem_in_place(struct bpf_map map, void key, if (l_old) { / Update value in-place / if (percpu) { - pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), + pcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old), value, onallcpus, map_flags); } else { - void inner_map_pptr = htab_elem_value(l_old, key_size); + void inner_map_pptr = htab_elem_value(htab, l_old); old_map_ptr = inner_map_pptr; WRITE_ONCE(inner_map_pptr, (void *)value); @@ -1456,20 +1568,20 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map map, void key, if (ret) goto err_lock_bucket; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(htab, head, hash, key, key_size); ret = check_flags(htab, l_old, map_flags); if (ret) goto err; if (l_old) { - bpf_lru_node_set_ref(&l_old->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l_old)); / per-cpu hash map can update value in-place / - pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), + pcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old), value, onallcpus, map_flags); } else { - pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size), + pcpu_init_value(htab, htab_elem_get_ptr(htab, l_new), value, onallcpus, map_flags); hlist_nulls_add_head_rcu(&l_new->hash_node, head); l_new = NULL; @@ -1480,7 +1592,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map map, void key, err_lock_bucket: if (l_new) { bpf_map_dec_elem_count(&htab->map); - bpf_lru_push_free(&htab->lru, &l_new->lru_node); + bpf_lru_push_free(&htab->lru, htab_elem_lru_node(l_new)); } return ret; } @@ -1521,7 +1633,7 @@ static long htab_map_delete_elem(struct bpf_map map, void key) if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(htab, head, hash, key, key_size); if (l) hlist_nulls_del_rcu(&l->hash_node); else @@ -1556,7 +1668,7 @@ static long htab_lru_map_delete_elem(struct bpf_map map, void key) if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(htab, head, hash, key, key_size); if (l) hlist_nulls_del_rcu(&l->hash_node); @@ -1602,7 +1714,7 @@ static void htab_free_malloced_internal_structs(struct bpf_htab htab) hlist_nulls_for_each_entry(l, n, head, hash_node) { /* We only free internal structs on uref dropping to zero / bpf_map_free_internal_structs(&htab->map, - htab_elem_value(l, htab->map.key_size)); + htab_elem_value(htab, l)); } cond_resched_rcu(); } @@ -1697,7 +1809,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map map, void key, if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(htab, head, hash, key, key_size); if (!l) { ret = -ENOENT; goto out_unlock; @@ -1708,14 +1820,14 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map map, void key, void __percpu pptr; int off = 0, cpu; - pptr = htab_elem_get_ptr(l, key_size); + pptr = htab_elem_get_ptr(htab, l); for_each_possible_cpu(cpu) { copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu)); check_and_init_map_value(&htab->map, value + off); off += roundup_value_size; } } else { - void src = htab_elem_value(l, map->key_size); + void src = htab_elem_value(htab, l); if (flags & BPF_F_LOCK) copy_map_value_locked(map, value, src, true); @@ -1898,13 +2010,13 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map, goto next_batch; hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { - memcpy(dst_key, l->key, key_size); + memcpy(dst_key, htab_elem_key(htab, l), key_size); if (is_percpu) { int off = 0, cpu; void __percpu pptr; - pptr = htab_elem_get_ptr(l, map->key_size); + pptr = htab_elem_get_ptr(htab, l); if (elem_map_flags & BPF_F_CPU) { cpu = elem_map_flags >> 32; copy_map_value(&htab->map, dst_val, per_cpu_ptr(pptr, cpu)); @@ -1918,7 +2030,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map, } } } else { - value = htab_elem_value(l, key_size); + value = htab_elem_value(htab, l); if (is_fd_htab(htab)) { struct bpf_map inner_map = value; @@ -2183,12 +2295,12 @@ static int __bpf_hash_map_seq_show(struct seq_file seq, struct htab_elem elem) ctx.meta = &meta; ctx.map = info->map; if (elem) { - ctx.key = elem->key; + ctx.key = htab_elem_key(info->htab, elem); if (!info->percpu_value_buf) { - ctx.value = htab_elem_value(elem, map->key_size); + ctx.value = htab_elem_value(info->htab, elem); } else { roundup_value_size = round_up(map->value_size, 8); - pptr = htab_elem_get_ptr(elem, map->key_size); + pptr = htab_elem_get_ptr(info->htab, elem); for_each_possible_cpu(cpu) { copy_map_value_long(map, info->percpu_value_buf + off, per_cpu_ptr(pptr, cpu)); @@ -2293,13 +2405,13 @@ static long bpf_for_each_hash_elem(struct bpf_map map, bpf_callback_t callback_ rcu_read_lock(); head = &b->head; hlist_nulls_for_each_entry_safe(elem, n, head, hash_node) { - key = elem->key; + key = htab_elem_key(htab, elem); if (is_percpu) { /* current cpu value for percpu map / - pptr = htab_elem_get_ptr(elem, map->key_size); + pptr = htab_elem_get_ptr(htab, elem); val = this_cpu_ptr(pptr); } else { - val = htab_elem_value(elem, map->key_size); + val = htab_elem_value(htab, elem); } num_elems++; ret = callback_fn((u64)(long)map, (u64)(long)key, @@ -2403,10 +2515,11 @@ const struct bpf_map_ops htab_lru_map_ops = { / Called from eBPF program / static void htab_percpu_map_lookup_elem(struct bpf_map map, void key) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l = __htab_map_lookup_elem(map, key); if (l) - return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); + return this_cpu_ptr(htab_elem_get_ptr(htab, l)); else return NULL; } @@ -2414,6 +2527,7 @@ static void htab_percpu_map_lookup_elem(struct bpf_map map, void key) / inline bpf_map_lookup_elem() call for per-CPU hashmap / static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct bpf_insn insn = insn_buf; if (!bpf_jit_supports_percpu_insn()) @@ -2424,7 +2538,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3); insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, - offsetof(struct htab_elem, key) + roundup(map->key_size, 8)); + htab->key_offset + roundup(map->key_size, 8)); insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); insn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); @@ -2433,6 +2547,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn static void htab_percpu_map_lookup_percpu_elem(struct bpf_map map, void key, u32 cpu) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l; if (cpu >= nr_cpu_ids) @@ -2440,18 +2555,19 @@ static void htab_percpu_map_lookup_percpu_elem(struct bpf_map map, void key, l = __htab_map_lookup_elem(map, key); if (l) - return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); + return per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu); else return NULL; } static void htab_lru_percpu_map_lookup_elem(struct bpf_map map, void key) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l = __htab_map_lookup_elem(map, key); if (l) { - bpf_lru_node_set_ref(&l->lru_node); - return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); + return this_cpu_ptr(htab_elem_get_ptr(htab, l)); } return NULL; @@ -2459,6 +2575,7 @@ static void htab_lru_percpu_map_lookup_elem(struct bpf_map map, void key) static void htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map map, void key, u32 cpu) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l; if (cpu >= nr_cpu_ids) @@ -2466,8 +2583,8 @@ static void htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map map, void k l = __htab_map_lookup_elem(map, key); if (l) { - bpf_lru_node_set_ref(&l->lru_node); - return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); + return per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu); } return NULL; @@ -2475,6 +2592,7 @@ static void htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map map, void k int bpf_percpu_hash_copy(struct bpf_map map, void key, void value, u64 map_flags) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l; void __percpu pptr; int ret = -ENOENT; @@ -2494,7 +2612,7 @@ int bpf_percpu_hash_copy(struct bpf_map map, void key, void value, u64 map_fl /* We do not mark LRU map element here in order to not mess up * eviction heuristics when user space does a map walk. / - pptr = htab_elem_get_ptr(l, map->key_size); + pptr = htab_elem_get_ptr(htab, l); if (map_flags & BPF_F_CPU) { cpu = map_flags >> 32; copy_map_value(map, value, per_cpu_ptr(pptr, cpu)); @@ -2532,6 +2650,7 @@ int bpf_percpu_hash_update(struct bpf_map map, void key, void value, static void htab_percpu_map_seq_show_elem(struct bpf_map map, void key, struct seq_file m) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l; void __percpu pptr; int cpu; @@ -2546,7 +2665,7 @@ static void htab_percpu_map_seq_show_elem(struct bpf_map map, void key, btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); seq_puts(m, ": {\n"); - pptr = htab_elem_get_ptr(l, map->key_size); + pptr = htab_elem_get_ptr(htab, l); for_each_possible_cpu(cpu) { seq_printf(m, "\tcpu%d: ", cpu); btf_type_seq_show(map->btf, map->btf_value_type_id, @@ -2620,7 +2739,7 @@ static void fd_htab_map_free(struct bpf_map map) head = select_bucket(htab, i); hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { - void ptr = fd_htab_map_get_ptr(map, l); + void ptr = fd_htab_map_get_ptr(htab, l); map->ops->map_fd_put_ptr(map, ptr, false); } @@ -2705,6 +2824,7 @@ static void htab_of_map_lookup_elem(struct bpf_map map, void key) static int htab_of_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct bpf_insn insn = insn_buf; const int ret = BPF_REG_0; @@ -2713,7 +2833,7 @@ static int htab_of_map_gen_lookup(struct bpf_map map, insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2); insn++ = BPF_ALU64_IMM(BPF_ADD, ret, - offsetof(struct htab_elem, key) + + htab->key_offset + round_up(map->key_size, 8)); insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0); @@ -3112,7 +3232,7 @@ static int rhtab_map_check_btf(struct bpf_map map, const struct btf btf, { struct bpf_rhtab rhtab = container_of(map, struct bpf_rhtab, map); - return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor); + return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor, offsetof(struct rhtab_elem, data)); } static void rhtab_map_free_internal_structs(struct bpf_map map) @@ -3533,3 +3653,16 @@ const struct bpf_map_ops rhtab_map_ops = { .map_btf_id = &rhtab_map_btf_ids[0], .iter_seq_info = &rhash_iter_seq_info, }; + +size_t bpf_htab_map_meta_size(void) +{ + return sizeof(struct bpf_htab); +} + +void bpf_htab_map_meta_init(struct bpf_map inner_map_meta, struct bpf_map inner_map) +{ + struct bpf_htab inner_htab_meta = container_of(inner_map_meta, struct bpf_htab, map); + struct bpf_htab inner_htab = container_of(inner_map, struct bpf_htab, map); + + inner_htab_meta->key_offset = inner_htab->key_offset; +} diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index d2cbab4bdf644a..b2db5d0c98bee2 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -7,6 +7,15 @@ #include "map_in_map.h" +static bool bpf_map_type_is_htab(enum bpf_map_type type) +{ + return type == BPF_MAP_TYPE_HASH \|\| + type == BPF_MAP_TYPE_PERCPU_HASH \|\| + type == BPF_MAP_TYPE_LRU_HASH \|\| + type == BPF_MAP_TYPE_LRU_PERCPU_HASH \|\| + type == BPF_MAP_TYPE_HASH_OF_MAPS; +} + struct bpf_map bpf_map_meta_alloc(int inner_map_ufd) { struct bpf_map inner_map, inner_map_meta; @@ -29,6 +38,8 @@ struct bpf_map bpf_map_meta_alloc(int inner_map_ufd) / In some cases verifier needs to access beyond just base map. / if (inner_map->ops == &array_map_ops \|\| inner_map->ops == &percpu_array_map_ops) inner_map_meta_size = sizeof(struct bpf_array); + else if (bpf_map_type_is_htab(inner_map->map_type)) + inner_map_meta_size = bpf_htab_map_meta_size(); inner_map_meta = kzalloc(inner_map_meta_size, GFP_USER); if (!inner_map_meta) @@ -70,6 +81,8 @@ struct bpf_map bpf_map_meta_alloc(int inner_map_ufd) inner_array_meta->index_mask = inner_array->index_mask; inner_array_meta->elem_size = inner_array->elem_size; inner_map_meta->bypass_spec_v1 = inner_map->bypass_spec_v1; + } else if (bpf_map_type_is_htab(inner_map->map_type)) { + bpf_htab_map_meta_init(inner_map_meta, inner_map); } return inner_map_meta; } diff --git a/kernel/bpf/map_in_map.h b/kernel/bpf/map_in_map.h index 7d61602354de80..1edc205772d8cd 100644 --- a/kernel/bpf/map_in_map.h +++ b/kernel/bpf/map_in_map.h @@ -15,5 +15,7 @@ void bpf_map_fd_get_ptr(struct bpf_map map, struct file map_file, int ufd); void bpf_map_fd_put_ptr(struct bpf_map map, void ptr, bool need_defer); u32 bpf_map_fd_sys_lookup_elem(void ptr); +size_t bpf_htab_map_meta_size(void); +void bpf_htab_map_meta_init(struct bpf_map inner_map_meta, struct bpf_map inner_map); #endif diff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c index 373c8d17ea557d..6bd4cb68c20c1c 100644 --- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c +++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c @@ -114,7 +114,7 @@ static inline int check_hash(void) VERIFY(check_default_noinline(&hash->map, map)); VERIFY(hash->n_buckets == MAX_ENTRIES); - VERIFY(hash->elem_size == 64); + VERIFY(hash->elem_size == 32); VERIFY(hash->count.counter == 0); VERIFY(bpf_map_sum_elem_count(map) == 0);
3/2	2026/07/23 22:38	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 18572 output: 37 thoughts: 583 Thoughts: Analyzing the BPF Hash Table Optimization Patch Okay, so I've got this patch – seems like a pretty significant one – touching `kernel/bpf/hashtab.c` and `kernel/bpf/map_in_map.c`. From the description, it looks like they're optimizing the memory layout of BPF hash tables, splitting `struct htab_elem` into multiple specialized structures (`htab_elem`, `htab_elem_lru`, `htab_elem_pcpu`) and also adding `key_offset` to `struct bpf_htab`. This is a functional change to the core hash map implementation, and that immediately tells me we need to treat this as high-priority for fuzzing. It's a prime target for uncovering subtle memory corruption or concurrency issues. My immediate next step is to drill down into the details of the code. Let's see what's been added, modified, or changed. I'm going to look closely at these new or changed functions: `htab_elem_lru_node`, `htab_elem_get_ptr_to_pptr`, `htab_elem_set_ptr_to_pptr`, `htab_elem_key`, `htab_elem_value`, `htab_elem_set_ptr`, `htab_elem_get_ptr`, `fd_htab_map_get_ptr`, `htab_has_hash`, `htab_elem_hash`, `htab_elem_set_hash`, `htab_map_alloc_check`, `htab_map_alloc`, `htab_map_gen_lookup`, `htab_lru_map_gen_lookup`, `htab_map_update_elem`, `htab_lru_map_update_elem`, `__htab_lru_percpu_map_update_elem`, `htab_map_delete_elem`, `htab_lru_map_delete_elem`, `__htab_map_lookup_and_delete_elem`, `__htab_map_lookup_and_delete_batch`, `bpf_percpu_hash_copy`, `htab_of_map_gen_lookup`, `bpf_htab_map_meta_size`, `bpf_htab_map_meta_init`, and `bpf_map_meta_alloc`. Any one of these, if mishandled, could have significant security implications. Now, let's nail down the configuration dependencies. `CONFIG_BPF_SYSCALL` will be the primary enabler for BPF, of course. Given the focus on hash maps, I'm going to quickly check if there are any specific configuration flags for BPF hash maps themselves. No dedicated config options, which makes sense - the hash table implementation is core BPF. I need to confirm these functions' interactions with the existing BPF hash map infrastructure. Time to search the source for `BPF_MAP_TYPE_HASH` and `BPF_MAP_TYPE_LRU_HASH` within `kernel/bpf/hashtab.c` to see how those map types are integrated with the new code. This will help understand the breadth of impact of this change.
4/2	2026/07/23 22:38	tool	grepper	0m Args: { "Expression": "BPF_MAP_TYPE_HASH", "PathPrefix": "kernel/bpf/hashtab.c" } Results: { "Output": "kernel/bpf/hashtab.c=225=static inline bool is_fd_htab(const struct bpf_htab *htab)\nkernel/bpf/hashtab.c-226-{\nkernel/bpf/hashtab.c:227:\treturn htab-\u003emap.map_type == BPF_MAP_TYPE_HASH_OF_MAPS;\nkernel/bpf/hashtab.c-228-}\n" }
5/2	2026/07/23 22:38	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 20021 output: 31 thoughts: 41 Thoughts: Here's my attempt at summarizing the thought process, tailored for an expert audience: Investigating Potential BPF Configuration Nuances Okay, before I dive too deep into the implications here, I need to take a quick peek at the BPF internals. Specifically, I should check `kernel/bpf/hashtab.c` and `kernel/bpf/map_in_map.c`. Those files are likely to contain the crucial low-level details regarding potential configurations or limitations that I need to be aware of. I'm looking for any specific configurations that might impact the behavior or performance I'm considering. Gotta make sure I'm not overlooking some subtle detail buried within those implementations. That level of detail is critical for fully understanding the landscape here.
6/2	2026/07/23 22:38	tool	grepper	0m Error: no matches Args: { "Expression": "CONFIG_", "PathPrefix": "kernel/bpf/hashtab.c" } Results: { "error": "no matches" }
7/2	2026/07/23 22:38	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 20267 output: 34 thoughts: 24 Thoughts: Here's my thought process, as if I were considering this myself: Investigating Configuration Dependencies Okay, it seems there are no specific configuration details explicitly defined within `hashtab.c`. That's not entirely unexpected, depending on the system design, but still good to confirm. Now, if the configuration isn't in the obvious place, where else could it be? The structure often dictates the organization of code. My instincts are telling me to look at `map_in_map.c`. It's possible that the configuration parameters are somehow managed within the nested map structure itself, or at least referenced from there. Let's see what we can dig up there.
8/2	2026/07/23 22:38	tool	grepper	0m Error: no matches Args: { "Expression": "CONFIG_", "PathPrefix": "kernel/bpf/map_in_map.c" } Results: { "error": "no matches" }
9/2	2026/07/23 22:38	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 20482 output: 278 thoughts: 1140 Thoughts: Analyzing BPF Hash Map Operations for Fuzzing Okay, first things first, this is definitely related to `CONFIG_BPF_SYSCALL`, that's the key enabling configuration. Let's dig in and figure out the most important functions to target for fuzzing. This patch is clearly touching BPF hash map operations. My initial thought is to grab all the relevant functions, and then refine. I'll need to focus on: `htab_map_alloc`, `htab_map_update_elem`, `htab_lru_map_update_elem`, `__htab_lru_percpu_map_update_elem`, `htab_map_delete_elem`, `htab_lru_map_delete_elem`, `__htab_map_lookup_and_delete_elem`, `__htab_map_lookup_and_delete_batch`, `bpf_map_meta_alloc`, and the various lookup functions: `htab_map_gen_lookup`, `htab_lru_map_gen_lookup`, `htab_percpu_map_gen_lookup`, and `htab_of_map_gen_lookup`. These seem like the core entry points for the changes. Alright, I'm going to refine that list a little. Wait, I remember that `htab_map_update_elem` is a pretty important hot-path function, the specific implementation for hash maps, so it's a prime target, especially considering fuzzing BPF map operations is standard practice. After that initial list I'll keep the ones I've listed previously and focus on the `bpf_map_meta_alloc` because that could be a significant change and thus a potential target for fuzzing. Now, let's see, `htab_map_update_elem` is static, which is usually not ideal for fuzzing. However, it's assigned to `map_ops`, so Syzkaller should be able to target it through the `bpf()` syscall. Yes, `FocusSymbols` are specifically those that can be targeted by syzkaller or are critical for coverage. So, the definitive list for my analysis now includes: `htab_map_alloc`, `htab_map_update_elem`, `htab_lru_map_update_elem`, `__htab_lru_percpu_map_update_elem`, `htab_map_delete_elem`, `htab_lru_map_delete_elem`, `__htab_map_lookup_and_delete_elem`, `__htab_map_lookup_and_delete_batch`, `bpf_map_meta_alloc`, `htab_map_gen_lookup`, `htab_lru_map_gen_lookup`, `htab_percpu_map_gen_lookup`, and `htab_of_map_gen_lookup`. I'll make sure to note these. Just verifying that `htab_map_gen_lookup` is indeed present in the diff. Good. Yes, this is definitely worth fuzzing. The patch is changing the memory layout and operations of BPF hash maps, including the LRU and per-CPU variants. It introduces new structures and offsets which in turn affects crucial operations like allocation, updates, deletions, and JIT lookup generation. Fuzzing is essential here to ensure memory safety and the correctness of these core BPF map operations given the substantial changes. My final settings will be: `WorthFuzzing: true`, `EnableConfigs: ["BPF_SYSCALL"]`, `FocusSymbols: ["htab_map_alloc", "htab_map_update_elem", "htab_lru_map_update_elem", "__htab_lru_percpu_map_update_elem", "htab_map_delete_elem", "htab_lru_map_delete_elem", "__htab_map_lookup_and_delete_elem", "__htab_map_lookup_and_delete_batch", "bpf_map_meta_alloc", "htab_map_gen_lookup", "htab_lru_map_gen_lookup", "htab_percpu_map_gen_lookup", "htab_of_map_gen_lookup"]`, and `Reasoning: "The patch functionally modifies the memory layout and operations of BPF hash maps (including LRU and per-CPU variants). It introduces new structures and offsets for hash map elements, which affects allocation, updates, deletions, and JIT lookup generation. Fuzzing is needed to ensure memory safety and correctness of these core BPF map operations."`
10/2	2026/07/23 22:39	tool	set-results	0m Args: { "EnableConfigs": [ "BPF_SYSCALL" ], "FocusSymbols": [ "htab_map_alloc", "htab_map_update_elem", "htab_lru_map_update_elem", "__htab_lru_percpu_map_update_elem", "htab_map_delete_elem", "htab_lru_map_delete_elem", "__htab_map_lookup_and_delete_elem", "__htab_map_lookup_and_delete_batch", "bpf_map_meta_alloc", "htab_map_gen_lookup", "htab_lru_map_gen_lookup", "htab_percpu_map_gen_lookup", "htab_of_map_gen_lookup" ], "Reasoning": "The patch functionally modifies the memory layout and operations of BPF hash maps (including LRU and per-CPU variants). It introduces new structures and offsets for hash map elements, which affects allocation, updates, deletions, and JIT lookup generation. Fuzzing is needed to ensure memory safety and correctness of these core BPF map operations.", "WorthFuzzing": true } Results: { "EnableConfigs": [ "BPF_SYSCALL" ], "FocusSymbols": [ "htab_map_alloc", "htab_map_update_elem", "htab_lru_map_update_elem", "__htab_lru_percpu_map_update_elem", "htab_map_delete_elem", "htab_lru_map_delete_elem", "__htab_map_lookup_and_delete_elem", "__htab_map_lookup_and_delete_batch", "bpf_map_meta_alloc", "htab_map_gen_lookup", "htab_lru_map_gen_lookup", "htab_percpu_map_gen_lookup", "htab_of_map_gen_lookup" ], "Reasoning": "The patch functionally modifies the memory layout and operations of BPF hash maps (including LRU and per-CPU variants). It introduces new structures and offsets for hash map elements, which affects allocation, updates, deletions, and JIT lookup generation. Fuzzing is needed to ensure memory safety and correctness of these core BPF map operations.", "WorthFuzzing": true }

Trajectory: (Expand/Collapse All)

Seq

Timestamp

Type

Name

Duration

0/0

2026/07/23 22:38

flow

patch-triage

Args:

null

Results:

{
  "EnableConfigs": [
    "BPF_SYSCALL"
  ],
  "FocusSymbols": [
    "htab_map_alloc",
    "htab_map_update_elem",
    "htab_lru_map_update_elem",
    "__htab_lru_percpu_map_update_elem",
    "htab_map_delete_elem",
    "htab_lru_map_delete_elem",
    "__htab_map_lookup_and_delete_elem",
    "__htab_map_lookup_and_delete_batch",
    "bpf_map_meta_alloc",
    "htab_map_gen_lookup",
    "htab_lru_map_gen_lookup",
    "htab_percpu_map_gen_lookup",
    "htab_of_map_gen_lookup"
  ],
  "Reasoning": "The patch functionally modifies the memory layout and operations of BPF hash maps (including LRU and per-CPU variants). It introduces new structures and offsets for hash map elements, which affects allocation, updates, deletions, and JIT lookup generation. Fuzzing is needed to ensure memory safety and correctness of these core BPF map operations.",
  "WorthFuzzing": true
}

1/1

2026/07/23 22:38

action

read-patch-diff

Args:

null

Results:

{
  "PatchDiff": "commit 243a270640e0dc901e4a78b599b0748d59bf7b63\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate:   Thu Jul 23 22:38:40 2026 +0000\n\n    syz-cluster: applied patch under review\n\ndiff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c\nindex 9f394e1aa2e85b..b8c30d40295879 100644\n--- a/kernel/bpf/hashtab.c\n+++ b/kernel/bpf/hashtab.c\n@@ -102,6 +102,7 @@ struct bpf_htab {\n \tbool use_percpu_counter;\n \tu32 n_buckets;\t/* number of hash buckets */\n \tu32 elem_size;\t/* size of each element in bytes */\n+\tu32 key_offset;\t/* offset of key in bytes */\n \tu32 hashrnd;\n };\n \n@@ -117,18 +118,46 @@ struct htab_elem {\n \t\t\t};\n \t\t};\n \t};\n+\tu32 hash; /* Only when htab_has_hash() */\n+} __aligned(8);\n+\n+struct htab_elem_lru {\n \tunion {\n-\t\t/* pointer to per-cpu pointer */\n-\t\tvoid *ptr_to_pptr;\n-\t\tstruct bpf_lru_node lru_node;\n+\t\tstruct hlist_nulls_node hash_node;\n+\t\tstruct {\n+\t\t\tvoid *padding;\n+\t\t\tunion {\n+\t\t\t\tstruct pcpu_freelist_node fnode;\n+\t\t\t\tstruct htab_elem_lru *batch_flink;\n+\t\t\t};\n+\t\t};\n \t};\n+\tstruct bpf_lru_node lru_node;\n \tu32 hash;\n-\tchar key[] __aligned(8);\n-};\n+} __aligned(8);\n+\n+/* Only for non-preallocated PCPU maps. Preallocated PCPU maps don't need\n+ * ptr_to_pptr, and use htab_elem.\n+ */\n+struct htab_elem_pcpu {\n+\tunion {\n+\t\tstruct hlist_nulls_node hash_node;\n+\t\tstruct {\n+\t\t\tvoid *padding;\n+\t\t\tunion {\n+\t\t\t\tstruct pcpu_freelist_node fnode;\n+\t\t\t\tstruct htab_elem_pcpu *batch_flink;\n+\t\t\t};\n+\t\t};\n+\t};\n+\tvoid *ptr_to_pptr;\n+\tu32 hash; /* Only when htab_has_hash() */\n+} __aligned(8);\n \n struct htab_btf_record {\n \tstruct btf_record *record;\n \tu32 key_size;\n+\tu32 key_offset;\n };\n \n static inline bool htab_is_prealloc(const struct bpf_htab *htab)\n@@ -136,6 +165,21 @@ static inline bool htab_is_prealloc(const struct bpf_htab *htab)\n \treturn !(htab-\u003emap.map_flags \u0026 BPF_F_NO_PREALLOC);\n }\n \n+static inline struct bpf_lru_node *htab_elem_lru_node(struct htab_elem *l)\n+{\n+\treturn \u0026((struct htab_elem_lru *)l)-\u003elru_node;\n+}\n+\n+static inline void *htab_elem_get_ptr_to_pptr(struct htab_elem *l)\n+{\n+\treturn ((struct htab_elem_pcpu *)l)-\u003eptr_to_pptr;\n+}\n+\n+static inline void htab_elem_set_ptr_to_pptr(struct htab_elem *l, void *ptr)\n+{\n+\t((struct htab_elem_pcpu *)l)-\u003eptr_to_pptr = ptr;\n+}\n+\n static void htab_init_buckets(struct bpf_htab *htab)\n {\n \tunsigned int i;\n@@ -183,25 +227,30 @@ static inline bool is_fd_htab(const struct bpf_htab *htab)\n \treturn htab-\u003emap.map_type == BPF_MAP_TYPE_HASH_OF_MAPS;\n }\n \n-static inline void *htab_elem_value(struct htab_elem *l, u32 key_size)\n+static inline void *htab_elem_key(struct bpf_htab *htab, struct htab_elem *l)\n+{\n+\treturn (void *)l + htab-\u003ekey_offset;\n+}\n+\n+static inline void *htab_elem_value(struct bpf_htab *htab, struct htab_elem *l)\n {\n-\treturn l-\u003ekey + round_up(key_size, 8);\n+\treturn htab_elem_key(htab, l) + round_up(htab-\u003emap.key_size, 8);\n }\n \n-static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,\n+static inline void htab_elem_set_ptr(struct bpf_htab *htab, struct htab_elem *l,\n \t\t\t\t     void __percpu *pptr)\n {\n-\t*(void __percpu **)htab_elem_value(l, key_size) = pptr;\n+\t*(void __percpu **)htab_elem_value(htab, l) = pptr;\n }\n \n-static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)\n+static inline void __percpu *htab_elem_get_ptr(struct bpf_htab *htab, struct htab_elem *l)\n {\n-\treturn *(void __percpu **)htab_elem_value(l, key_size);\n+\treturn *(void __percpu **)htab_elem_value(htab, l);\n }\n \n-static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)\n+static void *fd_htab_map_get_ptr(struct bpf_htab *htab, struct htab_elem *l)\n {\n-\treturn *(void **)htab_elem_value(l, map-\u003ekey_size);\n+\treturn *(void **)htab_elem_value(htab, l);\n }\n \n static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)\n@@ -209,6 +258,33 @@ static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)\n \treturn (struct htab_elem *) (htab-\u003eelems + i * (u64)htab-\u003eelem_size);\n }\n \n+static inline bool htab_has_hash(const struct bpf_htab *htab)\n+{\n+\treturn htab_is_lru(htab) || htab-\u003emap.key_size \u003e 8;\n+}\n+\n+static inline u32 htab_elem_hash(struct bpf_htab *htab, struct htab_elem *l)\n+{\n+\tif (htab_is_lru(htab))\n+\t\treturn ((struct htab_elem_lru *)l)-\u003ehash;\n+\telse if (htab_is_percpu(htab) \u0026\u0026 !htab_is_prealloc(htab))\n+\t\treturn ((struct htab_elem_pcpu *)l)-\u003ehash;\n+\telse\n+\t\treturn l-\u003ehash;\n+}\n+\n+static inline void htab_elem_set_hash(struct bpf_htab *htab, struct htab_elem *l, u32 hash)\n+{\n+\tif (!htab_has_hash(htab))\n+\t\treturn;\n+\tif (htab_is_lru(htab))\n+\t\t((struct htab_elem_lru *)l)-\u003ehash = hash;\n+\telse if (htab_is_percpu(htab) \u0026\u0026 !htab_is_prealloc(htab))\n+\t\t((struct htab_elem_pcpu *)l)-\u003ehash = hash;\n+\telse\n+\t\tl-\u003ehash = hash;\n+}\n+\n /* Both percpu and fd htab support in-place update, so no need for\n  * extra elem. LRU itself can remove the least used element, so\n  * there is no need for an extra elem during map_update.\n@@ -231,7 +307,7 @@ static void htab_free_prealloced_internal_structs(struct bpf_htab *htab)\n \n \t\telem = get_htab_elem(htab, i);\n \t\tbpf_map_free_internal_structs(\u0026htab-\u003emap,\n-\t\t\t\t\t      htab_elem_value(elem, htab-\u003emap.key_size));\n+\t\t\t\t\t      htab_elem_value(htab, elem));\n \t\tcond_resched();\n \t}\n }\n@@ -254,7 +330,7 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab)\n \n \t\telem = get_htab_elem(htab, i);\n \t\tif (htab_is_percpu(htab)) {\n-\t\t\tvoid __percpu *pptr = htab_elem_get_ptr(elem, htab-\u003emap.key_size);\n+\t\t\tvoid __percpu *pptr = htab_elem_get_ptr(htab, elem);\n \t\t\tint cpu;\n \n \t\t\tfor_each_possible_cpu(cpu) {\n@@ -263,7 +339,7 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab)\n \t\t\t}\n \t\t} else {\n \t\t\tbpf_obj_free_fields(htab-\u003emap.record,\n-\t\t\t\t\t    htab_elem_value(elem, htab-\u003emap.key_size));\n+\t\t\t\t\t    htab_elem_value(htab, elem));\n \t\t\tcond_resched();\n \t\t}\n \t\tcond_resched();\n@@ -280,8 +356,7 @@ static void htab_free_elems(struct bpf_htab *htab)\n \tfor (i = 0; i \u003c htab-\u003emap.max_entries; i++) {\n \t\tvoid __percpu *pptr;\n \n-\t\tpptr = htab_elem_get_ptr(get_htab_elem(htab, i),\n-\t\t\t\t\t htab-\u003emap.key_size);\n+\t\tpptr = htab_elem_get_ptr(htab, get_htab_elem(htab, i));\n \t\tfree_percpu(pptr);\n \t\tcond_resched();\n \t}\n@@ -308,8 +383,8 @@ static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key,\n \n \tif (node) {\n \t\tbpf_map_inc_elem_count(\u0026htab-\u003emap);\n-\t\tl = container_of(node, struct htab_elem, lru_node);\n-\t\tmemcpy(l-\u003ekey, key, htab-\u003emap.key_size);\n+\t\tl = (struct htab_elem *)container_of(node, struct htab_elem_lru, lru_node);\n+\t\tmemcpy(htab_elem_key(htab, l), key, htab-\u003emap.key_size);\n \t\treturn l;\n \t}\n \n@@ -340,8 +415,7 @@ static int prealloc_init(struct bpf_htab *htab)\n \t\t\t\t\t    GFP_USER | __GFP_NOWARN);\n \t\tif (!pptr)\n \t\t\tgoto free_elems;\n-\t\thtab_elem_set_ptr(get_htab_elem(htab, i), htab-\u003emap.key_size,\n-\t\t\t\t  pptr);\n+\t\thtab_elem_set_ptr(htab, get_htab_elem(htab, i), pptr);\n \t\tcond_resched();\n \t}\n \n@@ -349,8 +423,8 @@ static int prealloc_init(struct bpf_htab *htab)\n \tif (htab_is_lru(htab))\n \t\terr = bpf_lru_init(\u0026htab-\u003elru,\n \t\t\t\t   htab-\u003emap.map_flags \u0026 BPF_F_NO_COMMON_LRU,\n-\t\t\t\t   offsetof(struct htab_elem, hash) -\n-\t\t\t\t   offsetof(struct htab_elem, lru_node),\n+\t\t\t\t   offsetof(struct htab_elem_lru, hash) -\n+\t\t\t\t   offsetof(struct htab_elem_lru, lru_node),\n \t\t\t\t   htab_lru_map_delete_node,\n \t\t\t\t   htab);\n \telse\n@@ -361,7 +435,7 @@ static int prealloc_init(struct bpf_htab *htab)\n \n \tif (htab_is_lru(htab))\n \t\tbpf_lru_populate(\u0026htab-\u003elru, htab-\u003eelems,\n-\t\t\t\t offsetof(struct htab_elem, lru_node),\n+\t\t\t\t offsetof(struct htab_elem_lru, lru_node),\n \t\t\t\t htab-\u003eelem_size, num_entries);\n \telse\n \t\tpcpu_freelist_populate(\u0026htab-\u003efreelist,\n@@ -428,6 +502,25 @@ static int htab_map_alloc_check(union bpf_attr *attr)\n \tBUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=\n \t\t     offsetof(struct htab_elem, hash_node.pprev));\n \n+\t/* htab_elem pointers are used as base pointers for htab_elem_lru and\n+\t * htab_elem_pcpu. These structures must share a common initial sequence\n+\t * to ensure pointer casting between them is safe.\n+\t */\n+\tBUILD_BUG_ON(offsetof(struct htab_elem, hash_node) !=\n+\t\t     offsetof(struct htab_elem_lru, hash_node));\n+\tBUILD_BUG_ON(offsetof(struct htab_elem, hash_node) !=\n+\t\t     offsetof(struct htab_elem_pcpu, hash_node));\n+\n+\tBUILD_BUG_ON(offsetof(struct htab_elem, padding) !=\n+\t\t     offsetof(struct htab_elem_lru, padding));\n+\tBUILD_BUG_ON(offsetof(struct htab_elem, padding) !=\n+\t\t     offsetof(struct htab_elem_pcpu, padding));\n+\n+\tBUILD_BUG_ON(offsetof(struct htab_elem, fnode) !=\n+\t\t     offsetof(struct htab_elem_lru, fnode));\n+\tBUILD_BUG_ON(offsetof(struct htab_elem, fnode) !=\n+\t\t     offsetof(struct htab_elem_pcpu, fnode));\n+\n \tif (zero_seed \u0026\u0026 !capable(CAP_SYS_ADMIN))\n \t\t/* Guard against local DoS, and discourage production use. */\n \t\treturn -EPERM;\n@@ -476,7 +569,7 @@ static void htab_mem_dtor(void *obj, void *ctx)\n \tif (IS_ERR_OR_NULL(hrec-\u003erecord))\n \t\treturn;\n \n-\tmap_value = htab_elem_value(elem, hrec-\u003ekey_size);\n+\tmap_value = (void *)elem + hrec-\u003ekey_offset + round_up(hrec-\u003ekey_size, 8);\n \tbpf_obj_free_fields(hrec-\u003erecord, map_value);\n }\n \n@@ -502,7 +595,7 @@ static void htab_dtor_ctx_free(void *ctx)\n }\n \n static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,\n-\t\t\t   void (*dtor)(void *, void *))\n+\t\t\t   void (*dtor)(void *, void *), u32 key_offset)\n {\n \tstruct htab_btf_record *hrec;\n \tint err;\n@@ -515,6 +608,7 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,\n \tif (!hrec)\n \t\treturn -ENOMEM;\n \threc-\u003ekey_size = map-\u003ekey_size;\n+\threc-\u003ekey_offset = key_offset;\n \threc-\u003erecord = btf_record_dup(map-\u003erecord);\n \tif (IS_ERR(hrec-\u003erecord)) {\n \t\terr = PTR_ERR(hrec-\u003erecord);\n@@ -537,9 +631,9 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf,\n \t * populated in htab_map_alloc(), so it will always appear as NULL.\n \t */\n \tif (htab_is_percpu(htab))\n-\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003epcpu_ma, htab_pcpu_mem_dtor);\n+\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003epcpu_ma, htab_pcpu_mem_dtor, htab-\u003ekey_offset);\n \telse\n-\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003ema, htab_mem_dtor);\n+\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003ema, htab_mem_dtor, htab-\u003ekey_offset);\n }\n \n static struct bpf_map *htab_map_alloc(union bpf_attr *attr)\n@@ -583,8 +677,18 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)\n \n \thtab-\u003en_buckets = roundup_pow_of_two(htab-\u003emap.max_entries);\n \n-\thtab-\u003eelem_size = sizeof(struct htab_elem) +\n-\t\t\t  round_up(htab-\u003emap.key_size, 8);\n+\tif (htab_is_lru(htab))\n+\t\thtab-\u003ekey_offset = sizeof(struct htab_elem_lru);\n+\telse if (percpu \u0026\u0026 !prealloc)\n+\t\thtab-\u003ekey_offset = round_up(htab_has_hash(htab) ?\n+\t\t\t\t\t    sizeof(struct htab_elem_pcpu) :\n+\t\t\t\t\t    offsetof(struct htab_elem_pcpu, hash), 8);\n+\telse\n+\t\thtab-\u003ekey_offset = round_up(htab_has_hash(htab) ?\n+\t\t\t\t\t    sizeof(struct htab_elem) :\n+\t\t\t\t\t    offsetof(struct htab_elem, hash), 8);\n+\n+\thtab-\u003eelem_size = htab-\u003ekey_offset + round_up(htab-\u003emap.key_size, 8);\n \tif (percpu)\n \t\thtab-\u003eelem_size += sizeof(void *);\n \telse\n@@ -692,14 +796,16 @@ static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32\n }\n \n /* this lookup function can only be called with bucket lock taken */\n-static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash,\n+static struct htab_elem *lookup_elem_raw(struct bpf_htab *htab,\n+\t\t\t\t\t struct hlist_nulls_head *head, u32 hash,\n \t\t\t\t\t void *key, u32 key_size)\n {\n \tstruct hlist_nulls_node *n;\n \tstruct htab_elem *l;\n \n \thlist_nulls_for_each_entry_rcu(l, n, head, hash_node)\n-\t\tif (l-\u003ehash == hash \u0026\u0026 !memcmp(\u0026l-\u003ekey, key, key_size))\n+\t\tif ((!htab_has_hash(htab) || htab_elem_hash(htab, l) == hash) \u0026\u0026\n+\t\t    !memcmp(htab_elem_key(htab, l), key, key_size))\n \t\t\treturn l;\n \n \treturn NULL;\n@@ -709,7 +815,8 @@ static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash\n  * the unlikely event when elements moved from one bucket into another\n  * while link list is being walked\n  */\n-static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,\n+static struct htab_elem *lookup_nulls_elem_raw(struct bpf_htab *htab,\n+\t\t\t\t\t       struct hlist_nulls_head *head,\n \t\t\t\t\t       u32 hash, void *key,\n \t\t\t\t\t       u32 key_size, u32 n_buckets)\n {\n@@ -718,7 +825,8 @@ static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,\n \n again:\n \thlist_nulls_for_each_entry_rcu(l, n, head, hash_node)\n-\t\tif (l-\u003ehash == hash \u0026\u0026 !memcmp(\u0026l-\u003ekey, key, key_size))\n+\t\tif ((!htab_has_hash(htab) || htab_elem_hash(htab, l) == hash) \u0026\u0026\n+\t\t    !memcmp(htab_elem_key(htab, l), key, key_size))\n \t\t\treturn l;\n \n \tif (unlikely(get_nulls_value(n) != (hash \u0026 (n_buckets - 1))))\n@@ -747,17 +855,18 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)\n \n \thead = select_bucket(htab, hash);\n \n-\tl = lookup_nulls_elem_raw(head, hash, key, key_size, htab-\u003en_buckets);\n+\tl = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab-\u003en_buckets);\n \n \treturn l;\n }\n \n static void *htab_map_lookup_elem(struct bpf_map *map, void *key)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l = __htab_map_lookup_elem(map, key);\n \n \tif (l)\n-\t\treturn htab_elem_value(l, map-\u003ekey_size);\n+\t\treturn htab_elem_value(htab, l);\n \n \treturn NULL;\n }\n@@ -775,6 +884,7 @@ static void *htab_map_lookup_elem(struct bpf_map *map, void *key)\n  */\n static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn *insn = insn_buf;\n \tconst int ret = BPF_REG_0;\n \n@@ -783,7 +893,7 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)\n \t*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \t*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);\n \t*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,\n-\t\t\t\toffsetof(struct htab_elem, key) +\n+\t\t\t\thtab-\u003ekey_offset +\n \t\t\t\tround_up(map-\u003ekey_size, 8));\n \treturn insn - insn_buf;\n }\n@@ -791,12 +901,13 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)\n static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map,\n \t\t\t\t\t\t\tvoid *key, const bool mark)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l = __htab_map_lookup_elem(map, key);\n \n \tif (l) {\n \t\tif (mark)\n-\t\t\tbpf_lru_node_set_ref(\u0026l-\u003elru_node);\n-\t\treturn htab_elem_value(l, map-\u003ekey_size);\n+\t\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l));\n+\t\treturn htab_elem_value(htab, l);\n \t}\n \n \treturn NULL;\n@@ -815,6 +926,7 @@ static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key)\n static int htab_lru_map_gen_lookup(struct bpf_map *map,\n \t\t\t\t   struct bpf_insn *insn_buf)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn *insn = insn_buf;\n \tconst int ret = BPF_REG_0;\n \tconst int ref_reg = BPF_REG_1;\n@@ -824,15 +936,15 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map,\n \t*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \t*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4);\n \t*insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret,\n-\t\t\t      offsetof(struct htab_elem, lru_node) +\n+\t\t\t      offsetof(struct htab_elem_lru, lru_node) +\n \t\t\t      offsetof(struct bpf_lru_node, ref));\n \t*insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1);\n \t*insn++ = BPF_ST_MEM(BPF_B, ret,\n-\t\t\t     offsetof(struct htab_elem, lru_node) +\n+\t\t\t     offsetof(struct htab_elem_lru, lru_node) +\n \t\t\t     offsetof(struct bpf_lru_node, ref),\n \t\t\t     1);\n \t*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,\n-\t\t\t\toffsetof(struct htab_elem, key) +\n+\t\t\t\thtab-\u003ekey_offset +\n \t\t\t\tround_up(map-\u003ekey_size, 8));\n \treturn insn - insn_buf;\n }\n@@ -844,13 +956,13 @@ static void check_and_cancel_fields(struct bpf_htab *htab,\n \t\treturn;\n \n \tif (htab_is_percpu(htab)) {\n-\t\tvoid __percpu *pptr = htab_elem_get_ptr(elem, htab-\u003emap.key_size);\n+\t\tvoid __percpu *pptr = htab_elem_get_ptr(htab, elem);\n \t\tint cpu;\n \n \t\tfor_each_possible_cpu(cpu)\n \t\t\tbpf_obj_cancel_fields(\u0026htab-\u003emap, per_cpu_ptr(pptr, cpu));\n \t} else {\n-\t\tvoid *map_value = htab_elem_value(elem, htab-\u003emap.key_size);\n+\t\tvoid *map_value = htab_elem_value(htab, elem);\n \n \t\tbpf_obj_cancel_fields(\u0026htab-\u003emap, map_value);\n \t}\n@@ -869,8 +981,8 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)\n \tstruct bucket *b;\n \tint ret;\n \n-\ttgt_l = container_of(node, struct htab_elem, lru_node);\n-\tb = __select_bucket(htab, tgt_l-\u003ehash);\n+\ttgt_l = (struct htab_elem *)container_of(node, struct htab_elem_lru, lru_node);\n+\tb = __select_bucket(htab, htab_elem_hash(htab, tgt_l));\n \thead = \u0026b-\u003ehead;\n \n \tret = htab_lock_bucket(b, \u0026flags);\n@@ -912,7 +1024,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)\n \thead = select_bucket(htab, hash);\n \n \t/* lookup the key */\n-\tl = lookup_nulls_elem_raw(head, hash, key, key_size, htab-\u003en_buckets);\n+\tl = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab-\u003en_buckets);\n \n \tif (!l)\n \t\tgoto find_first_elem;\n@@ -923,7 +1035,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)\n \n \tif (next_l) {\n \t\t/* if next elem in this hash list is non-zero, just return it */\n-\t\tmemcpy(next_key, next_l-\u003ekey, key_size);\n+\t\tmemcpy(next_key, htab_elem_key(htab, next_l), key_size);\n \t\treturn 0;\n \t}\n \n@@ -941,7 +1053,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)\n \t\t\t\t\t  struct htab_elem, hash_node);\n \t\tif (next_l) {\n \t\t\t/* if it's not empty, just return it */\n-\t\t\tmemcpy(next_key, next_l-\u003ekey, key_size);\n+\t\t\tmemcpy(next_key, htab_elem_key(htab, next_l), key_size);\n \t\t\treturn 0;\n \t\t}\n \t}\n@@ -955,7 +1067,7 @@ static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)\n \tcheck_and_cancel_fields(htab, l);\n \n \tif (htab-\u003emap.map_type == BPF_MAP_TYPE_PERCPU_HASH)\n-\t\tbpf_mem_cache_free(\u0026htab-\u003epcpu_ma, l-\u003eptr_to_pptr);\n+\t\tbpf_mem_cache_free(\u0026htab-\u003epcpu_ma, htab_elem_get_ptr_to_pptr(l));\n \tbpf_mem_cache_free(\u0026htab-\u003ema, l);\n }\n \n@@ -965,7 +1077,7 @@ static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)\n \tvoid *ptr;\n \n \tif (map-\u003eops-\u003emap_fd_put_ptr) {\n-\t\tptr = fd_htab_map_get_ptr(map, l);\n+\t\tptr = fd_htab_map_get_ptr(htab, l);\n \t\tmap-\u003eops-\u003emap_fd_put_ptr(map, ptr, true);\n \t}\n }\n@@ -1117,10 +1229,10 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,\n \t\t}\n \t}\n \n-\tmemcpy(l_new-\u003ekey, key, key_size);\n+\tmemcpy(htab_elem_key(htab, l_new), key, key_size);\n \tif (percpu) {\n \t\tif (prealloc) {\n-\t\t\tpptr = htab_elem_get_ptr(l_new, key_size);\n+\t\t\tpptr = htab_elem_get_ptr(htab, l_new);\n \t\t} else {\n \t\t\t/* alloc_percpu zero-fills */\n \t\t\tvoid *ptr = bpf_mem_cache_alloc(\u0026htab-\u003epcpu_ma);\n@@ -1130,26 +1242,26 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,\n \t\t\t\tl_new = ERR_PTR(-ENOMEM);\n \t\t\t\tgoto dec_count;\n \t\t\t}\n-\t\t\tl_new-\u003eptr_to_pptr = ptr;\n+\t\t\thtab_elem_set_ptr_to_pptr(l_new, ptr);\n \t\t\tpptr = *(void __percpu **)ptr;\n \t\t}\n \n \t\tpcpu_init_value(htab, pptr, value, onallcpus, map_flags);\n \n \t\tif (!prealloc)\n-\t\t\thtab_elem_set_ptr(l_new, key_size, pptr);\n+\t\t\thtab_elem_set_ptr(htab, l_new, pptr);\n \t} else if (fd_htab_map_needs_adjust(htab)) {\n \t\tsize = round_up(size, 8);\n-\t\tmemcpy(htab_elem_value(l_new, key_size), value, size);\n+\t\tmemcpy(htab_elem_value(htab, l_new), value, size);\n \t} else if (map_flags \u0026 BPF_F_LOCK) {\n \t\tcopy_map_value_locked(\u0026htab-\u003emap,\n-\t\t\t\t      htab_elem_value(l_new, key_size),\n+\t\t\t\t      htab_elem_value(htab, l_new),\n \t\t\t\t      value, false);\n \t} else {\n-\t\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(l_new, key_size), value);\n+\t\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(htab, l_new), value);\n \t}\n \n-\tl_new-\u003ehash = hash;\n+\thtab_elem_set_hash(htab, l_new, hash);\n \treturn l_new;\n dec_count:\n \tdec_elem_count(htab);\n@@ -1199,7 +1311,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,\n \t\tif (unlikely(!btf_record_has_field(map-\u003erecord, BPF_SPIN_LOCK)))\n \t\t\treturn -EINVAL;\n \t\t/* find an element without taking the bucket lock */\n-\t\tl_old = lookup_nulls_elem_raw(head, hash, key, key_size,\n+\t\tl_old = lookup_nulls_elem_raw(htab, head, hash, key, key_size,\n \t\t\t\t\t      htab-\u003en_buckets);\n \t\tret = check_flags(htab, l_old, map_flags);\n \t\tif (ret)\n@@ -1207,7 +1319,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,\n \t\tif (l_old) {\n \t\t\t/* grab the element lock and update value in place */\n \t\t\tcopy_map_value_locked(map,\n-\t\t\t\t\t      htab_elem_value(l_old, key_size),\n+\t\t\t\t\t      htab_elem_value(htab, l_old),\n \t\t\t\t\t      value, false);\n \t\t\treturn 0;\n \t\t}\n@@ -1221,7 +1333,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,\n \tif (ret)\n \t\treturn ret;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n@@ -1235,7 +1347,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,\n \t\t * and update element in place\n \t\t */\n \t\tcopy_map_value_locked(map,\n-\t\t\t\t      htab_elem_value(l_old, key_size),\n+\t\t\t\t      htab_elem_value(htab, l_old),\n \t\t\t\t      value, false);\n \t\tret = 0;\n \t\tgoto err;\n@@ -1275,7 +1387,7 @@ static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem)\n {\n \tcheck_and_cancel_fields(htab, elem);\n \tbpf_map_dec_elem_count(\u0026htab-\u003emap);\n-\tbpf_lru_push_free(\u0026htab-\u003elru, \u0026elem-\u003elru_node);\n+\tbpf_lru_push_free(\u0026htab-\u003elru, htab_elem_lru_node(elem));\n }\n \n static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,\n@@ -1310,13 +1422,13 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value\n \tl_new = prealloc_lru_pop(htab, key, hash);\n \tif (!l_new)\n \t\treturn -ENOMEM;\n-\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(l_new, map-\u003ekey_size), value);\n+\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(htab, l_new), value);\n \n \tret = htab_lock_bucket(b, \u0026flags);\n \tif (ret)\n \t\tgoto err_lock_bucket;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n@@ -1327,7 +1439,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value\n \t */\n \thlist_nulls_add_head_rcu(\u0026l_new-\u003ehash_node, head);\n \tif (l_old) {\n-\t\tbpf_lru_node_set_ref(\u0026l_new-\u003elru_node);\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l_new));\n \t\thlist_nulls_del_rcu(\u0026l_old-\u003ehash_node);\n \t}\n \tret = 0;\n@@ -1383,7 +1495,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,\n \tif (ret)\n \t\treturn ret;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n@@ -1392,10 +1504,10 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,\n \tif (l_old) {\n \t\t/* Update value in-place */\n \t\tif (percpu) {\n-\t\t\tpcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),\n+\t\t\tpcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old),\n \t\t\t\t\tvalue, onallcpus, map_flags);\n \t\t} else {\n-\t\t\tvoid **inner_map_pptr = htab_elem_value(l_old, key_size);\n+\t\t\tvoid **inner_map_pptr = htab_elem_value(htab, l_old);\n \n \t\t\told_map_ptr = *inner_map_pptr;\n \t\t\tWRITE_ONCE(*inner_map_pptr, *(void **)value);\n@@ -1456,20 +1568,20 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,\n \tif (ret)\n \t\tgoto err_lock_bucket;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n \t\tgoto err;\n \n \tif (l_old) {\n-\t\tbpf_lru_node_set_ref(\u0026l_old-\u003elru_node);\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l_old));\n \n \t\t/* per-cpu hash map can update value in-place */\n-\t\tpcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),\n+\t\tpcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old),\n \t\t\t\tvalue, onallcpus, map_flags);\n \t} else {\n-\t\tpcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size),\n+\t\tpcpu_init_value(htab, htab_elem_get_ptr(htab, l_new),\n \t\t\t\tvalue, onallcpus, map_flags);\n \t\thlist_nulls_add_head_rcu(\u0026l_new-\u003ehash_node, head);\n \t\tl_new = NULL;\n@@ -1480,7 +1592,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,\n err_lock_bucket:\n \tif (l_new) {\n \t\tbpf_map_dec_elem_count(\u0026htab-\u003emap);\n-\t\tbpf_lru_push_free(\u0026htab-\u003elru, \u0026l_new-\u003elru_node);\n+\t\tbpf_lru_push_free(\u0026htab-\u003elru, htab_elem_lru_node(l_new));\n \t}\n \treturn ret;\n }\n@@ -1521,7 +1633,7 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key)\n \tif (ret)\n \t\treturn ret;\n \n-\tl = lookup_elem_raw(head, hash, key, key_size);\n+\tl = lookup_elem_raw(htab, head, hash, key, key_size);\n \tif (l)\n \t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n \telse\n@@ -1556,7 +1668,7 @@ static long htab_lru_map_delete_elem(struct bpf_map *map, void *key)\n \tif (ret)\n \t\treturn ret;\n \n-\tl = lookup_elem_raw(head, hash, key, key_size);\n+\tl = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tif (l)\n \t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n@@ -1602,7 +1714,7 @@ static void htab_free_malloced_internal_structs(struct bpf_htab *htab)\n \t\thlist_nulls_for_each_entry(l, n, head, hash_node) {\n \t\t\t/* We only free internal structs on uref dropping to zero */\n \t\t\tbpf_map_free_internal_structs(\u0026htab-\u003emap,\n-\t\t\t\t\t\t      htab_elem_value(l, htab-\u003emap.key_size));\n+\t\t\t\t\t\t      htab_elem_value(htab, l));\n \t\t}\n \t\tcond_resched_rcu();\n \t}\n@@ -1697,7 +1809,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,\n \tif (ret)\n \t\treturn ret;\n \n-\tl = lookup_elem_raw(head, hash, key, key_size);\n+\tl = lookup_elem_raw(htab, head, hash, key, key_size);\n \tif (!l) {\n \t\tret = -ENOENT;\n \t\tgoto out_unlock;\n@@ -1708,14 +1820,14 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,\n \t\tvoid __percpu *pptr;\n \t\tint off = 0, cpu;\n \n-\t\tpptr = htab_elem_get_ptr(l, key_size);\n+\t\tpptr = htab_elem_get_ptr(htab, l);\n \t\tfor_each_possible_cpu(cpu) {\n \t\t\tcopy_map_value_long(\u0026htab-\u003emap, value + off, per_cpu_ptr(pptr, cpu));\n \t\t\tcheck_and_init_map_value(\u0026htab-\u003emap, value + off);\n \t\t\toff += roundup_value_size;\n \t\t}\n \t} else {\n-\t\tvoid *src = htab_elem_value(l, map-\u003ekey_size);\n+\t\tvoid *src = htab_elem_value(htab, l);\n \n \t\tif (flags \u0026 BPF_F_LOCK)\n \t\t\tcopy_map_value_locked(map, value, src, true);\n@@ -1898,13 +2010,13 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,\n \t\tgoto next_batch;\n \n \thlist_nulls_for_each_entry_safe(l, n, head, hash_node) {\n-\t\tmemcpy(dst_key, l-\u003ekey, key_size);\n+\t\tmemcpy(dst_key, htab_elem_key(htab, l), key_size);\n \n \t\tif (is_percpu) {\n \t\t\tint off = 0, cpu;\n \t\t\tvoid __percpu *pptr;\n \n-\t\t\tpptr = htab_elem_get_ptr(l, map-\u003ekey_size);\n+\t\t\tpptr = htab_elem_get_ptr(htab, l);\n \t\t\tif (elem_map_flags \u0026 BPF_F_CPU) {\n \t\t\t\tcpu = elem_map_flags \u003e\u003e 32;\n \t\t\t\tcopy_map_value(\u0026htab-\u003emap, dst_val, per_cpu_ptr(pptr, cpu));\n@@ -1918,7 +2030,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,\n \t\t\t\t}\n \t\t\t}\n \t\t} else {\n-\t\t\tvalue = htab_elem_value(l, key_size);\n+\t\t\tvalue = htab_elem_value(htab, l);\n \t\t\tif (is_fd_htab(htab)) {\n \t\t\t\tstruct bpf_map **inner_map = value;\n \n@@ -2183,12 +2295,12 @@ static int __bpf_hash_map_seq_show(struct seq_file *seq, struct htab_elem *elem)\n \t\tctx.meta = \u0026meta;\n \t\tctx.map = info-\u003emap;\n \t\tif (elem) {\n-\t\t\tctx.key = elem-\u003ekey;\n+\t\t\tctx.key = htab_elem_key(info-\u003ehtab, elem);\n \t\t\tif (!info-\u003epercpu_value_buf) {\n-\t\t\t\tctx.value = htab_elem_value(elem, map-\u003ekey_size);\n+\t\t\t\tctx.value = htab_elem_value(info-\u003ehtab, elem);\n \t\t\t} else {\n \t\t\t\troundup_value_size = round_up(map-\u003evalue_size, 8);\n-\t\t\t\tpptr = htab_elem_get_ptr(elem, map-\u003ekey_size);\n+\t\t\t\tpptr = htab_elem_get_ptr(info-\u003ehtab, elem);\n \t\t\t\tfor_each_possible_cpu(cpu) {\n \t\t\t\t\tcopy_map_value_long(map, info-\u003epercpu_value_buf + off,\n \t\t\t\t\t\t\t    per_cpu_ptr(pptr, cpu));\n@@ -2293,13 +2405,13 @@ static long bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_\n \t\trcu_read_lock();\n \t\thead = \u0026b-\u003ehead;\n \t\thlist_nulls_for_each_entry_safe(elem, n, head, hash_node) {\n-\t\t\tkey = elem-\u003ekey;\n+\t\t\tkey = htab_elem_key(htab, elem);\n \t\t\tif (is_percpu) {\n \t\t\t\t/* current cpu value for percpu map */\n-\t\t\t\tpptr = htab_elem_get_ptr(elem, map-\u003ekey_size);\n+\t\t\t\tpptr = htab_elem_get_ptr(htab, elem);\n \t\t\t\tval = this_cpu_ptr(pptr);\n \t\t\t} else {\n-\t\t\t\tval = htab_elem_value(elem, map-\u003ekey_size);\n+\t\t\t\tval = htab_elem_value(htab, elem);\n \t\t\t}\n \t\t\tnum_elems++;\n \t\t\tret = callback_fn((u64)(long)map, (u64)(long)key,\n@@ -2403,10 +2515,11 @@ const struct bpf_map_ops htab_lru_map_ops = {\n /* Called from eBPF program */\n static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l = __htab_map_lookup_elem(map, key);\n \n \tif (l)\n-\t\treturn this_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size));\n+\t\treturn this_cpu_ptr(htab_elem_get_ptr(htab, l));\n \telse\n \t\treturn NULL;\n }\n@@ -2414,6 +2527,7 @@ static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key)\n /* inline bpf_map_lookup_elem() call for per-CPU hashmap */\n static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn *insn = insn_buf;\n \n \tif (!bpf_jit_supports_percpu_insn())\n@@ -2424,7 +2538,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn\n \t*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \t*insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3);\n \t*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0,\n-\t\t\t\toffsetof(struct htab_elem, key) + roundup(map-\u003ekey_size, 8));\n+\t\t\t\thtab-\u003ekey_offset + roundup(map-\u003ekey_size, 8));\n \t*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);\n \t*insn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);\n \n@@ -2433,6 +2547,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn\n \n static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l;\n \n \tif (cpu \u003e= nr_cpu_ids)\n@@ -2440,18 +2555,19 @@ static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key,\n \n \tl = __htab_map_lookup_elem(map, key);\n \tif (l)\n-\t\treturn per_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size), cpu);\n+\t\treturn per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu);\n \telse\n \t\treturn NULL;\n }\n \n static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l = __htab_map_lookup_elem(map, key);\n \n \tif (l) {\n-\t\tbpf_lru_node_set_ref(\u0026l-\u003elru_node);\n-\t\treturn this_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size));\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l));\n+\t\treturn this_cpu_ptr(htab_elem_get_ptr(htab, l));\n \t}\n \n \treturn NULL;\n@@ -2459,6 +2575,7 @@ static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key)\n \n static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l;\n \n \tif (cpu \u003e= nr_cpu_ids)\n@@ -2466,8 +2583,8 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k\n \n \tl = __htab_map_lookup_elem(map, key);\n \tif (l) {\n-\t\tbpf_lru_node_set_ref(\u0026l-\u003elru_node);\n-\t\treturn per_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size), cpu);\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l));\n+\t\treturn per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu);\n \t}\n \n \treturn NULL;\n@@ -2475,6 +2592,7 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k\n \n int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value, u64 map_flags)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l;\n \tvoid __percpu *pptr;\n \tint ret = -ENOENT;\n@@ -2494,7 +2612,7 @@ int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value, u64 map_fl\n \t/* We do not mark LRU map element here in order to not mess up\n \t * eviction heuristics when user space does a map walk.\n \t */\n-\tpptr = htab_elem_get_ptr(l, map-\u003ekey_size);\n+\tpptr = htab_elem_get_ptr(htab, l);\n \tif (map_flags \u0026 BPF_F_CPU) {\n \t\tcpu = map_flags \u003e\u003e 32;\n \t\tcopy_map_value(map, value, per_cpu_ptr(pptr, cpu));\n@@ -2532,6 +2650,7 @@ int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,\n static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key,\n \t\t\t\t\t  struct seq_file *m)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l;\n \tvoid __percpu *pptr;\n \tint cpu;\n@@ -2546,7 +2665,7 @@ static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key,\n \n \tbtf_type_seq_show(map-\u003ebtf, map-\u003ebtf_key_type_id, key, m);\n \tseq_puts(m, \": {\\n\");\n-\tpptr = htab_elem_get_ptr(l, map-\u003ekey_size);\n+\tpptr = htab_elem_get_ptr(htab, l);\n \tfor_each_possible_cpu(cpu) {\n \t\tseq_printf(m, \"\\tcpu%d: \", cpu);\n \t\tbtf_type_seq_show(map-\u003ebtf, map-\u003ebtf_value_type_id,\n@@ -2620,7 +2739,7 @@ static void fd_htab_map_free(struct bpf_map *map)\n \t\thead = select_bucket(htab, i);\n \n \t\thlist_nulls_for_each_entry_safe(l, n, head, hash_node) {\n-\t\t\tvoid *ptr = fd_htab_map_get_ptr(map, l);\n+\t\t\tvoid *ptr = fd_htab_map_get_ptr(htab, l);\n \n \t\t\tmap-\u003eops-\u003emap_fd_put_ptr(map, ptr, false);\n \t\t}\n@@ -2705,6 +2824,7 @@ static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key)\n static int htab_of_map_gen_lookup(struct bpf_map *map,\n \t\t\t\t  struct bpf_insn *insn_buf)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn *insn = insn_buf;\n \tconst int ret = BPF_REG_0;\n \n@@ -2713,7 +2833,7 @@ static int htab_of_map_gen_lookup(struct bpf_map *map,\n \t*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \t*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2);\n \t*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,\n-\t\t\t\toffsetof(struct htab_elem, key) +\n+\t\t\t\thtab-\u003ekey_offset +\n \t\t\t\tround_up(map-\u003ekey_size, 8));\n \t*insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);\n \n@@ -3112,7 +3232,7 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf,\n {\n \tstruct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);\n \n-\treturn bpf_ma_set_dtor(map, \u0026rhtab-\u003ema, rhtab_mem_dtor);\n+\treturn bpf_ma_set_dtor(map, \u0026rhtab-\u003ema, rhtab_mem_dtor, offsetof(struct rhtab_elem, data));\n }\n \n static void rhtab_map_free_internal_structs(struct bpf_map *map)\n@@ -3533,3 +3653,16 @@ const struct bpf_map_ops rhtab_map_ops = {\n \t.map_btf_id = \u0026rhtab_map_btf_ids[0],\n \t.iter_seq_info = \u0026rhash_iter_seq_info,\n };\n+\n+size_t bpf_htab_map_meta_size(void)\n+{\n+\treturn sizeof(struct bpf_htab);\n+}\n+\n+void bpf_htab_map_meta_init(struct bpf_map *inner_map_meta, struct bpf_map *inner_map)\n+{\n+\tstruct bpf_htab *inner_htab_meta = container_of(inner_map_meta, struct bpf_htab, map);\n+\tstruct bpf_htab *inner_htab = container_of(inner_map, struct bpf_htab, map);\n+\n+\tinner_htab_meta-\u003ekey_offset = inner_htab-\u003ekey_offset;\n+}\ndiff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c\nindex d2cbab4bdf644a..b2db5d0c98bee2 100644\n--- a/kernel/bpf/map_in_map.c\n+++ b/kernel/bpf/map_in_map.c\n@@ -7,6 +7,15 @@\n \n #include \"map_in_map.h\"\n \n+static bool bpf_map_type_is_htab(enum bpf_map_type type)\n+{\n+\treturn type == BPF_MAP_TYPE_HASH ||\n+\t       type == BPF_MAP_TYPE_PERCPU_HASH ||\n+\t       type == BPF_MAP_TYPE_LRU_HASH ||\n+\t       type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||\n+\t       type == BPF_MAP_TYPE_HASH_OF_MAPS;\n+}\n+\n struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)\n {\n \tstruct bpf_map *inner_map, *inner_map_meta;\n@@ -29,6 +38,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)\n \t/* In some cases verifier needs to access beyond just base map. */\n \tif (inner_map-\u003eops == \u0026array_map_ops || inner_map-\u003eops == \u0026percpu_array_map_ops)\n \t\tinner_map_meta_size = sizeof(struct bpf_array);\n+\telse if (bpf_map_type_is_htab(inner_map-\u003emap_type))\n+\t\tinner_map_meta_size = bpf_htab_map_meta_size();\n \n \tinner_map_meta = kzalloc(inner_map_meta_size, GFP_USER);\n \tif (!inner_map_meta)\n@@ -70,6 +81,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)\n \t\tinner_array_meta-\u003eindex_mask = inner_array-\u003eindex_mask;\n \t\tinner_array_meta-\u003eelem_size = inner_array-\u003eelem_size;\n \t\tinner_map_meta-\u003ebypass_spec_v1 = inner_map-\u003ebypass_spec_v1;\n+\t} else if (bpf_map_type_is_htab(inner_map-\u003emap_type)) {\n+\t\tbpf_htab_map_meta_init(inner_map_meta, inner_map);\n \t}\n \treturn inner_map_meta;\n }\ndiff --git a/kernel/bpf/map_in_map.h b/kernel/bpf/map_in_map.h\nindex 7d61602354de80..1edc205772d8cd 100644\n--- a/kernel/bpf/map_in_map.h\n+++ b/kernel/bpf/map_in_map.h\n@@ -15,5 +15,7 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, struct file *map_file,\n \t\t\t int ufd);\n void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer);\n u32 bpf_map_fd_sys_lookup_elem(void *ptr);\n+size_t bpf_htab_map_meta_size(void);\n+void bpf_htab_map_meta_init(struct bpf_map *inner_map_meta, struct bpf_map *inner_map);\n \n #endif\ndiff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c\nindex 373c8d17ea557d..6bd4cb68c20c1c 100644\n--- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c\n+++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c\n@@ -114,7 +114,7 @@ static inline int check_hash(void)\n \tVERIFY(check_default_noinline(\u0026hash-\u003emap, map));\n \n \tVERIFY(hash-\u003en_buckets == MAX_ENTRIES);\n-\tVERIFY(hash-\u003eelem_size == 64);\n+\tVERIFY(hash-\u003eelem_size == 32);\n \n \tVERIFY(hash-\u003ecount.counter == 0);\n \tVERIFY(bpf_map_sum_elem_count(map) == 0);\n"
}

2/1

2026/07/23 22:38

agent

patch-evaluator

Model:

best-expensive

Args:

null

Results:

{
  "EnableConfigs": [
    "BPF_SYSCALL"
  ],
  "FocusSymbols": [
    "htab_map_alloc",
    "htab_map_update_elem",
    "htab_lru_map_update_elem",
    "__htab_lru_percpu_map_update_elem",
    "htab_map_delete_elem",
    "htab_lru_map_delete_elem",
    "__htab_map_lookup_and_delete_elem",
    "__htab_map_lookup_and_delete_batch",
    "bpf_map_meta_alloc",
    "htab_map_gen_lookup",
    "htab_lru_map_gen_lookup",
    "htab_percpu_map_gen_lookup",
    "htab_of_map_gen_lookup"
  ],
  "Reasoning": "The patch functionally modifies the memory layout and operations of BPF hash maps (including LRU and per-CPU variants). It introduces new structures and offsets for hash map elements, which affects allocation, updates, deletions, and JIT lookup generation. Fuzzing is needed to ensure memory safety and correctness of these core BPF map operations.",
  "WorthFuzzing": true
}

Instruction:

You are an expert Linux kernel maintainer.
Your job is to review a provided patch series and determine
if it makes functional changes to the kernel that should be fuzzed.

IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in
your workspace. Do NOT rely on your internal knowledge of the kernel. You must actively
use your code access tools to examine the actual source code and confirm any assumptions.

Return WorthFuzzing=false if the patch only contains:
- Modifications to Documentation/, Kconfig files, or code comments.
- Purely decorative changes, such as logging (e.g., pr_err, printk) or tracepoints.
- Changes to numeric constants or macros that do not functionally alter execution flow.
- Code paths that are impossible to reach in virtualized environments like GCE or QEMU,
even when utilizing software-emulated hardware (e.g., usb gadget, mac80211_hwsim).

If it modifies reachable core kernel logic, drivers, or architectures, use your code search
tools to verify the code can be executed, then return WorthFuzzing=true.

When returning WorthFuzzing=true, you MUST ALSO:
1. Extract any specific kernel functions that should be heavily fuzzed into FocusSymbols.
   Avoid listing generic hot-path functions to prevent skewed test distributions.
2. Identify any specific CONFIG_ options required to properly test this new/modified feature.
   Go and look into the Kconfig files and check for ifdefs around the code, do not make assumptions.
   Do not list too generic configs (we already have them enabled). Only list those that
   specifically cover the modified code. List them in the EnableConfigs output array,
   and DO NOT add a 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4").
Prefer calling several tools at the same time to save round-trips.


Use set-results tool to provide results of the analysis.
It must be called exactly once before the final reply.
Ignore results of this tool.

Prompt:

For your convenience, here is the diff of the changes:
commit 243a270640e0dc901e4a78b599b0748d59bf7b63
Author: syz-cluster <triage@syzkaller.com>
Date:   Thu Jul 23 22:38:40 2026 +0000

    syz-cluster: applied patch under review

diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 9f394e1aa2e85b..b8c30d40295879 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -102,6 +102,7 @@ struct bpf_htab {
 	bool use_percpu_counter;
 	u32 n_buckets;	/* number of hash buckets */
 	u32 elem_size;	/* size of each element in bytes */
+	u32 key_offset;	/* offset of key in bytes */
 	u32 hashrnd;
 };
 
@@ -117,18 +118,46 @@ struct htab_elem {
 			};
 		};
 	};
+	u32 hash; /* Only when htab_has_hash() */
+} __aligned(8);
+
+struct htab_elem_lru {
 	union {
-		/* pointer to per-cpu pointer */
-		void *ptr_to_pptr;
-		struct bpf_lru_node lru_node;
+		struct hlist_nulls_node hash_node;
+		struct {
+			void *padding;
+			union {
+				struct pcpu_freelist_node fnode;
+				struct htab_elem_lru *batch_flink;
+			};
+		};
 	};
+	struct bpf_lru_node lru_node;
 	u32 hash;
-	char key[] __aligned(8);
-};
+} __aligned(8);
+
+/* Only for non-preallocated PCPU maps. Preallocated PCPU maps don't need
+ * ptr_to_pptr, and use htab_elem.
+ */
+struct htab_elem_pcpu {
+	union {
+		struct hlist_nulls_node hash_node;
+		struct {
+			void *padding;
+			union {
+				struct pcpu_freelist_node fnode;
+				struct htab_elem_pcpu *batch_flink;
+			};
+		};
+	};
+	void *ptr_to_pptr;
+	u32 hash; /* Only when htab_has_hash() */
+} __aligned(8);
 
 struct htab_btf_record {
 	struct btf_record *record;
 	u32 key_size;
+	u32 key_offset;
 };
 
 static inline bool htab_is_prealloc(const struct bpf_htab *htab)
@@ -136,6 +165,21 @@ static inline bool htab_is_prealloc(const struct bpf_htab *htab)
 	return !(htab->map.map_flags & BPF_F_NO_PREALLOC);
 }
 
+static inline struct bpf_lru_node *htab_elem_lru_node(struct htab_elem *l)
+{
+	return &((struct htab_elem_lru *)l)->lru_node;
+}
+
+static inline void *htab_elem_get_ptr_to_pptr(struct htab_elem *l)
+{
+	return ((struct htab_elem_pcpu *)l)->ptr_to_pptr;
+}
+
+static inline void htab_elem_set_ptr_to_pptr(struct htab_elem *l, void *ptr)
+{
+	((struct htab_elem_pcpu *)l)->ptr_to_pptr = ptr;
+}
+
 static void htab_init_buckets(struct bpf_htab *htab)
 {
 	unsigned int i;
@@ -183,25 +227,30 @@ static inline bool is_fd_htab(const struct bpf_htab *htab)
 	return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS;
 }
 
-static inline void *htab_elem_value(struct htab_elem *l, u32 key_size)
+static inline void *htab_elem_key(struct bpf_htab *htab, struct htab_elem *l)
+{
+	return (void *)l + htab->key_offset;
+}
+
+static inline void *htab_elem_value(struct bpf_htab *htab, struct htab_elem *l)
 {
-	return l->key + round_up(key_size, 8);
+	return htab_elem_key(htab, l) + round_up(htab->map.key_size, 8);
 }
 
-static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,
+static inline void htab_elem_set_ptr(struct bpf_htab *htab, struct htab_elem *l,
 				     void __percpu *pptr)
 {
-	*(void __percpu **)htab_elem_value(l, key_size) = pptr;
+	*(void __percpu **)htab_elem_value(htab, l) = pptr;
 }
 
-static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)
+static inline void __percpu *htab_elem_get_ptr(struct bpf_htab *htab, struct htab_elem *l)
 {
-	return *(void __percpu **)htab_elem_value(l, key_size);
+	return *(void __percpu **)htab_elem_value(htab, l);
 }
 
-static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)
+static void *fd_htab_map_get_ptr(struct bpf_htab *htab, struct htab_elem *l)
 {
-	return *(void **)htab_elem_value(l, map->key_size);
+	return *(void **)htab_elem_value(htab, l);
 }
 
 static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)
@@ -209,6 +258,33 @@ static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)
 	return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size);
 }
 
+static inline bool htab_has_hash(const struct bpf_htab *htab)
+{
+	return htab_is_lru(htab) || htab->map.key_size > 8;
+}
+
+static inline u32 htab_elem_hash(struct bpf_htab *htab, struct htab_elem *l)
+{
+	if (htab_is_lru(htab))
+		return ((struct htab_elem_lru *)l)->hash;
+	else if (htab_is_percpu(htab) && !htab_is_prealloc(htab))
+		return ((struct htab_elem_pcpu *)l)->hash;
+	else
+		return l->hash;
+}
+
+static inline void htab_elem_set_hash(struct bpf_htab *htab, struct htab_elem *l, u32 hash)
+{
+	if (!htab_has_hash(htab))
+		return;
+	if (htab_is_lru(htab))
+		((struct htab_elem_lru *)l)->hash = hash;
+	else if (htab_is_percpu(htab) && !htab_is_prealloc(htab))
+		((struct htab_elem_pcpu *)l)->hash = hash;
+	else
+		l->hash = hash;
+}
+
 /* Both percpu and fd htab support in-place update, so no need for
  * extra elem. LRU itself can remove the least used element, so
  * there is no need for an extra elem during map_update.
@@ -231,7 +307,7 @@ static void htab_free_prealloced_internal_structs(struct bpf_htab *htab)
 
 		elem = get_htab_elem(htab, i);
 		bpf_map_free_internal_structs(&htab->map,
-					      htab_elem_value(elem, htab->map.key_size));
+					      htab_elem_value(htab, elem));
 		cond_resched();
 	}
 }
@@ -254,7 +330,7 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab)
 
 		elem = get_htab_elem(htab, i);
 		if (htab_is_percpu(htab)) {
-			void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);
+			void __percpu *pptr = htab_elem_get_ptr(htab, elem);
 			int cpu;
 
 			for_each_possible_cpu(cpu) {
@@ -263,7 +339,7 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab)
 			}
 		} else {
 			bpf_obj_free_fields(htab->map.record,
-					    htab_elem_value(elem, htab->map.key_size));
+					    htab_elem_value(htab, elem));
 			cond_resched();
 		}
 		cond_resched();
@@ -280,8 +356,7 @@ static void htab_free_elems(struct bpf_htab *htab)
 	for (i = 0; i < htab->map.max_entries; i++) {
 		void __percpu *pptr;
 
-		pptr = htab_elem_get_ptr(get_htab_elem(htab, i),
-					 htab->map.key_size);
+		pptr = htab_elem_get_ptr(htab, get_htab_elem(htab, i));
 		free_percpu(pptr);
 		cond_resched();
 	}
@@ -308,8 +383,8 @@ static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key,
 
 	if (node) {
 		bpf_map_inc_elem_count(&htab->map);
-		l = container_of(node, struct htab_elem, lru_node);
-		memcpy(l->key, key, htab->map.key_size);
+		l = (struct htab_elem *)container_of(node, struct htab_elem_lru, lru_node);
+		memcpy(htab_elem_key(htab, l), key, htab->map.key_size);
 		return l;
 	}
 
@@ -340,8 +415,7 @@ static int prealloc_init(struct bpf_htab *htab)
 					    GFP_USER | __GFP_NOWARN);
 		if (!pptr)
 			goto free_elems;
-		htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size,
-				  pptr);
+		htab_elem_set_ptr(htab, get_htab_elem(htab, i), pptr);
 		cond_resched();
 	}
 
@@ -349,8 +423,8 @@ static int prealloc_init(struct bpf_htab *htab)
 	if (htab_is_lru(htab))
 		err = bpf_lru_init(&htab->lru,
 				   htab->map.map_flags & BPF_F_NO_COMMON_LRU,
-				   offsetof(struct htab_elem, hash) -
-				   offsetof(struct htab_elem, lru_node),
+				   offsetof(struct htab_elem_lru, hash) -
+				   offsetof(struct htab_elem_lru, lru_node),
 				   htab_lru_map_delete_node,
 				   htab);
 	else
@@ -361,7 +435,7 @@ static int prealloc_init(struct bpf_htab *htab)
 
 	if (htab_is_lru(htab))
 		bpf_lru_populate(&htab->lru, htab->elems,
-				 offsetof(struct htab_elem, lru_node),
+				 offsetof(struct htab_elem_lru, lru_node),
 				 htab->elem_size, num_entries);
 	else
 		pcpu_freelist_populate(&htab->freelist,
@@ -428,6 +502,25 @@ static int htab_map_alloc_check(union bpf_attr *attr)
 	BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=
 		     offsetof(struct htab_elem, hash_node.pprev));
 
+	/* htab_elem pointers are used as base pointers for htab_elem_lru and
+	 * htab_elem_pcpu. These structures must share a common initial sequence
+	 * to ensure pointer casting between them is safe.
+	 */
+	BUILD_BUG_ON(offsetof(struct htab_elem, hash_node) !=
+		     offsetof(struct htab_elem_lru, hash_node));
+	BUILD_BUG_ON(offsetof(struct htab_elem, hash_node) !=
+		     offsetof(struct htab_elem_pcpu, hash_node));
+
+	BUILD_BUG_ON(offsetof(struct htab_elem, padding) !=
+		     offsetof(struct htab_elem_lru, padding));
+	BUILD_BUG_ON(offsetof(struct htab_elem, padding) !=
+		     offsetof(struct htab_elem_pcpu, padding));
+
+	BUILD_BUG_ON(offsetof(struct htab_elem, fnode) !=
+		     offsetof(struct htab_elem_lru, fnode));
+	BUILD_BUG_ON(offsetof(struct htab_elem, fnode) !=
+		     offsetof(struct htab_elem_pcpu, fnode));
+
 	if (zero_seed && !capable(CAP_SYS_ADMIN))
 		/* Guard against local DoS, and discourage production use. */
 		return -EPERM;
@@ -476,7 +569,7 @@ static void htab_mem_dtor(void *obj, void *ctx)
 	if (IS_ERR_OR_NULL(hrec->record))
 		return;
 
-	map_value = htab_elem_value(elem, hrec->key_size);
+	map_value = (void *)elem + hrec->key_offset + round_up(hrec->key_size, 8);
 	bpf_obj_free_fields(hrec->record, map_value);
 }
 
@@ -502,7 +595,7 @@ static void htab_dtor_ctx_free(void *ctx)
 }
 
 static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,
-			   void (*dtor)(void *, void *))
+			   void (*dtor)(void *, void *), u32 key_offset)
 {
 	struct htab_btf_record *hrec;
 	int err;
@@ -515,6 +608,7 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,
 	if (!hrec)
 		return -ENOMEM;
 	hrec->key_size = map->key_size;
+	hrec->key_offset = key_offset;
 	hrec->record = btf_record_dup(map->record);
 	if (IS_ERR(hrec->record)) {
 		err = PTR_ERR(hrec->record);
@@ -537,9 +631,9 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf,
 	 * populated in htab_map_alloc(), so it will always appear as NULL.
 	 */
 	if (htab_is_percpu(htab))
-		return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor);
+		return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor, htab->key_offset);
 	else
-		return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor);
+		return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor, htab->key_offset);
 }
 
 static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
@@ -583,8 +677,18 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
 
 	htab->n_buckets = roundup_pow_of_two(htab->map.max_entries);
 
-	htab->elem_size = sizeof(struct htab_elem) +
-			  round_up(htab->map.key_size, 8);
+	if (htab_is_lru(htab))
+		htab->key_offset = sizeof(struct htab_elem_lru);
+	else if (percpu && !prealloc)
+		htab->key_offset = round_up(htab_has_hash(htab) ?
+					    sizeof(struct htab_elem_pcpu) :
+					    offsetof(struct htab_elem_pcpu, hash), 8);
+	else
+		htab->key_offset = round_up(htab_has_hash(htab) ?
+					    sizeof(struct htab_elem) :
+					    offsetof(struct htab_elem, hash), 8);
+
+	htab->elem_size = htab->key_offset + round_up(htab->map.key_size, 8);
 	if (percpu)
 		htab->elem_size += sizeof(void *);
 	else
@@ -692,14 +796,16 @@ static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32
 }
 
 /* this lookup function can only be called with bucket lock taken */
-static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash,
+static struct htab_elem *lookup_elem_raw(struct bpf_htab *htab,
+					 struct hlist_nulls_head *head, u32 hash,
 					 void *key, u32 key_size)
 {
 	struct hlist_nulls_node *n;
 	struct htab_elem *l;
 
 	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
-		if (l->hash == hash && !memcmp(&l->key, key, key_size))
+		if ((!htab_has_hash(htab) || htab_elem_hash(htab, l) == hash) &&
+		    !memcmp(htab_elem_key(htab, l), key, key_size))
 			return l;
 
 	return NULL;
@@ -709,7 +815,8 @@ static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash
  * the unlikely event when elements moved from one bucket into another
  * while link list is being walked
  */
-static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,
+static struct htab_elem *lookup_nulls_elem_raw(struct bpf_htab *htab,
+					       struct hlist_nulls_head *head,
 					       u32 hash, void *key,
 					       u32 key_size, u32 n_buckets)
 {
@@ -718,7 +825,8 @@ static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,
 
 again:
 	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
-		if (l->hash == hash && !memcmp(&l->key, key, key_size))
+		if ((!htab_has_hash(htab) || htab_elem_hash(htab, l) == hash) &&
+		    !memcmp(htab_elem_key(htab, l), key, key_size))
 			return l;
 
 	if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1))))
@@ -747,17 +855,18 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
 
 	head = select_bucket(htab, hash);
 
-	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);
+	l = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab->n_buckets);
 
 	return l;
 }
 
 static void *htab_map_lookup_elem(struct bpf_map *map, void *key)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l = __htab_map_lookup_elem(map, key);
 
 	if (l)
-		return htab_elem_value(l, map->key_size);
+		return htab_elem_value(htab, l);
 
 	return NULL;
 }
@@ -775,6 +884,7 @@ static void *htab_map_lookup_elem(struct bpf_map *map, void *key)
  */
 static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct bpf_insn *insn = insn_buf;
 	const int ret = BPF_REG_0;
 
@@ -783,7 +893,7 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
 	*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
 	*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,
-				offsetof(struct htab_elem, key) +
+				htab->key_offset +
 				round_up(map->key_size, 8));
 	return insn - insn_buf;
 }
@@ -791,12 +901,13 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
 static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map,
 							void *key, const bool mark)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l = __htab_map_lookup_elem(map, key);
 
 	if (l) {
 		if (mark)
-			bpf_lru_node_set_ref(&l->lru_node);
-		return htab_elem_value(l, map->key_size);
+			bpf_lru_node_set_ref(htab_elem_lru_node(l));
+		return htab_elem_value(htab, l);
 	}
 
 	return NULL;
@@ -815,6 +926,7 @@ static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key)
 static int htab_lru_map_gen_lookup(struct bpf_map *map,
 				   struct bpf_insn *insn_buf)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct bpf_insn *insn = insn_buf;
 	const int ret = BPF_REG_0;
 	const int ref_reg = BPF_REG_1;
@@ -824,15 +936,15 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map,
 	*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
 	*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4);
 	*insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret,
-			      offsetof(struct htab_elem, lru_node) +
+			      offsetof(struct htab_elem_lru, lru_node) +
 			      offsetof(struct bpf_lru_node, ref));
 	*insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1);
 	*insn++ = BPF_ST_MEM(BPF_B, ret,
-			     offsetof(struct htab_elem, lru_node) +
+			     offsetof(struct htab_elem_lru, lru_node) +
 			     offsetof(struct bpf_lru_node, ref),
 			     1);
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,
-				offsetof(struct htab_elem, key) +
+				htab->key_offset +
 				round_up(map->key_size, 8));
 	return insn - insn_buf;
 }
@@ -844,13 +956,13 @@ static void check_and_cancel_fields(struct bpf_htab *htab,
 		return;
 
 	if (htab_is_percpu(htab)) {
-		void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);
+		void __percpu *pptr = htab_elem_get_ptr(htab, elem);
 		int cpu;
 
 		for_each_possible_cpu(cpu)
 			bpf_obj_cancel_fields(&htab->map, per_cpu_ptr(pptr, cpu));
 	} else {
-		void *map_value = htab_elem_value(elem, htab->map.key_size);
+		void *map_value = htab_elem_value(htab, elem);
 
 		bpf_obj_cancel_fields(&htab->map, map_value);
 	}
@@ -869,8 +981,8 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)
 	struct bucket *b;
 	int ret;
 
-	tgt_l = container_of(node, struct htab_elem, lru_node);
-	b = __select_bucket(htab, tgt_l->hash);
+	tgt_l = (struct htab_elem *)container_of(node, struct htab_elem_lru, lru_node);
+	b = __select_bucket(htab, htab_elem_hash(htab, tgt_l));
 	head = &b->head;
 
 	ret = htab_lock_bucket(b, &flags);
@@ -912,7 +1024,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
 	head = select_bucket(htab, hash);
 
 	/* lookup the key */
-	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);
+	l = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab->n_buckets);
 
 	if (!l)
 		goto find_first_elem;
@@ -923,7 +1035,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
 
 	if (next_l) {
 		/* if next elem in this hash list is non-zero, just return it */
-		memcpy(next_key, next_l->key, key_size);
+		memcpy(next_key, htab_elem_key(htab, next_l), key_size);
 		return 0;
 	}
 
@@ -941,7 +1053,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
 					  struct htab_elem, hash_node);
 		if (next_l) {
 			/* if it's not empty, just return it */
-			memcpy(next_key, next_l->key, key_size);
+			memcpy(next_key, htab_elem_key(htab, next_l), key_size);
 			return 0;
 		}
 	}
@@ -955,7 +1067,7 @@ static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)
 	check_and_cancel_fields(htab, l);
 
 	if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH)
-		bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);
+		bpf_mem_cache_free(&htab->pcpu_ma, htab_elem_get_ptr_to_pptr(l));
 	bpf_mem_cache_free(&htab->ma, l);
 }
 
@@ -965,7 +1077,7 @@ static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)
 	void *ptr;
 
 	if (map->ops->map_fd_put_ptr) {
-		ptr = fd_htab_map_get_ptr(map, l);
+		ptr = fd_htab_map_get_ptr(htab, l);
 		map->ops->map_fd_put_ptr(map, ptr, true);
 	}
 }
@@ -1117,10 +1229,10 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
 		}
 	}
 
-	memcpy(l_new->key, key, key_size);
+	memcpy(htab_elem_key(htab, l_new), key, key_size);
 	if (percpu) {
 		if (prealloc) {
-			pptr = htab_elem_get_ptr(l_new, key_size);
+			pptr = htab_elem_get_ptr(htab, l_new);
 		} else {
 			/* alloc_percpu zero-fills */
 			void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma);
@@ -1130,26 +1242,26 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
 				l_new = ERR_PTR(-ENOMEM);
 				goto dec_count;
 			}
-			l_new->ptr_to_pptr = ptr;
+			htab_elem_set_ptr_to_pptr(l_new, ptr);
 			pptr = *(void __percpu **)ptr;
 		}
 
 		pcpu_init_value(htab, pptr, value, onallcpus, map_flags);
 
 		if (!prealloc)
-			htab_elem_set_ptr(l_new, key_size, pptr);
+			htab_elem_set_ptr(htab, l_new, pptr);
 	} else if (fd_htab_map_needs_adjust(htab)) {
 		size = round_up(size, 8);
-		memcpy(htab_elem_value(l_new, key_size), value, size);
+		memcpy(htab_elem_value(htab, l_new), value, size);
 	} else if (map_flags & BPF_F_LOCK) {
 		copy_map_value_locked(&htab->map,
-				      htab_elem_value(l_new, key_size),
+				      htab_elem_value(htab, l_new),
 				      value, false);
 	} else {
-		copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value);
+		copy_map_value(&htab->map, htab_elem_value(htab, l_new), value);
 	}
 
-	l_new->hash = hash;
+	htab_elem_set_hash(htab, l_new, hash);
 	return l_new;
 dec_count:
 	dec_elem_count(htab);
@@ -1199,7 +1311,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
 		if (unlikely(!btf_record_has_field(map->record, BPF_SPIN_LOCK)))
 			return -EINVAL;
 		/* find an element without taking the bucket lock */
-		l_old = lookup_nulls_elem_raw(head, hash, key, key_size,
+		l_old = lookup_nulls_elem_raw(htab, head, hash, key, key_size,
 					      htab->n_buckets);
 		ret = check_flags(htab, l_old, map_flags);
 		if (ret)
@@ -1207,7 +1319,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
 		if (l_old) {
 			/* grab the element lock and update value in place */
 			copy_map_value_locked(map,
-					      htab_elem_value(l_old, key_size),
+					      htab_elem_value(htab, l_old),
 					      value, false);
 			return 0;
 		}
@@ -1221,7 +1333,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
 	if (ret)
 		return ret;
 
-	l_old = lookup_elem_raw(head, hash, key, key_size);
+	l_old = lookup_elem_raw(htab, head, hash, key, key_size);
 
 	ret = check_flags(htab, l_old, map_flags);
 	if (ret)
@@ -1235,7 +1347,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
 		 * and update element in place
 		 */
 		copy_map_value_locked(map,
-				      htab_elem_value(l_old, key_size),
+				      htab_elem_value(htab, l_old),
 				      value, false);
 		ret = 0;
 		goto err;
@@ -1275,7 +1387,7 @@ static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem)
 {
 	check_and_cancel_fields(htab, elem);
 	bpf_map_dec_elem_count(&htab->map);
-	bpf_lru_push_free(&htab->lru, &elem->lru_node);
+	bpf_lru_push_free(&htab->lru, htab_elem_lru_node(elem));
 }
 
 static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,
@@ -1310,13 +1422,13 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value
 	l_new = prealloc_lru_pop(htab, key, hash);
 	if (!l_new)
 		return -ENOMEM;
-	copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value);
+	copy_map_value(&htab->map, htab_elem_value(htab, l_new), value);
 
 	ret = htab_lock_bucket(b, &flags);
 	if (ret)
 		goto err_lock_bucket;
 
-	l_old = lookup_elem_raw(head, hash, key, key_size);
+	l_old = lookup_elem_raw(htab, head, hash, key, key_size);
 
 	ret = check_flags(htab, l_old, map_flags);
 	if (ret)
@@ -1327,7 +1439,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value
 	 */
 	hlist_nulls_add_head_rcu(&l_new->hash_node, head);
 	if (l_old) {
-		bpf_lru_node_set_ref(&l_new->lru_node);
+		bpf_lru_node_set_ref(htab_elem_lru_node(l_new));
 		hlist_nulls_del_rcu(&l_old->hash_node);
 	}
 	ret = 0;
@@ -1383,7 +1495,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,
 	if (ret)
 		return ret;
 
-	l_old = lookup_elem_raw(head, hash, key, key_size);
+	l_old = lookup_elem_raw(htab, head, hash, key, key_size);
 
 	ret = check_flags(htab, l_old, map_flags);
 	if (ret)
@@ -1392,10 +1504,10 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,
 	if (l_old) {
 		/* Update value in-place */
 		if (percpu) {
-			pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),
+			pcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old),
 					value, onallcpus, map_flags);
 		} else {
-			void **inner_map_pptr = htab_elem_value(l_old, key_size);
+			void **inner_map_pptr = htab_elem_value(htab, l_old);
 
 			old_map_ptr = *inner_map_pptr;
 			WRITE_ONCE(*inner_map_pptr, *(void **)value);
@@ -1456,20 +1568,20 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
 	if (ret)
 		goto err_lock_bucket;
 
-	l_old = lookup_elem_raw(head, hash, key, key_size);
+	l_old = lookup_elem_raw(htab, head, hash, key, key_size);
 
 	ret = check_flags(htab, l_old, map_flags);
 	if (ret)
 		goto err;
 
 	if (l_old) {
-		bpf_lru_node_set_ref(&l_old->lru_node);
+		bpf_lru_node_set_ref(htab_elem_lru_node(l_old));
 
 		/* per-cpu hash map can update value in-place */
-		pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),
+		pcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old),
 				value, onallcpus, map_flags);
 	} else {
-		pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size),
+		pcpu_init_value(htab, htab_elem_get_ptr(htab, l_new),
 				value, onallcpus, map_flags);
 		hlist_nulls_add_head_rcu(&l_new->hash_node, head);
 		l_new = NULL;
@@ -1480,7 +1592,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
 err_lock_bucket:
 	if (l_new) {
 		bpf_map_dec_elem_count(&htab->map);
-		bpf_lru_push_free(&htab->lru, &l_new->lru_node);
+		bpf_lru_push_free(&htab->lru, htab_elem_lru_node(l_new));
 	}
 	return ret;
 }
@@ -1521,7 +1633,7 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key)
 	if (ret)
 		return ret;
 
-	l = lookup_elem_raw(head, hash, key, key_size);
+	l = lookup_elem_raw(htab, head, hash, key, key_size);
 	if (l)
 		hlist_nulls_del_rcu(&l->hash_node);
 	else
@@ -1556,7 +1668,7 @@ static long htab_lru_map_delete_elem(struct bpf_map *map, void *key)
 	if (ret)
 		return ret;
 
-	l = lookup_elem_raw(head, hash, key, key_size);
+	l = lookup_elem_raw(htab, head, hash, key, key_size);
 
 	if (l)
 		hlist_nulls_del_rcu(&l->hash_node);
@@ -1602,7 +1714,7 @@ static void htab_free_malloced_internal_structs(struct bpf_htab *htab)
 		hlist_nulls_for_each_entry(l, n, head, hash_node) {
 			/* We only free internal structs on uref dropping to zero */
 			bpf_map_free_internal_structs(&htab->map,
-						      htab_elem_value(l, htab->map.key_size));
+						      htab_elem_value(htab, l));
 		}
 		cond_resched_rcu();
 	}
@@ -1697,7 +1809,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
 	if (ret)
 		return ret;
 
-	l = lookup_elem_raw(head, hash, key, key_size);
+	l = lookup_elem_raw(htab, head, hash, key, key_size);
 	if (!l) {
 		ret = -ENOENT;
 		goto out_unlock;
@@ -1708,14 +1820,14 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
 		void __percpu *pptr;
 		int off = 0, cpu;
 
-		pptr = htab_elem_get_ptr(l, key_size);
+		pptr = htab_elem_get_ptr(htab, l);
 		for_each_possible_cpu(cpu) {
 			copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu));
 			check_and_init_map_value(&htab->map, value + off);
 			off += roundup_value_size;
 		}
 	} else {
-		void *src = htab_elem_value(l, map->key_size);
+		void *src = htab_elem_value(htab, l);
 
 		if (flags & BPF_F_LOCK)
 			copy_map_value_locked(map, value, src, true);
@@ -1898,13 +2010,13 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
 		goto next_batch;
 
 	hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {
-		memcpy(dst_key, l->key, key_size);
+		memcpy(dst_key, htab_elem_key(htab, l), key_size);
 
 		if (is_percpu) {
 			int off = 0, cpu;
 			void __percpu *pptr;
 
-			pptr = htab_elem_get_ptr(l, map->key_size);
+			pptr = htab_elem_get_ptr(htab, l);
 			if (elem_map_flags & BPF_F_CPU) {
 				cpu = elem_map_flags >> 32;
 				copy_map_value(&htab->map, dst_val, per_cpu_ptr(pptr, cpu));
@@ -1918,7 +2030,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
 				}
 			}
 		} else {
-			value = htab_elem_value(l, key_size);
+			value = htab_elem_value(htab, l);
 			if (is_fd_htab(htab)) {
 				struct bpf_map **inner_map = value;
 
@@ -2183,12 +2295,12 @@ static int __bpf_hash_map_seq_show(struct seq_file *seq, struct htab_elem *elem)
 		ctx.meta = &meta;
 		ctx.map = info->map;
 		if (elem) {
-			ctx.key = elem->key;
+			ctx.key = htab_elem_key(info->htab, elem);
 			if (!info->percpu_value_buf) {
-				ctx.value = htab_elem_value(elem, map->key_size);
+				ctx.value = htab_elem_value(info->htab, elem);
 			} else {
 				roundup_value_size = round_up(map->value_size, 8);
-				pptr = htab_elem_get_ptr(elem, map->key_size);
+				pptr = htab_elem_get_ptr(info->htab, elem);
 				for_each_possible_cpu(cpu) {
 					copy_map_value_long(map, info->percpu_value_buf + off,
 							    per_cpu_ptr(pptr, cpu));
@@ -2293,13 +2405,13 @@ static long bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_
 		rcu_read_lock();
 		head = &b->head;
 		hlist_nulls_for_each_entry_safe(elem, n, head, hash_node) {
-			key = elem->key;
+			key = htab_elem_key(htab, elem);
 			if (is_percpu) {
 				/* current cpu value for percpu map */
-				pptr = htab_elem_get_ptr(elem, map->key_size);
+				pptr = htab_elem_get_ptr(htab, elem);
 				val = this_cpu_ptr(pptr);
 			} else {
-				val = htab_elem_value(elem, map->key_size);
+				val = htab_elem_value(htab, elem);
 			}
 			num_elems++;
 			ret = callback_fn((u64)(long)map, (u64)(long)key,
@@ -2403,10 +2515,11 @@ const struct bpf_map_ops htab_lru_map_ops = {
 /* Called from eBPF program */
 static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l = __htab_map_lookup_elem(map, key);
 
 	if (l)
-		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));
+		return this_cpu_ptr(htab_elem_get_ptr(htab, l));
 	else
 		return NULL;
 }
@@ -2414,6 +2527,7 @@ static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key)
 /* inline bpf_map_lookup_elem() call for per-CPU hashmap */
 static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct bpf_insn *insn = insn_buf;
 
 	if (!bpf_jit_supports_percpu_insn())
@@ -2424,7 +2538,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn
 	*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
 	*insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3);
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0,
-				offsetof(struct htab_elem, key) + roundup(map->key_size, 8));
+				htab->key_offset + roundup(map->key_size, 8));
 	*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
 	*insn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
 
@@ -2433,6 +2547,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn
 
 static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l;
 
 	if (cpu >= nr_cpu_ids)
@@ -2440,18 +2555,19 @@ static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key,
 
 	l = __htab_map_lookup_elem(map, key);
 	if (l)
-		return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu);
+		return per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu);
 	else
 		return NULL;
 }
 
 static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l = __htab_map_lookup_elem(map, key);
 
 	if (l) {
-		bpf_lru_node_set_ref(&l->lru_node);
-		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));
+		bpf_lru_node_set_ref(htab_elem_lru_node(l));
+		return this_cpu_ptr(htab_elem_get_ptr(htab, l));
 	}
 
 	return NULL;
@@ -2459,6 +2575,7 @@ static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key)
 
 static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l;
 
 	if (cpu >= nr_cpu_ids)
@@ -2466,8 +2583,8 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k
 
 	l = __htab_map_lookup_elem(map, key);
 	if (l) {
-		bpf_lru_node_set_ref(&l->lru_node);
-		return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu);
+		bpf_lru_node_set_ref(htab_elem_lru_node(l));
+		return per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu);
 	}
 
 	return NULL;
@@ -2475,6 +2592,7 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k
 
 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value, u64 map_flags)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l;
 	void __percpu *pptr;
 	int ret = -ENOENT;
@@ -2494,7 +2612,7 @@ int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value, u64 map_fl
 	/* We do not mark LRU map element here in order to not mess up
 	 * eviction heuristics when user space does a map walk.
 	 */
-	pptr = htab_elem_get_ptr(l, map->key_size);
+	pptr = htab_elem_get_ptr(htab, l);
 	if (map_flags & BPF_F_CPU) {
 		cpu = map_flags >> 32;
 		copy_map_value(map, value, per_cpu_ptr(pptr, cpu));
@@ -2532,6 +2650,7 @@ int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
 static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key,
 					  struct seq_file *m)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l;
 	void __percpu *pptr;
 	int cpu;
@@ -2546,7 +2665,7 @@ static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key,
 
 	btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);
 	seq_puts(m, ": {\n");
-	pptr = htab_elem_get_ptr(l, map->key_size);
+	pptr = htab_elem_get_ptr(htab, l);
 	for_each_possible_cpu(cpu) {
 		seq_printf(m, "\tcpu%d: ", cpu);
 		btf_type_seq_show(map->btf, map->btf_value_type_id,
@@ -2620,7 +2739,7 @@ static void fd_htab_map_free(struct bpf_map *map)
 		head = select_bucket(htab, i);
 
 		hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {
-			void *ptr = fd_htab_map_get_ptr(map, l);
+			void *ptr = fd_htab_map_get_ptr(htab, l);
 
 			map->ops->map_fd_put_ptr(map, ptr, false);
 		}
@@ -2705,6 +2824,7 @@ static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key)
 static int htab_of_map_gen_lookup(struct bpf_map *map,
 				  struct bpf_insn *insn_buf)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct bpf_insn *insn = insn_buf;
 	const int ret = BPF_REG_0;
 
@@ -2713,7 +2833,7 @@ static int htab_of_map_gen_lookup(struct bpf_map *map,
 	*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
 	*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2);
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,
-				offsetof(struct htab_elem, key) +
+				htab->key_offset +
 				round_up(map->key_size, 8));
 	*insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);
 
@@ -3112,7 +3232,7 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf,
 {
 	struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
 
-	return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor);
+	return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor, offsetof(struct rhtab_elem, data));
 }
 
 static void rhtab_map_free_internal_structs(struct bpf_map *map)
@@ -3533,3 +3653,16 @@ const struct bpf_map_ops rhtab_map_ops = {
 	.map_btf_id = &rhtab_map_btf_ids[0],
 	.iter_seq_info = &rhash_iter_seq_info,
 };
+
+size_t bpf_htab_map_meta_size(void)
+{
+	return sizeof(struct bpf_htab);
+}
+
+void bpf_htab_map_meta_init(struct bpf_map *inner_map_meta, struct bpf_map *inner_map)
+{
+	struct bpf_htab *inner_htab_meta = container_of(inner_map_meta, struct bpf_htab, map);
+	struct bpf_htab *inner_htab = container_of(inner_map, struct bpf_htab, map);
+
+	inner_htab_meta->key_offset = inner_htab->key_offset;
+}
diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c
index d2cbab4bdf644a..b2db5d0c98bee2 100644
--- a/kernel/bpf/map_in_map.c
+++ b/kernel/bpf/map_in_map.c
@@ -7,6 +7,15 @@
 
 #include "map_in_map.h"
 
+static bool bpf_map_type_is_htab(enum bpf_map_type type)
+{
+	return type == BPF_MAP_TYPE_HASH ||
+	       type == BPF_MAP_TYPE_PERCPU_HASH ||
+	       type == BPF_MAP_TYPE_LRU_HASH ||
+	       type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
+	       type == BPF_MAP_TYPE_HASH_OF_MAPS;
+}
+
 struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
 {
 	struct bpf_map *inner_map, *inner_map_meta;
@@ -29,6 +38,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
 	/* In some cases verifier needs to access beyond just base map. */
 	if (inner_map->ops == &array_map_ops || inner_map->ops == &percpu_array_map_ops)
 		inner_map_meta_size = sizeof(struct bpf_array);
+	else if (bpf_map_type_is_htab(inner_map->map_type))
+		inner_map_meta_size = bpf_htab_map_meta_size();
 
 	inner_map_meta = kzalloc(inner_map_meta_size, GFP_USER);
 	if (!inner_map_meta)
@@ -70,6 +81,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
 		inner_array_meta->index_mask = inner_array->index_mask;
 		inner_array_meta->elem_size = inner_array->elem_size;
 		inner_map_meta->bypass_spec_v1 = inner_map->bypass_spec_v1;
+	} else if (bpf_map_type_is_htab(inner_map->map_type)) {
+		bpf_htab_map_meta_init(inner_map_meta, inner_map);
 	}
 	return inner_map_meta;
 }
diff --git a/kernel/bpf/map_in_map.h b/kernel/bpf/map_in_map.h
index 7d61602354de80..1edc205772d8cd 100644
--- a/kernel/bpf/map_in_map.h
+++ b/kernel/bpf/map_in_map.h
@@ -15,5 +15,7 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, struct file *map_file,
 			 int ufd);
 void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer);
 u32 bpf_map_fd_sys_lookup_elem(void *ptr);
+size_t bpf_htab_map_meta_size(void);
+void bpf_htab_map_meta_init(struct bpf_map *inner_map_meta, struct bpf_map *inner_map);
 
 #endif
diff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c
index 373c8d17ea557d..6bd4cb68c20c1c 100644
--- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c
+++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c
@@ -114,7 +114,7 @@ static inline int check_hash(void)
 	VERIFY(check_default_noinline(&hash->map, map));
 
 	VERIFY(hash->n_buckets == MAX_ENTRIES);
-	VERIFY(hash->elem_size == 64);
+	VERIFY(hash->elem_size == 32);
 
 	VERIFY(hash->count.counter == 0);
 	VERIFY(bpf_map_sum_elem_count(map) == 0);

3/2

2026/07/23 22:38

llm

patch-evaluator

Model:

gemini-3.1-pro-preview

Args:

null

Results:

null

Tokens:

							input: 18572
							output: 37
							thoughts: 583

Thoughts:

**Analyzing the BPF Hash Table Optimization Patch**

Okay, so I've got this patch – seems like a pretty significant one – touching `kernel/bpf/hashtab.c` and `kernel/bpf/map_in_map.c`. From the description, it looks like they're optimizing the memory layout of BPF hash tables, splitting `struct htab_elem` into multiple specialized structures (`htab_elem`, `htab_elem_lru`, `htab_elem_pcpu`) and also adding `key_offset` to `struct bpf_htab`. This is a *functional* change to the core hash map implementation, and that immediately tells me we need to treat this as high-priority for fuzzing. It's a prime target for uncovering subtle memory corruption or concurrency issues.

My immediate next step is to drill down into the details of the code. Let's see what's been added, modified, or changed. I'm going to look closely at these new or changed functions: `htab_elem_lru_node`, `htab_elem_get_ptr_to_pptr`, `htab_elem_set_ptr_to_pptr`, `htab_elem_key`, `htab_elem_value`, `htab_elem_set_ptr`, `htab_elem_get_ptr`, `fd_htab_map_get_ptr`, `htab_has_hash`, `htab_elem_hash`, `htab_elem_set_hash`, `htab_map_alloc_check`, `htab_map_alloc`, `htab_map_gen_lookup`, `htab_lru_map_gen_lookup`, `htab_map_update_elem`, `htab_lru_map_update_elem`, `__htab_lru_percpu_map_update_elem`, `htab_map_delete_elem`, `htab_lru_map_delete_elem`, `__htab_map_lookup_and_delete_elem`, `__htab_map_lookup_and_delete_batch`, `bpf_percpu_hash_copy`, `htab_of_map_gen_lookup`, `bpf_htab_map_meta_size`, `bpf_htab_map_meta_init`, and `bpf_map_meta_alloc`. Any one of these, if mishandled, could have significant security implications.

Now, let's nail down the configuration dependencies. `CONFIG_BPF_SYSCALL` will be the primary enabler for BPF, of course. Given the focus on hash maps, I'm going to quickly check if there are any specific configuration flags for BPF hash maps themselves. No dedicated config options, which makes sense - the hash table implementation is core BPF.

I need to confirm these functions' interactions with the existing BPF hash map infrastructure. Time to search the source for `BPF_MAP_TYPE_HASH` and `BPF_MAP_TYPE_LRU_HASH` within `kernel/bpf/hashtab.c` to see how those map types are integrated with the new code. This will help understand the breadth of impact of this change.

4/2

2026/07/23 22:38

tool

grepper

Args:

{
  "Expression": "BPF_MAP_TYPE_HASH",
  "PathPrefix": "kernel/bpf/hashtab.c"
}

Results:

{
  "Output": "kernel/bpf/hashtab.c=225=static inline bool is_fd_htab(const struct bpf_htab *htab)\nkernel/bpf/hashtab.c-226-{\nkernel/bpf/hashtab.c:227:\treturn htab-\u003emap.map_type == BPF_MAP_TYPE_HASH_OF_MAPS;\nkernel/bpf/hashtab.c-228-}\n"
}

5/2

2026/07/23 22:38

llm

patch-evaluator

Model:

gemini-3.1-pro-preview

Args:

null

Results:

null

Tokens:

							input: 20021
							output: 31
							thoughts: 41

Thoughts:

Here's my attempt at summarizing the thought process, tailored for an expert audience:

**Investigating Potential BPF Configuration Nuances**

Okay, before I dive too deep into the implications here, I need to take a quick peek at the BPF internals. Specifically, I should check `kernel/bpf/hashtab.c` and `kernel/bpf/map_in_map.c`. Those files are likely to contain the crucial low-level details regarding potential configurations or limitations that I need to be aware of. I'm looking for any specific configurations that might impact the behavior or performance I'm considering. Gotta make sure I'm not overlooking some subtle detail buried within those implementations. That level of detail is critical for fully understanding the landscape here.

6/2

2026/07/23 22:38

tool

grepper

Error:

no matches

Args:

{
  "Expression": "CONFIG_",
  "PathPrefix": "kernel/bpf/hashtab.c"
}

Results:

{
  "error": "no matches"
}

7/2

2026/07/23 22:38

llm

patch-evaluator

Model:

gemini-3.1-pro-preview

Args:

null

Results:

null

Tokens:

							input: 20267
							output: 34
							thoughts: 24

Thoughts:

Here's my thought process, as if I were considering this myself:

**Investigating Configuration Dependencies**

Okay, it seems there are no specific configuration details explicitly defined within `hashtab.c`. That's not entirely unexpected, depending on the system design, but still good to confirm. Now, if the configuration isn't in the obvious place, where else could it be? The structure often dictates the organization of code. My instincts are telling me to look at `map_in_map.c`. It's possible that the configuration parameters are somehow managed within the nested map structure itself, or at least referenced from there. Let's see what we can dig up there.

8/2

2026/07/23 22:38

tool

grepper

Error:

no matches

Args:

{
  "Expression": "CONFIG_",
  "PathPrefix": "kernel/bpf/map_in_map.c"
}

Results:

{
  "error": "no matches"
}

9/2

2026/07/23 22:38

llm

patch-evaluator

Model:

gemini-3.1-pro-preview

Args:

null

Results:

null

Tokens:

							input: 20482
							output: 278
							thoughts: 1140

Thoughts:

**Analyzing BPF Hash Map Operations for Fuzzing**

Okay, first things first, this is definitely related to `CONFIG_BPF_SYSCALL`, that's the key enabling configuration. Let's dig in and figure out the most important functions to target for fuzzing. This patch is clearly touching BPF hash map operations.

My initial thought is to grab all the relevant functions, and then refine. I'll need to focus on: `htab_map_alloc`, `htab_map_update_elem`, `htab_lru_map_update_elem`, `__htab_lru_percpu_map_update_elem`, `htab_map_delete_elem`, `htab_lru_map_delete_elem`, `__htab_map_lookup_and_delete_elem`, `__htab_map_lookup_and_delete_batch`, `bpf_map_meta_alloc`, and the various lookup functions: `htab_map_gen_lookup`, `htab_lru_map_gen_lookup`, `htab_percpu_map_gen_lookup`, and `htab_of_map_gen_lookup`. These seem like the core entry points for the changes.

Alright, I'm going to refine that list a little. Wait, I remember that `htab_map_update_elem` is a pretty important hot-path function, the specific implementation for hash maps, so it's a prime target, especially considering fuzzing BPF map operations is standard practice.

After that initial list I'll keep the ones I've listed previously and focus on the `bpf_map_meta_alloc` because that could be a significant change and thus a potential target for fuzzing.

Now, let's see, `htab_map_update_elem` is static, which is usually not ideal for fuzzing. However, it's assigned to `map_ops`, so Syzkaller *should* be able to target it through the `bpf()` syscall. Yes, `FocusSymbols` are specifically those that can be targeted by syzkaller or are critical for coverage.

So, the definitive list for my analysis now includes: `htab_map_alloc`, `htab_map_update_elem`, `htab_lru_map_update_elem`, `__htab_lru_percpu_map_update_elem`, `htab_map_delete_elem`, `htab_lru_map_delete_elem`, `__htab_map_lookup_and_delete_elem`, `__htab_map_lookup_and_delete_batch`, `bpf_map_meta_alloc`, `htab_map_gen_lookup`, `htab_lru_map_gen_lookup`, `htab_percpu_map_gen_lookup`, and `htab_of_map_gen_lookup`. I'll make sure to note these.

Just verifying that `htab_map_gen_lookup` is indeed present in the diff. Good.

Yes, this is definitely worth fuzzing. The patch is changing the memory layout and operations of BPF hash maps, including the LRU and per-CPU variants. It introduces new structures and offsets which in turn affects crucial operations like allocation, updates, deletions, and JIT lookup generation. Fuzzing is essential here to ensure memory safety and the correctness of these core BPF map operations given the substantial changes.

My final settings will be: `WorthFuzzing: true`, `EnableConfigs: ["BPF_SYSCALL"]`, `FocusSymbols: ["htab_map_alloc", "htab_map_update_elem", "htab_lru_map_update_elem", "__htab_lru_percpu_map_update_elem", "htab_map_delete_elem", "htab_lru_map_delete_elem", "__htab_map_lookup_and_delete_elem", "__htab_map_lookup_and_delete_batch", "bpf_map_meta_alloc", "htab_map_gen_lookup", "htab_lru_map_gen_lookup", "htab_percpu_map_gen_lookup", "htab_of_map_gen_lookup"]`, and `Reasoning: "The patch functionally modifies the memory layout and operations of BPF hash maps (including LRU and per-CPU variants). It introduces new structures and offsets for hash map elements, which affects allocation, updates, deletions, and JIT lookup generation. Fuzzing is needed to ensure memory safety and correctness of these core BPF map operations."`

10/2

2026/07/23 22:39

tool

set-results

Args:

{
  "EnableConfigs": [
    "BPF_SYSCALL"
  ],
  "FocusSymbols": [
    "htab_map_alloc",
    "htab_map_update_elem",
    "htab_lru_map_update_elem",
    "__htab_lru_percpu_map_update_elem",
    "htab_map_delete_elem",
    "htab_lru_map_delete_elem",
    "__htab_map_lookup_and_delete_elem",
    "__htab_map_lookup_and_delete_batch",
    "bpf_map_meta_alloc",
    "htab_map_gen_lookup",
    "htab_lru_map_gen_lookup",
    "htab_percpu_map_gen_lookup",
    "htab_of_map_gen_lookup"
  ],
  "Reasoning": "The patch functionally modifies the memory layout and operations of BPF hash maps (including LRU and per-CPU variants). It introduces new structures and offsets for hash map elements, which affects allocation, updates, deletions, and JIT lookup generation. Fuzzing is needed to ensure memory safety and correctness of these core BPF map operations.",
  "WorthFuzzing": true
}

Results:

{
  "EnableConfigs": [
    "BPF_SYSCALL"
  ],
  "FocusSymbols": [
    "htab_map_alloc",
    "htab_map_update_elem",
    "htab_lru_map_update_elem",
    "__htab_lru_percpu_map_update_elem",
    "htab_map_delete_elem",
    "htab_lru_map_delete_elem",
    "__htab_map_lookup_and_delete_elem",
    "__htab_map_lookup_and_delete_batch",
    "bpf_map_meta_alloc",
    "htab_map_gen_lookup",
    "htab_lru_map_gen_lookup",
    "htab_percpu_map_gen_lookup",
    "htab_of_map_gen_lookup"
  ],
  "Reasoning": "The patch functionally modifies the memory layout and operations of BPF hash maps (including LRU and per-CPU variants). It introduces new structures and offsets for hash map elements, which affects allocation, updates, deletions, and JIT lookup generation. Fuzzing is needed to ensure memory safety and correctness of these core BPF map operations.",
  "WorthFuzzing": true
}

LLM Calls Summary:
Total Calls	Total Tokens	Avg Tokens	Total Duration (Seconds)	Avg Duration (Seconds)

Tool Calls Summary:
Total Calls	Total Duration (Seconds)	Avg Duration (Seconds)