AI Trajectory

Trajectory: (Expand/Collapse All)
Seq	Timestamp	Type	Name	Duration
0/0	2026/08/05 22:52	flow	patch-triage	0m Args: null Results: { "EnableConfigs": [ "BPF_SYSCALL" ], "FocusSymbols": [ "htab_map_alloc", "htab_map_update_elem", "htab_lru_map_update_elem", "htab_map_lookup_elem", "htab_map_delete_elem", "bpf_map_meta_alloc" ], "Reasoning": "The patch optimizes the memory layout of BPF hash map elements by conditionally omitting the hash value and adjusting the key offset. This affects core BPF hash map operations like allocation, lookup, update, and deletion. Fuzzing is necessary to ensure these operations still work correctly with the new memory layout and that no out-of-bounds accesses or memory leaks occur.", "WorthFuzzing": true }
1/1	2026/08/05 22:52	action	read-patch-diff	0m Args: null Results: { "PatchDiff": "commit 6454a11c03dfaedb4b09f8575af8d03639fb5273\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate: Wed Aug 5 22:52:22 2026 +0000\n\n syz-cluster: applied patch under review\n\ndiff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c\nindex 9f394e1aa2e85..9967268d453d0 100644\n--- a/kernel/bpf/hashtab.c\n+++ b/kernel/bpf/hashtab.c\n@@ -100,13 +100,16 @@ struct bpf_htab {\n \tstruct percpu_counter pcount;\n \tatomic_t count;\n \tbool use_percpu_counter;\n+\tbool has_hash;\n \tu32 n_buckets;\t/* number of hash buckets /\n \tu32 elem_size;\t/ size of each element in bytes /\n+\tu32 key_offset;\t/ offset of key in bytes /\n \tu32 hashrnd;\n };\n \n / each htab element is struct htab_elem + key + value /\n-struct htab_elem {\n+struct htab_elem;\n+struct htab_node {\n \tunion {\n \t\tstruct hlist_nulls_node hash_node;\n \t\tstruct {\n@@ -117,18 +120,32 @@ struct htab_elem {\n \t\t\t};\n \t\t};\n \t};\n-\tunion {\n-\t\t/ pointer to per-cpu pointer /\n-\t\tvoid ptr_to_pptr;\n-\t\tstruct bpf_lru_node lru_node;\n-\t};\n-\tu32 hash;\n-\tchar key[] __aligned(8);\n+};\n+\n+struct htab_elem {\n+\tstruct htab_node node;\n+\tu8 data[] __aligned(8);\n+};\n+\n+struct htab_elem_lru {\n+\tstruct htab_node node;\n+\tstruct bpf_lru_node lru_node;\n+\tu8 data[] __aligned(8);\n+};\n+\n+/* Only for non-preallocated PCPU maps. Preallocated PCPU maps don't need\n+ * ptr_to_pptr, and use htab_elem.\n+ /\n+struct htab_elem_pcpu {\n+\tstruct htab_node node;\n+\tvoid ptr_to_pptr;\n+\tu8 data[] __aligned(8);\n };\n \n struct htab_btf_record {\n \tstruct btf_record record;\n \tu32 key_size;\n+\tu32 key_offset;\n };\n \n static inline bool htab_is_prealloc(const struct bpf_htab htab)\n@@ -136,6 +153,21 @@ static inline bool htab_is_prealloc(const struct bpf_htab htab)\n \treturn !(htab-\u003emap.map_flags \u0026 BPF_F_NO_PREALLOC);\n }\n \n+static inline struct bpf_lru_node htab_elem_lru_node(struct htab_elem l)\n+{\n+\treturn \u0026((struct htab_elem_lru )l)-\u003elru_node;\n+}\n+\n+static inline void htab_elem_get_ptr_to_pptr(struct htab_elem l)\n+{\n+\treturn ((struct htab_elem_pcpu )l)-\u003eptr_to_pptr;\n+}\n+\n+static inline void htab_elem_set_ptr_to_pptr(struct htab_elem l, void ptr)\n+{\n+\t((struct htab_elem_pcpu )l)-\u003eptr_to_pptr = ptr;\n+}\n+\n static void htab_init_buckets(struct bpf_htab htab)\n {\n \tunsigned int i;\n@@ -183,25 +215,30 @@ static inline bool is_fd_htab(const struct bpf_htab htab)\n \treturn htab-\u003emap.map_type == BPF_MAP_TYPE_HASH_OF_MAPS;\n }\n \n-static inline void htab_elem_value(struct htab_elem l, u32 key_size)\n+static inline void htab_elem_key(struct bpf_htab htab, struct htab_elem l)\n+{\n+\treturn (void )l + htab-\u003ekey_offset;\n+}\n+\n+static inline void htab_elem_value(struct bpf_htab htab, struct htab_elem l)\n {\n-\treturn l-\u003ekey + round_up(key_size, 8);\n+\treturn htab_elem_key(htab, l) + round_up(htab-\u003emap.key_size, 8);\n }\n \n-static inline void htab_elem_set_ptr(struct htab_elem l, u32 key_size,\n+static inline void htab_elem_set_ptr(struct bpf_htab htab, struct htab_elem l,\n \t\t\t\t void __percpu pptr)\n {\n-\t(void __percpu *)htab_elem_value(l, key_size) = pptr;\n+\t(void __percpu *)htab_elem_value(htab, l) = pptr;\n }\n \n-static inline void __percpu htab_elem_get_ptr(struct htab_elem l, u32 key_size)\n+static inline void __percpu htab_elem_get_ptr(struct bpf_htab htab, struct htab_elem l)\n {\n-\treturn (void __percpu )htab_elem_value(l, key_size);\n+\treturn (void __percpu *)htab_elem_value(htab, l);\n }\n \n-static void fd_htab_map_get_ptr(const struct bpf_map map, struct htab_elem l)\n+static void fd_htab_map_get_ptr(struct bpf_htab htab, struct htab_elem l)\n {\n-\treturn (void *)htab_elem_value(l, map-\u003ekey_size);\n+\treturn (void *)htab_elem_value(htab, l);\n }\n \n static struct htab_elem get_htab_elem(struct bpf_htab htab, int i)\n@@ -209,6 +246,33 @@ static struct htab_elem get_htab_elem(struct bpf_htab htab, int i)\n \treturn (struct htab_elem ) (htab-\u003eelems + i * (u64)htab-\u003eelem_size);\n }\n \n+static inline bool htab_has_hash(const struct bpf_htab htab)\n+{\n+\treturn htab-\u003ehas_hash;\n+}\n+\n+static inline u32 htab_elem_hash(struct bpf_htab htab, struct htab_elem l)\n+{\n+\tif (htab_is_lru(htab))\n+\t\treturn (u32 )((struct htab_elem_lru )l)-\u003edata;\n+\telse if (htab_is_percpu(htab) \u0026\u0026 !htab_is_prealloc(htab))\n+\t\treturn (u32 )((struct htab_elem_pcpu )l)-\u003edata;\n+\telse\n+\t\treturn (u32 )l-\u003edata;\n+}\n+\n+static inline void htab_elem_set_hash(struct bpf_htab htab, struct htab_elem l, u32 hash)\n+{\n+\tif (!htab_has_hash(htab))\n+\t\treturn;\n+\tif (htab_is_lru(htab))\n+\t\t(u32 )((struct htab_elem_lru )l)-\u003edata = hash;\n+\telse if (htab_is_percpu(htab) \u0026\u0026 !htab_is_prealloc(htab))\n+\t\t(u32 )((struct htab_elem_pcpu )l)-\u003edata = hash;\n+\telse\n+\t\t(u32 )l-\u003edata = hash;\n+}\n+\n / Both percpu and fd htab support in-place update, so no need for\n * extra elem. LRU itself can remove the least used element, so\n * there is no need for an extra elem during map_update.\n@@ -231,7 +295,7 @@ static void htab_free_prealloced_internal_structs(struct bpf_htab htab)\n \n \t\telem = get_htab_elem(htab, i);\n \t\tbpf_map_free_internal_structs(\u0026htab-\u003emap,\n-\t\t\t\t\t htab_elem_value(elem, htab-\u003emap.key_size));\n+\t\t\t\t\t htab_elem_value(htab, elem));\n \t\tcond_resched();\n \t}\n }\n@@ -254,7 +318,7 @@ static void htab_free_prealloced_fields(struct bpf_htab htab)\n \n \t\telem = get_htab_elem(htab, i);\n \t\tif (htab_is_percpu(htab)) {\n-\t\t\tvoid __percpu pptr = htab_elem_get_ptr(elem, htab-\u003emap.key_size);\n+\t\t\tvoid __percpu pptr = htab_elem_get_ptr(htab, elem);\n \t\t\tint cpu;\n \n \t\t\tfor_each_possible_cpu(cpu) {\n@@ -263,7 +327,7 @@ static void htab_free_prealloced_fields(struct bpf_htab htab)\n \t\t\t}\n \t\t} else {\n \t\t\tbpf_obj_free_fields(htab-\u003emap.record,\n-\t\t\t\t\t htab_elem_value(elem, htab-\u003emap.key_size));\n+\t\t\t\t\t htab_elem_value(htab, elem));\n \t\t\tcond_resched();\n \t\t}\n \t\tcond_resched();\n@@ -280,8 +344,7 @@ static void htab_free_elems(struct bpf_htab htab)\n \tfor (i = 0; i \u003c htab-\u003emap.max_entries; i++) {\n \t\tvoid __percpu pptr;\n \n-\t\tpptr = htab_elem_get_ptr(get_htab_elem(htab, i),\n-\t\t\t\t\t htab-\u003emap.key_size);\n+\t\tpptr = htab_elem_get_ptr(htab, get_htab_elem(htab, i));\n \t\tfree_percpu(pptr);\n \t\tcond_resched();\n \t}\n@@ -308,8 +371,8 @@ static struct htab_elem prealloc_lru_pop(struct bpf_htab htab, void key,\n \n \tif (node) {\n \t\tbpf_map_inc_elem_count(\u0026htab-\u003emap);\n-\t\tl = container_of(node, struct htab_elem, lru_node);\n-\t\tmemcpy(l-\u003ekey, key, htab-\u003emap.key_size);\n+\t\tl = (struct htab_elem )container_of(node, struct htab_elem_lru, lru_node);\n+\t\tmemcpy(htab_elem_key(htab, l), key, htab-\u003emap.key_size);\n \t\treturn l;\n \t}\n \n@@ -340,8 +403,7 @@ static int prealloc_init(struct bpf_htab htab)\n \t\t\t\t\t GFP_USER \| __GFP_NOWARN);\n \t\tif (!pptr)\n \t\t\tgoto free_elems;\n-\t\thtab_elem_set_ptr(get_htab_elem(htab, i), htab-\u003emap.key_size,\n-\t\t\t\t pptr);\n+\t\thtab_elem_set_ptr(htab, get_htab_elem(htab, i), pptr);\n \t\tcond_resched();\n \t}\n \n@@ -349,8 +411,8 @@ static int prealloc_init(struct bpf_htab htab)\n \tif (htab_is_lru(htab))\n \t\terr = bpf_lru_init(\u0026htab-\u003elru,\n \t\t\t\t htab-\u003emap.map_flags \u0026 BPF_F_NO_COMMON_LRU,\n-\t\t\t\t offsetof(struct htab_elem, hash) -\n-\t\t\t\t offsetof(struct htab_elem, lru_node),\n+\t\t\t\t offsetof(struct htab_elem_lru, data) -\n+\t\t\t\t offsetof(struct htab_elem_lru, lru_node),\n \t\t\t\t htab_lru_map_delete_node,\n \t\t\t\t htab);\n \telse\n@@ -361,11 +423,11 @@ static int prealloc_init(struct bpf_htab htab)\n \n \tif (htab_is_lru(htab))\n \t\tbpf_lru_populate(\u0026htab-\u003elru, htab-\u003eelems,\n-\t\t\t\t offsetof(struct htab_elem, lru_node),\n+\t\t\t\t offsetof(struct htab_elem_lru, lru_node),\n \t\t\t\t htab-\u003eelem_size, num_entries);\n \telse\n \t\tpcpu_freelist_populate(\u0026htab-\u003efreelist,\n-\t\t\t\t htab-\u003eelems + offsetof(struct htab_elem, fnode),\n+\t\t\t\t htab-\u003eelems + offsetof(struct htab_elem, node.fnode),\n \t\t\t\t htab-\u003eelem_size, num_entries);\n \n \treturn 0;\n@@ -401,7 +463,7 @@ static int alloc_extra_elems(struct bpf_htab htab)\n \t\t/ pop will succeed, since prealloc_init()\n \t\t * preallocated extra num_possible_cpus elements\n \t\t /\n-\t\tl_new = container_of(l, struct htab_elem, fnode);\n+\t\tl_new = container_of(l, struct htab_elem, node.fnode);\n \t\tper_cpu_ptr(pptr, cpu) = l_new;\n \t}\n \thtab-\u003eextra_elems = pptr;\n@@ -425,8 +487,8 @@ static int htab_map_alloc_check(union bpf_attr attr)\n \tbool zero_seed = (attr-\u003emap_flags \u0026 BPF_F_ZERO_SEED);\n \tint numa_node = bpf_map_attr_numa_node(attr);\n \n-\tBUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=\n-\t\t offsetof(struct htab_elem, hash_node.pprev));\n+\tBUILD_BUG_ON(offsetof(struct htab_node, fnode.next) !=\n+\t\t offsetof(struct htab_node, hash_node.pprev));\n \n \tif (zero_seed \u0026\u0026 !capable(CAP_SYS_ADMIN))\n \t\t/ Guard against local DoS, and discourage production use. /\n@@ -476,7 +538,7 @@ static void htab_mem_dtor(void obj, void ctx)\n \tif (IS_ERR_OR_NULL(hrec-\u003erecord))\n \t\treturn;\n \n-\tmap_value = htab_elem_value(elem, hrec-\u003ekey_size);\n+\tmap_value = (void )elem + hrec-\u003ekey_offset + round_up(hrec-\u003ekey_size, 8);\n \tbpf_obj_free_fields(hrec-\u003erecord, map_value);\n }\n \n@@ -502,7 +564,7 @@ static void htab_dtor_ctx_free(void ctx)\n }\n \n static int bpf_ma_set_dtor(struct bpf_map map, struct bpf_mem_alloc ma,\n-\t\t\t void (dtor)(void , void ))\n+\t\t\t void (dtor)(void , void ), u32 key_offset)\n {\n \tstruct htab_btf_record hrec;\n \tint err;\n@@ -515,6 +577,7 @@ static int bpf_ma_set_dtor(struct bpf_map map, struct bpf_mem_alloc ma,\n \tif (!hrec)\n \t\treturn -ENOMEM;\n \threc-\u003ekey_size = map-\u003ekey_size;\n+\threc-\u003ekey_offset = key_offset;\n \threc-\u003erecord = btf_record_dup(map-\u003erecord);\n \tif (IS_ERR(hrec-\u003erecord)) {\n \t\terr = PTR_ERR(hrec-\u003erecord);\n@@ -537,9 +600,9 @@ static int htab_map_check_btf(struct bpf_map map, const struct btf btf,\n \t * populated in htab_map_alloc(), so it will always appear as NULL.\n \t /\n \tif (htab_is_percpu(htab))\n-\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003epcpu_ma, htab_pcpu_mem_dtor);\n+\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003epcpu_ma, htab_pcpu_mem_dtor, htab-\u003ekey_offset);\n \telse\n-\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003ema, htab_mem_dtor);\n+\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003ema, htab_mem_dtor, htab-\u003ekey_offset);\n }\n \n static struct bpf_map htab_map_alloc(union bpf_attr attr)\n@@ -562,6 +625,13 @@ static struct bpf_map htab_map_alloc(union bpf_attr attr)\n \n \tbpf_map_init_from_attr(\u0026htab-\u003emap, attr);\n \n+\t/ Avoid hash memory use and comparisons where unnecessary.\n+\t * u32 hash reads are always atomic. If we elide them, key comparisons must also be atomic\n+\t * to avoid false positive key matches due to torn key reads / writes. This is only possible\n+\t * when the key fits within a word, so check key_size.\n+\t /\n+\thtab-\u003ehas_hash = htab_is_lru(htab) \|\| htab-\u003emap.key_size \u003e sizeof(unsigned long);\n+\n \tif (percpu_lru) {\n \t\t/ ensure each CPU's lru list has \u003e=1 elements.\n \t\t * since we are at it, make each lru list has the same\n@@ -583,8 +653,16 @@ static struct bpf_map htab_map_alloc(union bpf_attr attr)\n \n \thtab-\u003en_buckets = roundup_pow_of_two(htab-\u003emap.max_entries);\n \n-\thtab-\u003eelem_size = sizeof(struct htab_elem) +\n-\t\t\t round_up(htab-\u003emap.key_size, 8);\n+\tif (htab_is_lru(htab))\n+\t\thtab-\u003ekey_offset = offsetof(struct htab_elem_lru, data) + 8;\n+\telse if (percpu \u0026\u0026 !prealloc)\n+\t\thtab-\u003ekey_offset = offsetof(struct htab_elem_pcpu, data) +\n+\t\t\t\t (htab_has_hash(htab) ? 8 : 0);\n+\telse\n+\t\thtab-\u003ekey_offset = offsetof(struct htab_elem, data) +\n+\t\t\t\t (htab_has_hash(htab) ? 8 : 0);\n+\n+\thtab-\u003eelem_size = htab-\u003ekey_offset + round_up(htab-\u003emap.key_size, 8);\n \tif (percpu)\n \t\thtab-\u003eelem_size += sizeof(void );\n \telse\n@@ -692,15 +770,29 @@ static inline struct hlist_nulls_head select_bucket(struct bpf_htab htab, u32\n }\n \n / this lookup function can only be called with bucket lock taken /\n-static struct htab_elem lookup_elem_raw(struct hlist_nulls_head head, u32 hash,\n+static struct htab_elem lookup_elem_raw(struct bpf_htab htab,\n+\t\t\t\t\t struct hlist_nulls_head head, u32 hash,\n \t\t\t\t\t void key, u32 key_size)\n {\n \tstruct hlist_nulls_node n;\n \tstruct htab_elem l;\n \n-\thlist_nulls_for_each_entry_rcu(l, n, head, hash_node)\n-\t\tif (l-\u003ehash == hash \u0026\u0026 !memcmp(\u0026l-\u003ekey, key, key_size))\n-\t\t\treturn l;\n+\tif (htab_has_hash(htab)) {\n+\t\thlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)\n+\t\t\tif (htab_elem_hash(htab, l) == hash \u0026\u0026\n+\t\t\t !memcmp(htab_elem_key(htab, l), key, key_size))\n+\t\t\t\treturn l;\n+\t} else {\n+\t\t/ When hash is omitted, key comparisons must be atomic. Zero extend\n+\t\t * the caller's key to the word size to support an atomic compare.\n+\t\t /\n+\t\tunsigned long k = 0;\n+\n+\t\tmemcpy(\u0026k, key, key_size);\n+\t\thlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)\n+\t\t\tif (READ_ONCE((unsigned long )htab_elem_key(htab, l)) == k)\n+\t\t\t\treturn l;\n+\t}\n \n \treturn NULL;\n }\n@@ -709,7 +801,8 @@ static struct htab_elem lookup_elem_raw(struct hlist_nulls_head head, u32 hash\n the unlikely event when elements moved from one bucket into another\n * while link list is being walked\n /\n-static struct htab_elem lookup_nulls_elem_raw(struct hlist_nulls_head head,\n+static struct htab_elem lookup_nulls_elem_raw(struct bpf_htab htab,\n+\t\t\t\t\t struct hlist_nulls_head head,\n \t\t\t\t\t u32 hash, void key,\n \t\t\t\t\t u32 key_size, u32 n_buckets)\n {\n@@ -717,9 +810,20 @@ static struct htab_elem lookup_nulls_elem_raw(struct hlist_nulls_head head,\n \tstruct htab_elem l;\n \n again:\n-\thlist_nulls_for_each_entry_rcu(l, n, head, hash_node)\n-\t\tif (l-\u003ehash == hash \u0026\u0026 !memcmp(\u0026l-\u003ekey, key, key_size))\n-\t\t\treturn l;\n+\tif (htab_has_hash(htab)) {\n+\t\thlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)\n+\t\t\tif (htab_elem_hash(htab, l) == hash \u0026\u0026\n+\t\t\t !memcmp(htab_elem_key(htab, l), key, key_size))\n+\t\t\t\treturn l;\n+\t} else {\n+\t\t/* See lookup_elem_raw() comment above. /\n+\t\tunsigned long k = 0;\n+\n+\t\tmemcpy(\u0026k, key, key_size);\n+\t\thlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)\n+\t\t\tif (READ_ONCE((unsigned long )htab_elem_key(htab, l)) == k)\n+\t\t\t\treturn l;\n+\t}\n \n \tif (unlikely(get_nulls_value(n) != (hash \u0026 (n_buckets - 1))))\n \t\tgoto again;\n@@ -747,17 +851,18 @@ static void __htab_map_lookup_elem(struct bpf_map map, void key)\n \n \thead = select_bucket(htab, hash);\n \n-\tl = lookup_nulls_elem_raw(head, hash, key, key_size, htab-\u003en_buckets);\n+\tl = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab-\u003en_buckets);\n \n \treturn l;\n }\n \n static void htab_map_lookup_elem(struct bpf_map map, void key)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l = __htab_map_lookup_elem(map, key);\n \n \tif (l)\n-\t\treturn htab_elem_value(l, map-\u003ekey_size);\n+\t\treturn htab_elem_value(htab, l);\n \n \treturn NULL;\n }\n@@ -775,6 +880,7 @@ static void htab_map_lookup_elem(struct bpf_map map, void key)\n /\n static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn insn = insn_buf;\n \tconst int ret = BPF_REG_0;\n \n@@ -783,7 +889,7 @@ static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)\n \tinsn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \tinsn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);\n \tinsn++ = BPF_ALU64_IMM(BPF_ADD, ret,\n-\t\t\t\toffsetof(struct htab_elem, key) +\n+\t\t\t\thtab-\u003ekey_offset +\n \t\t\t\tround_up(map-\u003ekey_size, 8));\n \treturn insn - insn_buf;\n }\n@@ -791,12 +897,13 @@ static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)\n static __always_inline void __htab_lru_map_lookup_elem(struct bpf_map map,\n \t\t\t\t\t\t\tvoid key, const bool mark)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l = __htab_map_lookup_elem(map, key);\n \n \tif (l) {\n \t\tif (mark)\n-\t\t\tbpf_lru_node_set_ref(\u0026l-\u003elru_node);\n-\t\treturn htab_elem_value(l, map-\u003ekey_size);\n+\t\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l));\n+\t\treturn htab_elem_value(htab, l);\n \t}\n \n \treturn NULL;\n@@ -815,6 +922,7 @@ static void htab_lru_map_lookup_elem_sys(struct bpf_map map, void key)\n static int htab_lru_map_gen_lookup(struct bpf_map map,\n \t\t\t\t struct bpf_insn insn_buf)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn insn = insn_buf;\n \tconst int ret = BPF_REG_0;\n \tconst int ref_reg = BPF_REG_1;\n@@ -824,15 +932,15 @@ static int htab_lru_map_gen_lookup(struct bpf_map map,\n \tinsn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \tinsn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4);\n \tinsn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret,\n-\t\t\t offsetof(struct htab_elem, lru_node) +\n+\t\t\t offsetof(struct htab_elem_lru, lru_node) +\n \t\t\t offsetof(struct bpf_lru_node, ref));\n \tinsn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1);\n \tinsn++ = BPF_ST_MEM(BPF_B, ret,\n-\t\t\t offsetof(struct htab_elem, lru_node) +\n+\t\t\t offsetof(struct htab_elem_lru, lru_node) +\n \t\t\t offsetof(struct bpf_lru_node, ref),\n \t\t\t 1);\n \tinsn++ = BPF_ALU64_IMM(BPF_ADD, ret,\n-\t\t\t\toffsetof(struct htab_elem, key) +\n+\t\t\t\thtab-\u003ekey_offset +\n \t\t\t\tround_up(map-\u003ekey_size, 8));\n \treturn insn - insn_buf;\n }\n@@ -844,13 +952,13 @@ static void check_and_cancel_fields(struct bpf_htab htab,\n \t\treturn;\n \n \tif (htab_is_percpu(htab)) {\n-\t\tvoid __percpu pptr = htab_elem_get_ptr(elem, htab-\u003emap.key_size);\n+\t\tvoid __percpu pptr = htab_elem_get_ptr(htab, elem);\n \t\tint cpu;\n \n \t\tfor_each_possible_cpu(cpu)\n \t\t\tbpf_obj_cancel_fields(\u0026htab-\u003emap, per_cpu_ptr(pptr, cpu));\n \t} else {\n-\t\tvoid map_value = htab_elem_value(elem, htab-\u003emap.key_size);\n+\t\tvoid map_value = htab_elem_value(htab, elem);\n \n \t\tbpf_obj_cancel_fields(\u0026htab-\u003emap, map_value);\n \t}\n@@ -869,17 +977,17 @@ static bool htab_lru_map_delete_node(void arg, struct bpf_lru_node node)\n \tstruct bucket b;\n \tint ret;\n \n-\ttgt_l = container_of(node, struct htab_elem, lru_node);\n-\tb = __select_bucket(htab, tgt_l-\u003ehash);\n+\ttgt_l = (struct htab_elem )container_of(node, struct htab_elem_lru, lru_node);\n+\tb = __select_bucket(htab, htab_elem_hash(htab, tgt_l));\n \thead = \u0026b-\u003ehead;\n \n \tret = htab_lock_bucket(b, \u0026flags);\n \tif (ret)\n \t\treturn false;\n \n-\thlist_nulls_for_each_entry_rcu(l, n, head, hash_node)\n+\thlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)\n \t\tif (l == tgt_l) {\n-\t\t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n+\t\t\thlist_nulls_del_rcu(\u0026l-\u003enode.hash_node);\n \t\t\tbpf_map_dec_elem_count(\u0026htab-\u003emap);\n \t\t\tbreak;\n \t\t}\n@@ -912,18 +1020,19 @@ static int htab_map_get_next_key(struct bpf_map map, void key, void next_key)\n \thead = select_bucket(htab, hash);\n \n \t/ lookup the key /\n-\tl = lookup_nulls_elem_raw(head, hash, key, key_size, htab-\u003en_buckets);\n+\tl = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab-\u003en_buckets);\n \n \tif (!l)\n \t\tgoto find_first_elem;\n \n \t/ key was found, get next key in the same bucket /\n-\tnext_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(\u0026l-\u003ehash_node)),\n-\t\t\t\t struct htab_elem, hash_node);\n+\tnext_l = hlist_nulls_entry_safe(\n+\t\t\trcu_dereference_raw(hlist_nulls_next_rcu(\u0026l-\u003enode.hash_node)),\n+\t\t\tstruct htab_elem, node.hash_node);\n \n \tif (next_l) {\n \t\t/ if next elem in this hash list is non-zero, just return it /\n-\t\tmemcpy(next_key, next_l-\u003ekey, key_size);\n+\t\tmemcpy(next_key, htab_elem_key(htab, next_l), key_size);\n \t\treturn 0;\n \t}\n \n@@ -938,10 +1047,10 @@ static int htab_map_get_next_key(struct bpf_map map, void key, void next_key)\n \n \t\t/* pick first element in the bucket /\n \t\tnext_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_first_rcu(head)),\n-\t\t\t\t\t struct htab_elem, hash_node);\n+\t\t\t\t\t struct htab_elem, node.hash_node);\n \t\tif (next_l) {\n \t\t\t/ if it's not empty, just return it /\n-\t\t\tmemcpy(next_key, next_l-\u003ekey, key_size);\n+\t\t\tmemcpy(next_key, htab_elem_key(htab, next_l), key_size);\n \t\t\treturn 0;\n \t\t}\n \t}\n@@ -955,7 +1064,7 @@ static void htab_elem_free(struct bpf_htab htab, struct htab_elem l)\n \tcheck_and_cancel_fields(htab, l);\n \n \tif (htab-\u003emap.map_type == BPF_MAP_TYPE_PERCPU_HASH)\n-\t\tbpf_mem_cache_free(\u0026htab-\u003epcpu_ma, l-\u003eptr_to_pptr);\n+\t\tbpf_mem_cache_free(\u0026htab-\u003epcpu_ma, htab_elem_get_ptr_to_pptr(l));\n \tbpf_mem_cache_free(\u0026htab-\u003ema, l);\n }\n \n@@ -965,7 +1074,7 @@ static void htab_put_fd_value(struct bpf_htab htab, struct htab_elem l)\n \tvoid ptr;\n \n \tif (map-\u003eops-\u003emap_fd_put_ptr) {\n-\t\tptr = fd_htab_map_get_ptr(map, l);\n+\t\tptr = fd_htab_map_get_ptr(htab, l);\n \t\tmap-\u003eops-\u003emap_fd_put_ptr(map, ptr, true);\n \t}\n }\n@@ -1006,7 +1115,7 @@ static void free_htab_elem(struct bpf_htab htab, struct htab_elem l)\n \tif (htab_is_prealloc(htab)) {\n \t\tbpf_map_dec_elem_count(\u0026htab-\u003emap);\n \t\tcheck_and_cancel_fields(htab, l);\n-\t\tpcpu_freelist_push(\u0026htab-\u003efreelist, \u0026l-\u003efnode);\n+\t\tpcpu_freelist_push(\u0026htab-\u003efreelist, \u0026l-\u003enode.fnode);\n \t} else {\n \t\tdec_elem_count(htab);\n \t\thtab_elem_free(htab, l);\n@@ -1097,7 +1206,7 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void key,\n \t\t\tl = __pcpu_freelist_pop(\u0026htab-\u003efreelist);\n \t\t\tif (!l)\n \t\t\t\treturn ERR_PTR(-E2BIG);\n-\t\t\tl_new = container_of(l, struct htab_elem, fnode);\n+\t\t\tl_new = container_of(l, struct htab_elem, node.fnode);\n \t\t\tbpf_map_inc_elem_count(\u0026htab-\u003emap);\n \t\t}\n \t} else {\n@@ -1117,10 +1226,20 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void key,\n \t\t}\n \t}\n \n-\tmemcpy(l_new-\u003ekey, key, key_size);\n+\tif (htab_has_hash(htab)) {\n+\t\tmemcpy(htab_elem_key(htab, l_new), key, key_size);\n+\t} else {\n+\t\t/* Zero-extend key into k for an atomic write to support\n+\t\t * lockless RCU readers.\n+\t\t /\n+\t\tunsigned long k = 0;\n+\n+\t\tmemcpy(\u0026k, key, key_size);\n+\t\tWRITE_ONCE((unsigned long )htab_elem_key(htab, l_new), k);\n+\t}\n \tif (percpu) {\n \t\tif (prealloc) {\n-\t\t\tpptr = htab_elem_get_ptr(l_new, key_size);\n+\t\t\tpptr = htab_elem_get_ptr(htab, l_new);\n \t\t} else {\n \t\t\t/ alloc_percpu zero-fills /\n \t\t\tvoid ptr = bpf_mem_cache_alloc(\u0026htab-\u003epcpu_ma);\n@@ -1130,26 +1249,26 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void key,\n \t\t\t\tl_new = ERR_PTR(-ENOMEM);\n \t\t\t\tgoto dec_count;\n \t\t\t}\n-\t\t\tl_new-\u003eptr_to_pptr = ptr;\n+\t\t\thtab_elem_set_ptr_to_pptr(l_new, ptr);\n \t\t\tpptr = (void __percpu *)ptr;\n \t\t}\n \n \t\tpcpu_init_value(htab, pptr, value, onallcpus, map_flags);\n \n \t\tif (!prealloc)\n-\t\t\thtab_elem_set_ptr(l_new, key_size, pptr);\n+\t\t\thtab_elem_set_ptr(htab, l_new, pptr);\n \t} else if (fd_htab_map_needs_adjust(htab)) {\n \t\tsize = round_up(size, 8);\n-\t\tmemcpy(htab_elem_value(l_new, key_size), value, size);\n+\t\tmemcpy(htab_elem_value(htab, l_new), value, size);\n \t} else if (map_flags \u0026 BPF_F_LOCK) {\n \t\tcopy_map_value_locked(\u0026htab-\u003emap,\n-\t\t\t\t htab_elem_value(l_new, key_size),\n+\t\t\t\t htab_elem_value(htab, l_new),\n \t\t\t\t value, false);\n \t} else {\n-\t\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(l_new, key_size), value);\n+\t\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(htab, l_new), value);\n \t}\n \n-\tl_new-\u003ehash = hash;\n+\thtab_elem_set_hash(htab, l_new, hash);\n \treturn l_new;\n dec_count:\n \tdec_elem_count(htab);\n@@ -1199,7 +1318,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value,\n \t\tif (unlikely(!btf_record_has_field(map-\u003erecord, BPF_SPIN_LOCK)))\n \t\t\treturn -EINVAL;\n \t\t/* find an element without taking the bucket lock /\n-\t\tl_old = lookup_nulls_elem_raw(head, hash, key, key_size,\n+\t\tl_old = lookup_nulls_elem_raw(htab, head, hash, key, key_size,\n \t\t\t\t\t htab-\u003en_buckets);\n \t\tret = check_flags(htab, l_old, map_flags);\n \t\tif (ret)\n@@ -1207,7 +1326,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value,\n \t\tif (l_old) {\n \t\t\t/* grab the element lock and update value in place /\n \t\t\tcopy_map_value_locked(map,\n-\t\t\t\t\t htab_elem_value(l_old, key_size),\n+\t\t\t\t\t htab_elem_value(htab, l_old),\n \t\t\t\t\t value, false);\n \t\t\treturn 0;\n \t\t}\n@@ -1221,7 +1340,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value,\n \tif (ret)\n \t\treturn ret;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n@@ -1235,7 +1354,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value,\n \t\t and update element in place\n \t\t /\n \t\tcopy_map_value_locked(map,\n-\t\t\t\t htab_elem_value(l_old, key_size),\n+\t\t\t\t htab_elem_value(htab, l_old),\n \t\t\t\t value, false);\n \t\tret = 0;\n \t\tgoto err;\n@@ -1252,9 +1371,9 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value,\n \t/* add new element to the head of the list, so that\n \t * concurrent search will find it before old elem\n \t /\n-\thlist_nulls_add_head_rcu(\u0026l_new-\u003ehash_node, head);\n+\thlist_nulls_add_head_rcu(\u0026l_new-\u003enode.hash_node, head);\n \tif (l_old) {\n-\t\thlist_nulls_del_rcu(\u0026l_old-\u003ehash_node);\n+\t\thlist_nulls_del_rcu(\u0026l_old-\u003enode.hash_node);\n \n \t\t/ l_old has already been stashed in htab-\u003eextra_elems, cancel\n \t\t * its reusable special fields before it is available for reuse.\n@@ -1275,7 +1394,7 @@ static void htab_lru_push_free(struct bpf_htab htab, struct htab_elem elem)\n {\n \tcheck_and_cancel_fields(htab, elem);\n \tbpf_map_dec_elem_count(\u0026htab-\u003emap);\n-\tbpf_lru_push_free(\u0026htab-\u003elru, \u0026elem-\u003elru_node);\n+\tbpf_lru_push_free(\u0026htab-\u003elru, htab_elem_lru_node(elem));\n }\n \n static long htab_lru_map_update_elem(struct bpf_map map, void key, void value,\n@@ -1310,13 +1429,13 @@ static long htab_lru_map_update_elem(struct bpf_map map, void key, void value\n \tl_new = prealloc_lru_pop(htab, key, hash);\n \tif (!l_new)\n \t\treturn -ENOMEM;\n-\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(l_new, map-\u003ekey_size), value);\n+\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(htab, l_new), value);\n \n \tret = htab_lock_bucket(b, \u0026flags);\n \tif (ret)\n \t\tgoto err_lock_bucket;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n@@ -1325,10 +1444,10 @@ static long htab_lru_map_update_elem(struct bpf_map map, void key, void value\n \t/ add new element to the head of the list, so that\n \t * concurrent search will find it before old elem\n \t /\n-\thlist_nulls_add_head_rcu(\u0026l_new-\u003ehash_node, head);\n+\thlist_nulls_add_head_rcu(\u0026l_new-\u003enode.hash_node, head);\n \tif (l_old) {\n-\t\tbpf_lru_node_set_ref(\u0026l_new-\u003elru_node);\n-\t\thlist_nulls_del_rcu(\u0026l_old-\u003ehash_node);\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l_new));\n+\t\thlist_nulls_del_rcu(\u0026l_old-\u003enode.hash_node);\n \t}\n \tret = 0;\n \n@@ -1383,7 +1502,7 @@ static long htab_map_update_elem_in_place(struct bpf_map map, void key,\n \tif (ret)\n \t\treturn ret;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n@@ -1392,10 +1511,10 @@ static long htab_map_update_elem_in_place(struct bpf_map map, void key,\n \tif (l_old) {\n \t\t/ Update value in-place /\n \t\tif (percpu) {\n-\t\t\tpcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),\n+\t\t\tpcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old),\n \t\t\t\t\tvalue, onallcpus, map_flags);\n \t\t} else {\n-\t\t\tvoid inner_map_pptr = htab_elem_value(l_old, key_size);\n+\t\t\tvoid inner_map_pptr = htab_elem_value(htab, l_old);\n \n \t\t\told_map_ptr = inner_map_pptr;\n \t\t\tWRITE_ONCE(inner_map_pptr, (void *)value);\n@@ -1407,7 +1526,7 @@ static long htab_map_update_elem_in_place(struct bpf_map map, void key,\n \t\t\tret = PTR_ERR(l_new);\n \t\t\tgoto err;\n \t\t}\n-\t\thlist_nulls_add_head_rcu(\u0026l_new-\u003ehash_node, head);\n+\t\thlist_nulls_add_head_rcu(\u0026l_new-\u003enode.hash_node, head);\n \t}\n err:\n \thtab_unlock_bucket(b, flags);\n@@ -1456,22 +1575,22 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map map, void key,\n \tif (ret)\n \t\tgoto err_lock_bucket;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n \t\tgoto err;\n \n \tif (l_old) {\n-\t\tbpf_lru_node_set_ref(\u0026l_old-\u003elru_node);\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l_old));\n \n \t\t/ per-cpu hash map can update value in-place /\n-\t\tpcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),\n+\t\tpcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old),\n \t\t\t\tvalue, onallcpus, map_flags);\n \t} else {\n-\t\tpcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size),\n+\t\tpcpu_init_value(htab, htab_elem_get_ptr(htab, l_new),\n \t\t\t\tvalue, onallcpus, map_flags);\n-\t\thlist_nulls_add_head_rcu(\u0026l_new-\u003ehash_node, head);\n+\t\thlist_nulls_add_head_rcu(\u0026l_new-\u003enode.hash_node, head);\n \t\tl_new = NULL;\n \t}\n \tret = 0;\n@@ -1480,7 +1599,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map map, void key,\n err_lock_bucket:\n \tif (l_new) {\n \t\tbpf_map_dec_elem_count(\u0026htab-\u003emap);\n-\t\tbpf_lru_push_free(\u0026htab-\u003elru, \u0026l_new-\u003elru_node);\n+\t\tbpf_lru_push_free(\u0026htab-\u003elru, htab_elem_lru_node(l_new));\n \t}\n \treturn ret;\n }\n@@ -1521,9 +1640,9 @@ static long htab_map_delete_elem(struct bpf_map map, void key)\n \tif (ret)\n \t\treturn ret;\n \n-\tl = lookup_elem_raw(head, hash, key, key_size);\n+\tl = lookup_elem_raw(htab, head, hash, key, key_size);\n \tif (l)\n-\t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n+\t\thlist_nulls_del_rcu(\u0026l-\u003enode.hash_node);\n \telse\n \t\tret = -ENOENT;\n \n@@ -1556,10 +1675,10 @@ static long htab_lru_map_delete_elem(struct bpf_map map, void key)\n \tif (ret)\n \t\treturn ret;\n \n-\tl = lookup_elem_raw(head, hash, key, key_size);\n+\tl = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tif (l)\n-\t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n+\t\thlist_nulls_del_rcu(\u0026l-\u003enode.hash_node);\n \telse\n \t\tret = -ENOENT;\n \n@@ -1581,8 +1700,8 @@ static void delete_all_elements(struct bpf_htab htab)\n \t\tstruct hlist_nulls_node n;\n \t\tstruct htab_elem l;\n \n-\t\thlist_nulls_for_each_entry_safe(l, n, head, hash_node) {\n-\t\t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n+\t\thlist_nulls_for_each_entry_safe(l, n, head, node.hash_node) {\n+\t\t\thlist_nulls_del_rcu(\u0026l-\u003enode.hash_node);\n \t\t\thtab_elem_free(htab, l);\n \t\t}\n \t\tcond_resched();\n@@ -1599,10 +1718,10 @@ static void htab_free_malloced_internal_structs(struct bpf_htab htab)\n \t\tstruct hlist_nulls_node n;\n \t\tstruct htab_elem l;\n \n-\t\thlist_nulls_for_each_entry(l, n, head, hash_node) {\n+\t\thlist_nulls_for_each_entry(l, n, head, node.hash_node) {\n \t\t\t/ We only free internal structs on uref dropping to zero /\n \t\t\tbpf_map_free_internal_structs(\u0026htab-\u003emap,\n-\t\t\t\t\t\t htab_elem_value(l, htab-\u003emap.key_size));\n+\t\t\t\t\t\t htab_elem_value(htab, l));\n \t\t}\n \t\tcond_resched_rcu();\n \t}\n@@ -1697,7 +1816,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map map, void key,\n \tif (ret)\n \t\treturn ret;\n \n-\tl = lookup_elem_raw(head, hash, key, key_size);\n+\tl = lookup_elem_raw(htab, head, hash, key, key_size);\n \tif (!l) {\n \t\tret = -ENOENT;\n \t\tgoto out_unlock;\n@@ -1708,14 +1827,14 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map map, void key,\n \t\tvoid __percpu pptr;\n \t\tint off = 0, cpu;\n \n-\t\tpptr = htab_elem_get_ptr(l, key_size);\n+\t\tpptr = htab_elem_get_ptr(htab, l);\n \t\tfor_each_possible_cpu(cpu) {\n \t\t\tcopy_map_value_long(\u0026htab-\u003emap, value + off, per_cpu_ptr(pptr, cpu));\n \t\t\tcheck_and_init_map_value(\u0026htab-\u003emap, value + off);\n \t\t\toff += roundup_value_size;\n \t\t}\n \t} else {\n-\t\tvoid src = htab_elem_value(l, map-\u003ekey_size);\n+\t\tvoid src = htab_elem_value(htab, l);\n \n \t\tif (flags \u0026 BPF_F_LOCK)\n \t\t\tcopy_map_value_locked(map, value, src, true);\n@@ -1724,7 +1843,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map map, void key,\n \t\t/* Zeroing special fields in the temp buffer /\n \t\tcheck_and_init_map_value(map, value);\n \t}\n-\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n+\thlist_nulls_del_rcu(\u0026l-\u003enode.hash_node);\n \n out_unlock:\n \thtab_unlock_bucket(b, bflags);\n@@ -1860,7 +1979,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map,\n \t}\n \n \tbucket_cnt = 0;\n-\thlist_nulls_for_each_entry_rcu(l, n, head, hash_node)\n+\thlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)\n \t\tbucket_cnt++;\n \n \tif (bucket_cnt \u0026\u0026 !locked) {\n@@ -1897,14 +2016,14 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map,\n \tif (!locked)\n \t\tgoto next_batch;\n \n-\thlist_nulls_for_each_entry_safe(l, n, head, hash_node) {\n-\t\tmemcpy(dst_key, l-\u003ekey, key_size);\n+\thlist_nulls_for_each_entry_safe(l, n, head, node.hash_node) {\n+\t\tmemcpy(dst_key, htab_elem_key(htab, l), key_size);\n \n \t\tif (is_percpu) {\n \t\t\tint off = 0, cpu;\n \t\t\tvoid __percpu pptr;\n \n-\t\t\tpptr = htab_elem_get_ptr(l, map-\u003ekey_size);\n+\t\t\tpptr = htab_elem_get_ptr(htab, l);\n \t\t\tif (elem_map_flags \u0026 BPF_F_CPU) {\n \t\t\t\tcpu = elem_map_flags \u003e\u003e 32;\n \t\t\t\tcopy_map_value(\u0026htab-\u003emap, dst_val, per_cpu_ptr(pptr, cpu));\n@@ -1918,7 +2037,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map,\n \t\t\t\t}\n \t\t\t}\n \t\t} else {\n-\t\t\tvalue = htab_elem_value(l, key_size);\n+\t\t\tvalue = htab_elem_value(htab, l);\n \t\t\tif (is_fd_htab(htab)) {\n \t\t\t\tstruct bpf_map inner_map = value;\n \n@@ -1936,7 +2055,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map,\n \t\t\tcheck_and_init_map_value(map, dst_val);\n \t\t}\n \t\tif (do_delete) {\n-\t\t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n+\t\t\thlist_nulls_del_rcu(\u0026l-\u003enode.hash_node);\n \n \t\t\t/* bpf_lru_push_free() will acquire lru_lock, which\n \t\t\t * may cause deadlock. See comments in function\n@@ -1948,7 +2067,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map,\n \t\t\t lock being held and it violates the lock rule, so\n \t\t\t * invoke free_htab_elem() after unlock as well.\n \t\t\t /\n-\t\t\tl-\u003ebatch_flink = node_to_free;\n+\t\t\tl-\u003enode.batch_flink = node_to_free;\n \t\t\tnode_to_free = l;\n \t\t}\n \t\tdst_key += key_size;\n@@ -1960,7 +2079,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map,\n \n \twhile (node_to_free) {\n \t\tl = node_to_free;\n-\t\tnode_to_free = node_to_free-\u003ebatch_flink;\n+\t\tnode_to_free = node_to_free-\u003enode.batch_flink;\n \t\tif (is_lru_map)\n \t\t\thtab_lru_push_free(htab, l);\n \t\telse\n@@ -2108,8 +2227,8 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info info,\n \t\t/ no update/deletion on this bucket, prev_elem should be still valid\n \t\t * and we won't skip elements.\n \t\t /\n-\t\tn = rcu_dereference_raw(hlist_nulls_next_rcu(\u0026prev_elem-\u003ehash_node));\n-\t\telem = hlist_nulls_entry_safe(n, struct htab_elem, hash_node);\n+\t\tn = rcu_dereference_raw(hlist_nulls_next_rcu(\u0026prev_elem-\u003enode.hash_node));\n+\t\telem = hlist_nulls_entry_safe(n, struct htab_elem, node.hash_node);\n \t\tif (elem)\n \t\t\treturn elem;\n \n@@ -2125,7 +2244,7 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info info,\n \n \t\tcount = 0;\n \t\thead = \u0026b-\u003ehead;\n-\t\thlist_nulls_for_each_entry_rcu(elem, n, head, hash_node) {\n+\t\thlist_nulls_for_each_entry_rcu(elem, n, head, node.hash_node) {\n \t\t\tif (count \u003e= skip_elems) {\n \t\t\t\tinfo-\u003ebucket_id = i;\n \t\t\t\tinfo-\u003eskip_elems = count;\n@@ -2183,12 +2302,12 @@ static int __bpf_hash_map_seq_show(struct seq_file seq, struct htab_elem elem)\n \t\tctx.meta = \u0026meta;\n \t\tctx.map = info-\u003emap;\n \t\tif (elem) {\n-\t\t\tctx.key = elem-\u003ekey;\n+\t\t\tctx.key = htab_elem_key(info-\u003ehtab, elem);\n \t\t\tif (!info-\u003epercpu_value_buf) {\n-\t\t\t\tctx.value = htab_elem_value(elem, map-\u003ekey_size);\n+\t\t\t\tctx.value = htab_elem_value(info-\u003ehtab, elem);\n \t\t\t} else {\n \t\t\t\troundup_value_size = round_up(map-\u003evalue_size, 8);\n-\t\t\t\tpptr = htab_elem_get_ptr(elem, map-\u003ekey_size);\n+\t\t\t\tpptr = htab_elem_get_ptr(info-\u003ehtab, elem);\n \t\t\t\tfor_each_possible_cpu(cpu) {\n \t\t\t\t\tcopy_map_value_long(map, info-\u003epercpu_value_buf + off,\n \t\t\t\t\t\t\t per_cpu_ptr(pptr, cpu));\n@@ -2292,14 +2411,14 @@ static long bpf_for_each_hash_elem(struct bpf_map map, bpf_callback_t callback_\n \t\tb = \u0026htab-\u003ebuckets[i];\n \t\trcu_read_lock();\n \t\thead = \u0026b-\u003ehead;\n-\t\thlist_nulls_for_each_entry_safe(elem, n, head, hash_node) {\n-\t\t\tkey = elem-\u003ekey;\n+\t\thlist_nulls_for_each_entry_safe(elem, n, head, node.hash_node) {\n+\t\t\tkey = htab_elem_key(htab, elem);\n \t\t\tif (is_percpu) {\n \t\t\t\t/ current cpu value for percpu map /\n-\t\t\t\tpptr = htab_elem_get_ptr(elem, map-\u003ekey_size);\n+\t\t\t\tpptr = htab_elem_get_ptr(htab, elem);\n \t\t\t\tval = this_cpu_ptr(pptr);\n \t\t\t} else {\n-\t\t\t\tval = htab_elem_value(elem, map-\u003ekey_size);\n+\t\t\t\tval = htab_elem_value(htab, elem);\n \t\t\t}\n \t\t\tnum_elems++;\n \t\t\tret = callback_fn((u64)(long)map, (u64)(long)key,\n@@ -2403,10 +2522,11 @@ const struct bpf_map_ops htab_lru_map_ops = {\n / Called from eBPF program /\n static void htab_percpu_map_lookup_elem(struct bpf_map map, void key)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l = __htab_map_lookup_elem(map, key);\n \n \tif (l)\n-\t\treturn this_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size));\n+\t\treturn this_cpu_ptr(htab_elem_get_ptr(htab, l));\n \telse\n \t\treturn NULL;\n }\n@@ -2414,6 +2534,7 @@ static void htab_percpu_map_lookup_elem(struct bpf_map map, void key)\n / inline bpf_map_lookup_elem() call for per-CPU hashmap /\n static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn insn = insn_buf;\n \n \tif (!bpf_jit_supports_percpu_insn())\n@@ -2424,7 +2545,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn\n \tinsn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \tinsn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3);\n \tinsn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0,\n-\t\t\t\toffsetof(struct htab_elem, key) + roundup(map-\u003ekey_size, 8));\n+\t\t\t\thtab-\u003ekey_offset + roundup(map-\u003ekey_size, 8));\n \tinsn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);\n \tinsn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);\n \n@@ -2433,6 +2554,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn\n \n static void htab_percpu_map_lookup_percpu_elem(struct bpf_map map, void key, u32 cpu)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l;\n \n \tif (cpu \u003e= nr_cpu_ids)\n@@ -2440,18 +2562,19 @@ static void htab_percpu_map_lookup_percpu_elem(struct bpf_map map, void key,\n \n \tl = __htab_map_lookup_elem(map, key);\n \tif (l)\n-\t\treturn per_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size), cpu);\n+\t\treturn per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu);\n \telse\n \t\treturn NULL;\n }\n \n static void htab_lru_percpu_map_lookup_elem(struct bpf_map map, void key)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l = __htab_map_lookup_elem(map, key);\n \n \tif (l) {\n-\t\tbpf_lru_node_set_ref(\u0026l-\u003elru_node);\n-\t\treturn this_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size));\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l));\n+\t\treturn this_cpu_ptr(htab_elem_get_ptr(htab, l));\n \t}\n \n \treturn NULL;\n@@ -2459,6 +2582,7 @@ static void htab_lru_percpu_map_lookup_elem(struct bpf_map map, void key)\n \n static void htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map map, void key, u32 cpu)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l;\n \n \tif (cpu \u003e= nr_cpu_ids)\n@@ -2466,8 +2590,8 @@ static void htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map map, void k\n \n \tl = __htab_map_lookup_elem(map, key);\n \tif (l) {\n-\t\tbpf_lru_node_set_ref(\u0026l-\u003elru_node);\n-\t\treturn per_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size), cpu);\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l));\n+\t\treturn per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu);\n \t}\n \n \treturn NULL;\n@@ -2475,6 +2599,7 @@ static void htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map map, void k\n \n int bpf_percpu_hash_copy(struct bpf_map map, void key, void value, u64 map_flags)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l;\n \tvoid __percpu pptr;\n \tint ret = -ENOENT;\n@@ -2494,7 +2619,7 @@ int bpf_percpu_hash_copy(struct bpf_map map, void key, void value, u64 map_fl\n \t/* We do not mark LRU map element here in order to not mess up\n \t * eviction heuristics when user space does a map walk.\n \t /\n-\tpptr = htab_elem_get_ptr(l, map-\u003ekey_size);\n+\tpptr = htab_elem_get_ptr(htab, l);\n \tif (map_flags \u0026 BPF_F_CPU) {\n \t\tcpu = map_flags \u003e\u003e 32;\n \t\tcopy_map_value(map, value, per_cpu_ptr(pptr, cpu));\n@@ -2532,6 +2657,7 @@ int bpf_percpu_hash_update(struct bpf_map map, void key, void value,\n static void htab_percpu_map_seq_show_elem(struct bpf_map map, void key,\n \t\t\t\t\t struct seq_file m)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem l;\n \tvoid __percpu pptr;\n \tint cpu;\n@@ -2546,7 +2672,7 @@ static void htab_percpu_map_seq_show_elem(struct bpf_map map, void key,\n \n \tbtf_type_seq_show(map-\u003ebtf, map-\u003ebtf_key_type_id, key, m);\n \tseq_puts(m, \": {\\n\");\n-\tpptr = htab_elem_get_ptr(l, map-\u003ekey_size);\n+\tpptr = htab_elem_get_ptr(htab, l);\n \tfor_each_possible_cpu(cpu) {\n \t\tseq_printf(m, \"\\tcpu%d: \", cpu);\n \t\tbtf_type_seq_show(map-\u003ebtf, map-\u003ebtf_value_type_id,\n@@ -2619,8 +2745,8 @@ static void fd_htab_map_free(struct bpf_map map)\n \tfor (i = 0; i \u003c htab-\u003en_buckets; i++) {\n \t\thead = select_bucket(htab, i);\n \n-\t\thlist_nulls_for_each_entry_safe(l, n, head, hash_node) {\n-\t\t\tvoid ptr = fd_htab_map_get_ptr(map, l);\n+\t\thlist_nulls_for_each_entry_safe(l, n, head, node.hash_node) {\n+\t\t\tvoid ptr = fd_htab_map_get_ptr(htab, l);\n \n \t\t\tmap-\u003eops-\u003emap_fd_put_ptr(map, ptr, false);\n \t\t}\n@@ -2705,6 +2831,7 @@ static void htab_of_map_lookup_elem(struct bpf_map map, void key)\n static int htab_of_map_gen_lookup(struct bpf_map map,\n \t\t\t\t struct bpf_insn insn_buf)\n {\n+\tstruct bpf_htab htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn insn = insn_buf;\n \tconst int ret = BPF_REG_0;\n \n@@ -2713,7 +2840,7 @@ static int htab_of_map_gen_lookup(struct bpf_map map,\n \tinsn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \tinsn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2);\n \tinsn++ = BPF_ALU64_IMM(BPF_ADD, ret,\n-\t\t\t\toffsetof(struct htab_elem, key) +\n+\t\t\t\thtab-\u003ekey_offset +\n \t\t\t\tround_up(map-\u003ekey_size, 8));\n \tinsn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);\n \n@@ -3112,7 +3239,7 @@ static int rhtab_map_check_btf(struct bpf_map map, const struct btf btf,\n {\n \tstruct bpf_rhtab rhtab = container_of(map, struct bpf_rhtab, map);\n \n-\treturn bpf_ma_set_dtor(map, \u0026rhtab-\u003ema, rhtab_mem_dtor);\n+\treturn bpf_ma_set_dtor(map, \u0026rhtab-\u003ema, rhtab_mem_dtor, offsetof(struct rhtab_elem, data));\n }\n \n static void rhtab_map_free_internal_structs(struct bpf_map map)\n@@ -3533,3 +3660,16 @@ const struct bpf_map_ops rhtab_map_ops = {\n \t.map_btf_id = \u0026rhtab_map_btf_ids[0],\n \t.iter_seq_info = \u0026rhash_iter_seq_info,\n };\n+\n+size_t bpf_htab_map_meta_size(void)\n+{\n+\treturn sizeof(struct bpf_htab);\n+}\n+\n+void bpf_htab_map_meta_init(struct bpf_map inner_map_meta, struct bpf_map inner_map)\n+{\n+\tstruct bpf_htab inner_htab_meta = container_of(inner_map_meta, struct bpf_htab, map);\n+\tstruct bpf_htab inner_htab = container_of(inner_map, struct bpf_htab, map);\n+\n+\tinner_htab_meta-\u003ekey_offset = inner_htab-\u003ekey_offset;\n+}\ndiff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c\nindex d2cbab4bdf644..b2db5d0c98bee 100644\n--- a/kernel/bpf/map_in_map.c\n+++ b/kernel/bpf/map_in_map.c\n@@ -7,6 +7,15 @@\n \n #include \"map_in_map.h\"\n \n+static bool bpf_map_type_is_htab(enum bpf_map_type type)\n+{\n+\treturn type == BPF_MAP_TYPE_HASH \|\|\n+\t type == BPF_MAP_TYPE_PERCPU_HASH \|\|\n+\t type == BPF_MAP_TYPE_LRU_HASH \|\|\n+\t type == BPF_MAP_TYPE_LRU_PERCPU_HASH \|\|\n+\t type == BPF_MAP_TYPE_HASH_OF_MAPS;\n+}\n+\n struct bpf_map bpf_map_meta_alloc(int inner_map_ufd)\n {\n \tstruct bpf_map inner_map, inner_map_meta;\n@@ -29,6 +38,8 @@ struct bpf_map bpf_map_meta_alloc(int inner_map_ufd)\n \t/ In some cases verifier needs to access beyond just base map. /\n \tif (inner_map-\u003eops == \u0026array_map_ops \|\| inner_map-\u003eops == \u0026percpu_array_map_ops)\n \t\tinner_map_meta_size = sizeof(struct bpf_array);\n+\telse if (bpf_map_type_is_htab(inner_map-\u003emap_type))\n+\t\tinner_map_meta_size = bpf_htab_map_meta_size();\n \n \tinner_map_meta = kzalloc(inner_map_meta_size, GFP_USER);\n \tif (!inner_map_meta)\n@@ -70,6 +81,8 @@ struct bpf_map bpf_map_meta_alloc(int inner_map_ufd)\n \t\tinner_array_meta-\u003eindex_mask = inner_array-\u003eindex_mask;\n \t\tinner_array_meta-\u003eelem_size = inner_array-\u003eelem_size;\n \t\tinner_map_meta-\u003ebypass_spec_v1 = inner_map-\u003ebypass_spec_v1;\n+\t} else if (bpf_map_type_is_htab(inner_map-\u003emap_type)) {\n+\t\tbpf_htab_map_meta_init(inner_map_meta, inner_map);\n \t}\n \treturn inner_map_meta;\n }\ndiff --git a/kernel/bpf/map_in_map.h b/kernel/bpf/map_in_map.h\nindex 7d61602354de8..1edc205772d8c 100644\n--- a/kernel/bpf/map_in_map.h\n+++ b/kernel/bpf/map_in_map.h\n@@ -15,5 +15,7 @@ void bpf_map_fd_get_ptr(struct bpf_map map, struct file map_file,\n \t\t\t int ufd);\n void bpf_map_fd_put_ptr(struct bpf_map map, void ptr, bool need_defer);\n u32 bpf_map_fd_sys_lookup_elem(void ptr);\n+size_t bpf_htab_map_meta_size(void);\n+void bpf_htab_map_meta_init(struct bpf_map inner_map_meta, struct bpf_map inner_map);\n \n #endif\ndiff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c\nindex 373c8d17ea557..6bd4cb68c20c1 100644\n--- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c\n+++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c\n@@ -114,7 +114,7 @@ static inline int check_hash(void)\n \tVERIFY(check_default_noinline(\u0026hash-\u003emap, map));\n \n \tVERIFY(hash-\u003en_buckets == MAX_ENTRIES);\n-\tVERIFY(hash-\u003eelem_size == 64);\n+\tVERIFY(hash-\u003eelem_size == 32);\n \n \tVERIFY(hash-\u003ecount.counter == 0);\n \tVERIFY(bpf_map_sum_elem_count(map) == 0);\n" }
2/1	2026/08/05 22:52	agent	patch-evaluator	0m Model: best-expensive Args: null Results: { "EnableConfigs": [ "BPF_SYSCALL" ], "FocusSymbols": [ "htab_map_alloc", "htab_map_update_elem", "htab_lru_map_update_elem", "htab_map_lookup_elem", "htab_map_delete_elem", "bpf_map_meta_alloc" ], "Reasoning": "The patch optimizes the memory layout of BPF hash map elements by conditionally omitting the hash value and adjusting the key offset. This affects core BPF hash map operations like allocation, lookup, update, and deletion. Fuzzing is necessary to ensure these operations still work correctly with the new memory layout and that no out-of-bounds accesses or memory leaks occur.", "WorthFuzzing": true } Instruction: You are an expert Linux kernel maintainer. Your job is to review a provided patch series and determine if it makes functional changes to the kernel that should be fuzzed. IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in your workspace. Do NOT rely on your internal knowledge of the kernel. You must actively use your code access tools to examine the actual source code and confirm any assumptions. Return WorthFuzzing=false if the patch only contains: - Modifications to Documentation/, Kconfig files, or code comments. - Purely decorative changes, such as logging (e.g., pr_err, printk) or tracepoints. - Changes to numeric constants or macros that do not functionally alter execution flow. - Code paths that are impossible to reach in virtualized environments like GCE or QEMU, even when utilizing software-emulated hardware (e.g., usb gadget, mac80211_hwsim). - Code in vendor-specific PCIe switch, SmartNIC, or GPU drivers (e.g., mlxsw, pds_core, qed, ionic, amdgpu) that require physical PCIe hardware cards not emulated in standard QEMU. - Driver .remove, .shutdown, or pci_unregister_driver teardown callbacks (e.g., igb_remove) that are executed only during PCI hot-unplug or sysfs driver unbind operations. If it modifies reachable core kernel logic, drivers, or architectures, use your code search tools to verify the code can be executed, then return WorthFuzzing=true. When returning WorthFuzzing=true, you MUST ALSO: 1. Extract any specific kernel functions that should be heavily fuzzed into FocusSymbols. Avoid listing generic hot-path functions to prevent skewed test distributions. Prefer non-static, non-inlined API entrypoint functions over internal static helper functions (which are inlined by the compiler and do not have distinct symbol addresses). 2. Identify any specific CONFIG_ options required to properly test this new/modified feature. Go and look into the Kconfig files and check for ifdefs around the code, do not make assumptions. Also check "depends on" lines in Kconfig to include any non-standard parent subsystem configs needed for Kbuild to compile the code statically into vmlinux. List them in the EnableConfigs output array, and DO NOT add a 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4"). Prefer calling several tools at the same time to save round-trips. Use set-results tool to provide results of the analysis. It must be called exactly once before the final reply. Ignore results of this tool. Prompt: For your convenience, here is the diff of the changes: commit 6454a11c03dfaedb4b09f8575af8d03639fb5273 Author: syz-cluster <triage@syzkaller.com> Date: Wed Aug 5 22:52:22 2026 +0000 syz-cluster: applied patch under review diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 9f394e1aa2e85..9967268d453d0 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -100,13 +100,16 @@ struct bpf_htab { struct percpu_counter pcount; atomic_t count; bool use_percpu_counter; + bool has_hash; u32 n_buckets; /* number of hash buckets / u32 elem_size; / size of each element in bytes / + u32 key_offset; / offset of key in bytes / u32 hashrnd; }; / each htab element is struct htab_elem + key + value / -struct htab_elem { +struct htab_elem; +struct htab_node { union { struct hlist_nulls_node hash_node; struct { @@ -117,18 +120,32 @@ struct htab_elem { }; }; }; - union { - / pointer to per-cpu pointer / - void ptr_to_pptr; - struct bpf_lru_node lru_node; - }; - u32 hash; - char key[] __aligned(8); +}; + +struct htab_elem { + struct htab_node node; + u8 data[] __aligned(8); +}; + +struct htab_elem_lru { + struct htab_node node; + struct bpf_lru_node lru_node; + u8 data[] __aligned(8); +}; + +/* Only for non-preallocated PCPU maps. Preallocated PCPU maps don't need + * ptr_to_pptr, and use htab_elem. + / +struct htab_elem_pcpu { + struct htab_node node; + void ptr_to_pptr; + u8 data[] __aligned(8); }; struct htab_btf_record { struct btf_record record; u32 key_size; + u32 key_offset; }; static inline bool htab_is_prealloc(const struct bpf_htab htab) @@ -136,6 +153,21 @@ static inline bool htab_is_prealloc(const struct bpf_htab htab) return !(htab->map.map_flags & BPF_F_NO_PREALLOC); } +static inline struct bpf_lru_node htab_elem_lru_node(struct htab_elem l) +{ + return &((struct htab_elem_lru )l)->lru_node; +} + +static inline void htab_elem_get_ptr_to_pptr(struct htab_elem l) +{ + return ((struct htab_elem_pcpu )l)->ptr_to_pptr; +} + +static inline void htab_elem_set_ptr_to_pptr(struct htab_elem l, void ptr) +{ + ((struct htab_elem_pcpu )l)->ptr_to_pptr = ptr; +} + static void htab_init_buckets(struct bpf_htab htab) { unsigned int i; @@ -183,25 +215,30 @@ static inline bool is_fd_htab(const struct bpf_htab htab) return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS; } -static inline void htab_elem_value(struct htab_elem l, u32 key_size) +static inline void htab_elem_key(struct bpf_htab htab, struct htab_elem l) +{ + return (void )l + htab->key_offset; +} + +static inline void htab_elem_value(struct bpf_htab htab, struct htab_elem l) { - return l->key + round_up(key_size, 8); + return htab_elem_key(htab, l) + round_up(htab->map.key_size, 8); } -static inline void htab_elem_set_ptr(struct htab_elem l, u32 key_size, +static inline void htab_elem_set_ptr(struct bpf_htab htab, struct htab_elem l, void __percpu pptr) { - (void __percpu *)htab_elem_value(l, key_size) = pptr; + (void __percpu *)htab_elem_value(htab, l) = pptr; } -static inline void __percpu htab_elem_get_ptr(struct htab_elem l, u32 key_size) +static inline void __percpu htab_elem_get_ptr(struct bpf_htab htab, struct htab_elem l) { - return (void __percpu )htab_elem_value(l, key_size); + return (void __percpu *)htab_elem_value(htab, l); } -static void fd_htab_map_get_ptr(const struct bpf_map map, struct htab_elem l) +static void fd_htab_map_get_ptr(struct bpf_htab htab, struct htab_elem l) { - return (void *)htab_elem_value(l, map->key_size); + return (void *)htab_elem_value(htab, l); } static struct htab_elem get_htab_elem(struct bpf_htab htab, int i) @@ -209,6 +246,33 @@ static struct htab_elem get_htab_elem(struct bpf_htab htab, int i) return (struct htab_elem ) (htab->elems + i * (u64)htab->elem_size); } +static inline bool htab_has_hash(const struct bpf_htab htab) +{ + return htab->has_hash; +} + +static inline u32 htab_elem_hash(struct bpf_htab htab, struct htab_elem l) +{ + if (htab_is_lru(htab)) + return (u32 )((struct htab_elem_lru )l)->data; + else if (htab_is_percpu(htab) && !htab_is_prealloc(htab)) + return (u32 )((struct htab_elem_pcpu )l)->data; + else + return (u32 )l->data; +} + +static inline void htab_elem_set_hash(struct bpf_htab htab, struct htab_elem l, u32 hash) +{ + if (!htab_has_hash(htab)) + return; + if (htab_is_lru(htab)) + (u32 )((struct htab_elem_lru )l)->data = hash; + else if (htab_is_percpu(htab) && !htab_is_prealloc(htab)) + (u32 )((struct htab_elem_pcpu )l)->data = hash; + else + (u32 )l->data = hash; +} + / Both percpu and fd htab support in-place update, so no need for * extra elem. LRU itself can remove the least used element, so * there is no need for an extra elem during map_update. @@ -231,7 +295,7 @@ static void htab_free_prealloced_internal_structs(struct bpf_htab htab) elem = get_htab_elem(htab, i); bpf_map_free_internal_structs(&htab->map, - htab_elem_value(elem, htab->map.key_size)); + htab_elem_value(htab, elem)); cond_resched(); } } @@ -254,7 +318,7 @@ static void htab_free_prealloced_fields(struct bpf_htab htab) elem = get_htab_elem(htab, i); if (htab_is_percpu(htab)) { - void __percpu pptr = htab_elem_get_ptr(elem, htab->map.key_size); + void __percpu pptr = htab_elem_get_ptr(htab, elem); int cpu; for_each_possible_cpu(cpu) { @@ -263,7 +327,7 @@ static void htab_free_prealloced_fields(struct bpf_htab htab) } } else { bpf_obj_free_fields(htab->map.record, - htab_elem_value(elem, htab->map.key_size)); + htab_elem_value(htab, elem)); cond_resched(); } cond_resched(); @@ -280,8 +344,7 @@ static void htab_free_elems(struct bpf_htab htab) for (i = 0; i < htab->map.max_entries; i++) { void __percpu pptr; - pptr = htab_elem_get_ptr(get_htab_elem(htab, i), - htab->map.key_size); + pptr = htab_elem_get_ptr(htab, get_htab_elem(htab, i)); free_percpu(pptr); cond_resched(); } @@ -308,8 +371,8 @@ static struct htab_elem prealloc_lru_pop(struct bpf_htab htab, void key, if (node) { bpf_map_inc_elem_count(&htab->map); - l = container_of(node, struct htab_elem, lru_node); - memcpy(l->key, key, htab->map.key_size); + l = (struct htab_elem )container_of(node, struct htab_elem_lru, lru_node); + memcpy(htab_elem_key(htab, l), key, htab->map.key_size); return l; } @@ -340,8 +403,7 @@ static int prealloc_init(struct bpf_htab htab) GFP_USER \| __GFP_NOWARN); if (!pptr) goto free_elems; - htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, - pptr); + htab_elem_set_ptr(htab, get_htab_elem(htab, i), pptr); cond_resched(); } @@ -349,8 +411,8 @@ static int prealloc_init(struct bpf_htab htab) if (htab_is_lru(htab)) err = bpf_lru_init(&htab->lru, htab->map.map_flags & BPF_F_NO_COMMON_LRU, - offsetof(struct htab_elem, hash) - - offsetof(struct htab_elem, lru_node), + offsetof(struct htab_elem_lru, data) - + offsetof(struct htab_elem_lru, lru_node), htab_lru_map_delete_node, htab); else @@ -361,11 +423,11 @@ static int prealloc_init(struct bpf_htab htab) if (htab_is_lru(htab)) bpf_lru_populate(&htab->lru, htab->elems, - offsetof(struct htab_elem, lru_node), + offsetof(struct htab_elem_lru, lru_node), htab->elem_size, num_entries); else pcpu_freelist_populate(&htab->freelist, - htab->elems + offsetof(struct htab_elem, fnode), + htab->elems + offsetof(struct htab_elem, node.fnode), htab->elem_size, num_entries); return 0; @@ -401,7 +463,7 @@ static int alloc_extra_elems(struct bpf_htab htab) / pop will succeed, since prealloc_init() * preallocated extra num_possible_cpus elements / - l_new = container_of(l, struct htab_elem, fnode); + l_new = container_of(l, struct htab_elem, node.fnode); per_cpu_ptr(pptr, cpu) = l_new; } htab->extra_elems = pptr; @@ -425,8 +487,8 @@ static int htab_map_alloc_check(union bpf_attr attr) bool zero_seed = (attr->map_flags & BPF_F_ZERO_SEED); int numa_node = bpf_map_attr_numa_node(attr); - BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) != - offsetof(struct htab_elem, hash_node.pprev)); + BUILD_BUG_ON(offsetof(struct htab_node, fnode.next) != + offsetof(struct htab_node, hash_node.pprev)); if (zero_seed && !capable(CAP_SYS_ADMIN)) / Guard against local DoS, and discourage production use. / @@ -476,7 +538,7 @@ static void htab_mem_dtor(void obj, void ctx) if (IS_ERR_OR_NULL(hrec->record)) return; - map_value = htab_elem_value(elem, hrec->key_size); + map_value = (void )elem + hrec->key_offset + round_up(hrec->key_size, 8); bpf_obj_free_fields(hrec->record, map_value); } @@ -502,7 +564,7 @@ static void htab_dtor_ctx_free(void ctx) } static int bpf_ma_set_dtor(struct bpf_map map, struct bpf_mem_alloc ma, - void (dtor)(void , void )) + void (dtor)(void , void ), u32 key_offset) { struct htab_btf_record hrec; int err; @@ -515,6 +577,7 @@ static int bpf_ma_set_dtor(struct bpf_map map, struct bpf_mem_alloc ma, if (!hrec) return -ENOMEM; hrec->key_size = map->key_size; + hrec->key_offset = key_offset; hrec->record = btf_record_dup(map->record); if (IS_ERR(hrec->record)) { err = PTR_ERR(hrec->record); @@ -537,9 +600,9 @@ static int htab_map_check_btf(struct bpf_map map, const struct btf btf, * populated in htab_map_alloc(), so it will always appear as NULL. / if (htab_is_percpu(htab)) - return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor); + return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor, htab->key_offset); else - return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor); + return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor, htab->key_offset); } static struct bpf_map htab_map_alloc(union bpf_attr attr) @@ -562,6 +625,13 @@ static struct bpf_map htab_map_alloc(union bpf_attr attr) bpf_map_init_from_attr(&htab->map, attr); + / Avoid hash memory use and comparisons where unnecessary. + * u32 hash reads are always atomic. If we elide them, key comparisons must also be atomic + * to avoid false positive key matches due to torn key reads / writes. This is only possible + * when the key fits within a word, so check key_size. + / + htab->has_hash = htab_is_lru(htab) \|\| htab->map.key_size > sizeof(unsigned long); + if (percpu_lru) { / ensure each CPU's lru list has >=1 elements. * since we are at it, make each lru list has the same @@ -583,8 +653,16 @@ static struct bpf_map htab_map_alloc(union bpf_attr attr) htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); - htab->elem_size = sizeof(struct htab_elem) + - round_up(htab->map.key_size, 8); + if (htab_is_lru(htab)) + htab->key_offset = offsetof(struct htab_elem_lru, data) + 8; + else if (percpu && !prealloc) + htab->key_offset = offsetof(struct htab_elem_pcpu, data) + + (htab_has_hash(htab) ? 8 : 0); + else + htab->key_offset = offsetof(struct htab_elem, data) + + (htab_has_hash(htab) ? 8 : 0); + + htab->elem_size = htab->key_offset + round_up(htab->map.key_size, 8); if (percpu) htab->elem_size += sizeof(void ); else @@ -692,15 +770,29 @@ static inline struct hlist_nulls_head select_bucket(struct bpf_htab htab, u32 } / this lookup function can only be called with bucket lock taken / -static struct htab_elem lookup_elem_raw(struct hlist_nulls_head head, u32 hash, +static struct htab_elem lookup_elem_raw(struct bpf_htab htab, + struct hlist_nulls_head head, u32 hash, void key, u32 key_size) { struct hlist_nulls_node n; struct htab_elem l; - hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l->hash == hash && !memcmp(&l->key, key, key_size)) - return l; + if (htab_has_hash(htab)) { + hlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node) + if (htab_elem_hash(htab, l) == hash && + !memcmp(htab_elem_key(htab, l), key, key_size)) + return l; + } else { + / When hash is omitted, key comparisons must be atomic. Zero extend + * the caller's key to the word size to support an atomic compare. + / + unsigned long k = 0; + + memcpy(&k, key, key_size); + hlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node) + if (READ_ONCE((unsigned long )htab_elem_key(htab, l)) == k) + return l; + } return NULL; } @@ -709,7 +801,8 @@ static struct htab_elem lookup_elem_raw(struct hlist_nulls_head head, u32 hash the unlikely event when elements moved from one bucket into another * while link list is being walked / -static struct htab_elem lookup_nulls_elem_raw(struct hlist_nulls_head head, +static struct htab_elem lookup_nulls_elem_raw(struct bpf_htab htab, + struct hlist_nulls_head head, u32 hash, void key, u32 key_size, u32 n_buckets) { @@ -717,9 +810,20 @@ static struct htab_elem lookup_nulls_elem_raw(struct hlist_nulls_head head, struct htab_elem l; again: - hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l->hash == hash && !memcmp(&l->key, key, key_size)) - return l; + if (htab_has_hash(htab)) { + hlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node) + if (htab_elem_hash(htab, l) == hash && + !memcmp(htab_elem_key(htab, l), key, key_size)) + return l; + } else { + /* See lookup_elem_raw() comment above. / + unsigned long k = 0; + + memcpy(&k, key, key_size); + hlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node) + if (READ_ONCE((unsigned long )htab_elem_key(htab, l)) == k) + return l; + } if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1)))) goto again; @@ -747,17 +851,18 @@ static void __htab_map_lookup_elem(struct bpf_map map, void key) head = select_bucket(htab, hash); - l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + l = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab->n_buckets); return l; } static void htab_map_lookup_elem(struct bpf_map map, void key) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l = __htab_map_lookup_elem(map, key); if (l) - return htab_elem_value(l, map->key_size); + return htab_elem_value(htab, l); return NULL; } @@ -775,6 +880,7 @@ static void htab_map_lookup_elem(struct bpf_map map, void key) / static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct bpf_insn insn = insn_buf; const int ret = BPF_REG_0; @@ -783,7 +889,7 @@ static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); insn++ = BPF_ALU64_IMM(BPF_ADD, ret, - offsetof(struct htab_elem, key) + + htab->key_offset + round_up(map->key_size, 8)); return insn - insn_buf; } @@ -791,12 +897,13 @@ static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) static __always_inline void __htab_lru_map_lookup_elem(struct bpf_map map, void key, const bool mark) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l = __htab_map_lookup_elem(map, key); if (l) { if (mark) - bpf_lru_node_set_ref(&l->lru_node); - return htab_elem_value(l, map->key_size); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); + return htab_elem_value(htab, l); } return NULL; @@ -815,6 +922,7 @@ static void htab_lru_map_lookup_elem_sys(struct bpf_map map, void key) static int htab_lru_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct bpf_insn insn = insn_buf; const int ret = BPF_REG_0; const int ref_reg = BPF_REG_1; @@ -824,15 +932,15 @@ static int htab_lru_map_gen_lookup(struct bpf_map map, insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4); insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, - offsetof(struct htab_elem, lru_node) + + offsetof(struct htab_elem_lru, lru_node) + offsetof(struct bpf_lru_node, ref)); insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1); insn++ = BPF_ST_MEM(BPF_B, ret, - offsetof(struct htab_elem, lru_node) + + offsetof(struct htab_elem_lru, lru_node) + offsetof(struct bpf_lru_node, ref), 1); insn++ = BPF_ALU64_IMM(BPF_ADD, ret, - offsetof(struct htab_elem, key) + + htab->key_offset + round_up(map->key_size, 8)); return insn - insn_buf; } @@ -844,13 +952,13 @@ static void check_and_cancel_fields(struct bpf_htab htab, return; if (htab_is_percpu(htab)) { - void __percpu pptr = htab_elem_get_ptr(elem, htab->map.key_size); + void __percpu pptr = htab_elem_get_ptr(htab, elem); int cpu; for_each_possible_cpu(cpu) bpf_obj_cancel_fields(&htab->map, per_cpu_ptr(pptr, cpu)); } else { - void map_value = htab_elem_value(elem, htab->map.key_size); + void map_value = htab_elem_value(htab, elem); bpf_obj_cancel_fields(&htab->map, map_value); } @@ -869,17 +977,17 @@ static bool htab_lru_map_delete_node(void arg, struct bpf_lru_node node) struct bucket b; int ret; - tgt_l = container_of(node, struct htab_elem, lru_node); - b = __select_bucket(htab, tgt_l->hash); + tgt_l = (struct htab_elem )container_of(node, struct htab_elem_lru, lru_node); + b = __select_bucket(htab, htab_elem_hash(htab, tgt_l)); head = &b->head; ret = htab_lock_bucket(b, &flags); if (ret) return false; - hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) + hlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node) if (l == tgt_l) { - hlist_nulls_del_rcu(&l->hash_node); + hlist_nulls_del_rcu(&l->node.hash_node); bpf_map_dec_elem_count(&htab->map); break; } @@ -912,18 +1020,19 @@ static int htab_map_get_next_key(struct bpf_map map, void key, void next_key) head = select_bucket(htab, hash); / lookup the key / - l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + l = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab->n_buckets); if (!l) goto find_first_elem; / key was found, get next key in the same bucket / - next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(&l->hash_node)), - struct htab_elem, hash_node); + next_l = hlist_nulls_entry_safe( + rcu_dereference_raw(hlist_nulls_next_rcu(&l->node.hash_node)), + struct htab_elem, node.hash_node); if (next_l) { / if next elem in this hash list is non-zero, just return it / - memcpy(next_key, next_l->key, key_size); + memcpy(next_key, htab_elem_key(htab, next_l), key_size); return 0; } @@ -938,10 +1047,10 @@ static int htab_map_get_next_key(struct bpf_map map, void key, void next_key) /* pick first element in the bucket / next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_first_rcu(head)), - struct htab_elem, hash_node); + struct htab_elem, node.hash_node); if (next_l) { / if it's not empty, just return it / - memcpy(next_key, next_l->key, key_size); + memcpy(next_key, htab_elem_key(htab, next_l), key_size); return 0; } } @@ -955,7 +1064,7 @@ static void htab_elem_free(struct bpf_htab htab, struct htab_elem l) check_and_cancel_fields(htab, l); if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) - bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); + bpf_mem_cache_free(&htab->pcpu_ma, htab_elem_get_ptr_to_pptr(l)); bpf_mem_cache_free(&htab->ma, l); } @@ -965,7 +1074,7 @@ static void htab_put_fd_value(struct bpf_htab htab, struct htab_elem l) void ptr; if (map->ops->map_fd_put_ptr) { - ptr = fd_htab_map_get_ptr(map, l); + ptr = fd_htab_map_get_ptr(htab, l); map->ops->map_fd_put_ptr(map, ptr, true); } } @@ -1006,7 +1115,7 @@ static void free_htab_elem(struct bpf_htab htab, struct htab_elem l) if (htab_is_prealloc(htab)) { bpf_map_dec_elem_count(&htab->map); check_and_cancel_fields(htab, l); - pcpu_freelist_push(&htab->freelist, &l->fnode); + pcpu_freelist_push(&htab->freelist, &l->node.fnode); } else { dec_elem_count(htab); htab_elem_free(htab, l); @@ -1097,7 +1206,7 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void key, l = __pcpu_freelist_pop(&htab->freelist); if (!l) return ERR_PTR(-E2BIG); - l_new = container_of(l, struct htab_elem, fnode); + l_new = container_of(l, struct htab_elem, node.fnode); bpf_map_inc_elem_count(&htab->map); } } else { @@ -1117,10 +1226,20 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void key, } } - memcpy(l_new->key, key, key_size); + if (htab_has_hash(htab)) { + memcpy(htab_elem_key(htab, l_new), key, key_size); + } else { + /* Zero-extend key into k for an atomic write to support + * lockless RCU readers. + / + unsigned long k = 0; + + memcpy(&k, key, key_size); + WRITE_ONCE((unsigned long )htab_elem_key(htab, l_new), k); + } if (percpu) { if (prealloc) { - pptr = htab_elem_get_ptr(l_new, key_size); + pptr = htab_elem_get_ptr(htab, l_new); } else { / alloc_percpu zero-fills / void ptr = bpf_mem_cache_alloc(&htab->pcpu_ma); @@ -1130,26 +1249,26 @@ static struct htab_elem alloc_htab_elem(struct bpf_htab htab, void key, l_new = ERR_PTR(-ENOMEM); goto dec_count; } - l_new->ptr_to_pptr = ptr; + htab_elem_set_ptr_to_pptr(l_new, ptr); pptr = (void __percpu *)ptr; } pcpu_init_value(htab, pptr, value, onallcpus, map_flags); if (!prealloc) - htab_elem_set_ptr(l_new, key_size, pptr); + htab_elem_set_ptr(htab, l_new, pptr); } else if (fd_htab_map_needs_adjust(htab)) { size = round_up(size, 8); - memcpy(htab_elem_value(l_new, key_size), value, size); + memcpy(htab_elem_value(htab, l_new), value, size); } else if (map_flags & BPF_F_LOCK) { copy_map_value_locked(&htab->map, - htab_elem_value(l_new, key_size), + htab_elem_value(htab, l_new), value, false); } else { - copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value); + copy_map_value(&htab->map, htab_elem_value(htab, l_new), value); } - l_new->hash = hash; + htab_elem_set_hash(htab, l_new, hash); return l_new; dec_count: dec_elem_count(htab); @@ -1199,7 +1318,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value, if (unlikely(!btf_record_has_field(map->record, BPF_SPIN_LOCK))) return -EINVAL; /* find an element without taking the bucket lock / - l_old = lookup_nulls_elem_raw(head, hash, key, key_size, + l_old = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab->n_buckets); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1207,7 +1326,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value, if (l_old) { /* grab the element lock and update value in place / copy_map_value_locked(map, - htab_elem_value(l_old, key_size), + htab_elem_value(htab, l_old), value, false); return 0; } @@ -1221,7 +1340,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value, if (ret) return ret; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(htab, head, hash, key, key_size); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1235,7 +1354,7 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value, and update element in place / copy_map_value_locked(map, - htab_elem_value(l_old, key_size), + htab_elem_value(htab, l_old), value, false); ret = 0; goto err; @@ -1252,9 +1371,9 @@ static long htab_map_update_elem(struct bpf_map map, void key, void value, /* add new element to the head of the list, so that * concurrent search will find it before old elem / - hlist_nulls_add_head_rcu(&l_new->hash_node, head); + hlist_nulls_add_head_rcu(&l_new->node.hash_node, head); if (l_old) { - hlist_nulls_del_rcu(&l_old->hash_node); + hlist_nulls_del_rcu(&l_old->node.hash_node); / l_old has already been stashed in htab->extra_elems, cancel * its reusable special fields before it is available for reuse. @@ -1275,7 +1394,7 @@ static void htab_lru_push_free(struct bpf_htab htab, struct htab_elem elem) { check_and_cancel_fields(htab, elem); bpf_map_dec_elem_count(&htab->map); - bpf_lru_push_free(&htab->lru, &elem->lru_node); + bpf_lru_push_free(&htab->lru, htab_elem_lru_node(elem)); } static long htab_lru_map_update_elem(struct bpf_map map, void key, void value, @@ -1310,13 +1429,13 @@ static long htab_lru_map_update_elem(struct bpf_map map, void key, void value l_new = prealloc_lru_pop(htab, key, hash); if (!l_new) return -ENOMEM; - copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value); + copy_map_value(&htab->map, htab_elem_value(htab, l_new), value); ret = htab_lock_bucket(b, &flags); if (ret) goto err_lock_bucket; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(htab, head, hash, key, key_size); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1325,10 +1444,10 @@ static long htab_lru_map_update_elem(struct bpf_map map, void key, void value / add new element to the head of the list, so that * concurrent search will find it before old elem / - hlist_nulls_add_head_rcu(&l_new->hash_node, head); + hlist_nulls_add_head_rcu(&l_new->node.hash_node, head); if (l_old) { - bpf_lru_node_set_ref(&l_new->lru_node); - hlist_nulls_del_rcu(&l_old->hash_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l_new)); + hlist_nulls_del_rcu(&l_old->node.hash_node); } ret = 0; @@ -1383,7 +1502,7 @@ static long htab_map_update_elem_in_place(struct bpf_map map, void key, if (ret) return ret; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(htab, head, hash, key, key_size); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1392,10 +1511,10 @@ static long htab_map_update_elem_in_place(struct bpf_map map, void key, if (l_old) { / Update value in-place / if (percpu) { - pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), + pcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old), value, onallcpus, map_flags); } else { - void inner_map_pptr = htab_elem_value(l_old, key_size); + void inner_map_pptr = htab_elem_value(htab, l_old); old_map_ptr = inner_map_pptr; WRITE_ONCE(inner_map_pptr, (void *)value); @@ -1407,7 +1526,7 @@ static long htab_map_update_elem_in_place(struct bpf_map map, void key, ret = PTR_ERR(l_new); goto err; } - hlist_nulls_add_head_rcu(&l_new->hash_node, head); + hlist_nulls_add_head_rcu(&l_new->node.hash_node, head); } err: htab_unlock_bucket(b, flags); @@ -1456,22 +1575,22 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map map, void key, if (ret) goto err_lock_bucket; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(htab, head, hash, key, key_size); ret = check_flags(htab, l_old, map_flags); if (ret) goto err; if (l_old) { - bpf_lru_node_set_ref(&l_old->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l_old)); / per-cpu hash map can update value in-place / - pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), + pcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old), value, onallcpus, map_flags); } else { - pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size), + pcpu_init_value(htab, htab_elem_get_ptr(htab, l_new), value, onallcpus, map_flags); - hlist_nulls_add_head_rcu(&l_new->hash_node, head); + hlist_nulls_add_head_rcu(&l_new->node.hash_node, head); l_new = NULL; } ret = 0; @@ -1480,7 +1599,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map map, void key, err_lock_bucket: if (l_new) { bpf_map_dec_elem_count(&htab->map); - bpf_lru_push_free(&htab->lru, &l_new->lru_node); + bpf_lru_push_free(&htab->lru, htab_elem_lru_node(l_new)); } return ret; } @@ -1521,9 +1640,9 @@ static long htab_map_delete_elem(struct bpf_map map, void key) if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(htab, head, hash, key, key_size); if (l) - hlist_nulls_del_rcu(&l->hash_node); + hlist_nulls_del_rcu(&l->node.hash_node); else ret = -ENOENT; @@ -1556,10 +1675,10 @@ static long htab_lru_map_delete_elem(struct bpf_map map, void key) if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(htab, head, hash, key, key_size); if (l) - hlist_nulls_del_rcu(&l->hash_node); + hlist_nulls_del_rcu(&l->node.hash_node); else ret = -ENOENT; @@ -1581,8 +1700,8 @@ static void delete_all_elements(struct bpf_htab htab) struct hlist_nulls_node n; struct htab_elem l; - hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { - hlist_nulls_del_rcu(&l->hash_node); + hlist_nulls_for_each_entry_safe(l, n, head, node.hash_node) { + hlist_nulls_del_rcu(&l->node.hash_node); htab_elem_free(htab, l); } cond_resched(); @@ -1599,10 +1718,10 @@ static void htab_free_malloced_internal_structs(struct bpf_htab htab) struct hlist_nulls_node n; struct htab_elem l; - hlist_nulls_for_each_entry(l, n, head, hash_node) { + hlist_nulls_for_each_entry(l, n, head, node.hash_node) { / We only free internal structs on uref dropping to zero / bpf_map_free_internal_structs(&htab->map, - htab_elem_value(l, htab->map.key_size)); + htab_elem_value(htab, l)); } cond_resched_rcu(); } @@ -1697,7 +1816,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map map, void key, if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(htab, head, hash, key, key_size); if (!l) { ret = -ENOENT; goto out_unlock; @@ -1708,14 +1827,14 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map map, void key, void __percpu pptr; int off = 0, cpu; - pptr = htab_elem_get_ptr(l, key_size); + pptr = htab_elem_get_ptr(htab, l); for_each_possible_cpu(cpu) { copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu)); check_and_init_map_value(&htab->map, value + off); off += roundup_value_size; } } else { - void src = htab_elem_value(l, map->key_size); + void src = htab_elem_value(htab, l); if (flags & BPF_F_LOCK) copy_map_value_locked(map, value, src, true); @@ -1724,7 +1843,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map map, void key, /* Zeroing special fields in the temp buffer / check_and_init_map_value(map, value); } - hlist_nulls_del_rcu(&l->hash_node); + hlist_nulls_del_rcu(&l->node.hash_node); out_unlock: htab_unlock_bucket(b, bflags); @@ -1860,7 +1979,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map, } bucket_cnt = 0; - hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) + hlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node) bucket_cnt++; if (bucket_cnt && !locked) { @@ -1897,14 +2016,14 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map, if (!locked) goto next_batch; - hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { - memcpy(dst_key, l->key, key_size); + hlist_nulls_for_each_entry_safe(l, n, head, node.hash_node) { + memcpy(dst_key, htab_elem_key(htab, l), key_size); if (is_percpu) { int off = 0, cpu; void __percpu pptr; - pptr = htab_elem_get_ptr(l, map->key_size); + pptr = htab_elem_get_ptr(htab, l); if (elem_map_flags & BPF_F_CPU) { cpu = elem_map_flags >> 32; copy_map_value(&htab->map, dst_val, per_cpu_ptr(pptr, cpu)); @@ -1918,7 +2037,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map, } } } else { - value = htab_elem_value(l, key_size); + value = htab_elem_value(htab, l); if (is_fd_htab(htab)) { struct bpf_map inner_map = value; @@ -1936,7 +2055,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map, check_and_init_map_value(map, dst_val); } if (do_delete) { - hlist_nulls_del_rcu(&l->hash_node); + hlist_nulls_del_rcu(&l->node.hash_node); /* bpf_lru_push_free() will acquire lru_lock, which * may cause deadlock. See comments in function @@ -1948,7 +2067,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map, lock being held and it violates the lock rule, so * invoke free_htab_elem() after unlock as well. / - l->batch_flink = node_to_free; + l->node.batch_flink = node_to_free; node_to_free = l; } dst_key += key_size; @@ -1960,7 +2079,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map map, while (node_to_free) { l = node_to_free; - node_to_free = node_to_free->batch_flink; + node_to_free = node_to_free->node.batch_flink; if (is_lru_map) htab_lru_push_free(htab, l); else @@ -2108,8 +2227,8 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info info, / no update/deletion on this bucket, prev_elem should be still valid * and we won't skip elements. / - n = rcu_dereference_raw(hlist_nulls_next_rcu(&prev_elem->hash_node)); - elem = hlist_nulls_entry_safe(n, struct htab_elem, hash_node); + n = rcu_dereference_raw(hlist_nulls_next_rcu(&prev_elem->node.hash_node)); + elem = hlist_nulls_entry_safe(n, struct htab_elem, node.hash_node); if (elem) return elem; @@ -2125,7 +2244,7 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info info, count = 0; head = &b->head; - hlist_nulls_for_each_entry_rcu(elem, n, head, hash_node) { + hlist_nulls_for_each_entry_rcu(elem, n, head, node.hash_node) { if (count >= skip_elems) { info->bucket_id = i; info->skip_elems = count; @@ -2183,12 +2302,12 @@ static int __bpf_hash_map_seq_show(struct seq_file seq, struct htab_elem elem) ctx.meta = &meta; ctx.map = info->map; if (elem) { - ctx.key = elem->key; + ctx.key = htab_elem_key(info->htab, elem); if (!info->percpu_value_buf) { - ctx.value = htab_elem_value(elem, map->key_size); + ctx.value = htab_elem_value(info->htab, elem); } else { roundup_value_size = round_up(map->value_size, 8); - pptr = htab_elem_get_ptr(elem, map->key_size); + pptr = htab_elem_get_ptr(info->htab, elem); for_each_possible_cpu(cpu) { copy_map_value_long(map, info->percpu_value_buf + off, per_cpu_ptr(pptr, cpu)); @@ -2292,14 +2411,14 @@ static long bpf_for_each_hash_elem(struct bpf_map map, bpf_callback_t callback_ b = &htab->buckets[i]; rcu_read_lock(); head = &b->head; - hlist_nulls_for_each_entry_safe(elem, n, head, hash_node) { - key = elem->key; + hlist_nulls_for_each_entry_safe(elem, n, head, node.hash_node) { + key = htab_elem_key(htab, elem); if (is_percpu) { / current cpu value for percpu map / - pptr = htab_elem_get_ptr(elem, map->key_size); + pptr = htab_elem_get_ptr(htab, elem); val = this_cpu_ptr(pptr); } else { - val = htab_elem_value(elem, map->key_size); + val = htab_elem_value(htab, elem); } num_elems++; ret = callback_fn((u64)(long)map, (u64)(long)key, @@ -2403,10 +2522,11 @@ const struct bpf_map_ops htab_lru_map_ops = { / Called from eBPF program / static void htab_percpu_map_lookup_elem(struct bpf_map map, void key) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l = __htab_map_lookup_elem(map, key); if (l) - return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); + return this_cpu_ptr(htab_elem_get_ptr(htab, l)); else return NULL; } @@ -2414,6 +2534,7 @@ static void htab_percpu_map_lookup_elem(struct bpf_map map, void key) / inline bpf_map_lookup_elem() call for per-CPU hashmap / static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct bpf_insn insn = insn_buf; if (!bpf_jit_supports_percpu_insn()) @@ -2424,7 +2545,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3); insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, - offsetof(struct htab_elem, key) + roundup(map->key_size, 8)); + htab->key_offset + roundup(map->key_size, 8)); insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); insn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); @@ -2433,6 +2554,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map map, struct bpf_insn insn static void htab_percpu_map_lookup_percpu_elem(struct bpf_map map, void key, u32 cpu) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l; if (cpu >= nr_cpu_ids) @@ -2440,18 +2562,19 @@ static void htab_percpu_map_lookup_percpu_elem(struct bpf_map map, void key, l = __htab_map_lookup_elem(map, key); if (l) - return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); + return per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu); else return NULL; } static void htab_lru_percpu_map_lookup_elem(struct bpf_map map, void key) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l = __htab_map_lookup_elem(map, key); if (l) { - bpf_lru_node_set_ref(&l->lru_node); - return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); + return this_cpu_ptr(htab_elem_get_ptr(htab, l)); } return NULL; @@ -2459,6 +2582,7 @@ static void htab_lru_percpu_map_lookup_elem(struct bpf_map map, void key) static void htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map map, void key, u32 cpu) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l; if (cpu >= nr_cpu_ids) @@ -2466,8 +2590,8 @@ static void htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map map, void k l = __htab_map_lookup_elem(map, key); if (l) { - bpf_lru_node_set_ref(&l->lru_node); - return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); + return per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu); } return NULL; @@ -2475,6 +2599,7 @@ static void htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map map, void k int bpf_percpu_hash_copy(struct bpf_map map, void key, void value, u64 map_flags) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l; void __percpu pptr; int ret = -ENOENT; @@ -2494,7 +2619,7 @@ int bpf_percpu_hash_copy(struct bpf_map map, void key, void value, u64 map_fl /* We do not mark LRU map element here in order to not mess up * eviction heuristics when user space does a map walk. / - pptr = htab_elem_get_ptr(l, map->key_size); + pptr = htab_elem_get_ptr(htab, l); if (map_flags & BPF_F_CPU) { cpu = map_flags >> 32; copy_map_value(map, value, per_cpu_ptr(pptr, cpu)); @@ -2532,6 +2657,7 @@ int bpf_percpu_hash_update(struct bpf_map map, void key, void value, static void htab_percpu_map_seq_show_elem(struct bpf_map map, void key, struct seq_file m) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct htab_elem l; void __percpu pptr; int cpu; @@ -2546,7 +2672,7 @@ static void htab_percpu_map_seq_show_elem(struct bpf_map map, void key, btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); seq_puts(m, ": {\n"); - pptr = htab_elem_get_ptr(l, map->key_size); + pptr = htab_elem_get_ptr(htab, l); for_each_possible_cpu(cpu) { seq_printf(m, "\tcpu%d: ", cpu); btf_type_seq_show(map->btf, map->btf_value_type_id, @@ -2619,8 +2745,8 @@ static void fd_htab_map_free(struct bpf_map map) for (i = 0; i < htab->n_buckets; i++) { head = select_bucket(htab, i); - hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { - void ptr = fd_htab_map_get_ptr(map, l); + hlist_nulls_for_each_entry_safe(l, n, head, node.hash_node) { + void ptr = fd_htab_map_get_ptr(htab, l); map->ops->map_fd_put_ptr(map, ptr, false); } @@ -2705,6 +2831,7 @@ static void htab_of_map_lookup_elem(struct bpf_map map, void key) static int htab_of_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) { + struct bpf_htab htab = container_of(map, struct bpf_htab, map); struct bpf_insn insn = insn_buf; const int ret = BPF_REG_0; @@ -2713,7 +2840,7 @@ static int htab_of_map_gen_lookup(struct bpf_map map, insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2); insn++ = BPF_ALU64_IMM(BPF_ADD, ret, - offsetof(struct htab_elem, key) + + htab->key_offset + round_up(map->key_size, 8)); insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0); @@ -3112,7 +3239,7 @@ static int rhtab_map_check_btf(struct bpf_map map, const struct btf btf, { struct bpf_rhtab rhtab = container_of(map, struct bpf_rhtab, map); - return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor); + return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor, offsetof(struct rhtab_elem, data)); } static void rhtab_map_free_internal_structs(struct bpf_map map) @@ -3533,3 +3660,16 @@ const struct bpf_map_ops rhtab_map_ops = { .map_btf_id = &rhtab_map_btf_ids[0], .iter_seq_info = &rhash_iter_seq_info, }; + +size_t bpf_htab_map_meta_size(void) +{ + return sizeof(struct bpf_htab); +} + +void bpf_htab_map_meta_init(struct bpf_map inner_map_meta, struct bpf_map inner_map) +{ + struct bpf_htab inner_htab_meta = container_of(inner_map_meta, struct bpf_htab, map); + struct bpf_htab inner_htab = container_of(inner_map, struct bpf_htab, map); + + inner_htab_meta->key_offset = inner_htab->key_offset; +} diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index d2cbab4bdf644..b2db5d0c98bee 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -7,6 +7,15 @@ #include "map_in_map.h" +static bool bpf_map_type_is_htab(enum bpf_map_type type) +{ + return type == BPF_MAP_TYPE_HASH \|\| + type == BPF_MAP_TYPE_PERCPU_HASH \|\| + type == BPF_MAP_TYPE_LRU_HASH \|\| + type == BPF_MAP_TYPE_LRU_PERCPU_HASH \|\| + type == BPF_MAP_TYPE_HASH_OF_MAPS; +} + struct bpf_map bpf_map_meta_alloc(int inner_map_ufd) { struct bpf_map inner_map, inner_map_meta; @@ -29,6 +38,8 @@ struct bpf_map bpf_map_meta_alloc(int inner_map_ufd) / In some cases verifier needs to access beyond just base map. / if (inner_map->ops == &array_map_ops \|\| inner_map->ops == &percpu_array_map_ops) inner_map_meta_size = sizeof(struct bpf_array); + else if (bpf_map_type_is_htab(inner_map->map_type)) + inner_map_meta_size = bpf_htab_map_meta_size(); inner_map_meta = kzalloc(inner_map_meta_size, GFP_USER); if (!inner_map_meta) @@ -70,6 +81,8 @@ struct bpf_map bpf_map_meta_alloc(int inner_map_ufd) inner_array_meta->index_mask = inner_array->index_mask; inner_array_meta->elem_size = inner_array->elem_size; inner_map_meta->bypass_spec_v1 = inner_map->bypass_spec_v1; + } else if (bpf_map_type_is_htab(inner_map->map_type)) { + bpf_htab_map_meta_init(inner_map_meta, inner_map); } return inner_map_meta; } diff --git a/kernel/bpf/map_in_map.h b/kernel/bpf/map_in_map.h index 7d61602354de8..1edc205772d8c 100644 --- a/kernel/bpf/map_in_map.h +++ b/kernel/bpf/map_in_map.h @@ -15,5 +15,7 @@ void bpf_map_fd_get_ptr(struct bpf_map map, struct file map_file, int ufd); void bpf_map_fd_put_ptr(struct bpf_map map, void ptr, bool need_defer); u32 bpf_map_fd_sys_lookup_elem(void ptr); +size_t bpf_htab_map_meta_size(void); +void bpf_htab_map_meta_init(struct bpf_map inner_map_meta, struct bpf_map inner_map); #endif diff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c index 373c8d17ea557..6bd4cb68c20c1 100644 --- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c +++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c @@ -114,7 +114,7 @@ static inline int check_hash(void) VERIFY(check_default_noinline(&hash->map, map)); VERIFY(hash->n_buckets == MAX_ENTRIES); - VERIFY(hash->elem_size == 64); + VERIFY(hash->elem_size == 32); VERIFY(hash->count.counter == 0); VERIFY(bpf_map_sum_elem_count(map) == 0);
3/2	2026/08/05 22:52	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 22044 output: 55 thoughts: 715 Thoughts: Analyzing the BPF Hash Map Optimization Patch Okay, so I'm looking at this patch and my initial thought is, this is a pretty significant change. We're talking about a fundamental re-architecture of `struct htab_elem` in `kernel/bpf/hashtab.c` and also impacting `kernel/bpf/map_in_map.c`. They're introducing new structures: `htab_node`, `htab_elem`, `htab_elem_lru`, `htab_elem_pcpu`. That's not trivial. And, oh yeah, `has_hash` and `key_offset` are getting added to `struct bpf_htab` as well. The biggest thing that catches my attention is how they're conditionally including the hash value in the element depending on if it's an LRU map or if the key is wider than a word. That's a clever optimization, but it also creates potential for some really subtle bugs. Also, updating `map_in_map.c` to support meta-data for hash maps inside other maps... that impacts another core component. This absolutely screams for fuzzing. We need to be thorough with this. Now, let's figure out where to focus our fuzzing efforts. The patch modifies: `htab_map_alloc`, `htab_map_update_elem`, `htab_lru_map_update_elem`, `htab_map_delete_elem`, `htab_lru_map_delete_elem`, `htab_map_lookup_elem`, `htab_lru_map_lookup_elem`, `htab_map_get_next_key`, `htab_map_lookup_and_delete_elem`, `htab_map_lookup_and_delete_batch` and `bpf_map_meta_alloc`. Right, let's look at those functions, especially. Alright, checking `kernel/bpf/hashtab.c`. Those `htab_` functions are all standard BPF map operations. We can focus there. Given what I'm seeing, `htab_map_alloc`, `htab_map_update_elem`, `htab_lru_map_update_elem`, `htab_map_delete_elem`, `htab_lru_map_delete_elem`, `htab_map_lookup_elem`, `htab_lru_map_lookup_elem`, `__htab_map_lookup_and_delete_batch`, `bpf_map_meta_alloc` - those seem like the primary entry points to scrutinize. Okay, so most of them are static, but assigned to the `bpf_map_ops` structures. Ah, right. The advice is to prioritize non-static entry points. But even though they're static, they aren't* inlined because they're called through the function pointers in those `bpf_map_ops` structures. That means they still have distinct symbol addresses, and they should be testable. Great. Let's dig deeper into the code.
4/2	2026/08/05 22:52	tool	grepper	0m Args: { "Expression": "^(static )?(struct bpf_map \\\|int\|long\|void \\\|void) htab_map_alloc", "PathPrefix": "kernel/bpf/hashtab.c" } Results: { "Output": "kernel/bpf/hashtab.c=450=static int alloc_extra_elems(struct bpf_htab htab)\n--\nkernel/bpf/hashtab.c-473-/ Called from syscall /\nkernel/bpf/hashtab.c:474:static int htab_map_alloc_check(union bpf_attr attr)\nkernel/bpf/hashtab.c-475-{\n" }
5/2	2026/08/05 22:52	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 23615 output: 35 thoughts: 23 Thoughts: Okay, here's my interpretation of that thought process, tailored for an expert audience: Refining the Search Strategy Alright, that regex I initially crafted was proving a little too restrictive. It was probably overcomplicating things, leading to some false negatives. Rather than trying to maintain that complex pattern, I'm going to dial it back and adopt a simpler approach. My instincts are telling me to go directly for the core function I'm interested in: `htab_map_alloc`. A simple `grep` for that should be more efficient and get me the results I need without unnecessary overhead or the potential for missed occurrences due to overly specific pattern matching. Focusing on the function name itself should provide a cleaner and more focused result set for deeper analysis. It's a pragmatic shift, and I expect it will be more effective.
6/2	2026/08/05 22:52	tool	grepper	0m Args: { "Expression": "htab_map_alloc", "PathPrefix": "kernel/bpf/hashtab.c" } Results: { "Output": "kernel/bpf/hashtab.c=450=static int alloc_extra_elems(struct bpf_htab htab)\n--\nkernel/bpf/hashtab.c-473-/ Called from syscall /\nkernel/bpf/hashtab.c:474:static int htab_map_alloc_check(union bpf_attr attr)\nkernel/bpf/hashtab.c-475-{\n--\nkernel/bpf/hashtab.c=591=static int htab_map_check_btf(struct bpf_map map, const struct btf btf,\n--\nkernel/bpf/hashtab.c-599-\t * We must set the dtor using this callback, as map's BTF record is not\nkernel/bpf/hashtab.c:600:\t * populated in htab_map_alloc(), so it will always appear as NULL.\nkernel/bpf/hashtab.c-601-\t /\n--\nkernel/bpf/hashtab.c-607-\nkernel/bpf/hashtab.c:608:static struct bpf_map htab_map_alloc(union bpf_attr attr)\nkernel/bpf/hashtab.c-609-{\n--\nkernel/bpf/hashtab.c=2477=const struct bpf_map_ops htab_map_ops = {\nkernel/bpf/hashtab.c-2478-\t.map_meta_equal = bpf_map_meta_equal,\nkernel/bpf/hashtab.c:2479:\t.map_alloc_check = htab_map_alloc_check,\nkernel/bpf/hashtab.c:2480:\t.map_alloc = htab_map_alloc,\nkernel/bpf/hashtab.c-2481-\t.map_free = htab_map_free,\n--\nkernel/bpf/hashtab.c=2499=const struct bpf_map_ops htab_lru_map_ops = {\nkernel/bpf/hashtab.c-2500-\t.map_meta_equal = bpf_map_meta_equal,\nkernel/bpf/hashtab.c:2501:\t.map_alloc_check = htab_map_alloc_check,\nkernel/bpf/hashtab.c:2502:\t.map_alloc = htab_map_alloc,\nkernel/bpf/hashtab.c-2503-\t.map_free = htab_map_free,\n--\nkernel/bpf/hashtab.c=2687=const struct bpf_map_ops htab_percpu_map_ops = {\nkernel/bpf/hashtab.c-2688-\t.map_meta_equal = bpf_map_meta_equal,\nkernel/bpf/hashtab.c:2689:\t.map_alloc_check = htab_map_alloc_check,\nkernel/bpf/hashtab.c:2690:\t.map_alloc = htab_map_alloc,\nkernel/bpf/hashtab.c-2691-\t.map_free = htab_map_free,\n--\nkernel/bpf/hashtab.c=2709=const struct bpf_map_ops htab_lru_percpu_map_ops = {\nkernel/bpf/hashtab.c-2710-\t.map_meta_equal = bpf_map_meta_equal,\nkernel/bpf/hashtab.c:2711:\t.map_alloc_check = htab_map_alloc_check,\nkernel/bpf/hashtab.c:2712:\t.map_alloc = htab_map_alloc,\nkernel/bpf/hashtab.c-2713-\t.map_free = htab_map_free,\n--\nkernel/bpf/hashtab.c-2729-\nkernel/bpf/hashtab.c:2730:static int fd_htab_map_alloc_check(union bpf_attr attr)\nkernel/bpf/hashtab.c-2731-{\n--\nkernel/bpf/hashtab.c-2733-\t\treturn -EINVAL;\nkernel/bpf/hashtab.c:2734:\treturn htab_map_alloc_check(attr);\nkernel/bpf/hashtab.c-2735-}\n--\nkernel/bpf/hashtab.c=2802=static struct bpf_map htab_of_map_alloc(union bpf_attr attr)\n--\nkernel/bpf/hashtab.c-2809-\nkernel/bpf/hashtab.c:2810:\tmap = htab_map_alloc(attr);\nkernel/bpf/hashtab.c-2811-\tif (IS_ERR(map)) {\n--\nkernel/bpf/hashtab.c=2856=const struct bpf_map_ops htab_of_maps_map_ops = {\nkernel/bpf/hashtab.c:2857:\t.map_alloc_check = fd_htab_map_alloc_check,\nkernel/bpf/hashtab.c-2858-\t.map_alloc = htab_of_map_alloc,\n--\nkernel/bpf/hashtab.c=2914=static const struct rhashtable_params rhtab_params_long = {\n--\nkernel/bpf/hashtab.c-2921-\nkernel/bpf/hashtab.c:2922:static struct bpf_map rhtab_map_alloc(union bpf_attr attr)\nkernel/bpf/hashtab.c-2923-{\n--\nkernel/bpf/hashtab.c-2971-\nkernel/bpf/hashtab.c:2972:static int rhtab_map_alloc_check(union bpf_attr *attr)\nkernel/bpf/hashtab.c-2973-{\n--\nkernel/bpf/hashtab.c-2991-\nkernel/bpf/hashtab.c:2992:\treturn htab_map_alloc_check(attr);\nkernel/bpf/hashtab.c-2993-}\n--\nkernel/bpf/hashtab.c=3642=const struct bpf_map_ops rhtab_map_ops = {\nkernel/bpf/hashtab.c-3643-\t.map_meta_equal = bpf_map_meta_equal,\nkernel/bpf/hashtab.c:3644:\t.map_alloc_check = rhtab_map_alloc_check,\nkernel/bpf/hashtab.c:3645:\t.map_alloc = rhtab_map_alloc,\nkernel/bpf/hashtab.c-3646-\t.map_free = rhtab_map_free,\n" }
7/2	2026/08/05 22:52	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 25572 output: 40 thoughts: 165 Thoughts: My Assessment of the Hash Map Functions Okay, so I'm looking at these functions and immediately see they're static, which is interesting, but actually, that doesn't really matter. These are the core operations – the mapping functions – for hash maps. That makes them a very valuable target for the fuzzer. Let me see... if we include `htab_map_alloc`, `htab_map_update_elem`, `htab_lru_map_update_elem`, `htab_map_delete_elem`, `htab_lru_map_delete_elem`, `htab_map_lookup_elem`, `htab_lru_map_lookup_elem`, `__htab_map_lookup_and_delete_batch`, and `bpf_map_meta_alloc`, the fuzzer should be able to dig in there. Ah, wait a second. Hold on. Is `bpf_map_meta_alloc` actually static? That's not what I expected. Let me just quickly check that. Time to go look into `kernel/bpf/map_in_map.c`.
8/2	2026/08/05 22:52	tool	grepper	0m Args: { "Expression": "bpf_map_meta_alloc", "PathPrefix": "kernel/bpf/map_in_map.c" } Results: { "Output": "kernel/bpf/map_in_map.c=10=static bool bpf_map_type_is_htab(enum bpf_map_type type)\n--\nkernel/bpf/map_in_map.c-18-\nkernel/bpf/map_in_map.c:19:struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)\nkernel/bpf/map_in_map.c-20-{\n" }
9/2	2026/08/05 22:52	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 26154 output: 31 thoughts: 337 Thoughts: Analyzing BPF Map Allocation Functions Okay, so `bpf_map_meta_alloc` is definitely exported, that's straightforward. Now, about `htab_map_alloc`... it's declared static, which initially seems like a problem based on the prompt's preference for non-static entrypoints with distinct symbol addresses. But wait a minute, it's assigned to a function pointer! That means the compiler won't inline it, and it does have a distinct symbol. So, it should be fine to include it. Of course, I could play it safe and just list the syscall entry points like `map_create`, `map_update_elem`, etc. But that's a bit generic. The prompt specifically warns against generic hot-path functions to avoid biasing the results. And listing those syscalls would skew the test distribution. So, the optimal approach here is to list the specific `htab_map_alloc` functions: `htab_map_alloc`, `htab_map_update_elem`, `htab_lru_map_update_elem`, `htab_map_delete_elem`, `htab_lru_map_delete_elem`, `htab_map_lookup_elem`, `htab_lru_map_lookup_elem`, and `__htab_map_lookup_and_delete_batch`. That covers the core functionality without being overly generic. Now, just to be thorough, I should check for any CONFIG options we might need. BPF hash maps are a fundamental part of core BPF, so it's a safe bet that `CONFIG_BPF_SYSCALL` is the primary one, but let me just double check the details to be sure.
10/2	2026/08/05 22:52	tool	grepper	0m Error: no matches Args: { "Expression": "config BPF_SYSCALL", "PathPrefix": "init/Kconfig" } Results: { "error": "no matches" }
11/2	2026/08/05 22:52	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 26916 output: 27 thoughts: 16 Thoughts: Okay, let's dive into this. Here's my thought process on this: Investigating BPF Configuration Options Alright, I need to figure out where the configuration options for BPF are defined. My first instinct, and this is where I'd start in any kernel investigation, is to check the `Kconfig` files. Specifically, I'm thinking I need to look in `kernel/bpf/Kconfig`, or somewhere similar within the BPF subsystem directory. The `Kconfig` files are the bread and butter for any kernel developer because they define the configuration options that users can toggle using `make menuconfig` or similar tools. This should give me a good understanding of what's available and how it's structured.
12/2	2026/08/05 22:52	tool	grepper	0m Args: { "Expression": "config BPF_SYSCALL", "PathPrefix": "" } Results: { "Output": "kernel/bpf/Kconfig=25=menu \"BPF subsystem\"\nkernel/bpf/Kconfig-26-\nkernel/bpf/Kconfig:27:config BPF_SYSCALL\nkernel/bpf/Kconfig-28-\tbool \"Enable bpf() system call\"\n" }
13/2	2026/08/05 22:52	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 27199 output: 30 thoughts: 66 Thoughts: Okay, let's see, the core of this seems to revolve around `BPF_SYSCALL`, that's the primary configuration to be aware of. I should double-check to be sure there aren't any other configs that are related. Alright, the patch touches `kernel/bpf/hashtab.c` and `kernel/bpf/map_in_map.c`. Hmm, are there any specialized configurations, or perhaps dependencies that are unique to those files within this context? We need to determine if there are any specific configurations linked with the specific code paths altered in those modules.
14/2	2026/08/05 22:52	tool	grepper	0m Args: { "Expression": "hashtab\\.o", "PathPrefix": "kernel/bpf/Makefile" } Results: { "Output": "kernel/bpf/Makefile=10=obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o\nkernel/bpf/Makefile:11:obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o\nkernel/bpf/Makefile-12-obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o\n" }
15/2	2026/08/05 22:52	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 27601 output: 178 thoughts: 648 Thoughts: My Analysis of this BPF Hash Map Optimization Right, so I've confirmed that enabling `CONFIG_BPF_SYSCALL` is the crucial prerequisite here. Time to get to the heart of this patch. My immediate thought is to focus on a set of critical functions that represent the core operations affected by this memory layout optimization. I've compiled a shortlist, focusing on those involved with allocation, updates, lookups, and deletions. I've got `htab_map_alloc`, `htab_map_update_elem`, `htab_lru_map_update_elem`, `htab_map_delete_elem`, `htab_map_lookup_elem`, and `bpf_map_meta_alloc` earmarked as the key targets. I did a quick double check on `htab_map_update_elem` to be sure I'm targeting the proper function, it's static internally but exposed through the `htab_map_ops` structure. I'm keeping a close eye on any aliases or variations as well, but these seem to be the primary players. This patch is essentially a functional change to the core BPF hash map implementation, specifically aiming to optimize memory usage by conditionally removing the hash value from the element and adjusting the key offset. It's not just a tweak; it's a fundamental shift in how these maps manage memory. Given this, my expert intuition screams for thorough fuzzing. The change touches core operations like memory layout in a way that demands robust testing. We need to be absolutely certain that this memory optimization doesn't introduce vulnerabilities or unexpected behaviors during operations like allocation, lookup, update, and deletion, especially related to out-of-bounds access or memory corruption. I need to be 100% certain the keys and data are still mapped correctly, too. I'm flagging `htab_map_alloc`, `htab_map_update_elem`, `htab_lru_map_update_elem`, `htab_map_lookup_elem`, `htab_map_delete_elem`, and `bpf_map_meta_alloc` as the focus symbols, and I'll make sure `BPF_SYSCALL` is enabled for testing. Let's call `set-results`.
16/2	2026/08/05 22:53	tool	set-results	0m Args: { "EnableConfigs": [ "BPF_SYSCALL" ], "FocusSymbols": [ "htab_map_alloc", "htab_map_update_elem", "htab_lru_map_update_elem", "htab_map_lookup_elem", "htab_map_delete_elem", "bpf_map_meta_alloc" ], "Reasoning": "The patch optimizes the memory layout of BPF hash map elements by conditionally omitting the hash value and adjusting the key offset. This affects core BPF hash map operations like allocation, lookup, update, and deletion. Fuzzing is necessary to ensure these operations still work correctly with the new memory layout and that no out-of-bounds accesses or memory leaks occur.", "WorthFuzzing": true } Results: { "EnableConfigs": [ "BPF_SYSCALL" ], "FocusSymbols": [ "htab_map_alloc", "htab_map_update_elem", "htab_lru_map_update_elem", "htab_map_lookup_elem", "htab_map_delete_elem", "bpf_map_meta_alloc" ], "Reasoning": "The patch optimizes the memory layout of BPF hash map elements by conditionally omitting the hash value and adjusting the key offset. This affects core BPF hash map operations like allocation, lookup, update, and deletion. Fuzzing is necessary to ensure these operations still work correctly with the new memory layout and that no out-of-bounds accesses or memory leaks occur.", "WorthFuzzing": true }

{
  "PatchDiff": "commit 6454a11c03dfaedb4b09f8575af8d03639fb5273\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate:   Wed Aug 5 22:52:22 2026 +0000\n\n    syz-cluster: applied patch under review\n\ndiff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c\nindex 9f394e1aa2e85..9967268d453d0 100644\n--- a/kernel/bpf/hashtab.c\n+++ b/kernel/bpf/hashtab.c\n@@ -100,13 +100,16 @@ struct bpf_htab {\n \tstruct percpu_counter pcount;\n \tatomic_t count;\n \tbool use_percpu_counter;\n+\tbool has_hash;\n \tu32 n_buckets;\t/* number of hash buckets */\n \tu32 elem_size;\t/* size of each element in bytes */\n+\tu32 key_offset;\t/* offset of key in bytes */\n \tu32 hashrnd;\n };\n \n /* each htab element is struct htab_elem + key + value */\n-struct htab_elem {\n+struct htab_elem;\n+struct htab_node {\n \tunion {\n \t\tstruct hlist_nulls_node hash_node;\n \t\tstruct {\n@@ -117,18 +120,32 @@ struct htab_elem {\n \t\t\t};\n \t\t};\n \t};\n-\tunion {\n-\t\t/* pointer to per-cpu pointer */\n-\t\tvoid *ptr_to_pptr;\n-\t\tstruct bpf_lru_node lru_node;\n-\t};\n-\tu32 hash;\n-\tchar key[] __aligned(8);\n+};\n+\n+struct htab_elem {\n+\tstruct htab_node node;\n+\tu8 data[] __aligned(8);\n+};\n+\n+struct htab_elem_lru {\n+\tstruct htab_node node;\n+\tstruct bpf_lru_node lru_node;\n+\tu8 data[] __aligned(8);\n+};\n+\n+/* Only for non-preallocated PCPU maps. Preallocated PCPU maps don't need\n+ * ptr_to_pptr, and use htab_elem.\n+ */\n+struct htab_elem_pcpu {\n+\tstruct htab_node node;\n+\tvoid *ptr_to_pptr;\n+\tu8 data[] __aligned(8);\n };\n \n struct htab_btf_record {\n \tstruct btf_record *record;\n \tu32 key_size;\n+\tu32 key_offset;\n };\n \n static inline bool htab_is_prealloc(const struct bpf_htab *htab)\n@@ -136,6 +153,21 @@ static inline bool htab_is_prealloc(const struct bpf_htab *htab)\n \treturn !(htab-\u003emap.map_flags \u0026 BPF_F_NO_PREALLOC);\n }\n \n+static inline struct bpf_lru_node *htab_elem_lru_node(struct htab_elem *l)\n+{\n+\treturn \u0026((struct htab_elem_lru *)l)-\u003elru_node;\n+}\n+\n+static inline void *htab_elem_get_ptr_to_pptr(struct htab_elem *l)\n+{\n+\treturn ((struct htab_elem_pcpu *)l)-\u003eptr_to_pptr;\n+}\n+\n+static inline void htab_elem_set_ptr_to_pptr(struct htab_elem *l, void *ptr)\n+{\n+\t((struct htab_elem_pcpu *)l)-\u003eptr_to_pptr = ptr;\n+}\n+\n static void htab_init_buckets(struct bpf_htab *htab)\n {\n \tunsigned int i;\n@@ -183,25 +215,30 @@ static inline bool is_fd_htab(const struct bpf_htab *htab)\n \treturn htab-\u003emap.map_type == BPF_MAP_TYPE_HASH_OF_MAPS;\n }\n \n-static inline void *htab_elem_value(struct htab_elem *l, u32 key_size)\n+static inline void *htab_elem_key(struct bpf_htab *htab, struct htab_elem *l)\n+{\n+\treturn (void *)l + htab-\u003ekey_offset;\n+}\n+\n+static inline void *htab_elem_value(struct bpf_htab *htab, struct htab_elem *l)\n {\n-\treturn l-\u003ekey + round_up(key_size, 8);\n+\treturn htab_elem_key(htab, l) + round_up(htab-\u003emap.key_size, 8);\n }\n \n-static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,\n+static inline void htab_elem_set_ptr(struct bpf_htab *htab, struct htab_elem *l,\n \t\t\t\t     void __percpu *pptr)\n {\n-\t*(void __percpu **)htab_elem_value(l, key_size) = pptr;\n+\t*(void __percpu **)htab_elem_value(htab, l) = pptr;\n }\n \n-static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)\n+static inline void __percpu *htab_elem_get_ptr(struct bpf_htab *htab, struct htab_elem *l)\n {\n-\treturn *(void __percpu **)htab_elem_value(l, key_size);\n+\treturn *(void __percpu **)htab_elem_value(htab, l);\n }\n \n-static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)\n+static void *fd_htab_map_get_ptr(struct bpf_htab *htab, struct htab_elem *l)\n {\n-\treturn *(void **)htab_elem_value(l, map-\u003ekey_size);\n+\treturn *(void **)htab_elem_value(htab, l);\n }\n \n static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)\n@@ -209,6 +246,33 @@ static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)\n \treturn (struct htab_elem *) (htab-\u003eelems + i * (u64)htab-\u003eelem_size);\n }\n \n+static inline bool htab_has_hash(const struct bpf_htab *htab)\n+{\n+\treturn htab-\u003ehas_hash;\n+}\n+\n+static inline u32 htab_elem_hash(struct bpf_htab *htab, struct htab_elem *l)\n+{\n+\tif (htab_is_lru(htab))\n+\t\treturn *(u32 *)((struct htab_elem_lru *)l)-\u003edata;\n+\telse if (htab_is_percpu(htab) \u0026\u0026 !htab_is_prealloc(htab))\n+\t\treturn *(u32 *)((struct htab_elem_pcpu *)l)-\u003edata;\n+\telse\n+\t\treturn *(u32 *)l-\u003edata;\n+}\n+\n+static inline void htab_elem_set_hash(struct bpf_htab *htab, struct htab_elem *l, u32 hash)\n+{\n+\tif (!htab_has_hash(htab))\n+\t\treturn;\n+\tif (htab_is_lru(htab))\n+\t\t*(u32 *)((struct htab_elem_lru *)l)-\u003edata = hash;\n+\telse if (htab_is_percpu(htab) \u0026\u0026 !htab_is_prealloc(htab))\n+\t\t*(u32 *)((struct htab_elem_pcpu *)l)-\u003edata = hash;\n+\telse\n+\t\t*(u32 *)l-\u003edata = hash;\n+}\n+\n /* Both percpu and fd htab support in-place update, so no need for\n  * extra elem. LRU itself can remove the least used element, so\n  * there is no need for an extra elem during map_update.\n@@ -231,7 +295,7 @@ static void htab_free_prealloced_internal_structs(struct bpf_htab *htab)\n \n \t\telem = get_htab_elem(htab, i);\n \t\tbpf_map_free_internal_structs(\u0026htab-\u003emap,\n-\t\t\t\t\t      htab_elem_value(elem, htab-\u003emap.key_size));\n+\t\t\t\t\t      htab_elem_value(htab, elem));\n \t\tcond_resched();\n \t}\n }\n@@ -254,7 +318,7 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab)\n \n \t\telem = get_htab_elem(htab, i);\n \t\tif (htab_is_percpu(htab)) {\n-\t\t\tvoid __percpu *pptr = htab_elem_get_ptr(elem, htab-\u003emap.key_size);\n+\t\t\tvoid __percpu *pptr = htab_elem_get_ptr(htab, elem);\n \t\t\tint cpu;\n \n \t\t\tfor_each_possible_cpu(cpu) {\n@@ -263,7 +327,7 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab)\n \t\t\t}\n \t\t} else {\n \t\t\tbpf_obj_free_fields(htab-\u003emap.record,\n-\t\t\t\t\t    htab_elem_value(elem, htab-\u003emap.key_size));\n+\t\t\t\t\t    htab_elem_value(htab, elem));\n \t\t\tcond_resched();\n \t\t}\n \t\tcond_resched();\n@@ -280,8 +344,7 @@ static void htab_free_elems(struct bpf_htab *htab)\n \tfor (i = 0; i \u003c htab-\u003emap.max_entries; i++) {\n \t\tvoid __percpu *pptr;\n \n-\t\tpptr = htab_elem_get_ptr(get_htab_elem(htab, i),\n-\t\t\t\t\t htab-\u003emap.key_size);\n+\t\tpptr = htab_elem_get_ptr(htab, get_htab_elem(htab, i));\n \t\tfree_percpu(pptr);\n \t\tcond_resched();\n \t}\n@@ -308,8 +371,8 @@ static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key,\n \n \tif (node) {\n \t\tbpf_map_inc_elem_count(\u0026htab-\u003emap);\n-\t\tl = container_of(node, struct htab_elem, lru_node);\n-\t\tmemcpy(l-\u003ekey, key, htab-\u003emap.key_size);\n+\t\tl = (struct htab_elem *)container_of(node, struct htab_elem_lru, lru_node);\n+\t\tmemcpy(htab_elem_key(htab, l), key, htab-\u003emap.key_size);\n \t\treturn l;\n \t}\n \n@@ -340,8 +403,7 @@ static int prealloc_init(struct bpf_htab *htab)\n \t\t\t\t\t    GFP_USER | __GFP_NOWARN);\n \t\tif (!pptr)\n \t\t\tgoto free_elems;\n-\t\thtab_elem_set_ptr(get_htab_elem(htab, i), htab-\u003emap.key_size,\n-\t\t\t\t  pptr);\n+\t\thtab_elem_set_ptr(htab, get_htab_elem(htab, i), pptr);\n \t\tcond_resched();\n \t}\n \n@@ -349,8 +411,8 @@ static int prealloc_init(struct bpf_htab *htab)\n \tif (htab_is_lru(htab))\n \t\terr = bpf_lru_init(\u0026htab-\u003elru,\n \t\t\t\t   htab-\u003emap.map_flags \u0026 BPF_F_NO_COMMON_LRU,\n-\t\t\t\t   offsetof(struct htab_elem, hash) -\n-\t\t\t\t   offsetof(struct htab_elem, lru_node),\n+\t\t\t\t   offsetof(struct htab_elem_lru, data) -\n+\t\t\t\t   offsetof(struct htab_elem_lru, lru_node),\n \t\t\t\t   htab_lru_map_delete_node,\n \t\t\t\t   htab);\n \telse\n@@ -361,11 +423,11 @@ static int prealloc_init(struct bpf_htab *htab)\n \n \tif (htab_is_lru(htab))\n \t\tbpf_lru_populate(\u0026htab-\u003elru, htab-\u003eelems,\n-\t\t\t\t offsetof(struct htab_elem, lru_node),\n+\t\t\t\t offsetof(struct htab_elem_lru, lru_node),\n \t\t\t\t htab-\u003eelem_size, num_entries);\n \telse\n \t\tpcpu_freelist_populate(\u0026htab-\u003efreelist,\n-\t\t\t\t       htab-\u003eelems + offsetof(struct htab_elem, fnode),\n+\t\t\t\t       htab-\u003eelems + offsetof(struct htab_elem, node.fnode),\n \t\t\t\t       htab-\u003eelem_size, num_entries);\n \n \treturn 0;\n@@ -401,7 +463,7 @@ static int alloc_extra_elems(struct bpf_htab *htab)\n \t\t/* pop will succeed, since prealloc_init()\n \t\t * preallocated extra num_possible_cpus elements\n \t\t */\n-\t\tl_new = container_of(l, struct htab_elem, fnode);\n+\t\tl_new = container_of(l, struct htab_elem, node.fnode);\n \t\t*per_cpu_ptr(pptr, cpu) = l_new;\n \t}\n \thtab-\u003eextra_elems = pptr;\n@@ -425,8 +487,8 @@ static int htab_map_alloc_check(union bpf_attr *attr)\n \tbool zero_seed = (attr-\u003emap_flags \u0026 BPF_F_ZERO_SEED);\n \tint numa_node = bpf_map_attr_numa_node(attr);\n \n-\tBUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=\n-\t\t     offsetof(struct htab_elem, hash_node.pprev));\n+\tBUILD_BUG_ON(offsetof(struct htab_node, fnode.next) !=\n+\t\t     offsetof(struct htab_node, hash_node.pprev));\n \n \tif (zero_seed \u0026\u0026 !capable(CAP_SYS_ADMIN))\n \t\t/* Guard against local DoS, and discourage production use. */\n@@ -476,7 +538,7 @@ static void htab_mem_dtor(void *obj, void *ctx)\n \tif (IS_ERR_OR_NULL(hrec-\u003erecord))\n \t\treturn;\n \n-\tmap_value = htab_elem_value(elem, hrec-\u003ekey_size);\n+\tmap_value = (void *)elem + hrec-\u003ekey_offset + round_up(hrec-\u003ekey_size, 8);\n \tbpf_obj_free_fields(hrec-\u003erecord, map_value);\n }\n \n@@ -502,7 +564,7 @@ static void htab_dtor_ctx_free(void *ctx)\n }\n \n static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,\n-\t\t\t   void (*dtor)(void *, void *))\n+\t\t\t   void (*dtor)(void *, void *), u32 key_offset)\n {\n \tstruct htab_btf_record *hrec;\n \tint err;\n@@ -515,6 +577,7 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,\n \tif (!hrec)\n \t\treturn -ENOMEM;\n \threc-\u003ekey_size = map-\u003ekey_size;\n+\threc-\u003ekey_offset = key_offset;\n \threc-\u003erecord = btf_record_dup(map-\u003erecord);\n \tif (IS_ERR(hrec-\u003erecord)) {\n \t\terr = PTR_ERR(hrec-\u003erecord);\n@@ -537,9 +600,9 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf,\n \t * populated in htab_map_alloc(), so it will always appear as NULL.\n \t */\n \tif (htab_is_percpu(htab))\n-\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003epcpu_ma, htab_pcpu_mem_dtor);\n+\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003epcpu_ma, htab_pcpu_mem_dtor, htab-\u003ekey_offset);\n \telse\n-\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003ema, htab_mem_dtor);\n+\t\treturn bpf_ma_set_dtor(map, \u0026htab-\u003ema, htab_mem_dtor, htab-\u003ekey_offset);\n }\n \n static struct bpf_map *htab_map_alloc(union bpf_attr *attr)\n@@ -562,6 +625,13 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)\n \n \tbpf_map_init_from_attr(\u0026htab-\u003emap, attr);\n \n+\t/* Avoid hash memory use and comparisons where unnecessary.\n+\t * u32 hash reads are always atomic. If we elide them, key comparisons must also be atomic\n+\t * to avoid false positive key matches due to torn key reads / writes. This is only possible\n+\t * when the key fits within a word, so check key_size.\n+\t */\n+\thtab-\u003ehas_hash = htab_is_lru(htab) || htab-\u003emap.key_size \u003e sizeof(unsigned long);\n+\n \tif (percpu_lru) {\n \t\t/* ensure each CPU's lru list has \u003e=1 elements.\n \t\t * since we are at it, make each lru list has the same\n@@ -583,8 +653,16 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)\n \n \thtab-\u003en_buckets = roundup_pow_of_two(htab-\u003emap.max_entries);\n \n-\thtab-\u003eelem_size = sizeof(struct htab_elem) +\n-\t\t\t  round_up(htab-\u003emap.key_size, 8);\n+\tif (htab_is_lru(htab))\n+\t\thtab-\u003ekey_offset = offsetof(struct htab_elem_lru, data) + 8;\n+\telse if (percpu \u0026\u0026 !prealloc)\n+\t\thtab-\u003ekey_offset = offsetof(struct htab_elem_pcpu, data) +\n+\t\t\t\t   (htab_has_hash(htab) ? 8 : 0);\n+\telse\n+\t\thtab-\u003ekey_offset = offsetof(struct htab_elem, data) +\n+\t\t\t\t   (htab_has_hash(htab) ? 8 : 0);\n+\n+\thtab-\u003eelem_size = htab-\u003ekey_offset + round_up(htab-\u003emap.key_size, 8);\n \tif (percpu)\n \t\thtab-\u003eelem_size += sizeof(void *);\n \telse\n@@ -692,15 +770,29 @@ static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32\n }\n \n /* this lookup function can only be called with bucket lock taken */\n-static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash,\n+static struct htab_elem *lookup_elem_raw(struct bpf_htab *htab,\n+\t\t\t\t\t struct hlist_nulls_head *head, u32 hash,\n \t\t\t\t\t void *key, u32 key_size)\n {\n \tstruct hlist_nulls_node *n;\n \tstruct htab_elem *l;\n \n-\thlist_nulls_for_each_entry_rcu(l, n, head, hash_node)\n-\t\tif (l-\u003ehash == hash \u0026\u0026 !memcmp(\u0026l-\u003ekey, key, key_size))\n-\t\t\treturn l;\n+\tif (htab_has_hash(htab)) {\n+\t\thlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)\n+\t\t\tif (htab_elem_hash(htab, l) == hash \u0026\u0026\n+\t\t\t    !memcmp(htab_elem_key(htab, l), key, key_size))\n+\t\t\t\treturn l;\n+\t} else {\n+\t\t/* When hash is omitted, key comparisons must be atomic. Zero extend\n+\t\t * the caller's key to the word size to support an atomic compare.\n+\t\t */\n+\t\tunsigned long k = 0;\n+\n+\t\tmemcpy(\u0026k, key, key_size);\n+\t\thlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)\n+\t\t\tif (READ_ONCE(*(unsigned long *)htab_elem_key(htab, l)) == k)\n+\t\t\t\treturn l;\n+\t}\n \n \treturn NULL;\n }\n@@ -709,7 +801,8 @@ static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash\n  * the unlikely event when elements moved from one bucket into another\n  * while link list is being walked\n  */\n-static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,\n+static struct htab_elem *lookup_nulls_elem_raw(struct bpf_htab *htab,\n+\t\t\t\t\t       struct hlist_nulls_head *head,\n \t\t\t\t\t       u32 hash, void *key,\n \t\t\t\t\t       u32 key_size, u32 n_buckets)\n {\n@@ -717,9 +810,20 @@ static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,\n \tstruct htab_elem *l;\n \n again:\n-\thlist_nulls_for_each_entry_rcu(l, n, head, hash_node)\n-\t\tif (l-\u003ehash == hash \u0026\u0026 !memcmp(\u0026l-\u003ekey, key, key_size))\n-\t\t\treturn l;\n+\tif (htab_has_hash(htab)) {\n+\t\thlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)\n+\t\t\tif (htab_elem_hash(htab, l) == hash \u0026\u0026\n+\t\t\t    !memcmp(htab_elem_key(htab, l), key, key_size))\n+\t\t\t\treturn l;\n+\t} else {\n+\t\t/* See lookup_elem_raw() comment above. */\n+\t\tunsigned long k = 0;\n+\n+\t\tmemcpy(\u0026k, key, key_size);\n+\t\thlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)\n+\t\t\tif (READ_ONCE(*(unsigned long *)htab_elem_key(htab, l)) == k)\n+\t\t\t\treturn l;\n+\t}\n \n \tif (unlikely(get_nulls_value(n) != (hash \u0026 (n_buckets - 1))))\n \t\tgoto again;\n@@ -747,17 +851,18 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)\n \n \thead = select_bucket(htab, hash);\n \n-\tl = lookup_nulls_elem_raw(head, hash, key, key_size, htab-\u003en_buckets);\n+\tl = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab-\u003en_buckets);\n \n \treturn l;\n }\n \n static void *htab_map_lookup_elem(struct bpf_map *map, void *key)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l = __htab_map_lookup_elem(map, key);\n \n \tif (l)\n-\t\treturn htab_elem_value(l, map-\u003ekey_size);\n+\t\treturn htab_elem_value(htab, l);\n \n \treturn NULL;\n }\n@@ -775,6 +880,7 @@ static void *htab_map_lookup_elem(struct bpf_map *map, void *key)\n  */\n static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn *insn = insn_buf;\n \tconst int ret = BPF_REG_0;\n \n@@ -783,7 +889,7 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)\n \t*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \t*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);\n \t*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,\n-\t\t\t\toffsetof(struct htab_elem, key) +\n+\t\t\t\thtab-\u003ekey_offset +\n \t\t\t\tround_up(map-\u003ekey_size, 8));\n \treturn insn - insn_buf;\n }\n@@ -791,12 +897,13 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)\n static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map,\n \t\t\t\t\t\t\tvoid *key, const bool mark)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l = __htab_map_lookup_elem(map, key);\n \n \tif (l) {\n \t\tif (mark)\n-\t\t\tbpf_lru_node_set_ref(\u0026l-\u003elru_node);\n-\t\treturn htab_elem_value(l, map-\u003ekey_size);\n+\t\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l));\n+\t\treturn htab_elem_value(htab, l);\n \t}\n \n \treturn NULL;\n@@ -815,6 +922,7 @@ static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key)\n static int htab_lru_map_gen_lookup(struct bpf_map *map,\n \t\t\t\t   struct bpf_insn *insn_buf)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn *insn = insn_buf;\n \tconst int ret = BPF_REG_0;\n \tconst int ref_reg = BPF_REG_1;\n@@ -824,15 +932,15 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map,\n \t*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \t*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4);\n \t*insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret,\n-\t\t\t      offsetof(struct htab_elem, lru_node) +\n+\t\t\t      offsetof(struct htab_elem_lru, lru_node) +\n \t\t\t      offsetof(struct bpf_lru_node, ref));\n \t*insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1);\n \t*insn++ = BPF_ST_MEM(BPF_B, ret,\n-\t\t\t     offsetof(struct htab_elem, lru_node) +\n+\t\t\t     offsetof(struct htab_elem_lru, lru_node) +\n \t\t\t     offsetof(struct bpf_lru_node, ref),\n \t\t\t     1);\n \t*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,\n-\t\t\t\toffsetof(struct htab_elem, key) +\n+\t\t\t\thtab-\u003ekey_offset +\n \t\t\t\tround_up(map-\u003ekey_size, 8));\n \treturn insn - insn_buf;\n }\n@@ -844,13 +952,13 @@ static void check_and_cancel_fields(struct bpf_htab *htab,\n \t\treturn;\n \n \tif (htab_is_percpu(htab)) {\n-\t\tvoid __percpu *pptr = htab_elem_get_ptr(elem, htab-\u003emap.key_size);\n+\t\tvoid __percpu *pptr = htab_elem_get_ptr(htab, elem);\n \t\tint cpu;\n \n \t\tfor_each_possible_cpu(cpu)\n \t\t\tbpf_obj_cancel_fields(\u0026htab-\u003emap, per_cpu_ptr(pptr, cpu));\n \t} else {\n-\t\tvoid *map_value = htab_elem_value(elem, htab-\u003emap.key_size);\n+\t\tvoid *map_value = htab_elem_value(htab, elem);\n \n \t\tbpf_obj_cancel_fields(\u0026htab-\u003emap, map_value);\n \t}\n@@ -869,17 +977,17 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)\n \tstruct bucket *b;\n \tint ret;\n \n-\ttgt_l = container_of(node, struct htab_elem, lru_node);\n-\tb = __select_bucket(htab, tgt_l-\u003ehash);\n+\ttgt_l = (struct htab_elem *)container_of(node, struct htab_elem_lru, lru_node);\n+\tb = __select_bucket(htab, htab_elem_hash(htab, tgt_l));\n \thead = \u0026b-\u003ehead;\n \n \tret = htab_lock_bucket(b, \u0026flags);\n \tif (ret)\n \t\treturn false;\n \n-\thlist_nulls_for_each_entry_rcu(l, n, head, hash_node)\n+\thlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)\n \t\tif (l == tgt_l) {\n-\t\t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n+\t\t\thlist_nulls_del_rcu(\u0026l-\u003enode.hash_node);\n \t\t\tbpf_map_dec_elem_count(\u0026htab-\u003emap);\n \t\t\tbreak;\n \t\t}\n@@ -912,18 +1020,19 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)\n \thead = select_bucket(htab, hash);\n \n \t/* lookup the key */\n-\tl = lookup_nulls_elem_raw(head, hash, key, key_size, htab-\u003en_buckets);\n+\tl = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab-\u003en_buckets);\n \n \tif (!l)\n \t\tgoto find_first_elem;\n \n \t/* key was found, get next key in the same bucket */\n-\tnext_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(\u0026l-\u003ehash_node)),\n-\t\t\t\t  struct htab_elem, hash_node);\n+\tnext_l = hlist_nulls_entry_safe(\n+\t\t\trcu_dereference_raw(hlist_nulls_next_rcu(\u0026l-\u003enode.hash_node)),\n+\t\t\tstruct htab_elem, node.hash_node);\n \n \tif (next_l) {\n \t\t/* if next elem in this hash list is non-zero, just return it */\n-\t\tmemcpy(next_key, next_l-\u003ekey, key_size);\n+\t\tmemcpy(next_key, htab_elem_key(htab, next_l), key_size);\n \t\treturn 0;\n \t}\n \n@@ -938,10 +1047,10 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)\n \n \t\t/* pick first element in the bucket */\n \t\tnext_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_first_rcu(head)),\n-\t\t\t\t\t  struct htab_elem, hash_node);\n+\t\t\t\t\t  struct htab_elem, node.hash_node);\n \t\tif (next_l) {\n \t\t\t/* if it's not empty, just return it */\n-\t\t\tmemcpy(next_key, next_l-\u003ekey, key_size);\n+\t\t\tmemcpy(next_key, htab_elem_key(htab, next_l), key_size);\n \t\t\treturn 0;\n \t\t}\n \t}\n@@ -955,7 +1064,7 @@ static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)\n \tcheck_and_cancel_fields(htab, l);\n \n \tif (htab-\u003emap.map_type == BPF_MAP_TYPE_PERCPU_HASH)\n-\t\tbpf_mem_cache_free(\u0026htab-\u003epcpu_ma, l-\u003eptr_to_pptr);\n+\t\tbpf_mem_cache_free(\u0026htab-\u003epcpu_ma, htab_elem_get_ptr_to_pptr(l));\n \tbpf_mem_cache_free(\u0026htab-\u003ema, l);\n }\n \n@@ -965,7 +1074,7 @@ static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)\n \tvoid *ptr;\n \n \tif (map-\u003eops-\u003emap_fd_put_ptr) {\n-\t\tptr = fd_htab_map_get_ptr(map, l);\n+\t\tptr = fd_htab_map_get_ptr(htab, l);\n \t\tmap-\u003eops-\u003emap_fd_put_ptr(map, ptr, true);\n \t}\n }\n@@ -1006,7 +1115,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)\n \tif (htab_is_prealloc(htab)) {\n \t\tbpf_map_dec_elem_count(\u0026htab-\u003emap);\n \t\tcheck_and_cancel_fields(htab, l);\n-\t\tpcpu_freelist_push(\u0026htab-\u003efreelist, \u0026l-\u003efnode);\n+\t\tpcpu_freelist_push(\u0026htab-\u003efreelist, \u0026l-\u003enode.fnode);\n \t} else {\n \t\tdec_elem_count(htab);\n \t\thtab_elem_free(htab, l);\n@@ -1097,7 +1206,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,\n \t\t\tl = __pcpu_freelist_pop(\u0026htab-\u003efreelist);\n \t\t\tif (!l)\n \t\t\t\treturn ERR_PTR(-E2BIG);\n-\t\t\tl_new = container_of(l, struct htab_elem, fnode);\n+\t\t\tl_new = container_of(l, struct htab_elem, node.fnode);\n \t\t\tbpf_map_inc_elem_count(\u0026htab-\u003emap);\n \t\t}\n \t} else {\n@@ -1117,10 +1226,20 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,\n \t\t}\n \t}\n \n-\tmemcpy(l_new-\u003ekey, key, key_size);\n+\tif (htab_has_hash(htab)) {\n+\t\tmemcpy(htab_elem_key(htab, l_new), key, key_size);\n+\t} else {\n+\t\t/* Zero-extend key into k for an atomic write to support\n+\t\t * lockless RCU readers.\n+\t\t */\n+\t\tunsigned long k = 0;\n+\n+\t\tmemcpy(\u0026k, key, key_size);\n+\t\tWRITE_ONCE(*(unsigned long *)htab_elem_key(htab, l_new), k);\n+\t}\n \tif (percpu) {\n \t\tif (prealloc) {\n-\t\t\tpptr = htab_elem_get_ptr(l_new, key_size);\n+\t\t\tpptr = htab_elem_get_ptr(htab, l_new);\n \t\t} else {\n \t\t\t/* alloc_percpu zero-fills */\n \t\t\tvoid *ptr = bpf_mem_cache_alloc(\u0026htab-\u003epcpu_ma);\n@@ -1130,26 +1249,26 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,\n \t\t\t\tl_new = ERR_PTR(-ENOMEM);\n \t\t\t\tgoto dec_count;\n \t\t\t}\n-\t\t\tl_new-\u003eptr_to_pptr = ptr;\n+\t\t\thtab_elem_set_ptr_to_pptr(l_new, ptr);\n \t\t\tpptr = *(void __percpu **)ptr;\n \t\t}\n \n \t\tpcpu_init_value(htab, pptr, value, onallcpus, map_flags);\n \n \t\tif (!prealloc)\n-\t\t\thtab_elem_set_ptr(l_new, key_size, pptr);\n+\t\t\thtab_elem_set_ptr(htab, l_new, pptr);\n \t} else if (fd_htab_map_needs_adjust(htab)) {\n \t\tsize = round_up(size, 8);\n-\t\tmemcpy(htab_elem_value(l_new, key_size), value, size);\n+\t\tmemcpy(htab_elem_value(htab, l_new), value, size);\n \t} else if (map_flags \u0026 BPF_F_LOCK) {\n \t\tcopy_map_value_locked(\u0026htab-\u003emap,\n-\t\t\t\t      htab_elem_value(l_new, key_size),\n+\t\t\t\t      htab_elem_value(htab, l_new),\n \t\t\t\t      value, false);\n \t} else {\n-\t\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(l_new, key_size), value);\n+\t\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(htab, l_new), value);\n \t}\n \n-\tl_new-\u003ehash = hash;\n+\thtab_elem_set_hash(htab, l_new, hash);\n \treturn l_new;\n dec_count:\n \tdec_elem_count(htab);\n@@ -1199,7 +1318,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,\n \t\tif (unlikely(!btf_record_has_field(map-\u003erecord, BPF_SPIN_LOCK)))\n \t\t\treturn -EINVAL;\n \t\t/* find an element without taking the bucket lock */\n-\t\tl_old = lookup_nulls_elem_raw(head, hash, key, key_size,\n+\t\tl_old = lookup_nulls_elem_raw(htab, head, hash, key, key_size,\n \t\t\t\t\t      htab-\u003en_buckets);\n \t\tret = check_flags(htab, l_old, map_flags);\n \t\tif (ret)\n@@ -1207,7 +1326,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,\n \t\tif (l_old) {\n \t\t\t/* grab the element lock and update value in place */\n \t\t\tcopy_map_value_locked(map,\n-\t\t\t\t\t      htab_elem_value(l_old, key_size),\n+\t\t\t\t\t      htab_elem_value(htab, l_old),\n \t\t\t\t\t      value, false);\n \t\t\treturn 0;\n \t\t}\n@@ -1221,7 +1340,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,\n \tif (ret)\n \t\treturn ret;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n@@ -1235,7 +1354,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,\n \t\t * and update element in place\n \t\t */\n \t\tcopy_map_value_locked(map,\n-\t\t\t\t      htab_elem_value(l_old, key_size),\n+\t\t\t\t      htab_elem_value(htab, l_old),\n \t\t\t\t      value, false);\n \t\tret = 0;\n \t\tgoto err;\n@@ -1252,9 +1371,9 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,\n \t/* add new element to the head of the list, so that\n \t * concurrent search will find it before old elem\n \t */\n-\thlist_nulls_add_head_rcu(\u0026l_new-\u003ehash_node, head);\n+\thlist_nulls_add_head_rcu(\u0026l_new-\u003enode.hash_node, head);\n \tif (l_old) {\n-\t\thlist_nulls_del_rcu(\u0026l_old-\u003ehash_node);\n+\t\thlist_nulls_del_rcu(\u0026l_old-\u003enode.hash_node);\n \n \t\t/* l_old has already been stashed in htab-\u003eextra_elems, cancel\n \t\t * its reusable special fields before it is available for reuse.\n@@ -1275,7 +1394,7 @@ static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem)\n {\n \tcheck_and_cancel_fields(htab, elem);\n \tbpf_map_dec_elem_count(\u0026htab-\u003emap);\n-\tbpf_lru_push_free(\u0026htab-\u003elru, \u0026elem-\u003elru_node);\n+\tbpf_lru_push_free(\u0026htab-\u003elru, htab_elem_lru_node(elem));\n }\n \n static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,\n@@ -1310,13 +1429,13 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value\n \tl_new = prealloc_lru_pop(htab, key, hash);\n \tif (!l_new)\n \t\treturn -ENOMEM;\n-\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(l_new, map-\u003ekey_size), value);\n+\tcopy_map_value(\u0026htab-\u003emap, htab_elem_value(htab, l_new), value);\n \n \tret = htab_lock_bucket(b, \u0026flags);\n \tif (ret)\n \t\tgoto err_lock_bucket;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n@@ -1325,10 +1444,10 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value\n \t/* add new element to the head of the list, so that\n \t * concurrent search will find it before old elem\n \t */\n-\thlist_nulls_add_head_rcu(\u0026l_new-\u003ehash_node, head);\n+\thlist_nulls_add_head_rcu(\u0026l_new-\u003enode.hash_node, head);\n \tif (l_old) {\n-\t\tbpf_lru_node_set_ref(\u0026l_new-\u003elru_node);\n-\t\thlist_nulls_del_rcu(\u0026l_old-\u003ehash_node);\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l_new));\n+\t\thlist_nulls_del_rcu(\u0026l_old-\u003enode.hash_node);\n \t}\n \tret = 0;\n \n@@ -1383,7 +1502,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,\n \tif (ret)\n \t\treturn ret;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n@@ -1392,10 +1511,10 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,\n \tif (l_old) {\n \t\t/* Update value in-place */\n \t\tif (percpu) {\n-\t\t\tpcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),\n+\t\t\tpcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old),\n \t\t\t\t\tvalue, onallcpus, map_flags);\n \t\t} else {\n-\t\t\tvoid **inner_map_pptr = htab_elem_value(l_old, key_size);\n+\t\t\tvoid **inner_map_pptr = htab_elem_value(htab, l_old);\n \n \t\t\told_map_ptr = *inner_map_pptr;\n \t\t\tWRITE_ONCE(*inner_map_pptr, *(void **)value);\n@@ -1407,7 +1526,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,\n \t\t\tret = PTR_ERR(l_new);\n \t\t\tgoto err;\n \t\t}\n-\t\thlist_nulls_add_head_rcu(\u0026l_new-\u003ehash_node, head);\n+\t\thlist_nulls_add_head_rcu(\u0026l_new-\u003enode.hash_node, head);\n \t}\n err:\n \thtab_unlock_bucket(b, flags);\n@@ -1456,22 +1575,22 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,\n \tif (ret)\n \t\tgoto err_lock_bucket;\n \n-\tl_old = lookup_elem_raw(head, hash, key, key_size);\n+\tl_old = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tret = check_flags(htab, l_old, map_flags);\n \tif (ret)\n \t\tgoto err;\n \n \tif (l_old) {\n-\t\tbpf_lru_node_set_ref(\u0026l_old-\u003elru_node);\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l_old));\n \n \t\t/* per-cpu hash map can update value in-place */\n-\t\tpcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),\n+\t\tpcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old),\n \t\t\t\tvalue, onallcpus, map_flags);\n \t} else {\n-\t\tpcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size),\n+\t\tpcpu_init_value(htab, htab_elem_get_ptr(htab, l_new),\n \t\t\t\tvalue, onallcpus, map_flags);\n-\t\thlist_nulls_add_head_rcu(\u0026l_new-\u003ehash_node, head);\n+\t\thlist_nulls_add_head_rcu(\u0026l_new-\u003enode.hash_node, head);\n \t\tl_new = NULL;\n \t}\n \tret = 0;\n@@ -1480,7 +1599,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,\n err_lock_bucket:\n \tif (l_new) {\n \t\tbpf_map_dec_elem_count(\u0026htab-\u003emap);\n-\t\tbpf_lru_push_free(\u0026htab-\u003elru, \u0026l_new-\u003elru_node);\n+\t\tbpf_lru_push_free(\u0026htab-\u003elru, htab_elem_lru_node(l_new));\n \t}\n \treturn ret;\n }\n@@ -1521,9 +1640,9 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key)\n \tif (ret)\n \t\treturn ret;\n \n-\tl = lookup_elem_raw(head, hash, key, key_size);\n+\tl = lookup_elem_raw(htab, head, hash, key, key_size);\n \tif (l)\n-\t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n+\t\thlist_nulls_del_rcu(\u0026l-\u003enode.hash_node);\n \telse\n \t\tret = -ENOENT;\n \n@@ -1556,10 +1675,10 @@ static long htab_lru_map_delete_elem(struct bpf_map *map, void *key)\n \tif (ret)\n \t\treturn ret;\n \n-\tl = lookup_elem_raw(head, hash, key, key_size);\n+\tl = lookup_elem_raw(htab, head, hash, key, key_size);\n \n \tif (l)\n-\t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n+\t\thlist_nulls_del_rcu(\u0026l-\u003enode.hash_node);\n \telse\n \t\tret = -ENOENT;\n \n@@ -1581,8 +1700,8 @@ static void delete_all_elements(struct bpf_htab *htab)\n \t\tstruct hlist_nulls_node *n;\n \t\tstruct htab_elem *l;\n \n-\t\thlist_nulls_for_each_entry_safe(l, n, head, hash_node) {\n-\t\t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n+\t\thlist_nulls_for_each_entry_safe(l, n, head, node.hash_node) {\n+\t\t\thlist_nulls_del_rcu(\u0026l-\u003enode.hash_node);\n \t\t\thtab_elem_free(htab, l);\n \t\t}\n \t\tcond_resched();\n@@ -1599,10 +1718,10 @@ static void htab_free_malloced_internal_structs(struct bpf_htab *htab)\n \t\tstruct hlist_nulls_node *n;\n \t\tstruct htab_elem *l;\n \n-\t\thlist_nulls_for_each_entry(l, n, head, hash_node) {\n+\t\thlist_nulls_for_each_entry(l, n, head, node.hash_node) {\n \t\t\t/* We only free internal structs on uref dropping to zero */\n \t\t\tbpf_map_free_internal_structs(\u0026htab-\u003emap,\n-\t\t\t\t\t\t      htab_elem_value(l, htab-\u003emap.key_size));\n+\t\t\t\t\t\t      htab_elem_value(htab, l));\n \t\t}\n \t\tcond_resched_rcu();\n \t}\n@@ -1697,7 +1816,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,\n \tif (ret)\n \t\treturn ret;\n \n-\tl = lookup_elem_raw(head, hash, key, key_size);\n+\tl = lookup_elem_raw(htab, head, hash, key, key_size);\n \tif (!l) {\n \t\tret = -ENOENT;\n \t\tgoto out_unlock;\n@@ -1708,14 +1827,14 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,\n \t\tvoid __percpu *pptr;\n \t\tint off = 0, cpu;\n \n-\t\tpptr = htab_elem_get_ptr(l, key_size);\n+\t\tpptr = htab_elem_get_ptr(htab, l);\n \t\tfor_each_possible_cpu(cpu) {\n \t\t\tcopy_map_value_long(\u0026htab-\u003emap, value + off, per_cpu_ptr(pptr, cpu));\n \t\t\tcheck_and_init_map_value(\u0026htab-\u003emap, value + off);\n \t\t\toff += roundup_value_size;\n \t\t}\n \t} else {\n-\t\tvoid *src = htab_elem_value(l, map-\u003ekey_size);\n+\t\tvoid *src = htab_elem_value(htab, l);\n \n \t\tif (flags \u0026 BPF_F_LOCK)\n \t\t\tcopy_map_value_locked(map, value, src, true);\n@@ -1724,7 +1843,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,\n \t\t/* Zeroing special fields in the temp buffer */\n \t\tcheck_and_init_map_value(map, value);\n \t}\n-\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n+\thlist_nulls_del_rcu(\u0026l-\u003enode.hash_node);\n \n out_unlock:\n \thtab_unlock_bucket(b, bflags);\n@@ -1860,7 +1979,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,\n \t}\n \n \tbucket_cnt = 0;\n-\thlist_nulls_for_each_entry_rcu(l, n, head, hash_node)\n+\thlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)\n \t\tbucket_cnt++;\n \n \tif (bucket_cnt \u0026\u0026 !locked) {\n@@ -1897,14 +2016,14 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,\n \tif (!locked)\n \t\tgoto next_batch;\n \n-\thlist_nulls_for_each_entry_safe(l, n, head, hash_node) {\n-\t\tmemcpy(dst_key, l-\u003ekey, key_size);\n+\thlist_nulls_for_each_entry_safe(l, n, head, node.hash_node) {\n+\t\tmemcpy(dst_key, htab_elem_key(htab, l), key_size);\n \n \t\tif (is_percpu) {\n \t\t\tint off = 0, cpu;\n \t\t\tvoid __percpu *pptr;\n \n-\t\t\tpptr = htab_elem_get_ptr(l, map-\u003ekey_size);\n+\t\t\tpptr = htab_elem_get_ptr(htab, l);\n \t\t\tif (elem_map_flags \u0026 BPF_F_CPU) {\n \t\t\t\tcpu = elem_map_flags \u003e\u003e 32;\n \t\t\t\tcopy_map_value(\u0026htab-\u003emap, dst_val, per_cpu_ptr(pptr, cpu));\n@@ -1918,7 +2037,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,\n \t\t\t\t}\n \t\t\t}\n \t\t} else {\n-\t\t\tvalue = htab_elem_value(l, key_size);\n+\t\t\tvalue = htab_elem_value(htab, l);\n \t\t\tif (is_fd_htab(htab)) {\n \t\t\t\tstruct bpf_map **inner_map = value;\n \n@@ -1936,7 +2055,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,\n \t\t\tcheck_and_init_map_value(map, dst_val);\n \t\t}\n \t\tif (do_delete) {\n-\t\t\thlist_nulls_del_rcu(\u0026l-\u003ehash_node);\n+\t\t\thlist_nulls_del_rcu(\u0026l-\u003enode.hash_node);\n \n \t\t\t/* bpf_lru_push_free() will acquire lru_lock, which\n \t\t\t * may cause deadlock. See comments in function\n@@ -1948,7 +2067,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,\n \t\t\t * lock being held and it violates the lock rule, so\n \t\t\t * invoke free_htab_elem() after unlock as well.\n \t\t\t */\n-\t\t\tl-\u003ebatch_flink = node_to_free;\n+\t\t\tl-\u003enode.batch_flink = node_to_free;\n \t\t\tnode_to_free = l;\n \t\t}\n \t\tdst_key += key_size;\n@@ -1960,7 +2079,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,\n \n \twhile (node_to_free) {\n \t\tl = node_to_free;\n-\t\tnode_to_free = node_to_free-\u003ebatch_flink;\n+\t\tnode_to_free = node_to_free-\u003enode.batch_flink;\n \t\tif (is_lru_map)\n \t\t\thtab_lru_push_free(htab, l);\n \t\telse\n@@ -2108,8 +2227,8 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info *info,\n \t\t/* no update/deletion on this bucket, prev_elem should be still valid\n \t\t * and we won't skip elements.\n \t\t */\n-\t\tn = rcu_dereference_raw(hlist_nulls_next_rcu(\u0026prev_elem-\u003ehash_node));\n-\t\telem = hlist_nulls_entry_safe(n, struct htab_elem, hash_node);\n+\t\tn = rcu_dereference_raw(hlist_nulls_next_rcu(\u0026prev_elem-\u003enode.hash_node));\n+\t\telem = hlist_nulls_entry_safe(n, struct htab_elem, node.hash_node);\n \t\tif (elem)\n \t\t\treturn elem;\n \n@@ -2125,7 +2244,7 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info *info,\n \n \t\tcount = 0;\n \t\thead = \u0026b-\u003ehead;\n-\t\thlist_nulls_for_each_entry_rcu(elem, n, head, hash_node) {\n+\t\thlist_nulls_for_each_entry_rcu(elem, n, head, node.hash_node) {\n \t\t\tif (count \u003e= skip_elems) {\n \t\t\t\tinfo-\u003ebucket_id = i;\n \t\t\t\tinfo-\u003eskip_elems = count;\n@@ -2183,12 +2302,12 @@ static int __bpf_hash_map_seq_show(struct seq_file *seq, struct htab_elem *elem)\n \t\tctx.meta = \u0026meta;\n \t\tctx.map = info-\u003emap;\n \t\tif (elem) {\n-\t\t\tctx.key = elem-\u003ekey;\n+\t\t\tctx.key = htab_elem_key(info-\u003ehtab, elem);\n \t\t\tif (!info-\u003epercpu_value_buf) {\n-\t\t\t\tctx.value = htab_elem_value(elem, map-\u003ekey_size);\n+\t\t\t\tctx.value = htab_elem_value(info-\u003ehtab, elem);\n \t\t\t} else {\n \t\t\t\troundup_value_size = round_up(map-\u003evalue_size, 8);\n-\t\t\t\tpptr = htab_elem_get_ptr(elem, map-\u003ekey_size);\n+\t\t\t\tpptr = htab_elem_get_ptr(info-\u003ehtab, elem);\n \t\t\t\tfor_each_possible_cpu(cpu) {\n \t\t\t\t\tcopy_map_value_long(map, info-\u003epercpu_value_buf + off,\n \t\t\t\t\t\t\t    per_cpu_ptr(pptr, cpu));\n@@ -2292,14 +2411,14 @@ static long bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_\n \t\tb = \u0026htab-\u003ebuckets[i];\n \t\trcu_read_lock();\n \t\thead = \u0026b-\u003ehead;\n-\t\thlist_nulls_for_each_entry_safe(elem, n, head, hash_node) {\n-\t\t\tkey = elem-\u003ekey;\n+\t\thlist_nulls_for_each_entry_safe(elem, n, head, node.hash_node) {\n+\t\t\tkey = htab_elem_key(htab, elem);\n \t\t\tif (is_percpu) {\n \t\t\t\t/* current cpu value for percpu map */\n-\t\t\t\tpptr = htab_elem_get_ptr(elem, map-\u003ekey_size);\n+\t\t\t\tpptr = htab_elem_get_ptr(htab, elem);\n \t\t\t\tval = this_cpu_ptr(pptr);\n \t\t\t} else {\n-\t\t\t\tval = htab_elem_value(elem, map-\u003ekey_size);\n+\t\t\t\tval = htab_elem_value(htab, elem);\n \t\t\t}\n \t\t\tnum_elems++;\n \t\t\tret = callback_fn((u64)(long)map, (u64)(long)key,\n@@ -2403,10 +2522,11 @@ const struct bpf_map_ops htab_lru_map_ops = {\n /* Called from eBPF program */\n static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l = __htab_map_lookup_elem(map, key);\n \n \tif (l)\n-\t\treturn this_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size));\n+\t\treturn this_cpu_ptr(htab_elem_get_ptr(htab, l));\n \telse\n \t\treturn NULL;\n }\n@@ -2414,6 +2534,7 @@ static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key)\n /* inline bpf_map_lookup_elem() call for per-CPU hashmap */\n static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn *insn = insn_buf;\n \n \tif (!bpf_jit_supports_percpu_insn())\n@@ -2424,7 +2545,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn\n \t*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \t*insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3);\n \t*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0,\n-\t\t\t\toffsetof(struct htab_elem, key) + roundup(map-\u003ekey_size, 8));\n+\t\t\t\thtab-\u003ekey_offset + roundup(map-\u003ekey_size, 8));\n \t*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);\n \t*insn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);\n \n@@ -2433,6 +2554,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn\n \n static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l;\n \n \tif (cpu \u003e= nr_cpu_ids)\n@@ -2440,18 +2562,19 @@ static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key,\n \n \tl = __htab_map_lookup_elem(map, key);\n \tif (l)\n-\t\treturn per_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size), cpu);\n+\t\treturn per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu);\n \telse\n \t\treturn NULL;\n }\n \n static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l = __htab_map_lookup_elem(map, key);\n \n \tif (l) {\n-\t\tbpf_lru_node_set_ref(\u0026l-\u003elru_node);\n-\t\treturn this_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size));\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l));\n+\t\treturn this_cpu_ptr(htab_elem_get_ptr(htab, l));\n \t}\n \n \treturn NULL;\n@@ -2459,6 +2582,7 @@ static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key)\n \n static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l;\n \n \tif (cpu \u003e= nr_cpu_ids)\n@@ -2466,8 +2590,8 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k\n \n \tl = __htab_map_lookup_elem(map, key);\n \tif (l) {\n-\t\tbpf_lru_node_set_ref(\u0026l-\u003elru_node);\n-\t\treturn per_cpu_ptr(htab_elem_get_ptr(l, map-\u003ekey_size), cpu);\n+\t\tbpf_lru_node_set_ref(htab_elem_lru_node(l));\n+\t\treturn per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu);\n \t}\n \n \treturn NULL;\n@@ -2475,6 +2599,7 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k\n \n int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value, u64 map_flags)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l;\n \tvoid __percpu *pptr;\n \tint ret = -ENOENT;\n@@ -2494,7 +2619,7 @@ int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value, u64 map_fl\n \t/* We do not mark LRU map element here in order to not mess up\n \t * eviction heuristics when user space does a map walk.\n \t */\n-\tpptr = htab_elem_get_ptr(l, map-\u003ekey_size);\n+\tpptr = htab_elem_get_ptr(htab, l);\n \tif (map_flags \u0026 BPF_F_CPU) {\n \t\tcpu = map_flags \u003e\u003e 32;\n \t\tcopy_map_value(map, value, per_cpu_ptr(pptr, cpu));\n@@ -2532,6 +2657,7 @@ int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,\n static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key,\n \t\t\t\t\t  struct seq_file *m)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct htab_elem *l;\n \tvoid __percpu *pptr;\n \tint cpu;\n@@ -2546,7 +2672,7 @@ static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key,\n \n \tbtf_type_seq_show(map-\u003ebtf, map-\u003ebtf_key_type_id, key, m);\n \tseq_puts(m, \": {\\n\");\n-\tpptr = htab_elem_get_ptr(l, map-\u003ekey_size);\n+\tpptr = htab_elem_get_ptr(htab, l);\n \tfor_each_possible_cpu(cpu) {\n \t\tseq_printf(m, \"\\tcpu%d: \", cpu);\n \t\tbtf_type_seq_show(map-\u003ebtf, map-\u003ebtf_value_type_id,\n@@ -2619,8 +2745,8 @@ static void fd_htab_map_free(struct bpf_map *map)\n \tfor (i = 0; i \u003c htab-\u003en_buckets; i++) {\n \t\thead = select_bucket(htab, i);\n \n-\t\thlist_nulls_for_each_entry_safe(l, n, head, hash_node) {\n-\t\t\tvoid *ptr = fd_htab_map_get_ptr(map, l);\n+\t\thlist_nulls_for_each_entry_safe(l, n, head, node.hash_node) {\n+\t\t\tvoid *ptr = fd_htab_map_get_ptr(htab, l);\n \n \t\t\tmap-\u003eops-\u003emap_fd_put_ptr(map, ptr, false);\n \t\t}\n@@ -2705,6 +2831,7 @@ static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key)\n static int htab_of_map_gen_lookup(struct bpf_map *map,\n \t\t\t\t  struct bpf_insn *insn_buf)\n {\n+\tstruct bpf_htab *htab = container_of(map, struct bpf_htab, map);\n \tstruct bpf_insn *insn = insn_buf;\n \tconst int ret = BPF_REG_0;\n \n@@ -2713,7 +2840,7 @@ static int htab_of_map_gen_lookup(struct bpf_map *map,\n \t*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);\n \t*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2);\n \t*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,\n-\t\t\t\toffsetof(struct htab_elem, key) +\n+\t\t\t\thtab-\u003ekey_offset +\n \t\t\t\tround_up(map-\u003ekey_size, 8));\n \t*insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);\n \n@@ -3112,7 +3239,7 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf,\n {\n \tstruct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);\n \n-\treturn bpf_ma_set_dtor(map, \u0026rhtab-\u003ema, rhtab_mem_dtor);\n+\treturn bpf_ma_set_dtor(map, \u0026rhtab-\u003ema, rhtab_mem_dtor, offsetof(struct rhtab_elem, data));\n }\n \n static void rhtab_map_free_internal_structs(struct bpf_map *map)\n@@ -3533,3 +3660,16 @@ const struct bpf_map_ops rhtab_map_ops = {\n \t.map_btf_id = \u0026rhtab_map_btf_ids[0],\n \t.iter_seq_info = \u0026rhash_iter_seq_info,\n };\n+\n+size_t bpf_htab_map_meta_size(void)\n+{\n+\treturn sizeof(struct bpf_htab);\n+}\n+\n+void bpf_htab_map_meta_init(struct bpf_map *inner_map_meta, struct bpf_map *inner_map)\n+{\n+\tstruct bpf_htab *inner_htab_meta = container_of(inner_map_meta, struct bpf_htab, map);\n+\tstruct bpf_htab *inner_htab = container_of(inner_map, struct bpf_htab, map);\n+\n+\tinner_htab_meta-\u003ekey_offset = inner_htab-\u003ekey_offset;\n+}\ndiff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c\nindex d2cbab4bdf644..b2db5d0c98bee 100644\n--- a/kernel/bpf/map_in_map.c\n+++ b/kernel/bpf/map_in_map.c\n@@ -7,6 +7,15 @@\n \n #include \"map_in_map.h\"\n \n+static bool bpf_map_type_is_htab(enum bpf_map_type type)\n+{\n+\treturn type == BPF_MAP_TYPE_HASH ||\n+\t       type == BPF_MAP_TYPE_PERCPU_HASH ||\n+\t       type == BPF_MAP_TYPE_LRU_HASH ||\n+\t       type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||\n+\t       type == BPF_MAP_TYPE_HASH_OF_MAPS;\n+}\n+\n struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)\n {\n \tstruct bpf_map *inner_map, *inner_map_meta;\n@@ -29,6 +38,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)\n \t/* In some cases verifier needs to access beyond just base map. */\n \tif (inner_map-\u003eops == \u0026array_map_ops || inner_map-\u003eops == \u0026percpu_array_map_ops)\n \t\tinner_map_meta_size = sizeof(struct bpf_array);\n+\telse if (bpf_map_type_is_htab(inner_map-\u003emap_type))\n+\t\tinner_map_meta_size = bpf_htab_map_meta_size();\n \n \tinner_map_meta = kzalloc(inner_map_meta_size, GFP_USER);\n \tif (!inner_map_meta)\n@@ -70,6 +81,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)\n \t\tinner_array_meta-\u003eindex_mask = inner_array-\u003eindex_mask;\n \t\tinner_array_meta-\u003eelem_size = inner_array-\u003eelem_size;\n \t\tinner_map_meta-\u003ebypass_spec_v1 = inner_map-\u003ebypass_spec_v1;\n+\t} else if (bpf_map_type_is_htab(inner_map-\u003emap_type)) {\n+\t\tbpf_htab_map_meta_init(inner_map_meta, inner_map);\n \t}\n \treturn inner_map_meta;\n }\ndiff --git a/kernel/bpf/map_in_map.h b/kernel/bpf/map_in_map.h\nindex 7d61602354de8..1edc205772d8c 100644\n--- a/kernel/bpf/map_in_map.h\n+++ b/kernel/bpf/map_in_map.h\n@@ -15,5 +15,7 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, struct file *map_file,\n \t\t\t int ufd);\n void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer);\n u32 bpf_map_fd_sys_lookup_elem(void *ptr);\n+size_t bpf_htab_map_meta_size(void);\n+void bpf_htab_map_meta_init(struct bpf_map *inner_map_meta, struct bpf_map *inner_map);\n \n #endif\ndiff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c\nindex 373c8d17ea557..6bd4cb68c20c1 100644\n--- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c\n+++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c\n@@ -114,7 +114,7 @@ static inline int check_hash(void)\n \tVERIFY(check_default_noinline(\u0026hash-\u003emap, map));\n \n \tVERIFY(hash-\u003en_buckets == MAX_ENTRIES);\n-\tVERIFY(hash-\u003eelem_size == 64);\n+\tVERIFY(hash-\u003eelem_size == 32);\n \n \tVERIFY(hash-\u003ecount.counter == 0);\n \tVERIFY(bpf_map_sum_elem_count(map) == 0);\n"
}

For your convenience, here is the diff of the changes:
commit 6454a11c03dfaedb4b09f8575af8d03639fb5273
Author: syz-cluster <triage@syzkaller.com>
Date:   Wed Aug 5 22:52:22 2026 +0000

    syz-cluster: applied patch under review

diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 9f394e1aa2e85..9967268d453d0 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -100,13 +100,16 @@ struct bpf_htab {
 	struct percpu_counter pcount;
 	atomic_t count;
 	bool use_percpu_counter;
+	bool has_hash;
 	u32 n_buckets;	/* number of hash buckets */
 	u32 elem_size;	/* size of each element in bytes */
+	u32 key_offset;	/* offset of key in bytes */
 	u32 hashrnd;
 };
 
 /* each htab element is struct htab_elem + key + value */
-struct htab_elem {
+struct htab_elem;
+struct htab_node {
 	union {
 		struct hlist_nulls_node hash_node;
 		struct {
@@ -117,18 +120,32 @@ struct htab_elem {
 			};
 		};
 	};
-	union {
-		/* pointer to per-cpu pointer */
-		void *ptr_to_pptr;
-		struct bpf_lru_node lru_node;
-	};
-	u32 hash;
-	char key[] __aligned(8);
+};
+
+struct htab_elem {
+	struct htab_node node;
+	u8 data[] __aligned(8);
+};
+
+struct htab_elem_lru {
+	struct htab_node node;
+	struct bpf_lru_node lru_node;
+	u8 data[] __aligned(8);
+};
+
+/* Only for non-preallocated PCPU maps. Preallocated PCPU maps don't need
+ * ptr_to_pptr, and use htab_elem.
+ */
+struct htab_elem_pcpu {
+	struct htab_node node;
+	void *ptr_to_pptr;
+	u8 data[] __aligned(8);
 };
 
 struct htab_btf_record {
 	struct btf_record *record;
 	u32 key_size;
+	u32 key_offset;
 };
 
 static inline bool htab_is_prealloc(const struct bpf_htab *htab)
@@ -136,6 +153,21 @@ static inline bool htab_is_prealloc(const struct bpf_htab *htab)
 	return !(htab->map.map_flags & BPF_F_NO_PREALLOC);
 }
 
+static inline struct bpf_lru_node *htab_elem_lru_node(struct htab_elem *l)
+{
+	return &((struct htab_elem_lru *)l)->lru_node;
+}
+
+static inline void *htab_elem_get_ptr_to_pptr(struct htab_elem *l)
+{
+	return ((struct htab_elem_pcpu *)l)->ptr_to_pptr;
+}
+
+static inline void htab_elem_set_ptr_to_pptr(struct htab_elem *l, void *ptr)
+{
+	((struct htab_elem_pcpu *)l)->ptr_to_pptr = ptr;
+}
+
 static void htab_init_buckets(struct bpf_htab *htab)
 {
 	unsigned int i;
@@ -183,25 +215,30 @@ static inline bool is_fd_htab(const struct bpf_htab *htab)
 	return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS;
 }
 
-static inline void *htab_elem_value(struct htab_elem *l, u32 key_size)
+static inline void *htab_elem_key(struct bpf_htab *htab, struct htab_elem *l)
+{
+	return (void *)l + htab->key_offset;
+}
+
+static inline void *htab_elem_value(struct bpf_htab *htab, struct htab_elem *l)
 {
-	return l->key + round_up(key_size, 8);
+	return htab_elem_key(htab, l) + round_up(htab->map.key_size, 8);
 }
 
-static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,
+static inline void htab_elem_set_ptr(struct bpf_htab *htab, struct htab_elem *l,
 				     void __percpu *pptr)
 {
-	*(void __percpu **)htab_elem_value(l, key_size) = pptr;
+	*(void __percpu **)htab_elem_value(htab, l) = pptr;
 }
 
-static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)
+static inline void __percpu *htab_elem_get_ptr(struct bpf_htab *htab, struct htab_elem *l)
 {
-	return *(void __percpu **)htab_elem_value(l, key_size);
+	return *(void __percpu **)htab_elem_value(htab, l);
 }
 
-static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)
+static void *fd_htab_map_get_ptr(struct bpf_htab *htab, struct htab_elem *l)
 {
-	return *(void **)htab_elem_value(l, map->key_size);
+	return *(void **)htab_elem_value(htab, l);
 }
 
 static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)
@@ -209,6 +246,33 @@ static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)
 	return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size);
 }
 
+static inline bool htab_has_hash(const struct bpf_htab *htab)
+{
+	return htab->has_hash;
+}
+
+static inline u32 htab_elem_hash(struct bpf_htab *htab, struct htab_elem *l)
+{
+	if (htab_is_lru(htab))
+		return *(u32 *)((struct htab_elem_lru *)l)->data;
+	else if (htab_is_percpu(htab) && !htab_is_prealloc(htab))
+		return *(u32 *)((struct htab_elem_pcpu *)l)->data;
+	else
+		return *(u32 *)l->data;
+}
+
+static inline void htab_elem_set_hash(struct bpf_htab *htab, struct htab_elem *l, u32 hash)
+{
+	if (!htab_has_hash(htab))
+		return;
+	if (htab_is_lru(htab))
+		*(u32 *)((struct htab_elem_lru *)l)->data = hash;
+	else if (htab_is_percpu(htab) && !htab_is_prealloc(htab))
+		*(u32 *)((struct htab_elem_pcpu *)l)->data = hash;
+	else
+		*(u32 *)l->data = hash;
+}
+
 /* Both percpu and fd htab support in-place update, so no need for
  * extra elem. LRU itself can remove the least used element, so
  * there is no need for an extra elem during map_update.
@@ -231,7 +295,7 @@ static void htab_free_prealloced_internal_structs(struct bpf_htab *htab)
 
 		elem = get_htab_elem(htab, i);
 		bpf_map_free_internal_structs(&htab->map,
-					      htab_elem_value(elem, htab->map.key_size));
+					      htab_elem_value(htab, elem));
 		cond_resched();
 	}
 }
@@ -254,7 +318,7 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab)
 
 		elem = get_htab_elem(htab, i);
 		if (htab_is_percpu(htab)) {
-			void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);
+			void __percpu *pptr = htab_elem_get_ptr(htab, elem);
 			int cpu;
 
 			for_each_possible_cpu(cpu) {
@@ -263,7 +327,7 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab)
 			}
 		} else {
 			bpf_obj_free_fields(htab->map.record,
-					    htab_elem_value(elem, htab->map.key_size));
+					    htab_elem_value(htab, elem));
 			cond_resched();
 		}
 		cond_resched();
@@ -280,8 +344,7 @@ static void htab_free_elems(struct bpf_htab *htab)
 	for (i = 0; i < htab->map.max_entries; i++) {
 		void __percpu *pptr;
 
-		pptr = htab_elem_get_ptr(get_htab_elem(htab, i),
-					 htab->map.key_size);
+		pptr = htab_elem_get_ptr(htab, get_htab_elem(htab, i));
 		free_percpu(pptr);
 		cond_resched();
 	}
@@ -308,8 +371,8 @@ static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key,
 
 	if (node) {
 		bpf_map_inc_elem_count(&htab->map);
-		l = container_of(node, struct htab_elem, lru_node);
-		memcpy(l->key, key, htab->map.key_size);
+		l = (struct htab_elem *)container_of(node, struct htab_elem_lru, lru_node);
+		memcpy(htab_elem_key(htab, l), key, htab->map.key_size);
 		return l;
 	}
 
@@ -340,8 +403,7 @@ static int prealloc_init(struct bpf_htab *htab)
 					    GFP_USER | __GFP_NOWARN);
 		if (!pptr)
 			goto free_elems;
-		htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size,
-				  pptr);
+		htab_elem_set_ptr(htab, get_htab_elem(htab, i), pptr);
 		cond_resched();
 	}
 
@@ -349,8 +411,8 @@ static int prealloc_init(struct bpf_htab *htab)
 	if (htab_is_lru(htab))
 		err = bpf_lru_init(&htab->lru,
 				   htab->map.map_flags & BPF_F_NO_COMMON_LRU,
-				   offsetof(struct htab_elem, hash) -
-				   offsetof(struct htab_elem, lru_node),
+				   offsetof(struct htab_elem_lru, data) -
+				   offsetof(struct htab_elem_lru, lru_node),
 				   htab_lru_map_delete_node,
 				   htab);
 	else
@@ -361,11 +423,11 @@ static int prealloc_init(struct bpf_htab *htab)
 
 	if (htab_is_lru(htab))
 		bpf_lru_populate(&htab->lru, htab->elems,
-				 offsetof(struct htab_elem, lru_node),
+				 offsetof(struct htab_elem_lru, lru_node),
 				 htab->elem_size, num_entries);
 	else
 		pcpu_freelist_populate(&htab->freelist,
-				       htab->elems + offsetof(struct htab_elem, fnode),
+				       htab->elems + offsetof(struct htab_elem, node.fnode),
 				       htab->elem_size, num_entries);
 
 	return 0;
@@ -401,7 +463,7 @@ static int alloc_extra_elems(struct bpf_htab *htab)
 		/* pop will succeed, since prealloc_init()
 		 * preallocated extra num_possible_cpus elements
 		 */
-		l_new = container_of(l, struct htab_elem, fnode);
+		l_new = container_of(l, struct htab_elem, node.fnode);
 		*per_cpu_ptr(pptr, cpu) = l_new;
 	}
 	htab->extra_elems = pptr;
@@ -425,8 +487,8 @@ static int htab_map_alloc_check(union bpf_attr *attr)
 	bool zero_seed = (attr->map_flags & BPF_F_ZERO_SEED);
 	int numa_node = bpf_map_attr_numa_node(attr);
 
-	BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=
-		     offsetof(struct htab_elem, hash_node.pprev));
+	BUILD_BUG_ON(offsetof(struct htab_node, fnode.next) !=
+		     offsetof(struct htab_node, hash_node.pprev));
 
 	if (zero_seed && !capable(CAP_SYS_ADMIN))
 		/* Guard against local DoS, and discourage production use. */
@@ -476,7 +538,7 @@ static void htab_mem_dtor(void *obj, void *ctx)
 	if (IS_ERR_OR_NULL(hrec->record))
 		return;
 
-	map_value = htab_elem_value(elem, hrec->key_size);
+	map_value = (void *)elem + hrec->key_offset + round_up(hrec->key_size, 8);
 	bpf_obj_free_fields(hrec->record, map_value);
 }
 
@@ -502,7 +564,7 @@ static void htab_dtor_ctx_free(void *ctx)
 }
 
 static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,
-			   void (*dtor)(void *, void *))
+			   void (*dtor)(void *, void *), u32 key_offset)
 {
 	struct htab_btf_record *hrec;
 	int err;
@@ -515,6 +577,7 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,
 	if (!hrec)
 		return -ENOMEM;
 	hrec->key_size = map->key_size;
+	hrec->key_offset = key_offset;
 	hrec->record = btf_record_dup(map->record);
 	if (IS_ERR(hrec->record)) {
 		err = PTR_ERR(hrec->record);
@@ -537,9 +600,9 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf,
 	 * populated in htab_map_alloc(), so it will always appear as NULL.
 	 */
 	if (htab_is_percpu(htab))
-		return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor);
+		return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor, htab->key_offset);
 	else
-		return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor);
+		return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor, htab->key_offset);
 }
 
 static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
@@ -562,6 +625,13 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
 
 	bpf_map_init_from_attr(&htab->map, attr);
 
+	/* Avoid hash memory use and comparisons where unnecessary.
+	 * u32 hash reads are always atomic. If we elide them, key comparisons must also be atomic
+	 * to avoid false positive key matches due to torn key reads / writes. This is only possible
+	 * when the key fits within a word, so check key_size.
+	 */
+	htab->has_hash = htab_is_lru(htab) || htab->map.key_size > sizeof(unsigned long);
+
 	if (percpu_lru) {
 		/* ensure each CPU's lru list has >=1 elements.
 		 * since we are at it, make each lru list has the same
@@ -583,8 +653,16 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
 
 	htab->n_buckets = roundup_pow_of_two(htab->map.max_entries);
 
-	htab->elem_size = sizeof(struct htab_elem) +
-			  round_up(htab->map.key_size, 8);
+	if (htab_is_lru(htab))
+		htab->key_offset = offsetof(struct htab_elem_lru, data) + 8;
+	else if (percpu && !prealloc)
+		htab->key_offset = offsetof(struct htab_elem_pcpu, data) +
+				   (htab_has_hash(htab) ? 8 : 0);
+	else
+		htab->key_offset = offsetof(struct htab_elem, data) +
+				   (htab_has_hash(htab) ? 8 : 0);
+
+	htab->elem_size = htab->key_offset + round_up(htab->map.key_size, 8);
 	if (percpu)
 		htab->elem_size += sizeof(void *);
 	else
@@ -692,15 +770,29 @@ static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32
 }
 
 /* this lookup function can only be called with bucket lock taken */
-static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash,
+static struct htab_elem *lookup_elem_raw(struct bpf_htab *htab,
+					 struct hlist_nulls_head *head, u32 hash,
 					 void *key, u32 key_size)
 {
 	struct hlist_nulls_node *n;
 	struct htab_elem *l;
 
-	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
-		if (l->hash == hash && !memcmp(&l->key, key, key_size))
-			return l;
+	if (htab_has_hash(htab)) {
+		hlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)
+			if (htab_elem_hash(htab, l) == hash &&
+			    !memcmp(htab_elem_key(htab, l), key, key_size))
+				return l;
+	} else {
+		/* When hash is omitted, key comparisons must be atomic. Zero extend
+		 * the caller's key to the word size to support an atomic compare.
+		 */
+		unsigned long k = 0;
+
+		memcpy(&k, key, key_size);
+		hlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)
+			if (READ_ONCE(*(unsigned long *)htab_elem_key(htab, l)) == k)
+				return l;
+	}
 
 	return NULL;
 }
@@ -709,7 +801,8 @@ static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash
  * the unlikely event when elements moved from one bucket into another
  * while link list is being walked
  */
-static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,
+static struct htab_elem *lookup_nulls_elem_raw(struct bpf_htab *htab,
+					       struct hlist_nulls_head *head,
 					       u32 hash, void *key,
 					       u32 key_size, u32 n_buckets)
 {
@@ -717,9 +810,20 @@ static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,
 	struct htab_elem *l;
 
 again:
-	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
-		if (l->hash == hash && !memcmp(&l->key, key, key_size))
-			return l;
+	if (htab_has_hash(htab)) {
+		hlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)
+			if (htab_elem_hash(htab, l) == hash &&
+			    !memcmp(htab_elem_key(htab, l), key, key_size))
+				return l;
+	} else {
+		/* See lookup_elem_raw() comment above. */
+		unsigned long k = 0;
+
+		memcpy(&k, key, key_size);
+		hlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)
+			if (READ_ONCE(*(unsigned long *)htab_elem_key(htab, l)) == k)
+				return l;
+	}
 
 	if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1))))
 		goto again;
@@ -747,17 +851,18 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
 
 	head = select_bucket(htab, hash);
 
-	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);
+	l = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab->n_buckets);
 
 	return l;
 }
 
 static void *htab_map_lookup_elem(struct bpf_map *map, void *key)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l = __htab_map_lookup_elem(map, key);
 
 	if (l)
-		return htab_elem_value(l, map->key_size);
+		return htab_elem_value(htab, l);
 
 	return NULL;
 }
@@ -775,6 +880,7 @@ static void *htab_map_lookup_elem(struct bpf_map *map, void *key)
  */
 static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct bpf_insn *insn = insn_buf;
 	const int ret = BPF_REG_0;
 
@@ -783,7 +889,7 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
 	*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
 	*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,
-				offsetof(struct htab_elem, key) +
+				htab->key_offset +
 				round_up(map->key_size, 8));
 	return insn - insn_buf;
 }
@@ -791,12 +897,13 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
 static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map,
 							void *key, const bool mark)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l = __htab_map_lookup_elem(map, key);
 
 	if (l) {
 		if (mark)
-			bpf_lru_node_set_ref(&l->lru_node);
-		return htab_elem_value(l, map->key_size);
+			bpf_lru_node_set_ref(htab_elem_lru_node(l));
+		return htab_elem_value(htab, l);
 	}
 
 	return NULL;
@@ -815,6 +922,7 @@ static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key)
 static int htab_lru_map_gen_lookup(struct bpf_map *map,
 				   struct bpf_insn *insn_buf)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct bpf_insn *insn = insn_buf;
 	const int ret = BPF_REG_0;
 	const int ref_reg = BPF_REG_1;
@@ -824,15 +932,15 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map,
 	*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
 	*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4);
 	*insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret,
-			      offsetof(struct htab_elem, lru_node) +
+			      offsetof(struct htab_elem_lru, lru_node) +
 			      offsetof(struct bpf_lru_node, ref));
 	*insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1);
 	*insn++ = BPF_ST_MEM(BPF_B, ret,
-			     offsetof(struct htab_elem, lru_node) +
+			     offsetof(struct htab_elem_lru, lru_node) +
 			     offsetof(struct bpf_lru_node, ref),
 			     1);
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,
-				offsetof(struct htab_elem, key) +
+				htab->key_offset +
 				round_up(map->key_size, 8));
 	return insn - insn_buf;
 }
@@ -844,13 +952,13 @@ static void check_and_cancel_fields(struct bpf_htab *htab,
 		return;
 
 	if (htab_is_percpu(htab)) {
-		void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);
+		void __percpu *pptr = htab_elem_get_ptr(htab, elem);
 		int cpu;
 
 		for_each_possible_cpu(cpu)
 			bpf_obj_cancel_fields(&htab->map, per_cpu_ptr(pptr, cpu));
 	} else {
-		void *map_value = htab_elem_value(elem, htab->map.key_size);
+		void *map_value = htab_elem_value(htab, elem);
 
 		bpf_obj_cancel_fields(&htab->map, map_value);
 	}
@@ -869,17 +977,17 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)
 	struct bucket *b;
 	int ret;
 
-	tgt_l = container_of(node, struct htab_elem, lru_node);
-	b = __select_bucket(htab, tgt_l->hash);
+	tgt_l = (struct htab_elem *)container_of(node, struct htab_elem_lru, lru_node);
+	b = __select_bucket(htab, htab_elem_hash(htab, tgt_l));
 	head = &b->head;
 
 	ret = htab_lock_bucket(b, &flags);
 	if (ret)
 		return false;
 
-	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
+	hlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)
 		if (l == tgt_l) {
-			hlist_nulls_del_rcu(&l->hash_node);
+			hlist_nulls_del_rcu(&l->node.hash_node);
 			bpf_map_dec_elem_count(&htab->map);
 			break;
 		}
@@ -912,18 +1020,19 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
 	head = select_bucket(htab, hash);
 
 	/* lookup the key */
-	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);
+	l = lookup_nulls_elem_raw(htab, head, hash, key, key_size, htab->n_buckets);
 
 	if (!l)
 		goto find_first_elem;
 
 	/* key was found, get next key in the same bucket */
-	next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(&l->hash_node)),
-				  struct htab_elem, hash_node);
+	next_l = hlist_nulls_entry_safe(
+			rcu_dereference_raw(hlist_nulls_next_rcu(&l->node.hash_node)),
+			struct htab_elem, node.hash_node);
 
 	if (next_l) {
 		/* if next elem in this hash list is non-zero, just return it */
-		memcpy(next_key, next_l->key, key_size);
+		memcpy(next_key, htab_elem_key(htab, next_l), key_size);
 		return 0;
 	}
 
@@ -938,10 +1047,10 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
 
 		/* pick first element in the bucket */
 		next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_first_rcu(head)),
-					  struct htab_elem, hash_node);
+					  struct htab_elem, node.hash_node);
 		if (next_l) {
 			/* if it's not empty, just return it */
-			memcpy(next_key, next_l->key, key_size);
+			memcpy(next_key, htab_elem_key(htab, next_l), key_size);
 			return 0;
 		}
 	}
@@ -955,7 +1064,7 @@ static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)
 	check_and_cancel_fields(htab, l);
 
 	if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH)
-		bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);
+		bpf_mem_cache_free(&htab->pcpu_ma, htab_elem_get_ptr_to_pptr(l));
 	bpf_mem_cache_free(&htab->ma, l);
 }
 
@@ -965,7 +1074,7 @@ static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)
 	void *ptr;
 
 	if (map->ops->map_fd_put_ptr) {
-		ptr = fd_htab_map_get_ptr(map, l);
+		ptr = fd_htab_map_get_ptr(htab, l);
 		map->ops->map_fd_put_ptr(map, ptr, true);
 	}
 }
@@ -1006,7 +1115,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
 	if (htab_is_prealloc(htab)) {
 		bpf_map_dec_elem_count(&htab->map);
 		check_and_cancel_fields(htab, l);
-		pcpu_freelist_push(&htab->freelist, &l->fnode);
+		pcpu_freelist_push(&htab->freelist, &l->node.fnode);
 	} else {
 		dec_elem_count(htab);
 		htab_elem_free(htab, l);
@@ -1097,7 +1206,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
 			l = __pcpu_freelist_pop(&htab->freelist);
 			if (!l)
 				return ERR_PTR(-E2BIG);
-			l_new = container_of(l, struct htab_elem, fnode);
+			l_new = container_of(l, struct htab_elem, node.fnode);
 			bpf_map_inc_elem_count(&htab->map);
 		}
 	} else {
@@ -1117,10 +1226,20 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
 		}
 	}
 
-	memcpy(l_new->key, key, key_size);
+	if (htab_has_hash(htab)) {
+		memcpy(htab_elem_key(htab, l_new), key, key_size);
+	} else {
+		/* Zero-extend key into k for an atomic write to support
+		 * lockless RCU readers.
+		 */
+		unsigned long k = 0;
+
+		memcpy(&k, key, key_size);
+		WRITE_ONCE(*(unsigned long *)htab_elem_key(htab, l_new), k);
+	}
 	if (percpu) {
 		if (prealloc) {
-			pptr = htab_elem_get_ptr(l_new, key_size);
+			pptr = htab_elem_get_ptr(htab, l_new);
 		} else {
 			/* alloc_percpu zero-fills */
 			void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma);
@@ -1130,26 +1249,26 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
 				l_new = ERR_PTR(-ENOMEM);
 				goto dec_count;
 			}
-			l_new->ptr_to_pptr = ptr;
+			htab_elem_set_ptr_to_pptr(l_new, ptr);
 			pptr = *(void __percpu **)ptr;
 		}
 
 		pcpu_init_value(htab, pptr, value, onallcpus, map_flags);
 
 		if (!prealloc)
-			htab_elem_set_ptr(l_new, key_size, pptr);
+			htab_elem_set_ptr(htab, l_new, pptr);
 	} else if (fd_htab_map_needs_adjust(htab)) {
 		size = round_up(size, 8);
-		memcpy(htab_elem_value(l_new, key_size), value, size);
+		memcpy(htab_elem_value(htab, l_new), value, size);
 	} else if (map_flags & BPF_F_LOCK) {
 		copy_map_value_locked(&htab->map,
-				      htab_elem_value(l_new, key_size),
+				      htab_elem_value(htab, l_new),
 				      value, false);
 	} else {
-		copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value);
+		copy_map_value(&htab->map, htab_elem_value(htab, l_new), value);
 	}
 
-	l_new->hash = hash;
+	htab_elem_set_hash(htab, l_new, hash);
 	return l_new;
 dec_count:
 	dec_elem_count(htab);
@@ -1199,7 +1318,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
 		if (unlikely(!btf_record_has_field(map->record, BPF_SPIN_LOCK)))
 			return -EINVAL;
 		/* find an element without taking the bucket lock */
-		l_old = lookup_nulls_elem_raw(head, hash, key, key_size,
+		l_old = lookup_nulls_elem_raw(htab, head, hash, key, key_size,
 					      htab->n_buckets);
 		ret = check_flags(htab, l_old, map_flags);
 		if (ret)
@@ -1207,7 +1326,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
 		if (l_old) {
 			/* grab the element lock and update value in place */
 			copy_map_value_locked(map,
-					      htab_elem_value(l_old, key_size),
+					      htab_elem_value(htab, l_old),
 					      value, false);
 			return 0;
 		}
@@ -1221,7 +1340,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
 	if (ret)
 		return ret;
 
-	l_old = lookup_elem_raw(head, hash, key, key_size);
+	l_old = lookup_elem_raw(htab, head, hash, key, key_size);
 
 	ret = check_flags(htab, l_old, map_flags);
 	if (ret)
@@ -1235,7 +1354,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
 		 * and update element in place
 		 */
 		copy_map_value_locked(map,
-				      htab_elem_value(l_old, key_size),
+				      htab_elem_value(htab, l_old),
 				      value, false);
 		ret = 0;
 		goto err;
@@ -1252,9 +1371,9 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
 	/* add new element to the head of the list, so that
 	 * concurrent search will find it before old elem
 	 */
-	hlist_nulls_add_head_rcu(&l_new->hash_node, head);
+	hlist_nulls_add_head_rcu(&l_new->node.hash_node, head);
 	if (l_old) {
-		hlist_nulls_del_rcu(&l_old->hash_node);
+		hlist_nulls_del_rcu(&l_old->node.hash_node);
 
 		/* l_old has already been stashed in htab->extra_elems, cancel
 		 * its reusable special fields before it is available for reuse.
@@ -1275,7 +1394,7 @@ static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem)
 {
 	check_and_cancel_fields(htab, elem);
 	bpf_map_dec_elem_count(&htab->map);
-	bpf_lru_push_free(&htab->lru, &elem->lru_node);
+	bpf_lru_push_free(&htab->lru, htab_elem_lru_node(elem));
 }
 
 static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,
@@ -1310,13 +1429,13 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value
 	l_new = prealloc_lru_pop(htab, key, hash);
 	if (!l_new)
 		return -ENOMEM;
-	copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value);
+	copy_map_value(&htab->map, htab_elem_value(htab, l_new), value);
 
 	ret = htab_lock_bucket(b, &flags);
 	if (ret)
 		goto err_lock_bucket;
 
-	l_old = lookup_elem_raw(head, hash, key, key_size);
+	l_old = lookup_elem_raw(htab, head, hash, key, key_size);
 
 	ret = check_flags(htab, l_old, map_flags);
 	if (ret)
@@ -1325,10 +1444,10 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value
 	/* add new element to the head of the list, so that
 	 * concurrent search will find it before old elem
 	 */
-	hlist_nulls_add_head_rcu(&l_new->hash_node, head);
+	hlist_nulls_add_head_rcu(&l_new->node.hash_node, head);
 	if (l_old) {
-		bpf_lru_node_set_ref(&l_new->lru_node);
-		hlist_nulls_del_rcu(&l_old->hash_node);
+		bpf_lru_node_set_ref(htab_elem_lru_node(l_new));
+		hlist_nulls_del_rcu(&l_old->node.hash_node);
 	}
 	ret = 0;
 
@@ -1383,7 +1502,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,
 	if (ret)
 		return ret;
 
-	l_old = lookup_elem_raw(head, hash, key, key_size);
+	l_old = lookup_elem_raw(htab, head, hash, key, key_size);
 
 	ret = check_flags(htab, l_old, map_flags);
 	if (ret)
@@ -1392,10 +1511,10 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,
 	if (l_old) {
 		/* Update value in-place */
 		if (percpu) {
-			pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),
+			pcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old),
 					value, onallcpus, map_flags);
 		} else {
-			void **inner_map_pptr = htab_elem_value(l_old, key_size);
+			void **inner_map_pptr = htab_elem_value(htab, l_old);
 
 			old_map_ptr = *inner_map_pptr;
 			WRITE_ONCE(*inner_map_pptr, *(void **)value);
@@ -1407,7 +1526,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,
 			ret = PTR_ERR(l_new);
 			goto err;
 		}
-		hlist_nulls_add_head_rcu(&l_new->hash_node, head);
+		hlist_nulls_add_head_rcu(&l_new->node.hash_node, head);
 	}
 err:
 	htab_unlock_bucket(b, flags);
@@ -1456,22 +1575,22 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
 	if (ret)
 		goto err_lock_bucket;
 
-	l_old = lookup_elem_raw(head, hash, key, key_size);
+	l_old = lookup_elem_raw(htab, head, hash, key, key_size);
 
 	ret = check_flags(htab, l_old, map_flags);
 	if (ret)
 		goto err;
 
 	if (l_old) {
-		bpf_lru_node_set_ref(&l_old->lru_node);
+		bpf_lru_node_set_ref(htab_elem_lru_node(l_old));
 
 		/* per-cpu hash map can update value in-place */
-		pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),
+		pcpu_copy_value(htab, htab_elem_get_ptr(htab, l_old),
 				value, onallcpus, map_flags);
 	} else {
-		pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size),
+		pcpu_init_value(htab, htab_elem_get_ptr(htab, l_new),
 				value, onallcpus, map_flags);
-		hlist_nulls_add_head_rcu(&l_new->hash_node, head);
+		hlist_nulls_add_head_rcu(&l_new->node.hash_node, head);
 		l_new = NULL;
 	}
 	ret = 0;
@@ -1480,7 +1599,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
 err_lock_bucket:
 	if (l_new) {
 		bpf_map_dec_elem_count(&htab->map);
-		bpf_lru_push_free(&htab->lru, &l_new->lru_node);
+		bpf_lru_push_free(&htab->lru, htab_elem_lru_node(l_new));
 	}
 	return ret;
 }
@@ -1521,9 +1640,9 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key)
 	if (ret)
 		return ret;
 
-	l = lookup_elem_raw(head, hash, key, key_size);
+	l = lookup_elem_raw(htab, head, hash, key, key_size);
 	if (l)
-		hlist_nulls_del_rcu(&l->hash_node);
+		hlist_nulls_del_rcu(&l->node.hash_node);
 	else
 		ret = -ENOENT;
 
@@ -1556,10 +1675,10 @@ static long htab_lru_map_delete_elem(struct bpf_map *map, void *key)
 	if (ret)
 		return ret;
 
-	l = lookup_elem_raw(head, hash, key, key_size);
+	l = lookup_elem_raw(htab, head, hash, key, key_size);
 
 	if (l)
-		hlist_nulls_del_rcu(&l->hash_node);
+		hlist_nulls_del_rcu(&l->node.hash_node);
 	else
 		ret = -ENOENT;
 
@@ -1581,8 +1700,8 @@ static void delete_all_elements(struct bpf_htab *htab)
 		struct hlist_nulls_node *n;
 		struct htab_elem *l;
 
-		hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {
-			hlist_nulls_del_rcu(&l->hash_node);
+		hlist_nulls_for_each_entry_safe(l, n, head, node.hash_node) {
+			hlist_nulls_del_rcu(&l->node.hash_node);
 			htab_elem_free(htab, l);
 		}
 		cond_resched();
@@ -1599,10 +1718,10 @@ static void htab_free_malloced_internal_structs(struct bpf_htab *htab)
 		struct hlist_nulls_node *n;
 		struct htab_elem *l;
 
-		hlist_nulls_for_each_entry(l, n, head, hash_node) {
+		hlist_nulls_for_each_entry(l, n, head, node.hash_node) {
 			/* We only free internal structs on uref dropping to zero */
 			bpf_map_free_internal_structs(&htab->map,
-						      htab_elem_value(l, htab->map.key_size));
+						      htab_elem_value(htab, l));
 		}
 		cond_resched_rcu();
 	}
@@ -1697,7 +1816,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
 	if (ret)
 		return ret;
 
-	l = lookup_elem_raw(head, hash, key, key_size);
+	l = lookup_elem_raw(htab, head, hash, key, key_size);
 	if (!l) {
 		ret = -ENOENT;
 		goto out_unlock;
@@ -1708,14 +1827,14 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
 		void __percpu *pptr;
 		int off = 0, cpu;
 
-		pptr = htab_elem_get_ptr(l, key_size);
+		pptr = htab_elem_get_ptr(htab, l);
 		for_each_possible_cpu(cpu) {
 			copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu));
 			check_and_init_map_value(&htab->map, value + off);
 			off += roundup_value_size;
 		}
 	} else {
-		void *src = htab_elem_value(l, map->key_size);
+		void *src = htab_elem_value(htab, l);
 
 		if (flags & BPF_F_LOCK)
 			copy_map_value_locked(map, value, src, true);
@@ -1724,7 +1843,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
 		/* Zeroing special fields in the temp buffer */
 		check_and_init_map_value(map, value);
 	}
-	hlist_nulls_del_rcu(&l->hash_node);
+	hlist_nulls_del_rcu(&l->node.hash_node);
 
 out_unlock:
 	htab_unlock_bucket(b, bflags);
@@ -1860,7 +1979,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
 	}
 
 	bucket_cnt = 0;
-	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
+	hlist_nulls_for_each_entry_rcu(l, n, head, node.hash_node)
 		bucket_cnt++;
 
 	if (bucket_cnt && !locked) {
@@ -1897,14 +2016,14 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
 	if (!locked)
 		goto next_batch;
 
-	hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {
-		memcpy(dst_key, l->key, key_size);
+	hlist_nulls_for_each_entry_safe(l, n, head, node.hash_node) {
+		memcpy(dst_key, htab_elem_key(htab, l), key_size);
 
 		if (is_percpu) {
 			int off = 0, cpu;
 			void __percpu *pptr;
 
-			pptr = htab_elem_get_ptr(l, map->key_size);
+			pptr = htab_elem_get_ptr(htab, l);
 			if (elem_map_flags & BPF_F_CPU) {
 				cpu = elem_map_flags >> 32;
 				copy_map_value(&htab->map, dst_val, per_cpu_ptr(pptr, cpu));
@@ -1918,7 +2037,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
 				}
 			}
 		} else {
-			value = htab_elem_value(l, key_size);
+			value = htab_elem_value(htab, l);
 			if (is_fd_htab(htab)) {
 				struct bpf_map **inner_map = value;
 
@@ -1936,7 +2055,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
 			check_and_init_map_value(map, dst_val);
 		}
 		if (do_delete) {
-			hlist_nulls_del_rcu(&l->hash_node);
+			hlist_nulls_del_rcu(&l->node.hash_node);
 
 			/* bpf_lru_push_free() will acquire lru_lock, which
 			 * may cause deadlock. See comments in function
@@ -1948,7 +2067,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
 			 * lock being held and it violates the lock rule, so
 			 * invoke free_htab_elem() after unlock as well.
 			 */
-			l->batch_flink = node_to_free;
+			l->node.batch_flink = node_to_free;
 			node_to_free = l;
 		}
 		dst_key += key_size;
@@ -1960,7 +2079,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
 
 	while (node_to_free) {
 		l = node_to_free;
-		node_to_free = node_to_free->batch_flink;
+		node_to_free = node_to_free->node.batch_flink;
 		if (is_lru_map)
 			htab_lru_push_free(htab, l);
 		else
@@ -2108,8 +2227,8 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info *info,
 		/* no update/deletion on this bucket, prev_elem should be still valid
 		 * and we won't skip elements.
 		 */
-		n = rcu_dereference_raw(hlist_nulls_next_rcu(&prev_elem->hash_node));
-		elem = hlist_nulls_entry_safe(n, struct htab_elem, hash_node);
+		n = rcu_dereference_raw(hlist_nulls_next_rcu(&prev_elem->node.hash_node));
+		elem = hlist_nulls_entry_safe(n, struct htab_elem, node.hash_node);
 		if (elem)
 			return elem;
 
@@ -2125,7 +2244,7 @@ bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info *info,
 
 		count = 0;
 		head = &b->head;
-		hlist_nulls_for_each_entry_rcu(elem, n, head, hash_node) {
+		hlist_nulls_for_each_entry_rcu(elem, n, head, node.hash_node) {
 			if (count >= skip_elems) {
 				info->bucket_id = i;
 				info->skip_elems = count;
@@ -2183,12 +2302,12 @@ static int __bpf_hash_map_seq_show(struct seq_file *seq, struct htab_elem *elem)
 		ctx.meta = &meta;
 		ctx.map = info->map;
 		if (elem) {
-			ctx.key = elem->key;
+			ctx.key = htab_elem_key(info->htab, elem);
 			if (!info->percpu_value_buf) {
-				ctx.value = htab_elem_value(elem, map->key_size);
+				ctx.value = htab_elem_value(info->htab, elem);
 			} else {
 				roundup_value_size = round_up(map->value_size, 8);
-				pptr = htab_elem_get_ptr(elem, map->key_size);
+				pptr = htab_elem_get_ptr(info->htab, elem);
 				for_each_possible_cpu(cpu) {
 					copy_map_value_long(map, info->percpu_value_buf + off,
 							    per_cpu_ptr(pptr, cpu));
@@ -2292,14 +2411,14 @@ static long bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_
 		b = &htab->buckets[i];
 		rcu_read_lock();
 		head = &b->head;
-		hlist_nulls_for_each_entry_safe(elem, n, head, hash_node) {
-			key = elem->key;
+		hlist_nulls_for_each_entry_safe(elem, n, head, node.hash_node) {
+			key = htab_elem_key(htab, elem);
 			if (is_percpu) {
 				/* current cpu value for percpu map */
-				pptr = htab_elem_get_ptr(elem, map->key_size);
+				pptr = htab_elem_get_ptr(htab, elem);
 				val = this_cpu_ptr(pptr);
 			} else {
-				val = htab_elem_value(elem, map->key_size);
+				val = htab_elem_value(htab, elem);
 			}
 			num_elems++;
 			ret = callback_fn((u64)(long)map, (u64)(long)key,
@@ -2403,10 +2522,11 @@ const struct bpf_map_ops htab_lru_map_ops = {
 /* Called from eBPF program */
 static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l = __htab_map_lookup_elem(map, key);
 
 	if (l)
-		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));
+		return this_cpu_ptr(htab_elem_get_ptr(htab, l));
 	else
 		return NULL;
 }
@@ -2414,6 +2534,7 @@ static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key)
 /* inline bpf_map_lookup_elem() call for per-CPU hashmap */
 static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct bpf_insn *insn = insn_buf;
 
 	if (!bpf_jit_supports_percpu_insn())
@@ -2424,7 +2545,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn
 	*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
 	*insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3);
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0,
-				offsetof(struct htab_elem, key) + roundup(map->key_size, 8));
+				htab->key_offset + roundup(map->key_size, 8));
 	*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
 	*insn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
 
@@ -2433,6 +2554,7 @@ static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn
 
 static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l;
 
 	if (cpu >= nr_cpu_ids)
@@ -2440,18 +2562,19 @@ static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key,
 
 	l = __htab_map_lookup_elem(map, key);
 	if (l)
-		return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu);
+		return per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu);
 	else
 		return NULL;
 }
 
 static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l = __htab_map_lookup_elem(map, key);
 
 	if (l) {
-		bpf_lru_node_set_ref(&l->lru_node);
-		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));
+		bpf_lru_node_set_ref(htab_elem_lru_node(l));
+		return this_cpu_ptr(htab_elem_get_ptr(htab, l));
 	}
 
 	return NULL;
@@ -2459,6 +2582,7 @@ static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key)
 
 static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l;
 
 	if (cpu >= nr_cpu_ids)
@@ -2466,8 +2590,8 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k
 
 	l = __htab_map_lookup_elem(map, key);
 	if (l) {
-		bpf_lru_node_set_ref(&l->lru_node);
-		return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu);
+		bpf_lru_node_set_ref(htab_elem_lru_node(l));
+		return per_cpu_ptr(htab_elem_get_ptr(htab, l), cpu);
 	}
 
 	return NULL;
@@ -2475,6 +2599,7 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k
 
 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value, u64 map_flags)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l;
 	void __percpu *pptr;
 	int ret = -ENOENT;
@@ -2494,7 +2619,7 @@ int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value, u64 map_fl
 	/* We do not mark LRU map element here in order to not mess up
 	 * eviction heuristics when user space does a map walk.
 	 */
-	pptr = htab_elem_get_ptr(l, map->key_size);
+	pptr = htab_elem_get_ptr(htab, l);
 	if (map_flags & BPF_F_CPU) {
 		cpu = map_flags >> 32;
 		copy_map_value(map, value, per_cpu_ptr(pptr, cpu));
@@ -2532,6 +2657,7 @@ int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
 static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key,
 					  struct seq_file *m)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct htab_elem *l;
 	void __percpu *pptr;
 	int cpu;
@@ -2546,7 +2672,7 @@ static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key,
 
 	btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);
 	seq_puts(m, ": {\n");
-	pptr = htab_elem_get_ptr(l, map->key_size);
+	pptr = htab_elem_get_ptr(htab, l);
 	for_each_possible_cpu(cpu) {
 		seq_printf(m, "\tcpu%d: ", cpu);
 		btf_type_seq_show(map->btf, map->btf_value_type_id,
@@ -2619,8 +2745,8 @@ static void fd_htab_map_free(struct bpf_map *map)
 	for (i = 0; i < htab->n_buckets; i++) {
 		head = select_bucket(htab, i);
 
-		hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {
-			void *ptr = fd_htab_map_get_ptr(map, l);
+		hlist_nulls_for_each_entry_safe(l, n, head, node.hash_node) {
+			void *ptr = fd_htab_map_get_ptr(htab, l);
 
 			map->ops->map_fd_put_ptr(map, ptr, false);
 		}
@@ -2705,6 +2831,7 @@ static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key)
 static int htab_of_map_gen_lookup(struct bpf_map *map,
 				  struct bpf_insn *insn_buf)
 {
+	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct bpf_insn *insn = insn_buf;
 	const int ret = BPF_REG_0;
 
@@ -2713,7 +2840,7 @@ static int htab_of_map_gen_lookup(struct bpf_map *map,
 	*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
 	*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2);
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,
-				offsetof(struct htab_elem, key) +
+				htab->key_offset +
 				round_up(map->key_size, 8));
 	*insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);
 
@@ -3112,7 +3239,7 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf,
 {
 	struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
 
-	return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor);
+	return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor, offsetof(struct rhtab_elem, data));
 }
 
 static void rhtab_map_free_internal_structs(struct bpf_map *map)
@@ -3533,3 +3660,16 @@ const struct bpf_map_ops rhtab_map_ops = {
 	.map_btf_id = &rhtab_map_btf_ids[0],
 	.iter_seq_info = &rhash_iter_seq_info,
 };
+
+size_t bpf_htab_map_meta_size(void)
+{
+	return sizeof(struct bpf_htab);
+}
+
+void bpf_htab_map_meta_init(struct bpf_map *inner_map_meta, struct bpf_map *inner_map)
+{
+	struct bpf_htab *inner_htab_meta = container_of(inner_map_meta, struct bpf_htab, map);
+	struct bpf_htab *inner_htab = container_of(inner_map, struct bpf_htab, map);
+
+	inner_htab_meta->key_offset = inner_htab->key_offset;
+}
diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c
index d2cbab4bdf644..b2db5d0c98bee 100644
--- a/kernel/bpf/map_in_map.c
+++ b/kernel/bpf/map_in_map.c
@@ -7,6 +7,15 @@
 
 #include "map_in_map.h"
 
+static bool bpf_map_type_is_htab(enum bpf_map_type type)
+{
+	return type == BPF_MAP_TYPE_HASH ||
+	       type == BPF_MAP_TYPE_PERCPU_HASH ||
+	       type == BPF_MAP_TYPE_LRU_HASH ||
+	       type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
+	       type == BPF_MAP_TYPE_HASH_OF_MAPS;
+}
+
 struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
 {
 	struct bpf_map *inner_map, *inner_map_meta;
@@ -29,6 +38,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
 	/* In some cases verifier needs to access beyond just base map. */
 	if (inner_map->ops == &array_map_ops || inner_map->ops == &percpu_array_map_ops)
 		inner_map_meta_size = sizeof(struct bpf_array);
+	else if (bpf_map_type_is_htab(inner_map->map_type))
+		inner_map_meta_size = bpf_htab_map_meta_size();
 
 	inner_map_meta = kzalloc(inner_map_meta_size, GFP_USER);
 	if (!inner_map_meta)
@@ -70,6 +81,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
 		inner_array_meta->index_mask = inner_array->index_mask;
 		inner_array_meta->elem_size = inner_array->elem_size;
 		inner_map_meta->bypass_spec_v1 = inner_map->bypass_spec_v1;
+	} else if (bpf_map_type_is_htab(inner_map->map_type)) {
+		bpf_htab_map_meta_init(inner_map_meta, inner_map);
 	}
 	return inner_map_meta;
 }
diff --git a/kernel/bpf/map_in_map.h b/kernel/bpf/map_in_map.h
index 7d61602354de8..1edc205772d8c 100644
--- a/kernel/bpf/map_in_map.h
+++ b/kernel/bpf/map_in_map.h
@@ -15,5 +15,7 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, struct file *map_file,
 			 int ufd);
 void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer);
 u32 bpf_map_fd_sys_lookup_elem(void *ptr);
+size_t bpf_htab_map_meta_size(void);
+void bpf_htab_map_meta_init(struct bpf_map *inner_map_meta, struct bpf_map *inner_map);
 
 #endif
diff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c
index 373c8d17ea557..6bd4cb68c20c1 100644
--- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c
+++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c
@@ -114,7 +114,7 @@ static inline int check_hash(void)
 	VERIFY(check_default_noinline(&hash->map, map));
 
 	VERIFY(hash->n_buckets == MAX_ENTRIES);
-	VERIFY(hash->elem_size == 64);
+	VERIFY(hash->elem_size == 32);
 
 	VERIFY(hash->count.counter == 0);
 	VERIFY(bpf_map_sum_elem_count(map) == 0);