With the introduction of bpf_strstr, bpf_strnstr, bpf_strcasestr, and bpf_strcasecmp in Linux 6.17–6.19, BPF programs gained solid support for floating-offset substring search in null-terminated C-strings. However, in network packet inspection (TC/XDP, TLS SNI tagging, binary protocol delimiters) and security monitoring (e.g. BPF LSM hooks inspecting raw memory or binary execution buffers), data buffers are frequently not null-terminated and often contain embedded '\0' bytes. For these use cases, existing string kfuncs terminate prematurely at the first null byte. Currently, developers attempting to search variable-length binary payloads must choose between: 1. Writing a loop using bpf_loop() or open-coded iterators: Even with verifier inlining, this executes 1 subprogram call per byte and cannot leverage 64-bit word-at-a-time SIMD scanning, making it significantly slower than kernel-internal scanning. 2. Streaming raw buffers to user space via BPF ring buffers: This induces heavy memory copying and context-switch latency, often leading to ring buffer drops under high-throughput workloads. To address this gap, this RFC proposes two complementary kfuncs in kernel/bpf/helpers.c: - bpf_memmem(): Searches for a byte sequence within a flat memory slice. - bpf_dynptr_memmem(): Searches for a byte sequence within a dynamic pointer (struct bpf_dynptr). Both functions wrap an internal allocation-free scanning routine based on the kernel's optimized memchr() followed by memcmp(), returning the matched byte offset or -ENOENT. Buffer sizes are capped at BPF_MEMMEM_MAX_LEN (2 MB) and needle patterns at BPF_MEMMEM_MAX_NEEDLE_SZ (4 KB). To prevent pathological quadratic O(N * M) RCU CPU stalls, cumulative byte comparisons across memcmp calls are strictly bounded by BPF_MEMMEM_MAX_LEN. Verifier bounds safety is enforced via the standard __sz naming convention and dynptr type tracking. Both kfuncs are registered in common_btf_ids to allow access across all program types. Signed-off-by: Abhishek Khade --- kernel/bpf/helpers.c | 116 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 6788489a2..1f27ad21a 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -3406,6 +3406,120 @@ __bpf_kfunc int bpf_dynptr_memset(const struct bpf_dynptr *p, u64 offset, u64 si return 0; } +#define BPF_MEMMEM_MAX_LEN (2 * 1024 * 1024) +#define BPF_MEMMEM_MAX_NEEDLE_SZ 4096 + +/* In-kernel binary byte sequence search using memchr() and memcmp() */ +static void *__bpf_memmem(const void *haystack, size_t haystack_sz, + const void *needle, size_t needle_sz) +{ + const char *h = haystack; + const char *n = needle; + const char *end; + size_t total_cmp = 0; + + if (needle_sz == 0) + return (void *)haystack; + if (needle_sz > haystack_sz) + return NULL; + if (needle_sz == 1) + return memchr(haystack, n[0], haystack_sz); + + end = h + haystack_sz - needle_sz + 1; + while ((h = memchr(h, n[0], end - h)) != NULL) { + total_cmp += needle_sz; + if (total_cmp > BPF_MEMMEM_MAX_LEN) + return NULL; + if (memcmp(h, n, needle_sz) == 0) + return (void *)h; + h++; + } + + return NULL; +} + +/** + * bpf_memmem - search for needle byte pattern in a flat memory buffer + * @haystack: pointer to haystack memory buffer + * @haystack__sz: size of haystack buffer in bytes + * @needle: pointer to byte pattern to search for + * @needle__sz: size of pattern in bytes + * @flags: reserved for future options (must be 0) + * + * Return: + * * >=0 - 0-based byte offset of match in haystack + * * %-ENOENT - needle was not found in haystack + * * %-EINVAL - invalid flags or parameters + * * %-E2BIG - haystack exceeds 2 MB or needle exceeds 4 KB + */ +__bpf_kfunc long bpf_memmem(const void *haystack, u32 haystack__sz, + const void *needle, u32 needle__sz, + u64 flags) +{ + const void *match; + + if (flags) + return -EINVAL; + if (!haystack || !needle || needle__sz == 0 || haystack__sz == 0) + return -EINVAL; + if (needle__sz > haystack__sz) + return -EINVAL; + if (haystack__sz > BPF_MEMMEM_MAX_LEN || needle__sz > BPF_MEMMEM_MAX_NEEDLE_SZ) + return -E2BIG; + + match = __bpf_memmem(haystack, haystack__sz, needle, needle__sz); + if (!match) + return -ENOENT; + + return (long)((const char *)match - (const char *)haystack); +} + +/** + * bpf_dynptr_memmem - search for needle within a dynamic pointer + * @haystack: pointer to dynptr representing the search buffer + * @needle: pointer to pattern to search for + * @needle__sz: size of pattern in bytes + * @flags: reserved for future options (must be 0) + * + * Return: + * * >=0 - 0-based byte offset of match in dynptr buffer + * * %-ENOENT - needle was not found + * * %-EINVAL - invalid flags, non-linear buffer, or parameters + * * %-E2BIG - dynptr buffer exceeds 2 MB or needle exceeds 4 KB + */ +__bpf_kfunc long bpf_dynptr_memmem(const struct bpf_dynptr *haystack, + const void *needle, u32 needle__sz, + u64 flags) +{ + struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)haystack; + const void *data; + const void *match; + u64 size; + + if (flags) + return -EINVAL; + if (!ptr || !needle || needle__sz == 0) + return -EINVAL; + if (needle__sz > BPF_MEMMEM_MAX_NEEDLE_SZ) + return -E2BIG; + + size = __bpf_dynptr_size(ptr); + if (size == 0 || needle__sz > size) + return -EINVAL; + if (size > BPF_MEMMEM_MAX_LEN) + return -E2BIG; + + data = __bpf_dynptr_data(ptr, size); + if (!data) + return -EINVAL; + + match = __bpf_memmem(data, size, needle, needle__sz); + if (!match) + return -ENOENT; + + return (long)((const char *)match - (const char *)data); +} + __bpf_kfunc void *bpf_cast_to_kern_ctx(void *obj) { return obj; @@ -5214,6 +5328,8 @@ BTF_ID_FLAGS(func, bpf_dynptr_size) BTF_ID_FLAGS(func, bpf_dynptr_clone) BTF_ID_FLAGS(func, bpf_dynptr_copy) BTF_ID_FLAGS(func, bpf_dynptr_memset) +BTF_ID_FLAGS(func, bpf_memmem) +BTF_ID_FLAGS(func, bpf_dynptr_memmem) #ifdef CONFIG_NET BTF_ID_FLAGS(func, bpf_modify_return_test_tp) #endif -- 2.53.0