This patch adds support for crafting non-linear skbs in BPF test runs for tc programs, via a new flag BPF_F_TEST_SKB_NON_LINEAR. When this flag is set, only the L2 header is pulled in the linear area. This is particularly useful to test support for non-linear skbs in large codebases such as Cilium. We've had multiple bugs in the past few years where we were missing calls to bpf_skb_pull_data(). This support in BPF_PROG_TEST_RUN would allow us to automatically cover this case in our BPF tests. In addition to the selftests introduced later in the series, this patch was tested by setting BPF_F_TEST_SKB_NON_LINEAR for all tc selftests programs and checking test failures were expected. Suggested-by: Daniel Borkmann Signed-off-by: Paul Chaignon --- include/uapi/linux/bpf.h | 2 + net/bpf/test_run.c | 88 ++++++++++++++++++++++++---------- tools/include/uapi/linux/bpf.h | 2 + 3 files changed, 66 insertions(+), 26 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 233de8677382..d77c8bf4b131 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1448,6 +1448,8 @@ enum { #define BPF_F_TEST_XDP_LIVE_FRAMES (1U << 1) /* If set, apply CHECKSUM_COMPLETE to skb and validate the checksum */ #define BPF_F_TEST_SKB_CHECKSUM_COMPLETE (1U << 2) +/* If set, skb will be non-linear */ +#define BPF_F_TEST_SKB_NON_LINEAR (1U << 3) /* type for BPF_ENABLE_STATS */ enum bpf_stats_type { diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 4e595b7ad94f..645b7b5af08f 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -660,20 +660,29 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE) BTF_KFUNCS_END(test_sk_check_kfunc_ids) static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size, - u32 size, u32 headroom, u32 tailroom) + u32 size, u32 headroom, u32 tailroom, bool nonlinear) { void __user *data_in = u64_to_user_ptr(kattr->test.data_in); - void *data; + void *data, *dst; if (user_size < ETH_HLEN || user_size > PAGE_SIZE - headroom - tailroom) return ERR_PTR(-EINVAL); - size = SKB_DATA_ALIGN(size); - data = kzalloc(size + headroom + tailroom, GFP_USER); + /* In non-linear case, data_in is copied to the paged data */ + if (nonlinear) { + data = alloc_page(GFP_USER); + } else { + size = SKB_DATA_ALIGN(size); + data = kzalloc(size + headroom + tailroom, GFP_USER); + } if (!data) return ERR_PTR(-ENOMEM); - if (copy_from_user(data + headroom, data_in, user_size)) { + if (nonlinear) + dst = page_address(data); + else + dst = data + headroom; + if (copy_from_user(dst, data_in, user_size)) { kfree(data); return ERR_PTR(-EFAULT); } @@ -984,7 +993,7 @@ static struct proto bpf_dummy_proto = { int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr) { - bool is_l2 = false, is_direct_pkt_access = false; + bool is_l2 = false, is_direct_pkt_access = false, is_nonlinear = false; struct net *net = current->nsproxy->net_ns; struct net_device *dev = net->loopback_dev; u32 size = kattr->test.data_size_in; @@ -997,21 +1006,11 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, void *data; int ret; - if ((kattr->test.flags & ~BPF_F_TEST_SKB_CHECKSUM_COMPLETE) || + if ((kattr->test.flags & ~(BPF_F_TEST_SKB_CHECKSUM_COMPLETE | BPF_F_TEST_SKB_NON_LINEAR)) || kattr->test.cpu || kattr->test.batch_size) return -EINVAL; - data = bpf_test_init(kattr, kattr->test.data_size_in, - size, NET_SKB_PAD + NET_IP_ALIGN, - SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); - if (IS_ERR(data)) - return PTR_ERR(data); - - ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); - if (IS_ERR(ctx)) { - ret = PTR_ERR(ctx); - goto out; - } + is_nonlinear = kattr->test.flags & BPF_F_TEST_SKB_NON_LINEAR; switch (prog->type) { case BPF_PROG_TYPE_SCHED_CLS: @@ -1028,6 +1027,22 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, break; } + if (is_nonlinear && !is_l2) + return -EINVAL; + + data = bpf_test_init(kattr, kattr->test.data_size_in, + size, NET_SKB_PAD + NET_IP_ALIGN, + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), + is_nonlinear); + if (IS_ERR(data)) + return PTR_ERR(data); + + ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); + if (IS_ERR(ctx)) { + ret = PTR_ERR(ctx); + goto out; + } + sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1); if (!sk) { ret = -ENOMEM; @@ -1035,15 +1050,32 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, } sock_init_data(NULL, sk); - skb = slab_build_skb(data); + if (is_nonlinear) + skb = alloc_skb(NET_SKB_PAD + NET_IP_ALIGN + size + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), + GFP_USER); + else + skb = slab_build_skb(data); if (!skb) { ret = -ENOMEM; goto out; } + skb->sk = sk; skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); - __skb_put(skb, size); + + if (is_nonlinear) { + skb_fill_page_desc(skb, 0, data, 0, size); + skb->truesize += PAGE_SIZE; + skb->data_len = size; + skb->len = size; + + /* eth_type_trans expects the Ethernet header in the linear area. */ + __pskb_pull_tail(skb, ETH_HLEN); + } else { + __skb_put(skb, size); + } data = NULL; /* data released via kfree_skb */ @@ -1126,9 +1158,11 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, convert_skb_to___skb(skb, ctx); size = skb->len; - /* bpf program can never convert linear skb to non-linear */ - if (WARN_ON_ONCE(skb_is_nonlinear(skb))) + if (skb_is_nonlinear(skb)) { + /* bpf program can never convert linear skb to non-linear */ + WARN_ON_ONCE(!is_nonlinear); size = skb_headlen(skb); + } ret = bpf_test_finish(kattr, uattr, skb->data, NULL, size, retval, duration); if (!ret) @@ -1138,7 +1172,8 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, if (dev && dev != net->loopback_dev) dev_put(dev); kfree_skb(skb); - kfree(data); + if (data) + is_nonlinear ? __free_page(data) : kfree(data); if (sk) sk_free(sk); kfree(ctx); @@ -1264,7 +1299,7 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, size = max_data_sz; } - data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom); + data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom, false); if (IS_ERR(data)) { ret = PTR_ERR(data); goto free_ctx; @@ -1387,7 +1422,7 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, if (size < ETH_HLEN) return -EINVAL; - data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0); + data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0, false); if (IS_ERR(data)) return PTR_ERR(data); @@ -1660,7 +1695,8 @@ int bpf_prog_test_run_nf(struct bpf_prog *prog, data = bpf_test_init(kattr, kattr->test.data_size_in, size, NET_SKB_PAD + NET_IP_ALIGN, - SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), + false); if (IS_ERR(data)) return PTR_ERR(data); diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 233de8677382..d77c8bf4b131 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1448,6 +1448,8 @@ enum { #define BPF_F_TEST_XDP_LIVE_FRAMES (1U << 1) /* If set, apply CHECKSUM_COMPLETE to skb and validate the checksum */ #define BPF_F_TEST_SKB_CHECKSUM_COMPLETE (1U << 2) +/* If set, skb will be non-linear */ +#define BPF_F_TEST_SKB_NON_LINEAR (1U << 3) /* type for BPF_ENABLE_STATS */ enum bpf_stats_type { -- 2.43.0