When a TCP skb is queued to sk->sk_receive_queue, BPF SOCK_OPS prog can be called with BPF_SOCK_OPS_RCVQ_CB. In this hook, we want to parse the RPC descriptor in the skb and adjust sk->sk_rcvlowat based on the RPC frame size. However, we cannot access payload via bpf_sock_ops.data on modern NICs with TCP header/data split on as the payload is not placed in the linear area. Let's support bpf_skb_load_bytes() for BPF_SOCK_OPS_RCVQ_CB. Three notes: 1) bpf_sock_ops_kern.skb will be NULL when the BPF prog is invoked from recvmsg(). 2) Access to bpf_sock_ops.data will be disabled by passing 0 end_offset to bpf_skops_init_skb(). 3) ____bpf_skb_load_bytes() is called directly instead of __bpf_skb_load_bytes() to allow compilers to inline it instead of generating a tail-call. Signed-off-by: Kuniyuki Iwashima --- v2: Explain why using ____ version instead of __ --- net/core/filter.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/net/core/filter.c b/net/core/filter.c index 4a50fe2cd863..fa8a7c7d86eb 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -7760,6 +7760,38 @@ static const struct bpf_func_proto bpf_sk_assign_proto = { .arg3_type = ARG_ANYTHING, }; +BPF_CALL_4(bpf_sock_ops_skb_load_bytes, struct bpf_sock_ops_kern *, bpf_sock, + u32, offset, void *, to, u32, len) +{ + int err; + + if (bpf_sock->op != BPF_SOCK_OPS_RCVQ_CB) { + err = -EOPNOTSUPP; + goto err_clear; + } + + if (!bpf_sock->skb) { + err = -EPERM; + goto err_clear; + } + + return ____bpf_skb_load_bytes(bpf_sock->skb, offset, to, len); + +err_clear: + memset(to, 0, len); + return err; +} + +static const struct bpf_func_proto bpf_sock_ops_skb_load_bytes_proto = { + .func = bpf_sock_ops_skb_load_bytes, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_PTR_TO_UNINIT_MEM, + .arg4_type = ARG_CONST_SIZE, +}; + static const u8 *bpf_search_tcp_opt(const u8 *op, const u8 *opend, u8 search_kind, const u8 *magic, u8 magic_len, bool *eol) @@ -8616,6 +8648,8 @@ sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_get_netns_cookie: return &bpf_get_netns_cookie_sock_ops_proto; #ifdef CONFIG_INET + case BPF_FUNC_skb_load_bytes: + return &bpf_sock_ops_skb_load_bytes_proto; case BPF_FUNC_load_hdr_opt: return &bpf_sock_ops_load_hdr_opt_proto; case BPF_FUNC_store_hdr_opt: -- 2.54.0.746.g67dd491aae-goog