From: Vineet Gupta A fix of precision in the main pass. It stands alone in this series. Its tests come with the tests of sign extensions of linked registers. 1: w0 &= 63 2: w0 -= 63 3: w0 = (s8)w0 4: if w0 s< 0xffffffc1 goto pc+2 .. 7: w0 /= 0 div by zero w0 is in [-63, 0] before 3. Every value of that range fits in s8, so the sign extension is a no-op. The verifier drops the range all the same: after 3 w0 is in [-128, 127], is_branch_taken() cannot tell at 4, and 7 is verified though that branch is dead. coerce_subreg_to_size_sx() keeps a range if both bounds have the same bits above the low 8 or 16 bits. In a range that crosses 0 they differ: all ones in -63, all zeros in 0. The check before that one is on umax, which is huge when the range has a negative value. Return early if the 32-bit range fits in s8 or s16. A register that passed the check of umax has such a range, so that check goes. coerce_reg_to_size_sx() does the same in 64 bits since commit 0892ef2a7ef3 ("bpf: Representation for intervals with steps"). No verdict of the selftests changes. Fixes: 8100928c8814 ("bpf: Support new sign-extension mov insns") Signed-off-by: Vineet Gupta Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 20b1b9ff574c..5bffc1e288f6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6441,14 +6441,28 @@ static void set_sext32_default_val(struct bpf_reg_state *reg, int size) static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size) { s32 init_s32_max, init_s32_min, s32_max, s32_min, u32_val; + s32 field_smin, field_smax; u32 top_smax_value, top_smin_value; u32 num_bits = size * 8; + if (size == 1) { + field_smin = S8_MIN; + field_smax = S8_MAX; + } else { + /* size == 2 */ + field_smin = S16_MIN; + field_smax = S16_MAX; + } + /* - * If value fits the non-negative half of the signed N-bit range, - * sign extension is a noop. + * The range already fits the signed N-bit range, so (sN)v == v for every + * value of the low 32 bits and the sign extension is a no-op. + * + * This has to precede reg_step_reset() below: the base/step line stays. + * It is about all 64 bits: zext_32_to_64() in the caller discards it + * if the upper 32 bits are not zero. */ - if (reg_umax(reg) < (1ULL << (num_bits - 1))) + if (reg_s32_min(reg) >= field_smin && reg_s32_max(reg) <= field_smax) return; /* See comment at reg_step_check_unsigned(). */ -- 2.55.0 From: Alexei Starovoitov A fix of the representation, for the patches that take two registers with the same expression for the same value. Today two insns that compute distinct values get the same expression: r5 >>= r4; w5 = w5 (zext32 (>> r5 r4)) w5 >>= w4 (zext32 (>> r5 r4)) The second one shifts the low 32 bits of r5, by r4 & 31. Nothing goes wrong so far: the code that analyzes loops does not look into a shift or a division. cubic_root() of bpf_cubic.c is going to depend on it: shift = (a >> (b * 3)); if (shift >= 64) return 0; x = ((__u32)(((__u32)v[shift] + 10) << b)) >> 6; GCC tests one register and uses another one as the index: r5 >>= r4 w2 = w5 if w5 > 0x3f goto .. r4 = v ll r4 += r2 The prog loads because r5 fits in u32 in the state: 'w2 = w5' assigns one scalar id to r2 and r5, so the jump bounds both. Without scalar ids it is going to load because r5 is (>> a b) and r2 is (zext32 (>> a b)), which is r5 truncated to 32 bits. An ALU32 insn is (zext32 (op a b)) with the 64-bit expressions of its operands. That is exact for + - * | & ^ and for the value that << shifts: the low 32 bits of the result depend on the low 32 bits of the operands only. It is not exact for a right shift, a division, a modulo and a shift amount. Zero or sign extend these operands, mask the shift amount: w5 >>= w4 (zext32 (>> (zext32 r5) (& r4 31))) w5 s>>= 3 (zext32 (s>> (sext32 r5) 3)) w5 /= w4 (zext32 (/ (zext32 r5) (zext32 r4))) w5 s%= w4 (zext32 (s% (sext32 r5) (sext32 r4))) Every expression is a 64-bit operation now. (>> x 32) shifts by 32. Before the patch it could also be 'w1 >>= w2' with 32 in w2, which shifts by 0. Signed-off-by: Alexei Starovoitov --- kernel/bpf/scev.c | 46 +++++++++++++++---- .../selftests/bpf/progs/verifier_scev.c | 26 +++++------ 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 4187ce955be9..1cb4bc7c0f96 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -686,6 +686,38 @@ static int maybe_load_fp(struct scev *scev, struct env *e, struct bpf_insn *insn return replace_reg(scev, e, insn->dst_reg, id); } +/* + * The low 32 bits of a + b depend on the low 32 bits of a and b only: 'w1 += w2' is + * (zext32 (+ a b)). That doesn't hold for a right shift and for a division. Their operands are zero + * or sign extended and a shift amount is masked, so that every expression is a 64-bit operation. + */ +static void alu32_operands(struct scev *scev, u32 op, int *a, int *b) +{ + s64 imm = 0; + bool imm_b = *b >= 0 && is_imm(scev, *b, &imm); + + switch (op) { + case BPF_RSH: + case BPF_ARSH: + *a = expr1(scev, op == BPF_RSH ? ZEXT32 : SEXT32, *a); + fallthrough; + case BPF_LSH: + *b = imm_b ? imm_expr(scev, imm & 31) : + expr2(scev, BPF_AND, *b, imm_expr(scev, 31)); + break; + case BPF_DIV: + case BPF_MOD: + *a = expr1(scev, ZEXT32, *a); + *b = imm_b ? imm_expr(scev, (u32)imm) : expr1(scev, ZEXT32, *b); + break; + case SDIV: + case SMOD: + *a = expr1(scev, SEXT32, *a); + *b = imm_b ? imm_expr(scev, (s32)imm) : expr1(scev, SEXT32, *b); + break; + } +} + static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) { const bool little_endian = htons(0x3412) == 0x1234; @@ -700,7 +732,7 @@ static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) u32 src = insn->src_reg; u32 op, sext, l, r; bool need_bswap; - int i, id; + int i, id, a; s64 imm; /* reg2expr[12] is a stack slot, not r12 */ @@ -764,19 +796,17 @@ static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) else goto mark_dst_unknown; - /* - * 'r1 <<= 32; r1 >>= 32' is (u32)r1 for cpu v1 and v2. Only here both - * shifts are known to be 64-bit ones: the result of a 32-bit shift is - * (zext32 ...), and (>> x 32) is 'w1 >>= w2' with 32 in w2 as well, - * which shifts by 0. - */ + /* 'r1 <<= 32; r1 >>= 32' is (u32)r1 for cpu v1 and v2 */ if (class == BPF_ALU64 && op == BPF_RSH && id >= 0 && is_imm(scev, id, &imm) && imm == 32 && is_binop(scev, BPF_LSH, reg2expr[dst], &l, &r) && is_imm(scev, r, &imm) && imm == 32) return replace_reg(scev, e, dst, expr1(scev, ZEXT32, l)); - id = expr2(scev, op, reg2expr[dst], id); + a = reg2expr[dst]; + if (class == BPF_ALU) + alu32_operands(scev, op, &a, &id); + id = expr2(scev, op, a, id); if (class == BPF_ALU) id = expr1(scev, ZEXT32, id); diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 5c9c832fbbe0..f2a53fe5f6ad 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -334,20 +334,20 @@ __msg("w1 *= 2 {{.*}}; r1 r6 -> (zext32 (* r6 2))") __msg("w1 *= w7 {{.*}}; r1 r6 -> (zext32 (* r6 r7))") __msg("r1 /= 2 {{.*}}; r1 r6 -> (/ r6 2)") __msg("r1 /= r7 {{.*}}; r1 r6 -> (/ r6 r7)") -__msg("w1 /= 2 {{.*}}; r1 r6 -> (zext32 (/ r6 2))") -__msg("w1 /= w7 {{.*}}; r1 r6 -> (zext32 (/ r6 r7))") +__msg("w1 /= 2 {{.*}}; r1 r6 -> (zext32 (/ (zext32 r6) 2))") +__msg("w1 /= w7 {{.*}}; r1 r6 -> (zext32 (/ (zext32 r6) (zext32 r7)))") __msg("r1 s/= 2 {{.*}}; r1 r6 -> (s/ r6 2)") __msg("r1 s/= r7 {{.*}}; r1 r6 -> (s/ r6 r7)") -__msg("w1 s/= 2 {{.*}}; r1 r6 -> (zext32 (s/ r6 2))") -__msg("w1 s/= w7 {{.*}}; r1 r6 -> (zext32 (s/ r6 r7))") +__msg("w1 s/= 2 {{.*}}; r1 r6 -> (zext32 (s/ (sext32 r6) 2))") +__msg("w1 s/= w7 {{.*}}; r1 r6 -> (zext32 (s/ (sext32 r6) (sext32 r7)))") __msg("r1 %= 2 {{.*}}; r1 r6 -> (% r6 2)") __msg("r1 %= r7 {{.*}}; r1 r6 -> (% r6 r7)") -__msg("w1 %= 2 {{.*}}; r1 r6 -> (zext32 (% r6 2))") -__msg("w1 %= w7 {{.*}}; r1 r6 -> (zext32 (% r6 r7))") +__msg("w1 %= 2 {{.*}}; r1 r6 -> (zext32 (% (zext32 r6) 2))") +__msg("w1 %= w7 {{.*}}; r1 r6 -> (zext32 (% (zext32 r6) (zext32 r7)))") __msg("r1 s%= 2 {{.*}}; r1 r6 -> (s% r6 2)") __msg("r1 s%= r7 {{.*}}; r1 r6 -> (s% r6 r7)") -__msg("w1 s%= 2 {{.*}}; r1 r6 -> (zext32 (s% r6 2))") -__msg("w1 s%= w7 {{.*}}; r1 r6 -> (zext32 (s% r6 r7))") +__msg("w1 s%= 2 {{.*}}; r1 r6 -> (zext32 (s% (sext32 r6) 2))") +__msg("w1 s%= w7 {{.*}}; r1 r6 -> (zext32 (s% (sext32 r6) (sext32 r7)))") __msg("r1 |= 2 {{.*}}; r1 r6 -> (| r6 2)") __msg("r1 |= r7 {{.*}}; r1 r6 -> (| r6 r7)") __msg("w1 |= 2 {{.*}}; r1 r6 -> (zext32 (| r6 2))") @@ -363,15 +363,15 @@ __msg("w1 ^= w7 {{.*}}; r1 r6 -> (zext32 (^ r6 r7))") __msg("r1 <<= 2 {{.*}}; r1 r6 -> (<< r6 2)") __msg("r1 <<= r7 {{.*}}; r1 r6 -> (<< r6 r7)") __msg("w1 <<= 2 {{.*}}; r1 r6 -> (zext32 (<< r6 2))") -__msg("w1 <<= w7 {{.*}}; r1 r6 -> (zext32 (<< r6 r7))") +__msg("w1 <<= w7 {{.*}}; r1 r6 -> (zext32 (<< r6 (& r7 31)))") __msg("r1 >>= 2 {{.*}}; r1 r6 -> (>> r6 2)") __msg("r1 >>= r7 {{.*}}; r1 r6 -> (>> r6 r7)") -__msg("w1 >>= 2 {{.*}}; r1 r6 -> (zext32 (>> r6 2))") -__msg("w1 >>= w7 {{.*}}; r1 r6 -> (zext32 (>> r6 r7))") +__msg("w1 >>= 2 {{.*}}; r1 r6 -> (zext32 (>> (zext32 r6) 2))") +__msg("w1 >>= w7 {{.*}}; r1 r6 -> (zext32 (>> (zext32 r6) (& r7 31)))") __msg("r1 s>>= 2 {{.*}}; r1 r6 -> (s>> r6 2)") __msg("r1 s>>= r7 {{.*}}; r1 r6 -> (s>> r6 r7)") -__msg("w1 s>>= 2 {{.*}}; r1 r6 -> (zext32 (s>> r6 2))") -__msg("w1 s>>= w7 {{.*}}; r1 r6 -> (zext32 (s>> r6 r7))") +__msg("w1 s>>= 2 {{.*}}; r1 r6 -> (zext32 (s>> (sext32 r6) 2))") +__msg("w1 s>>= w7 {{.*}}; r1 r6 -> (zext32 (s>> (sext32 r6) (& r7 31)))") __msg("r1 = -r1 {{.*}}; r1 r6 -> (- r6)") __msg("w1 = -w1 {{.*}}; r1 r6 -> (zext32 (- r6))") __msg("r1 = 42 {{.*}}; r1 r6 -> 42") -- 2.55.0 From: Alexei Starovoitov This is about the cost of scalar evolution. The time in scev.c goes down by a constant factor: 1.7 per insn, 5 per subprog, 11 for the smallest prog. No expression changes, only the way expressions are stored and found. Scalar evolution analyzes loops only. A later patch computes expressions for whole subprogs, to find the registers that are linked at a conditional jump: j = i + 1; if (i > 100) return 0; use(buf[j]); Then every prog pays for it, with a loop or without, and 1.8 expressions are constructed per insn, not 0.13. Two things that a few loops do not feel would be a third of that cost: - Per expression: expressions are hash consed, and each bucket of exprs_ht is an array of ids of its own. That is a kvrealloc(), charged to the memcg, and a kvfree() for every second expression. The hash table has max(256, len / 4) buckets and never grows. - Per loop: the analysis starts with a lookup of (REG r) for all 267 registers and stack slots. Each subprog would do the same. The first one constructs them: 'r0 = 0; exit' would take 180 allocations. Chain the ids of a bucket through exprs_next[], an array next to exprs[]. exprs_ht[] grows with exprs[]. A load has these three arrays, which double. (REG r) has the fixed id 2 + r and is not in the hash table. What it would cost to compute expressions for all code of the selftests, Cilium, sched_ext, katran, bpfjailer and programs in Rust. The last two rows are against loops only: before after allocations in add_expr() per load 471 6 scev.c for 'r0 = 0; exit' 12.1 us 1.1 us scev.c for each more subprog 3.6 us 0.7 us scev.c per insn 485 ns 291 ns kernel instructions (perf stat) +6.2 % +4.2 % load time (veristat) +8.5 % +5.7 % The allocations go with the chains, the cost per subprog with the fixed ids, the cost of 'r0 = 0; exit' with both: the first lookups of (REG r) fill the buckets. Not joining dead registers takes most of what is left. The time in scev.c is linear in the number of insns before and after, so small progs gain most: 12 % of the load time up to 100 insns, 3 % from 1000 to 10000, 2 % above. With loops only, as today, the kernel spends 0.3 % fewer instructions on loading these progs. Signed-off-by: Alexei Starovoitov --- kernel/bpf/scev.c | 125 +++++++++++++++++++++++----------------------- 1 file changed, 62 insertions(+), 63 deletions(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 1cb4bc7c0f96..8f8ca69482ca 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -14,6 +14,8 @@ #define REGS_NUM BPF_SCEV_REGS_NUM #define UNKNOWN_EXPR_ID 0 #define OPAQUE_EXPR_ID 1 +/* (REG r). UNKNOWN, OPAQUE and (REG r) have fixed ids and are not in the hash table. */ +#define REG_EXPR_ID(r) (2 + (r)) /* * BPF instructions use 'code', 'src_reg', 'off' and 'imm' fields for instruction encoding. @@ -58,12 +60,6 @@ struct expr { }; }; -struct expr_bucket { - u32 cnt; - u32 cap; - u32 ids[]; -}; - struct env { bool empty; u32 reg2expr[REGS_NUM]; @@ -96,8 +92,11 @@ struct scev { * Expressions are identified by id, exprs_ht ensures that * each expression exists as a unique instance. * This allows for fast equivalence check: id1 === id2. + * exprs_ht[hash] is the first id of a chain, exprs_next[id] is the next one, 0 ends it. + * Both have exprs_cap elements, like exprs[]. */ - struct expr_bucket **exprs_ht; + u32 *exprs_ht; + u32 *exprs_next; struct scev_worklist worklist; struct insn_envs **envs; struct expr *exprs; @@ -106,7 +105,6 @@ struct scev { * instruction was visited on a current pass. */ u32 *discovered; - u32 exprs_ht_cnt; int exprs_cnt; int exprs_cap; int envs_cnt; @@ -125,44 +123,58 @@ static int expr_eq(struct expr *a, struct expr *b) return a->op == b->op && a->imm == b->imm; } -static int add_expr(struct scev *scev, struct expr e) +static void exprs_ht_add(struct scev *scev, u32 id) +{ + u32 *bucket = &scev->exprs_ht[expr_hash(&scev->exprs[id]) & (scev->exprs_cap - 1)]; + + scev->exprs_next[id] = *bucket; + *bucket = id; +} + +/* @cap is a power of two. No warning for a size that kvmalloc() rejects. */ +static int grow_exprs(struct scev *scev, u32 cap) { - struct expr_bucket *bucket; - u32 i, id, hash, new_cap; + gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_NOWARN; void *tmp; + u32 id; - hash = expr_hash(&e) & (scev->exprs_ht_cnt - 1); - bucket = scev->exprs_ht[hash]; + tmp = kvrealloc(scev->exprs, array_size(cap, sizeof(*scev->exprs)), gfp); + if (!tmp) + return -ENOMEM; + scev->exprs = tmp; + tmp = kvrealloc(scev->exprs_next, array_size(cap, sizeof(u32)), gfp); + if (!tmp) + return -ENOMEM; + scev->exprs_next = tmp; + tmp = kvrealloc(scev->exprs_ht, array_size(cap, sizeof(u32)), gfp); + if (!tmp) + return -ENOMEM; + scev->exprs_ht = tmp; + scev->exprs_cap = cap; + memset(scev->exprs_ht, 0, array_size(cap, sizeof(u32))); + for (id = REG_EXPR_ID(REGS_NUM); id < scev->exprs_cnt; id++) + exprs_ht_add(scev, id); + return 0; +} - if (bucket) { - for (i = 0; i < bucket->cnt; i++) { - id = bucket->ids[i]; - if (expr_eq(&e, &scev->exprs[id])) - return id; - } - } +static int add_expr(struct scev *scev, struct expr e) +{ + u32 id; - if (!bucket || bucket->cap == bucket->cnt) { - new_cap = bucket ? bucket->cap * 2 : 4; - bucket = kvrealloc(bucket, sizeof(*bucket) + sizeof(u32) * new_cap, GFP_KERNEL_ACCOUNT | __GFP_ZERO); - if (!bucket) - return -ENOMEM; - scev->exprs_ht[hash] = bucket; - bucket->cap = new_cap; - } + if (e.op == REG) + return REG_EXPR_ID(e.params[0]); - if (scev->exprs_cnt == scev->exprs_cap) { - new_cap = scev->exprs_cap ? scev->exprs_cap * 2 : 256; - tmp = kvrealloc(scev->exprs, sizeof(struct expr) * new_cap, GFP_KERNEL_ACCOUNT); - if (!tmp) - return -ENOMEM; - scev->exprs = tmp; - scev->exprs_cap = new_cap; - } + id = scev->exprs_ht[expr_hash(&e) & (scev->exprs_cap - 1)]; + for (; id; id = scev->exprs_next[id]) + if (expr_eq(&e, &scev->exprs[id])) + return id; + + if (scev->exprs_cnt == scev->exprs_cap && grow_exprs(scev, scev->exprs_cap * 2)) + return -ENOMEM; id = scev->exprs_cnt++; scev->exprs[id] = e; - bucket->ids[bucket->cnt++] = id; + exprs_ht_add(scev, id); return id; } @@ -182,11 +194,6 @@ static int expr1(struct scev *scev, u32 op, int a) return expr2(scev, op, a, 0); } -static int expr0(struct scev *scev, u32 op) -{ - return expr2(scev, op, 0, 0); -} - static int imm_expr(struct scev *scev, s64 value) { return add_expr(scev, (struct expr){ .op = IMM, .imm = value }); @@ -576,10 +583,9 @@ static int replace_reg(struct scev *scev, struct env *e, u32 reg, int id) return 0; } -static int setup_initial_loop_env(struct bpf_verifier_env *env, struct env *e, int insn_idx) +static void setup_initial_loop_env(struct env *e) { - struct scev *scev = env->scev; - int i, err; + int i; /* * Initial assignments are of form rX -> rX are needed as seeds for subsequent @@ -587,12 +593,8 @@ static int setup_initial_loop_env(struct bpf_verifier_env *env, struct env *e, i * join() will overwrite the them. compute_header_scevs() accounts for register * states at loop entry. */ - for (i = 0; i < REGS_NUM; i++) { - err = replace_reg(scev, e, i, expr1(scev, REG, i)); - if (err) - return err; - } - return 0; + for (i = 0; i < REGS_NUM; i++) + e->reg2expr[i] = REG_EXPR_ID(i); } static void forget_call_regs(struct env *e) @@ -1404,9 +1406,7 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) header_env = get_loop_env(scev, cur_header, cur_header); if (!header_env) goto nomem; - err = setup_initial_loop_env(env, header_env, cur_header); - if (err) - goto out; + setup_initial_loop_env(header_env); err = worklist_push(env, cur_header, cur_header); if (err) goto out; @@ -1966,10 +1966,8 @@ void bpf_free_scev(struct bpf_verifier_env *env) kfree(envs->entries[j].env); kfree(envs); } - if (scev->exprs_ht) - for (i = 0; i < scev->exprs_ht_cnt; i++) - kvfree(scev->exprs_ht[i]); kvfree(scev->exprs_ht); + kvfree(scev->exprs_next); kvfree(scev->worklist.data); kvfree(scev->envs); kvfree(scev->exprs); @@ -1981,6 +1979,7 @@ void bpf_free_scev(struct bpf_verifier_env *env) int bpf_init_scev(struct bpf_verifier_env *env) { struct scev *scev; + u32 r; scev = kzalloc(sizeof(struct scev), GFP_KERNEL_ACCOUNT); if (!scev) @@ -1988,13 +1987,13 @@ int bpf_init_scev(struct bpf_verifier_env *env) env->scev = scev; bpf_mark_reg_known_scalar(&scev->zero_reg, 0); bpf_mark_reg_unknown(&scev->unknown_reg); - /* A power of two for hash & (exprs_ht_cnt - 1)` indexing. */ - scev->exprs_ht_cnt = roundup_pow_of_two(max(256U, DIV_ROUND_UP(env->prog->len, 4))); - scev->exprs_ht = kvcalloc(scev->exprs_ht_cnt, sizeof(*scev->exprs_ht), GFP_KERNEL_ACCOUNT); - if (!scev->exprs_ht) - goto nomem; - if (expr0(scev, UNKNOWN) < 0 || expr0(scev, OPAQUE) < 0) + if (grow_exprs(scev, roundup_pow_of_two(REG_EXPR_ID(REGS_NUM)))) goto nomem; + scev->exprs[UNKNOWN_EXPR_ID] = (struct expr){ .op = UNKNOWN }; + scev->exprs[OPAQUE_EXPR_ID] = (struct expr){ .op = OPAQUE }; + for (r = 0; r < REGS_NUM; r++) + scev->exprs[REG_EXPR_ID(r)] = (struct expr){ .op = REG, .params = { r, 0 } }; + scev->exprs_cnt = REG_EXPR_ID(REGS_NUM); scev->envs = kvcalloc(env->prog->len, sizeof(*scev->envs), GFP_KERNEL_ACCOUNT); scev->discovered = kvcalloc(env->prog->len, sizeof(*scev->discovered), GFP_KERNEL_ACCOUNT); if (!scev->envs || !scev->discovered) -- 2.55.0 From: Alexei Starovoitov This is about the cost of scalar evolution: the work per join point. The time in scev.c goes down by a factor of 1.7, the number of expressions by 2.3. It is what pruned SSA does: no phi for a dead register. Expressions change, but only those that have no users: a register or stack slot that is dead at a join point is '?' there. join() constructs (any a b), which stands for a phi, for each register and stack slot that has distinct expressions on the two edges. That is a traversal of both and two or three calls of add_expr(). In loops it is rare. A later patch computes expressions for whole subprogs. There every 11th insn is a join point, and 25 of the 267 registers and stack slots differ. One of them is live. The others: - Branches leave temporaries in r0 - r5 and in the slots of their spills and buffers. Where two branches join, 7 of them are dead. - A subprog that is out of budget goes to the last exit of the subprog, on an edge that the verifier adds and the CFG does not have. Nothing is known on that edge, so every jump to that exit is joined with '?' in all 267. Only r0 is live there. These are 6 % of the join points and have 72 % of the dead registers and stack slots. 56 % of all expressions would be constructed for them, by 76 % of the calls of add_expr(). Ask reg_alive_at() first. '?' is the conservative answer: liveness is used to lose precision only, never to gain it. What it would cost to compute expressions for all code of the selftests, Cilium, sched_ext, katran, bpfjailer and programs in Rust. The last two rows are against loops only: before after expressions per insn 1.79 0.78 calls of add_expr() per insn 6.4 1.5 scev.c per insn 291 ns 170 ns kernel instructions (perf stat) +4.2 % +1.9 % load time (veristat) +5.7 % +3.2 % mem_peak of veristat is 0.7 % less. An edge to a join point cost up to 267 calls of mk_any(). It costs one per live register that differs. That is a bound per edge: the time in scev.c is linear in the number of insns before and after. Progs of 100 to 1000 insns gain most, 5 % of the load time. Above 10000 insns it is less than 1 %. With loops only, as today, it makes no difference that can be measured. Signed-off-by: Alexei Starovoitov --- kernel/bpf/scev.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 8f8ca69482ca..7031ab00a22a 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -989,8 +989,10 @@ next:; return id; } -static int join(struct scev *scev, struct env *acc, struct env *cur) +/* @acc is the env before the insn at @idx */ +static int join(struct bpf_verifier_env *env, struct env *acc, struct env *cur, int idx) { + struct scev *scev = env->scev; int i, id; if (acc->empty) { @@ -1001,7 +1003,9 @@ static int join(struct scev *scev, struct env *acc, struct env *cur) for (i = 0; i < REGS_NUM; i++) { if (!same_exprs(scev, acc->reg2expr[i], cur->reg2expr[i])) { - id = mk_any(scev, acc->reg2expr[i], cur->reg2expr[i]); + /* no phi for a dead register: pruned SSA */ + id = reg_alive_at(env, i, idx) ? + mk_any(scev, acc->reg2expr[i], cur->reg2expr[i]) : UNKNOWN_EXPR_ID; if (id < 0) return id; acc->reg2expr[i] = id; @@ -1339,7 +1343,7 @@ static int join_successor(struct bpf_verifier_env *env, int cur_header, int succ return -ENOMEM; if (log_level2) memcpy(old_env, succ_env, sizeof(*old_env)); - err = join(scev, succ_env, cur_env); + err = join(env, succ_env, cur_env, succ_idx); if (err) return err; err = worklist_push(env, cur_header, succ_idx); -- 2.55.0 From: Alexei Starovoitov A prerequisite of the patches that find linked registers. It is about the shape of an expression, not about cost: a constant offset has to stand at the root. These patches compare the expressions of registers at every conditional jump: r2 = r1 r2 += 1 r2 += 2 if r1 > 100 goto .. The jump bounds r2 too if r2 is known to be r1 + 3. The scalar id of r2 is dropped at the second '+='. Its expression has to be (+ r1 3), however many insns computed it. Today it is (+ (+ r1 1) 2). transfer() records what each insn does and folds nothing. simplify() is called later, and only for the registers at the header, the latches and the exits of a loop. It traverses the whole expression until nothing changes, and beyond depth 8 its answer is '?'. Simplify in expr2(). The operands of a new expression are simplified already, so simplify() handles the root only. Its rules construct their results with expr2() too, so one rule can enable the next, 8 deep at most: a prog can nest partial spills as deep as it likes. An expression beyond that limit is correct, but not as simple as it could be. This is no canonical form: operands are not ordered. transform_expr(), the repeated pass over a whole expression, goes away. What can be seen at once: GCC unrolls the loop of while_true() in loop3.c 8 times, with 'volatile __u64 i' on the stack. Before and after: *fp-16=(+ (+ ? ?) 7) / ? loop header at 5, non-linear latch at 76 *fp-16=(+ *fp-16 8) / (linear *fp-16 8) loop header at 5, header_count is [0..536870912] The loop gets an iteration count (header_count: how often the header is executed), 4406 -> 150 insns processed. And two edges that bring the same value, 'r6 += 1; r6 += 1' and 'r6 += 2', bring the same expression to a join point. No verdict changes. New rules. (+ x 0) is x. A fill that is not wider than the spill gets the low bits of the spilled value: (sext8 (spill16 x)) is (sext8 x). Only (zext32 (spill32 x)) had that. Little endian only, as before. No rule drops a register from an expression: collect_tested_regs() and collect_store_base_regs() see simplified expressions now. The level 2 log has (+ r6 -2) for 'r6 -= 2' and 42 for 'w1 = 42'. A by-product: the kernel spends 0.8 % fewer instructions (perf stat) on loading the selftests, Cilium, sched_ext, katran, bpfjailer and programs in Rust. Signed-off-by: Alexei Starovoitov --- kernel/bpf/scev.c | 112 +++++++++++------- .../selftests/bpf/progs/verifier_scev.c | 20 +++- 2 files changed, 84 insertions(+), 48 deletions(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 7031ab00a22a..4f02cd62305c 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -109,6 +109,7 @@ struct scev { int exprs_cap; int envs_cnt; int stack_sz; + int simplify_depth; struct expr_stack_elt expr_stack[EXPR_STACK_DEPTH]; u32 ids_buf[EXPR_STACK_DEPTH]; }; @@ -178,13 +179,30 @@ static int add_expr(struct scev *scev, struct expr e) return id; } +static int simplify(struct scev *scev, u32 id); + +/* + * An expression is simplified at construction. Its params are simplified already, so simplify() + * rewrites the root only, whatever the depth of the expression. + * The rules construct expressions too, and one rule can enable the next: the depth of (zext32 + * (spill32 (+ (spill32 (+ ..) 1)) 1)) is up to the prog. Beyond depth EXPR_STACK_DEPTH an + * expression is correct, but not simplified. + */ static int expr2(struct scev *scev, u32 op, int a, int b) { + int id; + if (a < 0) return a; if (b < 0) return b; - return add_expr(scev, (struct expr){ .op = op, .params = {a, b} }); + id = add_expr(scev, (struct expr){ .op = op, .params = {a, b} }); + if (id < 0 || scev->simplify_depth == EXPR_STACK_DEPTH) + return id; + scev->simplify_depth++; + id = simplify(scev, id); + scev->simplify_depth--; + return id; } static int expr1(struct scev *scev, u32 op, int a) @@ -251,14 +269,47 @@ static bool is_zext32(struct scev *scev, int id, u32 *left) return is_unop(scev, ZEXT32, id, left); } -/* The low half of the slot is the low half of @left. The other half is not known. */ -static bool is_spill32(struct scev *scev, int id, u32 *left) +/* (zextN x), (sextN x) and (spillN x) depend on the low N bits of x only */ +static u32 op_low_bits(u32 op) +{ + switch (op) { + case ZEXT8: + case SEXT8: + case SPILL8: + return 8; + case ZEXT16: + case SEXT16: + case SPILL16: + return 16; + case ZEXT32: + case SEXT32: + case SPILL32: + return 32; + } + return 0; +} + +/* The low @bits bits of the slot are those of @left. The other bits are not known. */ +static bool is_spill(struct scev *scev, int id, u32 *bits, u32 *left) { + u32 op = scev->exprs[id].op; + #ifdef __BIG_ENDIAN /* '*(u32 *)(r10 - 8) = rX' writes the high half, see mk_fill() */ return false; #endif - return is_unop(scev, SPILL32, id, left); + if (op != SPILL8 && op != SPILL16 && op != SPILL32) + return false; + *bits = op_low_bits(op); + *left = scev->exprs[id].params[0]; + return true; +} + +static bool is_spill32(struct scev *scev, int id, u32 *left) +{ + u32 bits; + + return is_spill(scev, id, &bits, left) && bits == 32; } static bool is_any(struct scev *scev, int id, u32 *left, u32 *right) @@ -1579,29 +1630,10 @@ static int transform_expr_once(struct scev *scev, u32 lvl, u32 root, void *priv, return id < 0 ? id : fn(scev, id, priv); } -static int transform_expr(struct scev *scev, u32 root, void *priv, - int (*fn)(struct scev *scev, u32 id, void *priv)) -{ - int i = 0, id_old, id_new = root; - - do { - /* - * The depth limit is somewhat arbitrary and is essentially a failsafe. - * oscillation should not really happen. Still, it seems harsh to error out. - */ - if (i++ == 4 * EXPR_STACK_DEPTH) - return UNKNOWN_EXPR_ID; - id_old = id_new; - id_new = transform_expr_once(scev, 0, id_old, priv, fn); - if (id_new < 0) - return id_new; - } while (id_old != id_new); - return id_new; -} - -static int simplify(struct scev *scev, u32 id, void *priv) +static int simplify(struct scev *scev, u32 id) { int a, b, l, r, t, base, slope; + u32 bits, op = scev->exprs[id].op; s64 imm1, imm2; /* (- x imm) -> (+ x -imm) */ @@ -1609,6 +1641,10 @@ static int simplify(struct scev *scev, u32 id, void *priv) is_imm(scev, r, &imm1) && imm1 != S64_MIN) return expr2(scev, BPF_ADD, l, imm_expr(scev, -imm1)); + /* (+ x 0) -> x */ + if (is_add(scev, id, &l, &r) && is_imm(scev, r, &imm1) && !imm1) + return l; + /* (+ (+ x imm) imm) -> (+ x imm) */ if (is_add(scev, id, &l, &r) && is_imm(scev, r, &imm2) && @@ -1658,12 +1694,14 @@ static int simplify(struct scev *scev, u32 id, void *priv) return expr1(scev, SPILL32, expr2(scev, BPF_ADD, a, b)); /* - * 'rX = *(u32 *)(r10 - 8)' loads what was stored: + * 'rX = *(u32 *)(r10 - 8)' fills what was spilled, a narrower fill the low bits of it: * (zext32 (spill32 $a)) -> (zext32 $a) + * (sext8 (spill16 $a)) -> (sext8 $a) * (zext32 (+ (spill32 $a) $b)) -> (zext32 (+ $a $b)) */ - if (is_zext32(scev, id, &t) && is_spill32(scev, t, &a)) - return expr1(scev, ZEXT32, a); + if (op_low_bits(op) && is_spill(scev, scev->exprs[id].params[0], &bits, &a) && + op_low_bits(op) <= bits) + return expr1(scev, op, a); if (is_zext32(scev, id, &t) && is_add(scev, t, &l, &b) && is_spill32(scev, l, &a)) return expr1(scev, ZEXT32, expr2(scev, BPF_ADD, a, b)); @@ -1692,10 +1730,7 @@ static int compute_header_scevs(struct bpf_verifier_env *env, struct env *header * ... at exit r1 is in {7,42} ... */ for (ra = 0; ra < REGS_NUM; ra++) { - ra_expr = transform_expr(scev, header_env->reg2expr[ra], NULL, simplify); - if (ra_expr < 0) - return ra_expr; - header_env->reg2expr[ra] = ra_expr; + ra_expr = header_env->reg2expr[ra]; /* rA = (+ rA IMM) -> (linear rA IMM) */ if (is_add(scev, ra_expr, &l, &r) && is_reg(scev, l, &rb) && @@ -1843,17 +1878,8 @@ static int compute_insn_scevs(struct bpf_verifier_env *env, struct env *eheader, int id, reg; for (reg = 0; reg < REGS_NUM; reg++) { - /* - * Simplify first. 'store, load, test' of a 32-bit counter on the stack with the - * equation of the header in it is deeper than EXPR_STACK_DEPTH. - */ - id = transform_expr(scev, einsn->reg2expr[reg], NULL, simplify); - if (id < 0) - return id; - id = transform_expr_once(scev, 0, id, eheader->reg2scev, instantiate_header_scevs); - if (id < 0) - return id; - id = transform_expr(scev, id, NULL, simplify); + id = transform_expr_once(scev, 0, einsn->reg2expr[reg], eheader->reg2scev, + instantiate_header_scevs); if (id < 0) return id; einsn->reg2scev[reg] = id; diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index f2a53fe5f6ad..3d73a7107cba 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -324,10 +324,11 @@ __msg("r1 += 2 {{.*}}; r1 r6 -> (+ r6 2)") __msg("r1 += r7 {{.*}}; r1 r6 -> (+ r6 r7)") __msg("w1 += 2 {{.*}}; r1 r6 -> (zext32 (+ r6 2))") __msg("w1 += w7 {{.*}}; r1 r6 -> (zext32 (+ r6 r7))") -__msg("r1 -= 2 {{.*}}; r1 r6 -> (- r6 2)") +__msg("r1 -= 2 {{.*}}; r1 r6 -> (+ r6 -2)") __msg("r1 -= r7 {{.*}}; r1 r6 -> (- r6 r7)") -__msg("w1 -= 2 {{.*}}; r1 r6 -> (zext32 (- r6 2))") +__msg("w1 -= 2 {{.*}}; r1 r6 -> (zext32 (+ r6 -2))") __msg("w1 -= w7 {{.*}}; r1 r6 -> (zext32 (- r6 r7))") +__msg("r1 -= 2 {{.*}}; r1 (+ r6 2) -> r6") __msg("r1 *= 2 {{.*}}; r1 r6 -> (* r6 2)") __msg("r1 *= r7 {{.*}}; r1 r6 -> (* r6 r7)") __msg("w1 *= 2 {{.*}}; r1 r6 -> (zext32 (* r6 2))") @@ -375,7 +376,7 @@ __msg("w1 s>>= w7 {{.*}}; r1 r6 -> (zext32 (s>> (sext32 r6) (& r7 31)))") __msg("r1 = -r1 {{.*}}; r1 r6 -> (- r6)") __msg("w1 = -w1 {{.*}}; r1 r6 -> (zext32 (- r6))") __msg("r1 = 42 {{.*}}; r1 r6 -> 42") -__msg("w1 = 42 {{.*}}; r1 r6 -> (zext32 42)") +__msg("w1 = 42 {{.*}}; r1 r6 -> 42") __msg("r1 = r7 {{.*}}; r1 r6 -> r7") __msg("w1 = w7 {{.*}}; r1 r6 -> (zext32 r7)") __msg("r1 = (s8)r6 {{.*}}; r1 r6 -> (sext8 r6)") @@ -417,6 +418,7 @@ __naked void alu_ops_exprs(void) ALU_OP("r1 -= r7") ALU_OP("w1 -= 2") ALU_OP("w1 -= w7") + ALU_OP("r1 += 2; r1 -= 2") ALU_OP("r1 *= 2") ALU_OP("r1 *= r7") ALU_OP("w1 *= 2") @@ -511,7 +513,7 @@ __msg("*(u16 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill16 1)") __msg("*(u8 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill8 1)") __msg("r1 = *(u64 *)(r10 -8) {{.*}}; r1 1 -> -8") #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -__msg("r1 = *(u32 *)(r10 -8) {{.*}}; r1 1 -> (zext32 -8)") +__msg("r1 = *(u32 *)(r10 -8) {{.*}}; r1 1 -> 4294967288") __msg("r1 = *(u16 *)(r10 -8) {{.*}}; r1 1 -> (zext16 -8)") __msg("r1 = *(u8 *)(r10 -8) {{.*}}; r1 1 -> (zext8 -8)") __msg("r1 = *(s32 *)(r10 -8) {{.*}}; r1 1 -> (sext32 -8)") @@ -519,7 +521,7 @@ __msg("r1 = *(s16 *)(r10 -8) {{.*}}; r1 1 -> (sext16 -8)") __msg("r1 = *(s8 *)(r10 -8) {{.*}}; r1 1 -> (sext8 -8)") __msg("r1 = load_acquire((u8 *)(r10 -8)) {{.*}}; r1 1 -> (zext8 -8)") __msg("r1 = load_acquire((u16 *)(r10 -8)) {{.*}}; r1 1 -> (zext16 -8)") -__msg("r1 = load_acquire((u32 *)(r10 -8)) {{.*}}; r1 1 -> (zext32 -8)") +__msg("r1 = load_acquire((u32 *)(r10 -8)) {{.*}}; r1 1 -> 4294967288") #else __msg("r1 = *(u32 *)(r10 -8) {{.*}}; r1 1 -> (>> -8 32)") __msg("r1 = *(u16 *)(r10 -8) {{.*}}; r1 1 -> (>> -8 48)") @@ -543,6 +545,11 @@ __msg("r1 = atomic64_xchg((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _" __msg("r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2) {{.*}}; r0 0 -> _, *fp-8 -8 -> _") __msg("*(u64 *)(r10 -8) = 42 {{.*}}; *fp-8 -8 -> 42") __msg("*(u8 *)(r10 -7) = r0 {{.*}}; *fp-8 -8 -> _") +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +/* a fill that is not wider than the spill reads what was spilled */ +__msg("r1 = *(s8 *)(r10 -8) {{.*}}; r1 r7 -> (sext8 r7)") +__msg("r1 = *(u16 *)(r10 -8) {{.*}}; r1 r7 -> (zext16 (spill8 r7))") +#endif __naked void store_load_exprs(void) { asm volatile ( @@ -579,6 +586,9 @@ __naked void store_load_exprs(void) ST_OP("*(u64 *)(r10 - 8) = 42") ST_OP("*(u8 *)(r10 - 7) = r0") + ST_OP("r1 = r7; *(u16 *)(r10 - 8) = r1; r1 = *(s8 *)(r10 - 8)") + ST_OP("r1 = r7; *(u8 *)(r10 - 8) = r1; r1 = *(u16 *)(r10 - 8)") + "r7 += 1;" "if r7 < 2 goto 1b;" "r0 = 0;" -- 2.55.0 From: Alexei Starovoitov A prerequisite of the patch that refines linked registers at conditional jumps: it turns the id of an expression into a value number. No users yet, no verdict changes. i = v->idx; j = i; if (i > 100) return 0; use(buf[j]); i and j hold the same value. The verifier knows it from the scalar id that 'j = i' assigns to both registers in a state. Scalar evolution could decide it statically, by comparing two expression ids. But a loaded value is '_', the result of a call is '_' too, and a value that is not tracked is '?'. All loads have the same id, so the same id in two registers is not the same value. Nor at a join point: 'x = c ? 1 : 2' and 'y = d ? 1 : 2' are (any 1 2). Give such a value an SSA name, struct expr_def: the def of a register or stack slot by the insn at idx, or by a phi before the insn at idx, a join point. forget_reg() makes the defs, join() the phis. struct expr has four bytes of padding for it. The level 2 log prints '%', the register or stack slot, '.', idx, and for a phi its operands: 1: call bpf_get_prandom_u32 r0 is %r0.1 2: r6 = r0 r6 is %r0.1: the same value 5: call bpf_get_prandom_u32 r0 is %r0.5 7: join point r0 is %r0.7=phi(_) r8 is %r8.7=phi(1, 2) The op of %r0.1 is '_', a trailing '?' says that it is '?'. phi(_) joins distinct '_', phi(?) distinct '?' or more than 8 values. A def is printed in full at every use: 'r0 += r8' at 7 is (+ %r0.7=phi(_) %r8.7=phi(1, 2)). Where it differs from SSA in a compiler: - Only a value that no expression describes gets a name. With a def, '?' and '_' are what LLVM calls SCEVUnknown: a leaf that stands for one value. Without a def '?' is SCEVCouldNotCompute. - An insn that may write defines too (may_def in liveness.c, a MemoryDef in MemorySSA): insn 8 may write fp-8, %fp-8.8? is what the slot holds after it. A nested loop defines at its header what it leaves. - The operands of a phi are a set, 8 at most. Phis are never congruent. - Without its backedges and with nested loops collapsed the body of a loop is acyclic: an insn executes at most once per iteration, so a def is one value per iteration. A name is unique within the env of one loop. - In the env of loop H a phi at H merges the backedge values only (the phi of the single latch that LoopSimplify would make). The header phi itself is (REG r). 'r6=%r6.1=phi(r6, (+ r6 1)) / ?' at header 1 is not {r6,+,1}: one backedge keeps r6, the other one adds 1. def.phi is not only for the log. r0 before an insn and after it are distinct values, so it is a part of the name. And later patches look through the op of a def, which the insn at idx computed, but not through a phi, whose op is what the joined values have in common. Op and operands stay, so the code that analyzes loops matches what it matched. struct expr::is_vn says that an id is a value number: all registers and stack slots of an env that have it hold the same value. That is (REG r), a constant, a def, an operation on such. 28 expected messages of verifier_scev.c follow the notation: 'r7=?' becomes 'r7=%r7.4?', 'r7=(any r7 100)' becomes 'r7=%r7.6=phi(r7, 100)'. Signed-off-by: Alexei Starovoitov --- kernel/bpf/scev.c | 313 ++++++++++++++---- .../selftests/bpf/progs/verifier_scev.c | 56 ++-- 2 files changed, 278 insertions(+), 91 deletions(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 4f02cd62305c..fdeedba90d66 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -52,8 +52,38 @@ enum expr_op { COPY_SCEV, }; +/* + * An SSA name for a value that no expression describes: the def of the SCEV register 'reg' + * - by the insn at 'idx'. An insn that may write defines too (may_def in liveness.c). A nested loop + * defines at its header what it leaves. + * - by a phi before the insn at 'idx': a join point, a loop header. + * Without its backedges and with nested loops collapsed the body of a loop is acyclic: an insn + * executes at most once per iteration, so a def is one value per iteration. A name is unique within + * the env of one loop. + * The operands of a phi are a set, 8 at most; a phi(a, b) among them is expanded. Every phi is a + * value number of its own. + * In the env of loop H a phi at H merges the backedge values only, the header phi itself is + * (REG r). In the env of the enclosing loop a phi at H merges the entering edges. + */ +struct expr_def { + u32 set:1; /* 0: the expression has no def */ + u32 phi:1; + u32 reg:9; + u32 idx:21; +}; +static_assert(REGS_NUM <= 1 << 9 && BPF_COMPLEXITY_LIMIT_INSNS <= 1 << 21); + +static const struct expr_def no_def; + struct expr { - u32 op; + u16 op; + /* A value number: all SCEV registers of an env that have this id hold the same value */ + bool is_vn; + /* + * UNKNOWN, OPAQUE and ANY don't describe a value. With a def the id stands for one value, + * see struct expr_def. Without a def every use of the id can be another value. + */ + struct expr_def def; union { u32 params[2]; s64 imm; @@ -116,12 +146,19 @@ struct scev { static u32 expr_hash(struct expr *e) { - return jhash_3words(e->op, e->params[0], e->params[1], 0); + /* def.phi is not hashed: the def by an insn and the phi before it share a chain */ + return jhash_3words(e->op, e->params[0], e->params[1], + e->def.idx * REGS_NUM + e->def.reg); +} + +static bool same_def(struct expr_def a, struct expr_def b) +{ + return a.set == b.set && a.phi == b.phi && a.reg == b.reg && a.idx == b.idx; } static int expr_eq(struct expr *a, struct expr *b) { - return a->op == b->op && a->imm == b->imm; + return a->op == b->op && a->imm == b->imm && same_def(a->def, b->def); } static void exprs_ht_add(struct scev *scev, u32 id) @@ -158,12 +195,41 @@ static int grow_exprs(struct scev *scev, u32 cap) return 0; } +static u32 op_params_num(u32 op); + +static bool expr_is_vn(struct scev *scev, struct expr *e) +{ + u32 i; + + if (e->def.set) + return true; + switch (e->op) { + case UNKNOWN: + case OPAQUE: + case ANY: + /* a partial spill: the other bits of the slot are not known */ + case SPILL8: + case SPILL16: + case SPILL32: + /* a function of the iteration number */ + case LINEAR_SCEV: + case COPY_SCEV: + return false; + } + for (i = 0; i < op_params_num(e->op); i++) + if (!scev->exprs[e->params[i]].is_vn) + return false; + return true; +} + static int add_expr(struct scev *scev, struct expr e) { u32 id; if (e.op == REG) return REG_EXPR_ID(e.params[0]); + if (!e.def.set && !e.imm && (e.op == UNKNOWN || e.op == OPAQUE)) + return e.op == UNKNOWN ? UNKNOWN_EXPR_ID : OPAQUE_EXPR_ID; id = scev->exprs_ht[expr_hash(&e) & (scev->exprs_cap - 1)]; for (; id; id = scev->exprs_next[id]) @@ -174,6 +240,7 @@ static int add_expr(struct scev *scev, struct expr e) return -ENOMEM; id = scev->exprs_cnt++; + e.is_vn = expr_is_vn(scev, &e); scev->exprs[id] = e; exprs_ht_add(scev, id); return id; @@ -217,6 +284,33 @@ static int imm_expr(struct scev *scev, s64 value) return add_expr(scev, (struct expr){ .op = IMM, .imm = value }); } +/* + * The insn at @idx defines @reg. Without its backedges and with nested loops collapsed the body is + * acyclic: an insn executes at most once per iteration, so a def is one value per iteration. + */ +static struct expr_def insn_def(int idx, u32 reg) +{ + return (struct expr_def){ .set = 1, .reg = reg, .idx = idx }; +} + +/* The phi of @reg before the insn at @idx. One value per iteration for the same reason. */ +static struct expr_def phi_def(int idx, u32 reg) +{ + return (struct expr_def){ .set = 1, .phi = 1, .reg = reg, .idx = idx }; +} + +/* @id with another def */ +static int expr_with_def(struct scev *scev, int id, struct expr_def def) +{ + struct expr e; + + if (id < 0) + return id; + e = scev->exprs[id]; + e.def = def; + return add_expr(scev, e); +} + static bool same_exprs(struct scev *scev, int id_a, int id_b) { return id_a == id_b; @@ -486,27 +580,69 @@ static bool expr_next(struct scev *scev, u32 *id, u32 *order) return expr_next(scev, id, order); } +/* + * An SSA name is printed as '%', the SCEV register, '.', the insn. A def is printed in full at + * every use: (+ %r8.7=phi(1, 2) 4). + * %r0.6 insn 6 defines r0. The value is OPAQUE, '_' without a def + * %fp-8.8? insn 8 may write fp-8. The value is UNKNOWN, '?' without a def + * %r8.7=phi(1, 2) the phi of r8 before insn 7 + * %r0.7=phi(_) a phi of distinct OPAQUE values + * %r6.9=phi(?) a phi of distinct UNKNOWN values, or of more than 8 values + * A name is unique within the env of one loop. In the env of loop H a phi at H merges the backedge + * values only, the header phi itself is (REG r): in 'scev at header 1: r6=%r6.1=phi(r6, (+ r6 1))' + * one backedge leaves r6 as it is, the other one adds 1. + * '%' in front of r or fp is a name, followed by a blank it is BPF_MOD. + */ +static void log_def(struct bpf_verifier_env *env, struct expr_def def) +{ + if (!def.set) + return; + if (def.reg < MAX_BPF_REG) + bpf_log(&env->log, "%%r%d.%d", def.reg, def.idx); + else + bpf_log(&env->log, "%%fp%d.%d", (MAX_BPF_REG - def.reg - 1) * 8, def.idx); +} + static void log_expr(struct bpf_verifier_env *env, u32 id) { struct bpf_verifier_log *log = &env->log; + bool operand, bare, first = false; struct scev *scev = env->scev; + u32 order, d, bare_any = 0; + struct expr_def def; struct expr *expr; const char *str; - u32 order; scev->stack_sz = 0; expr_stack_push(scev, id); while (expr_next(scev, &id, &order)) { - if ((order & PRE) && scev->stack_sz > 1) - bpf_log(log, " "); expr = &scev->exprs[id]; + def = expr->def; + /* the depth of @id in expr_stack[], expr_next() pops it before POST */ + d = order & PRE ? scev->stack_sz - 1 : scev->stack_sz; + operand = d && (bare_any & BIT(d - 1)); + /* No '(any' for the operands of phi(a, b). (any a b) among them is expanded. */ + if (order & PRE) { + bare_any &= ~BIT(d); + if (expr->op == ANY && (def.set ? def.phi : operand)) + bare_any |= BIT(d); + } + bare = bare_any & BIT(d); + if ((order & PRE) && (def.set || !bare)) { + if (d) + bpf_log(log, "%s", !operand ? " " : first ? "" : ", "); + log_def(env, def); + if (def.phi) + bpf_log(log, "=phi("); + first = bare; + } switch (expr->op) { case UNKNOWN: if (order & PRE) bpf_log(log, "?"); break; case OPAQUE: - if (order & PRE) + if ((order & PRE) && (!def.set || def.phi)) bpf_log(log, "_"); break; case REG: @@ -518,19 +654,23 @@ static void log_expr(struct bpf_verifier_env *env, u32 id) bpf_log(log, "%lld", expr->imm); break; default: - if (order & PRE) { + if ((order & PRE) && !bare) { str = op_str(expr->op); - bpf_log(log, "("); + bpf_log(log, def.set && !def.phi ? "=(" : "("); if (str) bpf_log(log, "%s", str); else bpf_log(log, "bad-expr-op %x", expr->op); } - if (order & DEPTH_LIMIT) - bpf_log(log, "..."); - if (order & POST) + if (order & DEPTH_LIMIT) { + bpf_log(log, bare && !def.set && !first ? ", ..." : "..."); + first = false; + } + if ((order & POST) && !bare) bpf_log(log, ")"); } + if ((order & POST) && def.phi) + bpf_log(log, ")"); } } @@ -570,7 +710,7 @@ static void print_env(struct bpf_verifier_env *env, struct env *e, u32 insn_idx) log_reg(env, i); bpf_log(log, "="); log_expr(env, e->reg2expr[i]); - if (e->reg2expr[i] != UNKNOWN_EXPR_ID) { + if (!is_op(scev, e->reg2expr[i], UNKNOWN)) { bpf_log(log, " / "); log_expr(env, e->reg2scev[i]); } @@ -634,6 +774,12 @@ static int replace_reg(struct scev *scev, struct env *e, u32 reg, int id) return 0; } +/* The insn at @idx defines @reg. The value is not tracked: @id is UNKNOWN or OPAQUE. */ +static int forget_reg(struct scev *scev, struct env *e, u32 reg, int id, int idx) +{ + return replace_reg(scev, e, reg, expr_with_def(scev, id, insn_def(idx, reg))); +} + static void setup_initial_loop_env(struct env *e) { int i; @@ -648,12 +794,20 @@ static void setup_initial_loop_env(struct env *e) e->reg2expr[i] = REG_EXPR_ID(i); } -static void forget_call_regs(struct env *e) +/* A subprog can return a pair in r0 and r2: a call defines r1 - r5 too if they are live after it */ +static int forget_call_regs(struct bpf_verifier_env *env, struct env *e, int idx) { - int i; + int i, err; - for (i = BPF_REG_0; i <= BPF_REG_5; i++) + for (i = BPF_REG_0; i <= BPF_REG_5; i++) { e->reg2expr[i] = UNKNOWN_EXPR_ID; + if (idx + 1 >= env->prog->len || !reg_alive_at(env, i, idx + 1)) + continue; + err = forget_reg(env->scev, e, i, UNKNOWN_EXPR_ID, idx); + if (err) + return err; + } + return 0; } static u32 spill_spi(struct bpf_insn *insn) @@ -673,16 +827,19 @@ static bool is_spill_off(int off) off >= -MAX_BPF_STACK_JIT; } -static void mark_opaque(struct env *e, int off, int size) +static int mark_opaque(struct scev *scev, struct env *e, int idx, int off, int size) { - int b, spi; + int b, spi, err; for (b = off; b < off + size; b++) { if (b >= 0 || b < -MAX_BPF_STACK_JIT) continue; spi = (-b - 1) / BPF_REG_SIZE; - e->reg2expr[MAX_BPF_REG + spi] = OPAQUE_EXPR_ID; + err = forget_reg(scev, e, MAX_BPF_REG + spi, OPAQUE_EXPR_ID, idx); + if (err) + return err; } + return 0; } static int mk_spill(struct scev *scev, u8 size, int id) @@ -716,7 +873,7 @@ static int mk_fill(struct scev *scev, int id, u8 code) return UNKNOWN_EXPR_ID; } -static int maybe_store_fp(struct scev *scev, struct env *e, struct bpf_insn *insn, int id) +static int maybe_store_fp(struct scev *scev, struct env *e, struct bpf_insn *insn, int idx, int id) { u8 size = BPF_SIZE(insn->code); @@ -724,19 +881,15 @@ static int maybe_store_fp(struct scev *scev, struct env *e, struct bpf_insn *ins return 0; if (is_spill_off(insn->off)) return replace_reg(scev, e, off_to_reg(insn), mk_spill(scev, size, id)); - mark_opaque(e, insn->off, bpf_size_to_bytes(size)); - return 0; + return mark_opaque(scev, e, idx, insn->off, bpf_size_to_bytes(size)); } -static int maybe_load_fp(struct scev *scev, struct env *e, struct bpf_insn *insn) +static int maybe_load_fp(struct scev *scev, struct env *e, struct bpf_insn *insn, int idx) { - int id; - - if (insn->src_reg == BPF_REG_FP && is_spill_off(insn->off)) - id = mk_fill(scev, e->reg2expr[off_to_reg(insn)], insn->code); - else - id = OPAQUE_EXPR_ID; - return replace_reg(scev, e, insn->dst_reg, id); + if (insn->src_reg != BPF_REG_FP || !is_spill_off(insn->off)) + return forget_reg(scev, e, insn->dst_reg, OPAQUE_EXPR_ID, idx); + return replace_reg(scev, e, insn->dst_reg, + mk_fill(scev, e->reg2expr[off_to_reg(insn)], insn->code)); } /* @@ -912,30 +1065,34 @@ static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) imm = env->insn_aux_data[idx + 1].const_reg_vals[dst]; return replace_reg(scev, e, dst, imm_expr(scev, imm)); } - return maybe_load_fp(scev, e, insn); + return maybe_load_fp(scev, e, insn, idx); default: goto mark_dst_unknown; } case BPF_STX: switch (mode) { case BPF_MEM: - return maybe_store_fp(scev, e, insn, reg2expr[src]); + return maybe_store_fp(scev, e, insn, idx, reg2expr[src]); case BPF_ATOMIC: if (insn->imm == BPF_LOAD_ACQ) - return maybe_load_fp(scev, e, insn); + return maybe_load_fp(scev, e, insn, idx); if (insn->imm == BPF_STORE_REL) { - return maybe_store_fp(scev, e, insn, reg2expr[src]); + return maybe_store_fp(scev, e, insn, idx, reg2expr[src]); } /* * verifier does not track other atomic ops precisely, * hence mark the results as opaque. */ - if (insn->dst_reg == BPF_REG_FP) - mark_opaque(e, insn->off, bpf_size_to_bytes(BPF_SIZE(insn->code))); + if (insn->dst_reg == BPF_REG_FP) { + id = mark_opaque(scev, e, idx, insn->off, + bpf_size_to_bytes(BPF_SIZE(insn->code))); + if (id < 0) + return id; + } if (insn->imm == BPF_CMPXCHG) - return replace_reg(scev, e, BPF_REG_0, OPAQUE_EXPR_ID); + return forget_reg(scev, e, BPF_REG_0, OPAQUE_EXPR_ID, idx); if (insn->imm & BPF_FETCH) - return replace_reg(scev, e, src, OPAQUE_EXPR_ID); + return forget_reg(scev, e, src, OPAQUE_EXPR_ID, idx); break; } break; @@ -944,15 +1101,17 @@ static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) id = imm_expr(scev, insn->imm); if (id < 0) return id; - return maybe_store_fp(scev, e, insn, id); + return maybe_store_fp(scev, e, insn, idx, id); } break; case BPF_JMP: case BPF_JMP32: if (opcode == BPF_CALL) { - forget_call_regs(e); + id = forget_call_regs(env, e, idx); + if (id < 0) + return id; /* made anew, like what is loaded. The parameter tells the two apart. */ - return replace_reg(scev, e, BPF_REG_0, expr1(scev, OPAQUE, 1)); + return forget_reg(scev, e, BPF_REG_0, expr1(scev, OPAQUE, 1), idx); } /* for non-CALL there are no changes in register states */ break; @@ -966,12 +1125,11 @@ static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) } /* map, map value, BTF id, function */ if (BPF_SIZE(insn->code) == BPF_DW) - return replace_reg(scev, e, dst, OPAQUE_EXPR_ID); + return forget_reg(scev, e, dst, OPAQUE_EXPR_ID, idx); goto mark_dst_unknown; case BPF_ABS: case BPF_IND: - forget_call_regs(e); - break; + return forget_call_regs(env, e, idx); default: goto mark_dst_unknown; } @@ -985,18 +1143,18 @@ static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) return 0; mark_dst_unknown: - reg2expr[dst] = UNKNOWN_EXPR_ID; - return 0; + return forget_reg(scev, e, dst, UNKNOWN_EXPR_ID, idx); } /* * Construct minimal 'ANY' expression by traversing 'a' and 'b' * and accumulating non-duplicated non-ANY entries. + * 'def' is the def of the result, no_def if it has none. */ -static int mk_any(struct scev *scev, int a, int b) +static int mk_any(struct scev *scev, int a, int b, struct expr_def def) { struct expr_stack_elt *elt; - struct expr *expr; + struct expr *expr, *other; u32 i, j, ids_buf_sz; u32 roots[2] = {a, b}; int id; @@ -1016,14 +1174,23 @@ static int mk_any(struct scev *scev, int a, int b) if (expr->op == ANY) { if (!expr_stack_push(scev, expr->params[1]) || !expr_stack_push(scev, expr->params[0])) - return UNKNOWN_EXPR_ID; + return expr_with_def(scev, UNKNOWN_EXPR_ID, def); } else { for (j = 0; j < ids_buf_sz; j++) { if (same_exprs(scev, scev->ids_buf[j], id)) goto next; + /* two defs of one shape: the shape without a def */ + other = &scev->exprs[scev->ids_buf[j]]; + if (other->op != expr->op || other->imm != expr->imm) + continue; + id = expr_with_def(scev, id, no_def); + if (id < 0) + return id; + scev->ids_buf[j] = id; + goto next; } if (ids_buf_sz == ARRAY_SIZE(scev->ids_buf)) - return UNKNOWN_EXPR_ID; + return expr_with_def(scev, UNKNOWN_EXPR_ID, def); scev->ids_buf[ids_buf_sz++] = id; } next:; @@ -1037,7 +1204,7 @@ next:; if (id < 0) return id; } - return id; + return expr_with_def(scev, id, def); } /* @acc is the env before the insn at @idx */ @@ -1056,7 +1223,8 @@ static int join(struct bpf_verifier_env *env, struct env *acc, struct env *cur, if (!same_exprs(scev, acc->reg2expr[i], cur->reg2expr[i])) { /* no phi for a dead register: pruned SSA */ id = reg_alive_at(env, i, idx) ? - mk_any(scev, acc->reg2expr[i], cur->reg2expr[i]) : UNKNOWN_EXPR_ID; + mk_any(scev, acc->reg2expr[i], cur->reg2expr[i], + phi_def(idx, i)) : UNKNOWN_EXPR_ID; if (id < 0) return id; acc->reg2expr[i] = id; @@ -1078,12 +1246,12 @@ static void forget_modified_regs(struct scev *scev, struct env *acc, struct env } /* For a loop H at `header`, mark any register modified as a result of H execution as unknown. */ -static void forget_non_invariants(struct bpf_verifier_env *env, struct env *acc, int header) +static int forget_non_invariants(struct bpf_verifier_env *env, struct env *acc, int header) { struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_loop *loop = aux[header].loop; struct scev *scev = env->scev; - int i, idx, h; + int i, idx, h, err; acc->empty = false; forget_modified_regs(scev, acc, find_header_env(scev, header)); @@ -1120,6 +1288,15 @@ static void forget_non_invariants(struct bpf_verifier_env *env, struct env *acc, */ forget_modified_regs(scev, acc, find_loop_env(scev, header, idx)); } + /* the nested loop defines them at its header */ + for (i = 0; i < REGS_NUM; i++) { + if (acc->reg2expr[i] != UNKNOWN_EXPR_ID) + continue; + err = forget_reg(scev, acc, i, UNKNOWN_EXPR_ID, header); + if (err) + return err; + } + return 0; } /* Order worklist in reverse post-order. */ @@ -1236,7 +1413,7 @@ static bool is_probe_read_helper(u32 func_id) * If instruction is an indirect write to stack, invalidate SCEVs for spi's * that this instruction can touch. */ -static void reset_scevs_at_indirect_writes(struct bpf_verifier_env *env, struct env *cur_env, int idx) +static int reset_scevs_at_indirect_writes(struct bpf_verifier_env *env, struct env *cur_env, int idx) { struct bpf_insn *insn = &env->prog->insnsi[idx]; struct scev *scev = env->scev; @@ -1244,19 +1421,24 @@ static void reset_scevs_at_indirect_writes(struct bpf_verifier_env *env, struct const unsigned long *mask; u32 spi, reg; bool opaque; + int err; /* Direct fp stores are fine. */ if ((class == BPF_ST || class == BPF_STX) && insn->dst_reg == BPF_REG_FP) - return; + return 0; mask = bpf_may_write_mask(env, idx); opaque = bpf_helper_call(insn) && is_probe_read_helper(insn->imm); for_each_set_bit(spi, mask, MAX_BPF_STACK_SLOTS) { reg = MAX_BPF_REG + spi; - if (cur_env->reg2expr[reg] == UNKNOWN_EXPR_ID) - continue; - replace_reg(scev, cur_env, reg, opaque ? OPAQUE_EXPR_ID : UNKNOWN_EXPR_ID); + /* '?' stays '?' */ + err = forget_reg(scev, cur_env, reg, + opaque && !is_op(scev, cur_env->reg2expr[reg], UNKNOWN) ? + OPAQUE_EXPR_ID : UNKNOWN_EXPR_ID, idx); + if (err) + return err; } + return 0; } /* @@ -1486,7 +1668,9 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) /* And what its jumps depend on. */ for_each_set_bit(r, nested_loop->tested_regs, BPF_SCEV_REGS_NUM) or_expr_regs(env, cur_env->reg2expr[r], cur_loop->tested_regs); - forget_non_invariants(env, cur_env, idx); + err = forget_non_invariants(env, cur_env, idx); + if (err) + goto out; if (log_level2) log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx); /* @@ -1512,7 +1696,9 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) err = transfer(env, cur_env, idx); if (err) goto out; - reset_scevs_at_indirect_writes(env, cur_env, idx); + err = reset_scevs_at_indirect_writes(env, cur_env, idx); + if (err) + goto out; if (log_level2) log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx); succ = bpf_insn_successors(env, idx); @@ -1778,7 +1964,7 @@ static int compute_header_scevs(struct bpf_verifier_env *env, struct env *header } /* rA = (any 1 2 3 4 ...) -> (any rA 1 2 3 4 ...) */ if (is_any_imm_reg_opaque(scev, ra_expr)) { - id = mk_any(scev, expr1(scev, REG, ra), ra_expr); + id = mk_any(scev, expr1(scev, REG, ra), ra_expr, no_def); if (id < 0) return id; header_env->reg2scev[ra] = id; @@ -2022,7 +2208,8 @@ int bpf_init_scev(struct bpf_verifier_env *env) scev->exprs[UNKNOWN_EXPR_ID] = (struct expr){ .op = UNKNOWN }; scev->exprs[OPAQUE_EXPR_ID] = (struct expr){ .op = OPAQUE }; for (r = 0; r < REGS_NUM; r++) - scev->exprs[REG_EXPR_ID(r)] = (struct expr){ .op = REG, .params = { r, 0 } }; + scev->exprs[REG_EXPR_ID(r)] = (struct expr){ .op = REG, .is_vn = true, + .params = { r, 0 } }; scev->exprs_cnt = REG_EXPR_ID(REGS_NUM); scev->envs = kvcalloc(env->prog->len, sizeof(*scev->envs), GFP_KERNEL_ACCOUNT); scev->discovered = kvcalloc(env->prog->len, sizeof(*scev->discovered), GFP_KERNEL_ACCOUNT); diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 3d73a7107cba..de7679656831 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -135,7 +135,7 @@ exit_%=: \ SEC("xdp") __success __log_level(2) -__msg("8: (7b) *(u64 *)(r1 +0) = r0 ; *fp-8 (+ *fp-8 1) -> ?") +__msg("8: (7b) *(u64 *)(r1 +0) = r0 ; *fp-8 (+ *fp-8 1) -> %fp-8.8?") __naked void indirect_write_invalidates_scev(void) { asm volatile (" \ @@ -165,7 +165,7 @@ __noinline int global_noop_counter(__u64 *p) SEC("xdp") __success __log_level(2) -__msg("8: (85) call pc+{{[0-9]+}} {{.*}}; *fp-8 (+ *fp-8 1) -> ?") +__msg("8: (85) call pc+{{[0-9]+}} {{.*}}; *fp-8 (+ *fp-8 1) -> %fp-8.8?") __naked void global_call_invalidates_scev(void) { asm volatile ( @@ -242,7 +242,7 @@ SEC("xdp") __success __retval(0) __log_level(2) __msg_next("scev at header 1:") -__msg_next(" r6=(any r6 (+ r6 1)) / ?") +__msg_next(" r6=%r6.1=phi(r6, (+ r6 1)) / ?") __naked void meet_disagrees(void) { asm volatile (" \ @@ -538,13 +538,13 @@ __msg("store_release((u8 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill8 1)") __msg("store_release((u16 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill16 1)") __msg("store_release((u32 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill32 1)") __msg("store_release((u64 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> 1") -__msg("lock *(u64 *)(r10 -8) += r1 {{.*}}; *fp-8 -8 -> _") -__msg("r1 = atomic64_fetch_add((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _") -__msg("r1 = atomic_fetch_add((u32 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _") -__msg("r1 = atomic64_xchg((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _") -__msg("r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2) {{.*}}; r0 0 -> _, *fp-8 -8 -> _") +__msg("lock *(u64 *)(r10 -8) += r1 {{.*}}; *fp-8 -8 -> %fp-8.{{[0-9]+}}") +__msg("r1 = atomic64_fetch_add((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> %r1.{{[0-9]+}}, *fp-8 -8 -> %fp-8.{{[0-9]+}}") +__msg("r1 = atomic_fetch_add((u32 *)(r10 -8), r1) {{.*}}; r1 1 -> %r1.{{[0-9]+}}, *fp-8 -8 -> %fp-8.{{[0-9]+}}") +__msg("r1 = atomic64_xchg((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> %r1.{{[0-9]+}}, *fp-8 -8 -> %fp-8.{{[0-9]+}}") +__msg("r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2) {{.*}}; r0 0 -> %r0.{{[0-9]+}}, *fp-8 -8 -> %fp-8.{{[0-9]+}}") __msg("*(u64 *)(r10 -8) = 42 {{.*}}; *fp-8 -8 -> 42") -__msg("*(u8 *)(r10 -7) = r0 {{.*}}; *fp-8 -8 -> _") +__msg("*(u8 *)(r10 -7) = r0 {{.*}}; *fp-8 -8 -> %fp-8.{{[0-9]+}}") #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ /* a fill that is not wider than the spill reads what was spilled */ __msg("r1 = *(s8 *)(r10 -8) {{.*}}; r1 r7 -> (sext8 r7)") @@ -617,10 +617,10 @@ __success __log_level(2) __msg_next("scev at header 1:") __msg_next(" r6=(+ r6 1) / (linear r6 1)") -__msg_next(" r7=?") +__msg_next(" r7=%r7.4?") __msg_next(" scev at latch 1:") __msg_next(" r6=(+ r6 1) / (linear r6 1)") -__msg_next(" r7=?") +__msg_next(" r7=%r7.4?") __msg_next("scev at header 4:") __msg_next(" r7=(+ r7 1) / (linear r7 1)") __msg_next(" scev at latch 4:") @@ -657,7 +657,7 @@ __msg_next(" backedge from 6, latch at 4") __msg_next(" exit from 4 to 1") __msg("scev at header 1:") __msg_next(" r6=(+ r6 1) / (linear r6 1)") -__msg_next(" r7=?") +__msg_next(" r7=%r7.4?") __msg_next("scev at header 4:") __msg_next(" r7=(+ r7 1) / (linear r7 1)") __msg_next(" scev at latch 4:") @@ -702,7 +702,7 @@ __msg_next(" backedge from 5, latch at 5") __msg_next(" exit from 4 to 2") __msg_next(" exit from 5 to 6") __msg("scev at header 2:") -__msg_next(" r6=?") +__msg_next(" r6=%r6.3?") __msg_next(" r7=(+ r7 1) / (linear r7 1)") __msg_next("scev at header 3:") __msg_next(" r6=(+ r6 1) / (linear r6 1)") @@ -737,8 +737,8 @@ __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) __msg("loop at 3{{$}}") __msg("scev at header 3:") -__msg_next(" r6=(any r6 (+ r6 1)) / ?") -__msg_next(" r7=(any (+ r7 1) 0) / ?") +__msg_next(" r6=%r6.3=phi(r6, (+ r6 1)) / ?") +__msg_next(" r7=%r7.3=phi((+ r7 1), 0) / ?") __msg_next(" r8=(+ r8 1) / (linear r8 1)") __msg("loop header at 3, can't compute iterations count") __naked void nested_loops_shared_header(void) @@ -1539,7 +1539,7 @@ __success __retval(7) __log_level(2) __msg("scev at header 1:") __msg_next("r6=(+ r6 1) / (linear r6 1)") -__msg_next("*fp-8=(any *fp-8 7) / (any *fp-8 7)") +__msg_next("*fp-8=%fp-8.3=phi(*fp-8, 7) / (any *fp-8 7)") __msg("loop header at 1, widening r6 to 0..2 step 1") __msg("3: (79) r0 = *(u64 *)(r10 -8) ; R0=scalar() R10=fp0 loop_stack=1") __naked void widen_any_unallocated_slot(void) @@ -1580,7 +1580,7 @@ __success __retval(7) __log_level(2) __msg("scev at header 2:") __msg_next("r6=(+ r6 1) / (linear r6 1)") -__msg_next("*fp-8=(any *fp-8 7) / (any *fp-8 7)") +__msg_next("*fp-8=%fp-8.5=phi(*fp-8, 7) / (any *fp-8 7)") __msg("loop header at 2, widening r6 to 0..2 step 1") __msg("3: (79) r1 = *(u64 *)(r10 -8) ; R1=scalar() R10=fp0 fp-8=m???????") __naked void widen_any_misc_slot(void) @@ -1683,7 +1683,7 @@ __success __log_level(2) __msg("scev at header 4:") __msg_next(" r6=(+ r6 1) / (linear r6 1)") -__msg_next(" *fp-8=?") +__msg_next(" *fp-8=%fp-8.7?") __msg("loop header at 4, widening r6 to 0..9 step 1") __naked void widen_subprog_probe_read(void) { @@ -2604,7 +2604,7 @@ __msg("loop at 1{{$}}") __msg("loop at 5, nested in 1, irreducible") __msg("loop at 11, nested in 5") __msg("scev at header 1:") -__msg_next(" r6=?") +__msg_next(" r6=%r6.5?") __msg("scev at header 11:") __msg_next(" r9=(+ r9 1) / (linear r9 1)") __msg("loop header at 11, widening r9 to 0..2 step 1") @@ -2851,7 +2851,7 @@ __success __log_level(2) __msg("scev at header 2:") __msg_next(" r6=(+ r6 1) / (linear r6 1)") -__msg_next(" r7=(any r7 100) / (any r7 100)") +__msg_next(" r7=%r7.6=phi(r7, 100) / (any r7 100)") __msg_next(" scev at latch 3:") __msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)") __msg_next(" r7=r7 / (any r7 100)") @@ -3126,12 +3126,12 @@ __naked void name(void) \ #define NOT_UNROLLED "widening r6 to 0..0xf423f step 1" /* 'if (...) cnt++' */ -PARTIAL_SPILL_TEST(partial_spill32_inc, u32, 10, "can't widen *fp-8, expr is (any *fp-8 (spill32 ", +PARTIAL_SPILL_TEST(partial_spill32_inc, u32, 10, "can't widen *fp-8, expr is %fp-8.{{[0-9]+}}=phi(*fp-8, (spill32 ", SPILL32_0, "if r6 & 1 goto 2f;" SPILL32_INC "2:") PARTIAL_SPILL_TEST(partial_spill16_inc, u16, 10, "can't widen *fp-8, expr is (spill16 ", "*(u16 *)(r10 - 8) = r1;", "r1 = *(u16 *)(r10 - 8); w1 += 1; *(u16 *)(r10 - 8) = r1;") /* A loop in the loop does it */ -PARTIAL_SPILL_TEST(partial_spill32_inc_in_nested_loop, u32, 10, "can't widen *fp-8, expr is ?", +PARTIAL_SPILL_TEST(partial_spill32_inc_in_nested_loop, u32, 10, "can't widen *fp-8, expr is %fp-8.{{[0-9]+}}?", SPILL32_0, "r7 = 0; 2:" SPILL32_INC "r7 += 1; if r7 < 3 goto 2b;") /* Two slots feed each other */ PARTIAL_SPILL_TEST(partial_spill32_two_slots, u32, 10, "can't widen *fp-8, expr is (spill32 ", @@ -3296,7 +3296,7 @@ NOT_LINEAR_TEST(zext32_or_nothing, "r7 >>= 8;", "if r8 == 0 goto +1; w7 = w7;", NOT_LINEAR_TEST(and_64_to_127, "r7 &= 63; r7 |= 64;", "r7 &= 127;", UNROLLED("(& r7 127)")) /* '>>' does not make a number greater, but iterations one by one tell more: 0 after 8 shifts */ NOT_LINEAR_TEST(rsh_on_one_path, "r7 &= 255;", "if r8 == 0 goto +1; r7 >>= 1;", - UNROLLED("(any r7 (>> r7 1))")) + UNROLLED("%r7.8=phi(r7, (>> r7 1))")) NOT_LINEAR_TEST(rsh_then_and, "r7 &= 255;", "r7 >>= 1; r7 &= r8;", UNROLLED("(& (>> r7 1) r8)")) NOT_LINEAR_TEST(lsh_grows, "r7 &= 15;", "r7 <<= 1;", UNROLLED("(<< r7 1)")) @@ -3458,9 +3458,9 @@ __failure __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) __msg("scev at header 9:") -__msg_next(" r6=(+ ? 1) / ?") +__msg_next(" r6=(+ %r6.13? 1) / ?") __msg_next(" scev at latch 20:") -__msg_next(" r6=(+ ? 1) / (+ ? 1)") +__msg_next(" r6=(+ %r6.13? 1) / (+ %r6.13? 1)") __msg_next("scev at header 13:") __msg_next(" r9=(+ r9 1) / (linear r9 1)") __msg_next(" scev at latch 16:") @@ -3534,7 +3534,7 @@ SEC("xdp") __success __retval(3) __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) -__msg("r6 = *(u64 *)(r10 -8) {{.*}}; r6 ? -> r6") +__msg("r6 = *(u64 *)(r10 -8) {{.*}}; r6 %r6.4? -> r6") __msg("scev at header 1:") __msg_next(" r6=(+ r6 1) / (linear r6 1)") __msg_next(" scev at latch 16:") @@ -3655,7 +3655,7 @@ __success __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) __msg("scev at header 4:") -__msg_next("r7=(any r7 8) / (any r7 8)") +__msg_next("r7=%r7.11=phi(r7, 8) / (any r7 8)") __msg("loop header at 4, can't widen r7, expr is {{.*}}, requires exact stack-offset tracking") __naked void no_widen_stack_fill_any(void) { @@ -4429,7 +4429,7 @@ SEC("socket") __success __retval(0) __log_level(2) __msg("header_count is 100000 ") -__msg("can't widen r6, expr is _") +__msg("can't widen r6, expr is %r6.13") __not_msg("widening r8") __msg("clamping r8 to 0..0x1869f step 1") __naked void index_loaded_in_previous_iteration(void) -- 2.55.0 From: Alexei Starovoitov A prerequisite of the patches that use scalar evolution at conditional jumps. Expressions exist in loops only. Compute them for all code. They have no users yet: no verdict changes, the prog below is still rejected. char *buf = bpf_map_lookup_elem(&map, &key); /* 128 bytes */ .. p = buf + i; /* i is a byte */ if (i > 100) return 0; *p = buf[0]; The jump bounds i. p was computed before: invalid access to map value, value_size=128 off=255 size=1 That p is buf + i is a static fact: it holds on every path. Treat the subprog as the outermost loop, which runs once: compute_scev_for_loop() with -1 for the header. (REG r) is the entry value of r. A loop is a single insn for the subprog, as a nested loop is for the enclosing one, so join_successor(), topmost_nested_loop(), forget_non_invariants() and the worklist work as they are. The body is acyclic: each basic block is visited once, and a def is one value per call of the subprog. The prog above, from the level 2 log after 'Computing SCEV for subprog at 0:': 2: r7 &= 255 r7 is (& %r0.0 255) 8: call bpf_map_lookup_elem#1 r0 is %r0.8 10: r6 = r0 11: r2 = r6 12: r2 += r7 r2 is (+ %r0.8 (& %r0.0 255)) 13: if r7 > 0x64 goto pc+2 %r0.8 is the def of r0 by insn 8. The verifier adds edges that the CFG does not have, see bpf_head_guard_target(): from may_goto before a loop header, and from a subprog that is out of budget to its last exit. Nothing is known on them: join_added_edges() joins '?' at their targets, as bpf_compute_const_regs() does. A subprog that has a loop with a truncated exit list is not analyzed: the exits are the successors of the loop as a single insn. The worklist is in reverse post-order: all predecessors come first. So the env of a basic block, 2140 bytes, is freed when the block is done, unless it starts with a header, a latch or an exit. That goes for loops too. Their envs stayed until the end of bpf_check(), so memory goes down. To load the selftests, Cilium, sched_ext, katran, bpfjailer and programs in Rust the kernel spends 1.9 % more instructions (perf stat), and load time goes up by 2 to 3 % (veristat). mem_peak of veristat goes down by 1.1 % for the C programs and by 7.3 % for those in Rust. reg_bounds.c looks for '18: (bf) r0 = r6' and ';' in the log. Scalar evolution prints that for code outside of loops now, after 't'. Signed-off-by: Alexei Starovoitov --- kernel/bpf/scev.c | 130 +++++++++++++++--- .../selftests/bpf/prog_tests/reg_bounds.c | 5 +- .../selftests/bpf/progs/verifier_scev.c | 4 +- 3 files changed, 118 insertions(+), 21 deletions(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index fdeedba90d66..f47faeb0ba53 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -286,7 +286,8 @@ static int imm_expr(struct scev *scev, s64 value) /* * The insn at @idx defines @reg. Without its backedges and with nested loops collapsed the body is - * acyclic: an insn executes at most once per iteration, so a def is one value per iteration. + * acyclic: an insn executes at most once per iteration, so a def is one value per iteration. A + * subprog is the outermost loop, which runs once. */ static struct expr_def insn_def(int idx, u32 reg) { @@ -766,6 +767,25 @@ static struct env *get_loop_env(struct scev *scev, int loop_header, int insn_idx return e; } +/* The env has no users any more */ +static void free_loop_env(struct scev *scev, int loop_header, int insn_idx) +{ + struct insn_envs *envs = scev->envs[insn_idx]; + u32 i; + + for (i = 0; i < envs->cnt; i++) { + if (envs->entries[i].loop_header != loop_header) + continue; + kfree(envs->entries[i].env); + envs->entries[i] = envs->entries[--envs->cnt]; + break; + } + if (envs->cnt) + return; + kfree(envs); + scev->envs[insn_idx] = NULL; +} + static int replace_reg(struct scev *scev, struct env *e, u32 reg, int id) { if (id < 0) @@ -1315,7 +1335,7 @@ static int worklist_push(struct bpf_verifier_env *env, int loop_header, int idx) { struct scev *scev = env->scev; struct scev_worklist *worklist = &scev->worklist; - u32 tag = (u32)loop_header + 1; + u32 tag = loop_header + 2; /* -1 is a subprog */ if (scev->discovered[idx] == tag) return 0; @@ -1599,11 +1619,52 @@ static bool can_compute_loop_scev(const struct bpf_loop *loop) return !loop->irreducible && !loop->nested_exits_overflow; } -static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) +/* A loop is a single instruction for the subprog too: no exit list may be truncated. */ +static bool can_compute_subprog_scev(struct bpf_verifier_env *env, struct bpf_subprog_info *sub) +{ + struct bpf_loop *loop; + int i; + + for (i = sub->start; i < (sub + 1)->start; i++) { + loop = env->insn_aux_data[i].loop; + if (loop && (loop->exits_overflow || loop->nested_exits_overflow)) + return false; + } + return true; +} + +/* + * The edges that the verifier adds, see bpf_head_guard_target(): bpf_insn_successors() doesn't have + * them. They leave all loops. The env at their source is not known: @cur_env is all '?'. So their + * target is a join point with a phi for every live register. + */ +static int join_added_edges(struct bpf_verifier_env *env, struct bpf_subprog_info *sub, + struct env *old_env, struct env *cur_env) +{ + int i, t, err; + + for (i = sub->start; i < (sub + 1)->start; i++) { + t = bpf_head_guard_target(env, i); + if (t < 0) + continue; + err = join_successor(env, -1, t, old_env, cur_env); + if (err) + return err; + } + return sub->bail_insn < 0 ? 0 : join_successor(env, -1, sub->bail_insn, old_env, cur_env); +} + +/* + * @cur_header and @first are a loop header. Or -1 and the first insn of a subprog: the subprog is + * the outermost loop, which runs once. (REG r) is the entry value of r, the loops in it are single + * instructions. + */ +static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header, int first) { struct scev_worklist *worklist = &env->scev->worklist; struct bpf_insn_aux_data *aux = env->insn_aux_data; - struct bpf_loop *cur_loop = aux[cur_header].loop; + struct bpf_loop *cur_loop = cur_header < 0 ? NULL : aux[cur_header].loop; + struct bpf_subprog_info *sub = bpf_find_containing_subprog(env, first); struct env *header_env; struct scev *scev = env->scev; struct bpf_loop *nested_loop; @@ -1611,17 +1672,21 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) struct env *old_env = NULL; struct bpf_iarray *succ; bool log_level2 = env->log.level & BPF_LOG_LEVEL2; - int s, i, err, idx, succ_idx; + int s, i, err, idx, succ_idx, start; u32 r; if (log_level2) - bpf_log(&env->log, "Computing SCEV for loop at %d:\n", cur_header); + bpf_log(&env->log, "Computing SCEV for %s at %d:\n", + cur_loop ? "loop" : "subprog", first); + + if (!cur_loop && !can_compute_subprog_scev(env, sub)) + return 0; /* * For irreducible loops, and for loops nesting a loop with a truncated * exit list, just assume that everything is clobbered for now. */ - if (!can_compute_loop_scev(cur_loop)) { + if (cur_loop && !can_compute_loop_scev(cur_loop)) { header_env = get_loop_env(scev, cur_header, cur_header); if (!header_env) return -ENOMEM; @@ -1640,17 +1705,30 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) if (!old_env) goto nomem; } - header_env = get_loop_env(scev, cur_header, cur_header); + header_env = get_loop_env(scev, cur_header, first); if (!header_env) goto nomem; setup_initial_loop_env(header_env); - err = worklist_push(env, cur_header, cur_header); + err = worklist_push(env, cur_header, first); if (err) goto out; + if (!cur_loop) { + /* no backedge overwrites it */ + header_env->empty = false; + err = join_added_edges(env, sub, old_env, cur_env); + if (err) + goto out; + } while (worklist->nr) { idx = *min_heap_peek_inline(worklist); min_heap_pop_inline(worklist, &worklist_cb, env->cfg.postorder_nums); + start = idx; + if (signal_pending(current)) { + err = -EAGAIN; + goto out; + } + cond_resched(); /* join_successor() maps nested-loop entries to their representative header. */ nested_loop = idx != cur_header ? aux[idx].loop : NULL; @@ -1662,12 +1740,16 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) */ if (log_level2) memcpy(old_env, cur_env, sizeof(*old_env)); - /* Pull the nested loop's stack-store base dependencies up. */ - for_each_set_bit(r, nested_loop->store_base_regs, BPF_SCEV_REGS_NUM) - or_expr_regs(env, cur_env->reg2expr[r], cur_loop->store_base_regs); - /* And what its jumps depend on. */ - for_each_set_bit(r, nested_loop->tested_regs, BPF_SCEV_REGS_NUM) - or_expr_regs(env, cur_env->reg2expr[r], cur_loop->tested_regs); + if (cur_loop) { + /* Pull the nested loop's stack-store base dependencies up. */ + for_each_set_bit(r, nested_loop->store_base_regs, BPF_SCEV_REGS_NUM) + or_expr_regs(env, cur_env->reg2expr[r], + cur_loop->store_base_regs); + /* And what its jumps depend on. */ + for_each_set_bit(r, nested_loop->tested_regs, BPF_SCEV_REGS_NUM) + or_expr_regs(env, cur_env->reg2expr[r], + cur_loop->tested_regs); + } err = forget_non_invariants(env, cur_env, idx); if (err) goto out; @@ -1691,8 +1773,12 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) for (;;) { if (log_level2) memcpy(old_env, cur_env, sizeof(*old_env)); - collect_store_base_regs(env, cur_env, idx, cur_loop->store_base_regs); - collect_tested_regs(env, cur_env, idx, cur_loop->tested_regs); + if (cur_loop) { + collect_store_base_regs(env, cur_env, idx, + cur_loop->store_base_regs); + collect_tested_regs(env, cur_env, idx, + cur_loop->tested_regs); + } err = transfer(env, cur_env, idx); if (err) goto out; @@ -1717,6 +1803,9 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) goto out; } } + /* Reverse post-order: all predecessors came first, the env has no users any more */ + if (!aux[start].need_scev) + free_loop_env(scev, cur_header, start); } err = 0; @@ -2124,7 +2213,12 @@ int bpf_compute_scev(struct bpf_verifier_env *env) idx = postorder[i]; if (!aux[idx].loop) continue; - err = compute_scev_for_loop(env, idx); + err = compute_scev_for_loop(env, idx, idx); + if (err) + return err; + } + for (i = 0; i < env->subprog_cnt; i++) { + err = compute_scev_for_loop(env, -1, env->subprog_info[i].start); if (err) return err; } diff --git a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c index 09f7930733be..c2079eb64201 100644 --- a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c +++ b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c @@ -1290,7 +1290,8 @@ static int parse_range_cmp_log(const char *log_buf, struct case_spec spec, * 18: (bf) r0 = r6 ; R0=... R6=... * Different verifier passes may print * 18: (bf) r0 = r6 - * as well, but never followed by ';'. + * as well, but never followed by ';'. Except for scalar evolution: + * t 18: (bf) r0 = r6 ; r0 ... -> ... * The branches are in the log in the order the verifier looked at them. */ q = log_buf; @@ -1299,7 +1300,7 @@ static int parse_range_cmp_log(const char *log_buf, struct case_spec spec, while (*s == ' ' || *s == '\t') s++; - if (*s == ';') + if (*s == ';' && (q == log_buf || q[-1] == '\n')) break; q = s; } diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index de7679656831..939f49a4ebdb 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -2346,12 +2346,14 @@ __naked __noinline __used static int returns_zero(void) * A loop that the compiler unrolls in part has an exit for every 'break' and 'return' in every * copy of its body. How many there are says nothing about the number of iterations. * Scalar evolution takes a nested loop for one insn, and its exits are the successors of this insn. - * The list of exits is not complete: the loop around has no formulas. + * The exit list is truncated: the enclosing loop has no SCEVs, the subprog is not analyzed. */ SEC("xdp") __success __log_level(2) __msg("loop at 1, too many exits, too many exits of a nested loop") __msg("loop at 2, nested in 1, too many exits") +__msg("Computing SCEV for subprog at 0:") +__msg_next("Computing SCEV for subprog at {{[0-9]+}}:") __msg("loop header at 1, unsupported loop: too many exits of a nested loop") __msg("loop header at 2, header_count is [0..4] ") __naked void loop_around_301_exits(void) -- 2.55.0 From: Alexei Starovoitov A fix of the declaration of a helper. This series needs it: scalar evolution keeps the expression of a stack slot across a call unless bpf_may_write_mask() of the call has the slot, and liveness.c computes that mask from the argument types of the helper. union bpf_attr attr = { .btf = .., .btf_size = .., .btf_log_level = 1 }; bpf_sys_bpf(BPF_BTF_LOAD, &attr, sizeof(attr)); BPF_BTF_LOAD and BPF_PROG_LOAD store log_true_size into the attr of the caller. When BPF_PROG_LOAD rejects func_info, line_info or core_relo records that are larger than it knows, it stores the record size that it expects. For the helper the attr is a kernel pointer, so copy_to_bpfptr_offset() writes to the memory of the program. arg2_type in struct bpf_func_proto bpf_sys_bpf_proto is ARG_PTR_TO_MEM | MEM_RDONLY. So the verifier keeps what it knows about an attr on the stack across the call: a slot that held 0 is taken for 0 after it. bpf_may_write_mask() of the call does not have the slots of the attr. And an attr in a map that the program cannot write to is accepted. Say MEM_WRITE. Fixes: 79a7f8bdb159 ("bpf: Introduce bpf_sys_bpf() helper and program type.") Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 9004f2673ccf..04a3b3c80c74 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -6692,7 +6692,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, - .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg3_type = ARG_MEM_SIZE, }; -- 2.55.0 From: Alexei Starovoitov A consistency check of the verifier, no new ability. Scalar evolution trusts bpf_may_write_mask(), the stack slots that an insn may write, and nothing checks it. The patches that follow refine stack slots at conditional jumps by their expressions, so more is going to depend on it. int n = 10, v; for (i = 0; i < n; i++) bpf_probe_read_kernel(&v, sizeof(v), p + i); With n on the stack scalar evolution computes the iteration count (header_count: how often the header is executed) if the call does not write n. It takes that from the mask of the call. liveness.c computes the masks before the main pass: it tracks the values derived from FP (struct arg_track) and looks at the argument types of helpers and kfuncs. A counted loop gets no may_goto, and i gets its range from the iteration count: nothing detects a wrong mask later. Let the main pass check the masks. It sees every write to a stack with the offset that it has in the state: a store in check_stack_write(), the memory that a helper or a kfunc writes in check_stack_range_initialized(). Compare the slots with the mask of the insn. If a subprog writes to the stack of its caller it is the mask of the callsite in the frame of the caller. A slot that is not in the mask is a bug of the verifier: verifier_bug_if(), "insn %d writes to fp%d outside of may_write_mask". The state can be less precise than the mask: struct st *cur = &first; for (..) { ..; cur = &rest; } cur->last = 1; The mask of the store has two slots. cur is spilled, and at the loop header the two offsets are joined to a range. So if the offset is not constant in the state only require that some slot of the range is in the mask. Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 36 +++++++++++++++++++ .../selftests/bpf/progs/verifier_raw_stack.c | 35 ++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5bffc1e288f6..ae3b751b99a1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4327,6 +4327,31 @@ static void invalidate_stack_slices(struct bpf_verifier_env *env, u32 frameno, })); } +/* + * Scalar evolution takes the may-defs of stack slots from bpf_may_write_mask(): a slot that is not + * in the mask holds the same value before and after the insn. The iteration count of a loop depends + * on that, and nothing detects a wrong one later. Here an insn, a helper or a kfunc writes to + * [min_off, max_off) of frame @frameno. For a frame of a caller the insn is the callsite. Without + * @fixed_off the range can be wider than the mask: two offsets become a range at a join point. Then + * the write is somewhere in the range. + */ +static int check_may_write_mask(struct bpf_verifier_env *env, u32 frameno, int min_off, int max_off, + bool fixed_off) +{ + struct bpf_verifier_state *st = env->cur_state; + int idx = frameno == st->curframe ? env->insn_idx : st->frame[frameno + 1]->callsite; + int spi, first = bpf_get_spi(max_off - 1), last = bpf_get_spi(min_off); + const unsigned long *mask = bpf_may_write_mask(env, idx); + + spi = fixed_off ? find_next_zero_bit(mask, last + 1, first) : last + 1; + if (!fixed_off && find_next_bit(mask, last + 1, first) > last) + spi = first; + if (verifier_bug_if(spi <= last, env, "insn %d writes to fp%d outside of may_write_mask", + idx, -(spi + 1) * BPF_REG_SIZE)) + return -EFAULT; + return 0; +} + /* check_stack_write dispatches to check_stack_write_fixed_off or * check_stack_write_var_off. * @@ -4345,6 +4370,10 @@ static int check_stack_write(struct bpf_verifier_env *env, int max_off = reg_smax(reg) + off + size; int err; + err = check_may_write_mask(env, state->frameno, min_off, max_off, + tnum_is_const(reg->var_off)); + if (err) + return err; if (tnum_is_const(reg->var_off)) { off += reg->var_off.value; err = check_stack_write_fixed_off(env, state, off, size, @@ -8032,6 +8061,13 @@ static int check_stack_range_initialized( max_off = reg_smax(reg) + off; } + if (clobber && access_size) { + err = check_may_write_mask(env, state->frameno, min_off, max_off + access_size, + tnum_is_const(reg->var_off)); + if (err) + return err; + } + /* Unprivileged outputs retain each byte's initialization state. */ if (raw_mode) { meta->arg_raw_mem.size[arg_slot] = access_size; diff --git a/tools/testing/selftests/bpf/progs/verifier_raw_stack.c b/tools/testing/selftests/bpf/progs/verifier_raw_stack.c index 0fe631411b9c..d0975a28a243 100644 --- a/tools/testing/selftests/bpf/progs/verifier_raw_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_raw_stack.c @@ -125,6 +125,41 @@ __naked void stack_skb_load_bytes_init(void) : __clobber_all); } +/* The pointer is fp-128 or fp-96. At the loop header the two offsets are joined to a range. */ +SEC("tc") +__description("raw_stack: store to one of two buffers") +__success __retval(0) +__naked void store_to_one_of_two_buffers(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r8 = r0; \ + r1 = 0; \ + *(u64*)(r10 - 144) = r1; \ + *(u64*)(r10 - 120) = r1; \ + *(u64*)(r10 - 88) = r1; \ + r3 = r10; \ + r3 += -128; \ + *(u64*)(r10 - 136) = r3; \ +l0_%=: r8 *= 8; \ + if w8 s> 100 goto l1_%=; \ + r3 = r10; \ + r3 += -96; \ + *(u64*)(r10 - 136) = r3; \ + r4 = *(u64*)(r10 - 144); \ + r4 += 1; \ + *(u64*)(r10 - 144) = r4; \ + if r4 < 3 goto l0_%=; \ +l1_%=: r3 = *(u64*)(r10 - 136); \ + r1 = 7; \ + *(u64*)(r3 + 8) = r1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + SEC("tc") __description("raw_stack: skb_load_bytes, spilled regs around bounds") __success __retval(0) -- 2.55.0 From: Alexei Starovoitov A fix of the main pass that stands alone. It is also a prerequisite of the patch that refines linked registers at conditional jumps: that patch intersects the range of one register with var_off of another one and asks reg_is_empty() if the intersection is empty. 1: r0 <<= 3 2: r0 += 5 3: if r0 s< -2 goto pc+2 4: if r0 s> 1 goto pc+1 5: r0 += 3 verifier bug: REG INVARIANTS VIOLATION (alu): range bounds violation The low three bits of r0 are 5. No value in [-2, 1] has them: the range and var_off don't overlap, the fall-through branch of insn 4 is dead. The verifier takes it with an empty register, and the addition at insn 5 trips over that. simulate_both_branches_taken() refines the registers as each branch does and drops a branch in which a range is empty. It relies on reg_bounds_sync() to empty the range. __update_reg64_bounds() steps through var_off from umin to umax, which are 0 and U64_MAX for a range that crosses 0. __update_reg32_bounds() uses only the smallest and the largest member of var_off. tnum_step() returns the next member of var_off. reg_is_empty() asks for the first one from the start of the range on and compares it with the end, in 64 and in 32 bits. A range that crosses 0 wraps around at U64_MAX or U32_MAX: tnum_overlaps_cnum() splits it there. The test is sufficient, not exact: 64 and 32 bits are checked each on its own. Two tests in verifier_bounds.c, with the values above shifted left by 32: no member of var_off is in [-2, 1] and the branch is dead, -3 is one in [-3, 1] and the branch is verified. Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 34 +++++++++++++++++-- .../selftests/bpf/progs/verifier_bounds.c | 33 ++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ae3b751b99a1..83249ecbdfb1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2325,6 +2325,36 @@ static bool range_bounds_violation(struct bpf_reg_state *reg) return cnum32_is_empty(reg->r32) || cnum64_is_empty(reg->r64); } +/* Do @t and [min, max] overlap: is a member of @t in the range? */ +static bool tnum_overlaps_urange(struct tnum t, u64 min, u64 max) +{ + u64 tnum_next = tnum_step(t, min); + + return (min & ~t.mask) == t.value || (tnum_next > min && tnum_next <= max); +} + +/* The same for the cnum [@base, @base + @size]. It wraps around at @ut_max. */ +static bool tnum_overlaps_cnum(struct tnum t, u64 base, u64 size, u64 ut_max) +{ + u64 end = (base + size) & ut_max; + + if (base <= end) + return tnum_overlaps_urange(t, base, end); + return tnum_overlaps_urange(t, base, ut_max) || tnum_overlaps_urange(t, 0, end); +} + +/* + * A range and var_off don't overlap. reg_bounds_sync() compares var_off with umin and umax: it + * doesn't see that [-2, 1], which crosses 0, has no member with 5 in its low three bits. + */ +static bool reg_is_empty(struct bpf_reg_state *reg) +{ + return range_bounds_violation(reg) || + !tnum_overlaps_cnum(reg->var_off, reg->r64.base, reg->r64.size, U64_MAX) || + !tnum_overlaps_cnum(tnum_subreg(reg->var_off), reg->r32.base, reg->r32.size, + U32_MAX); +} + static int reg_bounds_sanity_check(struct bpf_verifier_env *env, struct bpf_reg_state *reg, const char *ctx) { @@ -18952,7 +18982,7 @@ static int simulate_both_branches_taken(struct bpf_verifier_env *env, u8 opcode, * reg_states in the FALSE branch (i.e. reg1, reg2), the FALSE branch must be dead. Only * TRUE branch will be taken. */ - if (range_bounds_violation(&env->false_reg1) || range_bounds_violation(&env->false_reg2)) + if (reg_is_empty(&env->false_reg1) || reg_is_empty(&env->false_reg2)) return 1; /* Jump (TRUE) branch */ @@ -18964,7 +18994,7 @@ static int simulate_both_branches_taken(struct bpf_verifier_env *env, u8 opcode, * reg_states in the TRUE branch (i.e. true_reg1, true_reg2), the TRUE branch must be dead. * Only FALSE branch will be taken. */ - if (range_bounds_violation(&env->true_reg1) || range_bounds_violation(&env->true_reg2)) + if (reg_is_empty(&env->true_reg1) || reg_is_empty(&env->true_reg2)) return 0; /* Both branches are possible, we can't determine which one will be taken. */ diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index f55d58af8004..316e9377caa8 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -1995,6 +1995,39 @@ __naked void bounds_refinement_multiple_overlaps(void *ctx) : __clobber_all); } +/* + * r0 is x << 32, the low three bits of x are 5. No such x is in [-2, 1], -3 is in [-3, 1]. + * umin and umax are unbounded for a range that crosses 0. + */ +#define RANGE_CROSSES_0(name, smin) \ +__naked void name(void *ctx) \ +{ \ + asm volatile( \ + "call %[bpf_ktime_get_ns];" \ + "r0 <<= 35;" \ + "r1 = 0x500000000 ll;" \ + "r0 += r1;" \ + "r1 = " #smin " ll;" \ + "if r0 s< r1 goto 1f;" \ + "r1 = 0x100000000 ll;" \ + "if r0 s> r1 goto 1f;" \ + "r10 = 0;" \ +"1: exit;" \ + : \ + : __imm(bpf_ktime_get_ns) \ + : __clobber_all); \ +} + +SEC("socket") +__description("dead branch: tnum and a range that crosses 0 don't overlap") +__success +RANGE_CROSSES_0(range_crossing_0_no_tnum_overlap, -0x200000000) + +SEC("socket") +__description("tnum overlaps the negative part of a range that crosses 0") +__failure __msg("frame pointer is read only") +RANGE_CROSSES_0(range_crossing_0_tnum_overlap_below_0, -0x300000000) + SEC("socket") __success __naked void signed_unsigned_intersection32_case1(void *ctx) -- 2.55.0 From: Alexei Starovoitov A fix of the main pass for big endian that stands alone. The patches that follow refine more registers and stack slots at conditional jumps, so more progs would run into it. *(u64 *)(r10 - 8) = r1 r2 = *(u32 *)(r10 - 8) r2 gets the low 32 bits of r1 on little endian and the upper 32 bits on big endian. In two cases the verifier gives r2 the range of the low 32 bits on big endian too. The fill itself is right: check_stack_read_fixed_off() asks bpf_stack_narrow_access_ok() where the low bytes of a spill are. A fill from elsewhere gets an unbounded scalar. After the 32-bit spill '*(u32 *)(r10 - 8) = r1' the half-slot at r10 - 8 is STACK_SPILL and spilled_ptr describes r1, whose low 32 bits it holds on both byte orders. Two functions take a 64-bit spill for such a slot: - __clean_func_state() marks a dead half-slot STACK_POISON. If that is the one at r10 - 4, what remains of a 64-bit spill has the slot types of a 32-bit spill. - stacksafe() skips the bytes that are STACK_MISC in the old state. An old 32-bit spill and a current 64-bit spill are then compared by regsafe() of spilled_ptr. That is right on little endian, where the half-slot at r10 - 8 has the low 32 bits of a 64-bit spill. On big endian it has the upper 32 bits. Keep such a 64-bit spill whole in __clean_func_state(). In stacksafe() compare two spills only if the bytes of the old one are the low bytes of the current one. bpf_stack_narrow_access_ok() decides in both. Two tests in verifier_spill_fill.c, one per function. They load on little endian and are rejected on big endian. Fixes: be23266b4a08 ("bpf: 4-byte precise clean_verifier_state") Fixes: 6715df8d5d24 ("bpf: Allow reads from uninit stack") Signed-off-by: Alexei Starovoitov --- kernel/bpf/states.c | 19 ++++++++++++ .../selftests/bpf/progs/verifier_spill_fill.c | 31 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index efc0870b85c2..b60676349655 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -168,6 +168,16 @@ static void __clean_func_state(struct bpf_verifier_env *env, stype == STACK_IRQ_FLAG) continue; + /* + * When the half-slot closer to FP is dead, what remains of a + * 64-bit spill looks like a 32-bit spill: the low 32 bits of + * the value. On big endian the other half-slot holds the upper + * 32 bits. + */ + if (hi_live && bpf_spill_size(bpf_stack_slot(st, i)) == BPF_REG_SIZE && + !bpf_stack_narrow_access_ok(0, BPF_REG_SIZE / 2, BPF_REG_SIZE)) + continue; + /* * Only scalar spills can be degraded to raw stack bytes * when their high half is dead. Pointer spills need the @@ -446,6 +456,15 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, if (!cur_slot) return false; + /* + * The old spill is the first bytes of a wider spill. On big endian + * these are not the low bytes of the value. + */ + if (old_type == STACK_SPILL && bpf_is_spilled_reg(cur_slot) && + !bpf_stack_narrow_access_ok(0, bpf_spill_size(old_slot), + bpf_spill_size(cur_slot))) + return false; + /* * 64 and 32-bit scalar spills vs MISC/INVALID slots and vice versa. * Load from MISC/INVALID slots produces unbound scalar. diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index 0baa8e06b8d1..623ed5ef3630 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c @@ -1451,6 +1451,37 @@ __naked void imprecise_scalar_spill_half_dead(void) : __clobber_all); } +/* + * The first half-slot of a 64-bit spill holds its low 32 bits on little endian only. + * The test spills to r10 - 8. The first four bytes of the slot are an offset into one byte. + */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define FIRST_HALF_IS_LOW __success +#else +#define FIRST_HALF_IS_LOW __failure __msg("access may be outside object bounds") +#endif +#define FIRST_HALF(name, stores) \ +SEC("raw_tp") FIRST_HALF_IS_LOW \ +__naked void name(void) \ +{ \ + asm volatile ( \ + "call %[bpf_ktime_get_ns];" \ + stores \ + "r7 = *(u32 *)(r10 - 8); r1 = %[single_byte_buf] ll; r1 += r7; r0 = *(u8 *)(r1 + 0);" \ + "out_%=: r0 = 0; exit;" \ + : \ + : __imm(bpf_ktime_get_ns), __imm_addr(single_byte_buf) \ + : __clobber_all); \ +} + +/* The low 32 bits of the value are 0. The second half-slot is dead after the jump. */ +FIRST_HALF(first_half_of_spill_second_half_dead, + "r0 <<= 32; *(u64 *)(r10 - 8) = r0; if r0 == 0 goto out_%=;") +/* One path has a 32-bit spill of r6, whose low 32 bits are 0, the other one a 64-bit spill */ +FIRST_HALF(spill_of_64_bits_vs_spill_of_32_bits, + "r6 = r0; r6 <<= 32; if r0 > 5 goto 1f; *(u32 *)(r10 - 8) = r6; goto 2f;" + "1: r6 = 0x7ffffff000000000 ll; *(u64 *)(r10 - 8) = r6; 2:") + /* check valid spill/fill, ptr to tp buffer */ SEC("raw_tracepoint.w") __success -- 2.55.0 From: Alexei Starovoitov A new ability, and the first user of scalar evolution outside of loops. The linked registers of a conditional jump are the live registers and stack slots that are a tested register + a constant on every path to the jump. scev.c computes them before the main pass, and the main pass refines them together with the tested registers. Scalar ids do some of that today. They stay for now, later patches remove them. if (bpf_dynptr_from_skb(skb, 0, &ptr)) return NF_DROP; .. return NF_DROP; That is ip_check_defrag.c of the selftests. NF_DROP is 0, and GCC returns the 0 that the call returned: 27: call bpf_dynptr_from_skb 28: r5 = (s32)r0 29: if w0 != 0x0 goto pc-12 .. 46: r0 = r5 50: exit At program exit the register R0 has smin=-2147483648 smax=2147483647 should have been in [0, 1] The jump bounds r0. r5 got no scalar id at 28: r0 was not known to be below 2^31. Before insn 29 r0 is %r0.27, the def of r0 by insn 27, and r5 is (sext32 %r0.27). split_const_offset() strips added constants, extensions and partial spills: expr == base + off (mod 2^bits). Expressions with the same base, a value number, have a constant difference: const_diff(). Then find_linked_regs() links the registers: linked == tested + delta (mod 2^bits), here r5 == r0 (mod 2^32). Unlike scalar ids the links are not stored in a state and have no limit of five registers (LINKED_REGS_MAX). A link may hold modulo 2^N only, and it stays within one frame. At the header, a latch or an exit of a loop the difference may also be one that holds at the header in every iteration: it holds at loop entry and every backedge preserves it, const_diff_at_header(). refine_linked_regs() adds delta to the tested register and intersects the linked one with the result, as 'if r5 == r0' does, in 64 or 32 bits. Values that are equal modulo 2^8, 2^16 or 2^32 are equal if both fit in uN or both in sN: then all 64 bits are intersected. The intersection is empty: no execution gets here with this state (the path is infeasible), the register is left as it is. Types are read in the state: both have to be scalars. scev.c has the insns, no types and no states. Where that matters: - Arena: an ALU64 insn is converted to ALU32 at fixup if an operand is an arena pointer. In a prog with an arena its result gets a def too: %r2.12=(+ x 4) is x + 4 in 64 or in 32 bits. It is trusted modulo 2^32, and modulo 2^64 once the main pass has verified insn 12 without an arena pointer: prevent_zext is only ever set, and the def dominates every use. %r2.25=phi((+ x 4)) joins two such defs. The phi is not (+ x 4). - Slices: a store through what bpf_dynptr_slice_rdwr() returned may alias the buffer on the stack and is in no bpf_may_write_mask(). In a prog with that kfunc a slot that a call may write is a value number only after a store through r10, which invalidates the slices. 20 more progs of the selftests load when GCC builds them, among them profiler1-3, test_sk_lookup and setget_sockopt. The kernel spends 0.7 % more instructions on loading the selftests, Cilium, sched_ext, katran and bpfjailer, 1.5 % more on programs in Rust. scalars_double_add and subreg_equality_2 expected a missing link and load now. The arena tests of verifier_scev.c move to verifier_arena.c: libbpf binds an arena to all progs of an object, so every prog of verifier_scev.c had one. Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 4 + kernel/bpf/scev.c | 464 ++++++++++++++++-- kernel/bpf/verifier.c | 96 ++++ .../selftests/bpf/progs/verifier_arena.c | 28 ++ .../selftests/bpf/progs/verifier_bounds.c | 2 +- .../bpf/progs/verifier_linked_scalars.c | 5 +- .../selftests/bpf/progs/verifier_reg_equal.c | 11 +- .../selftests/bpf/progs/verifier_scalar_ids.c | 6 +- .../selftests/bpf/progs/verifier_scev.c | 36 -- 9 files changed, 569 insertions(+), 83 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 4771c55d8f41..18f0736fccb7 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1902,6 +1902,7 @@ u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); +bool bpf_is_dynptr_slice_rdwr_kfunc(struct bpf_insn *insn); int bpf_compute_const_regs(struct bpf_verifier_env *env); int bpf_prune_dead_branches(struct bpf_verifier_env *env); int bpf_check_cfg(struct bpf_verifier_env *env); @@ -2224,6 +2225,9 @@ int bpf_init_scev(struct bpf_verifier_env *env); void bpf_free_scev(struct bpf_verifier_env *env); int bpf_compute_scev(struct bpf_verifier_env *env); int bpf_scev_jump_counts(struct bpf_verifier_env *env, int header, int n, bool taken_leaves); +bool bpf_scev_next_linked_reg(struct bpf_verifier_env *env, int insn_idx, u32 regno, u32 *pos, + u32 *linked_regno, u32 *fill_off, s64 *delta, u32 *bits); +bool bpf_scev_in_slice_slots(struct bpf_verifier_env *env, int off, u32 size); int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st, struct bpf_loop_iters *iters); diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index f47faeb0ba53..dfbad71bba55 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -64,6 +64,10 @@ enum expr_op { * value number of its own. * In the env of loop H a phi at H merges the backedge values only, the header phi itself is * (REG r). In the env of the enclosing loop a phi at H merges the entering edges. + * Op and params of a def by an insn are what that insn computes, see alu64_with_def(): + * set_expr_base() looks through such an add, split_const_offset() checks prevent_zext of the insn. + * Those of a phi are the shape that its operands share, no insn computes them: nothing looks + * through a phi. */ struct expr_def { u32 set:1; /* 0: the expression has no def */ @@ -79,6 +83,8 @@ struct expr { u16 op; /* A value number: all SCEV registers of an env that have this id hold the same value */ bool is_vn; + /* expr == base + off (mod 2^bits), see split_const_offset() */ + u8 bits; /* * UNKNOWN, OPAQUE and ANY don't describe a value. With a def the id stands for one value, * see struct expr_def. Without a def every use of the id can be another value. @@ -88,6 +94,13 @@ struct expr { u32 params[2]; s64 imm; }; + u32 base; + /* + * id of the first add on the way to 'base' whose insn may still become ALU32, see + * prevent_zext. 0: none + */ + u32 arena_add; + s64 off; }; struct env { @@ -96,8 +109,17 @@ struct env { u32 reg2scev[REGS_NUM]; }; +/* A SCEV register and the id of its expression before a conditional jump */ +struct scev_linked_reg { + u32 reg; + u32 id; +}; + struct insn_envs { u32 cnt; + /* The linked registers of a jump, see find_linked_regs(): linked_cnt of scev->linked[] */ + u32 linked_first; + u32 linked_cnt; struct { int loop_header; struct env *env; @@ -118,6 +140,8 @@ struct scev { /* Read-only representatives for stack slots without a saved register. */ struct bpf_reg_state zero_reg; struct bpf_reg_state unknown_reg; + /* r -> (REG r): before the first insn of a loop every SCEV register is its header phi */ + struct env identity_env; /* * Expressions are identified by id, exprs_ht ensures that * each expression exists as a unique instance. @@ -130,6 +154,9 @@ struct scev { struct scev_worklist worklist; struct insn_envs **envs; struct expr *exprs; + struct scev_linked_reg *linked; + /* The stack slots that a store through a dynptr slice may alias, see expr_is_vn() */ + DECLARE_BITMAP(slice_slots, MAX_BPF_STACK_SLOTS); /* * Loops are analyzed one by one, this array keeps track if a particular * instruction was visited on a current pass. @@ -137,6 +164,8 @@ struct scev { u32 *discovered; int exprs_cnt; int exprs_cap; + u32 linked_cnt; + u32 linked_cap; int envs_cnt; int stack_sz; int simplify_depth; @@ -199,10 +228,17 @@ static u32 op_params_num(u32 op); static bool expr_is_vn(struct scev *scev, struct expr *e) { - u32 i; + u32 i, reg = e->def.reg; + /* + * bpf_dynptr_slice_rdwr() can return the buffer that it got. A store through that slice may + * alias the slots of the buffer and is in no bpf_may_write_mask(): it is no def, so a def + * of such a slot is no value number. A spill through r10 has the expression of the + * register, and invalidate_stack_slices() invalidates the slices there. The next slice + * comes from a call that may write to the slot. + */ if (e->def.set) - return true; + return reg < MAX_BPF_REG || !test_bit(reg - MAX_BPF_REG, scev->slice_slots); switch (e->op) { case UNKNOWN: case OPAQUE: @@ -222,6 +258,37 @@ static bool expr_is_vn(struct scev *scev, struct expr *e) return true; } +static u32 op_low_bits(u32 op); + +/* + * @e gets the id @id. (+ x imm), an extension and a partial spill of x have the base of x, which + * was constructed earlier. Every other expression is its own base. So split_const_offset() does not + * traverse the expression. + */ +static void set_expr_base(struct scev *scev, struct expr *e, u32 id) +{ + u32 low = op_low_bits(e->op); + struct expr *x; + s64 imm = 0; + + e->base = id; + e->off = 0; + e->bits = 64; + e->arena_add = 0; + /* nothing looks through a phi, see struct expr_def */ + if (e->def.phi) + return; + if (e->op == BPF_ADD && scev->exprs[e->params[1]].op == IMM) + imm = scev->exprs[e->params[1]].imm; + else if (!low || e->def.set) + return; + x = &scev->exprs[e->params[0]]; + e->base = x->base; + e->off = (u64)x->off + imm; + e->bits = low ? min(x->bits, low) : x->bits; + e->arena_add = e->def.set ? id : x->arena_add; +} + static int add_expr(struct scev *scev, struct expr e) { u32 id; @@ -241,6 +308,7 @@ static int add_expr(struct scev *scev, struct expr e) id = scev->exprs_cnt++; e.is_vn = expr_is_vn(scev, &e); + set_expr_base(scev, &e, id); scev->exprs[id] = e; exprs_ht_add(scev, id); return id; @@ -589,6 +657,9 @@ static bool expr_next(struct scev *scev, u32 *id, u32 *order) * %r8.7=phi(1, 2) the phi of r8 before insn 7 * %r0.7=phi(_) a phi of distinct OPAQUE values * %r6.9=phi(?) a phi of distinct UNKNOWN values, or of more than 8 values + * %r2.12=(+ x 4) insn 12 defines r2, see alu64_with_def() + * %r2.25=phi((+ x 4)) two adds of one shape, each may have been converted to ALU32: + * the phi is not (+ x 4) * A name is unique within the env of one loop. In the env of loop H a phi at H merges the backedge * values only, the header phi itself is (REG r): in 'scev at header 1: r6=%r6.1=phi(r6, (+ r6 1))' * one backedge leaves r6 as it is, the other one adds 1. @@ -753,7 +824,7 @@ static struct env *get_loop_env(struct scev *scev, int loop_header, int insn_idx if (!e) return NULL; - tmp = krealloc(envs, struct_size(envs, entries, cnt + 1), GFP_KERNEL_ACCOUNT); + tmp = krealloc(envs, struct_size(envs, entries, cnt + 1), GFP_KERNEL_ACCOUNT | __GFP_ZERO); if (!tmp) { kfree(e); return NULL; @@ -780,7 +851,7 @@ static void free_loop_env(struct scev *scev, int loop_header, int insn_idx) envs->entries[i] = envs->entries[--envs->cnt]; break; } - if (envs->cnt) + if (envs->cnt || envs->linked_cnt) return; kfree(envs); scev->envs[insn_idx] = NULL; @@ -832,7 +903,7 @@ static int forget_call_regs(struct bpf_verifier_env *env, struct env *e, int idx static u32 spill_spi(struct bpf_insn *insn) { - return -insn->off / BPF_REG_SIZE - 1; + return (-insn->off - 1) / BPF_REG_SIZE; } static u32 off_to_reg(struct bpf_insn *insn) @@ -873,18 +944,39 @@ static int mk_spill(struct scev *scev, u8 size, int id) return UNKNOWN_EXPR_ID; } -static int mk_fill(struct scev *scev, int id, u8 code) +/* The size of the spill in a slot with the expression @id. It holds the low bytes of *@x. */ +static u32 expr_spill_size(struct scev *scev, u32 id, u32 *x) { - bool sx = BPF_MODE(code) == BPF_MEMSX; + struct expr *e = &scev->exprs[id]; + bool partial = e->op == SPILL8 || e->op == SPILL16 || e->op == SPILL32; + + *x = partial ? e->params[0] : id; + return partial ? op_low_bits(e->op) / BITS_PER_BYTE : BPF_REG_SIZE; +} + +/* -ENOENT: the value of the fill is not known */ +static int mk_fill(struct scev *scev, int id, struct bpf_insn *insn) +{ + u32 x, spill_size = expr_spill_size(scev, id, &x); + u32 size = bpf_size_to_bytes(BPF_SIZE(insn->code)); + bool sx = BPF_MODE(insn->code) == BPF_MEMSX; + /* The fill reads the low bytes of the spill, as in check_stack_read_fixed_off() */ + if (size <= spill_size && bpf_stack_narrow_access_ok(insn->off, size, spill_size)) + id = x; + else if (!is_spill_off(insn->off)) + return -ENOENT; #ifdef __BIG_ENDIAN + /* No register becomes (spillN x): its low bits would not be those of x */ + else if (spill_size < BPF_REG_SIZE) + return -ENOENT; /* A narrow fill reads the high bits of the slot on big-endian. */ - if (BPF_SIZE(code) != BPF_DW) + else return expr2(scev, sx ? BPF_ARSH : BPF_RSH, id, - imm_expr(scev, 64 - BITS_PER_BYTE * bpf_size_to_bytes(BPF_SIZE(code)))); + imm_expr(scev, BITS_PER_BYTE * (BPF_REG_SIZE - size))); #endif - switch (BPF_SIZE(code)) { + switch (BPF_SIZE(insn->code)) { case BPF_B: return expr1(scev, sx ? SEXT8 : ZEXT8, id); case BPF_H: return expr1(scev, sx ? SEXT16 : ZEXT16, id); case BPF_W: return expr1(scev, sx ? SEXT32 : ZEXT32, id); @@ -906,10 +998,13 @@ static int maybe_store_fp(struct scev *scev, struct env *e, struct bpf_insn *ins static int maybe_load_fp(struct scev *scev, struct env *e, struct bpf_insn *insn, int idx) { - if (insn->src_reg != BPF_REG_FP || !is_spill_off(insn->off)) + int id = -ENOENT; + + if (insn->src_reg == BPF_REG_FP && insn->off < 0 && insn->off >= -MAX_BPF_STACK_JIT) + id = mk_fill(scev, e->reg2expr[off_to_reg(insn)], insn); + if (id == -ENOENT) return forget_reg(scev, e, insn->dst_reg, OPAQUE_EXPR_ID, idx); - return replace_reg(scev, e, insn->dst_reg, - mk_fill(scev, e->reg2expr[off_to_reg(insn)], insn->code)); + return replace_reg(scev, e, insn->dst_reg, id); } /* @@ -944,10 +1039,28 @@ static void alu32_operands(struct scev *scev, u32 op, int *a, int *b) } } +/* + * In a prog with an arena an ALU64 insn is converted to ALU32 at fixup if an operand is a pointer + * to the arena, see adjust_reg_min_max_vals(). Types are not known here. So @id, what the insn at + * @idx computes from @a, gets a def: the value of @dst after the insn, converted or not. See + * split_const_offset() for when the op can be trusted. Constant operands are scalars in every + * state. + */ +static int alu64_with_def(struct scev *scev, int id, int a, int idx, u32 dst) +{ + s64 imm; + + if (id < 0 || is_imm(scev, id, &imm)) + return id; + /* (+ x 0) is x only if the insn stays ALU64 */ + return expr_with_def(scev, id == a ? UNKNOWN_EXPR_ID : id, insn_def(idx, dst)); +} + static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) { const bool little_endian = htons(0x3412) == 0x1234; struct bpf_insn *insn = &env->prog->insnsi[idx]; + bool arena = env->prog->aux->arena; struct scev *scev = env->scev; u32 *reg2expr = e->reg2expr; u8 class = BPF_CLASS(insn->code); @@ -956,7 +1069,7 @@ static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) u8 mode = BPF_MODE(insn->code); u32 dst = insn->dst_reg; u32 src = insn->src_reg; - u32 op, sext, l, r; + u32 op, sext; bool need_bswap; int i, id, a; s64 imm; @@ -1022,13 +1135,6 @@ static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) else goto mark_dst_unknown; - /* 'r1 <<= 32; r1 >>= 32' is (u32)r1 for cpu v1 and v2 */ - if (class == BPF_ALU64 && op == BPF_RSH && id >= 0 && - is_imm(scev, id, &imm) && imm == 32 && - is_binop(scev, BPF_LSH, reg2expr[dst], &l, &r) && - is_imm(scev, r, &imm) && imm == 32) - return replace_reg(scev, e, dst, expr1(scev, ZEXT32, l)); - a = reg2expr[dst]; if (class == BPF_ALU) alu32_operands(scev, op, &a, &id); @@ -1036,6 +1142,8 @@ static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) if (class == BPF_ALU) id = expr1(scev, ZEXT32, id); + else if (arena) + id = alu64_with_def(scev, id, a, idx, dst); return replace_reg(scev, e, dst, id); @@ -1654,6 +1762,227 @@ static int join_added_edges(struct bpf_verifier_env *env, struct bpf_subprog_inf return sub->bail_insn < 0 ? 0 : join_successor(env, -1, sub->bail_insn, old_env, cur_env); } +/* + * @id == *@base + *@off (mod 2^*@bits): constant offsets, extensions and partial spills are + * stripped, see set_expr_base(). An add with a def holds modulo 2^32 whether its insn is converted + * to ALU32 or not, see alu64_with_def(). It holds modulo 2^64 once the main pass has set + * prevent_zext: it verified the insn without a pointer to the arena and rejects a state that brings + * one. The def dominates every use, and prevent_zext is only ever set: such an add is not checked + * again. + */ +static void split_const_offset(struct bpf_verifier_env *env, u32 id, u32 *base, s64 *off, u32 *bits) +{ + struct scev *scev = env->scev; + struct expr *add, *e = &scev->exprs[id]; + + *base = e->base; + *off = e->off; + *bits = e->bits; + while (e->arena_add) { + add = &scev->exprs[e->arena_add]; + if (!env->insn_aux_data[add->def.idx].prevent_zext) { + *bits = min(*bits, 32); + break; + } + e->arena_add = scev->exprs[add->params[0]].arena_add; + } +} + +static bool const_diff_at_header(struct bpf_verifier_env *env, int h, u32 ra, u32 rb, s64 *delta); + +/* + * @id_a and @id_b are expressions of one env of the loop at @h, or of a subprog: @h is -1. Do they + * have a constant difference, @id_b == @id_a + *@delta (mod 2^*@bits)? + */ +static bool const_diff(struct bpf_verifier_env *env, int h, u32 id_a, u32 id_b, s64 *delta, + u32 *bits) +{ + struct scev *scev = env->scev; + u32 base_a, base_b, bits_a, ra, rb; + s64 off_a, c; + + split_const_offset(env, id_a, &base_a, &off_a, &bits_a); + split_const_offset(env, id_b, &base_b, delta, bits); + *delta = (u64)*delta - off_a; + *bits = min(*bits, bits_a); + /* copies are equal in all bits */ + if (id_a == id_b && scev->exprs[id_a].is_vn) { + *bits = 64; + return true; + } + if (base_a == base_b) + return scev->exprs[base_a].is_vn; + if (h < 0 || !is_reg(scev, base_a, &ra) || !is_reg(scev, base_b, &rb) || + !const_diff_at_header(env, h, ra, rb, &c)) + return false; + *delta = (u64)*delta + c; + return true; +} + +/* + * old = i = 0; do { i++; if (old > 1021) break; c = buf[old]; old = i; } while (c); + * Does @rb == @ra + *@delta hold at the header of the loop at @h in every iteration? It does if it + * holds at loop entry, by the env of the enclosing loop at @h, and every backedge preserves it: the + * backedge values have this constant difference, or both are the header phi plus one constant. + * Loops nest LOOP_STACK_SIZE deep at most, see bpf_flow_place_guards(). + */ +static bool const_diff_at_header(struct bpf_verifier_env *env, int h, u32 ra, u32 rb, s64 *delta) +{ + int outer_header = env->insn_aux_data[h].loop_header; + struct env *entry_env = find_loop_env(env->scev, outer_header, h); + struct env *header_env = find_header_env(env->scev, h); + u32 base_a, base_b, bits_a, bits; + s64 off_a, c; + + if (!entry_env || !can_compute_loop_scev(env->insn_aux_data[h].loop) || + !const_diff(env, outer_header, entry_env->reg2expr[ra], entry_env->reg2expr[rb], + delta, &bits) || bits != 64) + return false; + if (const_diff(env, -1, header_env->reg2expr[ra], header_env->reg2expr[rb], &c, &bits)) + return bits == 64 && c == *delta; + split_const_offset(env, header_env->reg2expr[ra], &base_a, &off_a, &bits_a); + split_const_offset(env, header_env->reg2expr[rb], &base_b, &c, &bits); + return base_a == REG_EXPR_ID(ra) && base_b == REG_EXPR_ID(rb) && off_a == c && + bits_a == 64 && bits == 64; +} + +/* The first bytes of a stack slot are its high bits */ +static bool is_big_endian_slot(u32 reg) +{ + return IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) && reg >= MAX_BPF_REG; +} + +/* + * const_diff() for the SCEV register @reg, which has the expression @id_b. A fill from byte + * *@fill_off of a slot reads these bits. On big endian the low bytes of a spill are the last ones, + * and a narrow fill from the start of a slot is a right shift, see mk_fill(). + */ +static bool const_diff_reg(struct bpf_verifier_env *env, int h, u32 id_a, u32 reg, u32 id_b, + u32 *fill_off, s64 *delta, u32 *bits) +{ + struct scev *scev = env->scev; + struct expr *e; + u32 x; + s64 k; + + *fill_off = 0; + if (const_diff(env, h, id_a, id_b, delta, bits)) { + if (is_big_endian_slot(reg)) + *fill_off = expr_spill_size(scev, id_b, &x) - *bits / BITS_PER_BYTE; + return true; + } + if (!is_big_endian_slot(reg) || !scev->exprs[id_b].is_vn) + return false; + split_const_offset(env, id_a, &x, delta, bits); + e = &scev->exprs[x]; + if ((e->op != BPF_RSH && e->op != BPF_ARSH) || e->def.set || e->params[0] != id_b || + !is_imm(scev, e->params[1], &k) || (k != 32 && k != 48 && k != 56)) + return false; + *delta = -(u64)*delta; + *bits = min_t(u32, *bits, 64 - k); + *fill_off = (64 - k - *bits) / BITS_PER_BYTE; + return true; +} + +/* + * n = bpf_get_stack(..); m = n; if (n < 0) ..; use(m); + * The main pass refines the registers that the conditional jump at @idx tests. Find the live SCEV + * registers that are linked to one of them in @e, the env before the jump, and keep them with their + * expressions: what bpf_scev_next_linked_reg() needs of @e. + */ +static int find_linked_regs(struct bpf_verifier_env *env, struct env *e, int h, int idx) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx]; + struct bpf_insn *insn = &env->prog->insnsi[idx]; + u32 dst = e->reg2expr[insn->dst_reg], src = dst; + struct insn_envs *envs = env->scev->envs[idx]; + struct scev *scev = env->scev; + u32 r, id, fill_off, base, base_dst, base_src, bits, cap, tested = 0; + u32 first = scev->linked_cnt; + DECLARE_BITMAP(live, REGS_NUM) = {}; + void *tmp; + s64 delta; + + if (!bpf_insn_is_cond_jump(insn->code) || BPF_OP(insn->code) == BPF_JCOND) + return 0; + if (BPF_SRC(insn->code) == BPF_X) + src = e->reg2expr[insn->src_reg]; + bitmap_copy(live, aux->live_stack_before, MAX_BPF_STACK_SLOTS); + bitmap_shift_left(live, live, MAX_BPF_REG, REGS_NUM); + live[0] |= aux->live_regs_before; + split_const_offset(env, dst, &base_dst, &delta, &bits); + split_const_offset(env, src, &base_src, &delta, &bits); + for_each_set_bit(r, live, REGS_NUM) { + id = e->reg2expr[r]; + /* most are not linked */ + split_const_offset(env, id, &base, &delta, &bits); + if (base != base_dst && base != base_src && (h < 0 || !is_op(scev, base, REG)) && + !is_big_endian_slot(r)) + continue; + if (!const_diff_reg(env, h, dst, r, id, &fill_off, &delta, &bits) && + (src == dst || !const_diff_reg(env, h, src, r, id, &fill_off, &delta, &bits))) + continue; + if (scev->linked_cnt == scev->linked_cap) { + cap = max(scev->linked_cap * 2, 16); + tmp = kvrealloc(scev->linked, array_size(cap, sizeof(*scev->linked)), + GFP_KERNEL_ACCOUNT); + if (!tmp) + return -ENOMEM; + scev->linked = tmp; + scev->linked_cap = cap; + } + tested += r == insn->dst_reg || (src != dst && r == insn->src_reg); + scev->linked[scev->linked_cnt].reg = r; + scev->linked[scev->linked_cnt++].id = id; + } + /* nothing but the tested registers */ + if (scev->linked_cnt - first == tested) { + scev->linked_cnt = first; + return 0; + } + if (!envs) { + envs = kzalloc(sizeof(*envs), GFP_KERNEL_ACCOUNT); + if (!envs) + return -ENOMEM; + scev->envs[idx] = envs; + } + envs->linked_first = first; + envs->linked_cnt = scev->linked_cnt - first; + return 0; +} + +/* + * Find the next SCEV register that is linked to @regno at the conditional jump at @insn_idx: on + * every path to the jump *@linked_regno == @regno + *@delta (mod 2^*@bits). It is live there and of + * the same frame. It is a register, or MAX_BPF_REG + the index of a stack slot: a fill from byte + * *@fill_off of the slot reads these bits. *@pos is the position of the search, 0 at first. False: + * no more links are known. Types are not known here: the caller checks them in its state. + */ +bool bpf_scev_next_linked_reg(struct bpf_verifier_env *env, int insn_idx, u32 regno, u32 *pos, + u32 *linked_regno, u32 *fill_off, s64 *delta, u32 *bits) +{ + struct insn_envs *envs = env->scev->envs[insn_idx]; + int h = bpf_loop_at_index(env, insn_idx); + struct scev_linked_reg *linked; + u32 i; + + if (!envs || !envs->linked_cnt) + return false; + linked = &env->scev->linked[envs->linked_first]; + /* registers come first */ + for (i = 0; linked[i].reg != regno; i++) + if (i + 1 == envs->linked_cnt || linked[i].reg > regno) + return false; + for (; *pos < envs->linked_cnt; ++*pos) { + *linked_regno = linked[*pos].reg; + if (*pos != i && + const_diff_reg(env, h, linked[i].id, *linked_regno, linked[*pos].id, fill_off, + delta, bits)) + return true; + } + return false; +} + /* * @cur_header and @first are a loop header. Or -1 and the first insn of a subprog: the subprog is * the outermost loop, which runs once. (REG r) is the entry value of r, the loops in it are single @@ -1779,6 +2108,12 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header, i collect_tested_regs(env, cur_env, idx, cur_loop->tested_regs); } + /* the others wait for the envs of all loop headers */ + if (!aux[idx].need_scev) { + err = find_linked_regs(env, cur_env, -1, idx); + if (err) + goto out; + } err = transfer(env, cur_env, idx); if (err) goto out; @@ -1920,13 +2255,20 @@ static int simplify(struct scev *scev, u32 id) if (is_add(scev, id, &l, &r) && is_imm(scev, r, &imm1) && !imm1) return l; - /* (+ (+ x imm) imm) -> (+ x imm) */ + /* (+ (+ x imm) imm) -> (+ x imm). Not through a def, see alu64_with_def() */ if (is_add(scev, id, &l, &r) && is_imm(scev, r, &imm2) && is_add(scev, l, &a, &b) && - is_imm(scev, b, &imm1)) + is_imm(scev, b, &imm1) && !scev->exprs[l].def.set) return expr2(scev, BPF_ADD, a, imm_expr(scev, (s64)((u64)imm1 + (u64)imm2))); + /* 'r1 <<= 32; r1 >>= 32' is (u32)r1 for cpu v1 and v2: (>> (<< x 32) 32) -> (zext32 x) */ + if (is_binop(scev, BPF_RSH, id, &l, &r) && + is_imm(scev, r, &imm2) && imm2 == 32 && + is_binop(scev, BPF_LSH, l, &a, &b) && + is_imm(scev, b, &imm1) && imm1 == 32 && !scev->exprs[l].def.set) + return expr1(scev, ZEXT32, a); + /* (+ (linear base slope) imm) -> (linear (+ base imm) slope) */ if (is_add(scev, id, &l, &r) && is_linear(scev, l, &base, &slope) && @@ -1983,6 +2325,17 @@ static int simplify(struct scev *scev, u32 id) return id; } +static int instantiate_header_scevs(struct scev *scev, u32 id, void *priv) +{ + u32 reg, *reg2scev = priv; + + /* (reg r) -> ... equation at header ... */ + if (is_reg(scev, id, ®)) + return reg2scev[reg]; + /* keep expression shapes */ + return id; +} + static int compute_header_scevs(struct bpf_verifier_env *env, struct env *header_env) { struct scev *scev = env->scev; @@ -2006,6 +2359,12 @@ static int compute_header_scevs(struct bpf_verifier_env *env, struct env *header */ for (ra = 0; ra < REGS_NUM; ra++) { ra_expr = header_env->reg2expr[ra]; + /* Constructed again it has no defs, see alu64_with_def(): they hide the shape */ + if (env->prog->aux->arena) + ra_expr = transform_expr_once(scev, 0, ra_expr, scev->identity_env.reg2expr, + instantiate_header_scevs); + if (ra_expr < 0) + return ra_expr; /* rA = (+ rA IMM) -> (linear rA IMM) */ if (is_add(scev, ra_expr, &l, &r) && is_reg(scev, l, &rb) && @@ -2136,17 +2495,6 @@ static void scev_compute_linear_regs(struct scev *scev, struct bpf_loop *loop, } } -static int instantiate_header_scevs(struct scev *scev, u32 id, void *priv) -{ - u32 reg, *reg2scev = priv; - - /* (reg r) -> ... equation at header ... */ - if (is_reg(scev, id, ®)) - return reg2scev[reg]; - /* keep expression shapes */ - return id; -} - static int compute_insn_scevs(struct bpf_verifier_env *env, struct env *eheader, struct env *einsn) { struct scev *scev = env->scev; @@ -2196,6 +2544,27 @@ static int log_scevs(struct bpf_verifier_env *env) return 0; } +/* + * A slice comes from a call that may write to the buffer: the kfunc, or a subprog that calls it. + * See expr_is_vn(). + */ +static void find_slice_slots(struct bpf_verifier_env *env) +{ + struct bpf_insn *insn = env->prog->insnsi; + struct scev *scev = env->scev; + bool slices = false; + u32 i, spi; + + for (i = 0; i < env->prog->len; i++) + slices |= bpf_is_dynptr_slice_rdwr_kfunc(&insn[i]); + for (i = 0; slices && i < env->prog->len; i++) + if (BPF_CLASS(insn[i].code) == BPF_JMP && BPF_OP(insn[i].code) == BPF_CALL) + bitmap_or(scev->slice_slots, scev->slice_slots, bpf_may_write_mask(env, i), + MAX_BPF_STACK_SLOTS); + for_each_set_bit(spi, scev->slice_slots, MAX_BPF_STACK_SLOTS) + scev->exprs[REG_EXPR_ID(MAX_BPF_REG + spi)].is_vn = false; +} + int bpf_compute_scev(struct bpf_verifier_env *env) { struct bpf_insn_aux_data *aux = env->insn_aux_data; @@ -2204,6 +2573,7 @@ int bpf_compute_scev(struct bpf_verifier_env *env) int cnt = env->cfg.cur_postorder; int i, idx, err, header; + find_slice_slots(env); mark_latches(env); /* * Visit loop headers in postorder, to guarantee that scevs @@ -2232,6 +2602,9 @@ int bpf_compute_scev(struct bpf_verifier_env *env) idx = postorder[i]; if (!aux[idx].need_scev) continue; + if (signal_pending(current)) + return -EAGAIN; + cond_resched(); header = bpf_loop_at_index(env, idx); err = aux[idx].loop ? compute_header_scevs(env, find_header_env(scev, header)) @@ -2243,6 +2616,10 @@ int bpf_compute_scev(struct bpf_verifier_env *env) if (aux[idx].loop) scev_compute_linear_regs(scev, aux[idx].loop, find_header_env(scev, header)); + err = find_linked_regs(env, aux[idx].loop ? &scev->identity_env : + find_loop_env(scev, header, idx), header, idx); + if (err) + return err; } /* Now it's known which jumps test a counter: the places of a back edge are for them */ @@ -2259,6 +2636,17 @@ int bpf_compute_scev(struct bpf_verifier_env *env) return 0; } +/* Are all slots of [off, off + size) in slice_slots? */ +bool bpf_scev_in_slice_slots(struct bpf_verifier_env *env, int off, u32 size) +{ + int spi; + + for (spi = bpf_get_spi(off + size - 1); spi <= bpf_get_spi(off); spi++) + if (!test_bit(spi, env->scev->slice_slots)) + return false; + return true; +} + void bpf_free_scev(struct bpf_verifier_env *env) { struct scev *scev = env->scev; @@ -2281,6 +2669,7 @@ void bpf_free_scev(struct bpf_verifier_env *env) kvfree(scev->worklist.data); kvfree(scev->envs); kvfree(scev->exprs); + kvfree(scev->linked); kvfree(scev->discovered); kfree(scev); env->scev = NULL; @@ -2297,6 +2686,7 @@ int bpf_init_scev(struct bpf_verifier_env *env) env->scev = scev; bpf_mark_reg_known_scalar(&scev->zero_reg, 0); bpf_mark_reg_unknown(&scev->unknown_reg); + setup_initial_loop_env(&scev->identity_env); if (grow_exprs(scev, roundup_pow_of_two(REG_EXPR_ID(REGS_NUM)))) goto nomem; scev->exprs[UNKNOWN_EXPR_ID] = (struct expr){ .op = UNKNOWN }; @@ -2304,6 +2694,8 @@ int bpf_init_scev(struct bpf_verifier_env *env) for (r = 0; r < REGS_NUM; r++) scev->exprs[REG_EXPR_ID(r)] = (struct expr){ .op = REG, .is_vn = true, .params = { r, 0 } }; + for (r = 0; r < REG_EXPR_ID(REGS_NUM); r++) + set_expr_base(scev, &scev->exprs[r], r); scev->exprs_cnt = REG_EXPR_ID(REGS_NUM); scev->envs = kvcalloc(env->prog->len, sizeof(*scev->envs), GFP_KERNEL_ACCOUNT); scev->discovered = kvcalloc(env->prog->len, sizeof(*scev->discovered), GFP_KERNEL_ACCOUNT); @@ -2372,6 +2764,10 @@ static struct bpf_reg_state *scev_reg_for_spill32(struct bpf_func_state *st, u32 { struct bpf_stack_state *stack = scev_regno_to_stack(st, r); +#ifdef __BIG_ENDIAN + /* a 32-bit spill is in the upper 32 bits of the slot */ + return NULL; +#endif if (!stack || bpf_spill_size(stack) != 4 || stack->spilled_ptr.type != SCALAR_VALUE) return NULL; return &stack->spilled_ptr; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 83249ecbdfb1..d980e2c60805 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -15230,6 +15230,12 @@ bool bpf_is_throw_kfunc(struct bpf_insn *insn) insn->imm == special_kfunc_list[KF_bpf_throw]; } +bool bpf_is_dynptr_slice_rdwr_kfunc(struct bpf_insn *insn) +{ + return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && + insn->imm == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]; +} + static bool is_bpf_wq_set_callback_kfunc(u32 btf_id) { return btf_id == special_kfunc_list[KF_bpf_wq_set_callback]; @@ -16054,6 +16060,12 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg regs[BPF_REG_0].buf = meta->slice_buf; if (meta->slice_buf.size) env->seen_stack_slice = true; + if (verifier_bug_if(meta->slice_buf.size && + !type_is_rdonly_mem(regs[BPF_REG_0].type) && + !bpf_scev_in_slice_slots(env, meta->slice_buf.off, + meta->slice_buf.size), + env, "slice buffer is not in scev slice_slots")) + return -EFAULT; } else { return 0; } @@ -19730,6 +19742,82 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s } } +/* a == b (mod 2^N) implies a == b if both fit in uN or both fit in sN. N is @bits. */ +static bool regs_fit_in_bits(const struct bpf_reg_state *a, const struct bpf_reg_state *b, + u32 bits) +{ + u64 umax = GENMASK_ULL(bits - 1, 0); + s64 smax = umax >> 1; + + return (reg_umax(a) <= umax && reg_umax(b) <= umax) || + (reg_smin(a) >= -smax - 1 && reg_smax(a) <= smax && + reg_smin(b) >= -smax - 1 && reg_smax(b) <= smax); +} + +/* + * The state of the SCEV register @r. As in check_stack_read_fixed_off() a fill of @bits bits from + * byte @fill_off of the slot reads the low bits of the spill. + */ +static struct bpf_reg_state *scev_reg_for_fill(struct bpf_func_state *frame, u32 r, u32 fill_off, + u32 bits) +{ + struct bpf_stack_state *ss; + + if (r < MAX_BPF_REG) + return &frame->regs[r]; + if (r - MAX_BPF_REG >= bpf_stack_nr_slots(frame)) + return NULL; + ss = bpf_stack_slot(frame, r - MAX_BPF_REG); + if (!bpf_is_spilled_reg(ss) || bpf_spill_size(ss) * BITS_PER_BYTE < bits || + !bpf_stack_narrow_access_ok(fill_off, bits / BITS_PER_BYTE, bpf_spill_size(ss))) + return NULL; + return &ss->spilled_ptr; +} + +/* + * The jump at @idx refined @regno. A register that scalar evolution links to it holds + * @regno + delta on every path to the jump: intersect the two. The intersection is empty: no + * execution gets here with this state (the path is infeasible), the register is left as it is. + */ +static void refine_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *vstate, + int idx, u32 regno) +{ + struct bpf_func_state *frame = vstate->frame[vstate->curframe]; + struct bpf_reg_state *reg, implied, saved; + u32 pos, linked_regno, fill_off, bits; + s64 delta; + + if (frame->regs[regno].type != SCALAR_VALUE) + return; + for (pos = 0; + bpf_scev_next_linked_reg(env, idx, regno, &pos, &linked_regno, &fill_off, &delta, + &bits); + pos++) { + reg = scev_reg_for_fill(frame, linked_regno, fill_off, bits); + if (!reg || reg->type != SCALAR_VALUE) + continue; + + implied = frame->regs[regno]; + add_scalar_to_reg(&implied, delta); + if (regs_fit_in_bits(reg, &implied, bits)) + bits = 64; + if (bits < 32) + continue; + saved = *reg; + regs_refine_cond_op(reg, &implied, BPF_JEQ, bits == 32); + reg_bounds_sync(reg); + if (reg_is_empty(reg) || const_tnum_range_mismatch(reg) || + const_tnum_range_mismatch_32(reg)) { + *reg = saved; + continue; + } + if (linked_regno < MAX_BPF_REG) + mark_reg_scratched(env, linked_regno); + else + mark_stack_slot_scratched(env, linked_regno - MAX_BPF_REG); + } +} + /* It's known which way the jump at env->insn_idx goes */ static void mark_loop_exit_decided(struct bpf_verifier_env *env, struct bpf_verifier_state *st) { @@ -19902,6 +19990,14 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, sync_linked_regs(env, other_branch, &other_branch_regs[insn->dst_reg], &linked_regs); } + if (env->bpf_capable) { + if (BPF_SRC(insn->code) == BPF_X) { + refine_linked_regs(env, this_branch, *insn_idx, insn->src_reg); + refine_linked_regs(env, other_branch, *insn_idx, insn->src_reg); + } + refine_linked_regs(env, this_branch, *insn_idx, insn->dst_reg); + refine_linked_regs(env, other_branch, *insn_idx, insn->dst_reg); + } /* if one pointer register is compared to another pointer * register check if PTR_MAYBE_NULL could be lifted. diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index cadb13f023d2..bebb607000a5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -597,6 +597,34 @@ __naked void mixed_arena_scalar_alu64_arena_first(void) : __clobber_all); } +/* The offset of a pointer to arena is not tracked */ +#define ARENA_TEST(name, op, expect) \ +SEC("syscall") \ +__log_level(2) \ +expect \ +__naked void name(void) \ +{ \ + asm volatile ( \ + "r1 = %[arena] ll;" \ + "r6 = 0;" \ + "r6 = addr_space_cast(r6, 0, 1);" \ + "r7 = 0;" \ +"1: *(u64 *)(r6 + 0) = r7;" \ + op \ + "r7 += 1;" \ + "if r7 < 100 goto 1b;" \ + "r0 = 0;" \ + "exit;" \ + :: __imm_addr(arena) : __clobber_all); \ +} + +#define NOT_UNROLLED100 __msg("loop header at 5, widening r7 to 0..99 step 1") +/* A loop that moves the pointer is not unrolled */ +ARENA_TEST(arena_pointer_moves, "r6 += 8;", __success NOT_UNROLLED100) +/* The loop makes a number of it: the state that returns to the loop header tells */ +ARENA_TEST(arena_pointer_becomes_a_number, "if r7 != 5 goto +1; r6 = 4096;", + __failure NOT_UNROLLED100 __msg("R6 invalid mem access 'scalar'")) + SEC("syscall") __description("arena: a number is not an address in arena") __failure __msg("R6 invalid mem access 'scalar'") diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index 316e9377caa8..94d52154b0c6 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -1572,7 +1572,7 @@ l0_%=: r0 = 0; \ SEC("socket") __description("bounds deduction cross sign boundary, negative overlap") __success __log_level(2) -__msg("7: (1f) r0 -= r6 {{.*}} R0=scalar(smin=smin32=-655,smax=smax32=-146,umin=0xfffffffffffffd71,umax=0xffffffffffffff6e,umin32=0xfffffd71,umax32=0xffffff6e,var_off=(0xfffffffffffffc00; 0x3ff))") +__msg("7: (1f) r0 -= r6 {{.*}} R0=scalar(smin=smin32=-527,smax=smax32=-146,umin=0xfffffffffffffdf1,umax=0xffffffffffffff6e,umin32=0xfffffdf1,umax32=0xffffff6e,var_off=(0xfffffffffffffc00; 0x3ff))") __retval(0) __naked void bounds_deduct_negative_overlap(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c b/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c index caae67feaf5b..42b9e46a88af 100644 --- a/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c +++ b/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c @@ -190,10 +190,9 @@ l0_%=: \ : __clobber_all); } -/* Double ADD clears the ID (can't accumulate offsets) */ +/* Double ADD clears the ID. That r1 = r0 + 4 is known all the same: r0 /= 0 is dead code */ SEC("socket") -__failure -__msg("div by zero") +__success __naked void scalars_double_add(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_reg_equal.c b/tools/testing/selftests/bpf/progs/verifier_reg_equal.c index dc1d8c30fb0e..91daa5b323c9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_reg_equal.c +++ b/tools/testing/selftests/bpf/progs/verifier_reg_equal.c @@ -31,17 +31,16 @@ l1_%=: exit; \ } SEC("socket") -__description("check w reg not equal if r reg upper32 bits not 0") -__failure __msg("R1 !read_ok") +__description("check w reg equal if r reg upper32 bits not 0") +__success __naked void subreg_equality_2(void) { asm volatile (" \ call %[bpf_ktime_get_ns]; \ r2 = r0; \ - /* Upper 4-bytes of r2 may not be 0, thus insn \ - * w3 = w2 should not propagate reg id, and \ - * w2 < 9 comparison should not propagate \ - * the range for r3 either. \ + /* Upper 4-bytes of r2 may not be 0. r3 is the \ + * lower 4-bytes of r2, which is what w2 < 9 \ + * compares. \ */ \ w3 = w2; \ if w2 < 9 goto l0_%=; \ diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c index 9ea61b162736..819d0a1a6886 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c +++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c @@ -288,9 +288,9 @@ __success __log_level(2) /* * check that r5 is unlinked after 'if', collect_linked_regs() can't tie * more than 5 registers for a single insn and the register compared by - * the jump is not exempt from that. + * the jump is not exempt from that. The jump refines the copies all the same. */ -__msg("7: (25) if r5 > 0x7 goto pc+0 ; R5=scalar(smin=") +__msg("7: (25) if r5 > 0x7 goto pc+0 ; R0=" LINKED("1", 7, 0x7) " R1=" LINKED("1", 7, 0x7) " R2=" LINKED("1", 7, 0x7) " R3=" LINKED("1", 7, 0x7) " R4=" LINKED("1", 7, 0x7) " R5=scalar(smin=") __msg("12: (bf) r5 = r5 ; R5=scalar(id=2") __naked void linked_regs_too_many_regs(void) { @@ -808,7 +808,7 @@ __naked void linked_regs_and_subreg_def(void) * at runtime), which must be rejected. */ SEC("socket") -__failure __msg("R0 max value is outside of the allowed memory range") +__failure __msg("invalid access to map value, value_size=8 off=8 size=1") __naked void ldsx_fill_scalar_id_not_shared(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 939f49a4ebdb..70aeeadcd181 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -3307,42 +3307,6 @@ NOT_LINEAR_TEST(lsh_grows, "r7 &= 15;", "r7 <<= 1;", UNROLLED("(<< r7 1)")) NOT_LINEAR_TEST(same_constant, "r7 = 17;", "r7 &= 31;", NOT_CHANGED) NOT_LINEAR_TEST(same_wide_constant, "r7 = 0xffffffff00000011 ll;", "r7 &= -239;", NOT_CHANGED) -#ifdef __BPF_FEATURE_ADDR_SPACE_CAST -struct { - __uint(type, BPF_MAP_TYPE_ARENA); - __uint(map_flags, BPF_F_MMAPABLE); - __uint(max_entries, 1); -} arena SEC(".maps"); - -/* The offset of a pointer to arena is not tracked */ -#define ARENA_TEST(name, op, expect) \ -SEC("syscall") \ -__log_level(2) \ -expect \ -__naked void name(void) \ -{ \ - asm volatile ( \ - "r1 = %[arena] ll;" \ - "r6 = 0;" \ - "r6 = addr_space_cast(r6, 0, 1);" \ - "r7 = 0;" \ -"1: *(u64 *)(r6 + 0) = r7;" \ - op \ - "r7 += 1;" \ - "if r7 < 100 goto 1b;" \ - "r0 = 0;" \ - "exit;" \ - :: __imm_addr(arena) : __clobber_all); \ -} - -#define NOT_UNROLLED100 __msg("loop header at 5, widening r7 to 0..99 step 1") -/* A loop that moves the pointer is not unrolled */ -ARENA_TEST(arena_pointer_moves, "r6 += 8;", __success NOT_UNROLLED100) -/* The loop makes a number of it: the state that returns to the loop header tells */ -ARENA_TEST(arena_pointer_becomes_a_number, "if r7 != 5 goto +1; r6 = 4096;", - __failure NOT_UNROLLED100 __msg("R6 invalid mem access 'scalar'")) -#endif - /* * r7 = ctx->data; * r8 = r7 -- 2.55.0 From: Alexei Starovoitov Tests for the patch that refines the linked registers of a conditional jump. These are the live registers and stack slots that are a tested register + a constant on every path to the jump. The existing tests of linked scalars, spills and loops cover the mechanism: a copy, an added constant, the low 32 bits, a spill and its fill, two calls, a store through a pointer. These tests are for the hazards that only a static analysis has. scev.c computes the links before the main pass, from the insns alone, without types and without states. r8 is an unbounded scalar. Each prog computes a second value from it, tests one of the two and uses the other one as an offset into a map value of 64 bytes. What loads is run. - An add that becomes ALU32: 'r3 += 1' is converted to 'w3 += 1' at fixup if r3 is an arena pointer. r3 == r7 + 1 holds modulo 2^32 only, also after 'r3 = le64 r3; r7 = le64 r7' has turned both into scalars, and also for the phi of two such adds. Rejected. A scalar + two constants in a prog with an arena is linked modulo 2^64: loads. - A store through a dynptr slice may alias the buffer on the stack: a fill before the store and a fill after it are not linked. Rejected. - An edge that the verifier adds: each iteration of the outer loop saves r3, which is r8, and restores it. may_goto that the verifier adds before the header of the nested loop leaves both loops while r3 is r7. Rejected. - A link at a loop header: two induction variables that are linked at loop entry and have the same step are linked there. Negative test: the steps differ. - A pointer is not a scalar: r3 is the address in r1 as a scalar. A bound of r3 does not bound the offset of r1. Rejected. - Big endian: a 32-bit spill goes to the first half-slot. The low 32 bits of a 64-bit fill are that spill on little endian only, also if the slot is a phi. Loads there, rejected on big endian. - An empty intersection: the range that the jump implies and the known bits of the linked register don't overlap. The register is left as it is: loads with BPF_F_TEST_REG_INVARIANTS. - A link modulo 2^8: after 'r2 = (s8)r8; if r8 > 63 goto' both fit in s8, so r2 is refined. Loads. The tests with an arena are in verifier_arena.c, the others in verifier_scev.c. Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/progs/verifier_arena.c | 39 +++++++ .../selftests/bpf/progs/verifier_scev.c | 107 ++++++++++++++++++ 2 files changed, 146 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index bebb607000a5..bf50413f6c08 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -625,6 +625,45 @@ ARENA_TEST(arena_pointer_moves, "r6 += 8;", __success NOT_UNROLLED100) ARENA_TEST(arena_pointer_becomes_a_number, "if r7 != 5 goto +1; r6 = 4096;", __failure NOT_UNROLLED100 __msg("R6 invalid mem access 'scalar'")) +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, char[64]); +} map64 SEC(".maps"); + +/* + * r6: 64 bytes. r8: an unbounded scalar. r3, r7: an arena pointer and a copy of it. + * The test leaves an address in r1. + */ +#define ARENA_LINKED(name, expect, body) \ +SEC("syscall") expect __load_if_JITed() \ +__naked void name(void) \ +{ \ + asm volatile ( \ + "r1 = %[arena] ll; r1 = 0; *(u32 *)(r10 - 4) = r1; r2 = r10; r2 += -4;" \ + "r1 = %[map64] ll; call %[bpf_map_lookup_elem]; if r0 == 0 goto out_%=; r6 = r0;" \ + "call %[bpf_ktime_get_ns]; r8 = r0; r3 = addr_space_cast(r8, 0, 1); r7 = r3;" \ + body \ + "r0 = *(u8 *)(r1 + 0);" \ + "out_%=: r0 = 0; exit;" \ + : \ + : __imm(bpf_map_lookup_elem), __imm(bpf_ktime_get_ns), \ + __imm_addr(map64), __imm_addr(arena) \ + : __clobber_all); \ +} + +ARENA_LINKED(scalar_plus_two_constants, __success __retval(0), + "r2 = r8; r2 += 1; r2 += 1; if r8 > 61 goto out_%=; r1 = r6; r1 += r2;") +/* 'r3 += 1' is converted to 'w3 += 1': r3 is 0 and not 1 << 32 */ +ARENA_LINKED(arena_ptr_plus_constant, __failure, + "r3 += 1; r3 = le64 r3; r7 = le64 r7; r2 = 0xffffffff ll; if r7 != r2 goto out_%=;" + "r3 >>= 32; r3 -= 1; r1 = r6; r1 += r3;") +/* At the join point r3 is phi((+ x 1)). The insn there, an add to a scalar, does not define it. */ +ARENA_LINKED(arena_ptr_plus_constant_on_two_paths, __failure, + "if r8 == 0 goto 1f; r3 += 1; goto 2f; 1: r3 += 1; 2: r8 += 1; r3 = le64 r3; r7 = le64 r7;" + "r2 = 0xffffffff ll; if r7 != r2 goto out_%=; r3 >>= 32; r3 -= 1; r1 = r6; r1 += r3;") + SEC("syscall") __description("arena: a number is not an address in arena") __failure __msg("R6 invalid mem access 'scalar'") diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 70aeeadcd181..1047845c916b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -3307,6 +3307,113 @@ NOT_LINEAR_TEST(lsh_grows, "r7 &= 15;", "r7 <<= 1;", UNROLLED("(<< r7 1)")) NOT_LINEAR_TEST(same_constant, "r7 = 17;", "r7 &= 31;", NOT_CHANGED) NOT_LINEAR_TEST(same_wide_constant, "r7 = 0xffffffff00000011 ll;", "r7 &= -239;", NOT_CHANGED) +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, char[64]); +} map64 SEC(".maps"); + +/* + * A conditional jump refines the registers and stack slots that are linked to what it tests. + * r6: 64 bytes. r7: a scalar below 256. r8: an unbounded scalar. r9: the context. + * The test leaves an address in r1. + */ +#define LINKED_SEC(sec, name, expect, body) \ +SEC(sec) expect \ +__naked void name(void) \ +{ \ + asm volatile ( \ + "r9 = r1; r1 = 0; *(u32 *)(r10 - 4) = r1; r2 = r10; r2 += -4; r1 = %[map64] ll;" \ + "call %[bpf_map_lookup_elem]; if r0 == 0 goto out_%=; r6 = r0;" \ + "call %[bpf_get_prandom_u32]; r7 = r0; r7 &= 255;" \ + "call %[bpf_ktime_get_ns]; r8 = r0;" \ + body \ + "r0 = *(u8 *)(r1 + 0);" \ + "out_%=: r0 = 0; exit;" \ + : \ + : __imm(bpf_map_lookup_elem), __imm(bpf_get_prandom_u32), __imm(bpf_ktime_get_ns), \ + __imm(bpf_dynptr_from_skb), __imm(bpf_dynptr_slice_rdwr), \ + __imm(bpf_ringbuf_reserve), __imm(bpf_ringbuf_discard), \ + __imm_addr(map64), __imm_addr(ringbuf) \ + : __clobber_all); \ +} +#define LINKED(name, expect, body) LINKED_SEC("socket", name, expect, body) + +#define VALID __success __retval(0) +#define UNBOUND __failure __msg("unbounded min value") + +/* r3 is the address in r1 as a scalar. It does not bound the offset of r1. */ +LINKED(pointer_is_not_scalar, __failure, + "r7 &= 127; r1 = r6; r1 += r7; r3 = r1; r3 = le64 r3; if r3 < 64 goto out_%=; r1 += -64;") + +LINKED(sign_extension_that_fits, VALID, "r2 = (s8)r8; if r8 > 63 goto out_%=; r1 = r6; r1 += r2;") + +/* The low 3 bits of r3 are 5. It is r8 + 5 once r8 fits in s8: no overlap with [-2, 1]. */ +LINKED(empty_intersection, VALID __flag(BPF_F_TEST_REG_INVARIANTS), + "r3 = (s8)r8; if r3 & 7 goto out_%=; r3 += 5; if r3 s> 100 goto out_%=;" + "if r3 s< -100 goto out_%=; if r8 s< -7 goto out_%=; if r8 s> -4 goto out_%=;" + "r3 += 3; r1 = r6;") + +/* The low bytes of the spill to r10 - 16 come first or last */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define FIRST_BYTES_ARE_LOW VALID +#else +#define FIRST_BYTES_ARE_LOW __failure __msg("R1 unbounded memory access") +#endif + +/* r2 is the low 32 bits of the slot. The 32-bit spill is in its first half-slot. */ +LINKED(spill_32_bits_fill_64_take_low_half, FIRST_BYTES_ARE_LOW, + "*(u64 *)(r10 - 16) = r7; *(u32 *)(r10 - 16) = r8; r2 = *(u64 *)(r10 - 16); w2 = w2;" + "w3 = w8; if r2 > 63 goto out_%=; r1 = r6; r1 += r3;") +/* The slot is a phi: two paths spill distinct values to its first half-slot */ +LINKED(spill_32_bits_on_two_paths_fill_64_take_low_half, FIRST_BYTES_ARE_LOW, + "*(u64 *)(r10 - 16) = r7; *(u32 *)(r10 - 16) = r8; if r7 > 9 goto l_%=;" + "r3 = r8; r3 >>= 1; *(u32 *)(r10 - 16) = r3;" + "l_%=: r2 = *(u64 *)(r10 - 16); w2 = w2; if r2 > 63 goto out_%=;" + "r3 = *(u32 *)(r10 - 16); r1 = r6; r1 += r3;") + +/* r0 is r10 - 24 when the first 8 bytes of the packet are not in one piece */ +LINKED_SEC("tc", slot_written_through_a_slice, UNBOUND, + "r1 = r9; r2 = 0; r3 = r10; r3 += -16; call %[bpf_dynptr_from_skb];" + "r1 = r10; r1 += -16; r2 = 0; r3 = r10; r3 += -24; r4 = 8;" + "call %[bpf_dynptr_slice_rdwr]; if r0 == 0 goto out_%=;" + "r3 = *(u64 *)(r10 - 24); *(u64 *)(r0 + 0) = r8; r2 = *(u64 *)(r10 - 24);" + "if r3 > 63 goto out_%=; r1 = r6; r1 += r2;") + +/* BTF of the kfuncs that only asm calls */ +void __kfunc_btf_root(void) +{ + bpf_dynptr_from_skb(0, 0, 0); + bpf_dynptr_slice_rdwr(0, 0, 0, 0); +} + +/* Linked at the header in every iteration: r2 is r3 + 2 at loop entry, both have the step 1 */ +#define LOOP_HEADER(enter, step) \ + enter "l_%=: if r3 > 61 goto out_%=; r1 = r6; r1 += r2; r0 = *(u8 *)(r1 + 0);" step \ + "if r0 != 0 goto l_%=;" + +LINKED(linked_at_loop_header, VALID, + LOOP_HEADER("r3 = r8; r2 = r8; r2 += 1; r2 += 1;", "r2 += 1; r3 += 1;")) +LINKED(linked_at_loop_entry_only, __failure, + LOOP_HEADER("r3 = r8; r2 = r8; r2 += 1; r2 += 1;", "r2 += 2; r3 += 1;")) + +/* + * No exit of the inner loop dominates its backedges: may_goto is added before its header. The prog + * holds a reference, so may_goto goes to the exit of the outer loop: an edge that the verifier + * adds. Every iteration of the outer loop restores r3. At the header of the inner one it is r7. + */ +LINKED(added_edge, UNBOUND, + "r1 = %[ringbuf] ll; r2 = 8; r3 = 0; call %[bpf_ringbuf_reserve]; if r0 == 0 goto out_%=;" + "*(u64 *)(r10 - 16) = r0; r3 = r8;" + "o_%=: r2 = r3; r3 = r7;" + "i_%=: r1 = *(u8 *)(r6 + 0); if r1 == 1 goto a_%=; if r1 == 2 goto b_%=; goto i_%=;" + "a_%=: goto i_%=;" + "b_%=: r3 = r2; r1 = *(u8 *)(r6 + 1); if r1 == 3 goto o_%=;" + "r4 = r6; if r8 > 63 goto r_%=; r4 += r3;" + "r_%=: *(u64 *)(r10 - 24) = r4; r1 = *(u64 *)(r10 - 16); r2 = 0;" + "call %[bpf_ringbuf_discard]; r1 = *(u64 *)(r10 - 24);") + /* * r7 = ctx->data; * r8 = r7 -- 2.55.0 From: Vineet Gupta Tests for two patches of this series. Two progs are for coerce_subreg_to_size_sx(), which keeps the 32-bit range across a no-op sign extension. Five are for refine_linked_regs(), which refines the linked registers of a conditional jump: the live registers and stack slots that are a tested register + a constant on every path to the jump, here modulo 2^32. call bpf_get_prandom_u32 r7 = (s32)r0 if w0 != -1 goto .. GCC keeps an int that a call returned that way. The jump tests the low 32 bits of r0, the prog goes on with r7. 'w0 = (s8)w0' and 'w0 = (s16)w0' keep a 32-bit range that crosses 0 and fits in s8 or s16. After 'r7 = (s32)r6' r7 == r6 (mod 2^32). A test of the low 32 bits of the source refines r7. The source is the def of r0 by a call, or (u32)(r6 + 1), which is 'i += 1' of an int as GCC emits it. Three negative tests get to 'r0 /= 0' and are rejected: - A test of r7 does not bound the upper 32 bits of r6 + 3. - If r6 is a u32 then 'r7 s<= 100' keeps the values from 2^31 on in r6. - If r6 is a multiple of 3 then r7 need not be one: from 2^31 on it is r6 - 2^32. Signed-off-by: Vineet Gupta Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/progs/verifier_movsx.c | 186 ++++++++++++++++++ 1 file changed, 186 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_movsx.c b/tools/testing/selftests/bpf/progs/verifier_movsx.c index 2f4a678c31c2..6f90dac78fcd 100644 --- a/tools/testing/selftests/bpf/progs/verifier_movsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_movsx.c @@ -202,6 +202,192 @@ l0_%=: \ : __clobber_all); } +/* + * A 32-bit range that already fits the signed N-bit range is unchanged by the sign + * extension. Both of these straddle zero. The guard that follows can only be + * resolved statically if the range survives, and it protects a div by zero. + */ +SEC("socket") +__description("MOV32SX, S8, negative range is preserved") +__success __success_unpriv __retval(0) +__naked void mov32sx_s8_negative_range(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w0 &= 0x3f; \ + w0 -= 0x3f; \ + /* w0 is [-63, 0], already a valid s8 */ \ + w0 = (s8)w0; \ + if w0 s< -0x3f goto l0_%=; \ + w0 = 0; \ + exit; \ +l0_%=: \ + /* dead unless the range was widened */ \ + w0 /= 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__description("MOV32SX, S16, negative range is preserved") +__success __success_unpriv __retval(0) +__naked void mov32sx_s16_negative_range(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w0 &= 0xfff; \ + w0 -= 0xfff; \ + /* w0 is [-4095, 0], already a valid s16 */ \ + w0 = (s16)w0; \ + if w0 s< -0xfff goto l0_%=; \ + w0 = 0; \ + exit; \ +l0_%=: \ + /* dead unless the range was widened */ \ + w0 /= 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * A 32-bit sign extension keeps the low 32 bits, so refining the source + * reaches the destination. The source is a call return, which is the shape + * bpf-gcc builds for "int err = helper(); if (err) ..". + */ +SEC("socket") +__success __retval(0) +__naked void sext_mov_unlinked_src(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r7 = (s32)r0; /* r0 as the call defines it */ \ + if w0 != -1 goto l0_%=; /* refines r0, propagates to r7 */ \ + if r7 == -1 goto l0_%=; /* sign-extended, not 0xffffffff */ \ + r0 /= 0; \ +l0_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * The propagation must not run backwards over the upper 32 bits: a sign extension + * knows nothing about the upper bits of its source. r8's low 32 bits do become + * 2, r8 == 2 does not follow. + */ +SEC("socket") +__failure __msg("div by zero") +__naked void sext_no_sync_from_subreg_base(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r7 = (s32)r6; \ + r8 = r6; \ + r8 += 3; \ + if r7 != -1 goto 1f; /* pins the low 32 bits of r8 only */ \ + if r8 == 2 goto 1f; /* taken only if r8 wrongly refined */ \ + r0 /= 0; \ +1: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * r2 is (u32)(r6 + 1) and r7 is (s32)(r6 + 1), so w2 == 1 means r7 == 1. + * bpf-gcc builds "i += 1" for an int induction variable into exactly this shape. + */ +SEC("socket") +__success __retval(0) +__naked void sext_sync_from_zext_sibling(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r6 <<= 32; \ + call %[bpf_get_prandom_u32]; \ + r6 |= r0; /* r6 = full 64-bit unknown */ \ + r2 = r6; \ + w2 += 1; /* r2 = (u32)(r6 + 1) */ \ + r7 = (s32)r2; /* r7 = (s32)(r6 + 1) */ \ + if w2 != 1 goto 1f; /* propagates to r7 */ \ + if r7 == 1 goto 1f; \ + r0 /= 0; \ +1: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * "w6 = w6" leaves r6 in [0, U32_MAX]. A signed compare on its sign extension + * must not hand r6 the sign-extended value: that drops every r6 >= 2^31 from + * the range and makes the guarded div provably dead. + */ +SEC("socket") +__failure __msg("div by zero") +__naked void sext_narrow_base_keeps_high_values(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + w6 = w6; /* r6 in [0, U32_MAX] */ \ + r7 = (s32)r6; \ + r8 = 1; \ + r8 <<= 31; /* 0x80000000 */ \ + if r7 s> 100 goto 1f; /* r6 is [0,100] or >= 2^31 */ \ + if r6 < r8 goto 1f; /* reachable: r6 may be >= 2^31 */ \ + r0 /= 0; \ +1: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * A base/step line does not survive the sign extension: it subtracts 2^32, + * and 2^32 is not a multiple of 3. r6 = 0x80000007 is a multiple of 3, so + * r7 = -2147483641 is reachable. + */ +SEC("socket") +__failure __msg("div by zero") +__naked void sext_drops_stale_step(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + w6 = w6; \ + r6 *= 3; /* base 0, step 3 */ \ + r7 = (s32)r6; \ + r8 = 1; \ + r8 <<= 31; \ + r9 = r8; \ + r9 += 7; \ + if r6 < r8 goto 1f; \ + if r6 > r9 goto 1f; /* r6 in [0x80000000, 0x80000007] */ \ + if r7 != -2147483641 goto 1f; /* reachable: r6 == 0x80000007 */ \ + r0 /= 0; \ +1: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + SEC("socket") __description("MOV64SX, S16, R10 Sign Extension") __failure __msg("R1 type=scalar expected=fp, pkt, pkt_meta, map_key, map_value, mem, ringbuf_mem, buf, trusted_ptr_") -- 2.55.0 From: Alexei Starovoitov A new ability. It extends the patch that refines linked registers at conditional jumps to 'p = buf + i' that is computed before i is bounded. Nothing links a pointer to its index today. while (buf[i]) { if (i < 0 || i > 32) break; if (bpf_map_lookup_elem(&names, buf + i)) return 0; while (buf[i++] && i < 32) ; } Strings are stored back to back in a map value of 64 bytes, a key has 32. clang computes 'buf + i' for the load at the end of an iteration and tests i at the start of the next one. The jumps bound r8, the offset of r2 stays: 27: r2 = r6 28: r2 += r8 30: r1 = *(u8 *)(r2 +0) 36: if r8 s> 0x20 goto pc-4 38: if r8 s< 0x0 goto pc-6 41: call bpf_map_lookup_elem#1 invalid access to map value, value_size=64 off=33 size=32 The linked registers of a conditional jump are the live registers and stack slots that are a tested register + a constant on every path to the jump. bpf_scev_next_linked_reg() gets one more result. A linked register can be a pointer: *base_regno + regno + delta. *base_regno is the address base, as in store_base_regs. Here r2 == r6 + r8. So 'r2 = r6; r2 += r8' after the jump would change no value at run time, and a state that is sound before two insns is sound after them. recompute_ptr() applies their transfer function. Type, object and offset of r2 are taken from r6 as it is now; how r2 was computed does not matter. The range is that of r8 after the jump. match_linked_add(): the expression of r2 is (+ x y), y has a constant difference from the tested register, and a live register or stack slot has one from x. At a loop header r2 is a phi. There ADD_SCEV has its operands: scev at header 35: r2=(+ r6 %r8.21?) / (add r6 r8) r8=%r8.21? (add r6 r8): r2 == r6 + r8 at the header in every iteration. It holds at loop entry, where r2 is r6 and r8 is 0. Every backedge preserves it: the backedge values, before the '/', are r6 + %r8.21? and %r8.21?, the def of r8 by the nested loop at 21. r6 and r8 are the header phis here, not the entry values as below LINEAR_SCEV. recompute_ptr() replaces r2 only if - r6 is a pointer that may have a variable offset, and no packet pointer, - type, object and ids of r6 are those of r2, - the new range is a subset of the old one, - the loader has CAP_PERFMON: the cpu can speculate past the jump with an out-of-bounds index. The old offset is replaced, not intersected: a state cannot distinguish two values of a hash map, so the two offsets may belong to two objects. In a prog with an arena the add has a def, %r2.28=(add r6 r8), and is trusted once the main pass has verified insn 28 without an arena pointer. Not found: 'buf += i', which overwrites the base, and an add whose base is dead at the jump. No register holds buf there. Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 4 +- kernel/bpf/scev.c | 174 +++++++++++++++++++++++++++++++---- kernel/bpf/verifier.c | 49 +++++++++- 3 files changed, 204 insertions(+), 23 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 18f0736fccb7..651174144f5f 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -2226,7 +2226,9 @@ void bpf_free_scev(struct bpf_verifier_env *env); int bpf_compute_scev(struct bpf_verifier_env *env); int bpf_scev_jump_counts(struct bpf_verifier_env *env, int header, int n, bool taken_leaves); bool bpf_scev_next_linked_reg(struct bpf_verifier_env *env, int insn_idx, u32 regno, u32 *pos, - u32 *linked_regno, u32 *fill_off, s64 *delta, u32 *bits); + u32 *linked_regno, u32 *fill_off, int *base_regno, s64 *delta, + u32 *bits); +bool bpf_is_widenable_reg_type(const struct bpf_reg_state *reg); bool bpf_scev_in_slice_slots(struct bpf_verifier_env *env, int off, u32 size); int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st, diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index dfbad71bba55..0bb52ca8aa56 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -50,6 +50,11 @@ enum expr_op { * equal when the loop is entered. Not known otherwise. */ COPY_SCEV, + /* + * (add rB rC) is the SCEV of rA: rA == rB + rC at the header in every iteration. rB and rC + * are the header phis here, not the entry values as below LINEAR_SCEV. + */ + ADD_SCEV, }; /* @@ -65,9 +70,9 @@ enum expr_op { * In the env of loop H a phi at H merges the backedge values only, the header phi itself is * (REG r). In the env of the enclosing loop a phi at H merges the entering edges. * Op and params of a def by an insn are what that insn computes, see alu64_with_def(): - * set_expr_base() looks through such an add, split_const_offset() checks prevent_zext of the insn. - * Those of a phi are the shape that its operands share, no insn computes them: nothing looks - * through a phi. + * set_expr_base() and match_linked_add() look through such an add, split_const_offset() and + * match_linked_add() check prevent_zext of the insn. Those of a phi are the shape that its operands + * share, no insn computes them: nothing looks through a phi. */ struct expr_def { u32 set:1; /* 0: the expression has no def */ @@ -250,6 +255,8 @@ static bool expr_is_vn(struct scev *scev, struct expr *e) /* a function of the iteration number */ case LINEAR_SCEV: case COPY_SCEV: + /* a relation between header phis */ + case ADD_SCEV: return false; } for (i = 0; i < op_params_num(e->op); i++) @@ -553,6 +560,7 @@ static const char *op_str(u32 op) case ANY: return "any"; case LINEAR_SCEV: return "linear"; case COPY_SCEV: return "copy"; + case ADD_SCEV: return "add"; } return NULL; } @@ -576,6 +584,7 @@ static u32 op_params_num(u32 op) case ANY: case LINEAR_SCEV: case COPY_SCEV: + case ADD_SCEV: return 2; case BPF_NEG: case SEXT8: @@ -1846,6 +1855,41 @@ static bool const_diff_at_header(struct bpf_verifier_env *env, int h, u32 ra, u3 bits_a == 64 && bits == 64; } +/* + * p = buf + i; if (i > 100) return 0; *p = 0; + * @tested is the expression of the tested register and @id that of p, in one env as for + * const_diff(). Is @id an add, and one operand @tested + a constant? Then + * @id == *@other + @tested + *@delta (mod 2^*@bits). A header phi can be an add, see ADD_SCEV. For + * an add with a def see split_const_offset(). Which operand is the pointer is not known here. + */ +static bool match_linked_add(struct bpf_verifier_env *env, int h, u32 tested, u32 id, u32 *other, + s64 *delta, u32 *bits) +{ + struct scev *scev = env->scev; + u32 reg, r, b, op = BPF_ADD; + struct expr_def def; + s64 c; + + split_const_offset(env, id, &id, delta, bits); + if (h >= 0 && is_reg(scev, id, ®)) { + id = find_header_env(scev, h)->reg2scev[reg]; + op = ADD_SCEV; + } + def = scev->exprs[id].def; + if (!is_binop(scev, op, id, other, &r) || def.phi) + return false; + if (def.set && !env->insn_aux_data[def.idx].prevent_zext) + *bits = min(*bits, 32); + if (!const_diff(env, h, tested, r, &c, &b)) { + swap(*other, r); + if (!const_diff(env, h, tested, r, &c, &b)) + return false; + } + *delta = (u64)*delta + c; + *bits = min(*bits, b); + return true; +} + /* The first bytes of a stack slot are its high bits */ static bool is_big_endian_slot(u32 reg) { @@ -1897,9 +1941,10 @@ static int find_linked_regs(struct bpf_verifier_env *env, struct env *e, int h, u32 dst = e->reg2expr[insn->dst_reg], src = dst; struct insn_envs *envs = env->scev->envs[idx]; struct scev *scev = env->scev; - u32 r, id, fill_off, base, base_dst, base_src, bits, cap, tested = 0; + u32 r, q, other, id, fill_off, base, base_dst, base_src, bits, cap, tested = 0; u32 first = scev->linked_cnt; DECLARE_BITMAP(live, REGS_NUM) = {}; + DECLARE_BITMAP(found, REGS_NUM) = {}; void *tmp; s64 delta; @@ -1916,12 +1961,27 @@ static int find_linked_regs(struct bpf_verifier_env *env, struct env *e, int h, id = e->reg2expr[r]; /* most are not linked */ split_const_offset(env, id, &base, &delta, &bits); - if (base != base_dst && base != base_src && (h < 0 || !is_op(scev, base, REG)) && - !is_big_endian_slot(r)) + if (base != base_dst && base != base_src && !is_op(scev, base, BPF_ADD) && + (h < 0 || !is_op(scev, base, REG)) && !is_big_endian_slot(r)) continue; - if (!const_diff_reg(env, h, dst, r, id, &fill_off, &delta, &bits) && - (src == dst || !const_diff_reg(env, h, src, r, id, &fill_off, &delta, &bits))) + if (const_diff_reg(env, h, dst, r, id, &fill_off, &delta, &bits) || + (src != dst && const_diff_reg(env, h, src, r, id, &fill_off, &delta, &bits))) { + __set_bit(r, found); + continue; + } + if (!match_linked_add(env, h, dst, id, &other, &delta, &bits) && + (src == dst || !match_linked_add(env, h, src, id, &other, &delta, &bits))) continue; + /* an add is of use only with a live register that holds its other operand */ + for_each_set_bit(q, live, REGS_NUM) { + if (q == r || !const_diff(env, h, e->reg2expr[q], other, &delta, &bits)) + continue; + __set_bit(r, found); + __set_bit(q, found); + break; + } + } + for_each_set_bit(r, found, REGS_NUM) { if (scev->linked_cnt == scev->linked_cap) { cap = max(scev->linked_cap * 2, 16); tmp = kvrealloc(scev->linked, array_size(cap, sizeof(*scev->linked)), @@ -1933,7 +1993,7 @@ static int find_linked_regs(struct bpf_verifier_env *env, struct env *e, int h, } tested += r == insn->dst_reg || (src != dst && r == insn->src_reg); scev->linked[scev->linked_cnt].reg = r; - scev->linked[scev->linked_cnt++].id = id; + scev->linked[scev->linked_cnt++].id = e->reg2expr[r]; } /* nothing but the tested registers */ if (scev->linked_cnt - first == tested) { @@ -1957,14 +2017,18 @@ static int find_linked_regs(struct bpf_verifier_env *env, struct env *e, int h, * the same frame. It is a register, or MAX_BPF_REG + the index of a stack slot: a fill from byte * *@fill_off of the slot reads these bits. *@pos is the position of the search, 0 at first. False: * no more links are known. Types are not known here: the caller checks them in its state. + * A linked register can be a pointer: *base_regno + regno + delta. *base_regno is the address base, + * as in store_base_regs. It is -1 for a link without a base. */ bool bpf_scev_next_linked_reg(struct bpf_verifier_env *env, int insn_idx, u32 regno, u32 *pos, - u32 *linked_regno, u32 *fill_off, s64 *delta, u32 *bits) + u32 *linked_regno, u32 *fill_off, int *base_regno, s64 *delta, + u32 *bits) { struct insn_envs *envs = env->scev->envs[insn_idx]; int h = bpf_loop_at_index(env, insn_idx); struct scev_linked_reg *linked; - u32 i; + u32 i, j, other, b; + s64 c; if (!envs || !envs->linked_cnt) return false; @@ -1975,10 +2039,22 @@ bool bpf_scev_next_linked_reg(struct bpf_verifier_env *env, int insn_idx, u32 re return false; for (; *pos < envs->linked_cnt; ++*pos) { *linked_regno = linked[*pos].reg; - if (*pos != i && - const_diff_reg(env, h, linked[i].id, *linked_regno, linked[*pos].id, fill_off, + *base_regno = -1; + if (*pos == i) + continue; + if (const_diff_reg(env, h, linked[i].id, *linked_regno, linked[*pos].id, fill_off, delta, bits)) return true; + if (!match_linked_add(env, h, linked[i].id, linked[*pos].id, &other, delta, bits)) + continue; + for (j = 0; j < envs->linked_cnt; j++) { + if (j == *pos || !const_diff(env, h, linked[j].id, other, &c, &b)) + continue; + *base_regno = linked[j].reg; + *delta = (u64)*delta + c; + *bits = min(*bits, b); + return true; + } } return false; } @@ -2445,6 +2521,64 @@ static int compute_header_scevs(struct bpf_verifier_env *env, struct env *header return 0; } +/* + * Find a SCEV register with the entry value @x and the backedge value @l, and one with @y and @r. + * All that match are copies of each other at the header: take the first. + */ +static bool find_operands(struct env *entry_env, struct env *header_env, u32 x, u32 y, u32 l, u32 r, + u32 *rb, u32 *rc) +{ + for (*rb = 0; *rb < REGS_NUM; ++*rb) + if (entry_env->reg2expr[*rb] == x && header_env->reg2expr[*rb] == l) + break; + for (*rc = 0; *rc < REGS_NUM; ++*rc) + if (entry_env->reg2expr[*rc] == y && header_env->reg2expr[*rc] == r) + break; + return *rb < REGS_NUM && *rc < REGS_NUM; +} + +/* + * p = buf; i = 0; while (*p) { if (i > 512) break; use(p); i += len(p); p = buf + i; } + * rA == rB + rC holds at the header in every iteration: it holds at loop entry and every backedge + * preserves it. The backedge values are rA = (+ l r), rB = l, rC = r. The entry values, by the env + * of the enclosing loop at @h, are rA = x, rB = x, rC = 0 or rA = (+ x y), rB = x, rC = y. + * (add rB rC) keeps the def of (+ l r), see alu64_with_def(). + */ +static int compute_header_adds(struct bpf_verifier_env *env, int h) +{ + struct scev *scev = env->scev; + struct env *entry_env = find_loop_env(scev, env->insn_aux_data[h].loop_header, h); + struct env *header_env = find_header_env(scev, h); + u32 ra, rb, rc, l, r, x, y, entry_expr; + struct expr_def def; + int id, zero = imm_expr(scev, 0); + + if (zero < 0) + return zero; + for (ra = 0; entry_env && ra < REGS_NUM; ra++) { + entry_expr = entry_env->reg2expr[ra]; + if (header_env->reg2scev[ra] != UNKNOWN_EXPR_ID || + !is_add(scev, header_env->reg2expr[ra], &l, &r) || + !scev->exprs[entry_expr].is_vn || !scev->exprs[l].is_vn || + !scev->exprs[r].is_vn) + continue; + if (!find_operands(entry_env, header_env, entry_expr, zero, l, r, &rb, &rc) && + !find_operands(entry_env, header_env, zero, entry_expr, l, r, &rb, &rc) && + (scev->exprs[entry_expr].def.set || !is_add(scev, entry_expr, &x, &y) || + (!find_operands(entry_env, header_env, x, y, l, r, &rb, &rc) && + !find_operands(entry_env, header_env, y, x, l, r, &rb, &rc)))) + continue; + /* expr2() can move exprs[] */ + def = scev->exprs[header_env->reg2expr[ra]].def; + id = expr_with_def(scev, expr2(scev, ADD_SCEV, REG_EXPR_ID(rb), REG_EXPR_ID(rc)), + def); + if (id < 0) + return id; + header_env->reg2scev[ra] = id; + } + return 0; +} + /* * 'sum += a[i]', 'if (..) { len -= n; len -= 2; }': is 'id', what an iteration makes of * register 'r', the register plus or minus something, or the register as it was? @@ -2613,9 +2747,13 @@ int bpf_compute_scev(struct bpf_verifier_env *env) find_loop_env(scev, header, idx)); if (err) return err; - if (aux[idx].loop) + if (aux[idx].loop) { scev_compute_linear_regs(scev, aux[idx].loop, find_header_env(scev, header)); + err = compute_header_adds(env, idx); + if (err) + return err; + } err = find_linked_regs(env, aux[idx].loop ? &scev->identity_env : find_loop_env(scev, header, idx), header, idx); if (err) @@ -3850,7 +3988,7 @@ static bool is_const_scev_reg(struct bpf_verifier_env *env, int insn_idx, u32 r) } /* Main logic in verifier.c forbids varying offsets for certain register types. */ -static bool is_widenable_reg_type(const struct bpf_reg_state *reg) +bool bpf_is_widenable_reg_type(const struct bpf_reg_state *reg) { if (type_may_be_null(reg->type)) return false; @@ -3897,7 +4035,7 @@ static bool is_widenable_any(struct bpf_verifier_env *env, struct bpf_func_state if (!is_any(scev, id, &l, &rr)) return false; reg = scev_reg_for_read(env, loop_entry, r); - if (!reg || !is_widenable_reg_type(reg)) + if (!reg || !bpf_is_widenable_reg_type(reg)) return false; scev->stack_sz = 0; @@ -4430,7 +4568,7 @@ int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_sta } /* If SCEV for r is (linear ) */ if (reg && is_linear_at_header(scev, cur_func, r_scev, r, &coefs) && - is_widenable_reg_type(reg)) { + bpf_is_widenable_reg_type(reg)) { /* 32-bit ALU operations on pointers convert them to scalars */ if (coefs.zext32 && reg->type != SCALAR_VALUE) goto cant_widen; @@ -4773,7 +4911,7 @@ int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur * bpf_compute_loop_iters() gives no range to a pointer to arena that moves, and * it does not look at dead registers. Set a range only where it would. */ - if (!is_widenable_reg_type(entry_reg) || + if (!bpf_is_widenable_reg_type(entry_reg) || (coefs.zext32 && entry_reg->type != SCALAR_VALUE)) continue; clamp: diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d980e2c60805..970211dfe38c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19774,6 +19774,39 @@ static struct bpf_reg_state *scev_reg_for_fill(struct bpf_func_state *frame, u32 return &ss->spilled_ptr; } +/* + * p = buf + i; if (i > 100) return 0; *p = 0; + * p is @dst_reg, x is @ptr_reg, i is @off_reg. Scalar evolution says that p == x + i. So the insns + * 'p = x; p += i' are a no-op at run time, and a state that is sound before them is sound after + * them: apply their transfer function. Type, object and offset of p are taken from x as it is now; + * how p was computed does not matter. + * The old p is replaced, not intersected: a state does not distinguish two values of one hash map, + * and the intersection of an offset into the one with an offset into the other holds for neither. + * It is replaced only if the new range is a subset of the old one. + */ +static bool recompute_ptr(struct bpf_verifier_env *env, struct bpf_reg_state *dst_reg, + const struct bpf_reg_state *ptr_reg, const struct bpf_reg_state *off_reg) +{ + struct bpf_reg_state sum = *ptr_reg; + + /* the cpu can speculate past the jump with an out-of-bounds index */ + if (!env->bypass_spec_v1) + return false; + /* 'p += i' assigns a new id to a packet pointer */ + if (ptr_reg->type == SCALAR_VALUE || type_is_pkt_pointer(ptr_reg->type) || + !bpf_is_widenable_reg_type(ptr_reg)) + return false; + __mark_reg32_unbounded(&sum); + scalar_step_add(&sum, ptr_reg, off_reg); + sum.r64 = cnum64_add(ptr_reg->r64, off_reg->r64); + sum.var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); + reg_bounds_sync(&sum); + if (!bpf_reg_union_compatible(dst_reg, &sum) || !cnum64_is_subset(dst_reg->r64, sum.r64)) + return false; + *dst_reg = sum; + return true; +} + /* * The jump at @idx refined @regno. A register that scalar evolution links to it holds * @regno + delta on every path to the jump: intersect the two. The intersection is empty: no @@ -19783,22 +19816,29 @@ static void refine_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier int idx, u32 regno) { struct bpf_func_state *frame = vstate->frame[vstate->curframe]; - struct bpf_reg_state *reg, implied, saved; + struct bpf_reg_state *reg, *base_reg, implied, saved; u32 pos, linked_regno, fill_off, bits; + int base_regno; s64 delta; if (frame->regs[regno].type != SCALAR_VALUE) return; for (pos = 0; - bpf_scev_next_linked_reg(env, idx, regno, &pos, &linked_regno, &fill_off, &delta, - &bits); + bpf_scev_next_linked_reg(env, idx, regno, &pos, &linked_regno, &fill_off, &base_regno, + &delta, &bits); pos++) { reg = scev_reg_for_fill(frame, linked_regno, fill_off, bits); - if (!reg || reg->type != SCALAR_VALUE) + if (!reg || (base_regno < 0 && reg->type != SCALAR_VALUE)) continue; implied = frame->regs[regno]; add_scalar_to_reg(&implied, delta); + if (base_regno >= 0) { + base_reg = scev_reg_for_fill(frame, base_regno, 0, 64); + if (base_reg && bits == 64 && recompute_ptr(env, reg, base_reg, &implied)) + goto scratch; + continue; + } if (regs_fit_in_bits(reg, &implied, bits)) bits = 64; if (bits < 32) @@ -19811,6 +19851,7 @@ static void refine_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier *reg = saved; continue; } +scratch: if (linked_regno < MAX_BPF_REG) mark_reg_scratched(env, linked_regno); else -- 2.55.0 From: Alexei Starovoitov Tests for the patch that recomputes a pointer after the jump that bounds its index. Since that patch a linked register of a conditional jump can be a pointer: an address base + the tested register + a constant on every path to the jump. Each prog adds an index to a pointer to 64 bytes, tests the index and reads through the result. What loads is run. - The add comes before the jump: loads. - Only the low 32 bits of the index are tested. Rejected. - base + index at a loop header: the add is at the end of an iteration, the jump at the header. Negative tests: it holds at loop entry only, on the backedge only. - In a prog with an arena 'r1 += r2' is trusted once the main pass has verified it without an arena pointer (prevent_zext): loads. The test with an arena is in verifier_arena.c, the others in verifier_scev.c. Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/progs/verifier_arena.c | 3 +++ .../selftests/bpf/progs/verifier_scev.c | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index bf50413f6c08..3931a1d9b6da 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -663,6 +663,9 @@ ARENA_LINKED(arena_ptr_plus_constant, __failure, ARENA_LINKED(arena_ptr_plus_constant_on_two_paths, __failure, "if r8 == 0 goto 1f; r3 += 1; goto 2f; 1: r3 += 1; 2: r8 += 1; r3 = le64 r3; r7 = le64 r7;" "r2 = 0xffffffff ll; if r7 != r2 goto out_%=; r3 >>= 32; r3 -= 1; r1 = r6; r1 += r3;") +/* 'r1 += r2' was verified without a pointer to arena: it stays ALU64 */ +ARENA_LINKED(index_tested_after_the_add, __success __retval(0), + "r2 = r8; r2 &= 255; r1 = r6; r1 += r2; if r2 > 63 goto out_%=; r0 = *(u8 *)(r6 + 0);") SEC("syscall") __description("arena: a number is not an address in arena") diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 1047845c916b..8ffbe7bbdb0a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -3341,12 +3341,37 @@ __naked void name(void) \ #define LINKED(name, expect, body) LINKED_SEC("socket", name, expect, body) #define VALID __success __retval(0) +#define MAP_OOB __failure __msg("invalid access to map value") #define UNBOUND __failure __msg("unbounded min value") /* r3 is the address in r1 as a scalar. It does not bound the offset of r1. */ LINKED(pointer_is_not_scalar, __failure, "r7 &= 127; r1 = r6; r1 += r7; r3 = r1; r3 = le64 r3; if r3 < 64 goto out_%=; r1 += -64;") +/* + * p = buf + i; if (i > 63) return 0; c = *p; + * The jump bounds i. r1 is r6 + r7 and r6 is live: r1 is recomputed. + */ +#define USE_R6 "r0 = *(u8 *)(r6 + 0);" + +LINKED(index_tested_after_the_add, VALID, "r1 = r6; r1 += r7; if r7 > 63 goto out_%=;" USE_R6) +LINKED(low_32_bits_of_the_index_tested, __failure __msg("R1 unbounded memory access"), + "r2 = 0x1000000ff ll; r2 &= r8; r1 = r6; r1 += r2; w3 = w2; if r3 > 63 goto out_%=;" USE_R6) + +/* + * p = buf; i = 0; do { if (i > 62) break; i = *(u16 *)p; p = buf + i; } while (i); + * r1 == r6 + r2 at the header in every iteration: it holds at loop entry, the backedge preserves it + */ +#define LOOP(add) \ + "l_%=: if r2 > 62 goto out_%=; r2 = *(u16 *)(r1 + 0);" add \ + "if r2 != 0 goto l_%=; r1 = r6;" + +LINKED(base_plus_index_at_loop_header, VALID, "r1 = r6; r2 = 0;" LOOP("r1 = r6; r1 += r2;")) +LINKED(base_plus_index_at_loop_entry_only, MAP_OOB, + "r1 = r6; r2 = 0;" LOOP("r1 = r6; r1 += r2; r2 >>= 1;")) +LINKED(base_plus_index_on_backedge_only, MAP_OOB, + "r1 = r6; r1 += r7; r2 = r8; r2 &= 255;" LOOP("r1 = r6; r1 += r2;")) + LINKED(sign_extension_that_fits, VALID, "r2 = (s8)r8; if r8 > 63 goto out_%=; r1 = r6; r1 += r2;") /* The low 3 bits of r3 are 5. It is r8 + 5 once r8 fits in s8: no overlap with [-2, 1]. */ -- 2.55.0 From: Alexei Starovoitov A prerequisite of the removal of scalar ids, no new ability. scev.c has two users of them: scev_regs_equal() reads them, link_copies() assigns them. Both are for a loop like this one. It has an iteration count (header_count: how often the header is executed) before and after the patch: u32 i = 0; while (buf[i++] && i < 1023) ; return buf[i]; clang keeps the value of i from before the increment in r2, and the linear latch tests r2: r2 = 0 r1 = 0 l: r1 += 1 if r2 > 1021 goto out r2 += r0 w3 = *(u8 *)(r2 + 0) r2 = r1 if w3 != 0 goto l out: r0 += r1 r1 and r2 have the same backedge value. r2 is a copy of the induction variable r1 if their entry values are equal too (at the header the two are congruent induction variables, LLVM: replaceCongruentIVs()). scev.c used to analyze loops only, so (copy r2 (linear r1 1)), COPY_SCEV, leaves the entry values to the main pass: scev_regs_equal() accepts the same constant or the same scalar id in the state at loop entry. By now scev.c computes expressions for whole subprogs, the code before the loop included. const_diff_at_header() decides if r2 == r1 holds at the header in every iteration: it holds at loop entry and every backedge preserves it. Then r2 is an induction variable of its own, (linear r2 1), and no code after compute_header_scevs() has to know that it is a copy. link_copies() assigns one scalar id to r1 and r2 at the header, so that the test of r2 bounds r1 at 'out'. The linked registers of a conditional jump do that without an id: the live registers and stack slots that are a tested register + a constant on every path to the jump. At the header, the latches and the exits of a loop they include what const_diff_at_header() finds. Remove COPY_SCEV, is_copy(), copy_reg, scev_regs_equal(), link_copies(). Precision that is lost: - Two registers enter the loop with the same constant, and only the state knows it, not scev.c. The loop has no iteration count. - At a jump of the loop that is no latch and no exit a test of r1 does not refine r2: const_diff_at_header() is not used there. copy_less and copy_more of verifier_loop_latch.c stored at the copy - 1 and + 1 after such a test. They store at the copy now. No other verdict changes in the selftests, Cilium, sched_ext, katran, bpfjailer and the programs in Rust. The new test copy_ahead enters the loop with 0 in the induction variable and 1 in the copy: no iteration count. Signed-off-by: Alexei Starovoitov --- kernel/bpf/scev.c | 107 +++--------------- .../selftests/bpf/progs/verifier_loop_latch.c | 8 +- 2 files changed, 24 insertions(+), 91 deletions(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 0bb52ca8aa56..2792217754be 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -45,11 +45,6 @@ enum expr_op { * bpf_loop_at_index(). */ LINEAR_SCEV, - /* - * param[1], given that register param[0] and the register that param[1] starts from are - * equal when the loop is entered. Not known otherwise. - */ - COPY_SCEV, /* * (add rB rC) is the SCEV of rA: rA == rB + rC at the header in every iteration. rB and rC * are the header phis here, not the entry values as below LINEAR_SCEV. @@ -254,7 +249,6 @@ static bool expr_is_vn(struct scev *scev, struct expr *e) case SPILL32: /* a function of the iteration number */ case LINEAR_SCEV: - case COPY_SCEV: /* a relation between header phis */ case ADD_SCEV: return false; @@ -492,13 +486,6 @@ static bool is_linear(struct scev *scev, int id, u32 *base, u32 *slope) return is_binop(scev, LINEAR_SCEV, id, base, slope); } -static bool is_copy(struct scev *scev, int id, u32 *reg, u32 *orig) -{ - u32 t; - - return is_binop(scev, COPY_SCEV, id, &t, orig) && is_reg(scev, t, reg); -} - static bool is_linear_or_linear32(struct scev *scev, int id, u32 *base, u32 *slope, bool *zext32) { u32 t; @@ -559,7 +546,6 @@ static const char *op_str(u32 op) case SPILL32: return "spill32"; case ANY: return "any"; case LINEAR_SCEV: return "linear"; - case COPY_SCEV: return "copy"; case ADD_SCEV: return "add"; } return NULL; @@ -583,7 +569,6 @@ static u32 op_params_num(u32 op) case SMOD: case ANY: case LINEAR_SCEV: - case COPY_SCEV: case ADD_SCEV: return 2; case BPF_NEG: @@ -2412,9 +2397,10 @@ static int instantiate_header_scevs(struct scev *scev, u32 id, void *priv) return id; } -static int compute_header_scevs(struct bpf_verifier_env *env, struct env *header_env) +static int compute_header_scevs(struct bpf_verifier_env *env, int h) { struct scev *scev = env->scev; + struct env *header_env = find_header_env(scev, h); int l, r, t, id, ra, rb, ra_expr; bool zext32; s64 imm; @@ -2499,7 +2485,7 @@ static int compute_header_scevs(struct bpf_verifier_env *env, struct env *header /* * do { i++; if (old > 1021) break; c = buf[old]; old = i; } while (c); * rA = (+ rB IMM) and rB = (+ rB IMM): an iteration leaves the same number in both. If - * they are equal when the loop is entered rA is at the header what rB is. + * they are equal at loop entry too, rA is an induction variable with the same step. */ for (ra = 0; ra < REGS_NUM; ra++) { if (header_env->reg2scev[ra] != UNKNOWN_EXPR_ID || @@ -2509,9 +2495,12 @@ static int compute_header_scevs(struct bpf_verifier_env *env, struct env *header id = header_env->reg2scev[rb]; if (header_env->reg2expr[rb] != header_env->reg2expr[ra] || !is_linear_or_linear32(scev, id, &l, &r, &zext32) || - !is_reg(scev, l, &t) || t != rb) + !is_reg(scev, l, &t) || t != rb || + !const_diff_at_header(env, h, rb, ra, &imm) || imm) continue; - id = expr2(scev, COPY_SCEV, expr1(scev, REG, ra), id); + id = expr2(scev, LINEAR_SCEV, expr1(scev, REG, ra), r); + if (zext32) + id = expr1(scev, ZEXT32, id); if (id < 0) return id; header_env->reg2scev[ra] = id; @@ -2741,7 +2730,7 @@ int bpf_compute_scev(struct bpf_verifier_env *env) cond_resched(); header = bpf_loop_at_index(env, idx); err = aux[idx].loop - ? compute_header_scevs(env, find_header_env(scev, header)) + ? compute_header_scevs(env, header) : compute_insn_scevs(env, find_header_env(scev, header), find_loop_env(scev, header, idx)); @@ -2884,19 +2873,6 @@ static struct bpf_reg_state *scev_reg_for_linear(struct bpf_func_state *st, u32 return &stack->spilled_ptr; } -/* Do the registers or stack slots @a and @b hold the same number in @st? */ -static bool scev_regs_equal(struct bpf_func_state *st, u32 a, u32 b) -{ - const struct bpf_reg_state *ra = scev_reg_for_linear(st, a); - const struct bpf_reg_state *rb = scev_reg_for_linear(st, b); - - if (!ra || !rb || ra->type != SCALAR_VALUE || rb->type != SCALAR_VALUE) - return false; - if (tnum_is_const(ra->var_off) && tnum_is_const(rb->var_off)) - return ra->var_off.value == rb->var_off.value; - return ra->id && ra->id == rb->id && ra->delta == rb->delta; -} - /* '*(u32 *)(r10 - 8) = rX' left a number in the low half of the slot. */ static struct bpf_reg_state *scev_reg_for_spill32(struct bpf_func_state *st, u32 r) { @@ -3014,8 +2990,7 @@ static bool scev_reg_alive(struct bpf_verifier_env *env, struct bpf_verifier_sta * 'i' is the loop iteration number, starting from 0. * 'zext32' indicates the latch is for (zext32 (linear ...)) SCEV. * 'jmp32' indicates that the latch is a JMP32 instruction. - * 'base_reg' is the register in . 'copy_reg' is -1, or is made from that register, - * which has to be equal to 'base_reg' at the entry of the loop, see COPY_SCEV. + * 'base_reg' is the register in . */ struct linear_latch { u32 insn_idx; @@ -3023,7 +2998,6 @@ struct linear_latch { u32 step_expr; u32 bound_expr; u32 base_reg; - int copy_reg; u8 op; bool zext32; bool jmp32; @@ -3047,7 +3021,7 @@ static int match_linear_latch(struct bpf_verifier_env *env, struct env *latch_env; u32 src_reg_scev; u32 dst_reg_scev; - u32 t, l, r, op; + u32 l, r, op; int id; op = BPF_OP(insn->code); @@ -3077,11 +3051,6 @@ static int match_linear_latch(struct bpf_verifier_env *env, return -EFAULT; dst_reg_scev = latch_env->reg2scev[insn->dst_reg]; src_reg_scev = latch_env->reg2scev[insn->src_reg]; - latch->copy_reg = -1; - if (is_copy(scev, dst_reg_scev, &latch->copy_reg, &t)) - dst_reg_scev = t; - else if (BPF_SRC(insn->code) == BPF_X && is_copy(scev, src_reg_scev, &latch->copy_reg, &t)) - src_reg_scev = t; /* * Make sure (linear ...) is in dst_reg_scev: * 'if ' ⇔ @@ -3779,8 +3748,6 @@ static bool compute_max_iters(struct bpf_verifier_env *env, iters->iv_reg = -1; - if (latch->copy_reg >= 0 && !scev_regs_equal(st, latch->copy_reg, latch->base_reg)) - return false; if (!eval_expr(env, scev, st, latch->base_expr, &initial_val) || !eval_expr(env, scev, st, latch->step_expr, &step_val) || !eval_expr(env, scev, st, latch->bound_expr, &bound_val)) @@ -4116,49 +4083,15 @@ static bool is_simple_linear(struct scev *scev, u32 id, struct linear_coefs *out } /* - * Is @r 'what it is in @st + slope * k' at the header in iteration k? @id is its SCEV there, @st is - * the state that entered the loop. At a jump in the loop @id can be what another register was at - * the header. @st says nothing about @r then. + * Is @r 'its entry value + slope * k' at the header in iteration k? @id is its SCEV there. At a + * jump in the loop @id can be linear in the entry value of another register: the entry value of @r + * does not bound it then. */ -static bool is_linear_at_header(struct scev *scev, struct bpf_func_state *st, u32 id, u32 r, - struct linear_coefs *out) +static bool is_linear_at_header(struct scev *scev, u32 id, u32 r, struct linear_coefs *out) { - u32 copy, orig; - - if (is_copy(scev, id, ©, &orig)) - return copy == r && is_simple_linear(scev, orig, out) && - scev_regs_equal(st, r, out->base_reg); return is_simple_linear(scev, id, out) && out->base_reg == r; } -/* - * @st is at the header of a loop that @entry entered. A copy of an induction variable is equal to - * it there in every iteration. They got their ranges one by one: give them one id, like 'r2 = r1' - * does, so that a test of one of them is a test of both. - */ -static void link_copies(struct bpf_verifier_env *env, struct env *header_env, - struct bpf_func_state *entry, struct bpf_func_state *st) -{ - struct bpf_reg_state *reg, *orig; - struct linear_coefs coefs; - u32 r; - - for (r = 0; r < REGS_NUM; r++) { - if (!is_linear_at_header(env->scev, entry, header_env->reg2scev[r], r, &coefs) || - coefs.base_reg == r) - continue; - reg = scev_reg_for_linear(st, r); - orig = scev_reg_for_linear(st, coefs.base_reg); - if (!reg || !orig || reg->type != SCALAR_VALUE || orig->type != SCALAR_VALUE || - tnum_is_const(orig->var_off)) - continue; - if (!orig->id) - orig->id = ++env->id_gen; - reg->id = orig->id; - reg->delta = orig->delta; - } -} - /* * A 32-bit counter that lives on the stack: the SCEV of slot @r is * (spill32 (linear (zext32 @r) slope)) and there is a 32-bit number in the slot in @st. @@ -4567,7 +4500,7 @@ int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_sta continue; } /* If SCEV for r is (linear ) */ - if (reg && is_linear_at_header(scev, cur_func, r_scev, r, &coefs) && + if (reg && is_linear_at_header(scev, r_scev, r, &coefs) && bpf_is_widenable_reg_type(reg)) { /* 32-bit ALU operations on pointers convert them to scalars */ if (coefs.zext32 && reg->type != SCALAR_VALUE) @@ -4783,7 +4716,7 @@ int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state } /* If SCEV for r is (linear )*/ - if (spill32 || is_linear_at_header(scev, entry_func, r_expr, r, &coefs)) { + if (spill32 || is_linear_at_header(scev, r_expr, r, &coefs)) { /* the range is derived from @loop_entry: bpf_clamp_scev_regs() */ if (back) continue; @@ -4831,7 +4764,6 @@ int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state mark_scev_reg_scratched(env, r); } } - link_copies(env, header_env, entry_func, cur_func); return 1; } @@ -4888,7 +4820,7 @@ int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur goto clamp; /* If SCEV for r is (linear )*/ - if (!is_linear_at_header(scev, entry_st, r_scev, r, &coefs) && + if (!is_linear_at_header(scev, r_scev, r, &coefs) && !(behind && is_moved_linear(scev, r_scev, &coefs, &moved) && coefs.base_reg == r)) continue; @@ -4958,8 +4890,5 @@ int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur return err; mark_scev_reg_scratched(env, r); } - /* behind the jump the variable may have moved on: its copy is what it was at the head */ - if (!behind) - link_copies(env, header_env, entry_st, cur_func_state); return 0; } diff --git a/tools/testing/selftests/bpf/progs/verifier_loop_latch.c b/tools/testing/selftests/bpf/progs/verifier_loop_latch.c index 35bb3e16db3d..805e3f3ae091 100644 --- a/tools/testing/selftests/bpf/progs/verifier_loop_latch.c +++ b/tools/testing/selftests/bpf/progs/verifier_loop_latch.c @@ -271,6 +271,10 @@ LOOP(copy_bad, "r8 = 0; r9 = r8;", COPY(2047)) BAD("a copy of the counter, another number at the entry") LOOP(copy_other_const, "r8 = 2047; r9 = 0; r6 += 2047;", COPY(2046)) +/* the latch exits one iteration later than for an induction variable that starts from 1 */ +NO_COUNT("a copy of the counter, one more at the entry") +LOOP(copy_ahead, "r8 = 0; r9 = 1;", COPY(2047)) + OK("a copy of the counter, a range", "[0..2]") LOOP(copy_range, "r5 = r6; r8 = r7; r8 &= 1; r8 += 2045; r9 = r8;", COPY(2045) AT_R8) @@ -297,13 +301,13 @@ LOOP(copy_bounds_both, "r5 = r6; if r7 > 2047 goto 9f; r8 = r7; r9 = r8;", "r4 = r5; r4 += r9; *(u8 *)(r4 " off ") = r1; 2: r9 = r8; if r7 != 0 goto 1b;" OK("a copy of the counter, one less behind the jump", "[0..2048]") -LOOP(copy_less, "r5 = r6; r8 = 0; r9 = r8;", NOT_EQUAL(1, "> 2046", "< 2", "- 1")) +LOOP(copy_less, "r5 = r6; r8 = 0; r9 = r8;", NOT_EQUAL(1, "> 2046", "< 2", "+ 0")) BAD("a copy of the counter, one less behind the jump, not equal") LOOP(copy_less_bad, "r5 = r6; r8 = 0; r9 = r8;", NOT_EQUAL(1, "> 2046", "< 2", "- 2")) OK("a copy of the counter, one more behind the jump", "[0..2047]") -LOOP(copy_more, "r5 = r6; r8 = 2047; r9 = r8;", NOT_EQUAL(-1, "s< 2", "> 2045", "+ 1")) +LOOP(copy_more, "r5 = r6; r8 = 2047; r9 = r8;", NOT_EQUAL(-1, "s< 2", "> 2045", "+ 0")) BAD("a copy of the counter, one more behind the jump, not equal") LOOP(copy_more_bad, "r5 = r6; r8 = 2047; r9 = r8;", NOT_EQUAL(-1, "s< 2", "> 2045", "+ 2")) -- 2.55.0 From: Alexei Starovoitov A prerequisite of the removal of scalar ids. bpfj_proc_file_open of bpfjailer loads before and after the patch. Today it loads because of a scalar id, and scev.c has no replacement for that id in an irreducible loop: for (int i = BPFJ_MAX_POD_PER_PID - 1; i >= 0; --i) { if (i >= num_pods) { continue; } struct bpfj_pod __arena* pod = actor->pods[i]; That is role_gate.h. i lives on the stack (clang, -mcpu=v4): 171: r1 = *(u64 *)(r10 -40) 172: if w1 > 0x7 goto pc-10 173: r1 = *(u64 *)(r10 -40) 174: r1 <<= 3 The loop is entered at 163 and at 171: 'loop at 171, irreducible' in the level 2 log. scev.c does not support such a loop: nothing is known at its header, its body is not analyzed. The linked registers of a conditional jump, the live registers and stack slots that are a tested register + a constant on every path to the jump, are computed by scev.c. So the jump at 172 has none. Only the scalar id that r1 and fp-40 share bounds what insn 173 fills. The basic block is enough to see that r1 == fp-40 at 172: local value numbering. find_linked_regs_in_bb() visits the basic blocks of unsupported loops, see can_compute_loop_scev(), one by one. At the start of a basic block every live-in register and stack slot is a phi whose operands are not known: fp-40 is %fp-40.171=phi(?). Insn 171 copies that to r1. These expressions are for find_linked_regs() only, the level 2 log does not print them. The verifier adds edges that the CFG does not have: may_goto before a loop header, a subprog that is out of budget, see bpf_head_guard_target(). They lead out of all loops, so a basic block of a loop is entered at its first insn only. Two tests: a link within a basic block is found. What the fall-through predecessor computed is not used: it is not the only predecessor. Signed-off-by: Alexei Starovoitov --- kernel/bpf/scev.c | 52 ++++++++++++++++++- .../selftests/bpf/progs/verifier_scev.c | 13 +++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 2792217754be..db60585c00c8 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -56,7 +56,8 @@ enum expr_op { * An SSA name for a value that no expression describes: the def of the SCEV register 'reg' * - by the insn at 'idx'. An insn that may write defines too (may_def in liveness.c). A nested loop * defines at its header what it leaves. - * - by a phi before the insn at 'idx': a join point, a loop header. + * - by a phi before the insn at 'idx': a join point, a loop header, in an unsupported loop the + * start of a basic block. * Without its backedges and with nested loops collapsed the body of a loop is acyclic: an insn * executes at most once per iteration, so a def is one value per iteration. A name is unique within * the env of one loop. @@ -650,7 +651,8 @@ static bool expr_next(struct scev *scev, u32 *id, u32 *order) * %fp-8.8? insn 8 may write fp-8. The value is UNKNOWN, '?' without a def * %r8.7=phi(1, 2) the phi of r8 before insn 7 * %r0.7=phi(_) a phi of distinct OPAQUE values - * %r6.9=phi(?) a phi of distinct UNKNOWN values, or of more than 8 values + * %r6.9=phi(?) a phi of distinct UNKNOWN values, or of more than 8 values, or + * a live-in of a basic block * %r2.12=(+ x 4) insn 12 defines r2, see alu64_with_def() * %r2.25=phi((+ x 4)) two adds of one shape, each may have been converted to ALU32: * the phi is not (+ x 4) @@ -1996,6 +1998,49 @@ static int find_linked_regs(struct bpf_verifier_env *env, struct env *e, int h, return 0; } +/* + * for (..) { if (i >= n) ..; pod = actor->pods[i]; .. } + * with i on the stack is 'fill, test, fill again'. clang can emit an irreducible loop for it. + * Unsupported loops, see can_compute_loop_scev(), get local value numbering: a live-in of a basic + * block is a phi whose operands are not known, %r6.9=phi(?). + */ +static int find_linked_regs_in_bb(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_insn *insn = env->prog->insnsi; + bool bb_start = true; + int i, h, err = 0; + struct env *e; + u32 r; + + e = kzalloc(sizeof(*e), GFP_KERNEL_ACCOUNT); + if (!e) + return -ENOMEM; + for (i = 0; i < env->prog->len && !err; i += bpf_is_ldimm64(&insn[i]) ? 2 : 1) { + h = bpf_loop_at_index(env, i); + if (h < 0 || can_compute_loop_scev(aux[h].loop)) { + bb_start = true; + continue; + } + if (signal_pending(current)) + err = -EAGAIN; + cond_resched(); + for (r = 0; bb_start && r < REGS_NUM && !err; r++) { + e->reg2expr[r] = UNKNOWN_EXPR_ID; + if (reg_alive_at(env, r, i)) + err = replace_reg(env->scev, e, r, + expr_with_def(env->scev, UNKNOWN_EXPR_ID, + phi_def(i, r))); + } + err = err ?: find_linked_regs(env, e, -1, i); + err = err ?: transfer(env, e, i); + err = err ?: reset_scevs_at_indirect_writes(env, e, i); + bb_start = aux[i].bb_end || bpf_insn_successors(env, i)->cnt != 1; + } + kfree(e); + return err; +} + /* * Find the next SCEV register that is linked to @regno at the conditional jump at @insn_idx: on * every path to the jump *@linked_regno == @regno + *@delta (mod 2^*@bits). It is live there and of @@ -2715,6 +2760,9 @@ int bpf_compute_scev(struct bpf_verifier_env *env) if (err) return err; } + err = find_linked_regs_in_bb(env); + if (err) + return err; /* * Compute scevs from exprs collected on a previous step. Iterate instructions in diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 8ffbe7bbdb0a..49f9a976a44c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -3423,6 +3423,19 @@ LINKED(linked_at_loop_header, VALID, LINKED(linked_at_loop_entry_only, __failure, LOOP_HEADER("r3 = r8; r2 = r8; r2 += 1; r2 += 1;", "r2 += 2; r3 += 1;")) +/* + * In an irreducible loop registers are linked within a basic block only. In the second test the + * fall-through predecessor of 'm' makes r2 a copy of r8. The other predecessor does not. + */ +LINKED(irreducible_loop, VALID, + "if r7 > 5 goto m_%=; l_%=: r7 += 1;" + "m_%=: r2 = r8; r2 += 1; r2 += 1; if r8 > 61 goto out_%=; r1 = r6; r1 += r2;" + "r0 = *(u8 *)(r1 + 0); if r7 < 10 goto l_%=;") +LINKED(irreducible_loop_fallthrough_pred, UNBOUND, + "r2 = r7; if r7 > 5 goto m_%=; l_%=: r7 += 1; r2 = r8;" + "m_%=: if r8 > 63 goto out_%=; r1 = r6; r1 += r2;" + "r0 = *(u8 *)(r1 + 0); if r7 < 10 goto l_%=;") + /* * No exit of the inner loop dominates its backedges: may_goto is added before its header. The prog * holds a reference, so may_goto goes to the exit of the outer loop: an edge that the verifier -- 2.55.0 From: Alexei Starovoitov A removal: sync_linked_regs(), the code that scalar ids exist for, goes. The ids themselves go in later patches. Precision is lost where only a state knows that two scalars are linked: across frames, and on one of several paths. One selftest has to change. After a conditional jump sync_linked_regs() copies the range of the tested register, plus delta, to all scalars with its id. refine_linked_regs() does that job without ids. It refines the linked registers of the jump: the live registers and stack slots that are a tested register + a constant on every path to the jump. Scalar evolution computes them before the main pass. There is no limit of 5 registers, and a second '+=' does not drop the link. Both run at every jump, sync_linked_regs() first. Remove it, collect_linked_regs() and struct linked_regs. Scalar ids are still assigned, no jump reads them. Precision that is lost: - Linked registers belong to the frame of the jump. Scalar ids also linked registers and stack slots of the callers. A subprog that tests its argument refined the register that the caller keeps. A field of a struct on the stack of the caller that a subprog loads, tests and loads again was refined in the slot. - That two arguments hold the same value is known at the callsite, not in the subprog. The selftest "regalloc in callee" passes the same value in r1 and r2 and is rejected now. Its 'r2 = r0' before the call becomes 'r2 = r1' in the callee. - 'r2 = r1' on one of two paths to a jump. A link holds on every path, so r2 is not linked. The id worked until the state of that path was joined with the other one. The first iteration of a loop is such a path. - refine_linked_regs() runs for loaders with CAP_BPF, sync_linked_regs() ran for everybody. No other prog of the selftests, Cilium, sched_ext, katran, bpfjailer and of the programs in Rust changes its verdict. Precision that is gained: sync_linked_regs() overwrites ranges, refine_linked_regs() intersects them. With n in [0, 19] and 'u64 ent[16]' clang emits for 'n = n > 4 ? n - 4 : 0; e->ent[n] = 1;': 10: w2 = w6 11: w2 += -4 14: if w2 > w6 goto pc+1 If the jump is not taken w2 did not wrap: it is at most 19. r2 and r6 have the same id, so sync_linked_regs() overwrites r2 with r6 - 4 modulo 2^32, which can be 0xfffffffc: R0 unbounded memory access, make sure to bounds check any such access With refine_linked_regs() alone r2 is in [0, 15] and the prog loads. The selftest linked_regs_too_many_regs sees all six registers refined. Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 161 ------------------ .../bpf/progs/verifier_array_access.c | 2 +- .../selftests/bpf/progs/verifier_regalloc.c | 2 +- .../selftests/bpf/progs/verifier_scalar_ids.c | 13 +- 4 files changed, 5 insertions(+), 173 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 970211dfe38c..103e83cd23ef 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3578,30 +3578,6 @@ static void mark_non_stack_access(struct bpf_verifier_env *env, int idx) env->insn_aux_data[idx].non_stack_access = true; } -#define LINKED_REGS_MAX 5 - -struct linked_reg { - u8 frameno; - union { - u16 spi; - u16 regno; - }; - bool is_reg; -}; - -struct linked_regs { - int cnt; - struct linked_reg entries[LINKED_REGS_MAX]; -}; - -static struct linked_reg *linked_regs_push(struct linked_regs *s) -{ - if (s->cnt < LINKED_REGS_MAX) - return &s->entries[s->cnt++]; - - return NULL; -} - const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn) { const struct btf_type *func; @@ -19629,119 +19605,6 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn, return true; } -static void __collect_linked_regs(struct linked_regs *reg_set, struct bpf_reg_state *reg, - u32 id, u32 frameno, u32 spi_or_reg, bool is_reg) -{ - struct linked_reg *e; - - if (reg->type != SCALAR_VALUE || (reg->id & ~BPF_ADD_CONST) != id) - return; - - e = linked_regs_push(reg_set); - if (e) { - e->frameno = frameno; - e->is_reg = is_reg; - e->regno = spi_or_reg; - } else { - clear_scalar_id(reg); - } -} - -/* For all R being scalar registers or spilled scalar registers - * in verifier state, save R in linked_regs if R->id == id. - * If there are too many Rs sharing same id, reset id for leftover Rs. - */ -static void collect_linked_regs(struct bpf_verifier_env *env, - struct bpf_verifier_state *vstate, - u32 id, - struct linked_regs *linked_regs) -{ - struct bpf_insn_aux_data *aux = env->insn_aux_data; - struct bpf_func_state *func; - struct bpf_reg_state *reg; - u16 live_regs; - int i, j; - - id = id & ~BPF_ADD_CONST; - for (i = vstate->curframe; i >= 0; i--) { - live_regs = aux[bpf_frame_insn_idx(vstate, i)].live_regs_before; - func = vstate->frame[i]; - for (j = 0; j < BPF_REG_FP; j++) { - if (!(live_regs & BIT(j))) - continue; - reg = &func->regs[j]; - __collect_linked_regs(linked_regs, reg, id, i, j, true); - } - for (j = 0; j < bpf_stack_nr_slots(func); j++) { - struct bpf_stack_state *ss = bpf_stack_slot(func, j); - - if (!bpf_is_spilled_reg(ss)) - continue; - reg = &ss->spilled_ptr; - __collect_linked_regs(linked_regs, reg, id, i, j, false); - } - } -} - -/* For all R in linked_regs, copy known_reg range into R - * if R->id == known_reg->id. - */ -static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *vstate, - struct bpf_reg_state *known_reg, struct linked_regs *linked_regs) -{ - struct bpf_reg_state fake_reg; - struct bpf_reg_state *reg; - struct linked_reg *e; - int i; - - for (i = 0; i < linked_regs->cnt; ++i) { - e = &linked_regs->entries[i]; - reg = e->is_reg ? &vstate->frame[e->frameno]->regs[e->regno] - : &bpf_stack_slot(vstate->frame[e->frameno], e->spi)->spilled_ptr; - if (reg->type != SCALAR_VALUE || reg == known_reg) - continue; - if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST)) - continue; - /* - * Skip mixed 32/64-bit links: the delta relationship doesn't - * hold across different ALU widths. - */ - if (((reg->id ^ known_reg->id) & BPF_ADD_CONST) == BPF_ADD_CONST) - continue; - if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) || - reg->delta == known_reg->delta) { - *reg = *known_reg; - } else { - s32 saved_off = reg->delta; - u32 saved_id = reg->id; - - fake_reg.type = SCALAR_VALUE; - __mark_reg_known(&fake_reg, (s64)reg->delta - (s64)known_reg->delta); - - /* reg = known_reg; reg += delta */ - *reg = *known_reg; - /* - * Must preserve off and id, otherwise another sync_linked_regs() - * will be incorrect. - */ - reg->delta = saved_off; - reg->id = saved_id; - - scalar_step_add(reg, reg, &fake_reg); - scalar32_min_max_add(reg, &fake_reg); - scalar_min_max_add(reg, &fake_reg); - reg->var_off = tnum_add(reg->var_off, fake_reg.var_off); - if ((reg->id | known_reg->id) & BPF_ADD_CONST32) - zext_32_to_64(reg); - reg_bounds_sync(reg); - } - if (e->is_reg) - mark_reg_scratched(env, e->regno); - else - mark_stack_slot_scratched(env, e->spi); - } -} - /* a == b (mod 2^N) implies a == b if both fit in uN or both fit in sN. N is @bits. */ static bool regs_fit_in_bits(const struct bpf_reg_state *a, const struct bpf_reg_state *b, u32 bits) @@ -19886,7 +19749,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; struct bpf_reg_state *eq_branch_regs; - struct linked_regs linked_regs = {}; u8 opcode = BPF_OP(insn->code); bool is_jmp32; int pred = -1; @@ -19955,16 +19817,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, __mark_reg_known(src_reg, insn->imm); } - /* - * Collect the linked registers before env->{true,false}_reg{1,2} setup, - * otherwise ids dropped by collect_linked_regs() would be resurrected - * when env->{true,false}_reg{1,2} are copied back. - */ - if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id) - collect_linked_regs(env, this_branch, src_reg->id, &linked_regs); - if (dst_reg->type == SCALAR_VALUE && dst_reg->id) - collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); - is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; env->false_reg1 = *dst_reg; env->false_reg2 = *src_reg; @@ -20018,19 +19870,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, if (BPF_SRC(insn->code) == BPF_X) other_branch_regs[insn->src_reg] = env->true_reg2; - if (BPF_SRC(insn->code) == BPF_X && - src_reg->type == SCALAR_VALUE && src_reg->id && - !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { - sync_linked_regs(env, this_branch, src_reg, &linked_regs); - sync_linked_regs(env, other_branch, &other_branch_regs[insn->src_reg], - &linked_regs); - } - if (dst_reg->type == SCALAR_VALUE && dst_reg->id && - !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { - sync_linked_regs(env, this_branch, dst_reg, &linked_regs); - sync_linked_regs(env, other_branch, &other_branch_regs[insn->dst_reg], - &linked_regs); - } if (env->bpf_capable) { if (BPF_SRC(insn->code) == BPF_X) { refine_linked_regs(env, this_branch, *insn_idx, insn->src_reg); diff --git a/tools/testing/selftests/bpf/progs/verifier_array_access.c b/tools/testing/selftests/bpf/progs/verifier_array_access.c index 7f98822889ca..29bed1e496af 100644 --- a/tools/testing/selftests/bpf/progs/verifier_array_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_array_access.c @@ -762,7 +762,7 @@ l0_%=: *(u32*)(r10 - 8) = r6; \ : __clobber_all); } -/* The slot has the id of r6, so the compare sets the range of the slot too */ +/* The slot holds the low 32 bits of r6, so the compare refines the slot too */ SEC("socket") __description("valid map access into an array using a key that is compared after the spill") __success __retval(0) diff --git a/tools/testing/selftests/bpf/progs/verifier_regalloc.c b/tools/testing/selftests/bpf/progs/verifier_regalloc.c index ee5ddea87c91..cf26617b6ba1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_regalloc.c +++ b/tools/testing/selftests/bpf/progs/verifier_regalloc.c @@ -303,7 +303,6 @@ __naked void regalloc_in_callee(void) r7 = r0; \ call %[bpf_get_prandom_u32]; \ r1 = r0; \ - r2 = r0; \ r3 = r7; \ call regalloc_in_callee__1; \ l0_%=: exit; \ @@ -318,6 +317,7 @@ static __naked __noinline __attribute__((used)) void regalloc_in_callee__1(void) { asm volatile (" \ + r2 = r1; \ if r1 s> 20 goto l0_%=; \ if r2 s< 0 goto l0_%=; \ r3 += r1; \ diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c index 819d0a1a6886..e22120e5858f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c +++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c @@ -285,13 +285,8 @@ __naked void precision_two_ids(void) SEC("socket") __success __log_level(2) -/* - * check that r5 is unlinked after 'if', collect_linked_regs() can't tie - * more than 5 registers for a single insn and the register compared by - * the jump is not exempt from that. The jump refines the copies all the same. - */ -__msg("7: (25) if r5 > 0x7 goto pc+0 ; R0=" LINKED("1", 7, 0x7) " R1=" LINKED("1", 7, 0x7) " R2=" LINKED("1", 7, 0x7) " R3=" LINKED("1", 7, 0x7) " R4=" LINKED("1", 7, 0x7) " R5=scalar(smin=") -__msg("12: (bf) r5 = r5 ; R5=scalar(id=2") +/* there is no limit of 5 registers for a single insn */ +__msg("7: (25) if r5 > 0x7 goto pc+0 ; R0=" LINKED("1", 7, 0x7) " R1=" LINKED("1", 7, 0x7) " R2=" LINKED("1", 7, 0x7) " R3=" LINKED("1", 7, 0x7) " R4=" LINKED("1", 7, 0x7) " R5=" LINKED("1", 7, 0x7)) __naked void linked_regs_too_many_regs(void) { asm volatile ( @@ -304,14 +299,12 @@ __naked void linked_regs_too_many_regs(void) "r3 = r0;" "r4 = r0;" "r5 = r0;" - /* r{0-4} fill the record, r5 does not fit and is unlinked */ "if r5 > 7 goto +0;" - /* keep r{1-4} live */ + /* keep r{1-5} live */ "r1 = r1;" "r2 = r2;" "r3 = r3;" "r4 = r4;" - /* make r5 appear in the log */ "r5 = r5;" "r7 = r10;" "r7 += r0;" -- 2.55.0 From: Alexei Starovoitov A prerequisite of the removal of scalar ids. flow_join_makes_up() is the last code that learns something about the prog from them: which registers and stack slots hold copies of a scalar. Recognize the copies by their ranges. The function is a heuristic that decides which states are joined. So this patch is about the number of states. Soundness does not depend on it. err = update(); ret = err < 0 ? DROP : 0; .. if (err < 0) goto out; .. if (ret < 0) .. The states (err >= 0, ret = 0) and (err < 0, ret = DROP) get to a join point before the jumps. Their join loses the correlation of err and ret: it has err < 0 with ret = 0. So two states stay apart if two scalars that later jumps test each have disjoint ranges in the two states, as err and ret do. Copies of a scalar are treated as one scalar there. A jump that tests one of them refines the others, so their join makes up nothing, and compilers keep a value in a register and in stack slots at the same time. Copies are recognized by their id, in one of the two states: if (one && (!id || id != (ra->id & ~BPF_ADD_CONST))) return true; If every copy is a scalar of its own, too many states stay apart. The init prog of a sched_ext scheduler in Rust goes from 36 to 512 states at one insn and takes twice as long to load. At 512 flow_has_room() is false, and states are joined by force. Copies have the same range and tnum, in each of the two states. Treat scalars with that property as copies. Two scalars that are not copies can have it too. Then two states are joined that stayed apart. veristat over the selftests, Cilium, sched_ext, katran, bpfjailer and programs in Rust: no verdict changes, 199 of 32913 progs move, by 0.01 % of the insns in all. The scheduler stays at 36 states. Signed-off-by: Alexei Starovoitov --- kernel/bpf/flow.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c index c9458b3ad114..778288acb5ba 100644 --- a/kernel/bpf/flow.c +++ b/kernel/bpf/flow.c @@ -2603,10 +2603,9 @@ static bool flow_join_makes_up(const struct bpf_flow *flow, struct bpf_verifier_ struct bpf_verifier_state *b) { struct bpf_func_state *fa = a->frame[a->curframe], *fb = b->frame[b->curframe]; - const struct bpf_reg_state *ra, *rb; + const struct bpf_reg_state *ra, *rb, *one_a = NULL, *one_b = NULL; const struct flow_need *tested = flow->need[FLOW_NEED_JUMP], *need; - u32 fr, i, id = 0; - bool one = false; + u32 fr, i, len = offsetof(struct bpf_reg_state, id); if (!tested) return false; @@ -2623,11 +2622,11 @@ static bool flow_join_makes_up(const struct bpf_flow *flow, struct bpf_verifier_ continue; if (flow_gap(ra, rb)) return true; - /* copies of a number are one number */ - if (one && (!id || id != (ra->id & ~BPF_ADD_CONST))) + /* copies of a scalar count once: same range and tnum in each state */ + if (one_a && (memcmp(ra, one_a, len) || memcmp(rb, one_b, len))) return true; - one = true; - id = ra->id & ~BPF_ADD_CONST; + one_a = ra; + one_b = rb; } } if (!flow_needed_consts_differ(flow, FLOW_NEED_FLAG, a, b)) -- 2.55.0 From: Alexei Starovoitov A removal. Nothing learns anything about a prog from a scalar id any more. The code that is left only maintains the ids. Stop assigning them. No verdict changes, the number of states goes down. r2 = r1 r1 and r2 get id 7 r2 += 4 id 7 | BPF_ADD_CONST64, delta 4 *(u64 *)(r10 - 8) = r1 fp-8 has id 7 if r1 > 100 goto .. all three were refined A scalar id in a state said that scalars hold the same value, or differ by delta. Each insn that moves or changes a scalar had to assign, keep or clear it. regsafe() and the join of states had to map the ids of one state to those of the other. What does that job now: - After a conditional jump refine_linked_regs() refines the linked registers of the jump: the live registers and stack slots that are a tested register + a constant on every path to the jump. Scalar evolution computes them before the main pass. - Scalar evolution compares expressions to find a copy of an induction variable: const_diff_at_header(). - flow_join_makes_up() recognizes copies of a scalar by their ranges. Remove assign_scalar_id_before_mov(), BPF_ADD_CONST and delta in adjust_reg_min_max_vals(), the ids that a spill and a fill keep or clear, the callers of clear_scalar_id(), bpf_clear_singular_ids() and its caller. get_reg_width() has no user left. The ids of pointers, references, dynptrs and iterators stay. States that differed in scalar ids only are one state now. veristat over the selftests, Cilium, sched_ext, katran, bpfjailer and programs in Rust: 2.3 % fewer insns and 3.7 % fewer states, in Rust 5.3 % and 6.3 %. No verdict changes. The kernel spends 0.7 % fewer instructions on loading the C programs and 4.9 % fewer on those in Rust. Tests that change: - Log lines that pin 'id=' of a scalar lose it. The id of a packet pointer is smaller: ids are assigned less often. - two_old_ids_one_cur_id maps the scalar ids of two states: removed. Signed-off-by: Alexei Starovoitov --- kernel/bpf/flow.c | 2 - kernel/bpf/scev.c | 8 +- kernel/bpf/verifier.c | 247 +----------------- .../selftests/bpf/progs/verifier_bounds.c | 2 +- .../bpf/progs/verifier_bounds_step.c | 6 +- .../bpf/progs/verifier_direct_packet_access.c | 2 +- .../selftests/bpf/progs/verifier_scalar_ids.c | 55 +--- 7 files changed, 19 insertions(+), 303 deletions(-) diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c index 778288acb5ba..abbadba1e79f 100644 --- a/kernel/bpf/flow.c +++ b/kernel/bpf/flow.c @@ -2924,8 +2924,6 @@ static struct flow_state *flow_arrive(struct bpf_verifier_env *env, struct bpf_v flow_free_state(env, s); return ERR_PTR(err); } - /* when the registers that it was shared with are dead, or are those of the caller */ - bpf_clear_singular_ids(env, &s->st); return s; } diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index db60585c00c8..34abb4ab0c46 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -3940,7 +3940,7 @@ static bool is_unbound_scev_reg(struct bpf_verifier_env *env, const struct bpf_f { const struct bpf_reg_state *reg = scev_reg_for_read(env, st, r); - return reg && reg->type == SCALAR_VALUE && !reg->id && reg->step <= 1 && + return reg && reg->type == SCALAR_VALUE && reg->step <= 1 && tnum_is_unknown(reg->var_off) && reg->r64.size == U64_MAX && reg->r32.size == U32_MAX; } @@ -4026,9 +4026,6 @@ bool bpf_is_widenable_reg_type(const struct bpf_reg_state *reg) static void scratch_widened_reg_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { switch (base_type(reg->type)) { - case SCALAR_VALUE: - clear_scalar_id(reg); - break; case PTR_TO_PACKET: case PTR_TO_PACKET_META: reg->id = ++env->id_gen; @@ -4746,7 +4743,7 @@ int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state if (back && (!val || !entry_val || !bpf_reg_union_compatible(val, entry_val))) continue; if (test_bit(r, iters->mask_regs)) { - /* The range stays. It's no longer equal to the registers with its id. */ + /* the range stays */ err = scev_reg_for_write(env, cur_func, r, ®); if (!err && back) err = bpf_reg_union(env, reg, entry_val); @@ -4758,7 +4755,6 @@ int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state bpf_log(log, " stays in 0..%llu, it loses bits only\n", reg_umax(reg)); } - scratch_widened_reg_id(env, reg); mark_scev_reg_scratched(env, r); continue; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 103e83cd23ef..3d1be990123b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3634,27 +3634,6 @@ static bool __is_pointer_value(bool allow_ptr_leaks, return is_pointer_regtype(reg->type); } -static void assign_scalar_id_before_mov(struct bpf_verifier_env *env, - struct bpf_reg_state *src_reg) -{ - if (src_reg->type != SCALAR_VALUE) - return; - /* - * The verifier is processing rX = rY insn and - * rY->id has special linked register already. - * Cleared it, since multiple rX += const are not supported. - */ - if (src_reg->id & BPF_ADD_CONST) - clear_scalar_id(src_reg); - /* - * Ensure that src_reg has a valid ID that will be copied to - * dst_reg and then will be used by sync_linked_regs() to - * propagate min/max range. - */ - if (!src_reg->id && !tnum_is_const(src_reg->var_off)) - src_reg->id = ++env->id_gen; -} - static void save_register_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi, struct bpf_reg_state *reg, @@ -3681,11 +3660,6 @@ static bool is_bpf_st_mem(struct bpf_insn *insn) return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM; } -static int get_reg_width(struct bpf_reg_state *reg) -{ - return fls64(reg_umax(reg)); -} - /* See comment for mark_fastcall_pattern_for_call() */ static void check_fastcall_stack_contract(struct bpf_verifier_env *env, struct bpf_func_state *state, int insn_idx, int off) @@ -3786,16 +3760,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, mark_stack_slot_scratched(env, spi); if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) { - bool reg_value_fits; - - reg_value_fits = get_reg_width(reg) <= BITS_PER_BYTE * size; - /* Make sure that reg had an ID to build a relation on spill. */ - if (reg_value_fits) - assign_scalar_id_before_mov(env, reg); save_register_state(env, state, spi, reg, size); - /* Break the relation on a narrowing spill. */ - if (!reg_value_fits) - clear_scalar_id(&ss->spilled_ptr); } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) && env->bpf_capable) { struct bpf_reg_state *tmp_reg = &env->fake_reg[0]; @@ -4098,19 +4063,8 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, if (dst_regno < 0) return 0; - if (env->bpf_capable && size == 4 && spill_size == 4 && - get_reg_width(reg) <= 32) - /* Ensure stack slot has an ID to build a relation - * with the destination register on fill. - */ - assign_scalar_id_before_mov(env, reg); + /* coerce_reg_to_size will adjust the boundaries. */ state->regs[dst_regno] = *reg; - - /* Break the relation on a narrowing fill. - * coerce_reg_to_size will adjust the boundaries. - */ - if (get_reg_width(reg) > size * BITS_PER_BYTE) - clear_scalar_id(&state->regs[dst_regno]); } else { int spill_cnt = 0, zero_cnt = 0; @@ -4155,11 +4109,6 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } } else if (dst_regno >= 0) { /* restore register state from stack */ - if (env->bpf_capable) - /* Ensure stack slot has an ID to build a relation - * with the destination register on fill. - */ - assign_scalar_id_before_mov(env, reg); state->regs[dst_regno] = *reg; } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { /* If dst_regno==-1, the caller is asking us whether @@ -7718,18 +7667,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b /* b/h/w load zero-extends, mark upper bits as known 0 */ coerce_reg_to_size(®s[value_regno], size); } else { - /* - * Sign-extension can change the register value relative - * to a scalar it is linked with by id (e.g. a zero- - * extending fill of the same spilled stack slot), thus - * drop the shared id in that case. - */ - bool no_sext = reg_umax(®s[value_regno]) < - (1ULL << (size * BITS_PER_BYTE - 1)); - coerce_reg_to_size_sx(®s[value_regno], size); - if (!no_sext) - clear_scalar_id(®s[value_regno]); } } @@ -11772,16 +11710,11 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i callee->entry_rcu_locks = entry.active_rcu_locks; callee->entry_preempt_locks = entry.active_preempt_locks; for (i = BPF_REG_1; i <= BPF_REG_5; i++) { - /* what a scalar is linked to stays with the caller, so does the delta to it */ - if (callee->regs[i].type == SCALAR_VALUE) - clear_scalar_id(&callee->regs[i]); callee->regs[i].id = inherited_id(state, callee->regs[i].id); callee->regs[i].parent_id = inherited_id(state, callee->regs[i].parent_id); callee->regs[i].map_uid = inherited_id(state, callee->regs[i].map_uid); } for (i = 0; i < callee->out_stack_arg_cnt; i++) { - if (callee->stack_arg_regs[i].type == SCALAR_VALUE) - clear_scalar_id(&callee->stack_arg_regs[i]); callee->stack_arg_regs[i].id = inherited_id(state, callee->stack_arg_regs[i].id); callee->stack_arg_regs[i].parent_id = inherited_id(state, callee->stack_arg_regs[i].parent_id); @@ -11827,19 +11760,18 @@ static int exit_func_on_its_own(struct bpf_verifier_env *env) static u32 fresh_id(struct bpf_verifier_env *env, struct bpf_idmap *idmap, u32 id) { - u32 flags = id & BPF_ADD_CONST, i; + u32 i; - id &= ~BPF_ADD_CONST; if (!id) return 0; for (i = 0; i < idmap->cnt; i++) if (idmap->map[i].old == id) - return idmap->map[i].cur | flags; + return idmap->map[i].cur; if (idmap->cnt == BPF_ID_MAP_SIZE) return 0; idmap->map[idmap->cnt].old = id; idmap->map[idmap->cnt].cur = ++env->id_gen; - return idmap->map[idmap->cnt++].cur | flags; + return idmap->map[idmap->cnt++].cur; } /* @@ -17909,13 +17841,6 @@ static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *ins /* Apply bswap if alu64 or switch between big-endian and little-endian machines */ bool need_bswap = alu64 || (to_le == is_big_endian); - /* - * If the register is mutated, manually reset its scalar ID to break - * any existing ties and avoid incorrect bounds propagation. - */ - if (need_bswap || insn->imm == 16 || insn->imm == 32) - clear_scalar_id(dst_reg); - if (need_bswap) { if (insn->imm == 16) dst_reg->var_off = tnum_bswap16(dst_reg->var_off); @@ -18016,7 +17941,6 @@ static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *ins * like any other. At this insn the branch is a part of the state that is looked at: * it must not be compared with it. */ - clear_scalar_id(reg); return adjust_scalar_min_max_vals(env, insn, reg, *src_reg); } @@ -18226,10 +18150,8 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; - bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); struct bpf_insn_aux_data *aux = cur_aux(env); u8 opcode = BPF_OP(insn->code); - int err; dst_reg = ®s[insn->dst_reg]; if (BPF_SRC(insn->code) == BPF_X) @@ -18336,69 +18258,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, verbose(env, "verifier internal error: no src_reg\n"); return -EFAULT; } - /* - * For alu32 linked register tracking, we need to check dst_reg's - * umax_value before the ALU operation. After adjust_scalar_min_max_vals(), - * alu32 ops will have zero-extended the result, making umax_value <= U32_MAX. - */ - u64 dst_umax = reg_umax(dst_reg); - - err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); - if (err) - return err; - /* - * Compilers can generate the code - * r1 = r2 - * r1 += 0x1 - * if r2 < 1000 goto ... - * use r1 in memory access - * So remember constant delta between r2 and r1 and update r1 after - * 'if' condition. - */ - if (env->bpf_capable && - (BPF_OP(insn->code) == BPF_ADD || BPF_OP(insn->code) == BPF_SUB) && - dst_reg->id && is_reg_const(src_reg, alu32) && - !(BPF_SRC(insn->code) == BPF_X && insn->src_reg == insn->dst_reg)) { - u64 val = reg_const_value(src_reg, alu32); - s32 off; - - if (!alu32 && ((s64)val < S32_MIN || (s64)val > S32_MAX)) - goto clear_id; - - if (alu32 && (dst_umax > U32_MAX)) - goto clear_id; - - off = (s32)val; - - if (BPF_OP(insn->code) == BPF_SUB) { - /* Negating S32_MIN would overflow */ - if (off == S32_MIN) - goto clear_id; - off = -off; - } - - if (dst_reg->id & BPF_ADD_CONST) { - /* - * If the register already went through rX += val - * we cannot accumulate another val into rx->off. - */ -clear_id: - clear_scalar_id(dst_reg); - } else { - if (alu32) - dst_reg->id |= BPF_ADD_CONST32; - else - dst_reg->id |= BPF_ADD_CONST64; - dst_reg->delta = off; - } - } else { - /* - * Make sure ID is cleared otherwise dst_reg min/max could be - * incorrectly propagated into other registers by sync_linked_regs() - */ - clear_scalar_id(dst_reg); - } - return 0; + return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); } /* @@ -18612,7 +18472,6 @@ void bpf_reg_union_range(struct bpf_reg_state *acc, const struct bpf_reg_state * /* * Extends acc's scalar range to include src's scalar range. * Caller must ensure that other kinds of metadata is compatible. - * Caller must clear acc's scalar ID. */ int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc, const struct bpf_reg_state *src) @@ -18753,7 +18612,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) /* case: R1 = R2 * copy register state to dest reg */ - assign_scalar_id_before_mov(env, src_reg); *dst_reg = *src_reg; } else { /* case: R1 = (s8, s16 s32)R2 */ @@ -18763,14 +18621,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) insn->src_reg); return -EACCES; } else if (src_reg->type == SCALAR_VALUE) { - bool no_sext; - - no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1)); - if (no_sext) - assign_scalar_id_before_mov(env, src_reg); *dst_reg = *src_reg; - if (!no_sext) - clear_scalar_id(dst_reg); coerce_reg_to_size_sx(dst_reg, insn->off >> 3); } else { mark_reg_unknown(env, regs, insn->dst_reg); @@ -18784,29 +18635,10 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) insn->src_reg); return -EACCES; } else if (src_reg->type == SCALAR_VALUE) { - if (insn->off == 0) { - bool is_src_reg_u32 = get_reg_width(src_reg) <= 32; - - if (is_src_reg_u32) - assign_scalar_id_before_mov(env, src_reg); - *dst_reg = *src_reg; - /* Make sure ID is cleared if src_reg is not in u32 - * range otherwise dst_reg min/max could be incorrectly - * propagated into src_reg by sync_linked_regs() - */ - if (!is_src_reg_u32) - clear_scalar_id(dst_reg); - } else { - /* case: W1 = (s8, s16)W2 */ - bool no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1)); - - if (no_sext) - assign_scalar_id_before_mov(env, src_reg); - *dst_reg = *src_reg; - if (!no_sext) - clear_scalar_id(dst_reg); + *dst_reg = *src_reg; + /* case: W1 = (s8, s16)W2 */ + if (insn->off) coerce_subreg_to_size_sx(dst_reg, insn->off >> 3); - } } else { mark_reg_unknown(env, regs, insn->dst_reg); @@ -20730,69 +20562,6 @@ static void adjust_btf_func(struct bpf_verifier_env *env) aux->func_info[i].insn_off = env->subprog_info[i].start; } -/* Find id in idset and increment its count, or add new entry */ -static void idset_cnt_inc(struct bpf_idset *idset, u32 id) -{ - u32 i; - - for (i = 0; i < idset->num_ids; i++) { - if (idset->entries[i].id == id) { - idset->entries[i].cnt++; - return; - } - } - /* New id */ - if (idset->num_ids < BPF_ID_MAP_SIZE) { - idset->entries[idset->num_ids].id = id; - idset->entries[idset->num_ids].cnt = 1; - idset->num_ids++; - } -} - -/* Find id in idset and return its count, or 0 if not found */ -static u32 idset_cnt_get(struct bpf_idset *idset, u32 id) -{ - u32 i; - - for (i = 0; i < idset->num_ids; i++) { - if (idset->entries[i].id == id) - return idset->entries[i].cnt; - } - return 0; -} - -/* - * Clear singular scalar ids in a state. - * A register with a non-zero id is called singular if no other register shares - * the same base id. Such registers can be treated as independent (id=0). - */ -void bpf_clear_singular_ids(struct bpf_verifier_env *env, - struct bpf_verifier_state *st) -{ - struct bpf_idset *idset = &env->idset_scratch; - struct bpf_func_state *func; - struct bpf_reg_state *reg; - - idset->num_ids = 0; - - bpf_for_each_reg_in_vstate(st, func, reg, ({ - if (reg->type != SCALAR_VALUE) - continue; - if (!reg->id) - continue; - idset_cnt_inc(idset, reg->id & ~BPF_ADD_CONST); - })); - - bpf_for_each_reg_in_vstate(st, func, reg, ({ - if (reg->type != SCALAR_VALUE) - continue; - if (!reg->id) - continue; - if (idset_cnt_get(idset, reg->id & ~BPF_ADD_CONST) == 1) - clear_scalar_id(reg); - })); -} - /* Return true if it's OK to have the same insn return a different type. */ static bool reg_type_mismatch_ok(enum bpf_reg_type type) { diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index 94d52154b0c6..b78e7fc96fa7 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -1476,7 +1476,7 @@ __naked void sub64_full_overflow(void) SEC("socket") __description("64-bit subtraction, partial overflow, result in unbounded reg") __success __log_level(2) -__msg("3: (1f) r3 -= r2 {{.*}} R3=scalar(id=1-1)") +__msg("3: (1f) r3 -= r2 {{.*}} R3=scalar()") __retval(0) __naked void sub64_partial_overflow(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c index 51b703058cf7..b42717a825c6 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c @@ -299,7 +299,7 @@ SEC("socket") __success __log_level(2) __msg("r0 += 8 {{.*}}smin=umin=smin32=umin32=8,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") __msg("from 4 to 6: R0=scalar(smin=umin=smin32=umin32=32,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") -__msg("r1 = r0 {{.*}}R1=scalar(id={{[0-9]+}},smin=umin=smin32=umin32=104,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") +__msg("r1 = r0 {{.*}}R1=scalar(smin=umin=smin32=umin32=104,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") __msg("r1 += 24 {{.*}}R1=scalar({{.*}}smin=umin=smin32=umin32=128,smax=umax=smax32=umax32=200,{{.*}}step=8+24)") __naked void step_bounds_resync_after_arith(void) { @@ -336,10 +336,10 @@ l_out_%=: \ */ SEC("socket") __success __log_level(2) -__msg("r1 = (s16)r0 {{.*}}R1=scalar(id={{[0-9]+}},smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)") +__msg("r1 = (s16)r0 {{.*}}R1=scalar(smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)") __msg("r2 = (s16)r0 {{.*}}R2=scalar(smin=smin32=-24,smax=smax32=144,{{.*}}step=0+24)") __msg("r3 = (s8)r0 {{.*}}R3=scalar(smin=smin32=-128,smax=smax32=127)") -__msg("w4 = (s16)w1 {{.*}}R4=scalar(id={{[0-9]+}},smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)") +__msg("w4 = (s16)w1 {{.*}}R4=scalar(smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)") __msg("w5 = (s8)w1 {{.*}}R5=scalar({{.*}}smin32=-128,smax32=127,var_off=(0x0; 0xffffffff))") __naked void step_movsx_bounds_on_line(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c index 6c1d2014b95b..212c08a27058 100644 --- a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c @@ -569,7 +569,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("direct packet access: test23 (x += pkt_ptr, 4)") -__failure __msg("invalid access to packet, off=31 size=8, R5(id=3,off=31,r=0)") +__failure __msg("invalid access to packet, off=31 size=8, R5(id=1,off=31,r=0)") __flag(BPF_F_ANY_ALIGNMENT) __naked void test23_x_pkt_ptr_4(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c index e22120e5858f..56c2412608a5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c +++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c @@ -11,11 +11,10 @@ struct { __type(value, long long); } map_hash_8b SEC(".maps"); -/* scalar in [0, max] that shares id with other registers, as it is in the log */ -#define LINKED(id, max, mask) \ - "scalar(id=" id ",smin=smin32=0,smax=umax=smax32=umax32=" #max ",var_off=(0x0; " #mask "))" +/* scalar in [0, max], as it is in the log. @id: the registers that hold copies of one value. */ #define ALONE(max, mask) \ "scalar(smin=smin32=0,smax=umax=smax32=umax32=" #max ",var_off=(0x0; " #mask "))" +#define LINKED(id, max, mask) ALONE(max, mask) /* * Registers r{0,1,2} have the same scalar ID. @@ -151,7 +150,7 @@ __naked void linked_regs_broken_link(void) SEC("socket") __success __log_level(2) __msg("13: (25) if r1 > 0x7 goto pc+0 ; R1=" ALONE(7, 0x7)) -__msg("10: R0=0 R6=scalar(id={{.*}}umax32=255,{{.*}} R7=scalar(id={{.*}}umax32=255,") +__msg("10: R0=0 R6=scalar({{.*}}umax32=255,{{.*}} R7=scalar({{.*}}umax32=255,") __msg("5: R0=0 R6=" ALONE(255, 0xff)) __naked void precision_many_frames(void) { @@ -317,7 +316,7 @@ __naked void linked_regs_too_many_regs(void) SEC("socket") __failure __log_level(2) -__msg("5: (3d) if r8 >= r0 goto pc+3 ; R0=scalar(umin=1) R7=scalar(id=1,umax=0xfffffffffffffffe) R8=scalar(id=1,umax=0xfffffffffffffffe)") +__msg("5: (3d) if r8 >= r0 goto pc+3 ; R0=scalar(umin=1) R7=scalar(umax=0xfffffffffffffffe) R8=scalar(umax=0xfffffffffffffffe)") __msg("div by zero") __naked void linked_regs_broken_link_2(void) { @@ -680,52 +679,6 @@ __naked void two_nil_old_ids_one_cur_id(void) : __clobber_all); } -/* - * Check that two different scalar IDs in the state that is there can't be - * mapped to the same scalar ID in the state that arrives. - */ -SEC("socket") -__success __log_level(2) -__not_msg("from 10 to 11: within") -__msg("11: R0=0 R6=" ALONE(255, 0xff) " R7=" ALONE(255, 0xff)) -__naked void two_old_ids_one_cur_id(void) -{ - asm volatile ( - /* Give unique scalar IDs to r{6,7} */ - "call %[bpf_ktime_get_ns];" - "r0 &= 0xff;" - "r6 = r0;" - "r8 = r0;" - "call %[bpf_ktime_get_ns];" - "r0 &= 0xff;" - "r7 = r0;" - "r9 = r0;" - "r0 = 0;" - /* Maybe make r{6,7} IDs identical */ - "if r6 <= r7 goto l1_%=;" - "r6 = r7;" -"l1_%=:" - /* - * Get here in two states: - * - first: r6{.id=A}, r7{.id=B} - * - second: r6{.id=B}, r7{.id=B} - * Currently we don't want to consider the second one to be within - * the first one. Thus they are joined. - */ - "r2 = r10;" - "r2 += r6;" - "r2 += r7;" - /* - * keep r8 and r9 live, otherwise r6->id and r7->id - * will become singular and reset to zero before if r6 <= r7 - */ - "r9 += r8;" - "exit;" - : - : __imm(bpf_ktime_get_ns) - : __clobber_all); -} - SEC("socket") /* Note the flag, see verifier.c:opt_subreg_zext_lo32_rnd_hi32() */ __flag(BPF_F_TEST_RND_HI32) -- 2.55.0 From: Alexei Starovoitov Removal of dead code. Scalars have no ids any more, so every check of a scalar id in regsafe() passes. No prog is verified differently. 4: r6 = r7 5: if r6 > 10 goto .. 6: use r7 While a jump refined all scalars with the id of the tested register, a state in which r6 and r7 have the same id was stronger than a state with the same ranges and two ids: in the first one insn 5 bounds r7. So regsafe() did not stop at the ranges. check_scalar_ids() mapped the ids of the old state to those of the current one, an id that the current state did not have got a temporary one, BPF_ADD_CONST and delta had to be equal. Now no state says that two scalars hold the same value. What insn 5 refines are its linked registers. Scalar evolution computes them before the main pass, and a link holds on every path to insn 5, whatever state gets there. For two scalars range_within() and tnum_in() are enough. Remove check_scalar_ids() and the table of cases in regsafe(). Signed-off-by: Alexei Starovoitov --- kernel/bpf/states.c | 108 ++------------------------------------------ 1 file changed, 3 insertions(+), 105 deletions(-) diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index b60676349655..48359e789faa 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -58,7 +58,7 @@ static bool check_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap) return true; /* a name of what the caller has, see BPF_INHERITED_ID */ - if (((old_id | cur_id) & ~BPF_ADD_CONST) >= BPF_INHERITED_ID) + if ((old_id | cur_id) >= BPF_INHERITED_ID) return old_id == cur_id; for (i = 0; i < idmap->cnt; i++) { @@ -85,49 +85,6 @@ static bool check_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap) return false; } -/* - * Compare scalar register IDs for state equivalence. - * - * When old_id == 0, the old register is independent - not linked to any - * other register. Any linking in the current state only adds constraints, - * making it more restrictive. Since the old state didn't rely on any ID - * relationships for this register, it's always safe to accept cur regardless - * of its ID. Hence, return true immediately. - * - * When old_id != 0 but cur_id == 0, we need to ensure that different - * independent registers in cur don't incorrectly satisfy the ID matching - * requirements of linked registers in old. - * - * Example: if old has r6.id=X and r7.id=X (linked), but cur has r6.id=0 - * and r7.id=0 (both independent), without temp IDs both would map old_id=X - * to cur_id=0 and pass. With temp IDs: r6 maps X->temp1, r7 tries to map - * X->temp2, but X is already mapped to temp1, so the check fails correctly. - * - * When old_id has BPF_ADD_CONST set, the compound id (base | flag) and the - * base id (flag stripped) must both map consistently. Example: old has - * r2.id=A, r3.id=A|flag (r3 = r2 + delta), cur has r2.id=B, r3.id=C|flag - * (r3 derived from unrelated r4). Without the base check, idmap gets two - * independent entries A->B and A|flag->C|flag, missing that A->C conflicts - * with A->B. The base ID cross-check catches this. - */ -static bool check_scalar_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap) -{ - if (!old_id) - return true; - - cur_id = cur_id ? cur_id : ++idmap->tmp_id_gen; - - if (!check_ids(old_id, cur_id, idmap)) - return false; - if (old_id & BPF_ADD_CONST) { - old_id &= ~BPF_ADD_CONST; - cur_id &= ~BPF_ADD_CONST; - if (!check_ids(old_id, cur_id, idmap)) - return false; - } - return true; -} - static void __clean_func_state(struct bpf_verifier_env *env, struct bpf_func_state *st, u16 live_regs, int frame) @@ -272,65 +229,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, /* explore_alu_limits disables tnum_in() and range_within() * logic and requires everything to be strict */ - return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && - check_scalar_ids(rold->id, rcur->id, idmap); + return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0; } - /* - * Linked register tracking uses rold->id to detect relationships. - * When rold->id == 0, the register is independent and any linking - * in rcur only adds constraints. When rold->id != 0, we must verify - * id mapping and (for BPF_ADD_CONST) offset consistency. - * - * +------------------+-----------+------------------+---------------+ - * | | rold->id | rold + ADD_CONST | rold->id == 0 | - * |------------------+-----------+------------------+---------------| - * | rcur->id | range,ids | false | range | - * | rcur + ADD_CONST | false | range,ids,off | range | - * | rcur->id == 0 | range,ids | false | range | - * +------------------+-----------+------------------+---------------+ - * - * Why check_ids() for scalar registers? - * - * Consider the following BPF code: - * 1: r6 = ... unbound scalar, ID=a ... - * 2: r7 = ... unbound scalar, ID=b ... - * 3: if (r6 > r7) goto +1 - * 4: r6 = r7 - * 5: if (r6 > X) goto ... - * 6: ... memory operation using r7 ... - * - * First verification path is [1-6]: - * - at (4) same bpf_reg_state::id (b) would be assigned to r6 and r7; - * - at (5) r6 would be marked <= X, sync_linked_regs() would also mark - * r7 <= X, because r6 and r7 share same id. - * Next verification path is [1-4, 6]. - * - * Instruction (6) would be reached in two states: - * I. r6{.id=b}, r7{.id=b} via path 1-6; - * II. r6{.id=a}, r7{.id=b} via path 1-4, 6. - * - * Use check_ids() to distinguish these states. - * --- - * Also verify that new value satisfies old value range knowledge. - */ - - /* - * ADD_CONST flags must match exactly: BPF_ADD_CONST32 and - * BPF_ADD_CONST64 have different linking semantics in - * sync_linked_regs() (alu32 zero-extends, alu64 does not), - * so one doesn't stand for the other. - */ - if (rold->id && - (rold->id & BPF_ADD_CONST) != (rcur->id & BPF_ADD_CONST)) - return false; - - /* Both have offset linkage: offsets must match */ - if ((rold->id & BPF_ADD_CONST) && rold->delta != rcur->delta) - return false; - - if (!check_scalar_ids(rold->id, rcur->id, idmap)) - return false; - return range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off); case PTR_TO_MAP_KEY: case PTR_TO_MAP_VALUE: @@ -468,8 +368,7 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, /* * 64 and 32-bit scalar spills vs MISC/INVALID slots and vice versa. * Load from MISC/INVALID slots produces unbound scalar. - * Construct a fake register for such stack and call - * regsafe() to ensure scalar ids are compared. + * Construct a fake register for such stack and call regsafe(). */ if (im == 0 || im == 4) { old_reg = scalar_reg_for_stack(env, old_slot, im); @@ -736,7 +635,6 @@ static void reset_idmap_scratch(struct bpf_verifier_env *env) { struct bpf_idmap *idmap = &env->idmap_scratch; - idmap->tmp_id_gen = env->id_gen; idmap->cnt = 0; } -- 2.55.0 From: Alexei Starovoitov Removal of dead code, as in regsafe(). Scalars have no ids any more. No prog is verified differently. Two states are joined in two passes. The first one records which id of one state goes with which id of the other: jid_note(). The second one computes the registers. A scalar kept its id only if the pair was good: the registers that share the id in one state share it in the other one, with the same BPF_ADD_CONST and delta, see scalar_link_consistent(). A link that one state did not have was dropped. Now the join of two scalars is the join of their ranges. Remove scalar_link_consistent() and the scalars from jid_note() and jid_good(). All other callers of jid_note() pass true for 'consistent': the parameter goes. Signed-off-by: Alexei Starovoitov --- kernel/bpf/join.c | 54 ++++++++++++++--------------------------------- 1 file changed, 16 insertions(+), 38 deletions(-) diff --git a/kernel/bpf/join.c b/kernel/bpf/join.c index cc4494aef187..2b8c94b7dc50 100644 --- a/kernel/bpf/join.c +++ b/kernel/bpf/join.c @@ -75,18 +75,16 @@ static void jid_mark_bad(struct bpf_join_state *js, u32 a, u32 b) js->pairs[i].bad = true; } -static void jid_note(struct bpf_join_state *js, u32 ida, u32 idb, bool consistent) +static void jid_note(struct bpf_join_state *js, u32 a, u32 b) { - u32 a = ida & ~BPF_ADD_CONST, b = idb & ~BPF_ADD_CONST; + bool consistent, other = false; struct join_idpair *p; - bool other = false; u32 i; if (!a || !b) return; /* a name of what the caller has, see BPF_INHERITED_ID */ - if (a != b && (a | b) >= BPF_INHERITED_ID) - consistent = false; + consistent = a == b || (a | b) < BPF_INHERITED_ID; for (i = 0; i < js->npairs; i++) { p = &js->pairs[i]; if (p->a == a && p->b == b) { @@ -111,15 +109,12 @@ static void jid_note(struct bpf_join_state *js, u32 ida, u32 idb, bool consisten p->bad = !consistent; } -static bool jid_good(struct bpf_join_state *js, u32 ida, u32 idb) +static bool jid_good(struct bpf_join_state *js, u32 a, u32 b) { - u32 a = ida & ~BPF_ADD_CONST, b = idb & ~BPF_ADD_CONST; u32 i; if (!a || !b || js->pairs_overflow) return false; - if ((ida ^ idb) & BPF_ADD_CONST) - return false; for (i = 0; i < js->npairs; i++) if (js->pairs[i].a == a && js->pairs[i].b == b) return !js->pairs[i].bad; @@ -199,15 +194,6 @@ static bool reg_value_equal(const struct bpf_reg_state *ra, const struct bpf_reg return !memcmp(ra, rb, offsetof(struct bpf_reg_state, id)); } -static bool scalar_link_consistent(const struct bpf_reg_state *ra, const struct bpf_reg_state *rb) -{ - if ((ra->id ^ rb->id) & BPF_ADD_CONST) - return false; - if ((ra->id & BPF_ADD_CONST) && ra->delta != rb->delta) - return false; - return true; -} - /* * Join two registers, regsafe() documents what each type needs. * Pass 1 records id pairs, pass 2 computes @dst, which is a copy of @ra on entry. @@ -229,16 +215,8 @@ static bool join_reg(struct bpf_join_state *js, struct bpf_reg_state *dst, case SCALAR_VALUE: if (js->env->explore_alu_limits && !reg_value_equal(ra, rb)) return false; - if (pass == 1) { - jid_note(js, ra->id, rb->id, scalar_link_consistent(ra, rb)); - return true; - } - join_range(js, dst, ra, rb); - /* a link that exists on one side only is dropped */ - if (ra->id && !jid_good(js, ra->id, rb->id)) { - dst->id = 0; - dst->delta = 0; - } + if (pass == 2) + join_range(js, dst, ra, rb); return true; case PTR_TO_MAP_KEY: case PTR_TO_MAP_VALUE: @@ -269,7 +247,7 @@ static bool join_reg(struct bpf_join_state *js, struct bpf_reg_state *dst, if (ra->r64.base != rb->r64.base) return false; if (pass == 1) { - jid_note(js, ra->id, rb->id, true); + jid_note(js, ra->id, rb->id); return true; } if (!jid_strict_ok(js, ra->id, rb->id)) @@ -291,9 +269,9 @@ static bool join_reg(struct bpf_join_state *js, struct bpf_reg_state *dst, } if (pass == 1) { - jid_note(js, ra->id, rb->id, true); - jid_note(js, ra->parent_id, rb->parent_id, true); - jid_note(js, ra->map_uid, rb->map_uid, true); + jid_note(js, ra->id, rb->id); + jid_note(js, ra->parent_id, rb->parent_id); + jid_note(js, ra->map_uid, rb->map_uid); return true; } if (!jid_strict_ok(js, ra->id, rb->id) || @@ -350,8 +328,8 @@ static bool join_special_slot(struct bpf_join_state *js, const struct bpf_stack_ break; } if (pass == 1) { - jid_note(js, ra->id, rb->id, true); - jid_note(js, ra->parent_id, rb->parent_id, true); + jid_note(js, ra->id, rb->id); + jid_note(js, ra->parent_id, rb->parent_id); return true; } return jid_strict_ok(js, ra->id, rb->id) && @@ -563,12 +541,12 @@ static void join_refs_note(struct bpf_join_state *js, const struct bpf_verifier_ { int i; - jid_note(js, a->active_irq_id, b->active_irq_id, true); - jid_note(js, a->active_lock_id, b->active_lock_id, true); + jid_note(js, a->active_irq_id, b->active_irq_id); + jid_note(js, a->active_lock_id, b->active_lock_id); for (i = 0; i < a->acquired_refs; i++) { - jid_note(js, a->refs[i].id, b->refs[i].id, true); + jid_note(js, a->refs[i].id, b->refs[i].id); if (a->refs[i].type == REF_TYPE_PTR) - jid_note(js, a->refs[i].parent_id, b->refs[i].parent_id, true); + jid_note(js, a->refs[i].parent_id, b->refs[i].parent_id); } } -- 2.55.0 From: Alexei Starovoitov Removal of dead code, the rest of it. Nothing sets the id of a scalar. No prog is verified differently, no line of a log changes. Remove BPF_ADD_CONST*, what prints 'id=N+M', clear_scalar_id(), struct bpf_idset, idset_scratch, tmp_id_gen and the declaration of bpf_clear_singular_ids(). 'delta' was the constant between two scalars with the same id. For a pointer it is always 0, and log.c printed it for pointers only if it was not. Its four bytes stay, as 'unused': they lie between 'type' and a union that is aligned to 8 bytes, and regsafe() and the join compare the bytes before var_off with memcmp(). Padding there has no defined content. Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 46 +++++++----------------------------- kernel/bpf/log.c | 10 ++------ 2 files changed, 10 insertions(+), 46 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 651174144f5f..cee25ec87403 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -60,10 +60,8 @@ struct bpf_stack_range { struct bpf_reg_state { /* Ordering of fields matters. See regsafe() */ enum bpf_reg_type type; - /* - * Constant delta between "linked" scalars with the same ID. - */ - s32 delta; + /* 0. regsafe() compares the bytes before var_off: there is no hole among them. */ + u32 unused; union { /* valid when type == PTR_TO_PACKET */ int range; @@ -179,18 +177,9 @@ struct bpf_reg_state { * the dynptr to determine proper reference freeing. * Similarly to dynptrs, we use ID to track "belonging" of a reference * to a specific instance of bpf_iter. + * A scalar has no id. The scalars that it is linked to are static: computed before the + * main pass, see bpf_scev_next_linked_reg(). */ - /* - * Upper bit of ID is used to remember relationship between "linked" - * registers. Example: - * r1 = r2; both will have r1->id == r2->id == N - * r1 += 10; r1->id == N | BPF_ADD_CONST and r1->delta == 10 - * r3 = r2; both will have r3->id == r2->id == N - * w3 += 10; r3->id == N | BPF_ADD_CONST32 and r3->delta == 10 - */ -#define BPF_ADD_CONST64 (1U << 31) -#define BPF_ADD_CONST32 (1U << 30) -#define BPF_ADD_CONST (BPF_ADD_CONST64 | BPF_ADD_CONST32) u32 id; /* * Tracks the parent object this register was derived from. @@ -1112,27 +1101,18 @@ struct bpf_id_pair { * comes out for it, only if the same places have the same names. * - bpf_flow_return() puts the ids of the call that waits back. All other ids get new values. * - What comes out has the ranges of all calls that share the state at the entry. So a name is good - * for what holds whatever the ranges are: the same object. Other ids have no name: a number - * loses its id, any other argument keeps the frames of the caller, see arg_needs_caller(). - * - env->id_gen stays below BPF_MAX_ID. So no id is a name or has a bit of BPF_ADD_CONST. + * for what holds whatever the ranges are: the same object. Other ids have no name: such an + * argument keeps the frames of the caller, see arg_needs_caller(). + * - env->id_gen stays below BPF_MAX_ID. So no id is a name. */ #define BPF_INHERITED_ID 0x20000000 #define BPF_MAX_ID (BPF_INHERITED_ID / 2) struct bpf_idmap { - u32 tmp_id_gen; u32 cnt; struct bpf_id_pair map[BPF_ID_MAP_SIZE]; }; -struct bpf_idset { - u32 num_ids; - struct { - u32 id; - u32 cnt; - } entries[BPF_ID_MAP_SIZE]; -}; - struct bpf_liveness; struct scev; @@ -1278,10 +1258,7 @@ struct bpf_verifier_env { * via callx. Allocated when the first such edge is recorded. */ unsigned long *callx_edges; - union { - struct bpf_idmap idmap_scratch; - struct bpf_idset idset_scratch; - }; + struct bpf_idmap idmap_scratch; struct { int *insn_state; int *insn_stack; @@ -1447,12 +1424,6 @@ void bpf_verbose_snum32(struct bpf_verifier_env *env, s32 num); bpf_log(&env->log, "verifier bug: " fmt "\n", ##args); \ }) -static inline void clear_scalar_id(struct bpf_reg_state *reg) -{ - reg->id = 0; - reg->delta = 0; -} - static inline void mark_calls_callback(struct bpf_verifier_env *env, int idx) { env->insn_aux_data[idx].calls_callback = true; @@ -1589,7 +1560,6 @@ void bpf_mark_reg_not_init(const struct bpf_verifier_env *env, void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm); void bpf_mark_reg_unknown(struct bpf_reg_state *reg); int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size); -void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st); static inline int bpf_get_spi(s32 off) { diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 84659a757b43..ffbec587a916 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -696,7 +696,7 @@ static void print_reg_state(struct bpf_verifier_env *env, if (state->frameno != reg->frameno) verbose(env, "[%d]", reg->frameno); if (tnum_is_const(reg->var_off)) { - bpf_verbose_snum(env, reg->var_off.value + reg->delta); + bpf_verbose_snum(env, reg->var_off.value); return; } } @@ -704,9 +704,7 @@ static void print_reg_state(struct bpf_verifier_env *env, verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id)); verbose(env, "("); if (reg->id) - verbose_a("id=%d", reg->id & ~BPF_ADD_CONST); - if (reg->id & BPF_ADD_CONST) - verbose(env, "%+d", reg->delta); + verbose_a("id=%d", reg->id); if (reg->parent_id) verbose_a("parent_id=%d", reg->parent_id); if (type_is_non_owning_ref(reg->type)) @@ -720,10 +718,6 @@ static void print_reg_state(struct bpf_verifier_env *env, reg->map_ptr->key_size, reg->map_ptr->value_size); } - if (t != SCALAR_VALUE && reg->delta) { - verbose_a("off="); - bpf_verbose_snum(env, reg->delta); - } if (type_is_pkt_pointer(t)) { verbose_a("r="); verbose_unum(env, reg->range); -- 2.55.0 From: Alexei Starovoitov A new ability: the recurrence (sext32 (linear rA IMM)), next to (zext32 (linear ..)) and (spill32 (linear ..)). A loop over 'int i' as GCC compiles it becomes a counted loop. test_rhash_update_elements() of rhash.c has 'for (i = 0; i < 128; ++i)'. GCC adds to an int in 32 bits and keeps the sign extension of the result: 53: r3 = r7 54: w3 += 1 55: r7 = (s32)r3 56: if w3 == 0x80 goto pc+27 scev at header 57: r7=(sext32 (zext32 (+ r7 1))) / ? The backedge value of r7, before the '/', has no closed form. No iteration count (header_count: how often the header is executed) is computed. header_count = backedge-taken count + 1. The loop is walked: the verifier unrolls the loop, one state per iteration; a counted loop is summarized, one state for all iterations. A long loop that is walked ends with "more than 1024 states that cannot be joined". It loaded when the verifier explored each path. GCC compiles some 50 loops of the selftests that way, most of them with i in a register. The latch tests the 32-bit add, which is supported. What is missing is to look through the extension, and the range of all 64 bits: - An extension and a partial spill truncate their operand. How it was extended or spilled before does not matter, below an add too: (sext32 (zext32 a)) is (sext32 a). Two rules of simplify() say so for all widths, in place of six. - The backedge value (sext32 (+ rA IMM)) is (sext32 (linear rA IMM)) at the header. Its range is the sign extension of the range of the low 32 bits. A wrap is handled, not excluded: the range is all of s32 then. - iv_range, the range that the latch condition implies, is a range of the zero-extended low 32 bits. iv_bounds() ignores it if bit 31 can be set. - For i in a stack slot, zero or sign extended, little endian only: iterate_wakeupsources() of plain_c_test_wakeup_source.c has a 64-bit spill and a 32-bit fill of i. The half-slot with the upper 32 bits is dead, see the new bpf_half_slot_live_before(), and the states don't have it: 'fp-16=pppp1000'. A recurrence over 64 bits has no entry value there. (spill32 (linear (zext32 rA) IMM)) describes the low 32 bits only, so it holds too: use it. With the patch: r7=(sext32 (+ r7 1)) / (sext32 (linear r7 1)) loop header at 57, header_count is [0..128] The selftests as GCC builds them: no verdict changes, 2.6 % fewer insns. test_rhash_update_elements (rhash.c) 6074 -> 105 iterate_wakeupsources (plain_c_test_wakeup_source.c) 4402 -> 131 nested_loops (loop1.c) 4558 -> 702 test1 (timer.c) 3973 -> 173 Three loops of these selftests keep a sign-extended i in a stack slot. Two load before and after with the same insns. The third, in dump_task_sleepable() of bpf_iter_tasks.c, is still rejected with the message above: one of its two backedges has no linear latch, so the loop has no iteration count. Fixes: 5395aa9bbeb5 ("bpf: Verify programs as a fixpoint over the control flow graph") Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 1 + kernel/bpf/liveness.c | 13 ++++ kernel/bpf/scev.c | 145 +++++++++++++++++++---------------- 3 files changed, 95 insertions(+), 64 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index cee25ec87403..566787a0196c 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -2049,6 +2049,7 @@ int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_ const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx); bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx); bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi); +bool bpf_half_slot_live_before(struct bpf_verifier_env *env, u32 insn_idx, u32 half_spi); int bpf_compute_live_registers(struct bpf_verifier_env *env); #define BPF_MAP_KEY_POISON (1ULL << 63) diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index e930d928afa5..fff5e7fcae18 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -760,6 +760,19 @@ static void compute_may_write_masks(struct bpf_verifier_env *env) } } +/* live_stack_before of the insn at @insn_idx for one half-slot */ +bool bpf_half_slot_live_before(struct bpf_verifier_env *env, u32 insn_idx, u32 half_spi) +{ + struct func_instance *instance; + int bkt; + + hash_for_each(env->liveness->func_instances, bkt, instance, hl_node) + if (insn_idx - instance->subprog_start < instance->insn_cnt && + is_live_before(instance, insn_idx, instance->depth, half_spi)) + return true; + return false; +} + const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx) { return env->insn_aux_data[insn_idx].may_write_mask; diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 34abb4ab0c46..befb2e8c9c27 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -477,6 +477,22 @@ static bool is_spill32(struct scev *scev, int id, u32 *left) return is_spill(scev, id, &bits, left) && bits == 32; } +/* (zextN x), (sextN x) or (spillN x) with N >= @bits: the low @bits bits are those of @left */ +static bool is_low_bits(struct scev *scev, int id, u32 bits, u32 *left) +{ + u32 op = scev->exprs[id].op; + +#ifdef __BIG_ENDIAN + /* see is_spill() */ + if (op == SPILL8 || op == SPILL16 || op == SPILL32) + return false; +#endif + if (op_low_bits(op) < bits) + return false; + *left = scev->exprs[id].params[0]; + return true; +} + static bool is_any(struct scev *scev, int id, u32 *left, u32 *right) { return is_binop(scev, ANY, id, left, right); @@ -752,6 +768,19 @@ static bool reg_alive_at(struct bpf_verifier_env *env, u32 reg, u32 insn_idx) : test_bit(reg - MAX_BPF_REG, aux[insn_idx].live_stack_before); } +/* The first bytes of a stack slot are its high bits */ +static bool is_big_endian_slot(u32 reg) +{ + return IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) && reg >= MAX_BPF_REG; +} + +/* Are the upper 32 bits of @reg live before the insn at @insn_idx? */ +static bool hi32_alive_at(struct bpf_verifier_env *env, u32 reg, u32 insn_idx) +{ + return reg < MAX_BPF_REG || is_big_endian_slot(reg) || + bpf_half_slot_live_before(env, insn_idx, (reg - MAX_BPF_REG) * 2); +} + static void print_env(struct bpf_verifier_env *env, struct env *e, u32 insn_idx) { struct bpf_verifier_log *log = &env->log; @@ -1877,12 +1906,6 @@ static bool match_linked_add(struct bpf_verifier_env *env, int h, u32 tested, u3 return true; } -/* The first bytes of a stack slot are its high bits */ -static bool is_big_endian_slot(u32 reg) -{ - return IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) && reg >= MAX_BPF_REG; -} - /* * const_diff() for the SCEV register @reg, which has the expression @id_b. A fill from byte * *@fill_off of a slot reads these bits. On big endian the low bytes of a spill are the last ones, @@ -2349,7 +2372,7 @@ static int transform_expr_once(struct scev *scev, u32 lvl, u32 root, void *priv, static int simplify(struct scev *scev, u32 id) { int a, b, l, r, t, base, slope; - u32 bits, op = scev->exprs[id].op; + u32 op = scev->exprs[id].op, bits = op_low_bits(op); s64 imm1, imm2; /* (- x imm) -> (+ x -imm) */ @@ -2392,41 +2415,21 @@ static int simplify(struct scev *scev, u32 id) return imm_expr(scev, (u64)(u32)imm1); /* - * (zext32 (+ (zext32 $a) $b)) -> (zext32 (+ $a $b)) - * (zext32 (+ $a (zext32 $b))) -> (zext32 (+ $a $b)) - */ - if ((is_zext32(scev, id, &t) && is_add(scev, t, &l, &b) && is_zext32(scev, l, &a)) || - (is_zext32(scev, id, &t) && is_add(scev, t, &a, &r) && is_zext32(scev, r, &b))) { - t = expr2(scev, BPF_ADD, a, b); - t = expr1(scev, ZEXT32, t); - return t; - } - - /* (zext32 (zext32 X)) -> (zext32 X) */ - if (is_zext32(scev, id, &a) && is_zext32(scev, a, &b)) - return a; - - /* - * '*(u32 *)(r10 - 8) = rX' stores the low half of rX: - * (spill32 (zext32 $a)) -> (spill32 $a) - * (spill32 (+ (zext32 $a) $b)) -> (spill32 (+ $a $b)) - */ - if (is_spill32(scev, id, &t) && is_zext32(scev, t, &a)) - return expr1(scev, SPILL32, a); - if (is_spill32(scev, id, &t) && is_add(scev, t, &l, &b) && is_zext32(scev, l, &a)) - return expr1(scev, SPILL32, expr2(scev, BPF_ADD, a, b)); - - /* - * 'rX = *(u32 *)(r10 - 8)' fills what was spilled, a narrower fill the low bits of it: - * (zext32 (spill32 $a)) -> (zext32 $a) + * An extension and a partial spill depend on the low bits of their operand only: look + * through an extension and a partial spill that is not narrower, below an add too: + * (sext32 (zext32 $a)) -> (sext32 $a) * (sext8 (spill16 $a)) -> (sext8 $a) - * (zext32 (+ (spill32 $a) $b)) -> (zext32 (+ $a $b)) + * (zext32 (+ (sext32 $a) $b)) -> (zext32 (+ $a $b)) + * (spill32 (+ $a (zext32 $b))) -> (spill32 (+ $a $b)) */ - if (op_low_bits(op) && is_spill(scev, scev->exprs[id].params[0], &bits, &a) && - op_low_bits(op) <= bits) + if (!bits) + return id; + t = scev->exprs[id].params[0]; + if (is_low_bits(scev, t, bits, &a)) return expr1(scev, op, a); - if (is_zext32(scev, id, &t) && is_add(scev, t, &l, &b) && is_spill32(scev, l, &a)) - return expr1(scev, ZEXT32, expr2(scev, BPF_ADD, a, b)); + if (is_add(scev, t, &l, &r) && + (is_low_bits(scev, l, bits, &l) || is_low_bits(scev, r, bits, &r))) + return expr1(scev, op, expr2(scev, BPF_ADD, l, r)); return id; } @@ -2449,6 +2452,7 @@ static int compute_header_scevs(struct bpf_verifier_env *env, int h) int l, r, t, id, ra, rb, ra_expr; bool zext32; s64 imm; + u32 op; /* * Note: @@ -2483,30 +2487,24 @@ static int compute_header_scevs(struct bpf_verifier_env *env, int h) header_env->reg2scev[ra] = id; continue; } - /* rA = (is_zext32 (+ rA IMM)) -> (is_zext32 (linear rA IMM)) */ - if (is_zext32(scev, ra_expr, &t) && - is_add(scev, t, &l, &r) && - is_reg(scev, l, &rb) && - is_imm(scev, r, &imm) && - ra == rb) { - id = expr2(scev, LINEAR_SCEV, l, r); - id = expr1(scev, ZEXT32, id); - if (id < 0) - return id; - header_env->reg2scev[ra] = id; - continue; - } /* + * A 32-bit induction variable: + * rA = (zext32 (+ rA IMM)) -> (zext32 (linear rA IMM)) + * rA = (sext32 (+ rA IMM)) -> (sext32 (linear rA IMM)) * rA = (spill32 (+ rA IMM)) -> (spill32 (linear (zext32 rA) IMM)) - * The low half of the slot counts. Nothing is known about the other half. + * The last SCEV leaves the upper 32 bits of the slot unknown, so it holds for the + * first two as well. They get it if these bits are dead: the states have a + * 32-bit spill then, see bpf_clean_verifier_state(). */ - if (is_spill32(scev, ra_expr, &t) && + if (is_low_bits(scev, ra_expr, 32, &t) && is_add(scev, t, &l, &r) && is_reg(scev, l, &rb) && is_imm(scev, r, &imm) && ra == rb) { - id = expr2(scev, LINEAR_SCEV, expr1(scev, ZEXT32, l), r); - id = expr1(scev, SPILL32, id); + op = hi32_alive_at(env, ra, h) ? scev->exprs[ra_expr].op : SPILL32; + if (op == SPILL32) + l = expr1(scev, ZEXT32, l); + id = expr1(scev, op, expr2(scev, LINEAR_SCEV, l, r)); if (id < 0) return id; header_env->reg2scev[ra] = id; @@ -4105,8 +4103,10 @@ struct linear_coefs { u32 base_reg; s64 slope; bool zext32; - /* the low half of a stack slot */ + /* with zext32: the low 32 bits of a stack slot */ bool spill32; + /* with zext32: the low 32 bits are sign extended */ + bool sext32; }; static bool is_simple_slope(struct scev *scev, u32 id, s64 *slope) @@ -4122,8 +4122,12 @@ static bool is_simple_linear(struct scev *scev, u32 id, struct linear_coefs *out u32 base, slope; out->spill32 = false; - return is_linear_or_linear32(scev, id, &base, &slope, &out->zext32) && - is_reg(scev, base, &out->base_reg) && + out->sext32 = is_unop(scev, SEXT32, id, &id); + if (!is_linear_or_linear32(scev, id, &base, &slope, &out->zext32) || + (out->sext32 && out->zext32)) + return false; + out->zext32 |= out->sext32; + return is_reg(scev, base, &out->base_reg) && is_simple_slope(scev, slope, &out->slope); } @@ -4155,6 +4159,7 @@ static struct bpf_reg_state *spill32_counter(struct scev *scev, struct bpf_func_ return NULL; out->zext32 = true; out->spill32 = true; + out->sext32 = false; return scev_reg_for_spill32(st, r); } @@ -4163,6 +4168,11 @@ static struct cnum64 cnum64_zext32(struct cnum32 r32) return cnum64_from_urange(cnum32_umin(r32), cnum32_umax(r32)); } +static struct cnum64 cnum64_sext32(struct cnum32 r32) +{ + return cnum64_from_srange(cnum32_smin(r32), cnum32_smax(r32)); +} + /* Compute the range an induction variable in `reg` spans over the loop. */ static void linear_bounds(struct bpf_reg_state *reg, u32 max_header_count, struct linear_coefs *coefs, struct bounds *out) @@ -4192,12 +4202,14 @@ static void linear_bounds(struct bpf_reg_state *reg, u32 max_header_count, r32 = CNUM32_UNBOUNDED; if (coefs->zext32) { r32 = cnum32_from_cnum64(r64); - r64 = cnum64_zext32(r32); + r64 = coefs->sext32 ? cnum64_sext32(r32) : cnum64_zext32(r32); /* * 32-bit iteration overflow is equivalent to subtracting N * 2**32: * (base + slope * k) % slope ≠ (base + slope * k - N * 2**32) % slope. */ - overflow |= cnum32_contains(r32, 0) && cnum32_contains(r32, U32_MAX); + overflow |= coefs->sext32 ? + cnum32_contains(r32, S32_MAX) && cnum32_contains(r32, S32_MIN) : + cnum32_contains(r32, 0) && cnum32_contains(r32, U32_MAX); /* * The entry and zero-extended values must agree on how they define `base`, * otherwise `base + slope * k` would define different sets of points between @@ -4228,10 +4240,14 @@ static void linear_bounds(struct bpf_reg_state *reg, u32 max_header_count, out->r32 = r32; } -static void iv_bounds(struct bounds *bounds, struct cnum64 iv_range) +static void iv_bounds(struct bounds *bounds, const struct linear_coefs *coefs, + struct cnum64 iv_range) { struct cnum64 r64 = cnum64_intersect(bounds->r64, iv_range); + /* The latch bounds the zero extension. With bit 31 set the sign extension differs. */ + if (coefs->sext32 && cnum64_umax(iv_range) > S32_MAX) + return; if (!cnum64_is_empty(r64)) bounds->r64 = r64; } @@ -4771,7 +4787,7 @@ int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state return err; linear_bounds(reg, iters->max_header_count, &coefs, &bounds); if (coefs.base_reg == iters->iv_reg) - iv_bounds(&bounds, iters->iv_range); + iv_bounds(&bounds, &coefs, iters->iv_range); if (log->level & BPF_LOG_LEVEL2) { bpf_log(log, "loop header at %d, widening ", insn_idx); log_reg(env, r); @@ -4817,6 +4833,7 @@ static bool is_moved_linear(struct scev *scev, u32 id, struct linear_coefs *out, u32 base, slope, l, r; out->spill32 = false; + out->sext32 = false; return is_linear_or_linear32(scev, id, &base, &slope, &out->zext32) && !out->zext32 && is_add(scev, base, &l, &r) && is_reg(scev, l, &out->base_reg) && is_imm(scev, r, moved) && is_simple_slope(scev, slope, &out->slope); @@ -4893,7 +4910,7 @@ int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur clamp: linear_bounds(entry_reg, max_header_count, &coefs, &bounds); if (coefs.base_reg == entry->iv_reg && !moved) - iv_bounds(&bounds, entry->iv_range); + iv_bounds(&bounds, &coefs, entry->iv_range); if (moved) { /* a sum that can wrap has another residue, see scalar_step_add() */ if (check_add_overflow(cnum64_smin(bounds.r64), moved, &sum) || -- 2.55.0 From: Alexei Starovoitov Tests for (sext32 (linear rA IMM)), the recurrence of 'int i' as GCC compiles it. 'w1 = w6; w1 += 1; r6 = (s32)r1; if w1 != 4 goto' is '++i != 4' as GCC emits it. An index that is not in 'char buf[8]' gets to a read of r5, which is not initialized. r9 is the number of iterations and is what the prog returns. What loads is run. - sext32_iv: i in a register goes from -4 to 3. sext32_iv_hi32_dead: i in a stack slot, the half-slot with the upper 32 bits is dead. Both are counted loops with header_count 8. The negative tests, *_oob, have one iteration more. - sext32_iv_wraps: i wraps at INT32_MAX in the loop. Its range is all of s32, the index is out of bounds. - sext32_iv_unsigned_bound: 'i < n' with n above INT32_MAX. The range that the latch condition implies is not the range of i. Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/progs/verifier_scev.c | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 49f9a976a44c..6e72900826bb 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -1169,6 +1169,50 @@ RANGE_TEST(range_ge_dir, "", 10, 20, 2, 3, 5, PRE("<"), NO_COUNT) RANGE_TEST(range_ne_step2, "", 0, 3, 2, 3, 8, PRE("=="), NO_COUNT) RANGE_TEST(range_ne_behind, "", 0, 4, 1, 3, 8, PRE("=="), NO_COUNT) +/* + * char buf[8]; + * for (int i = first; ...; i++) + * sum += buf[i - first]; + * + * GCC adds in 32 bits and keeps the sign extension of the result: a sign-extended 32-bit induction + * variable. r6 or fp-16 is i, r7 is first, r9 counts the iterations. r8 is 0, which the verifier + * doesn't know. In place of an out-of-bounds access to buf r5 is read, which is not initialized. + */ +#define SEXT32_TEST(name, _first, loop, expect) \ +SEC("socket") \ +__log_level(2) \ +expect \ +__naked void name(void) \ +{ \ + asm volatile ( \ + "r6 = %[first] ll; r7 = r6; r8 = 0; r9 = 0;" \ + "*(u64 *)(r10 - 8) = r8; lock *(u64 *)(r10 - 8) += r8; r8 = *(u64 *)(r10 - 8);" \ + "*(u64 *)(r10 - 16) = r6;" \ + loop \ + "2: r0 = r9; exit;" \ + : \ + : __imm_const(first, _first) \ + : __clobber_all); \ +} +#define BUF(i) "r9 += 1; r2 = " i "; r2 -= r7; if r2 <= 7 goto +1; r0 = *(u64 *)(r5 + 0);" +#define OOB __failure __msg("R5 !read_ok") +#define ALL_OF_S32 "0xffffffff80000000..0x7fffffff " +#define SEXT32_INC "w1 = w6; w1 += 1; r6 = (s32)r1;" +#define SEXT32_LOOP(bound) "1:" BUF("r6") SEXT32_INC "if w1 != " #bound " goto 1b;" + +SEXT32_TEST(sext32_iv, -4, SEXT32_LOOP(4), __success __retval(8) + __msg(" r6=(sext32 (+ r6 1)) / (sext32 (linear r6 1))") + __msg("header_count is 8 ") __msg("widening r6 to -4..3 ")) +SEXT32_TEST(sext32_iv_oob, -4, SEXT32_LOOP(5), OOB) +/* i wraps from INT32_MAX to INT32_MIN. The latch tests 0x7ffffffd .. 0x80000004. */ +SEXT32_TEST(sext32_iv_wraps, INT32_MAX - 3, SEXT32_LOOP(0x80000004), + __msg("header_count is 8 ") __msg("widening r6 to " ALL_OF_S32) OOB) +/* 'i < n' implies the unsigned range 1 .. 0x80000000. Its last value is negative in r6. */ +SEXT32_TEST(sext32_iv_unsigned_bound, 0, + "r8 &= 1; r6 += r8; r4 = 0x80000001 ll;" + "1:" SEXT32_INC "if w1 < w4 goto 1b; if r6 s>= 0 goto 2f; r0 = *(u64 *)(r5 + 0);", + __msg("widening r6 to " ALL_OF_S32) OOB) + /* * Two back edges: * @@ -3197,6 +3241,14 @@ RANGE_TEST(spill32_first_is_precise, "", 0, 0, 1, 0, 1, RANGE_TEST(spill32_first_is_64bit, "", 0, 0, 1, 5, 5, "*(u64 *)(r10 - 8) = r6; 1:" LIMIT_ITERATIONS SPILL32_INC "if w1 != w7 goto 1b;", SPILL32_COUNTED(5, "0..4")) +/* A 64-bit spill of i. Its upper 32 bits are dead: the low 32 bits are the induction variable. */ +#define SEXT32_SPILL_LOOP(bound) \ + "1:" BUF("*(u32 *)(r10 - 16)") "w1 = *(u32 *)(r10 - 16); w1 += 1; r3 = (s32)r1;" \ + "*(u64 *)(r10 - 16) = r3; if w1 != " #bound " goto 1b;" +SEXT32_TEST(sext32_iv_hi32_dead, 0, SEXT32_SPILL_LOOP(8), __success __retval(8) + __msg(" *fp-16=(sext32 (+ *fp-16 1)) / (spill32 (linear (zext32 *fp-16) 1))") + __msg("header_count is 8 ") __msg("widening *fp-16 to 0..7 ")) +SEXT32_TEST(sext32_iv_hi32_dead_oob, 0, SEXT32_SPILL_LOOP(9), OOB) /* The counter makes the address of a spill. Such a loop is not summarized, as with a register. */ PARTIAL_SPILL_TEST(spill32_is_a_stack_offset, u32, 4, "can't widen *fp-8, expr is (spill32 (+ *fp-8 8)), requires exact stack-offset tracking", -- 2.55.0