Adds flow_dissect_fast_mpls() — straight-line read of one MPLS label entry (ETH_P_MPLS_UC 0x8847 / ETH_P_MPLS_MC 0x8848), extracting label / TC / TTL / BoS and writing to FLOW_DISSECTOR_KEY_MPLS lse[0] when the dissector requests it. Byte-identical with the slow path's __skb_flow_dissect_mpls() called with lse_index=0, bos=1: the slow path returns FLOW_DISSECT_RET_OUT_GOOD when BoS is set without descending into the inner packet (MPLS flow hashing is on the label stack, not the inner 5-tuple), so this fast-path does the same — no IP tail-call. Multi-label stacks (BoS=0 on the first label) defer to the slow path. A depth-2+ variant can land later, mirroring the vlan -> qinq staging used for the VLAN gates. Why this matters: MPLS is the dominant SP backbone L2.5 and the existing slow-path traverses an iteration loop per label even when the consumer just wants the label-stack flow key. The fast-path saves the iteration overhead for the common single-label case while keeping the slow path as escape for multi-label and the entropy-label case (which can only fire from label 2+ anyway). Gated by: - new DEFINE_STATIC_KEY_FALSE(flow_dissector_mpls_key) - new sysctl /proc/sys/net/flow_dissector/mpls (default 0) - dispatcher case htons(ETH_P_MPLS_UC) / htons(ETH_P_MPLS_MC) with static_branch_unlikely(&flow_dissector_mpls_key) guard When the sysctl is 0, dispatcher cost is one not-taken JMP per matching packet — same per-shape pattern as the other fast-path patches. Documentation/admin-guide/sysctl/net.rst grows an `mpls` subsection under /proc/sys/net/flow_dissector/. The helper also mirrors the slow path's out_good terminal writes: for the standard flow_keys dissectors (which request no MPLS key), the slow path returns OUT_GOOD after the first LSE with nhoff advanced past it, so thoff, n_proto (the MPLS ethertype) and ip_proto (0) are still written at the exit label. Skipping those writes is observable in struct flow_keys and would break the byte-identical contract; the in-tree KUnit equivalence test added later in this series catches exactly this. Assisted-by: Claude:claude-fable-5 sparse smatch Signed-off-by: Dave Seddon --- Documentation/admin-guide/sysctl/net.rst | 19 ++++++ include/net/flow_dissector.h | 1 + net/core/flow_dissector.c | 83 ++++++++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/Documentation/admin-guide/sysctl/net.rst b/Documentation/admin-guide/sysctl/net.rst index 1096e6ddca39..1ee5419ffaeb 100644 --- a/Documentation/admin-guide/sysctl/net.rst +++ b/Documentation/admin-guide/sysctl/net.rst @@ -534,6 +534,25 @@ code != 0). Default: 0 +mpls +~~~~ + +Single-label MPLS frames (EtherType ``ETH_P_MPLS_UC`` 0x8847 or +``ETH_P_MPLS_MC`` 0x8848 with bottom-of-stack bit set on the first +label). The fast-path reads the 4-byte label entry, extracts +label / TC / TTL / BoS, and writes them to FLOW_DISSECTOR_KEY_MPLS +lse[0] when requested. Byte-identical with the slow path's +__skb_flow_dissect_mpls() for lse_index=0, bos=1: the slow path +returns OUT_GOOD on BoS without descending into the inner packet +(MPLS hashing is on the label stack, not inner 5-tuple), so this +fast-path does the same. + +Multi-label stacks (BoS=0 on the first label) defer to slow path — +extending to depth-2 or more is a follow-up, mirroring the +vlan -> qinq staging used earlier in this series. + +Default: 0 + 3. /proc/sys/net/unix - Parameters for Unix domain sockets ---------------------------------------------------------- diff --git a/include/net/flow_dissector.h b/include/net/flow_dissector.h index df85162959cc..8df7821ba769 100644 --- a/include/net/flow_dissector.h +++ b/include/net/flow_dissector.h @@ -432,6 +432,7 @@ extern struct static_key_false flow_dissector_eth_ip_key; extern struct static_key_false flow_dissector_vlan_key; extern struct static_key_false flow_dissector_qinq_key; extern struct static_key_false flow_dissector_pppoe_key; +extern struct static_key_false flow_dissector_mpls_key; /* struct flow_keys_digest: * diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 21a3f5e0ea05..0ec1a6a1bb0e 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -48,6 +48,8 @@ DEFINE_STATIC_KEY_FALSE(flow_dissector_qinq_key); EXPORT_SYMBOL(flow_dissector_qinq_key); DEFINE_STATIC_KEY_FALSE(flow_dissector_pppoe_key); EXPORT_SYMBOL(flow_dissector_pppoe_key); +DEFINE_STATIC_KEY_FALSE(flow_dissector_mpls_key); +EXPORT_SYMBOL(flow_dissector_mpls_key); /* IPv4 version/IHL byte of an option-less header: version 4, IHL 5. */ #define FLOW_DIS_IPV4_VIHL_NOOPT 0x45 @@ -1396,6 +1398,73 @@ static bool flow_dissect_fast_pppoe(const struct sk_buff *skb, nhoff, hlen); } +/* Single MPLS label (BoS=1): write FLOW_DISSECTOR_KEY_MPLS lse[0] + * when requested and stop -- the slow path returns OUT_GOOD on BoS + * without descending into the inner packet, so the fast path must + * not tail-call either. Multi-label stacks defer. + */ +static bool flow_dissect_fast_mpls(const struct sk_buff *skb, + struct flow_dissector *flow_dissector, + void *target_container, + const void *data, + __be16 proto, int nhoff, int hlen) +{ + struct flow_dissector_key_control *key_control; + struct flow_dissector_key_basic *key_basic; + struct flow_dissector_key_mpls *key_mpls; + struct flow_dissector_mpls_lse *lse; + const struct mpls_label *hdr; + u32 entry, label, bos; + + if (unlikely(hlen - nhoff < (int)sizeof(*hdr))) + return false; + hdr = (const struct mpls_label *)((const u8 *)data + nhoff); + entry = ntohl(hdr->entry); + bos = (entry & MPLS_LS_S_MASK) >> MPLS_LS_S_SHIFT; + + /* Multi-label stack: defer. */ + if (!bos) + return false; + + /* Neither MPLS key requested: the slow path returns OUT_GOOD without + * writing the key; skipping the write block matches it. + */ + label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT; + + if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) { + key_mpls = skb_flow_dissector_target(flow_dissector, + FLOW_DISSECTOR_KEY_MPLS, + target_container); + lse = &key_mpls->ls[0]; + lse->mpls_ttl = (entry & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT; + lse->mpls_bos = bos; + lse->mpls_tc = (entry & MPLS_LS_TC_MASK) >> MPLS_LS_TC_SHIFT; + lse->mpls_label = label; + dissector_set_mpls_lse(key_mpls, 0); + } + + /* Mirror the slow path's out_good terminal: nhoff advances past the + * LSE, then thoff/basic are written from the final proto/ip_proto. + */ + nhoff += (int)sizeof(*hdr); + if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_CONTROL)) { + key_control = skb_flow_dissector_target(flow_dissector, + FLOW_DISSECTOR_KEY_CONTROL, + target_container); + key_control->thoff = min_t(u16, nhoff, + skb ? skb->len : hlen); + } + if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_BASIC)) { + key_basic = skb_flow_dissector_target(flow_dissector, + FLOW_DISSECTOR_KEY_BASIC, + target_container); + key_basic->n_proto = proto; + key_basic->ip_proto = 0; + } + + return true; +} + /* Top-level dispatcher: eligibility check (only the two standard * dissectors and flag subset) + per-proto switch with per-shape * static_branch gating. Each case's branch is a forward not-taken JMP @@ -1449,6 +1518,13 @@ static bool flow_dissect_fast(const struct sk_buff *skb, return flow_dissect_fast_pppoe(skb, flow_dissector, target_container, data, nhoff, hlen); + case htons(ETH_P_MPLS_UC): + case htons(ETH_P_MPLS_MC): + if (!static_branch_unlikely(&flow_dissector_mpls_key)) + return false; + return flow_dissect_fast_mpls(skb, flow_dissector, + target_container, data, + proto, nhoff, hlen); default: return false; } @@ -2598,6 +2674,13 @@ static struct ctl_table flow_dissector_sysctl_table[] = { .mode = 0644, .proc_handler = proc_do_static_key, }, + { + .procname = "mpls", + .data = &flow_dissector_mpls_key.key, + .maxlen = sizeof(flow_dissector_mpls_key), + .mode = 0644, + .proc_handler = proc_do_static_key, + }, }; static int __init flow_dissector_sysctl_init(void) -- 2.54.0