From: Shay Drory The eswitch source-port match metadata is laid out as 4 bits of PFNUM and 12 bits of a unique id (ESW_SOURCE_PORT_METADATA_BITS). PFNUM is the native_port_num reported in HCA_CAP, so all VFs/SFs of the same PF share a single PFNUM. Each VF/SF eswitch allocates its id from its own vport_metadata_ida, restarting from the lowest value, so two VFs/SFs of the same PF can end up with identical metadata. Once their eswitches share an FDB (virt LAG), that metadata no longer uniquely identifies the source port. Add a dedicated allocator for nested (virt LAG) eswitches that replaces the PFNUM field with the VF/SF's slot in the LAG (its xarray index in ldev->fns). That index is unique per function within the LAG, so combined with a per-VF/SF id it yields unique metadata regardless of which PF the VFs/SFs belong to. The split is 8 bits for the virt LAG index - which fits the U8_MAX cap on the LAG xarray - and the remaining 8 bits for the id; the two still sum to ESW_SOURCE_PORT_METADATA_BITS, so nothing downstream of the match changes. Describe the nested layout next to the reg_c_0 layout in include/linux/mlx5/eswitch.h. The PFNUM value 0xf is reserved for internal port offload, which stamps it into reg_c_0 and matches it with the same mask used for vport metadata. That offload is gated on switchdev mode only, so it can be used on a nested eswitch as well. Cap the LAG index accordingly. Select the nested layout for a non-PF eswitch, unless it is virt LAG capable but not a LAG member, e.g. one refused as a late joiner and left standalone. Such a device keeps the PFNUM layout: it does not pair with the eswitches of the LAG members, so its metadata is never compared against theirs. PFs, which never report the virt LAG capabilities, keep the PFNUM layout. Signed-off-by: Shay Drory Reviewed-by: Moshe Shemesh Reviewed-by: Mark Bloch Signed-off-by: Tariq Toukan --- .../mellanox/mlx5/core/eswitch_offloads.c | 48 +++++++++++++++++++ .../net/ethernet/mellanox/mlx5/core/lag/lag.c | 17 +++++++ .../ethernet/mellanox/mlx5/core/mlx5_core.h | 1 + include/linux/mlx5/eswitch.h | 4 ++ 4 files changed, 70 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 1323114ce7f9..3c1355f91090 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -3650,6 +3650,48 @@ static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw) return MLX5_ESW_METADATA_RSVD_UPLINK; } +/* Nested (virt LAG) source-port metadata layout: the upper ESW_NESTED_IDX_BITS + * hold the VF/SF's slot in the LAG and the lower ESW_NESTED_VPORT_BITS hold a + * vport id unique within its eswitch. The two fields still sum to + * ESW_SOURCE_PORT_METADATA_BITS. The LAG slot (0-254, per the xa_alloc() limit + * in mlx5_ldev_add_mdev()) fits in ESW_NESTED_IDX_BITS; slots 0xf0 and up would + * drive the PFNUM bits to the 0xf reserved by internal port offload, so only + * 0-239 are used. + */ +#define ESW_NESTED_VPORT_BITS 8 +#define ESW_NESTED_IDX_BITS (ESW_SOURCE_PORT_METADATA_BITS - ESW_NESTED_VPORT_BITS) + +static u32 mlx5_nested_esw_match_metadata_alloc(struct mlx5_eswitch *esw) +{ + u32 vport_end_ida = (1 << ESW_NESTED_VPORT_BITS) - 1; + u32 max_idx = (1 << ESW_NESTED_IDX_BITS) - + (1 << (ESW_NESTED_IDX_BITS - ESW_PFNUM_BITS)) - 1; + int idx; + int id; + + idx = mlx5_lag_get_member_idx(esw->dev); + if (idx < 0 || idx > max_idx) + return 0; + + /* Use only non-zero id's (2-255). */ + id = ida_alloc_range(&esw->offloads.vport_metadata_ida, + MLX5_ESW_METADATA_RSVD_UPLINK + 1, + vport_end_ida, GFP_KERNEL); + if (id < 0) + return 0; + id = (idx << ESW_NESTED_VPORT_BITS) | id; + return id; +} + +static void mlx5_nested_esw_match_metadata_free(struct mlx5_eswitch *esw, + u32 metadata) +{ + u32 vport_bit_mask = (1 << ESW_NESTED_VPORT_BITS) - 1; + + /* Metadata holds only ESW_NESTED_VPORT_BITS of actual ida id. */ + ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); +} + u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) { u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1; @@ -3658,6 +3700,9 @@ u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) int pf_num; int id; + if (!mlx5_core_is_pf(esw->dev) && !mlx5_virt_lag_nonmember(esw->dev)) + return mlx5_nested_esw_match_metadata_alloc(esw); + /* Only 4 bits of pf_num */ pf_num = mlx5_sd_pf_num_get(esw->dev); if (pf_num < 0 || pf_num > max_pf_num) @@ -3678,6 +3723,9 @@ void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) { u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1; + if (!mlx5_core_is_pf(esw->dev) && !mlx5_virt_lag_nonmember(esw->dev)) + return mlx5_nested_esw_match_metadata_free(esw, metadata); + /* Metadata contains only 12 bits of actual ida id */ ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index abcab76fced0..193fbe47fef1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -516,6 +516,23 @@ int mlx5_lag_get_dev_seq(struct mlx5_core_dev *dev) } EXPORT_SYMBOL(mlx5_lag_get_dev_seq); +int mlx5_lag_get_member_idx(struct mlx5_core_dev *dev) +{ + struct mlx5_lag *ldev = mlx5_lag_dev(dev); + struct lag_fn *fn; + int idx; + + if (!ldev) + return -ENOENT; + + xa_lock(&ldev->fns); + fn = mlx5_lag_fn_by_dev(ldev, dev); + idx = fn ? fn->idx : -ENOENT; + xa_unlock(&ldev->fns); + + return idx; +} + /* seq 0 = master, then all remaining devices */ static int mlx5_lag_get_dev_index_by_seq_all(struct mlx5_lag *ldev, int seq) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index d84d0424d04e..f5f105762d88 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -275,6 +275,7 @@ void mlx5_lag_add_mdev(struct mlx5_core_dev *dev); void mlx5_lag_remove_mdev(struct mlx5_core_dev *dev); void mlx5_lag_disable_change(struct mlx5_core_dev *dev); void mlx5_lag_enable_change(struct mlx5_core_dev *dev); +int mlx5_lag_get_member_idx(struct mlx5_core_dev *dev); int mlx5_events_init(struct mlx5_core_dev *dev); void mlx5_events_cleanup(struct mlx5_core_dev *dev); diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h index 03d3620141c8..06d7d634de0d 100644 --- a/include/linux/mlx5/eswitch.h +++ b/include/linux/mlx5/eswitch.h @@ -95,6 +95,10 @@ bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw); * unique non-zero vport id (range 1-4095). The rest (lowest 16 bits) is left * for user data objects managed by a common mapping context. * PFNUM + VPORT comprise the SOURCE_PORT matching. + * + * A nested (virt LAG) VF/SF eswitch splits the SOURCE_PORT bits differently: + * the highest 8 bits are its slot in the LAG (range 0-239) and the next 8 + * bits are a unique non-zero vport id (range 2-255). */ #define ESW_VPORT_BITS 12 #define ESW_PFNUM_BITS 4 -- 2.44.0