The hclge_main.c file has become very large, so the fd code has been moved to a separate hclge_fd.c file. This patch only moves the code and does not modify any functionality. Signed-off-by: Jijie Shao --- drivers/net/ethernet/hisilicon/hns3/Makefile | 1 + .../ethernet/hisilicon/hns3/hns3pf/hclge_fd.c | 2575 ++++++++++++++++ .../ethernet/hisilicon/hns3/hns3pf/hclge_fd.h | 33 + .../hisilicon/hns3/hns3pf/hclge_main.c | 2656 +---------------- .../hisilicon/hns3/hns3pf/hclge_main.h | 2 + 5 files changed, 2654 insertions(+), 2613 deletions(-) create mode 100644 drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_fd.c create mode 100644 drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_fd.h diff --git a/drivers/net/ethernet/hisilicon/hns3/Makefile b/drivers/net/ethernet/hisilicon/hns3/Makefile index e8af26da1fc1..5785d4c5709e 100644 --- a/drivers/net/ethernet/hisilicon/hns3/Makefile +++ b/drivers/net/ethernet/hisilicon/hns3/Makefile @@ -24,5 +24,6 @@ hclgevf-objs = hns3vf/hclgevf_main.o hns3vf/hclgevf_mbx.o hns3vf/hclgevf_devlin obj-$(CONFIG_HNS3_HCLGE) += hclge.o hclge-common.o hclge-objs = hns3pf/hclge_main.o hns3pf/hclge_mdio.o hns3pf/hclge_tm.o hns3pf/hclge_regs.o \ hns3pf/hclge_mbx.o hns3pf/hclge_err.o hns3pf/hclge_debugfs.o hns3pf/hclge_ptp.o hns3pf/hclge_devlink.o \ + hns3pf/hclge_fd.o hclge-$(CONFIG_HNS3_DCB) += hns3pf/hclge_dcb.o diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_fd.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_fd.c new file mode 100644 index 000000000000..d27074ce9e9f --- /dev/null +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_fd.c @@ -0,0 +1,2575 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright (c) 2026 Hisilicon Limited. + +#include +#include +#include +#include "hclge_fd.h" +#include "hclge_main.h" + +static const struct key_info meta_data_key_info[] = { + { PACKET_TYPE_ID, 6 }, + { IP_FRAGEMENT, 1 }, + { ROCE_TYPE, 1 }, + { NEXT_KEY, 5 }, + { VLAN_NUMBER, 2 }, + { SRC_VPORT, 12 }, + { DST_VPORT, 12 }, + { TUNNEL_PACKET, 1 }, +}; + +static const struct key_info tuple_key_info[] = { + { OUTER_DST_MAC, 48, KEY_OPT_MAC, -1, -1 }, + { OUTER_SRC_MAC, 48, KEY_OPT_MAC, -1, -1 }, + { OUTER_VLAN_TAG_FST, 16, KEY_OPT_LE16, -1, -1 }, + { OUTER_VLAN_TAG_SEC, 16, KEY_OPT_LE16, -1, -1 }, + { OUTER_ETH_TYPE, 16, KEY_OPT_LE16, -1, -1 }, + { OUTER_L2_RSV, 16, KEY_OPT_LE16, -1, -1 }, + { OUTER_IP_TOS, 8, KEY_OPT_U8, -1, -1 }, + { OUTER_IP_PROTO, 8, KEY_OPT_U8, -1, -1 }, + { OUTER_SRC_IP, 32, KEY_OPT_IP, -1, -1 }, + { OUTER_DST_IP, 32, KEY_OPT_IP, -1, -1 }, + { OUTER_L3_RSV, 16, KEY_OPT_LE16, -1, -1 }, + { OUTER_SRC_PORT, 16, KEY_OPT_LE16, -1, -1 }, + { OUTER_DST_PORT, 16, KEY_OPT_LE16, -1, -1 }, + { OUTER_L4_RSV, 32, KEY_OPT_LE32, -1, -1 }, + { OUTER_TUN_VNI, 24, KEY_OPT_VNI, + offsetof(struct hclge_fd_rule, tuples.outer_tun_vni), + offsetof(struct hclge_fd_rule, tuples_mask.outer_tun_vni) }, + { OUTER_TUN_FLOW_ID, 8, KEY_OPT_U8, -1, -1 }, + { INNER_DST_MAC, 48, KEY_OPT_MAC, + offsetof(struct hclge_fd_rule, tuples.dst_mac), + offsetof(struct hclge_fd_rule, tuples_mask.dst_mac) }, + { INNER_SRC_MAC, 48, KEY_OPT_MAC, + offsetof(struct hclge_fd_rule, tuples.src_mac), + offsetof(struct hclge_fd_rule, tuples_mask.src_mac) }, + { INNER_VLAN_TAG_FST, 16, KEY_OPT_LE16, + offsetof(struct hclge_fd_rule, tuples.vlan_tag1), + offsetof(struct hclge_fd_rule, tuples_mask.vlan_tag1) }, + { INNER_VLAN_TAG_SEC, 16, KEY_OPT_LE16, -1, -1 }, + { INNER_ETH_TYPE, 16, KEY_OPT_LE16, + offsetof(struct hclge_fd_rule, tuples.ether_proto), + offsetof(struct hclge_fd_rule, tuples_mask.ether_proto) }, + { INNER_L2_RSV, 16, KEY_OPT_LE16, + offsetof(struct hclge_fd_rule, tuples.l2_user_def), + offsetof(struct hclge_fd_rule, tuples_mask.l2_user_def) }, + { INNER_IP_TOS, 8, KEY_OPT_U8, + offsetof(struct hclge_fd_rule, tuples.ip_tos), + offsetof(struct hclge_fd_rule, tuples_mask.ip_tos) }, + { INNER_IP_PROTO, 8, KEY_OPT_U8, + offsetof(struct hclge_fd_rule, tuples.ip_proto), + offsetof(struct hclge_fd_rule, tuples_mask.ip_proto) }, + { INNER_SRC_IP, 32, KEY_OPT_IP, + offsetof(struct hclge_fd_rule, tuples.src_ip), + offsetof(struct hclge_fd_rule, tuples_mask.src_ip) }, + { INNER_DST_IP, 32, KEY_OPT_IP, + offsetof(struct hclge_fd_rule, tuples.dst_ip), + offsetof(struct hclge_fd_rule, tuples_mask.dst_ip) }, + { INNER_L3_RSV, 16, KEY_OPT_LE16, + offsetof(struct hclge_fd_rule, tuples.l3_user_def), + offsetof(struct hclge_fd_rule, tuples_mask.l3_user_def) }, + { INNER_SRC_PORT, 16, KEY_OPT_LE16, + offsetof(struct hclge_fd_rule, tuples.src_port), + offsetof(struct hclge_fd_rule, tuples_mask.src_port) }, + { INNER_DST_PORT, 16, KEY_OPT_LE16, + offsetof(struct hclge_fd_rule, tuples.dst_port), + offsetof(struct hclge_fd_rule, tuples_mask.dst_port) }, + { INNER_L4_RSV, 32, KEY_OPT_LE32, + offsetof(struct hclge_fd_rule, tuples.l4_user_def), + offsetof(struct hclge_fd_rule, tuples_mask.l4_user_def) }, +}; + +static void hclge_sync_fd_state(struct hclge_dev *hdev) +{ + if (hlist_empty(&hdev->fd_rule_list)) + hdev->fd_active_type = HCLGE_FD_RULE_NONE; +} + +static void hclge_fd_inc_rule_cnt(struct hclge_dev *hdev, u16 location) +{ + if (!test_bit(location, hdev->fd_bmap)) { + set_bit(location, hdev->fd_bmap); + hdev->hclge_fd_rule_num++; + } +} + +static void hclge_fd_dec_rule_cnt(struct hclge_dev *hdev, u16 location) +{ + if (test_bit(location, hdev->fd_bmap)) { + clear_bit(location, hdev->fd_bmap); + hdev->hclge_fd_rule_num--; + } +} + +static void hclge_fd_free_node(struct hclge_dev *hdev, + struct hclge_fd_rule *rule) +{ + hlist_del(&rule->rule_node); + kfree(rule); + hclge_sync_fd_state(hdev); +} + +static void hclge_update_fd_rule_node(struct hclge_dev *hdev, + struct hclge_fd_rule *old_rule, + struct hclge_fd_rule *new_rule, + enum HCLGE_FD_NODE_STATE state) +{ + switch (state) { + case HCLGE_FD_TO_ADD: + case HCLGE_FD_ACTIVE: + /* 1) if the new state is TO_ADD, just replace the old rule + * with the same location, no matter its state, because the + * new rule will be configured to the hardware. + * 2) if the new state is ACTIVE, it means the new rule + * has been configured to the hardware, so just replace + * the old rule node with the same location. + * 3) for it doesn't add a new node to the list, so it's + * unnecessary to update the rule number and fd_bmap. + */ + new_rule->rule_node.next = old_rule->rule_node.next; + new_rule->rule_node.pprev = old_rule->rule_node.pprev; + memcpy(old_rule, new_rule, sizeof(*old_rule)); + kfree(new_rule); + break; + case HCLGE_FD_DELETED: + hclge_fd_dec_rule_cnt(hdev, old_rule->location); + hclge_fd_free_node(hdev, old_rule); + break; + case HCLGE_FD_TO_DEL: + /* if new request is TO_DEL, and old rule is existent + * 1) the state of old rule is TO_DEL, we need do nothing, + * because we delete rule by location, other rule content + * is unnecessary. + * 2) the state of old rule is ACTIVE, we need to change its + * state to TO_DEL, so the rule will be deleted when periodic + * task being scheduled. + * 3) the state of old rule is TO_ADD, it means the rule hasn't + * been added to hardware, so we just delete the rule node from + * fd_rule_list directly. + */ + if (old_rule->state == HCLGE_FD_TO_ADD) { + hclge_fd_dec_rule_cnt(hdev, old_rule->location); + hclge_fd_free_node(hdev, old_rule); + return; + } + old_rule->state = HCLGE_FD_TO_DEL; + break; + } +} + +static struct hclge_fd_rule *hclge_find_fd_rule(struct hlist_head *hlist, + u16 location, + struct hclge_fd_rule **parent) +{ + struct hclge_fd_rule *rule; + struct hlist_node *node; + + hlist_for_each_entry_safe(rule, node, hlist, rule_node) { + if (rule->location == location) + return rule; + else if (rule->location > location) + return NULL; + /* record the parent node, use to keep the nodes in fd_rule_list + * in ascend order. + */ + *parent = rule; + } + + return NULL; +} + +/* insert fd rule node in ascend order according to rule->location */ +static void hclge_fd_insert_rule_node(struct hlist_head *hlist, + struct hclge_fd_rule *rule, + struct hclge_fd_rule *parent) +{ + INIT_HLIST_NODE(&rule->rule_node); + + if (parent) + hlist_add_behind(&rule->rule_node, &parent->rule_node); + else + hlist_add_head(&rule->rule_node, hlist); +} + +static int hclge_fd_set_user_def_cmd(struct hclge_dev *hdev, + struct hclge_fd_user_def_cfg *cfg) +{ + struct hclge_fd_user_def_cfg_cmd *req; + struct hclge_desc desc; + u16 data = 0; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_USER_DEF_OP, false); + + req = (struct hclge_fd_user_def_cfg_cmd *)desc.data; + + hnae3_set_bit(data, HCLGE_FD_USER_DEF_EN_B, cfg[0].ref_cnt > 0); + hnae3_set_field(data, HCLGE_FD_USER_DEF_OFT_M, + HCLGE_FD_USER_DEF_OFT_S, cfg[0].offset); + req->ol2_cfg = cpu_to_le16(data); + + data = 0; + hnae3_set_bit(data, HCLGE_FD_USER_DEF_EN_B, cfg[1].ref_cnt > 0); + hnae3_set_field(data, HCLGE_FD_USER_DEF_OFT_M, + HCLGE_FD_USER_DEF_OFT_S, cfg[1].offset); + req->ol3_cfg = cpu_to_le16(data); + + data = 0; + hnae3_set_bit(data, HCLGE_FD_USER_DEF_EN_B, cfg[2].ref_cnt > 0); + hnae3_set_field(data, HCLGE_FD_USER_DEF_OFT_M, + HCLGE_FD_USER_DEF_OFT_S, cfg[2].offset); + req->ol4_cfg = cpu_to_le16(data); + + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + dev_err(&hdev->pdev->dev, + "failed to set fd user def data, ret= %d\n", ret); + return ret; +} + +static void hclge_sync_fd_user_def_cfg(struct hclge_dev *hdev, bool locked) +{ + int ret; + + if (!test_and_clear_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state)) + return; + + if (!locked) + spin_lock_bh(&hdev->fd_rule_lock); + + ret = hclge_fd_set_user_def_cmd(hdev, hdev->fd_cfg.user_def_cfg); + if (ret) + set_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state); + + if (!locked) + spin_unlock_bh(&hdev->fd_rule_lock); +} + +static int hclge_fd_check_user_def_refcnt(struct hclge_dev *hdev, + struct hclge_fd_rule *rule) +{ + struct hlist_head *hlist = &hdev->fd_rule_list; + struct hclge_fd_rule *fd_rule, *parent = NULL; + struct hclge_fd_user_def_info *info, *old_info; + struct hclge_fd_user_def_cfg *cfg; + + if (!rule || rule->rule_type != HCLGE_FD_EP_ACTIVE || + rule->ep.user_def.layer == HCLGE_FD_USER_DEF_NONE) + return 0; + + /* for valid layer is start from 1, so need minus 1 to get the cfg */ + cfg = &hdev->fd_cfg.user_def_cfg[rule->ep.user_def.layer - 1]; + info = &rule->ep.user_def; + + if (!cfg->ref_cnt || cfg->offset == info->offset) + return 0; + + if (cfg->ref_cnt > 1) + goto error; + + fd_rule = hclge_find_fd_rule(hlist, rule->location, &parent); + if (fd_rule) { + old_info = &fd_rule->ep.user_def; + if (info->layer == old_info->layer) + return 0; + } + +error: + dev_err(&hdev->pdev->dev, + "No available offset for layer%d fd rule, each layer only support one user def offset.\n", + info->layer + 1); + return -ENOSPC; +} + +static void hclge_fd_inc_user_def_refcnt(struct hclge_dev *hdev, + struct hclge_fd_rule *rule) +{ + struct hclge_fd_user_def_cfg *cfg; + + if (!rule || rule->rule_type != HCLGE_FD_EP_ACTIVE || + rule->ep.user_def.layer == HCLGE_FD_USER_DEF_NONE) + return; + + cfg = &hdev->fd_cfg.user_def_cfg[rule->ep.user_def.layer - 1]; + if (!cfg->ref_cnt) { + cfg->offset = rule->ep.user_def.offset; + set_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state); + } + cfg->ref_cnt++; +} + +static void hclge_fd_dec_user_def_refcnt(struct hclge_dev *hdev, + struct hclge_fd_rule *rule) +{ + struct hclge_fd_user_def_cfg *cfg; + + if (!rule || rule->rule_type != HCLGE_FD_EP_ACTIVE || + rule->ep.user_def.layer == HCLGE_FD_USER_DEF_NONE) + return; + + cfg = &hdev->fd_cfg.user_def_cfg[rule->ep.user_def.layer - 1]; + if (!cfg->ref_cnt) + return; + + cfg->ref_cnt--; + if (!cfg->ref_cnt) { + cfg->offset = 0; + set_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state); + } +} + +static void hclge_update_fd_list(struct hclge_dev *hdev, + enum HCLGE_FD_NODE_STATE state, u16 location, + struct hclge_fd_rule *new_rule) +{ + struct hlist_head *hlist = &hdev->fd_rule_list; + struct hclge_fd_rule *fd_rule, *parent = NULL; + + fd_rule = hclge_find_fd_rule(hlist, location, &parent); + if (fd_rule) { + hclge_fd_dec_user_def_refcnt(hdev, fd_rule); + if (state == HCLGE_FD_ACTIVE) + hclge_fd_inc_user_def_refcnt(hdev, new_rule); + hclge_sync_fd_user_def_cfg(hdev, true); + + hclge_update_fd_rule_node(hdev, fd_rule, new_rule, state); + return; + } + + /* it's unlikely to fail here, because we have checked the rule + * exist before. + */ + if (unlikely(state == HCLGE_FD_TO_DEL || state == HCLGE_FD_DELETED)) { + dev_warn(&hdev->pdev->dev, + "failed to delete fd rule %u, it's inexistent\n", + location); + return; + } + + hclge_fd_inc_user_def_refcnt(hdev, new_rule); + hclge_sync_fd_user_def_cfg(hdev, true); + + hclge_fd_insert_rule_node(hlist, new_rule, parent); + hclge_fd_inc_rule_cnt(hdev, new_rule->location); + + if (state == HCLGE_FD_TO_ADD) { + set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state); + hclge_task_schedule(hdev, 0); + } +} + +static int hclge_get_fd_mode(struct hclge_dev *hdev, u8 *fd_mode) +{ + struct hclge_get_fd_mode_cmd *req; + struct hclge_desc desc; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_MODE_CTRL, true); + + req = (struct hclge_get_fd_mode_cmd *)desc.data; + + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, "get fd mode fail, ret=%d\n", ret); + return ret; + } + + *fd_mode = req->mode; + + return ret; +} + +static int hclge_get_fd_allocation(struct hclge_dev *hdev, + u32 *stage1_entry_num, + u32 *stage2_entry_num, + u16 *stage1_counter_num, + u16 *stage2_counter_num) +{ + struct hclge_get_fd_allocation_cmd *req; + struct hclge_desc desc; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_GET_ALLOCATION, true); + + req = (struct hclge_get_fd_allocation_cmd *)desc.data; + + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, "query fd allocation fail, ret=%d\n", + ret); + return ret; + } + + *stage1_entry_num = le32_to_cpu(req->stage1_entry_num); + *stage2_entry_num = le32_to_cpu(req->stage2_entry_num); + *stage1_counter_num = le16_to_cpu(req->stage1_counter_num); + *stage2_counter_num = le16_to_cpu(req->stage2_counter_num); + + return ret; +} + +static int hclge_set_fd_key_config(struct hclge_dev *hdev, + enum HCLGE_FD_STAGE stage_num) +{ + struct hclge_set_fd_key_config_cmd *req; + struct hclge_fd_key_cfg *stage; + struct hclge_desc desc; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_KEY_CONFIG, false); + + req = (struct hclge_set_fd_key_config_cmd *)desc.data; + stage = &hdev->fd_cfg.key_cfg[stage_num]; + req->stage = stage_num; + req->key_select = stage->key_sel; + req->inner_sipv6_word_en = stage->inner_sipv6_word_en; + req->inner_dipv6_word_en = stage->inner_dipv6_word_en; + req->outer_sipv6_word_en = stage->outer_sipv6_word_en; + req->outer_dipv6_word_en = stage->outer_dipv6_word_en; + req->tuple_mask = cpu_to_le32(~stage->tuple_active); + req->meta_data_mask = cpu_to_le32(~stage->meta_data_active); + + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + dev_err(&hdev->pdev->dev, "set fd key fail, ret=%d\n", ret); + + return ret; +} + +static void hclge_fd_disable_user_def(struct hclge_dev *hdev) +{ + struct hclge_fd_user_def_cfg *cfg = hdev->fd_cfg.user_def_cfg; + + spin_lock_bh(&hdev->fd_rule_lock); + memset(cfg, 0, sizeof(hdev->fd_cfg.user_def_cfg)); + spin_unlock_bh(&hdev->fd_rule_lock); + + hclge_fd_set_user_def_cmd(hdev, cfg); +} + +int hclge_init_fd_config(struct hclge_dev *hdev) +{ +#define LOW_2_WORDS 0x03 + struct hclge_fd_key_cfg *key_cfg; + int ret; + + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return 0; + + ret = hclge_get_fd_mode(hdev, &hdev->fd_cfg.fd_mode); + if (ret) + return ret; + + switch (hdev->fd_cfg.fd_mode) { + case HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1: + hdev->fd_cfg.max_key_length = MAX_KEY_LENGTH; + break; + case HCLGE_FD_MODE_DEPTH_4K_WIDTH_200B_STAGE_1: + hdev->fd_cfg.max_key_length = MAX_KEY_LENGTH / 2; + break; + default: + dev_err(&hdev->pdev->dev, + "Unsupported flow director mode %u\n", + hdev->fd_cfg.fd_mode); + return -EOPNOTSUPP; + } + + key_cfg = &hdev->fd_cfg.key_cfg[HCLGE_FD_STAGE_1]; + key_cfg->key_sel = HCLGE_FD_KEY_BASE_ON_TUPLE; + key_cfg->inner_sipv6_word_en = LOW_2_WORDS; + key_cfg->inner_dipv6_word_en = LOW_2_WORDS; + key_cfg->outer_sipv6_word_en = 0; + key_cfg->outer_dipv6_word_en = 0; + + key_cfg->tuple_active = BIT(INNER_VLAN_TAG_FST) | BIT(INNER_ETH_TYPE) | + BIT(INNER_IP_PROTO) | BIT(INNER_IP_TOS) | + BIT(INNER_SRC_IP) | BIT(INNER_DST_IP) | + BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT); + + /* If use max 400bit key, we can support tuples for ether type */ + if (hdev->fd_cfg.fd_mode == HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1) { + key_cfg->tuple_active |= BIT(INNER_DST_MAC) | + BIT(INNER_SRC_MAC) | + BIT(OUTER_TUN_VNI); + if (hdev->ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) + key_cfg->tuple_active |= HCLGE_FD_TUPLE_USER_DEF_TUPLES; + } + + /* roce_type is used to filter roce frames + * dst_vport is used to specify the rule + */ + key_cfg->meta_data_active = BIT(ROCE_TYPE) | BIT(DST_VPORT); + + ret = hclge_get_fd_allocation(hdev, + &hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1], + &hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_2], + &hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1], + &hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_2]); + if (ret) + return ret; + + return hclge_set_fd_key_config(hdev, HCLGE_FD_STAGE_1); +} + +static int hclge_fd_tcam_config(struct hclge_dev *hdev, u8 stage, bool sel_x, + int loc, u8 *key, bool is_add) +{ + struct hclge_fd_tcam_config_1_cmd *req1; + struct hclge_fd_tcam_config_2_cmd *req2; + struct hclge_fd_tcam_config_3_cmd *req3; + struct hclge_desc desc[3]; + int ret; + + hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_FD_TCAM_OP, false); + desc[0].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT); + hclge_cmd_setup_basic_desc(&desc[1], HCLGE_OPC_FD_TCAM_OP, false); + desc[1].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT); + hclge_cmd_setup_basic_desc(&desc[2], HCLGE_OPC_FD_TCAM_OP, false); + + req1 = (struct hclge_fd_tcam_config_1_cmd *)desc[0].data; + req2 = (struct hclge_fd_tcam_config_2_cmd *)desc[1].data; + req3 = (struct hclge_fd_tcam_config_3_cmd *)desc[2].data; + + req1->stage = stage; + req1->xy_sel = sel_x ? 1 : 0; + hnae3_set_bit(req1->port_info, HCLGE_FD_EPORT_SW_EN_B, 0); + req1->index = cpu_to_le32(loc); + req1->entry_vld = sel_x ? is_add : 0; + + if (key) { + memcpy(req1->tcam_data, &key[0], sizeof(req1->tcam_data)); + memcpy(req2->tcam_data, &key[sizeof(req1->tcam_data)], + sizeof(req2->tcam_data)); + memcpy(req3->tcam_data, &key[sizeof(req1->tcam_data) + + sizeof(req2->tcam_data)], sizeof(req3->tcam_data)); + } + + ret = hclge_cmd_send(&hdev->hw, desc, 3); + if (ret) + dev_err(&hdev->pdev->dev, + "config tcam key fail, ret=%d\n", + ret); + + return ret; +} + +static int hclge_fd_ad_config(struct hclge_dev *hdev, u8 stage, int loc, + struct hclge_fd_ad_data *action) +{ + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); + struct hclge_fd_ad_config_cmd *req; + struct hclge_desc desc; + u64 ad_data = 0; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_AD_OP, false); + + req = (struct hclge_fd_ad_config_cmd *)desc.data; + req->index = cpu_to_le32(loc); + req->stage = stage; + + hnae3_set_bit(ad_data, HCLGE_FD_AD_WR_RULE_ID_B, + action->write_rule_id_to_bd); + hnae3_set_field(ad_data, HCLGE_FD_AD_RULE_ID_M, HCLGE_FD_AD_RULE_ID_S, + action->rule_id); + if (test_bit(HNAE3_DEV_SUPPORT_FD_FORWARD_TC_B, ae_dev->caps)) { + hnae3_set_bit(ad_data, HCLGE_FD_AD_TC_OVRD_B, + action->override_tc); + hnae3_set_field(ad_data, HCLGE_FD_AD_TC_SIZE_M, + HCLGE_FD_AD_TC_SIZE_S, (u32)action->tc_size); + } + hnae3_set_bit(ad_data, HCLGE_FD_AD_QID_H_B, + action->queue_id >= HCLGE_TQP_MAX_SIZE_DEV_V2 ? 1 : 0); + hnae3_set_bit(ad_data, HCLGE_FD_AD_COUNTER_NUM_H_B, + action->counter_id >= HCLGE_FD_COUNTER_MAX_SIZE_DEV_V2 ? + 1 : 0); + ad_data <<= 32; + hnae3_set_bit(ad_data, HCLGE_FD_AD_DROP_B, action->drop_packet); + hnae3_set_bit(ad_data, HCLGE_FD_AD_DIRECT_QID_B, + action->forward_to_direct_queue); + hnae3_set_field(ad_data, HCLGE_FD_AD_QID_L_M, HCLGE_FD_AD_QID_L_S, + action->queue_id); + hnae3_set_bit(ad_data, HCLGE_FD_AD_USE_COUNTER_B, action->use_counter); + hnae3_set_field(ad_data, HCLGE_FD_AD_COUNTER_NUM_L_M, + HCLGE_FD_AD_COUNTER_NUM_L_S, action->counter_id); + hnae3_set_bit(ad_data, HCLGE_FD_AD_NXT_STEP_B, action->use_next_stage); + hnae3_set_field(ad_data, HCLGE_FD_AD_NXT_KEY_M, HCLGE_FD_AD_NXT_KEY_S, + action->next_input_key); + + req->ad_data = cpu_to_le64(ad_data); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + dev_err(&hdev->pdev->dev, "fd ad config fail, ret=%d\n", ret); + + return ret; +} + +static bool hclge_fd_convert_tuple(u32 tuple_bit, u8 *key_x, u8 *key_y, + struct hclge_fd_rule *rule) +{ + int offset, moffset, ip_offset; + enum HCLGE_FD_KEY_OPT key_opt; + u16 tmp_x_s, tmp_y_s; + u32 tmp_x_l, tmp_y_l; + u8 *p = (u8 *)rule; + int i; + + if (rule->unused_tuple & BIT(tuple_bit)) + return true; + + key_opt = tuple_key_info[tuple_bit].key_opt; + offset = tuple_key_info[tuple_bit].offset; + moffset = tuple_key_info[tuple_bit].moffset; + + switch (key_opt) { + case KEY_OPT_U8: + calc_x(*key_x, p[offset], p[moffset]); + calc_y(*key_y, p[offset], p[moffset]); + + return true; + case KEY_OPT_LE16: + calc_x(tmp_x_s, *(u16 *)(&p[offset]), *(u16 *)(&p[moffset])); + calc_y(tmp_y_s, *(u16 *)(&p[offset]), *(u16 *)(&p[moffset])); + *(__le16 *)key_x = cpu_to_le16(tmp_x_s); + *(__le16 *)key_y = cpu_to_le16(tmp_y_s); + + return true; + case KEY_OPT_LE32: + calc_x(tmp_x_l, *(u32 *)(&p[offset]), *(u32 *)(&p[moffset])); + calc_y(tmp_y_l, *(u32 *)(&p[offset]), *(u32 *)(&p[moffset])); + *(__le32 *)key_x = cpu_to_le32(tmp_x_l); + *(__le32 *)key_y = cpu_to_le32(tmp_y_l); + + return true; + case KEY_OPT_MAC: + for (i = 0; i < ETH_ALEN; i++) { + calc_x(key_x[ETH_ALEN - 1 - i], p[offset + i], + p[moffset + i]); + calc_y(key_y[ETH_ALEN - 1 - i], p[offset + i], + p[moffset + i]); + } + + return true; + case KEY_OPT_IP: + ip_offset = IPV4_INDEX * sizeof(u32); + calc_x(tmp_x_l, *(u32 *)(&p[offset + ip_offset]), + *(u32 *)(&p[moffset + ip_offset])); + calc_y(tmp_y_l, *(u32 *)(&p[offset + ip_offset]), + *(u32 *)(&p[moffset + ip_offset])); + *(__le32 *)key_x = cpu_to_le32(tmp_x_l); + *(__le32 *)key_y = cpu_to_le32(tmp_y_l); + + return true; + case KEY_OPT_VNI: + calc_x(tmp_x_l, *(u32 *)(&p[offset]), *(u32 *)(&p[moffset])); + calc_y(tmp_y_l, *(u32 *)(&p[offset]), *(u32 *)(&p[moffset])); + for (i = 0; i < HCLGE_VNI_LENGTH; i++) { + key_x[i] = ((__force u32)cpu_to_le32(tmp_x_l) >> + (i * BITS_PER_BYTE)) & 0xFF; + key_y[i] = ((__force u32)cpu_to_le32(tmp_y_l) >> + (i * BITS_PER_BYTE)) & 0xFF; + } + + return true; + default: + return false; + } +} + +static void hclge_fd_convert_meta_data(struct hclge_fd_key_cfg *key_cfg, + __le32 *key_x, __le32 *key_y, + struct hclge_fd_rule *rule) +{ + u32 tuple_bit, meta_data = 0, tmp_x, tmp_y, port_number; + u8 cur_pos = 0, tuple_size, shift_bits; + unsigned int i; + + for (i = 0; i < MAX_META_DATA; i++) { + tuple_size = meta_data_key_info[i].key_length; + tuple_bit = key_cfg->meta_data_active & BIT(i); + + switch (tuple_bit) { + case BIT(ROCE_TYPE): + hnae3_set_bit(meta_data, cur_pos, NIC_PACKET); + cur_pos += tuple_size; + break; + case BIT(DST_VPORT): + port_number = hclge_get_port_number(HOST_PORT, 0, + rule->vf_id, 0); + hnae3_set_field(meta_data, + GENMASK(cur_pos + tuple_size, cur_pos), + cur_pos, port_number); + cur_pos += tuple_size; + break; + default: + break; + } + } + + calc_x(tmp_x, meta_data, 0xFFFFFFFF); + calc_y(tmp_y, meta_data, 0xFFFFFFFF); + shift_bits = sizeof(meta_data) * 8 - cur_pos; + + *key_x = cpu_to_le32(tmp_x << shift_bits); + *key_y = cpu_to_le32(tmp_y << shift_bits); +} + +/* A complete key is combined with meta data key and tuple key. + * Meta data key is stored at the MSB region, and tuple key is stored at + * the LSB region, unused bits will be filled 0. + */ +static int hclge_config_key(struct hclge_dev *hdev, u8 stage, + struct hclge_fd_rule *rule) +{ + struct hclge_fd_key_cfg *key_cfg = &hdev->fd_cfg.key_cfg[stage]; + u8 key_x[MAX_KEY_BYTES], key_y[MAX_KEY_BYTES]; + u8 *cur_key_x, *cur_key_y; + u8 meta_data_region; + u8 tuple_size; + int ret; + u32 i; + + memset(key_x, 0, sizeof(key_x)); + memset(key_y, 0, sizeof(key_y)); + cur_key_x = key_x; + cur_key_y = key_y; + + for (i = 0; i < MAX_TUPLE; i++) { + bool tuple_valid; + + tuple_size = tuple_key_info[i].key_length / 8; + if (!(key_cfg->tuple_active & BIT(i))) + continue; + + tuple_valid = hclge_fd_convert_tuple(i, cur_key_x, + cur_key_y, rule); + if (tuple_valid) { + cur_key_x += tuple_size; + cur_key_y += tuple_size; + } + } + + meta_data_region = hdev->fd_cfg.max_key_length / 8 - + MAX_META_DATA_LENGTH / 8; + + hclge_fd_convert_meta_data(key_cfg, + (__le32 *)(key_x + meta_data_region), + (__le32 *)(key_y + meta_data_region), + rule); + + ret = hclge_fd_tcam_config(hdev, stage, false, rule->location, key_y, + true); + if (ret) { + dev_err(&hdev->pdev->dev, + "fd key_y config fail, loc=%u, ret=%d\n", + rule->queue_id, ret); + return ret; + } + + ret = hclge_fd_tcam_config(hdev, stage, true, rule->location, key_x, + true); + if (ret) + dev_err(&hdev->pdev->dev, + "fd key_x config fail, loc=%u, ret=%d\n", + rule->queue_id, ret); + return ret; +} + +static int hclge_config_action(struct hclge_dev *hdev, u8 stage, + struct hclge_fd_rule *rule) +{ + struct hclge_vport *vport = hdev->vport; + struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo; + struct hclge_fd_ad_data ad_data; + + memset(&ad_data, 0, sizeof(struct hclge_fd_ad_data)); + ad_data.ad_id = rule->location; + + if (rule->action == HCLGE_FD_ACTION_DROP_PACKET) { + ad_data.drop_packet = true; + } else if (rule->action == HCLGE_FD_ACTION_SELECT_TC) { + ad_data.override_tc = true; + ad_data.queue_id = + kinfo->tc_info.tqp_offset[rule->cls_flower.tc]; + ad_data.tc_size = + ilog2(kinfo->tc_info.tqp_count[rule->cls_flower.tc]); + } else { + ad_data.forward_to_direct_queue = true; + ad_data.queue_id = rule->queue_id; + } + + if (hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1]) { + ad_data.use_counter = true; + ad_data.counter_id = rule->vf_id % + hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1]; + } else { + ad_data.use_counter = false; + ad_data.counter_id = 0; + } + + ad_data.use_next_stage = false; + ad_data.next_input_key = 0; + + ad_data.write_rule_id_to_bd = true; + ad_data.rule_id = rule->location; + + return hclge_fd_ad_config(hdev, stage, ad_data.ad_id, &ad_data); +} + +static int hclge_fd_check_tcpip4_tuple(struct ethtool_tcpip4_spec *spec, + u32 *unused_tuple) +{ + if (!spec || !unused_tuple) + return -EINVAL; + + *unused_tuple |= BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC); + + if (!spec->ip4src) + *unused_tuple |= BIT(INNER_SRC_IP); + + if (!spec->ip4dst) + *unused_tuple |= BIT(INNER_DST_IP); + + if (!spec->psrc) + *unused_tuple |= BIT(INNER_SRC_PORT); + + if (!spec->pdst) + *unused_tuple |= BIT(INNER_DST_PORT); + + if (!spec->tos) + *unused_tuple |= BIT(INNER_IP_TOS); + + return 0; +} + +static int hclge_fd_check_ip4_tuple(struct ethtool_usrip4_spec *spec, + u32 *unused_tuple) +{ + if (!spec || !unused_tuple) + return -EINVAL; + + *unused_tuple |= BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC) | + BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT); + + if (!spec->ip4src) + *unused_tuple |= BIT(INNER_SRC_IP); + + if (!spec->ip4dst) + *unused_tuple |= BIT(INNER_DST_IP); + + if (!spec->tos) + *unused_tuple |= BIT(INNER_IP_TOS); + + if (!spec->proto) + *unused_tuple |= BIT(INNER_IP_PROTO); + + if (spec->l4_4_bytes) + return -EOPNOTSUPP; + + if (spec->ip_ver != ETH_RX_NFC_IP4) + return -EOPNOTSUPP; + + return 0; +} + +static int hclge_fd_check_tcpip6_tuple(struct ethtool_tcpip6_spec *spec, + u32 *unused_tuple) +{ + if (!spec || !unused_tuple) + return -EINVAL; + + *unused_tuple |= BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC); + + /* check whether src/dst ip address used */ + if (ipv6_addr_any((struct in6_addr *)spec->ip6src)) + *unused_tuple |= BIT(INNER_SRC_IP); + + if (ipv6_addr_any((struct in6_addr *)spec->ip6dst)) + *unused_tuple |= BIT(INNER_DST_IP); + + if (!spec->psrc) + *unused_tuple |= BIT(INNER_SRC_PORT); + + if (!spec->pdst) + *unused_tuple |= BIT(INNER_DST_PORT); + + if (!spec->tclass) + *unused_tuple |= BIT(INNER_IP_TOS); + + return 0; +} + +static int hclge_fd_check_ip6_tuple(struct ethtool_usrip6_spec *spec, + u32 *unused_tuple) +{ + if (!spec || !unused_tuple) + return -EINVAL; + + *unused_tuple |= BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC) | + BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT); + + /* check whether src/dst ip address used */ + if (ipv6_addr_any((struct in6_addr *)spec->ip6src)) + *unused_tuple |= BIT(INNER_SRC_IP); + + if (ipv6_addr_any((struct in6_addr *)spec->ip6dst)) + *unused_tuple |= BIT(INNER_DST_IP); + + if (!spec->l4_proto) + *unused_tuple |= BIT(INNER_IP_PROTO); + + if (!spec->tclass) + *unused_tuple |= BIT(INNER_IP_TOS); + + if (spec->l4_4_bytes) + return -EOPNOTSUPP; + + return 0; +} + +static int hclge_fd_check_ether_tuple(struct ethhdr *spec, u32 *unused_tuple) +{ + if (!spec || !unused_tuple) + return -EINVAL; + + *unused_tuple |= BIT(INNER_SRC_IP) | BIT(INNER_DST_IP) | + BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT) | + BIT(INNER_IP_TOS) | BIT(INNER_IP_PROTO); + + if (is_zero_ether_addr(spec->h_source)) + *unused_tuple |= BIT(INNER_SRC_MAC); + + if (is_zero_ether_addr(spec->h_dest)) + *unused_tuple |= BIT(INNER_DST_MAC); + + if (!spec->h_proto) + *unused_tuple |= BIT(INNER_ETH_TYPE); + + return 0; +} + +static int hclge_fd_check_ext_tuple(struct hclge_dev *hdev, + struct ethtool_rx_flow_spec *fs, + u32 *unused_tuple) +{ + if (fs->flow_type & FLOW_EXT) { + if (fs->h_ext.vlan_etype) { + dev_err(&hdev->pdev->dev, "vlan-etype is not supported!\n"); + return -EOPNOTSUPP; + } + + if (!fs->h_ext.vlan_tci) + *unused_tuple |= BIT(INNER_VLAN_TAG_FST); + + if (fs->m_ext.vlan_tci && + be16_to_cpu(fs->h_ext.vlan_tci) >= VLAN_N_VID) { + dev_err(&hdev->pdev->dev, + "failed to config vlan_tci, invalid vlan_tci: %u, max is %d.\n", + ntohs(fs->h_ext.vlan_tci), VLAN_N_VID - 1); + return -EINVAL; + } + } else { + *unused_tuple |= BIT(INNER_VLAN_TAG_FST); + } + + if (fs->flow_type & FLOW_MAC_EXT) { + if (hdev->fd_cfg.fd_mode != + HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1) { + dev_err(&hdev->pdev->dev, + "FLOW_MAC_EXT is not supported in current fd mode!\n"); + return -EOPNOTSUPP; + } + + if (is_zero_ether_addr(fs->h_ext.h_dest)) + *unused_tuple |= BIT(INNER_DST_MAC); + else + *unused_tuple &= ~BIT(INNER_DST_MAC); + } + + return 0; +} + +static int hclge_fd_get_user_def_layer(u32 flow_type, u32 *unused_tuple, + struct hclge_fd_user_def_info *info) +{ + switch (flow_type) { + case ETHER_FLOW: + info->layer = HCLGE_FD_USER_DEF_L2; + *unused_tuple &= ~BIT(INNER_L2_RSV); + break; + case IP_USER_FLOW: + case IPV6_USER_FLOW: + info->layer = HCLGE_FD_USER_DEF_L3; + *unused_tuple &= ~BIT(INNER_L3_RSV); + break; + case TCP_V4_FLOW: + case UDP_V4_FLOW: + case TCP_V6_FLOW: + case UDP_V6_FLOW: + info->layer = HCLGE_FD_USER_DEF_L4; + *unused_tuple &= ~BIT(INNER_L4_RSV); + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static bool hclge_fd_is_user_def_all_masked(struct ethtool_rx_flow_spec *fs) +{ + return be32_to_cpu(fs->m_ext.data[1] | fs->m_ext.data[0]) == 0; +} + +static int hclge_fd_parse_user_def_field(struct hclge_dev *hdev, + struct ethtool_rx_flow_spec *fs, + u32 *unused_tuple, + struct hclge_fd_user_def_info *info) +{ + u32 tuple_active = hdev->fd_cfg.key_cfg[HCLGE_FD_STAGE_1].tuple_active; + u32 flow_type = fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT); + u16 data, offset, data_mask, offset_mask; + int ret; + + info->layer = HCLGE_FD_USER_DEF_NONE; + *unused_tuple |= HCLGE_FD_TUPLE_USER_DEF_TUPLES; + + if (!(fs->flow_type & FLOW_EXT) || hclge_fd_is_user_def_all_masked(fs)) + return 0; + + /* user-def data from ethtool is 64 bit value, the bit0~15 is used + * for data, and bit32~47 is used for offset. + */ + data = be32_to_cpu(fs->h_ext.data[1]) & HCLGE_FD_USER_DEF_DATA; + data_mask = be32_to_cpu(fs->m_ext.data[1]) & HCLGE_FD_USER_DEF_DATA; + offset = be32_to_cpu(fs->h_ext.data[0]) & HCLGE_FD_USER_DEF_OFFSET; + offset_mask = be32_to_cpu(fs->m_ext.data[0]) & HCLGE_FD_USER_DEF_OFFSET; + + if (!(tuple_active & HCLGE_FD_TUPLE_USER_DEF_TUPLES)) { + dev_err(&hdev->pdev->dev, "user-def bytes are not supported\n"); + return -EOPNOTSUPP; + } + + if (offset > HCLGE_FD_MAX_USER_DEF_OFFSET) { + dev_err(&hdev->pdev->dev, + "user-def offset[%u] should be no more than %u\n", + offset, HCLGE_FD_MAX_USER_DEF_OFFSET); + return -EINVAL; + } + + if (offset_mask != HCLGE_FD_USER_DEF_OFFSET_UNMASK) { + dev_err(&hdev->pdev->dev, "user-def offset can't be masked\n"); + return -EINVAL; + } + + ret = hclge_fd_get_user_def_layer(flow_type, unused_tuple, info); + if (ret) { + dev_err(&hdev->pdev->dev, + "unsupported flow type for user-def bytes, ret = %d\n", + ret); + return ret; + } + + info->data = data; + info->data_mask = data_mask; + info->offset = offset; + + return 0; +} + +static int hclge_fd_check_spec(struct hclge_dev *hdev, + struct ethtool_rx_flow_spec *fs, + u32 *unused_tuple, + struct hclge_fd_user_def_info *info) +{ + u32 flow_type; + int ret; + + if (fs->location >= hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) { + dev_err(&hdev->pdev->dev, + "failed to config fd rules, invalid rule location: %u, max is %u\n.", + fs->location, + hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1] - 1); + return -EINVAL; + } + + ret = hclge_fd_parse_user_def_field(hdev, fs, unused_tuple, info); + if (ret) + return ret; + + flow_type = fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT); + switch (flow_type) { + case SCTP_V4_FLOW: + case TCP_V4_FLOW: + case UDP_V4_FLOW: + ret = hclge_fd_check_tcpip4_tuple(&fs->h_u.tcp_ip4_spec, + unused_tuple); + break; + case IP_USER_FLOW: + ret = hclge_fd_check_ip4_tuple(&fs->h_u.usr_ip4_spec, + unused_tuple); + break; + case SCTP_V6_FLOW: + case TCP_V6_FLOW: + case UDP_V6_FLOW: + ret = hclge_fd_check_tcpip6_tuple(&fs->h_u.tcp_ip6_spec, + unused_tuple); + break; + case IPV6_USER_FLOW: + ret = hclge_fd_check_ip6_tuple(&fs->h_u.usr_ip6_spec, + unused_tuple); + break; + case ETHER_FLOW: + if (hdev->fd_cfg.fd_mode != + HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1) { + dev_err(&hdev->pdev->dev, + "ETHER_FLOW is not supported in current fd mode!\n"); + return -EOPNOTSUPP; + } + + ret = hclge_fd_check_ether_tuple(&fs->h_u.ether_spec, + unused_tuple); + break; + default: + dev_err(&hdev->pdev->dev, + "unsupported protocol type, protocol type = %#x\n", + flow_type); + return -EOPNOTSUPP; + } + + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to check flow union tuple, ret = %d\n", + ret); + return ret; + } + + return hclge_fd_check_ext_tuple(hdev, fs, unused_tuple); +} + +static void hclge_fd_get_tcpip4_tuple(struct ethtool_rx_flow_spec *fs, + struct hclge_fd_rule *rule, u8 ip_proto) +{ + rule->tuples.src_ip[IPV4_INDEX] = + be32_to_cpu(fs->h_u.tcp_ip4_spec.ip4src); + rule->tuples_mask.src_ip[IPV4_INDEX] = + be32_to_cpu(fs->m_u.tcp_ip4_spec.ip4src); + + rule->tuples.dst_ip[IPV4_INDEX] = + be32_to_cpu(fs->h_u.tcp_ip4_spec.ip4dst); + rule->tuples_mask.dst_ip[IPV4_INDEX] = + be32_to_cpu(fs->m_u.tcp_ip4_spec.ip4dst); + + rule->tuples.src_port = be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc); + rule->tuples_mask.src_port = be16_to_cpu(fs->m_u.tcp_ip4_spec.psrc); + + rule->tuples.dst_port = be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst); + rule->tuples_mask.dst_port = be16_to_cpu(fs->m_u.tcp_ip4_spec.pdst); + + rule->tuples.ip_tos = fs->h_u.tcp_ip4_spec.tos; + rule->tuples_mask.ip_tos = fs->m_u.tcp_ip4_spec.tos; + + rule->tuples.ether_proto = ETH_P_IP; + rule->tuples_mask.ether_proto = 0xFFFF; + + rule->tuples.ip_proto = ip_proto; + rule->tuples_mask.ip_proto = 0xFF; +} + +static void hclge_fd_get_ip4_tuple(struct ethtool_rx_flow_spec *fs, + struct hclge_fd_rule *rule) +{ + rule->tuples.src_ip[IPV4_INDEX] = + be32_to_cpu(fs->h_u.usr_ip4_spec.ip4src); + rule->tuples_mask.src_ip[IPV4_INDEX] = + be32_to_cpu(fs->m_u.usr_ip4_spec.ip4src); + + rule->tuples.dst_ip[IPV4_INDEX] = + be32_to_cpu(fs->h_u.usr_ip4_spec.ip4dst); + rule->tuples_mask.dst_ip[IPV4_INDEX] = + be32_to_cpu(fs->m_u.usr_ip4_spec.ip4dst); + + rule->tuples.ip_tos = fs->h_u.usr_ip4_spec.tos; + rule->tuples_mask.ip_tos = fs->m_u.usr_ip4_spec.tos; + + rule->tuples.ip_proto = fs->h_u.usr_ip4_spec.proto; + rule->tuples_mask.ip_proto = fs->m_u.usr_ip4_spec.proto; + + rule->tuples.ether_proto = ETH_P_IP; + rule->tuples_mask.ether_proto = 0xFFFF; +} + +static void hclge_fd_get_tcpip6_tuple(struct ethtool_rx_flow_spec *fs, + struct hclge_fd_rule *rule, u8 ip_proto) +{ + ipv6_addr_be32_to_cpu(rule->tuples.src_ip, + fs->h_u.tcp_ip6_spec.ip6src); + ipv6_addr_be32_to_cpu(rule->tuples_mask.src_ip, + fs->m_u.tcp_ip6_spec.ip6src); + + ipv6_addr_be32_to_cpu(rule->tuples.dst_ip, + fs->h_u.tcp_ip6_spec.ip6dst); + ipv6_addr_be32_to_cpu(rule->tuples_mask.dst_ip, + fs->m_u.tcp_ip6_spec.ip6dst); + + rule->tuples.src_port = be16_to_cpu(fs->h_u.tcp_ip6_spec.psrc); + rule->tuples_mask.src_port = be16_to_cpu(fs->m_u.tcp_ip6_spec.psrc); + + rule->tuples.dst_port = be16_to_cpu(fs->h_u.tcp_ip6_spec.pdst); + rule->tuples_mask.dst_port = be16_to_cpu(fs->m_u.tcp_ip6_spec.pdst); + + rule->tuples.ether_proto = ETH_P_IPV6; + rule->tuples_mask.ether_proto = 0xFFFF; + + rule->tuples.ip_tos = fs->h_u.tcp_ip6_spec.tclass; + rule->tuples_mask.ip_tos = fs->m_u.tcp_ip6_spec.tclass; + + rule->tuples.ip_proto = ip_proto; + rule->tuples_mask.ip_proto = 0xFF; +} + +static void hclge_fd_get_ip6_tuple(struct ethtool_rx_flow_spec *fs, + struct hclge_fd_rule *rule) +{ + ipv6_addr_be32_to_cpu(rule->tuples.src_ip, + fs->h_u.usr_ip6_spec.ip6src); + ipv6_addr_be32_to_cpu(rule->tuples_mask.src_ip, + fs->m_u.usr_ip6_spec.ip6src); + + ipv6_addr_be32_to_cpu(rule->tuples.dst_ip, + fs->h_u.usr_ip6_spec.ip6dst); + ipv6_addr_be32_to_cpu(rule->tuples_mask.dst_ip, + fs->m_u.usr_ip6_spec.ip6dst); + + rule->tuples.ip_proto = fs->h_u.usr_ip6_spec.l4_proto; + rule->tuples_mask.ip_proto = fs->m_u.usr_ip6_spec.l4_proto; + + rule->tuples.ip_tos = fs->h_u.tcp_ip6_spec.tclass; + rule->tuples_mask.ip_tos = fs->m_u.tcp_ip6_spec.tclass; + + rule->tuples.ether_proto = ETH_P_IPV6; + rule->tuples_mask.ether_proto = 0xFFFF; +} + +static void hclge_fd_get_ether_tuple(struct ethtool_rx_flow_spec *fs, + struct hclge_fd_rule *rule) +{ + ether_addr_copy(rule->tuples.src_mac, fs->h_u.ether_spec.h_source); + ether_addr_copy(rule->tuples_mask.src_mac, fs->m_u.ether_spec.h_source); + + ether_addr_copy(rule->tuples.dst_mac, fs->h_u.ether_spec.h_dest); + ether_addr_copy(rule->tuples_mask.dst_mac, fs->m_u.ether_spec.h_dest); + + rule->tuples.ether_proto = be16_to_cpu(fs->h_u.ether_spec.h_proto); + rule->tuples_mask.ether_proto = be16_to_cpu(fs->m_u.ether_spec.h_proto); +} + +static void hclge_fd_get_user_def_tuple(struct hclge_fd_user_def_info *info, + struct hclge_fd_rule *rule) +{ + switch (info->layer) { + case HCLGE_FD_USER_DEF_L2: + rule->tuples.l2_user_def = info->data; + rule->tuples_mask.l2_user_def = info->data_mask; + break; + case HCLGE_FD_USER_DEF_L3: + rule->tuples.l3_user_def = info->data; + rule->tuples_mask.l3_user_def = info->data_mask; + break; + case HCLGE_FD_USER_DEF_L4: + rule->tuples.l4_user_def = (u32)info->data << 16; + rule->tuples_mask.l4_user_def = (u32)info->data_mask << 16; + break; + default: + break; + } + + rule->ep.user_def = *info; +} + +static int hclge_fd_get_tuple(struct ethtool_rx_flow_spec *fs, + struct hclge_fd_rule *rule, + struct hclge_fd_user_def_info *info) +{ + u32 flow_type = fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT); + + switch (flow_type) { + case SCTP_V4_FLOW: + hclge_fd_get_tcpip4_tuple(fs, rule, IPPROTO_SCTP); + break; + case TCP_V4_FLOW: + hclge_fd_get_tcpip4_tuple(fs, rule, IPPROTO_TCP); + break; + case UDP_V4_FLOW: + hclge_fd_get_tcpip4_tuple(fs, rule, IPPROTO_UDP); + break; + case IP_USER_FLOW: + hclge_fd_get_ip4_tuple(fs, rule); + break; + case SCTP_V6_FLOW: + hclge_fd_get_tcpip6_tuple(fs, rule, IPPROTO_SCTP); + break; + case TCP_V6_FLOW: + hclge_fd_get_tcpip6_tuple(fs, rule, IPPROTO_TCP); + break; + case UDP_V6_FLOW: + hclge_fd_get_tcpip6_tuple(fs, rule, IPPROTO_UDP); + break; + case IPV6_USER_FLOW: + hclge_fd_get_ip6_tuple(fs, rule); + break; + case ETHER_FLOW: + hclge_fd_get_ether_tuple(fs, rule); + break; + default: + return -EOPNOTSUPP; + } + + if (fs->flow_type & FLOW_EXT) { + rule->tuples.vlan_tag1 = be16_to_cpu(fs->h_ext.vlan_tci); + rule->tuples_mask.vlan_tag1 = be16_to_cpu(fs->m_ext.vlan_tci); + hclge_fd_get_user_def_tuple(info, rule); + } + + if (fs->flow_type & FLOW_MAC_EXT) { + ether_addr_copy(rule->tuples.dst_mac, fs->h_ext.h_dest); + ether_addr_copy(rule->tuples_mask.dst_mac, fs->m_ext.h_dest); + } + + return 0; +} + +static int hclge_fd_config_rule(struct hclge_dev *hdev, + struct hclge_fd_rule *rule) +{ + int ret; + + ret = hclge_config_action(hdev, HCLGE_FD_STAGE_1, rule); + if (ret) + return ret; + + return hclge_config_key(hdev, HCLGE_FD_STAGE_1, rule); +} + +static int hclge_add_fd_entry_common(struct hclge_dev *hdev, + struct hclge_fd_rule *rule) +{ + int ret; + + spin_lock_bh(&hdev->fd_rule_lock); + + if (hdev->fd_active_type != rule->rule_type && + (hdev->fd_active_type == HCLGE_FD_TC_FLOWER_ACTIVE || + hdev->fd_active_type == HCLGE_FD_EP_ACTIVE)) { + dev_err(&hdev->pdev->dev, + "mode conflict(new type %d, active type %d), please delete existent rules first\n", + rule->rule_type, hdev->fd_active_type); + spin_unlock_bh(&hdev->fd_rule_lock); + return -EINVAL; + } + + ret = hclge_fd_check_user_def_refcnt(hdev, rule); + if (ret) + goto out; + + ret = hclge_clear_arfs_rules(hdev); + if (ret) + goto out; + + ret = hclge_fd_config_rule(hdev, rule); + if (ret) + goto out; + + rule->state = HCLGE_FD_ACTIVE; + hdev->fd_active_type = rule->rule_type; + hclge_update_fd_list(hdev, rule->state, rule->location, rule); + +out: + spin_unlock_bh(&hdev->fd_rule_lock); + return ret; +} + +bool hclge_is_cls_flower_active(struct hnae3_handle *handle) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + + return hdev->fd_active_type == HCLGE_FD_TC_FLOWER_ACTIVE; +} + +static int hclge_fd_parse_ring_cookie(struct hclge_dev *hdev, u64 ring_cookie, + u16 *vport_id, u8 *action, u16 *queue_id) +{ + struct hclge_vport *vport = hdev->vport; + + if (ring_cookie == RX_CLS_FLOW_DISC) { + *action = HCLGE_FD_ACTION_DROP_PACKET; + } else { + u32 ring = ethtool_get_flow_spec_ring(ring_cookie); + u8 vf = ethtool_get_flow_spec_ring_vf(ring_cookie); + u16 tqps; + + /* To keep consistent with user's configuration, minus 1 when + * printing 'vf', because vf id from ethtool is added 1 for vf. + */ + if (vf > hdev->num_req_vfs) { + dev_err(&hdev->pdev->dev, + "Error: vf id (%u) should be less than %u\n", + vf - 1U, hdev->num_req_vfs); + return -EINVAL; + } + + *vport_id = vf ? hdev->vport[vf].vport_id : vport->vport_id; + tqps = hdev->vport[vf].nic.kinfo.num_tqps; + + if (ring >= tqps) { + dev_err(&hdev->pdev->dev, + "Error: queue id (%u) > max tqp num (%u)\n", + ring, tqps - 1U); + return -EINVAL; + } + + *action = HCLGE_FD_ACTION_SELECT_QUEUE; + *queue_id = ring; + } + + return 0; +} + +int hclge_add_fd_entry(struct hnae3_handle *handle, struct ethtool_rxnfc *cmd) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + struct hclge_fd_user_def_info info; + u16 dst_vport_id = 0, q_index = 0; + struct ethtool_rx_flow_spec *fs; + struct hclge_fd_rule *rule; + u32 unused = 0; + u8 action; + int ret; + + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) { + dev_err(&hdev->pdev->dev, + "flow table director is not supported\n"); + return -EOPNOTSUPP; + } + + if (!hdev->fd_en) { + dev_err(&hdev->pdev->dev, + "please enable flow director first\n"); + return -EOPNOTSUPP; + } + + fs = (struct ethtool_rx_flow_spec *)&cmd->fs; + + ret = hclge_fd_check_spec(hdev, fs, &unused, &info); + if (ret) + return ret; + + ret = hclge_fd_parse_ring_cookie(hdev, fs->ring_cookie, &dst_vport_id, + &action, &q_index); + if (ret) + return ret; + + rule = kzalloc_obj(*rule); + if (!rule) + return -ENOMEM; + + ret = hclge_fd_get_tuple(fs, rule, &info); + if (ret) { + kfree(rule); + return ret; + } + + rule->flow_type = fs->flow_type; + rule->location = fs->location; + rule->unused_tuple = unused; + rule->vf_id = dst_vport_id; + rule->queue_id = q_index; + rule->action = action; + rule->rule_type = HCLGE_FD_EP_ACTIVE; + + ret = hclge_add_fd_entry_common(hdev, rule); + if (ret) + kfree(rule); + + return ret; +} + +int hclge_del_fd_entry(struct hnae3_handle *handle, struct ethtool_rxnfc *cmd) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + struct ethtool_rx_flow_spec *fs; + int ret; + + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return -EOPNOTSUPP; + + fs = (struct ethtool_rx_flow_spec *)&cmd->fs; + + if (fs->location >= hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) + return -EINVAL; + + spin_lock_bh(&hdev->fd_rule_lock); + if (hdev->fd_active_type == HCLGE_FD_TC_FLOWER_ACTIVE || + !test_bit(fs->location, hdev->fd_bmap)) { + dev_err(&hdev->pdev->dev, + "Delete fail, rule %u is inexistent\n", fs->location); + spin_unlock_bh(&hdev->fd_rule_lock); + return -ENOENT; + } + + ret = hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, fs->location, + NULL, false); + if (ret) + goto out; + + hclge_update_fd_list(hdev, HCLGE_FD_DELETED, fs->location, NULL); + +out: + spin_unlock_bh(&hdev->fd_rule_lock); + return ret; +} + +static void hclge_clear_fd_rules_in_list(struct hclge_dev *hdev, + bool clear_list) +{ + struct hclge_fd_rule *rule; + struct hlist_node *node; + u16 location; + + spin_lock_bh(&hdev->fd_rule_lock); + + for_each_set_bit(location, hdev->fd_bmap, + hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) + hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, location, + NULL, false); + + if (clear_list) { + hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, + rule_node) { + hlist_del(&rule->rule_node); + kfree(rule); + } + hdev->fd_active_type = HCLGE_FD_RULE_NONE; + hdev->hclge_fd_rule_num = 0; + bitmap_zero(hdev->fd_bmap, + hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]); + } + + spin_unlock_bh(&hdev->fd_rule_lock); +} + +void hclge_del_all_fd_entries(struct hclge_dev *hdev) +{ + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return; + + hclge_clear_fd_rules_in_list(hdev, true); + hclge_fd_disable_user_def(hdev); +} + +int hclge_restore_fd_entries(struct hnae3_handle *handle) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + struct hclge_fd_rule *rule; + struct hlist_node *node; + + /* Return ok here, because reset error handling will check this + * return value. If error is returned here, the reset process will + * fail. + */ + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return 0; + + /* if fd is disabled, should not restore it when reset */ + if (!hdev->fd_en) + return 0; + + spin_lock_bh(&hdev->fd_rule_lock); + hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) { + if (rule->state == HCLGE_FD_ACTIVE) + rule->state = HCLGE_FD_TO_ADD; + } + spin_unlock_bh(&hdev->fd_rule_lock); + set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state); + + return 0; +} + +int hclge_get_fd_rule_cnt(struct hnae3_handle *handle, + struct ethtool_rxnfc *cmd) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev) || + hclge_is_cls_flower_active(handle)) + return -EOPNOTSUPP; + + cmd->rule_cnt = hdev->hclge_fd_rule_num; + cmd->data = hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]; + + return 0; +} + +static void hclge_fd_get_tcpip4_info(struct hclge_fd_rule *rule, + struct ethtool_tcpip4_spec *spec, + struct ethtool_tcpip4_spec *spec_mask) +{ + spec->ip4src = cpu_to_be32(rule->tuples.src_ip[IPV4_INDEX]); + spec_mask->ip4src = rule->unused_tuple & BIT(INNER_SRC_IP) ? + 0 : cpu_to_be32(rule->tuples_mask.src_ip[IPV4_INDEX]); + + spec->ip4dst = cpu_to_be32(rule->tuples.dst_ip[IPV4_INDEX]); + spec_mask->ip4dst = rule->unused_tuple & BIT(INNER_DST_IP) ? + 0 : cpu_to_be32(rule->tuples_mask.dst_ip[IPV4_INDEX]); + + spec->psrc = cpu_to_be16(rule->tuples.src_port); + spec_mask->psrc = rule->unused_tuple & BIT(INNER_SRC_PORT) ? + 0 : cpu_to_be16(rule->tuples_mask.src_port); + + spec->pdst = cpu_to_be16(rule->tuples.dst_port); + spec_mask->pdst = rule->unused_tuple & BIT(INNER_DST_PORT) ? + 0 : cpu_to_be16(rule->tuples_mask.dst_port); + + spec->tos = rule->tuples.ip_tos; + spec_mask->tos = rule->unused_tuple & BIT(INNER_IP_TOS) ? + 0 : rule->tuples_mask.ip_tos; +} + +static void hclge_fd_get_ip4_info(struct hclge_fd_rule *rule, + struct ethtool_usrip4_spec *spec, + struct ethtool_usrip4_spec *spec_mask) +{ + spec->ip4src = cpu_to_be32(rule->tuples.src_ip[IPV4_INDEX]); + spec_mask->ip4src = rule->unused_tuple & BIT(INNER_SRC_IP) ? + 0 : cpu_to_be32(rule->tuples_mask.src_ip[IPV4_INDEX]); + + spec->ip4dst = cpu_to_be32(rule->tuples.dst_ip[IPV4_INDEX]); + spec_mask->ip4dst = rule->unused_tuple & BIT(INNER_DST_IP) ? + 0 : cpu_to_be32(rule->tuples_mask.dst_ip[IPV4_INDEX]); + + spec->tos = rule->tuples.ip_tos; + spec_mask->tos = rule->unused_tuple & BIT(INNER_IP_TOS) ? + 0 : rule->tuples_mask.ip_tos; + + spec->proto = rule->tuples.ip_proto; + spec_mask->proto = rule->unused_tuple & BIT(INNER_IP_PROTO) ? + 0 : rule->tuples_mask.ip_proto; + + spec->ip_ver = ETH_RX_NFC_IP4; +} + +static void hclge_fd_get_tcpip6_info(struct hclge_fd_rule *rule, + struct ethtool_tcpip6_spec *spec, + struct ethtool_tcpip6_spec *spec_mask) +{ + ipv6_addr_cpu_to_be32(spec->ip6src, rule->tuples.src_ip); + ipv6_addr_cpu_to_be32(spec->ip6dst, rule->tuples.dst_ip); + if (rule->unused_tuple & BIT(INNER_SRC_IP)) + memset(spec_mask->ip6src, 0, sizeof(spec_mask->ip6src)); + else + ipv6_addr_cpu_to_be32(spec_mask->ip6src, + rule->tuples_mask.src_ip); + + if (rule->unused_tuple & BIT(INNER_DST_IP)) + memset(spec_mask->ip6dst, 0, sizeof(spec_mask->ip6dst)); + else + ipv6_addr_cpu_to_be32(spec_mask->ip6dst, + rule->tuples_mask.dst_ip); + + spec->tclass = rule->tuples.ip_tos; + spec_mask->tclass = rule->unused_tuple & BIT(INNER_IP_TOS) ? + 0 : rule->tuples_mask.ip_tos; + + spec->psrc = cpu_to_be16(rule->tuples.src_port); + spec_mask->psrc = rule->unused_tuple & BIT(INNER_SRC_PORT) ? + 0 : cpu_to_be16(rule->tuples_mask.src_port); + + spec->pdst = cpu_to_be16(rule->tuples.dst_port); + spec_mask->pdst = rule->unused_tuple & BIT(INNER_DST_PORT) ? + 0 : cpu_to_be16(rule->tuples_mask.dst_port); +} + +static void hclge_fd_get_ip6_info(struct hclge_fd_rule *rule, + struct ethtool_usrip6_spec *spec, + struct ethtool_usrip6_spec *spec_mask) +{ + ipv6_addr_cpu_to_be32(spec->ip6src, rule->tuples.src_ip); + ipv6_addr_cpu_to_be32(spec->ip6dst, rule->tuples.dst_ip); + if (rule->unused_tuple & BIT(INNER_SRC_IP)) + memset(spec_mask->ip6src, 0, sizeof(spec_mask->ip6src)); + else + ipv6_addr_cpu_to_be32(spec_mask->ip6src, + rule->tuples_mask.src_ip); + + if (rule->unused_tuple & BIT(INNER_DST_IP)) + memset(spec_mask->ip6dst, 0, sizeof(spec_mask->ip6dst)); + else + ipv6_addr_cpu_to_be32(spec_mask->ip6dst, + rule->tuples_mask.dst_ip); + + spec->tclass = rule->tuples.ip_tos; + spec_mask->tclass = rule->unused_tuple & BIT(INNER_IP_TOS) ? + 0 : rule->tuples_mask.ip_tos; + + spec->l4_proto = rule->tuples.ip_proto; + spec_mask->l4_proto = rule->unused_tuple & BIT(INNER_IP_PROTO) ? + 0 : rule->tuples_mask.ip_proto; +} + +static void hclge_fd_get_ether_info(struct hclge_fd_rule *rule, + struct ethhdr *spec, + struct ethhdr *spec_mask) +{ + ether_addr_copy(spec->h_source, rule->tuples.src_mac); + ether_addr_copy(spec->h_dest, rule->tuples.dst_mac); + + if (rule->unused_tuple & BIT(INNER_SRC_MAC)) + eth_zero_addr(spec_mask->h_source); + else + ether_addr_copy(spec_mask->h_source, rule->tuples_mask.src_mac); + + if (rule->unused_tuple & BIT(INNER_DST_MAC)) + eth_zero_addr(spec_mask->h_dest); + else + ether_addr_copy(spec_mask->h_dest, rule->tuples_mask.dst_mac); + + spec->h_proto = cpu_to_be16(rule->tuples.ether_proto); + spec_mask->h_proto = rule->unused_tuple & BIT(INNER_ETH_TYPE) ? + 0 : cpu_to_be16(rule->tuples_mask.ether_proto); +} + +static void hclge_fd_get_user_def_info(struct ethtool_rx_flow_spec *fs, + struct hclge_fd_rule *rule) +{ + if ((rule->unused_tuple & HCLGE_FD_TUPLE_USER_DEF_TUPLES) == + HCLGE_FD_TUPLE_USER_DEF_TUPLES) { + fs->h_ext.data[0] = 0; + fs->h_ext.data[1] = 0; + fs->m_ext.data[0] = 0; + fs->m_ext.data[1] = 0; + } else { + fs->h_ext.data[0] = cpu_to_be32(rule->ep.user_def.offset); + fs->h_ext.data[1] = cpu_to_be32(rule->ep.user_def.data); + fs->m_ext.data[0] = + cpu_to_be32(HCLGE_FD_USER_DEF_OFFSET_UNMASK); + fs->m_ext.data[1] = cpu_to_be32(rule->ep.user_def.data_mask); + } +} + +static void hclge_fd_get_ext_info(struct ethtool_rx_flow_spec *fs, + struct hclge_fd_rule *rule) +{ + if (fs->flow_type & FLOW_EXT) { + fs->h_ext.vlan_tci = cpu_to_be16(rule->tuples.vlan_tag1); + fs->m_ext.vlan_tci = + rule->unused_tuple & BIT(INNER_VLAN_TAG_FST) ? + 0 : cpu_to_be16(rule->tuples_mask.vlan_tag1); + + hclge_fd_get_user_def_info(fs, rule); + } + + if (fs->flow_type & FLOW_MAC_EXT) { + ether_addr_copy(fs->h_ext.h_dest, rule->tuples.dst_mac); + if (rule->unused_tuple & BIT(INNER_DST_MAC)) + eth_zero_addr(fs->m_u.ether_spec.h_dest); + else + ether_addr_copy(fs->m_u.ether_spec.h_dest, + rule->tuples_mask.dst_mac); + } +} + +static struct hclge_fd_rule *hclge_get_fd_rule(struct hclge_dev *hdev, + u16 location) +{ + struct hclge_fd_rule *rule = NULL; + struct hlist_node *node2; + + hlist_for_each_entry_safe(rule, node2, &hdev->fd_rule_list, rule_node) { + if (rule->location == location) + return rule; + else if (rule->location > location) + return NULL; + } + + return NULL; +} + +static void hclge_fd_get_ring_cookie(struct ethtool_rx_flow_spec *fs, + struct hclge_fd_rule *rule) +{ + if (rule->action == HCLGE_FD_ACTION_DROP_PACKET) { + fs->ring_cookie = RX_CLS_FLOW_DISC; + } else { + u64 vf_id; + + fs->ring_cookie = rule->queue_id; + vf_id = rule->vf_id; + vf_id <<= ETHTOOL_RX_FLOW_SPEC_RING_VF_OFF; + fs->ring_cookie |= vf_id; + } +} + +int hclge_get_fd_rule_info(struct hnae3_handle *handle, + struct ethtool_rxnfc *cmd) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_fd_rule *rule = NULL; + struct hclge_dev *hdev = vport->back; + struct ethtool_rx_flow_spec *fs; + + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return -EOPNOTSUPP; + + fs = (struct ethtool_rx_flow_spec *)&cmd->fs; + + spin_lock_bh(&hdev->fd_rule_lock); + + rule = hclge_get_fd_rule(hdev, fs->location); + if (!rule) { + spin_unlock_bh(&hdev->fd_rule_lock); + return -ENOENT; + } + + fs->flow_type = rule->flow_type; + switch (fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) { + case SCTP_V4_FLOW: + case TCP_V4_FLOW: + case UDP_V4_FLOW: + hclge_fd_get_tcpip4_info(rule, &fs->h_u.tcp_ip4_spec, + &fs->m_u.tcp_ip4_spec); + break; + case IP_USER_FLOW: + hclge_fd_get_ip4_info(rule, &fs->h_u.usr_ip4_spec, + &fs->m_u.usr_ip4_spec); + break; + case SCTP_V6_FLOW: + case TCP_V6_FLOW: + case UDP_V6_FLOW: + hclge_fd_get_tcpip6_info(rule, &fs->h_u.tcp_ip6_spec, + &fs->m_u.tcp_ip6_spec); + break; + case IPV6_USER_FLOW: + hclge_fd_get_ip6_info(rule, &fs->h_u.usr_ip6_spec, + &fs->m_u.usr_ip6_spec); + break; + /* The flow type of fd rule has been checked before adding in to rule + * list. As other flow types have been handled, it must be ETHER_FLOW + * for the default case + */ + default: + hclge_fd_get_ether_info(rule, &fs->h_u.ether_spec, + &fs->m_u.ether_spec); + break; + } + + hclge_fd_get_ext_info(fs, rule); + + hclge_fd_get_ring_cookie(fs, rule); + + spin_unlock_bh(&hdev->fd_rule_lock); + + return 0; +} + +int hclge_get_all_rules(struct hnae3_handle *handle, + struct ethtool_rxnfc *cmd, u32 *rule_locs) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + struct hclge_fd_rule *rule; + struct hlist_node *node2; + u32 cnt = 0; + + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return -EOPNOTSUPP; + + cmd->data = hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]; + + spin_lock_bh(&hdev->fd_rule_lock); + hlist_for_each_entry_safe(rule, node2, + &hdev->fd_rule_list, rule_node) { + if (cnt == cmd->rule_cnt) { + spin_unlock_bh(&hdev->fd_rule_lock); + return -EMSGSIZE; + } + + if (rule->state == HCLGE_FD_TO_DEL) + continue; + + rule_locs[cnt] = rule->location; + cnt++; + } + + spin_unlock_bh(&hdev->fd_rule_lock); + + cmd->rule_cnt = cnt; + + return 0; +} + +static void hclge_fd_get_flow_tuples(const struct flow_keys *fkeys, + struct hclge_fd_rule_tuples *tuples) +{ +#define flow_ip6_src fkeys->addrs.v6addrs.src.in6_u.u6_addr32 +#define flow_ip6_dst fkeys->addrs.v6addrs.dst.in6_u.u6_addr32 + + tuples->ether_proto = be16_to_cpu(fkeys->basic.n_proto); + tuples->ip_proto = fkeys->basic.ip_proto; + tuples->dst_port = be16_to_cpu(fkeys->ports.dst); + + if (fkeys->basic.n_proto == htons(ETH_P_IP)) { + tuples->src_ip[3] = be32_to_cpu(fkeys->addrs.v4addrs.src); + tuples->dst_ip[3] = be32_to_cpu(fkeys->addrs.v4addrs.dst); + } else { + int i; + + for (i = 0; i < IPV6_ADDR_WORDS; i++) { + tuples->src_ip[i] = be32_to_cpu(flow_ip6_src[i]); + tuples->dst_ip[i] = be32_to_cpu(flow_ip6_dst[i]); + } + } +} + +/* traverse all rules, check whether an existed rule has the same tuples */ +static struct hclge_fd_rule * +hclge_fd_search_flow_keys(struct hclge_dev *hdev, + const struct hclge_fd_rule_tuples *tuples) +{ + struct hclge_fd_rule *rule = NULL; + struct hlist_node *node; + + hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) { + if (!memcmp(tuples, &rule->tuples, sizeof(*tuples))) + return rule; + } + + return NULL; +} + +static void hclge_fd_build_arfs_rule(const struct hclge_fd_rule_tuples *tuples, + struct hclge_fd_rule *rule) +{ + rule->unused_tuple = BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC) | + BIT(INNER_VLAN_TAG_FST) | BIT(INNER_IP_TOS) | + BIT(INNER_SRC_PORT); + rule->action = 0; + rule->vf_id = 0; + rule->rule_type = HCLGE_FD_ARFS_ACTIVE; + rule->state = HCLGE_FD_TO_ADD; + if (tuples->ether_proto == ETH_P_IP) { + if (tuples->ip_proto == IPPROTO_TCP) + rule->flow_type = TCP_V4_FLOW; + else + rule->flow_type = UDP_V4_FLOW; + } else { + if (tuples->ip_proto == IPPROTO_TCP) + rule->flow_type = TCP_V6_FLOW; + else + rule->flow_type = UDP_V6_FLOW; + } + memcpy(&rule->tuples, tuples, sizeof(rule->tuples)); + memset(&rule->tuples_mask, 0xFF, sizeof(rule->tuples_mask)); +} + +int hclge_add_fd_entry_by_arfs(struct hnae3_handle *handle, u16 queue_id, + u16 flow_id, struct flow_keys *fkeys) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_fd_rule_tuples new_tuples = {}; + struct hclge_dev *hdev = vport->back; + struct hclge_fd_rule *rule; + u16 bit_id; + + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return -EOPNOTSUPP; + + /* when there is already fd rule existed add by user, + * arfs should not work + */ + spin_lock_bh(&hdev->fd_rule_lock); + if (hdev->fd_active_type != HCLGE_FD_ARFS_ACTIVE && + hdev->fd_active_type != HCLGE_FD_RULE_NONE) { + spin_unlock_bh(&hdev->fd_rule_lock); + return -EOPNOTSUPP; + } + + hclge_fd_get_flow_tuples(fkeys, &new_tuples); + + /* check is there flow director filter existed for this flow, + * if not, create a new filter for it; + * if filter exist with different queue id, modify the filter; + * if filter exist with same queue id, do nothing + */ + rule = hclge_fd_search_flow_keys(hdev, &new_tuples); + if (!rule) { + bit_id = find_first_zero_bit(hdev->fd_bmap, MAX_FD_FILTER_NUM); + if (bit_id >= hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) { + spin_unlock_bh(&hdev->fd_rule_lock); + return -ENOSPC; + } + + rule = kzalloc_obj(*rule, GFP_ATOMIC); + if (!rule) { + spin_unlock_bh(&hdev->fd_rule_lock); + return -ENOMEM; + } + + rule->location = bit_id; + rule->arfs.flow_id = flow_id; + rule->queue_id = queue_id; + hclge_fd_build_arfs_rule(&new_tuples, rule); + hclge_update_fd_list(hdev, rule->state, rule->location, rule); + hdev->fd_active_type = HCLGE_FD_ARFS_ACTIVE; + } else if (rule->queue_id != queue_id) { + rule->queue_id = queue_id; + rule->state = HCLGE_FD_TO_ADD; + set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state); + hclge_task_schedule(hdev, 0); + } + spin_unlock_bh(&hdev->fd_rule_lock); + return rule->location; +} + +void hclge_rfs_filter_expire(struct hclge_dev *hdev) +{ +#ifdef CONFIG_RFS_ACCEL + struct hnae3_handle *handle = &hdev->vport[0].nic; + struct hclge_fd_rule *rule; + struct hlist_node *node; + + spin_lock_bh(&hdev->fd_rule_lock); + if (hdev->fd_active_type != HCLGE_FD_ARFS_ACTIVE) { + spin_unlock_bh(&hdev->fd_rule_lock); + return; + } + hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) { + if (rule->state != HCLGE_FD_ACTIVE) + continue; + if (rps_may_expire_flow(handle->netdev, rule->queue_id, + rule->arfs.flow_id, rule->location)) { + rule->state = HCLGE_FD_TO_DEL; + set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state); + } + } + spin_unlock_bh(&hdev->fd_rule_lock); +#endif +} + +/* make sure being called after lock up with fd_rule_lock */ +int hclge_clear_arfs_rules(struct hclge_dev *hdev) +{ +#ifdef CONFIG_RFS_ACCEL + struct hclge_fd_rule *rule; + struct hlist_node *node; + int ret; + + if (hdev->fd_active_type != HCLGE_FD_ARFS_ACTIVE) + return 0; + + hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) { + switch (rule->state) { + case HCLGE_FD_TO_DEL: + case HCLGE_FD_ACTIVE: + ret = hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, + rule->location, NULL, false); + if (ret) + return ret; + fallthrough; + case HCLGE_FD_TO_ADD: + hclge_fd_dec_rule_cnt(hdev, rule->location); + hlist_del(&rule->rule_node); + kfree(rule); + break; + default: + break; + } + } + hclge_sync_fd_state(hdev); + +#endif + return 0; +} + +static void hclge_get_cls_key_basic(const struct flow_rule *flow, + struct hclge_fd_rule *rule) +{ + if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_BASIC)) { + struct flow_match_basic match; + u16 ethtype_key, ethtype_mask; + + flow_rule_match_basic(flow, &match); + ethtype_key = ntohs(match.key->n_proto); + ethtype_mask = ntohs(match.mask->n_proto); + + if (ethtype_key == ETH_P_ALL) { + ethtype_key = 0; + ethtype_mask = 0; + } + rule->tuples.ether_proto = ethtype_key; + rule->tuples_mask.ether_proto = ethtype_mask; + rule->tuples.ip_proto = match.key->ip_proto; + rule->tuples_mask.ip_proto = match.mask->ip_proto; + } else { + rule->unused_tuple |= BIT(INNER_IP_PROTO); + rule->unused_tuple |= BIT(INNER_ETH_TYPE); + } +} + +static void hclge_get_cls_key_mac(const struct flow_rule *flow, + struct hclge_fd_rule *rule) +{ + if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { + struct flow_match_eth_addrs match; + + flow_rule_match_eth_addrs(flow, &match); + ether_addr_copy(rule->tuples.dst_mac, match.key->dst); + ether_addr_copy(rule->tuples_mask.dst_mac, match.mask->dst); + ether_addr_copy(rule->tuples.src_mac, match.key->src); + ether_addr_copy(rule->tuples_mask.src_mac, match.mask->src); + if (is_zero_ether_addr(match.key->dst)) + rule->unused_tuple |= BIT(INNER_DST_MAC); + if (is_zero_ether_addr(match.key->src)) + rule->unused_tuple |= BIT(INNER_SRC_MAC); + } else { + rule->unused_tuple |= BIT(INNER_DST_MAC); + rule->unused_tuple |= BIT(INNER_SRC_MAC); + } +} + +static void hclge_get_cls_key_vlan(const struct flow_rule *flow, + struct hclge_fd_rule *rule) +{ + if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_VLAN)) { + struct flow_match_vlan match; + + flow_rule_match_vlan(flow, &match); + rule->tuples.vlan_tag1 = match.key->vlan_id | + (match.key->vlan_priority << VLAN_PRIO_SHIFT); + rule->tuples_mask.vlan_tag1 = match.mask->vlan_id | + (match.mask->vlan_priority << VLAN_PRIO_SHIFT); + } else { + rule->unused_tuple |= BIT(INNER_VLAN_TAG_FST); + } +} + +static int hclge_get_cls_key_ip(const struct flow_rule *flow, + struct hclge_fd_rule *rule, + struct netlink_ext_ack *extack) +{ + u16 addr_type = 0; + + if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_CONTROL)) { + struct flow_match_control match; + + flow_rule_match_control(flow, &match); + addr_type = match.key->addr_type; + + if (flow_rule_has_control_flags(match.mask->flags, extack)) + return -EOPNOTSUPP; + } + + if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { + struct flow_match_ipv4_addrs match; + + flow_rule_match_ipv4_addrs(flow, &match); + rule->tuples.src_ip[IPV4_INDEX] = be32_to_cpu(match.key->src); + rule->tuples_mask.src_ip[IPV4_INDEX] = + be32_to_cpu(match.mask->src); + rule->tuples.dst_ip[IPV4_INDEX] = be32_to_cpu(match.key->dst); + rule->tuples_mask.dst_ip[IPV4_INDEX] = + be32_to_cpu(match.mask->dst); + if (!match.key->src) + rule->unused_tuple |= BIT(INNER_SRC_IP); + if (!match.key->dst) + rule->unused_tuple |= BIT(INNER_DST_IP); + } else if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { + struct flow_match_ipv6_addrs match; + + flow_rule_match_ipv6_addrs(flow, &match); + ipv6_addr_be32_to_cpu(rule->tuples.src_ip, + match.key->src.s6_addr32); + ipv6_addr_be32_to_cpu(rule->tuples_mask.src_ip, + match.mask->src.s6_addr32); + ipv6_addr_be32_to_cpu(rule->tuples.dst_ip, + match.key->dst.s6_addr32); + ipv6_addr_be32_to_cpu(rule->tuples_mask.dst_ip, + match.mask->dst.s6_addr32); + if (ipv6_addr_any((struct in6_addr *)match.key->src.s6_addr32)) + rule->unused_tuple |= BIT(INNER_SRC_IP); + if (ipv6_addr_any((struct in6_addr *)match.key->dst.s6_addr32)) + rule->unused_tuple |= BIT(INNER_DST_IP); + } else { + rule->unused_tuple |= BIT(INNER_SRC_IP); + rule->unused_tuple |= BIT(INNER_DST_IP); + } + + return 0; +} + +static void hclge_get_cls_key_port(const struct flow_rule *flow, + struct hclge_fd_rule *rule) +{ + if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_PORTS)) { + struct flow_match_ports match; + + flow_rule_match_ports(flow, &match); + + rule->tuples.src_port = be16_to_cpu(match.key->src); + rule->tuples_mask.src_port = be16_to_cpu(match.mask->src); + rule->tuples.dst_port = be16_to_cpu(match.key->dst); + rule->tuples_mask.dst_port = be16_to_cpu(match.mask->dst); + } else { + rule->unused_tuple |= BIT(INNER_SRC_PORT); + rule->unused_tuple |= BIT(INNER_DST_PORT); + } +} + +static int hclge_get_cls_enc_keyid(struct hclge_dev *hdev, + const struct flow_rule *flow, + struct hclge_fd_rule *rule) +{ + if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_ENC_KEYID)) { + struct flow_match_enc_keyid match; + + flow_rule_match_enc_keyid(flow, &match); + + /* vni is only 24 bits and must be greater than 0, + * and it can not be masked. + */ + if (be32_to_cpu(match.mask->keyid) != + HCLGE_FD_VXLAN_VNI_UNMASK || + be32_to_cpu(match.key->keyid) >= VXLAN_N_VID || + !match.key->keyid) { + dev_err(&hdev->pdev->dev, "invalid enc_keyid\n"); + return -EINVAL; + } + + rule->tuples.outer_tun_vni = be32_to_cpu(match.key->keyid); + rule->tuples_mask.outer_tun_vni = + be32_to_cpu(match.mask->keyid); + } else { + rule->unused_tuple |= BIT(OUTER_TUN_VNI); + } + + return 0; +} + +static void hclge_get_cls_key_ip_tos(const struct flow_rule *flow, + struct hclge_fd_rule *rule) +{ + if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_IP)) { + struct flow_match_ip match; + + flow_rule_match_ip(flow, &match); + + rule->tuples.ip_tos = match.key->tos; + rule->tuples_mask.ip_tos = match.mask->tos; + if (!rule->tuples.ip_tos) + rule->unused_tuple |= BIT(INNER_IP_TOS); + } else { + rule->unused_tuple |= BIT(INNER_IP_TOS); + } +} + +static int hclge_get_tc_flower_action(struct hclge_dev *hdev, + struct flow_cls_offload *cls_flower, + struct hclge_fd_rule *rule) +{ + struct flow_rule *flow = flow_cls_offload_flow_rule(cls_flower); + struct hnae3_handle *handle = &hdev->vport[0].nic; + struct flow_action *action = &flow->action; + struct flow_action_entry *act; + int tc; + + if (!flow_action_has_entries(&flow->action)) { + tc = tc_classid_to_hwtc(handle->netdev, cls_flower->classid); + if (tc < 0 || tc > hdev->tc_max) { + dev_err(&hdev->pdev->dev, + "invalid traffic class: %d\n", tc); + return -EINVAL; + } + + rule->action = HCLGE_FD_ACTION_SELECT_TC; + rule->cls_flower.tc = tc; + return 0; + } + + act = &action->entries[0]; + switch (act->id) { + case FLOW_ACTION_RX_QUEUE_MAPPING: + if (act->rx_queue >= handle->kinfo.num_tqps) { + dev_err(&hdev->pdev->dev, + "queue id (%u) should be less than %u\n", + act->rx_queue, handle->kinfo.num_tqps - 1); + return -EINVAL; + } + + rule->queue_id = act->rx_queue; + rule->action = HCLGE_FD_ACTION_SELECT_QUEUE; + return 0; + case FLOW_ACTION_DROP: + rule->action = HCLGE_FD_ACTION_DROP_PACKET; + return 0; + default: + dev_err(&hdev->pdev->dev, "unsupported action(%d)\n", act->id); + return -EOPNOTSUPP; + } +} + +static int hclge_parse_cls_flower(struct hclge_dev *hdev, + struct flow_cls_offload *cls_flower, + struct hclge_fd_rule *rule) +{ + struct flow_rule *flow = flow_cls_offload_flow_rule(cls_flower); + struct netlink_ext_ack *extack = cls_flower->common.extack; + int ret; + + /* not support any user def tuples */ + rule->unused_tuple |= HCLGE_FD_TUPLE_USER_DEF_TUPLES; + + hclge_get_cls_key_basic(flow, rule); + hclge_get_cls_key_mac(flow, rule); + hclge_get_cls_key_vlan(flow, rule); + + ret = hclge_get_cls_key_ip(flow, rule, extack); + if (ret) + return ret; + + hclge_get_cls_key_port(flow, rule); + hclge_get_cls_key_ip_tos(flow, rule); + + return hclge_get_cls_enc_keyid(hdev, flow, rule); +} + +static int hclge_check_cls_flower(struct hclge_dev *hdev, + struct flow_cls_offload *cls_flower) +{ + struct flow_rule *flow = flow_cls_offload_flow_rule(cls_flower); + struct flow_dissector *dissector = flow->match.dissector; + u32 prio = cls_flower->common.prio; + + if (prio == 0 || + prio > hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) { + dev_err(&hdev->pdev->dev, + "prio %u should be in range[1, %u]\n", + prio, hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]); + return -EINVAL; + } + + if (test_bit(prio - 1, hdev->fd_bmap)) { + dev_err(&hdev->pdev->dev, "prio %u is already used\n", prio); + return -EINVAL; + } + + if (dissector->used_keys & + ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) { + dev_err(&hdev->pdev->dev, "unsupported key set: %#llx\n", + dissector->used_keys); + return -EOPNOTSUPP; + } + + /* driver will parses classid into an action */ + if (cls_flower->classid && flow_action_has_entries(&flow->action)) { + dev_err(&hdev->pdev->dev, + "please not set classid and action together\n"); + return -EOPNOTSUPP; + } + + if (!flow_action_has_entries(&flow->action) && !cls_flower->classid) { + dev_err(&hdev->pdev->dev, "please set action or classid\n"); + return -EINVAL; + } + + if (flow_action_has_entries(&flow->action) && + !flow_offload_has_one_action(&flow->action)) { + dev_err(&hdev->pdev->dev, "unsupported multiple actions\n"); + return -EOPNOTSUPP; + } + + return 0; +} + +int hclge_add_cls_flower(struct hnae3_handle *handle, + struct flow_cls_offload *cls_flower) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + struct hclge_fd_rule *rule; + int ret; + + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) { + dev_err(&hdev->pdev->dev, + "cls flower is not supported\n"); + return -EOPNOTSUPP; + } + + ret = hclge_check_cls_flower(hdev, cls_flower); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to check cls flower params, ret = %d\n", ret); + return ret; + } + + rule = kzalloc_obj(*rule); + if (!rule) + return -ENOMEM; + + ret = hclge_parse_cls_flower(hdev, cls_flower, rule); + if (ret) { + kfree(rule); + return ret; + } + + ret = hclge_get_tc_flower_action(hdev, cls_flower, rule); + if (ret) { + kfree(rule); + return ret; + } + + rule->location = cls_flower->common.prio - 1; + rule->vf_id = 0; + rule->cls_flower.cookie = cls_flower->cookie; + rule->rule_type = HCLGE_FD_TC_FLOWER_ACTIVE; + + ret = hclge_add_fd_entry_common(hdev, rule); + if (ret) + kfree(rule); + + return ret; +} + +static struct hclge_fd_rule *hclge_find_cls_flower(struct hclge_dev *hdev, + unsigned long cookie) +{ + struct hclge_fd_rule *rule; + struct hlist_node *node; + + hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) { + if (rule->cls_flower.cookie == cookie) + return rule; + } + + return NULL; +} + +int hclge_del_cls_flower(struct hnae3_handle *handle, + struct flow_cls_offload *cls_flower) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + struct hclge_fd_rule *rule; + int ret; + + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return -EOPNOTSUPP; + + spin_lock_bh(&hdev->fd_rule_lock); + + rule = hclge_find_cls_flower(hdev, cls_flower->cookie); + if (!rule) { + spin_unlock_bh(&hdev->fd_rule_lock); + return -EINVAL; + } + + ret = hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, rule->location, + NULL, false); + if (ret) { + /* if tcam config fail, set rule state to TO_DEL, + * so the rule will be deleted when periodic + * task being scheduled. + */ + hclge_update_fd_list(hdev, HCLGE_FD_TO_DEL, + rule->location, NULL); + set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state); + spin_unlock_bh(&hdev->fd_rule_lock); + return ret; + } + + hclge_update_fd_list(hdev, HCLGE_FD_DELETED, rule->location, NULL); + spin_unlock_bh(&hdev->fd_rule_lock); + + return 0; +} + +static void hclge_sync_fd_list(struct hclge_dev *hdev, struct hlist_head *hlist) +{ + struct hclge_fd_rule *rule; + struct hlist_node *node; + int ret = 0; + + if (!test_and_clear_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state)) + return; + + spin_lock_bh(&hdev->fd_rule_lock); + + hlist_for_each_entry_safe(rule, node, hlist, rule_node) { + switch (rule->state) { + case HCLGE_FD_TO_ADD: + ret = hclge_fd_config_rule(hdev, rule); + if (ret) + goto out; + rule->state = HCLGE_FD_ACTIVE; + break; + case HCLGE_FD_TO_DEL: + ret = hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, + rule->location, NULL, false); + if (ret) + goto out; + hclge_fd_dec_rule_cnt(hdev, rule->location); + hclge_fd_free_node(hdev, rule); + break; + default: + break; + } + } + +out: + if (ret) + set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state); + + spin_unlock_bh(&hdev->fd_rule_lock); +} + +void hclge_sync_fd_table(struct hclge_dev *hdev) +{ + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return; + + if (test_and_clear_bit(HCLGE_STATE_FD_CLEAR_ALL, &hdev->state)) { + bool clear_list = hdev->fd_active_type == HCLGE_FD_ARFS_ACTIVE; + + hclge_clear_fd_rules_in_list(hdev, clear_list); + } + + hclge_sync_fd_user_def_cfg(hdev, false); + + hclge_sync_fd_list(hdev, &hdev->fd_rule_list); +} + +void hclge_enable_fd(struct hnae3_handle *handle, bool enable) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + + hdev->fd_en = enable; + + if (!enable) + set_bit(HCLGE_STATE_FD_CLEAR_ALL, &hdev->state); + else + hclge_restore_fd_entries(handle); + + hclge_task_schedule(hdev, 0); +} diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_fd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_fd.h new file mode 100644 index 000000000000..e0c03699d47e --- /dev/null +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_fd.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2024 Hisilicon Limited. */ + +#ifndef __HCLGE_FD_H +#define __HCLGE_FD_H + +struct hnae3_handle; +struct hclge_dev; + +int hclge_init_fd_config(struct hclge_dev *hdev); +int hclge_add_fd_entry(struct hnae3_handle *handle, struct ethtool_rxnfc *cmd); +int hclge_del_fd_entry(struct hnae3_handle *handle, struct ethtool_rxnfc *cmd); +void hclge_del_all_fd_entries(struct hclge_dev *hdev); +int hclge_restore_fd_entries(struct hnae3_handle *handle); +int hclge_get_fd_rule_cnt(struct hnae3_handle *handle, + struct ethtool_rxnfc *cmd); +int hclge_get_fd_rule_info(struct hnae3_handle *handle, + struct ethtool_rxnfc *cmd); +int hclge_get_all_rules(struct hnae3_handle *handle, + struct ethtool_rxnfc *cmd, u32 *rule_locs); +void hclge_enable_fd(struct hnae3_handle *handle, bool enable); +int hclge_add_fd_entry_by_arfs(struct hnae3_handle *handle, u16 queue_id, + u16 flow_id, struct flow_keys *fkeys); +int hclge_add_cls_flower(struct hnae3_handle *handle, + struct flow_cls_offload *cls_flower); +int hclge_del_cls_flower(struct hnae3_handle *handle, + struct flow_cls_offload *cls_flower); +bool hclge_is_cls_flower_active(struct hnae3_handle *handle); +int hclge_clear_arfs_rules(struct hclge_dev *hdev); +void hclge_sync_fd_table(struct hclge_dev *hdev); +void hclge_rfs_filter_expire(struct hclge_dev *hdev); + +#endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index a7320eca41d3..2f1984930da2 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -16,9 +16,9 @@ #include #include -#include #include "hclge_cmd.h" #include "hclge_dcb.h" +#include "hclge_fd.h" #include "hclge_main.h" #include "hclge_mbx.h" #include "hclge_mdio.h" @@ -52,8 +52,6 @@ static int hclge_init_vlan_config(struct hclge_dev *hdev); static void hclge_sync_vlan_filter(struct hclge_dev *hdev); static int hclge_reset_ae_dev(struct hnae3_ae_dev *ae_dev); static bool hclge_get_hw_reset_stat(struct hnae3_handle *handle); -static void hclge_rfs_filter_expire(struct hclge_dev *hdev); -static int hclge_clear_arfs_rules(struct hclge_dev *hdev); static enum hnae3_reset_type hclge_get_reset_level(struct hnae3_ae_dev *ae_dev, unsigned long *addr); static int hclge_set_default_loopback(struct hclge_dev *hdev); @@ -61,7 +59,6 @@ static int hclge_set_default_loopback(struct hclge_dev *hdev); static void hclge_sync_mac_table(struct hclge_dev *hdev); static void hclge_restore_hw_table(struct hclge_dev *hdev); static void hclge_sync_promisc_mode(struct hclge_dev *hdev); -static void hclge_sync_fd_table(struct hclge_dev *hdev); static void hclge_update_fec_stats(struct hclge_dev *hdev); static int hclge_mac_link_status_wait(struct hclge_dev *hdev, int link_ret, int wait_cnt); @@ -311,78 +308,6 @@ static const struct hclge_mac_mgr_tbl_entry_cmd hclge_mgr_table[] = { }, }; -static const struct key_info meta_data_key_info[] = { - { PACKET_TYPE_ID, 6 }, - { IP_FRAGEMENT, 1 }, - { ROCE_TYPE, 1 }, - { NEXT_KEY, 5 }, - { VLAN_NUMBER, 2 }, - { SRC_VPORT, 12 }, - { DST_VPORT, 12 }, - { TUNNEL_PACKET, 1 }, -}; - -static const struct key_info tuple_key_info[] = { - { OUTER_DST_MAC, 48, KEY_OPT_MAC, -1, -1 }, - { OUTER_SRC_MAC, 48, KEY_OPT_MAC, -1, -1 }, - { OUTER_VLAN_TAG_FST, 16, KEY_OPT_LE16, -1, -1 }, - { OUTER_VLAN_TAG_SEC, 16, KEY_OPT_LE16, -1, -1 }, - { OUTER_ETH_TYPE, 16, KEY_OPT_LE16, -1, -1 }, - { OUTER_L2_RSV, 16, KEY_OPT_LE16, -1, -1 }, - { OUTER_IP_TOS, 8, KEY_OPT_U8, -1, -1 }, - { OUTER_IP_PROTO, 8, KEY_OPT_U8, -1, -1 }, - { OUTER_SRC_IP, 32, KEY_OPT_IP, -1, -1 }, - { OUTER_DST_IP, 32, KEY_OPT_IP, -1, -1 }, - { OUTER_L3_RSV, 16, KEY_OPT_LE16, -1, -1 }, - { OUTER_SRC_PORT, 16, KEY_OPT_LE16, -1, -1 }, - { OUTER_DST_PORT, 16, KEY_OPT_LE16, -1, -1 }, - { OUTER_L4_RSV, 32, KEY_OPT_LE32, -1, -1 }, - { OUTER_TUN_VNI, 24, KEY_OPT_VNI, - offsetof(struct hclge_fd_rule, tuples.outer_tun_vni), - offsetof(struct hclge_fd_rule, tuples_mask.outer_tun_vni) }, - { OUTER_TUN_FLOW_ID, 8, KEY_OPT_U8, -1, -1 }, - { INNER_DST_MAC, 48, KEY_OPT_MAC, - offsetof(struct hclge_fd_rule, tuples.dst_mac), - offsetof(struct hclge_fd_rule, tuples_mask.dst_mac) }, - { INNER_SRC_MAC, 48, KEY_OPT_MAC, - offsetof(struct hclge_fd_rule, tuples.src_mac), - offsetof(struct hclge_fd_rule, tuples_mask.src_mac) }, - { INNER_VLAN_TAG_FST, 16, KEY_OPT_LE16, - offsetof(struct hclge_fd_rule, tuples.vlan_tag1), - offsetof(struct hclge_fd_rule, tuples_mask.vlan_tag1) }, - { INNER_VLAN_TAG_SEC, 16, KEY_OPT_LE16, -1, -1 }, - { INNER_ETH_TYPE, 16, KEY_OPT_LE16, - offsetof(struct hclge_fd_rule, tuples.ether_proto), - offsetof(struct hclge_fd_rule, tuples_mask.ether_proto) }, - { INNER_L2_RSV, 16, KEY_OPT_LE16, - offsetof(struct hclge_fd_rule, tuples.l2_user_def), - offsetof(struct hclge_fd_rule, tuples_mask.l2_user_def) }, - { INNER_IP_TOS, 8, KEY_OPT_U8, - offsetof(struct hclge_fd_rule, tuples.ip_tos), - offsetof(struct hclge_fd_rule, tuples_mask.ip_tos) }, - { INNER_IP_PROTO, 8, KEY_OPT_U8, - offsetof(struct hclge_fd_rule, tuples.ip_proto), - offsetof(struct hclge_fd_rule, tuples_mask.ip_proto) }, - { INNER_SRC_IP, 32, KEY_OPT_IP, - offsetof(struct hclge_fd_rule, tuples.src_ip), - offsetof(struct hclge_fd_rule, tuples_mask.src_ip) }, - { INNER_DST_IP, 32, KEY_OPT_IP, - offsetof(struct hclge_fd_rule, tuples.dst_ip), - offsetof(struct hclge_fd_rule, tuples_mask.dst_ip) }, - { INNER_L3_RSV, 16, KEY_OPT_LE16, - offsetof(struct hclge_fd_rule, tuples.l3_user_def), - offsetof(struct hclge_fd_rule, tuples_mask.l3_user_def) }, - { INNER_SRC_PORT, 16, KEY_OPT_LE16, - offsetof(struct hclge_fd_rule, tuples.src_port), - offsetof(struct hclge_fd_rule, tuples_mask.src_port) }, - { INNER_DST_PORT, 16, KEY_OPT_LE16, - offsetof(struct hclge_fd_rule, tuples.dst_port), - offsetof(struct hclge_fd_rule, tuples_mask.dst_port) }, - { INNER_L4_RSV, 32, KEY_OPT_LE32, - offsetof(struct hclge_fd_rule, tuples.l4_user_def), - offsetof(struct hclge_fd_rule, tuples_mask.l4_user_def) }, -}; - /** * hclge_cmd_send - send command to command queue * @hw: pointer to the hw struct @@ -5183,605 +5108,80 @@ static void hclge_request_update_promisc_mode(struct hnae3_handle *handle) set_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, &vport->state); } -static void hclge_sync_fd_state(struct hclge_dev *hdev) -{ - if (hlist_empty(&hdev->fd_rule_list)) - hdev->fd_active_type = HCLGE_FD_RULE_NONE; -} - -static void hclge_fd_inc_rule_cnt(struct hclge_dev *hdev, u16 location) -{ - if (!test_bit(location, hdev->fd_bmap)) { - set_bit(location, hdev->fd_bmap); - hdev->hclge_fd_rule_num++; - } -} - -static void hclge_fd_dec_rule_cnt(struct hclge_dev *hdev, u16 location) -{ - if (test_bit(location, hdev->fd_bmap)) { - clear_bit(location, hdev->fd_bmap); - hdev->hclge_fd_rule_num--; - } -} - -static void hclge_fd_free_node(struct hclge_dev *hdev, - struct hclge_fd_rule *rule) -{ - hlist_del(&rule->rule_node); - kfree(rule); - hclge_sync_fd_state(hdev); -} - -static void hclge_update_fd_rule_node(struct hclge_dev *hdev, - struct hclge_fd_rule *old_rule, - struct hclge_fd_rule *new_rule, - enum HCLGE_FD_NODE_STATE state) -{ - switch (state) { - case HCLGE_FD_TO_ADD: - case HCLGE_FD_ACTIVE: - /* 1) if the new state is TO_ADD, just replace the old rule - * with the same location, no matter its state, because the - * new rule will be configured to the hardware. - * 2) if the new state is ACTIVE, it means the new rule - * has been configured to the hardware, so just replace - * the old rule node with the same location. - * 3) for it doesn't add a new node to the list, so it's - * unnecessary to update the rule number and fd_bmap. - */ - new_rule->rule_node.next = old_rule->rule_node.next; - new_rule->rule_node.pprev = old_rule->rule_node.pprev; - memcpy(old_rule, new_rule, sizeof(*old_rule)); - kfree(new_rule); - break; - case HCLGE_FD_DELETED: - hclge_fd_dec_rule_cnt(hdev, old_rule->location); - hclge_fd_free_node(hdev, old_rule); - break; - case HCLGE_FD_TO_DEL: - /* if new request is TO_DEL, and old rule is existent - * 1) the state of old rule is TO_DEL, we need do nothing, - * because we delete rule by location, other rule content - * is unncessary. - * 2) the state of old rule is ACTIVE, we need to change its - * state to TO_DEL, so the rule will be deleted when periodic - * task being scheduled. - * 3) the state of old rule is TO_ADD, it means the rule hasn't - * been added to hardware, so we just delete the rule node from - * fd_rule_list directly. - */ - if (old_rule->state == HCLGE_FD_TO_ADD) { - hclge_fd_dec_rule_cnt(hdev, old_rule->location); - hclge_fd_free_node(hdev, old_rule); - return; - } - old_rule->state = HCLGE_FD_TO_DEL; - break; - } -} - -static struct hclge_fd_rule *hclge_find_fd_rule(struct hlist_head *hlist, - u16 location, - struct hclge_fd_rule **parent) -{ - struct hclge_fd_rule *rule; - struct hlist_node *node; - - hlist_for_each_entry_safe(rule, node, hlist, rule_node) { - if (rule->location == location) - return rule; - else if (rule->location > location) - return NULL; - /* record the parent node, use to keep the nodes in fd_rule_list - * in ascend order. - */ - *parent = rule; - } - - return NULL; -} - -/* insert fd rule node in ascend order according to rule->location */ -static void hclge_fd_insert_rule_node(struct hlist_head *hlist, - struct hclge_fd_rule *rule, - struct hclge_fd_rule *parent) -{ - INIT_HLIST_NODE(&rule->rule_node); - - if (parent) - hlist_add_behind(&rule->rule_node, &parent->rule_node); - else - hlist_add_head(&rule->rule_node, hlist); -} - -static int hclge_fd_set_user_def_cmd(struct hclge_dev *hdev, - struct hclge_fd_user_def_cfg *cfg) -{ - struct hclge_fd_user_def_cfg_cmd *req; - struct hclge_desc desc; - u16 data = 0; - int ret; - - hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_USER_DEF_OP, false); - - req = (struct hclge_fd_user_def_cfg_cmd *)desc.data; - - hnae3_set_bit(data, HCLGE_FD_USER_DEF_EN_B, cfg[0].ref_cnt > 0); - hnae3_set_field(data, HCLGE_FD_USER_DEF_OFT_M, - HCLGE_FD_USER_DEF_OFT_S, cfg[0].offset); - req->ol2_cfg = cpu_to_le16(data); - - data = 0; - hnae3_set_bit(data, HCLGE_FD_USER_DEF_EN_B, cfg[1].ref_cnt > 0); - hnae3_set_field(data, HCLGE_FD_USER_DEF_OFT_M, - HCLGE_FD_USER_DEF_OFT_S, cfg[1].offset); - req->ol3_cfg = cpu_to_le16(data); - - data = 0; - hnae3_set_bit(data, HCLGE_FD_USER_DEF_EN_B, cfg[2].ref_cnt > 0); - hnae3_set_field(data, HCLGE_FD_USER_DEF_OFT_M, - HCLGE_FD_USER_DEF_OFT_S, cfg[2].offset); - req->ol4_cfg = cpu_to_le16(data); - - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - dev_err(&hdev->pdev->dev, - "failed to set fd user def data, ret= %d\n", ret); - return ret; -} - -static void hclge_sync_fd_user_def_cfg(struct hclge_dev *hdev, bool locked) +static bool hclge_get_hw_reset_stat(struct hnae3_handle *handle) { - int ret; - - if (!test_and_clear_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state)) - return; - - if (!locked) - spin_lock_bh(&hdev->fd_rule_lock); - - ret = hclge_fd_set_user_def_cmd(hdev, hdev->fd_cfg.user_def_cfg); - if (ret) - set_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state); + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; - if (!locked) - spin_unlock_bh(&hdev->fd_rule_lock); + return hclge_read_dev(&hdev->hw, HCLGE_GLOBAL_RESET_REG) || + hclge_read_dev(&hdev->hw, HCLGE_FUN_RST_ING); } -static int hclge_fd_check_user_def_refcnt(struct hclge_dev *hdev, - struct hclge_fd_rule *rule) +static bool hclge_get_cmdq_stat(struct hnae3_handle *handle) { - struct hlist_head *hlist = &hdev->fd_rule_list; - struct hclge_fd_rule *fd_rule, *parent = NULL; - struct hclge_fd_user_def_info *info, *old_info; - struct hclge_fd_user_def_cfg *cfg; - - if (!rule || rule->rule_type != HCLGE_FD_EP_ACTIVE || - rule->ep.user_def.layer == HCLGE_FD_USER_DEF_NONE) - return 0; - - /* for valid layer is start from 1, so need minus 1 to get the cfg */ - cfg = &hdev->fd_cfg.user_def_cfg[rule->ep.user_def.layer - 1]; - info = &rule->ep.user_def; - - if (!cfg->ref_cnt || cfg->offset == info->offset) - return 0; - - if (cfg->ref_cnt > 1) - goto error; - - fd_rule = hclge_find_fd_rule(hlist, rule->location, &parent); - if (fd_rule) { - old_info = &fd_rule->ep.user_def; - if (info->layer == old_info->layer) - return 0; - } + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; -error: - dev_err(&hdev->pdev->dev, - "No available offset for layer%d fd rule, each layer only support one user def offset.\n", - info->layer + 1); - return -ENOSPC; + return test_bit(HCLGE_COMM_STATE_CMD_DISABLE, &hdev->hw.hw.comm_state); } -static void hclge_fd_inc_user_def_refcnt(struct hclge_dev *hdev, - struct hclge_fd_rule *rule) +static bool hclge_ae_dev_resetting(struct hnae3_handle *handle) { - struct hclge_fd_user_def_cfg *cfg; - - if (!rule || rule->rule_type != HCLGE_FD_EP_ACTIVE || - rule->ep.user_def.layer == HCLGE_FD_USER_DEF_NONE) - return; + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; - cfg = &hdev->fd_cfg.user_def_cfg[rule->ep.user_def.layer - 1]; - if (!cfg->ref_cnt) { - cfg->offset = rule->ep.user_def.offset; - set_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state); - } - cfg->ref_cnt++; + return test_bit(HCLGE_STATE_RST_HANDLING, &hdev->state); } -static void hclge_fd_dec_user_def_refcnt(struct hclge_dev *hdev, - struct hclge_fd_rule *rule) +static unsigned long hclge_ae_dev_reset_cnt(struct hnae3_handle *handle) { - struct hclge_fd_user_def_cfg *cfg; - - if (!rule || rule->rule_type != HCLGE_FD_EP_ACTIVE || - rule->ep.user_def.layer == HCLGE_FD_USER_DEF_NONE) - return; - - cfg = &hdev->fd_cfg.user_def_cfg[rule->ep.user_def.layer - 1]; - if (!cfg->ref_cnt) - return; + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; - cfg->ref_cnt--; - if (!cfg->ref_cnt) { - cfg->offset = 0; - set_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state); - } + return hdev->rst_stats.hw_reset_done_cnt; } -static void hclge_update_fd_list(struct hclge_dev *hdev, - enum HCLGE_FD_NODE_STATE state, u16 location, - struct hclge_fd_rule *new_rule) +static void hclge_cfg_mac_mode(struct hclge_dev *hdev, bool enable) { - struct hlist_head *hlist = &hdev->fd_rule_list; - struct hclge_fd_rule *fd_rule, *parent = NULL; - - fd_rule = hclge_find_fd_rule(hlist, location, &parent); - if (fd_rule) { - hclge_fd_dec_user_def_refcnt(hdev, fd_rule); - if (state == HCLGE_FD_ACTIVE) - hclge_fd_inc_user_def_refcnt(hdev, new_rule); - hclge_sync_fd_user_def_cfg(hdev, true); - - hclge_update_fd_rule_node(hdev, fd_rule, new_rule, state); - return; - } - - /* it's unlikely to fail here, because we have checked the rule - * exist before. - */ - if (unlikely(state == HCLGE_FD_TO_DEL || state == HCLGE_FD_DELETED)) { - dev_warn(&hdev->pdev->dev, - "failed to delete fd rule %u, it's inexistent\n", - location); - return; - } - - hclge_fd_inc_user_def_refcnt(hdev, new_rule); - hclge_sync_fd_user_def_cfg(hdev, true); - - hclge_fd_insert_rule_node(hlist, new_rule, parent); - hclge_fd_inc_rule_cnt(hdev, new_rule->location); - - if (state == HCLGE_FD_TO_ADD) { - set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state); - hclge_task_schedule(hdev, 0); - } -} +#define HCLGE_LINK_STATUS_WAIT_CNT 3 -static int hclge_get_fd_mode(struct hclge_dev *hdev, u8 *fd_mode) -{ - struct hclge_get_fd_mode_cmd *req; struct hclge_desc desc; + struct hclge_config_mac_mode_cmd *req = + (struct hclge_config_mac_mode_cmd *)desc.data; + u32 loop_en = 0; int ret; - hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_MODE_CTRL, true); - - req = (struct hclge_get_fd_mode_cmd *)desc.data; + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CONFIG_MAC_MODE, false); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) { - dev_err(&hdev->pdev->dev, "get fd mode fail, ret=%d\n", ret); - return ret; + if (enable) { + hnae3_set_bit(loop_en, HCLGE_MAC_TX_EN_B, 1U); + hnae3_set_bit(loop_en, HCLGE_MAC_RX_EN_B, 1U); + hnae3_set_bit(loop_en, HCLGE_MAC_PAD_TX_B, 1U); + hnae3_set_bit(loop_en, HCLGE_MAC_PAD_RX_B, 1U); + hnae3_set_bit(loop_en, HCLGE_MAC_FCS_TX_B, 1U); + hnae3_set_bit(loop_en, HCLGE_MAC_RX_FCS_B, 1U); + hnae3_set_bit(loop_en, HCLGE_MAC_RX_FCS_STRIP_B, 1U); + hnae3_set_bit(loop_en, HCLGE_MAC_TX_OVERSIZE_TRUNCATE_B, 1U); + hnae3_set_bit(loop_en, HCLGE_MAC_RX_OVERSIZE_TRUNCATE_B, 1U); + hnae3_set_bit(loop_en, HCLGE_MAC_TX_UNDER_MIN_ERR_B, 1U); } - *fd_mode = req->mode; - - return ret; -} - -static int hclge_get_fd_allocation(struct hclge_dev *hdev, - u32 *stage1_entry_num, - u32 *stage2_entry_num, - u16 *stage1_counter_num, - u16 *stage2_counter_num) -{ - struct hclge_get_fd_allocation_cmd *req; - struct hclge_desc desc; - int ret; - - hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_GET_ALLOCATION, true); - - req = (struct hclge_get_fd_allocation_cmd *)desc.data; + req->txrx_pad_fcs_loop_en = cpu_to_le32(loop_en); ret = hclge_cmd_send(&hdev->hw, &desc, 1); if (ret) { - dev_err(&hdev->pdev->dev, "query fd allocation fail, ret=%d\n", - ret); - return ret; - } - - *stage1_entry_num = le32_to_cpu(req->stage1_entry_num); - *stage2_entry_num = le32_to_cpu(req->stage2_entry_num); - *stage1_counter_num = le16_to_cpu(req->stage1_counter_num); - *stage2_counter_num = le16_to_cpu(req->stage2_counter_num); - - return ret; -} - -static int hclge_set_fd_key_config(struct hclge_dev *hdev, - enum HCLGE_FD_STAGE stage_num) -{ - struct hclge_set_fd_key_config_cmd *req; - struct hclge_fd_key_cfg *stage; - struct hclge_desc desc; - int ret; - - hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_KEY_CONFIG, false); - - req = (struct hclge_set_fd_key_config_cmd *)desc.data; - stage = &hdev->fd_cfg.key_cfg[stage_num]; - req->stage = stage_num; - req->key_select = stage->key_sel; - req->inner_sipv6_word_en = stage->inner_sipv6_word_en; - req->inner_dipv6_word_en = stage->inner_dipv6_word_en; - req->outer_sipv6_word_en = stage->outer_sipv6_word_en; - req->outer_dipv6_word_en = stage->outer_dipv6_word_en; - req->tuple_mask = cpu_to_le32(~stage->tuple_active); - req->meta_data_mask = cpu_to_le32(~stage->meta_data_active); - - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - dev_err(&hdev->pdev->dev, "set fd key fail, ret=%d\n", ret); - - return ret; -} - -static void hclge_fd_disable_user_def(struct hclge_dev *hdev) -{ - struct hclge_fd_user_def_cfg *cfg = hdev->fd_cfg.user_def_cfg; - - spin_lock_bh(&hdev->fd_rule_lock); - memset(cfg, 0, sizeof(hdev->fd_cfg.user_def_cfg)); - spin_unlock_bh(&hdev->fd_rule_lock); - - hclge_fd_set_user_def_cmd(hdev, cfg); -} - -static int hclge_init_fd_config(struct hclge_dev *hdev) -{ -#define LOW_2_WORDS 0x03 - struct hclge_fd_key_cfg *key_cfg; - int ret; - - if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) - return 0; - - ret = hclge_get_fd_mode(hdev, &hdev->fd_cfg.fd_mode); - if (ret) - return ret; - - switch (hdev->fd_cfg.fd_mode) { - case HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1: - hdev->fd_cfg.max_key_length = MAX_KEY_LENGTH; - break; - case HCLGE_FD_MODE_DEPTH_4K_WIDTH_200B_STAGE_1: - hdev->fd_cfg.max_key_length = MAX_KEY_LENGTH / 2; - break; - default: dev_err(&hdev->pdev->dev, - "Unsupported flow director mode %u\n", - hdev->fd_cfg.fd_mode); - return -EOPNOTSUPP; - } - - key_cfg = &hdev->fd_cfg.key_cfg[HCLGE_FD_STAGE_1]; - key_cfg->key_sel = HCLGE_FD_KEY_BASE_ON_TUPLE; - key_cfg->inner_sipv6_word_en = LOW_2_WORDS; - key_cfg->inner_dipv6_word_en = LOW_2_WORDS; - key_cfg->outer_sipv6_word_en = 0; - key_cfg->outer_dipv6_word_en = 0; - - key_cfg->tuple_active = BIT(INNER_VLAN_TAG_FST) | BIT(INNER_ETH_TYPE) | - BIT(INNER_IP_PROTO) | BIT(INNER_IP_TOS) | - BIT(INNER_SRC_IP) | BIT(INNER_DST_IP) | - BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT); - - /* If use max 400bit key, we can support tuples for ether type */ - if (hdev->fd_cfg.fd_mode == HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1) { - key_cfg->tuple_active |= BIT(INNER_DST_MAC) | - BIT(INNER_SRC_MAC) | - BIT(OUTER_TUN_VNI); - if (hdev->ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) - key_cfg->tuple_active |= HCLGE_FD_TUPLE_USER_DEF_TUPLES; - } - - /* roce_type is used to filter roce frames - * dst_vport is used to specify the rule - */ - key_cfg->meta_data_active = BIT(ROCE_TYPE) | BIT(DST_VPORT); - - ret = hclge_get_fd_allocation(hdev, - &hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1], - &hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_2], - &hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1], - &hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_2]); - if (ret) - return ret; - - return hclge_set_fd_key_config(hdev, HCLGE_FD_STAGE_1); -} - -static int hclge_fd_tcam_config(struct hclge_dev *hdev, u8 stage, bool sel_x, - int loc, u8 *key, bool is_add) -{ - struct hclge_fd_tcam_config_1_cmd *req1; - struct hclge_fd_tcam_config_2_cmd *req2; - struct hclge_fd_tcam_config_3_cmd *req3; - struct hclge_desc desc[3]; - int ret; - - hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_FD_TCAM_OP, false); - desc[0].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT); - hclge_cmd_setup_basic_desc(&desc[1], HCLGE_OPC_FD_TCAM_OP, false); - desc[1].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT); - hclge_cmd_setup_basic_desc(&desc[2], HCLGE_OPC_FD_TCAM_OP, false); - - req1 = (struct hclge_fd_tcam_config_1_cmd *)desc[0].data; - req2 = (struct hclge_fd_tcam_config_2_cmd *)desc[1].data; - req3 = (struct hclge_fd_tcam_config_3_cmd *)desc[2].data; - - req1->stage = stage; - req1->xy_sel = sel_x ? 1 : 0; - hnae3_set_bit(req1->port_info, HCLGE_FD_EPORT_SW_EN_B, 0); - req1->index = cpu_to_le32(loc); - req1->entry_vld = sel_x ? is_add : 0; - - if (key) { - memcpy(req1->tcam_data, &key[0], sizeof(req1->tcam_data)); - memcpy(req2->tcam_data, &key[sizeof(req1->tcam_data)], - sizeof(req2->tcam_data)); - memcpy(req3->tcam_data, &key[sizeof(req1->tcam_data) + - sizeof(req2->tcam_data)], sizeof(req3->tcam_data)); + "mac enable fail, ret =%d.\n", ret); + return; } - ret = hclge_cmd_send(&hdev->hw, desc, 3); - if (ret) - dev_err(&hdev->pdev->dev, - "config tcam key fail, ret=%d\n", - ret); - - return ret; -} - -static int hclge_fd_ad_config(struct hclge_dev *hdev, u8 stage, int loc, - struct hclge_fd_ad_data *action) -{ - struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); - struct hclge_fd_ad_config_cmd *req; - struct hclge_desc desc; - u64 ad_data = 0; - int ret; - - hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_AD_OP, false); - - req = (struct hclge_fd_ad_config_cmd *)desc.data; - req->index = cpu_to_le32(loc); - req->stage = stage; - - hnae3_set_bit(ad_data, HCLGE_FD_AD_WR_RULE_ID_B, - action->write_rule_id_to_bd); - hnae3_set_field(ad_data, HCLGE_FD_AD_RULE_ID_M, HCLGE_FD_AD_RULE_ID_S, - action->rule_id); - if (test_bit(HNAE3_DEV_SUPPORT_FD_FORWARD_TC_B, ae_dev->caps)) { - hnae3_set_bit(ad_data, HCLGE_FD_AD_TC_OVRD_B, - action->override_tc); - hnae3_set_field(ad_data, HCLGE_FD_AD_TC_SIZE_M, - HCLGE_FD_AD_TC_SIZE_S, (u32)action->tc_size); - } - hnae3_set_bit(ad_data, HCLGE_FD_AD_QID_H_B, - action->queue_id >= HCLGE_TQP_MAX_SIZE_DEV_V2 ? 1 : 0); - hnae3_set_bit(ad_data, HCLGE_FD_AD_COUNTER_NUM_H_B, - action->counter_id >= HCLGE_FD_COUNTER_MAX_SIZE_DEV_V2 ? - 1 : 0); - ad_data <<= 32; - hnae3_set_bit(ad_data, HCLGE_FD_AD_DROP_B, action->drop_packet); - hnae3_set_bit(ad_data, HCLGE_FD_AD_DIRECT_QID_B, - action->forward_to_direct_queue); - hnae3_set_field(ad_data, HCLGE_FD_AD_QID_L_M, HCLGE_FD_AD_QID_L_S, - action->queue_id); - hnae3_set_bit(ad_data, HCLGE_FD_AD_USE_COUNTER_B, action->use_counter); - hnae3_set_field(ad_data, HCLGE_FD_AD_COUNTER_NUM_L_M, - HCLGE_FD_AD_COUNTER_NUM_L_S, action->counter_id); - hnae3_set_bit(ad_data, HCLGE_FD_AD_NXT_STEP_B, action->use_next_stage); - hnae3_set_field(ad_data, HCLGE_FD_AD_NXT_KEY_M, HCLGE_FD_AD_NXT_KEY_S, - action->next_input_key); - - req->ad_data = cpu_to_le64(ad_data); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - dev_err(&hdev->pdev->dev, "fd ad config fail, ret=%d\n", ret); - - return ret; -} - -static bool hclge_fd_convert_tuple(u32 tuple_bit, u8 *key_x, u8 *key_y, - struct hclge_fd_rule *rule) -{ - int offset, moffset, ip_offset; - enum HCLGE_FD_KEY_OPT key_opt; - u16 tmp_x_s, tmp_y_s; - u32 tmp_x_l, tmp_y_l; - u8 *p = (u8 *)rule; - int i; - - if (rule->unused_tuple & BIT(tuple_bit)) - return true; - - key_opt = tuple_key_info[tuple_bit].key_opt; - offset = tuple_key_info[tuple_bit].offset; - moffset = tuple_key_info[tuple_bit].moffset; - - switch (key_opt) { - case KEY_OPT_U8: - calc_x(*key_x, p[offset], p[moffset]); - calc_y(*key_y, p[offset], p[moffset]); - - return true; - case KEY_OPT_LE16: - calc_x(tmp_x_s, *(u16 *)(&p[offset]), *(u16 *)(&p[moffset])); - calc_y(tmp_y_s, *(u16 *)(&p[offset]), *(u16 *)(&p[moffset])); - *(__le16 *)key_x = cpu_to_le16(tmp_x_s); - *(__le16 *)key_y = cpu_to_le16(tmp_y_s); - - return true; - case KEY_OPT_LE32: - calc_x(tmp_x_l, *(u32 *)(&p[offset]), *(u32 *)(&p[moffset])); - calc_y(tmp_y_l, *(u32 *)(&p[offset]), *(u32 *)(&p[moffset])); - *(__le32 *)key_x = cpu_to_le32(tmp_x_l); - *(__le32 *)key_y = cpu_to_le32(tmp_y_l); - - return true; - case KEY_OPT_MAC: - for (i = 0; i < ETH_ALEN; i++) { - calc_x(key_x[ETH_ALEN - 1 - i], p[offset + i], - p[moffset + i]); - calc_y(key_y[ETH_ALEN - 1 - i], p[offset + i], - p[moffset + i]); - } - - return true; - case KEY_OPT_IP: - ip_offset = IPV4_INDEX * sizeof(u32); - calc_x(tmp_x_l, *(u32 *)(&p[offset + ip_offset]), - *(u32 *)(&p[moffset + ip_offset])); - calc_y(tmp_y_l, *(u32 *)(&p[offset + ip_offset]), - *(u32 *)(&p[moffset + ip_offset])); - *(__le32 *)key_x = cpu_to_le32(tmp_x_l); - *(__le32 *)key_y = cpu_to_le32(tmp_y_l); - - return true; - case KEY_OPT_VNI: - calc_x(tmp_x_l, *(u32 *)(&p[offset]), *(u32 *)(&p[moffset])); - calc_y(tmp_y_l, *(u32 *)(&p[offset]), *(u32 *)(&p[moffset])); - for (i = 0; i < HCLGE_VNI_LENGTH; i++) { - key_x[i] = ((__force u32)cpu_to_le32(tmp_x_l) >> - (i * BITS_PER_BYTE)) & 0xFF; - key_y[i] = ((__force u32)cpu_to_le32(tmp_y_l) >> - (i * BITS_PER_BYTE)) & 0xFF; - } - - return true; - default: - return false; - } + if (!enable) + hclge_mac_link_status_wait(hdev, HCLGE_LINK_STATUS_DOWN, + HCLGE_LINK_STATUS_WAIT_CNT); } -static u32 hclge_get_port_number(enum HLCGE_PORT_TYPE port_type, u8 pf_id, - u8 vf_id, u8 network_port_id) +u32 hclge_get_port_number(enum HLCGE_PORT_TYPE port_type, u8 pf_id, + u8 vf_id, u8 network_port_id) { u32 port_number = 0; @@ -5800,1976 +5200,6 @@ static u32 hclge_get_port_number(enum HLCGE_PORT_TYPE port_type, u8 pf_id, return port_number; } -static void hclge_fd_convert_meta_data(struct hclge_fd_key_cfg *key_cfg, - __le32 *key_x, __le32 *key_y, - struct hclge_fd_rule *rule) -{ - u32 tuple_bit, meta_data = 0, tmp_x, tmp_y, port_number; - u8 cur_pos = 0, tuple_size, shift_bits; - unsigned int i; - - for (i = 0; i < MAX_META_DATA; i++) { - tuple_size = meta_data_key_info[i].key_length; - tuple_bit = key_cfg->meta_data_active & BIT(i); - - switch (tuple_bit) { - case BIT(ROCE_TYPE): - hnae3_set_bit(meta_data, cur_pos, NIC_PACKET); - cur_pos += tuple_size; - break; - case BIT(DST_VPORT): - port_number = hclge_get_port_number(HOST_PORT, 0, - rule->vf_id, 0); - hnae3_set_field(meta_data, - GENMASK(cur_pos + tuple_size, cur_pos), - cur_pos, port_number); - cur_pos += tuple_size; - break; - default: - break; - } - } - - calc_x(tmp_x, meta_data, 0xFFFFFFFF); - calc_y(tmp_y, meta_data, 0xFFFFFFFF); - shift_bits = sizeof(meta_data) * 8 - cur_pos; - - *key_x = cpu_to_le32(tmp_x << shift_bits); - *key_y = cpu_to_le32(tmp_y << shift_bits); -} - -/* A complete key is combined with meta data key and tuple key. - * Meta data key is stored at the MSB region, and tuple key is stored at - * the LSB region, unused bits will be filled 0. - */ -static int hclge_config_key(struct hclge_dev *hdev, u8 stage, - struct hclge_fd_rule *rule) -{ - struct hclge_fd_key_cfg *key_cfg = &hdev->fd_cfg.key_cfg[stage]; - u8 key_x[MAX_KEY_BYTES], key_y[MAX_KEY_BYTES]; - u8 *cur_key_x, *cur_key_y; - u8 meta_data_region; - u8 tuple_size; - int ret; - u32 i; - - memset(key_x, 0, sizeof(key_x)); - memset(key_y, 0, sizeof(key_y)); - cur_key_x = key_x; - cur_key_y = key_y; - - for (i = 0; i < MAX_TUPLE; i++) { - bool tuple_valid; - - tuple_size = tuple_key_info[i].key_length / 8; - if (!(key_cfg->tuple_active & BIT(i))) - continue; - - tuple_valid = hclge_fd_convert_tuple(i, cur_key_x, - cur_key_y, rule); - if (tuple_valid) { - cur_key_x += tuple_size; - cur_key_y += tuple_size; - } - } - - meta_data_region = hdev->fd_cfg.max_key_length / 8 - - MAX_META_DATA_LENGTH / 8; - - hclge_fd_convert_meta_data(key_cfg, - (__le32 *)(key_x + meta_data_region), - (__le32 *)(key_y + meta_data_region), - rule); - - ret = hclge_fd_tcam_config(hdev, stage, false, rule->location, key_y, - true); - if (ret) { - dev_err(&hdev->pdev->dev, - "fd key_y config fail, loc=%u, ret=%d\n", - rule->queue_id, ret); - return ret; - } - - ret = hclge_fd_tcam_config(hdev, stage, true, rule->location, key_x, - true); - if (ret) - dev_err(&hdev->pdev->dev, - "fd key_x config fail, loc=%u, ret=%d\n", - rule->queue_id, ret); - return ret; -} - -static int hclge_config_action(struct hclge_dev *hdev, u8 stage, - struct hclge_fd_rule *rule) -{ - struct hclge_vport *vport = hdev->vport; - struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo; - struct hclge_fd_ad_data ad_data; - - memset(&ad_data, 0, sizeof(struct hclge_fd_ad_data)); - ad_data.ad_id = rule->location; - - if (rule->action == HCLGE_FD_ACTION_DROP_PACKET) { - ad_data.drop_packet = true; - } else if (rule->action == HCLGE_FD_ACTION_SELECT_TC) { - ad_data.override_tc = true; - ad_data.queue_id = - kinfo->tc_info.tqp_offset[rule->cls_flower.tc]; - ad_data.tc_size = - ilog2(kinfo->tc_info.tqp_count[rule->cls_flower.tc]); - } else { - ad_data.forward_to_direct_queue = true; - ad_data.queue_id = rule->queue_id; - } - - if (hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1]) { - ad_data.use_counter = true; - ad_data.counter_id = rule->vf_id % - hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1]; - } else { - ad_data.use_counter = false; - ad_data.counter_id = 0; - } - - ad_data.use_next_stage = false; - ad_data.next_input_key = 0; - - ad_data.write_rule_id_to_bd = true; - ad_data.rule_id = rule->location; - - return hclge_fd_ad_config(hdev, stage, ad_data.ad_id, &ad_data); -} - -static int hclge_fd_check_tcpip4_tuple(struct ethtool_tcpip4_spec *spec, - u32 *unused_tuple) -{ - if (!spec || !unused_tuple) - return -EINVAL; - - *unused_tuple |= BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC); - - if (!spec->ip4src) - *unused_tuple |= BIT(INNER_SRC_IP); - - if (!spec->ip4dst) - *unused_tuple |= BIT(INNER_DST_IP); - - if (!spec->psrc) - *unused_tuple |= BIT(INNER_SRC_PORT); - - if (!spec->pdst) - *unused_tuple |= BIT(INNER_DST_PORT); - - if (!spec->tos) - *unused_tuple |= BIT(INNER_IP_TOS); - - return 0; -} - -static int hclge_fd_check_ip4_tuple(struct ethtool_usrip4_spec *spec, - u32 *unused_tuple) -{ - if (!spec || !unused_tuple) - return -EINVAL; - - *unused_tuple |= BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC) | - BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT); - - if (!spec->ip4src) - *unused_tuple |= BIT(INNER_SRC_IP); - - if (!spec->ip4dst) - *unused_tuple |= BIT(INNER_DST_IP); - - if (!spec->tos) - *unused_tuple |= BIT(INNER_IP_TOS); - - if (!spec->proto) - *unused_tuple |= BIT(INNER_IP_PROTO); - - if (spec->l4_4_bytes) - return -EOPNOTSUPP; - - if (spec->ip_ver != ETH_RX_NFC_IP4) - return -EOPNOTSUPP; - - return 0; -} - -static int hclge_fd_check_tcpip6_tuple(struct ethtool_tcpip6_spec *spec, - u32 *unused_tuple) -{ - if (!spec || !unused_tuple) - return -EINVAL; - - *unused_tuple |= BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC); - - /* check whether src/dst ip address used */ - if (ipv6_addr_any((struct in6_addr *)spec->ip6src)) - *unused_tuple |= BIT(INNER_SRC_IP); - - if (ipv6_addr_any((struct in6_addr *)spec->ip6dst)) - *unused_tuple |= BIT(INNER_DST_IP); - - if (!spec->psrc) - *unused_tuple |= BIT(INNER_SRC_PORT); - - if (!spec->pdst) - *unused_tuple |= BIT(INNER_DST_PORT); - - if (!spec->tclass) - *unused_tuple |= BIT(INNER_IP_TOS); - - return 0; -} - -static int hclge_fd_check_ip6_tuple(struct ethtool_usrip6_spec *spec, - u32 *unused_tuple) -{ - if (!spec || !unused_tuple) - return -EINVAL; - - *unused_tuple |= BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC) | - BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT); - - /* check whether src/dst ip address used */ - if (ipv6_addr_any((struct in6_addr *)spec->ip6src)) - *unused_tuple |= BIT(INNER_SRC_IP); - - if (ipv6_addr_any((struct in6_addr *)spec->ip6dst)) - *unused_tuple |= BIT(INNER_DST_IP); - - if (!spec->l4_proto) - *unused_tuple |= BIT(INNER_IP_PROTO); - - if (!spec->tclass) - *unused_tuple |= BIT(INNER_IP_TOS); - - if (spec->l4_4_bytes) - return -EOPNOTSUPP; - - return 0; -} - -static int hclge_fd_check_ether_tuple(struct ethhdr *spec, u32 *unused_tuple) -{ - if (!spec || !unused_tuple) - return -EINVAL; - - *unused_tuple |= BIT(INNER_SRC_IP) | BIT(INNER_DST_IP) | - BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT) | - BIT(INNER_IP_TOS) | BIT(INNER_IP_PROTO); - - if (is_zero_ether_addr(spec->h_source)) - *unused_tuple |= BIT(INNER_SRC_MAC); - - if (is_zero_ether_addr(spec->h_dest)) - *unused_tuple |= BIT(INNER_DST_MAC); - - if (!spec->h_proto) - *unused_tuple |= BIT(INNER_ETH_TYPE); - - return 0; -} - -static int hclge_fd_check_ext_tuple(struct hclge_dev *hdev, - struct ethtool_rx_flow_spec *fs, - u32 *unused_tuple) -{ - if (fs->flow_type & FLOW_EXT) { - if (fs->h_ext.vlan_etype) { - dev_err(&hdev->pdev->dev, "vlan-etype is not supported!\n"); - return -EOPNOTSUPP; - } - - if (!fs->h_ext.vlan_tci) - *unused_tuple |= BIT(INNER_VLAN_TAG_FST); - - if (fs->m_ext.vlan_tci && - be16_to_cpu(fs->h_ext.vlan_tci) >= VLAN_N_VID) { - dev_err(&hdev->pdev->dev, - "failed to config vlan_tci, invalid vlan_tci: %u, max is %d.\n", - ntohs(fs->h_ext.vlan_tci), VLAN_N_VID - 1); - return -EINVAL; - } - } else { - *unused_tuple |= BIT(INNER_VLAN_TAG_FST); - } - - if (fs->flow_type & FLOW_MAC_EXT) { - if (hdev->fd_cfg.fd_mode != - HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1) { - dev_err(&hdev->pdev->dev, - "FLOW_MAC_EXT is not supported in current fd mode!\n"); - return -EOPNOTSUPP; - } - - if (is_zero_ether_addr(fs->h_ext.h_dest)) - *unused_tuple |= BIT(INNER_DST_MAC); - else - *unused_tuple &= ~BIT(INNER_DST_MAC); - } - - return 0; -} - -static int hclge_fd_get_user_def_layer(u32 flow_type, u32 *unused_tuple, - struct hclge_fd_user_def_info *info) -{ - switch (flow_type) { - case ETHER_FLOW: - info->layer = HCLGE_FD_USER_DEF_L2; - *unused_tuple &= ~BIT(INNER_L2_RSV); - break; - case IP_USER_FLOW: - case IPV6_USER_FLOW: - info->layer = HCLGE_FD_USER_DEF_L3; - *unused_tuple &= ~BIT(INNER_L3_RSV); - break; - case TCP_V4_FLOW: - case UDP_V4_FLOW: - case TCP_V6_FLOW: - case UDP_V6_FLOW: - info->layer = HCLGE_FD_USER_DEF_L4; - *unused_tuple &= ~BIT(INNER_L4_RSV); - break; - default: - return -EOPNOTSUPP; - } - - return 0; -} - -static bool hclge_fd_is_user_def_all_masked(struct ethtool_rx_flow_spec *fs) -{ - return be32_to_cpu(fs->m_ext.data[1] | fs->m_ext.data[0]) == 0; -} - -static int hclge_fd_parse_user_def_field(struct hclge_dev *hdev, - struct ethtool_rx_flow_spec *fs, - u32 *unused_tuple, - struct hclge_fd_user_def_info *info) -{ - u32 tuple_active = hdev->fd_cfg.key_cfg[HCLGE_FD_STAGE_1].tuple_active; - u32 flow_type = fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT); - u16 data, offset, data_mask, offset_mask; - int ret; - - info->layer = HCLGE_FD_USER_DEF_NONE; - *unused_tuple |= HCLGE_FD_TUPLE_USER_DEF_TUPLES; - - if (!(fs->flow_type & FLOW_EXT) || hclge_fd_is_user_def_all_masked(fs)) - return 0; - - /* user-def data from ethtool is 64 bit value, the bit0~15 is used - * for data, and bit32~47 is used for offset. - */ - data = be32_to_cpu(fs->h_ext.data[1]) & HCLGE_FD_USER_DEF_DATA; - data_mask = be32_to_cpu(fs->m_ext.data[1]) & HCLGE_FD_USER_DEF_DATA; - offset = be32_to_cpu(fs->h_ext.data[0]) & HCLGE_FD_USER_DEF_OFFSET; - offset_mask = be32_to_cpu(fs->m_ext.data[0]) & HCLGE_FD_USER_DEF_OFFSET; - - if (!(tuple_active & HCLGE_FD_TUPLE_USER_DEF_TUPLES)) { - dev_err(&hdev->pdev->dev, "user-def bytes are not supported\n"); - return -EOPNOTSUPP; - } - - if (offset > HCLGE_FD_MAX_USER_DEF_OFFSET) { - dev_err(&hdev->pdev->dev, - "user-def offset[%u] should be no more than %u\n", - offset, HCLGE_FD_MAX_USER_DEF_OFFSET); - return -EINVAL; - } - - if (offset_mask != HCLGE_FD_USER_DEF_OFFSET_UNMASK) { - dev_err(&hdev->pdev->dev, "user-def offset can't be masked\n"); - return -EINVAL; - } - - ret = hclge_fd_get_user_def_layer(flow_type, unused_tuple, info); - if (ret) { - dev_err(&hdev->pdev->dev, - "unsupported flow type for user-def bytes, ret = %d\n", - ret); - return ret; - } - - info->data = data; - info->data_mask = data_mask; - info->offset = offset; - - return 0; -} - -static int hclge_fd_check_spec(struct hclge_dev *hdev, - struct ethtool_rx_flow_spec *fs, - u32 *unused_tuple, - struct hclge_fd_user_def_info *info) -{ - u32 flow_type; - int ret; - - if (fs->location >= hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) { - dev_err(&hdev->pdev->dev, - "failed to config fd rules, invalid rule location: %u, max is %u\n.", - fs->location, - hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1] - 1); - return -EINVAL; - } - - ret = hclge_fd_parse_user_def_field(hdev, fs, unused_tuple, info); - if (ret) - return ret; - - flow_type = fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT); - switch (flow_type) { - case SCTP_V4_FLOW: - case TCP_V4_FLOW: - case UDP_V4_FLOW: - ret = hclge_fd_check_tcpip4_tuple(&fs->h_u.tcp_ip4_spec, - unused_tuple); - break; - case IP_USER_FLOW: - ret = hclge_fd_check_ip4_tuple(&fs->h_u.usr_ip4_spec, - unused_tuple); - break; - case SCTP_V6_FLOW: - case TCP_V6_FLOW: - case UDP_V6_FLOW: - ret = hclge_fd_check_tcpip6_tuple(&fs->h_u.tcp_ip6_spec, - unused_tuple); - break; - case IPV6_USER_FLOW: - ret = hclge_fd_check_ip6_tuple(&fs->h_u.usr_ip6_spec, - unused_tuple); - break; - case ETHER_FLOW: - if (hdev->fd_cfg.fd_mode != - HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1) { - dev_err(&hdev->pdev->dev, - "ETHER_FLOW is not supported in current fd mode!\n"); - return -EOPNOTSUPP; - } - - ret = hclge_fd_check_ether_tuple(&fs->h_u.ether_spec, - unused_tuple); - break; - default: - dev_err(&hdev->pdev->dev, - "unsupported protocol type, protocol type = %#x\n", - flow_type); - return -EOPNOTSUPP; - } - - if (ret) { - dev_err(&hdev->pdev->dev, - "failed to check flow union tuple, ret = %d\n", - ret); - return ret; - } - - return hclge_fd_check_ext_tuple(hdev, fs, unused_tuple); -} - -static void hclge_fd_get_tcpip4_tuple(struct ethtool_rx_flow_spec *fs, - struct hclge_fd_rule *rule, u8 ip_proto) -{ - rule->tuples.src_ip[IPV4_INDEX] = - be32_to_cpu(fs->h_u.tcp_ip4_spec.ip4src); - rule->tuples_mask.src_ip[IPV4_INDEX] = - be32_to_cpu(fs->m_u.tcp_ip4_spec.ip4src); - - rule->tuples.dst_ip[IPV4_INDEX] = - be32_to_cpu(fs->h_u.tcp_ip4_spec.ip4dst); - rule->tuples_mask.dst_ip[IPV4_INDEX] = - be32_to_cpu(fs->m_u.tcp_ip4_spec.ip4dst); - - rule->tuples.src_port = be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc); - rule->tuples_mask.src_port = be16_to_cpu(fs->m_u.tcp_ip4_spec.psrc); - - rule->tuples.dst_port = be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst); - rule->tuples_mask.dst_port = be16_to_cpu(fs->m_u.tcp_ip4_spec.pdst); - - rule->tuples.ip_tos = fs->h_u.tcp_ip4_spec.tos; - rule->tuples_mask.ip_tos = fs->m_u.tcp_ip4_spec.tos; - - rule->tuples.ether_proto = ETH_P_IP; - rule->tuples_mask.ether_proto = 0xFFFF; - - rule->tuples.ip_proto = ip_proto; - rule->tuples_mask.ip_proto = 0xFF; -} - -static void hclge_fd_get_ip4_tuple(struct ethtool_rx_flow_spec *fs, - struct hclge_fd_rule *rule) -{ - rule->tuples.src_ip[IPV4_INDEX] = - be32_to_cpu(fs->h_u.usr_ip4_spec.ip4src); - rule->tuples_mask.src_ip[IPV4_INDEX] = - be32_to_cpu(fs->m_u.usr_ip4_spec.ip4src); - - rule->tuples.dst_ip[IPV4_INDEX] = - be32_to_cpu(fs->h_u.usr_ip4_spec.ip4dst); - rule->tuples_mask.dst_ip[IPV4_INDEX] = - be32_to_cpu(fs->m_u.usr_ip4_spec.ip4dst); - - rule->tuples.ip_tos = fs->h_u.usr_ip4_spec.tos; - rule->tuples_mask.ip_tos = fs->m_u.usr_ip4_spec.tos; - - rule->tuples.ip_proto = fs->h_u.usr_ip4_spec.proto; - rule->tuples_mask.ip_proto = fs->m_u.usr_ip4_spec.proto; - - rule->tuples.ether_proto = ETH_P_IP; - rule->tuples_mask.ether_proto = 0xFFFF; -} - -static void hclge_fd_get_tcpip6_tuple(struct ethtool_rx_flow_spec *fs, - struct hclge_fd_rule *rule, u8 ip_proto) -{ - ipv6_addr_be32_to_cpu(rule->tuples.src_ip, - fs->h_u.tcp_ip6_spec.ip6src); - ipv6_addr_be32_to_cpu(rule->tuples_mask.src_ip, - fs->m_u.tcp_ip6_spec.ip6src); - - ipv6_addr_be32_to_cpu(rule->tuples.dst_ip, - fs->h_u.tcp_ip6_spec.ip6dst); - ipv6_addr_be32_to_cpu(rule->tuples_mask.dst_ip, - fs->m_u.tcp_ip6_spec.ip6dst); - - rule->tuples.src_port = be16_to_cpu(fs->h_u.tcp_ip6_spec.psrc); - rule->tuples_mask.src_port = be16_to_cpu(fs->m_u.tcp_ip6_spec.psrc); - - rule->tuples.dst_port = be16_to_cpu(fs->h_u.tcp_ip6_spec.pdst); - rule->tuples_mask.dst_port = be16_to_cpu(fs->m_u.tcp_ip6_spec.pdst); - - rule->tuples.ether_proto = ETH_P_IPV6; - rule->tuples_mask.ether_proto = 0xFFFF; - - rule->tuples.ip_tos = fs->h_u.tcp_ip6_spec.tclass; - rule->tuples_mask.ip_tos = fs->m_u.tcp_ip6_spec.tclass; - - rule->tuples.ip_proto = ip_proto; - rule->tuples_mask.ip_proto = 0xFF; -} - -static void hclge_fd_get_ip6_tuple(struct ethtool_rx_flow_spec *fs, - struct hclge_fd_rule *rule) -{ - ipv6_addr_be32_to_cpu(rule->tuples.src_ip, - fs->h_u.usr_ip6_spec.ip6src); - ipv6_addr_be32_to_cpu(rule->tuples_mask.src_ip, - fs->m_u.usr_ip6_spec.ip6src); - - ipv6_addr_be32_to_cpu(rule->tuples.dst_ip, - fs->h_u.usr_ip6_spec.ip6dst); - ipv6_addr_be32_to_cpu(rule->tuples_mask.dst_ip, - fs->m_u.usr_ip6_spec.ip6dst); - - rule->tuples.ip_proto = fs->h_u.usr_ip6_spec.l4_proto; - rule->tuples_mask.ip_proto = fs->m_u.usr_ip6_spec.l4_proto; - - rule->tuples.ip_tos = fs->h_u.tcp_ip6_spec.tclass; - rule->tuples_mask.ip_tos = fs->m_u.tcp_ip6_spec.tclass; - - rule->tuples.ether_proto = ETH_P_IPV6; - rule->tuples_mask.ether_proto = 0xFFFF; -} - -static void hclge_fd_get_ether_tuple(struct ethtool_rx_flow_spec *fs, - struct hclge_fd_rule *rule) -{ - ether_addr_copy(rule->tuples.src_mac, fs->h_u.ether_spec.h_source); - ether_addr_copy(rule->tuples_mask.src_mac, fs->m_u.ether_spec.h_source); - - ether_addr_copy(rule->tuples.dst_mac, fs->h_u.ether_spec.h_dest); - ether_addr_copy(rule->tuples_mask.dst_mac, fs->m_u.ether_spec.h_dest); - - rule->tuples.ether_proto = be16_to_cpu(fs->h_u.ether_spec.h_proto); - rule->tuples_mask.ether_proto = be16_to_cpu(fs->m_u.ether_spec.h_proto); -} - -static void hclge_fd_get_user_def_tuple(struct hclge_fd_user_def_info *info, - struct hclge_fd_rule *rule) -{ - switch (info->layer) { - case HCLGE_FD_USER_DEF_L2: - rule->tuples.l2_user_def = info->data; - rule->tuples_mask.l2_user_def = info->data_mask; - break; - case HCLGE_FD_USER_DEF_L3: - rule->tuples.l3_user_def = info->data; - rule->tuples_mask.l3_user_def = info->data_mask; - break; - case HCLGE_FD_USER_DEF_L4: - rule->tuples.l4_user_def = (u32)info->data << 16; - rule->tuples_mask.l4_user_def = (u32)info->data_mask << 16; - break; - default: - break; - } - - rule->ep.user_def = *info; -} - -static int hclge_fd_get_tuple(struct ethtool_rx_flow_spec *fs, - struct hclge_fd_rule *rule, - struct hclge_fd_user_def_info *info) -{ - u32 flow_type = fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT); - - switch (flow_type) { - case SCTP_V4_FLOW: - hclge_fd_get_tcpip4_tuple(fs, rule, IPPROTO_SCTP); - break; - case TCP_V4_FLOW: - hclge_fd_get_tcpip4_tuple(fs, rule, IPPROTO_TCP); - break; - case UDP_V4_FLOW: - hclge_fd_get_tcpip4_tuple(fs, rule, IPPROTO_UDP); - break; - case IP_USER_FLOW: - hclge_fd_get_ip4_tuple(fs, rule); - break; - case SCTP_V6_FLOW: - hclge_fd_get_tcpip6_tuple(fs, rule, IPPROTO_SCTP); - break; - case TCP_V6_FLOW: - hclge_fd_get_tcpip6_tuple(fs, rule, IPPROTO_TCP); - break; - case UDP_V6_FLOW: - hclge_fd_get_tcpip6_tuple(fs, rule, IPPROTO_UDP); - break; - case IPV6_USER_FLOW: - hclge_fd_get_ip6_tuple(fs, rule); - break; - case ETHER_FLOW: - hclge_fd_get_ether_tuple(fs, rule); - break; - default: - return -EOPNOTSUPP; - } - - if (fs->flow_type & FLOW_EXT) { - rule->tuples.vlan_tag1 = be16_to_cpu(fs->h_ext.vlan_tci); - rule->tuples_mask.vlan_tag1 = be16_to_cpu(fs->m_ext.vlan_tci); - hclge_fd_get_user_def_tuple(info, rule); - } - - if (fs->flow_type & FLOW_MAC_EXT) { - ether_addr_copy(rule->tuples.dst_mac, fs->h_ext.h_dest); - ether_addr_copy(rule->tuples_mask.dst_mac, fs->m_ext.h_dest); - } - - return 0; -} - -static int hclge_fd_config_rule(struct hclge_dev *hdev, - struct hclge_fd_rule *rule) -{ - int ret; - - ret = hclge_config_action(hdev, HCLGE_FD_STAGE_1, rule); - if (ret) - return ret; - - return hclge_config_key(hdev, HCLGE_FD_STAGE_1, rule); -} - -static int hclge_add_fd_entry_common(struct hclge_dev *hdev, - struct hclge_fd_rule *rule) -{ - int ret; - - spin_lock_bh(&hdev->fd_rule_lock); - - if (hdev->fd_active_type != rule->rule_type && - (hdev->fd_active_type == HCLGE_FD_TC_FLOWER_ACTIVE || - hdev->fd_active_type == HCLGE_FD_EP_ACTIVE)) { - dev_err(&hdev->pdev->dev, - "mode conflict(new type %d, active type %d), please delete existent rules first\n", - rule->rule_type, hdev->fd_active_type); - spin_unlock_bh(&hdev->fd_rule_lock); - return -EINVAL; - } - - ret = hclge_fd_check_user_def_refcnt(hdev, rule); - if (ret) - goto out; - - ret = hclge_clear_arfs_rules(hdev); - if (ret) - goto out; - - ret = hclge_fd_config_rule(hdev, rule); - if (ret) - goto out; - - rule->state = HCLGE_FD_ACTIVE; - hdev->fd_active_type = rule->rule_type; - hclge_update_fd_list(hdev, rule->state, rule->location, rule); - -out: - spin_unlock_bh(&hdev->fd_rule_lock); - return ret; -} - -static bool hclge_is_cls_flower_active(struct hnae3_handle *handle) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - - return hdev->fd_active_type == HCLGE_FD_TC_FLOWER_ACTIVE; -} - -static int hclge_fd_parse_ring_cookie(struct hclge_dev *hdev, u64 ring_cookie, - u16 *vport_id, u8 *action, u16 *queue_id) -{ - struct hclge_vport *vport = hdev->vport; - - if (ring_cookie == RX_CLS_FLOW_DISC) { - *action = HCLGE_FD_ACTION_DROP_PACKET; - } else { - u32 ring = ethtool_get_flow_spec_ring(ring_cookie); - u8 vf = ethtool_get_flow_spec_ring_vf(ring_cookie); - u16 tqps; - - /* To keep consistent with user's configuration, minus 1 when - * printing 'vf', because vf id from ethtool is added 1 for vf. - */ - if (vf > hdev->num_req_vfs) { - dev_err(&hdev->pdev->dev, - "Error: vf id (%u) should be less than %u\n", - vf - 1U, hdev->num_req_vfs); - return -EINVAL; - } - - *vport_id = vf ? hdev->vport[vf].vport_id : vport->vport_id; - tqps = hdev->vport[vf].nic.kinfo.num_tqps; - - if (ring >= tqps) { - dev_err(&hdev->pdev->dev, - "Error: queue id (%u) > max tqp num (%u)\n", - ring, tqps - 1U); - return -EINVAL; - } - - *action = HCLGE_FD_ACTION_SELECT_QUEUE; - *queue_id = ring; - } - - return 0; -} - -static int hclge_add_fd_entry(struct hnae3_handle *handle, - struct ethtool_rxnfc *cmd) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - struct hclge_fd_user_def_info info; - u16 dst_vport_id = 0, q_index = 0; - struct ethtool_rx_flow_spec *fs; - struct hclge_fd_rule *rule; - u32 unused = 0; - u8 action; - int ret; - - if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) { - dev_err(&hdev->pdev->dev, - "flow table director is not supported\n"); - return -EOPNOTSUPP; - } - - if (!hdev->fd_en) { - dev_err(&hdev->pdev->dev, - "please enable flow director first\n"); - return -EOPNOTSUPP; - } - - fs = (struct ethtool_rx_flow_spec *)&cmd->fs; - - ret = hclge_fd_check_spec(hdev, fs, &unused, &info); - if (ret) - return ret; - - ret = hclge_fd_parse_ring_cookie(hdev, fs->ring_cookie, &dst_vport_id, - &action, &q_index); - if (ret) - return ret; - - rule = kzalloc_obj(*rule); - if (!rule) - return -ENOMEM; - - ret = hclge_fd_get_tuple(fs, rule, &info); - if (ret) { - kfree(rule); - return ret; - } - - rule->flow_type = fs->flow_type; - rule->location = fs->location; - rule->unused_tuple = unused; - rule->vf_id = dst_vport_id; - rule->queue_id = q_index; - rule->action = action; - rule->rule_type = HCLGE_FD_EP_ACTIVE; - - ret = hclge_add_fd_entry_common(hdev, rule); - if (ret) - kfree(rule); - - return ret; -} - -static int hclge_del_fd_entry(struct hnae3_handle *handle, - struct ethtool_rxnfc *cmd) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - struct ethtool_rx_flow_spec *fs; - int ret; - - if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) - return -EOPNOTSUPP; - - fs = (struct ethtool_rx_flow_spec *)&cmd->fs; - - if (fs->location >= hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) - return -EINVAL; - - spin_lock_bh(&hdev->fd_rule_lock); - if (hdev->fd_active_type == HCLGE_FD_TC_FLOWER_ACTIVE || - !test_bit(fs->location, hdev->fd_bmap)) { - dev_err(&hdev->pdev->dev, - "Delete fail, rule %u is inexistent\n", fs->location); - spin_unlock_bh(&hdev->fd_rule_lock); - return -ENOENT; - } - - ret = hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, fs->location, - NULL, false); - if (ret) - goto out; - - hclge_update_fd_list(hdev, HCLGE_FD_DELETED, fs->location, NULL); - -out: - spin_unlock_bh(&hdev->fd_rule_lock); - return ret; -} - -static void hclge_clear_fd_rules_in_list(struct hclge_dev *hdev, - bool clear_list) -{ - struct hclge_fd_rule *rule; - struct hlist_node *node; - u16 location; - - spin_lock_bh(&hdev->fd_rule_lock); - - for_each_set_bit(location, hdev->fd_bmap, - hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) - hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, location, - NULL, false); - - if (clear_list) { - hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, - rule_node) { - hlist_del(&rule->rule_node); - kfree(rule); - } - hdev->fd_active_type = HCLGE_FD_RULE_NONE; - hdev->hclge_fd_rule_num = 0; - bitmap_zero(hdev->fd_bmap, - hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]); - } - - spin_unlock_bh(&hdev->fd_rule_lock); -} - -static void hclge_del_all_fd_entries(struct hclge_dev *hdev) -{ - if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) - return; - - hclge_clear_fd_rules_in_list(hdev, true); - hclge_fd_disable_user_def(hdev); -} - -static int hclge_restore_fd_entries(struct hnae3_handle *handle) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - struct hclge_fd_rule *rule; - struct hlist_node *node; - - /* Return ok here, because reset error handling will check this - * return value. If error is returned here, the reset process will - * fail. - */ - if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) - return 0; - - /* if fd is disabled, should not restore it when reset */ - if (!hdev->fd_en) - return 0; - - spin_lock_bh(&hdev->fd_rule_lock); - hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) { - if (rule->state == HCLGE_FD_ACTIVE) - rule->state = HCLGE_FD_TO_ADD; - } - spin_unlock_bh(&hdev->fd_rule_lock); - set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state); - - return 0; -} - -static int hclge_get_fd_rule_cnt(struct hnae3_handle *handle, - struct ethtool_rxnfc *cmd) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - - if (!hnae3_ae_dev_fd_supported(hdev->ae_dev) || hclge_is_cls_flower_active(handle)) - return -EOPNOTSUPP; - - cmd->rule_cnt = hdev->hclge_fd_rule_num; - cmd->data = hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]; - - return 0; -} - -static void hclge_fd_get_tcpip4_info(struct hclge_fd_rule *rule, - struct ethtool_tcpip4_spec *spec, - struct ethtool_tcpip4_spec *spec_mask) -{ - spec->ip4src = cpu_to_be32(rule->tuples.src_ip[IPV4_INDEX]); - spec_mask->ip4src = rule->unused_tuple & BIT(INNER_SRC_IP) ? - 0 : cpu_to_be32(rule->tuples_mask.src_ip[IPV4_INDEX]); - - spec->ip4dst = cpu_to_be32(rule->tuples.dst_ip[IPV4_INDEX]); - spec_mask->ip4dst = rule->unused_tuple & BIT(INNER_DST_IP) ? - 0 : cpu_to_be32(rule->tuples_mask.dst_ip[IPV4_INDEX]); - - spec->psrc = cpu_to_be16(rule->tuples.src_port); - spec_mask->psrc = rule->unused_tuple & BIT(INNER_SRC_PORT) ? - 0 : cpu_to_be16(rule->tuples_mask.src_port); - - spec->pdst = cpu_to_be16(rule->tuples.dst_port); - spec_mask->pdst = rule->unused_tuple & BIT(INNER_DST_PORT) ? - 0 : cpu_to_be16(rule->tuples_mask.dst_port); - - spec->tos = rule->tuples.ip_tos; - spec_mask->tos = rule->unused_tuple & BIT(INNER_IP_TOS) ? - 0 : rule->tuples_mask.ip_tos; -} - -static void hclge_fd_get_ip4_info(struct hclge_fd_rule *rule, - struct ethtool_usrip4_spec *spec, - struct ethtool_usrip4_spec *spec_mask) -{ - spec->ip4src = cpu_to_be32(rule->tuples.src_ip[IPV4_INDEX]); - spec_mask->ip4src = rule->unused_tuple & BIT(INNER_SRC_IP) ? - 0 : cpu_to_be32(rule->tuples_mask.src_ip[IPV4_INDEX]); - - spec->ip4dst = cpu_to_be32(rule->tuples.dst_ip[IPV4_INDEX]); - spec_mask->ip4dst = rule->unused_tuple & BIT(INNER_DST_IP) ? - 0 : cpu_to_be32(rule->tuples_mask.dst_ip[IPV4_INDEX]); - - spec->tos = rule->tuples.ip_tos; - spec_mask->tos = rule->unused_tuple & BIT(INNER_IP_TOS) ? - 0 : rule->tuples_mask.ip_tos; - - spec->proto = rule->tuples.ip_proto; - spec_mask->proto = rule->unused_tuple & BIT(INNER_IP_PROTO) ? - 0 : rule->tuples_mask.ip_proto; - - spec->ip_ver = ETH_RX_NFC_IP4; -} - -static void hclge_fd_get_tcpip6_info(struct hclge_fd_rule *rule, - struct ethtool_tcpip6_spec *spec, - struct ethtool_tcpip6_spec *spec_mask) -{ - ipv6_addr_cpu_to_be32(spec->ip6src, rule->tuples.src_ip); - ipv6_addr_cpu_to_be32(spec->ip6dst, rule->tuples.dst_ip); - if (rule->unused_tuple & BIT(INNER_SRC_IP)) - memset(spec_mask->ip6src, 0, sizeof(spec_mask->ip6src)); - else - ipv6_addr_cpu_to_be32(spec_mask->ip6src, - rule->tuples_mask.src_ip); - - if (rule->unused_tuple & BIT(INNER_DST_IP)) - memset(spec_mask->ip6dst, 0, sizeof(spec_mask->ip6dst)); - else - ipv6_addr_cpu_to_be32(spec_mask->ip6dst, - rule->tuples_mask.dst_ip); - - spec->tclass = rule->tuples.ip_tos; - spec_mask->tclass = rule->unused_tuple & BIT(INNER_IP_TOS) ? - 0 : rule->tuples_mask.ip_tos; - - spec->psrc = cpu_to_be16(rule->tuples.src_port); - spec_mask->psrc = rule->unused_tuple & BIT(INNER_SRC_PORT) ? - 0 : cpu_to_be16(rule->tuples_mask.src_port); - - spec->pdst = cpu_to_be16(rule->tuples.dst_port); - spec_mask->pdst = rule->unused_tuple & BIT(INNER_DST_PORT) ? - 0 : cpu_to_be16(rule->tuples_mask.dst_port); -} - -static void hclge_fd_get_ip6_info(struct hclge_fd_rule *rule, - struct ethtool_usrip6_spec *spec, - struct ethtool_usrip6_spec *spec_mask) -{ - ipv6_addr_cpu_to_be32(spec->ip6src, rule->tuples.src_ip); - ipv6_addr_cpu_to_be32(spec->ip6dst, rule->tuples.dst_ip); - if (rule->unused_tuple & BIT(INNER_SRC_IP)) - memset(spec_mask->ip6src, 0, sizeof(spec_mask->ip6src)); - else - ipv6_addr_cpu_to_be32(spec_mask->ip6src, - rule->tuples_mask.src_ip); - - if (rule->unused_tuple & BIT(INNER_DST_IP)) - memset(spec_mask->ip6dst, 0, sizeof(spec_mask->ip6dst)); - else - ipv6_addr_cpu_to_be32(spec_mask->ip6dst, - rule->tuples_mask.dst_ip); - - spec->tclass = rule->tuples.ip_tos; - spec_mask->tclass = rule->unused_tuple & BIT(INNER_IP_TOS) ? - 0 : rule->tuples_mask.ip_tos; - - spec->l4_proto = rule->tuples.ip_proto; - spec_mask->l4_proto = rule->unused_tuple & BIT(INNER_IP_PROTO) ? - 0 : rule->tuples_mask.ip_proto; -} - -static void hclge_fd_get_ether_info(struct hclge_fd_rule *rule, - struct ethhdr *spec, - struct ethhdr *spec_mask) -{ - ether_addr_copy(spec->h_source, rule->tuples.src_mac); - ether_addr_copy(spec->h_dest, rule->tuples.dst_mac); - - if (rule->unused_tuple & BIT(INNER_SRC_MAC)) - eth_zero_addr(spec_mask->h_source); - else - ether_addr_copy(spec_mask->h_source, rule->tuples_mask.src_mac); - - if (rule->unused_tuple & BIT(INNER_DST_MAC)) - eth_zero_addr(spec_mask->h_dest); - else - ether_addr_copy(spec_mask->h_dest, rule->tuples_mask.dst_mac); - - spec->h_proto = cpu_to_be16(rule->tuples.ether_proto); - spec_mask->h_proto = rule->unused_tuple & BIT(INNER_ETH_TYPE) ? - 0 : cpu_to_be16(rule->tuples_mask.ether_proto); -} - -static void hclge_fd_get_user_def_info(struct ethtool_rx_flow_spec *fs, - struct hclge_fd_rule *rule) -{ - if ((rule->unused_tuple & HCLGE_FD_TUPLE_USER_DEF_TUPLES) == - HCLGE_FD_TUPLE_USER_DEF_TUPLES) { - fs->h_ext.data[0] = 0; - fs->h_ext.data[1] = 0; - fs->m_ext.data[0] = 0; - fs->m_ext.data[1] = 0; - } else { - fs->h_ext.data[0] = cpu_to_be32(rule->ep.user_def.offset); - fs->h_ext.data[1] = cpu_to_be32(rule->ep.user_def.data); - fs->m_ext.data[0] = - cpu_to_be32(HCLGE_FD_USER_DEF_OFFSET_UNMASK); - fs->m_ext.data[1] = cpu_to_be32(rule->ep.user_def.data_mask); - } -} - -static void hclge_fd_get_ext_info(struct ethtool_rx_flow_spec *fs, - struct hclge_fd_rule *rule) -{ - if (fs->flow_type & FLOW_EXT) { - fs->h_ext.vlan_tci = cpu_to_be16(rule->tuples.vlan_tag1); - fs->m_ext.vlan_tci = - rule->unused_tuple & BIT(INNER_VLAN_TAG_FST) ? - 0 : cpu_to_be16(rule->tuples_mask.vlan_tag1); - - hclge_fd_get_user_def_info(fs, rule); - } - - if (fs->flow_type & FLOW_MAC_EXT) { - ether_addr_copy(fs->h_ext.h_dest, rule->tuples.dst_mac); - if (rule->unused_tuple & BIT(INNER_DST_MAC)) - eth_zero_addr(fs->m_u.ether_spec.h_dest); - else - ether_addr_copy(fs->m_u.ether_spec.h_dest, - rule->tuples_mask.dst_mac); - } -} - -static struct hclge_fd_rule *hclge_get_fd_rule(struct hclge_dev *hdev, - u16 location) -{ - struct hclge_fd_rule *rule = NULL; - struct hlist_node *node2; - - hlist_for_each_entry_safe(rule, node2, &hdev->fd_rule_list, rule_node) { - if (rule->location == location) - return rule; - else if (rule->location > location) - return NULL; - } - - return NULL; -} - -static void hclge_fd_get_ring_cookie(struct ethtool_rx_flow_spec *fs, - struct hclge_fd_rule *rule) -{ - if (rule->action == HCLGE_FD_ACTION_DROP_PACKET) { - fs->ring_cookie = RX_CLS_FLOW_DISC; - } else { - u64 vf_id; - - fs->ring_cookie = rule->queue_id; - vf_id = rule->vf_id; - vf_id <<= ETHTOOL_RX_FLOW_SPEC_RING_VF_OFF; - fs->ring_cookie |= vf_id; - } -} - -static int hclge_get_fd_rule_info(struct hnae3_handle *handle, - struct ethtool_rxnfc *cmd) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_fd_rule *rule = NULL; - struct hclge_dev *hdev = vport->back; - struct ethtool_rx_flow_spec *fs; - - if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) - return -EOPNOTSUPP; - - fs = (struct ethtool_rx_flow_spec *)&cmd->fs; - - spin_lock_bh(&hdev->fd_rule_lock); - - rule = hclge_get_fd_rule(hdev, fs->location); - if (!rule) { - spin_unlock_bh(&hdev->fd_rule_lock); - return -ENOENT; - } - - fs->flow_type = rule->flow_type; - switch (fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) { - case SCTP_V4_FLOW: - case TCP_V4_FLOW: - case UDP_V4_FLOW: - hclge_fd_get_tcpip4_info(rule, &fs->h_u.tcp_ip4_spec, - &fs->m_u.tcp_ip4_spec); - break; - case IP_USER_FLOW: - hclge_fd_get_ip4_info(rule, &fs->h_u.usr_ip4_spec, - &fs->m_u.usr_ip4_spec); - break; - case SCTP_V6_FLOW: - case TCP_V6_FLOW: - case UDP_V6_FLOW: - hclge_fd_get_tcpip6_info(rule, &fs->h_u.tcp_ip6_spec, - &fs->m_u.tcp_ip6_spec); - break; - case IPV6_USER_FLOW: - hclge_fd_get_ip6_info(rule, &fs->h_u.usr_ip6_spec, - &fs->m_u.usr_ip6_spec); - break; - /* The flow type of fd rule has been checked before adding in to rule - * list. As other flow types have been handled, it must be ETHER_FLOW - * for the default case - */ - default: - hclge_fd_get_ether_info(rule, &fs->h_u.ether_spec, - &fs->m_u.ether_spec); - break; - } - - hclge_fd_get_ext_info(fs, rule); - - hclge_fd_get_ring_cookie(fs, rule); - - spin_unlock_bh(&hdev->fd_rule_lock); - - return 0; -} - -static int hclge_get_all_rules(struct hnae3_handle *handle, - struct ethtool_rxnfc *cmd, u32 *rule_locs) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - struct hclge_fd_rule *rule; - struct hlist_node *node2; - u32 cnt = 0; - - if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) - return -EOPNOTSUPP; - - cmd->data = hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]; - - spin_lock_bh(&hdev->fd_rule_lock); - hlist_for_each_entry_safe(rule, node2, - &hdev->fd_rule_list, rule_node) { - if (cnt == cmd->rule_cnt) { - spin_unlock_bh(&hdev->fd_rule_lock); - return -EMSGSIZE; - } - - if (rule->state == HCLGE_FD_TO_DEL) - continue; - - rule_locs[cnt] = rule->location; - cnt++; - } - - spin_unlock_bh(&hdev->fd_rule_lock); - - cmd->rule_cnt = cnt; - - return 0; -} - -static void hclge_fd_get_flow_tuples(const struct flow_keys *fkeys, - struct hclge_fd_rule_tuples *tuples) -{ -#define flow_ip6_src fkeys->addrs.v6addrs.src.in6_u.u6_addr32 -#define flow_ip6_dst fkeys->addrs.v6addrs.dst.in6_u.u6_addr32 - - tuples->ether_proto = be16_to_cpu(fkeys->basic.n_proto); - tuples->ip_proto = fkeys->basic.ip_proto; - tuples->dst_port = be16_to_cpu(fkeys->ports.dst); - - if (fkeys->basic.n_proto == htons(ETH_P_IP)) { - tuples->src_ip[3] = be32_to_cpu(fkeys->addrs.v4addrs.src); - tuples->dst_ip[3] = be32_to_cpu(fkeys->addrs.v4addrs.dst); - } else { - int i; - - for (i = 0; i < IPV6_ADDR_WORDS; i++) { - tuples->src_ip[i] = be32_to_cpu(flow_ip6_src[i]); - tuples->dst_ip[i] = be32_to_cpu(flow_ip6_dst[i]); - } - } -} - -/* traverse all rules, check whether an existed rule has the same tuples */ -static struct hclge_fd_rule * -hclge_fd_search_flow_keys(struct hclge_dev *hdev, - const struct hclge_fd_rule_tuples *tuples) -{ - struct hclge_fd_rule *rule = NULL; - struct hlist_node *node; - - hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) { - if (!memcmp(tuples, &rule->tuples, sizeof(*tuples))) - return rule; - } - - return NULL; -} - -static void hclge_fd_build_arfs_rule(const struct hclge_fd_rule_tuples *tuples, - struct hclge_fd_rule *rule) -{ - rule->unused_tuple = BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC) | - BIT(INNER_VLAN_TAG_FST) | BIT(INNER_IP_TOS) | - BIT(INNER_SRC_PORT); - rule->action = 0; - rule->vf_id = 0; - rule->rule_type = HCLGE_FD_ARFS_ACTIVE; - rule->state = HCLGE_FD_TO_ADD; - if (tuples->ether_proto == ETH_P_IP) { - if (tuples->ip_proto == IPPROTO_TCP) - rule->flow_type = TCP_V4_FLOW; - else - rule->flow_type = UDP_V4_FLOW; - } else { - if (tuples->ip_proto == IPPROTO_TCP) - rule->flow_type = TCP_V6_FLOW; - else - rule->flow_type = UDP_V6_FLOW; - } - memcpy(&rule->tuples, tuples, sizeof(rule->tuples)); - memset(&rule->tuples_mask, 0xFF, sizeof(rule->tuples_mask)); -} - -static int hclge_add_fd_entry_by_arfs(struct hnae3_handle *handle, u16 queue_id, - u16 flow_id, struct flow_keys *fkeys) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_fd_rule_tuples new_tuples = {}; - struct hclge_dev *hdev = vport->back; - struct hclge_fd_rule *rule; - u16 bit_id; - - if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) - return -EOPNOTSUPP; - - /* when there is already fd rule existed add by user, - * arfs should not work - */ - spin_lock_bh(&hdev->fd_rule_lock); - if (hdev->fd_active_type != HCLGE_FD_ARFS_ACTIVE && - hdev->fd_active_type != HCLGE_FD_RULE_NONE) { - spin_unlock_bh(&hdev->fd_rule_lock); - return -EOPNOTSUPP; - } - - hclge_fd_get_flow_tuples(fkeys, &new_tuples); - - /* check is there flow director filter existed for this flow, - * if not, create a new filter for it; - * if filter exist with different queue id, modify the filter; - * if filter exist with same queue id, do nothing - */ - rule = hclge_fd_search_flow_keys(hdev, &new_tuples); - if (!rule) { - bit_id = find_first_zero_bit(hdev->fd_bmap, MAX_FD_FILTER_NUM); - if (bit_id >= hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) { - spin_unlock_bh(&hdev->fd_rule_lock); - return -ENOSPC; - } - - rule = kzalloc_obj(*rule, GFP_ATOMIC); - if (!rule) { - spin_unlock_bh(&hdev->fd_rule_lock); - return -ENOMEM; - } - - rule->location = bit_id; - rule->arfs.flow_id = flow_id; - rule->queue_id = queue_id; - hclge_fd_build_arfs_rule(&new_tuples, rule); - hclge_update_fd_list(hdev, rule->state, rule->location, rule); - hdev->fd_active_type = HCLGE_FD_ARFS_ACTIVE; - } else if (rule->queue_id != queue_id) { - rule->queue_id = queue_id; - rule->state = HCLGE_FD_TO_ADD; - set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state); - hclge_task_schedule(hdev, 0); - } - spin_unlock_bh(&hdev->fd_rule_lock); - return rule->location; -} - -static void hclge_rfs_filter_expire(struct hclge_dev *hdev) -{ -#ifdef CONFIG_RFS_ACCEL - struct hnae3_handle *handle = &hdev->vport[0].nic; - struct hclge_fd_rule *rule; - struct hlist_node *node; - - spin_lock_bh(&hdev->fd_rule_lock); - if (hdev->fd_active_type != HCLGE_FD_ARFS_ACTIVE) { - spin_unlock_bh(&hdev->fd_rule_lock); - return; - } - hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) { - if (rule->state != HCLGE_FD_ACTIVE) - continue; - if (rps_may_expire_flow(handle->netdev, rule->queue_id, - rule->arfs.flow_id, rule->location)) { - rule->state = HCLGE_FD_TO_DEL; - set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state); - } - } - spin_unlock_bh(&hdev->fd_rule_lock); -#endif -} - -/* make sure being called after lock up with fd_rule_lock */ -static int hclge_clear_arfs_rules(struct hclge_dev *hdev) -{ -#ifdef CONFIG_RFS_ACCEL - struct hclge_fd_rule *rule; - struct hlist_node *node; - int ret; - - if (hdev->fd_active_type != HCLGE_FD_ARFS_ACTIVE) - return 0; - - hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) { - switch (rule->state) { - case HCLGE_FD_TO_DEL: - case HCLGE_FD_ACTIVE: - ret = hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, - rule->location, NULL, false); - if (ret) - return ret; - fallthrough; - case HCLGE_FD_TO_ADD: - hclge_fd_dec_rule_cnt(hdev, rule->location); - hlist_del(&rule->rule_node); - kfree(rule); - break; - default: - break; - } - } - hclge_sync_fd_state(hdev); - -#endif - return 0; -} - -static void hclge_get_cls_key_basic(const struct flow_rule *flow, - struct hclge_fd_rule *rule) -{ - if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_BASIC)) { - struct flow_match_basic match; - u16 ethtype_key, ethtype_mask; - - flow_rule_match_basic(flow, &match); - ethtype_key = ntohs(match.key->n_proto); - ethtype_mask = ntohs(match.mask->n_proto); - - if (ethtype_key == ETH_P_ALL) { - ethtype_key = 0; - ethtype_mask = 0; - } - rule->tuples.ether_proto = ethtype_key; - rule->tuples_mask.ether_proto = ethtype_mask; - rule->tuples.ip_proto = match.key->ip_proto; - rule->tuples_mask.ip_proto = match.mask->ip_proto; - } else { - rule->unused_tuple |= BIT(INNER_IP_PROTO); - rule->unused_tuple |= BIT(INNER_ETH_TYPE); - } -} - -static void hclge_get_cls_key_mac(const struct flow_rule *flow, - struct hclge_fd_rule *rule) -{ - if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { - struct flow_match_eth_addrs match; - - flow_rule_match_eth_addrs(flow, &match); - ether_addr_copy(rule->tuples.dst_mac, match.key->dst); - ether_addr_copy(rule->tuples_mask.dst_mac, match.mask->dst); - ether_addr_copy(rule->tuples.src_mac, match.key->src); - ether_addr_copy(rule->tuples_mask.src_mac, match.mask->src); - if (is_zero_ether_addr(match.key->dst)) - rule->unused_tuple |= BIT(INNER_DST_MAC); - if (is_zero_ether_addr(match.key->src)) - rule->unused_tuple |= BIT(INNER_SRC_MAC); - } else { - rule->unused_tuple |= BIT(INNER_DST_MAC); - rule->unused_tuple |= BIT(INNER_SRC_MAC); - } -} - -static void hclge_get_cls_key_vlan(const struct flow_rule *flow, - struct hclge_fd_rule *rule) -{ - if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_VLAN)) { - struct flow_match_vlan match; - - flow_rule_match_vlan(flow, &match); - rule->tuples.vlan_tag1 = match.key->vlan_id | - (match.key->vlan_priority << VLAN_PRIO_SHIFT); - rule->tuples_mask.vlan_tag1 = match.mask->vlan_id | - (match.mask->vlan_priority << VLAN_PRIO_SHIFT); - } else { - rule->unused_tuple |= BIT(INNER_VLAN_TAG_FST); - } -} - -static int hclge_get_cls_key_ip(const struct flow_rule *flow, - struct hclge_fd_rule *rule, - struct netlink_ext_ack *extack) -{ - u16 addr_type = 0; - - if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_CONTROL)) { - struct flow_match_control match; - - flow_rule_match_control(flow, &match); - addr_type = match.key->addr_type; - - if (flow_rule_has_control_flags(match.mask->flags, extack)) - return -EOPNOTSUPP; - } - - if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { - struct flow_match_ipv4_addrs match; - - flow_rule_match_ipv4_addrs(flow, &match); - rule->tuples.src_ip[IPV4_INDEX] = be32_to_cpu(match.key->src); - rule->tuples_mask.src_ip[IPV4_INDEX] = - be32_to_cpu(match.mask->src); - rule->tuples.dst_ip[IPV4_INDEX] = be32_to_cpu(match.key->dst); - rule->tuples_mask.dst_ip[IPV4_INDEX] = - be32_to_cpu(match.mask->dst); - if (!match.key->src) - rule->unused_tuple |= BIT(INNER_SRC_IP); - if (!match.key->dst) - rule->unused_tuple |= BIT(INNER_DST_IP); - } else if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { - struct flow_match_ipv6_addrs match; - - flow_rule_match_ipv6_addrs(flow, &match); - ipv6_addr_be32_to_cpu(rule->tuples.src_ip, - match.key->src.s6_addr32); - ipv6_addr_be32_to_cpu(rule->tuples_mask.src_ip, - match.mask->src.s6_addr32); - ipv6_addr_be32_to_cpu(rule->tuples.dst_ip, - match.key->dst.s6_addr32); - ipv6_addr_be32_to_cpu(rule->tuples_mask.dst_ip, - match.mask->dst.s6_addr32); - if (ipv6_addr_any((struct in6_addr *)match.key->src.s6_addr32)) - rule->unused_tuple |= BIT(INNER_SRC_IP); - if (ipv6_addr_any((struct in6_addr *)match.key->dst.s6_addr32)) - rule->unused_tuple |= BIT(INNER_DST_IP); - } else { - rule->unused_tuple |= BIT(INNER_SRC_IP); - rule->unused_tuple |= BIT(INNER_DST_IP); - } - - return 0; -} - -static void hclge_get_cls_key_port(const struct flow_rule *flow, - struct hclge_fd_rule *rule) -{ - if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_PORTS)) { - struct flow_match_ports match; - - flow_rule_match_ports(flow, &match); - - rule->tuples.src_port = be16_to_cpu(match.key->src); - rule->tuples_mask.src_port = be16_to_cpu(match.mask->src); - rule->tuples.dst_port = be16_to_cpu(match.key->dst); - rule->tuples_mask.dst_port = be16_to_cpu(match.mask->dst); - } else { - rule->unused_tuple |= BIT(INNER_SRC_PORT); - rule->unused_tuple |= BIT(INNER_DST_PORT); - } -} - -static int hclge_get_cls_enc_keyid(struct hclge_dev *hdev, - const struct flow_rule *flow, - struct hclge_fd_rule *rule) -{ - if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_ENC_KEYID)) { - struct flow_match_enc_keyid match; - - flow_rule_match_enc_keyid(flow, &match); - - /* vni is only 24 bits and must be greater than 0, - * and it can not be masked. - */ - if (be32_to_cpu(match.mask->keyid) != - HCLGE_FD_VXLAN_VNI_UNMASK || - be32_to_cpu(match.key->keyid) >= VXLAN_N_VID || - !match.key->keyid) { - dev_err(&hdev->pdev->dev, "invalid enc_keyid\n"); - return -EINVAL; - } - - rule->tuples.outer_tun_vni = be32_to_cpu(match.key->keyid); - rule->tuples_mask.outer_tun_vni = - be32_to_cpu(match.mask->keyid); - } else { - rule->unused_tuple |= BIT(OUTER_TUN_VNI); - } - - return 0; -} - -static void hclge_get_cls_key_ip_tos(const struct flow_rule *flow, - struct hclge_fd_rule *rule) -{ - if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_IP)) { - struct flow_match_ip match; - - flow_rule_match_ip(flow, &match); - - rule->tuples.ip_tos = match.key->tos; - rule->tuples_mask.ip_tos = match.mask->tos; - if (!rule->tuples.ip_tos) - rule->unused_tuple |= BIT(INNER_IP_TOS); - } else { - rule->unused_tuple |= BIT(INNER_IP_TOS); - } -} - -static int hclge_get_tc_flower_action(struct hclge_dev *hdev, - struct flow_cls_offload *cls_flower, - struct hclge_fd_rule *rule) -{ - struct flow_rule *flow = flow_cls_offload_flow_rule(cls_flower); - struct hnae3_handle *handle = &hdev->vport[0].nic; - struct flow_action *action = &flow->action; - struct flow_action_entry *act; - int tc; - - if (!flow_action_has_entries(&flow->action)) { - tc = tc_classid_to_hwtc(handle->netdev, cls_flower->classid); - if (tc < 0 || tc > hdev->tc_max) { - dev_err(&hdev->pdev->dev, - "invalid traffic class: %d\n", tc); - return -EINVAL; - } - - rule->action = HCLGE_FD_ACTION_SELECT_TC; - rule->cls_flower.tc = tc; - return 0; - } - - act = &action->entries[0]; - switch (act->id) { - case FLOW_ACTION_RX_QUEUE_MAPPING: - if (act->rx_queue >= handle->kinfo.num_tqps) { - dev_err(&hdev->pdev->dev, - "queue id (%u) should be less than %u\n", - act->rx_queue, handle->kinfo.num_tqps - 1); - return -EINVAL; - } - - rule->queue_id = act->rx_queue; - rule->action = HCLGE_FD_ACTION_SELECT_QUEUE; - return 0; - case FLOW_ACTION_DROP: - rule->action = HCLGE_FD_ACTION_DROP_PACKET; - return 0; - default: - dev_err(&hdev->pdev->dev, "unsupported action(%d)\n", act->id); - return -EOPNOTSUPP; - } -} - -static int hclge_parse_cls_flower(struct hclge_dev *hdev, - struct flow_cls_offload *cls_flower, - struct hclge_fd_rule *rule) -{ - struct flow_rule *flow = flow_cls_offload_flow_rule(cls_flower); - struct netlink_ext_ack *extack = cls_flower->common.extack; - int ret; - - /* not support any user def tuples */ - rule->unused_tuple |= HCLGE_FD_TUPLE_USER_DEF_TUPLES; - - hclge_get_cls_key_basic(flow, rule); - hclge_get_cls_key_mac(flow, rule); - hclge_get_cls_key_vlan(flow, rule); - - ret = hclge_get_cls_key_ip(flow, rule, extack); - if (ret) - return ret; - - hclge_get_cls_key_port(flow, rule); - hclge_get_cls_key_ip_tos(flow, rule); - - return hclge_get_cls_enc_keyid(hdev, flow, rule); -} - -static int hclge_check_cls_flower(struct hclge_dev *hdev, - struct flow_cls_offload *cls_flower) -{ - struct flow_rule *flow = flow_cls_offload_flow_rule(cls_flower); - struct flow_dissector *dissector = flow->match.dissector; - u32 prio = cls_flower->common.prio; - - if (prio == 0 || - prio > hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) { - dev_err(&hdev->pdev->dev, - "prio %u should be in range[1, %u]\n", - prio, hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]); - return -EINVAL; - } - - if (test_bit(prio - 1, hdev->fd_bmap)) { - dev_err(&hdev->pdev->dev, "prio %u is already used\n", prio); - return -EINVAL; - } - - if (dissector->used_keys & - ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | - BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | - BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | - BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | - BIT_ULL(FLOW_DISSECTOR_KEY_IP) | - BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) { - dev_err(&hdev->pdev->dev, "unsupported key set: %#llx\n", - dissector->used_keys); - return -EOPNOTSUPP; - } - - /* driver will parses classid into an action */ - if (cls_flower->classid && flow_action_has_entries(&flow->action)) { - dev_err(&hdev->pdev->dev, - "please not set classid and action together\n"); - return -EOPNOTSUPP; - } - - if (!flow_action_has_entries(&flow->action) && !cls_flower->classid) { - dev_err(&hdev->pdev->dev, "please set action or classid\n"); - return -EINVAL; - } - - if (flow_action_has_entries(&flow->action) && - !flow_offload_has_one_action(&flow->action)) { - dev_err(&hdev->pdev->dev, "unsupported multiple actions\n"); - return -EOPNOTSUPP; - } - - return 0; -} - -static int hclge_add_cls_flower(struct hnae3_handle *handle, - struct flow_cls_offload *cls_flower) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - struct hclge_fd_rule *rule; - int ret; - - if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) { - dev_err(&hdev->pdev->dev, - "cls flower is not supported\n"); - return -EOPNOTSUPP; - } - - ret = hclge_check_cls_flower(hdev, cls_flower); - if (ret) { - dev_err(&hdev->pdev->dev, - "failed to check cls flower params, ret = %d\n", ret); - return ret; - } - - rule = kzalloc_obj(*rule); - if (!rule) - return -ENOMEM; - - ret = hclge_parse_cls_flower(hdev, cls_flower, rule); - if (ret) { - kfree(rule); - return ret; - } - - ret = hclge_get_tc_flower_action(hdev, cls_flower, rule); - if (ret) { - kfree(rule); - return ret; - } - - rule->location = cls_flower->common.prio - 1; - rule->vf_id = 0; - rule->cls_flower.cookie = cls_flower->cookie; - rule->rule_type = HCLGE_FD_TC_FLOWER_ACTIVE; - - ret = hclge_add_fd_entry_common(hdev, rule); - if (ret) - kfree(rule); - - return ret; -} - -static struct hclge_fd_rule *hclge_find_cls_flower(struct hclge_dev *hdev, - unsigned long cookie) -{ - struct hclge_fd_rule *rule; - struct hlist_node *node; - - hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) { - if (rule->cls_flower.cookie == cookie) - return rule; - } - - return NULL; -} - -static int hclge_del_cls_flower(struct hnae3_handle *handle, - struct flow_cls_offload *cls_flower) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - struct hclge_fd_rule *rule; - int ret; - - if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) - return -EOPNOTSUPP; - - spin_lock_bh(&hdev->fd_rule_lock); - - rule = hclge_find_cls_flower(hdev, cls_flower->cookie); - if (!rule) { - spin_unlock_bh(&hdev->fd_rule_lock); - return -EINVAL; - } - - ret = hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, rule->location, - NULL, false); - if (ret) { - /* if tcam config fail, set rule state to TO_DEL, - * so the rule will be deleted when periodic - * task being scheduled. - */ - hclge_update_fd_list(hdev, HCLGE_FD_TO_DEL, rule->location, NULL); - set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state); - spin_unlock_bh(&hdev->fd_rule_lock); - return ret; - } - - hclge_update_fd_list(hdev, HCLGE_FD_DELETED, rule->location, NULL); - spin_unlock_bh(&hdev->fd_rule_lock); - - return 0; -} - -static void hclge_sync_fd_list(struct hclge_dev *hdev, struct hlist_head *hlist) -{ - struct hclge_fd_rule *rule; - struct hlist_node *node; - int ret = 0; - - if (!test_and_clear_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state)) - return; - - spin_lock_bh(&hdev->fd_rule_lock); - - hlist_for_each_entry_safe(rule, node, hlist, rule_node) { - switch (rule->state) { - case HCLGE_FD_TO_ADD: - ret = hclge_fd_config_rule(hdev, rule); - if (ret) - goto out; - rule->state = HCLGE_FD_ACTIVE; - break; - case HCLGE_FD_TO_DEL: - ret = hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, - rule->location, NULL, false); - if (ret) - goto out; - hclge_fd_dec_rule_cnt(hdev, rule->location); - hclge_fd_free_node(hdev, rule); - break; - default: - break; - } - } - -out: - if (ret) - set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state); - - spin_unlock_bh(&hdev->fd_rule_lock); -} - -static void hclge_sync_fd_table(struct hclge_dev *hdev) -{ - if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) - return; - - if (test_and_clear_bit(HCLGE_STATE_FD_CLEAR_ALL, &hdev->state)) { - bool clear_list = hdev->fd_active_type == HCLGE_FD_ARFS_ACTIVE; - - hclge_clear_fd_rules_in_list(hdev, clear_list); - } - - hclge_sync_fd_user_def_cfg(hdev, false); - - hclge_sync_fd_list(hdev, &hdev->fd_rule_list); -} - -static bool hclge_get_hw_reset_stat(struct hnae3_handle *handle) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - - return hclge_read_dev(&hdev->hw, HCLGE_GLOBAL_RESET_REG) || - hclge_read_dev(&hdev->hw, HCLGE_FUN_RST_ING); -} - -static bool hclge_get_cmdq_stat(struct hnae3_handle *handle) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - - return test_bit(HCLGE_COMM_STATE_CMD_DISABLE, &hdev->hw.hw.comm_state); -} - -static bool hclge_ae_dev_resetting(struct hnae3_handle *handle) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - - return test_bit(HCLGE_STATE_RST_HANDLING, &hdev->state); -} - -static unsigned long hclge_ae_dev_reset_cnt(struct hnae3_handle *handle) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - - return hdev->rst_stats.hw_reset_done_cnt; -} - -static void hclge_enable_fd(struct hnae3_handle *handle, bool enable) -{ - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - - hdev->fd_en = enable; - - if (!enable) - set_bit(HCLGE_STATE_FD_CLEAR_ALL, &hdev->state); - else - hclge_restore_fd_entries(handle); - - hclge_task_schedule(hdev, 0); -} - -static void hclge_cfg_mac_mode(struct hclge_dev *hdev, bool enable) -{ -#define HCLGE_LINK_STATUS_WAIT_CNT 3 - - struct hclge_desc desc; - struct hclge_config_mac_mode_cmd *req = - (struct hclge_config_mac_mode_cmd *)desc.data; - u32 loop_en = 0; - int ret; - - hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CONFIG_MAC_MODE, false); - - if (enable) { - hnae3_set_bit(loop_en, HCLGE_MAC_TX_EN_B, 1U); - hnae3_set_bit(loop_en, HCLGE_MAC_RX_EN_B, 1U); - hnae3_set_bit(loop_en, HCLGE_MAC_PAD_TX_B, 1U); - hnae3_set_bit(loop_en, HCLGE_MAC_PAD_RX_B, 1U); - hnae3_set_bit(loop_en, HCLGE_MAC_FCS_TX_B, 1U); - hnae3_set_bit(loop_en, HCLGE_MAC_RX_FCS_B, 1U); - hnae3_set_bit(loop_en, HCLGE_MAC_RX_FCS_STRIP_B, 1U); - hnae3_set_bit(loop_en, HCLGE_MAC_TX_OVERSIZE_TRUNCATE_B, 1U); - hnae3_set_bit(loop_en, HCLGE_MAC_RX_OVERSIZE_TRUNCATE_B, 1U); - hnae3_set_bit(loop_en, HCLGE_MAC_TX_UNDER_MIN_ERR_B, 1U); - } - - req->txrx_pad_fcs_loop_en = cpu_to_le32(loop_en); - - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) { - dev_err(&hdev->pdev->dev, - "mac enable fail, ret =%d.\n", ret); - return; - } - - if (!enable) - hclge_mac_link_status_wait(hdev, HCLGE_LINK_STATUS_DOWN, - HCLGE_LINK_STATUS_WAIT_CNT); -} - static int hclge_config_switch_param(struct hclge_dev *hdev, int vfid, u8 switch_param, u8 param_mask) { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index ccb19d960690..87adeb64e6ea 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -1179,4 +1179,6 @@ int hclge_mac_update_stats(struct hclge_dev *hdev); struct hclge_vport *hclge_get_vf_vport(struct hclge_dev *hdev, int vf); int hclge_inform_vf_reset(struct hclge_vport *vport, u16 reset_type); int hclge_query_scc_version(struct hclge_dev *hdev, u32 *scc_version); +u32 hclge_get_port_number(enum HLCGE_PORT_TYPE port_type, u8 pf_id, + u8 vf_id, u8 network_port_id); #endif -- 2.33.0