Because of the performance and security implications, the use of PISA features is opt-in and configurable per device with a fine granularity. Add boolean attributes to struct device exposed under /sys/devices/*/*/pisa_*: - pisa_coherent: enable PISA for coherent allocations - pisa_rx_bufs: enable PISA for RX buffers / page_pool - pisa_tx_bufs: enable PISA for TX socket page frags - pisa_tx_skb_heads: enable PISA slab cache for small TX skb heads - pisa_tx_zerocopy: migrate MSG_ZEROCOPY user pages into PISA Signed-off-by: Luigi Rizzo --- drivers/base/core.c | 75 ++++++++++++++++++++++++++++++++++++++++++ include/linux/device.h | 15 +++++++++ include/linux/pisa.h | 27 +++++++++++++++ mm/pisa-alloc.c | 21 ++++++++++++ mm/pisa-kunit.c | 71 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 209 insertions(+) diff --git a/drivers/base/core.c b/drivers/base/core.c index cbe05197d4a0b..df93b6b6e125e 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -2907,6 +2908,65 @@ static ssize_t removable_show(struct device *dev, const struct device_attribute } static const DEVICE_ATTR_RO(removable); +/* Per-device configuration to selectively enable PISA */ +static ssize_t pisa_attr_store(struct device *dev, const char *buf, size_t count, + bool *field, int (*setter)(struct device *, bool)) +{ + bool val; + int ret = kstrtobool(buf, &val); + + if (ret < 0) + return ret; + if (val) { + ret = pisa_meta_init(); + if (ret < 0) + return ret; + } + if (setter) { + ret = setter(dev, val); + if (ret < 0) + return ret; + } else { + WRITE_ONCE(*field, val); + } + return count; +} + +#define PISA_SET_ATTR(name, setter) \ +static ssize_t name##_show(struct device *dev, \ + struct device_attribute *attr, char *buf) \ +{ \ + return sysfs_emit(buf, "%u\n", READ_ONCE(dev->name)); \ +} \ +static ssize_t name##_store(struct device *dev, \ + struct device_attribute *attr, \ + const char *buf, size_t count) \ +{ \ + return pisa_attr_store(dev, buf, count, &dev->name, setter); \ +} \ +static DEVICE_ATTR_RW(name) + +#define PISA_BOOL_ATTR(name) PISA_SET_ATTR(name, NULL) + +PISA_BOOL_ATTR(pisa_coherent); +PISA_BOOL_ATTR(pisa_rx_bufs); +PISA_BOOL_ATTR(pisa_tx_bufs); +PISA_BOOL_ATTR(pisa_tx_skb_heads); +PISA_BOOL_ATTR(pisa_tx_zerocopy); + +static struct attribute *dev_attr_pisa[] = { + &dev_attr_pisa_coherent.attr, + &dev_attr_pisa_rx_bufs.attr, + &dev_attr_pisa_tx_bufs.attr, + &dev_attr_pisa_tx_skb_heads.attr, + &dev_attr_pisa_tx_zerocopy.attr, + NULL, +}; + +static const struct attribute_group dev_attr_pisa_group = { + .attrs = dev_attr_pisa, +}; + int device_add_groups(struct device *dev, const struct attribute_group *const *groups) { @@ -3014,6 +3074,18 @@ static int device_add_attrs(struct device *dev) goto err_remove_dev_removable; } + if (IS_ENABLED(CONFIG_PISA) && dev->dma_mask) { + error = device_add_group(dev, &dev_attr_pisa_group); + if (error) { + if (dev->physical_location) { + device_remove_group(dev, &dev_attr_physical_location_group); + kfree(dev->physical_location); + dev->physical_location = NULL; + } + goto err_remove_dev_removable; + } + } + return 0; err_remove_dev_removable: @@ -3039,6 +3111,9 @@ static void device_remove_attrs(struct device *dev) const struct class *class = dev->class; const struct device_type *type = dev->type; + if (IS_ENABLED(CONFIG_PISA) && dev->dma_mask) + device_remove_group(dev, &dev_attr_pisa_group); + if (dev->physical_location) { device_remove_group(dev, &dev_attr_physical_location_group); kfree(dev->physical_location); diff --git a/include/linux/device.h b/include/linux/device.h index 429e39f647cf9..e705df38e49af 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -695,6 +695,15 @@ enum struct_device_flags { * @removable: Whether the device can be removed from the system. This * should be set by the subsystem / bus driver that discovered * the device. + * @pisa_coherent: Back coherent dma_alloc_attrs() with PISA + * (sysfs pisa_coherent). + * @pisa_rx_bufs: Use PISA allocators for RX buffers (sysfs pisa_rx_bufs). + * @pisa_tx_bufs: Use PISA allocators for socket TX page fragments + * (sysfs pisa_tx_bufs). + * @pisa_tx_skb_heads: Use PISA slab cache for small TX skb heads + * (sysfs pisa_tx_skb_heads). + * @pisa_tx_zerocopy: Migrate MSG_ZEROCOPY user pages into PISA + * (sysfs pisa_tx_zerocopy). * @flags: DEV_FLAG_XXX flags. Use atomic bitfield operations to modify. * * At the lowest level, every device in a Linux system is represented by an @@ -799,6 +808,12 @@ struct device { enum device_removable removable; + bool pisa_coherent; + bool pisa_rx_bufs; + bool pisa_tx_bufs; + bool pisa_tx_skb_heads; + bool pisa_tx_zerocopy; + DECLARE_BITMAP(flags, DEV_FLAG_COUNT); }; diff --git a/include/linux/pisa.h b/include/linux/pisa.h index cc33e181df097..5222cb207257f 100644 --- a/include/linux/pisa.h +++ b/include/linux/pisa.h @@ -84,6 +84,7 @@ static inline u64 pisa_iova_size(void) } int pisa_meta_init(void); +bool __pisa_has_free_blocks_rx(unsigned int order, int nid); bool __pisa_page_can_recycle(const struct page *page); static inline bool pisa_enabled(void) @@ -142,6 +143,21 @@ static inline struct page *pisa_virt_to_page(const void *addr) return is_pisa_page(page) ? page : NULL; } +/* Hook for __page_pool_alloc_netmems_slow() (guarded by @pisa_key). */ +static inline gfp_t pisa_rx_gfp(const struct device *dev) +{ + return pisa_enabled() && dev && READ_ONCE(dev->pisa_rx_bufs) ? + __GFP_PISA_RW : 0; +} + +/* Hook for NIC driver RX buffer allocation (guarded by @pisa_key). */ +static inline bool pisa_has_free_blocks_rx(const struct device *dev, + unsigned int order, int nid) +{ + return pisa_enabled() && dev && READ_ONCE(dev->pisa_rx_bufs) && + __pisa_has_free_blocks_rx(order, nid); +} + /* Hook for skb_page_frag_refill() and __pisa_free_page() (@pisa_recycle_check_key). */ static inline bool pisa_page_can_recycle(const struct page *page) { @@ -158,6 +174,17 @@ static inline bool is_pisa_page(const struct page *page) { return false; } static inline bool is_pisa_phys(phys_addr_t phys) { return false; } static inline struct page *pisa_virt_to_page(const void *addr) { return NULL; } +static inline gfp_t pisa_rx_gfp(const struct device *dev) +{ + return 0; +} + +static inline bool pisa_has_free_blocks_rx(const struct device *dev, + unsigned int order, int nid) +{ + return false; +} + static inline bool pisa_page_can_recycle(const struct page *page) { return true; diff --git a/mm/pisa-alloc.c b/mm/pisa-alloc.c index 01d8bbc8d72ff..ff8565c3206ad 100644 --- a/mm/pisa-alloc.c +++ b/mm/pisa-alloc.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -374,6 +375,26 @@ unsigned long __pisa_alloc_pages_bulk(gfp_t gfp, unsigned int order, return pisa_alloc_bulk(pa, gfp & ~__GFP_PISA_RW, nr_pages, page_array, frozen); } +/** + * __pisa_has_free_blocks_rx - Check if per-node RX allocator of @order on @nid has free blocks + * @order: Block order + * @nid: Target NUMA node (or NUMA_NO_NODE) + * + * Return: true if the per-node RX allocator has a free block available, or + * the node's reservoir has a PISA page to give it. + */ +bool __pisa_has_free_blocks_rx(unsigned int order, int nid) +{ + struct pisa_alloc *pa = pisa_node_pa(nid, order, false); + + return pa && + (!list_empty_careful(&pa->partial) || + !list_empty_careful(&pa->partial_dirty) || + !list_empty_careful(&pa->idle) || + READ_ONCE(pisa_nodes[pa->nid].nr_pages)); +} +EXPORT_SYMBOL(__pisa_has_free_blocks_rx); + /* * Split a non-compound page of @old_order into 1 << (@old_order - @new_order) * pages of @new_order (compound if @new_order > 0), all frozen at refcount 0. diff --git a/mm/pisa-kunit.c b/mm/pisa-kunit.c index eb51007e66de5..2adaeb1ffe43e 100644 --- a/mm/pisa-kunit.c +++ b/mm/pisa-kunit.c @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -443,6 +445,74 @@ static void test_map_and_domain_lifecycle(struct kunit *test) } #endif /* CONFIG_IOMMU_DMA */ +static void kunit_check_pisa_attr(struct kunit *test, struct device *dev, + const char *name, bool *field) +{ + const struct sysfs_ops *ops = dev->kobj.ktype->sysfs_ops; + struct kernfs_node *kn; + char *buf; + + buf = (char *)__get_free_page(GFP_KERNEL); + KUNIT_EXPECT_NOT_NULL(test, buf); + if (!buf) + return; + kn = kernfs_find_and_get(dev->kobj.sd, name); + KUNIT_EXPECT_NOT_NULL(test, kn); + if (!kn) + goto out_free; + + KUNIT_EXPECT_EQ(test, ops->show(&dev->kobj, kn->priv, buf), 2); + KUNIT_EXPECT_STREQ(test, buf, "0\n"); + KUNIT_EXPECT_FALSE(test, READ_ONCE(*field)); + + KUNIT_EXPECT_EQ(test, ops->store(&dev->kobj, kn->priv, "1\n", 2), 2); + KUNIT_EXPECT_TRUE(test, READ_ONCE(*field)); + KUNIT_EXPECT_EQ(test, ops->show(&dev->kobj, kn->priv, buf), 2); + KUNIT_EXPECT_STREQ(test, buf, "1\n"); + + KUNIT_EXPECT_LT(test, ops->store(&dev->kobj, kn->priv, "bad", 3), 0); + KUNIT_EXPECT_TRUE(test, READ_ONCE(*field)); + + KUNIT_EXPECT_EQ(test, ops->store(&dev->kobj, kn->priv, "0\n", 2), 2); + KUNIT_EXPECT_FALSE(test, READ_ONCE(*field)); + + kernfs_put(kn); +out_free: + free_page((unsigned long)buf); +} + +static void kunit_attr_dev_release(struct device *dev) +{ + kfree(dev); +} + +static void test_dev_pisa_attrs(struct kunit *test) +{ + struct device *dev; + + if (!IS_ENABLED(CONFIG_SYSFS)) + kunit_skip(test, "requires CONFIG_SYSFS"); + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dev); + dev->init_name = "pisa_attr_kunit"; + dev->release = kunit_attr_dev_release; + dev->coherent_dma_mask = DMA_BIT_MASK(64); + dev->dma_mask = &dev->coherent_dma_mask; + + if (device_register(dev)) { + put_device(dev); + KUNIT_FAIL(test, "device_register failed"); + return; + } + kunit_check_pisa_attr(test, dev, "pisa_coherent", &dev->pisa_coherent); + kunit_check_pisa_attr(test, dev, "pisa_rx_bufs", &dev->pisa_rx_bufs); + kunit_check_pisa_attr(test, dev, "pisa_tx_zerocopy", &dev->pisa_tx_zerocopy); + kunit_check_pisa_attr(test, dev, "pisa_tx_bufs", &dev->pisa_tx_bufs); + kunit_check_pisa_attr(test, dev, "pisa_tx_skb_heads", &dev->pisa_tx_skb_heads); + device_unregister(dev); +} + static void test_reservoir_and_scrubber(struct kunit *test) { KUNIT_ASSERT_EQ(test, pisa_meta_init(), 0); @@ -459,6 +529,7 @@ static struct kunit_case pisa_meta_test_cases[] = { KUNIT_CASE(test_iova_helpers), KUNIT_CASE(test_map_and_domain_lifecycle), #endif + KUNIT_CASE(test_dev_pisa_attrs), KUNIT_CASE(test_reservoir_and_scrubber), {} }; -- 2.56.1.809.ga0b61035b5-goog