Give each target a token bucket and consult it once per message in netconsole_write(). The bucket is created with a zero interval, which struct ratelimit_state treats as unlimited, and nothing can set a nonzero one yet, so no target changes behaviour. Skip the bucket while oops_in_progress is set, so a limit configured for steady-state logging never truncates an oops, BUG() or panic(). The configfs files that expose it come next. ___ratelimit() only trylocks its own raw spinlock, so it is safe with target_list_lock held and interrupts disabled, and safe from NMI. Signed-off-by: Breno Leitao --- drivers/net/netconsole.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index b358e5c367351..2ad514afa2b89 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -49,6 +49,7 @@ #include #include #include +#include MODULE_AUTHOR("Matt Mackall "); MODULE_DESCRIPTION("Console driver for network interfaces"); @@ -175,6 +176,7 @@ struct netcons_userdata { * @sysdata: Cached, formatted string of append * @sysdata_fields: Sysdata features enabled. * @msgcounter: Message sent counter. + * @ratelimit: Opaque structure to ratelimit messages * @stats: Packet send stats for the target. Used for debugging. * @state: State of the target. * Visible from userspace (read-write). @@ -219,6 +221,7 @@ struct netconsole_target { u32 sysdata_fields; /* protected by target_list_lock */ u32 msgcounter; + struct ratelimit_state ratelimit; #endif struct netconsole_target_stats stats; enum target_state state; @@ -282,6 +285,21 @@ static void dynamic_netconsole_mutex_unlock(void) mutex_unlock(&dynamic_netconsole_mutex); } +static void netconsole_ratelimit_init(struct netconsole_target *nt) +{ + ratelimit_state_init(&nt->ratelimit, 0, DEFAULT_RATELIMIT_BURST); + ratelimit_set_flags(&nt->ratelimit, RATELIMIT_MSG_ON_RELEASE); +} + +static bool netconsole_ratelimited(struct netconsole_target *nt) +{ + /* A limit meant for steady-state logging must not eat a crash dump. */ + if (oops_in_progress) + return false; + + return !__ratelimit(&nt->ratelimit); +} + #else /* !CONFIG_NETCONSOLE_DYNAMIC */ static int __init dynamic_netconsole_init(void) @@ -318,6 +336,15 @@ static void dynamic_netconsole_mutex_unlock(void) { } +static void netconsole_ratelimit_init(struct netconsole_target *nt) +{ +} + +static bool netconsole_ratelimited(struct netconsole_target *nt) +{ + return false; +} + #endif /* CONFIG_NETCONSOLE_DYNAMIC */ /* Check if the target was bound by mac address. */ @@ -686,6 +713,7 @@ static struct netconsole_target *alloc_and_init(void) nt->remote_port = 6666; eth_broadcast_addr(nt->remote_mac); nt->state = STATE_DISABLED; + netconsole_ratelimit_init(nt); INIT_WORK(&nt->resume_wq, process_resume_target); /* Set up the skb pool primitives once; enabling only refills it. */ skb_queue_head_init(&nt->skb_pool); @@ -2482,6 +2510,9 @@ static void netconsole_write(struct nbcon_write_context *wctxt, bool extended) !netif_running(nt->np.dev)) continue; + if (netconsole_ratelimited(nt)) + continue; + /* If nbcon_enter_unsafe() fails, just return given netconsole * lost the ownership, and iterating over the targets will not * be able to re-acquire. -- 2.53.0-Meta The per-target token bucket has no interface, so every target is still unlimited. Expose the interval as ratelimit_interval_ms through configfs. It defaults to zero, so a target stays unlimited until an administrator sets one. Signed-off-by: Breno Leitao --- drivers/net/netconsole.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index 2ad514afa2b89..6f94495d0abc3 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -940,6 +940,14 @@ static ssize_t transmit_errors_show(struct config_item *item, char *buf) return sysfs_emit(buf, "%llu\n", xmit_drop_count + enomem_count); } +static ssize_t ratelimit_interval_ms_show(struct config_item *item, char *buf) +{ + struct netconsole_target *nt = to_target(item); + + return sysfs_emit(buf, "%u\n", + jiffies_to_msecs(READ_ONCE(nt->ratelimit.interval))); +} + /* configfs helper to display if cpu_nr sysdata feature is enabled */ static ssize_t sysdata_cpu_nr_enabled_show(struct config_item *item, char *buf) { @@ -1335,6 +1343,29 @@ static ssize_t remote_mac_store(struct config_item *item, const char *buf, return ret; } +static ssize_t ratelimit_interval_ms_store(struct config_item *item, + const char *buf, size_t count) +{ + struct netconsole_target *nt = to_target(item); + unsigned int interval; + unsigned long jifs; + ssize_t ret; + + ret = kstrtouint(buf, 10, &interval); + if (ret) + return ret; + + jifs = msecs_to_jiffies(interval); + if (jifs > INT_MAX) + return -ERANGE; + + dynamic_netconsole_mutex_lock(); + WRITE_ONCE(nt->ratelimit.interval, jifs); + dynamic_netconsole_mutex_unlock(); + + return count; +} + struct userdatum { struct config_item item; char value[MAX_EXTRADATA_VALUE_LEN]; @@ -1699,6 +1730,7 @@ CONFIGFS_ATTR_RO(, local_mac); CONFIGFS_ATTR(, remote_mac); CONFIGFS_ATTR(, release); CONFIGFS_ATTR_RO(, transmit_errors); +CONFIGFS_ATTR(, ratelimit_interval_ms); static struct configfs_attribute *netconsole_target_attrs[] = { &attr_enabled, @@ -1712,6 +1744,7 @@ static struct configfs_attribute *netconsole_target_attrs[] = { &attr_local_mac, &attr_remote_mac, &attr_transmit_errors, + &attr_ratelimit_interval_ms, NULL, }; -- 2.53.0-Meta A target that sets ratelimit_interval_ms runs with the ratelimit default of 10 messages per interval, which is either too coarse or too generous depending on how chatty the target is. Expose it as ratelimit_burst through configfs. Signed-off-by: Breno Leitao --- drivers/net/netconsole.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index 6f94495d0abc3..6d60a5188bf13 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -948,6 +948,13 @@ static ssize_t ratelimit_interval_ms_show(struct config_item *item, char *buf) jiffies_to_msecs(READ_ONCE(nt->ratelimit.interval))); } +static ssize_t ratelimit_burst_show(struct config_item *item, char *buf) +{ + struct netconsole_target *nt = to_target(item); + + return sysfs_emit(buf, "%d\n", READ_ONCE(nt->ratelimit.burst)); +} + /* configfs helper to display if cpu_nr sysdata feature is enabled */ static ssize_t sysdata_cpu_nr_enabled_show(struct config_item *item, char *buf) { @@ -1366,6 +1373,27 @@ static ssize_t ratelimit_interval_ms_store(struct config_item *item, return count; } +static ssize_t ratelimit_burst_store(struct config_item *item, const char *buf, + size_t count) +{ + struct netconsole_target *nt = to_target(item); + unsigned int burst; + ssize_t ret; + + ret = kstrtouint(buf, 10, &burst); + if (ret) + return ret; + + if (burst > INT_MAX) + return -ERANGE; + + dynamic_netconsole_mutex_lock(); + WRITE_ONCE(nt->ratelimit.burst, burst); + dynamic_netconsole_mutex_unlock(); + + return count; +} + struct userdatum { struct config_item item; char value[MAX_EXTRADATA_VALUE_LEN]; @@ -1731,6 +1759,7 @@ CONFIGFS_ATTR(, remote_mac); CONFIGFS_ATTR(, release); CONFIGFS_ATTR_RO(, transmit_errors); CONFIGFS_ATTR(, ratelimit_interval_ms); +CONFIGFS_ATTR(, ratelimit_burst); static struct configfs_attribute *netconsole_target_attrs[] = { &attr_enabled, @@ -1745,6 +1774,7 @@ static struct configfs_attribute *netconsole_target_attrs[] = { &attr_remote_mac, &attr_transmit_errors, &attr_ratelimit_interval_ms, + &attr_ratelimit_burst, NULL, }; -- 2.53.0-Meta Keep the number of messages dropped since the last report on the target and send it to the receiver as soon as a message gets through again: netconsole: 45 messages dropped by rate limit netconsole formats that record itself rather than calling printk(), which would feed the console it is currently servicing. Nothing here has a printk sequence number, so the extended header carries a zero. The timestamp comes from local_clock(), the same clock printk stamps its records with, but it is taken when the notice goes out rather than when the message was logged. It can therefore read a few microseconds later than the message it precedes. The notice rides on the next message the bucket lets through, so a target that goes quiet right after a burst of drops only reports them once the host logs again, and a target with ratelimit_burst set to zero never reports at all. Signed-off-by: Breno Leitao --- drivers/net/netconsole.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index 6d60a5188bf13..0af2e5b4335c0 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -50,6 +50,7 @@ #include #include #include +#include MODULE_AUTHOR("Matt Mackall "); MODULE_DESCRIPTION("Console driver for network interfaces"); @@ -177,6 +178,7 @@ struct netcons_userdata { * @sysdata_fields: Sysdata features enabled. * @msgcounter: Message sent counter. * @ratelimit: Opaque structure to ratelimit messages + * @pending_drops: Messages dropped since the last notice was sent. * @stats: Packet send stats for the target. Used for debugging. * @state: State of the target. * Visible from userspace (read-write). @@ -221,6 +223,7 @@ struct netconsole_target { u32 sysdata_fields; /* protected by target_list_lock */ u32 msgcounter; + u32 pending_drops; struct ratelimit_state ratelimit; #endif struct netconsole_target_stats stats; @@ -297,7 +300,20 @@ static bool netconsole_ratelimited(struct netconsole_target *nt) if (oops_in_progress) return false; - return !__ratelimit(&nt->ratelimit); + if (__ratelimit(&nt->ratelimit)) + return false; + + nt->pending_drops++; + + return true; +} + +static u32 netconsole_take_drops(struct netconsole_target *nt) +{ + u32 drops = nt->pending_drops; + + nt->pending_drops = 0; + return drops; } #else /* !CONFIG_NETCONSOLE_DYNAMIC */ @@ -345,6 +361,11 @@ static bool netconsole_ratelimited(struct netconsole_target *nt) return false; } +static u32 netconsole_take_drops(struct netconsole_target *nt) +{ + return 0; +} + #endif /* CONFIG_NETCONSOLE_DYNAMIC */ /* Check if the target was bound by mac address. */ @@ -2554,6 +2575,34 @@ static void send_msg_udp(struct netconsole_target *nt, const char *msg, } } +static void send_ratelimit_notice(struct netconsole_target *nt, bool extended) +{ + int len = 0; + u64 ts_usec; + u32 drops; + + drops = netconsole_take_drops(nt); + if (!drops) + return; + + if (extended) { + /* append the extended headers */ + if (nt->release) + len = scnprintf(nt->buf, sizeof(nt->buf), "%s,", + init_utsname()->release); + + ts_usec = div_u64(local_clock(), NSEC_PER_USEC); + len += scnprintf(nt->buf + len, sizeof(nt->buf) - len, + "%u,0,%llu,-;", LOGLEVEL_WARNING, ts_usec); + } + + len += scnprintf(nt->buf + len, sizeof(nt->buf) - len, + "netconsole: %u messages dropped by rate limit\n", + drops); + + send_udp(nt, nt->buf, len); +} + /** * netconsole_write - Generic function to send a msg to all targets * @wctxt: nbcon write context @@ -2583,6 +2632,8 @@ static void netconsole_write(struct nbcon_write_context *wctxt, bool extended) if (!nbcon_enter_unsafe(wctxt)) return; + send_ratelimit_notice(nt, extended); + if (extended) send_ext_msg_udp(nt, wctxt); else -- 2.53.0-Meta Describe the per-target token bucket and the two configfs files that drive it: ratelimit_interval_ms and ratelimit_burst. Spell out the two properties that are not obvious from the file names. The limit is accounted per message rather than per packet, so a message split into several ncfrag packets is never truncated by the bucket running dry halfway through. Signed-off-by: Breno Leitao --- Documentation/networking/netconsole.rst | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/Documentation/networking/netconsole.rst b/Documentation/networking/netconsole.rst index 4ab5d7b05cf10..7369cd89ae76c 100644 --- a/Documentation/networking/netconsole.rst +++ b/Documentation/networking/netconsole.rst @@ -177,6 +177,49 @@ You can modify these targets in runtime by creating the following targets:: cat cmdline1/remote_ip 10.0.0.3 +Rate limiting +------------- + +Netconsole hands every console message to every enabled target, so a host that +logs continuously can saturate the receiving agent. Each target carries a token +bucket that drops messages once the configured rate is exceeded, controlled by +two files in the target directory: + + ===================== ================================================ + ratelimit_interval_ms Length of the accounting interval, in + milliseconds. Zero, the default, sends + everything. + ratelimit_burst Messages allowed per interval. Defaults to + 10; zero drops every message. + ===================== ================================================ + +Unlike most target parameters, both knobs can be written while the target is +enabled, which is when a flooding target most likely needs them. + +The limit is applied per message, not per packet, so a message big enough to be +split into several `ncfrag` packets is either sent whole or not at all. + +Crash output bypasses the bucket. While an oops, BUG() or panic() is in +progress every message is sent, whatever the limit says, so a small burst +cannot cost you part of a crash dump. + +A drop leaves nothing on the wire, so the receiver is told what it missed as +soon as a message gets through again:: + + netconsole: 45 messages dropped by rate limit + +The notice only travels with the next message the bucket allows. A host that +goes quiet right after being limited reports the drops later, when it logs +again, and a target with a `ratelimit_burst` of zero never reports them. + +netconsole generates the notice itself instead of logging it, so on an +extended target the record carries a sequence number of zero. + +Capping a target at 500 messages a minute:: + + echo 60000 > ratelimit_interval_ms + echo 500 > ratelimit_burst + Append User Data ---------------- -- 2.53.0-Meta Check that a new target starts unlimited and that a configured burst caps what the target transmits. Dropping the interval back to zero has to restore unlimited delivery and report what was lost to the receiver. Only an upper bound is checked on the number of messages received. netconsole is best-effort UDP and unrelated kernel messages draw from the same bucket, so what arrives is not an exact count, while the cap is. Signed-off-by: Breno Leitao --- .../selftests/drivers/net/netconsole/Makefile | 1 + .../drivers/net/netconsole/netcons_ratelimit.sh | 160 +++++++++++++++++++++ 2 files changed, 161 insertions(+) diff --git a/tools/testing/selftests/drivers/net/netconsole/Makefile b/tools/testing/selftests/drivers/net/netconsole/Makefile index f0674c0017fc4..04cce40f162e9 100644 --- a/tools/testing/selftests/drivers/net/netconsole/Makefile +++ b/tools/testing/selftests/drivers/net/netconsole/Makefile @@ -10,6 +10,7 @@ TEST_PROGS := \ netcons_cmdline.sh \ netcons_fragmented_msg.sh \ netcons_overflow.sh \ + netcons_ratelimit.sh \ netcons_resume.sh \ netcons_sysdata.sh \ netcons_torture.sh \ diff --git a/tools/testing/selftests/drivers/net/netconsole/netcons_ratelimit.sh b/tools/testing/selftests/drivers/net/netconsole/netcons_ratelimit.sh new file mode 100755 index 0000000000000..38dd1599fe57d --- /dev/null +++ b/tools/testing/selftests/drivers/net/netconsole/netcons_ratelimit.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-2.0 + +# This test exercises the per-target rate limit. It configures a small burst +# over an interval long enough that the bucket is never refilled, sends many +# more messages than the burst allows, and checks that the target stops +# transmitting once the bucket is empty. +# +# Clearing the interval has to restore unlimited delivery and tell the +# receiver how many messages it missed, which is verified last. +# +# Author: Breno Leitao + +set -euo pipefail + +SCRIPTDIR=$(dirname "$(readlink -e "${BASH_SOURCE[0]}")") + +source "${SCRIPTDIR}"/../lib/sh/lib_netcons.sh + +# Messages sent while the limit is in place, comfortably above BURST so that +# the bucket is drained +MSG_COUNT=50 +BURST=5 +# Long enough that the bucket is not refilled while the test runs +INTERVAL_MS=60000 +# Default the target starts with, as documented in netconsole.rst +DEFAULT_BURST=10 +# What the target sends once it can transmit again +DROP_NOTICE="messages dropped by rate limit" + +# The content of kmsg will be saved to the following file +OUTPUT_FILE="/tmp/${TARGET}" + +function count_msgs() { + local FILE="${1}" + + if [ ! -f "${FILE}" ] + then + echo 0 + return + fi + + # grep exits 1 on no match, which is a valid result here + grep -c "${MSG}" "${FILE}" || true +} + +function send_msgs() { + local COUNT="${1}" + local I + + for I in $(seq "${COUNT}") + do + echo "${MSG}: ${TARGET} ${I}" > /dev/kmsg + done +} + +# A freshly created target has to be unlimited, otherwise every existing +# netconsole user would start dropping messages after an upgrade +function check_defaults() { + local INTERVAL BURST_DEFAULT + + INTERVAL=$(cat "${NETCONS_PATH}"/ratelimit_interval_ms) + BURST_DEFAULT=$(cat "${NETCONS_PATH}"/ratelimit_burst) + + if [ "${INTERVAL}" -ne 0 ] || + [ "${BURST_DEFAULT}" -ne "${DEFAULT_BURST}" ] + then + echo "FAIL: unexpected rate limit defaults:" \ + "interval=${INTERVAL} burst=${BURST_DEFAULT}" >&2 + exit "${ksft_fail}" + fi +} + +function check_limited() { + local RECEIVED + + RECEIVED=$(count_msgs "${OUTPUT_FILE}") + + # Unrelated kernel messages share the bucket, so fewer than BURST of + # ours can get through, but never more + if [ "${RECEIVED}" -gt "${BURST}" ] + then + echo "FAIL: received ${RECEIVED} messages with ratelimit_burst=${BURST}" >&2 + cat "${OUTPUT_FILE}" >&2 + exit "${ksft_fail}" + fi +} + +# The notice below travels ahead of the message that reopened the bucket, so +# waiting for the file to appear is not enough +function msg_received() { + grep -q "${MSG}" "${OUTPUT_FILE}" 2> /dev/null +} + +# The messages lost above have to be reported to the receiver +function check_drops_reported() { + if ! grep -q "${DROP_NOTICE}" "${OUTPUT_FILE}" + then + echo "FAIL: no rate limit notice in ${OUTPUT_FILE}" >&2 + cat "${OUTPUT_FILE}" >&2 + exit "${ksft_fail}" + fi +} + +# ========== # +# Start here # +# ========== # + +modprobe netdevsim 2> /dev/null || true +modprobe netconsole 2> /dev/null || true + +# Check for basic system dependency and exit if not found +check_for_dependencies +# Remove the namespace, interfaces and netconsole target on exit +trap cleanup EXIT + +# Set current loglevel to KERN_INFO(6), and default to KERN_NOTICE(5) +echo "6 5" > /proc/sys/kernel/printk +# Create one namespace and two interfaces +set_network +# Create a dynamic target for netconsole +create_dynamic_target + +check_defaults + +# Set the burst before the interval, so that no message escapes while the +# target still carries the default burst +echo "${BURST}" > "${NETCONS_PATH}"/ratelimit_burst +echo "${INTERVAL_MS}" > "${NETCONS_PATH}"/ratelimit_interval_ms + +listen_port_and_save_to "${OUTPUT_FILE}" & +wait_for_port "${NAMESPACE}" "${PORT}" "ipv4" +send_msgs "${MSG_COUNT}" +# This half of the test is about messages that never arrive, so there is +# nothing to busywait on +sleep 1 +pkill_socat +check_limited +rm -f "${OUTPUT_FILE}" + +# Dropping the interval back to zero has to make the target unlimited again +echo 0 > "${NETCONS_PATH}"/ratelimit_interval_ms + +listen_port_and_save_to "${OUTPUT_FILE}" & +wait_for_port "${NAMESPACE}" "${PORT}" "ipv4" +send_msgs 1 +if ! busywait "${BUSYWAIT_TIMEOUT}" msg_received +then + echo "FAIL: Timed out waiting (${BUSYWAIT_TIMEOUT} ms) for netconsole" \ + "message in ${OUTPUT_FILE} after clearing the rate limit" >&2 + exit "${ksft_fail}" +fi +validate_msg "${OUTPUT_FILE}" +check_drops_reported +pkill_socat +rm -f "${OUTPUT_FILE}" + +trap - EXIT +cleanup +exit "${ksft_pass}" -- 2.53.0-Meta