A future patch will enable NFSD to sign filehandles by appending a Message Authentication Code(MAC). To do this, NFSD requires a secret 128-bit key that can persist across reboots. A persisted key allows the server to accept filehandles after a restart. Enable NFSD to be configured with this key via the netlink interface. Link: https://lore.kernel.org/linux-nfs/cover.1771961922.git.bcodding@hammerspace.com Signed-off-by: Benjamin Coddington Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- Documentation/netlink/specs/nfsd.yaml | 6 +++++ fs/nfsd/netlink.c | 5 ++-- fs/nfsd/netns.h | 1 + fs/nfsd/nfsctl.c | 38 ++++++++++++++++++++++++++- fs/nfsd/trace.h | 22 ++++++++++++++++ include/uapi/linux/nfsd_netlink.h | 1 + 6 files changed, 70 insertions(+), 3 deletions(-) diff --git a/Documentation/netlink/specs/nfsd.yaml b/Documentation/netlink/specs/nfsd.yaml index badb2fe57c98..d348648033d9 100644 --- a/Documentation/netlink/specs/nfsd.yaml +++ b/Documentation/netlink/specs/nfsd.yaml @@ -81,6 +81,11 @@ attribute-sets: - name: min-threads type: u32 + - + name: fh-key + type: binary + checks: + exact-len: 16 - name: version attributes: @@ -163,6 +168,7 @@ operations: - leasetime - scope - min-threads + - fh-key - name: threads-get doc: get the number of running threads diff --git a/fs/nfsd/netlink.c b/fs/nfsd/netlink.c index 887525964451..4e08c1a6b394 100644 --- a/fs/nfsd/netlink.c +++ b/fs/nfsd/netlink.c @@ -24,12 +24,13 @@ const struct nla_policy nfsd_version_nl_policy[NFSD_A_VERSION_ENABLED + 1] = { }; /* NFSD_CMD_THREADS_SET - do */ -static const struct nla_policy nfsd_threads_set_nl_policy[NFSD_A_SERVER_MIN_THREADS + 1] = { +static const struct nla_policy nfsd_threads_set_nl_policy[NFSD_A_SERVER_FH_KEY + 1] = { [NFSD_A_SERVER_THREADS] = { .type = NLA_U32, }, [NFSD_A_SERVER_GRACETIME] = { .type = NLA_U32, }, [NFSD_A_SERVER_LEASETIME] = { .type = NLA_U32, }, [NFSD_A_SERVER_SCOPE] = { .type = NLA_NUL_STRING, }, [NFSD_A_SERVER_MIN_THREADS] = { .type = NLA_U32, }, + [NFSD_A_SERVER_FH_KEY] = NLA_POLICY_EXACT_LEN(16), }; /* NFSD_CMD_VERSION_SET - do */ @@ -58,7 +59,7 @@ static const struct genl_split_ops nfsd_nl_ops[] = { .cmd = NFSD_CMD_THREADS_SET, .doit = nfsd_nl_threads_set_doit, .policy = nfsd_threads_set_nl_policy, - .maxattr = NFSD_A_SERVER_MIN_THREADS, + .maxattr = NFSD_A_SERVER_MAX, .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, }, { diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index 3a89d4708e8a..6ad3fe5d7e12 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -227,6 +227,7 @@ struct nfsd_net { spinlock_t local_clients_lock; struct list_head local_clients; #endif + siphash_key_t *fh_key; }; /* Simple check to find out if a given net was properly initialized */ diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index a745b97b45fb..032ab44feb70 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -1581,6 +1581,32 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, return ret; } +/** + * nfsd_nl_fh_key_set - helper to copy fh_key from userspace + * @attr: nlattr NFSD_A_SERVER_FH_KEY + * @nn: nfsd_net + * + * Callers should hold nfsd_mutex, returns 0 on success or negative errno. + * Callers must ensure the server is shut down (sv_nrthreads == 0), + * userspace documentation asserts the key may only be set when the server + * is not running. + */ +static int nfsd_nl_fh_key_set(const struct nlattr *attr, struct nfsd_net *nn) +{ + siphash_key_t *fh_key = nn->fh_key; + + if (!fh_key) { + fh_key = kmalloc(sizeof(siphash_key_t), GFP_KERNEL); + if (!fh_key) + return -ENOMEM; + nn->fh_key = fh_key; + } + + fh_key->key[0] = get_unaligned_le64(nla_data(attr)); + fh_key->key[1] = get_unaligned_le64(nla_data(attr) + 8); + return 0; +} + /** * nfsd_nl_threads_set_doit - set the number of running threads * @skb: reply buffer @@ -1622,7 +1648,8 @@ int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info) if (info->attrs[NFSD_A_SERVER_GRACETIME] || info->attrs[NFSD_A_SERVER_LEASETIME] || - info->attrs[NFSD_A_SERVER_SCOPE]) { + info->attrs[NFSD_A_SERVER_SCOPE] || + info->attrs[NFSD_A_SERVER_FH_KEY]) { ret = -EBUSY; if (nn->nfsd_serv && nn->nfsd_serv->sv_nrthreads) goto out_unlock; @@ -1651,6 +1678,14 @@ int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info) attr = info->attrs[NFSD_A_SERVER_SCOPE]; if (attr) scope = nla_data(attr); + + attr = info->attrs[NFSD_A_SERVER_FH_KEY]; + if (attr) { + ret = nfsd_nl_fh_key_set(attr, nn); + trace_nfsd_ctl_fh_key_set((const char *)nn->fh_key, ret); + if (ret) + goto out_unlock; + } } attr = info->attrs[NFSD_A_SERVER_MIN_THREADS]; @@ -2250,6 +2285,7 @@ static __net_exit void nfsd_net_exit(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); + kfree_sensitive(nn->fh_key); nfsd_proc_stat_shutdown(net); percpu_counter_destroy_many(nn->counter, NFSD_STATS_COUNTERS_NUM); nfsd_idmap_shutdown(net); diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index d1d0b0dd0545..185a998996a0 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -2240,6 +2240,28 @@ TRACE_EVENT(nfsd_end_grace, ) ); +TRACE_EVENT(nfsd_ctl_fh_key_set, + TP_PROTO( + const char *key, + int result + ), + TP_ARGS(key, result), + TP_STRUCT__entry( + __field(u32, key_hash) + __field(int, result) + ), + TP_fast_assign( + if (key) + __entry->key_hash = ~crc32_le(0xFFFFFFFF, key, 16); + else + __entry->key_hash = 0; + __entry->result = result; + ), + TP_printk("key=0x%08x result=%d", + __entry->key_hash, __entry->result + ) +); + DECLARE_EVENT_CLASS(nfsd_copy_class, TP_PROTO( const struct nfsd4_copy *copy diff --git a/include/uapi/linux/nfsd_netlink.h b/include/uapi/linux/nfsd_netlink.h index e9efbc9e63d8..97c7447f4d14 100644 --- a/include/uapi/linux/nfsd_netlink.h +++ b/include/uapi/linux/nfsd_netlink.h @@ -36,6 +36,7 @@ enum { NFSD_A_SERVER_LEASETIME, NFSD_A_SERVER_SCOPE, NFSD_A_SERVER_MIN_THREADS, + NFSD_A_SERVER_FH_KEY, __NFSD_A_SERVER_MAX, NFSD_A_SERVER_MAX = (__NFSD_A_SERVER_MAX - 1) -- 2.53.0 In order to signal that filehandles on this export should be signed, add a "sign_fh" export option. Filehandle signing can help the server defend against certain filehandle guessing attacks. Setting the "sign_fh" export option sets NFSEXP_SIGN_FH. In a future patch NFSD uses this signal to append a MAC onto filehandles for that export. While we're in here, tidy a few stray expflags to more closely align to the export flag order. Link: https://lore.kernel.org/linux-nfs/cover.1771961922.git.bcodding@hammerspace.com Signed-off-by: Benjamin Coddington Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/export.c | 5 +++-- include/uapi/linux/nfsd/export.h | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 8e8a76a44ff0..7f4a51b832ef 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -1362,13 +1362,14 @@ static struct flags { { NFSEXP_ASYNC, {"async", "sync"}}, { NFSEXP_GATHERED_WRITES, {"wdelay", "no_wdelay"}}, { NFSEXP_NOREADDIRPLUS, {"nordirplus", ""}}, + { NFSEXP_SECURITY_LABEL, {"security_label", ""}}, + { NFSEXP_SIGN_FH, {"sign_fh", ""}}, { NFSEXP_NOHIDE, {"nohide", ""}}, - { NFSEXP_CROSSMOUNT, {"crossmnt", ""}}, { NFSEXP_NOSUBTREECHECK, {"no_subtree_check", ""}}, { NFSEXP_NOAUTHNLM, {"insecure_locks", ""}}, + { NFSEXP_CROSSMOUNT, {"crossmnt", ""}}, { NFSEXP_V4ROOT, {"v4root", ""}}, { NFSEXP_PNFS, {"pnfs", ""}}, - { NFSEXP_SECURITY_LABEL, {"security_label", ""}}, { 0, {"", ""}} }; diff --git a/include/uapi/linux/nfsd/export.h b/include/uapi/linux/nfsd/export.h index a73ca3703abb..de647cf166c3 100644 --- a/include/uapi/linux/nfsd/export.h +++ b/include/uapi/linux/nfsd/export.h @@ -34,7 +34,7 @@ #define NFSEXP_GATHERED_WRITES 0x0020 #define NFSEXP_NOREADDIRPLUS 0x0040 #define NFSEXP_SECURITY_LABEL 0x0080 -/* 0x100 currently unused */ +#define NFSEXP_SIGN_FH 0x0100 #define NFSEXP_NOHIDE 0x0200 #define NFSEXP_NOSUBTREECHECK 0x0400 #define NFSEXP_NOAUTHNLM 0x0800 /* Don't authenticate NLM requests - just trust */ @@ -55,7 +55,7 @@ #define NFSEXP_PNFS 0x20000 /* All flags that we claim to support. (Note we don't support NOACL.) */ -#define NFSEXP_ALLFLAGS 0x3FEFF +#define NFSEXP_ALLFLAGS 0x3FFFF /* The flags that may vary depending on security flavor: */ #define NFSEXP_SECINFO_FLAGS (NFSEXP_READONLY | NFSEXP_ROOTSQUASH \ -- 2.53.0 NFS clients may bypass restrictive directory permissions by using open_by_handle() (or other available OS system call) to guess the filehandles for files below that directory. In order to harden knfsd servers against this attack, create a method to sign and verify filehandles using SipHash-2-4 as a MAC (Message Authentication Code). According to https://cr.yp.to/siphash/siphash-20120918.pdf, SipHash can be used as a MAC, and our use of SipHash-2-4 provides a low 1 in 2^64 chance of forgery. Filehandles that have been signed cannot be tampered with, nor can clients reasonably guess correct filehandles and hashes that may exist in parts of the filesystem they cannot access due to directory permissions. Append the 8 byte SipHash to encoded filehandles for exports that have set the "sign_fh" export option. Filehandles received from clients are verified by comparing the appended hash to the expected hash. If the MAC does not match the server responds with NFS error _STALE. If unsigned filehandles are received for an export with "sign_fh" they are rejected with NFS error _STALE. Signed-off-by: Benjamin Coddington Reviewed-by: Jeff Layton --- Documentation/filesystems/nfs/exporting.rst | 85 +++++++++++++ fs/nfsd/Kconfig | 2 +- fs/nfsd/nfsfh.c | 127 +++++++++++++++++++- fs/nfsd/trace.h | 1 + 4 files changed, 210 insertions(+), 5 deletions(-) diff --git a/Documentation/filesystems/nfs/exporting.rst b/Documentation/filesystems/nfs/exporting.rst index a01d9b9b5bc3..4aa59b0bf253 100644 --- a/Documentation/filesystems/nfs/exporting.rst +++ b/Documentation/filesystems/nfs/exporting.rst @@ -206,3 +206,88 @@ following flags are defined: all of an inode's dirty data on last close. Exports that behave this way should set EXPORT_OP_FLUSH_ON_CLOSE so that NFSD knows to skip waiting for writeback when closing such files. + +Signed Filehandles +------------------ + +To protect against filehandle guessing attacks, the Linux NFS server can be +configured to sign filehandles with a Message Authentication Code (MAC). + +Standard NFS filehandles are often predictable. If an attacker can guess +a valid filehandle for a file they do not have permission to access via +directory traversal, they may be able to bypass path-based permissions +(though they still remain subject to inode-level permissions). + +Signed filehandles prevent this by appending a MAC to the filehandle +before it is sent to the client. Upon receiving a filehandle back from a +client, the server re-calculates the MAC using its internal key and +verifies it against the one provided. If the signatures do not match, +the server treats the filehandle as invalid (returning NFS[34]ERR_STALE). + +Note that signing filehandles provides integrity and authenticity but +not confidentiality. The contents of the filehandle remain visible to +the client; they simply cannot be forged or modified. + +Configuration +~~~~~~~~~~~~~ + +To enable signed filehandles, the administrator must provide a signing +key to the kernel and enable the "sign_fh" export option. + +1. Providing a Key + The signing key is managed via the nfsd netlink interface. This key + is per-network-namespace and must be set before any exports using + "sign_fh" become active. + +2. Export Options + The feature is controlled on a per-export basis in /etc/exports: + + sign_fh + Enables signing for all filehandles generated under this export. + + no_sign_fh + (Default) Disables signing. + +Key Management and Rotation +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The security of this mechanism relies entirely on the secrecy of the +signing key. + +Initial Setup: + The key should be generated using a high-quality random source and + loaded early in the boot process or during the nfs-server startup + sequence. + +Changing Keys: + If a key is changed while clients have active mounts, existing + filehandles held by those clients will become invalid, resulting in + "Stale file handle" errors on the client side. + +Safe Rotation: + Currently, there is no mechanism for "graceful" key rotation + (maintaining multiple valid keys). Changing the key is an atomic + operation that immediately invalidates all previous signatures. + +Transitioning Exports +~~~~~~~~~~~~~~~~~~~~~ + +When adding or removing the "sign_fh" flag from an active export, the +following behaviors should be expected: + ++-------------------+---------------------------------------------------+ +| Change | Result for Existing Clients | ++===================+===================================================+ +| Adding sign_fh | Clients holding unsigned filehandles will find | +| | them rejected, as the server now expects a | +| | signature. | ++-------------------+---------------------------------------------------+ +| Removing sign_fh | Clients holding signed filehandles will find them | +| | rejected, as the server now expects the | +| | filehandle to end at its traditional boundary | +| | without a MAC. | ++-------------------+---------------------------------------------------+ + +Because filehandles are often cached persistently by clients, adding or +removing this option should generally be done during a scheduled maintenance +window involving a NFS client unmount/remount. diff --git a/fs/nfsd/Kconfig b/fs/nfsd/Kconfig index fc0e87eaa257..ffb76761d6a8 100644 --- a/fs/nfsd/Kconfig +++ b/fs/nfsd/Kconfig @@ -7,6 +7,7 @@ config NFSD select CRC32 select CRYPTO_LIB_MD5 if NFSD_LEGACY_CLIENT_TRACKING select CRYPTO_LIB_SHA256 if NFSD_V4 + select CRYPTO # required by RPCSEC_GSS_KRB5 and signed filehandles select LOCKD select SUNRPC select EXPORTFS @@ -78,7 +79,6 @@ config NFSD_V4 depends on NFSD && PROC_FS select FS_POSIX_ACL select RPCSEC_GSS_KRB5 - select CRYPTO # required by RPCSEC_GSS_KRB5 select GRACE_PERIOD select NFS_V4_2_SSC_HELPER if NFS_V4_2 help diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 68b629fbaaeb..383d04596627 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -11,6 +11,7 @@ #include #include +#include #include "nfsd.h" #include "vfs.h" #include "auth.h" @@ -140,6 +141,110 @@ static inline __be32 check_pseudo_root(struct dentry *dentry, return nfs_ok; } +/* Size of a file handle MAC, in 4-octet words */ +#define FH_MAC_WORDS (sizeof(__le64) / 4) + +static bool fh_append_mac(struct svc_fh *fhp, struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct knfsd_fh *fh = &fhp->fh_handle; + siphash_key_t *fh_key = nn->fh_key; + __le64 hash; + + if (!fh_key) + goto out_no_key; + if (fh->fh_size + sizeof(hash) > fhp->fh_maxsize) + goto out_no_space; + + hash = cpu_to_le64(siphash(&fh->fh_raw, fh->fh_size, fh_key)); + memcpy(&fh->fh_raw[fh->fh_size], &hash, sizeof(hash)); + fh->fh_size += sizeof(hash); + return true; + +out_no_key: + pr_warn_ratelimited("NFSD: unable to sign filehandles, fh_key not set.\n"); + return false; + +out_no_space: + pr_warn_ratelimited("NFSD: unable to sign filehandles, fh_size %zu would be greater than fh_maxsize %d.\n", + fh->fh_size + sizeof(hash), fhp->fh_maxsize); + return false; +} + +/* + * Verify that the filehandle's MAC was hashed from this filehandle + * given the server's fh_key: + */ +static bool fh_verify_mac(struct svc_fh *fhp, struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct knfsd_fh *fh = &fhp->fh_handle; + siphash_key_t *fh_key = nn->fh_key; + __le64 hash; + + if (!fh_key) { + pr_warn_ratelimited("NFSD: unable to verify signed filehandles, fh_key not set.\n"); + return false; + } + + hash = cpu_to_le64(siphash(&fh->fh_raw, fh->fh_size - sizeof(hash), fh_key)); + return crypto_memneq(&fh->fh_raw[fh->fh_size - sizeof(hash)], + &hash, sizeof(hash)) == 0; +} + +/* + * Append an 8-byte MAC to the filehandle hashed from the server's fh_key: + */ +#define FH_MAC_WORDS sizeof(__le64)/4 +static bool fh_append_mac(struct svc_fh *fhp, struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct knfsd_fh *fh = &fhp->fh_handle; + siphash_key_t *fh_key = nn->fh_key; + __le64 hash; + + if (!(fhp->fh_export->ex_flags & NFSEXP_SIGN_FH)) + return true; + + if (!fh_key) { + pr_warn_ratelimited("NFSD: unable to sign filehandles, fh_key not set.\n"); + return false; + } + + if (fh->fh_size + sizeof(hash) > fhp->fh_maxsize) { + pr_warn_ratelimited("NFSD: unable to sign filehandles, fh_size %d would be greater" + " than fh_maxsize %d.\n", (int)(fh->fh_size + sizeof(hash)), fhp->fh_maxsize); + return false; + } + + hash = cpu_to_le64(siphash(&fh->fh_raw, fh->fh_size, fh_key)); + memcpy(&fh->fh_raw[fh->fh_size], &hash, sizeof(hash)); + fh->fh_size += sizeof(hash); + + return true; +} + +/* + * Verify that the filehandle's MAC was hashed from this filehandle + * given the server's fh_key: + */ +static bool fh_verify_mac(struct svc_fh *fhp, struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct knfsd_fh *fh = &fhp->fh_handle; + siphash_key_t *fh_key = nn->fh_key; + __le64 hash; + + if (!fh_key) { + pr_warn_ratelimited("NFSD: unable to verify signed filehandles, fh_key not set.\n"); + return false; + } + + hash = cpu_to_le64(siphash(&fh->fh_raw, fh->fh_size - sizeof(hash), fh_key)); + return crypto_memneq(&fh->fh_raw[fh->fh_size - sizeof(hash)], + &hash, sizeof(hash)) == 0; +} + /* * Use the given filehandle to look up the corresponding export and * dentry. On success, the results are used to set fh_export and @@ -236,13 +341,21 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct net *net, /* * Look up the dentry using the NFS file handle. */ - error = nfserr_badhandle; - fileid_type = fh->fh_fileid_type; + error = nfserr_stale; - if (fileid_type == FILEID_ROOT) + if (fileid_type == FILEID_ROOT) { + /* We don't sign or verify the root, no per-file identity */ dentry = dget(exp->ex_path.dentry); - else { + } else { + if (exp->ex_flags & NFSEXP_SIGN_FH) { + if (!fh_verify_mac(fhp, net)) { + trace_nfsd_set_fh_dentry_badmac(rqstp, fhp, -ESTALE); + goto out; + } + data_left -= FH_MAC_WORDS; + } + dentry = exportfs_decode_fh_raw(exp->ex_path.mnt, fid, data_left, fileid_type, 0, nfsd_acceptable, exp); @@ -258,6 +371,8 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct net *net, } } } + + error = nfserr_badhandle; if (dentry == NULL) goto out; if (IS_ERR(dentry)) { @@ -498,6 +613,10 @@ static void _fh_update(struct svc_fh *fhp, struct svc_export *exp, fhp->fh_handle.fh_fileid_type = fileid_type > 0 ? fileid_type : FILEID_INVALID; fhp->fh_handle.fh_size += maxsize * 4; + + if (exp->ex_flags & NFSEXP_SIGN_FH) + if (!fh_append_mac(fhp, exp->cd->net)) + fhp->fh_handle.fh_fileid_type = FILEID_INVALID; } else { fhp->fh_handle.fh_fileid_type = FILEID_ROOT; } diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 185a998996a0..5ad38f50836d 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -373,6 +373,7 @@ DEFINE_EVENT_CONDITION(nfsd_fh_err_class, nfsd_##name, \ DEFINE_NFSD_FH_ERR_EVENT(set_fh_dentry_badexport); DEFINE_NFSD_FH_ERR_EVENT(set_fh_dentry_badhandle); +DEFINE_NFSD_FH_ERR_EVENT(set_fh_dentry_badmac); TRACE_EVENT(nfsd_exp_find_key, TP_PROTO(const struct svc_expkey *key, -- 2.53.0