This symbol is used from the Nova driver, so it needs to be exported to avoid a build failure when building Nova as a module. ERROR: modpost: "vrealloc_node_align_noprof" [drivers/gpu/nova-core/nova_core.ko] undefined! ERROR: modpost: "vrealloc_node_align_noprof" [samples/rust/rust_dma.ko] undefined! This error is only triggered if inlining of helpers into Rust is enabled. Signed-off-by: Alice Ryhl --- mm/vmalloc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 798b2ed21e46059f341ed0d46c7fe56bbe357b22..e086d00f04d61ebd481cb84e3dfea51d8a2ffc57 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -4200,6 +4200,7 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align return n; } +EXPORT_SYMBOL(vrealloc_node_align_noprof); #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) -- 2.52.0.158.g65b55ccf14-goog From: Gary Guo Because of LLVM inling checks, it's generally not possible to inline a C helper into Rust code, even with LTO: * LLVM doesn't want to inline functions compiled with `-fno-delete-null-pointer-checks` with code compiled without. The C CGUs all have this enabled and Rust CGUs don't. Inlining is okay since this is one of the hardening features that does not change the ABI, and we shouldn't have null pointer dereferences in these helpers. * LLVM doesn't want to inline functions with different list of builtins. C side has `-fno-builtin-wcslen`; `wcslen` is not a Rust builtin, so they should be compatible, but LLVM does not perform inlining due to attributes mismatch. * clang and Rust doesn't have the exact target string. Clang generates `+cmov,+cx8,+fxsr` but Rust doesn't enable them (in fact, Rust will complain if `-Ctarget-feature=+cmov,+cx8,+fxsr` is used). x86-64 always enable these features, so they are in fact the same target string, but LLVM doesn't understand this and so inlining is inhibited. This can be bypassed with `--ignore-tti-inline-compatible`, but this is a hidden option. To fix this, we can add __always_inline on every helper, which skips these LLVM inlining checks. For this purpose, introduce a new __rust_helper macro that needs to be added to every helper. The actual additions of __rust_helper can happen in separate patches. A "flag day" change is not required since missing annotations do not lead to anything worse than missing inlining. Signed-off-by: Gary Guo Signed-off-by: Alice Ryhl --- rust/helpers/atomic.c | 5 ----- rust/helpers/helpers.c | 31 +++++++++++++++++++++++++++++++ scripts/atomic/gen-rust-atomic-helpers.sh | 5 ----- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/rust/helpers/atomic.c b/rust/helpers/atomic.c index cf06b7ef9a1c559e8d7bdfc2bcd2aeb8951c29d1..a48605628ed73ac32aae2e6280481407a670e88f 100644 --- a/rust/helpers/atomic.c +++ b/rust/helpers/atomic.c @@ -11,11 +11,6 @@ #include -// TODO: Remove this after INLINE_HELPERS support is added. -#ifndef __rust_helper -#define __rust_helper -#endif - __rust_helper int rust_helper_atomic_read(const atomic_t *v) { diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 551da6c9b5064c324d6f62bafcec672c6c6f5bee..d0e2f1f9b449b2248cfbddcee1e8bf9becc8a2f9 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -7,6 +7,37 @@ * Sorted alphabetically. */ +#include + +#ifdef __BINDGEN__ +// Omit `inline` for bindgen as it ignores inline functions. +#define __rust_helper +#else +// The helper functions are all inline functions. +// +// We use `__always_inline` here to bypass LLVM inlining checks, in case the +// helpers are inlined directly into Rust CGUs. +// +// The LLVM inlining checks are false positives: +// * LLVM doesn't want to inline functions compiled with +// `-fno-delete-null-pointer-checks` with code compiled without. +// The C CGUs all have this enabled and Rust CGUs don't. Inlining is okay +// since this is one of the hardening features that does not change the ABI, +// and we shouldn't have null pointer dereferences in these helpers. +// * LLVM doesn't want to inline functions with different list of builtins. C +// side has `-fno-builtin-wcslen`; `wcslen` is not a Rust builtin, so they +// should be compatible, but LLVM does not perform inlining due to attributes +// mismatch. +// * clang and Rust doesn't have the exact target string. Clang generates +// `+cmov,+cx8,+fxsr` but Rust doesn't enable them (in fact, Rust will +// complain if `-Ctarget-feature=+cmov,+cx8,+fxsr` is used). x86-64 always +// enable these features, so they are in fact the same target string, but +// LLVM doesn't understand this and so inlining is inhibited. This can be +// bypassed with `--ignore-tti-inline-compatible`, but this is a hidden +// option. +#define __rust_helper __always_inline +#endif + #include "atomic.c" #include "auxiliary.c" #include "barrier.c" diff --git a/scripts/atomic/gen-rust-atomic-helpers.sh b/scripts/atomic/gen-rust-atomic-helpers.sh index 45b1e100ed7c63108ee6cb07e48a17668f860d47..a3732153af29f415e397e17cab6e75cb5d7efafc 100755 --- a/scripts/atomic/gen-rust-atomic-helpers.sh +++ b/scripts/atomic/gen-rust-atomic-helpers.sh @@ -47,11 +47,6 @@ cat << EOF #include -// TODO: Remove this after INLINE_HELPERS support is added. -#ifndef __rust_helper -#define __rust_helper -#endif - EOF grep '^[a-z]' "$1" | while read name meta args; do -- 2.52.0.158.g65b55ccf14-goog From: Gary Guo This config detects if Rust and Clang have matching LLVM major version. All IR or bitcode operations (e.g. LTO) rely on LLVM major version to be matching, otherwise it may generate errors, or worse, miscompile silently due to change of IR semantics. It's usually suggested to use the exact same LLVM version, but this can be difficult to guarantee. Rust's suggestion [1] is also major-version only, so I think this check is sufficient for the kernel. Link: https://doc.rust-lang.org/rustc/linker-plugin-lto.html [1] Reviewed-by: Andreas Hindborg Signed-off-by: Gary Guo Signed-off-by: Matthew Maurer Signed-off-by: Alice Ryhl --- init/Kconfig | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/init/Kconfig b/init/Kconfig index cab3ad28ca49e7ac930207c9cde8d431d55dc7af..d35d0ddee573d09997087a0f99dee7c7c9a66ae3 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -82,6 +82,21 @@ config RUSTC_LLVM_VERSION int default $(rustc-llvm-version) +config RUSTC_LLVM_MAJOR_VERSION + int + default $(shell,expr $(rustc-llvm-version) / 10000) + +config RUSTC_CLANG_LLVM_COMPATIBLE + bool + default y if CC_IS_CLANG && RUSTC_LLVM_MAJOR_VERSION = $(shell,expr $(cc-version) / 10000) + help + This indicates whether Rust and Clang use LLVM of the same major + version. + + Operations involving handling LLVM IR or bitcode (e.g. cross-language + LTO) requires the same LLVM major version to work properly. For best + compatibility it is recommended that the exact same LLVM is used. + config CC_CAN_LINK bool default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(USERCFLAGS) $(USERLDFLAGS) $(m64-flag)) if 64BIT -- 2.52.0.158.g65b55ccf14-goog From: Gary Guo A new experimental Kconfig option, `RUST_INLINE_HELPERS` is added to allow C helpers (which were created to allow Rust to call into inline/macro C functions without having to re-implement the logic in Rust) to be inlined into Rust crates without performing global LTO. If the option is enabled, the following is performed: * For helpers, instead of compiling them to an object file to be linked into vmlinux, they're compiled to LLVM IR. * The LLVM IR is compiled to bitcode (This is step is not necessary, but is a performance optimisation to prevent LLVM from always have to reparse the same IR). * When a Rust crate is compiled, instead of generating an object file, we ask LLVM bitcode to be generated. * llvm-link is invoked with --internalize to combine the helper bitcode with the crate bitcode. This step is similar to LTO, but this is much faster since it only needs to inline the helpers. * clang is invoked to turn the combined bitcode into a final object file. The --internalize flag tells llvm-link to treat all symbols in helpers.bc using `internal` linkage. This matches the behavior of `clang` on `static inline` functions, and avoids exporting the symbol from the object file. To ensure that RUST_INLINE_HELPERS is not incompatible with BTF, we pass the -g0 flag when building helpers. See commit 5daa0c35a1f0 ("rust: Disallow BTF generation with Rust + LTO") for details. We have an intended triple mismatch of `aarch64-unknown-none` vs `aarch64-unknown-linux-gnu`, so we suppress the warning. Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Co-developed-by: Matthew Maurer Signed-off-by: Matthew Maurer Signed-off-by: Gary Guo Co-developed-by: Alice Ryhl Signed-off-by: Alice Ryhl --- Makefile | 4 +++- lib/Kconfig.debug | 15 +++++++++++++++ rust/Makefile | 26 ++++++++++++++++++++++---- rust/exports.c | 5 ++++- scripts/Makefile.build | 5 ++++- 5 files changed, 48 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 96ddbaae7e12de71bcfabef4639de3a13a6e4815..5834bfd568548d1bee34b328dccce5d60f85526f 100644 --- a/Makefile +++ b/Makefile @@ -517,6 +517,8 @@ OBJCOPY = $(LLVM_PREFIX)llvm-objcopy$(LLVM_SUFFIX) OBJDUMP = $(LLVM_PREFIX)llvm-objdump$(LLVM_SUFFIX) READELF = $(LLVM_PREFIX)llvm-readelf$(LLVM_SUFFIX) STRIP = $(LLVM_PREFIX)llvm-strip$(LLVM_SUFFIX) +LLVM_LINK = $(LLVM_PREFIX)llvm-link$(LLVM_SUFFIX) +LLVM_AS = $(LLVM_PREFIX)llvm-as$(LLVM_SUFFIX) else CC = $(CROSS_COMPILE)gcc LD = $(CROSS_COMPILE)ld @@ -625,7 +627,7 @@ export RUSTC_BOOTSTRAP := 1 export CLIPPY_CONF_DIR := $(srctree) export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC HOSTPKG_CONFIG -export RUSTC RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN +export RUSTC RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN LLVM_LINK LLVM_AS export HOSTRUSTC KBUILD_HOSTRUSTFLAGS export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 3034e294d50df55c4003c5fa5df442f59e711bd8..e63c5eb57b049aff988419ccd12dfd99d59f5080 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -3427,6 +3427,21 @@ config RUST_KERNEL_DOCTESTS If unsure, say N. +config RUST_INLINE_HELPERS + bool "Inline C helpers into Rust crates (EXPERIMENTAL)" + depends on RUST && RUSTC_CLANG_LLVM_COMPATIBLE + depends on EXPERT + help + Links C helpers into Rust crates through LLVM IR. + + If this option is enabled, instead of generating object files directly, + rustc is asked to produce LLVM IR, which is then linked together with + the LLVM IR of C helpers, before object file is generated. + + This requires a matching LLVM version for Clang and rustc. + + If unsure, say N. + endmenu # "Rust" endmenu # Kernel hacking diff --git a/rust/Makefile b/rust/Makefile index d7d19c21b671dea10242b1772a8bcf0bf5dcc1cd..2344e2662ce29280582215954132c09f63cd8c9d 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -6,15 +6,19 @@ rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc obj-$(CONFIG_RUST) += core.o compiler_builtins.o ffi.o always-$(CONFIG_RUST) += exports_core_generated.h +ifdef CONFIG_RUST_INLINE_HELPERS +always-$(CONFIG_RUST) += helpers/helpers.bc +else +obj-$(CONFIG_RUST) += helpers/helpers.o +always-$(CONFIG_RUST) += exports_helpers_generated.h +endif # Missing prototypes are expected in the helpers since these are exported # for Rust only, thus there is no header nor prototypes. -obj-$(CONFIG_RUST) += helpers/helpers.o CFLAGS_REMOVE_helpers/helpers.o = -Wmissing-prototypes -Wmissing-declarations always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs obj-$(CONFIG_RUST) += bindings.o pin_init.o kernel.o -always-$(CONFIG_RUST) += exports_helpers_generated.h \ - exports_bindings_generated.h exports_kernel_generated.h +always-$(CONFIG_RUST) += exports_bindings_generated.h exports_kernel_generated.h always-$(CONFIG_RUST) += uapi/uapi_generated.rs obj-$(CONFIG_RUST) += uapi.o @@ -468,6 +472,13 @@ $(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; $(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers/helpers.c FORCE $(call if_changed_dep,bindgen) +quiet_cmd_rust_helper = HELPER $@ + cmd_rust_helper = \ + $(CC) $(filter-out $(CFLAGS_REMOVE_helpers/helpers.o), $(c_flags)) -c -g0 $< -emit-llvm -o $@ + +$(obj)/helpers/helpers.bc: $(obj)/helpers/helpers.c FORCE + +$(call if_changed_dep,rust_helper) + rust_exports = $(NM) -p --defined-only $(1) | awk '$$2~/(T|R|D|B)/ && $$3!~/__(pfx|cfi|odr_asan)/ { printf $(2),$$3 }' quiet_cmd_exports = EXPORTS $@ @@ -547,11 +558,13 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L OBJTREE=$(abspath $(objtree)) \ $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ $(filter-out $(skip_flags),$(rust_flags)) $(rustc_target_flags) \ - --emit=dep-info=$(depfile) --emit=obj=$@ \ + --emit=dep-info=$(depfile) --emit=$(if $(link_helper),llvm-bc=$(patsubst %.o,%.bc,$@),obj=$@) \ --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \ --crate-type rlib -L$(objtree)/$(obj) \ --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ --sysroot=/dev/null \ + $(if $(link_helper),;$(LLVM_LINK) --internalize --suppress-warnings $(patsubst %.o,%.bc,$@) $(obj)/helpers/helpers.bc -o $(patsubst %.o,%.m.bc,$@); \ + $(CC) $(CLANG_FLAGS) $(KBUILD_CFLAGS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@) \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \ $(cmd_objtool) @@ -678,4 +691,9 @@ $(obj)/kernel.o: $(obj)/kernel/generated_arch_warn_asm.rs $(obj)/kernel/generate endif endif +ifdef CONFIG_RUST_INLINE_HELPERS +$(obj)/kernel.o: private link_helper = 1 +$(obj)/kernel.o: $(obj)/helpers/helpers.bc +endif + endif # CONFIG_RUST diff --git a/rust/exports.c b/rust/exports.c index 587f0e776aba52854080f15aa91094b55996c072..1b52460b0f4eeef6df9081abb9b7e054a28c3c21 100644 --- a/rust/exports.c +++ b/rust/exports.c @@ -16,10 +16,13 @@ #define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym) #include "exports_core_generated.h" -#include "exports_helpers_generated.h" #include "exports_bindings_generated.h" #include "exports_kernel_generated.h" +#ifndef CONFIG_RUST_INLINE_HELPERS +#include "exports_helpers_generated.h" +#endif + // For modules using `rust/build_error.rs`. #ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW EXPORT_SYMBOL_RUST_GPL(rust_build_error); diff --git a/scripts/Makefile.build b/scripts/Makefile.build index d0ee33a487be95f8ba9a5c964ebecfbebc6c4bf8..04eaf2b4fbca2245f904a6dc7875cb3275aa7df6 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -343,7 +343,10 @@ rust_common_cmd = \ # would not match each other. quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_o_rs = $(rust_common_cmd) --emit=obj=$@ $< $(cmd_objtool) + cmd_rustc_o_rs = $(rust_common_cmd) --emit=$(if $(CONFIG_RUST_INLINE_HELPERS),llvm-bc=$(patsubst %.o,%.bc,$@),obj=$@) $< \ + $(if $(CONFIG_RUST_INLINE_HELPERS),;$(LLVM_LINK) --internalize --suppress-warnings $(patsubst %.o,%.bc,$@) $(objtree)/rust/helpers/helpers.bc -o $(patsubst %.o,%.m.bc,$@); \ + $(CC) $(CLANG_FLAGS) $(KBUILD_CFLAGS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@) \ + $(cmd_objtool) define rule_rustc_o_rs $(call cmd_and_fixdep,rustc_o_rs) -- 2.52.0.158.g65b55ccf14-goog