With nullfs as the real rootfs userspace can pivot_root() away from the initramfs and unmount it: chdir(rootfs); pivot_root(".", "."); umount2(".", MNT_DETACH); That's what systemd does and the initramfs is supposed to go away at that point together with all the memory it uses. But on every machine with more than one possible cpu the initramfs superblock stays around forever (cf. [1]). With ramfs as rootfs the unpacked initramfs shows up as Unevictable in /proc/meminfo, with tmpfs as Shmem, and it is never freed. In the reports that's between 70MB and 150MB. The idle threads of the secondary cpus are created by fork_idle() from smp_init(). That runs in kernel_init(), i.e., in PID 1, after init_userspace_fs() moved PID 1 from nullfs into the initramfs. fork_idle() doesn't pass CLONE_FS so each idle thread gets a private copy of PID 1's fs_struct and with it a reference to the initramfs as root and as pwd. pivot_root() relies on chroot_fs_refs() to move everyone whose root or pwd is the old root over to the new root. Idle threads aren't on the task list though so chroot_fs_refs() never finds them. They never exit either. So they keep the initramfs mount and with it the superblock alive for good. I overlooked this when I added nullfs. Before that the rootfs could never be unmounted anyway so it didn't matter what pinned it. Idle threads are kernel threads and init_task, the idle thread of the boot cpu, already uses init_fs. So let the idle threads of the other cpus share init_fs as well instead of copying PID 1's filesystem state. Note that kernels before 7.3 need a different backport. There PID 1 still shares init_fs with all kthreads so passing CLONE_FS in fork_idle() is enough. They also have a second pin: devtmpfsd unshares a copy of the initial mount namespace, initramfs mount included. That was fixed in commit 66d2faeccbff ("devtmpfs: create private mount namespace"). Reproducer, as /init in an initramfs together with a 128MB ballast file, booted with root=/dev/sda to get ramfs and with -smp 4: // SPDX-License-Identifier: GPL-2.0 /* initramfs /init: switch root like systemd, check the old root */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include static void meminfo(void) { char line[256]; FILE *f = fopen("/proc/meminfo", "re"); while (f && fgets(line, sizeof(line), f)) if (!strncmp(line, "Unevictable:", 12) || !strncmp(line, "Shmem:", 6)) fputs(line, stdout); if (f) fclose(f); } int main(int argc, char *argv[]) { struct stat st; char dev[32], buf[32]; unsigned int maj, min; int fd, in, out; long r; if (argc > 1) { sleep(2); meminfo(); /* ustat() only finds live superblocks */ r = syscall(SYS_ustat, (unsigned)atoi(argv[1]), buf); printf("initramfs superblock %s\n", r ? "freed" : "pinned"); fflush(stdout); sleep(1); reboot(RB_POWER_OFF); } mount("proc", "/proc", "proc", 0, NULL); mount("devtmpfs", "/dev", "devtmpfs", 0, NULL); /* drop the fds console_on_rootfs() opened in the initramfs */ fd = open("/dev/console", O_RDWR); dup2(fd, 0); dup2(fd, 1); dup2(fd, 2); close(fd); setvbuf(stdout, NULL, _IOLBF, 0); meminfo(); stat("/", &st); maj = major(st.st_dev); min = minor(st.st_dev); /* new_encode_dev(), what ustat() expects */ snprintf(dev, sizeof(dev), "%u", (min & 0xff) | (maj << 8) | ((min & ~0xffu) << 12)); mkdir("/sysroot", 0755); mount("tmpfs", "/sysroot", "tmpfs", 0, NULL); mkdir("/sysroot/proc", 0755); mkdir("/sysroot/dev", 0755); in = open("/init", O_RDONLY); out = open("/sysroot/init", O_WRONLY | O_CREAT, 0755); fstat(in, &st); sendfile(out, in, NULL, st.st_size); close(in); close(out); mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, NULL); mount("/proc", "/sysroot/proc", NULL, MS_BIND | MS_REC, NULL); mount("/dev", "/sysroot/dev", NULL, MS_BIND | MS_REC, NULL); chdir("/sysroot"); if (syscall(SYS_pivot_root, ".", ".") || umount2(".", MNT_DETACH)) perror("pivot_root"); execl("/init", "/init", dev, NULL); return 1; } Before: Unevictable: 131812 kB Shmem: 0 kB Unevictable: 131812 kB Shmem: 792 kB initramfs superblock pinned After: Unevictable: 131812 kB Shmem: 0 kB Unevictable: 0 kB Shmem: 792 kB initramfs superblock freed With -smp 1 the superblock is freed without this patch as well. [1]: https://github.com/systemd/systemd/issues/43911 Fixes: 576ee5dfd459 ("fs: add immutable rootfs") Signed-off-by: Christian Brauner (Amutable) --- kernel/fork.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index 947136cdd115..33760ece3bb3 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1621,6 +1621,7 @@ static int copy_exec_state(u64 clone_flags, struct task_struct *tsk) enum copy_fs_flags_t { COPY_FS_UMH = (1 << 0), + COPY_FS_IDLE = (1 << 1), }; static int copy_fs(u64 clone_flags, struct task_struct *tsk, @@ -1640,13 +1641,14 @@ static int copy_fs(u64 clone_flags, struct task_struct *tsk, if (current->nsproxy->mnt_ns != &init_mnt_ns) return -EINVAL; fs = userspace_init_fs; + } else if (flags & COPY_FS_IDLE) { + fs = &init_fs; } else { fs = current->fs; VFS_WARN_ON_ONCE(current->fs != current->real_fs); } if (clone_flags & CLONE_FS) { - /* tsk->fs is already what we want */ read_seqlock_excl(&fs->seq); /* "users" and "in_exec" locked for check_unsafe_exec() */ if (fs->in_exec) { @@ -1655,6 +1657,7 @@ static int copy_fs(u64 clone_flags, struct task_struct *tsk, } fs->users++; read_sequnlock_excl(&fs->seq); + tsk->real_fs = tsk->fs = fs; return 0; } tsk->real_fs = tsk->fs = copy_fs_struct(fs); @@ -2304,7 +2307,9 @@ __latent_entropy struct task_struct *copy_process( retval = copy_files(clone_flags, p, args->no_files); if (retval) goto bad_fork_cleanup_semundo; - retval = copy_fs(clone_flags, p, args->umh ? COPY_FS_UMH : 0); + retval = copy_fs(clone_flags, p, + (args->umh ? COPY_FS_UMH : 0) | + (args->idle ? COPY_FS_IDLE : 0)); if (retval) goto bad_fork_cleanup_files; retval = copy_sighand(clone_flags, p); @@ -2676,7 +2681,7 @@ struct task_struct * __init fork_idle(int cpu) { struct task_struct *task; struct kernel_clone_args args = { - .flags = CLONE_VM, + .flags = CLONE_VM | CLONE_FS, .fn = &idle_dummy, .fn_arg = NULL, .kthread = 1, -- 2.53.0