The first task that is spawned into a cell is the init of its pid
namespace. When it exits, the kernel kills everything else in that
namespace, as it does for every pid namespace. That is also the end of the
cell.

Notify the cell from exit_notify(), after zap_pid_ns_processes(). The cell
records the exit status and queues a work item for the rest. The work item
kills whatever may still be in the budget, waits until the budget is no
longer populated, tears the cell down, and only then sets the state to
DEAD.

The waiting is needed because the init itself keeps the budget populated
until it has been scheduled out for the last time.

Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
 include/linux/cell.h |  7 +++++++
 kernel/cell/core.c   | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 kernel/exit.c        | 14 ++++++++++++++
 3 files changed, 69 insertions(+)

diff --git a/include/linux/cell.h b/include/linux/cell.h
index a68638b7319d..86999efc083d 100644
--- a/include/linux/cell.h
+++ b/include/linux/cell.h
@@ -228,4 +228,11 @@ static inline bool embryo_pidfd_is_empty(const struct file *file) { return false
 static inline void embryo_pidfd_release(struct file *file) { }
 #endif
 
+/* The end of a cell. */
+#ifdef CONFIG_CELLS
+void cell_init_exited(struct cell *cell, struct task_struct *tsk);
+#else
+static inline void cell_init_exited(struct cell *cell, struct task_struct *tsk) { }
+#endif
+
 #endif /* _LINUX_CELL_H */
diff --git a/kernel/cell/core.c b/kernel/cell/core.c
index b9494b5fd7b9..c780094ad794 100644
--- a/kernel/cell/core.c
+++ b/kernel/cell/core.c
@@ -172,6 +172,53 @@ static void cell_kill_workfn(struct work_struct *work)
 	cell_put(cell);
 }
 
+static void cell_exit_workfn(struct work_struct *work)
+{
+	struct cell *cell = container_of(to_delayed_work(work), struct cell,
+					 exit_work);
+
+	scoped_guard(mutex, &cell->lock) {
+		if (cell->state == CELL_DEAD)
+			break;
+		WRITE_ONCE(cell->state, CELL_DYING);
+		/*
+		 * DEAD means gone. zap_pid_ns_processes() has reaped everything
+		 * else in the pid namespace, but the init itself is still on
+		 * its way out. It populates the budget until it is off the cpu
+		 * for good. Come back until then.
+		 */
+		if (cgroup_cell_kill(cell->budget)) {
+			queue_delayed_work(system_dfl_wq, &cell->exit_work,
+					   max(HZ / 100, 1));
+			return;
+		}
+		cell_teardown(cell);
+	}
+	cell_put(cell);
+}
+
+/**
+ * cell_init_exited - the init of the pid namespace of a cell is gone
+ * @cell: the cell
+ * @tsk:  the last thread of that init, which is the caller
+ *
+ * With its init everything else in the pid namespace is gone, and so the cell.
+ * This notes the exit status and owes the teardown to a work item.
+ *
+ * Context: The exit path of @tsk, after zap_pid_ns_processes(), with no locks
+ * held. The cell has not been torn down, since only this sets that off once a
+ * cell has run anything.
+ */
+void cell_init_exited(struct cell *cell, struct task_struct *tsk)
+{
+	cell->exit_status = (tsk->signal->flags & SIGNAL_GROUP_EXIT) ?
+			    tsk->signal->group_exit_code :
+			    tsk->group_leader->exit_code;
+	cell_get(cell);
+	if (!queue_delayed_work(system_dfl_wq, &cell->exit_work, 0))
+		cell_put(cell);
+}
+
 /* The cellfd. */
 
 static int cell_file_open(struct inode *inode, struct file *file)
@@ -286,6 +333,7 @@ SYSCALL_DEFINE2(cell_create, int, parent, unsigned int, flags)
 	mutex_init(&cell->lock);
 	init_waitqueue_head(&cell->wait);
 	INIT_WORK(&cell->kill_work, cell_kill_workfn);
+	INIT_DELAYED_WORK(&cell->exit_work, cell_exit_workfn);
 	strscpy(cell->hostname, "(none)");
 
 	/* The scratch credential is only here to carry the new namespace. */
diff --git a/kernel/exit.c b/kernel/exit.c
index d5e7bc2af804..1ae82e7ac29f 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/mm.h>
+#include <linux/cell.h>
 #include <linux/slab.h>
 #include <linux/sched/autogroup.h>
 #include <linux/sched/mm.h>
@@ -741,11 +742,20 @@ static void forget_original_parent(struct task_struct *father,
  */
 static void exit_notify(struct task_struct *tsk, int group_dead)
 {
+	struct cell *cell = NULL;
 	bool autoreap;
 	struct task_struct *p, *n;
 	LIST_HEAD(dead);
 
 	write_lock_irq(&tasklist_lock);
+	/*
+	 * Is this the end of the init of a cell? Asked of the pid namespace
+	 * and not of the task, and asked now: once the lock is dropped the
+	 * parent may reap us, pids and all. The cell holds on to its pid
+	 * namespace until it is torn down, which only this sets off.
+	 */
+	if (group_dead && is_child_reaper(task_tgid(tsk)))
+		cell = userns_cell(ns_of_pid(task_tgid(tsk))->user_ns);
 	forget_original_parent(tsk, &dead);
 
 	if (group_dead)
@@ -776,6 +786,10 @@ static void exit_notify(struct task_struct *tsk, int group_dead)
 		wake_up_process(tsk->signal->group_exec_task);
 	write_unlock_irq(&tasklist_lock);
 
+	/* With its init, and so everything else in it, gone, so is a cell. */
+	if (cell)
+		cell_init_exited(cell, tsk);
+
 	list_for_each_entry_safe(p, n, &dead, ptrace_entry) {
 		list_del_init(&p->ptrace_entry);
 		release_task(p);

-- 
2.55.0