pipe_poll() unconditionally sets ->poll_usage on the first call, forcing anon_pipe_write() to wake up readers on every write even if the pipe was not empty. The reason is that some legacy epoll(EPOLLET) users depend on historical per-write wakeups, see commit 3a34b13a88ca ("pipe: make pipe writes always wake up readers"). Test-case: #include #include #include int main(void) { int pfd[2], efd; struct epoll_event evt = { .events = EPOLLIN | EPOLLET }; pipe(pfd); efd = epoll_create1(0); epoll_ctl(efd, EPOLL_CTL_ADD, pfd[0], &evt); for (int i = 0; i < 2; ++i) { write(pfd[1], "", 1); assert(epoll_wait(efd, &evt, 1, 0) == 1); } return 0; } it fails if WRITE_ONCE(poll_usage, true) is removed from pipe_poll(). However, without EPOLLET in .events, it does not need the extra wakeup and succeeds even if write() is called only once before the main loop. Currently io_uring without (unsupported) IORING_POLL_ADD_LEVEL always sets EPOLLET, and in IORING_POLL_ADD_MULTI mode it depends on per-write wakeups the same way: #include #include #include #include #include #include int main(void) { struct io_uring_params p = {}; int fd, pfd[2]; pipe(pfd); fd = syscall(SYS_io_uring_setup, 2, &p); assert(fd >= 0); void *ring = mmap(0, p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe), PROT_READ | PROT_WRITE, MAP_SHARED, fd, IORING_OFF_SQ_RING); assert(ring != MAP_FAILED); *(unsigned *)(ring + p.sq_off.tail) = 1; struct io_uring_sqe *sqes = mmap(0, p.sq_entries * sizeof(*sqes), PROT_READ | PROT_WRITE, MAP_SHARED, fd, IORING_OFF_SQES); assert(sqes != MAP_FAILED); sqes[0].opcode = IORING_OP_POLL_ADD; sqes[0].fd = pfd[0]; sqes[0].len = IORING_POLL_ADD_MULTI; sqes[0].poll32_events = EPOLLIN; syscall(SYS_io_uring_enter, fd, 1, 0, 0, 0, 0); unsigned *cq_head = ring + p.cq_off.head; unsigned *cq_tail = ring + p.cq_off.tail; for (int i = 0; i < 2; ++i) { write(pfd[1], "", 1); syscall(SYS_io_uring_enter, fd, 0, 0, IORING_ENTER_GETEVENTS, 0, 0); assert(*cq_tail == ++*cq_head); } return 0; } the 2nd assert() in the main loop fails without ->poll_usage == true. Rename ->poll_usage to ->pseudo_edgetrigger to make the purpose clearer, update the comments, and change pipe_poll() to set ->pseudo_edgetrigger only if wait->_key & EPOLLET is true. This check should catch both users, and this way poll/select and epoll without EPOLLET users will not pay for the extra wakeup. Signed-off-by: Oleg Nesterov --- fs/pipe.c | 20 +++++++++++++------- include/linux/pipe_fs_i.h | 4 ++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/fs/pipe.c b/fs/pipe.c index 429b0714ec57..84a81db39f8a 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -686,10 +686,9 @@ anon_pipe_write(struct kiocb *iocb, struct iov_iter *from) * how (for example) the GNU make jobserver uses small writes to * wake up pending jobs * - * Epoll nonsensically wants a wakeup whether the pipe - * was already empty or not. + * ->pseudo_edgetrigger enables per-write wakeups, see pipe_poll() */ - if (was_empty || pipe->poll_usage) + if (was_empty || READ_ONCE(pipe->pseudo_edgetrigger)) wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); if (wake_next_writer) @@ -752,7 +751,6 @@ static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } } -/* No kernel lock held - fine */ static __poll_t pipe_poll(struct file *filp, poll_table *wait) { @@ -760,9 +758,17 @@ pipe_poll(struct file *filp, poll_table *wait) struct pipe_inode_info *pipe = filp->private_data; union pipe_index idx; - /* Epoll has some historical nasty semantics, this enables them */ - if (unlikely(!READ_ONCE(pipe->poll_usage))) - WRITE_ONCE(pipe->poll_usage, true); + /* + * Legacy epoll(EPOLLET) users depend on historical per-write wakeups, + * see 3a34b13a88ca ("pipe: make pipe writes always wake up readers") + * and the ->pseudo_edgetrigger check in anon_pipe_write(). + * Currently io_uring sets EPOLLET for multishot polls, so it gets the + * same behaviour. + */ + if ((filp->f_mode & FMODE_READ) && + wait && (wait->_key & EPOLLET) && + unlikely(!READ_ONCE(pipe->pseudo_edgetrigger))) + WRITE_ONCE(pipe->pseudo_edgetrigger, true); /* * Reading pipe state only -- no need for acquiring the semaphore. diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index 7f6a92ac9704..1acc76581af5 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -74,7 +74,7 @@ union pipe_index { * @files: number of struct file referring this pipe (protected by ->i_lock) * @r_counter: reader counter * @w_counter: writer counter - * @poll_usage: is this pipe used for epoll, which has crazy wakeups? + * @pseudo_edgetrigger: has an EPOLLET consumer, enable per-write wakeups * @fasync_readers: reader side fasync * @fasync_writers: writer side fasync * @bufs: the circular array of pipe buffers @@ -95,7 +95,7 @@ struct pipe_inode_info { unsigned int files; unsigned int r_counter; unsigned int w_counter; - bool poll_usage; + bool pseudo_edgetrigger; #ifdef CONFIG_WATCH_QUEUE bool note_loss; #endif -- 2.52.0