From 97f6b8d41cda1693b9319c7c2bbfa045f1b261b2 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 15 Oct 2026 19:35:12 +0200 Subject: [PATCH 1/6] fuse: warn when invalidation or folio laundering fails fuse_reverse_inval_inode() ignored the result of invalidate_inode_pages2_range(), so folios left in the page cache after a notify went unnoticed. Log a rate-limited warning when the invalidate fails, and when the write in fuse_launder_folio() fails. --- fs/fuse/file.c | 4 ++++ fs/fuse/inode.c | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 021abd8d7f052f..bc7cbb0e74670f 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3438,6 +3438,10 @@ static int fuse_launder_folio(struct folio *folio) err = fuse_iomap_writeback_submit(&wpc, err); if (!err) folio_wait_writeback(folio); + else + pr_warn_ratelimited("fuse: launder write failed, nodeid %llu pos %lld: %d\n", + get_node_id(wpc.inode), + folio_pos(folio), err); } return err; } diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 9d57846259056e..56cf6d2b8fded6 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1030,6 +1030,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, fc->writeback_cache && fc->dlm; bool hot = false, has_writer = false, latched = false; bool track_latch; + int ierr; pg_start = offset >> PAGE_SHIFT; if (len <= 0) @@ -1149,10 +1150,14 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * notified range. */ if (fuse_inode_force_dio(inode)) - invalidate_inode_pages2_range(inode->i_mapping, 0, -1); + ierr = invalidate_inode_pages2_range(inode->i_mapping, + 0, -1); else - invalidate_inode_pages2_range(inode->i_mapping, - pg_start, pg_end); + ierr = invalidate_inode_pages2_range(inode->i_mapping, + pg_start, pg_end); + if (ierr) + pr_warn_ratelimited("FUSE: inode %llu invalidate failed, folios left in page cache: %d\n", + nodeid, ierr); if (latched) pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", From ec2431cba592bc5d474cb7f0cfd5cf83685a2e03 Mon Sep 17 00:00:00 2001 From: Sven Oehme Date: Fri, 16 Oct 2026 00:15:15 +0000 Subject: [PATCH 2/6] fuse: do not send the zeroed EOF folio tail in iomap writeback iomap_writeback_handle_eof() zeroes the folio straddling i_size past that size before ->writeback_range() runs, and passes the size on as end_pos. fuse_iomap_writeback_range() ignored end_pos and queued the whole block, relying on the crop in fuse_send_writepage() to drop the bytes past EOF. That crop reads i_size again when the request is sent. Under DLM with the writeback cache, i_size is only this node's view: each writer claims its own extension under fi->lock while the inode lock is held shared. A partial EOF folio can hold another node's bytes, read in by the read-modify-write read under the write grant. If a concurrent local writer extends i_size between the zeroing and the crop, the crop no longer trims the folio and the zeros are written over the other node's acknowledged data. RED-51747 hits this with three nodes writing interleaved 47008-byte blocks of one file: the higher block's head reads back as zeros up to the page boundary. Traces show the lower writer's flush leaving with "own block + zeros" while the page it read from the server, and the i_size it zeroed at, were correct; a higher-offset write had extended i_size 40 us before the request was sent. Send only up to end_pos and still report the whole range as handled, so the bytes iomap zeroed never reach the server. The 6.12 tree avoided this because its writeback crops at i_size without zeroing. Signed-off-by: Sven Oehme Co-Authored-By: Claude Opus 5.5 --- fs/fuse/file.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index bc7cbb0e74670f..617234e0a569d4 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3322,9 +3322,18 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); loff_t offset = offset_in_folio(folio, pos); + unsigned int send_len = len; WARN_ON_ONCE(!data); + /* + * iomap_writeback_handle_eof() zeroed this folio past end_pos (the local + * i_size). Under DLM that tail can hold another node's data, and i_size + * may grow before the send crop runs, so never send past end_pos. + */ + if (pos + len > end_pos) + send_len = end_pos - pos; + if (!data->ff) { data->ff = fuse_write_file_get(fi); if (!data->ff) @@ -3332,7 +3341,7 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, } if (wpa) { - bool send = fuse_folios_need_send(fc, pos, len, ap, + bool send = fuse_folios_need_send(fc, pos, send_len, ap, data->nr_bytes, true); if (!send) { @@ -3346,7 +3355,7 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, if (!send) send = fuse_writeback_reached_alignment(fc, pos, - data->nr_bytes + len, wpc->wbc); + data->nr_bytes + send_len, wpc->wbc); if (send) { fuse_writepages_send(inode, data); @@ -3365,8 +3374,8 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, } fuse_writepage_args_page_fill(wpa, folio, ap->num_folios, - offset, len); - data->nr_bytes += len; + offset, send_len); + data->nr_bytes += send_len; ap->num_folios++; if (!data->wpa) From a8c7c7d3fc2a1a66d48c1f048f2172d4863d475b Mon Sep 17 00:00:00 2001 From: Yong Ze Chen Date: Fri, 16 Oct 2026 06:19:42 +0000 Subject: [PATCH 3/6] fuse: raise i_size over bytes a DLM read reply returned Under DLM with the writeback cache, a node's i_size only covers its own writes. When two nodes share a page at a block boundary, the lower writer's read before its partial write returns the upper writer's bytes from the server, but they lie past the local i_size. iomap_writeback_handle_eof() then zeroes them in the cached folio as if they were past EOF, the folio stays uptodate, and if i_size grows before the send is cropped the zeroes overwrite the other node's data on the server (RED-51747). A read reply proves the file extends at least to pos + res, since the server clips reads at its size. Raise i_size to cover those bytes in fuse_do_readfolio() and fuse_readpages_end(), so the folio tail is never treated as past EOF. The raise must not resurrect bytes a truncate removed while the read was in flight. attr_version cannot guard this, because every revoke bumps it, including the revoke of the range being read. Add fi->size_gen instead, bumped only by a local truncate and by a whole-file invalidate, which is how a remote truncate reaches the client, and drop the raise when it changed. Co-Authored-By: Claude Opus 5.5 --- fs/fuse/dir.c | 14 ++++++++++++++ fs/fuse/file.c | 41 +++++++++++++++++++++++++++++++++++++++++ fs/fuse/fuse_i.h | 14 ++++++++++++++ fs/fuse/inode.c | 6 ++++++ 4 files changed, 75 insertions(+) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 80e32e49bcdc8b..f2f25397f1320d 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2595,7 +2595,11 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, } if (is_truncate) { + /* Drop read replies that may predate the truncate. */ + spin_lock(&fi->lock); + fuse_bump_size_gen(fi); set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); + spin_unlock(&fi->lock); if (trust_local_cmtime && attr->ia_size != inode->i_size) attr->ia_valid |= ATTR_MTIME | ATTR_CTIME; } @@ -2692,6 +2696,11 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, fuse_range_lock_release(fi, &rlock); } + if (is_truncate) { + spin_lock(&fi->lock); + fuse_bump_size_gen(fi); + spin_unlock(&fi->lock); + } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); out: if (fault_blocked) @@ -2700,6 +2709,11 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, return 0; error: + if (is_truncate) { + spin_lock(&fi->lock); + fuse_bump_size_gen(fi); + spin_unlock(&fi->lock); + } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); if (fault_blocked) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 617234e0a569d4..6a63f598208f4b 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1020,6 +1020,36 @@ static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read, } } +/* + * Under DLM with the writeback cache, i_size covers only this node's own + * writes. A read reply proves the file extends at least to the bytes it + * returned (the server clips reads at its size), and they may be another + * node's data in a page this node shares. Raise i_size over them, so that + * iomap_writeback_handle_eof() never zeroes them as if they were past EOF. + * + * Every revoke bumps attr_version, including the revoke of the very range + * being read, so attr_version cannot gate this. size_gen moves only for a + * local truncate or a whole-file invalidate, which is how a remote truncate + * reaches us; a reply that may predate either is dropped. + */ +static void fuse_dlm_read_extend_size(struct inode *inode, loff_t end, + u64 size_gen) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + + if (!fc->dlm || !fc->writeback_cache) + return; + + spin_lock(&fi->lock); + if (fi->size_gen == size_gen && end > inode->i_size && + !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) { + fi->attr_version = atomic64_inc_return(&fc->attr_version); + i_size_write(inode, end); + } + spin_unlock(&fi->lock); +} + static int fuse_do_readfolio(struct file *file, struct folio *folio, size_t off, size_t len) { @@ -1040,8 +1070,10 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, }; ssize_t res; u64 attr_ver; + u64 size_gen; attr_ver = fuse_get_attr_version(fm->fc); + size_gen = READ_ONCE(get_fuse_inode(inode)->size_gen); /* Don't overflow end offset */ if (pos + (desc.length - 1) == LLONG_MAX) @@ -1069,6 +1101,9 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, if (res < desc.length) fuse_short_read(inode, attr_ver, res, &ia.ap); + if (res > 0) + fuse_dlm_read_extend_size(inode, pos + res, size_gen); + return 0; } @@ -1300,6 +1335,11 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, if (!err && num_read < count) fuse_short_read(inode, ia->read.attr_ver, num_read, ap); + if (!err && num_read) + fuse_dlm_read_extend_size(inode, + ia->read.in.offset + num_read, + ia->read.size_gen); + fuse_invalidate_atime(inode); for (i = 0; i < ap->num_folios; i++) { @@ -1337,6 +1377,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, fuse_read_args_fill(ia, file, pos, count, FUSE_READ); ia->read.attr_ver = fuse_get_attr_version(fm->fc); + ia->read.size_gen = READ_ONCE(get_fuse_inode(file_inode(file))->size_gen); if (async) { ia->ff = fuse_file_get(ff); ap->args.end = fuse_readpages_end; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 1c07c28abb9c6a..c92e16d1b2d9da 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -200,6 +200,13 @@ struct fuse_inode { /** Version of last attribute change */ u64 attr_version; + /* + * Bumped under fi->lock whenever the file may have shrunk behind a + * read in flight: a local truncate, or a whole-file invalidate (the + * shape a remote truncate takes). See fuse_dlm_read_extend_size(). + */ + u64 size_gen; + union { /* read/write io cache (regular file only) */ struct { @@ -1256,6 +1263,12 @@ static inline u64 fuse_get_attr_version(struct fuse_conn *fc) return atomic64_read(&fc->attr_version); } +static inline void fuse_bump_size_gen(struct fuse_inode *fi) +{ + lockdep_assert_held(&fi->lock); + WRITE_ONCE(fi->size_gen, fi->size_gen + 1); +} + static inline u64 fuse_get_evict_ctr(struct fuse_conn *fc) { return atomic64_read(&fc->evict_ctr); @@ -1348,6 +1361,7 @@ struct fuse_io_args { struct { struct fuse_read_in in; u64 attr_ver; + u64 size_gen; } read; struct { struct fuse_write_in in; diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 56cf6d2b8fded6..06699b0e7f3df4 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1015,6 +1015,12 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, fi = get_fuse_inode(inode); spin_lock(&fi->lock); fi->attr_version = atomic64_inc_return(&fc->attr_version); + /* + * A remote truncate reaches us as a whole-file invalidate; read + * replies in flight may predate it (see fuse_dlm_read_extend_size()). + */ + if (offset <= 0 && len <= 0) + fuse_bump_size_gen(fi); spin_unlock(&fi->lock); if (fc->inval_inode_entries) From 51698eca627799fb9d67e346e2b74395cf0b4145 Mon Sep 17 00:00:00 2001 From: Sven Oehme Date: Fri, 16 Oct 2026 06:29:06 +0000 Subject: [PATCH 4/6] fuse: add tracepoints for i_size changes, writeback ranges and READ replies Under DLM with the writeback cache, i_size is only this node's view of the file, and whether writeback treats bytes as past EOF depends on what i_size was at that exact moment. When RED-51747 lost another node's bytes to zeroes, the local i_size at writeback had to be reconstructed from the order of DLM grants, because nothing records it. Add three tracepoints: fuse_isize_change every i_size change and why: the cached-write extension claim, its retraction, or a size taken from an attribute reply fuse_writeback_range each writeback chunk with pos, len, bytes sent, iomap's end_pos and the current i_size fuse_readfolio_reply each fuse_do_readfolio() READ with the length asked, the bytes returned and i_size With these, the RED-51747 race shows up directly as a writeback chunk whose end_pos is below i_size while its whole block is sent, preceded by a READ reply that returned bytes past the local EOF. Signed-off-by: Sven Oehme Co-Authored-By: Claude Opus 5.5 --- fs/fuse/file.c | 11 ++++- fs/fuse/fuse_i.h | 6 +++ fs/fuse/fuse_trace.h | 108 +++++++++++++++++++++++++++++++++++++++++++ fs/fuse/inode.c | 7 ++- 4 files changed, 130 insertions(+), 2 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 6a63f598208f4b..77c15932603f25 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -8,6 +8,7 @@ #include "fuse_i.h" #include "fuse_dlm_cache.h" +#include "fuse_trace.h" #include #include @@ -1081,6 +1082,7 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); res = fuse_simple_request(fm, &ia.ap.args); + trace_fuse_readfolio_reply(inode, pos, desc.length, res); if (res < 0) { /* * Please refer to Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt @@ -2573,6 +2575,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) orig_size = i_size_read(inode); if (end > orig_size) { i_size_write(inode, end); + trace_fuse_isize_change(inode, orig_size, end, + FUSE_ISIZE_WRITE_CLAIM); extended = true; } spin_unlock(&fi->lock); @@ -2597,8 +2601,11 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (reached < end) { spin_lock(&fi->lock); - if (i_size_read(inode) == end) + if (i_size_read(inode) == end) { i_size_write(inode, reached); + trace_fuse_isize_change(inode, end, reached, + FUSE_ISIZE_WRITE_RETRACT); + } spin_unlock(&fi->lock); } } @@ -3375,6 +3382,8 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, if (pos + len > end_pos) send_len = end_pos - pos; + trace_fuse_writeback_range(inode, pos, len, end_pos, send_len); + if (!data->ff) { data->ff = fuse_write_file_get(fi); if (!data->ff) diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index c92e16d1b2d9da..8ee68d2a46d5e7 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1238,6 +1238,12 @@ static inline struct fuse_mount *get_fuse_mount(struct inode *inode) return get_fuse_mount_super(inode->i_sb); } +enum fuse_isize_reason { + FUSE_ISIZE_WRITE_CLAIM, + FUSE_ISIZE_WRITE_RETRACT, + FUSE_ISIZE_ATTR, +}; + static inline struct fuse_conn *get_fuse_conn(struct inode *inode) { return get_fuse_mount_super(inode->i_sb)->fc; diff --git a/fs/fuse/fuse_trace.h b/fs/fuse/fuse_trace.h index e81c93b9614627..5d5045354fec37 100644 --- a/fs/fuse/fuse_trace.h +++ b/fs/fuse/fuse_trace.h @@ -150,6 +150,114 @@ TRACE_EVENT(fuse_request_end, __entry->unique, __entry->len, __entry->error) ); +#define FUSE_ISIZE_REASONS \ + EM( FUSE_ISIZE_WRITE_CLAIM, "write_claim") \ + EM( FUSE_ISIZE_WRITE_RETRACT, "write_retract")\ + EMe(FUSE_ISIZE_ATTR, "attr") + +#undef EM +#undef EMe +#define EM(a, b) TRACE_DEFINE_ENUM(a); +#define EMe(a, b) TRACE_DEFINE_ENUM(a); +FUSE_ISIZE_REASONS +#undef EM +#undef EMe +#define EM(a, b) { a, b }, +#define EMe(a, b) { a, b } + +/* + * Under DLM i_size is this node's view only; recording every change with its + * reason is what lets a lost update be tied to the local EOF at writeback. + */ +TRACE_EVENT(fuse_isize_change, + TP_PROTO(struct inode *inode, loff_t old_size, loff_t new_size, + int reason), + + TP_ARGS(inode, old_size, new_size, reason), + + TP_STRUCT__entry( + __field(dev_t, connection) + __field(uint64_t, nodeid) + __field(loff_t, old_size) + __field(loff_t, new_size) + __field(int, reason) + ), + + TP_fast_assign( + __entry->connection = get_fuse_conn(inode)->dev; + __entry->nodeid = get_fuse_inode(inode)->nodeid; + __entry->old_size = old_size; + __entry->new_size = new_size; + __entry->reason = reason; + ), + + TP_printk("connection %u nodeid %llu i_size %lld -> %lld (%s)", + __entry->connection, __entry->nodeid, __entry->old_size, + __entry->new_size, + __print_symbolic(__entry->reason, FUSE_ISIZE_REASONS)) +); + +TRACE_EVENT(fuse_writeback_range, + TP_PROTO(struct inode *inode, u64 pos, unsigned int len, + u64 end_pos, unsigned int send_len), + + TP_ARGS(inode, pos, len, end_pos, send_len), + + TP_STRUCT__entry( + __field(dev_t, connection) + __field(uint64_t, nodeid) + __field(uint64_t, pos) + __field(uint32_t, len) + __field(uint32_t, send_len) + __field(uint64_t, end_pos) + __field(loff_t, i_size) + ), + + TP_fast_assign( + __entry->connection = get_fuse_conn(inode)->dev; + __entry->nodeid = get_fuse_inode(inode)->nodeid; + __entry->pos = pos; + __entry->len = len; + __entry->send_len = send_len; + __entry->end_pos = end_pos; + __entry->i_size = i_size_read(inode); + ), + + TP_printk("connection %u nodeid %llu pos %llu len %u send %u end_pos %llu i_size %lld", + __entry->connection, __entry->nodeid, __entry->pos, + __entry->len, __entry->send_len, __entry->end_pos, + __entry->i_size) +); + +TRACE_EVENT(fuse_readfolio_reply, + TP_PROTO(struct inode *inode, loff_t pos, size_t len, + ssize_t res), + + TP_ARGS(inode, pos, len, res), + + TP_STRUCT__entry( + __field(dev_t, connection) + __field(uint64_t, nodeid) + __field(loff_t, pos) + __field(size_t, len) + __field(ssize_t, res) + __field(loff_t, i_size) + ), + + TP_fast_assign( + __entry->connection = get_fuse_conn(inode)->dev; + __entry->nodeid = get_fuse_inode(inode)->nodeid; + __entry->pos = pos; + __entry->len = len; + __entry->res = res; + __entry->i_size = i_size_read(inode); + ), + + TP_printk("connection %u nodeid %llu pos %lld len %zu res %zd i_size %lld", + __entry->connection, __entry->nodeid, __entry->pos, + __entry->len, __entry->res, __entry->i_size) +); + #endif /* _TRACE_FUSE_H */ #undef TRACE_INCLUDE_PATH diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 06699b0e7f3df4..78dc270ecab73e 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -10,6 +10,7 @@ #include "fuse_dlm_cache.h" #include "fuse_dev_i.h" #include "dev_uring_i.h" +#include "fuse_trace.h" #include #include @@ -684,8 +685,12 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr * attr->size coming from server can be stale. We cannot trust it. * Only update i_size if SIZE was actually returned by the server. */ - if (have_size && !(cache_mask & STATX_SIZE)) + if (have_size && !(cache_mask & STATX_SIZE)) { i_size_write(inode, attr->size); + if (oldsize != attr->size) + trace_fuse_isize_change(inode, oldsize, attr->size, + FUSE_ISIZE_ATTR); + } spin_unlock(&fi->lock); /* From b523bff1871ef0eddc1d1a0ac23a178c8415386c Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 16 Oct 2026 13:11:36 +0200 Subject: [PATCH 5/6] fuse: bound the attribute timeout of an inode a DLM lock was taken on Under DLM the server's attr_valid can keep the attributes of a locked inode valid indefinitely. Once a DLM lock was taken on an inode, cap its i_time to dlm_attr_timeout seconds so a GETATTR goes out at most that long after the lock. The first attribute reply from the server after the lock ends the bound; a reply to a request sent before it is discarded by bumping attr_version. Default 30s, module parameter, 0 keeps the server's timeout. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 2 ++ fs/fuse/fuse_dlm_cache.c | 2 ++ fs/fuse/fuse_i.h | 9 +++++++++ fs/fuse/inode.c | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 77c15932603f25..957ec0df917e06 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3569,6 +3569,8 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l fuse_abort_conn(fc); err = -EINVAL; } + if (!err && fc->dlm) + fuse_dlm_attr_timeout(get_fuse_inode(inode)); return err; } diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index c32ece043ba9bb..deb7181e04a8b8 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -830,6 +830,8 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, if (err) return err; + fuse_dlm_attr_timeout(fi); + if (inarg.start < outarg.start || inarg.end > outarg.end) { /* fuse server is seriously broken */ pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 8ee68d2a46d5e7..c5b78819284e47 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -350,6 +350,12 @@ enum { * See fuse_reverse_inval_inode()/fuse_file_io_open(). */ FUSE_I_FORCE_DIO, + /* + * A DLM lock was taken on this inode and no attribute reply has + * arrived since: i_time is bounded to dlm_attr_timeout seconds from + * the lock, see fuse_dlm_attr_timeout(). + */ + FUSE_I_DLM_LOCKED, }; struct fuse_conn; @@ -1656,6 +1662,9 @@ struct inode *fuse_ilookup(struct fuse_conn *fc, u64 nodeid, int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len); +/* A DLM lock was taken on @fi: bound its attribute timeout until the next reply */ +void fuse_dlm_attr_timeout(struct fuse_inode *fi); + /** * File-system tells the kernel to invalidate parent attributes and * the dentry matching parent/name. diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 78dc270ecab73e..479fee45410426 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -55,6 +55,39 @@ module_param(enable_notify_dio, bool, 0644); MODULE_PARM_DESC(enable_notify_dio, "Latch a contended inode to direct IO on an invalidation notify storm"); +/* + * Attribute timeout of an inode once a DLM lock was taken on it, in + * seconds: i_time is bounded to it until the next attribute reply from + * the server, so a GETATTR goes out at most this long after the lock. + * 0 leaves the server's timeout alone. + */ +static unsigned int __read_mostly dlm_attr_timeout = 30; +module_param(dlm_attr_timeout, uint, 0644); +MODULE_PARM_DESC(dlm_attr_timeout, + "Seconds until the attributes of an inode a DLM lock was taken on are fetched again (0 = server's timeout)"); + +void fuse_dlm_attr_timeout(struct fuse_inode *fi) +{ + struct fuse_conn *fc = get_fuse_conn(&fi->inode); + unsigned int timeout = READ_ONCE(dlm_attr_timeout); + u64 valid; + + if (!timeout) + return; + valid = get_jiffies_64() + secs_to_jiffies(timeout); + + spin_lock(&fi->lock); + /* + * Only a reply to a request sent after the lock may end the bound: + * one to an earlier request is discarded by fuse_change_attributes_i(). + */ + if (!test_and_set_bit(FUSE_I_DLM_LOCKED, &fi->state)) + fi->attr_version = atomic64_inc_return(&fc->attr_version); + if (time_after64(fi->i_time, valid)) + fi->i_time = valid; + spin_unlock(&fi->lock); +} + static struct kmem_cache *fuse_inode_cachep; struct list_head fuse_conn_list; DEFINE_MUTEX(fuse_mutex); @@ -402,6 +435,9 @@ void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, lockdep_assert_held(&fi->lock); + /* The first attribute reply after a DLM lock ends the bounded timeout */ + clear_bit(FUSE_I_DLM_LOCKED, &fi->state); + if (sx) { return fuse_change_attributes_common_sx(inode, attr, sx, attr_valid, cache_mask, From 89f712f62f0826553c41593300a9ace49648890c Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Wed, 21 Oct 2026 10:36:31 +0200 Subject: [PATCH 6/6] fuse: select per-mode, DAX and passthrough operations at inode init and open Split file, address space, inode and vm operations into wt and wb variants chosen in fuse_init_file_inode()/fuse_init_common() from fc->writeback_cache. Drop fc->dlm, now unread; -ENOSYS to a DLM request aborts the connection. DAX is fixed per inode, passthrough per open. Install DAX file operations from fuse_dax_inode_init() and switch a passthrough open to passthrough file operations in fuse_finish_open(), so the cached read/write/mmap/splice paths no longer test for either. Each keeps the open, flush and release of its cache mode. Split fuse_do_setattr() the same way: fuse_wt_do_setattr() and fuse_wb_do_setattr() share the checks, the request and the exit, and the writeback variant truncates through fuse_wb_truncate_pagecache(). A failed write_inode_now() before the request no longer leaks the DAX invalidate lock. Only a read without a grant, the writethrough read_folio, is bounced by the server with -EDEADLK/-EAGAIN. fuse_wt_read_folio() reads outside iomap and returns AOP_TRUNCATED_PAGE after the unlock, which iomap_read_folio() cannot pass on; fuse_do_readfolio() returns the server's code unchanged. The writeback cache reads under a grant, so the dlm_retry_tasks retry of the iomap write path goes. Signed-off-by: Horst Birthelmer --- fs/fuse/dax.c | 10 + fs/fuse/dir.c | 492 ++++++---- fs/fuse/file.c | 1888 ++++++++++++++++++++++---------------- fs/fuse/fuse_dlm_cache.c | 20 +- fs/fuse/fuse_dlm_cache.h | 4 +- fs/fuse/fuse_i.h | 31 +- fs/fuse/inode.c | 71 +- fs/fuse/passthrough.c | 2 +- 8 files changed, 1492 insertions(+), 1026 deletions(-) diff --git a/fs/fuse/dax.c b/fs/fuse/dax.c index f6cf00a8938cc7..ff0dbba01373a8 100644 --- a/fs/fuse/dax.c +++ b/fs/fuse/dax.c @@ -678,6 +678,9 @@ ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to) struct inode *inode = file_inode(iocb->ki_filp); ssize_t ret; + if (fuse_is_bad(inode)) + return -EIO; + if (iocb->ki_flags & IOCB_NOWAIT) { if (!inode_trylock_shared(inode)) return -EAGAIN; @@ -718,6 +721,9 @@ ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) struct inode *inode = file_inode(iocb->ki_filp); ssize_t ret; + if (fuse_is_bad(inode)) + return -EIO; + if (iocb->ki_flags & IOCB_NOWAIT) { if (!inode_trylock(inode)) return -EAGAIN; @@ -1317,10 +1323,14 @@ static bool fuse_should_enable_dax(struct inode *inode, unsigned int flags) void fuse_dax_inode_init(struct inode *inode, unsigned int flags) { + struct fuse_conn *fc = get_fuse_conn(inode); + if (!fuse_should_enable_dax(inode, flags)) return; inode->i_flags |= S_DAX; + inode->i_fop = fc->writeback_cache ? &fuse_wb_dax_file_operations : + &fuse_wt_dax_file_operations; inode->i_data.a_ops = &fuse_dax_file_aops; } diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index f2f25397f1320d..f45899119f9c2f 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -1833,7 +1833,6 @@ static int fuse_update_get_attr(struct mnt_idmap *idmap, struct inode *inode, int err = 0; bool sync; u32 inval_mask = READ_ONCE(fi->inval_mask); - u32 cache_mask = fuse_get_cache_mask(inode); u32 mandatory_sx_mask = request_mask & STATX_BASIC_STATS; u32 sx_mask = request_mask; @@ -1849,7 +1848,7 @@ static int fuse_update_get_attr(struct mnt_idmap *idmap, struct inode *inode, sync = true; else if (flags & AT_STATX_DONT_SYNC) sync = false; - else if (sx_mask & inval_mask & ~cache_mask) + else if (sx_mask & inval_mask) sync = true; else sync = time_before64(fi->i_time, get_jiffies_64()); @@ -2483,132 +2482,68 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) * may fail, in which case we don't want to touch the mapping. * vmtruncate() doesn't allow for this case, so do the rlimit checking * and the actual truncation by hand. + * + * fuse_wt_do_setattr() and fuse_wb_do_setattr() are the two cache modes + * around the shared checks, request and exit below. */ -int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, - struct iattr *attr, struct file *file) + +/* + * The VFS checks and, before a DAX truncate, the layout break under the + * invalidate lock; *fault_blocked tells the caller to drop that lock. + */ +static int fuse_setattr_begin(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr, bool *fault_blocked) { struct inode *inode = d_inode(dentry); - struct fuse_mount *fm = get_fuse_mount(inode); - struct fuse_conn *fc = fm->fc; - struct fuse_inode *fi = get_fuse_inode(inode); - struct address_space *mapping = inode->i_mapping; - FUSE_ARGS(args); - struct fuse_setattr_in inarg; - struct fuse_attr_out outarg; - bool is_truncate = false; - bool is_wb = fc->writeback_cache && S_ISREG(inode->i_mode); - loff_t oldsize; int err; - bool trust_local_cmtime = is_wb; - bool fault_blocked = false; - u64 attr_version; - if (!fc->default_permissions) + if (!get_fuse_conn(inode)->default_permissions) attr->ia_valid |= ATTR_FORCE; err = setattr_prepare(idmap, dentry, attr); if (err) return err; - if (attr->ia_valid & ATTR_SIZE) { - if (WARN_ON(!S_ISREG(inode->i_mode))) - return -EIO; - is_truncate = true; - } + if (!(attr->ia_valid & ATTR_SIZE)) + return 0; + + if (WARN_ON(!S_ISREG(inode->i_mode))) + return -EIO; - if (FUSE_IS_DAX(inode) && is_truncate) { - filemap_invalidate_lock(mapping); - fault_blocked = true; + if (FUSE_IS_DAX(inode)) { + filemap_invalidate_lock(inode->i_mapping); err = fuse_dax_break_layouts(inode, 0, -1); if (err) { - filemap_invalidate_unlock(mapping); + filemap_invalidate_unlock(inode->i_mapping); return err; } + *fault_blocked = true; } - if (attr->ia_valid & ATTR_OPEN) { - /* This is coming from open(..., ... | O_TRUNC); */ - WARN_ON(!(attr->ia_valid & ATTR_SIZE)); - WARN_ON(attr->ia_size != 0); - if (fc->atomic_o_trunc) { - struct fuse_range_lock rlock; - bool range_locked = fc->writeback_cache && fc->dlm; - - /* - * No need to send request to userspace, since actual - * truncation has already been done by OPEN. But still - * need to truncate page cache. - * - * Revoke and drop under the full-range IO range lock, - * like the NOTIFY invalidate path - * (fuse_reverse_inval_inode()): a reader/writer that - * already reached LOCKED state must not have the lock - * tree and the cache yanked mid-hold, or it would - * repopulate the truncated range trusting a grant that - * no longer exists. fuse_range_lock_acquire_locked() - * ignores an overlapping INIT range (a read/write with - * only a DLM request in flight), so waiting here is - * bounded. Blocking is also safe: we hold i_rwsem - * exclusive, so no cached writer can be waiting on this - * range lock (the write path takes i_rwsem before it, - * the read path never takes i_rwsem at all). Only - * meaningful under DLM with the writeback cache; see - * fuse_range_lock.h. - */ - if (range_locked) - fuse_range_lock_acquire_locked(fi, &rlock, 0, ~0ULL, - FUSE_RANGE_LOCK_WRITE); - if (fc->dlm && fc->writeback_cache) - fuse_dlm_cache_release_locks(fi); - spin_lock(&fi->lock); - i_size_write(inode, 0); - spin_unlock(&fi->lock); - truncate_pagecache(inode, 0); - if (range_locked) - fuse_range_lock_release(fi, &rlock); - goto out; - } - file = NULL; - } - - /* Flush dirty data/metadata before non-truncate SETATTR */ - if (is_wb && - attr->ia_valid & - (ATTR_MODE | ATTR_UID | ATTR_GID | ATTR_MTIME_SET | - ATTR_TIMES_SET)) { - err = write_inode_now(inode, true); - if (err) - return err; - } - - /* - * Write dirty folios back before a truncating SETATTR, like the - * atomic-O_TRUNC case in fuse_open(): under DLM the server's - * truncate revokes this client's locks, and the invalidate that - * follows would otherwise launder dirty folios with writes the - * server holds behind the same truncate. - */ - if (is_truncate && is_wb && fc->dlm) { - err = filemap_write_and_wait(mapping); - if (err) - goto error; - } + return 0; +} - if (is_truncate) { - /* Drop read replies that may predate the truncate. */ - spin_lock(&fi->lock); - fuse_bump_size_gen(fi); - set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); - spin_unlock(&fi->lock); - if (trust_local_cmtime && attr->ia_size != inode->i_size) - attr->ia_valid |= ATTR_MTIME | ATTR_CTIME; - } +/* + * Build, send and validate the SETATTR. With @trust_local_cmtime the + * times in @attr are the kernel's and are not replaced by the server's. + */ +static int fuse_setattr_send(struct mnt_idmap *idmap, struct inode *inode, + struct iattr *attr, struct file *file, + bool trust_local_cmtime, + struct fuse_attr_out *outarg, u64 *attr_version) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_conn *fc = fm->fc; + FUSE_ARGS(args); + struct fuse_setattr_in inarg; + int err; memset(&inarg, 0, sizeof(inarg)); - memset(&outarg, 0, sizeof(outarg)); + memset(outarg, 0, sizeof(*outarg)); iattr_to_fattr(idmap, fc, attr, &inarg, trust_local_cmtime); if (file) { struct fuse_file *ff = file->private_data; + inarg.valid |= FATTR_FH; inarg.fh = ff->fh; } @@ -2628,32 +2563,127 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, inarg.valid |= FATTR_KILL_SUIDGID; } - attr_version = fuse_get_attr_version(fm->fc); - fuse_setattr_fill(fc, &args, inode, &inarg, &outarg); + *attr_version = fuse_get_attr_version(fc); + fuse_setattr_fill(fc, &args, inode, &inarg, outarg); err = fuse_simple_request(fm, &args); if (err) { if (err == -EINTR) fuse_invalidate_attr(inode); - goto error; + return err; } - if (fuse_invalid_attr(&outarg.attr) || - inode_wrong_type(inode, outarg.attr.mode)) { + if (fuse_invalid_attr(&outarg->attr) || + inode_wrong_type(inode, outarg->attr.mode)) { fuse_make_bad(inode); - err = -EIO; - goto error; + return -EIO; } - spin_lock(&fi->lock); - /* the kernel maintains i_mtime locally */ - if (trust_local_cmtime) { - if (attr->ia_valid & ATTR_MTIME) - inode_set_mtime_to_ts(inode, attr->ia_mtime); - if (attr->ia_valid & ATTR_CTIME) - inode_set_ctime_to_ts(inode, attr->ia_ctime); - /* FIXME: clear I_DIRTY_SYNC? */ + return 0; +} + +/* + * Reads issued while the size was unstable may no longer raise it, + * attribute replies may set it again, and the DAX layout block goes. + */ +static void fuse_setattr_end(struct inode *inode, bool is_truncate, + bool fault_blocked) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + + if (is_truncate) { + spin_lock(&fi->lock); + fuse_bump_size_gen(fi); + spin_unlock(&fi->lock); + } + clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); + if (fault_blocked) + filemap_invalidate_unlock(inode->i_mapping); +} + +/* + * Drop the page cache from @size on, and the DLM records over it first, + * under the full-range IO range lock like the NOTIFY invalidate path + * (fuse_reverse_inval_inode()): a reader/writer that already reached + * LOCKED state must not have the lock tree and the cache yanked mid-hold, + * or it would repopulate the truncated range trusting a grant that no + * longer exists. fuse_range_lock_acquire_locked() ignores an overlapping + * INIT range (a read/write with only a DLM request in flight), so waiting + * here is bounded. Blocking is also safe: the caller holds i_rwsem + * exclusive, so no cached writer can be waiting on this range lock (the + * write path takes i_rwsem before it, the read path never takes i_rwsem + * at all). See fuse_range_lock.h. + */ +static void fuse_wb_truncate_pagecache(struct inode *inode, loff_t size) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_range_lock rlock; + + fuse_range_lock_acquire_locked(fi, &rlock, 0, ~0ULL, + FUSE_RANGE_LOCK_WRITE); + fuse_dlm_unlock_range(fi, size & PAGE_MASK, -1); + truncate_pagecache(inode, size); + invalidate_inode_pages2(inode->i_mapping); + fuse_range_lock_release(fi, &rlock); +} + +/* + * Directories, symlinks, special files and regular files without the + * writeback cache: the server's reply is the truth for times and size. + */ +static int fuse_wt_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr, struct file *file) +{ + struct inode *inode = d_inode(dentry); + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct address_space *mapping = inode->i_mapping; + struct fuse_attr_out outarg; + bool is_truncate; + bool fault_blocked = false; + loff_t oldsize; + u64 attr_version; + int err; + + err = fuse_setattr_begin(idmap, dentry, attr, &fault_blocked); + if (err) + return err; + is_truncate = attr->ia_valid & ATTR_SIZE; + + if (attr->ia_valid & ATTR_OPEN) { + /* This is coming from open(..., ... | O_TRUNC); */ + WARN_ON(!(attr->ia_valid & ATTR_SIZE)); + WARN_ON(attr->ia_size != 0); + if (fc->atomic_o_trunc) { + /* + * No need to send request to userspace, since actual + * truncation has already been done by OPEN. But still + * need to truncate page cache. + */ + spin_lock(&fi->lock); + i_size_write(inode, 0); + spin_unlock(&fi->lock); + truncate_pagecache(inode, 0); + if (fault_blocked) + filemap_invalidate_unlock(mapping); + return 0; + } + file = NULL; + } + + if (is_truncate) { + /* Drop read replies that may predate the truncate. */ + spin_lock(&fi->lock); + fuse_bump_size_gen(fi); + set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); + spin_unlock(&fi->lock); } + err = fuse_setattr_send(idmap, inode, attr, file, false, &outarg, + &attr_version); + if (err) + goto out; + + spin_lock(&fi->lock); if (fi->attr_version > attr_version) { /* * Apply attributes, for example for fsnotify_change(), but set @@ -2661,68 +2691,160 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ outarg.attr_valid = outarg.attr_valid_nsec = 0; } - fuse_change_attributes_common(inode, &outarg.attr, NULL, - ATTR_TIMEOUT(&outarg), - fuse_get_cache_mask(inode), 0); + ATTR_TIMEOUT(&outarg), 0, 0); oldsize = inode->i_size; - /* see the comment in fuse_change_attributes() */ - if (!is_wb || is_truncate) - i_size_write(inode, outarg.attr.size); - + i_size_write(inode, outarg.attr.size); spin_unlock(&fi->lock); - if ((is_truncate || !is_wb) && - S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { - struct fuse_range_lock rlock; - bool range_locked = fc->writeback_cache && fc->dlm; + if (S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { + truncate_pagecache(inode, outarg.attr.size); + invalidate_inode_pages2(mapping); + } +out: + fuse_setattr_end(inode, is_truncate, fault_blocked); + return err; +} - /* - * Revoke and drop under the full-range IO range lock; see - * the atomic-O_TRUNC branch above. i_rwsem is held - * exclusive here as well (setattr), so waiting out - * in-progress LOCKED IO cannot deadlock. Only meaningful - * under DLM with the writeback cache; see fuse_range_lock.h. - */ - if (range_locked) +/* + * A regular file under the writeback cache: the kernel owns the times, + * dirty data is written out before the server sees the change, and the + * page cache is truncated under the IO range lock once the DLM records + * over it are gone. + */ +static int fuse_wb_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr, struct file *file) +{ + struct inode *inode = d_inode(dentry); + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct address_space *mapping = inode->i_mapping; + struct fuse_attr_out outarg; + bool is_truncate; + bool fault_blocked = false; + loff_t oldsize; + u64 attr_version; + int err; + + err = fuse_setattr_begin(idmap, dentry, attr, &fault_blocked); + if (err) + return err; + is_truncate = attr->ia_valid & ATTR_SIZE; + + if (attr->ia_valid & ATTR_OPEN) { + /* This is coming from open(..., ... | O_TRUNC); */ + WARN_ON(!(attr->ia_valid & ATTR_SIZE)); + WARN_ON(attr->ia_size != 0); + if (fc->atomic_o_trunc) { + struct fuse_range_lock rlock; + + /* + * No need to send request to userspace, since actual + * truncation has already been done by OPEN. But still + * need to truncate page cache. + * + * Under the range lock for the reasons given at + * fuse_wb_truncate_pagecache(); every DLM record + * goes, and i_size is set once the IO the lock waits + * for is done. + */ fuse_range_lock_acquire_locked(fi, &rlock, 0, ~0ULL, FUSE_RANGE_LOCK_WRITE); - if (fc->dlm && fc->writeback_cache) - fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1); - - truncate_pagecache(inode, outarg.attr.size); - invalidate_inode_pages2(mapping); - if (range_locked) + fuse_dlm_cache_release_locks(fi); + spin_lock(&fi->lock); + i_size_write(inode, 0); + spin_unlock(&fi->lock); + truncate_pagecache(inode, 0); fuse_range_lock_release(fi, &rlock); + if (fault_blocked) + filemap_invalidate_unlock(mapping); + return 0; + } + file = NULL; + } + + /* Flush dirty data/metadata before non-truncate SETATTR */ + if (attr->ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID | + ATTR_MTIME_SET | ATTR_TIMES_SET)) { + err = write_inode_now(inode, true); + if (err) + goto out; } if (is_truncate) { + /* + * Write dirty folios back before a truncating SETATTR, like + * the atomic-O_TRUNC case in fuse_wb_open(): under DLM the + * server's truncate revokes this client's locks, and the + * invalidate that follows would otherwise launder dirty + * folios with writes the server holds behind the same + * truncate. + */ + err = filemap_write_and_wait(mapping); + if (err) + goto out; + + /* Drop read replies that may predate the truncate. */ spin_lock(&fi->lock); fuse_bump_size_gen(fi); + set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); spin_unlock(&fi->lock); + if (attr->ia_size != inode->i_size) + attr->ia_valid |= ATTR_MTIME | ATTR_CTIME; } - clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); -out: - if (fault_blocked) - filemap_invalidate_unlock(mapping); - return 0; + err = fuse_setattr_send(idmap, inode, attr, file, true, &outarg, + &attr_version); + if (err) + goto out; -error: - if (is_truncate) { - spin_lock(&fi->lock); - fuse_bump_size_gen(fi); - spin_unlock(&fi->lock); + spin_lock(&fi->lock); + /* the kernel maintains i_mtime locally */ + if (attr->ia_valid & ATTR_MTIME) + inode_set_mtime_to_ts(inode, attr->ia_mtime); + if (attr->ia_valid & ATTR_CTIME) + inode_set_ctime_to_ts(inode, attr->ia_ctime); + /* FIXME: clear I_DIRTY_SYNC? */ + + if (fi->attr_version > attr_version) { + /* + * Apply attributes, for example for fsnotify_change(), but set + * attribute timeout to zero. + */ + outarg.attr_valid = outarg.attr_valid_nsec = 0; } - clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); + fuse_change_attributes_common(inode, &outarg.attr, NULL, + ATTR_TIMEOUT(&outarg), 0, 0); + oldsize = inode->i_size; + /* see the comment in fuse_change_attributes() */ + if (is_truncate) + i_size_write(inode, outarg.attr.size); + spin_unlock(&fi->lock); - if (fault_blocked) - filemap_invalidate_unlock(mapping); + if (is_truncate && oldsize != outarg.attr.size) + fuse_wb_truncate_pagecache(inode, outarg.attr.size); +out: + fuse_setattr_end(inode, is_truncate, fault_blocked); return err; } -static int fuse_setattr(struct mnt_idmap *idmap, struct dentry *entry, - struct iattr *attr) +/* + * Entry for callers outside the inode operations (the direct IO rollback in + * file.c), which do not know which mode the inode's operations were chosen + * for. + */ +int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr, struct file *file) +{ + struct inode *inode = d_inode(dentry); + + if (get_fuse_conn(inode)->writeback_cache && S_ISREG(inode->i_mode)) + return fuse_wb_do_setattr(idmap, dentry, attr, file); + return fuse_wt_do_setattr(idmap, dentry, attr, file); +} + +static int fuse_setattr_common(struct mnt_idmap *idmap, struct dentry *entry, + struct iattr *attr, bool is_wb) { struct inode *inode = d_inode(entry); struct fuse_conn *fc = get_fuse_conn(inode); @@ -2768,7 +2890,10 @@ static int fuse_setattr(struct mnt_idmap *idmap, struct dentry *entry, if (!attr->ia_valid) return 0; - ret = fuse_do_setattr(idmap, entry, attr, file); + if (is_wb) + ret = fuse_wb_do_setattr(idmap, entry, attr, file); + else + ret = fuse_wt_do_setattr(idmap, entry, attr, file); if (!ret) { /* * If filesystem supports acls it may have updated acl xattrs in @@ -2784,6 +2909,20 @@ static int fuse_setattr(struct mnt_idmap *idmap, struct dentry *entry, return ret; } +/* Directories, symlinks, special files and writethrough regular files. */ +static int fuse_wt_setattr(struct mnt_idmap *idmap, struct dentry *entry, + struct iattr *attr) +{ + return fuse_setattr_common(idmap, entry, attr, false); +} + +/* Regular files under the writeback cache. */ +static int fuse_wb_setattr(struct mnt_idmap *idmap, struct dentry *entry, + struct iattr *attr) +{ + return fuse_setattr_common(idmap, entry, attr, true); +} + static int fuse_getattr(struct mnt_idmap *idmap, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags) @@ -2819,7 +2958,7 @@ static const struct inode_operations fuse_dir_inode_operations = { .rmdir = fuse_rmdir, .rename = fuse_rename2, .link = fuse_link, - .setattr = fuse_setattr, + .setattr = fuse_wt_setattr, .create = fuse_create, .atomic_open = fuse_atomic_open, .tmpfile = fuse_tmpfile, @@ -2845,8 +2984,22 @@ static const struct file_operations fuse_dir_operations = { .compat_ioctl = fuse_dir_compat_ioctl, }; -static const struct inode_operations fuse_common_inode_operations = { - .setattr = fuse_setattr, +/* Special files, and regular files without the writeback cache. */ +static const struct inode_operations fuse_wt_common_inode_operations = { + .setattr = fuse_wt_setattr, + .permission = fuse_permission, + .getattr = fuse_getattr, + .listxattr = fuse_listxattr, + .get_inode_acl = fuse_get_inode_acl, + .get_acl = fuse_get_acl, + .set_acl = fuse_set_acl, + .fileattr_get = fuse_fileattr_get, + .fileattr_set = fuse_fileattr_set, +}; + +/* Regular files under the writeback cache. */ +static const struct inode_operations fuse_wb_common_inode_operations = { + .setattr = fuse_wb_setattr, .permission = fuse_permission, .getattr = fuse_getattr, .listxattr = fuse_listxattr, @@ -2858,7 +3011,7 @@ static const struct inode_operations fuse_common_inode_operations = { }; static const struct inode_operations fuse_symlink_inode_operations = { - .setattr = fuse_setattr, + .setattr = fuse_wt_setattr, .get_link = fuse_get_link, .getattr = fuse_getattr, .listxattr = fuse_listxattr, @@ -2866,7 +3019,16 @@ static const struct inode_operations fuse_symlink_inode_operations = { void fuse_init_common(struct inode *inode) { - inode->i_op = &fuse_common_inode_operations; + struct fuse_conn *fc = get_fuse_conn(inode); + + /* + * Only a regular file has a page cache to write back; the mode is + * known from the INIT reply, so it is decided here once. + */ + if (S_ISREG(inode->i_mode) && fc->writeback_cache) + inode->i_op = &fuse_wb_common_inode_operations; + else + inode->i_op = &fuse_wt_common_inode_operations; } void fuse_init_dir(struct inode *inode) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 957ec0df917e06..139fbece650379 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -326,6 +326,27 @@ static void fuse_link_write_file(struct file *file) spin_unlock(&fi->lock); } +#ifdef CONFIG_FUSE_PASSTHROUGH +static const struct file_operations fuse_wt_passthrough_file_operations; +static const struct file_operations fuse_wb_passthrough_file_operations; + +/* + * Passthrough is decided per open, by the FUSE_OPEN reply, so the file + * swaps to the passthrough operations of its cache mode once the backing + * file is set up. + */ +static void fuse_passthrough_set_fops(struct file *file, struct fuse_conn *fc) +{ + replace_fops(file, fc->writeback_cache ? + &fuse_wb_passthrough_file_operations : + &fuse_wt_passthrough_file_operations); +} +#else +static void fuse_passthrough_set_fops(struct file *file, struct fuse_conn *fc) +{ +} +#endif + int fuse_finish_open(struct inode *inode, struct file *file) { struct fuse_file *ff = file->private_data; @@ -336,6 +357,9 @@ int fuse_finish_open(struct inode *inode, struct file *file) if (err) return err; + if (fuse_file_passthrough(ff)) + fuse_passthrough_set_fops(file, fc); + if (ff->open_flags & FOPEN_STREAM) stream_open(inode, file); else if (ff->open_flags & FOPEN_NONSEEKABLE) @@ -364,18 +388,15 @@ static void fuse_truncate_update_attr(struct inode *inode) fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); } -static int fuse_open(struct inode *inode, struct file *file) +/* + * Common prologue of ->open: the VFS checks, and the write access that + * handle_truncate() would otherwise take after the server has already + * truncated on FUSE_OPEN (before it checks for a running executable). + */ +static int fuse_open_begin(struct inode *inode, struct file *file, + bool is_truncate) { - struct fuse_mount *fm = get_fuse_mount(inode); - struct fuse_inode *fi = get_fuse_inode(inode); - struct fuse_conn *fc = fm->fc; - struct fuse_file *ff; - struct fuse_range_lock rlock; int err; - bool is_truncate = (file->f_flags & O_TRUNC) && fc->atomic_o_trunc; - bool is_wb_truncate = is_truncate && fc->writeback_cache; - bool dax_truncate = is_truncate && FUSE_IS_DAX(inode); - bool range_locked = is_wb_truncate && fc->dlm; if (fuse_is_bad(inode)) return -EIO; @@ -384,17 +405,106 @@ static int fuse_open(struct inode *inode, struct file *file) if (err) return err; - /* - * The server truncates on FUSE_OPEN, before handle_truncate() checks - * for a running executable. - */ if (is_truncate) { err = get_write_access(inode); if (err) return err; } - if (is_wb_truncate || dax_truncate) + return 0; +} + +/* Send FUSE_OPEN and set the file up on success. */ +static int fuse_open_send(struct inode *inode, struct file *file, + bool is_truncate) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + int err; + + err = fuse_do_open(fm, get_node_id(inode), file, false); + if (err) + return err; + + err = fuse_finish_open(inode, file); + if (err) + fuse_sync_release(fi, file->private_data, file->f_flags); + else if (is_truncate) + fuse_truncate_update_attr(inode); + + return err; +} + +/* Bring the page cache in line with the file the server just opened. */ +static void fuse_open_finish_cache(struct inode *inode, struct file *file, + bool is_truncate) +{ + struct fuse_file *ff = file->private_data; + + if (is_truncate) { + truncate_pagecache(inode, 0); + fuse_truncate_server_times(inode, file); + } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { + invalidate_inode_pages2(inode->i_mapping); + } +} + +/* + * Whether the inode is DAX is fixed when it is set up, so the DAX opens + * pass it as a constant rather than testing the inode. + */ +static int __fuse_wt_open(struct inode *inode, struct file *file, bool dax) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + int err; + bool is_truncate = (file->f_flags & O_TRUNC) && fc->atomic_o_trunc; + bool dax_truncate = is_truncate && dax; + + err = fuse_open_begin(inode, file, is_truncate); + if (err) + return err; + + if (dax_truncate) { + inode_lock(inode); + filemap_invalidate_lock(inode->i_mapping); + err = fuse_dax_break_layouts(inode, 0, -1); + if (err) + goto out_inode_unlock; + fuse_set_nowrite(inode); + } + + err = fuse_open_send(inode, file, is_truncate); + + if (dax_truncate) + fuse_release_nowrite(inode); + + if (!err) + fuse_open_finish_cache(inode, file, is_truncate); +out_inode_unlock: + if (dax_truncate) { + filemap_invalidate_unlock(inode->i_mapping); + inode_unlock(inode); + } + if (is_truncate) + put_write_access(inode); + + return err; +} + +static int __fuse_wb_open(struct inode *inode, struct file *file, bool dax) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_range_lock rlock; + int err; + bool is_truncate = (file->f_flags & O_TRUNC) && fc->atomic_o_trunc; + bool dax_truncate = is_truncate && dax; + + err = fuse_open_begin(inode, file, is_truncate); + if (err) + return err; + + if (is_truncate) inode_lock(inode); /* @@ -405,7 +515,7 @@ static int fuse_open(struct inode *inode, struct file *file) * while the truncate waits for the invalidate to finish. i_rwsem * keeps cached writers from redirtying the cache until the OPEN. */ - if (range_locked) { + if (is_truncate) { err = filemap_write_and_wait(inode->i_mapping); if (err) goto out_inode_unlock; @@ -415,54 +525,57 @@ static int fuse_open(struct inode *inode, struct file *file) filemap_invalidate_lock(inode->i_mapping); err = fuse_dax_break_layouts(inode, 0, -1); if (err) - goto out_inode_unlock; + goto out_invalidate_unlock; } - if (range_locked) { + if (is_truncate) { err = fuse_range_lock_acquire_init(fi, &rlock, 0, ~0ULL, FUSE_RANGE_LOCK_WRITE); - if (err) { - if (dax_truncate) - filemap_invalidate_unlock(inode->i_mapping); - goto out_inode_unlock; - } - } else if (is_wb_truncate || dax_truncate) { - fuse_set_nowrite(inode); - } - - err = fuse_do_open(fm, get_node_id(inode), file, false); - if (!err) { - ff = file->private_data; - err = fuse_finish_open(inode, file); if (err) - fuse_sync_release(fi, ff, file->f_flags); - else if (is_truncate) - fuse_truncate_update_attr(inode); + goto out_invalidate_unlock; } - if (range_locked) + err = fuse_open_send(inode, file, is_truncate); + + if (is_truncate) fuse_range_lock_release(fi, &rlock); - else if (is_wb_truncate || dax_truncate) - fuse_release_nowrite(inode); - if (!err) { - if (is_truncate) { - truncate_pagecache(inode, 0); - fuse_truncate_server_times(inode, file); - } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) - invalidate_inode_pages2(inode->i_mapping); - } + if (!err) + fuse_open_finish_cache(inode, file, is_truncate); +out_invalidate_unlock: if (dax_truncate) filemap_invalidate_unlock(inode->i_mapping); out_inode_unlock: - if (is_wb_truncate || dax_truncate) + if (is_truncate) { inode_unlock(inode); - if (is_truncate) put_write_access(inode); + } return err; } +static int fuse_wt_open(struct inode *inode, struct file *file) +{ + return __fuse_wt_open(inode, file, false); +} + +static int fuse_wb_open(struct inode *inode, struct file *file) +{ + return __fuse_wb_open(inode, file, false); +} + +#ifdef CONFIG_FUSE_DAX +static int fuse_wt_dax_open(struct inode *inode, struct file *file) +{ + return __fuse_wt_open(inode, file, true); +} + +static int fuse_wb_dax_open(struct inode *inode, struct file *file) +{ + return __fuse_wb_open(inode, file, true); +} +#endif + static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, unsigned int flags, int opcode, bool sync) { @@ -573,21 +686,23 @@ void fuse_release_common(struct file *file, bool isdir) (fl_owner_t) file, isdir); } -static int fuse_release(struct inode *inode, struct file *file) +static int fuse_wt_release(struct inode *inode, struct file *file) { - struct fuse_conn *fc = get_fuse_conn(inode); + fuse_release_common(file, false); + + /* return value is ignored by VFS */ + return 0; +} +static int fuse_wb_release(struct inode *inode, struct file *file) +{ /* * Dirty pages might remain despite write_inode_now() call from - * fuse_flush() due to writes racing with the close. + * fuse_wb_flush() due to writes racing with the close. */ - if (fc->writeback_cache) - write_inode_now(inode, 1); - - fuse_release_common(file, false); + write_inode_now(inode, 1); - /* return value is ignored by VFS */ - return 0; + return fuse_wt_release(inode, file); } void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, @@ -643,7 +758,11 @@ static void fuse_sync_writes(struct inode *inode) fuse_release_nowrite(inode); } -static int fuse_flush(struct file *file, fl_owner_t id) +/* + * Write the inode out and send FUSE_FLUSH. Dirty folios exist in both + * modes: writethrough gets them from shared mappings. + */ +static int fuse_flush_common(struct file *file, fl_owner_t id) { struct inode *inode = file_inode(file); struct fuse_mount *fm = get_fuse_mount(inode); @@ -652,12 +771,6 @@ static int fuse_flush(struct file *file, fl_owner_t id) FUSE_ARGS(args); int err; - if (fuse_is_bad(inode)) - return -EIO; - - if (ff->open_flags & FOPEN_NOFLUSH && !fm->fc->writeback_cache) - return 0; - err = write_inode_now(inode, 1); if (err) return err; @@ -666,9 +779,8 @@ static int fuse_flush(struct file *file, fl_owner_t id) if (err) return err; - err = 0; if (fm->fc->no_flush) - goto inval_attr_out; + return 0; memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; @@ -685,13 +797,37 @@ static int fuse_flush(struct file *file, fl_owner_t id) fm->fc->no_flush = 1; err = 0; } + return err; +} + +static int fuse_wt_flush(struct file *file, fl_owner_t id) +{ + struct fuse_file *ff = file->private_data; + + if (fuse_is_bad(file_inode(file))) + return -EIO; + + if (ff->open_flags & FOPEN_NOFLUSH) + return 0; + + return fuse_flush_common(file, id); +} + +static int fuse_wb_flush(struct file *file, fl_owner_t id) +{ + struct inode *inode = file_inode(file); + int err; + + if (fuse_is_bad(inode)) + return -EIO; + + err = fuse_flush_common(file, id); -inval_attr_out: /* - * In memory i_blocks is not maintained by fuse, if writeback cache is - * enabled, i_blocks from cached attr may not be accurate. + * In memory i_blocks is not maintained by fuse, so with the writeback + * cache i_blocks from cached attr may not be accurate. */ - if (!err && fm->fc->writeback_cache) + if (!err) fuse_invalidate_attr_mask(inode, STATX_BLOCKS); return err; } @@ -1005,27 +1141,30 @@ static void fuse_read_update_size(struct inode *inode, loff_t size, spin_unlock(&fi->lock); } -static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read, - struct fuse_args_pages *ap) -{ - struct fuse_conn *fc = get_fuse_conn(inode); +/* + * What a READ reply says about i_size, which the two cache modes read + * differently. @pos and @asked are the request, @got what came back; + * @attr_ver and @size_gen were sampled before the request was sent. + */ +typedef void (*fuse_read_size_fn)(struct inode *inode, loff_t pos, + size_t asked, size_t got, u64 attr_ver, + u64 size_gen); - /* - * If writeback_cache is enabled, a short read means there's a hole in - * the file. Some data after the hole is in page cache, but has not - * reached the client fs yet. So the hole is not present there. - */ - if (!fc->writeback_cache) { - loff_t pos = folio_pos(ap->folios[0]) + num_read; - fuse_read_update_size(inode, pos, attr_ver); - } +/* Without the writeback cache a short read means EOF: shrink i_size to it. */ +static void fuse_wt_read_size(struct inode *inode, loff_t pos, size_t asked, + size_t got, u64 attr_ver, u64 size_gen) +{ + if (got < asked) + fuse_read_update_size(inode, pos + got, attr_ver); } /* * Under DLM with the writeback cache, i_size covers only this node's own - * writes. A read reply proves the file extends at least to the bytes it - * returned (the server clips reads at its size), and they may be another - * node's data in a page this node shares. Raise i_size over them, so that + * writes, and a short read means a hole: data after it is in the page cache + * but has not reached the server yet, so i_size is left alone. A read reply + * does prove the file extends at least to the bytes it returned (the server + * clips reads at its size), and they may be another node's data in a page + * this node shares. Raise i_size over them, so that * iomap_writeback_handle_eof() never zeroes them as if they were past EOF. * * Every revoke bumps attr_version, including the revoke of the very range @@ -1033,13 +1172,14 @@ static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read, * local truncate or a whole-file invalidate, which is how a remote truncate * reaches us; a reply that may predate either is dropped. */ -static void fuse_dlm_read_extend_size(struct inode *inode, loff_t end, - u64 size_gen) +static void fuse_wb_read_size(struct inode *inode, loff_t pos, size_t asked, + size_t got, u64 attr_ver, u64 size_gen) { struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); + loff_t end = pos + got; - if (!fc->dlm || !fc->writeback_cache) + if (!got) return; spin_lock(&fi->lock); @@ -1052,7 +1192,8 @@ static void fuse_dlm_read_extend_size(struct inode *inode, loff_t end, } static int fuse_do_readfolio(struct file *file, struct folio *folio, - size_t off, size_t len) + size_t off, size_t len, + fuse_read_size_fn read_size) { struct inode *inode = folio->mapping->host; struct fuse_mount *fm = get_fuse_mount(inode); @@ -1083,28 +1224,10 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); res = fuse_simple_request(fm, &ia.ap.args); trace_fuse_readfolio_reply(inode, pos, desc.length, res); - if (res < 0) { - /* - * Please refer to Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt - * why READ can return -EDEADLK from DLM subsystem. - * - * -EDEADLK: Preferred error code indicating DLM lock ordering violation - * (would cause deadlock with page lock) - * -EAGAIN: Legacy error code, maintained for backward compatibility - */ - if ((res == -EDEADLK || res == -EAGAIN) && fm->fc->dlm) - res = AOP_TRUNCATED_PAGE; + if (res < 0) return res; - } - /* - * Short read means EOF. If file size is larger, truncate it - */ - if (res < desc.length) - fuse_short_read(inode, attr_ver, res, &ia.ap); - - if (res > 0) - fuse_dlm_read_extend_size(inode, pos + res, size_gen); + read_size(inode, pos, desc.length, res, attr_ver, size_gen); return 0; } @@ -1123,6 +1246,10 @@ static const struct iomap_ops fuse_iomap_ops = { .iomap_begin = fuse_iomap_begin, }; +typedef void (*fuse_args_end_fn)(struct fuse_mount *fm, + struct fuse_args *args, int err); + +/* @end and @dlm_covered carry the cache mode of the inode. */ struct fuse_fill_read_data { struct file *file; @@ -1131,6 +1258,8 @@ struct fuse_fill_read_data { struct fuse_io_args *ia; unsigned int nr_bytes; loff_t covered; + fuse_args_end_fn end; + bool dlm_covered; }; /* forward declarations */ @@ -1138,7 +1267,8 @@ static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos, unsigned len, struct fuse_args_pages *ap, unsigned cur_bytes, bool write); static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, - unsigned int count, bool async); + unsigned int count, bool async, + fuse_args_end_fn end); static int fuse_handle_readahead(struct folio *folio, struct readahead_control *rac, @@ -1160,7 +1290,7 @@ static int fuse_handle_readahead(struct folio *folio, * no grant covers whole: iomap sends what is batched, unlocks this * folio and read_pages() drops the rest. */ - if (fc->dlm && fc->writeback_cache && + if (data->dlm_covered && data->covered < folio_pos(folio) + folio_size(folio)) { struct fuse_inode *fi = get_fuse_inode(folio->mapping->host); loff_t fpos = folio_pos(folio); @@ -1175,7 +1305,7 @@ static int fuse_handle_readahead(struct folio *folio, if (ia && fuse_folios_need_send(fc, pos, len, &ia->ap, data->nr_bytes, false)) { fuse_send_readpages(ia, data->file, data->nr_bytes, - fc->async_read); + fc->async_read, data->end); data->nr_bytes = 0; data->ia = NULL; ia = NULL; @@ -1221,11 +1351,13 @@ static int fuse_iomap_read_folio_range_async(const struct iomap_iter *iter, ret = fuse_handle_readahead(folio, ctx->rac, data, pos, len); } else { /* - * for non-readahead read requests, do reads synchronously - * since it's not guaranteed that the server can handle - * out-of-order reads + * for non-readahead read requests, do reads synchronously + * since it's not guaranteed that the server can handle + * out-of-order reads. Only the writeback cache reads a + * folio through iomap, see fuse_wt_read_folio(). */ - ret = fuse_do_readfolio(file, folio, off, len); + ret = fuse_do_readfolio(file, folio, off, len, + fuse_wb_read_size); if (!ret) iomap_finish_folio_read(folio, off, len, ret); } @@ -1238,7 +1370,7 @@ static void fuse_iomap_read_submit(struct iomap_read_folio_ctx *ctx) if (data->ia) fuse_send_readpages(data->ia, data->file, data->nr_bytes, - data->fc->async_read); + data->fc->async_read, data->end); } static const struct iomap_read_ops fuse_iomap_read_ops = { @@ -1246,7 +1378,37 @@ static const struct iomap_read_ops fuse_iomap_read_ops = { .submit_read = fuse_iomap_read_submit, }; -static int fuse_read_folio(struct file *file, struct folio *folio) +/* + * Without a grant the server may answer a READ with -EDEADLK (or the legacy + * -EAGAIN) instead of waiting for a lock while this node holds the folio + * lock, see Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt. + * The VFS retries from scratch on AOP_TRUNCATED_PAGE, returned after the + * unlock. iomap_read_folio() cannot pass that on (a positive status is + * -EIO to iomap_iter()), so the writethrough read stays off iomap; the + * writeback cache reads under a grant and is never bounced. + */ +static int fuse_wt_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + int err; + + err = -EIO; + if (fuse_is_bad(inode)) + goto out; + + err = fuse_do_readfolio(file, folio, 0, folio_size(folio), + fuse_wt_read_size); + if (!err) + folio_mark_uptodate(folio); + fuse_invalidate_atime(inode); +out: + folio_unlock(folio); + if (err == -EDEADLK || err == -EAGAIN) + return AOP_TRUNCATED_PAGE; + return err; +} + +static int fuse_wb_read_folio(struct file *file, struct folio *folio) { struct inode *inode = folio->mapping->host; struct fuse_fill_read_data data = { @@ -1256,7 +1418,6 @@ static int fuse_read_folio(struct file *file, struct folio *folio) .cur_folio = folio, .ops = &fuse_iomap_read_ops, .read_ctx = &data, - }; if (fuse_is_bad(inode)) { @@ -1269,55 +1430,24 @@ static int fuse_read_folio(struct file *file, struct folio *folio) return 0; } +/* + * Only the writeback cache writes through iomap, and + * fuse_wb_cache_write_iter() holds the grant over the written range across + * this read, so the server never bounces it (see fuse_wt_read_folio() for a + * read without a grant). + */ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, struct folio *folio, loff_t pos, size_t len) { struct file *file = iter->private; - struct inode *inode = file_inode(file); - struct fuse_conn *fc = get_fuse_conn(inode); size_t off = offset_in_folio(folio, pos); - int ret; - - ret = fuse_do_readfolio(file, folio, off, len); - /* - * TEMPORARY WORKAROUND for iomap write deadlock: - * - * When FUSE server returns -EDEADLK (or legacy -EAGAIN) due to DLM - * lock contention, fuse_do_readfolio() converts it to AOP_TRUNCATED_PAGE - * and unlocks the folio (per AOP_TRUNCATED_PAGE contract). - * - * However, iomap doesn't understand AOP_TRUNCATED_PAGE. - * We need to: - * 1. Mark the retry flag (caller stored it in xarray) - * 2. Convert to -EAGAIN so iomap sees an error - * 3. Let fuse_cache_write_iter() detect and retry - * - * This breaks the ABBA deadlock: - * - Folio is unlocked (page invalidation can proceed) - * - Write will be retried at higher level - * - * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE support. - */ - if (ret == AOP_TRUNCATED_PAGE) { - struct fuse_dlm_retry *retry; - unsigned long task_key = (unsigned long)current; - - retry = xa_load(&fc->dlm_retry_tasks, task_key); - if (retry) { - retry->retry_needed = true; - } - - /* Convert to -EAGAIN for iomap */ - ret = -EAGAIN; - } - - return ret; + return fuse_do_readfolio(file, folio, off, len, fuse_wb_read_size); } -static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, - int err) +static void fuse_readpages_end_common(struct fuse_args *args, int err, + fuse_read_size_fn read_size) { int i; struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); @@ -1331,16 +1461,9 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, mapping = ap->folios[0]->mapping; inode = mapping->host; - /* - * Short read means EOF. If file size is larger, truncate it - */ - if (!err && num_read < count) - fuse_short_read(inode, ia->read.attr_ver, num_read, ap); - - if (!err && num_read) - fuse_dlm_read_extend_size(inode, - ia->read.in.offset + num_read, - ia->read.size_gen); + if (!err) + read_size(inode, ia->read.in.offset, count, num_read, + ia->read.attr_ver, ia->read.size_gen); fuse_invalidate_atime(inode); @@ -1355,8 +1478,21 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, fuse_io_free(ia); } +static void fuse_wt_readpages_end(struct fuse_mount *fm, + struct fuse_args *args, int err) +{ + fuse_readpages_end_common(args, err, fuse_wt_read_size); +} + +static void fuse_wb_readpages_end(struct fuse_mount *fm, + struct fuse_args *args, int err) +{ + fuse_readpages_end_common(args, err, fuse_wb_read_size); +} + static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, - unsigned int count, bool async) + unsigned int count, bool async, + fuse_args_end_fn end) { struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; @@ -1382,7 +1518,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, ia->read.size_gen = READ_ONCE(get_fuse_inode(file_inode(file))->size_gen); if (async) { ia->ff = fuse_file_get(ff); - ap->args.end = fuse_readpages_end; + ap->args.end = end; err = fuse_simple_background(fm, &ap->args, GFP_KERNEL); if (!err) return; @@ -1390,16 +1526,23 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, res = fuse_simple_request(fm, &ap->args); err = res < 0 ? res : 0; } - fuse_readpages_end(fm, &ap->args, err); + end(fm, &ap->args, err); } -static void fuse_readahead(struct readahead_control *rac) +/* + * @dlm_covered: fill only what a DLM grant covers, which is what the + * writeback cache needs; without it there are no grants to check. + */ +static void fuse_readahead_common(struct readahead_control *rac, + fuse_args_end_fn end, bool dlm_covered) { struct inode *inode = rac->mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_fill_read_data data = { .file = rac->file, .fc = fc, + .end = end, + .dlm_covered = dlm_covered, }; struct iomap_read_folio_ctx ctx = { .ops = &fuse_iomap_read_ops, @@ -1413,6 +1556,16 @@ static void fuse_readahead(struct readahead_control *rac) iomap_readahead(&fuse_iomap_ops, &ctx, NULL); } +static void fuse_wt_readahead(struct readahead_control *rac) +{ + fuse_readahead_common(rac, fuse_wt_readpages_end, false); +} + +static void fuse_wb_readahead(struct readahead_control *rac) +{ + fuse_readahead_common(rac, fuse_wb_readpages_end, true); +} + static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); /* @@ -1429,7 +1582,7 @@ static bool fuse_size_needs_server(struct inode *inode) { struct fuse_conn *fc = get_fuse_conn(inode); - return fc->dlm && fc->writeback_cache && S_ISREG(inode->i_mode); + return fc->writeback_cache && S_ISREG(inode->i_mode); } /* @@ -1474,7 +1627,29 @@ static size_t fuse_read_fault_window(struct kiocb *iocb, size_t count, return min_t(size_t, size - offs, count); } -static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) +static ssize_t fuse_wt_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file->f_mapping->host; + struct fuse_conn *fc = get_fuse_conn(inode); + size_t count = iov_iter_count(to); + + /* + * In auto invalidate mode, always update attributes on read. + * Otherwise, only update if we attempt to read past EOF (to ensure + * i_size is up to date). + */ + if (fc->auto_inval_data || (iocb->ki_pos + count > i_size_read(inode))) { + int err = fuse_update_attributes(inode, file, STATX_SIZE); + + if (err) + return err; + } + + return generic_file_read_iter(iocb, to); +} + +static ssize_t fuse_wb_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; @@ -1482,8 +1657,8 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_range_lock rlock; size_t count = iov_iter_count(to); - bool range_locked = false; ssize_t res; + int err; /* * In auto invalidate mode, always update attributes on read. @@ -1491,8 +1666,6 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) * i_size is up to date). */ if (fc->auto_inval_data || (iocb->ki_pos + count > i_size_read(inode))) { - int err; - if (fuse_read_needs_size(inode, iocb->ki_pos, count)) err = fuse_update_attributes_sync(inode, file, STATX_SIZE); @@ -1502,6 +1675,9 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) return err; } + if (!count) + return generic_file_read_iter(iocb, to); + /* * Reserve the IO range lock in INIT state over the bytes this read * will touch before requesting the DLM read lock below: an INIT @@ -1509,39 +1685,27 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) * invalidate whose range overlaps ours can run to completion -- * revoking the grant requested below and dropping its page-cache * range -- without waiting out this (unbounded, cluster round trip) - * request. Only meaningful under DLM with the writeback cache: - * without DLM there is no round trip to protect against, and the - * truncate / invalidate paths' fuse_range_lock_acquire_locked() - * calls simply find no overlapping node to wait on; without the - * writeback cache there is no DLM-covered write to race against. + * request. */ - if (fc->writeback_cache && fc->dlm && count) { - int err = fuse_range_lock_acquire_init(fi, &rlock, - iocb->ki_pos, - iocb->ki_pos + count - 1, - FUSE_RANGE_LOCK_READ); - if (err) - return err; - range_locked = true; - } + err = fuse_range_lock_acquire_init(fi, &rlock, iocb->ki_pos, + iocb->ki_pos + count - 1, + FUSE_RANGE_LOCK_READ); + if (err) + return err; /* - * If we have dlm support acquire a read lock for the area we are - * reading from. Passing the range lock through moves it to READY - * as part of processing the reply -- as soon as the grant (or a - * "no DLM" reply) comes back, rather than only later, right below. + * Acquire a DLM read lock for the area we are reading from. Passing + * the range lock through moves it to READY as part of processing the + * reply -- as soon as the grant comes back, rather than only later, + * right below. */ - if (fc->writeback_cache && fc->dlm) - fuse_get_dlm_lock(file, iocb->ki_pos, iov_iter_count(to), - FUSE_PAGE_LOCK_READ, - range_locked ? &rlock : NULL); + fuse_get_dlm_lock(file, iocb->ki_pos, count, FUSE_PAGE_LOCK_READ, + &rlock); /* * Ensure the range lock is LOCKED before touching the page cache: * the DLM reply above already moved it to READY when it was taken, - * so this call promotes it the rest of the way. It still does the - * full INIT-to-LOCKED transition itself when no DLM request was - * made above (no writeback cache or no dlm), and blocks only if a + * so this call promotes it the rest of the way, and blocks only if a * NOTIFY invalidate is currently draining an overlapping range -- * once granted it fences any *new* overlapping invalidate until the * range lock is released below. An invalidate that instead ran to @@ -1553,28 +1717,22 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) * requesting a fresh grant if the race happened, so by the time we * get here the page cache this read is about to see is consistent. */ - if (range_locked) { - int err = fuse_range_lock_mark_locked(fi, &rlock); - - if (err) { - fuse_range_lock_release(fi, &rlock); - return err; - } - - if (fuse_inode_force_dio(inode)) { - fuse_range_lock_release(fi, &rlock); - return fuse_direct_read_iter(iocb, to); - } + err = fuse_range_lock_mark_locked(fi, &rlock); + if (err) { + fuse_range_lock_release(fi, &rlock); + return err; } - if (!range_locked) - return generic_file_read_iter(iocb, to); + if (fuse_inode_force_dio(inode)) { + fuse_range_lock_release(fi, &rlock); + return fuse_direct_read_iter(iocb, to); + } /* * Nothing held once the range lock is LOCKED may wait for the - * server (see fuse_cache_write_iter), and faulting on the user + * server (see fuse_wb_cache_write_iter), and faulting on the user * buffer can do exactly that: a buffer backed by a shared-writable - * mapping of this (or any) fuse file sends fuse_filemap_fault() + * mapping of this (or any) fuse file sends fuse_wb_filemap_fault() * after a WRITE grant, and granting it can first require the * server to revoke this read's own READ grant -- whose * invalidation then waits on this read's LOCKED range. ABBA @@ -1598,7 +1756,6 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) for (;;) { size_t remain; size_t window; - int err; pagefault_disable(); res = generic_file_read_iter(iocb, to); @@ -2013,77 +2170,6 @@ static const struct iomap_write_ops fuse_iomap_write_ops = { .read_folio_range = fuse_iomap_read_folio_range, }; -static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, - struct iov_iter *from, - struct file *file) -{ - struct fuse_conn *fc = get_fuse_conn(file_inode(file)); - ssize_t written, total_written = 0; - - /* - * TEMPORARY WORKAROUND for iomap write deadlock: - * - * Stack-allocate retry state and register it before calling - * iomap. If fuse_iomap_read_folio_range() encounters - * AOP_TRUNCATED_PAGE, it will mark retry_needed. - * - * Stack allocation ensures no memory leaks - the state is - * valid for the duration of this function call and is - * automatically cleaned up. - */ - struct fuse_dlm_retry retry_state = { - .retry_needed = false, - }; - unsigned long task_key = (unsigned long)current; - int xa_ret; - - xa_ret = xa_err(xa_store(&fc->dlm_retry_tasks, task_key, - &retry_state, GFP_KERNEL)); - if (xa_ret) - return xa_ret; - -retry: - /* - * Reset before each iomap call so retry_needed only reflects what - * happened in the most recent call. iomap may set retry_needed - * during an internal iteration that then recovers and completes - * the write fully; without the reset the flag would survive into - * the next iteration with iov_iter already drained, and iomap - * would re-enter with len==0 and livelock on a 0-length mapping. - */ - retry_state.retry_needed = false; - - /* - * Use iomap so that we can do granular uptodate reads - * and granular dirty tracking for large folios. - */ - written = iomap_file_buffered_write(iocb, from, &fuse_iomap_ops, - &fuse_iomap_write_ops, file); - - if (written > 0) - total_written += written; - - /* - * If DLM lock contention occurred (AOP_TRUNCATED_PAGE), - * retry the entire write operation. - * - * The folio has been unlocked by fuse_do_readfolio(), - * breaking the ABBA deadlock with page invalidation. - * - * Keep the entry in xarray and reuse it for the retry. - * - * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE - * retry support. - */ - if (retry_state.retry_needed && iov_iter_count(from)) - goto retry; - - /* Remove from xarray now that we're done */ - xa_erase(&fc->dlm_retry_tasks, task_key); - - return written < 0 ? written : total_written; -} - static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from); /* @@ -2093,21 +2179,14 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) * Buffered writes normally hold the inode rwsem exclusively, serialising all * writers even on disjoint ranges. The DLM-serialised iomap writeback path is * the exception: the DLM already excludes cluster-wide, and i_size is committed - * under fi->lock rather than the inode rwsem (see fuse_cache_write_iter()), so - * disjoint writers (MPI-IO / IOR) may share the lock. Mirrors - * fuse_dio_wr_exclusive_lock() for the direct path. + * under fi->lock rather than the inode rwsem (see + * fuse_wb_cache_write_iter()), so disjoint writers (MPI-IO / IOR) may share + * the lock. Mirrors fuse_dio_wr_exclusive_lock() for the direct path. */ static bool fuse_cache_wr_exclusive_lock(struct kiocb *iocb, bool writeback) { - struct inode *inode = file_inode(iocb->ki_filp); - struct fuse_conn *fc = get_fuse_conn(inode); - /* Only the DLM-serialised iomap writeback path relaxes the lock. */ - if (!fc->dlm || !writeback) - return true; - - /* O_DIRECT writes fall back to generic_file_direct_write(). */ - if (iocb->ki_flags & IOCB_DIRECT) + if (!writeback) return true; /* Append needs the eventual EOF - always needs an exclusive lock. */ @@ -2126,11 +2205,10 @@ static void fuse_cache_wr_unlock(struct inode *inode, bool exclusive) } /* - * Request the DLM write lock covering a cached write. -ENOSYS cleared - * fc->dlm: the server has no DLM, proceed as a plain cached write. Any - * other failure means the cache would be dirtied without DLM coverage - - * the caller must fail the write instead. A granted-but-unrecorded - * lock (positive return) is covered cluster-wide; proceed regardless. + * Request the DLM write lock covering a cached write. A failure means the + * cache would be dirtied without DLM coverage - the caller must fail the + * write instead. A granted-but-unrecorded lock (positive return) is covered + * cluster-wide; proceed regardless. * * @rlock: passed straight through to fuse_get_dlm_lock(), which may * leave it READY or LOCKED; the caller must not wait for the server @@ -2142,7 +2220,7 @@ static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len, int err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE, rlock); - return (err < 0 && err != -ENOSYS) ? err : 0; + return err < 0 ? err : 0; } /* @@ -2185,7 +2263,57 @@ static bool fuse_write_stream_update(struct fuse_inode *fi, size_t len) return run >= FUSE_WRITE_STREAM_RUN; } -static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) +/* + * Buffered write without the writeback cache: every write goes to the + * server from here under the exclusive inode lock, and nothing is left + * dirty behind it. + */ +static ssize_t fuse_wt_cache_write_iter(struct kiocb *iocb, + struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t written = 0; + ssize_t err, count; + + if (fuse_inode_force_dio(inode)) + return fuse_direct_write_iter(iocb, from); + + inode_lock(inode); + + err = count = generic_write_checks(iocb, from); + if (err <= 0) + goto out; + + err = kiocb_modified(iocb); + if (err) + goto out; + + if (fuse_inode_force_dio(inode)) { + inode_unlock(inode); + return fuse_direct_write_iter(iocb, from); + } + + task_io_account_write(count); + + if (iocb->ki_flags & IOCB_DIRECT) { + written = generic_file_direct_write(iocb, from); + if (written < 0 || !iov_iter_count(from)) + goto out; + written = direct_write_fallback(iocb, from, written, + fuse_perform_write(iocb, from, 0)); + } else { + written = fuse_perform_write(iocb, from, 0); + } +out: + inode_unlock(inode); + if (written > 0) + written = generic_write_sync(iocb, written); + + return written ? written : err; +} + +static ssize_t fuse_wb_cache_write_iter(struct kiocb *iocb, + struct iov_iter *from) { struct file *file = iocb->ki_filp; struct mnt_idmap *idmap = file_mnt_idmap(file); @@ -2203,6 +2331,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) bool exclusive = true; loff_t dlm_pos = 0; size_t dlm_len = 0; + u32 request_mask; /* * An O_DIRECT write (e.g. set with F_SETFL) on a cached inode under @@ -2213,48 +2342,40 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * range lock until the WRITE returns. Send it as a plain direct * write instead, which takes neither. */ - if (fuse_inode_force_dio(inode) || - ((iocb->ki_flags & IOCB_DIRECT) && fc->writeback_cache && fc->dlm)) + if (fuse_inode_force_dio(inode) || (iocb->ki_flags & IOCB_DIRECT)) return fuse_direct_write_iter(iocb, from); - if (fc->writeback_cache) { - /* Update mode for SUID clearing, and also update size if the file - * is opened with O_APPEND mode. - */ - u32 request_mask = (file->f_flags & O_APPEND) ? - (STATX_SIZE | STATX_MODE) : STATX_MODE; - err = fuse_update_attributes(mapping->host, file, request_mask); - if (err) - return err; + /* Update mode for SUID clearing, and also update size if the file + * is opened with O_APPEND mode. + */ + request_mask = (file->f_flags & O_APPEND) ? + (STATX_SIZE | STATX_MODE) : STATX_MODE; + err = fuse_update_attributes(mapping->host, file, request_mask); + if (err) + return err; - /* - * A write that drops suid/sgid stays off the writeback path, - * so it holds i_rwsem exclusive. - * - * With handle_killpriv_v2 that is because the server does the - * killing from the WRITE itself. Without it, fuse_setattr() - * has to ask the server, and fuse_do_setattr() freezes - * writepages around that SETATTR: fuse_set_nowrite() asserts - * BUG_ON(fi->writectr < 0), which assumes an exclusive - * i_rwsem, and the DLM-relaxed buffered write path below holds - * it only shared. Two writers can both see the bits set - * before either has cleared them, and the second one would - * then oops inside spin_lock(&fi->lock). - * - * Only the DLM path needs the detour: everywhere else the - * buffered write already holds i_rwsem exclusive, so the two - * writers cannot overlap in the first place. - * - * The bits are read without the inode lock here, so a server - * attribute update can still set them between this test and - * file_remove_privs(). That leaves the same race, but only - * for writers whose mode changed underneath them, rather than - * for every write to a suid file. - */ - if (!(fc->handle_killpriv_v2 || fc->dlm) || - !setattr_should_drop_suidgid(idmap, file_inode(file))) - writeback = true; - } + /* + * A write that drops suid/sgid stays off the writeback path, so it + * holds i_rwsem exclusive and is written through. + * + * With handle_killpriv_v2 that is because the server does the + * killing from the WRITE itself. Without it, fuse_wb_setattr() + * has to ask the server, and fuse_wb_do_setattr() freezes + * writepages around that SETATTR: fuse_set_nowrite() asserts + * BUG_ON(fi->writectr < 0), which assumes an exclusive + * i_rwsem, and the DLM-relaxed buffered write path below holds + * it only shared. Two writers can both see the bits set + * before either has cleared them, and the second one would + * then oops inside spin_lock(&fi->lock). + * + * The bits are read without the inode lock here, so a server + * attribute update can still set them between this test and + * file_remove_privs(). That leaves the same race, but only + * for writers whose mode changed underneath them, rather than + * for every write to a suid file. + */ + if (!setattr_should_drop_suidgid(idmap, file_inode(file))) + writeback = true; /* * Every write that can be cached feeds the rolling mean, streamed or @@ -2263,7 +2384,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * had a chance to clamp it, which is the record the writer is working * with. */ - if (writeback && !(iocb->ki_flags & IOCB_DIRECT)) + if (writeback) stream = fuse_write_stream_update(fi, iov_iter_count(from)); /* @@ -2304,7 +2425,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * Only the append case must wait for the lock: its range depends * on i_size, which is stable only under the exclusive inode lock. */ - if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { + if (writeback && !(iocb->ki_flags & IOCB_APPEND)) { dlm_pos = iocb->ki_pos; dlm_len = iov_iter_count(from); @@ -2337,22 +2458,11 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) return err; } - - /* - * The request above may have found that the server has no DLM - * at all, in which case it cleared fc->dlm. The relaxed shared - * lock was chosen just before, while fc->dlm still read 1, and - * it is only sound under DLM: the shared path claims the i_size - * extension up front and commits i_size here rather than in - * iomap, and the grant is what keeps another node off the bytes - * meanwhile. Re-decide now, while no lock is held yet. - */ - exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); } /* note that this small code dup will save us a lot of headache later * when appends are done concurrently without using parallel direct writes */ - if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { + if (writeback && (iocb->ki_flags & IOCB_APPEND)) { /* * An append write lands at the current EOF no matter what * ki_pos holds: generic_write_checks() rewrites ki_pos to @@ -2386,7 +2496,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * range. Re-lock where the write really lands; dlm_pos tracks it * so the exact-range request further down covers the same range. */ - if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND) && + if (writeback && (iocb->ki_flags & IOCB_APPEND) && iocb->ki_pos != dlm_pos) { fuse_range_lock_release(fi, &rlock); @@ -2422,7 +2532,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * lock moves to LOCKED below, instead of leaving them next to the * write itself. kiocb_modified() -> file_remove_privs() is the one * that reaches the server: without handle_killpriv[_v2] - * fuse_setattr() kills the bits by asking it (a FUSE_GETATTR to + * fuse_wb_setattr() kills the bits by asking it (a FUSE_GETATTR to * refresh the mode, then a FUSE_SETATTR, which for a writeback * inode first flushes and freezes writepages), and * security_inode_killpriv() can drop the capability xattr with @@ -2444,13 +2554,9 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * touch -- generic_write_checks() may have trimmed count below the * provisional dlm_len -- in INIT state so an invalidate that is * already draining an overlapping range is not waited on here - * either. Only reacquired while still on the DLM path: a - * redfs_cache_wr_dlm_lock() call above may have found the server - * has no DLM and cleared fc->dlm, in which case the exclusive - * i_rwsem already serializes this write against invalidation and - * the range lock is not needed. + * either. */ - if (writeback && fc->dlm) { + if (writeback) { dlm_pos = iocb->ki_pos; dlm_len = count; @@ -2487,7 +2593,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * (see out:). This is what gives invalidation an exact conflict * test instead of draining every in-flight cached reader/writer on * the inode. It also serializes us against a concurrent - * fuse_cache_read_iter() on an overlapping range, and against + * fuse_wb_cache_read_iter() on an overlapping range, and against * another shared-locked writer on an overlapping range. */ if (range_locked) { @@ -2505,13 +2611,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) task_io_account_write(count); - if (iocb->ki_flags & IOCB_DIRECT) { - written = generic_file_direct_write(iocb, from); - if (written < 0 || !iov_iter_count(from)) - goto out; - written = direct_write_fallback(iocb, from, written, - fuse_perform_write(iocb, from, 0)); - } else if (through) { + if (through) { loff_t pos = iocb->ki_pos; /* @@ -2586,7 +2686,14 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) pagecache_isize_extended(inode, orig_size, pos); } - written = fuse_writeback_write_iter(iocb, from, file); + /* + * Use iomap so that we can do granular uptodate reads + * and granular dirty tracking for large folios. + */ + written = iomap_file_buffered_write(iocb, from, + &fuse_iomap_ops, + &fuse_iomap_write_ops, + file); /* * Reconcile the speculative extension with what was actually @@ -2951,7 +3058,17 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) return res; } -static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +typedef ssize_t (*fuse_rw_iter_t)(struct kiocb *, struct iov_iter *); + +/* + * The per-open routing shared by both cache modes: direct IO is decided by + * the open and the remote-modify latch, only the cached path differs between + * writethrough and writeback. DAX inodes and passthrough opens have + * operations of their own. + */ +static ssize_t fuse_file_read_iter_common(struct kiocb *iocb, + struct iov_iter *to, + fuse_rw_iter_t cache_read_iter) { struct file *file = iocb->ki_filp; struct fuse_file *ff = file->private_data; @@ -2960,19 +3077,60 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) if (fuse_is_bad(inode)) return -EIO; - if (FUSE_IS_DAX(inode)) - return fuse_dax_read_iter(iocb, to); - - /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) return fuse_direct_read_iter(iocb, to); - else if (fuse_file_passthrough(ff)) - return fuse_passthrough_read_iter(iocb, to); - else - return fuse_cache_read_iter(iocb, to); + + return cache_read_iter(iocb, to); +} + +static ssize_t fuse_wt_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + return fuse_file_read_iter_common(iocb, to, fuse_wt_cache_read_iter); +} + +static ssize_t fuse_wb_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + return fuse_file_read_iter_common(iocb, to, fuse_wb_cache_read_iter); +} + +static ssize_t fuse_file_write_iter_common(struct kiocb *iocb, + struct iov_iter *from, + fuse_rw_iter_t cache_write_iter) +{ + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); + + if (fuse_is_bad(inode)) + return -EIO; + + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) + return fuse_direct_write_iter(iocb, from); + + return cache_write_iter(iocb, from); +} + +static ssize_t fuse_wt_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + return fuse_file_write_iter_common(iocb, from, + fuse_wt_cache_write_iter); +} + +static ssize_t fuse_wb_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + return fuse_file_write_iter_common(iocb, from, + fuse_wb_cache_write_iter); } -static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +#ifdef CONFIG_FUSE_PASSTHROUGH +/* + * A passthrough open may also carry FOPEN_DIRECT_IO, which sends its reads + * and writes to the server instead of the backing file; mmap still goes to + * the backing file. The remote-modify latch routes the same way: it can be + * set by a direct IO writer of the inode before the passthrough open. + */ +static ssize_t fuse_passthrough_file_read_iter(struct kiocb *iocb, + struct iov_iter *to) { struct file *file = iocb->ki_filp; struct fuse_file *ff = file->private_data; @@ -2981,42 +3139,52 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) if (fuse_is_bad(inode)) return -EIO; - if (FUSE_IS_DAX(inode)) - return fuse_dax_write_iter(iocb, from); + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) + return fuse_direct_read_iter(iocb, to); + + return fuse_passthrough_read_iter(iocb, to); +} + +static ssize_t fuse_passthrough_file_write_iter(struct kiocb *iocb, + struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); + + if (fuse_is_bad(inode)) + return -EIO; - /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) return fuse_direct_write_iter(iocb, from); - else if (fuse_file_passthrough(ff)) - return fuse_passthrough_write_iter(iocb, from); - else - return fuse_cache_write_iter(iocb, from); + + return fuse_passthrough_write_iter(iocb, from); } -static ssize_t fuse_splice_read(struct file *in, loff_t *ppos, - struct pipe_inode_info *pipe, size_t len, - unsigned int flags) +static ssize_t fuse_passthrough_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) { struct fuse_file *ff = in->private_data; - /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO)) - return fuse_passthrough_splice_read(in, ppos, pipe, len, flags); - else + if (ff->open_flags & FOPEN_DIRECT_IO) return filemap_splice_read(in, ppos, pipe, len, flags); + + return fuse_passthrough_splice_read(in, ppos, pipe, len, flags); } -static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out, - loff_t *ppos, size_t len, unsigned int flags) +static ssize_t fuse_passthrough_file_splice_write(struct pipe_inode_info *pipe, + struct file *out, loff_t *ppos, + size_t len, unsigned int flags) { struct fuse_file *ff = out->private_data; - /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO)) - return fuse_passthrough_splice_write(pipe, out, ppos, len, flags); - else + if (ff->open_flags & FOPEN_DIRECT_IO) return iter_file_splice_write(pipe, out, ppos, len, flags); + + return fuse_passthrough_splice_write(pipe, out, ppos, len, flags); } +#endif static void fuse_writepage_free(struct fuse_writepage_args *wpa) { @@ -3555,21 +3723,21 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l args.out_args[0].value = &outarg; err = fuse_simple_request(fm, &args); if (err == -ENOSYS) { - fc->dlm = 0; - err = 0; + /* a server without DLM cannot serve a writeback cache */ + pr_warn("fuse: server does not implement FUSE_DLM_WB_LOCK, aborting connection\n"); + fuse_abort_conn(fc); + return -EIO; } if (!err && - fc->dlm && - (outarg.start > inarg.start || - outarg.end < inarg.end)) { + (outarg.start > inarg.start || outarg.end < inarg.end)) { /* fuse server is seriously broken */ pr_warn("fuse: dlm lock request for %llu:%llu bytes returned %llu:%llu bytes\n", inarg.start, inarg.end, outarg.start, outarg.end); fuse_abort_conn(fc); err = -EINVAL; } - if (!err && fc->dlm) + if (!err) fuse_dlm_attr_timeout(get_fuse_inode(inode)); return err; } @@ -3618,72 +3786,10 @@ static pgoff_t fuse_fault_batch_base(pgoff_t index) * - sync(2) * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER */ -static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) +static vm_fault_t fuse_page_mkwrite_finish(struct vm_fault *vmf) { struct folio *folio = page_folio(vmf->page); - struct file *file = vmf->vma->vm_file; - struct inode *inode = file_inode(file); - struct fuse_mount *fm = get_fuse_mount(inode); - loff_t pos = vmf->pgoff << PAGE_SHIFT; - - if (fm->fc->writeback_cache && fm->fc->dlm) { - /* - * Writenotify routes every clean-to-dirty transition of a - * shared mapping through here, but the WRITE grant itself - * is taken by fuse_filemap_fault(): shared-writable VMAs - * get no ->map_pages under DLM (see fuse_file_mmap()), so - * every PTE in such a VMA was installed by a fault that - * recorded a WRITE grant for its batch window. Only - * verify here, and only without blocking -- nothing may - * sleep on the range lock or the server while mmap_lock - * (or the per-VMA lock) is held: the buffered write path - * holds the range lock LOCKED while faulting in its - * source buffer (which takes mmap_lock), so with a - * pending mmap_lock writer queued between the two, - * blocking here deadlocks three ways -- the cycle - * 94effa2 ("fuse: do not block the fault path under - * mmap_lock") removed from the fault path -- and - * page_mkwrite has no VM_FAULT_RETRY protocol to drop - * the lock and come back with. - * - * Same fence as fuse_filemap_fault()'s fast path: only - * once LOCKED can the recorded grant be trusted. On a - * miss, NOPAGE. A contended trylock re-faults with - * mmap_lock released in between, so the holder can make - * progress; a verified miss means a revoke is in flight - * (only an invalidate removes a recorded grant), whose - * unmap_mapping_range() clears the PTE, so the re-fault - * goes through fuse_filemap_fault(), which re-takes the - * grant with the proper drop-and-retry dance. - */ - struct fuse_inode *fi = get_fuse_inode(inode); - struct fuse_range_lock rlock; - bool held = false; - - if (fuse_range_lock_try_acquire_init(fi, &rlock, pos, - pos + PAGE_SIZE - 1, - FUSE_RANGE_LOCK_WRITE)) { - bool locked = - fuse_range_lock_try_mark_locked(fi, &rlock); - - held = locked && - fuse_dlm_lock_is_held(fi, pos, PAGE_SIZE, - FUSE_PAGE_LOCK_WRITE); - if (locked && !held) - pr_warn_ratelimited("fuse: mkwrite with no WRITE grant, nodeid %llu idx %lu\n", - fi->nodeid, vmf->pgoff); - fuse_range_lock_release(fi, &rlock); - } - - if (!held) - return VM_FAULT_NOPAGE; - } else if (fm->fc->dlm) { - size_t length = PAGE_SIZE; - int err = fuse_get_page_mkwrite_lock(file, pos, length); - if (err < 0) { - return vmf_error(err); - } - } + struct inode *inode = file_inode(vmf->vma->vm_file); file_update_time(vmf->vma->vm_file); folio_lock(folio); @@ -3696,6 +3802,70 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) return VM_FAULT_LOCKED; } +static vm_fault_t fuse_wt_page_mkwrite(struct vm_fault *vmf) +{ + struct file *file = vmf->vma->vm_file; + loff_t pos = vmf->pgoff << PAGE_SHIFT; + int err; + + err = fuse_get_page_mkwrite_lock(file, pos, PAGE_SIZE); + if (err < 0) + return vmf_error(err); + + return fuse_page_mkwrite_finish(vmf); +} + +/* + * Writenotify routes every clean-to-dirty transition of a shared mapping + * through here, but the WRITE grant itself is taken by + * fuse_wb_filemap_fault(): shared-writable VMAs get no ->map_pages under + * DLM (see fuse_wb_mmap()), so every PTE in such a VMA was installed by a + * fault that recorded a WRITE grant for its batch window. Only verify + * here, and only without blocking -- nothing may sleep on the range lock + * or the server while mmap_lock (or the per-VMA lock) is held: the + * buffered write path holds the range lock LOCKED while faulting in its + * source buffer (which takes mmap_lock), so with a pending mmap_lock + * writer queued between the two, blocking here deadlocks three ways -- + * the cycle 94effa2 ("fuse: do not block the fault path under mmap_lock") + * removed from the fault path -- and page_mkwrite has no VM_FAULT_RETRY + * protocol to drop the lock and come back with. + * + * Same fence as fuse_wb_filemap_fault()'s fast path: only once LOCKED can + * the recorded grant be trusted. On a miss, NOPAGE. A contended trylock + * re-faults with mmap_lock released in between, so the holder can make + * progress; a verified miss means a revoke is in flight (only an + * invalidate removes a recorded grant), whose unmap_mapping_range() clears + * the PTE, so the re-fault goes through fuse_wb_filemap_fault(), which + * re-takes the grant with the proper drop-and-retry dance. + */ +static vm_fault_t fuse_wb_page_mkwrite(struct vm_fault *vmf) +{ + struct inode *inode = file_inode(vmf->vma->vm_file); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_range_lock rlock; + loff_t pos = vmf->pgoff << PAGE_SHIFT; + bool held = false; + + if (fuse_range_lock_try_acquire_init(fi, &rlock, pos, + pos + PAGE_SIZE - 1, + FUSE_RANGE_LOCK_WRITE)) { + bool locked = fuse_range_lock_try_mark_locked(fi, &rlock); + + held = locked && + fuse_dlm_lock_is_held(fi, pos, PAGE_SIZE, + FUSE_PAGE_LOCK_WRITE); + if (locked && !held) + pr_warn_ratelimited("fuse: mkwrite with no WRITE grant, nodeid %llu idx %lu\n", + fi->nodeid, vmf->pgoff); + fuse_range_lock_release(fi, &rlock); + } + + if (!held) + return VM_FAULT_NOPAGE; + + return fuse_page_mkwrite_finish(vmf); +} + /* * Does filemap_fault() need a synchronous READ before it can map * @index? True when the folio is absent or not uptodate and the fault @@ -3735,7 +3905,7 @@ static bool fuse_fault_needs_read(struct inode *inode, pgoff_t index) * * A short read (EOF) leaves the tails zeroed (page_zeroing), which is * what the page cache stores for the EOF pages anyway. i_size is - * deliberately NOT shrunk here (no fuse_short_read()), to keep this + * deliberately NOT shrunk here (no fuse_wt_read_size()), to keep this * reader free of attribute side effects. * * On success returns the number of folios in @folios (>= 1, @index @@ -3872,306 +4042,300 @@ static void fuse_fault_install_folio(struct inode *inode, folio_put(stale); } -static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) +static vm_fault_t fuse_wb_filemap_fault(struct vm_fault *vmf) { struct file *file = vmf->vma->vm_file; struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_range_lock rlock; + loff_t pos = vmf->pgoff << PAGE_SHIFT; + /* + * Grant and populate in whole batch windows. A one-page + * grant breaks spanning accesses (one instruction + * touching two pages): the neighbour's fault finds no + * recorded grant, and its own fuse_get_dlm_lock() round + * trip costs a revoke that drops the page just mapped -- + * a deterministic ping-pong (the generic/091 livelock). + * With the grant covering the same window the populate + * read fills, the neighbour's fault passes the fast + * path's is_held() and residency checks and maps with no + * server op at all. + */ + loff_t bpos = (loff_t)fuse_fault_batch_base(vmf->pgoff) << PAGE_SHIFT; + size_t blen = (size_t)FUSE_FAULT_BATCH_PAGES << PAGE_SHIFT; + enum fuse_page_lock_mode mode = FUSE_PAGE_LOCK_READ; + enum fuse_range_lock_mode range_mode = FUSE_RANGE_LOCK_READ; + vm_fault_t ret; + int tries; + int err; - if (fc->writeback_cache && fc->dlm) { - struct fuse_inode *fi = get_fuse_inode(inode); - struct fuse_range_lock rlock; - loff_t pos = vmf->pgoff << PAGE_SHIFT; - /* - * Grant and populate in whole batch windows. A one-page - * grant breaks spanning accesses (one instruction - * touching two pages): the neighbour's fault finds no - * recorded grant, and its own fuse_get_dlm_lock() round - * trip costs a revoke that drops the page just mapped -- - * a deterministic ping-pong (the generic/091 livelock). - * With the grant covering the same window the populate - * read fills, the neighbour's fault passes the fast - * path's is_held() and residency checks and maps with no - * server op at all. - */ - loff_t bpos = (loff_t)fuse_fault_batch_base(vmf->pgoff) - << PAGE_SHIFT; - size_t blen = (size_t)FUSE_FAULT_BATCH_PAGES << PAGE_SHIFT; - enum fuse_page_lock_mode mode = FUSE_PAGE_LOCK_READ; - enum fuse_range_lock_mode range_mode = FUSE_RANGE_LOCK_READ; - vm_fault_t ret; - int tries; - int err; - - if ((vmf->vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == - (VM_SHARED | VM_MAYWRITE)) { - mode = FUSE_PAGE_LOCK_WRITE; - range_mode = FUSE_RANGE_LOCK_WRITE; - } + if ((vmf->vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == + (VM_SHARED | VM_MAYWRITE)) { + mode = FUSE_PAGE_LOCK_WRITE; + range_mode = FUSE_RANGE_LOCK_WRITE; + } - /* - * Fast path: range lock uncontended, the grant already - * recorded, and the folio already resident. Nothing here - * blocks, so it is safe under mmap_lock (or the per-VMA - * lock). LOCKED first, then is_held(): only once LOCKED, - * which fences any concurrent revoke, can the recorded - * grant be trusted -- same order as fuse_get_dlm_lock()'s - * own fast path. The folio check keeps filemap_fault() - * from issuing a synchronous READ with the range LOCKED: - * nothing held once LOCKED may wait on the server (see the - * blocking path below), so a missing folio falls through - * to the slow path, which populates it under INIT. - */ - if (fuse_range_lock_try_acquire_init(fi, &rlock, pos, - pos + PAGE_SIZE - 1, - range_mode)) { - if (fuse_range_lock_try_mark_locked(fi, &rlock) && - fuse_dlm_lock_is_held(fi, pos, PAGE_SIZE, mode) && - !fuse_fault_needs_read(inode, vmf->pgoff)) { - ret = filemap_fault(vmf); - fuse_range_lock_release(fi, &rlock); - return ret; - } + /* + * Fast path: range lock uncontended, the grant already + * recorded, and the folio already resident. Nothing here + * blocks, so it is safe under mmap_lock (or the per-VMA + * lock). LOCKED first, then is_held(): only once LOCKED, + * which fences any concurrent revoke, can the recorded + * grant be trusted -- same order as fuse_get_dlm_lock()'s + * own fast path. The folio check keeps filemap_fault() + * from issuing a synchronous READ with the range LOCKED: + * nothing held once LOCKED may wait on the server (see the + * blocking path below), so a missing folio falls through + * to the slow path, which populates it under INIT. + */ + if (fuse_range_lock_try_acquire_init(fi, &rlock, pos, + pos + PAGE_SIZE - 1, + range_mode)) { + if (fuse_range_lock_try_mark_locked(fi, &rlock) && + fuse_dlm_lock_is_held(fi, pos, PAGE_SIZE, mode) && + !fuse_fault_needs_read(inode, vmf->pgoff)) { + ret = filemap_fault(vmf); fuse_range_lock_release(fi, &rlock); + return ret; } + fuse_range_lock_release(fi, &rlock); + } - /* - * Slow path: the range lock or the grant needs waiting -- - * a range conflict, an invalidation draining the range, or - * a DLM round trip to the server. Blocking here while - * holding mmap_lock recreates the ABBA the read/write - * paths set up the other way around: they hold the range - * lock while faulting in their user buffers, which takes - * mmap_lock. Drop the fault lock first, do the blocking - * work with only a file reference pinning the inode, then - * ask for the fault to be retried; the retry takes the - * fast path above off the now-recorded grant. The range - * lock is not held across the retry (there would be - * nowhere to release it), so the retried fault can still - * miss and come back here; each pass leaves a recorded - * grant behind, which only a real invalidation takes away - * again. - * - * FAULT_FLAG_TRIED does not demote a pass to the blocking - * path below: retries may repeat (4064b98 "mm: allow - * VM_FAULT_RETRY for multiple times"), and blocking on - * the range lock with mmap_lock held closes a three-party - * cycle whenever the lock holder is a buffered write - * faulting in its source buffer (which waits on - * mmap_lock) behind a queued mmap_lock writer. A miss on - * the retried fast path only means the range is contended - * again; waiting for it must happen with mmap_lock - * dropped, every time. - * - * The flip side is that an error met with mmap_lock - * dropped cannot be reported: only VM_FAULT_RETRY (or - * COMPLETED) may be returned then, and a retried fault - * that keeps failing comes straight back here. An - * aborted connection fails every request, so the task - * would refault forever: it burns CPU in this path, - * only SIGKILL ends it, and its mapping keeps the mount - * busy. Check it while the fault lock is still held, - * where SIGBUS can be returned. - */ - if (!READ_ONCE(fc->connected)) - return VM_FAULT_SIGBUS; + /* + * Slow path: the range lock or the grant needs waiting -- + * a range conflict, an invalidation draining the range, or + * a DLM round trip to the server. Blocking here while + * holding mmap_lock recreates the ABBA the read/write + * paths set up the other way around: they hold the range + * lock while faulting in their user buffers, which takes + * mmap_lock. Drop the fault lock first, do the blocking + * work with only a file reference pinning the inode, then + * ask for the fault to be retried; the retry takes the + * fast path above off the now-recorded grant. The range + * lock is not held across the retry (there would be + * nowhere to release it), so the retried fault can still + * miss and come back here; each pass leaves a recorded + * grant behind, which only a real invalidation takes away + * again. + * + * FAULT_FLAG_TRIED does not demote a pass to the blocking + * path below: retries may repeat (4064b98 "mm: allow + * VM_FAULT_RETRY for multiple times"), and blocking on + * the range lock with mmap_lock held closes a three-party + * cycle whenever the lock holder is a buffered write + * faulting in its source buffer (which waits on + * mmap_lock) behind a queued mmap_lock writer. A miss on + * the retried fast path only means the range is contended + * again; waiting for it must happen with mmap_lock + * dropped, every time. + * + * The flip side is that an error met with mmap_lock + * dropped cannot be reported: only VM_FAULT_RETRY (or + * COMPLETED) may be returned then, and a retried fault + * that keeps failing comes straight back here. An + * aborted connection fails every request, so the task + * would refault forever: it burns CPU in this path, + * only SIGKILL ends it, and its mapping keeps the mount + * busy. Check it while the fault lock is still held, + * where SIGBUS can be returned. + */ + if (!READ_ONCE(fc->connected)) + return VM_FAULT_SIGBUS; - if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { - if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) - return VM_FAULT_RETRY; + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { + if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) + return VM_FAULT_RETRY; - file = get_file(file); - release_fault_lock(vmf); + file = get_file(file); + release_fault_lock(vmf); - err = fuse_range_lock_acquire_init(fi, &rlock, bpos, - bpos + blen - 1, - range_mode); - if (!err) { - /* - * A hard error is not failed here, it - * cannot be with the fault lock dropped: - * the retried fault meets it again. A - * lost connection ends at the check above. - */ - fuse_get_dlm_lock(file, bpos, blen, - mode, &rlock); - /* - * Populate the folios too, or the retried - * fault's fast path would refuse the grant - * it just recorded (missing folio) and come - * straight back here. Read under INIT, - * install under LOCKED; see the blocking - * path below. - */ - if (fuse_fault_needs_read(inode, - vmf->pgoff)) { - struct folio *folios[FUSE_FAULT_BATCH_PAGES]; - pgoff_t fbase; - int nr, i; - - fuse_range_lock_mark_init(fi, &rlock); - nr = fuse_fault_read_private_folios( - file, inode, vmf->pgoff, - &fbase, folios); - if (nr > 0) { - bool locked = !fuse_range_lock_mark_locked(fi, &rlock); - - for (i = 0; i < nr; i++) { - if (locked) - fuse_fault_install_folio(inode, folios[i], fbase + i); - folio_put(folios[i]); - } + err = fuse_range_lock_acquire_init(fi, &rlock, bpos, + bpos + blen - 1, + range_mode); + if (!err) { + /* + * A hard error is not failed here, it + * cannot be with the fault lock dropped: + * the retried fault meets it again. A + * lost connection ends at the check above. + */ + fuse_get_dlm_lock(file, bpos, blen, mode, &rlock); + /* + * Populate the folios too, or the retried + * fault's fast path would refuse the grant + * it just recorded (missing folio) and come + * straight back here. Read under INIT, + * install under LOCKED; see the blocking + * path below. + */ + if (fuse_fault_needs_read(inode, vmf->pgoff)) { + struct folio *folios[FUSE_FAULT_BATCH_PAGES]; + pgoff_t fbase; + int nr, i; + + fuse_range_lock_mark_init(fi, &rlock); + nr = fuse_fault_read_private_folios( + file, inode, vmf->pgoff, + &fbase, folios); + if (nr > 0) { + bool locked = !fuse_range_lock_mark_locked(fi, &rlock); + + for (i = 0; i < nr; i++) { + if (locked) + fuse_fault_install_folio(inode, folios[i], fbase + i); + folio_put(folios[i]); } } - fuse_range_lock_release(fi, &rlock); } - fput(file); - return VM_FAULT_RETRY; + fuse_range_lock_release(fi, &rlock); } + fput(file); + return VM_FAULT_RETRY; + } - /* - * Retry not allowed (a caller without - * FAULT_FLAG_ALLOW_RETRY): block in place. The waits are - * killable, and a same-task fault-in never conflicts with - * its own IO's range lock, so this cannot deadlock on - * itself; it can still stack behind a pending mmap_lock - * writer the way any blocking fault can. - * - * Nothing held once the range lock is LOCKED may wait for - * the server (see fuse_cache_read_iter), and - * filemap_fault() on a non-resident folio does exactly - * that: a synchronous READ the server can queue behind a - * revoke already in flight for this range, whose - * invalidation then waits on this LOCKED range. ABBA - * through the server (the generic/091 hang). Nor can the - * page-cache read path be used with the range lock merely - * demoted: it keeps the folio locked in the mapping across - * the READ, and the invalidation blocks on that folio lock - * instead -- same cycle, one lock over. So read into a - * private folio with the range lock demoted to INIT - * (nothing the invalidation needs is held), then promote - * and install it, and keep the range LOCKED from the - * install through filemap_fault() so the READ's own - * revoke notification (the server revokes the requester's - * grant to serve the READ) cannot drop the folio before - * the PTE is mapped; see fuse_fault_install_folio(). - */ - err = fuse_range_lock_acquire_init(fi, &rlock, bpos, - bpos + blen - 1, - range_mode); - if (err) - return vmf_error(err); + /* + * Retry not allowed (a caller without + * FAULT_FLAG_ALLOW_RETRY): block in place. The waits are + * killable, and a same-task fault-in never conflicts with + * its own IO's range lock, so this cannot deadlock on + * itself; it can still stack behind a pending mmap_lock + * writer the way any blocking fault can. + * + * Nothing held once the range lock is LOCKED may wait for + * the server (see fuse_wb_cache_read_iter), and + * filemap_fault() on a non-resident folio does exactly + * that: a synchronous READ the server can queue behind a + * revoke already in flight for this range, whose + * invalidation then waits on this LOCKED range. ABBA + * through the server (the generic/091 hang). Nor can the + * page-cache read path be used with the range lock merely + * demoted: it keeps the folio locked in the mapping across + * the READ, and the invalidation blocks on that folio lock + * instead -- same cycle, one lock over. So read into a + * private folio with the range lock demoted to INIT + * (nothing the invalidation needs is held), then promote + * and install it, and keep the range LOCKED from the + * install through filemap_fault() so the READ's own + * revoke notification (the server revokes the requester's + * grant to serve the READ) cannot drop the folio before + * the PTE is mapped; see fuse_fault_install_folio(). + */ + err = fuse_range_lock_acquire_init(fi, &rlock, bpos, bpos + blen - 1, + range_mode); + if (err) + return vmf_error(err); - for (tries = 0; ; tries++) { - struct folio *folios[FUSE_FAULT_BATCH_PAGES]; - pgoff_t fbase; - int nr, i; + for (tries = 0; ; tries++) { + struct folio *folios[FUSE_FAULT_BATCH_PAGES]; + pgoff_t fbase; + int nr, i; - err = fuse_get_dlm_lock(file, bpos, blen, mode, - &rlock); - if (err < 0 && err != -ENOSYS) { - fuse_range_lock_release(fi, &rlock); - return vmf_error(err); - } + err = fuse_get_dlm_lock(file, bpos, blen, mode, &rlock); + if (err < 0) { + fuse_range_lock_release(fi, &rlock); + return vmf_error(err); + } - err = fuse_range_lock_mark_locked(fi, &rlock); - if (err) { - fuse_range_lock_release(fi, &rlock); - return vmf_error(err); - } + err = fuse_range_lock_mark_locked(fi, &rlock); + if (err) { + fuse_range_lock_release(fi, &rlock); + return vmf_error(err); + } - if (!fuse_fault_needs_read(inode, vmf->pgoff)) - break; + if (!fuse_fault_needs_read(inode, vmf->pgoff)) + break; + /* + * A pass loops only when the folio went missing + * again between the previous install and this + * pass's residency check (a revoke landed in the + * gap), or on a bounced READ. Each successful + * install exits the loop under LOCKED, so a storm + * costs refaults, not loop passes; still killable + * in case the server bounces READs indefinitely. + */ + if (fatal_signal_pending(current)) { + fuse_range_lock_release(fi, &rlock); + return vmf_error(-EINTR); + } + if (tries && (tries % 64) == 0) + pr_warn_ratelimited("fuse: fault populate slow, nodeid %llu idx %lu: %d passes\n", + fi->nodeid, vmf->pgoff, tries); + + fuse_range_lock_mark_init(fi, &rlock); + nr = fuse_fault_read_private_folios(file, inode, vmf->pgoff, + &fbase, folios); + if (nr < 0) { /* - * A pass loops only when the folio went missing - * again between the previous install and this - * pass's residency check (a revoke landed in the - * gap), or on a bounced READ. Each successful - * install exits the loop under LOCKED, so a storm - * costs refaults, not loop passes; still killable - * in case the server bounces READs indefinitely. + * -EDEADLK/-EAGAIN is the server bouncing + * a READ it cannot serve yet (a revoke in + * flight); back off and retry, the same + * thing AOP_TRUNCATED_PAGE does for the + * page-cache read path. Anything else is + * a real read error: let filemap_fault() + * run and turn it into SIGBUS -- the + * server answers (with the error) rather + * than queueing, so nothing hangs. */ - if (fatal_signal_pending(current)) { - fuse_range_lock_release(fi, &rlock); - return vmf_error(-EINTR); + if (nr == -EDEADLK || nr == -EAGAIN) { + msleep_interruptible(2); + continue; } - if (tries && (tries % 64) == 0) - pr_warn_ratelimited("fuse: fault populate slow, nodeid %llu idx %lu: %d passes\n", - fi->nodeid, vmf->pgoff, - tries); - - fuse_range_lock_mark_init(fi, &rlock); - nr = fuse_fault_read_private_folios(file, inode, - vmf->pgoff, - &fbase, folios); - if (nr < 0) { - /* - * -EDEADLK/-EAGAIN is the server bouncing - * a READ it cannot serve yet (a revoke in - * flight); back off and retry, the same - * thing AOP_TRUNCATED_PAGE does for the - * page-cache read path. Anything else is - * a real read error: let filemap_fault() - * run and turn it into SIGBUS -- the - * server answers (with the error) rather - * than queueing, so nothing hangs. - */ - if (nr == -EDEADLK || nr == -EAGAIN) { - msleep_interruptible(2); - continue; - } - err = fuse_range_lock_mark_locked(fi, &rlock); - if (err) { - fuse_range_lock_release(fi, &rlock); - return vmf_error(err); - } - break; - } - err = fuse_range_lock_mark_locked(fi, &rlock); if (err) { - for (i = 0; i < nr; i++) - folio_put(folios[i]); fuse_range_lock_release(fi, &rlock); return vmf_error(err); } - for (i = 0; i < nr; i++) { - fuse_fault_install_folio(inode, folios[i], - fbase + i); - folio_put(folios[i]); - } - /* - * Exit under LOCKED: filemap_fault() below maps - * the just-installed folio without a server wait - * (present and uptodate), and the range lock -- - * spanning the whole batch window -- keeps any - * pending revoke's page drop out until the PTE is - * in place. If the install lost to -EEXIST on a - * !uptodate foreign folio it could not fill (a - * read in flight holds the folio lock), loop - * instead: mapping it would trigger a - * synchronous READ with the range LOCKED, and - * the in-flight read resolves the index anyway. - */ - if (!fuse_fault_needs_read(inode, vmf->pgoff)) - break; - fuse_range_lock_mark_init(fi, &rlock); + break; } - ret = filemap_fault(vmf); - fuse_range_lock_release(fi, &rlock); - return ret; + err = fuse_range_lock_mark_locked(fi, &rlock); + if (err) { + for (i = 0; i < nr; i++) + folio_put(folios[i]); + fuse_range_lock_release(fi, &rlock); + return vmf_error(err); + } + for (i = 0; i < nr; i++) { + fuse_fault_install_folio(inode, folios[i], fbase + i); + folio_put(folios[i]); + } + /* + * Exit under LOCKED: filemap_fault() below maps + * the just-installed folio without a server wait + * (present and uptodate), and the range lock -- + * spanning the whole batch window -- keeps any + * pending revoke's page drop out until the PTE is + * in place. If the install lost to -EEXIST on a + * !uptodate foreign folio it could not fill (a + * read in flight holds the folio lock), loop + * instead: mapping it would trigger a + * synchronous READ with the range LOCKED, and + * the in-flight read resolves the index anyway. + */ + if (!fuse_fault_needs_read(inode, vmf->pgoff)) + break; + fuse_range_lock_mark_init(fi, &rlock); } - return filemap_fault(vmf); + ret = filemap_fault(vmf); + fuse_range_lock_release(fi, &rlock); + return ret; } -static const struct vm_operations_struct fuse_file_vm_ops = { +static const struct vm_operations_struct fuse_wt_file_vm_ops = { + .close = fuse_vma_close, + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = fuse_wt_page_mkwrite, +}; + +static const struct vm_operations_struct fuse_wb_file_vm_ops = { .close = fuse_vma_close, - .fault = fuse_filemap_fault, + .fault = fuse_wb_filemap_fault, .map_pages = filemap_map_pages, - .page_mkwrite = fuse_page_mkwrite, + .page_mkwrite = fuse_wb_page_mkwrite, }; /* @@ -4181,35 +4345,34 @@ static const struct vm_operations_struct fuse_file_vm_ops = { * grant (read(2), readahead) could take its first write in * ->page_mkwrite with no WRITE grant ever requested -- and * page_mkwrite cannot take one there (no retry protocol; see - * fuse_page_mkwrite()). Without fault-around, every first touch goes - * through fuse_filemap_fault(), which records a WRITE grant for the + * fuse_wb_page_mkwrite()). Without fault-around, every first touch goes + * through fuse_wb_filemap_fault(), which records a WRITE grant for the * batch window before any PTE is installed. */ -static const struct vm_operations_struct fuse_file_wb_dlm_vm_ops = { +static const struct vm_operations_struct fuse_wb_dlm_vm_ops = { .close = fuse_vma_close, - .fault = fuse_filemap_fault, - .page_mkwrite = fuse_page_mkwrite, + .fault = fuse_wb_filemap_fault, + .page_mkwrite = fuse_wb_page_mkwrite, }; -static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) +/* + * ->mmap for both cache modes; @vm_ops is what a cached mapping gets, which + * is the only thing the two modes decide differently. + */ +static int fuse_file_mmap_common(struct file *file, struct vm_area_struct *vma, + const struct vm_operations_struct *vm_ops) { struct fuse_file *ff = file->private_data; struct fuse_conn *fc = ff->fm->fc; struct inode *inode = file_inode(file); int rc; - /* DAX mmap is superior to direct_io mmap */ - if (FUSE_IS_DAX(inode)) - return fuse_dax_mmap(file, vma); - /* - * If inode is in passthrough io mode, because it has some file open - * in passthrough mode, either mmap to backing file or fail mmap, - * because mixing cached mmap and passthrough io mode is not allowed. + * If inode is in passthrough io mode, because it has some other file + * open in passthrough mode, fail mmap, because mixing cached mmap and + * passthrough io mode is not allowed. */ - if (fuse_file_passthrough(ff)) - return fuse_passthrough_mmap(file, vma); - else if (fuse_inode_backing(get_fuse_inode(inode))) + if (fuse_inode_backing(get_fuse_inode(inode))) return -ENODEV; /* @@ -4270,14 +4433,25 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) fuse_link_write_file(file); file_accessed(file); - if (fc->writeback_cache && fc->dlm && - (vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) - vma->vm_ops = &fuse_file_wb_dlm_vm_ops; - else - vma->vm_ops = &fuse_file_vm_ops; + vma->vm_ops = vm_ops; return 0; } +static int fuse_wt_mmap(struct file *file, struct vm_area_struct *vma) +{ + return fuse_file_mmap_common(file, vma, &fuse_wt_file_vm_ops); +} + +static int fuse_wb_mmap(struct file *file, struct vm_area_struct *vma) +{ + const struct vm_operations_struct *vm_ops = &fuse_wb_file_vm_ops; + + if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) + vm_ops = &fuse_wb_dlm_vm_ops; + + return fuse_file_mmap_common(file, vma, vm_ops); +} + static int convert_fuse_file_lock(struct fuse_conn *fc, const struct fuse_file_lock *ffl, struct file_lock *fl) @@ -5059,20 +5233,151 @@ static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off, return ret; } -static const struct file_operations fuse_file_operations = { +/* + * Regular-file operations without the writeback cache: every write goes to + * the server as it is made, and no DLM range locking is involved. + */ +static const struct file_operations fuse_wt_file_operations = { + .llseek = fuse_file_llseek, + .read_iter = fuse_wt_read_iter, + .write_iter = fuse_wt_write_iter, + .mmap = fuse_wt_mmap, + .open = fuse_wt_open, + .flush = fuse_wt_flush, + .release = fuse_wt_release, + .fsync = fuse_fsync, + .lock = fuse_file_lock, + .get_unmapped_area = thp_get_unmapped_area, + .flock = fuse_file_flock, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .unlocked_ioctl = fuse_file_ioctl, + .compat_ioctl = fuse_file_compat_ioctl, + .poll = fuse_file_poll, + .fallocate = fuse_file_fallocate, + .copy_file_range = fuse_copy_file_range, + .setlease = generic_setlease, +}; + +/* + * Regular-file operations with the writeback cache, which is kept coherent + * across nodes by the DLM range locks. + */ +static const struct file_operations fuse_wb_file_operations = { + .llseek = fuse_file_llseek, + .read_iter = fuse_wb_read_iter, + .write_iter = fuse_wb_write_iter, + .mmap = fuse_wb_mmap, + .open = fuse_wb_open, + .flush = fuse_wb_flush, + .release = fuse_wb_release, + .fsync = fuse_fsync, + .lock = fuse_file_lock, + .get_unmapped_area = thp_get_unmapped_area, + .flock = fuse_file_flock, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .unlocked_ioctl = fuse_file_ioctl, + .compat_ioctl = fuse_file_compat_ioctl, + .poll = fuse_file_poll, + .fallocate = fuse_file_fallocate, + .copy_file_range = fuse_copy_file_range, + .setlease = generic_setlease, +}; + +#ifdef CONFIG_FUSE_DAX +/* + * DAX inodes bypass the page cache for data: reads, writes and mappings go + * to the DAX window. Open, flush and release keep the cache mode's + * handling of the server-side truncate and of the FLUSH. Installed by + * fuse_dax_inode_init(). + */ +const struct file_operations fuse_wt_dax_file_operations = { + .llseek = fuse_file_llseek, + .read_iter = fuse_dax_read_iter, + .write_iter = fuse_dax_write_iter, + .mmap = fuse_dax_mmap, + .open = fuse_wt_dax_open, + .flush = fuse_wt_flush, + .release = fuse_wt_release, + .fsync = fuse_fsync, + .lock = fuse_file_lock, + .get_unmapped_area = thp_get_unmapped_area, + .flock = fuse_file_flock, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .unlocked_ioctl = fuse_file_ioctl, + .compat_ioctl = fuse_file_compat_ioctl, + .poll = fuse_file_poll, + .fallocate = fuse_file_fallocate, + .copy_file_range = fuse_copy_file_range, + .setlease = generic_setlease, +}; + +const struct file_operations fuse_wb_dax_file_operations = { + .llseek = fuse_file_llseek, + .read_iter = fuse_dax_read_iter, + .write_iter = fuse_dax_write_iter, + .mmap = fuse_dax_mmap, + .open = fuse_wb_dax_open, + .flush = fuse_wb_flush, + .release = fuse_wb_release, + .fsync = fuse_fsync, + .lock = fuse_file_lock, + .get_unmapped_area = thp_get_unmapped_area, + .flock = fuse_file_flock, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .unlocked_ioctl = fuse_file_ioctl, + .compat_ioctl = fuse_file_compat_ioctl, + .poll = fuse_file_poll, + .fallocate = fuse_file_fallocate, + .copy_file_range = fuse_copy_file_range, + .setlease = generic_setlease, +}; +#endif + +#ifdef CONFIG_FUSE_PASSTHROUGH +/* + * Passthrough opens do their IO on the backing file. No ->open: the file + * is switched to these by fuse_finish_open(), after FUSE_OPEN has named the + * backing file. Release takes the backing file down through + * fuse_prepare_release(). + */ +static const struct file_operations fuse_wt_passthrough_file_operations = { + .llseek = fuse_file_llseek, + .read_iter = fuse_passthrough_file_read_iter, + .write_iter = fuse_passthrough_file_write_iter, + .mmap = fuse_passthrough_mmap, + .flush = fuse_wt_flush, + .release = fuse_wt_release, + .fsync = fuse_fsync, + .lock = fuse_file_lock, + .get_unmapped_area = thp_get_unmapped_area, + .flock = fuse_file_flock, + .splice_read = fuse_passthrough_file_splice_read, + .splice_write = fuse_passthrough_file_splice_write, + .unlocked_ioctl = fuse_file_ioctl, + .compat_ioctl = fuse_file_compat_ioctl, + .poll = fuse_file_poll, + .fallocate = fuse_file_fallocate, + .copy_file_range = fuse_copy_file_range, + .setlease = generic_setlease, +}; + +static const struct file_operations fuse_wb_passthrough_file_operations = { .llseek = fuse_file_llseek, - .read_iter = fuse_file_read_iter, - .write_iter = fuse_file_write_iter, - .mmap = fuse_file_mmap, - .open = fuse_open, - .flush = fuse_flush, - .release = fuse_release, + .read_iter = fuse_passthrough_file_read_iter, + .write_iter = fuse_passthrough_file_write_iter, + .mmap = fuse_passthrough_mmap, + .flush = fuse_wb_flush, + .release = fuse_wb_release, .fsync = fuse_fsync, .lock = fuse_file_lock, .get_unmapped_area = thp_get_unmapped_area, .flock = fuse_file_flock, - .splice_read = fuse_splice_read, - .splice_write = fuse_splice_write, + .splice_read = fuse_passthrough_file_splice_read, + .splice_write = fuse_passthrough_file_splice_write, .unlocked_ioctl = fuse_file_ioctl, .compat_ioctl = fuse_file_compat_ioctl, .poll = fuse_file_poll, @@ -5080,10 +5385,30 @@ static const struct file_operations fuse_file_operations = { .copy_file_range = fuse_copy_file_range, .setlease = generic_setlease, }; +#endif + +/* + * The two modes read differently -- what a reply says about i_size, and + * whether a folio may be filled without a DLM grant -- and write back the + * same way: writethrough still has dirty folios from shared mappings. + */ +static const struct address_space_operations fuse_wt_file_aops = { + .read_folio = fuse_wt_read_folio, + .readahead = fuse_wt_readahead, + .writepages = fuse_writepages, + .launder_folio = fuse_launder_folio, + .dirty_folio = iomap_dirty_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, + .is_partially_uptodate = iomap_is_partially_uptodate, + .migrate_folio = filemap_migrate_folio, + .bmap = fuse_bmap, + .direct_IO = fuse_direct_IO, +}; -static const struct address_space_operations fuse_file_aops = { - .read_folio = fuse_read_folio, - .readahead = fuse_readahead, +static const struct address_space_operations fuse_wb_file_aops = { + .read_folio = fuse_wb_read_folio, + .readahead = fuse_wb_readahead, .writepages = fuse_writepages, .launder_folio = fuse_launder_folio, .dirty_folio = iomap_dirty_folio, @@ -5100,10 +5425,19 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); - inode->i_fop = &fuse_file_operations; - inode->i_data.a_ops = &fuse_file_aops; - if (fc->writeback_cache) + /* + * The cache mode is fixed by the INIT reply, before any inode exists, + * so each inode can be given the operations of its mode outright. + * fuse_dax_inode_init() below replaces them on a DAX inode. + */ + if (fc->writeback_cache) { + inode->i_fop = &fuse_wb_file_operations; + inode->i_data.a_ops = &fuse_wb_file_aops; mapping_set_writeback_may_deadlock_on_reclaim(&inode->i_data); + } else { + inode->i_fop = &fuse_wt_file_operations; + inode->i_data.a_ops = &fuse_wt_file_aops; + } INIT_LIST_HEAD(&fi->write_files); INIT_LIST_HEAD(&fi->queued_writes); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index deb7181e04a8b8..fd0b9e47da2574 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -715,13 +715,12 @@ static void fuse_get_dlm_lock_complete(struct fuse_mount *fm, return; /* - * error is 0 (a grant) or -ENOSYS (the server has no DLM, so the - * range is treated as covered) here: either way fuse_get_dlm_lock() - * goes on to consider the range usable. Any other error fails the - * IO instead, and the caller releases the still-INIT rlock directly - * without ever touching the page cache under it, so leave it alone. + * error is 0 (a grant) here and fuse_get_dlm_lock() goes on to + * consider the range usable. Any error fails the IO instead, and the + * caller releases the still-INIT rlock directly without ever touching + * the page cache under it, so leave it alone. */ - if (error && error != -ENOSYS) + if (error) return; fuse_range_lock_mark_ready(dargs->fi, dargs->rlock); @@ -822,9 +821,10 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, args->complete = fuse_get_dlm_lock_complete; err = fuse_simple_request(fm, args); if (err == -ENOSYS) { - /* fuse server does not support dlm, save the info */ - fc->dlm = 0; - return err; + /* a server without DLM cannot serve a writeback cache */ + pr_warn("fuse: server does not implement FUSE_DLM_WB_LOCK, aborting connection\n"); + fuse_abort_conn(fc); + return -EIO; } if (err) @@ -911,7 +911,7 @@ void fuse_range_lock_tree_init(struct fuse_inode *inode) * * A range held by @lock's own task never conflicts: the only way that * happens is the task faulting in its own user buffer mid-IO, where - * fuse_filemap_fault() takes a range lock on the faulting page while + * fuse_wb_filemap_fault() takes a range lock on the faulting page while * the read/write path already holds one covering it. Waiting here * would then deadlock on ourselves, and skipping is safe: the outer * hold already gives invalidation an overlapping range to conflict diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 3d97ad84d94cdd..e48a611f859987 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -29,7 +29,7 @@ struct fuse_inode; * share: byte-exact ranges would let both of them work on it at once. * * - Let attribute/BRL invalidation (fuse_reverse_inval_inode(), - * truncate in fuse_do_setattr()) block only on in-progress IO that + * fuse_wb_truncate_pagecache()) block only on in-progress IO that * overlaps with the range being invalidated, instead of serializing * with all IO on the inode. * @@ -132,7 +132,7 @@ struct fuse_range_lock { * conflict with each other: the only way a task acquires a range * overlapping its own is re-entering through the page fault path * while faulting in its own user buffer mid-IO (see - * fuse_filemap_fault()), and the outer hold already fences + * fuse_wb_filemap_fault()), and the outer hold already fences * invalidation over the overlap for as long as both are held. */ struct task_struct *owner; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index c5b78819284e47..75258c0e184497 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -203,7 +203,7 @@ struct fuse_inode { /* * Bumped under fi->lock whenever the file may have shrunk behind a * read in flight: a local truncate, or a whole-file invalidate (the - * shape a remote truncate takes). See fuse_dlm_read_extend_size(). + * shape a remote truncate takes). See fuse_wb_read_size(). */ u64 size_gen; @@ -793,17 +793,6 @@ struct fuse_sync_bucket { struct rcu_head rcu; }; -/** - * DLM retry tracking for iomap write deadlock workaround. - * - * Temporary workaround until mainline iomap gains AOP_TRUNCATED_PAGE - * retry support. Tracks tasks that need to retry write operations due - * to DLM lock contention (-EAGAIN from FUSE server). - */ -struct fuse_dlm_retry { - bool retry_needed; -}; - /** * A Fuse connection. * @@ -1088,9 +1077,6 @@ struct fuse_conn { /* Is statx not implemented by fs? */ unsigned int no_statx:1; - /* do we have support for dlm in the fs? */ - unsigned int dlm:1; - /* Is extended lookup implemented by fs? */ unsigned int lookupx:1; @@ -1185,13 +1171,6 @@ struct fuse_conn { /* The foffset alignment in PAGE */ unsigned int alignment_pages; - - /** - * XArray tracking tasks that need DLM retry. - * Maps task pointer -> struct fuse_dlm_retry. - * Temporary workaround for iomap write deadlock. - */ - struct xarray dlm_retry_tasks; }; /* @@ -1454,8 +1433,6 @@ void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, u64 attr_valid, u32 cache_mask, u64 evict_ctr); -u32 fuse_get_cache_mask(struct inode *inode); - /** * Initialize the client device */ @@ -1769,6 +1746,10 @@ void fuse_dax_dontcache(struct inode *inode, unsigned int flags); bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment); void fuse_dax_cancel_work(struct fuse_conn *fc); +/* file.c: the regular-file operations of a DAX inode, per cache mode */ +extern const struct file_operations fuse_wt_dax_file_operations; +extern const struct file_operations fuse_wb_dax_file_operations; + /* ioctl.c */ long fuse_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg); long fuse_file_compat_ioctl(struct file *file, unsigned int cmd, @@ -1867,7 +1848,7 @@ ssize_t fuse_passthrough_splice_read(struct file *in, loff_t *ppos, ssize_t fuse_passthrough_splice_write(struct pipe_inode_info *pipe, struct file *out, loff_t *ppos, size_t len, unsigned int flags); -ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma); +int fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma); #ifdef CONFIG_SYSCTL extern int fuse_sysctl_register(void); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 479fee45410426..9e6113c6336f91 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -519,26 +519,12 @@ void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, inode->i_flags &= ~S_NOSEC; } -u32 fuse_get_cache_mask(struct inode *inode) -{ - struct fuse_conn *fc = get_fuse_conn(inode); - - if (!fc->writeback_cache || !S_ISREG(inode->i_mode) || fc->dlm) - return 0; - - return STATX_MTIME | STATX_CTIME | STATX_SIZE; -} - /* * Which cached attributes survive a server reply. * - * Without DLM this is fuse_get_cache_mask(): with the writeback cache on, - * writes update mtime and ctime and may extend i_size locally, the server - * knows about none of it, so the cached values win. - * - * With DLM the server is the authority (fuse_get_cache_mask() returns 0), - * because another node may have changed the file behind us and only the - * server can say so. That holds for the parts of the file we do not own. A + * The server is the authority, because another node may have changed the + * file behind us and only the server can say so. That holds for the parts + * of the file we do not own. A * write grant means no other node can touch the range until we are revoked, * so anything the server reports about it is at best as new as what we have, * and older if we still have unwritten data there. Keep the cached values @@ -578,13 +564,12 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, { struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); - u32 cache_mask = fuse_get_cache_mask(inode); + u32 cache_mask = 0; loff_t size = i_size_read(inode); *size_seen = size; - if (cache_mask || !fc->dlm || !fc->writeback_cache || - !S_ISREG(inode->i_mode)) + if (!fc->writeback_cache || !S_ISREG(inode->i_mode)) return cache_mask; if (!fuse_dlm_write_grant_exists(fi)) @@ -604,16 +589,14 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, /* * Whether a cached time stamp stays over the one the server reported. * - * Without dlm the kernel owns the stamps of a writeback inode and the - * server's are ignored. With dlm the cached stamp is kept only while the - * page cache is dirty, because the local writes set it and the server has - * not seen them. That makes it a placeholder, not an authority: a later + * The cached stamp is kept only while the page cache is dirty, because the + * local writes set it and the server has not seen them. That makes it a + * placeholder, not an authority: a later * stamp from the server records a modification the server did see, our own * earlier writeback or another node's write, and stat() has to report it. * The dirty data stamps again when it is written back. */ -static bool fuse_cached_time_wins(struct fuse_conn *fc, - const struct timespec64 *cached, +static bool fuse_cached_time_wins(const struct timespec64 *cached, u64 sec, u32 nsec, bool have) { struct timespec64 reported = { @@ -621,7 +604,7 @@ static bool fuse_cached_time_wins(struct fuse_conn *fc, .tv_nsec = min_t(u32, nsec, NSEC_PER_SEC - 1), }; - if (!fc->dlm || !have) + if (!have) return true; return timespec64_compare(cached, &reported) >= 0; @@ -649,11 +632,11 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr * fi->lock alone, so a write landing past the reported size between * the decision and truncate_pagecache() would have its dirty folios * dropped. Hold the IO range lock over that range across the - * truncate, as fuse_do_setattr() does: a write claiming past the - * size waits in fuse_range_lock_mark_locked() until the truncate is - * done, and one already there holds this reply until its claim is in - * i_size, which the size_seen test below notices. A grant a waiting - * writer records meanwhile covers nothing dirty yet. + * truncate, as fuse_wb_truncate_pagecache() does: a write claiming + * past the size waits in fuse_range_lock_mark_locked() until the + * truncate is done, and one already there holds this reply until its + * claim is in i_size, which the size_seen test below notices. A + * grant a waiting writer records meanwhile covers nothing dirty yet. * * Only for a shrink that is going to be applied. A larger size * truncates nothing under dlm (see below), and the lock would hold @@ -661,7 +644,7 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr */ cache_mask = fuse_attr_cache_mask(inode, attr, have_size, &size_seen); if (have_size && !(cache_mask & STATX_SIZE) && S_ISREG(inode->i_mode) && - fc->dlm && fc->writeback_cache && (loff_t)attr->size < size_seen) { + fc->writeback_cache && (loff_t)attr->size < size_seen) { fuse_range_lock_acquire_locked(fi, &rlock, attr->size, ~0ULL, FUSE_RANGE_LOCK_WRITE); range_locked = true; @@ -684,7 +667,7 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr if (cache_mask & STATX_MTIME) { struct timespec64 cached = inode_get_mtime(inode); - if (fuse_cached_time_wins(fc, &cached, attr->mtime, + if (fuse_cached_time_wins(&cached, attr->mtime, attr->mtimensec, have_mtime)) { attr->mtime = cached.tv_sec; attr->mtimensec = cached.tv_nsec; @@ -695,7 +678,7 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr if (cache_mask & STATX_CTIME) { struct timespec64 cached = inode_get_ctime(inode); - if (fuse_cached_time_wins(fc, &cached, attr->ctime, + if (fuse_cached_time_wins(&cached, attr->ctime, attr->ctimensec, have_ctime)) { attr->ctime = cached.tv_sec; attr->ctimensec = cached.tv_nsec; @@ -751,7 +734,7 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr * right now. A smaller one runs under the range lock. */ if ((loff_t)attr->size < oldsize || - !(fc->dlm && fc->writeback_cache)) + !fc->writeback_cache) truncate_pagecache(inode, attr->size); if (!fc->explicit_inval_data) inval = true; @@ -1058,7 +1041,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, fi->attr_version = atomic64_inc_return(&fc->attr_version); /* * A remote truncate reaches us as a whole-file invalidate; read - * replies in flight may predate it (see fuse_dlm_read_extend_size()). + * replies in flight may predate it (see fuse_wb_read_size()). */ if (offset <= 0 && len <= 0) fuse_bump_size_gen(fi); @@ -1074,7 +1057,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, security_inode_invalidate_secctx(inode); if (offset >= 0) { bool range_locked = S_ISREG(inode->i_mode) && - fc->writeback_cache && fc->dlm; + fc->writeback_cache; bool hot = false, has_writer = false, latched = false; bool track_latch; int ierr; @@ -1123,7 +1106,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * files; elsewhere the invalidate just drops the range * (best-effort), as before. An mmapped inode is never * latched: a mapping needs the page cache, and - * fuse_file_mmap() reverts any latch it races with. + * fuse_wb_mmap() reverts any latch it races with. */ if (range_locked) { lock_start = offset; @@ -1134,12 +1117,11 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, FUSE_RANGE_LOCK_WRITE); } - if (fc->dlm && fc->writeback_cache) + if (fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); track_latch = S_ISREG(inode->i_mode) && fc->writeback_cache && - fc->dlm && !FUSE_IS_DAX(inode) && - !fuse_inode_backing(fi); + !FUSE_IS_DAX(inode) && !fuse_inode_backing(fi); if (track_latch) { spin_lock(&fi->lock); hot = fuse_notify_inval_hot(fi); @@ -1166,7 +1148,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * The LOCKED range lock keeps cached writers from dirtying * the range again; anything left dirty is still laundered * below. Wait with _keep_errors so that a write-back error - * stays in AS_EIO/AS_ENOSPC for fuse_flush() to report at + * stays in AS_EIO/AS_ENOSPC for fuse_wb_flush() to report at * close; filemap_write_and_wait_range() would clear it. */ if (range_locked) { @@ -1623,13 +1605,11 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, fc->blocked = 0; fc->initialized = 0; fc->connected = 1; - fc->dlm = 1; fc->lookupx = 1; /* module option for now */ fc->compound_open_getattr = enable_compound; - xa_init(&fc->dlm_retry_tasks); atomic64_set(&fc->attr_version, 1); atomic64_set(&fc->evict_ctr, 1); get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); @@ -1682,7 +1662,6 @@ void fuse_conn_put(struct fuse_conn *fc) } if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) fuse_backing_files_free(fc); - xa_destroy(&fc->dlm_retry_tasks); call_rcu(&fc->rcu, delayed_release); } EXPORT_SYMBOL_GPL(fuse_conn_put); diff --git a/fs/fuse/passthrough.c b/fs/fuse/passthrough.c index 72de97c03d0eeb..2b66afc2d43177 100644 --- a/fs/fuse/passthrough.c +++ b/fs/fuse/passthrough.c @@ -129,7 +129,7 @@ ssize_t fuse_passthrough_splice_write(struct pipe_inode_info *pipe, return ret; } -ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma) +int fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma) { struct fuse_file *ff = file->private_data; struct file *backing_file = fuse_file_passthrough(ff);