diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 88f23acd541c13..1b2076130617a7 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2092,7 +2092,7 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, bool is_wb = fc->writeback_cache && S_ISREG(inode->i_mode); loff_t oldsize; int err; - bool trust_local_cmtime = is_wb; + bool trust_local_cmtime = is_wb && !fc->dlm; bool fault_blocked = false; u64 attr_version; diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 83fb69070eb014..a848956b892af5 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -20,7 +20,11 @@ #include #include #include +#include #include +#include +#include +#include int sb_init_dio_done_wq(struct super_block *sb); @@ -1201,6 +1205,32 @@ static bool fuse_read_needs_size(struct inode *inode, loff_t pos, size_t len) return fuse_size_needs_server(inode) && pos + len > i_size_read(inode); } +/* + * How much of the user buffer to fault in before retrying a cached read. + * + * While the read is still making progress, fault a multi-page window, + * capped the way gfs2's should_fault_in_pages() does. After a retry + * copies nothing, drop to a single page so one large fault-in that does + * not stick gets a smaller try. The caller stops if that page also makes + * no progress: fault_in of an already resident page succeeds, so retrying + * it without a stop would spin. + */ +static size_t fuse_read_fault_window(struct kiocb *iocb, size_t count, + bool single_page) +{ + size_t size = PAGE_SIZE; + size_t offs = offset_in_page(iocb->ki_pos); + size_t nr_dirtied; + + if (!single_page) { + nr_dirtied = max(current->nr_dirtied_pause - + current->nr_dirtied, 8); + size = min_t(size_t, SZ_1M, nr_dirtied << PAGE_SHIFT); + } + + return min_t(size_t, size - offs, count); +} + static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; @@ -1209,6 +1239,10 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) struct fuse_inode *fi = get_fuse_inode(inode); struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; size_t count = iov_iter_count(to); + int tries = FUSE_DLM_READ_RETRIES; + size_t prev_count = 0; + bool single_page = false; + ssize_t total = 0; ssize_t res; int lock_err = 0; @@ -1236,6 +1270,10 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) iov_iter_count(to), FUSE_PAGE_LOCK_READ); + /* wb_sem is NULL on non-writeback+dlm mounts (gate inactive). */ + if (!wb_sem) + return generic_file_read_iter(iocb, to); + /* * Fence the cache-serving read against a NOTIFY invalidate so we never * hand back a folio the server has just superseded. The gate read side @@ -1243,16 +1281,34 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) * Re-check the forced-DIO latch under it: if a storm latched us while we * waited on a pending writer, reroute to direct like the buffered write * path, so we do not repopulate the cache the latch just dropped. - * wb_sem is NULL on non-writeback+dlm mounts (gate inactive). + * + * Nothing held under the gate may wait for the server, and faulting + * on the user buffer can do exactly that: a buffer backed by a + * shared-writable mapping of this (or any) fuse file sends + * fuse_filemap_fault() after a WRITE grant, and granting it can + * first require the server to revoke this read's own READ grant, + * whose invalidation then waits on the gate this read holds. ABBA + * through the server, and request_wait_answer() does not die to + * SIGKILL once the request is in userspace. + * + * So copy with page faults disabled (a non-resident user page makes + * the copy come back short instead of faulting), and fault the + * buffer in with the gate dropped: nothing is held then that the + * revoke's invalidation needs. The grant may be revoked while the + * gate is dropped, so re-cover it before re-entering, which + * re-validates it like the first pass. Same shape as + * gfs2_file_read_iter() around its glock. */ - if (wb_sem) { - int tries = FUSE_DLM_READ_RETRIES; + for (;;) { + size_t remain; + size_t window; retry: percpu_down_read(wb_sem); if (fuse_inode_force_dio(inode)) { percpu_up_read(wb_sem); - return fuse_direct_read_iter(iocb, to); + res = fuse_direct_read_iter(iocb, to); + break; } /* * The DLM lock was requested before entering the gate, and @@ -1265,8 +1321,8 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) * hold. A failed or unrecorded request falls through * unlocked, as before: the retry is taken even then (the * latch must be re-checked under the re-entered gate), so - * lock_err has to stay sticky across it -- seeded by the - * pre-gate request above -- or a grant that failed would + * lock_err has to stay sticky across it, seeded by the + * pre-gate request above, or a grant that failed would * be re-requested forever. The retry is also bounded: a * remote writer can revoke each successful grant before * the gate is re-entered, and a reader-only inode has no @@ -1284,14 +1340,49 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) FUSE_PAGE_LOCK_READ); goto retry; } - } - - res = generic_file_read_iter(iocb, to); - if (wb_sem) + pagefault_disable(); + res = generic_file_read_iter(iocb, to); + pagefault_enable(); percpu_up_read(wb_sem); + if (res > 0) + total += res; - return res; + if (res != -EFAULT && + !(res >= 0 && iov_iter_count(to) && + iocb->ki_pos < i_size_read(inode))) + break; + if (!user_backed_iter(to)) + break; + + remain = iov_iter_count(to); + if (!remain) + break; + if (remain == prev_count) { + if (single_page) { + pr_warn_ratelimited("fuse: cached read made no progress after a one-page fault-in, nodeid %llu pos %lld size %lld remain %zu res %zd\n", + fi->nodeid, iocb->ki_pos, + i_size_read(inode), remain, res); + break; + } + single_page = true; + } else { + single_page = false; + } + prev_count = remain; + + window = fuse_read_fault_window(iocb, remain, single_page); + if (!window) + break; + window -= fault_in_iov_iter_writeable(to, window); + if (!window) + break; + if (!lock_err && fc->dlm) + lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, remain, + FUSE_PAGE_LOCK_READ); + } + + return total > 0 ? total : res; } static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, @@ -2776,6 +2867,61 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l } return err; } + +/* + * Batch size for fuse_fault_read_private_folios(): one fault-around + * window (64k with 4k pages), centred on the faulted page. A + * spanning access (one instruction touching the tail of one page and + * the head of the next, which any unaligned copy does constantly) + * needs BOTH pages mapped at once to retire, and every server round + * trip gets this client's grant revoked and its page cache dropped + * (the server does not exempt the requester, and the notification + * runs one op behind). A one-page grant-and-populate therefore + * ping-pongs forever: mapping each page costs a round trip whose + * revoke drops the other. A window covering both pages makes the + * neighbour's fault a pure fast-path hit (recorded grant, resident + * folio, NO server op and no new revoke) and the spanning access + * completes. Centred, not aligned: an aligned window has edges + * (every 16th page boundary), and a spanning access sitting on an + * edge gets two one-page-covering windows and ping-pongs just the + * same (generic/263); a centred window always covers both of the + * faulted page's neighbours, so no boundary has the problem. + */ +#define FUSE_FAULT_BATCH_PAGES 16 + +/* First page of the populate/grant window centred on @index. */ +static pgoff_t fuse_fault_batch_base(pgoff_t index) +{ + if (index < FUSE_FAULT_BATCH_PAGES / 2) + return 0; + return index - FUSE_FAULT_BATCH_PAGES / 2; +} + +/* + * The coherency gate (fi->wb_inval_rwsem) is the fault path's fence + * against a revoke: a NOTIFY invalidate and a local truncate revoke + * the grant and drop the pages under its write side, so a grant seen + * under the read side stays for the whole hold. The gate is NULL + * when its allocation failed; nothing fences the revoke on such an + * inode anywhere, so the helpers pass. + */ +static bool fuse_wb_gate_tryenter(struct percpu_rw_semaphore *sem) +{ + return !sem || percpu_down_read_trylock(sem); +} + +static void fuse_wb_gate_enter(struct percpu_rw_semaphore *sem) +{ + if (sem) + percpu_down_read(sem); +} + +static void fuse_wb_gate_exit(struct percpu_rw_semaphore *sem) +{ + if (sem) + percpu_up_read(sem); +} + /* * Wait for writeback against this page to complete before allowing it * to be marked dirty again, and hence written back again, possibly @@ -2797,9 +2943,54 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) struct file *file = vmf->vma->vm_file; struct inode *inode = file_inode(file); struct fuse_mount *fm = get_fuse_mount(inode); + loff_t pos = vmf->pgoff << PAGE_SHIFT; - if (fm->fc->dlm) { - loff_t pos = vmf->pgoff << PAGE_SHIFT; + if (fm->fc->writeback_cache && fm->fc->dlm) { + /* + * Writenotify routes every clean-to-dirty transition of a + * shared mapping through here, but the WRITE grant itself + * is taken by fuse_filemap_fault(): shared-writable VMAs + * get no ->map_pages under DLM (see fuse_file_mmap()), so + * every PTE in such a VMA was installed by a fault that + * recorded a WRITE grant for its batch window. Only + * verify here, and only without blocking: nothing may + * sleep on the gate or the server while mmap_lock (or the + * per-VMA lock) is held. The buffered read and write + * paths hold the gate while faulting in their user + * buffers (which takes mmap_lock), so with a pending + * mmap_lock writer queued between the two, blocking here + * deadlocks three ways, the cycle fuse_filemap_fault() + * avoids with its drop-and-retry, and page_mkwrite has no + * VM_FAULT_RETRY protocol to drop the lock and come back + * with. + * + * Same fence as fuse_filemap_fault()'s fast path: only + * under the gate can the recorded grant be trusted. On a + * miss, NOPAGE. A contended trylock re-faults with + * mmap_lock released in between, so the gate writer can + * make progress; a verified miss means a revoke or a + * truncate is in flight (only those remove a recorded + * grant, and both drop the pages under the gate write + * side), whose unmap clears the PTE, so the re-fault goes + * through fuse_filemap_fault(), which re-takes the grant + * with the proper drop-and-retry dance. + */ + struct fuse_inode *fi = get_fuse_inode(inode); + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + bool held = false; + + if (fuse_wb_gate_tryenter(wb_sem)) { + held = fuse_dlm_lock_is_held(fi, pos, PAGE_SIZE, + FUSE_PAGE_LOCK_WRITE); + if (!held) + pr_warn_ratelimited("fuse: mkwrite with no WRITE grant, nodeid %llu idx %lu\n", + fi->nodeid, vmf->pgoff); + fuse_wb_gate_exit(wb_sem); + } + + if (!held) + return VM_FAULT_NOPAGE; + } else if (fm->fc->dlm) { size_t length = PAGE_SIZE; int err = fuse_get_page_mkwrite_lock(file, pos, length); if (err < 0) { @@ -2818,13 +3009,469 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) return VM_FAULT_LOCKED; } +/* + * Does filemap_fault() need a synchronous READ before it can map + * @index? True when the page is absent or not uptodate and the fault + * is inside i_size: past EOF, filemap_fault() returns SIGBUS before + * ever touching the page, so no READ can happen there. The lookup + * never blocks, so this is safe under mmap_lock (or the per-VMA lock) + * and with the gate in any state. + */ +static bool fuse_fault_needs_read(struct inode *inode, pgoff_t index) +{ + struct folio *folio; + bool uptodate; + + if (((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) + return false; + + folio = filemap_get_folio(inode->i_mapping, index); + if (IS_ERR(folio)) + return true; + + uptodate = folio_test_uptodate(folio); + folio_put(folio); + return !uptodate; +} + +/* + * Read a batch of pages centred on @index into freshly allocated + * folios that are NOT in the page cache. Unlike read_cache_folio(), + * nothing contested is held while the READ waits on the server: no + * page-cache folio lock (an invalidation laundering the range would + * block on it, while the server holds this READ behind the revoke + * that triggered the invalidation, the same cycle as the gate one, + * one lock over), and the caller is outside the gate. The folios are + * returned with a reference each and no mapping; the caller installs + * them with fuse_fault_install_folio() under the gate. + * + * A short read (EOF) leaves the tails zeroed (page_zeroing), which is + * what the page cache stores for the EOF pages anyway. i_size is + * deliberately NOT shrunk here (no fuse_short_read()), to keep this + * reader free of attribute side effects. + * + * On success returns the number of folios in @folios (>= 1, @index + * always covered) and sets @basep to the first folio's page index; + * negative error otherwise. + */ +static int fuse_fault_read_private_folios(struct file *file, + struct inode *inode, + pgoff_t index, pgoff_t *basep, + struct folio **folios) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_page_desc descs[FUSE_FAULT_BATCH_PAGES]; + struct page *pages[FUSE_FAULT_BATCH_PAGES]; + struct fuse_io_args ia = { + .ap.args.page_zeroing = true, + .ap.args.out_pages = true, + .ap.descs = descs, + .ap.pages = pages, + }; + pgoff_t base = fuse_fault_batch_base(index); + loff_t isize = i_size_read(inode); + loff_t pos = (loff_t)base << PAGE_SHIFT; + pgoff_t last = (isize - 1) >> PAGE_SHIFT; + unsigned int nr, i; + size_t count; + ssize_t res; + + /* Caller checked index < i_size; clip the window to EOF. */ + nr = min_t(pgoff_t, FUSE_FAULT_BATCH_PAGES, last - base + 1); + + for (i = 0; i < nr; i++) { + folios[i] = filemap_alloc_folio( + mapping_gfp_mask(inode->i_mapping), 0); + if (!folios[i]) + break; + pages[i] = &folios[i]->page; + descs[i].offset = 0; + descs[i].length = PAGE_SIZE; + } + if (i <= index - base) { + /* Not even the faulted page: fail with what we freed. */ + while (i--) + folio_put(folios[i]); + return -ENOMEM; + } + nr = i; + ia.ap.num_pages = nr; + count = (size_t)nr << PAGE_SHIFT; + + /* Don't overflow end offset */ + if (pos + (count - 1) == LLONG_MAX) { + count--; + descs[nr - 1].length--; + } + + fuse_read_args_fill(&ia, file, pos, count, FUSE_READ); + res = fuse_simple_request(fm, &ia.ap.args); + if (res < 0) { + for (i = 0; i < nr; i++) + folio_put(folios[i]); + return res; + } + + *basep = base; + return nr; +} + +/* + * Install a folio filled by fuse_fault_read_private_folios() into the + * page cache. Only called under the gate. filemap_add_folio() + * returns the folio locked; it is held only over the uptodate + * marking, never over a server wait. If another path populated the + * index meanwhile (-EEXIST) with an uptodate folio, that copy is just + * as good. A resident !uptodate folio, though, may have NO read in + * flight to ever complete it: an errored readahead (the server + * bounces READs behind a revoke in flight) leaves exactly that: + * unlocked, resident, !uptodate. The buffered read path would lock + * and re-read it, but this path reads privately and would -EEXIST + * against it on every pass, and the caller's residency re-check would + * loop forever (the generic/120 livelock). So fill it from the + * just-read data under a trylock: trylock never waits (no server wait + * is allowed under the gate), and losing the trylock means a read IS + * in flight, which resolves the index on its own. Does not consume + * the caller's reference. + * + * No freshness test gates the install, deliberately. The server + * revokes this client's own grant to serve the private READ (it does + * not exempt the requester), so the READ's own revoke notification is + * in flight more or less whenever this is called, and any "nothing + * invalidated this inode across the READ window" check fails on every + * pass: the probe destroys what it measures, and the fault livelocks + * re-reading forever. Instead the caller holds the gate from here + * through filemap_fault(), so the notification (which takes the gate + * write side to drop pages) cannot remove the folio before the PTE is + * mapped. A revoke that lands after the release drops the folio and + * unmaps the PTE, and the next access refaults and re-reads: coherent + * at the protocol's notification-latency granularity, same as every + * other cached page. + */ +static void fuse_fault_install_folio(struct inode *inode, + struct folio *folio, pgoff_t index) +{ + struct folio *stale; + + if (!filemap_add_folio(inode->i_mapping, folio, index, + mapping_gfp_mask(inode->i_mapping))) { + folio_mark_uptodate(folio); + folio_unlock(folio); + return; + } + + stale = filemap_get_folio(inode->i_mapping, index); + if (IS_ERR(stale)) + return; + if (!folio_test_uptodate(stale) && !folio_test_large(stale) && + folio_trylock(stale)) { + /* Re-check under the lock; skip if truncated away. */ + if (!folio_test_uptodate(stale) && + stale->mapping == inode->i_mapping) { + void *src = kmap_local_folio(folio, 0); + void *dst = kmap_local_folio(stale, 0); + + memcpy(dst, src, PAGE_SIZE); + kunmap_local(dst); + kunmap_local(src); + flush_dcache_folio(stale); + folio_mark_uptodate(stale); + } + folio_unlock(stale); + } + folio_put(stale); +} + +/* + * Install the folios a private read returned, under the gate, and + * drop the caller's references. + */ +static void fuse_fault_install_folios(struct inode *inode, + struct folio **folios, int nr, + pgoff_t base) +{ + int i; + + for (i = 0; i < nr; i++) { + fuse_fault_install_folio(inode, folios[i], base + i); + folio_put(folios[i]); + } +} + +static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) +{ + struct file *file = vmf->vma->vm_file; + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + + if (fc->writeback_cache && fc->dlm) { + struct fuse_inode *fi = get_fuse_inode(inode); + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + loff_t pos = vmf->pgoff << PAGE_SHIFT; + /* + * Grant and populate in whole batch windows. A one-page + * grant breaks spanning accesses (one instruction + * touching two pages): the neighbour's fault finds no + * recorded grant, and its own fuse_get_dlm_lock() round + * trip costs a revoke that drops the page just mapped, + * a deterministic ping-pong (the generic/091 livelock). + * With the grant covering the same window the populate + * read fills, the neighbour's fault passes the fast + * path's is_held() and residency checks and maps with no + * server op at all. + */ + loff_t bpos = (loff_t)fuse_fault_batch_base(vmf->pgoff) + << PAGE_SHIFT; + size_t blen = (size_t)FUSE_FAULT_BATCH_PAGES << PAGE_SHIFT; + enum fuse_page_lock_mode mode = FUSE_PAGE_LOCK_READ; + struct folio *folios[FUSE_FAULT_BATCH_PAGES]; + pgoff_t fbase; + vm_fault_t ret; + int tries; + int err; + int nr; + + if ((vmf->vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == + (VM_SHARED | VM_MAYWRITE)) + mode = FUSE_PAGE_LOCK_WRITE; + + /* + * Fast path: gate uncontended, the grant already + * recorded, and the folio already resident. Nothing here + * blocks, so it is safe under mmap_lock (or the per-VMA + * lock). Gate first, then is_held(): only under the + * gate, which fences any concurrent revoke, can the + * recorded grant be trusted, the same order as the + * buffered read and write paths. The folio check keeps + * filemap_fault() from issuing a synchronous READ under + * the gate: nothing held under it may wait on the server + * (see the blocking path below), so a missing folio falls + * through to the slow path, which populates it outside. + */ + if (fuse_wb_gate_tryenter(wb_sem)) { + if (fuse_dlm_lock_is_held(fi, pos, PAGE_SIZE, mode) && + !fuse_fault_needs_read(inode, vmf->pgoff)) { + ret = filemap_fault(vmf); + fuse_wb_gate_exit(wb_sem); + return ret; + } + fuse_wb_gate_exit(wb_sem); + } + + /* + * Slow path: the gate or the grant needs waiting, an + * invalidation draining the inode or a DLM round trip to + * the server. Blocking here while holding mmap_lock + * recreates the ABBA the read/write paths set up the + * other way around: they hold the gate while faulting in + * their user buffers, which takes mmap_lock. Drop the + * fault lock first, do the blocking work with only a file + * reference pinning the inode, then ask for the fault to + * be retried; the retry takes the fast path above off the + * now-recorded grant. The gate is not held across the + * retry (there would be nowhere to release it), so the + * retried fault can still miss and come back here; each + * pass leaves a recorded grant behind, which only a real + * invalidation takes away again. + * + * FAULT_FLAG_TRIED does not demote a pass to the blocking + * path below: retries may repeat (4064b98 "mm: allow + * VM_FAULT_RETRY for multiple times"), and blocking on + * the gate with mmap_lock held closes a three-party cycle + * whenever the gate reader is a buffered write faulting + * in its source buffer (which waits on mmap_lock) behind + * a queued mmap_lock writer. A miss on the retried fast + * path only means the gate is contended again; waiting + * for it must happen with mmap_lock dropped, every time. + * + * The flip side is that an error met with mmap_lock + * dropped cannot be reported: only VM_FAULT_RETRY (or + * COMPLETED) may be returned then, and a retried fault + * that keeps failing comes straight back here. An + * aborted connection fails every request, so the task + * would refault forever: it burns CPU in this path, + * only SIGKILL ends it, and its mapping keeps the mount + * busy. Check it while the fault lock is still held, + * where SIGBUS can be returned. + */ + if (!READ_ONCE(fc->connected)) + return VM_FAULT_SIGBUS; + + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { + if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) + return VM_FAULT_RETRY; + + file = get_file(file); + release_fault_lock(vmf); + + /* + * A hard error is not failed here, it cannot be + * with the fault lock dropped: the retried fault + * meets it again. A lost connection ends at the + * check above. + */ + fuse_get_dlm_lock(file, bpos, blen, mode); + /* + * Populate the folios too, or the retried fault's + * fast path would refuse the grant it just + * recorded (missing folio) and come straight back + * here. Read outside the gate, install under it; + * see the blocking path below. + */ + if (fuse_fault_needs_read(inode, vmf->pgoff)) { + nr = fuse_fault_read_private_folios( + file, inode, vmf->pgoff, &fbase, + folios); + if (nr > 0) { + fuse_wb_gate_enter(wb_sem); + fuse_fault_install_folios(inode, folios, + nr, fbase); + fuse_wb_gate_exit(wb_sem); + } + } + fput(file); + return VM_FAULT_RETRY; + } + + /* + * Retry not allowed (a caller without + * FAULT_FLAG_ALLOW_RETRY): block in place. It can stack + * behind a pending mmap_lock writer the way any blocking + * fault can. + * + * Nothing held under the gate may wait for the server + * (see fuse_cache_read_iter), and filemap_fault() on a + * non-resident folio does exactly that: a synchronous + * READ the server can queue behind a revoke already in + * flight for this range, whose invalidation then waits + * on the gate. ABBA through the server (the generic/091 + * hang). Nor can the page-cache read path be used with + * the gate merely dropped: it keeps the folio locked in + * the mapping across the READ, and the invalidation + * blocks on that folio lock instead, the same cycle one + * lock over. So read into private folios outside the + * gate (nothing the invalidation needs is held), then + * enter it and install them, and hold the gate from the + * install through filemap_fault() so the READ's own + * revoke notification (the server revokes the requester's + * grant to serve the READ) cannot drop the folio before + * the PTE is mapped; see fuse_fault_install_folio(). + */ + for (tries = 0; ; tries++) { + err = fuse_get_dlm_lock(file, bpos, blen, mode); + if (err < 0 && err != -ENOSYS) + return vmf_error(err); + + fuse_wb_gate_enter(wb_sem); + /* + * A pass loops only when the grant or the folio + * went missing again between the previous pass + * and this one (a revoke landed in the gap), or + * on a bounced READ. Each successful install + * exits the loop under the gate, so a storm costs + * refaults, not loop passes; still killable in + * case the server bounces READs indefinitely. + */ + if (fatal_signal_pending(current)) { + fuse_wb_gate_exit(wb_sem); + return vmf_error(-EINTR); + } + if (tries && (tries % 64) == 0) + pr_warn_ratelimited("fuse: fault populate slow, nodeid %llu idx %lu: %d passes\n", + fi->nodeid, vmf->pgoff, + tries); + + /* + * The request ran outside the gate and the + * invalidate just waited on revokes under its + * write side: re-validate, as the IO paths do. + * An unrecorded grant (err > 0) cannot show here + * and is not re-requested. + */ + if (!err && fc->dlm && + !fuse_dlm_lock_is_held(fi, pos, PAGE_SIZE, mode)) { + fuse_wb_gate_exit(wb_sem); + continue; + } + + if (!fuse_fault_needs_read(inode, vmf->pgoff)) + break; + + fuse_wb_gate_exit(wb_sem); + nr = fuse_fault_read_private_folios(file, inode, + vmf->pgoff, + &fbase, folios); + if (nr < 0) { + /* + * -EDEADLK/-EAGAIN is the server bouncing + * a READ it cannot serve yet (a revoke in + * flight); back off and retry, the same + * thing AOP_TRUNCATED_PAGE does for the + * page-cache read path. Anything else is + * a real read error: let filemap_fault() + * run and turn it into SIGBUS; the server + * answers (with the error) rather than + * queueing, so nothing hangs. + */ + if (nr == -EDEADLK || nr == -EAGAIN) { + msleep_interruptible(2); + continue; + } + fuse_wb_gate_enter(wb_sem); + break; + } + + fuse_wb_gate_enter(wb_sem); + fuse_fault_install_folios(inode, folios, nr, fbase); + /* + * Exit under the gate: filemap_fault() below maps + * the just-installed folio without a server wait + * (present and uptodate), and the gate keeps any + * pending revoke's page drop out until the PTE is + * in place. If the install lost to -EEXIST on a + * !uptodate foreign folio it could not fill (a + * read in flight holds the folio lock), loop + * instead: mapping it would trigger a + * synchronous READ under the gate, and the + * in-flight read resolves the index anyway. + */ + if (!fuse_fault_needs_read(inode, vmf->pgoff)) + break; + fuse_wb_gate_exit(wb_sem); + } + + ret = filemap_fault(vmf); + fuse_wb_gate_exit(wb_sem); + return ret; + } + + return filemap_fault(vmf); +} + static const struct vm_operations_struct fuse_file_vm_ops = { .close = fuse_vma_close, - .fault = filemap_fault, + .fault = fuse_filemap_fault, .map_pages = filemap_map_pages, .page_mkwrite = fuse_page_mkwrite, }; +/* + * Shared-writable VMAs under the writeback cache with DLM: no + * ->map_pages. Fault-around installs PTEs for any resident folio in + * its window without calling ->fault, so a folio read in under a READ + * grant (read(2), readahead) could take its first write in + * ->page_mkwrite with no WRITE grant ever requested, and page_mkwrite + * cannot take one there (no retry protocol; see fuse_page_mkwrite()). + * Without fault-around, every first touch goes through + * fuse_filemap_fault(), which records a WRITE grant for the batch + * window before any PTE is installed. + */ +static const struct vm_operations_struct fuse_file_wb_dlm_vm_ops = { + .close = fuse_vma_close, + .fault = fuse_filemap_fault, + .page_mkwrite = fuse_page_mkwrite, +}; + static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) { struct fuse_file *ff = file->private_data; @@ -2891,7 +3538,11 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) fuse_link_write_file(file); file_accessed(file); - vma->vm_ops = &fuse_file_vm_ops; + if (fc->writeback_cache && fc->dlm && + (vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) + vma->vm_ops = &fuse_file_wb_dlm_vm_ops; + else + vma->vm_ops = &fuse_file_vm_ops; return 0; }