swap_pager_getpages(): some pages from ma[] might be bogus

Same as vnode_pager_generic_getpages_async(), swap_pager_getpages() must
handle a possibility of the provided page run to include bogus_page on
some positions, when called from sendfile_swapin().  The swap pager is
used for tmpfs vnodes.

In particular, the bogus page must not be used for pindex calculation,
we better not update the flags on it or wait for the flag clearing, and
we must not call vm_page_valid() because the function expects busy page.

This was bisected down to 72ddb6de1028426 (unix: increase
net.local.(stream|seqpacket).(recv|send)space to 64 KiB),
which is somewhat surprising, but apparently reasonable because it
allowed the run of more than one page for page-in from the swap pager,
which now might include valid pages replaced by bogus one.

In collaboration with:	pho
Reviewed by:	glebius, markj
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
Differential revision:	https://reviews.freebsd.org/D54713

1 files changed, 29 insertions, 8 deletions

diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c
index 012d89db3310..f6d201309349 100644
--- a/sys/vm/swap_pager.c
+++ b/sys/vm/swap_pager.c
@@ -1362,14 +1362,22 @@ static int
 swap_pager_getpages_locked(struct pctrie_iter *blks, vm_object_t object,
     vm_page_t *ma, int count, int *a_rbehind, int *a_rahead, struct buf *bp)
 {
+	vm_page_t m;
 	vm_pindex_t pindex;
-	int rahead, rbehind;
+	int i, rahead, rbehind;
 
 	VM_OBJECT_ASSERT_WLOCKED(object);
 
 	KASSERT((object->flags & OBJ_SWAP) != 0,
 	    ("%s: object not swappable", __func__));
-	pindex = ma[0]->pindex;
+	for (i = 0; i < count; i++) {
+		m = ma[i];
+		if (m != bogus_page) {
+			pindex = m->pindex - i;
+			break;
+		}
+	}
+	MPASS(i != count);
 	if (!swp_pager_haspage_iter(pindex, &rbehind, &rahead, blks)) {
 		VM_OBJECT_WUNLOCK(object);
 		uma_zfree(swrbuf_zone, bp);
@@ -1399,8 +1407,11 @@ swap_pager_getpages_locked(struct pctrie_iter *blks, vm_object_t object,
 	KASSERT(bp->b_npages <= PBUF_PAGES,
 	    ("bp_npages %d (rb %d c %d ra %d) not less than PBUF_PAGES %jd",
 	    bp->b_npages, rbehind, count, rahead, (uintmax_t)PBUF_PAGES));
-	for (int i = 0; i < bp->b_npages; i++)
-		bp->b_pages[i]->oflags |= VPO_SWAPINPROG;
+	for (i = 0; i < bp->b_npages; i++) {
+		m = bp->b_pages[i];
+		if (m != bogus_page)
+			m->oflags |= VPO_SWAPINPROG;
+	}
 	bp->b_blkno = swp_pager_meta_lookup(blks, pindex - rbehind);
 	KASSERT(bp->b_blkno != SWAPBLK_NONE,
 	    ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex));
@@ -1448,8 +1459,14 @@ swap_pager_getpages_locked(struct pctrie_iter *blks, vm_object_t object,
 	 */
 	VM_OBJECT_WLOCK(object);
 	/* This could be implemented more efficiently with aflags */
-	while ((ma[0]->oflags & VPO_SWAPINPROG) != 0) {
-		ma[0]->oflags |= VPO_SWAPSLEEP;
+	for (i = 0; i < count; i++) {
+		m = ma[i];
+		if (m != bogus_page)
+			break;
+	}
+	MPASS(i != count);
+	while ((m->oflags & VPO_SWAPINPROG) != 0) {
+		m->oflags |= VPO_SWAPSLEEP;
 		VM_CNT_INC(v_intrans);
 		if (VM_OBJECT_SLEEP(object, &object->handle, PSWP,
 		    "swread", hz * 20)) {
@@ -1463,9 +1480,10 @@ swap_pager_getpages_locked(struct pctrie_iter *blks, vm_object_t object,
 	/*
 	 * If we had an unrecoverable read error pages will not be valid.
 	 */
-	for (int i = 0; i < count; i++)
-		if (ma[i]->valid != VM_PAGE_BITS_ALL)
+	for (i = 0; i < count; i++) {
+		if (ma[i] != bogus_page && ma[i]->valid != VM_PAGE_BITS_ALL)
 			return (VM_PAGER_ERROR);
+	}
 
 	return (VM_PAGER_OK);
 
@@ -1730,6 +1748,9 @@ swp_pager_async_iodone(struct buf *bp)
 	for (i = 0; i < bp->b_npages; ++i) {
 		vm_page_t m = bp->b_pages[i];
 
+		if (m == bogus_page)
+			continue;
+
 		m->oflags &= ~VPO_SWAPINPROG;
 		if (m->oflags & VPO_SWAPSLEEP) {
 			m->oflags &= ~VPO_SWAPSLEEP;