summaryrefslogtreecommitdiff
path: root/fs/proc
diff options
context:
space:
mode:
Diffstat (limited to 'fs/proc')
-rw-r--r--fs/proc/array.c15
-rw-r--r--fs/proc/base.c182
-rw-r--r--fs/proc/fd.c33
-rw-r--r--fs/proc/generic.c9
-rw-r--r--fs/proc/internal.h6
-rw-r--r--fs/proc/namespaces.c16
-rw-r--r--fs/proc/page.c2
-rw-r--r--fs/proc/proc_net.c2
-rw-r--r--fs/proc/task_mmu.c61
9 files changed, 184 insertions, 142 deletions
diff --git a/fs/proc/array.c b/fs/proc/array.c
index 038d4b57127f..38f96af7b521 100644
--- a/fs/proc/array.c
+++ b/fs/proc/array.c
@@ -481,7 +481,11 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns,
unsigned long flags;
int exit_code = task->exit_code;
struct signal_struct *sig = task->signal;
- unsigned int seq = 1;
+ int ret;
+
+ ret = down_read_killable(&task->signal->exec_update_lock);
+ if (ret)
+ return ret;
state = *get_task_state(task);
vsize = eip = esp = 0;
@@ -538,10 +542,7 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns,
if (permitted && (!whole || num_threads < 2))
wchan = !task_is_running(task);
- do {
- seq++; /* 2 on the 1st/lockless path, otherwise odd */
- flags = read_seqbegin_or_lock_irqsave(&sig->stats_lock, &seq);
-
+ scoped_seqlock_read (&sig->stats_lock, ss_lock_irqsave) {
cmin_flt = sig->cmin_flt;
cmaj_flt = sig->cmaj_flt;
cutime = sig->cutime;
@@ -563,8 +564,7 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns,
}
rcu_read_unlock();
}
- } while (need_seqretry(&sig->stats_lock, seq));
- done_seqretry_irqrestore(&sig->stats_lock, seq, flags);
+ }
if (whole) {
thread_group_cputime_adjusted(task, &utime, &stime);
@@ -661,6 +661,7 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns,
seq_puts(m, " 0");
seq_putc(m, '\n');
+ up_read(&task->signal->exec_update_lock);
if (mm)
mmput(mm);
return 0;
diff --git a/fs/proc/base.c b/fs/proc/base.c
index a314fd9af17c..c8d783f3b2ac 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -218,33 +218,24 @@ static int get_task_root(struct task_struct *task, struct path *root)
return result;
}
-static int proc_cwd_link(struct dentry *dentry, struct path *path)
+static int proc_cwd_link(struct dentry *dentry, struct path *path,
+ struct task_struct *task)
{
- struct task_struct *task = get_proc_task(d_inode(dentry));
int result = -ENOENT;
- if (task) {
- task_lock(task);
- if (task->fs) {
- get_fs_pwd(task->fs, path);
- result = 0;
- }
- task_unlock(task);
- put_task_struct(task);
+ task_lock(task);
+ if (task->fs) {
+ get_fs_pwd(task->fs, path);
+ result = 0;
}
+ task_unlock(task);
return result;
}
-static int proc_root_link(struct dentry *dentry, struct path *path)
+static int proc_root_link(struct dentry *dentry, struct path *path,
+ struct task_struct *task)
{
- struct task_struct *task = get_proc_task(d_inode(dentry));
- int result = -ENOENT;
-
- if (task) {
- result = get_task_root(task, path);
- put_task_struct(task);
- }
- return result;
+ return get_task_root(task, path);
}
/*
@@ -423,18 +414,24 @@ static int proc_pid_wchan(struct seq_file *m, struct pid_namespace *ns,
{
unsigned long wchan;
char symname[KSYM_NAME_LEN];
+ int err;
+ err = down_read_killable(&task->signal->exec_update_lock);
+ if (err)
+ return err;
if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS))
goto print0;
wchan = get_wchan(task);
if (wchan && !lookup_symbol_name(wchan, symname)) {
seq_puts(m, symname);
+ up_read(&task->signal->exec_update_lock);
return 0;
}
print0:
seq_putc(m, '0');
+ up_read(&task->signal->exec_update_lock);
return 0;
}
#endif /* CONFIG_KALLSYMS */
@@ -704,24 +701,7 @@ static int proc_pid_syscall(struct seq_file *m, struct pid_namespace *ns,
/* Here the fs part begins */
/************************************************************************/
-/* permission checks */
-static bool proc_fd_access_allowed(struct inode *inode)
-{
- struct task_struct *task;
- bool allowed = false;
- /* Allow access to a task's file descriptors if it is us or we
- * may use ptrace attach to the process and find out that
- * information.
- */
- task = get_proc_task(inode);
- if (task) {
- allowed = ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS);
- put_task_struct(task);
- }
- return allowed;
-}
-
-int proc_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int proc_nochmod_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
struct iattr *attr)
{
int error;
@@ -794,7 +774,7 @@ static int proc_pid_permission(struct mnt_idmap *idmap,
static const struct inode_operations proc_def_inode_operations = {
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
};
static int proc_single_show(struct seq_file *m, void *v)
@@ -1777,16 +1757,12 @@ static const struct file_operations proc_pid_set_comm_operations = {
.release = single_release,
};
-static int proc_exe_link(struct dentry *dentry, struct path *exe_path)
+static int proc_exe_link(struct dentry *dentry, struct path *exe_path,
+ struct task_struct *task)
{
- struct task_struct *task;
struct file *exe_file;
- task = get_proc_task(d_inode(dentry));
- if (!task)
- return -ENOENT;
exe_file = get_task_exe_file(task);
- put_task_struct(task);
if (exe_file) {
*exe_path = exe_file->f_path;
path_get(&exe_file->f_path);
@@ -1796,26 +1772,42 @@ static int proc_exe_link(struct dentry *dentry, struct path *exe_path)
return -ENOENT;
}
+static int call_proc_get_link(struct dentry *dentry, struct inode *inode, struct path *path_out)
+{
+ struct task_struct *task;
+ int ret;
+
+ task = get_proc_task(inode);
+ if (!task)
+ return -ENOENT;
+ ret = down_read_killable(&task->signal->exec_update_lock);
+ if (ret)
+ goto out_put_task;
+ if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) {
+ ret = -EACCES;
+ goto out;
+ }
+ ret = PROC_I(inode)->op.proc_get_link(dentry, path_out, task);
+
+out:
+ up_read(&task->signal->exec_update_lock);
+out_put_task:
+ put_task_struct(task);
+ return ret;
+}
+
static const char *proc_pid_get_link(struct dentry *dentry,
struct inode *inode,
struct delayed_call *done)
{
struct path path;
- int error = -EACCES;
+ int error;
if (!dentry)
return ERR_PTR(-ECHILD);
-
- /* Are we allowed to snoop on the tasks file descriptors? */
- if (!proc_fd_access_allowed(inode))
- goto out;
-
- error = PROC_I(inode)->op.proc_get_link(dentry, &path);
- if (error)
- goto out;
-
- error = nd_jump_link(&path);
-out:
+ error = call_proc_get_link(dentry, inode, &path);
+ if (!error)
+ error = nd_jump_link(&path);
return ERR_PTR(error);
}
@@ -1849,24 +1841,18 @@ static int proc_pid_readlink(struct dentry * dentry, char __user * buffer, int b
struct inode *inode = d_inode(dentry);
struct path path;
- /* Are we allowed to snoop on the tasks file descriptors? */
- if (!proc_fd_access_allowed(inode))
- goto out;
-
- error = PROC_I(inode)->op.proc_get_link(dentry, &path);
- if (error)
- goto out;
-
- error = do_proc_readlink(&path, buffer, buflen);
- path_put(&path);
-out:
+ error = call_proc_get_link(dentry, inode, &path);
+ if (!error) {
+ error = do_proc_readlink(&path, buffer, buflen);
+ path_put(&path);
+ }
return error;
}
const struct inode_operations proc_pid_link_inode_operations = {
.readlink = proc_pid_readlink,
.get_link = proc_pid_get_link,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
};
@@ -2250,21 +2236,16 @@ static const struct dentry_operations tid_map_files_dentry_operations = {
.d_delete = pid_delete_dentry,
};
-static int map_files_get_link(struct dentry *dentry, struct path *path)
+static int map_files_get_link(struct dentry *dentry, struct path *path,
+ struct task_struct *task)
{
unsigned long vm_start, vm_end;
struct vm_area_struct *vma;
- struct task_struct *task;
struct mm_struct *mm;
int rc;
rc = -ENOENT;
- task = get_proc_task(d_inode(dentry));
- if (!task)
- goto out;
-
mm = get_task_mm(task);
- put_task_struct(task);
if (!mm)
goto out;
@@ -2319,7 +2300,7 @@ proc_map_files_get_link(struct dentry *dentry,
static const struct inode_operations proc_map_files_link_inode_operations = {
.readlink = proc_pid_readlink,
.get_link = proc_map_files_get_link,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
};
static struct dentry *
@@ -2360,17 +2341,15 @@ static struct dentry *proc_map_files_lookup(struct inode *dir,
if (!task)
goto out;
- result = ERR_PTR(-EACCES);
- if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS))
- goto out_put_task;
-
result = ERR_PTR(-ENOENT);
if (dname_to_vma_addr(dentry, &vm_start, &vm_end))
goto out_put_task;
- mm = get_task_mm(task);
- if (!mm)
+ mm = mm_access(task, PTRACE_MODE_READ_FSCREDS);
+ if (IS_ERR(mm)) {
+ result = ERR_CAST(mm);
goto out_put_task;
+ }
result = ERR_PTR(-EINTR);
if (mmap_read_lock_killable(mm))
@@ -2398,7 +2377,7 @@ out:
static const struct inode_operations proc_map_files_inode_operations = {
.lookup = proc_map_files_lookup,
.permission = proc_fd_permission,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
};
static int
@@ -2420,23 +2399,22 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx)
if (!task)
goto out;
- ret = -EACCES;
- if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS))
- goto out_put_task;
-
ret = 0;
if (!dir_emit_dots(file, ctx))
goto out_put_task;
- mm = get_task_mm(task);
- if (!mm)
+ mm = mm_access(task, PTRACE_MODE_READ_FSCREDS);
+ if (IS_ERR(mm)) {
+ ret = PTR_ERR(mm);
+ /* if the task has no mm, the directory should just be empty */
+ if (ret == -ESRCH)
+ ret = 0;
goto out_put_task;
+ }
ret = mmap_read_lock_killable(mm);
- if (ret) {
- mmput(mm);
- goto out_put_task;
- }
+ if (ret)
+ goto out_put_mm;
nr_files = 0;
@@ -2462,8 +2440,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx)
if (!p) {
ret = -ENOMEM;
mmap_read_unlock(mm);
- mmput(mm);
- goto out_put_task;
+ goto out_put_mm;
}
p->start = vma->vm_start;
@@ -2471,7 +2448,6 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx)
p->mode = vma->vm_file->f_mode;
}
mmap_read_unlock(mm);
- mmput(mm);
for (i = 0; i < nr_files; i++) {
char buf[4 * sizeof(long) + 2]; /* max: %lx-%lx\0 */
@@ -2488,6 +2464,8 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx)
ctx->pos++;
}
+out_put_mm:
+ mmput(mm);
out_put_task:
put_task_struct(task);
out:
@@ -2885,7 +2863,7 @@ static struct dentry *proc_##LSM##_attr_dir_lookup(struct inode *dir, \
static const struct inode_operations proc_##LSM##_attr_dir_inode_ops = { \
.lookup = proc_##LSM##_attr_dir_lookup, \
.getattr = pid_getattr, \
- .setattr = proc_setattr, \
+ .setattr = proc_nochmod_setattr, \
}
#ifdef CONFIG_SECURITY_SMACK
@@ -2944,7 +2922,7 @@ static struct dentry *proc_attr_dir_lookup(struct inode *dir,
static const struct inode_operations proc_attr_dir_inode_operations = {
.lookup = proc_attr_dir_lookup,
.getattr = pid_getattr,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
};
#endif
@@ -3460,7 +3438,7 @@ static struct dentry *proc_tgid_base_lookup(struct inode *dir, struct dentry *de
static const struct inode_operations proc_tgid_base_inode_operations = {
.lookup = proc_tgid_base_lookup,
.getattr = pid_getattr,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
.permission = proc_pid_permission,
};
@@ -3659,7 +3637,7 @@ static int proc_tid_comm_permission(struct mnt_idmap *idmap,
}
static const struct inode_operations proc_tid_comm_inode_operations = {
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
.permission = proc_tid_comm_permission,
};
@@ -3788,7 +3766,7 @@ static const struct file_operations proc_tid_base_operations = {
static const struct inode_operations proc_tid_base_inode_operations = {
.lookup = proc_tid_base_lookup,
.getattr = pid_getattr,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
};
static struct dentry *proc_task_instantiate(struct dentry *dentry,
@@ -4001,7 +3979,7 @@ static loff_t proc_dir_llseek(struct file *file, loff_t offset, int whence)
static const struct inode_operations proc_task_inode_operations = {
.lookup = proc_task_lookup,
.getattr = proc_task_getattr,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
.permission = proc_pid_permission,
};
diff --git a/fs/proc/fd.c b/fs/proc/fd.c
index 9eeccff49b2a..5d6dac7fd1c8 100644
--- a/fs/proc/fd.c
+++ b/fs/proc/fd.c
@@ -102,7 +102,7 @@ static int proc_fdinfo_permission(struct mnt_idmap *idmap, struct inode *inode,
static const struct inode_operations proc_fdinfo_file_inode_operations = {
.permission = proc_fdinfo_permission,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
};
static const struct file_operations proc_fdinfo_file_operations = {
@@ -171,24 +171,19 @@ static const struct dentry_operations tid_fd_dentry_operations = {
.d_delete = pid_delete_dentry,
};
-static int proc_fd_link(struct dentry *dentry, struct path *path)
+static int proc_fd_link(struct dentry *dentry, struct path *path,
+ struct task_struct *task)
{
- struct task_struct *task;
int ret = -ENOENT;
-
- task = get_proc_task(d_inode(dentry));
- if (task) {
- unsigned int fd = proc_fd(d_inode(dentry));
- struct file *fd_file;
-
- fd_file = fget_task(task, fd);
- if (fd_file) {
- *path = fd_file->f_path;
- path_get(&fd_file->f_path);
- ret = 0;
- fput(fd_file);
- }
- put_task_struct(task);
+ unsigned int fd = proc_fd(d_inode(dentry));
+ struct file *fd_file;
+
+ fd_file = fget_task(task, fd);
+ if (fd_file) {
+ *path = fd_file->f_path;
+ path_get(&fd_file->f_path);
+ ret = 0;
+ fput(fd_file);
}
return ret;
@@ -361,7 +356,7 @@ const struct inode_operations proc_fd_inode_operations = {
.lookup = proc_lookupfd,
.permission = proc_fd_permission,
.getattr = proc_fd_getattr,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
};
static struct dentry *proc_fdinfo_instantiate(struct dentry *dentry,
@@ -402,7 +397,7 @@ static int proc_fdinfo_iterate(struct file *file, struct dir_context *ctx)
const struct inode_operations proc_fdinfo_inode_operations = {
.lookup = proc_lookupfdinfo,
.permission = proc_fdinfo_permission,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
};
const struct file_operations proc_fdinfo_operations = {
diff --git a/fs/proc/generic.c b/fs/proc/generic.c
index 501889856461..12b817f4ffcf 100644
--- a/fs/proc/generic.c
+++ b/fs/proc/generic.c
@@ -112,6 +112,8 @@ static bool pde_subdir_insert(struct proc_dir_entry *dir,
/* Add new node and rebalance tree. */
rb_link_node(&de->subdir_node, parent, new);
rb_insert_color(&de->subdir_node, root);
+ if (S_ISDIR(de->mode))
+ dir->nlink++;
return true;
}
@@ -404,7 +406,6 @@ struct proc_dir_entry *proc_register(struct proc_dir_entry *dir,
write_unlock(&proc_subdir_lock);
goto out_free_inum;
}
- dir->nlink++;
write_unlock(&proc_subdir_lock);
return dp;
@@ -702,6 +703,8 @@ static void pde_erase(struct proc_dir_entry *pde, struct proc_dir_entry *parent)
{
rb_erase(&pde->subdir_node, &parent->subdir);
RB_CLEAR_NODE(&pde->subdir_node);
+ if (S_ISDIR(pde->mode))
+ parent->nlink--;
}
/*
@@ -727,8 +730,6 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent)
de = NULL;
} else {
pde_erase(de, parent);
- if (S_ISDIR(de->mode))
- parent->nlink--;
}
}
write_unlock(&proc_subdir_lock);
@@ -787,8 +788,6 @@ int remove_proc_subtree(const char *name, struct proc_dir_entry *parent)
continue;
}
next = de->parent;
- if (S_ISDIR(de->mode))
- next->nlink--;
write_unlock(&proc_subdir_lock);
proc_entry_rundown(de);
diff --git a/fs/proc/internal.h b/fs/proc/internal.h
index d1598576506c..77c347d3b6ad 100644
--- a/fs/proc/internal.h
+++ b/fs/proc/internal.h
@@ -107,7 +107,7 @@ extern struct kmem_cache *proc_dir_entry_cache;
void pde_free(struct proc_dir_entry *pde);
union proc_op {
- int (*proc_get_link)(struct dentry *, struct path *);
+ int (*proc_get_link)(struct dentry *, struct path *, struct task_struct *);
int (*proc_show)(struct seq_file *m,
struct pid_namespace *ns, struct pid *pid,
struct task_struct *task);
@@ -257,8 +257,8 @@ extern int proc_pid_statm(struct seq_file *, struct pid_namespace *,
extern const struct dentry_operations pid_dentry_operations;
extern int pid_getattr(struct mnt_idmap *, const struct path *,
struct kstat *, u32, unsigned int);
-extern int proc_setattr(struct mnt_idmap *, struct dentry *,
- struct iattr *);
+int proc_nochmod_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+ struct iattr *attr);
extern void proc_pid_evict_inode(struct proc_inode *);
extern struct inode *proc_pid_make_inode(struct super_block *, struct task_struct *, umode_t);
extern void pid_update_inode(struct task_struct *, struct inode *);
diff --git a/fs/proc/namespaces.c b/fs/proc/namespaces.c
index ea2b597fd92c..2f46f1396744 100644
--- a/fs/proc/namespaces.c
+++ b/fs/proc/namespaces.c
@@ -55,6 +55,10 @@ static const char *proc_ns_get_link(struct dentry *dentry,
if (!task)
return ERR_PTR(-EACCES);
+ error = down_read_killable(&task->signal->exec_update_lock);
+ if (error)
+ goto out_put_task;
+
if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS))
goto out;
@@ -64,6 +68,8 @@ static const char *proc_ns_get_link(struct dentry *dentry,
error = nd_jump_link(&ns_path);
out:
+ up_read(&task->signal->exec_update_lock);
+out_put_task:
put_task_struct(task);
return ERR_PTR(error);
}
@@ -80,11 +86,17 @@ static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int bufl
if (!task)
return res;
+ res = down_read_killable(&task->signal->exec_update_lock);
+ if (res)
+ goto out_put_task;
+
if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) {
res = ns_get_name(name, sizeof(name), task, ns_ops);
if (res >= 0)
res = readlink_copy(buffer, buflen, name, strlen(name));
}
+ up_read(&task->signal->exec_update_lock);
+out_put_task:
put_task_struct(task);
return res;
}
@@ -92,7 +104,7 @@ static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int bufl
static const struct inode_operations proc_ns_link_inode_operations = {
.readlink = proc_ns_readlink,
.get_link = proc_ns_get_link,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
};
static struct dentry *proc_ns_instantiate(struct dentry *dentry,
@@ -178,5 +190,5 @@ out_no_task:
const struct inode_operations proc_ns_dir_inode_operations = {
.lookup = proc_ns_dir_lookup,
.getattr = pid_getattr,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
};
diff --git a/fs/proc/page.c b/fs/proc/page.c
index fc64f23e05e5..f612524c92a8 100644
--- a/fs/proc/page.c
+++ b/fs/proc/page.c
@@ -174,7 +174,7 @@ u64 stable_page_flags(const struct page *page)
u |= 1 << KPF_MMAP;
if (is_anon) {
u |= 1 << KPF_ANON;
- if (mapping & FOLIO_MAPPING_KSM)
+ if ((mapping & FOLIO_MAPPING_FLAGS) == FOLIO_MAPPING_KSM)
u |= 1 << KPF_KSM;
}
diff --git a/fs/proc/proc_net.c b/fs/proc/proc_net.c
index 52f0b75cbce2..184cddeb8215 100644
--- a/fs/proc/proc_net.c
+++ b/fs/proc/proc_net.c
@@ -322,7 +322,7 @@ static int proc_tgid_net_getattr(struct mnt_idmap *idmap,
const struct inode_operations proc_net_inode_operations = {
.lookup = proc_tgid_net_lookup,
.getattr = proc_tgid_net_getattr,
- .setattr = proc_setattr,
+ .setattr = proc_nochmod_setattr,
};
static int proc_tgid_net_readdir(struct file *file, struct dir_context *ctx)
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 4c5adfd4fc1f..38b9d47426ca 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -2834,7 +2834,7 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask,
if (~categories & PAGE_IS_WRITTEN)
goto out_unlock;
- if (end != start + HPAGE_SIZE) {
+ if (end != start + huge_page_size(hstate_vma(vma))) {
/* Partial HugeTLB page WP isn't possible. */
pagemap_scan_backout_range(p, start, end);
p->arg.walk_end = start;
@@ -2851,8 +2851,62 @@ out_unlock:
return ret;
}
+
+/*
+ * Write-protect the unpopulated hugetlb entries covering [addr, end) by
+ * installing uffd-wp markers inline, exactly as pagemap_scan_hugetlb_entry()
+ * does for populated entries.
+ *
+ * walk_hugetlb_range() currently calls ->pte_hole() once per huge page, so the
+ * loop normally runs a single iteration; it is written to cover the full range
+ * in case the walker ever coalesces adjacent holes.
+ *
+ * The obvious route -- uffd_wp_range() -> hugetlb_change_protection() --
+ * cannot be used here: it takes hugetlb_vma_lock_write(), but the page-table
+ * walker (walk_hugetlb_range()) already holds hugetlb_vma_lock_read() on the
+ * same VMA, so the scanning thread would deadlock against itself. PMD sharing
+ * is disabled on uffd-wp VMAs (hugetlb_unshare_all_pmds() at registration), so
+ * the vma lock guards nothing that matters for these entries anyway.
+ */
+static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma,
+ unsigned long addr, unsigned long end)
+{
+ struct hstate *h = hstate_vma(vma);
+ unsigned long psize = huge_page_size(h);
+ struct mm_struct *mm = vma->vm_mm;
+ spinlock_t *ptl;
+ pte_t *ptep;
+ pte_t pte;
+
+ for (addr = ALIGN_DOWN(addr, psize); addr < end; addr += psize) {
+ ptep = huge_pte_alloc(mm, vma, addr, psize);
+ if (!ptep)
+ return -ENOMEM;
+
+ i_mmap_lock_write(vma->vm_file->f_mapping);
+ ptl = huge_pte_lock(h, mm, ptep);
+ pte = huge_ptep_get(mm, addr, ptep);
+ make_uffd_wp_huge_pte(vma, addr, ptep, pte);
+ /*
+ * A none entry has no cached translation, so installing the
+ * marker needs no TLB flush. Flush only if a fault populated
+ * the entry between huge_pte_alloc() and the page table lock.
+ */
+ if (!huge_pte_none(pte))
+ flush_hugetlb_tlb_range(vma, addr, addr + psize);
+ spin_unlock(ptl);
+ i_mmap_unlock_write(vma->vm_file->f_mapping);
+ }
+
+ return 0;
+}
#else
#define pagemap_scan_hugetlb_entry NULL
+static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma,
+ unsigned long addr, unsigned long end)
+{
+ return 0;
+}
#endif
static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end,
@@ -2872,7 +2926,10 @@ static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end,
if (~p->arg.flags & PM_SCAN_WP_MATCHING)
return ret;
- err = uffd_wp_range(vma, addr, end - addr, true);
+ if (is_vm_hugetlb_page(vma))
+ err = pagemap_scan_hugetlb_hole_wp(vma, addr, end);
+ else
+ err = uffd_wp_range(vma, addr, end - addr, true);
if (err < 0)
ret = err;
generated by cgit v1.2.3 (git 2.25.1) at 2026-07-25 09:50:54 +0000