3 * Copyright (C) 1992, 1993 Krishna Balasubramanian
4 * Many improvements/fixes by Bruno Haible.
5 * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994.
6 * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli.
8 * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
9 * BIGMEM support, Andrea Arcangeli <andrea@suse.de>
10 * SMP thread shm, Jean-Luc Boyard <jean-luc.boyard@siemens.fr>
11 * HIGHMEM support, Ingo Molnar <mingo@redhat.com>
12 * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <cr@sap.com>
13 * Shared /dev/zero support, Kanoj Sarcar <kanoj@sgi.com>
14 * Move the mm functionality over to mm/shmem.c, Christoph Rohland <cr@sap.com>
16 * support for audit of ipc object properties and permission changes
17 * Dustin Kirkland <dustin.kirkland@us.ibm.com>
20 #include <linux/config.h>
21 #include <linux/slab.h>
23 #include <linux/hugetlb.h>
24 #include <linux/shm.h>
25 #include <linux/init.h>
26 #include <linux/file.h>
27 #include <linux/mman.h>
28 #include <linux/shmem_fs.h>
29 #include <linux/security.h>
30 #include <linux/syscalls.h>
31 #include <linux/audit.h>
32 #include <linux/capability.h>
33 #include <linux/ptrace.h>
34 #include <linux/seq_file.h>
35 #include <linux/mutex.h>
36 #include <linux/vs_base.h>
37 #include <linux/vs_context.h>
38 #include <linux/vs_limit.h>
40 #include <asm/uaccess.h>
44 static struct file_operations shm_file_operations;
45 static struct vm_operations_struct shm_vm_ops;
47 static struct ipc_ids shm_ids;
49 #define shm_lock(id) ((struct shmid_kernel*)ipc_lock(&shm_ids,id))
50 #define shm_unlock(shp) ipc_unlock(&(shp)->shm_perm)
51 #define shm_get(id) ((struct shmid_kernel*)ipc_get(&shm_ids,id))
52 #define shm_buildid(id, seq) \
53 ipc_buildid(&shm_ids, id, seq)
55 static int newseg (key_t key, int shmflg, size_t size);
56 static void shm_open (struct vm_area_struct *shmd);
57 static void shm_close (struct vm_area_struct *shmd);
59 static int sysvipc_shm_proc_show(struct seq_file *s, void *it);
62 size_t shm_ctlmax = SHMMAX;
63 size_t shm_ctlall = SHMALL;
64 int shm_ctlmni = SHMMNI;
66 static int shm_tot; /* total number of shared memory pages */
68 void __init shm_init (void)
70 ipc_init_ids(&shm_ids, 1);
71 ipc_init_proc_interface("sysvipc/shm",
72 " key shmid perms size cpid lpid nattch uid gid cuid cgid atime dtime ctime\n",
74 sysvipc_shm_proc_show);
77 static inline int shm_checkid(struct shmid_kernel *s, int id)
79 if (ipc_checkid(&shm_ids,&s->shm_perm,id))
84 static inline struct shmid_kernel *shm_rmid(int id)
86 return (struct shmid_kernel *)ipc_rmid(&shm_ids,id);
89 static inline int shm_addid(struct shmid_kernel *shp)
91 return ipc_addid(&shm_ids, &shp->shm_perm, shm_ctlmni);
96 static inline void shm_inc (int id) {
97 struct shmid_kernel *shp;
101 shp->shm_atim = get_seconds();
102 shp->shm_lprid = current->tgid;
107 /* This is called by fork, once for every shm attach. */
108 static void shm_open (struct vm_area_struct *shmd)
110 shm_inc (shmd->vm_file->f_dentry->d_inode->i_ino);
114 * shm_destroy - free the struct shmid_kernel
116 * @shp: struct to free
118 * It has to be called with shp and shm_ids.mutex locked,
119 * but returns with shp unlocked and freed.
121 static void shm_destroy (struct shmid_kernel *shp)
123 struct vx_info *vxi = lookup_vx_info(shp->shm_perm.xid);
124 int numpages = (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
126 vx_ipcshm_sub(vxi, shp, numpages);
131 if (!is_file_hugepages(shp->shm_file))
132 shmem_lock(shp->shm_file, 0, shp->mlock_user);
134 user_shm_unlock(shp->shm_file->f_dentry->d_inode->i_size,
136 fput (shp->shm_file);
137 security_shm_free(shp);
143 * remove the attach descriptor shmd.
144 * free memory for segment if it is marked destroyed.
145 * The descriptor has already been removed from the current->mm->mmap list
146 * and will later be kfree()d.
148 static void shm_close (struct vm_area_struct *shmd)
150 struct file * file = shmd->vm_file;
151 int id = file->f_dentry->d_inode->i_ino;
152 struct shmid_kernel *shp;
154 mutex_lock(&shm_ids.mutex);
155 /* remove from the list of attaches of the shm segment */
158 shp->shm_lprid = current->tgid;
159 shp->shm_dtim = get_seconds();
161 if(shp->shm_nattch == 0 &&
162 shp->shm_perm.mode & SHM_DEST)
166 mutex_unlock(&shm_ids.mutex);
169 static int shm_mmap(struct file * file, struct vm_area_struct * vma)
173 ret = shmem_mmap(file, vma);
175 vma->vm_ops = &shm_vm_ops;
176 if (!(vma->vm_flags & VM_WRITE))
177 vma->vm_flags &= ~VM_MAYWRITE;
178 shm_inc(file->f_dentry->d_inode->i_ino);
184 static struct file_operations shm_file_operations = {
187 .get_unmapped_area = shmem_get_unmapped_area,
191 static struct vm_operations_struct shm_vm_ops = {
192 .open = shm_open, /* callback for a new vm-area open */
193 .close = shm_close, /* callback for when the vm-area is released */
194 .nopage = shmem_nopage,
195 #if defined(CONFIG_NUMA) && defined(CONFIG_SHMEM)
196 .set_policy = shmem_set_policy,
197 .get_policy = shmem_get_policy,
201 static int newseg (key_t key, int shmflg, size_t size)
204 struct shmid_kernel *shp;
205 int numpages = (size + PAGE_SIZE -1) >> PAGE_SHIFT;
210 if (size < SHMMIN || size > shm_ctlmax)
213 if (shm_tot + numpages >= shm_ctlall)
215 if (!vx_ipcshm_avail(current->vx_info, numpages))
218 shp = ipc_rcu_alloc(sizeof(*shp));
222 shp->shm_perm.key = key;
223 shp->shm_perm.xid = vx_current_xid();
224 shp->shm_perm.mode = (shmflg & S_IRWXUGO);
225 shp->mlock_user = NULL;
227 shp->shm_perm.security = NULL;
228 error = security_shm_alloc(shp);
234 if (shmflg & SHM_HUGETLB) {
235 /* hugetlb_zero_setup takes care of mlock user accounting */
236 file = hugetlb_zero_setup(size);
237 shp->mlock_user = current->user;
239 int acctflag = VM_ACCOUNT;
241 * Do not allow no accounting for OVERCOMMIT_NEVER, even
244 if ((shmflg & SHM_NORESERVE) &&
245 sysctl_overcommit_memory != OVERCOMMIT_NEVER)
247 sprintf (name, "SYSV%08x", key);
248 file = shmem_file_setup(name, size, acctflag);
250 error = PTR_ERR(file);
259 shp->shm_cprid = current->tgid;
261 shp->shm_atim = shp->shm_dtim = 0;
262 shp->shm_ctim = get_seconds();
263 shp->shm_segsz = size;
265 shp->id = shm_buildid(id,shp->shm_perm.seq);
266 shp->shm_file = file;
267 file->f_dentry->d_inode->i_ino = shp->id;
269 /* Hugetlb ops would have already been assigned. */
270 if (!(shmflg & SHM_HUGETLB))
271 file->f_op = &shm_file_operations;
274 vx_ipcshm_add(current->vx_info, key, numpages);
281 security_shm_free(shp);
286 asmlinkage long sys_shmget (key_t key, size_t size, int shmflg)
288 struct shmid_kernel *shp;
291 mutex_lock(&shm_ids.mutex);
292 if (key == IPC_PRIVATE) {
293 err = newseg(key, shmflg, size);
294 } else if ((id = ipc_findkey(&shm_ids, key)) == -1) {
295 if (!(shmflg & IPC_CREAT))
298 err = newseg(key, shmflg, size);
299 } else if ((shmflg & IPC_CREAT) && (shmflg & IPC_EXCL)) {
304 if (shp->shm_segsz < size)
306 else if (ipcperms(&shp->shm_perm, shmflg))
309 int shmid = shm_buildid(id, shp->shm_perm.seq);
310 err = security_shm_associate(shp, shmflg);
316 mutex_unlock(&shm_ids.mutex);
321 static inline unsigned long copy_shmid_to_user(void __user *buf, struct shmid64_ds *in, int version)
325 return copy_to_user(buf, in, sizeof(*in));
330 ipc64_perm_to_ipc_perm(&in->shm_perm, &out.shm_perm);
331 out.shm_segsz = in->shm_segsz;
332 out.shm_atime = in->shm_atime;
333 out.shm_dtime = in->shm_dtime;
334 out.shm_ctime = in->shm_ctime;
335 out.shm_cpid = in->shm_cpid;
336 out.shm_lpid = in->shm_lpid;
337 out.shm_nattch = in->shm_nattch;
339 return copy_to_user(buf, &out, sizeof(out));
352 static inline unsigned long copy_shmid_from_user(struct shm_setbuf *out, void __user *buf, int version)
357 struct shmid64_ds tbuf;
359 if (copy_from_user(&tbuf, buf, sizeof(tbuf)))
362 out->uid = tbuf.shm_perm.uid;
363 out->gid = tbuf.shm_perm.gid;
364 out->mode = tbuf.shm_perm.mode;
370 struct shmid_ds tbuf_old;
372 if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
375 out->uid = tbuf_old.shm_perm.uid;
376 out->gid = tbuf_old.shm_perm.gid;
377 out->mode = tbuf_old.shm_perm.mode;
386 static inline unsigned long copy_shminfo_to_user(void __user *buf, struct shminfo64 *in, int version)
390 return copy_to_user(buf, in, sizeof(*in));
395 if(in->shmmax > INT_MAX)
396 out.shmmax = INT_MAX;
398 out.shmmax = (int)in->shmmax;
400 out.shmmin = in->shmmin;
401 out.shmmni = in->shmmni;
402 out.shmseg = in->shmseg;
403 out.shmall = in->shmall;
405 return copy_to_user(buf, &out, sizeof(out));
412 static void shm_get_stat(unsigned long *rss, unsigned long *swp)
419 for (i = 0; i <= shm_ids.max_id; i++) {
420 struct shmid_kernel *shp;
427 inode = shp->shm_file->f_dentry->d_inode;
429 if (is_file_hugepages(shp->shm_file)) {
430 struct address_space *mapping = inode->i_mapping;
431 *rss += (HPAGE_SIZE/PAGE_SIZE)*mapping->nrpages;
433 struct shmem_inode_info *info = SHMEM_I(inode);
434 spin_lock(&info->lock);
435 *rss += inode->i_mapping->nrpages;
436 *swp += info->swapped;
437 spin_unlock(&info->lock);
442 asmlinkage long sys_shmctl (int shmid, int cmd, struct shmid_ds __user *buf)
444 struct shm_setbuf setbuf;
445 struct shmid_kernel *shp;
448 if (cmd < 0 || shmid < 0) {
453 version = ipc_parse_version(&cmd);
455 switch (cmd) { /* replace with proc interface ? */
458 struct shminfo64 shminfo;
460 err = security_shm_shmctl(NULL, cmd);
464 memset(&shminfo,0,sizeof(shminfo));
465 shminfo.shmmni = shminfo.shmseg = shm_ctlmni;
466 shminfo.shmmax = shm_ctlmax;
467 shminfo.shmall = shm_ctlall;
469 shminfo.shmmin = SHMMIN;
470 if(copy_shminfo_to_user (buf, &shminfo, version))
472 /* reading a integer is always atomic */
480 struct shm_info shm_info;
482 err = security_shm_shmctl(NULL, cmd);
486 memset(&shm_info,0,sizeof(shm_info));
487 mutex_lock(&shm_ids.mutex);
488 shm_info.used_ids = shm_ids.in_use;
489 shm_get_stat (&shm_info.shm_rss, &shm_info.shm_swp);
490 shm_info.shm_tot = shm_tot;
491 shm_info.swap_attempts = 0;
492 shm_info.swap_successes = 0;
493 err = shm_ids.max_id;
494 mutex_unlock(&shm_ids.mutex);
495 if(copy_to_user (buf, &shm_info, sizeof(shm_info))) {
500 err = err < 0 ? 0 : err;
506 struct shmid64_ds tbuf;
508 memset(&tbuf, 0, sizeof(tbuf));
509 shp = shm_lock(shmid);
513 } else if(cmd==SHM_STAT) {
515 if (shmid > shm_ids.max_id)
517 result = shm_buildid(shmid, shp->shm_perm.seq);
519 err = shm_checkid(shp,shmid);
525 if (ipcperms (&shp->shm_perm, S_IRUGO))
527 err = security_shm_shmctl(shp, cmd);
530 kernel_to_ipc64_perm(&shp->shm_perm, &tbuf.shm_perm);
531 tbuf.shm_segsz = shp->shm_segsz;
532 tbuf.shm_atime = shp->shm_atim;
533 tbuf.shm_dtime = shp->shm_dtim;
534 tbuf.shm_ctime = shp->shm_ctim;
535 tbuf.shm_cpid = shp->shm_cprid;
536 tbuf.shm_lpid = shp->shm_lprid;
537 if (!is_file_hugepages(shp->shm_file))
538 tbuf.shm_nattch = shp->shm_nattch;
540 tbuf.shm_nattch = file_count(shp->shm_file) - 1;
542 if(copy_shmid_to_user (buf, &tbuf, version))
551 shp = shm_lock(shmid);
556 err = shm_checkid(shp,shmid);
560 err = audit_ipc_obj(&(shp->shm_perm));
564 if (!capable(CAP_IPC_LOCK)) {
566 if (current->euid != shp->shm_perm.uid &&
567 current->euid != shp->shm_perm.cuid)
569 if (cmd == SHM_LOCK &&
570 !current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur)
574 err = security_shm_shmctl(shp, cmd);
579 struct user_struct * user = current->user;
580 if (!is_file_hugepages(shp->shm_file)) {
581 err = shmem_lock(shp->shm_file, 1, user);
583 shp->shm_perm.mode |= SHM_LOCKED;
584 shp->mlock_user = user;
587 } else if (!is_file_hugepages(shp->shm_file)) {
588 shmem_lock(shp->shm_file, 0, shp->mlock_user);
589 shp->shm_perm.mode &= ~SHM_LOCKED;
590 shp->mlock_user = NULL;
598 * We cannot simply remove the file. The SVID states
599 * that the block remains until the last person
600 * detaches from it, then is deleted. A shmat() on
601 * an RMID segment is legal in older Linux and if
602 * we change it apps break...
604 * Instead we set a destroyed flag, and then blow
605 * the name away when the usage hits zero.
607 mutex_lock(&shm_ids.mutex);
608 shp = shm_lock(shmid);
612 err = shm_checkid(shp, shmid);
616 err = audit_ipc_obj(&(shp->shm_perm));
620 if (current->euid != shp->shm_perm.uid &&
621 current->euid != shp->shm_perm.cuid &&
622 !capable(CAP_SYS_ADMIN)) {
627 err = security_shm_shmctl(shp, cmd);
631 if (shp->shm_nattch){
632 shp->shm_perm.mode |= SHM_DEST;
633 /* Do not find it any more */
634 shp->shm_perm.key = IPC_PRIVATE;
638 mutex_unlock(&shm_ids.mutex);
644 if (copy_shmid_from_user (&setbuf, buf, version)) {
648 mutex_lock(&shm_ids.mutex);
649 shp = shm_lock(shmid);
653 err = shm_checkid(shp,shmid);
656 err = audit_ipc_obj(&(shp->shm_perm));
659 err = audit_ipc_set_perm(0, setbuf.uid, setbuf.gid, setbuf.mode, &(shp->shm_perm));
663 if (current->euid != shp->shm_perm.uid &&
664 current->euid != shp->shm_perm.cuid &&
665 !capable(CAP_SYS_ADMIN)) {
669 err = security_shm_shmctl(shp, cmd);
673 shp->shm_perm.uid = setbuf.uid;
674 shp->shm_perm.gid = setbuf.gid;
675 shp->shm_perm.mode = (shp->shm_perm.mode & ~S_IRWXUGO)
676 | (setbuf.mode & S_IRWXUGO);
677 shp->shm_ctim = get_seconds();
690 mutex_unlock(&shm_ids.mutex);
699 * Fix shmaddr, allocate descriptor, map shm, add attach descriptor to lists.
701 * NOTE! Despite the name, this is NOT a direct system call entrypoint. The
702 * "raddr" thing points to kernel space, and there has to be a wrapper around
705 long do_shmat(int shmid, char __user *shmaddr, int shmflg, ulong *raddr)
707 struct shmid_kernel *shp;
714 unsigned long o_flags;
721 } else if ((addr = (ulong)shmaddr)) {
722 if (addr & (SHMLBA-1)) {
723 if (shmflg & SHM_RND)
724 addr &= ~(SHMLBA-1); /* round down */
726 #ifndef __ARCH_FORCE_SHMLBA
727 if (addr & ~PAGE_MASK)
731 flags = MAP_SHARED | MAP_FIXED;
733 if ((shmflg & SHM_REMAP))
739 if (shmflg & SHM_RDONLY) {
744 prot = PROT_READ | PROT_WRITE;
746 acc_mode = S_IRUGO | S_IWUGO;
748 if (shmflg & SHM_EXEC) {
754 * We cannot rely on the fs check since SYSV IPC does have an
755 * additional creator id...
757 shp = shm_lock(shmid);
762 err = shm_checkid(shp,shmid);
767 if (ipcperms(&shp->shm_perm, acc_mode)) {
773 err = security_shm_shmat(shp, shmaddr, shmflg);
779 file = shp->shm_file;
780 size = i_size_read(file->f_dentry->d_inode);
784 down_write(¤t->mm->mmap_sem);
785 if (addr && !(shmflg & SHM_REMAP)) {
786 user_addr = ERR_PTR(-EINVAL);
787 if (find_vma_intersection(current->mm, addr, addr + size))
790 * If shm segment goes below stack, make sure there is some
791 * space left for the stack to grow (at least 4 pages).
793 if (addr < current->mm->start_stack &&
794 addr > current->mm->start_stack - size - PAGE_SIZE * 5)
798 user_addr = (void*) do_mmap (file, addr, size, prot, flags, 0);
801 up_write(¤t->mm->mmap_sem);
803 mutex_lock(&shm_ids.mutex);
804 shp = shm_lock(shmid);
807 if(shp->shm_nattch == 0 &&
808 shp->shm_perm.mode & SHM_DEST)
812 mutex_unlock(&shm_ids.mutex);
814 *raddr = (unsigned long) user_addr;
816 if (IS_ERR(user_addr))
817 err = PTR_ERR(user_addr);
822 asmlinkage long sys_shmat(int shmid, char __user *shmaddr, int shmflg)
827 err = do_shmat(shmid, shmaddr, shmflg, &ret);
830 force_successful_syscall_return();
835 * detach and kill segment if marked destroyed.
836 * The work is done in shm_close.
838 asmlinkage long sys_shmdt(char __user *shmaddr)
840 struct mm_struct *mm = current->mm;
841 struct vm_area_struct *vma, *next;
842 unsigned long addr = (unsigned long)shmaddr;
844 int retval = -EINVAL;
846 if (addr & ~PAGE_MASK)
849 down_write(&mm->mmap_sem);
852 * This function tries to be smart and unmap shm segments that
853 * were modified by partial mlock or munmap calls:
854 * - It first determines the size of the shm segment that should be
855 * unmapped: It searches for a vma that is backed by shm and that
856 * started at address shmaddr. It records it's size and then unmaps
858 * - Then it unmaps all shm vmas that started at shmaddr and that
859 * are within the initially determined size.
860 * Errors from do_munmap are ignored: the function only fails if
861 * it's called with invalid parameters or if it's called to unmap
862 * a part of a vma. Both calls in this function are for full vmas,
863 * the parameters are directly copied from the vma itself and always
864 * valid - therefore do_munmap cannot fail. (famous last words?)
867 * If it had been mremap()'d, the starting address would not
868 * match the usual checks anyway. So assume all vma's are
869 * above the starting address given.
871 vma = find_vma(mm, addr);
877 * Check if the starting address would match, i.e. it's
878 * a fragment created by mprotect() and/or munmap(), or it
879 * otherwise it starts at this address with no hassles.
881 if ((vma->vm_ops == &shm_vm_ops || is_vm_hugetlb_page(vma)) &&
882 (vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) {
885 size = vma->vm_file->f_dentry->d_inode->i_size;
886 do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start);
888 * We discovered the size of the shm segment, so
889 * break out of here and fall through to the next
890 * loop that uses the size information to stop
891 * searching for matching vma's.
901 * We need look no further than the maximum address a fragment
902 * could possibly have landed at. Also cast things to loff_t to
903 * prevent overflows and make comparisions vs. equal-width types.
905 size = PAGE_ALIGN(size);
906 while (vma && (loff_t)(vma->vm_end - addr) <= size) {
909 /* finding a matching vma now does not alter retval */
910 if ((vma->vm_ops == &shm_vm_ops || is_vm_hugetlb_page(vma)) &&
911 (vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff)
913 do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start);
917 up_write(&mm->mmap_sem);
921 #ifdef CONFIG_PROC_FS
922 static int sysvipc_shm_proc_show(struct seq_file *s, void *it)
924 struct shmid_kernel *shp = it;
927 #define SMALL_STRING "%10d %10d %4o %10u %5u %5u %5d %5u %5u %5u %5u %10lu %10lu %10lu\n"
928 #define BIG_STRING "%10d %10d %4o %21u %5u %5u %5d %5u %5u %5u %5u %10lu %10lu %10lu\n"
930 if (!vx_check(shp->shm_perm.xid, VX_IDENT))
933 if (sizeof(size_t) <= sizeof(int))
934 format = SMALL_STRING;
937 return seq_printf(s, format,
944 is_file_hugepages(shp->shm_file) ? (file_count(shp->shm_file) - 1) : shp->shm_nattch,