kernel/fork.c

   1 /*
   2  *  linux/kernel/fork.c
   3  *
   4  *  Copyright (C) 1991, 1992  Linus Torvalds
   5  */
   6
   7 /*
   8  *  'fork.c' contains the help-routines for the 'fork' system call
   9  * (see also entry.S and others).
  10  * Fork is rather simple, once you get the hang of it, but the memory
  11  * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
  12  */
  13
  14 #include <linux/config.h>
  15 #include <linux/slab.h>
  16 #include <linux/init.h>
  17 #include <linux/unistd.h>
  18 #include <linux/smp_lock.h>
  19 #include <linux/module.h>
  20 #include <linux/vmalloc.h>
  21 #include <linux/completion.h>
  22 #include <linux/namespace.h>
  23 #include <linux/personality.h>
  24 #include <linux/mempolicy.h>
  25 #include <linux/sem.h>
  26 #include <linux/file.h>
  27 #include <linux/key.h>
  28 #include <linux/binfmts.h>
  29 #include <linux/mman.h>
  30 #include <linux/fs.h>
  31 #include <linux/cpu.h>
  32 #include <linux/security.h>
  33 #include <linux/swap.h>
  34 #include <linux/syscalls.h>
  35 #include <linux/jiffies.h>
  36 #include <linux/futex.h>
  37 #include <linux/ptrace.h>
  38 #include <linux/mount.h>
  39 #include <linux/audit.h>
  40 #include <linux/profile.h>
  41 #include <linux/rmap.h>
  42 #include <linux/ckrm_events.h>
  43 #include <linux/ckrm_tsk.h>
  44 #include <linux/ckrm_tc.h>
  45 #include <linux/ckrm_mem_inline.h>
  46 #include <linux/vs_network.h>
  47 #include <linux/vs_limit.h>
  48 #include <linux/vs_memory.h>
  49
  50 #include <asm/pgtable.h>
  51 #include <asm/pgalloc.h>
  52 #include <asm/uaccess.h>
  53 #include <asm/mmu_context.h>
  54 #include <asm/cacheflush.h>
  55 #include <asm/tlbflush.h>
  56
  57 /* The idle threads do not count..
  58  * Protected by write_lock_irq(&tasklist_lock)
  59  */
  60 int nr_threads;
  61
  62 int max_threads;
  63 unsigned long total_forks;      /* Handle normal Linux uptimes. */
  64
  65 DEFINE_PER_CPU(unsigned long, process_counts) = 0;
  66
  67 rwlock_t tasklist_lock __cacheline_aligned = RW_LOCK_UNLOCKED;  /* outer */
  68
  69 EXPORT_SYMBOL(tasklist_lock);
  70
  71 int nr_processes(void)
  72 {
  73         int cpu;
  74         int total = 0;
  75
  76         for_each_online_cpu(cpu)
  77                 total += per_cpu(process_counts, cpu);
  78
  79         return total;
  80 }
  81
  82 #ifndef __HAVE_ARCH_TASK_STRUCT_ALLOCATOR
  83 # define alloc_task_struct()    kmem_cache_alloc(task_struct_cachep, GFP_KERNEL)
  84 # define free_task_struct(tsk)  kmem_cache_free(task_struct_cachep, (tsk))
  85 static kmem_cache_t *task_struct_cachep;
  86 #endif
  87
  88 void free_task(struct task_struct *tsk)
  89 {
  90         free_thread_info(tsk->thread_info);
  91         clr_vx_info(&tsk->vx_info);
  92         clr_nx_info(&tsk->nx_info);
  93         free_task_struct(tsk);
  94 }
  95 EXPORT_SYMBOL(free_task);
  96
  97 void __put_task_struct(struct task_struct *tsk)
  98 {
  99         WARN_ON(!(tsk->exit_state & (EXIT_DEAD | EXIT_ZOMBIE)));
 100         WARN_ON(atomic_read(&tsk->usage));
 101         WARN_ON(tsk == current);
 102
 103         if (unlikely(tsk->audit_context))
 104                 audit_free(tsk);
 105         security_task_free(tsk);
 106         free_uid(tsk->user);
 107         put_group_info(tsk->group_info);
 108
 109         if (!profile_handoff_task(tsk))
 110                 free_task(tsk);
 111 }
 112
 113 void __init fork_init(unsigned long mempages)
 114 {
 115 #ifndef __HAVE_ARCH_TASK_STRUCT_ALLOCATOR
 116 #ifndef ARCH_MIN_TASKALIGN
 117 #define ARCH_MIN_TASKALIGN      L1_CACHE_BYTES
 118 #endif
 119         /* create a slab on which task_structs can be allocated */
 120         task_struct_cachep =
 121                 kmem_cache_create("task_struct", sizeof(struct task_struct),
 122                         ARCH_MIN_TASKALIGN, SLAB_PANIC, NULL, NULL);
 123 #endif
 124
 125         /*
 126          * The default maximum number of threads is set to a safe
 127          * value: the thread structures can take up at most half
 128          * of memory.
 129          */
 130         max_threads = mempages / (8 * THREAD_SIZE / PAGE_SIZE);
 131
 132         /*
 133          * we need to allow at least 20 threads to boot a system
 134          */
 135         if(max_threads < 20)
 136                 max_threads = 20;
 137
 138         init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
 139         init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
 140 }
 141
 142 static struct task_struct *dup_task_struct(struct task_struct *orig)
 143 {
 144         struct task_struct *tsk;
 145         struct thread_info *ti;
 146
 147         prepare_to_copy(orig);
 148
 149         tsk = alloc_task_struct();
 150         if (!tsk)
 151                 return NULL;
 152
 153         ti = alloc_thread_info(tsk);
 154         if (!ti) {
 155                 free_task_struct(tsk);
 156                 return NULL;
 157         }
 158
 159         *ti = *orig->thread_info;
 160         *tsk = *orig;
 161         tsk->thread_info = ti;
 162         ti->task = tsk;
 163
 164         ckrm_cb_newtask(tsk);
 165         ckrm_task_mm_init(tsk);
 166         /* One for us, one for whoever does the "release_task()" (usually parent) */
 167         atomic_set(&tsk->usage,2);
 168         return tsk;
 169 }
 170
 171 #ifdef CONFIG_MMU
 172 static inline int dup_mmap(struct mm_struct * mm, struct mm_struct * oldmm)
 173 {
 174         struct vm_area_struct * mpnt, *tmp, **pprev;
 175         struct rb_node **rb_link, *rb_parent;
 176         int retval;
 177         unsigned long charge;
 178         struct mempolicy *pol;
 179
 180         down_write(&oldmm->mmap_sem);
 181         flush_cache_mm(current->mm);
 182         mm->locked_vm = 0;
 183         mm->mmap = NULL;
 184         mm->mmap_cache = NULL;
 185         mm->free_area_cache = oldmm->mmap_base;
 186         mm->map_count = 0;
 187         mm->rss = 0;
 188         mm->anon_rss = 0;
 189         cpus_clear(mm->cpu_vm_mask);
 190         mm->mm_rb = RB_ROOT;
 191         rb_link = &mm->mm_rb.rb_node;
 192         rb_parent = NULL;
 193         pprev = &mm->mmap;
 194
 195         for (mpnt = current->mm->mmap ; mpnt ; mpnt = mpnt->vm_next) {
 196                 struct file *file;
 197
 198                 if (mpnt->vm_flags & VM_DONTCOPY) {
 199                         __vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
 200                                                         -vma_pages(mpnt));
 201                         continue;
 202                 }
 203                 charge = 0;
 204                 if (mpnt->vm_flags & VM_ACCOUNT) {
 205                         unsigned int len = (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT;
 206                         if (security_vm_enough_memory(len))
 207                                 goto fail_nomem;
 208                         charge = len;
 209                 }
 210                 tmp = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
 211                 if (!tmp)
 212                         goto fail_nomem;
 213                 *tmp = *mpnt;
 214                 pol = mpol_copy(vma_policy(mpnt));
 215                 retval = PTR_ERR(pol);
 216                 if (IS_ERR(pol))
 217                         goto fail_nomem_policy;
 218                 vma_set_policy(tmp, pol);
 219                 tmp->vm_flags &= ~VM_LOCKED;
 220                 tmp->vm_mm = mm;
 221                 tmp->vm_next = NULL;
 222                 anon_vma_link(tmp);
 223                 file = tmp->vm_file;
 224                 if (file) {
 225                         struct inode *inode = file->f_dentry->d_inode;
 226                         get_file(file);
 227                         if (tmp->vm_flags & VM_DENYWRITE)
 228                                 atomic_dec(&inode->i_writecount);
 229
 230                         /* insert tmp into the share list, just after mpnt */
 231                         spin_lock(&file->f_mapping->i_mmap_lock);
 232                         flush_dcache_mmap_lock(file->f_mapping);
 233                         vma_prio_tree_add(tmp, mpnt);
 234                         flush_dcache_mmap_unlock(file->f_mapping);
 235                         spin_unlock(&file->f_mapping->i_mmap_lock);
 236                 }
 237
 238                 /*
 239                  * Link in the new vma and copy the page table entries:
 240                  * link in first so that swapoff can see swap entries,
 241                  * and try_to_unmap_one's find_vma find the new vma.
 242                  */
 243                 spin_lock(&mm->page_table_lock);
 244                 *pprev = tmp;
 245                 pprev = &tmp->vm_next;
 246
 247                 __vma_link_rb(mm, tmp, rb_link, rb_parent);
 248                 rb_link = &tmp->vm_rb.rb_right;
 249                 rb_parent = &tmp->vm_rb;
 250
 251                 mm->map_count++;
 252                 retval = copy_page_range(mm, current->mm, tmp);
 253                 spin_unlock(&mm->page_table_lock);
 254
 255                 if (tmp->vm_ops && tmp->vm_ops->open)
 256                         tmp->vm_ops->open(tmp);
 257
 258                 if (retval)
 259                         goto out;
 260         }
 261         retval = 0;
 262
 263 out:
 264         flush_tlb_mm(current->mm);
 265         up_write(&oldmm->mmap_sem);
 266         return retval;
 267 fail_nomem_policy:
 268         kmem_cache_free(vm_area_cachep, tmp);
 269 fail_nomem:
 270         retval = -ENOMEM;
 271         vm_unacct_memory(charge);
 272         goto out;
 273 }
 274
 275 static inline int mm_alloc_pgd(struct mm_struct * mm)
 276 {
 277         mm->pgd = pgd_alloc(mm);
 278         if (unlikely(!mm->pgd))
 279                 return -ENOMEM;
 280         return 0;
 281 }
 282
 283 static inline void mm_free_pgd(struct mm_struct * mm)
 284 {
 285         pgd_free(mm->pgd);
 286 }
 287 #else
 288 #define dup_mmap(mm, oldmm)     (0)
 289 #define mm_alloc_pgd(mm)        (0)
 290 #define mm_free_pgd(mm)
 291 #endif /* CONFIG_MMU */
 292
 293 spinlock_t mmlist_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 294
 295 #define allocate_mm()   (kmem_cache_alloc(mm_cachep, SLAB_KERNEL))
 296 #define free_mm(mm)     (kmem_cache_free(mm_cachep, (mm)))
 297
 298 #include <linux/init_task.h>
 299
 300 static struct mm_struct * mm_init(struct mm_struct * mm)
 301 {
 302         atomic_set(&mm->mm_users, 1);
 303         atomic_set(&mm->mm_count, 1);
 304         init_rwsem(&mm->mmap_sem);
 305         INIT_LIST_HEAD(&mm->mmlist);
 306         mm->core_waiters = 0;
 307         mm->nr_ptes = 0;
 308         spin_lock_init(&mm->page_table_lock);
 309         rwlock_init(&mm->ioctx_list_lock);
 310         mm->ioctx_list = NULL;
 311         mm->default_kioctx = (struct kioctx)INIT_KIOCTX(mm->default_kioctx, *mm);
 312         mm->free_area_cache = TASK_UNMAPPED_BASE;
 313         ckrm_mm_init(mm);
 314
 315         if (likely(!mm_alloc_pgd(mm))) {
 316                 mm->def_flags = 0;
 317                 set_vx_info(&mm->mm_vx_info, current->vx_info);
 318                 return mm;
 319         }
 320         free_mm(mm);
 321         return NULL;
 322 }
 323
 324 /*
 325  * Allocate and initialize an mm_struct.
 326  */
 327 struct mm_struct * mm_alloc(void)
 328 {
 329         struct mm_struct * mm;
 330
 331         mm = allocate_mm();
 332         if (mm) {
 333                 memset(mm, 0, sizeof(*mm));
 334                 mm = mm_init(mm);
 335                 ckrm_mm_setclass(mm, ckrm_get_mem_class(current));
 336         }
 337         return mm;
 338 }
 339
 340 /*
 341  * Called when the last reference to the mm
 342  * is dropped: either by a lazy thread or by
 343  * mmput. Free the page directory and the mm.
 344  */
 345 void fastcall __mmdrop(struct mm_struct *mm)
 346 {
 347         BUG_ON(mm == &init_mm);
 348         mm_free_pgd(mm);
 349         destroy_context(mm);
 350         ckrm_mm_clearclass(mm);
 351         clr_vx_info(&mm->mm_vx_info);
 352         free_mm(mm);
 353 }
 354
 355 /*
 356  * Decrement the use count and release all resources for an mm.
 357  */
 358 void mmput(struct mm_struct *mm)
 359 {
 360         if (atomic_dec_and_test(&mm->mm_users)) {
 361                 exit_aio(mm);
 362                 exit_mmap(mm);
 363                 if (!list_empty(&mm->mmlist)) {
 364                         spin_lock(&mmlist_lock);
 365                         list_del(&mm->mmlist);
 366                         spin_unlock(&mmlist_lock);
 367                 }
 368                 put_swap_token(mm);
 369                 mmdrop(mm);
 370         }
 371 }
 372 EXPORT_SYMBOL_GPL(mmput);
 373
 374 /**
 375  * get_task_mm - acquire a reference to the task's mm
 376  *
 377  * Returns %NULL if the task has no mm.  Checks PF_BORROWED_MM (meaning
 378  * this kernel workthread has transiently adopted a user mm with use_mm,
 379  * to do its AIO) is not set and if so returns a reference to it, after
 380  * bumping up the use count.  User must release the mm via mmput()
 381  * after use.  Typically used by /proc and ptrace.
 382  */
 383 struct mm_struct *get_task_mm(struct task_struct *task)
 384 {
 385         struct mm_struct *mm;
 386
 387         task_lock(task);
 388         mm = task->mm;
 389         if (mm) {
 390                 if (task->flags & PF_BORROWED_MM)
 391                         mm = NULL;
 392                 else
 393                         atomic_inc(&mm->mm_users);
 394         }
 395         task_unlock(task);
 396         return mm;
 397 }
 398 EXPORT_SYMBOL_GPL(get_task_mm);
 399
 400 /* Please note the differences between mmput and mm_release.
 401  * mmput is called whenever we stop holding onto a mm_struct,
 402  * error success whatever.
 403  *
 404  * mm_release is called after a mm_struct has been removed
 405  * from the current process.
 406  *
 407  * This difference is important for error handling, when we
 408  * only half set up a mm_struct for a new process and need to restore
 409  * the old one.  Because we mmput the new mm_struct before
 410  * restoring the old one. . .
 411  * Eric Biederman 10 January 1998
 412  */
 413 void mm_release(struct task_struct *tsk, struct mm_struct *mm)
 414 {
 415         struct completion *vfork_done = tsk->vfork_done;
 416
 417         /* Get rid of any cached register state */
 418         deactivate_mm(tsk, mm);
 419
 420         /* notify parent sleeping on vfork() */
 421         if (vfork_done) {
 422                 tsk->vfork_done = NULL;
 423                 complete(vfork_done);
 424         }
 425         if (tsk->clear_child_tid && atomic_read(&mm->mm_users) > 1) {
 426                 u32 __user * tidptr = tsk->clear_child_tid;
 427                 tsk->clear_child_tid = NULL;
 428
 429                 /*
 430                  * We don't check the error code - if userspace has
 431                  * not set up a proper pointer then tough luck.
 432                  */
 433                 put_user(0, tidptr);
 434                 sys_futex(tidptr, FUTEX_WAKE, 1, NULL, NULL, 0);
 435         }
 436 }
 437
 438 static int copy_mm(unsigned long clone_flags, struct task_struct * tsk)
 439 {
 440         struct mm_struct * mm, *oldmm;
 441         int retval;
 442
 443         tsk->min_flt = tsk->maj_flt = 0;
 444         tsk->nvcsw = tsk->nivcsw = 0;
 445
 446         tsk->mm = NULL;
 447         tsk->active_mm = NULL;
 448
 449         /*
 450          * Are we cloning a kernel thread?
 451          *
 452          * We need to steal a active VM for that..
 453          */
 454         oldmm = current->mm;
 455         if (!oldmm)
 456                 return 0;
 457
 458         if (clone_flags & CLONE_VM) {
 459                 atomic_inc(&oldmm->mm_users);
 460                 mm = oldmm;
 461                 /*
 462                  * There are cases where the PTL is held to ensure no
 463                  * new threads start up in user mode using an mm, which
 464                  * allows optimizing out ipis; the tlb_gather_mmu code
 465                  * is an example.
 466                  */
 467                 spin_unlock_wait(&oldmm->page_table_lock);
 468                 goto good_mm;
 469         }
 470
 471         retval = -ENOMEM;
 472         mm = allocate_mm();
 473         if (!mm)
 474                 goto fail_nomem;
 475
 476         /* Copy the current MM stuff.. */
 477         memcpy(mm, oldmm, sizeof(*mm));
 478         mm->mm_vx_info = NULL;
 479         if (!mm_init(mm))
 480                 goto fail_nomem;
 481
 482         if (init_new_context(tsk,mm))
 483                 goto fail_nocontext;
 484
 485         retval = dup_mmap(mm, oldmm);
 486         if (retval)
 487                 goto free_pt;
 488
 489 good_mm:
 490         ckrm_mm_setclass(mm, oldmm->memclass);
 491         tsk->mm = mm;
 492         tsk->active_mm = mm;
 493         ckrm_mm_setclass(mm, oldmm->memclass);
 494         ckrm_task_mm_set(mm, tsk);
 495         return 0;
 496
 497 free_pt:
 498         mmput(mm);
 499 fail_nomem:
 500         return retval;
 501
 502 fail_nocontext:
 503         /*
 504          * If init_new_context() failed, we cannot use mmput() to free the mm
 505          * because it calls destroy_context()
 506          */
 507         mm_free_pgd(mm);
 508         free_mm(mm);
 509         return retval;
 510 }
 511
 512 static inline struct fs_struct *__copy_fs_struct(struct fs_struct *old)
 513 {
 514         struct fs_struct *fs = kmem_cache_alloc(fs_cachep, GFP_KERNEL);
 515         /* We don't need to lock fs - think why ;-) */
 516         if (fs) {
 517                 atomic_set(&fs->count, 1);
 518                 rwlock_init(&fs->lock);
 519                 fs->umask = old->umask;
 520                 read_lock(&old->lock);
 521                 fs->rootmnt = mntget(old->rootmnt);
 522                 fs->root = dget(old->root);
 523                 fs->pwdmnt = mntget(old->pwdmnt);
 524                 fs->pwd = dget(old->pwd);
 525                 if (old->altroot) {
 526                         fs->altrootmnt = mntget(old->altrootmnt);
 527                         fs->altroot = dget(old->altroot);
 528                 } else {
 529                         fs->altrootmnt = NULL;
 530                         fs->altroot = NULL;
 531                 }
 532                 read_unlock(&old->lock);
 533         }
 534         return fs;
 535 }
 536
 537 struct fs_struct *copy_fs_struct(struct fs_struct *old)
 538 {
 539         return __copy_fs_struct(old);
 540 }
 541
 542 EXPORT_SYMBOL_GPL(copy_fs_struct);
 543
 544 static inline int copy_fs(unsigned long clone_flags, struct task_struct * tsk)
 545 {
 546         if (clone_flags & CLONE_FS) {
 547                 atomic_inc(&current->fs->count);
 548                 return 0;
 549         }
 550         tsk->fs = __copy_fs_struct(current->fs);
 551         if (!tsk->fs)
 552                 return -ENOMEM;
 553         return 0;
 554 }
 555
 556 static int count_open_files(struct files_struct *files, int size)
 557 {
 558         int i;
 559
 560         /* Find the last open fd */
 561         for (i = size/(8*sizeof(long)); i > 0; ) {
 562                 if (files->open_fds->fds_bits[--i])
 563                         break;
 564         }
 565         i = (i+1) * 8 * sizeof(long);
 566         return i;
 567 }
 568
 569 static int copy_files(unsigned long clone_flags, struct task_struct * tsk)
 570 {
 571         struct files_struct *oldf, *newf;
 572         struct file **old_fds, **new_fds;
 573         int open_files, nfds, size, i, error = 0;
 574
 575         /*
 576          * A background process may not have any files ...
 577          */
 578         oldf = current->files;
 579         if (!oldf)
 580                 goto out;
 581
 582         if (clone_flags & CLONE_FILES) {
 583                 atomic_inc(&oldf->count);
 584                 goto out;
 585         }
 586
 587         /*
 588          * Note: we may be using current for both targets (See exec.c)
 589          * This works because we cache current->files (old) as oldf. Don't
 590          * break this.
 591          */
 592         tsk->files = NULL;
 593         error = -ENOMEM;
 594         newf = kmem_cache_alloc(files_cachep, SLAB_KERNEL);
 595         if (!newf)
 596                 goto out;
 597
 598         atomic_set(&newf->count, 1);
 599
 600         spin_lock_init(&newf->file_lock);
 601         newf->next_fd       = 0;
 602         newf->max_fds       = NR_OPEN_DEFAULT;
 603         newf->max_fdset     = __FD_SETSIZE;
 604         newf->close_on_exec = &newf->close_on_exec_init;
 605         newf->open_fds      = &newf->open_fds_init;
 606         newf->fd            = &newf->fd_array[0];
 607
 608         /* We don't yet have the oldf readlock, but even if the old
 609            fdset gets grown now, we'll only copy up to "size" fds */
 610         size = oldf->max_fdset;
 611         if (size > __FD_SETSIZE) {
 612                 newf->max_fdset = 0;
 613                 spin_lock(&newf->file_lock);
 614                 error = expand_fdset(newf, size-1);
 615                 spin_unlock(&newf->file_lock);
 616                 if (error)
 617                         goto out_release;
 618         }
 619         spin_lock(&oldf->file_lock);
 620
 621         open_files = count_open_files(oldf, size);
 622
 623         /*
 624          * Check whether we need to allocate a larger fd array.
 625          * Note: we're not a clone task, so the open count won't
 626          * change.
 627          */
 628         nfds = NR_OPEN_DEFAULT;
 629         if (open_files > nfds) {
 630                 spin_unlock(&oldf->file_lock);
 631                 newf->max_fds = 0;
 632                 spin_lock(&newf->file_lock);
 633                 error = expand_fd_array(newf, open_files-1);
 634                 spin_unlock(&newf->file_lock);
 635                 if (error)
 636                         goto out_release;
 637                 nfds = newf->max_fds;
 638                 spin_lock(&oldf->file_lock);
 639         }
 640
 641         old_fds = oldf->fd;
 642         new_fds = newf->fd;
 643
 644         memcpy(newf->open_fds->fds_bits, oldf->open_fds->fds_bits, open_files/8);
 645         memcpy(newf->close_on_exec->fds_bits, oldf->close_on_exec->fds_bits, open_files/8);
 646
 647         for (i = open_files; i != 0; i--) {
 648                 struct file *f = *old_fds++;
 649                 if (f) {
 650                         get_file(f);
 651                 } else {
 652                         /*
 653                          * The fd may be claimed in the fd bitmap but not yet
 654                          * instantiated in the files array if a sibling thread
 655                          * is partway through open().  So make sure that this
 656                          * fd is available to the new process.
 657                          */
 658                         FD_CLR(open_files - i, newf->open_fds);
 659                 }
 660                 *new_fds++ = f;
 661         }
 662         spin_unlock(&oldf->file_lock);
 663
 664         /* compute the remainder to be cleared */
 665         size = (newf->max_fds - open_files) * sizeof(struct file *);
 666
 667         /* This is long word aligned thus could use a optimized version */
 668         memset(new_fds, 0, size);
 669
 670         if (newf->max_fdset > open_files) {
 671                 int left = (newf->max_fdset-open_files)/8;
 672                 int start = open_files / (8 * sizeof(unsigned long));
 673
 674                 memset(&newf->open_fds->fds_bits[start], 0, left);
 675                 memset(&newf->close_on_exec->fds_bits[start], 0, left);
 676         }
 677
 678         tsk->files = newf;
 679         error = 0;
 680 out:
 681         return error;
 682
 683 out_release:
 684         free_fdset (newf->close_on_exec, newf->max_fdset);
 685         free_fdset (newf->open_fds, newf->max_fdset);
 686         kmem_cache_free(files_cachep, newf);
 687         goto out;
 688 }
 689
 690 /*
 691  *      Helper to unshare the files of the current task.
 692  *      We don't want to expose copy_files internals to
 693  *      the exec layer of the kernel.
 694  */
 695
 696 int unshare_files(void)
 697 {
 698         struct files_struct *files  = current->files;
 699         int rc;
 700
 701         if(!files)
 702                 BUG();
 703
 704         /* This can race but the race causes us to copy when we don't
 705            need to and drop the copy */
 706         if(atomic_read(&files->count) == 1)
 707         {
 708                 atomic_inc(&files->count);
 709                 return 0;
 710         }
 711         rc = copy_files(0, current);
 712         if(rc)
 713                 current->files = files;
 714         return rc;
 715 }
 716
 717 EXPORT_SYMBOL(unshare_files);
 718
 719 static inline int copy_sighand(unsigned long clone_flags, struct task_struct * tsk)
 720 {
 721         struct sighand_struct *sig;
 722
 723         if (clone_flags & (CLONE_SIGHAND | CLONE_THREAD)) {
 724                 atomic_inc(&current->sighand->count);
 725                 return 0;
 726         }
 727         sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
 728         tsk->sighand = sig;
 729         if (!sig)
 730                 return -ENOMEM;
 731         spin_lock_init(&sig->siglock);
 732         atomic_set(&sig->count, 1);
 733         memcpy(sig->action, current->sighand->action, sizeof(sig->action));
 734         return 0;
 735 }
 736
 737 static inline int copy_signal(unsigned long clone_flags, struct task_struct * tsk)
 738 {
 739         struct signal_struct *sig;
 740
 741         if (clone_flags & CLONE_THREAD) {
 742                 atomic_inc(&current->signal->count);
 743                 atomic_inc(&current->signal->live);
 744                 return 0;
 745         }
 746         sig = kmem_cache_alloc(signal_cachep, GFP_KERNEL);
 747         tsk->signal = sig;
 748         if (!sig)
 749                 return -ENOMEM;
 750         atomic_set(&sig->count, 1);
 751         atomic_set(&sig->live, 1);
 752         sig->group_exit = 0;
 753         sig->group_exit_code = 0;
 754         sig->group_exit_task = NULL;
 755         sig->group_stop_count = 0;
 756         sig->stop_state = 0;
 757         sig->curr_target = NULL;
 758         init_sigpending(&sig->shared_pending);
 759         INIT_LIST_HEAD(&sig->posix_timers);
 760
 761         sig->tty = current->signal->tty;
 762         sig->pgrp = process_group(current);
 763         sig->session = current->signal->session;
 764         sig->leader = 0;        /* session leadership doesn't inherit */
 765         sig->tty_old_pgrp = 0;
 766
 767         sig->utime = sig->stime = sig->cutime = sig->cstime = 0;
 768         sig->nvcsw = sig->nivcsw = sig->cnvcsw = sig->cnivcsw = 0;
 769         sig->min_flt = sig->maj_flt = sig->cmin_flt = sig->cmaj_flt = 0;
 770
 771         task_lock(current->group_leader);
 772         memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
 773         task_unlock(current->group_leader);
 774
 775         return 0;
 776 }
 777
 778 static inline void copy_flags(unsigned long clone_flags, struct task_struct *p)
 779 {
 780         unsigned long new_flags = p->flags;
 781
 782         new_flags &= ~PF_SUPERPRIV;
 783         new_flags |= PF_FORKNOEXEC;
 784         if (!(clone_flags & CLONE_PTRACE))
 785                 p->ptrace = 0;
 786         p->flags = new_flags;
 787 }
 788
 789 asmlinkage long sys_set_tid_address(int __user *tidptr)
 790 {
 791         current->clear_child_tid = tidptr;
 792
 793         return current->pid;
 794 }
 795
 796 /*
 797  * This creates a new process as a copy of the old one,
 798  * but does not actually start it yet.
 799  *
 800  * It copies the registers, and all the appropriate
 801  * parts of the process environment (as per the clone
 802  * flags). The actual kick-off is left to the caller.
 803  */
 804 static task_t *copy_process(unsigned long clone_flags,
 805                                  unsigned long stack_start,
 806                                  struct pt_regs *regs,
 807                                  unsigned long stack_size,
 808                                  int __user *parent_tidptr,
 809                                  int __user *child_tidptr,
 810                                  int pid)
 811 {
 812         int retval;
 813         struct task_struct *p = NULL;
 814         struct vx_info *vxi;
 815
 816         if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
 817                 return ERR_PTR(-EINVAL);
 818
 819         /*
 820          * Thread groups must share signals as well, and detached threads
 821          * can only be started up within the thread group.
 822          */
 823         if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
 824                 return ERR_PTR(-EINVAL);
 825
 826         /*
 827          * Shared signal handlers imply shared VM. By way of the above,
 828          * thread groups also imply shared VM. Blocking this case allows
 829          * for various simplifications in other code.
 830          */
 831         if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
 832                 return ERR_PTR(-EINVAL);
 833
 834         retval = security_task_create(clone_flags);
 835         if (retval)
 836                 goto fork_out;
 837
 838         retval = -ENOMEM;
 839         p = dup_task_struct(current);
 840         if (!p)
 841                 goto fork_out;
 842         p->tux_info = NULL;
 843
 844         p->vx_info = NULL;
 845         set_vx_info(&p->vx_info, current->vx_info);
 846         p->nx_info = NULL;
 847         set_nx_info(&p->nx_info, current->nx_info);
 848
 849         /* check vserver memory */
 850         if (p->mm && !(clone_flags & CLONE_VM)) {
 851                 if (vx_vmpages_avail(p->mm, p->mm->total_vm))
 852                         vx_pages_add(p->mm->mm_vx_info, RLIMIT_AS, p->mm->total_vm);
 853                 else
 854                         goto bad_fork_free;
 855         }
 856         if (p->mm && vx_flags(VXF_FORK_RSS, 0)) {
 857                 if (!vx_rsspages_avail(p->mm, p->mm->rss))
 858                         goto bad_fork_cleanup_vm;
 859         }
 860
 861         p->vx_info = NULL;
 862         set_vx_info(&p->vx_info, current->vx_info);
 863         p->nx_info = NULL;
 864         set_nx_info(&p->nx_info, current->nx_info);
 865
 866         /* check vserver memory */
 867         if (p->mm && !(clone_flags & CLONE_VM)) {
 868                 if (vx_vmpages_avail(p->mm, p->mm->total_vm))
 869                         vx_pages_add(p->mm->mm_vx_info, RLIMIT_AS, p->mm->total_vm);
 870                 else
 871                         goto bad_fork_free;
 872         }
 873         if (p->mm && vx_flags(VXF_FORK_RSS, 0)) {
 874                 if (!vx_rsspages_avail(p->mm, p->mm->rss))
 875                         goto bad_fork_cleanup_vm;
 876         }
 877
 878         retval = -EAGAIN;
 879         if (!vx_nproc_avail(1))
 880                 goto bad_fork_cleanup_vm;
 881
 882         if (atomic_read(&p->user->processes) >=
 883                         p->signal->rlim[RLIMIT_NPROC].rlim_cur) {
 884                 if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RESOURCE) &&
 885                                 p->user != &root_user)
 886                         goto bad_fork_cleanup_vm;
 887         }
 888
 889         atomic_inc(&p->user->__count);
 890         atomic_inc(&p->user->processes);
 891         get_group_info(p->group_info);
 892
 893         /*
 894          * If multiple threads are within copy_process(), then this check
 895          * triggers too late. This doesn't hurt, the check is only there
 896          * to stop root fork bombs.
 897          */
 898         if (nr_threads >= max_threads)
 899                 goto bad_fork_cleanup_count;
 900
 901         if (!try_module_get(p->thread_info->exec_domain->module))
 902                 goto bad_fork_cleanup_count;
 903
 904         if (p->binfmt && !try_module_get(p->binfmt->module))
 905                 goto bad_fork_cleanup_put_domain;
 906
 907         init_delays(p);
 908         p->did_exec = 0;
 909         copy_flags(clone_flags, p);
 910         p->pid = pid;
 911         retval = -EFAULT;
 912         if (clone_flags & CLONE_PARENT_SETTID)
 913                 if (put_user(p->pid, parent_tidptr))
 914                         goto bad_fork_cleanup;
 915
 916         p->proc_dentry = NULL;
 917
 918         INIT_LIST_HEAD(&p->children);
 919         INIT_LIST_HEAD(&p->sibling);
 920         init_waitqueue_head(&p->wait_chldexit);
 921         p->vfork_done = NULL;
 922         spin_lock_init(&p->alloc_lock);
 923         spin_lock_init(&p->proc_lock);
 924
 925         clear_tsk_thread_flag(p, TIF_SIGPENDING);
 926         init_sigpending(&p->pending);
 927
 928         p->it_real_value = p->it_virt_value = p->it_prof_value = 0;
 929         p->it_real_incr = p->it_virt_incr = p->it_prof_incr = 0;
 930         init_timer(&p->real_timer);
 931         p->real_timer.data = (unsigned long) p;
 932
 933         p->utime = p->stime = 0;
 934         p->lock_depth = -1;             /* -1 = no lock */
 935         do_posix_clock_monotonic_gettime(&p->start_time);
 936         p->security = NULL;
 937         p->io_context = NULL;
 938         p->io_wait = NULL;
 939         p->audit_context = NULL;
 940 #ifdef CONFIG_NUMA
 941         p->mempolicy = mpol_copy(p->mempolicy);
 942         if (IS_ERR(p->mempolicy)) {
 943                 retval = PTR_ERR(p->mempolicy);
 944                 p->mempolicy = NULL;
 945                 goto bad_fork_cleanup;
 946         }
 947 #endif
 948
 949         p->tgid = p->pid;
 950         if (clone_flags & CLONE_THREAD)
 951                 p->tgid = current->tgid;
 952
 953         if ((retval = security_task_alloc(p)))
 954                 goto bad_fork_cleanup_policy;
 955         if ((retval = audit_alloc(p)))
 956                 goto bad_fork_cleanup_security;
 957         /* copy all the process information */
 958         if ((retval = copy_semundo(clone_flags, p)))
 959                 goto bad_fork_cleanup_audit;
 960         if ((retval = copy_files(clone_flags, p)))
 961                 goto bad_fork_cleanup_semundo;
 962         if ((retval = copy_fs(clone_flags, p)))
 963                 goto bad_fork_cleanup_files;
 964         if ((retval = copy_sighand(clone_flags, p)))
 965                 goto bad_fork_cleanup_fs;
 966         if ((retval = copy_signal(clone_flags, p)))
 967                 goto bad_fork_cleanup_sighand;
 968         if ((retval = copy_mm(clone_flags, p)))
 969                 goto bad_fork_cleanup_signal;
 970         if ((retval = copy_keys(clone_flags, p)))
 971                 goto bad_fork_cleanup_mm;
 972         if ((retval = copy_namespace(clone_flags, p)))
 973                 goto bad_fork_cleanup_keys;
 974         retval = copy_thread(0, clone_flags, stack_start, stack_size, p, regs);
 975         if (retval)
 976                 goto bad_fork_cleanup_namespace;
 977
 978         p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
 979         /*
 980          * Clear TID on mm_release()?
 981          */
 982         p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr: NULL;
 983
 984         /*
 985          * Syscall tracing should be turned off in the child regardless
 986          * of CLONE_PTRACE.
 987          */
 988         clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE);
 989
 990         /* Our parent execution domain becomes current domain
 991            These must match for thread signalling to apply */
 992
 993         p->parent_exec_id = p->self_exec_id;
 994
 995         /* ok, now we should be set up.. */
 996         p->exit_signal = (clone_flags & CLONE_THREAD) ? -1 : (clone_flags & CSIGNAL);
 997         p->pdeath_signal = 0;
 998         p->exit_state = 0;
 999
1000         /* Perform scheduler related setup */
1001         sched_fork(p);
1002
1003         /*
1004          * Ok, make it visible to the rest of the system.
1005          * We dont wake it up yet.
1006          */
1007         p->group_leader = p;
1008         INIT_LIST_HEAD(&p->ptrace_children);
1009         INIT_LIST_HEAD(&p->ptrace_list);
1010
1011         /* Need tasklist lock for parent etc handling! */
1012         write_lock_irq(&tasklist_lock);
1013
1014         /*
1015          * The task hasn't been attached yet, so cpus_allowed mask cannot
1016          * have changed. The cpus_allowed mask of the parent may have
1017          * changed after it was copied first time, and it may then move to
1018          * another CPU - so we re-copy it here and set the child's CPU to
1019          * the parent's CPU. This avoids alot of nasty races.
1020          */
1021         p->cpus_allowed = current->cpus_allowed;
1022         set_task_cpu(p, smp_processor_id());
1023
1024         /*
1025          * Check for pending SIGKILL! The new thread should not be allowed
1026          * to slip out of an OOM kill. (or normal SIGKILL.)
1027          */
1028         if (sigismember(&current->pending.signal, SIGKILL)) {
1029                 write_unlock_irq(&tasklist_lock);
1030                 retval = -EINTR;
1031                 goto bad_fork_cleanup_namespace;
1032         }
1033
1034         /* CLONE_PARENT re-uses the old parent */
1035         if (clone_flags & (CLONE_PARENT|CLONE_THREAD))
1036                 p->real_parent = current->real_parent;
1037         else
1038                 p->real_parent = current;
1039         p->parent = p->real_parent;
1040
1041         if (clone_flags & CLONE_THREAD) {
1042                 spin_lock(&current->sighand->siglock);
1043                 /*
1044                  * Important: if an exit-all has been started then
1045                  * do not create this new thread - the whole thread
1046                  * group is supposed to exit anyway.
1047                  */
1048                 if (current->signal->group_exit) {
1049                         spin_unlock(&current->sighand->siglock);
1050                         write_unlock_irq(&tasklist_lock);
1051                         retval = -EAGAIN;
1052                         goto bad_fork_cleanup_namespace;
1053                 }
1054                 p->group_leader = current->group_leader;
1055
1056                 if (current->signal->group_stop_count > 0) {
1057                         /*
1058                          * There is an all-stop in progress for the group.
1059                          * We ourselves will stop as soon as we check signals.
1060                          * Make the new thread part of that group stop too.
1061                          */
1062                         current->signal->group_stop_count++;
1063                         set_tsk_thread_flag(p, TIF_SIGPENDING);
1064                 }
1065
1066                 spin_unlock(&current->sighand->siglock);
1067         }
1068
1069         SET_LINKS(p);
1070         if (unlikely(p->ptrace & PT_PTRACED))
1071                 __ptrace_link(p, current->parent);
1072
1073         attach_pid(p, PIDTYPE_PID, p->pid);
1074         attach_pid(p, PIDTYPE_TGID, p->tgid);
1075         if (thread_group_leader(p)) {
1076                 attach_pid(p, PIDTYPE_PGID, process_group(p));
1077                 attach_pid(p, PIDTYPE_SID, p->signal->session);
1078                 if (p->pid)
1079                         __get_cpu_var(process_counts)++;
1080         }
1081
1082         p->ioprio = current->ioprio;
1083         nr_threads++;
1084         /* p is copy of current */
1085         vxi = p->vx_info;
1086         if (vxi) {
1087                 atomic_inc(&vxi->cvirt.nr_threads);
1088                 vx_nproc_inc(p);
1089         }
1090         write_unlock_irq(&tasklist_lock);
1091         retval = 0;
1092
1093 fork_out:
1094         if (retval)
1095                 return ERR_PTR(retval);
1096         return p;
1097
1098 bad_fork_cleanup_namespace:
1099         exit_namespace(p);
1100 bad_fork_cleanup_keys:
1101         exit_keys(p);
1102 bad_fork_cleanup_mm:
1103         if (p->mm)
1104                 mmput(p->mm);
1105 bad_fork_cleanup_signal:
1106         exit_signal(p);
1107 bad_fork_cleanup_sighand:
1108         exit_sighand(p);
1109 bad_fork_cleanup_fs:
1110         exit_fs(p); /* blocking */
1111 bad_fork_cleanup_files:
1112         exit_files(p); /* blocking */
1113 bad_fork_cleanup_semundo:
1114         exit_sem(p);
1115 bad_fork_cleanup_audit:
1116         audit_free(p);
1117 bad_fork_cleanup_security:
1118         security_task_free(p);
1119 bad_fork_cleanup_policy:
1120 #ifdef CONFIG_NUMA
1121         mpol_free(p->mempolicy);
1122 #endif
1123 bad_fork_cleanup:
1124         if (p->binfmt)
1125                 module_put(p->binfmt->module);
1126 bad_fork_cleanup_put_domain:
1127         module_put(p->thread_info->exec_domain->module);
1128 bad_fork_cleanup_count:
1129         put_group_info(p->group_info);
1130         atomic_dec(&p->user->processes);
1131         free_uid(p->user);
1132 bad_fork_cleanup_vm:
1133         if (p->mm && !(clone_flags & CLONE_VM))
1134                 vx_pages_sub(p->mm->mm_vx_info, RLIMIT_AS, p->mm->total_vm);
1135 bad_fork_free:
1136         free_task(p);
1137         goto fork_out;
1138 }
1139
1140 struct pt_regs * __devinit __attribute__((weak)) idle_regs(struct pt_regs *regs)
1141 {
1142         memset(regs, 0, sizeof(struct pt_regs));
1143         return regs;
1144 }
1145
1146 task_t * __devinit fork_idle(int cpu)
1147 {
1148         task_t *task;
1149         struct pt_regs regs;
1150
1151         task = copy_process(CLONE_VM, 0, idle_regs(&regs), 0, NULL, NULL, 0);
1152         if (!task)
1153                 return ERR_PTR(-ENOMEM);
1154         init_idle(task, cpu);
1155         unhash_process(task);
1156         return task;
1157 }
1158
1159 static inline int fork_traceflag (unsigned clone_flags)
1160 {
1161         if (clone_flags & CLONE_UNTRACED)
1162                 return 0;
1163         else if (clone_flags & CLONE_VFORK) {
1164                 if (current->ptrace & PT_TRACE_VFORK)
1165                         return PTRACE_EVENT_VFORK;
1166         } else if ((clone_flags & CSIGNAL) != SIGCHLD) {
1167                 if (current->ptrace & PT_TRACE_CLONE)
1168                         return PTRACE_EVENT_CLONE;
1169         } else if (current->ptrace & PT_TRACE_FORK)
1170                 return PTRACE_EVENT_FORK;
1171
1172         return 0;
1173 }
1174
1175 /*
1176  *  Ok, this is the main fork-routine.
1177  *
1178  * It copies the process, and if successful kick-starts
1179  * it and waits for it to finish using the VM if required.
1180  */
1181 long do_fork(unsigned long clone_flags,
1182               unsigned long stack_start,
1183               struct pt_regs *regs,
1184               unsigned long stack_size,
1185               int __user *parent_tidptr,
1186               int __user *child_tidptr)
1187 {
1188         struct task_struct *p;
1189         int trace = 0;
1190         long pid = alloc_pidmap();
1191
1192         if (pid < 0)
1193                 return -EAGAIN;
1194         if (unlikely(current->ptrace)) {
1195                 trace = fork_traceflag (clone_flags);
1196                 if (trace)
1197                         clone_flags |= CLONE_PTRACE;
1198         }
1199
1200         if (numtasks_get_ref(current->taskclass, 0) == 0) {
1201                 return -ENOMEM;
1202         }
1203         p = copy_process(clone_flags, stack_start, regs, stack_size, parent_tidptr, child_tidptr, pid);
1204
1205         /*
1206          * Do this prior waking up the new thread - the thread pointer
1207          * might get invalid after that point, if the thread exits quickly.
1208          */
1209         if (!IS_ERR(p)) {
1210                 struct completion vfork;
1211
1212                 ckrm_cb_fork(p);
1213
1214                 if (clone_flags & CLONE_VFORK) {
1215                         p->vfork_done = &vfork;
1216                         init_completion(&vfork);
1217                 }
1218
1219                 if ((p->ptrace & PT_PTRACED) || (clone_flags & CLONE_STOPPED)) {
1220                         /*
1221                          * We'll start up with an immediate SIGSTOP.
1222                          */
1223                         sigaddset(&p->pending.signal, SIGSTOP);
1224                         set_tsk_thread_flag(p, TIF_SIGPENDING);
1225                 }
1226
1227                 if (!(clone_flags & CLONE_STOPPED))
1228                         wake_up_new_task(p, clone_flags);
1229                 else
1230                         p->state = TASK_STOPPED;
1231                 ++total_forks;
1232
1233                 if (unlikely (trace)) {
1234                         current->ptrace_message = pid;
1235                         ptrace_notify ((trace << 8) | SIGTRAP);
1236                 }
1237
1238                 if (clone_flags & CLONE_VFORK) {
1239                         wait_for_completion(&vfork);
1240                         if (unlikely (current->ptrace & PT_TRACE_VFORK_DONE))
1241                                 ptrace_notify ((PTRACE_EVENT_VFORK_DONE << 8) | SIGTRAP);
1242                 }
1243         } else {
1244                 numtasks_put_ref(current->taskclass);
1245                 free_pidmap(pid);
1246                 pid = PTR_ERR(p);
1247         }
1248         return pid;
1249 }
1250
1251 /* SLAB cache for signal_struct structures (tsk->signal) */
1252 kmem_cache_t *signal_cachep;
1253
1254 /* SLAB cache for sighand_struct structures (tsk->sighand) */
1255 kmem_cache_t *sighand_cachep;
1256
1257 /* SLAB cache for files_struct structures (tsk->files) */
1258 kmem_cache_t *files_cachep;
1259
1260 /* SLAB cache for fs_struct structures (tsk->fs) */
1261 kmem_cache_t *fs_cachep;
1262
1263 /* SLAB cache for vm_area_struct structures */
1264 kmem_cache_t *vm_area_cachep;
1265
1266 /* SLAB cache for mm_struct structures (tsk->mm) */
1267 kmem_cache_t *mm_cachep;
1268
1269 void __init proc_caches_init(void)
1270 {
1271         sighand_cachep = kmem_cache_create("sighand_cache",
1272                         sizeof(struct sighand_struct), 0,
1273                         SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
1274         signal_cachep = kmem_cache_create("signal_cache",
1275                         sizeof(struct signal_struct), 0,
1276                         SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
1277         files_cachep = kmem_cache_create("files_cache",
1278                         sizeof(struct files_struct), 0,
1279                         SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
1280         fs_cachep = kmem_cache_create("fs_cache",
1281                         sizeof(struct fs_struct), 0,
1282                         SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
1283         vm_area_cachep = kmem_cache_create("vm_area_struct",
1284                         sizeof(struct vm_area_struct), 0,
1285                         SLAB_PANIC, NULL, NULL);
1286         mm_cachep = kmem_cache_create("mm_struct",
1287                         sizeof(struct mm_struct), 0,
1288                         SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
1289 }