kernel/fork.c

   1 /*
   2  *  linux/kernel/fork.c
   3  *
   4  *  Copyright (C) 1991, 1992  Linus Torvalds
   5  */
   6
   7 /*
   8  *  'fork.c' contains the help-routines for the 'fork' system call
   9  * (see also entry.S and others).
  10  * Fork is rather simple, once you get the hang of it, but the memory
  11  * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
  12  */
  13
  14 #include <linux/config.h>
  15 #include <linux/slab.h>
  16 #include <linux/init.h>
  17 #include <linux/unistd.h>
  18 #include <linux/smp_lock.h>
  19 #include <linux/module.h>
  20 #include <linux/vmalloc.h>
  21 #include <linux/completion.h>
  22 #include <linux/namespace.h>
  23 #include <linux/personality.h>
  24 #include <linux/mempolicy.h>
  25 #include <linux/sem.h>
  26 #include <linux/file.h>
  27 #include <linux/binfmts.h>
  28 #include <linux/mman.h>
  29 #include <linux/fs.h>
  30 #include <linux/cpu.h>
  31 #include <linux/security.h>
  32 #include <linux/swap.h>
  33 #include <linux/syscalls.h>
  34 #include <linux/jiffies.h>
  35 #include <linux/futex.h>
  36 #include <linux/ptrace.h>
  37 #include <linux/mount.h>
  38 #include <linux/audit.h>
  39 #include <linux/profile.h>
  40 #include <linux/rmap.h>
  41 #include <linux/vs_network.h>
  42 #include <linux/vs_limit.h>
  43 #include <linux/vs_memory.h>
  44
  45 #include <asm/pgtable.h>
  46 #include <asm/pgalloc.h>
  47 #include <asm/uaccess.h>
  48 #include <asm/mmu_context.h>
  49 #include <asm/cacheflush.h>
  50 #include <asm/tlbflush.h>
  51
  52 /* The idle threads do not count..
  53  * Protected by write_lock_irq(&tasklist_lock)
  54  */
  55 int nr_threads;
  56
  57 int max_threads;
  58 unsigned long total_forks;      /* Handle normal Linux uptimes. */
  59
  60 DEFINE_PER_CPU(unsigned long, process_counts) = 0;
  61
  62 rwlock_t tasklist_lock __cacheline_aligned = RW_LOCK_UNLOCKED;  /* outer */
  63
  64 EXPORT_SYMBOL(tasklist_lock);
  65
  66 int nr_processes(void)
  67 {
  68         int cpu;
  69         int total = 0;
  70
  71         for_each_online_cpu(cpu)
  72                 total += per_cpu(process_counts, cpu);
  73
  74         return total;
  75 }
  76
  77 #ifndef __HAVE_ARCH_TASK_STRUCT_ALLOCATOR
  78 # define alloc_task_struct()    kmem_cache_alloc(task_struct_cachep, GFP_KERNEL)
  79 # define free_task_struct(tsk)  kmem_cache_free(task_struct_cachep, (tsk))
  80 static kmem_cache_t *task_struct_cachep;
  81 #endif
  82
  83 void free_task(struct task_struct *tsk)
  84 {
  85         free_thread_info(tsk->thread_info);
  86         clr_vx_info(&tsk->vx_info);
  87         clr_nx_info(&tsk->nx_info);
  88         free_task_struct(tsk);
  89 }
  90 EXPORT_SYMBOL(free_task);
  91
  92 void __put_task_struct(struct task_struct *tsk)
  93 {
  94         WARN_ON(!(tsk->state & (TASK_DEAD | TASK_ZOMBIE)));
  95         WARN_ON(atomic_read(&tsk->usage));
  96         WARN_ON(tsk == current);
  97
  98         if (unlikely(tsk->audit_context))
  99                 audit_free(tsk);
 100         security_task_free(tsk);
 101         free_uid(tsk->user);
 102         put_group_info(tsk->group_info);
 103
 104         if (!profile_handoff_task(tsk))
 105                 free_task(tsk);
 106 }
 107
 108 void fastcall add_wait_queue(wait_queue_head_t *q, wait_queue_t * wait)
 109 {
 110         unsigned long flags;
 111
 112         wait->flags &= ~WQ_FLAG_EXCLUSIVE;
 113         spin_lock_irqsave(&q->lock, flags);
 114         __add_wait_queue(q, wait);
 115         spin_unlock_irqrestore(&q->lock, flags);
 116 }
 117
 118 EXPORT_SYMBOL(add_wait_queue);
 119
 120 void fastcall add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t * wait)
 121 {
 122         unsigned long flags;
 123
 124         wait->flags |= WQ_FLAG_EXCLUSIVE;
 125         spin_lock_irqsave(&q->lock, flags);
 126         __add_wait_queue_tail(q, wait);
 127         spin_unlock_irqrestore(&q->lock, flags);
 128 }
 129
 130 EXPORT_SYMBOL(add_wait_queue_exclusive);
 131
 132 void fastcall remove_wait_queue(wait_queue_head_t *q, wait_queue_t * wait)
 133 {
 134         unsigned long flags;
 135
 136         spin_lock_irqsave(&q->lock, flags);
 137         __remove_wait_queue(q, wait);
 138         spin_unlock_irqrestore(&q->lock, flags);
 139 }
 140
 141 EXPORT_SYMBOL(remove_wait_queue);
 142
 143
 144 /*
 145  * Note: we use "set_current_state()" _after_ the wait-queue add,
 146  * because we need a memory barrier there on SMP, so that any
 147  * wake-function that tests for the wait-queue being active
 148  * will be guaranteed to see waitqueue addition _or_ subsequent
 149  * tests in this thread will see the wakeup having taken place.
 150  *
 151  * The spin_unlock() itself is semi-permeable and only protects
 152  * one way (it only protects stuff inside the critical region and
 153  * stops them from bleeding out - it would still allow subsequent
 154  * loads to move into the the critical region).
 155  */
 156 void fastcall prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state)
 157 {
 158         unsigned long flags;
 159
 160         wait->flags &= ~WQ_FLAG_EXCLUSIVE;
 161         spin_lock_irqsave(&q->lock, flags);
 162         if (list_empty(&wait->task_list))
 163                 __add_wait_queue(q, wait);
 164         /*
 165          * don't alter the task state if this is just going to
 166          * queue an async wait queue callback
 167          */
 168         if (is_sync_wait(wait))
 169                 set_current_state(state);
 170         spin_unlock_irqrestore(&q->lock, flags);
 171 }
 172
 173 EXPORT_SYMBOL(prepare_to_wait);
 174
 175 void fastcall
 176 prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state)
 177 {
 178         unsigned long flags;
 179
 180         wait->flags |= WQ_FLAG_EXCLUSIVE;
 181         spin_lock_irqsave(&q->lock, flags);
 182         if (list_empty(&wait->task_list))
 183                 __add_wait_queue_tail(q, wait);
 184         /*
 185          * don't alter the task state if this is just going to
 186          * queue an async wait queue callback
 187          */
 188         if (is_sync_wait(wait))
 189                 set_current_state(state);
 190         spin_unlock_irqrestore(&q->lock, flags);
 191 }
 192
 193 EXPORT_SYMBOL(prepare_to_wait_exclusive);
 194
 195 void fastcall finish_wait(wait_queue_head_t *q, wait_queue_t *wait)
 196 {
 197         unsigned long flags;
 198
 199         __set_current_state(TASK_RUNNING);
 200         /*
 201          * We can check for list emptiness outside the lock
 202          * IFF:
 203          *  - we use the "careful" check that verifies both
 204          *    the next and prev pointers, so that there cannot
 205          *    be any half-pending updates in progress on other
 206          *    CPU's that we haven't seen yet (and that might
 207          *    still change the stack area.
 208          * and
 209          *  - all other users take the lock (ie we can only
 210          *    have _one_ other CPU that looks at or modifies
 211          *    the list).
 212          */
 213         if (!list_empty_careful(&wait->task_list)) {
 214                 spin_lock_irqsave(&q->lock, flags);
 215                 list_del_init(&wait->task_list);
 216                 spin_unlock_irqrestore(&q->lock, flags);
 217         }
 218 }
 219
 220 EXPORT_SYMBOL(finish_wait);
 221
 222 int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key)
 223 {
 224         int ret = default_wake_function(wait, mode, sync, key);
 225
 226         if (ret)
 227                 list_del_init(&wait->task_list);
 228         return ret;
 229 }
 230
 231 EXPORT_SYMBOL(autoremove_wake_function);
 232
 233 void __init fork_init(unsigned long mempages)
 234 {
 235 #ifndef __HAVE_ARCH_TASK_STRUCT_ALLOCATOR
 236 #ifndef ARCH_MIN_TASKALIGN
 237 #define ARCH_MIN_TASKALIGN      L1_CACHE_BYTES
 238 #endif
 239         /* create a slab on which task_structs can be allocated */
 240         task_struct_cachep =
 241                 kmem_cache_create("task_struct", sizeof(struct task_struct),
 242                         ARCH_MIN_TASKALIGN, SLAB_PANIC, NULL, NULL);
 243 #endif
 244
 245         /*
 246          * The default maximum number of threads is set to a safe
 247          * value: the thread structures can take up at most half
 248          * of memory.
 249          */
 250         max_threads = mempages / (THREAD_SIZE/PAGE_SIZE) / 8;
 251         /*
 252          * we need to allow at least 20 threads to boot a system
 253          */
 254         if(max_threads < 20)
 255                 max_threads = 20;
 256
 257         init_task.rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
 258         init_task.rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
 259 }
 260
 261 static struct task_struct *dup_task_struct(struct task_struct *orig)
 262 {
 263         struct task_struct *tsk;
 264         struct thread_info *ti;
 265
 266         prepare_to_copy(orig);
 267
 268         tsk = alloc_task_struct();
 269         if (!tsk)
 270                 return NULL;
 271
 272         ti = alloc_thread_info(tsk);
 273         if (!ti) {
 274                 free_task_struct(tsk);
 275                 return NULL;
 276         }
 277
 278         *ti = *orig->thread_info;
 279         *tsk = *orig;
 280         tsk->thread_info = ti;
 281         ti->task = tsk;
 282
 283         /* One for us, one for whoever does the "release_task()" (usually parent) */
 284         atomic_set(&tsk->usage,2);
 285         return tsk;
 286 }
 287
 288 #ifdef CONFIG_MMU
 289 static inline int dup_mmap(struct mm_struct * mm, struct mm_struct * oldmm)
 290 {
 291         struct vm_area_struct * mpnt, *tmp, **pprev;
 292         struct rb_node **rb_link, *rb_parent;
 293         int retval;
 294         unsigned long charge;
 295         struct mempolicy *pol;
 296
 297         down_write(&oldmm->mmap_sem);
 298         flush_cache_mm(current->mm);
 299         mm->locked_vm = 0;
 300         mm->mmap = NULL;
 301         mm->mmap_cache = NULL;
 302         mm->free_area_cache = oldmm->mmap_base;
 303         mm->map_count = 0;
 304         mm->rss = 0;
 305         cpus_clear(mm->cpu_vm_mask);
 306         mm->mm_rb = RB_ROOT;
 307         rb_link = &mm->mm_rb.rb_node;
 308         rb_parent = NULL;
 309         pprev = &mm->mmap;
 310
 311         /*
 312          * Add it to the mmlist after the parent.
 313          * Doing it this way means that we can order the list,
 314          * and fork() won't mess up the ordering significantly.
 315          * Add it first so that swapoff can see any swap entries.
 316          */
 317         spin_lock(&mmlist_lock);
 318         list_add(&mm->mmlist, &current->mm->mmlist);
 319         mmlist_nr++;
 320         spin_unlock(&mmlist_lock);
 321
 322         for (mpnt = current->mm->mmap ; mpnt ; mpnt = mpnt->vm_next) {
 323                 struct file *file;
 324
 325                 if (mpnt->vm_flags & VM_DONTCOPY) {
 326                         __vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
 327                                                         -vma_pages(mpnt));
 328                         continue;
 329                 }
 330                 charge = 0;
 331                 if (mpnt->vm_flags & VM_ACCOUNT) {
 332                         unsigned int len = (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT;
 333                         if (security_vm_enough_memory(len))
 334                                 goto fail_nomem;
 335                         charge = len;
 336                 }
 337                 tmp = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
 338                 if (!tmp)
 339                         goto fail_nomem;
 340                 *tmp = *mpnt;
 341                 pol = mpol_copy(vma_policy(mpnt));
 342                 retval = PTR_ERR(pol);
 343                 if (IS_ERR(pol))
 344                         goto fail_nomem_policy;
 345                 vma_set_policy(tmp, pol);
 346                 tmp->vm_flags &= ~VM_LOCKED;
 347                 tmp->vm_mm = mm;
 348                 tmp->vm_next = NULL;
 349                 anon_vma_link(tmp);
 350                 file = tmp->vm_file;
 351                 if (file) {
 352                         struct inode *inode = file->f_dentry->d_inode;
 353                         get_file(file);
 354                         if (tmp->vm_flags & VM_DENYWRITE)
 355                                 atomic_dec(&inode->i_writecount);
 356
 357                         /* insert tmp into the share list, just after mpnt */
 358                         spin_lock(&file->f_mapping->i_mmap_lock);
 359                         flush_dcache_mmap_lock(file->f_mapping);
 360                         vma_prio_tree_add(tmp, mpnt);
 361                         flush_dcache_mmap_unlock(file->f_mapping);
 362                         spin_unlock(&file->f_mapping->i_mmap_lock);
 363                 }
 364
 365                 /*
 366                  * Link in the new vma and copy the page table entries:
 367                  * link in first so that swapoff can see swap entries,
 368                  * and try_to_unmap_one's find_vma find the new vma.
 369                  */
 370                 spin_lock(&mm->page_table_lock);
 371                 *pprev = tmp;
 372                 pprev = &tmp->vm_next;
 373
 374                 __vma_link_rb(mm, tmp, rb_link, rb_parent);
 375                 rb_link = &tmp->vm_rb.rb_right;
 376                 rb_parent = &tmp->vm_rb;
 377
 378                 mm->map_count++;
 379                 retval = copy_page_range(mm, current->mm, tmp);
 380                 spin_unlock(&mm->page_table_lock);
 381
 382                 if (tmp->vm_ops && tmp->vm_ops->open)
 383                         tmp->vm_ops->open(tmp);
 384
 385                 if (retval)
 386                         goto out;
 387         }
 388         retval = 0;
 389
 390 out:
 391         flush_tlb_mm(current->mm);
 392         up_write(&oldmm->mmap_sem);
 393         return retval;
 394 fail_nomem_policy:
 395         kmem_cache_free(vm_area_cachep, tmp);
 396 fail_nomem:
 397         retval = -ENOMEM;
 398         vm_unacct_memory(charge);
 399         goto out;
 400 }
 401
 402 static inline int mm_alloc_pgd(struct mm_struct * mm)
 403 {
 404         mm->pgd = pgd_alloc(mm);
 405         if (unlikely(!mm->pgd))
 406                 return -ENOMEM;
 407         return 0;
 408 }
 409
 410 static inline void mm_free_pgd(struct mm_struct * mm)
 411 {
 412         pgd_free(mm->pgd);
 413 }
 414 #else
 415 #define dup_mmap(mm, oldmm)     (0)
 416 #define mm_alloc_pgd(mm)        (0)
 417 #define mm_free_pgd(mm)
 418 #endif /* CONFIG_MMU */
 419
 420 spinlock_t mmlist_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 421 int mmlist_nr;
 422
 423 #define allocate_mm()   (kmem_cache_alloc(mm_cachep, SLAB_KERNEL))
 424 #define free_mm(mm)     (kmem_cache_free(mm_cachep, (mm)))
 425
 426 #include <linux/init_task.h>
 427
 428 static struct mm_struct * mm_init(struct mm_struct * mm)
 429 {
 430         atomic_set(&mm->mm_users, 1);
 431         atomic_set(&mm->mm_count, 1);
 432         init_rwsem(&mm->mmap_sem);
 433         mm->core_waiters = 0;
 434         mm->page_table_lock = SPIN_LOCK_UNLOCKED;
 435         mm->ioctx_list_lock = RW_LOCK_UNLOCKED;
 436         mm->ioctx_list = NULL;
 437         mm->default_kioctx = (struct kioctx)INIT_KIOCTX(mm->default_kioctx, *mm);
 438         mm->free_area_cache = TASK_UNMAPPED_BASE;
 439
 440         if (likely(!mm_alloc_pgd(mm))) {
 441                 mm->def_flags = 0;
 442                 set_vx_info(&mm->mm_vx_info, current->vx_info);
 443                 return mm;
 444         }
 445         free_mm(mm);
 446         return NULL;
 447 }
 448
 449 /*
 450  * Allocate and initialize an mm_struct.
 451  */
 452 struct mm_struct * mm_alloc(void)
 453 {
 454         struct mm_struct * mm;
 455
 456         mm = allocate_mm();
 457         if (mm) {
 458                 memset(mm, 0, sizeof(*mm));
 459                 mm = mm_init(mm);
 460         }
 461         return mm;
 462 }
 463
 464 /*
 465  * Called when the last reference to the mm
 466  * is dropped: either by a lazy thread or by
 467  * mmput. Free the page directory and the mm.
 468  */
 469 void fastcall __mmdrop(struct mm_struct *mm)
 470 {
 471         BUG_ON(mm == &init_mm);
 472         mm_free_pgd(mm);
 473         destroy_context(mm);
 474         clr_vx_info(&mm->mm_vx_info);
 475         free_mm(mm);
 476 }
 477
 478 /*
 479  * Decrement the use count and release all resources for an mm.
 480  */
 481 void mmput(struct mm_struct *mm)
 482 {
 483         if (atomic_dec_and_lock(&mm->mm_users, &mmlist_lock)) {
 484                 list_del(&mm->mmlist);
 485                 mmlist_nr--;
 486                 spin_unlock(&mmlist_lock);
 487                 exit_aio(mm);
 488                 exit_mmap(mm);
 489                 put_swap_token(mm);
 490                 mmdrop(mm);
 491         }
 492 }
 493 EXPORT_SYMBOL_GPL(mmput);
 494
 495 /**
 496  * get_task_mm - acquire a reference to the task's mm
 497  *
 498  * Returns %NULL if the task has no mm.  Checks if the use count
 499  * of the mm is non-zero and if so returns a reference to it, after
 500  * bumping up the use count.  User must release the mm via mmput()
 501  * after use.  Typically used by /proc and ptrace.
 502  *
 503  * If the use count is zero, it means that this mm is going away,
 504  * so return %NULL.  This only happens in the case of an AIO daemon
 505  * which has temporarily adopted an mm (see use_mm), in the course
 506  * of its final mmput, before exit_aio has completed.
 507  */
 508 struct mm_struct *get_task_mm(struct task_struct *task)
 509 {
 510         struct mm_struct *mm;
 511
 512         task_lock(task);
 513         mm = task->mm;
 514         if (mm) {
 515                 spin_lock(&mmlist_lock);
 516                 if (!atomic_read(&mm->mm_users))
 517                         mm = NULL;
 518                 else
 519                         atomic_inc(&mm->mm_users);
 520                 spin_unlock(&mmlist_lock);
 521         }
 522         task_unlock(task);
 523         return mm;
 524 }
 525 EXPORT_SYMBOL_GPL(get_task_mm);
 526
 527 /* Please note the differences between mmput and mm_release.
 528  * mmput is called whenever we stop holding onto a mm_struct,
 529  * error success whatever.
 530  *
 531  * mm_release is called after a mm_struct has been removed
 532  * from the current process.
 533  *
 534  * This difference is important for error handling, when we
 535  * only half set up a mm_struct for a new process and need to restore
 536  * the old one.  Because we mmput the new mm_struct before
 537  * restoring the old one. . .
 538  * Eric Biederman 10 January 1998
 539  */
 540 void mm_release(struct task_struct *tsk, struct mm_struct *mm)
 541 {
 542         struct completion *vfork_done = tsk->vfork_done;
 543
 544         /* Get rid of any cached register state */
 545         deactivate_mm(tsk, mm);
 546
 547         /* notify parent sleeping on vfork() */
 548         if (vfork_done) {
 549                 tsk->vfork_done = NULL;
 550                 complete(vfork_done);
 551         }
 552         if (tsk->clear_child_tid && atomic_read(&mm->mm_users) > 1) {
 553                 u32 __user * tidptr = tsk->clear_child_tid;
 554                 tsk->clear_child_tid = NULL;
 555
 556                 /*
 557                  * We don't check the error code - if userspace has
 558                  * not set up a proper pointer then tough luck.
 559                  */
 560                 put_user(0, tidptr);
 561                 sys_futex(tidptr, FUTEX_WAKE, 1, NULL, NULL, 0);
 562         }
 563 }
 564
 565 static int copy_mm(unsigned long clone_flags, struct task_struct * tsk)
 566 {
 567         struct mm_struct * mm, *oldmm;
 568         int retval;
 569
 570         tsk->min_flt = tsk->maj_flt = 0;
 571         tsk->nvcsw = tsk->nivcsw = 0;
 572
 573         tsk->mm = NULL;
 574         tsk->active_mm = NULL;
 575
 576         /*
 577          * Are we cloning a kernel thread?
 578          *
 579          * We need to steal a active VM for that..
 580          */
 581         oldmm = current->mm;
 582         if (!oldmm)
 583                 return 0;
 584
 585         if (clone_flags & CLONE_VM) {
 586                 atomic_inc(&oldmm->mm_users);
 587                 mm = oldmm;
 588                 /*
 589                  * There are cases where the PTL is held to ensure no
 590                  * new threads start up in user mode using an mm, which
 591                  * allows optimizing out ipis; the tlb_gather_mmu code
 592                  * is an example.
 593                  */
 594                 spin_unlock_wait(&oldmm->page_table_lock);
 595                 goto good_mm;
 596         }
 597
 598         retval = -ENOMEM;
 599         mm = allocate_mm();
 600         if (!mm)
 601                 goto fail_nomem;
 602
 603         /* Copy the current MM stuff.. */
 604         memcpy(mm, oldmm, sizeof(*mm));
 605         mm->mm_vx_info = NULL;
 606         if (!mm_init(mm))
 607                 goto fail_nomem;
 608
 609         if (init_new_context(tsk,mm))
 610                 goto fail_nocontext;
 611
 612         retval = dup_mmap(mm, oldmm);
 613         if (retval)
 614                 goto free_pt;
 615
 616 good_mm:
 617         tsk->mm = mm;
 618         tsk->active_mm = mm;
 619         return 0;
 620
 621 free_pt:
 622         mmput(mm);
 623 fail_nomem:
 624         return retval;
 625
 626 fail_nocontext:
 627         /*
 628          * If init_new_context() failed, we cannot use mmput() to free the mm
 629          * because it calls destroy_context()
 630          */
 631         mm_free_pgd(mm);
 632         free_mm(mm);
 633         return retval;
 634 }
 635
 636 static inline struct fs_struct *__copy_fs_struct(struct fs_struct *old)
 637 {
 638         struct fs_struct *fs = kmem_cache_alloc(fs_cachep, GFP_KERNEL);
 639         /* We don't need to lock fs - think why ;-) */
 640         if (fs) {
 641                 atomic_set(&fs->count, 1);
 642                 fs->lock = RW_LOCK_UNLOCKED;
 643                 fs->umask = old->umask;
 644                 read_lock(&old->lock);
 645                 fs->rootmnt = mntget(old->rootmnt);
 646                 fs->root = dget(old->root);
 647                 fs->pwdmnt = mntget(old->pwdmnt);
 648                 fs->pwd = dget(old->pwd);
 649                 if (old->altroot) {
 650                         fs->altrootmnt = mntget(old->altrootmnt);
 651                         fs->altroot = dget(old->altroot);
 652                 } else {
 653                         fs->altrootmnt = NULL;
 654                         fs->altroot = NULL;
 655                 }
 656                 read_unlock(&old->lock);
 657         }
 658         return fs;
 659 }
 660
 661 struct fs_struct *copy_fs_struct(struct fs_struct *old)
 662 {
 663         return __copy_fs_struct(old);
 664 }
 665
 666 EXPORT_SYMBOL_GPL(copy_fs_struct);
 667
 668 static inline int copy_fs(unsigned long clone_flags, struct task_struct * tsk)
 669 {
 670         if (clone_flags & CLONE_FS) {
 671                 atomic_inc(&current->fs->count);
 672                 return 0;
 673         }
 674         tsk->fs = __copy_fs_struct(current->fs);
 675         if (!tsk->fs)
 676                 return -ENOMEM;
 677         return 0;
 678 }
 679
 680 static int count_open_files(struct files_struct *files, int size)
 681 {
 682         int i;
 683
 684         /* Find the last open fd */
 685         for (i = size/(8*sizeof(long)); i > 0; ) {
 686                 if (files->open_fds->fds_bits[--i])
 687                         break;
 688         }
 689         i = (i+1) * 8 * sizeof(long);
 690         return i;
 691 }
 692
 693 static int copy_files(unsigned long clone_flags, struct task_struct * tsk)
 694 {
 695         struct files_struct *oldf, *newf;
 696         struct file **old_fds, **new_fds;
 697         int open_files, nfds, size, i, error = 0;
 698
 699         /*
 700          * A background process may not have any files ...
 701          */
 702         oldf = current->files;
 703         if (!oldf)
 704                 goto out;
 705
 706         if (clone_flags & CLONE_FILES) {
 707                 atomic_inc(&oldf->count);
 708                 goto out;
 709         }
 710
 711         /*
 712          * Note: we may be using current for both targets (See exec.c)
 713          * This works because we cache current->files (old) as oldf. Don't
 714          * break this.
 715          */
 716         tsk->files = NULL;
 717         error = -ENOMEM;
 718         newf = kmem_cache_alloc(files_cachep, SLAB_KERNEL);
 719         if (!newf)
 720                 goto out;
 721
 722         atomic_set(&newf->count, 1);
 723
 724         newf->file_lock     = SPIN_LOCK_UNLOCKED;
 725         newf->next_fd       = 0;
 726         newf->max_fds       = NR_OPEN_DEFAULT;
 727         newf->max_fdset     = __FD_SETSIZE;
 728         newf->close_on_exec = &newf->close_on_exec_init;
 729         newf->open_fds      = &newf->open_fds_init;
 730         newf->fd            = &newf->fd_array[0];
 731
 732         /* We don't yet have the oldf readlock, but even if the old
 733            fdset gets grown now, we'll only copy up to "size" fds */
 734         size = oldf->max_fdset;
 735         if (size > __FD_SETSIZE) {
 736                 newf->max_fdset = 0;
 737                 spin_lock(&newf->file_lock);
 738                 error = expand_fdset(newf, size-1);
 739                 spin_unlock(&newf->file_lock);
 740                 if (error)
 741                         goto out_release;
 742         }
 743         spin_lock(&oldf->file_lock);
 744
 745         open_files = count_open_files(oldf, size);
 746
 747         /*
 748          * Check whether we need to allocate a larger fd array.
 749          * Note: we're not a clone task, so the open count won't
 750          * change.
 751          */
 752         nfds = NR_OPEN_DEFAULT;
 753         if (open_files > nfds) {
 754                 spin_unlock(&oldf->file_lock);
 755                 newf->max_fds = 0;
 756                 spin_lock(&newf->file_lock);
 757                 error = expand_fd_array(newf, open_files-1);
 758                 spin_unlock(&newf->file_lock);
 759                 if (error)
 760                         goto out_release;
 761                 nfds = newf->max_fds;
 762                 spin_lock(&oldf->file_lock);
 763         }
 764
 765         old_fds = oldf->fd;
 766         new_fds = newf->fd;
 767
 768         memcpy(newf->open_fds->fds_bits, oldf->open_fds->fds_bits, open_files/8);
 769         memcpy(newf->close_on_exec->fds_bits, oldf->close_on_exec->fds_bits, open_files/8);
 770
 771         for (i = open_files; i != 0; i--) {
 772                 struct file *f = *old_fds++;
 773                 if (f)
 774                         get_file(f);
 775                 *new_fds++ = f;
 776         }
 777         spin_unlock(&oldf->file_lock);
 778
 779         /* compute the remainder to be cleared */
 780         size = (newf->max_fds - open_files) * sizeof(struct file *);
 781
 782         /* This is long word aligned thus could use a optimized version */
 783         memset(new_fds, 0, size);
 784
 785         if (newf->max_fdset > open_files) {
 786                 int left = (newf->max_fdset-open_files)/8;
 787                 int start = open_files / (8 * sizeof(unsigned long));
 788
 789                 memset(&newf->open_fds->fds_bits[start], 0, left);
 790                 memset(&newf->close_on_exec->fds_bits[start], 0, left);
 791         }
 792
 793         tsk->files = newf;
 794         error = 0;
 795 out:
 796         return error;
 797
 798 out_release:
 799         free_fdset (newf->close_on_exec, newf->max_fdset);
 800         free_fdset (newf->open_fds, newf->max_fdset);
 801         kmem_cache_free(files_cachep, newf);
 802         goto out;
 803 }
 804
 805 /*
 806  *      Helper to unshare the files of the current task.
 807  *      We don't want to expose copy_files internals to
 808  *      the exec layer of the kernel.
 809  */
 810
 811 int unshare_files(void)
 812 {
 813         struct files_struct *files  = current->files;
 814         int rc;
 815
 816         if(!files)
 817                 BUG();
 818
 819         /* This can race but the race causes us to copy when we don't
 820            need to and drop the copy */
 821         if(atomic_read(&files->count) == 1)
 822         {
 823                 atomic_inc(&files->count);
 824                 return 0;
 825         }
 826         rc = copy_files(0, current);
 827         if(rc)
 828                 current->files = files;
 829         return rc;
 830 }
 831
 832 EXPORT_SYMBOL(unshare_files);
 833
 834 static inline int copy_sighand(unsigned long clone_flags, struct task_struct * tsk)
 835 {
 836         struct sighand_struct *sig;
 837
 838         if (clone_flags & (CLONE_SIGHAND | CLONE_THREAD)) {
 839                 atomic_inc(&current->sighand->count);
 840                 return 0;
 841         }
 842         sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
 843         tsk->sighand = sig;
 844         if (!sig)
 845                 return -ENOMEM;
 846         spin_lock_init(&sig->siglock);
 847         atomic_set(&sig->count, 1);
 848         memcpy(sig->action, current->sighand->action, sizeof(sig->action));
 849         return 0;
 850 }
 851
 852 static inline int copy_signal(unsigned long clone_flags, struct task_struct * tsk)
 853 {
 854         struct signal_struct *sig;
 855
 856         if (clone_flags & CLONE_THREAD) {
 857                 atomic_inc(&current->signal->count);
 858                 return 0;
 859         }
 860         sig = kmem_cache_alloc(signal_cachep, GFP_KERNEL);
 861         tsk->signal = sig;
 862         if (!sig)
 863                 return -ENOMEM;
 864         atomic_set(&sig->count, 1);
 865         sig->group_exit = 0;
 866         sig->group_exit_code = 0;
 867         sig->group_exit_task = NULL;
 868         sig->group_stop_count = 0;
 869         sig->curr_target = NULL;
 870         init_sigpending(&sig->shared_pending);
 871         INIT_LIST_HEAD(&sig->posix_timers);
 872
 873         sig->tty = current->signal->tty;
 874         sig->pgrp = process_group(current);
 875         sig->session = current->signal->session;
 876         sig->leader = 0;        /* session leadership doesn't inherit */
 877         sig->tty_old_pgrp = 0;
 878
 879         sig->utime = sig->stime = sig->cutime = sig->cstime = 0;
 880         sig->nvcsw = sig->nivcsw = sig->cnvcsw = sig->cnivcsw = 0;
 881         sig->min_flt = sig->maj_flt = sig->cmin_flt = sig->cmaj_flt = 0;
 882
 883         return 0;
 884 }
 885
 886 static inline void copy_flags(unsigned long clone_flags, struct task_struct *p)
 887 {
 888         unsigned long new_flags = p->flags;
 889
 890         new_flags &= ~PF_SUPERPRIV;
 891         new_flags |= PF_FORKNOEXEC;
 892         if (!(clone_flags & CLONE_PTRACE))
 893                 p->ptrace = 0;
 894         p->flags = new_flags;
 895 }
 896
 897 asmlinkage long sys_set_tid_address(int __user *tidptr)
 898 {
 899         current->clear_child_tid = tidptr;
 900
 901         return current->pid;
 902 }
 903
 904 /*
 905  * This creates a new process as a copy of the old one,
 906  * but does not actually start it yet.
 907  *
 908  * It copies the registers, and all the appropriate
 909  * parts of the process environment (as per the clone
 910  * flags). The actual kick-off is left to the caller.
 911  */
 912 static task_t *copy_process(unsigned long clone_flags,
 913                                  unsigned long stack_start,
 914                                  struct pt_regs *regs,
 915                                  unsigned long stack_size,
 916                                  int __user *parent_tidptr,
 917                                  int __user *child_tidptr,
 918                                  int pid)
 919 {
 920         int retval;
 921         struct task_struct *p = NULL;
 922         struct vx_info *vxi;
 923
 924         if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
 925                 return ERR_PTR(-EINVAL);
 926
 927         /*
 928          * Thread groups must share signals as well, and detached threads
 929          * can only be started up within the thread group.
 930          */
 931         if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
 932                 return ERR_PTR(-EINVAL);
 933
 934         /*
 935          * Shared signal handlers imply shared VM. By way of the above,
 936          * thread groups also imply shared VM. Blocking this case allows
 937          * for various simplifications in other code.
 938          */
 939         if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
 940                 return ERR_PTR(-EINVAL);
 941
 942         retval = security_task_create(clone_flags);
 943         if (retval)
 944                 goto fork_out;
 945
 946         retval = -ENOMEM;
 947         p = dup_task_struct(current);
 948         if (!p)
 949                 goto fork_out;
 950
 951         p->vx_info = NULL;
 952         set_vx_info(&p->vx_info, current->vx_info);
 953         p->nx_info = NULL;
 954         set_nx_info(&p->nx_info, current->nx_info);
 955
 956         /* check vserver memory */
 957         if (p->mm && !(clone_flags & CLONE_VM)) {
 958                 if (vx_vmpages_avail(p->mm, p->mm->total_vm))
 959                         vx_pages_add(p->mm->mm_vx_info, RLIMIT_AS, p->mm->total_vm);
 960                 else
 961                         goto bad_fork_free;
 962         }
 963         if (p->mm && vx_flags(VXF_FORK_RSS, 0)) {
 964                 if (!vx_rsspages_avail(p->mm, p->mm->rss))
 965                         goto bad_fork_cleanup_vm;
 966         }
 967
 968         retval = -EAGAIN;
 969         if (!vx_nproc_avail(1))
 970                 goto bad_fork_cleanup_vm;
 971
 972         if (atomic_read(&p->user->processes) >=
 973                         p->rlim[RLIMIT_NPROC].rlim_cur) {
 974                 if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RESOURCE) &&
 975                                 p->user != &root_user)
 976                         goto bad_fork_cleanup_vm;
 977         }
 978
 979         atomic_inc(&p->user->__count);
 980         atomic_inc(&p->user->processes);
 981         get_group_info(p->group_info);
 982
 983         /*
 984          * If multiple threads are within copy_process(), then this check
 985          * triggers too late. This doesn't hurt, the check is only there
 986          * to stop root fork bombs.
 987          */
 988         if (nr_threads >= max_threads)
 989                 goto bad_fork_cleanup_count;
 990
 991         if (!try_module_get(p->thread_info->exec_domain->module))
 992                 goto bad_fork_cleanup_count;
 993
 994         if (p->binfmt && !try_module_get(p->binfmt->module))
 995                 goto bad_fork_cleanup_put_domain;
 996
 997         p->did_exec = 0;
 998         copy_flags(clone_flags, p);
 999         p->pid = pid;
1000         retval = -EFAULT;
1001         if (clone_flags & CLONE_PARENT_SETTID)
1002                 if (put_user(p->pid, parent_tidptr))
1003                         goto bad_fork_cleanup;
1004
1005         p->proc_dentry = NULL;
1006
1007         INIT_LIST_HEAD(&p->children);
1008         INIT_LIST_HEAD(&p->sibling);
1009         init_waitqueue_head(&p->wait_chldexit);
1010         p->vfork_done = NULL;
1011         spin_lock_init(&p->alloc_lock);
1012         spin_lock_init(&p->proc_lock);
1013
1014         clear_tsk_thread_flag(p, TIF_SIGPENDING);
1015         init_sigpending(&p->pending);
1016
1017         p->it_real_value = p->it_virt_value = p->it_prof_value = 0;
1018         p->it_real_incr = p->it_virt_incr = p->it_prof_incr = 0;
1019         init_timer(&p->real_timer);
1020         p->real_timer.data = (unsigned long) p;
1021
1022         p->utime = p->stime = 0;
1023         p->lock_depth = -1;             /* -1 = no lock */
1024         do_posix_clock_monotonic_gettime(&p->start_time);
1025         p->security = NULL;
1026         p->io_context = NULL;
1027         p->io_wait = NULL;
1028         p->audit_context = NULL;
1029 #ifdef CONFIG_NUMA
1030         p->mempolicy = mpol_copy(p->mempolicy);
1031         if (IS_ERR(p->mempolicy)) {
1032                 retval = PTR_ERR(p->mempolicy);
1033                 p->mempolicy = NULL;
1034                 goto bad_fork_cleanup;
1035         }
1036 #endif
1037
1038         if ((retval = security_task_alloc(p)))
1039                 goto bad_fork_cleanup_policy;
1040         if ((retval = audit_alloc(p)))
1041                 goto bad_fork_cleanup_security;
1042         /* copy all the process information */
1043         if ((retval = copy_semundo(clone_flags, p)))
1044                 goto bad_fork_cleanup_audit;
1045         if ((retval = copy_files(clone_flags, p)))
1046                 goto bad_fork_cleanup_semundo;
1047         if ((retval = copy_fs(clone_flags, p)))
1048                 goto bad_fork_cleanup_files;
1049         if ((retval = copy_sighand(clone_flags, p)))
1050                 goto bad_fork_cleanup_fs;
1051         if ((retval = copy_signal(clone_flags, p)))
1052                 goto bad_fork_cleanup_sighand;
1053         if ((retval = copy_mm(clone_flags, p)))
1054                 goto bad_fork_cleanup_signal;
1055         if ((retval = copy_namespace(clone_flags, p)))
1056                 goto bad_fork_cleanup_mm;
1057         retval = copy_thread(0, clone_flags, stack_start, stack_size, p, regs);
1058         if (retval)
1059                 goto bad_fork_cleanup_namespace;
1060
1061         p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
1062         /*
1063          * Clear TID on mm_release()?
1064          */
1065         p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr: NULL;
1066
1067         /*
1068          * Syscall tracing should be turned off in the child regardless
1069          * of CLONE_PTRACE.
1070          */
1071         clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE);
1072
1073         /* Our parent execution domain becomes current domain
1074            These must match for thread signalling to apply */
1075
1076         p->parent_exec_id = p->self_exec_id;
1077
1078         /* ok, now we should be set up.. */
1079         p->exit_signal = (clone_flags & CLONE_THREAD) ? -1 : (clone_flags & CSIGNAL);
1080         p->pdeath_signal = 0;
1081
1082         /* Perform scheduler related setup */
1083         sched_fork(p);
1084
1085         /*
1086          * Ok, make it visible to the rest of the system.
1087          * We dont wake it up yet.
1088          */
1089         p->tgid = p->pid;
1090         p->group_leader = p;
1091         INIT_LIST_HEAD(&p->ptrace_children);
1092         INIT_LIST_HEAD(&p->ptrace_list);
1093
1094         /* Need tasklist lock for parent etc handling! */
1095         write_lock_irq(&tasklist_lock);
1096
1097         /*
1098          * The task hasn't been attached yet, so cpus_allowed mask cannot
1099          * have changed. The cpus_allowed mask of the parent may have
1100          * changed after it was copied first time, and it may then move to
1101          * another CPU - so we re-copy it here and set the child's CPU to
1102          * the parent's CPU. This avoids alot of nasty races.
1103          */
1104         p->cpus_allowed = current->cpus_allowed;
1105         set_task_cpu(p, smp_processor_id());
1106
1107         /*
1108          * Check for pending SIGKILL! The new thread should not be allowed
1109          * to slip out of an OOM kill. (or normal SIGKILL.)
1110          */
1111         if (sigismember(&current->pending.signal, SIGKILL)) {
1112                 write_unlock_irq(&tasklist_lock);
1113                 retval = -EINTR;
1114                 goto bad_fork_cleanup_namespace;
1115         }
1116
1117         /* CLONE_PARENT re-uses the old parent */
1118         if (clone_flags & (CLONE_PARENT|CLONE_THREAD))
1119                 p->real_parent = current->real_parent;
1120         else
1121                 p->real_parent = current;
1122         p->parent = p->real_parent;
1123
1124         if (clone_flags & CLONE_THREAD) {
1125                 spin_lock(&current->sighand->siglock);
1126                 /*
1127                  * Important: if an exit-all has been started then
1128                  * do not create this new thread - the whole thread
1129                  * group is supposed to exit anyway.
1130                  */
1131                 if (current->signal->group_exit) {
1132                         spin_unlock(&current->sighand->siglock);
1133                         write_unlock_irq(&tasklist_lock);
1134                         retval = -EAGAIN;
1135                         goto bad_fork_cleanup_namespace;
1136                 }
1137                 p->tgid = current->tgid;
1138                 p->group_leader = current->group_leader;
1139
1140                 if (current->signal->group_stop_count > 0) {
1141                         /*
1142                          * There is an all-stop in progress for the group.
1143                          * We ourselves will stop as soon as we check signals.
1144                          * Make the new thread part of that group stop too.
1145                          */
1146                         current->signal->group_stop_count++;
1147                         set_tsk_thread_flag(p, TIF_SIGPENDING);
1148                 }
1149
1150                 spin_unlock(&current->sighand->siglock);
1151         }
1152
1153         SET_LINKS(p);
1154         if (unlikely(p->ptrace & PT_PTRACED))
1155                 __ptrace_link(p, current->parent);
1156
1157         attach_pid(p, PIDTYPE_PID, p->pid);
1158         attach_pid(p, PIDTYPE_TGID, p->tgid);
1159         if (thread_group_leader(p)) {
1160                 attach_pid(p, PIDTYPE_PGID, process_group(p));
1161                 attach_pid(p, PIDTYPE_SID, p->signal->session);
1162                 if (p->pid)
1163                         __get_cpu_var(process_counts)++;
1164         }
1165
1166         nr_threads++;
1167         /* p is copy of current */
1168         vxi = p->vx_info;
1169         if (vxi) {
1170                 atomic_inc(&vxi->cvirt.nr_threads);
1171                 vx_nproc_inc(p);
1172         }
1173         write_unlock_irq(&tasklist_lock);
1174         retval = 0;
1175
1176 fork_out:
1177         if (retval)
1178                 return ERR_PTR(retval);
1179         return p;
1180
1181 bad_fork_cleanup_namespace:
1182         exit_namespace(p);
1183 bad_fork_cleanup_mm:
1184         if (p->mm)
1185                 mmput(p->mm);
1186 bad_fork_cleanup_signal:
1187         exit_signal(p);
1188 bad_fork_cleanup_sighand:
1189         exit_sighand(p);
1190 bad_fork_cleanup_fs:
1191         exit_fs(p); /* blocking */
1192 bad_fork_cleanup_files:
1193         exit_files(p); /* blocking */
1194 bad_fork_cleanup_semundo:
1195         exit_sem(p);
1196 bad_fork_cleanup_audit:
1197         audit_free(p);
1198 bad_fork_cleanup_security:
1199         security_task_free(p);
1200 bad_fork_cleanup_policy:
1201 #ifdef CONFIG_NUMA
1202         mpol_free(p->mempolicy);
1203 #endif
1204 bad_fork_cleanup:
1205         if (p->binfmt)
1206                 module_put(p->binfmt->module);
1207 bad_fork_cleanup_put_domain:
1208         module_put(p->thread_info->exec_domain->module);
1209 bad_fork_cleanup_count:
1210         put_group_info(p->group_info);
1211         atomic_dec(&p->user->processes);
1212         free_uid(p->user);
1213 bad_fork_cleanup_vm:
1214         if (p->mm && !(clone_flags & CLONE_VM))
1215                 vx_pages_sub(p->mm->mm_vx_info, RLIMIT_AS, p->mm->total_vm);
1216 bad_fork_free:
1217         free_task(p);
1218         goto fork_out;
1219 }
1220
1221 struct pt_regs * __devinit __attribute__((weak)) idle_regs(struct pt_regs *regs)
1222 {
1223         memset(regs, 0, sizeof(struct pt_regs));
1224         return regs;
1225 }
1226
1227 task_t * __devinit fork_idle(int cpu)
1228 {
1229         task_t *task;
1230         struct pt_regs regs;
1231
1232         task = copy_process(CLONE_VM, 0, idle_regs(&regs), 0, NULL, NULL, 0);
1233         if (!task)
1234                 return ERR_PTR(-ENOMEM);
1235         init_idle(task, cpu);
1236         unhash_process(task);
1237         return task;
1238 }
1239
1240 static inline int fork_traceflag (unsigned clone_flags)
1241 {
1242         if (clone_flags & CLONE_UNTRACED)
1243                 return 0;
1244         else if (clone_flags & CLONE_VFORK) {
1245                 if (current->ptrace & PT_TRACE_VFORK)
1246                         return PTRACE_EVENT_VFORK;
1247         } else if ((clone_flags & CSIGNAL) != SIGCHLD) {
1248                 if (current->ptrace & PT_TRACE_CLONE)
1249                         return PTRACE_EVENT_CLONE;
1250         } else if (current->ptrace & PT_TRACE_FORK)
1251                 return PTRACE_EVENT_FORK;
1252
1253         return 0;
1254 }
1255
1256 /*
1257  *  Ok, this is the main fork-routine.
1258  *
1259  * It copies the process, and if successful kick-starts
1260  * it and waits for it to finish using the VM if required.
1261  */
1262 long do_fork(unsigned long clone_flags,
1263               unsigned long stack_start,
1264               struct pt_regs *regs,
1265               unsigned long stack_size,
1266               int __user *parent_tidptr,
1267               int __user *child_tidptr)
1268 {
1269         struct task_struct *p;
1270         int trace = 0;
1271         long pid = alloc_pidmap();
1272
1273         if (pid < 0)
1274                 return -EAGAIN;
1275         if (unlikely(current->ptrace)) {
1276                 trace = fork_traceflag (clone_flags);
1277                 if (trace)
1278                         clone_flags |= CLONE_PTRACE;
1279         }
1280
1281         p = copy_process(clone_flags, stack_start, regs, stack_size, parent_tidptr, child_tidptr, pid);
1282         /*
1283          * Do this prior waking up the new thread - the thread pointer
1284          * might get invalid after that point, if the thread exits quickly.
1285          */
1286         if (!IS_ERR(p)) {
1287                 struct completion vfork;
1288
1289                 if (clone_flags & CLONE_VFORK) {
1290                         p->vfork_done = &vfork;
1291                         init_completion(&vfork);
1292                 }
1293
1294                 if ((p->ptrace & PT_PTRACED) || (clone_flags & CLONE_STOPPED)) {
1295                         /*
1296                          * We'll start up with an immediate SIGSTOP.
1297                          */
1298                         sigaddset(&p->pending.signal, SIGSTOP);
1299                         set_tsk_thread_flag(p, TIF_SIGPENDING);
1300                 }
1301
1302                 if (!(clone_flags & CLONE_STOPPED))
1303                         wake_up_new_task(p, clone_flags);
1304                 else
1305                         p->state = TASK_STOPPED;
1306                 ++total_forks;
1307
1308                 if (unlikely (trace)) {
1309                         current->ptrace_message = pid;
1310                         ptrace_notify ((trace << 8) | SIGTRAP);
1311                 }
1312
1313                 if (clone_flags & CLONE_VFORK) {
1314                         wait_for_completion(&vfork);
1315                         if (unlikely (current->ptrace & PT_TRACE_VFORK_DONE))
1316                                 ptrace_notify ((PTRACE_EVENT_VFORK_DONE << 8) | SIGTRAP);
1317                 }
1318         } else {
1319                 free_pidmap(pid);
1320                 pid = PTR_ERR(p);
1321         }
1322         return pid;
1323 }
1324
1325 /* SLAB cache for signal_struct structures (tsk->signal) */
1326 kmem_cache_t *signal_cachep;
1327
1328 /* SLAB cache for sighand_struct structures (tsk->sighand) */
1329 kmem_cache_t *sighand_cachep;
1330
1331 /* SLAB cache for files_struct structures (tsk->files) */
1332 kmem_cache_t *files_cachep;
1333
1334 /* SLAB cache for fs_struct structures (tsk->fs) */
1335 kmem_cache_t *fs_cachep;
1336
1337 /* SLAB cache for vm_area_struct structures */
1338 kmem_cache_t *vm_area_cachep;
1339
1340 /* SLAB cache for mm_struct structures (tsk->mm) */
1341 kmem_cache_t *mm_cachep;
1342
1343 void __init proc_caches_init(void)
1344 {
1345         sighand_cachep = kmem_cache_create("sighand_cache",
1346                         sizeof(struct sighand_struct), 0,
1347                         SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
1348         signal_cachep = kmem_cache_create("signal_cache",
1349                         sizeof(struct signal_struct), 0,
1350                         SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
1351         files_cachep = kmem_cache_create("files_cache",
1352                         sizeof(struct files_struct), 0,
1353                         SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
1354         fs_cachep = kmem_cache_create("fs_cache",
1355                         sizeof(struct fs_struct), 0,
1356                         SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
1357         vm_area_cachep = kmem_cache_create("vm_area_struct",
1358                         sizeof(struct vm_area_struct), 0,
1359                         SLAB_PANIC, NULL, NULL);
1360         mm_cachep = kmem_cache_create("mm_struct",
1361                         sizeof(struct mm_struct), 0,
1362                         SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
1363 }