+struct kmem_cache *anon_vma_cachep;
+
+static inline void validate_anon_vma(struct vm_area_struct *find_vma)
+{
+#ifdef CONFIG_DEBUG_VM
+ struct anon_vma *anon_vma = find_vma->anon_vma;
+ struct vm_area_struct *vma;
+ unsigned int mapcount = 0;
+ int found = 0;
+
+ list_for_each_entry(vma, &anon_vma->head, anon_vma_node) {
+ mapcount++;
+ BUG_ON(mapcount > 100000);
+ if (vma == find_vma)
+ found = 1;
+ }
+ BUG_ON(!found);
+#endif
+}
+
+/* This must be called under the mmap_sem. */
+int anon_vma_prepare(struct vm_area_struct *vma)
+{
+ struct anon_vma *anon_vma = vma->anon_vma;
+
+ might_sleep();
+ if (unlikely(!anon_vma)) {
+ struct mm_struct *mm = vma->vm_mm;
+ struct anon_vma *allocated, *locked;
+
+ anon_vma = find_mergeable_anon_vma(vma);
+ if (anon_vma) {
+ allocated = NULL;
+ locked = anon_vma;
+ spin_lock(&locked->lock);
+ } else {
+ anon_vma = anon_vma_alloc();
+ if (unlikely(!anon_vma))
+ return -ENOMEM;
+ allocated = anon_vma;
+ locked = NULL;
+ }
+
+ /* page_table_lock to protect against threads */
+ spin_lock(&mm->page_table_lock);
+ if (likely(!vma->anon_vma)) {
+ vma->anon_vma = anon_vma;
+ list_add(&vma->anon_vma_node, &anon_vma->head);
+ allocated = NULL;
+ }
+ spin_unlock(&mm->page_table_lock);
+
+ if (locked)
+ spin_unlock(&locked->lock);
+ if (unlikely(allocated))
+ anon_vma_free(allocated);
+ }
+ return 0;
+}
+
+void __anon_vma_merge(struct vm_area_struct *vma, struct vm_area_struct *next)
+{
+ BUG_ON(vma->anon_vma != next->anon_vma);
+ list_del(&next->anon_vma_node);
+}
+
+void __anon_vma_link(struct vm_area_struct *vma)
+{
+ struct anon_vma *anon_vma = vma->anon_vma;
+
+ if (anon_vma) {
+ list_add(&vma->anon_vma_node, &anon_vma->head);
+ validate_anon_vma(vma);
+ }
+}
+
+void anon_vma_link(struct vm_area_struct *vma)
+{
+ struct anon_vma *anon_vma = vma->anon_vma;
+
+ if (anon_vma) {
+ spin_lock(&anon_vma->lock);
+ list_add(&vma->anon_vma_node, &anon_vma->head);
+ validate_anon_vma(vma);
+ spin_unlock(&anon_vma->lock);
+ }
+}
+
+void anon_vma_unlink(struct vm_area_struct *vma)
+{
+ struct anon_vma *anon_vma = vma->anon_vma;
+ int empty;
+
+ if (!anon_vma)
+ return;
+
+ spin_lock(&anon_vma->lock);
+ validate_anon_vma(vma);
+ list_del(&vma->anon_vma_node);
+
+ /* We must garbage collect the anon_vma if it's empty */
+ empty = list_empty(&anon_vma->head);
+ spin_unlock(&anon_vma->lock);
+
+ if (empty)
+ anon_vma_free(anon_vma);
+}
+
+static void anon_vma_ctor(void *data, struct kmem_cache *cachep,
+ unsigned long flags)
+{
+ if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
+ SLAB_CTOR_CONSTRUCTOR) {
+ struct anon_vma *anon_vma = data;
+
+ spin_lock_init(&anon_vma->lock);
+ INIT_LIST_HEAD(&anon_vma->head);
+ }
+}
+
+void __init anon_vma_init(void)
+{
+ anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
+ 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC, anon_vma_ctor, NULL);
+}
+