include/asm-x86_64/mach-xen/asm/pgalloc.h

   1 #ifndef _X86_64_PGALLOC_H
   2 #define _X86_64_PGALLOC_H
   3
   4 #include <asm/fixmap.h>
   5 #include <asm/pda.h>
   6 #include <linux/threads.h>
   7 #include <linux/mm.h>
   8 #include <asm/io.h>             /* for phys_to_virt and page_to_pseudophys */
   9
  10 #define arch_add_exec_range(mm, limit) \
  11                 do { (void)(mm), (void)(limit); } while (0)
  12 #define arch_flush_exec_range(mm) \
  13                 do { (void)(mm); } while (0)
  14 #define arch_remove_exec_range(mm, limit) \
  15                 do { (void)(mm), (void)(limit); } while (0)
  16
  17 #include <xen/features.h>
  18 void make_page_readonly(void *va, unsigned int feature);
  19 void make_page_writable(void *va, unsigned int feature);
  20 void make_pages_readonly(void *va, unsigned int nr, unsigned int feature);
  21 void make_pages_writable(void *va, unsigned int nr, unsigned int feature);
  22
  23 #define __user_pgd(pgd) ((pgd) + PTRS_PER_PGD)
  24
  25 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte)
  26 {
  27         set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte)));
  28 }
  29
  30 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte)
  31 {
  32         if (unlikely((mm)->context.pinned)) {
  33                 BUG_ON(HYPERVISOR_update_va_mapping(
  34                                (unsigned long)__va(page_to_pfn(pte) << PAGE_SHIFT),
  35                                pfn_pte(page_to_pfn(pte), PAGE_KERNEL_RO), 0));
  36                 set_pmd(pmd, __pmd(_PAGE_TABLE | (page_to_pfn(pte) << PAGE_SHIFT)));
  37         } else {
  38                 *(pmd) = __pmd(_PAGE_TABLE | (page_to_pfn(pte) << PAGE_SHIFT));
  39         }
  40 }
  41
  42 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
  43 {
  44         if (unlikely((mm)->context.pinned)) {
  45                 BUG_ON(HYPERVISOR_update_va_mapping(
  46                                (unsigned long)pmd,
  47                                pfn_pte(virt_to_phys(pmd)>>PAGE_SHIFT,
  48                                        PAGE_KERNEL_RO), 0));
  49                 set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
  50         } else {
  51                 *(pud) =  __pud(_PAGE_TABLE | __pa(pmd));
  52         }
  53 }
  54
  55 /*
  56  * We need to use the batch mode here, but pgd_pupulate() won't be
  57  * be called frequently.
  58  */
  59 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
  60 {
  61         if (unlikely((mm)->context.pinned)) {
  62                 BUG_ON(HYPERVISOR_update_va_mapping(
  63                                (unsigned long)pud,
  64                                pfn_pte(virt_to_phys(pud)>>PAGE_SHIFT,
  65                                        PAGE_KERNEL_RO), 0));
  66                 set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud)));
  67                 set_pgd(__user_pgd(pgd), __pgd(_PAGE_TABLE | __pa(pud)));
  68         } else {
  69                 *(pgd) =  __pgd(_PAGE_TABLE | __pa(pud));
  70                 *(__user_pgd(pgd)) = *(pgd);
  71         }
  72 }
  73
  74 static inline void pmd_free(pmd_t *pmd)
  75 {
  76         pte_t *ptep = virt_to_ptep(pmd);
  77
  78         if (!pte_write(*ptep)) {
  79                 BUG_ON(HYPERVISOR_update_va_mapping(
  80                         (unsigned long)pmd,
  81                         pfn_pte(virt_to_phys(pmd)>>PAGE_SHIFT, PAGE_KERNEL),
  82                         0));
  83         }
  84         free_page((unsigned long)pmd);
  85 }
  86
  87 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
  88 {
  89         pmd_t *pmd = (pmd_t *) get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
  90         return pmd;
  91 }
  92
  93 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
  94 {
  95         pud_t *pud = (pud_t *) get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
  96         return pud;
  97 }
  98
  99 static inline void pud_free(pud_t *pud)
 100 {
 101         pte_t *ptep = virt_to_ptep(pud);
 102
 103         if (!pte_write(*ptep)) {
 104                 BUG_ON(HYPERVISOR_update_va_mapping(
 105                         (unsigned long)pud,
 106                         pfn_pte(virt_to_phys(pud)>>PAGE_SHIFT, PAGE_KERNEL),
 107                         0));
 108         }
 109         free_page((unsigned long)pud);
 110 }
 111
 112 static inline void pgd_list_add(pgd_t *pgd)
 113 {
 114         struct page *page = virt_to_page(pgd);
 115
 116         spin_lock(&pgd_lock);
 117         page->index = (pgoff_t)pgd_list;
 118         if (pgd_list)
 119                 pgd_list->private = (unsigned long)&page->index;
 120         pgd_list = page;
 121         page->private = (unsigned long)&pgd_list;
 122         spin_unlock(&pgd_lock);
 123 }
 124
 125 static inline void pgd_list_del(pgd_t *pgd)
 126 {
 127         struct page *next, **pprev, *page = virt_to_page(pgd);
 128
 129         spin_lock(&pgd_lock);
 130         next = (struct page *)page->index;
 131         pprev = (struct page **)page->private;
 132         *pprev = next;
 133         if (next)
 134                 next->private = (unsigned long)pprev;
 135         spin_unlock(&pgd_lock);
 136 }
 137
 138 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 139 {
 140         /*
 141          * We allocate two contiguous pages for kernel and user.
 142          */
 143         unsigned boundary;
 144         pgd_t *pgd = (pgd_t *)__get_free_pages(GFP_KERNEL|__GFP_REPEAT, 1);
 145
 146         if (!pgd)
 147                 return NULL;
 148         pgd_list_add(pgd);
 149         /*
 150          * Copy kernel pointers in from init.
 151          * Could keep a freelist or slab cache of those because the kernel
 152          * part never changes.
 153          */
 154         boundary = pgd_index(__PAGE_OFFSET);
 155         memset(pgd, 0, boundary * sizeof(pgd_t));
 156         memcpy(pgd + boundary,
 157                init_level4_pgt + boundary,
 158                (PTRS_PER_PGD - boundary) * sizeof(pgd_t));
 159
 160         memset(__user_pgd(pgd), 0, PAGE_SIZE); /* clean up user pgd */
 161         /*
 162          * Set level3_user_pgt for vsyscall area
 163          */
 164         set_pgd(__user_pgd(pgd) + pgd_index(VSYSCALL_START),
 165                 mk_kernel_pgd(__pa_symbol(level3_user_pgt)));
 166         return pgd;
 167 }
 168
 169 static inline void pgd_free(pgd_t *pgd)
 170 {
 171         pte_t *ptep = virt_to_ptep(pgd);
 172
 173         if (!pte_write(*ptep)) {
 174                 xen_pgd_unpin(__pa(pgd));
 175                 BUG_ON(HYPERVISOR_update_va_mapping(
 176                                (unsigned long)pgd,
 177                                pfn_pte(virt_to_phys(pgd)>>PAGE_SHIFT, PAGE_KERNEL),
 178                                0));
 179         }
 180
 181         ptep = virt_to_ptep(__user_pgd(pgd));
 182
 183         if (!pte_write(*ptep)) {
 184                 xen_pgd_unpin(__pa(__user_pgd(pgd)));
 185                 BUG_ON(HYPERVISOR_update_va_mapping(
 186                                (unsigned long)__user_pgd(pgd),
 187                                pfn_pte(virt_to_phys(__user_pgd(pgd))>>PAGE_SHIFT,
 188                                        PAGE_KERNEL),
 189                                0));
 190         }
 191
 192         pgd_list_del(pgd);
 193         free_pages((unsigned long)pgd, 1);
 194 }
 195
 196 static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
 197 {
 198         pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
 199         if (pte)
 200                 make_page_readonly(pte, XENFEAT_writable_page_tables);
 201
 202         return pte;
 203 }
 204
 205 static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
 206 {
 207         struct page *pte;
 208
 209         pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
 210         return pte;
 211 }
 212
 213 /* Should really implement gc for free page table pages. This could be
 214    done with a reference count in struct page. */
 215
 216 static inline void pte_free_kernel(pte_t *pte)
 217 {
 218         BUG_ON((unsigned long)pte & (PAGE_SIZE-1));
 219         make_page_writable(pte, XENFEAT_writable_page_tables);
 220         free_page((unsigned long)pte);
 221 }
 222
 223 extern void pte_free(struct page *pte);
 224
 225 //#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
 226 //#define __pmd_free_tlb(tlb,x)   tlb_remove_page((tlb),virt_to_page(x))
 227 //#define __pud_free_tlb(tlb,x)   tlb_remove_page((tlb),virt_to_page(x))
 228
 229 #define __pte_free_tlb(tlb,x)   pte_free((x))
 230 #define __pmd_free_tlb(tlb,x)   pmd_free((x))
 231 #define __pud_free_tlb(tlb,x)   pud_free((x))
 232
 233 #endif /* _X86_64_PGALLOC_H */