include/asm-x86_64/mach-xen/asm/pgalloc.h

   1 #ifndef _X86_64_PGALLOC_H
   2 #define _X86_64_PGALLOC_H
   3
   4 #include <asm/fixmap.h>
   5 #include <asm/pda.h>
   6 #include <linux/threads.h>
   7 #include <linux/mm.h>
   8 #include <asm/io.h>             /* for phys_to_virt and page_to_pseudophys */
   9
  10 #include <xen/features.h>
  11
  12 #define arch_add_exec_range(mm, limit) \
  13                 do { (void)(mm), (void)(limit); } while (0)
  14 #define arch_flush_exec_range(mm) \
  15                 do { (void)(mm); } while (0)
  16 #define arch_remove_exec_range(mm, limit) \
  17                 do { (void)(mm), (void)(limit); } while (0)
  18
  19 void make_page_readonly(void *va, unsigned int feature);
  20 void make_page_writable(void *va, unsigned int feature);
  21 void make_pages_readonly(void *va, unsigned int nr, unsigned int feature);
  22 void make_pages_writable(void *va, unsigned int nr, unsigned int feature);
  23
  24 #define __user_pgd(pgd) ((pgd) + PTRS_PER_PGD)
  25
  26 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte)
  27 {
  28         set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte)));
  29 }
  30
  31 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte)
  32 {
  33         if (unlikely((mm)->context.pinned)) {
  34                 BUG_ON(HYPERVISOR_update_va_mapping(
  35                                (unsigned long)__va(page_to_pfn(pte) << PAGE_SHIFT),
  36                                pfn_pte(page_to_pfn(pte), PAGE_KERNEL_RO), 0));
  37                 set_pmd(pmd, __pmd(_PAGE_TABLE | (page_to_pfn(pte) << PAGE_SHIFT)));
  38         } else {
  39                 *(pmd) = __pmd(_PAGE_TABLE | (page_to_pfn(pte) << PAGE_SHIFT));
  40         }
  41 }
  42
  43 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
  44 {
  45         if (unlikely((mm)->context.pinned)) {
  46                 BUG_ON(HYPERVISOR_update_va_mapping(
  47                                (unsigned long)pmd,
  48                                pfn_pte(virt_to_phys(pmd)>>PAGE_SHIFT,
  49                                        PAGE_KERNEL_RO), 0));
  50                 set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
  51         } else {
  52                 *(pud) =  __pud(_PAGE_TABLE | __pa(pmd));
  53         }
  54 }
  55
  56 /*
  57  * We need to use the batch mode here, but pgd_pupulate() won't be
  58  * be called frequently.
  59  */
  60 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
  61 {
  62         if (unlikely((mm)->context.pinned)) {
  63                 BUG_ON(HYPERVISOR_update_va_mapping(
  64                                (unsigned long)pud,
  65                                pfn_pte(virt_to_phys(pud)>>PAGE_SHIFT,
  66                                        PAGE_KERNEL_RO), 0));
  67                 set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud)));
  68                 set_pgd(__user_pgd(pgd), __pgd(_PAGE_TABLE | __pa(pud)));
  69         } else {
  70                 *(pgd) =  __pgd(_PAGE_TABLE | __pa(pud));
  71                 *(__user_pgd(pgd)) = *(pgd);
  72         }
  73 }
  74
  75 static inline void pmd_free(pmd_t *pmd)
  76 {
  77         pte_t *ptep = virt_to_ptep(pmd);
  78
  79         if (!pte_write(*ptep)) {
  80                 BUG_ON(HYPERVISOR_update_va_mapping(
  81                         (unsigned long)pmd,
  82                         pfn_pte(virt_to_phys(pmd)>>PAGE_SHIFT, PAGE_KERNEL),
  83                         0));
  84         }
  85         free_page((unsigned long)pmd);
  86 }
  87
  88 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
  89 {
  90         pmd_t *pmd = (pmd_t *) get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
  91         return pmd;
  92 }
  93
  94 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
  95 {
  96         pud_t *pud = (pud_t *) get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
  97         return pud;
  98 }
  99
 100 static inline void pud_free(pud_t *pud)
 101 {
 102         pte_t *ptep = virt_to_ptep(pud);
 103
 104         if (!pte_write(*ptep)) {
 105                 BUG_ON(HYPERVISOR_update_va_mapping(
 106                         (unsigned long)pud,
 107                         pfn_pte(virt_to_phys(pud)>>PAGE_SHIFT, PAGE_KERNEL),
 108                         0));
 109         }
 110         free_page((unsigned long)pud);
 111 }
 112
 113 static inline void pgd_list_add(pgd_t *pgd)
 114 {
 115         struct page *page = virt_to_page(pgd);
 116
 117         spin_lock(&pgd_lock);
 118         page->index = (pgoff_t)pgd_list;
 119         if (pgd_list)
 120                 pgd_list->private = (unsigned long)&page->index;
 121         pgd_list = page;
 122         page->private = (unsigned long)&pgd_list;
 123         spin_unlock(&pgd_lock);
 124 }
 125
 126 static inline void pgd_list_del(pgd_t *pgd)
 127 {
 128         struct page *next, **pprev, *page = virt_to_page(pgd);
 129
 130         spin_lock(&pgd_lock);
 131         next = (struct page *)page->index;
 132         pprev = (struct page **)page->private;
 133         *pprev = next;
 134         if (next)
 135                 next->private = (unsigned long)pprev;
 136         spin_unlock(&pgd_lock);
 137 }
 138
 139 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 140 {
 141         /*
 142          * We allocate two contiguous pages for kernel and user.
 143          */
 144         unsigned boundary;
 145         pgd_t *pgd = (pgd_t *)__get_free_pages(GFP_KERNEL|__GFP_REPEAT, 1);
 146
 147         if (!pgd)
 148                 return NULL;
 149         pgd_list_add(pgd);
 150         /*
 151          * Copy kernel pointers in from init.
 152          * Could keep a freelist or slab cache of those because the kernel
 153          * part never changes.
 154          */
 155         boundary = pgd_index(__PAGE_OFFSET);
 156         memset(pgd, 0, boundary * sizeof(pgd_t));
 157         memcpy(pgd + boundary,
 158                init_level4_pgt + boundary,
 159                (PTRS_PER_PGD - boundary) * sizeof(pgd_t));
 160
 161         memset(__user_pgd(pgd), 0, PAGE_SIZE); /* clean up user pgd */
 162         /*
 163          * Set level3_user_pgt for vsyscall area
 164          */
 165         set_pgd(__user_pgd(pgd) + pgd_index(VSYSCALL_START),
 166                 mk_kernel_pgd(__pa_symbol(level3_user_pgt)));
 167         return pgd;
 168 }
 169
 170 static inline void pgd_free(pgd_t *pgd)
 171 {
 172         pte_t *ptep = virt_to_ptep(pgd);
 173
 174         if (!pte_write(*ptep)) {
 175                 xen_pgd_unpin(__pa(pgd));
 176                 BUG_ON(HYPERVISOR_update_va_mapping(
 177                                (unsigned long)pgd,
 178                                pfn_pte(virt_to_phys(pgd)>>PAGE_SHIFT, PAGE_KERNEL),
 179                                0));
 180         }
 181
 182         ptep = virt_to_ptep(__user_pgd(pgd));
 183
 184         if (!pte_write(*ptep)) {
 185                 xen_pgd_unpin(__pa(__user_pgd(pgd)));
 186                 BUG_ON(HYPERVISOR_update_va_mapping(
 187                                (unsigned long)__user_pgd(pgd),
 188                                pfn_pte(virt_to_phys(__user_pgd(pgd))>>PAGE_SHIFT,
 189                                        PAGE_KERNEL),
 190                                0));
 191         }
 192
 193         pgd_list_del(pgd);
 194         free_pages((unsigned long)pgd, 1);
 195 }
 196
 197 static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
 198 {
 199         pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
 200         if (pte)
 201                 make_page_readonly(pte, XENFEAT_writable_page_tables);
 202
 203         return pte;
 204 }
 205
 206 static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
 207 {
 208         struct page *pte;
 209
 210         pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
 211         return pte;
 212 }
 213
 214 /* Should really implement gc for free page table pages. This could be
 215    done with a reference count in struct page. */
 216
 217 static inline void pte_free_kernel(pte_t *pte)
 218 {
 219         BUG_ON((unsigned long)pte & (PAGE_SIZE-1));
 220         make_page_writable(pte, XENFEAT_writable_page_tables);
 221         free_page((unsigned long)pte);
 222 }
 223
 224 extern void pte_free(struct page *pte);
 225
 226 //#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
 227 //#define __pmd_free_tlb(tlb,x)   tlb_remove_page((tlb),virt_to_page(x))
 228 //#define __pud_free_tlb(tlb,x)   tlb_remove_page((tlb),virt_to_page(x))
 229
 230 #define __pte_free_tlb(tlb,x)   pte_free((x))
 231 #define __pmd_free_tlb(tlb,x)   pmd_free((x))
 232 #define __pud_free_tlb(tlb,x)   pud_free((x))
 233
 234 #endif /* _X86_64_PGALLOC_H */