1 #ifndef _X86_64_PGALLOC_H
2 #define _X86_64_PGALLOC_H
4 #include <asm/fixmap.h>
6 #include <linux/threads.h>
8 #include <asm/io.h> /* for phys_to_virt and page_to_pseudophys */
10 #include <xen/features.h>
12 #define arch_add_exec_range(mm, limit) \
13 do { (void)(mm), (void)(limit); } while (0)
14 #define arch_flush_exec_range(mm) \
15 do { (void)(mm); } while (0)
16 #define arch_remove_exec_range(mm, limit) \
17 do { (void)(mm), (void)(limit); } while (0)
19 void make_page_readonly(void *va, unsigned int feature);
20 void make_page_writable(void *va, unsigned int feature);
21 void make_pages_readonly(void *va, unsigned int nr, unsigned int feature);
22 void make_pages_writable(void *va, unsigned int nr, unsigned int feature);
24 #define __user_pgd(pgd) ((pgd) + PTRS_PER_PGD)
26 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte)
28 set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte)));
31 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
33 if (unlikely((mm)->context.pinned)) {
34 BUG_ON(HYPERVISOR_update_va_mapping(
36 pfn_pte(virt_to_phys(pmd)>>PAGE_SHIFT,
38 set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
40 *(pud) = __pud(_PAGE_TABLE | __pa(pmd));
45 * We need to use the batch mode here, but pgd_pupulate() won't be
46 * be called frequently.
48 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
50 if (unlikely((mm)->context.pinned)) {
51 BUG_ON(HYPERVISOR_update_va_mapping(
53 pfn_pte(virt_to_phys(pud)>>PAGE_SHIFT,
55 set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud)));
56 set_pgd(__user_pgd(pgd), __pgd(_PAGE_TABLE | __pa(pud)));
58 *(pgd) = __pgd(_PAGE_TABLE | __pa(pud));
59 *(__user_pgd(pgd)) = *(pgd);
63 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte)
65 if (unlikely((mm)->context.pinned)) {
66 BUG_ON(HYPERVISOR_update_va_mapping(
67 (unsigned long)__va(page_to_pfn(pte) << PAGE_SHIFT),
68 pfn_pte(page_to_pfn(pte), PAGE_KERNEL_RO), 0));
69 set_pmd(pmd, __pmd(_PAGE_TABLE | (page_to_pfn(pte) << PAGE_SHIFT)));
71 *(pmd) = __pmd(_PAGE_TABLE | (page_to_pfn(pte) << PAGE_SHIFT));
75 static inline void pmd_free(pmd_t *pmd)
77 pte_t *ptep = virt_to_ptep(pmd);
79 if (!pte_write(*ptep)) {
80 BUG_ON(HYPERVISOR_update_va_mapping(
82 pfn_pte(virt_to_phys(pmd)>>PAGE_SHIFT, PAGE_KERNEL),
85 free_page((unsigned long)pmd);
88 static inline pmd_t *pmd_alloc_one (struct mm_struct *mm, unsigned long addr)
90 return (pmd_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
93 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
95 return (pud_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
98 static inline void pud_free (pud_t *pud)
100 pte_t *ptep = virt_to_ptep(pud);
102 if (!pte_write(*ptep)) {
103 BUG_ON(HYPERVISOR_update_va_mapping(
105 pfn_pte(virt_to_phys(pud)>>PAGE_SHIFT, PAGE_KERNEL),
108 free_page((unsigned long)pud);
111 static inline void pgd_list_add(pgd_t *pgd)
113 struct page *page = virt_to_page(pgd);
115 spin_lock(&pgd_lock);
116 page->index = (pgoff_t)pgd_list;
118 pgd_list->private = (unsigned long)&page->index;
120 page->private = (unsigned long)&pgd_list;
121 spin_unlock(&pgd_lock);
124 static inline void pgd_list_del(pgd_t *pgd)
126 struct page *next, **pprev, *page = virt_to_page(pgd);
128 spin_lock(&pgd_lock);
129 next = (struct page *)page->index;
130 pprev = (struct page **)page->private;
133 next->private = (unsigned long)pprev;
134 spin_unlock(&pgd_lock);
137 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
140 * We allocate two contiguous pages for kernel and user.
143 pgd_t *pgd = (pgd_t *)__get_free_pages(GFP_KERNEL|__GFP_REPEAT, 1);
149 * Copy kernel pointers in from init.
150 * Could keep a freelist or slab cache of those because the kernel
151 * part never changes.
153 boundary = pgd_index(__PAGE_OFFSET);
154 memset(pgd, 0, boundary * sizeof(pgd_t));
155 memcpy(pgd + boundary,
156 init_level4_pgt + boundary,
157 (PTRS_PER_PGD - boundary) * sizeof(pgd_t));
159 memset(__user_pgd(pgd), 0, PAGE_SIZE); /* clean up user pgd */
161 * Set level3_user_pgt for vsyscall area
163 set_pgd(__user_pgd(pgd) + pgd_index(VSYSCALL_START),
164 mk_kernel_pgd(__pa_symbol(level3_user_pgt)));
168 static inline void pgd_free(pgd_t *pgd)
170 pte_t *ptep = virt_to_ptep(pgd);
172 if (!pte_write(*ptep)) {
173 xen_pgd_unpin(__pa(pgd));
174 BUG_ON(HYPERVISOR_update_va_mapping(
176 pfn_pte(virt_to_phys(pgd)>>PAGE_SHIFT, PAGE_KERNEL),
180 ptep = virt_to_ptep(__user_pgd(pgd));
182 if (!pte_write(*ptep)) {
183 xen_pgd_unpin(__pa(__user_pgd(pgd)));
184 BUG_ON(HYPERVISOR_update_va_mapping(
185 (unsigned long)__user_pgd(pgd),
186 pfn_pte(virt_to_phys(__user_pgd(pgd))>>PAGE_SHIFT,
192 free_pages((unsigned long)pgd, 1);
195 static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
197 pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
199 make_page_readonly(pte, XENFEAT_writable_page_tables);
204 static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
208 pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
212 /* Should really implement gc for free page table pages. This could be
213 done with a reference count in struct page. */
215 static inline void pte_free_kernel(pte_t *pte)
217 BUG_ON((unsigned long)pte & (PAGE_SIZE-1));
218 make_page_writable(pte, XENFEAT_writable_page_tables);
219 free_page((unsigned long)pte);
222 extern void pte_free(struct page *pte);
224 //#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
225 //#define __pmd_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x))
226 //#define __pud_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x))
228 #define __pte_free_tlb(tlb,x) pte_free((x))
229 #define __pmd_free_tlb(tlb,x) pmd_free((x))
230 #define __pud_free_tlb(tlb,x) pud_free((x))
232 #endif /* _X86_64_PGALLOC_H */