1 #ifndef _I386_PGALLOC_H
2 #define _I386_PGALLOC_H
4 #include <linux/config.h>
5 #include <asm/processor.h>
6 #include <asm/fixmap.h>
7 #include <linux/threads.h>
9 #define pgd_quicklist (current_cpu_data.pgd_quick)
10 #define pmd_quicklist (current_cpu_data.pmd_quick)
11 #define pte_quicklist (current_cpu_data.pte_quick)
12 #define pgtable_cache_size (current_cpu_data.pgtable_cache_sz)
14 #define pmd_populate(mm, pmd, pte) \
15 set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte)))
18 * Allocate and free page tables.
21 #if defined (CONFIG_X86_PAE)
23 * We can't include <linux/slab.h> here, thus these uglinesses.
27 extern struct kmem_cache_s *pae_pgd_cachep;
28 extern void *kmem_cache_alloc(struct kmem_cache_s *, int);
29 extern void kmem_cache_free(struct kmem_cache_s *, void *);
32 static inline pgd_t *get_pgd_slow(void)
35 pgd_t *pgd = kmem_cache_alloc(pae_pgd_cachep, GFP_KERNEL);
38 for (i = 0; i < USER_PTRS_PER_PGD; i++) {
39 unsigned long pmd = __get_free_page(GFP_KERNEL);
43 set_pgd(pgd + i, __pgd(1 + __pa(pmd)));
45 memcpy(pgd + USER_PTRS_PER_PGD,
46 swapper_pg_dir + USER_PTRS_PER_PGD,
47 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
51 for (i--; i >= 0; i--)
52 free_page((unsigned long)__va(pgd_val(pgd[i])-1));
53 kmem_cache_free(pae_pgd_cachep, pgd);
59 static inline pgd_t *get_pgd_slow(void)
61 pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL);
64 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
65 memcpy(pgd + USER_PTRS_PER_PGD,
66 swapper_pg_dir + USER_PTRS_PER_PGD,
67 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
72 #endif /* CONFIG_X86_PAE */
74 static inline pgd_t *get_pgd_fast(void)
78 if ((ret = pgd_quicklist) != NULL) {
79 pgd_quicklist = (unsigned long *)(*ret);
83 ret = (unsigned long *)get_pgd_slow();
87 static inline void free_pgd_fast(pgd_t *pgd)
89 *(unsigned long *)pgd = (unsigned long) pgd_quicklist;
90 pgd_quicklist = (unsigned long *) pgd;
94 static inline void free_pgd_slow(pgd_t *pgd)
96 #if defined(CONFIG_X86_PAE)
99 for (i = 0; i < USER_PTRS_PER_PGD; i++)
100 free_page((unsigned long)__va(pgd_val(pgd[i])-1));
101 kmem_cache_free(pae_pgd_cachep, pgd);
103 free_page((unsigned long)pgd);
107 static inline pte_t *pte_alloc_one(struct mm_struct *mm, unsigned long address)
111 pte = (pte_t *) __get_free_page(GFP_KERNEL);
117 static inline pte_t *pte_alloc_one_fast(struct mm_struct *mm,
118 unsigned long address)
122 if ((ret = (unsigned long *)pte_quicklist) != NULL) {
123 pte_quicklist = (unsigned long *)(*ret);
125 pgtable_cache_size--;
130 static inline void pte_free_fast(pte_t *pte)
132 *(unsigned long *)pte = (unsigned long) pte_quicklist;
133 pte_quicklist = (unsigned long *) pte;
134 pgtable_cache_size++;
137 static __inline__ void pte_free_slow(pte_t *pte)
139 free_page((unsigned long)pte);
142 #define pte_free(pte) pte_free_fast(pte)
143 #define pgd_free(pgd) free_pgd_slow(pgd)
144 #define pgd_alloc(mm) get_pgd_fast()
147 * allocating and freeing a pmd is trivial: the 1-entry pmd is
148 * inside the pgd, so has no extra memory associated with it.
149 * (In the PAE case we free the pmds as part of the pgd.)
152 #define pmd_alloc_one_fast(mm, addr) ({ BUG(); ((pmd_t *)1); })
153 #define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); })
154 #define pmd_free_slow(x) do { } while (0)
155 #define pmd_free_fast(x) do { } while (0)
156 #define pmd_free(x) do { } while (0)
157 #define pgd_populate(mm, pmd, pte) BUG()
159 extern int do_check_pgt_cache(int, int);
164 * - flush_tlb() flushes the current mm struct TLBs
165 * - flush_tlb_all() flushes all processes TLBs
166 * - flush_tlb_mm(mm) flushes the specified mm context TLB's
167 * - flush_tlb_page(vma, vmaddr) flushes one page
168 * - flush_tlb_range(mm, start, end) flushes a range of pages
169 * - flush_tlb_pgtables(mm, start, end) flushes a range of page tables
171 * ..but the i386 has somewhat limited tlb flushing capabilities,
172 * and page-granular flushes are available only on i486 and up.
177 #define flush_tlb() __flush_tlb()
178 #define flush_tlb_all() __flush_tlb_all()
179 #define local_flush_tlb() __flush_tlb()
181 static inline void flush_tlb_mm(struct mm_struct *mm)
183 if (mm == current->active_mm)
187 static inline void flush_tlb_page(struct vm_area_struct *vma,
190 if (vma->vm_mm == current->active_mm)
191 __flush_tlb_one(addr);
194 static inline void flush_tlb_range(struct mm_struct *mm,
195 unsigned long start, unsigned long end)
197 if (mm == current->active_mm)
205 #define local_flush_tlb() \
208 extern void flush_tlb_all(void);
209 extern void flush_tlb_current_task(void);
210 extern void flush_tlb_mm(struct mm_struct *);
211 extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
213 #define flush_tlb() flush_tlb_current_task()
215 static inline void flush_tlb_range(struct mm_struct * mm, unsigned long start, unsigned long end)
220 #define TLBSTATE_OK 1
221 #define TLBSTATE_LAZY 2
225 struct mm_struct *active_mm;
227 } ____cacheline_aligned;
228 extern struct tlb_state cpu_tlbstate[NR_CPUS];
230 #endif /* CONFIG_SMP */
232 static inline void flush_tlb_pgtables(struct mm_struct *mm,
233 unsigned long start, unsigned long end)
238 #endif /* _I386_PGALLOC_H */