return atomic_inc_not_zero(&page->_count);
}
+static inline struct page *compound_head(struct page *page)
+{
+ /*
+ * We could avoid the PageCompound(page) check if
+ * we would not overload PageTail().
+ *
+ * This check has to be done in several performance critical
+ * paths of the slab etc. IMHO PageTail deserves its own flag.
+ */
+ if (unlikely(PageCompound(page) && PageTail(page)))
+ return page->first_page;
+ return page;
+}
+
static inline int page_count(struct page *page)
{
- if (unlikely(PageCompound(page)))
- page = (struct page *)page_private(page);
- return atomic_read(&page->_count);
+ return atomic_read(&compound_head(page)->_count);
}
static inline void get_page(struct page *page)
{
- if (unlikely(PageCompound(page)))
- page = (struct page *)page_private(page);
+ page = compound_head(page);
VM_BUG_ON(atomic_read(&page->_count) == 0);
atomic_inc(&page->_count);
}
return (compound_page_dtor *)page[1].lru.next;
}
+static inline int compound_order(struct page *page)
+{
+ if (!PageCompound(page) || PageTail(page))
+ return 0;
+ return (unsigned long)page[1].lru.prev;
+}
+
+static inline void set_compound_order(struct page *page, unsigned long order)
+{
+ page[1].lru.prev = (void *)order;
+}
+
/*
* Multiple processes may "see" the same page. E.g. for untouched
* mappings of /dev/null, all processes see the same page full of
extern void show_free_areas(void);
#ifdef CONFIG_SHMEM
-struct page *shmem_nopage(struct vm_area_struct *vma,
- unsigned long address, int *type);
int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new);
struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
unsigned long addr);
int shmem_lock(struct file *file, int lock, struct user_struct *user);
#else
-#define shmem_nopage filemap_nopage
-
static inline int shmem_lock(struct file *file, int lock,
struct user_struct *user)
{
}
#endif
struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags);
-extern int shmem_mmap(struct file *file, struct vm_area_struct *vma);
int shmem_zero_setup(struct vm_area_struct *);
extern pte_t *FASTCALL(get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl));
+#ifdef __PAGETABLE_PUD_FOLDED
+static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
+ unsigned long address)
+{
+ return 0;
+}
+#else
int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
+#endif
+
+#ifdef __PAGETABLE_PMD_FOLDED
+static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
+ unsigned long address)
+{
+ return 0;
+}
+#else
int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
+#endif
+
int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
#define FOLL_GET 0x04 /* do get_page on page */
#define FOLL_ANON 0x08 /* give ZERO_PAGE if no pgtable */
+typedef int (*pte_fn_t)(pte_t *pte, struct page *pmd_page, unsigned long addr,
+ void *data);
+extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
+ unsigned long size, pte_fn_t fn, void *data);
+
#ifdef CONFIG_PROC_FS
void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
#else