Move free pages between lists on steal

[powerpc.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index fea1e3b..d575a3e 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -680,6 +680,72 @@ static int fallbacks[MIGRATE_TYPES][MIGRATE_TYPES-1] = {
        [MIGRATE_MOVABLE]   = { MIGRATE_UNMOVABLE },
 };
 
+/*
+ * Move the free pages in a range to the free lists of the requested type.
+ * Note that start_page and end_pages are not aligned in a MAX_ORDER_NR_PAGES
+ * boundary. If alignment is required, use move_freepages_block()
+ */
+int move_freepages(struct zone *zone,
+                       struct page *start_page, struct page *end_page,
+                       int migratetype)
+{
+       struct page *page;
+       unsigned long order;
+       int blocks_moved = 0;
+
+#ifndef CONFIG_HOLES_IN_ZONE
+       /*
+        * page_zone is not safe to call in this context when
+        * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
+        * anyway as we check zone boundaries in move_freepages_block().
+        * Remove at a later date when no bug reports exist related to
+        * CONFIG_PAGE_GROUP_BY_MOBILITY
+        */
+       BUG_ON(page_zone(start_page) != page_zone(end_page));
+#endif
+
+       for (page = start_page; page <= end_page;) {
+               if (!pfn_valid_within(page_to_pfn(page))) {
+                       page++;
+                       continue;
+               }
+
+               if (!PageBuddy(page)) {
+                       page++;
+                       continue;
+               }
+
+               order = page_order(page);
+               list_del(&page->lru);
+               list_add(&page->lru,
+                       &zone->free_area[order].free_list[migratetype]);
+               page += 1 << order;
+               blocks_moved++;
+       }
+
+       return blocks_moved;
+}
+
+int move_freepages_block(struct zone *zone, struct page *page, int migratetype)
+{
+       unsigned long start_pfn, end_pfn;
+       struct page *start_page, *end_page;
+
+       start_pfn = page_to_pfn(page);
+       start_pfn = start_pfn & ~(MAX_ORDER_NR_PAGES-1);
+       start_page = pfn_to_page(start_pfn);
+       end_page = start_page + MAX_ORDER_NR_PAGES - 1;
+       end_pfn = start_pfn + MAX_ORDER_NR_PAGES - 1;
+
+       /* Do not cross zone boundaries */
+       if (start_pfn < zone->zone_start_pfn)
+               start_page = page;
+       if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages)
+               return 0;
+
+       return move_freepages(zone, start_page, end_page, migratetype);
+}
+
 /* Remove an element from the buddy allocator from the fallback list */
 static struct page *__rmqueue_fallback(struct zone *zone, int order,
                                                int start_migratetype)
@@ -704,11 +770,13 @@ static struct page *__rmqueue_fallback(struct zone *zone, int order,
                        area->nr_free--;
 
                        /*
-                        * If breaking a large block of pages, place the buddies
-                        * on the preferred allocation list
+                        * If breaking a large block of pages, move all free
+                        * pages to the preferred allocation list
                         */
-                       if (unlikely(current_order >= MAX_ORDER / 2))
+                       if (unlikely(current_order >= MAX_ORDER / 2)) {
                                migratetype = start_migratetype;
+                               move_freepages_block(zone, page, migratetype);
+                       }
 
                        /* Remove the page from the freelists */
                        list_del(&page->lru);
@@ -876,7 +944,9 @@ void mark_free_pages(struct zone *zone)
        }
        spin_unlock_irqrestore(&zone->lock, flags);
 }
+#endif /* CONFIG_PM */
 
+#if defined(CONFIG_HIBERNATION) || defined(CONFIG_PAGE_GROUP_BY_MOBILITY)
 /*
  * Spill all of this CPU's per-cpu pages back into the buddy allocator.
  */
@@ -888,7 +958,28 @@ void drain_local_pages(void)
        __drain_pages(smp_processor_id());
        local_irq_restore(flags);       
 }
-#endif /* CONFIG_HIBERNATION */
+
+void smp_drain_local_pages(void *arg)
+{
+       drain_local_pages();
+}
+
+/*
+ * Spill all the per-cpu pages from all CPUs back into the buddy allocator
+ */
+void drain_all_local_pages(void)
+{
+       unsigned long flags;
+
+       local_irq_save(flags);
+       __drain_pages(smp_processor_id());
+       local_irq_restore(flags);
+
+       smp_call_function(smp_drain_local_pages, NULL, 0, 1);
+}
+#else
+void drain_all_local_pages(void) {}
+#endif /* CONFIG_HIBERNATION || CONFIG_PAGE_GROUP_BY_MOBILITY */
 
 /*
  * Free a 0-order page
@@ -1480,6 +1571,9 @@ nofail_alloc:
 
        cond_resched();
 
+       if (order != 0)
+               drain_all_local_pages();
+
        if (likely(did_some_progress)) {
                page = get_page_from_freelist(gfp_mask, order,
                                                zonelist, alloc_flags);