172 lines
4.8 KiB
Diff
172 lines
4.8 KiB
Diff
From 532c38e1e5fb5c27b6574adbd5ec0abafa87e590 Mon Sep 17 00:00:00 2001
|
|
Message-Id: <532c38e1e5fb5c27b6574adbd5ec0abafa87e590.1596234183.git.zanussi@kernel.org>
|
|
In-Reply-To: <378ee68279f6a7631221f2670a9298620148690d.1596234183.git.zanussi@kernel.org>
|
|
References: <378ee68279f6a7631221f2670a9298620148690d.1596234183.git.zanussi@kernel.org>
|
|
From: Peter Zijlstra <peterz@infradead.org>
|
|
Date: Mon, 28 May 2018 15:24:20 +0200
|
|
Subject: [PATCH 072/329] Split IRQ-off and zone->lock while freeing pages from
|
|
PCP list #1
|
|
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.19/older/patches-4.19.135-rt60.tar.xz
|
|
|
|
Split the IRQ-off section while accessing the PCP list from zone->lock
|
|
while freeing pages.
|
|
Introcude isolate_pcp_pages() which separates the pages from the PCP
|
|
list onto a temporary list and then free the temporary list via
|
|
free_pcppages_bulk().
|
|
|
|
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
|
|
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
|
|
---
|
|
mm/page_alloc.c | 82 +++++++++++++++++++++++++++++++------------------
|
|
1 file changed, 52 insertions(+), 30 deletions(-)
|
|
|
|
--- a/mm/page_alloc.c
|
|
+++ b/mm/page_alloc.c
|
|
@@ -1095,7 +1095,7 @@
|
|
}
|
|
|
|
/*
|
|
- * Frees a number of pages from the PCP lists
|
|
+ * Frees a number of pages which have been collected from the pcp lists.
|
|
* Assumes all pages on list are in same zone, and of same order.
|
|
* count is the number of pages to free.
|
|
*
|
|
@@ -1106,14 +1106,41 @@
|
|
* pinned" detection logic.
|
|
*/
|
|
static void free_pcppages_bulk(struct zone *zone, int count,
|
|
- struct per_cpu_pages *pcp)
|
|
+ struct list_head *head)
|
|
+{
|
|
+ bool isolated_pageblocks;
|
|
+ struct page *page, *tmp;
|
|
+ unsigned long flags;
|
|
+
|
|
+ spin_lock_irqsave(&zone->lock, flags);
|
|
+ isolated_pageblocks = has_isolate_pageblock(zone);
|
|
+
|
|
+ /*
|
|
+ * Use safe version since after __free_one_page(),
|
|
+ * page->lru.next will not point to original list.
|
|
+ */
|
|
+ list_for_each_entry_safe(page, tmp, head, lru) {
|
|
+ int mt = get_pcppage_migratetype(page);
|
|
+ /* MIGRATE_ISOLATE page should not go to pcplists */
|
|
+ VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
|
|
+ /* Pageblock could have been isolated meanwhile */
|
|
+ if (unlikely(isolated_pageblocks))
|
|
+ mt = get_pageblock_migratetype(page);
|
|
+
|
|
+ __free_one_page(page, page_to_pfn(page), zone, 0, mt);
|
|
+ trace_mm_page_pcpu_drain(page, 0, mt);
|
|
+ }
|
|
+ spin_unlock_irqrestore(&zone->lock, flags);
|
|
+}
|
|
+
|
|
+static void isolate_pcp_pages(int count, struct per_cpu_pages *pcp,
|
|
+ struct list_head *dst)
|
|
+
|
|
{
|
|
int migratetype = 0;
|
|
int batch_free = 0;
|
|
int prefetch_nr = 0;
|
|
- bool isolated_pageblocks;
|
|
- struct page *page, *tmp;
|
|
- LIST_HEAD(head);
|
|
+ struct page *page;
|
|
|
|
/*
|
|
* Ensure proper count is passed which otherwise would stuck in the
|
|
@@ -1150,7 +1177,7 @@
|
|
if (bulkfree_pcp_prepare(page))
|
|
continue;
|
|
|
|
- list_add_tail(&page->lru, &head);
|
|
+ list_add_tail(&page->lru, dst);
|
|
|
|
/*
|
|
* We are going to put the page back to the global
|
|
@@ -1165,26 +1192,6 @@
|
|
prefetch_buddy(page);
|
|
} while (--count && --batch_free && !list_empty(list));
|
|
}
|
|
-
|
|
- spin_lock(&zone->lock);
|
|
- isolated_pageblocks = has_isolate_pageblock(zone);
|
|
-
|
|
- /*
|
|
- * Use safe version since after __free_one_page(),
|
|
- * page->lru.next will not point to original list.
|
|
- */
|
|
- list_for_each_entry_safe(page, tmp, &head, lru) {
|
|
- int mt = get_pcppage_migratetype(page);
|
|
- /* MIGRATE_ISOLATE page should not go to pcplists */
|
|
- VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
|
|
- /* Pageblock could have been isolated meanwhile */
|
|
- if (unlikely(isolated_pageblocks))
|
|
- mt = get_pageblock_migratetype(page);
|
|
-
|
|
- __free_one_page(page, page_to_pfn(page), zone, 0, mt);
|
|
- trace_mm_page_pcpu_drain(page, 0, mt);
|
|
- }
|
|
- spin_unlock(&zone->lock);
|
|
}
|
|
|
|
static void free_one_page(struct zone *zone,
|
|
@@ -2545,13 +2552,18 @@
|
|
{
|
|
unsigned long flags;
|
|
int to_drain, batch;
|
|
+ LIST_HEAD(dst);
|
|
|
|
local_irq_save(flags);
|
|
batch = READ_ONCE(pcp->batch);
|
|
to_drain = min(pcp->count, batch);
|
|
if (to_drain > 0)
|
|
- free_pcppages_bulk(zone, to_drain, pcp);
|
|
+ isolate_pcp_pages(to_drain, pcp, &dst);
|
|
+
|
|
local_irq_restore(flags);
|
|
+
|
|
+ if (to_drain > 0)
|
|
+ free_pcppages_bulk(zone, to_drain, &dst);
|
|
}
|
|
#endif
|
|
|
|
@@ -2567,14 +2579,21 @@
|
|
unsigned long flags;
|
|
struct per_cpu_pageset *pset;
|
|
struct per_cpu_pages *pcp;
|
|
+ LIST_HEAD(dst);
|
|
+ int count;
|
|
|
|
local_irq_save(flags);
|
|
pset = per_cpu_ptr(zone->pageset, cpu);
|
|
|
|
pcp = &pset->pcp;
|
|
- if (pcp->count)
|
|
- free_pcppages_bulk(zone, pcp->count, pcp);
|
|
+ count = pcp->count;
|
|
+ if (count)
|
|
+ isolate_pcp_pages(count, pcp, &dst);
|
|
+
|
|
local_irq_restore(flags);
|
|
+
|
|
+ if (count)
|
|
+ free_pcppages_bulk(zone, count, &dst);
|
|
}
|
|
|
|
/*
|
|
@@ -2796,7 +2815,10 @@
|
|
pcp->count++;
|
|
if (pcp->count >= pcp->high) {
|
|
unsigned long batch = READ_ONCE(pcp->batch);
|
|
- free_pcppages_bulk(zone, batch, pcp);
|
|
+ LIST_HEAD(dst);
|
|
+
|
|
+ isolate_pcp_pages(batch, pcp, &dst);
|
|
+ free_pcppages_bulk(zone, batch, &dst);
|
|
}
|
|
}
|
|
|