@ -82,19 +82,6 @@ static void __put_compound_page(struct page *page)
static void put_compound_page ( struct page * page )
{
/*
* hugetlbfs pages cannot be split from under us . If this is a
* hugetlbfs page , check refcount on head page and release the page if
* the refcount becomes zero .
*/
if ( PageHuge ( page ) ) {
page = compound_head ( page ) ;
if ( put_page_testzero ( page ) )
__put_compound_page ( page ) ;
return ;
}
if ( unlikely ( PageTail ( page ) ) ) {
/* __split_huge_page_refcount can run under us */
struct page * page_head = compound_trans_head ( page ) ;
@ -111,14 +98,31 @@ static void put_compound_page(struct page *page)
* still hot on arches that do not support
* this_cpu_cmpxchg_double ( ) .
*/
if ( PageSlab ( page_head ) ) {
if ( PageTail ( page ) ) {
if ( PageSlab ( page_head ) | | PageHeadHuge ( page_head ) ) {
if ( likely ( PageTail ( page ) ) ) {
/*
* __split_huge_page_refcount
* cannot race here .
*/
VM_BUG_ON ( ! PageHead ( page_head ) ) ;
atomic_dec ( & page - > _mapcount ) ;
if ( put_page_testzero ( page_head ) )
VM_BUG_ON ( 1 ) ;
atomic_dec ( & page - > _mapcount ) ;
goto skip_lock_tail ;
if ( put_page_testzero ( page_head ) )
__put_compound_page ( page_head ) ;
return ;
} else
/*
* __split_huge_page_refcount
* run before us , " page " was a
* THP tail . The split
* page_head has been freed
* and reallocated as slab or
* hugetlbfs page of smaller
* order ( only possible if
* reallocated as slab on
* x86 ) .
*/
goto skip_lock ;
}
/*
@ -132,8 +136,27 @@ static void put_compound_page(struct page *page)
/* __split_huge_page_refcount run before us */
compound_unlock_irqrestore ( page_head , flags ) ;
skip_lock :
if ( put_page_testzero ( page_head ) )
__put_single_page ( page_head ) ;
if ( put_page_testzero ( page_head ) ) {
/*
* The head page may have been
* freed and reallocated as a
* compound page of smaller
* order and then freed again .
* All we know is that it
* cannot have become : a THP
* page , a compound page of
* higher order , a tail page .
* That is because we still
* hold the refcount of the
* split THP tail and
* page_head was the THP head
* before the split .
*/
if ( PageHead ( page_head ) )
__put_compound_page ( page_head ) ;
else
__put_single_page ( page_head ) ;
}
out_put_single :
if ( put_page_testzero ( page ) )
__put_single_page ( page ) ;
@ -155,7 +178,6 @@ out_put_single:
VM_BUG_ON ( atomic_read ( & page - > _count ) ! = 0 ) ;
compound_unlock_irqrestore ( page_head , flags ) ;
skip_lock_tail :
if ( put_page_testzero ( page_head ) ) {
if ( PageHead ( page_head ) )
__put_compound_page ( page_head ) ;
@ -198,51 +220,52 @@ bool __get_page_tail(struct page *page)
* proper PT lock that already serializes against
* split_huge_page ( ) .
*/
unsigned long flags ;
bool got = false ;
struct page * page_head ;
/*
* If this is a hugetlbfs page it cannot be split under us . Simply
* increment refcount for the head page .
*/
if ( PageHuge ( page ) ) {
page_head = compound_head ( page ) ;
atomic_inc ( & page_head - > _count ) ;
got = true ;
} else {
unsigned long flags ;
struct page * page_head = compound_trans_head ( page ) ;
page_head = compound_trans_head ( page ) ;
if ( likely ( page ! = page_head & &
get_page_unless_zero ( page_head ) ) ) {
/* Ref to put_compound_page() comment. */
if ( PageSlab ( page_head ) ) {
if ( likely ( PageTail ( page ) ) ) {
__get_page_tail_foll ( page , false ) ;
return true ;
} else {
put_page ( page_head ) ;
return false ;
}
}
/*
* page_head wasn ' t a dangling pointer but it
* may not be a head page anymore by the time
* we obtain the lock . That is ok as long as it
* can ' t be freed from under us .
*/
flags = compound_lock_irqsave ( page_head ) ;
/* here __split_huge_page_refcount won't run anymore */
if ( likely ( page ! = page_head & & get_page_unless_zero ( page_head ) ) ) {
/* Ref to put_compound_page() comment. */
if ( PageSlab ( page_head ) | | PageHeadHuge ( page_head ) ) {
if ( likely ( PageTail ( page ) ) ) {
/*
* This is a hugetlbfs page or a slab
* page . __split_huge_page_refcount
* cannot race here .
*/
VM_BUG_ON ( ! PageHead ( page_head ) ) ;
__get_page_tail_foll ( page , false ) ;
got = true ;
}
compound_unlock_irqrestore ( page_head , flags ) ;
if ( unlikely ( ! got ) )
return true ;
} else {
/*
* __split_huge_page_refcount run
* before us , " page " was a THP
* tail . The split page_head has been
* freed and reallocated as slab or
* hugetlbfs page of smaller order
* ( only possible if reallocated as
* slab on x86 ) .
*/
put_page ( page_head ) ;
return false ;
}
}
/*
* page_head wasn ' t a dangling pointer but it
* may not be a head page anymore by the time
* we obtain the lock . That is ok as long as it
* can ' t be freed from under us .
*/
flags = compound_lock_irqsave ( page_head ) ;
/* here __split_huge_page_refcount won't run anymore */
if ( likely ( PageTail ( page ) ) ) {
__get_page_tail_foll ( page , false ) ;
got = true ;
}
compound_unlock_irqrestore ( page_head , flags ) ;
if ( unlikely ( ! got ) )
put_page ( page_head ) ;
}
return got ;
}