Line data Source code
1 : /*
2 : * Macros for manipulating and testing page->flags
3 : */
4 :
5 : #ifndef PAGE_FLAGS_H
6 : #define PAGE_FLAGS_H
7 :
8 : #include <linux/types.h>
9 : #include <linux/bug.h>
10 : #include <linux/mmdebug.h>
11 : #ifndef __GENERATING_BOUNDS_H
12 : #include <linux/mm_types.h>
13 : #include <generated/bounds.h>
14 : #endif /* !__GENERATING_BOUNDS_H */
15 :
16 : /*
17 : * Various page->flags bits:
18 : *
19 : * PG_reserved is set for special pages, which can never be swapped out. Some
20 : * of them might not even exist (eg empty_bad_page)...
21 : *
22 : * The PG_private bitflag is set on pagecache pages if they contain filesystem
23 : * specific data (which is normally at page->private). It can be used by
24 : * private allocations for its own usage.
25 : *
26 : * During initiation of disk I/O, PG_locked is set. This bit is set before I/O
27 : * and cleared when writeback _starts_ or when read _completes_. PG_writeback
28 : * is set before writeback starts and cleared when it finishes.
29 : *
30 : * PG_locked also pins a page in pagecache, and blocks truncation of the file
31 : * while it is held.
32 : *
33 : * page_waitqueue(page) is a wait queue of all tasks waiting for the page
34 : * to become unlocked.
35 : *
36 : * PG_uptodate tells whether the page's contents is valid. When a read
37 : * completes, the page becomes uptodate, unless a disk I/O error happened.
38 : *
39 : * PG_referenced, PG_reclaim are used for page reclaim for anonymous and
40 : * file-backed pagecache (see mm/vmscan.c).
41 : *
42 : * PG_error is set to indicate that an I/O error occurred on this page.
43 : *
44 : * PG_arch_1 is an architecture specific page state bit. The generic code
45 : * guarantees that this bit is cleared for a page when it first is entered into
46 : * the page cache.
47 : *
48 : * PG_highmem pages are not permanently mapped into the kernel virtual address
49 : * space, they need to be kmapped separately for doing IO on the pages. The
50 : * struct page (these bits with information) are always mapped into kernel
51 : * address space...
52 : *
53 : * PG_hwpoison indicates that a page got corrupted in hardware and contains
54 : * data with incorrect ECC bits that triggered a machine check. Accessing is
55 : * not safe since it may cause another machine check. Don't touch!
56 : */
57 :
58 : /*
59 : * Don't use the *_dontuse flags. Use the macros. Otherwise you'll break
60 : * locked- and dirty-page accounting.
61 : *
62 : * The page flags field is split into two parts, the main flags area
63 : * which extends from the low bits upwards, and the fields area which
64 : * extends from the high bits downwards.
65 : *
66 : * | FIELD | ... | FLAGS |
67 : * N-1 ^ 0
68 : * (NR_PAGEFLAGS)
69 : *
70 : * The fields area is reserved for fields mapping zone, node (for NUMA) and
71 : * SPARSEMEM section (for variants of SPARSEMEM that require section ids like
72 : * SPARSEMEM_EXTREME with !SPARSEMEM_VMEMMAP).
73 : */
74 : enum pageflags {
75 : PG_locked, /* Page is locked. Don't touch. */
76 : PG_error,
77 : PG_referenced,
78 : PG_uptodate,
79 : PG_dirty,
80 : PG_lru,
81 : PG_active,
82 : PG_slab,
83 : PG_owner_priv_1, /* Owner use. If pagecache, fs may use*/
84 : PG_arch_1,
85 : PG_reserved,
86 : PG_private, /* If pagecache, has fs-private data */
87 : PG_private_2, /* If pagecache, has fs aux data */
88 : PG_writeback, /* Page is under writeback */
89 : #ifdef CONFIG_PAGEFLAGS_EXTENDED
90 : PG_head, /* A head page */
91 : PG_tail, /* A tail page */
92 : #else
93 : PG_compound, /* A compound page */
94 : #endif
95 : PG_swapcache, /* Swap page: swp_entry_t in private */
96 : PG_mappedtodisk, /* Has blocks allocated on-disk */
97 : PG_reclaim, /* To be reclaimed asap */
98 : PG_swapbacked, /* Page is backed by RAM/swap */
99 : PG_unevictable, /* Page is "unevictable" */
100 : #ifdef CONFIG_MMU
101 : PG_mlocked, /* Page is vma mlocked */
102 : #endif
103 : #ifdef CONFIG_ARCH_USES_PG_UNCACHED
104 : PG_uncached, /* Page has been mapped as uncached */
105 : #endif
106 : #ifdef CONFIG_MEMORY_FAILURE
107 : PG_hwpoison, /* hardware poisoned page. Don't touch */
108 : #endif
109 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
110 : PG_compound_lock,
111 : #endif
112 : __NR_PAGEFLAGS,
113 :
114 : /* Filesystems */
115 : PG_checked = PG_owner_priv_1,
116 :
117 : /* Two page bits are conscripted by FS-Cache to maintain local caching
118 : * state. These bits are set on pages belonging to the netfs's inodes
119 : * when those inodes are being locally cached.
120 : */
121 : PG_fscache = PG_private_2, /* page backed by cache */
122 :
123 : /* XEN */
124 : PG_pinned = PG_owner_priv_1,
125 : PG_savepinned = PG_dirty,
126 :
127 : /* SLOB */
128 : PG_slob_free = PG_private,
129 : };
130 :
131 : #ifndef __GENERATING_BOUNDS_H
132 :
133 : /*
134 : * Macros to create function definitions for page flags
135 : */
136 : #define TESTPAGEFLAG(uname, lname) \
137 : static inline int Page##uname(const struct page *page) \
138 : { return test_bit(PG_##lname, &page->flags); }
139 :
140 : #define SETPAGEFLAG(uname, lname) \
141 : static inline void SetPage##uname(struct page *page) \
142 : { set_bit(PG_##lname, &page->flags); }
143 :
144 : #define CLEARPAGEFLAG(uname, lname) \
145 : static inline void ClearPage##uname(struct page *page) \
146 : { clear_bit(PG_##lname, &page->flags); }
147 :
148 : #define __SETPAGEFLAG(uname, lname) \
149 : static inline void __SetPage##uname(struct page *page) \
150 : { __set_bit(PG_##lname, &page->flags); }
151 :
152 : #define __CLEARPAGEFLAG(uname, lname) \
153 : static inline void __ClearPage##uname(struct page *page) \
154 : { __clear_bit(PG_##lname, &page->flags); }
155 :
156 : #define TESTSETFLAG(uname, lname) \
157 : static inline int TestSetPage##uname(struct page *page) \
158 : { return test_and_set_bit(PG_##lname, &page->flags); }
159 :
160 : #define TESTCLEARFLAG(uname, lname) \
161 : static inline int TestClearPage##uname(struct page *page) \
162 : { return test_and_clear_bit(PG_##lname, &page->flags); }
163 :
164 : #define __TESTCLEARFLAG(uname, lname) \
165 : static inline int __TestClearPage##uname(struct page *page) \
166 : { return __test_and_clear_bit(PG_##lname, &page->flags); }
167 :
168 : #define PAGEFLAG(uname, lname) TESTPAGEFLAG(uname, lname) \
169 : SETPAGEFLAG(uname, lname) CLEARPAGEFLAG(uname, lname)
170 :
171 : #define __PAGEFLAG(uname, lname) TESTPAGEFLAG(uname, lname) \
172 : __SETPAGEFLAG(uname, lname) __CLEARPAGEFLAG(uname, lname)
173 :
174 : #define TESTSCFLAG(uname, lname) \
175 : TESTSETFLAG(uname, lname) TESTCLEARFLAG(uname, lname)
176 :
177 : #define TESTPAGEFLAG_FALSE(uname) \
178 : static inline int Page##uname(const struct page *page) { return 0; }
179 :
180 : #define SETPAGEFLAG_NOOP(uname) \
181 : static inline void SetPage##uname(struct page *page) { }
182 :
183 : #define CLEARPAGEFLAG_NOOP(uname) \
184 : static inline void ClearPage##uname(struct page *page) { }
185 :
186 : #define __CLEARPAGEFLAG_NOOP(uname) \
187 : static inline void __ClearPage##uname(struct page *page) { }
188 :
189 : #define TESTSETFLAG_FALSE(uname) \
190 : static inline int TestSetPage##uname(struct page *page) { return 0; }
191 :
192 : #define TESTCLEARFLAG_FALSE(uname) \
193 : static inline int TestClearPage##uname(struct page *page) { return 0; }
194 :
195 : #define __TESTCLEARFLAG_FALSE(uname) \
196 : static inline int __TestClearPage##uname(struct page *page) { return 0; }
197 :
198 : #define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \
199 : SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)
200 :
201 : #define TESTSCFLAG_FALSE(uname) \
202 : TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)
203 :
204 : struct page; /* forward declaration */
205 :
206 : TESTPAGEFLAG(Locked, locked)
207 : PAGEFLAG(Error, error) TESTCLEARFLAG(Error, error)
208 : PAGEFLAG(Referenced, referenced) TESTCLEARFLAG(Referenced, referenced)
209 : __SETPAGEFLAG(Referenced, referenced)
210 : PAGEFLAG(Dirty, dirty) TESTSCFLAG(Dirty, dirty) __CLEARPAGEFLAG(Dirty, dirty)
211 : PAGEFLAG(LRU, lru) __CLEARPAGEFLAG(LRU, lru)
212 : PAGEFLAG(Active, active) __CLEARPAGEFLAG(Active, active)
213 : TESTCLEARFLAG(Active, active)
214 : __PAGEFLAG(Slab, slab)
215 : PAGEFLAG(Checked, checked) /* Used by some filesystems */
216 : PAGEFLAG(Pinned, pinned) TESTSCFLAG(Pinned, pinned) /* Xen */
217 : PAGEFLAG(SavePinned, savepinned); /* Xen */
218 : PAGEFLAG(Reserved, reserved) __CLEARPAGEFLAG(Reserved, reserved)
219 : PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked)
220 : __SETPAGEFLAG(SwapBacked, swapbacked)
221 :
222 : __PAGEFLAG(SlobFree, slob_free)
223 :
224 : /*
225 : * Private page markings that may be used by the filesystem that owns the page
226 : * for its own purposes.
227 : * - PG_private and PG_private_2 cause releasepage() and co to be invoked
228 : */
229 : PAGEFLAG(Private, private) __SETPAGEFLAG(Private, private)
230 : __CLEARPAGEFLAG(Private, private)
231 1308900 : PAGEFLAG(Private2, private_2) TESTSCFLAG(Private2, private_2)
232 : PAGEFLAG(OwnerPriv1, owner_priv_1) TESTCLEARFLAG(OwnerPriv1, owner_priv_1)
233 :
234 : /*
235 : * Only test-and-set exist for PG_writeback. The unconditional operators are
236 : * risky: they bypass page accounting.
237 : */
238 : TESTPAGEFLAG(Writeback, writeback) TESTSCFLAG(Writeback, writeback)
239 : PAGEFLAG(MappedToDisk, mappedtodisk)
240 :
241 : /* PG_readahead is only used for reads; PG_reclaim is only for writes */
242 : PAGEFLAG(Reclaim, reclaim) TESTCLEARFLAG(Reclaim, reclaim)
243 : PAGEFLAG(Readahead, reclaim) TESTCLEARFLAG(Readahead, reclaim)
244 :
245 : #ifdef CONFIG_HIGHMEM
246 : /*
247 : * Must use a macro here due to header dependency issues. page_zone() is not
248 : * available at this point.
249 : */
250 : #define PageHighMem(__p) is_highmem(page_zone(__p))
251 : #else
252 : PAGEFLAG_FALSE(HighMem)
253 : #endif
254 :
255 : #ifdef CONFIG_SWAP
256 : PAGEFLAG(SwapCache, swapcache)
257 : #else
258 : PAGEFLAG_FALSE(SwapCache)
259 : #endif
260 :
261 : PAGEFLAG(Unevictable, unevictable) __CLEARPAGEFLAG(Unevictable, unevictable)
262 : TESTCLEARFLAG(Unevictable, unevictable)
263 :
264 : #ifdef CONFIG_MMU
265 : PAGEFLAG(Mlocked, mlocked) __CLEARPAGEFLAG(Mlocked, mlocked)
266 : TESTSCFLAG(Mlocked, mlocked) __TESTCLEARFLAG(Mlocked, mlocked)
267 : #else
268 : PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
269 : TESTSCFLAG_FALSE(Mlocked) __TESTCLEARFLAG_FALSE(Mlocked)
270 : #endif
271 :
272 : #ifdef CONFIG_ARCH_USES_PG_UNCACHED
273 : PAGEFLAG(Uncached, uncached)
274 : #else
275 : PAGEFLAG_FALSE(Uncached)
276 : #endif
277 :
278 : #ifdef CONFIG_MEMORY_FAILURE
279 : PAGEFLAG(HWPoison, hwpoison)
280 : TESTSCFLAG(HWPoison, hwpoison)
281 : #define __PG_HWPOISON (1UL << PG_hwpoison)
282 : #else
283 : PAGEFLAG_FALSE(HWPoison)
284 : #define __PG_HWPOISON 0
285 : #endif
286 :
287 : u64 stable_page_flags(struct page *page);
288 :
289 : static inline int PageUptodate(struct page *page)
290 : {
291 : int ret = test_bit(PG_uptodate, &(page)->flags);
292 :
293 : /*
294 : * Must ensure that the data we read out of the page is loaded
295 : * _after_ we've loaded page->flags to check for PageUptodate.
296 : * We can skip the barrier if the page is not uptodate, because
297 : * we wouldn't be reading anything from it.
298 : *
299 : * See SetPageUptodate() for the other side of the story.
300 : */
301 3286188 : if (ret)
302 1555361 : smp_rmb();
303 :
304 : return ret;
305 : }
306 :
307 : static inline void __SetPageUptodate(struct page *page)
308 : {
309 : smp_wmb();
310 : __set_bit(PG_uptodate, &(page)->flags);
311 : }
312 :
313 : static inline void SetPageUptodate(struct page *page)
314 : {
315 : /*
316 : * Memory barrier must be issued before setting the PG_uptodate bit,
317 : * so that all previous stores issued in order to bring the page
318 : * uptodate are actually visible before PageUptodate becomes true.
319 : */
320 2026534 : smp_wmb();
321 : set_bit(PG_uptodate, &(page)->flags);
322 : }
323 :
324 : CLEARPAGEFLAG(Uptodate, uptodate)
325 :
326 : extern void cancel_dirty_page(struct page *page, unsigned int account_size);
327 :
328 : int test_clear_page_writeback(struct page *page);
329 : int __test_set_page_writeback(struct page *page, bool keep_write);
330 :
331 : #define test_set_page_writeback(page) \
332 : __test_set_page_writeback(page, false)
333 : #define test_set_page_writeback_keepwrite(page) \
334 : __test_set_page_writeback(page, true)
335 :
336 : static inline void set_page_writeback(struct page *page)
337 : {
338 1492754 : test_set_page_writeback(page);
339 : }
340 :
341 : static inline void set_page_writeback_keepwrite(struct page *page)
342 : {
343 : test_set_page_writeback_keepwrite(page);
344 : }
345 :
346 : #ifdef CONFIG_PAGEFLAGS_EXTENDED
347 : /*
348 : * System with lots of page flags available. This allows separate
349 : * flags for PageHead() and PageTail() checks of compound pages so that bit
350 : * tests can be used in performance sensitive paths. PageCompound is
351 : * generally not used in hot code paths except arch/powerpc/mm/init_64.c
352 : * and arch/powerpc/kvm/book3s_64_vio_hv.c which use it to detect huge pages
353 : * and avoid handling those in real mode.
354 : */
355 : __PAGEFLAG(Head, head) CLEARPAGEFLAG(Head, head)
356 : __PAGEFLAG(Tail, tail)
357 :
358 : static inline int PageCompound(struct page *page)
359 : {
360 : return page->flags & ((1L << PG_head) | (1L << PG_tail));
361 :
362 : }
363 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
364 : static inline void ClearPageCompound(struct page *page)
365 : {
366 : BUG_ON(!PageHead(page));
367 : ClearPageHead(page);
368 : }
369 : #endif
370 :
371 : #define PG_head_mask ((1L << PG_head))
372 :
373 : #else
374 : /*
375 : * Reduce page flag use as much as possible by overlapping
376 : * compound page flags with the flags used for page cache pages. Possible
377 : * because PageCompound is always set for compound pages and not for
378 : * pages on the LRU and/or pagecache.
379 : */
380 : TESTPAGEFLAG(Compound, compound)
381 : __SETPAGEFLAG(Head, compound) __CLEARPAGEFLAG(Head, compound)
382 :
383 : /*
384 : * PG_reclaim is used in combination with PG_compound to mark the
385 : * head and tail of a compound page. This saves one page flag
386 : * but makes it impossible to use compound pages for the page cache.
387 : * The PG_reclaim bit would have to be used for reclaim or readahead
388 : * if compound pages enter the page cache.
389 : *
390 : * PG_compound & PG_reclaim => Tail page
391 : * PG_compound & ~PG_reclaim => Head page
392 : */
393 : #define PG_head_mask ((1L << PG_compound))
394 : #define PG_head_tail_mask ((1L << PG_compound) | (1L << PG_reclaim))
395 :
396 : static inline int PageHead(struct page *page)
397 : {
398 : return ((page->flags & PG_head_tail_mask) == PG_head_mask);
399 : }
400 :
401 : static inline int PageTail(struct page *page)
402 : {
403 : return ((page->flags & PG_head_tail_mask) == PG_head_tail_mask);
404 : }
405 :
406 : static inline void __SetPageTail(struct page *page)
407 : {
408 : page->flags |= PG_head_tail_mask;
409 : }
410 :
411 : static inline void __ClearPageTail(struct page *page)
412 : {
413 : page->flags &= ~PG_head_tail_mask;
414 : }
415 :
416 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
417 : static inline void ClearPageCompound(struct page *page)
418 : {
419 : BUG_ON((page->flags & PG_head_tail_mask) != (1 << PG_compound));
420 : clear_bit(PG_compound, &page->flags);
421 : }
422 : #endif
423 :
424 : #endif /* !PAGEFLAGS_EXTENDED */
425 :
426 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
427 : /*
428 : * PageHuge() only returns true for hugetlbfs pages, but not for
429 : * normal or transparent huge pages.
430 : *
431 : * PageTransHuge() returns true for both transparent huge and
432 : * hugetlbfs pages, but not normal pages. PageTransHuge() can only be
433 : * called only in the core VM paths where hugetlbfs pages can't exist.
434 : */
435 : static inline int PageTransHuge(struct page *page)
436 : {
437 : VM_BUG_ON_PAGE(PageTail(page), page);
438 : return PageHead(page);
439 : }
440 :
441 : /*
442 : * PageTransCompound returns true for both transparent huge pages
443 : * and hugetlbfs pages, so it should only be called when it's known
444 : * that hugetlbfs pages aren't involved.
445 : */
446 : static inline int PageTransCompound(struct page *page)
447 : {
448 : return PageCompound(page);
449 : }
450 :
451 : /*
452 : * PageTransTail returns true for both transparent huge pages
453 : * and hugetlbfs pages, so it should only be called when it's known
454 : * that hugetlbfs pages aren't involved.
455 : */
456 : static inline int PageTransTail(struct page *page)
457 : {
458 : return PageTail(page);
459 : }
460 :
461 : #else
462 :
463 : static inline int PageTransHuge(struct page *page)
464 : {
465 : return 0;
466 : }
467 :
468 : static inline int PageTransCompound(struct page *page)
469 : {
470 : return 0;
471 : }
472 :
473 : static inline int PageTransTail(struct page *page)
474 : {
475 : return 0;
476 : }
477 : #endif
478 :
479 : /*
480 : * If network-based swap is enabled, sl*b must keep track of whether pages
481 : * were allocated from pfmemalloc reserves.
482 : */
483 : static inline int PageSlabPfmemalloc(struct page *page)
484 : {
485 : VM_BUG_ON_PAGE(!PageSlab(page), page);
486 : return PageActive(page);
487 : }
488 :
489 : static inline void SetPageSlabPfmemalloc(struct page *page)
490 : {
491 : VM_BUG_ON_PAGE(!PageSlab(page), page);
492 : SetPageActive(page);
493 : }
494 :
495 : static inline void __ClearPageSlabPfmemalloc(struct page *page)
496 : {
497 : VM_BUG_ON_PAGE(!PageSlab(page), page);
498 : __ClearPageActive(page);
499 : }
500 :
501 : static inline void ClearPageSlabPfmemalloc(struct page *page)
502 : {
503 : VM_BUG_ON_PAGE(!PageSlab(page), page);
504 : ClearPageActive(page);
505 : }
506 :
507 : #ifdef CONFIG_MMU
508 : #define __PG_MLOCKED (1 << PG_mlocked)
509 : #else
510 : #define __PG_MLOCKED 0
511 : #endif
512 :
513 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
514 : #define __PG_COMPOUND_LOCK (1 << PG_compound_lock)
515 : #else
516 : #define __PG_COMPOUND_LOCK 0
517 : #endif
518 :
519 : /*
520 : * Flags checked when a page is freed. Pages being freed should not have
521 : * these flags set. It they are, there is a problem.
522 : */
523 : #define PAGE_FLAGS_CHECK_AT_FREE \
524 : (1 << PG_lru | 1 << PG_locked | \
525 : 1 << PG_private | 1 << PG_private_2 | \
526 : 1 << PG_writeback | 1 << PG_reserved | \
527 : 1 << PG_slab | 1 << PG_swapcache | 1 << PG_active | \
528 : 1 << PG_unevictable | __PG_MLOCKED | __PG_HWPOISON | \
529 : __PG_COMPOUND_LOCK)
530 :
531 : /*
532 : * Flags checked when a page is prepped for return by the page allocator.
533 : * Pages being prepped should not have any flags set. It they are set,
534 : * there has been a kernel bug or struct page corruption.
535 : */
536 : #define PAGE_FLAGS_CHECK_AT_PREP ((1 << NR_PAGEFLAGS) - 1)
537 :
538 : #define PAGE_FLAGS_PRIVATE \
539 : (1 << PG_private | 1 << PG_private_2)
540 : /**
541 : * page_has_private - Determine if page has private stuff
542 : * @page: The page to be checked
543 : *
544 : * Determine if a page has private stuff, indicating that release routines
545 : * should be invoked upon it.
546 : */
547 : static inline int page_has_private(struct page *page)
548 : {
549 0 : return !!(page->flags & PAGE_FLAGS_PRIVATE);
550 : }
551 :
552 : #endif /* !__GENERATING_BOUNDS_H */
553 :
554 : #endif /* PAGE_FLAGS_H */
|
