summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKent Overstreet <kent.overstreet@gmail.com>2016-09-19 17:11:53 -0800
committerKent Overstreet <kent.overstreet@gmail.com>2018-01-30 19:22:24 -0500
commitfb90e6877cf17bafd77ef377469ad7a385d9e3bc (patch)
tree6e078a11648e48fe4b22808a2e534a9bcbc89fb1
parentcbcbe33aeff53ef92c7b361a6f44f87011189119 (diff)
mm: Real pagecache iterators
Introduce for_each_pagecache_page() and related macros, with the goal of replacing most/all uses of pagevec_lookup(). For the most part this shouldn't be a functional change. The one functional difference with the new macros is that they now take an @end parameter, so we're able to avoid grabbing pages in __find_get_pages() that we'll never use. This patch only does some of the conversions, the ones I was able to easily test myself - the conversions are mechanical but tricky enough they generally warrent testing. Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Al Viro <viro@zeniv.linux.org.uk>
-rw-r--r--include/linux/pagemap.h5
-rw-r--r--include/linux/pagevec.h67
-rw-r--r--include/linux/radix-tree.h25
-rw-r--r--mm/filemap.c147
4 files changed, 227 insertions, 17 deletions
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 79b36f57c3ba..3659c8923d8e 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -350,6 +350,11 @@ static inline struct page *grab_cache_page_nowait(struct address_space *mapping,
struct page *find_get_entry(struct address_space *mapping, pgoff_t offset);
struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset);
+
+unsigned __find_get_pages(struct address_space *mapping,
+ pgoff_t start, pgoff_t end,
+ unsigned nr_entries, struct page **entries,
+ pgoff_t *indices, unsigned flags);
unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
unsigned int nr_entries, struct page **entries,
pgoff_t *indices);
diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h
index b45d391b4540..e60d74148d0b 100644
--- a/include/linux/pagevec.h
+++ b/include/linux/pagevec.h
@@ -22,10 +22,6 @@ struct pagevec {
void __pagevec_release(struct pagevec *pvec);
void __pagevec_lru_add(struct pagevec *pvec);
-unsigned pagevec_lookup_entries(struct pagevec *pvec,
- struct address_space *mapping,
- pgoff_t start, unsigned nr_entries,
- pgoff_t *indices);
void pagevec_remove_exceptionals(struct pagevec *pvec);
unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
pgoff_t start, unsigned nr_pages);
@@ -69,4 +65,67 @@ static inline void pagevec_release(struct pagevec *pvec)
__pagevec_release(pvec);
}
+struct pagecache_iter {
+ unsigned nr;
+ unsigned idx;
+ pgoff_t index;
+ struct page *pages[PAGEVEC_SIZE];
+ pgoff_t indices[PAGEVEC_SIZE];
+};
+
+static inline void pagecache_iter_init(struct pagecache_iter *iter,
+ pgoff_t start)
+{
+ iter->nr = 0;
+ iter->idx = 0;
+ iter->index = start;
+}
+
+void __pagecache_iter_release(struct pagecache_iter *iter);
+
+/**
+ * pagecache_iter_release - release cached pages from pagacache_iter
+ *
+ * Must be called if breaking out of for_each_pagecache_page() etc. early - not
+ * needed if pagecache_iter_next() returned NULL and loop terminated normally
+ */
+static inline void pagecache_iter_release(struct pagecache_iter *iter)
+{
+ if (iter->nr)
+ __pagecache_iter_release(iter);
+}
+
+struct page *pagecache_iter_next(struct pagecache_iter *iter,
+ struct address_space *mapping,
+ pgoff_t end, pgoff_t *index,
+ unsigned flags);
+
+#define __pagecache_iter_for_each(_iter, _mapping, _start, _end, \
+ _page, _index, _flags) \
+ for (pagecache_iter_init((_iter), (_start)); \
+ ((_page) = pagecache_iter_next((_iter), (_mapping), \
+ (_end), (_index), (_flags)));)
+
+#define for_each_pagecache_page(_iter, _mapping, _start, _end, _page) \
+ __pagecache_iter_for_each((_iter), (_mapping), (_start), (_end),\
+ (_page), NULL, 0)
+
+#define for_each_pagecache_page_contig(_iter, _mapping, _start, _end, _page)\
+ __pagecache_iter_for_each((_iter), (_mapping), (_start), (_end),\
+ (_page), NULL, RADIX_TREE_ITER_CONTIG)
+
+#define for_each_pagecache_tag(_iter, _mapping, _tag, _start, _end, _page)\
+ __pagecache_iter_for_each((_iter), (_mapping), (_start), (_end),\
+ (_page), NULL, RADIX_TREE_ITER_TAGGED|(_tag))
+
+#define for_each_pagecache_entry(_iter, _mapping, _start, _end, _page, _index)\
+ __pagecache_iter_for_each((_iter), (_mapping), (_start), (_end),\
+ (_page), &(_index), RADIX_TREE_ITER_EXCEPTIONAL)
+
+#define for_each_pagecache_entry_tag(_iter, _mapping, _tag, \
+ _start, _end, _page, _index) \
+ __pagecache_iter_for_each((_iter), (_mapping), (_start), (_end),\
+ (_page), &(_index), RADIX_TREE_ITER_EXCEPTIONAL|\
+ RADIX_TREE_ITER_TAGGED|(_tag))
+
#endif /* _LINUX_PAGEVEC_H */
diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h
index 3e5735064b71..4277f960e79c 100644
--- a/include/linux/radix-tree.h
+++ b/include/linux/radix-tree.h
@@ -364,6 +364,8 @@ enum {
RADIX_TREE_ITER_TAG_MASK = 0x0f, /* tag index in lower nybble */
RADIX_TREE_ITER_TAGGED = 0x10, /* lookup tagged slots */
RADIX_TREE_ITER_CONTIG = 0x20, /* stop at first hole */
+ RADIX_TREE_ITER_EXCEPTIONAL = 0x40 /* include exceptional entries */
+ /* used by __find_get_pages() */
};
/**
@@ -564,6 +566,11 @@ static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot,
return slot;
}
+#define __radix_tree_for_each_slot(slot, root, iter, start, flags) \
+ for (slot = radix_tree_iter_init(iter, start) ; \
+ slot || (slot = radix_tree_next_chunk(root, iter, flags)); \
+ slot = radix_tree_next_slot(slot, iter, flags))
+
/**
* radix_tree_for_each_slot - iterate over non-empty slots
*
@@ -575,9 +582,7 @@ static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot,
* @slot points to radix tree slot, @iter->index contains its index.
*/
#define radix_tree_for_each_slot(slot, root, iter, start) \
- for (slot = radix_tree_iter_init(iter, start) ; \
- slot || (slot = radix_tree_next_chunk(root, iter, 0)) ; \
- slot = radix_tree_next_slot(slot, iter, 0))
+ __radix_tree_for_each_slot(slot, root, iter, start, 0)
/**
* radix_tree_for_each_contig - iterate over contiguous slots
@@ -590,11 +595,8 @@ static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot,
* @slot points to radix tree slot, @iter->index contains its index.
*/
#define radix_tree_for_each_contig(slot, root, iter, start) \
- for (slot = radix_tree_iter_init(iter, start) ; \
- slot || (slot = radix_tree_next_chunk(root, iter, \
- RADIX_TREE_ITER_CONTIG)) ; \
- slot = radix_tree_next_slot(slot, iter, \
- RADIX_TREE_ITER_CONTIG))
+ __radix_tree_for_each_slot(slot, root, iter, start, \
+ RADIX_TREE_ITER_CONTIG)
/**
* radix_tree_for_each_tagged - iterate over tagged slots
@@ -608,10 +610,7 @@ static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot,
* @slot points to radix tree slot, @iter->index contains its index.
*/
#define radix_tree_for_each_tagged(slot, root, iter, start, tag) \
- for (slot = radix_tree_iter_init(iter, start) ; \
- slot || (slot = radix_tree_next_chunk(root, iter, \
- RADIX_TREE_ITER_TAGGED | tag)) ; \
- slot = radix_tree_next_slot(slot, iter, \
- RADIX_TREE_ITER_TAGGED | tag))
+ __radix_tree_for_each_slot(slot, root, iter, start, \
+ RADIX_TREE_ITER_TAGGED|tag)
#endif /* _LINUX_RADIX_TREE_H */
diff --git a/mm/filemap.c b/mm/filemap.c
index b36e1ca1334d..33b4e359a7ea 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -1561,6 +1561,108 @@ no_page:
EXPORT_SYMBOL(pagecache_get_page);
/**
+ * __find_get_pages - gang pagecache lookup, internal mechanism
+ * @mapping: The address_space to search
+ * @start: The starting page cache index
+ * @end: Page cache index to stop at (inclusive)
+ * @nr_entries: The maximum number of entries
+ * @entries: Where the resulting entries are placed
+ * @indices: If non NULL, indices of corresponding entries placed here
+ * @flags: radix tree iter flags and tag (if supplied)
+ *
+ * Don't use directly - see wrappers in pagemap.h
+ *
+ * Possible values for flags (may be used in combination):
+ *
+ * 0: find_get_pages()
+ * RADIX_TREE_ITER_TAGGED|tag: find_get_pages_tag()
+ * RADIX_TREE_ITER_CONTIG: find_get_pages_contig()
+ * RADIX_TREE_ITER_EXCEPTIONAL: find_get_entries()
+ */
+unsigned __find_get_pages(struct address_space *mapping,
+ pgoff_t start, pgoff_t end,
+ unsigned nr_entries, struct page **entries,
+ pgoff_t *indices, unsigned flags)
+{
+ struct radix_tree_iter iter;
+ void **slot;
+ unsigned ret = 0;
+
+ if (unlikely(!nr_entries || start > end))
+ return 0;
+
+ rcu_read_lock();
+ __radix_tree_for_each_slot(slot, &mapping->page_tree,
+ &iter, start, flags) {
+ struct page *head, *page;
+
+ if (iter.index > end)
+ break;
+repeat:
+ page = radix_tree_deref_slot(slot);
+ if (unlikely(!page))
+ goto no_entry;
+
+ if (radix_tree_exception(page)) {
+ if (radix_tree_deref_retry(page)) {
+ slot = radix_tree_iter_retry(&iter);
+ continue;
+ }
+
+ /*
+ * A shadow entry of a recently evicted page,
+ * or a swap entry from shmem/tmpfs. Skip
+ * over it.
+ */
+ if (flags & RADIX_TREE_ITER_EXCEPTIONAL)
+ goto export;
+
+ goto no_entry;
+ }
+
+ head = compound_head(page);
+ if (!page_cache_get_speculative(head))
+ goto repeat;
+
+ /* The page was split under us? */
+ if (compound_head(page) != head) {
+ put_page(head);
+ goto repeat;
+ }
+
+ /* Has the page moved? */
+ if (unlikely(page != *slot)) {
+ put_page(head);
+ goto repeat;
+ }
+
+ /*
+ * must check mapping and index after taking the ref.
+ * otherwise we can get both false positives and false
+ * negatives, which is just confusing to the caller.
+ */
+ if ((flags & RADIX_TREE_ITER_CONTIG) &&
+ (page->mapping == NULL || page_to_pgoff(page) != iter.index)) {
+ put_page(page);
+ break;
+ }
+export:
+ if (indices)
+ indices[ret] = iter.index;
+ entries[ret] = page;
+ if (++ret == nr_entries)
+ break;
+ continue;
+no_entry:
+ if (flags & RADIX_TREE_ITER_CONTIG)
+ break;
+ }
+ rcu_read_unlock();
+ return ret;
+}
+EXPORT_SYMBOL(__find_get_pages);
+
+/**
* find_get_entries - gang pagecache lookup
* @mapping: The address_space to search
* @start: The starting page cache index
@@ -1941,6 +2043,51 @@ export:
}
EXPORT_SYMBOL(find_get_entries_tag);
+void __pagecache_iter_release(struct pagecache_iter *iter)
+{
+ lru_add_drain();
+ release_pages(iter->pages, iter->nr, 0);
+ iter->nr = 0;
+ iter->idx = 0;
+}
+EXPORT_SYMBOL(__pagecache_iter_release);
+
+/**
+ * pagecache_iter_next - get next page from pagecache iterator and advance
+ * iterator
+ * @iter: The iterator to advance
+ * @mapping: The address_space to search
+ * @end: Page cache index to stop at (inclusive)
+ * @index: if non NULL, index of page or entry will be returned here
+ * @flags: radix tree iter flags and tag for __find_get_pages()
+ */
+struct page *pagecache_iter_next(struct pagecache_iter *iter,
+ struct address_space *mapping,
+ pgoff_t end, pgoff_t *index,
+ unsigned flags)
+{
+ struct page *page;
+
+ if (iter->idx >= iter->nr) {
+ pagecache_iter_release(iter);
+ cond_resched();
+
+ iter->nr = __find_get_pages(mapping, iter->index, end,
+ PAGEVEC_SIZE, iter->pages,
+ iter->indices, flags);
+ if (!iter->nr)
+ return NULL;
+ }
+
+ iter->index = iter->indices[iter->idx] + 1;
+ if (index)
+ *index = iter->indices[iter->idx];
+ page = iter->pages[iter->idx];
+ iter->idx++;
+ return page;
+}
+EXPORT_SYMBOL(pagecache_iter_next);
+
/*
* CD/DVDs are error prone. When a medium error occurs, the driver may fail
* a _large_ part of the i/o request. Imagine the worst scenario: