summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--fs/fscache/cookie.c294
-rw-r--r--fs/fscache/internal.h7
-rw-r--r--fs/fscache/netfs.c81
-rw-r--r--include/linux/fscache.h8
-rw-r--r--include/trace/events/fscache.h8
5 files changed, 279 insertions, 119 deletions
diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c
index 8ca9a932d225..7dc55b93a830 100644
--- a/fs/fscache/cookie.c
+++ b/fs/fscache/cookie.c
@@ -21,6 +21,9 @@ struct kmem_cache *fscache_cookie_jar;
static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
+#define fscache_cookie_hash_shift 15
+static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
+
static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
loff_t object_size);
static int fscache_alloc_object(struct fscache_cache *cache,
@@ -28,6 +31,44 @@ static int fscache_alloc_object(struct fscache_cache *cache,
static int fscache_attach_object(struct fscache_cookie *cookie,
struct fscache_object *object);
+static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
+{
+ struct hlist_node *object;
+ const u8 *k;
+ unsigned loop;
+
+ pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n",
+ prefix, cookie, cookie->parent, cookie->flags,
+ atomic_read(&cookie->n_children),
+ atomic_read(&cookie->n_active));
+ pr_err("%c-cookie d=%p n=%p\n",
+ prefix, cookie->def, cookie->netfs_data);
+
+ object = READ_ONCE(cookie->backing_objects.first);
+ if (object)
+ pr_err("%c-cookie o=%p\n",
+ prefix, hlist_entry(object, struct fscache_object, cookie_link));
+
+ pr_err("%c-key=[%u] '", prefix, cookie->key_len);
+ k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
+ cookie->inline_key : cookie->key;
+ for (loop = 0; loop < cookie->key_len; loop++)
+ pr_cont("%02x", k[loop]);
+ pr_cont("'\n");
+}
+
+void fscache_free_cookie(struct fscache_cookie *cookie)
+{
+ if (cookie) {
+ BUG_ON(!hlist_empty(&cookie->backing_objects));
+ if (cookie->aux_len > sizeof(cookie->inline_aux))
+ kfree(cookie->aux);
+ if (cookie->key_len > sizeof(cookie->inline_key))
+ kfree(cookie->key);
+ kmem_cache_free(fscache_cookie_jar, cookie);
+ }
+}
+
/*
* initialise an cookie jar slab element prior to any use
*/
@@ -42,6 +83,170 @@ void fscache_cookie_init_once(void *_cookie)
}
/*
+ * Set the index key in a cookie. The cookie struct has space for a 12-byte
+ * key plus length and hash, but if that's not big enough, it's instead a
+ * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
+ * the key data.
+ */
+static int fscache_set_key(struct fscache_cookie *cookie,
+ const void *index_key, size_t index_key_len)
+{
+ unsigned long long h;
+ u32 *buf;
+ int i;
+
+ cookie->key_len = index_key_len;
+
+ if (index_key_len > sizeof(cookie->inline_key)) {
+ buf = kzalloc(index_key_len, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+ cookie->key = buf;
+ } else {
+ buf = (u32 *)cookie->inline_key;
+ buf[0] = 0;
+ buf[1] = 0;
+ buf[2] = 0;
+ }
+
+ memcpy(buf, index_key, index_key_len);
+
+ /* Calculate a hash and combine this with the length in the first word
+ * or first half word
+ */
+ h = (unsigned long)cookie->parent;
+ h += index_key_len + cookie->type;
+ for (i = 0; i < (index_key_len + sizeof(u32) - 1) / sizeof(u32); i++)
+ h += buf[i];
+
+ cookie->key_hash = h ^ (h >> 32);
+ return 0;
+}
+
+static long fscache_compare_cookie(const struct fscache_cookie *a,
+ const struct fscache_cookie *b)
+{
+ const void *ka, *kb;
+
+ if (a->key_hash != b->key_hash)
+ return (long)a->key_hash - (long)b->key_hash;
+ if (a->parent != b->parent)
+ return (long)a->parent - (long)b->parent;
+ if (a->key_len != b->key_len)
+ return (long)a->key_len - (long)b->key_len;
+ if (a->type != b->type)
+ return (long)a->type - (long)b->type;
+
+ if (a->key_len <= sizeof(a->inline_key)) {
+ ka = &a->inline_key;
+ kb = &b->inline_key;
+ } else {
+ ka = a->key;
+ kb = b->key;
+ }
+ return memcmp(ka, kb, a->key_len);
+}
+
+/*
+ * Allocate a cookie.
+ */
+struct fscache_cookie *fscache_alloc_cookie(
+ struct fscache_cookie *parent,
+ const struct fscache_cookie_def *def,
+ const void *index_key, size_t index_key_len,
+ const void *aux_data, size_t aux_data_len,
+ void *netfs_data,
+ loff_t object_size)
+{
+ struct fscache_cookie *cookie;
+
+ /* allocate and initialise a cookie */
+ cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL);
+ if (!cookie)
+ return NULL;
+
+ cookie->key_len = index_key_len;
+ cookie->aux_len = aux_data_len;
+
+ if (fscache_set_key(cookie, index_key, index_key_len) < 0)
+ goto nomem;
+
+ if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
+ memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
+ } else {
+ cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
+ if (!cookie->aux)
+ goto nomem;
+ }
+
+ atomic_set(&cookie->usage, 1);
+ atomic_set(&cookie->n_children, 0);
+
+ /* We keep the active count elevated until relinquishment to prevent an
+ * attempt to wake up every time the object operations queue quiesces.
+ */
+ atomic_set(&cookie->n_active, 1);
+
+ cookie->def = def;
+ cookie->parent = parent;
+ cookie->netfs_data = netfs_data;
+ cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET);
+ cookie->type = def->type;
+
+ /* radix tree insertion won't use the preallocation pool unless it's
+ * told it may not wait */
+ INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
+ return cookie;
+
+nomem:
+ fscache_free_cookie(cookie);
+ return NULL;
+}
+
+/*
+ * Attempt to insert the new cookie into the hash. If there's a collision, we
+ * return the old cookie if it's not in use and an error otherwise.
+ */
+struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
+{
+ struct fscache_cookie *cursor;
+ struct hlist_bl_head *h;
+ struct hlist_bl_node *p;
+ unsigned int bucket;
+
+ bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
+ h = &fscache_cookie_hash[bucket];
+
+ hlist_bl_lock(h);
+ hlist_bl_for_each_entry(cursor, p, h, hash_link) {
+ if (fscache_compare_cookie(candidate, cursor) == 0)
+ goto collision;
+ }
+
+ __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
+ fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
+ atomic_inc(&candidate->parent->n_children);
+ hlist_bl_add_head(&candidate->hash_link, h);
+ hlist_bl_unlock(h);
+ return candidate;
+
+collision:
+ if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
+ trace_fscache_cookie(cursor, fscache_cookie_collision,
+ atomic_read(&cursor->usage));
+ pr_err("Duplicate cookie detected\n");
+ fscache_print_cookie(cursor, 'O');
+ fscache_print_cookie(candidate, 'N');
+ hlist_bl_unlock(h);
+ return NULL;
+ }
+
+ fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
+ hlist_bl_unlock(h);
+ return cursor;
+}
+
+/*
* request a cookie to represent an object (index, datafile, xattr, etc)
* - parent specifies the parent object
* - the top level index cookie for each netfs is stored in the fscache_netfs
@@ -65,7 +270,7 @@ struct fscache_cookie *__fscache_acquire_cookie(
loff_t object_size,
bool enable)
{
- struct fscache_cookie *cookie;
+ struct fscache_cookie *candidate, *cookie;
BUG_ON(!def);
@@ -95,53 +300,24 @@ struct fscache_cookie *__fscache_acquire_cookie(
BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
parent->type != FSCACHE_COOKIE_TYPE_INDEX);
- /* allocate and initialise a cookie */
- cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL);
- if (!cookie) {
+ candidate = fscache_alloc_cookie(parent, def,
+ index_key, index_key_len,
+ aux_data, aux_data_len,
+ netfs_data, object_size);
+ if (!candidate) {
fscache_stat(&fscache_n_acquires_oom);
_leave(" [ENOMEM]");
return NULL;
}
- cookie->key_len = index_key_len;
- cookie->aux_len = aux_data_len;
-
- if (cookie->key_len <= sizeof(cookie->inline_key)) {
- memcpy(cookie->inline_key, index_key, cookie->key_len);
- } else {
- cookie->key = kmemdup(index_key, cookie->key_len, GFP_KERNEL);
- if (!cookie->key)
- goto nomem;
- }
-
- if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
- memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
- } else {
- cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
- if (!cookie->aux)
- goto nomem;
+ cookie = fscache_hash_cookie(candidate);
+ if (!cookie) {
+ trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
+ goto out;
}
- atomic_set(&cookie->usage, 1);
- atomic_set(&cookie->n_children, 0);
-
- /* We keep the active count elevated until relinquishment to prevent an
- * attempt to wake up every time the object operations queue quiesces.
- */
- atomic_set(&cookie->n_active, 1);
-
- fscache_cookie_get(parent, fscache_cookie_get_acquire_parent);
- atomic_inc(&parent->n_children);
-
- cookie->def = def;
- cookie->parent = parent;
- cookie->netfs_data = netfs_data;
- cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET);
- cookie->type = def->type;
-
- /* radix tree insertion won't use the preallocation pool unless it's
- * told it may not wait */
- INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
+ if (cookie == candidate)
+ candidate = NULL;
switch (cookie->type) {
case FSCACHE_COOKIE_TYPE_INDEX:
@@ -178,16 +354,10 @@ struct fscache_cookie *__fscache_acquire_cookie(
}
fscache_stat(&fscache_n_acquires_ok);
- _leave(" = %p", cookie);
- return cookie;
-nomem:
- if (cookie->aux_len > sizeof(cookie->inline_aux))
- kfree(cookie->aux);
- if (cookie->key_len > sizeof(cookie->inline_key))
- kfree(cookie->key);
- kmem_cache_free(fscache_cookie_jar, cookie);
- return NULL;
+out:
+ fscache_free_cookie(candidate);
+ return cookie;
}
EXPORT_SYMBOL(__fscache_acquire_cookie);
@@ -679,6 +849,22 @@ void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
EXPORT_SYMBOL(__fscache_relinquish_cookie);
/*
+ * Remove a cookie from the hash table.
+ */
+static void fscache_unhash_cookie(struct fscache_cookie *cookie)
+{
+ struct hlist_bl_head *h;
+ unsigned int bucket;
+
+ bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
+ h = &fscache_cookie_hash[bucket];
+
+ hlist_bl_lock(h);
+ hlist_bl_del(&cookie->hash_link);
+ hlist_bl_unlock(h);
+}
+
+/*
* Drop a reference to a cookie.
*/
void fscache_cookie_put(struct fscache_cookie *cookie,
@@ -698,12 +884,8 @@ void fscache_cookie_put(struct fscache_cookie *cookie,
BUG_ON(usage < 0);
parent = cookie->parent;
- BUG_ON(!hlist_empty(&cookie->backing_objects));
- if (cookie->aux_len > sizeof(cookie->inline_aux))
- kfree(cookie->aux);
- if (cookie->key_len > sizeof(cookie->inline_key))
- kfree(cookie->key);
- kmem_cache_free(fscache_cookie_jar, cookie);
+ fscache_unhash_cookie(cookie);
+ fscache_free_cookie(cookie);
cookie = parent;
where = fscache_cookie_put_parent;
diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h
index 5f905a499306..500650f938fe 100644
--- a/fs/fscache/internal.h
+++ b/fs/fscache/internal.h
@@ -49,7 +49,14 @@ extern struct fscache_cache *fscache_select_cache_for_object(
*/
extern struct kmem_cache *fscache_cookie_jar;
+extern void fscache_free_cookie(struct fscache_cookie *);
extern void fscache_cookie_init_once(void *);
+extern struct fscache_cookie *fscache_alloc_cookie(struct fscache_cookie *,
+ const struct fscache_cookie_def *,
+ const void *, size_t,
+ const void *, size_t,
+ void *, loff_t);
+extern struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *);
extern void fscache_cookie_put(struct fscache_cookie *,
enum fscache_cookie_trace);
diff --git a/fs/fscache/netfs.c b/fs/fscache/netfs.c
index a5998dfab7e7..c2f605483cc5 100644
--- a/fs/fscache/netfs.c
+++ b/fs/fscache/netfs.c
@@ -14,85 +14,51 @@
#include <linux/slab.h>
#include "internal.h"
-static LIST_HEAD(fscache_netfs_list);
-
/*
* register a network filesystem for caching
*/
int __fscache_register_netfs(struct fscache_netfs *netfs)
{
- struct fscache_netfs *ptr;
- struct fscache_cookie *cookie;
- int ret;
+ struct fscache_cookie *candidate, *cookie;
_enter("{%s}", netfs->name);
- INIT_LIST_HEAD(&netfs->link);
-
/* allocate a cookie for the primary index */
- cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
-
- if (!cookie) {
+ candidate = fscache_alloc_cookie(&fscache_fsdef_index,
+ &fscache_fsdef_netfs_def,
+ netfs->name, strlen(netfs->name),
+ &netfs->version, sizeof(netfs->version),
+ netfs, 0);
+ if (!candidate) {
_leave(" = -ENOMEM");
return -ENOMEM;
}
- cookie->key_len = strlen(netfs->name);
- if (cookie->key_len <= sizeof(cookie->inline_key)) {
- memcpy(cookie->inline_key, netfs->name, strlen(netfs->name));
- } else {
- ret = -ENOMEM;
- cookie->key = kmemdup(netfs->name, cookie->key_len, GFP_KERNEL);
- if (!cookie->key)
- goto nomem;
- }
-
- cookie->aux_len = sizeof(netfs->version);
- memcpy(cookie->inline_aux, &netfs->version, cookie->aux_len);
-
- /* initialise the primary index cookie */
- atomic_set(&cookie->usage, 1);
- atomic_set(&cookie->n_children, 0);
- atomic_set(&cookie->n_active, 1);
-
- cookie->def = &fscache_fsdef_netfs_def;
- cookie->parent = &fscache_fsdef_index;
- cookie->netfs_data = netfs;
- cookie->flags = 1 << FSCACHE_COOKIE_ENABLED;
- cookie->type = FSCACHE_COOKIE_TYPE_INDEX;
-
- spin_lock_init(&cookie->lock);
- spin_lock_init(&cookie->stores_lock);
- INIT_HLIST_HEAD(&cookie->backing_objects);
+ candidate->flags = 1 << FSCACHE_COOKIE_ENABLED;
/* check the netfs type is not already present */
- down_write(&fscache_addremove_sem);
-
- ret = -EEXIST;
- list_for_each_entry(ptr, &fscache_netfs_list, link) {
- if (strcmp(ptr->name, netfs->name) == 0)
- goto already_registered;
+ cookie = fscache_hash_cookie(candidate);
+ if (!cookie)
+ goto already_registered;
+ if (cookie != candidate) {
+ trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
+ fscache_free_cookie(candidate);
}
fscache_cookie_get(cookie->parent, fscache_cookie_get_register_netfs);
atomic_inc(&cookie->parent->n_children);
netfs->primary_index = cookie;
- list_add(&netfs->link, &fscache_netfs_list);
- ret = 0;
pr_notice("Netfs '%s' registered for caching\n", netfs->name);
trace_fscache_netfs(netfs);
+ _leave(" = 0");
+ return 0;
already_registered:
- up_write(&fscache_addremove_sem);
-
-nomem:
- if (ret < 0)
- kmem_cache_free(fscache_cookie_jar, cookie);
-
- _leave(" = %d", ret);
- return ret;
+ fscache_cookie_put(candidate, fscache_cookie_put_dup_netfs);
+ _leave(" = -EEXIST");
+ return -EEXIST;
}
EXPORT_SYMBOL(__fscache_register_netfs);
@@ -104,15 +70,8 @@ void __fscache_unregister_netfs(struct fscache_netfs *netfs)
{
_enter("{%s.%u}", netfs->name, netfs->version);
- down_write(&fscache_addremove_sem);
-
- list_del(&netfs->link);
fscache_relinquish_cookie(netfs->primary_index, NULL, false);
-
- up_write(&fscache_addremove_sem);
-
- pr_notice("Netfs '%s' unregistered from caching\n",
- netfs->name);
+ pr_notice("Netfs '%s' unregistered from caching\n", netfs->name);
_leave("");
}
diff --git a/include/linux/fscache.h b/include/linux/fscache.h
index eb38f39cf832..84b90a79d75a 100644
--- a/include/linux/fscache.h
+++ b/include/linux/fscache.h
@@ -22,6 +22,7 @@
#include <linux/list.h>
#include <linux/pagemap.h>
#include <linux/pagevec.h>
+#include <linux/list_bl.h>
#if defined(CONFIG_FSCACHE) || defined(CONFIG_FSCACHE_MODULE)
#define fscache_available() (1)
@@ -124,7 +125,6 @@ struct fscache_netfs {
uint32_t version; /* indexing version */
const char *name; /* filesystem name */
struct fscache_cookie *primary_index;
- struct list_head link; /* internal link */
};
/*
@@ -143,6 +143,7 @@ struct fscache_cookie {
struct hlist_head backing_objects; /* object(s) backing this file/index */
const struct fscache_cookie_def *def; /* definition */
struct fscache_cookie *parent; /* parent of this entry */
+ struct hlist_bl_node hash_link; /* Link in hash table */
void *netfs_data; /* back pointer to netfs */
struct radix_tree_root stores; /* pages to be stored on this cookie */
#define FSCACHE_COOKIE_PENDING_TAG 0 /* pages tag: pending write to cache */
@@ -156,11 +157,14 @@ struct fscache_cookie {
#define FSCACHE_COOKIE_RELINQUISHED 4 /* T if cookie has been relinquished */
#define FSCACHE_COOKIE_ENABLED 5 /* T if cookie is enabled */
#define FSCACHE_COOKIE_ENABLEMENT_LOCK 6 /* T if cookie is being en/disabled */
-#define FSCACHE_COOKIE_AUX_UPDATED 7 /* T if the auxiliary data was updated */
+#define FSCACHE_COOKIE_AUX_UPDATED 8 /* T if the auxiliary data was updated */
+#define FSCACHE_COOKIE_ACQUIRED 9 /* T if cookie is in use */
+#define FSCACHE_COOKIE_RELINQUISHING 10 /* T if cookie is being relinquished */
u8 type; /* Type of object */
u8 key_len; /* Length of index key */
u8 aux_len; /* Length of auxiliary data */
+ u32 key_hash; /* Hash of parent, type, key, len */
union {
void *key; /* Index key */
u8 inline_key[16]; /* - If the key is short enough */
diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h
index 82c060fe6635..686cfe997ed2 100644
--- a/include/trace/events/fscache.h
+++ b/include/trace/events/fscache.h
@@ -24,10 +24,14 @@
#define __FSCACHE_DECLARE_TRACE_ENUMS_ONCE_ONLY
enum fscache_cookie_trace {
+ fscache_cookie_collision,
+ fscache_cookie_discard,
fscache_cookie_get_acquire_parent,
fscache_cookie_get_attach_object,
+ fscache_cookie_get_reacquire,
fscache_cookie_get_register_netfs,
fscache_cookie_put_acquire_nobufs,
+ fscache_cookie_put_dup_netfs,
fscache_cookie_put_relinquish,
fscache_cookie_put_object,
fscache_cookie_put_parent,
@@ -86,10 +90,14 @@ enum fscache_page_op_trace {
* Declare tracing information enums and their string mappings for display.
*/
#define fscache_cookie_traces \
+ EM(fscache_cookie_collision, "*COLLISION*") \
+ EM(fscache_cookie_discard, "DISCARD") \
EM(fscache_cookie_get_acquire_parent, "GET prn") \
EM(fscache_cookie_get_attach_object, "GET obj") \
+ EM(fscache_cookie_get_reacquire, "GET raq") \
EM(fscache_cookie_get_register_netfs, "GET net") \
EM(fscache_cookie_put_acquire_nobufs, "PUT nbf") \
+ EM(fscache_cookie_put_dup_netfs, "PUT dnt") \
EM(fscache_cookie_put_relinquish, "PUT rlq") \
EM(fscache_cookie_put_object, "PUT obj") \
E_(fscache_cookie_put_parent, "PUT prn")