X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=fs%2Fdcache.c;h=f938ad50ffc6178e66d25a693b0617fdfd341e3d;hb=c7b5ebbddf7bcd3651947760f423e3783bbe6573;hp=613ff66dacde2d1b7ed339da5b3b1fa46b59b871;hpb=9213980e6a70d8473e0ffd4b39ab5b6caaba9ff5;p=linux-2.6.git diff --git a/fs/dcache.c b/fs/dcache.c index 613ff66da..f938ad50f 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -30,10 +30,12 @@ #include #include #include +#include -#define DCACHE_PARANOIA 1 /* #define DCACHE_DEBUG 1 */ +int sysctl_vfs_cache_pressure = 100; + spinlock_t dcache_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED; seqlock_t rename_lock __cacheline_aligned_in_smp = SEQLOCK_UNLOCKED; @@ -64,9 +66,9 @@ struct dentry_stat_t dentry_stat = { .age_limit = 45, }; -static void d_callback(void *arg) +static void d_callback(struct rcu_head *head) { - struct dentry * dentry = (struct dentry *)arg; + struct dentry * dentry = container_of(head, struct dentry, d_rcu); if (dname_external(dentry)) kfree(dentry->d_name.name); @@ -81,7 +83,7 @@ static void d_free(struct dentry *dentry) { if (dentry->d_op && dentry->d_op->d_release) dentry->d_op->d_release(dentry); - call_rcu(&dentry->d_rcu, d_callback, dentry); + call_rcu(&dentry->d_rcu, d_callback); } /* @@ -142,6 +144,8 @@ void dput(struct dentry *dentry) return; repeat: + if (atomic_read(&dentry->d_count) == 1) + might_sleep(); if (!atomic_dec_and_lock(&dentry->d_count, &dcache_lock)) return; @@ -284,12 +288,11 @@ struct dentry * dget_locked(struct dentry *dentry) * any other hashed alias over that one. */ -struct dentry * d_find_alias(struct inode *inode) +static struct dentry * __d_find_alias(struct inode *inode, int want_discon) { struct list_head *head, *next, *tmp; struct dentry *alias, *discon_alias=NULL; - spin_lock(&dcache_lock); head = &inode->i_dentry; next = inode->i_dentry.next; while (next != head) { @@ -300,19 +303,26 @@ struct dentry * d_find_alias(struct inode *inode) if (!d_unhashed(alias)) { if (alias->d_flags & DCACHE_DISCONNECTED) discon_alias = alias; - else { + else if (!want_discon) { __dget_locked(alias); - spin_unlock(&dcache_lock); return alias; } } } if (discon_alias) __dget_locked(discon_alias); - spin_unlock(&dcache_lock); return discon_alias; } +struct dentry * d_find_alias(struct inode *inode) +{ + struct dentry *de; + spin_lock(&dcache_lock); + de = __d_find_alias(inode, 0); + spin_unlock(&dcache_lock); + return de; +} + /* * Try to kill dentries associated with this inode. * WARNING: you must own a reference to inode. @@ -611,7 +621,7 @@ void shrink_dcache_parent(struct dentry * parent) * * Prune the dentries that are anonymous * - * parsing d_hash list does not read_barrier_depends() as it + * parsing d_hash list does not hlist_for_each_rcu() as it * done under dcache_lock. * */ @@ -626,7 +636,7 @@ void shrink_dcache_anon(struct hlist_head *head) struct dentry *this = hlist_entry(lp, struct dentry, d_hash); if (!list_empty(&this->d_lru)) { dentry_stat.nr_unused--; - list_del(&this->d_lru); + list_del_init(&this->d_lru); } /* @@ -663,7 +673,7 @@ static int shrink_dcache_memory(int nr, unsigned int gfp_mask) return -1; prune_dcache(nr); } - return dentry_stat.nr_unused; + return (dentry_stat.nr_unused / 100) * sysctl_vfs_cache_pressure; } /** @@ -831,33 +841,27 @@ struct dentry * d_alloc_anon(struct inode *inode) tmp->d_parent = tmp; /* make sure dput doesn't croak */ spin_lock(&dcache_lock); - if (S_ISDIR(inode->i_mode) && !list_empty(&inode->i_dentry)) { - /* A directory can only have one dentry. - * This (now) has one, so use it. - */ - res = list_entry(inode->i_dentry.next, struct dentry, d_alias); - __dget_locked(res); - } else { + res = __d_find_alias(inode, 0); + if (!res) { /* attach a disconnected dentry */ res = tmp; tmp = NULL; - if (res) { - spin_lock(&res->d_lock); - res->d_sb = inode->i_sb; - res->d_parent = res; - res->d_inode = inode; + spin_lock(&res->d_lock); + res->d_sb = inode->i_sb; + res->d_parent = res; + res->d_inode = inode; + + /* + * Set d_bucket to an "impossible" bucket address so + * that d_move() doesn't get a false positive + */ + res->d_bucket = NULL; + res->d_flags |= DCACHE_DISCONNECTED; + res->d_flags &= ~DCACHE_UNHASHED; + list_add(&res->d_alias, &inode->i_dentry); + hlist_add_head(&res->d_hash, &inode->i_sb->s_anon); + spin_unlock(&res->d_lock); - /* - * Set d_bucket to an "impossible" bucket address so - * that d_move() doesn't get a false positive - */ - res->d_bucket = NULL; - res->d_flags |= DCACHE_DISCONNECTED; - res->d_flags &= ~DCACHE_UNHASHED; - list_add(&res->d_alias, &inode->i_dentry); - hlist_add_head(&res->d_hash, &inode->i_sb->s_anon); - spin_unlock(&res->d_lock); - } inode = NULL; /* don't drop reference */ } spin_unlock(&dcache_lock); @@ -879,7 +883,7 @@ struct dentry * d_alloc_anon(struct inode *inode) * DCACHE_DISCONNECTED), then d_move that in place of the given dentry * and return it, else simply d_add the inode to the dentry and return NULL. * - * This is (will be) needed in the lookup routine of any filesystem that is exportable + * This is needed in the lookup routine of any filesystem that is exportable * (via knfsd) so that we can build dcache paths to directories effectively. * * If a dentry was found and moved, then it is returned. Otherwise NULL @@ -890,11 +894,11 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) { struct dentry *new = NULL; - if (inode && S_ISDIR(inode->i_mode)) { + if (inode) { spin_lock(&dcache_lock); - if (!list_empty(&inode->i_dentry)) { - new = list_entry(inode->i_dentry.next, struct dentry, d_alias); - __dget_locked(new); + new = __d_find_alias(inode, 1); + if (new) { + BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED)); spin_unlock(&dcache_lock); security_d_instantiate(new, inode); d_rehash(dentry); @@ -968,11 +972,10 @@ struct dentry * __d_lookup(struct dentry * parent, struct qstr * name) rcu_read_lock(); - hlist_for_each (node, head) { + hlist_for_each_rcu(node, head) { struct dentry *dentry; struct qstr *qstr; - smp_read_barrier_depends(); dentry = hlist_entry(node, struct dentry, d_hash); smp_rmb(); @@ -999,8 +1002,7 @@ struct dentry * __d_lookup(struct dentry * parent, struct qstr * name) if (dentry->d_parent != parent) goto next; - qstr = &dentry->d_name; - smp_read_barrier_depends(); + qstr = rcu_dereference(&dentry->d_name); if (parent->d_op && parent->d_op->d_compare) { if (parent->d_op->d_compare(parent, qstr, name)) goto next; @@ -1053,7 +1055,7 @@ int d_validate(struct dentry *dentry, struct dentry *dparent) spin_lock(&dcache_lock); base = d_hash(dparent, dentry->d_name.hash); hlist_for_each(lhp,base) { - /* read_barrier_depends() not required for d_hash list + /* hlist_for_each_rcu() not required for d_hash list * as it is parsed under dcache_lock */ if (dentry == hlist_entry(lhp, struct dentry, d_hash)) { @@ -1561,13 +1563,25 @@ static int __init set_dhash_entries(char *str) } __setup("dhash_entries=", set_dhash_entries); -static void __init dcache_init(unsigned long mempages) +static void __init dcache_init_early(void) { - struct hlist_head *d; - unsigned long order; - unsigned int nr_hash; - int i; + int loop; + + dentry_hashtable = + alloc_large_system_hash("Dentry cache", + sizeof(struct hlist_head), + dhash_entries, + 13, + 0, + &d_hash_shift, + &d_hash_mask); + + for (loop = 0; loop < (1 << d_hash_shift); loop++) + INIT_HLIST_HEAD(&dentry_hashtable[loop]); +} +static void __init dcache_init(unsigned long mempages) +{ /* * A constructor could be added for stable state like the lists, * but it is probably not worth it because of the cache nature @@ -1580,45 +1594,6 @@ static void __init dcache_init(unsigned long mempages) NULL, NULL); set_shrinker(DEFAULT_SEEKS, shrink_dcache_memory); - - if (!dhash_entries) - dhash_entries = PAGE_SHIFT < 13 ? - mempages >> (13 - PAGE_SHIFT) : - mempages << (PAGE_SHIFT - 13); - - dhash_entries *= sizeof(struct hlist_head); - for (order = 0; ((1UL << order) << PAGE_SHIFT) < dhash_entries; order++) - ; - - do { - unsigned long tmp; - - nr_hash = (1UL << order) * PAGE_SIZE / - sizeof(struct hlist_head); - d_hash_mask = (nr_hash - 1); - - tmp = nr_hash; - d_hash_shift = 0; - while ((tmp >>= 1UL) != 0UL) - d_hash_shift++; - - dentry_hashtable = (struct hlist_head *) - __get_free_pages(GFP_ATOMIC, order); - } while (dentry_hashtable == NULL && --order >= 0); - - printk(KERN_INFO "Dentry cache hash table entries: %d (order: %ld, %ld bytes)\n", - nr_hash, order, (PAGE_SIZE << order)); - - if (!dentry_hashtable) - panic("Failed to allocate dcache hash table\n"); - - d = dentry_hashtable; - i = nr_hash; - do { - INIT_HLIST_HEAD(d); - d++; - i--; - } while (i); } /* SLAB cache for __getname() consumers */ @@ -1632,6 +1607,12 @@ EXPORT_SYMBOL(d_genocide); extern void bdev_cache_init(void); extern void chrdev_init(void); +void __init vfs_caches_init_early(void) +{ + dcache_init_early(); + inode_init_early(); +} + void __init vfs_caches_init(unsigned long mempages) { unsigned long reserve;