X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=fs%2Fdcache.c;h=11dc83092d4aa48c78650f6dee4b48b834d25d7e;hb=7dc8dcbeec756d7489be41a49cc92c27333e135a;hp=b90d3813fed7506e6a63419eb7d01dd63a07a3e5;hpb=bc77d24c47b89f1e0efed0b8e4be5f8aad102883;p=linux-2.6.git diff --git a/fs/dcache.c b/fs/dcache.c index b90d3813f..11dc83092 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -15,9 +15,11 @@ */ #include +#include #include #include #include +#include #include #include #include @@ -30,12 +32,15 @@ #include #include #include +#include -#define DCACHE_PARANOIA 1 /* #define DCACHE_DEBUG 1 */ -spinlock_t dcache_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED; -seqlock_t rename_lock __cacheline_aligned_in_smp = SEQLOCK_UNLOCKED; +int sysctl_vfs_cache_pressure = 100; +EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure); + + __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock); +static seqlock_t rename_lock __cacheline_aligned_in_smp = SEQLOCK_UNLOCKED; EXPORT_SYMBOL(dcache_lock); @@ -64,9 +69,9 @@ struct dentry_stat_t dentry_stat = { .age_limit = 45, }; -static void d_callback(void *arg) +static void d_callback(struct rcu_head *head) { - struct dentry * dentry = (struct dentry *)arg; + struct dentry * dentry = container_of(head, struct dentry, d_u.d_rcu); if (dname_external(dentry)) kfree(dentry->d_name.name); @@ -81,11 +86,7 @@ static void d_free(struct dentry *dentry) { if (dentry->d_op && dentry->d_op->d_release) dentry->d_op->d_release(dentry); - if (dentry->d_extra_attributes) { - kfree(dentry->d_extra_attributes); - dentry->d_extra_attributes = NULL; - } - call_rcu(&dentry->d_rcu, d_callback, dentry); + call_rcu(&dentry->d_u.d_rcu, d_callback); } /* @@ -93,7 +94,7 @@ static void d_free(struct dentry *dentry) * d_iput() operation if defined. * Called with dcache_lock and per dentry lock held, drops both. */ -static inline void dentry_iput(struct dentry * dentry) +static void dentry_iput(struct dentry * dentry) { struct inode *inode = dentry->d_inode; if (inode) { @@ -101,6 +102,8 @@ static inline void dentry_iput(struct dentry * dentry) list_del_init(&dentry->d_alias); spin_unlock(&dentry->d_lock); spin_unlock(&dcache_lock); + if (!inode->i_nlink) + fsnotify_inoderemove(inode); if (dentry->d_op && dentry->d_op->d_iput) dentry->d_op->d_iput(dentry, inode); else @@ -146,6 +149,8 @@ void dput(struct dentry *dentry) return; repeat: + if (atomic_read(&dentry->d_count) == 1) + might_sleep(); if (!atomic_dec_and_lock(&dentry->d_count, &dcache_lock)) return; @@ -155,7 +160,7 @@ repeat: spin_unlock(&dcache_lock); return; } - + /* * AV: ->d_delete() is _NOT_ allowed to block now. */ @@ -188,7 +193,7 @@ kill_it: { list_del(&dentry->d_lru); dentry_stat.nr_unused--; } - list_del(&dentry->d_child); + list_del(&dentry->d_u.d_child); dentry_stat.nr_dentry--; /* For d_free, below */ /*drops the locks, at that point nobody can reach this dentry */ dentry_iput(dentry); @@ -278,22 +283,25 @@ struct dentry * dget_locked(struct dentry *dentry) /** * d_find_alias - grab a hashed alias of inode * @inode: inode in question + * @want_discon: flag, used by d_splice_alias, to request + * that only a DISCONNECTED alias be returned. * - * If inode has a hashed alias - acquire the reference to alias and - * return it. Otherwise return NULL. Notice that if inode is a directory - * there can be only one alias and it can be unhashed only if it has - * no children. + * If inode has a hashed alias, or is a directory and has any alias, + * acquire the reference to alias and return it. Otherwise return NULL. + * Notice that if inode is a directory there can be only one alias and + * it can be unhashed only if it has no children, or if it is the root + * of a filesystem. * * If the inode has a DCACHE_DISCONNECTED alias, then prefer - * any other hashed alias over that one. + * any other hashed alias over that one unless @want_discon is set, + * in which case only return a DCACHE_DISCONNECTED alias. */ -struct dentry * d_find_alias(struct inode *inode) +static struct dentry * __d_find_alias(struct inode *inode, int want_discon) { struct list_head *head, *next, *tmp; struct dentry *alias, *discon_alias=NULL; - spin_lock(&dcache_lock); head = &inode->i_dentry; next = inode->i_dentry.next; while (next != head) { @@ -301,41 +309,49 @@ struct dentry * d_find_alias(struct inode *inode) next = tmp->next; prefetch(next); alias = list_entry(tmp, struct dentry, d_alias); - if (!d_unhashed(alias)) { + if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) { if (alias->d_flags & DCACHE_DISCONNECTED) discon_alias = alias; - else { + else if (!want_discon) { __dget_locked(alias); - spin_unlock(&dcache_lock); return alias; } } } if (discon_alias) __dget_locked(discon_alias); - spin_unlock(&dcache_lock); return discon_alias; } +struct dentry * d_find_alias(struct inode *inode) +{ + struct dentry *de; + spin_lock(&dcache_lock); + de = __d_find_alias(inode, 0); + spin_unlock(&dcache_lock); + return de; +} + /* * Try to kill dentries associated with this inode. * WARNING: you must own a reference to inode. */ void d_prune_aliases(struct inode *inode) { - struct list_head *tmp, *head = &inode->i_dentry; + struct dentry *dentry; restart: spin_lock(&dcache_lock); - tmp = head; - while ((tmp = tmp->next) != head) { - struct dentry *dentry = list_entry(tmp, struct dentry, d_alias); + list_for_each_entry(dentry, &inode->i_dentry, d_alias) { + spin_lock(&dentry->d_lock); if (!atomic_read(&dentry->d_count)) { __dget_locked(dentry); __d_drop(dentry); + spin_unlock(&dentry->d_lock); spin_unlock(&dcache_lock); dput(dentry); goto restart; } + spin_unlock(&dentry->d_lock); } spin_unlock(&dcache_lock); } @@ -351,7 +367,7 @@ static inline void prune_one_dentry(struct dentry * dentry) struct dentry * parent; __d_drop(dentry); - list_del(&dentry->d_child); + list_del(&dentry->d_u.d_child); dentry_stat.nr_dentry--; /* For d_free, below */ dentry_iput(dentry); parent = dentry->d_parent; @@ -381,6 +397,8 @@ static void prune_dcache(int count) struct dentry *dentry; struct list_head *tmp; + cond_resched_lock(&dcache_lock); + tmp = dentry_unused.prev; if (tmp == &dentry_unused) break; @@ -444,10 +462,7 @@ void shrink_dcache_sb(struct super_block * sb) * superblock to the most recent end of the unused list. */ spin_lock(&dcache_lock); - next = dentry_unused.next; - while (next != &dentry_unused) { - tmp = next; - next = tmp->next; + list_for_each_safe(tmp, next, &dentry_unused) { dentry = list_entry(tmp, struct dentry, d_lru); if (dentry->d_sb != sb) continue; @@ -459,10 +474,7 @@ void shrink_dcache_sb(struct super_block * sb) * Pass two ... free the dentries for this superblock. */ repeat: - next = dentry_unused.next; - while (next != &dentry_unused) { - tmp = next; - next = tmp->next; + list_for_each_safe(tmp, next, &dentry_unused) { dentry = list_entry(tmp, struct dentry, d_lru); if (dentry->d_sb != sb) continue; @@ -506,7 +518,7 @@ repeat: resume: while (next != &this_parent->d_subdirs) { struct list_head *tmp = next; - struct dentry *dentry = list_entry(tmp, struct dentry, d_child); + struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child); next = tmp->next; /* Have we found a mount point ? */ if (d_mountpoint(dentry)) @@ -520,7 +532,7 @@ resume: * All done at this level ... ascend and resume the search. */ if (this_parent != parent) { - next = this_parent->d_child.next; + next = this_parent->d_u.d_child.next; this_parent = this_parent->d_parent; goto resume; } @@ -537,6 +549,13 @@ positive: * list for prune_dcache(). We descend to the next level * whenever the d_subdirs list is non-empty and continue * searching. + * + * It returns zero iff there are no unused children, + * otherwise it returns the number of children moved to + * the end of the unused list. This may not be the total + * number of unused children, because select_parent can + * drop the lock and return early due to latency + * constraints. */ static int select_parent(struct dentry * parent) { @@ -550,7 +569,7 @@ repeat: resume: while (next != &this_parent->d_subdirs) { struct list_head *tmp = next; - struct dentry *dentry = list_entry(tmp, struct dentry, d_child); + struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child); next = tmp->next; if (!list_empty(&dentry->d_lru)) { @@ -566,6 +585,15 @@ resume: dentry_stat.nr_unused++; found++; } + + /* + * We can return to the caller if we have found some (this + * ensures forward progress). We'll be coming back to find + * the rest. + */ + if (found && need_resched()) + goto out; + /* * Descend a level if the d_subdirs list is non-empty. */ @@ -582,7 +610,7 @@ dentry->d_parent->d_name.name, dentry->d_name.name, found); * All done at this level ... ascend and resume the search. */ if (this_parent != parent) { - next = this_parent->d_child.next; + next = this_parent->d_u.d_child.next; this_parent = this_parent->d_parent; #ifdef DCACHE_DEBUG printk(KERN_DEBUG "select_parent: ascending to %s/%s, found=%d\n", @@ -590,6 +618,7 @@ this_parent->d_parent->d_name.name, this_parent->d_name.name, found); #endif goto resume; } +out: spin_unlock(&dcache_lock); return found; } @@ -615,7 +644,7 @@ void shrink_dcache_parent(struct dentry * parent) * * Prune the dentries that are anonymous * - * parsing d_hash list does not read_barrier_depends() as it + * parsing d_hash list does not hlist_for_each_entry_rcu() as it * done under dcache_lock. * */ @@ -630,7 +659,7 @@ void shrink_dcache_anon(struct hlist_head *head) struct dentry *this = hlist_entry(lp, struct dentry, d_hash); if (!list_empty(&this->d_lru)) { dentry_stat.nr_unused--; - list_del(&this->d_lru); + list_del_init(&this->d_lru); } /* @@ -660,14 +689,14 @@ void shrink_dcache_anon(struct hlist_head *head) * * In this case we return -1 to tell the caller that we baled. */ -static int shrink_dcache_memory(int nr, unsigned int gfp_mask) +static int shrink_dcache_memory(int nr, gfp_t gfp_mask) { if (nr) { if (!(gfp_mask & __GFP_FS)) return -1; prune_dcache(nr); } - return dentry_stat.nr_unused; + return (dentry_stat.nr_unused / 100) * sysctl_vfs_cache_pressure; } /** @@ -707,16 +736,16 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) atomic_set(&dentry->d_count, 1); dentry->d_flags = DCACHE_UNHASHED; - dentry->d_lock = SPIN_LOCK_UNLOCKED; + spin_lock_init(&dentry->d_lock); dentry->d_inode = NULL; dentry->d_parent = NULL; dentry->d_sb = NULL; dentry->d_op = NULL; dentry->d_fsdata = NULL; - dentry->d_extra_attributes = NULL; dentry->d_mounted = 0; +#ifdef CONFIG_PROFILING dentry->d_cookie = NULL; - dentry->d_bucket = NULL; +#endif INIT_HLIST_NODE(&dentry->d_hash); INIT_LIST_HEAD(&dentry->d_lru); INIT_LIST_HEAD(&dentry->d_subdirs); @@ -726,18 +755,28 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) dentry->d_parent = dget(parent); dentry->d_sb = parent->d_sb; } else { - INIT_LIST_HEAD(&dentry->d_child); + INIT_LIST_HEAD(&dentry->d_u.d_child); } spin_lock(&dcache_lock); if (parent) - list_add(&dentry->d_child, &parent->d_subdirs); + list_add(&dentry->d_u.d_child, &parent->d_subdirs); dentry_stat.nr_dentry++; spin_unlock(&dcache_lock); return dentry; } +struct dentry *d_alloc_name(struct dentry *parent, const char *name) +{ + struct qstr q; + + q.name = name; + q.len = strlen(name); + q.hash = full_name_hash(q.name, q.len); + return d_alloc(parent, &q); +} + /** * d_instantiate - fill in inode information for a dentry * @entry: dentry to complete @@ -764,6 +803,59 @@ void d_instantiate(struct dentry *entry, struct inode * inode) security_d_instantiate(entry, inode); } +/** + * d_instantiate_unique - instantiate a non-aliased dentry + * @entry: dentry to instantiate + * @inode: inode to attach to this dentry + * + * Fill in inode information in the entry. On success, it returns NULL. + * If an unhashed alias of "entry" already exists, then we return the + * aliased dentry instead and drop one reference to inode. + * + * Note that in order to avoid conflicts with rename() etc, the caller + * had better be holding the parent directory semaphore. + * + * This also assumes that the inode count has been incremented + * (or otherwise set) by the caller to indicate that it is now + * in use by the dcache. + */ +struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode) +{ + struct dentry *alias; + int len = entry->d_name.len; + const char *name = entry->d_name.name; + unsigned int hash = entry->d_name.hash; + + BUG_ON(!list_empty(&entry->d_alias)); + spin_lock(&dcache_lock); + if (!inode) + goto do_negative; + list_for_each_entry(alias, &inode->i_dentry, d_alias) { + struct qstr *qstr = &alias->d_name; + + if (qstr->hash != hash) + continue; + if (alias->d_parent != entry->d_parent) + continue; + if (qstr->len != len) + continue; + if (memcmp(qstr->name, name, len)) + continue; + dget_locked(alias); + spin_unlock(&dcache_lock); + BUG_ON(!d_unhashed(alias)); + iput(inode); + return alias; + } + list_add(&entry->d_alias, &inode->i_dentry); +do_negative: + entry->d_inode = inode; + spin_unlock(&dcache_lock); + security_d_instantiate(entry, inode); + return NULL; +} +EXPORT_SYMBOL(d_instantiate_unique); + /** * d_alloc_root - allocate root dentry * @root_inode: inode to allocate the root for @@ -836,33 +928,21 @@ struct dentry * d_alloc_anon(struct inode *inode) tmp->d_parent = tmp; /* make sure dput doesn't croak */ spin_lock(&dcache_lock); - if (S_ISDIR(inode->i_mode) && !list_empty(&inode->i_dentry)) { - /* A directory can only have one dentry. - * This (now) has one, so use it. - */ - res = list_entry(inode->i_dentry.next, struct dentry, d_alias); - __dget_locked(res); - } else { + res = __d_find_alias(inode, 0); + if (!res) { /* attach a disconnected dentry */ res = tmp; tmp = NULL; - if (res) { - spin_lock(&res->d_lock); - res->d_sb = inode->i_sb; - res->d_parent = res; - res->d_inode = inode; + spin_lock(&res->d_lock); + res->d_sb = inode->i_sb; + res->d_parent = res; + res->d_inode = inode; + res->d_flags |= DCACHE_DISCONNECTED; + res->d_flags &= ~DCACHE_UNHASHED; + list_add(&res->d_alias, &inode->i_dentry); + hlist_add_head(&res->d_hash, &inode->i_sb->s_anon); + spin_unlock(&res->d_lock); - /* - * Set d_bucket to an "impossible" bucket address so - * that d_move() doesn't get a false positive - */ - res->d_bucket = NULL; - res->d_flags |= DCACHE_DISCONNECTED; - res->d_flags &= ~DCACHE_UNHASHED; - list_add(&res->d_alias, &inode->i_dentry); - hlist_add_head(&res->d_hash, &inode->i_sb->s_anon); - spin_unlock(&res->d_lock); - } inode = NULL; /* don't drop reference */ } spin_unlock(&dcache_lock); @@ -884,7 +964,7 @@ struct dentry * d_alloc_anon(struct inode *inode) * DCACHE_DISCONNECTED), then d_move that in place of the given dentry * and return it, else simply d_add the inode to the dentry and return NULL. * - * This is (will be) needed in the lookup routine of any filesystem that is exportable + * This is needed in the lookup routine of any filesystem that is exportable * (via knfsd) so that we can build dcache paths to directories effectively. * * If a dentry was found and moved, then it is returned. Otherwise NULL @@ -895,11 +975,11 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) { struct dentry *new = NULL; - if (inode && S_ISDIR(inode->i_mode)) { + if (inode) { spin_lock(&dcache_lock); - if (!list_empty(&inode->i_dentry)) { - new = list_entry(inode->i_dentry.next, struct dentry, d_alias); - __dget_locked(new); + new = __d_find_alias(inode, 1); + if (new) { + BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED)); spin_unlock(&dcache_lock); security_d_instantiate(new, inode); d_rehash(dentry); @@ -970,18 +1050,13 @@ struct dentry * __d_lookup(struct dentry * parent, struct qstr * name) struct hlist_head *head = d_hash(parent,hash); struct dentry *found = NULL; struct hlist_node *node; + struct dentry *dentry; rcu_read_lock(); - hlist_for_each (node, head) { - struct dentry *dentry; + hlist_for_each_entry_rcu(dentry, node, head, d_hash) { struct qstr *qstr; - smp_read_barrier_depends(); - dentry = hlist_entry(node, struct dentry, d_hash); - - smp_rmb(); - if (dentry->d_name.hash != hash) continue; if (dentry->d_parent != parent) @@ -989,13 +1064,6 @@ struct dentry * __d_lookup(struct dentry * parent, struct qstr * name) spin_lock(&dentry->d_lock); - /* - * If lookup ends up in a different bucket due to concurrent - * rename, fail it - */ - if (unlikely(dentry->d_bucket != head)) - goto terminate; - /* * Recheck the dentry after taking the lock - d_move may have * changed things. Don't bother checking the hash because we're @@ -1004,8 +1072,11 @@ struct dentry * __d_lookup(struct dentry * parent, struct qstr * name) if (dentry->d_parent != parent) goto next; + /* + * It is safe to compare names since d_move() cannot + * change the qstr (protected by d_lock). + */ qstr = &dentry->d_name; - smp_read_barrier_depends(); if (parent->d_op && parent->d_op->d_compare) { if (parent->d_op->d_compare(parent, qstr, name)) goto next; @@ -1020,7 +1091,6 @@ struct dentry * __d_lookup(struct dentry * parent, struct qstr * name) atomic_inc(&dentry->d_count); found = dentry; } -terminate: spin_unlock(&dentry->d_lock); break; next: @@ -1058,7 +1128,7 @@ int d_validate(struct dentry *dentry, struct dentry *dparent) spin_lock(&dcache_lock); base = d_hash(dparent, dentry->d_name.hash); hlist_for_each(lhp,base) { - /* read_barrier_depends() not required for d_hash list + /* hlist_for_each_entry_rcu() not required for d_hash list * as it is parsed under dcache_lock */ if (dentry == hlist_entry(lhp, struct dentry, d_hash)) { @@ -1095,13 +1165,16 @@ out: void d_delete(struct dentry * dentry) { + int isdir = 0; /* * Are we the only user? */ spin_lock(&dcache_lock); spin_lock(&dentry->d_lock); + isdir = S_ISDIR(dentry->d_inode->i_mode); if (atomic_read(&dentry->d_count) == 1) { dentry_iput(dentry); + fsnotify_nameremove(dentry, isdir); return; } @@ -1110,6 +1183,15 @@ void d_delete(struct dentry * dentry) spin_unlock(&dentry->d_lock); spin_unlock(&dcache_lock); + + fsnotify_nameremove(dentry, isdir); +} + +static void __d_rehash(struct dentry * entry, struct hlist_head *list) +{ + + entry->d_flags &= ~DCACHE_UNHASHED; + hlist_add_head_rcu(&entry->d_hash, list); } /** @@ -1125,10 +1207,8 @@ void d_rehash(struct dentry * entry) spin_lock(&dcache_lock); spin_lock(&entry->d_lock); - entry->d_flags &= ~DCACHE_UNHASHED; + __d_rehash(entry, list); spin_unlock(&entry->d_lock); - entry->d_bucket = list; - hlist_add_head_rcu(&entry->d_hash, list); spin_unlock(&dcache_lock); } @@ -1206,6 +1286,8 @@ static void switch_names(struct dentry *dentry, struct dentry *target) void d_move(struct dentry * dentry, struct dentry * target) { + struct hlist_head *list; + if (!dentry->d_inode) printk(KERN_WARNING "VFS: moving negative dcache entry\n"); @@ -1225,33 +1307,21 @@ void d_move(struct dentry * dentry, struct dentry * target) /* Move the dentry to the target hash queue, if on different bucket */ if (dentry->d_flags & DCACHE_UNHASHED) goto already_unhashed; - if (dentry->d_bucket != target->d_bucket) { - hlist_del_rcu(&dentry->d_hash); + + hlist_del_rcu(&dentry->d_hash); + already_unhashed: - dentry->d_bucket = target->d_bucket; - hlist_add_head_rcu(&dentry->d_hash, target->d_bucket); - dentry->d_flags &= ~DCACHE_UNHASHED; - } + list = d_hash(target->d_parent, target->d_name.hash); + __d_rehash(dentry, list); /* Unhash the target: dput() will then get rid of it */ __d_drop(target); - /* flush any possible attributes */ - if (dentry->d_extra_attributes) { - kfree(dentry->d_extra_attributes); - dentry->d_extra_attributes = NULL; - } - if (target->d_extra_attributes) { - kfree(target->d_extra_attributes); - target->d_extra_attributes = NULL; - } - - list_del(&dentry->d_child); - list_del(&target->d_child); + list_del(&dentry->d_u.d_child); + list_del(&target->d_u.d_child); /* Switch the names.. */ switch_names(dentry, target); - smp_wmb(); do_switch(dentry->d_name.len, target->d_name.len); do_switch(dentry->d_name.hash, target->d_name.hash); @@ -1259,15 +1329,15 @@ already_unhashed: if (IS_ROOT(dentry)) { dentry->d_parent = target->d_parent; target->d_parent = target; - INIT_LIST_HEAD(&target->d_child); + INIT_LIST_HEAD(&target->d_u.d_child); } else { do_switch(dentry->d_parent, target->d_parent); /* And add them back to the (new) parent lists */ - list_add(&target->d_child, &target->d_parent->d_subdirs); + list_add(&target->d_u.d_child, &target->d_parent->d_subdirs); } - list_add(&dentry->d_child, &dentry->d_parent->d_subdirs); + list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs); spin_unlock(&target->d_lock); spin_unlock(&dentry->d_lock); write_sequnlock(&rename_lock); @@ -1290,7 +1360,7 @@ already_unhashed: * * "buflen" should be positive. Caller holds the dcache_lock. */ -char * __d_path( struct dentry *dentry, struct vfsmount *vfsmnt, +static char * __d_path( struct dentry *dentry, struct vfsmount *vfsmnt, struct dentry *root, struct vfsmount *rootmnt, char *buffer, int buflen) { @@ -1358,8 +1428,6 @@ Elong: return ERR_PTR(-ENAMETOOLONG); } -EXPORT_SYMBOL_GPL(__d_path); - /* write full pathname into buffer and return start of pathname */ char * d_path(struct dentry *dentry, struct vfsmount *vfsmnt, char *buf, int buflen) @@ -1470,7 +1538,6 @@ int is_subdir(struct dentry * new_dentry, struct dentry * old_dentry) struct dentry * saved = new_dentry; unsigned long seq; - result = 0; /* need rcu_readlock to protect against the d_parent trashing due to * d_move */ @@ -1478,6 +1545,7 @@ int is_subdir(struct dentry * new_dentry, struct dentry * old_dentry) do { /* for restarting inner loop in case of seq retry */ new_dentry = saved; + result = 0; seq = read_seqbegin(&rename_lock); for (;;) { if (new_dentry != old_dentry) { @@ -1507,7 +1575,7 @@ repeat: resume: while (next != &this_parent->d_subdirs) { struct list_head *tmp = next; - struct dentry *dentry = list_entry(tmp, struct dentry, d_child); + struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child); next = tmp->next; if (d_unhashed(dentry)||!dentry->d_inode) continue; @@ -1518,7 +1586,7 @@ resume: atomic_dec(&dentry->d_count); } if (this_parent != root) { - next = this_parent->d_child.next; + next = this_parent->d_u.d_child.next; atomic_dec(&this_parent->d_count); this_parent = this_parent->d_parent; goto resume; @@ -1578,29 +1646,33 @@ static int __init set_dhash_entries(char *str) } __setup("dhash_entries=", set_dhash_entries); -void flush_dentry_attributes (void) +static void __init dcache_init_early(void) { - struct hlist_node *tmp; - struct dentry *dentry; - int i; + int loop; - spin_lock(&dcache_lock); - for (i = 0; i <= d_hash_mask; i++) - hlist_for_each_entry(dentry, tmp, dentry_hashtable+i, d_hash) { - kfree(dentry->d_extra_attributes); - dentry->d_extra_attributes = NULL; - } - spin_unlock(&dcache_lock); -} + /* If hashes are distributed across NUMA nodes, defer + * hash allocation until vmalloc space is available. + */ + if (hashdist) + return; -EXPORT_SYMBOL_GPL(flush_dentry_attributes); + dentry_hashtable = + alloc_large_system_hash("Dentry cache", + sizeof(struct hlist_head), + dhash_entries, + 13, + HASH_EARLY, + &d_hash_shift, + &d_hash_mask, + 0); + + for (loop = 0; loop < (1 << d_hash_shift); loop++) + INIT_HLIST_HEAD(&dentry_hashtable[loop]); +} static void __init dcache_init(unsigned long mempages) { - struct hlist_head *d; - unsigned long order; - unsigned int nr_hash; - int i; + int loop; /* * A constructor could be added for stable state like the lists, @@ -1615,47 +1687,22 @@ static void __init dcache_init(unsigned long mempages) set_shrinker(DEFAULT_SEEKS, shrink_dcache_memory); - if (!dhash_entries) - dhash_entries = PAGE_SHIFT < 13 ? - mempages >> (13 - PAGE_SHIFT) : - mempages << (PAGE_SHIFT - 13); - - dhash_entries *= sizeof(struct hlist_head); - for (order = 0; ((1UL << order) << PAGE_SHIFT) < dhash_entries; order++) - ; - - if (order > 5) - order = 5; - - do { - unsigned long tmp; - - nr_hash = (1UL << order) * PAGE_SIZE / - sizeof(struct hlist_head); - d_hash_mask = (nr_hash - 1); - - tmp = nr_hash; - d_hash_shift = 0; - while ((tmp >>= 1UL) != 0UL) - d_hash_shift++; - - dentry_hashtable = (struct hlist_head *) - __get_free_pages(GFP_ATOMIC, order); - } while (dentry_hashtable == NULL && --order >= 0); - - printk(KERN_INFO "Dentry cache hash table entries: %d (order: %ld, %ld bytes)\n", - nr_hash, order, (PAGE_SIZE << order)); - - if (!dentry_hashtable) - panic("Failed to allocate dcache hash table\n"); + /* Hash may have been set up in dcache_init_early */ + if (!hashdist) + return; - d = dentry_hashtable; - i = nr_hash; - do { - INIT_HLIST_HEAD(d); - d++; - i--; - } while (i); + dentry_hashtable = + alloc_large_system_hash("Dentry cache", + sizeof(struct hlist_head), + dhash_entries, + 13, + 0, + &d_hash_shift, + &d_hash_mask, + 0); + + for (loop = 0; loop < (1 << d_hash_shift); loop++) + INIT_HLIST_HEAD(&dentry_hashtable[loop]); } /* SLAB cache for __getname() consumers */ @@ -1669,6 +1716,12 @@ EXPORT_SYMBOL(d_genocide); extern void bdev_cache_init(void); extern void chrdev_init(void); +void __init vfs_caches_init_early(void) +{ + dcache_init_early(); + inode_init_early(); +} + void __init vfs_caches_init(unsigned long mempages) { unsigned long reserve; @@ -1683,7 +1736,7 @@ void __init vfs_caches_init(unsigned long mempages) SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL); filp_cachep = kmem_cache_create("filp", sizeof(struct file), 0, - SLAB_HWCACHE_ALIGN|SLAB_PANIC, filp_ctor, filp_dtor); + SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL); dcache_init(mempages); inode_init(mempages); @@ -1711,8 +1764,6 @@ EXPORT_SYMBOL(dget_locked); EXPORT_SYMBOL(dput); EXPORT_SYMBOL(find_inode_number); EXPORT_SYMBOL(have_submounts); -EXPORT_SYMBOL(is_subdir); EXPORT_SYMBOL(names_cachep); -EXPORT_SYMBOL(shrink_dcache_anon); EXPORT_SYMBOL(shrink_dcache_parent); EXPORT_SYMBOL(shrink_dcache_sb);