X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=include%2Flinux%2Flist.h;h=611059d633f4a9fcf2ec435309564b4b4bb0b126;hb=97bf2856c6014879bd04983a3e9dfcdac1e7fe85;hp=9e0c971ded6f7284328c747d03e45e401ac59944;hpb=5273a3df6485dc2ad6aa7ddd441b9a21970f003b;p=linux-2.6.git diff --git a/include/linux/list.h b/include/linux/list.h index 9e0c971de..611059d63 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -4,17 +4,10 @@ #ifdef __KERNEL__ #include +#include #include #include -/* - * These are non-NULL pointers that will result in page faults - * under normal circumstances, used to verify that nobody uses - * non-initialized list entries. - */ -#define LIST_POISON1 ((void *) 0x00100100) -#define LIST_POISON2 ((void *) 0x00200200) - /* * Simple doubly linked list implementation. * @@ -34,9 +27,11 @@ struct list_head { #define LIST_HEAD(name) \ struct list_head name = LIST_HEAD_INIT(name) -#define INIT_LIST_HEAD(ptr) do { \ - (ptr)->next = (ptr); (ptr)->prev = (ptr); \ -} while (0) +static inline void INIT_LIST_HEAD(struct list_head *list) +{ + list->next = list; + list->prev = list; +} /* * Insert a new entry between two known consecutive entries. @@ -44,6 +39,7 @@ struct list_head { * This is only for internal list manipulation where we know * the prev/next entries already! */ +#ifndef CONFIG_DEBUG_LIST static inline void __list_add(struct list_head *new, struct list_head *prev, struct list_head *next) @@ -53,6 +49,11 @@ static inline void __list_add(struct list_head *new, new->prev = prev; prev->next = new; } +#else +extern void __list_add(struct list_head *new, + struct list_head *prev, + struct list_head *next); +#endif /** * list_add - add a new entry @@ -62,10 +63,15 @@ static inline void __list_add(struct list_head *new, * Insert a new entry after the specified head. * This is good for implementing stacks. */ +#ifndef CONFIG_DEBUG_LIST static inline void list_add(struct list_head *new, struct list_head *head) { __list_add(new, head, head->next); } +#else +extern void list_add(struct list_head *new, struct list_head *head); +#endif + /** * list_add_tail - add a new entry @@ -158,12 +164,16 @@ static inline void __list_del(struct list_head * prev, struct list_head * next) * Note: list_empty on entry does not return true after this, the entry is * in an undefined state. */ +#ifndef CONFIG_DEBUG_LIST static inline void list_del(struct list_head *entry) { __list_del(entry->prev, entry->next); entry->next = LIST_POISON1; entry->prev = LIST_POISON2; } +#else +extern void list_del(struct list_head *entry); +#endif /** * list_del_rcu - deletes entry from list without re-initialization @@ -185,7 +195,7 @@ static inline void list_del(struct list_head *entry) * list_for_each_entry_rcu(). * * Note that the caller is not permitted to immediately free - * the newly deleted entry. Instead, either synchronize_kernel() + * the newly deleted entry. Instead, either synchronize_rcu() * or call_rcu() must be used to defer freeing until an RCU * grace period has elapsed. */ @@ -195,6 +205,47 @@ static inline void list_del_rcu(struct list_head *entry) entry->prev = LIST_POISON2; } +/** + * list_replace - replace old entry by new one + * @old : the element to be replaced + * @new : the new element to insert + * Note: if 'old' was empty, it will be overwritten. + */ +static inline void list_replace(struct list_head *old, + struct list_head *new) +{ + new->next = old->next; + new->next->prev = new; + new->prev = old->prev; + new->prev->next = new; +} + +static inline void list_replace_init(struct list_head *old, + struct list_head *new) +{ + list_replace(old, new); + INIT_LIST_HEAD(old); +} + +/** + * list_replace_rcu - replace old entry by new one + * @old : the element to be replaced + * @new : the new element to insert + * + * The @old entry will be replaced with the @new entry atomically. + * Note: @old should not be empty. + */ +static inline void list_replace_rcu(struct list_head *old, + struct list_head *new) +{ + new->next = old->next; + new->prev = old->prev; + smp_wmb(); + new->next->prev = new; + new->prev->next = new; + old->prev = LIST_POISON2; +} + /** * list_del_init - deletes entry from list and reinitialize it. * @entry: the element to delete from the list. @@ -228,6 +279,17 @@ static inline void list_move_tail(struct list_head *list, list_add_tail(list, head); } +/** + * list_is_last - tests whether @list is the last entry in list @head + * @list: the entry to test + * @head: the head of the list + */ +static inline int list_is_last(const struct list_head *list, + const struct list_head *head) +{ + return list->next == head; +} + /** * list_empty - tests whether a list is empty * @head: the list to test. @@ -238,16 +300,17 @@ static inline int list_empty(const struct list_head *head) } /** - * list_empty_careful - tests whether a list is - * empty _and_ checks that no other CPU might be - * in the process of still modifying either member + * list_empty_careful - tests whether a list is empty and not being modified + * @head: the list to test + * + * Description: + * tests whether a list is empty _and_ checks that no other CPU might be + * in the process of modifying either member (next or prev) * * NOTE: using list_empty_careful() without synchronization * can only be safe if the only activity that can happen * to the list entry is list_del_init(). Eg. it cannot be used * if another CPU could re-list_add() it. - * - * @head: the list to test. */ static inline int list_empty_careful(const struct list_head *head) { @@ -307,16 +370,16 @@ static inline void list_splice_init(struct list_head *list, /** * list_for_each - iterate over a list - * @pos: the &struct list_head to use as a loop counter. + * @pos: the &struct list_head to use as a loop cursor. * @head: the head for your list. */ #define list_for_each(pos, head) \ - for (pos = (head)->next, prefetch(pos->next); pos != (head); \ - pos = pos->next, prefetch(pos->next)) + for (pos = (head)->next; prefetch(pos->next), pos != (head); \ + pos = pos->next) /** * __list_for_each - iterate over a list - * @pos: the &struct list_head to use as a loop counter. + * @pos: the &struct list_head to use as a loop cursor. * @head: the head for your list. * * This variant differs from list_for_each() in that it's the @@ -329,16 +392,16 @@ static inline void list_splice_init(struct list_head *list, /** * list_for_each_prev - iterate over a list backwards - * @pos: the &struct list_head to use as a loop counter. + * @pos: the &struct list_head to use as a loop cursor. * @head: the head for your list. */ #define list_for_each_prev(pos, head) \ - for (pos = (head)->prev, prefetch(pos->prev); pos != (head); \ - pos = pos->prev, prefetch(pos->prev)) + for (pos = (head)->prev; prefetch(pos->prev), pos != (head); \ + pos = pos->prev) /** - * list_for_each_safe - iterate over a list safe against removal of list entry - * @pos: the &struct list_head to use as a loop counter. + * list_for_each_safe - iterate over a list safe against removal of list entry + * @pos: the &struct list_head to use as a loop cursor. * @n: another &struct list_head to use as temporary storage * @head: the head for your list. */ @@ -348,57 +411,66 @@ static inline void list_splice_init(struct list_head *list, /** * list_for_each_entry - iterate over list of given type - * @pos: the type * to use as a loop counter. + * @pos: the type * to use as a loop cursor. * @head: the head for your list. * @member: the name of the list_struct within the struct. */ #define list_for_each_entry(pos, head, member) \ - for (pos = list_entry((head)->next, typeof(*pos), member), \ - prefetch(pos->member.next); \ - &pos->member != (head); \ - pos = list_entry(pos->member.next, typeof(*pos), member), \ - prefetch(pos->member.next)) + for (pos = list_entry((head)->next, typeof(*pos), member); \ + prefetch(pos->member.next), &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) /** * list_for_each_entry_reverse - iterate backwards over list of given type. - * @pos: the type * to use as a loop counter. + * @pos: the type * to use as a loop cursor. * @head: the head for your list. * @member: the name of the list_struct within the struct. */ #define list_for_each_entry_reverse(pos, head, member) \ - for (pos = list_entry((head)->prev, typeof(*pos), member), \ - prefetch(pos->member.prev); \ - &pos->member != (head); \ - pos = list_entry(pos->member.prev, typeof(*pos), member), \ - prefetch(pos->member.prev)) + for (pos = list_entry((head)->prev, typeof(*pos), member); \ + prefetch(pos->member.prev), &pos->member != (head); \ + pos = list_entry(pos->member.prev, typeof(*pos), member)) /** - * list_prepare_entry - prepare a pos entry for use as a start point in - * list_for_each_entry_continue + * list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue * @pos: the type * to use as a start point * @head: the head of the list * @member: the name of the list_struct within the struct. + * + * Prepares a pos entry for use as a start point in list_for_each_entry_continue. */ #define list_prepare_entry(pos, head, member) \ ((pos) ? : list_entry(head, typeof(*pos), member)) /** - * list_for_each_entry_continue - iterate over list of given type - * continuing after existing point - * @pos: the type * to use as a loop counter. + * list_for_each_entry_continue - continue iteration over list of given type + * @pos: the type * to use as a loop cursor. * @head: the head for your list. * @member: the name of the list_struct within the struct. + * + * Continue to iterate over list of given type, continuing after + * the current position. */ #define list_for_each_entry_continue(pos, head, member) \ - for (pos = list_entry(pos->member.next, typeof(*pos), member), \ - prefetch(pos->member.next); \ - &pos->member != (head); \ - pos = list_entry(pos->member.next, typeof(*pos), member), \ - prefetch(pos->member.next)) + for (pos = list_entry(pos->member.next, typeof(*pos), member); \ + prefetch(pos->member.next), &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) + +/** + * list_for_each_entry_from - iterate over list of given type from the current point + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Iterate over list of given type, continuing from current position. + */ +#define list_for_each_entry_from(pos, head, member) \ + for (; prefetch(pos->member.next), &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) /** * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry - * @pos: the type * to use as a loop counter. + * @pos: the type * to use as a loop cursor. * @n: another type * to use as temporary storage * @head: the head for your list. * @member: the name of the list_struct within the struct. @@ -409,9 +481,56 @@ static inline void list_splice_init(struct list_head *list, &pos->member != (head); \ pos = n, n = list_entry(n->member.next, typeof(*n), member)) +/** + * list_for_each_entry_safe_continue + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Iterate over list of given type, continuing after current point, + * safe against removal of list entry. + */ +#define list_for_each_entry_safe_continue(pos, n, head, member) \ + for (pos = list_entry(pos->member.next, typeof(*pos), member), \ + n = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + +/** + * list_for_each_entry_safe_from + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Iterate over list of given type from current point, safe against + * removal of list entry. + */ +#define list_for_each_entry_safe_from(pos, n, head, member) \ + for (n = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + +/** + * list_for_each_entry_safe_reverse + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Iterate backwards over list of given type, safe against removal + * of list entry. + */ +#define list_for_each_entry_safe_reverse(pos, n, head, member) \ + for (pos = list_entry((head)->prev, typeof(*pos), member), \ + n = list_entry(pos->member.prev, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.prev, typeof(*n), member)) + /** * list_for_each_rcu - iterate over an rcu-protected list - * @pos: the &struct list_head to use as a loop counter. + * @pos: the &struct list_head to use as a loop cursor. * @head: the head for your list. * * This list-traversal primitive may safely run concurrently with @@ -419,31 +538,35 @@ static inline void list_splice_init(struct list_head *list, * as long as the traversal is guarded by rcu_read_lock(). */ #define list_for_each_rcu(pos, head) \ - for (pos = (head)->next, prefetch(pos->next); pos != (head); \ - pos = pos->next, ({ smp_read_barrier_depends(); 0;}), prefetch(pos->next)) + for (pos = (head)->next; \ + prefetch(rcu_dereference(pos)->next), pos != (head); \ + pos = pos->next) #define __list_for_each_rcu(pos, head) \ - for (pos = (head)->next; pos != (head); \ - pos = pos->next, ({ smp_read_barrier_depends(); 0;})) + for (pos = (head)->next; \ + rcu_dereference(pos) != (head); \ + pos = pos->next) /** - * list_for_each_safe_rcu - iterate over an rcu-protected list safe - * against removal of list entry - * @pos: the &struct list_head to use as a loop counter. + * list_for_each_safe_rcu + * @pos: the &struct list_head to use as a loop cursor. * @n: another &struct list_head to use as temporary storage * @head: the head for your list. * + * Iterate over an rcu-protected list, safe against removal of list entry. + * * This list-traversal primitive may safely run concurrently with * the _rcu list-mutation primitives such as list_add_rcu() * as long as the traversal is guarded by rcu_read_lock(). */ #define list_for_each_safe_rcu(pos, n, head) \ - for (pos = (head)->next, n = pos->next; pos != (head); \ - pos = n, ({ smp_read_barrier_depends(); 0;}), n = pos->next) + for (pos = (head)->next; \ + n = rcu_dereference(pos)->next, pos != (head); \ + pos = n) /** * list_for_each_entry_rcu - iterate over rcu list of given type - * @pos: the type * to use as a loop counter. + * @pos: the type * to use as a loop cursor. * @head: the head for your list. * @member: the name of the list_struct within the struct. * @@ -451,28 +574,28 @@ static inline void list_splice_init(struct list_head *list, * the _rcu list-mutation primitives such as list_add_rcu() * as long as the traversal is guarded by rcu_read_lock(). */ -#define list_for_each_entry_rcu(pos, head, member) \ - for (pos = list_entry((head)->next, typeof(*pos), member), \ - prefetch(pos->member.next); \ - &pos->member != (head); \ - pos = list_entry(pos->member.next, typeof(*pos), member), \ - ({ smp_read_barrier_depends(); 0;}), \ - prefetch(pos->member.next)) +#define list_for_each_entry_rcu(pos, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member); \ + prefetch(rcu_dereference(pos)->member.next), \ + &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) /** - * list_for_each_continue_rcu - iterate over an rcu-protected list - * continuing after existing point. - * @pos: the &struct list_head to use as a loop counter. + * list_for_each_continue_rcu + * @pos: the &struct list_head to use as a loop cursor. * @head: the head for your list. * + * Iterate over an rcu-protected list, continuing after current point. + * * This list-traversal primitive may safely run concurrently with * the _rcu list-mutation primitives such as list_add_rcu() * as long as the traversal is guarded by rcu_read_lock(). */ #define list_for_each_continue_rcu(pos, head) \ - for ((pos) = (pos)->next, prefetch((pos)->next); (pos) != (head); \ - (pos) = (pos)->next, ({ smp_read_barrier_depends(); 0;}), prefetch((pos)->next)) + for ((pos) = (pos)->next; \ + prefetch(rcu_dereference((pos))->next), (pos) != (head); \ + (pos) = (pos)->next) /* * Double linked lists with a single pointer list head. @@ -492,7 +615,11 @@ struct hlist_node { #define HLIST_HEAD_INIT { .first = NULL } #define HLIST_HEAD(name) struct hlist_head name = { .first = NULL } #define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL) -#define INIT_HLIST_NODE(ptr) ((ptr)->next = NULL, (ptr)->pprev = NULL) +static inline void INIT_HLIST_NODE(struct hlist_node *h) +{ + h->next = NULL; + h->pprev = NULL; +} static inline int hlist_unhashed(const struct hlist_node *h) { @@ -547,13 +674,32 @@ static inline void hlist_del_rcu(struct hlist_node *n) static inline void hlist_del_init(struct hlist_node *n) { - if (n->pprev) { + if (!hlist_unhashed(n)) { __hlist_del(n); INIT_HLIST_NODE(n); } } -#define hlist_del_rcu_init hlist_del_init +/** + * hlist_replace_rcu - replace old entry by new one + * @old : the element to be replaced + * @new : the new element to insert + * + * The @old entry will be replaced with the @new entry atomically. + */ +static inline void hlist_replace_rcu(struct hlist_node *old, + struct hlist_node *new) +{ + struct hlist_node *next = old->next; + + new->next = next; + new->pprev = old->pprev; + smp_wmb(); + if (next) + new->next->pprev = &new->next; + *new->pprev = new; + old->pprev = LIST_POISON2; +} static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) { @@ -567,23 +713,23 @@ static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) /** - * hlist_add_head_rcu - adds the specified element to the specified hlist, - * while permitting racing traversals. + * hlist_add_head_rcu * @n: the element to add to the hash list. * @h: the list to add to. * + * Description: + * Adds the specified element to the specified hlist, + * while permitting racing traversals. + * * The caller must take whatever precautions are necessary * (such as holding appropriate locks) to avoid racing * with another list-mutation primitive, such as hlist_add_head_rcu() * or hlist_del_rcu(), running on this same list. * However, it is perfectly legal to run concurrently with * the _rcu list-traversal primitives, such as - * hlist_for_each_entry(), but only if smp_read_barrier_depends() - * is used to prevent memory-consistency problems on Alpha CPUs. - * Regardless of the type of CPU, the list-traversal primitive - * must be guarded by rcu_read_lock(). - * - * OK, so why don't we have an hlist_for_each_entry_rcu()??? + * hlist_for_each_entry_rcu(), used to prevent memory-consistency + * problems on Alpha CPUs. Regardless of the type of CPU, the + * list-traversal primitive must be guarded by rcu_read_lock(). */ static inline void hlist_add_head_rcu(struct hlist_node *n, struct hlist_head *h) @@ -618,21 +764,77 @@ static inline void hlist_add_after(struct hlist_node *n, next->next->pprev = &next->next; } +/** + * hlist_add_before_rcu + * @n: the new element to add to the hash list. + * @next: the existing element to add the new element before. + * + * Description: + * Adds the specified element to the specified hlist + * before the specified node while permitting racing traversals. + * + * The caller must take whatever precautions are necessary + * (such as holding appropriate locks) to avoid racing + * with another list-mutation primitive, such as hlist_add_head_rcu() + * or hlist_del_rcu(), running on this same list. + * However, it is perfectly legal to run concurrently with + * the _rcu list-traversal primitives, such as + * hlist_for_each_entry_rcu(), used to prevent memory-consistency + * problems on Alpha CPUs. + */ +static inline void hlist_add_before_rcu(struct hlist_node *n, + struct hlist_node *next) +{ + n->pprev = next->pprev; + n->next = next; + smp_wmb(); + next->pprev = &n->next; + *(n->pprev) = n; +} + +/** + * hlist_add_after_rcu + * @prev: the existing element to add the new element after. + * @n: the new element to add to the hash list. + * + * Description: + * Adds the specified element to the specified hlist + * after the specified node while permitting racing traversals. + * + * The caller must take whatever precautions are necessary + * (such as holding appropriate locks) to avoid racing + * with another list-mutation primitive, such as hlist_add_head_rcu() + * or hlist_del_rcu(), running on this same list. + * However, it is perfectly legal to run concurrently with + * the _rcu list-traversal primitives, such as + * hlist_for_each_entry_rcu(), used to prevent memory-consistency + * problems on Alpha CPUs. + */ +static inline void hlist_add_after_rcu(struct hlist_node *prev, + struct hlist_node *n) +{ + n->next = prev->next; + n->pprev = &prev->next; + smp_wmb(); + prev->next = n; + if (n->next) + n->next->pprev = &n->next; +} + #define hlist_entry(ptr, type, member) container_of(ptr,type,member) -/* Cannot easily do prefetch unfortunately */ #define hlist_for_each(pos, head) \ for (pos = (head)->first; pos && ({ prefetch(pos->next); 1; }); \ pos = pos->next) #define hlist_for_each_safe(pos, n, head) \ - for (pos = (head)->first; n = pos ? pos->next : 0, pos; \ + for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \ pos = n) /** * hlist_for_each_entry - iterate over list of given type - * @tpos: the type * to use as a loop counter. - * @pos: the &struct hlist_node to use as a loop counter. + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. * @head: the head for your list. * @member: the name of the hlist_node within the struct. */ @@ -643,9 +845,9 @@ static inline void hlist_add_after(struct hlist_node *n, pos = pos->next) /** - * hlist_for_each_entry_continue - iterate over a hlist continuing after existing point - * @tpos: the type * to use as a loop counter. - * @pos: the &struct hlist_node to use as a loop counter. + * hlist_for_each_entry_continue - iterate over a hlist continuing after current point + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. * @member: the name of the hlist_node within the struct. */ #define hlist_for_each_entry_continue(tpos, pos, member) \ @@ -655,9 +857,9 @@ static inline void hlist_add_after(struct hlist_node *n, pos = pos->next) /** - * hlist_for_each_entry_from - iterate over a hlist continuing from existing point - * @tpos: the type * to use as a loop counter. - * @pos: the &struct hlist_node to use as a loop counter. + * hlist_for_each_entry_from - iterate over a hlist continuing from current point + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. * @member: the name of the hlist_node within the struct. */ #define hlist_for_each_entry_from(tpos, pos, member) \ @@ -667,8 +869,8 @@ static inline void hlist_add_after(struct hlist_node *n, /** * hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry - * @tpos: the type * to use as a loop counter. - * @pos: the &struct hlist_node to use as a loop counter. + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. * @n: another &struct hlist_node to use as temporary storage * @head: the head for your list. * @member: the name of the hlist_node within the struct. @@ -678,6 +880,24 @@ static inline void hlist_add_after(struct hlist_node *n, pos && ({ n = pos->next; 1; }) && \ ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ pos = n) + +/** + * hlist_for_each_entry_rcu - iterate over rcu list of given type + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the hlist_node within the struct. + * + * This list-traversal primitive may safely run concurrently with + * the _rcu list-mutation primitives such as hlist_add_head_rcu() + * as long as the traversal is guarded by rcu_read_lock(). + */ +#define hlist_for_each_entry_rcu(tpos, pos, head, member) \ + for (pos = (head)->first; \ + rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) && \ + ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ + pos = pos->next) + #else #warning "don't include kernel headers in userspace" #endif /* __KERNEL__ */