| Message ID | 20051015002649.GA28555@gondor.apana.org.au |
|---|---|
| State | New, archived |
| Headers | show |
| Series |
|
| Related | show |
On Sat, Oct 15, 2005 at 10:26:49AM +1000, Herbert Xu wrote: > Hi: > > It seems that all the list_*_rcu primitives are missing a memory barrier > on the very first dereference. For example, > > #define list_for_each_rcu(pos, head) \ > for (pos = (head)->next; prefetch(pos->next), pos != (head); \ > pos = rcu_dereference(pos->next)) > > It will go something like: > > pos = (head)->next > > prefetch(pos->next) > > pos != (head) > > do stuff > > We're missing a barrier here. > > pos = rcu_dereference(pos->next) > > fetch pos->next > > barrier given by rcu_dereference(pos->next) > > store pos > > Without the missing barrier, the pos->next value may turn out to be > stale. In fact, if "do stuff" were also dereferencing pos and relying > on list_for_each_rcu to provide the barrier then it may also break. > > So here is a patch to make sure that we have a barrier for the first > element in the list. Good catch!!! I wonder if Alpha SMP Linux machines have noticed... Some comments interspersed. > Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> > > Cheers, > -- > Visit Openswan at http://www.openswan.org/ > Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au> > Home Page: http://gondor.apana.org.au/~herbert/ > PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt > diff --git a/include/linux/list.h b/include/linux/list.h > --- a/include/linux/list.h > +++ b/include/linux/list.h > @@ -442,12 +442,15 @@ static inline void list_splice_init(stru > * as long as the traversal is guarded by rcu_read_lock(). > */ > #define list_for_each_rcu(pos, head) \ > - for (pos = (head)->next; prefetch(pos->next), pos != (head); \ > - pos = rcu_dereference(pos->next)) > + for (pos = (head)->next; \ > + pos = rcu_dereference(pos), \ > + prefetch(pos->next), pos != (head); \ > + pos = pos->next) Why not something like the following? Seems a bit simpler to me. #define list_for_each_rcu(pos, head) \ for (pos = rcu_dereference((head)->next); \ prefetch(pos->next), pos != (head); \ pos = rcu_dereference(pos->next)) > #define __list_for_each_rcu(pos, head) \ > - for (pos = (head)->next; pos != (head); \ > - pos = rcu_dereference(pos->next)) > + for (pos = (head)->next; \ > + rcu_dereference(pos) != (head); \ > + pos = pos->next) > > /** > * list_for_each_safe_rcu - iterate over an rcu-protected list safe > @@ -461,8 +464,9 @@ static inline void list_splice_init(stru > * as long as the traversal is guarded by rcu_read_lock(). > */ > #define list_for_each_safe_rcu(pos, n, head) \ > - for (pos = (head)->next, n = pos->next; pos != (head); \ > - pos = rcu_dereference(n), n = pos->next) > + for (pos = (head)->next; \ > + n = rcu_dereference(pos)->next, pos != (head); \ > + pos = n) > > /** > * list_for_each_entry_rcu - iterate over rcu list of given type > @@ -474,11 +478,11 @@ static inline void list_splice_init(stru > * the _rcu list-mutation primitives such as list_add_rcu() > * as long as the traversal is guarded by rcu_read_lock(). > */ > -#define list_for_each_entry_rcu(pos, head, member) \ > - for (pos = list_entry((head)->next, typeof(*pos), member); \ > - prefetch(pos->member.next), &pos->member != (head); \ > - pos = rcu_dereference(list_entry(pos->member.next, \ > - typeof(*pos), member))) > +#define list_for_each_entry_rcu(pos, head, member) \ > + for (pos = list_entry((head)->next, typeof(*pos), member); \ > + pos = rcu_dereference(pos), \ > + prefetch(pos->member.next), &pos->member != (head); \ > + pos = list_entry(pos->member.next, typeof(*pos), member)) > > > /** > @@ -492,8 +496,10 @@ static inline void list_splice_init(stru > * as long as the traversal is guarded by rcu_read_lock(). > */ > #define list_for_each_continue_rcu(pos, head) \ > - for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \ > - (pos) = rcu_dereference((pos)->next)) > + for ((pos) = (pos)->next; \ > + (pos) = rcu_dereference((pos)), \ > + prefetch((pos)->next), (pos) != (head); \ > + (pos) = (pos)->next) The above hurts my head -- childhood trauma due to having to use a FORTRAN compiler that required "I=I" at odd intervals in order to generate correct code... How about the following? #define list_for_each_continue_rcu(pos, head) \ for ((pos) = (pos)->next; \ prefetch(rcu_dereference(pos)->next), (pos) != (head); \ (pos) = (pos)->next) > /* > * Double linked lists with a single pointer list head. > @@ -696,8 +702,9 @@ static inline void hlist_add_after_rcu(s > pos = n) > > #define hlist_for_each_rcu(pos, head) \ > - for ((pos) = (head)->first; pos && ({ prefetch((pos)->next); 1; }); \ > - (pos) = rcu_dereference((pos)->next)) > + for ((pos) = (head)->first; \ > + rcu_dereference((pos)) && ({ prefetch((pos)->next); 1; }); \ > + (pos) = (pos)->next) > > /** > * hlist_for_each_entry - iterate over list of given type > @@ -762,9 +769,9 @@ static inline void hlist_add_after_rcu(s > */ > #define hlist_for_each_entry_rcu(tpos, pos, head, member) \ > for (pos = (head)->first; \ > - pos && ({ prefetch(pos->next); 1;}) && \ > + rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) && \ > ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ > - pos = rcu_dereference(pos->next)) > + pos = pos->next) > > #else > #warning "don't include kernel headers in userspace" - To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
On Fri, Oct 14, 2005 at 07:03:25PM -0700, Paul E. McKenney wrote: > > > diff --git a/include/linux/list.h b/include/linux/list.h > > --- a/include/linux/list.h > > +++ b/include/linux/list.h > > @@ -442,12 +442,15 @@ static inline void list_splice_init(stru > > * as long as the traversal is guarded by rcu_read_lock(). > > */ > > #define list_for_each_rcu(pos, head) \ > > - for (pos = (head)->next; prefetch(pos->next), pos != (head); \ > > - pos = rcu_dereference(pos->next)) > > + for (pos = (head)->next; \ > > + pos = rcu_dereference(pos), \ > > + prefetch(pos->next), pos != (head); \ > > + pos = pos->next) > > Why not something like the following? Seems a bit simpler to me. > > #define list_for_each_rcu(pos, head) \ > for (pos = rcu_dereference((head)->next); \ > prefetch(pos->next), pos != (head); \ > pos = rcu_dereference(pos->next)) In this case your version is indeed more concise. However, in most of the other for_each macros having it in the loop conditional looks more natural. So in order to be consistent throughout list.h, I'd like to keep the rcu_dereference in the loop conditional. > > @@ -492,8 +496,10 @@ static inline void list_splice_init(stru > > * as long as the traversal is guarded by rcu_read_lock(). > > */ > > #define list_for_each_continue_rcu(pos, head) \ > > - for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \ > > - (pos) = rcu_dereference((pos)->next)) > > + for ((pos) = (pos)->next; \ > > + (pos) = rcu_dereference((pos)), \ > > + prefetch((pos)->next), (pos) != (head); \ > > + (pos) = (pos)->next) > > The above hurts my head -- childhood trauma due to having to use a > FORTRAN compiler that required "I=I" at odd intervals in order to > generate correct code... How about the following? > > #define list_for_each_continue_rcu(pos, head) \ > for ((pos) = (pos)->next; \ > prefetch(rcu_dereference(pos)->next), (pos) != (head); \ > (pos) = (pos)->next) I chose to keep it out of prefetch because normally the argument to prefetch does not have any side-effects. Even though today's prefetch is an inline function which does respect side-effects, there is always a possibility that someone somewhere might decide to implement prefetch as a macro. Besides, the expression i = foo(i) where foo has side-effects is pretty normal. Cheers,
Hi Paul: On Sat, Oct 15, 2005 at 12:39:18PM +1000, herbert wrote: > > Besides, the expression > > i = foo(i) > > where foo has side-effects is pretty normal. Actually I've changed my mind on this. I think your version is better because the side-effect of rcu_dereference will cause the above assignment to occur twice when i refers to a memory-backed variable. Since all current prefetch implementations are safe as far as side-effects are concerned, here is an updated version that doesn't do i = foo(i). Andrew, please replace the previous version with this. Thanks,
On Sat, Oct 15, 2005 at 01:22:41PM +1000, Herbert Xu wrote: > Hi Paul: > > On Sat, Oct 15, 2005 at 12:39:18PM +1000, herbert wrote: > > > > Besides, the expression > > > > i = foo(i) > > > > where foo has side-effects is pretty normal. > > Actually I've changed my mind on this. I think your version is > better because the side-effect of rcu_dereference will cause the > above assignment to occur twice when i refers to a memory-backed > variable. > > Since all current prefetch implementations are safe as far as > side-effects are concerned, here is an updated version that > doesn't do i = foo(i). > > Andrew, please replace the previous version with this. Looks great to me! Thanx, Paul Acked-by: <paulmck@us.ibm.com> > Thanks, > -- > Visit Openswan at http://www.openswan.org/ > Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au> > Home Page: http://gondor.apana.org.au/~herbert/ > PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt > diff --git a/include/linux/list.h b/include/linux/list.h > --- a/include/linux/list.h > +++ b/include/linux/list.h > @@ -442,12 +442,14 @@ static inline void list_splice_init(stru > * as long as the traversal is guarded by rcu_read_lock(). > */ > #define list_for_each_rcu(pos, head) \ > - for (pos = (head)->next; prefetch(pos->next), pos != (head); \ > - pos = rcu_dereference(pos->next)) > + for (pos = (head)->next; \ > + prefetch(rcu_dereference(pos)->next), pos != (head); \ > + pos = pos->next) > > #define __list_for_each_rcu(pos, head) \ > - for (pos = (head)->next; pos != (head); \ > - pos = rcu_dereference(pos->next)) > + for (pos = (head)->next; \ > + rcu_dereference(pos) != (head); \ > + pos = pos->next) > > /** > * list_for_each_safe_rcu - iterate over an rcu-protected list safe > @@ -461,8 +463,9 @@ static inline void list_splice_init(stru > * as long as the traversal is guarded by rcu_read_lock(). > */ > #define list_for_each_safe_rcu(pos, n, head) \ > - for (pos = (head)->next, n = pos->next; pos != (head); \ > - pos = rcu_dereference(n), n = pos->next) > + for (pos = (head)->next; \ > + n = rcu_dereference(pos)->next, pos != (head); \ > + pos = n) > > /** > * list_for_each_entry_rcu - iterate over rcu list of given type > @@ -474,11 +477,11 @@ static inline void list_splice_init(stru > * the _rcu list-mutation primitives such as list_add_rcu() > * as long as the traversal is guarded by rcu_read_lock(). > */ > -#define list_for_each_entry_rcu(pos, head, member) \ > - for (pos = list_entry((head)->next, typeof(*pos), member); \ > - prefetch(pos->member.next), &pos->member != (head); \ > - pos = rcu_dereference(list_entry(pos->member.next, \ > - typeof(*pos), member))) > +#define list_for_each_entry_rcu(pos, head, member) \ > + for (pos = list_entry((head)->next, typeof(*pos), member); \ > + prefetch(rcu_dereference(pos)->member.next), \ > + &pos->member != (head); \ > + pos = list_entry(pos->member.next, typeof(*pos), member)) > > > /** > @@ -492,8 +495,9 @@ static inline void list_splice_init(stru > * as long as the traversal is guarded by rcu_read_lock(). > */ > #define list_for_each_continue_rcu(pos, head) \ > - for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \ > - (pos) = rcu_dereference((pos)->next)) > + for ((pos) = (pos)->next; \ > + prefetch(rcu_dereference((pos))->next), (pos) != (head); \ > + (pos) = (pos)->next) > > /* > * Double linked lists with a single pointer list head. > @@ -696,8 +700,9 @@ static inline void hlist_add_after_rcu(s > pos = n) > > #define hlist_for_each_rcu(pos, head) \ > - for ((pos) = (head)->first; pos && ({ prefetch((pos)->next); 1; }); \ > - (pos) = rcu_dereference((pos)->next)) > + for ((pos) = (head)->first; \ > + rcu_dereference((pos)) && ({ prefetch((pos)->next); 1; }); \ > + (pos) = (pos)->next) > > /** > * hlist_for_each_entry - iterate over list of given type > @@ -762,9 +767,9 @@ static inline void hlist_add_after_rcu(s > */ > #define hlist_for_each_entry_rcu(tpos, pos, head, member) \ > for (pos = (head)->first; \ > - pos && ({ prefetch(pos->next); 1;}) && \ > + rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) && \ > ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ > - pos = rcu_dereference(pos->next)) > + pos = pos->next) > > #else > #warning "don't include kernel headers in userspace" - To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
diff --git a/include/linux/list.h b/include/linux/list.h --- a/include/linux/list.h +++ b/include/linux/list.h @@ -442,12 +442,15 @@ static inline void list_splice_init(stru * as long as the traversal is guarded by rcu_read_lock(). */ #define list_for_each_rcu(pos, head) \ - for (pos = (head)->next; prefetch(pos->next), pos != (head); \ - pos = rcu_dereference(pos->next)) + for (pos = (head)->next; \ + pos = rcu_dereference(pos), \ + prefetch(pos->next), pos != (head); \ + pos = pos->next) #define __list_for_each_rcu(pos, head) \ - for (pos = (head)->next; pos != (head); \ - pos = rcu_dereference(pos->next)) + for (pos = (head)->next; \ + rcu_dereference(pos) != (head); \ + pos = pos->next) /** * list_for_each_safe_rcu - iterate over an rcu-protected list safe @@ -461,8 +464,9 @@ static inline void list_splice_init(stru * as long as the traversal is guarded by rcu_read_lock(). */ #define list_for_each_safe_rcu(pos, n, head) \ - for (pos = (head)->next, n = pos->next; pos != (head); \ - pos = rcu_dereference(n), n = pos->next) + for (pos = (head)->next; \ + n = rcu_dereference(pos)->next, pos != (head); \ + pos = n) /** * list_for_each_entry_rcu - iterate over rcu list of given type @@ -474,11 +478,11 @@ static inline void list_splice_init(stru * the _rcu list-mutation primitives such as list_add_rcu() * as long as the traversal is guarded by rcu_read_lock(). */ -#define list_for_each_entry_rcu(pos, head, member) \ - for (pos = list_entry((head)->next, typeof(*pos), member); \ - prefetch(pos->member.next), &pos->member != (head); \ - pos = rcu_dereference(list_entry(pos->member.next, \ - typeof(*pos), member))) +#define list_for_each_entry_rcu(pos, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member); \ + pos = rcu_dereference(pos), \ + prefetch(pos->member.next), &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) /** @@ -492,8 +496,10 @@ static inline void list_splice_init(stru * as long as the traversal is guarded by rcu_read_lock(). */ #define list_for_each_continue_rcu(pos, head) \ - for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \ - (pos) = rcu_dereference((pos)->next)) + for ((pos) = (pos)->next; \ + (pos) = rcu_dereference((pos)), \ + prefetch((pos)->next), (pos) != (head); \ + (pos) = (pos)->next) /* * Double linked lists with a single pointer list head. @@ -696,8 +702,9 @@ static inline void hlist_add_after_rcu(s pos = n) #define hlist_for_each_rcu(pos, head) \ - for ((pos) = (head)->first; pos && ({ prefetch((pos)->next); 1; }); \ - (pos) = rcu_dereference((pos)->next)) + for ((pos) = (head)->first; \ + rcu_dereference((pos)) && ({ prefetch((pos)->next); 1; }); \ + (pos) = (pos)->next) /** * hlist_for_each_entry - iterate over list of given type @@ -762,9 +769,9 @@ static inline void hlist_add_after_rcu(s */ #define hlist_for_each_entry_rcu(tpos, pos, head, member) \ for (pos = (head)->first; \ - pos && ({ prefetch(pos->next); 1;}) && \ + rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) && \ ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = rcu_dereference(pos->next)) + pos = pos->next) #else #warning "don't include kernel headers in userspace"
Hi: It seems that all the list_*_rcu primitives are missing a memory barrier on the very first dereference. For example, #define list_for_each_rcu(pos, head) \ for (pos = (head)->next; prefetch(pos->next), pos != (head); \ pos = rcu_dereference(pos->next)) It will go something like: pos = (head)->next prefetch(pos->next) pos != (head) do stuff We're missing a barrier here. pos = rcu_dereference(pos->next) fetch pos->next barrier given by rcu_dereference(pos->next) store pos Without the missing barrier, the pos->next value may turn out to be stale. In fact, if "do stuff" were also dereferencing pos and relying on list_for_each_rcu to provide the barrier then it may also break. So here is a patch to make sure that we have a barrier for the first element in the list. Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> Cheers,