[PATCH v2 0/3] Toyrcu: replace plain accesses with READ_ONCE() and

[PATCH v2 0/3] Toyrcu: replace plain accesses with READ_ONCE() and

[Junchang Wang]

Hi Paul and Akira,

This patch set replaces plain accesses and ACCESS_ONCE() with READ_ONCE()
and WRITE_ONCE() in Toy RCU examples. I will send another patch updating
tex file accordingly on top of this patch set.

--
Junchang Wang (3):
  rcu: Use READ_ONCE() and WRITE_ONCE() for shared variable rcu_gp_ctr
  rcu_nest: Use READ_ONCE() and WRITE_ONCE() for shared variable
    rcu_gp_ctr
  rcu_qs: Use READ_ONCE() AND WRITE_ONCE() for shared variable
    rcu_gp_ctr

 CodeSamples/defer/rcu.c      | 2 +-
 CodeSamples/defer/rcu.h      | 4 ++--
 CodeSamples/defer/rcu_nest.c | 2 +-
 CodeSamples/defer/rcu_nest.h | 2 +-
 CodeSamples/defer/rcu_qs.c   | 2 +-
 CodeSamples/defer/rcu_qs.h   | 4 ++--
 6 files changed, 8 insertions(+), 8 deletions(-)

-- 
2.7.4

[PATCH v2 1/3] rcu: Use READ_ONCE() and WRITE_ONCE() for shared variable rcu_gp_ctr

[Junchang Wang]

Signed-off-by: Junchang Wang <junchangwang@gmail.com>
Reviewed-by: Akira Yokosawa <akiyks@gmail.com>
---
 CodeSamples/defer/rcu.c | 2 +-
 CodeSamples/defer/rcu.h | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CodeSamples/defer/rcu.c b/CodeSamples/defer/rcu.c
index 4b868cc..66792e5 100644
--- a/CodeSamples/defer/rcu.c
+++ b/CodeSamples/defer/rcu.c
@@ -35,7 +35,7 @@ void synchronize_rcu(void)
 
 	/* Advance to a new grace-period number, enforce ordering. */
 
-	rcu_gp_ctr += 2;
+	WRITE_ONCE(rcu_gp_ctr, rcu_gp_ctr + 2);
 	smp_mb();
 
 	/*
diff --git a/CodeSamples/defer/rcu.h b/CodeSamples/defer/rcu.h
index f493f8b..b49ab14 100644
--- a/CodeSamples/defer/rcu.h
+++ b/CodeSamples/defer/rcu.h
@@ -41,7 +41,7 @@ static inline void rcu_read_lock(void)
 	 * periodic per-thread processing.)
 	 */
 
-	__get_thread_var(rcu_reader_gp) = rcu_gp_ctr + 1;
+	__get_thread_var(rcu_reader_gp) = READ_ONCE(rcu_gp_ctr) + 1;
 	smp_mb();
 }
 
@@ -55,7 +55,7 @@ static inline void rcu_read_unlock(void)
 	 */
 
 	smp_mb();
-	__get_thread_var(rcu_reader_gp) = rcu_gp_ctr;
+	__get_thread_var(rcu_reader_gp) = READ_ONCE(rcu_gp_ctr);
 }
 
 extern void synchronize_rcu(void);
-- 
2.7.4

[PATCH v2 2/3] rcu_nest: Use READ_ONCE() and WRITE_ONCE() for shared variable rcu_gp_ctr

[Junchang Wang]

Signed-off-by: Junchang Wang <junchangwang@gmail.com>
Reviewed-by: Akira Yokosawa <akiyks@gmail.com>
---
 CodeSamples/defer/rcu_nest.c | 2 +-
 CodeSamples/defer/rcu_nest.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CodeSamples/defer/rcu_nest.c b/CodeSamples/defer/rcu_nest.c
index 64e4087..733b01a 100644
--- a/CodeSamples/defer/rcu_nest.c
+++ b/CodeSamples/defer/rcu_nest.c
@@ -35,7 +35,7 @@ void synchronize_rcu(void)
 
 	/* Advance to a new grace-period number, enforce ordering. */
 
-	rcu_gp_ctr += RCU_GP_CTR_BOTTOM_BIT;
+	WRITE_ONCE(rcu_gp_ctr, rcu_gp_ctr + RCU_GP_CTR_BOTTOM_BIT);
 	smp_mb();
 
 	/*
diff --git a/CodeSamples/defer/rcu_nest.h b/CodeSamples/defer/rcu_nest.h
index bcc4cde..64d9679 100644
--- a/CodeSamples/defer/rcu_nest.h
+++ b/CodeSamples/defer/rcu_nest.h
@@ -52,7 +52,7 @@ static void rcu_read_lock(void)
 retry:
 	tmp = *rrgp;
 	if ((tmp & RCU_GP_CTR_NEST_MASK) == 0)
-		tmp = rcu_gp_ctr;
+		tmp = READ_ONCE(rcu_gp_ctr);
 	tmp++;
 	*rrgp = tmp;
 	smp_mb();
-- 
2.7.4

[PATCH v2 3/3] rcu_qs: Use READ_ONCE() AND WRITE_ONCE() for shared variable rcu_gp_ctr

[Junchang Wang]

Signed-off-by: Junchang Wang <junchangwang@gmail.com>
Reviewed-by: Akira Yokosawa <akiyks@gmail.com>
---
 CodeSamples/defer/rcu_qs.c | 2 +-
 CodeSamples/defer/rcu_qs.h | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CodeSamples/defer/rcu_qs.c b/CodeSamples/defer/rcu_qs.c
index 27e45f7..f3d0cd6 100644
--- a/CodeSamples/defer/rcu_qs.c
+++ b/CodeSamples/defer/rcu_qs.c
@@ -41,7 +41,7 @@ void synchronize_rcu(void)
 
 	/* Advance to a new grace-period number, enforce ordering. */
 
-	rcu_gp_ctr += 2;
+	WRITE_ONCE(rcu_gp_ctr, rcu_gp_ctr + 2);
 	smp_mb();
 
 	/*
diff --git a/CodeSamples/defer/rcu_qs.h b/CodeSamples/defer/rcu_qs.h
index 3bb45a4..d8be43e 100644
--- a/CodeSamples/defer/rcu_qs.h
+++ b/CodeSamples/defer/rcu_qs.h
@@ -59,14 +59,14 @@ static void rcu_read_unlock(void)
 static void rcu_quiescent_state(void)
 {
 	smp_mb();
-	__get_thread_var(rcu_reader_qs_gp) = ACCESS_ONCE(rcu_gp_ctr) + 1;
+	__get_thread_var(rcu_reader_qs_gp) = READ_ONCE(rcu_gp_ctr) + 1;
 	smp_mb();
 }
 
 static void rcu_thread_offline(void)
 {
 	smp_mb();
-	__get_thread_var(rcu_reader_qs_gp) = ACCESS_ONCE(rcu_gp_ctr);
+	__get_thread_var(rcu_reader_qs_gp) = READ_ONCE(rcu_gp_ctr);
 	smp_mb();
 }
 
-- 
2.7.4

Re: [PATCH v2 0/3] Toyrcu: replace plain accesses with READ_ONCE() and

[Akira Yokosawa]

Hi Junchang,

On Thu, 25 Apr 2019 16:29:51 +0800, Junchang Wang wrote:
> Hi Paul and Akira,
> 
> This patch set replaces plain accesses and ACCESS_ONCE() with READ_ONCE()
> and WRITE_ONCE() in Toy RCU examples. I will send another patch updating
> tex file accordingly on top of this patch set.

As far as accesses to rcu_gp_ctr, this patch set looks good to me.

In updating toyrcu.tex file, I suggested to extract the snippets from
code under CodeSamples/defer, but some snippets in toyrcu.tex consist of
code from both .h and .c. Current scheme does not support such merging
of snippets from multiple source files.

Your option is to manually convert verbbox environment to Verbatim
environment. Sorry I suggested infeasible extractions the other day.
rcu_qs.c and rcu_qs.h are in separate snippets, and they can be
extracted by fcvextract.pl. In this case, commit 4a41491166cd
("defer/hazptr: Extract snippet from hazptr.c") should be a good example,
especially the "gobbleblank=yes" option.

If you have any question, feel free to ask me. I'm looking forward to
seeing your update.

Paul, in the sources Junchang has touched, per-thread variable
"rcu_reader_gp" is updated by its owner, but is read accessed from
for_each_thread loops in updater side. So I think accesses to each of
them need to be annotated by WRITE_ONCE (in owner) and READ_ONCE (in updater).
I looked at only the diffs in the 1st patch set and didn't notice the raciness
of them.

Macros for accessing per-thread variables are expanded to array references,
and there are "barrier()"s in these loops.
So I think there is little chance those accesses would be optimized by
currently available compilers.

Do you see the need to annotate the accesses to per-thread variable?

        Thanks, Akira 

> 
> --
> Junchang Wang (3):
>   rcu: Use READ_ONCE() and WRITE_ONCE() for shared variable rcu_gp_ctr
>   rcu_nest: Use READ_ONCE() and WRITE_ONCE() for shared variable
>     rcu_gp_ctr
>   rcu_qs: Use READ_ONCE() AND WRITE_ONCE() for shared variable
>     rcu_gp_ctr
> 
>  CodeSamples/defer/rcu.c      | 2 +-
>  CodeSamples/defer/rcu.h      | 4 ++--
>  CodeSamples/defer/rcu_nest.c | 2 +-
>  CodeSamples/defer/rcu_nest.h | 2 +-
>  CodeSamples/defer/rcu_qs.c   | 2 +-
>  CodeSamples/defer/rcu_qs.h   | 4 ++--
>  6 files changed, 8 insertions(+), 8 deletions(-)
> 

Re: [PATCH v2 0/3] Toyrcu: replace plain accesses with READ_ONCE() and

[Junchang Wang]

On Thu, Apr 25, 2019 at 11:38 PM Akira Yokosawa <akiyks@gmail.com> wrote:
>
> Hi Junchang,
>
> On Thu, 25 Apr 2019 16:29:51 +0800, Junchang Wang wrote:
> > Hi Paul and Akira,
> >
> > This patch set replaces plain accesses and ACCESS_ONCE() with READ_ONCE()
> > and WRITE_ONCE() in Toy RCU examples. I will send another patch updating
> > tex file accordingly on top of this patch set.
>
> As far as accesses to rcu_gp_ctr, this patch set looks good to me.
>
> In updating toyrcu.tex file, I suggested to extract the snippets from
> code under CodeSamples/defer, but some snippets in toyrcu.tex consist of
> code from both .h and .c. Current scheme does not support such merging
> of snippets from multiple source files.
>
> Your option is to manually convert verbbox environment to Verbatim
> environment. Sorry I suggested infeasible extractions the other day.
> rcu_qs.c and rcu_qs.h are in separate snippets, and they can be
> extracted by fcvextract.pl. In this case, commit 4a41491166cd
> ("defer/hazptr: Extract snippet from hazptr.c") should be a good example,
> especially the "gobbleblank=yes" option.
>
> If you have any question, feel free to ask me. I'm looking forward to
> seeing your update.

Hi Akira,

Thanks for the note. Would be happy to submit a new patch for
toyrcu.tex according to your suggestions soon.

>
> Paul, in the sources Junchang has touched, per-thread variable
> "rcu_reader_gp" is updated by its owner, but is read accessed from
> for_each_thread loops in updater side. So I think accesses to each of
> them need to be annotated by WRITE_ONCE (in owner) and READ_ONCE (in updater).
> I looked at only the diffs in the 1st patch set and didn't notice the raciness
> of them.
>
> Macros for accessing per-thread variables are expanded to array references,
> and there are "barrier()"s in these loops.
> So I think there is little chance those accesses would be optimized by
> currently available compilers.
>

Great catch! My understanding is that one of the nice benefits of
READ_ONCE/WRITE_ONCE is that they explicitly notify programmers the
spots where there might be racy accesses. The more we eliminate them,
the harder a programmer verifies the code. Of course, that's my
current point of view. I would be happy to see your choices.

Thanks,
--Junchang

> Do you see the need to annotate the accesses to per-thread variable?
>
>         Thanks, Akira
>
> >
> > --
> > Junchang Wang (3):
> >   rcu: Use READ_ONCE() and WRITE_ONCE() for shared variable rcu_gp_ctr
> >   rcu_nest: Use READ_ONCE() and WRITE_ONCE() for shared variable
> >     rcu_gp_ctr
> >   rcu_qs: Use READ_ONCE() AND WRITE_ONCE() for shared variable
> >     rcu_gp_ctr
> >
> >  CodeSamples/defer/rcu.c      | 2 +-
> >  CodeSamples/defer/rcu.h      | 4 ++--
> >  CodeSamples/defer/rcu_nest.c | 2 +-
> >  CodeSamples/defer/rcu_nest.h | 2 +-
> >  CodeSamples/defer/rcu_qs.c   | 2 +-
> >  CodeSamples/defer/rcu_qs.h   | 4 ++--
> >  6 files changed, 8 insertions(+), 8 deletions(-)
> >
>

Re: [PATCH v2 0/3] Toyrcu: replace plain accesses with READ_ONCE() and

[Paul E. McKenney]

On Fri, Apr 26, 2019 at 12:38:40AM +0900, Akira Yokosawa wrote:
> Hi Junchang,
> 
> On Thu, 25 Apr 2019 16:29:51 +0800, Junchang Wang wrote:
> > Hi Paul and Akira,
> > 
> > This patch set replaces plain accesses and ACCESS_ONCE() with READ_ONCE()
> > and WRITE_ONCE() in Toy RCU examples. I will send another patch updating
> > tex file accordingly on top of this patch set.
> 
> As far as accesses to rcu_gp_ctr, this patch set looks good to me.
> 
> In updating toyrcu.tex file, I suggested to extract the snippets from
> code under CodeSamples/defer, but some snippets in toyrcu.tex consist of
> code from both .h and .c. Current scheme does not support such merging
> of snippets from multiple source files.
> 
> Your option is to manually convert verbbox environment to Verbatim
> environment. Sorry I suggested infeasible extractions the other day.
> rcu_qs.c and rcu_qs.h are in separate snippets, and they can be
> extracted by fcvextract.pl. In this case, commit 4a41491166cd
> ("defer/hazptr: Extract snippet from hazptr.c") should be a good example,
> especially the "gobbleblank=yes" option.

Given that these are toy implementations, pleaee feel free to reorganize
the code to make snippet extraction work better.  (Please check to see
if anything else uses these -- I do not believe so, but memory can be
a tricky thing sometimes.)

> If you have any question, feel free to ask me. I'm looking forward to
> seeing your update.
> 
> Paul, in the sources Junchang has touched, per-thread variable
> "rcu_reader_gp" is updated by its owner, but is read accessed from
> for_each_thread loops in updater side. So I think accesses to each of
> them need to be annotated by WRITE_ONCE (in owner) and READ_ONCE (in updater).
> I looked at only the diffs in the 1st patch set and didn't notice the raciness
> of them.

Agreed, WRITE_ONCE() by owner and READ_ONCE() by updater.

> Macros for accessing per-thread variables are expanded to array references,
> and there are "barrier()"s in these loops.
> So I think there is little chance those accesses would be optimized by
> currently available compilers.
> 
> Do you see the need to annotate the accesses to per-thread variable?

If a per-thread variable is accessed by multiple threads, it would be
good to annotate the accesses as needed.  That said, I have no idea
how well annotation interacts with the per-thread accesses, so some
experimentation might be required.

							Thnax, Paul

>         Thanks, Akira 
> 
> > 
> > --
> > Junchang Wang (3):
> >   rcu: Use READ_ONCE() and WRITE_ONCE() for shared variable rcu_gp_ctr
> >   rcu_nest: Use READ_ONCE() and WRITE_ONCE() for shared variable
> >     rcu_gp_ctr
> >   rcu_qs: Use READ_ONCE() AND WRITE_ONCE() for shared variable
> >     rcu_gp_ctr
> > 
> >  CodeSamples/defer/rcu.c      | 2 +-
> >  CodeSamples/defer/rcu.h      | 4 ++--
> >  CodeSamples/defer/rcu_nest.c | 2 +-
> >  CodeSamples/defer/rcu_nest.h | 2 +-
> >  CodeSamples/defer/rcu_qs.c   | 2 +-
> >  CodeSamples/defer/rcu_qs.h   | 4 ++--
> >  6 files changed, 8 insertions(+), 8 deletions(-)
> > 
> 

Re: [PATCH v2 0/3] Toyrcu: replace plain accesses with READ_ONCE() and

[Akira Yokosawa]

Hi Junchang, Paul,

On Fri, 26 Apr 2019 06:43:42 -0700, Paul E. McKenney wrote:
> On Fri, Apr 26, 2019 at 12:38:40AM +0900, Akira Yokosawa wrote:
>> Hi Junchang,
>>
>> On Thu, 25 Apr 2019 16:29:51 +0800, Junchang Wang wrote:
>>> Hi Paul and Akira,
>>>
>>> This patch set replaces plain accesses and ACCESS_ONCE() with READ_ONCE()
>>> and WRITE_ONCE() in Toy RCU examples. I will send another patch updating
>>> tex file accordingly on top of this patch set.
>>
>> As far as accesses to rcu_gp_ctr, this patch set looks good to me.
>>
>> In updating toyrcu.tex file, I suggested to extract the snippets from
>> code under CodeSamples/defer, but some snippets in toyrcu.tex consist of
>> code from both .h and .c. Current scheme does not support such merging
>> of snippets from multiple source files.

As a matter of fact, it turns out this is actually possible.

Please see the patch below, which converts Listing B.1.

Junchang, if it is not too late, you can use the same approach to
snippets from rcu.[ch] and rcu_nest.[ch].

        Thanks, Akira

-----8<-----------------------
From 3b3c6d99bcd580385175bf2a062dcb038f0cfc9e Mon Sep 17 00:00:00 2001
From: Akira Yokosawa <akiyks@gmail.com>
Date: Sat, 27 Apr 2019 22:34:40 +0900
Subject: [PATCH] toyrcu: Convert Listing B.1 to new scheme

This commit is an example where code extracted from two source
files under CodeSamples/ is merged into a code snippet.
Extra vertical space needs to be canceled by the \vspace*{} command
between the two \input{} commands.
\fvset{firstnumber=last} is to continue line numbering.
\fvset{firstnumber=auto} restores the default setting.

Also tweak positioning option of listing environment.

Signed-off-by: Akira Yokosawa <akiyks@gmail.com>
---
 CodeSamples/defer/rcu_lock.c |  2 ++
 CodeSamples/defer/rcu_lock.h |  2 ++
 appendix/toyrcu/toyrcu.tex   | 26 ++++----------------------
 3 files changed, 8 insertions(+), 22 deletions(-)

diff --git a/CodeSamples/defer/rcu_lock.c b/CodeSamples/defer/rcu_lock.c
index 8d45ac89..08b87fbd 100644
--- a/CodeSamples/defer/rcu_lock.c
+++ b/CodeSamples/defer/rcu_lock.c
@@ -20,12 +20,14 @@
 
 #include "../api.h"
 #include "rcu_lock.h"
+//\begin{snippet}[labelbase=ln:defer:rcu_lock:synchronize,commandchars=\\\[\]]
 
 void synchronize_rcu(void)
 {
 	spin_lock(&rcu_gp_lock);
 	spin_unlock(&rcu_gp_lock);
 }
+//\end{snippet}
 
 #ifdef TEST
 #include "rcutorture.h"
diff --git a/CodeSamples/defer/rcu_lock.h b/CodeSamples/defer/rcu_lock.h
index 224ea0e4..bf6f9b3c 100644
--- a/CodeSamples/defer/rcu_lock.h
+++ b/CodeSamples/defer/rcu_lock.h
@@ -26,6 +26,7 @@ static void rcu_init(void)
 {
 }
 
+//\begin{snippet}[labelbase=ln:defer:rcu_lock:lock_unlock,commandchars=\\\[\]]
 static void rcu_read_lock(void)
 {
 	spin_lock(&rcu_gp_lock);
@@ -35,5 +36,6 @@ static void rcu_read_unlock(void)
 {
 	spin_unlock(&rcu_gp_lock);
 }
+//\end{snippet}
 
 extern void synchronize_rcu(void);
diff --git a/appendix/toyrcu/toyrcu.tex b/appendix/toyrcu/toyrcu.tex
index c0a45a85..f7d6ec7f 100644
--- a/appendix/toyrcu/toyrcu.tex
+++ b/appendix/toyrcu/toyrcu.tex
@@ -30,29 +30,11 @@ provides a summary and a list of desirable RCU properties.
 
 \section{Lock-Based RCU}
 \label{sec:app:toyrcu:Lock-Based RCU}
+\NoIndentAfterThis
 
-\begin{listing}[bp]
-{ \scriptsize
-\begin{verbbox}
-  1 static void rcu_read_lock(void)
-  2 {
-  3   spin_lock(&rcu_gp_lock);
-  4 }
-  5
-  6 static void rcu_read_unlock(void)
-  7 {
-  8   spin_unlock(&rcu_gp_lock);
-  9 }
- 10
- 11 void synchronize_rcu(void)
- 12 {
- 13   spin_lock(&rcu_gp_lock);
- 14   spin_unlock(&rcu_gp_lock);
- 15 }
-\end{verbbox}
-}
-\centering
-\theverbbox
+\begin{listing}[htbp]
+\input{CodeSamples/defer/rcu_lock@lock_unlock.fcv}\vspace*{-11pt}\fvset{firstnumber=last}
+\input{CodeSamples/defer/rcu_lock@synchronize.fcv}\fvset{firstnumber=auto}
 \caption{Lock-Based RCU Implementation}
 \label{lst:app:toyrcu:Lock-Based RCU Implementation}
 \end{listing}
-- 
2.17.1

Re: [PATCH v2 0/3] Toyrcu: replace plain accesses with READ_ONCE() and

[Junchang Wang]

On Sat, Apr 27, 2019 at 10:27 PM Akira Yokosawa <akiyks@gmail.com> wrote:
>
> Hi Junchang, Paul,
>
> On Fri, 26 Apr 2019 06:43:42 -0700, Paul E. McKenney wrote:
> > On Fri, Apr 26, 2019 at 12:38:40AM +0900, Akira Yokosawa wrote:
> >> Hi Junchang,
> >>
> >> On Thu, 25 Apr 2019 16:29:51 +0800, Junchang Wang wrote:
> >>> Hi Paul and Akira,
> >>>
> >>> This patch set replaces plain accesses and ACCESS_ONCE() with READ_ONCE()
> >>> and WRITE_ONCE() in Toy RCU examples. I will send another patch updating
> >>> tex file accordingly on top of this patch set.
> >>
> >> As far as accesses to rcu_gp_ctr, this patch set looks good to me.
> >>
> >> In updating toyrcu.tex file, I suggested to extract the snippets from
> >> code under CodeSamples/defer, but some snippets in toyrcu.tex consist of
> >> code from both .h and .c. Current scheme does not support such merging
> >> of snippets from multiple source files.
>
> As a matter of fact, it turns out this is actually possible.
>
> Please see the patch below, which converts Listing B.1.
>
> Junchang, if it is not too late, you can use the same approach to
> snippets from rcu.[ch] and rcu_nest.[ch].
>

Hi Akira,

Thanks for the patch, which works well for rcu.[ch]. I can apply this
scheme to other examples and submit the patch set tomorrow.

Thanks,
--Junchang

>         Thanks, Akira
>
> -----8<-----------------------
> From 3b3c6d99bcd580385175bf2a062dcb038f0cfc9e Mon Sep 17 00:00:00 2001
> From: Akira Yokosawa <akiyks@gmail.com>
> Date: Sat, 27 Apr 2019 22:34:40 +0900
> Subject: [PATCH] toyrcu: Convert Listing B.1 to new scheme
>
> This commit is an example where code extracted from two source
> files under CodeSamples/ is merged into a code snippet.
> Extra vertical space needs to be canceled by the \vspace*{} command
> between the two \input{} commands.
> \fvset{firstnumber=last} is to continue line numbering.
> \fvset{firstnumber=auto} restores the default setting.
>
> Also tweak positioning option of listing environment.
>
> Signed-off-by: Akira Yokosawa <akiyks@gmail.com>
> ---
>  CodeSamples/defer/rcu_lock.c |  2 ++
>  CodeSamples/defer/rcu_lock.h |  2 ++
>  appendix/toyrcu/toyrcu.tex   | 26 ++++----------------------
>  3 files changed, 8 insertions(+), 22 deletions(-)
>
> diff --git a/CodeSamples/defer/rcu_lock.c b/CodeSamples/defer/rcu_lock.c
> index 8d45ac89..08b87fbd 100644
> --- a/CodeSamples/defer/rcu_lock.c
> +++ b/CodeSamples/defer/rcu_lock.c
> @@ -20,12 +20,14 @@
>
>  #include "../api.h"
>  #include "rcu_lock.h"
> +//\begin{snippet}[labelbase=ln:defer:rcu_lock:synchronize,commandchars=\\\[\]]
>
>  void synchronize_rcu(void)
>  {
>         spin_lock(&rcu_gp_lock);
>         spin_unlock(&rcu_gp_lock);
>  }
> +//\end{snippet}
>
>  #ifdef TEST
>  #include "rcutorture.h"
> diff --git a/CodeSamples/defer/rcu_lock.h b/CodeSamples/defer/rcu_lock.h
> index 224ea0e4..bf6f9b3c 100644
> --- a/CodeSamples/defer/rcu_lock.h
> +++ b/CodeSamples/defer/rcu_lock.h
> @@ -26,6 +26,7 @@ static void rcu_init(void)
>  {
>  }
>
> +//\begin{snippet}[labelbase=ln:defer:rcu_lock:lock_unlock,commandchars=\\\[\]]
>  static void rcu_read_lock(void)
>  {
>         spin_lock(&rcu_gp_lock);
> @@ -35,5 +36,6 @@ static void rcu_read_unlock(void)
>  {
>         spin_unlock(&rcu_gp_lock);
>  }
> +//\end{snippet}
>
>  extern void synchronize_rcu(void);
> diff --git a/appendix/toyrcu/toyrcu.tex b/appendix/toyrcu/toyrcu.tex
> index c0a45a85..f7d6ec7f 100644
> --- a/appendix/toyrcu/toyrcu.tex
> +++ b/appendix/toyrcu/toyrcu.tex
> @@ -30,29 +30,11 @@ provides a summary and a list of desirable RCU properties.
>
>  \section{Lock-Based RCU}
>  \label{sec:app:toyrcu:Lock-Based RCU}
> +\NoIndentAfterThis
>
> -\begin{listing}[bp]
> -{ \scriptsize
> -\begin{verbbox}
> -  1 static void rcu_read_lock(void)
> -  2 {
> -  3   spin_lock(&rcu_gp_lock);
> -  4 }
> -  5
> -  6 static void rcu_read_unlock(void)
> -  7 {
> -  8   spin_unlock(&rcu_gp_lock);
> -  9 }
> - 10
> - 11 void synchronize_rcu(void)
> - 12 {
> - 13   spin_lock(&rcu_gp_lock);
> - 14   spin_unlock(&rcu_gp_lock);
> - 15 }
> -\end{verbbox}
> -}
> -\centering
> -\theverbbox
> +\begin{listing}[htbp]
> +\input{CodeSamples/defer/rcu_lock@lock_unlock.fcv}\vspace*{-11pt}\fvset{firstnumber=last}
> +\input{CodeSamples/defer/rcu_lock@synchronize.fcv}\fvset{firstnumber=auto}
>  \caption{Lock-Based RCU Implementation}
>  \label{lst:app:toyrcu:Lock-Based RCU Implementation}
>  \end{listing}
> --
> 2.17.1
>