[PATCH RFC 0/2] docs: Deal with some Sphinx deprecation warnings

[PATCH RFC 0/2] docs: Deal with some Sphinx deprecation warnings

[Jonathan Corbet]

The Sphinx folks are deprecating some interfaces in the upcoming 2.0
release; one immediate result of that is a bunch of warnings that show up
when building with 1.8.  These two patches make those warnings go away,
but at a cost:

 - It introduces a couple of Sphinx version checks, which are always
   ugly, but the alternative would be to stop supporting versions
   before 1.7.  For now, I think we can carry that cruft.

 - The second patch causes the build to fail horribly on newer
   Sphinx installations.  The change to switch_source_input() seems
   to make the parser much more finicky, increasing warnings and
   eventually failing the build altogether.  In particular, it will
   scream about problems in .rst files that are not included in the
   TOC tree at all.  The complaints appear to be legitimate, but it's
   a bunch of stuff to clean up.

I've tested these with 1.4 and 1.8, but not various versions in between.

Jonathan Corbet (2):
  doc: Cope with Sphinx logging deprecations
  doc: Cope with the deprecation of AutoReporter

 Documentation/sphinx/kerneldoc.py | 48 ++++++++++++++++++++++++-------
 Documentation/sphinx/kernellog.py | 28 ++++++++++++++++++
 Documentation/sphinx/kfigure.py   | 38 +++++++++++++-----------
 3 files changed, 87 insertions(+), 27 deletions(-)
 create mode 100644 Documentation/sphinx/kernellog.py

-- 
2.21.0

[PATCH 1/2] doc: Cope with Sphinx logging deprecations

[Jonathan Corbet]

Recent versions of sphinx will emit messages like:

  /stuff/k/git/kernel/Documentation/sphinx/kerneldoc.py:103:
     RemovedInSphinx20Warning: app.warning() is now deprecated.
     Use sphinx.util.logging instead.

Switch to sphinx.util.logging to make this unsightly message go away.
Alas, that interface was only added in version 1.6, so we have to add a
version check to keep things working with older sphinxes.
---
 Documentation/sphinx/kerneldoc.py | 12 ++++++----
 Documentation/sphinx/kernellog.py | 28 +++++++++++++++++++++++
 Documentation/sphinx/kfigure.py   | 38 ++++++++++++++++++-------------
 3 files changed, 58 insertions(+), 20 deletions(-)
 create mode 100644 Documentation/sphinx/kernellog.py

diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py
index 9d0a7f08f93b..e8891e63e001 100644
--- a/Documentation/sphinx/kerneldoc.py
+++ b/Documentation/sphinx/kerneldoc.py
@@ -39,6 +39,8 @@ from docutils.statemachine import ViewList
 from docutils.parsers.rst import directives, Directive
 from sphinx.ext.autodoc import AutodocReporter
 
+import kernellog
+
 __version__  = '1.0'
 
 class KernelDocDirective(Directive):
@@ -90,7 +92,8 @@ class KernelDocDirective(Directive):
         cmd += [filename]
 
         try:
-            env.app.verbose('calling kernel-doc \'%s\'' % (" ".join(cmd)))
+            kernellog.verbose(env.app,
+                              'calling kernel-doc \'%s\'' % (" ".join(cmd)))
 
             p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             out, err = p.communicate()
@@ -100,7 +103,8 @@ class KernelDocDirective(Directive):
             if p.returncode != 0:
                 sys.stderr.write(err)
 
-                env.app.warn('kernel-doc \'%s\' failed with return code %d' % (" ".join(cmd), p.returncode))
+                kernellog.warn(env.app,
+                               'kernel-doc \'%s\' failed with return code %d' % (" ".join(cmd), p.returncode))
                 return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))]
             elif env.config.kerneldoc_verbosity > 0:
                 sys.stderr.write(err)
@@ -132,8 +136,8 @@ class KernelDocDirective(Directive):
             return node.children
 
         except Exception as e:  # pylint: disable=W0703
-            env.app.warn('kernel-doc \'%s\' processing failed with: %s' %
-                         (" ".join(cmd), str(e)))
+            kernellog.warn(app, 'kernel-doc \'%s\' processing failed with: %s' %
+                           (" ".join(cmd), str(e)))
             return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))]
 
 def setup(app):
diff --git a/Documentation/sphinx/kernellog.py b/Documentation/sphinx/kernellog.py
new file mode 100644
index 000000000000..af924f51a7dc
--- /dev/null
+++ b/Documentation/sphinx/kernellog.py
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Sphinx has deprecated its older logging interface, but the replacement
+# only goes back to 1.6.  So here's a wrapper layer to keep around for
+# as long as we support 1.4.
+#
+import sphinx
+
+if sphinx.__version__[:3] >= '1.6':
+    UseLogging = True
+    from sphinx.util import logging
+    logger = logging.getLogger('kerneldoc')
+else:
+    UseLogging = False
+
+def warn(app, message):
+    if UseLogging:
+        logger.warning(message)
+    else:
+        app.warn(message)
+
+def verbose(app, message):
+    if UseLogging:
+        logger.verbose(message)
+    else:
+        app.verbose(message)
+
+
diff --git a/Documentation/sphinx/kfigure.py b/Documentation/sphinx/kfigure.py
index b97228d2cc0e..448c450fe70c 100644
--- a/Documentation/sphinx/kfigure.py
+++ b/Documentation/sphinx/kfigure.py
@@ -60,6 +60,8 @@ import sphinx
 from sphinx.util.nodes import clean_astext
 from six import iteritems
 
+import kernellog
+
 PY3 = sys.version_info[0] == 3
 
 if PY3:
@@ -171,20 +173,20 @@ def setupTools(app):
     This function is called once, when the builder is initiated.
     """
     global dot_cmd, convert_cmd   # pylint: disable=W0603
-    app.verbose("kfigure: check installed tools ...")
+    kernellog.verbose(app, "kfigure: check installed tools ...")
 
     dot_cmd = which('dot')
     convert_cmd = which('convert')
 
     if dot_cmd:
-        app.verbose("use dot(1) from: " + dot_cmd)
+        kernellog.verbose(app, "use dot(1) from: " + dot_cmd)
     else:
-        app.warn("dot(1) not found, for better output quality install "
-                 "graphviz from http://www.graphviz.org")
+        kernellog.warn(app, "dot(1) not found, for better output quality install "
+                       "graphviz from http://www.graphviz.org")
     if convert_cmd:
-        app.verbose("use convert(1) from: " + convert_cmd)
+        kernellog.verbose(app, "use convert(1) from: " + convert_cmd)
     else:
-        app.warn(
+        kernellog.warn(app,
             "convert(1) not found, for SVG to PDF conversion install "
             "ImageMagick (https://www.imagemagick.org)")
 
@@ -225,7 +227,8 @@ def convert_image(img_node, translator, src_fname=None):
     if in_ext == '.dot':
 
         if not dot_cmd:
-            app.verbose("dot from graphviz not available / include DOT raw.")
+            kernellog.verbose(app,
+                              "dot from graphviz not available / include DOT raw.")
             img_node.replace_self(file2literal(src_fname))
 
         elif translator.builder.format == 'latex':
@@ -252,7 +255,8 @@ def convert_image(img_node, translator, src_fname=None):
 
         if translator.builder.format == 'latex':
             if convert_cmd is None:
-                app.verbose("no SVG to PDF conversion available / include SVG raw.")
+                kernellog.verbose(app,
+                                  "no SVG to PDF conversion available / include SVG raw.")
                 img_node.replace_self(file2literal(src_fname))
             else:
                 dst_fname = path.join(translator.builder.outdir, fname + '.pdf')
@@ -265,18 +269,19 @@ def convert_image(img_node, translator, src_fname=None):
         _name = dst_fname[len(translator.builder.outdir) + 1:]
 
         if isNewer(dst_fname, src_fname):
-            app.verbose("convert: {out}/%s already exists and is newer" % _name)
+            kernellog.verbose(app,
+                              "convert: {out}/%s already exists and is newer" % _name)
 
         else:
             ok = False
             mkdir(path.dirname(dst_fname))
 
             if in_ext == '.dot':
-                app.verbose('convert DOT to: {out}/' + _name)
+                kernellog.verbose(app, 'convert DOT to: {out}/' + _name)
                 ok = dot2format(app, src_fname, dst_fname)
 
             elif in_ext == '.svg':
-                app.verbose('convert SVG to: {out}/' + _name)
+                kernellog.verbose(app, 'convert SVG to: {out}/' + _name)
                 ok = svg2pdf(app, src_fname, dst_fname)
 
             if not ok:
@@ -305,7 +310,8 @@ def dot2format(app, dot_fname, out_fname):
     with open(out_fname, "w") as out:
         exit_code = subprocess.call(cmd, stdout = out)
         if exit_code != 0:
-            app.warn("Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
+            kernellog.warn(app,
+                          "Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
     return bool(exit_code == 0)
 
 def svg2pdf(app, svg_fname, pdf_fname):
@@ -322,7 +328,7 @@ def svg2pdf(app, svg_fname, pdf_fname):
     # use stdout and stderr from parent
     exit_code = subprocess.call(cmd)
     if exit_code != 0:
-        app.warn("Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
+        kernellog.warn(app, "Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
     return bool(exit_code == 0)
 
 
@@ -415,15 +421,15 @@ def visit_kernel_render(self, node):
     app = self.builder.app
     srclang = node.get('srclang')
 
-    app.verbose('visit kernel-render node lang: "%s"' % (srclang))
+    kernellog.verbose(app, 'visit kernel-render node lang: "%s"' % (srclang))
 
     tmp_ext = RENDER_MARKUP_EXT.get(srclang, None)
     if tmp_ext is None:
-        app.warn('kernel-render: "%s" unknown / include raw.' % (srclang))
+        kernellog.warn(app, 'kernel-render: "%s" unknown / include raw.' % (srclang))
         return
 
     if not dot_cmd and tmp_ext == '.dot':
-        app.verbose("dot from graphviz not available / include raw.")
+        kernellog.verbose(app, "dot from graphviz not available / include raw.")
         return
 
     literal_block = node[0]
-- 
2.21.0

[PATCH 2/2] doc: Cope with the deprecation of AutoReporter

[Jonathan Corbet]

AutoReporter is going away; recent versions of sphinx emit a warning like:

  /stuff/k/git/kernel/Documentation/sphinx/kerneldoc.py:125:
      RemovedInSphinx20Warning: AutodocReporter is now deprecated.
      Use sphinx.util.docutils.switch_source_input() instead.

Make the switch.  But switch_source_input() only showed up in 1.7, so we
have to do ugly version checks to keep things working in older versions.
---
 Documentation/sphinx/kerneldoc.py | 38 ++++++++++++++++++++++++-------
 1 file changed, 30 insertions(+), 8 deletions(-)

diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py
index e8891e63e001..d3216f7b4170 100644
--- a/Documentation/sphinx/kerneldoc.py
+++ b/Documentation/sphinx/kerneldoc.py
@@ -37,7 +37,17 @@ import glob
 from docutils import nodes, statemachine
 from docutils.statemachine import ViewList
 from docutils.parsers.rst import directives, Directive
-from sphinx.ext.autodoc import AutodocReporter
+
+#
+# AutodocReporter is only good up to Sphinx 1.7
+#
+import sphinx
+
+Use_SSI = sphinx.__version__[:3] >= '1.7'
+if Use_SSI:
+    from sphinx.util.docutils import switch_source_input
+else:
+    from sphinx.ext.autodoc import AutodocReporter
 
 import kernellog
 
@@ -125,13 +135,7 @@ class KernelDocDirective(Directive):
                     lineoffset += 1
 
             node = nodes.section()
-            buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter
-            self.state.memo.reporter = AutodocReporter(result, self.state.memo.reporter)
-            self.state.memo.title_styles, self.state.memo.section_level = [], 0
-            try:
-                self.state.nested_parse(result, 0, node, match_titles=1)
-            finally:
-                self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf
+            self.do_parse(result, node)
 
             return node.children
 
@@ -140,6 +144,24 @@ class KernelDocDirective(Directive):
                            (" ".join(cmd), str(e)))
             return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))]
 
+    def do_parse(self, result, node):
+        if Use_SSI:
+            save = self.state.memo.title_styles, self.state.memo.section_level
+            try:
+                with switch_source_input(self.state, result):
+                    self.state.nested_parse(result, 0, node, match_titles=1)
+            finally:
+                self.state.memo.title_styles, self.state.memo.section_level = save
+        else:
+            save = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter
+            self.state.memo.reporter = AutodocReporter(result, self.state.memo.reporter)
+            self.state.memo.title_styles, self.state.memo.section_level = [], 0
+            try:
+                self.state.nested_parse(result, 0, node, match_titles=1)
+            finally:
+                self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = save
+
+
 def setup(app):
     app.add_config_value('kerneldoc_bin', None, 'env')
     app.add_config_value('kerneldoc_srctree', None, 'env')
-- 
2.21.0

Re: [PATCH RFC 0/2] docs: Deal with some Sphinx deprecation warnings

[Jani Nikula]

On Tue, 21 May 2019, Jonathan Corbet <corbet@lwn.net> wrote:
> The Sphinx folks are deprecating some interfaces in the upcoming 2.0
> release; one immediate result of that is a bunch of warnings that show up
> when building with 1.8.  These two patches make those warnings go away,
> but at a cost:
>
>  - It introduces a couple of Sphinx version checks, which are always
>    ugly, but the alternative would be to stop supporting versions
>    before 1.7.  For now, I think we can carry that cruft.

Frankly, I'd just require Sphinx 1.7+, available even in Debian stable
through stretch-backports.

>  - The second patch causes the build to fail horribly on newer
>    Sphinx installations.  The change to switch_source_input() seems
>    to make the parser much more finicky, increasing warnings and
>    eventually failing the build altogether.  In particular, it will
>    scream about problems in .rst files that are not included in the
>    TOC tree at all.  The complaints appear to be legitimate, but it's
>    a bunch of stuff to clean up.

I can understand Sphinx complaining that a file is not included in a TOC
tree, but I don't understand why it goes on to parse them anyway.

BR,
Jani.


>
> I've tested these with 1.4 and 1.8, but not various versions in between.
>
> Jonathan Corbet (2):
>   doc: Cope with Sphinx logging deprecations
>   doc: Cope with the deprecation of AutoReporter
>
>  Documentation/sphinx/kerneldoc.py | 48 ++++++++++++++++++++++++-------
>  Documentation/sphinx/kernellog.py | 28 ++++++++++++++++++
>  Documentation/sphinx/kfigure.py   | 38 +++++++++++++-----------
>  3 files changed, 87 insertions(+), 27 deletions(-)
>  create mode 100644 Documentation/sphinx/kernellog.py

-- 
Jani Nikula, Intel Open Source Graphics Center

Re: [PATCH 2/2] doc: Cope with the deprecation of AutoReporter

[Jani Nikula]

On Tue, 21 May 2019, Jonathan Corbet <corbet@lwn.net> wrote:
> AutoReporter is going away; recent versions of sphinx emit a warning like:
>
>   /stuff/k/git/kernel/Documentation/sphinx/kerneldoc.py:125:
>       RemovedInSphinx20Warning: AutodocReporter is now deprecated.
>       Use sphinx.util.docutils.switch_source_input() instead.
>
> Make the switch.  But switch_source_input() only showed up in 1.7, so we
> have to do ugly version checks to keep things working in older versions.
> ---
>  Documentation/sphinx/kerneldoc.py | 38 ++++++++++++++++++++++++-------
>  1 file changed, 30 insertions(+), 8 deletions(-)
>
> diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py
> index e8891e63e001..d3216f7b4170 100644
> --- a/Documentation/sphinx/kerneldoc.py
> +++ b/Documentation/sphinx/kerneldoc.py
> @@ -37,7 +37,17 @@ import glob
>  from docutils import nodes, statemachine
>  from docutils.statemachine import ViewList
>  from docutils.parsers.rst import directives, Directive
> -from sphinx.ext.autodoc import AutodocReporter
> +
> +#
> +# AutodocReporter is only good up to Sphinx 1.7
> +#
> +import sphinx
> +
> +Use_SSI = sphinx.__version__[:3] >= '1.7'
> +if Use_SSI:
> +    from sphinx.util.docutils import switch_source_input
> +else:
> +    from sphinx.ext.autodoc import AutodocReporter
>  
>  import kernellog
>  
> @@ -125,13 +135,7 @@ class KernelDocDirective(Directive):
>                      lineoffset += 1
>  
>              node = nodes.section()
> -            buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter
> -            self.state.memo.reporter = AutodocReporter(result, self.state.memo.reporter)
> -            self.state.memo.title_styles, self.state.memo.section_level = [], 0
> -            try:
> -                self.state.nested_parse(result, 0, node, match_titles=1)
> -            finally:
> -                self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf
> +            self.do_parse(result, node)
>  
>              return node.children
>  
> @@ -140,6 +144,24 @@ class KernelDocDirective(Directive):
>                             (" ".join(cmd), str(e)))
>              return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))]
>  
> +    def do_parse(self, result, node):
> +        if Use_SSI:
> +            save = self.state.memo.title_styles, self.state.memo.section_level
> +            try:
> +                with switch_source_input(self.state, result):
> +                    self.state.nested_parse(result, 0, node, match_titles=1)

IIUC you don't need to save the state anymore, so the above two lines
should be sufficient when using switch_source_input.

BR,
Jani.


> +            finally:
> +                self.state.memo.title_styles, self.state.memo.section_level = save
> +        else:
> +            save = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter
> +            self.state.memo.reporter = AutodocReporter(result, self.state.memo.reporter)
> +            self.state.memo.title_styles, self.state.memo.section_level = [], 0
> +            try:
> +                self.state.nested_parse(result, 0, node, match_titles=1)
> +            finally:
> +                self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = save
> +
> +
>  def setup(app):
>      app.add_config_value('kerneldoc_bin', None, 'env')
>      app.add_config_value('kerneldoc_srctree', None, 'env')

-- 
Jani Nikula, Intel Open Source Graphics Center

Re: [PATCH RFC 0/2] docs: Deal with some Sphinx deprecation warnings

[Oleksandr Natalenko]

On Tue, May 21, 2019 at 03:17:12PM -0600, Jonathan Corbet wrote:
> The Sphinx folks are deprecating some interfaces in the upcoming 2.0
> release; one immediate result of that is a bunch of warnings that show up
> when building with 1.8.  These two patches make those warnings go away,
> but at a cost:

A minor correction, if I may and if I understand this correctly: 2.0 is
not an upcoming release, but a current one (2.0.1, to be precise), and
this means that in some distros (like, Arch [1]) `make htmldocs` is
already broken for quite some time.

[1] https://bugs.archlinux.org/task/59688

> 
>  - It introduces a couple of Sphinx version checks, which are always
>    ugly, but the alternative would be to stop supporting versions
>    before 1.7.  For now, I think we can carry that cruft.
> 
>  - The second patch causes the build to fail horribly on newer
>    Sphinx installations.  The change to switch_source_input() seems
>    to make the parser much more finicky, increasing warnings and
>    eventually failing the build altogether.  In particular, it will
>    scream about problems in .rst files that are not included in the
>    TOC tree at all.  The complaints appear to be legitimate, but it's
>    a bunch of stuff to clean up.
> 
> I've tested these with 1.4 and 1.8, but not various versions in between.
> 
> Jonathan Corbet (2):
>   doc: Cope with Sphinx logging deprecations
>   doc: Cope with the deprecation of AutoReporter
> 
>  Documentation/sphinx/kerneldoc.py | 48 ++++++++++++++++++++++++-------
>  Documentation/sphinx/kernellog.py | 28 ++++++++++++++++++
>  Documentation/sphinx/kfigure.py   | 38 +++++++++++++-----------
>  3 files changed, 87 insertions(+), 27 deletions(-)
>  create mode 100644 Documentation/sphinx/kernellog.py
> 
> -- 
> 2.21.0
> 

-- 
  Best regards,
    Oleksandr Natalenko (post-factum)
    Senior Software Maintenance Engineer

Re: [PATCH RFC 0/2] docs: Deal with some Sphinx deprecation warnings

[Oleksandr Natalenko]

On Wed, May 22, 2019 at 11:43:54AM +0200, Oleksandr Natalenko wrote:
> On Tue, May 21, 2019 at 03:17:12PM -0600, Jonathan Corbet wrote:
> > The Sphinx folks are deprecating some interfaces in the upcoming 2.0
> > release; one immediate result of that is a bunch of warnings that show up
> > when building with 1.8.  These two patches make those warnings go away,
> > but at a cost:
> 
> A minor correction, if I may and if I understand this correctly: 2.0 is
> not an upcoming release, but a current one (2.0.1, to be precise), and
> this means that in some distros (like, Arch [1]) `make htmldocs` is
> already broken for quite some time.
> 
> [1] https://bugs.archlinux.org/task/59688

^^ this was the initial Bug for introducing the doc, but it got reverted
in [2].

[2] https://git.archlinux.org/svntogit/packages.git/commit/trunk?h=packages/linux&id=cfe52e9aa8168d9571bedf8a376e2cfbd25223fd

> 
> > 
> >  - It introduces a couple of Sphinx version checks, which are always
> >    ugly, but the alternative would be to stop supporting versions
> >    before 1.7.  For now, I think we can carry that cruft.
> > 
> >  - The second patch causes the build to fail horribly on newer
> >    Sphinx installations.  The change to switch_source_input() seems
> >    to make the parser much more finicky, increasing warnings and
> >    eventually failing the build altogether.  In particular, it will
> >    scream about problems in .rst files that are not included in the
> >    TOC tree at all.  The complaints appear to be legitimate, but it's
> >    a bunch of stuff to clean up.
> > 
> > I've tested these with 1.4 and 1.8, but not various versions in between.
> > 
> > Jonathan Corbet (2):
> >   doc: Cope with Sphinx logging deprecations
> >   doc: Cope with the deprecation of AutoReporter
> > 
> >  Documentation/sphinx/kerneldoc.py | 48 ++++++++++++++++++++++++-------
> >  Documentation/sphinx/kernellog.py | 28 ++++++++++++++++++
> >  Documentation/sphinx/kfigure.py   | 38 +++++++++++++-----------
> >  3 files changed, 87 insertions(+), 27 deletions(-)
> >  create mode 100644 Documentation/sphinx/kernellog.py
> > 
> > -- 
> > 2.21.0
> > 
> 
> -- 
>   Best regards,
>     Oleksandr Natalenko (post-factum)
>     Senior Software Maintenance Engineer

-- 
  Best regards,
    Oleksandr Natalenko (post-factum)
    Senior Software Maintenance Engineer

Re: [PATCH RFC 0/2] docs: Deal with some Sphinx deprecation warnings

[Mauro Carvalho Chehab]

Em Wed, 22 May 2019 10:36:45 +0300
Jani Nikula <jani.nikula@linux.intel.com> escreveu:

> On Tue, 21 May 2019, Jonathan Corbet <corbet@lwn.net> wrote:
> > The Sphinx folks are deprecating some interfaces in the upcoming 2.0
> > release; one immediate result of that is a bunch of warnings that show up
> > when building with 1.8.  These two patches make those warnings go away,
> > but at a cost:
> >
> >  - It introduces a couple of Sphinx version checks, which are always
> >    ugly, but the alternative would be to stop supporting versions
> >    before 1.7.  For now, I think we can carry that cruft.  
> 
> Frankly, I'd just require Sphinx 1.7+, available even in Debian stable
> through stretch-backports.

We can raise the bar and force a 1.7 or even 1.8 (Fedora 30 comes
with version 1.8.4), but I would prefer to keep support older versions,
at least while we don't depend on some new features introduced after
the version we're using, and while our extensions won't require a major
rework due to a new version.
> 
> >  - The second patch causes the build to fail horribly on newer
> >    Sphinx installations.  The change to switch_source_input() seems
> >    to make the parser much more finicky, increasing warnings and
> >    eventually failing the build altogether.  In particular, it will
> >    scream about problems in .rst files that are not included in the
> >    TOC tree at all.  The complaints appear to be legitimate, but it's
> >    a bunch of stuff to clean up.  

There is a flag to cleanup the warning about a file not included at
a TOC tree (:orphan:), but it will still try to parse it. There's also
a conf.py way of doing it. For example, you can exclude an entire dir:

	exclude_patterns = ['includes/*.rst']

But using exclude_patterns will likely be too messy.

> I can understand Sphinx complaining that a file is not included in a TOC
> tree, but I don't understand why it goes on to parse them anyway.

Yeah, this is, IMHO, a very bad behavior.

> 
> BR,
> Jani.
> 
> 
> >
> > I've tested these with 1.4 and 1.8, but not various versions in between.
> >
> > Jonathan Corbet (2):
> >   doc: Cope with Sphinx logging deprecations
> >   doc: Cope with the deprecation of AutoReporter
> >
> >  Documentation/sphinx/kerneldoc.py | 48 ++++++++++++++++++++++++-------
> >  Documentation/sphinx/kernellog.py | 28 ++++++++++++++++++
> >  Documentation/sphinx/kfigure.py   | 38 +++++++++++++-----------
> >  3 files changed, 87 insertions(+), 27 deletions(-)
> >  create mode 100644 Documentation/sphinx/kernellog.py  
> 



Thanks,
Mauro

Re: [PATCH RFC 0/2] docs: Deal with some Sphinx deprecation warnings

[Markus Heiser]

22.05.19 um 12:19 schrieb Mauro Carvalho Chehab:
> Em Wed, 22 May 2019 10:36:45 +0300
> Jani Nikula <jani.nikula@linux.intel.com> escreveu:
> 
>> On Tue, 21 May 2019, Jonathan Corbet <corbet@lwn.net> wrote:
>>> The Sphinx folks are deprecating some interfaces in the upcoming 2.0
>>> release; one immediate result of that is a bunch of warnings that show up
>>> when building with 1.8.  These two patches make those warnings go away,
>>> but at a cost:
>>>
>>>   - It introduces a couple of Sphinx version checks, which are always
>>>     ugly, but the alternative would be to stop supporting versions
>>>     before 1.7.  For now, I think we can carry that cruft.
>>
>> Frankly, I'd just require Sphinx 1.7+, available even in Debian stable
>> through stretch-backports.
> 
> We can raise the bar and force a 1.7 or even 1.8 (Fedora 30 comes
> with version 1.8.4), but I would prefer to keep support older versions,
> at least while we don't depend on some new features introduced after
> the version we're using, and while our extensions won't require a major
> rework due to a new version.

Lets use 1.7 :

- no need for Use_SSI wrapper
- new log should work with 1.7 [1] --> no need for kernellog.py and
   additional imports, instead include on top of python modules ::

     from sphinx.util import logging
     logger = logging.getLogger('kerneldoc')

[1] 
https://github.com/sphinx-doc/sphinx/commit/6d4e6454093953943e79d4db6efeb17390870e62


BTW we can drop other (old) sphinx-version issues e.g.
Documentation/conf.py  which fails with version 2.0:

diff --git a/Documentation/conf.py b/Documentation/conf.py
index 72647a38b5c2..ba82715b6715 100644
--- a/Documentation/conf.py
+++ b/Documentation/conf.py
@@ -34,13 +34,7 @@ needs_sphinx = '1.3'
  # Add any Sphinx extension module names here, as strings. They can be
  # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
  # ones.
-extensions = ['kerneldoc', 'rstFlatTable', 'kernel_include', 'cdomain', 
'kfigure', 'sphinx.ext.ifconfig']
-
-# The name of the math extension changed on Sphinx 1.4
-if major == 1 and minor > 3:
-    extensions.append("sphinx.ext.imgmath")
-else:
-    extensions.append("sphinx.ext.pngmath")
+extensions = ['kerneldoc', 'rstFlatTable', 'kernel_include', 'cdomain', 
'kfigure', 'sphinx.ext.ifconfig', 'sphinx.ext.imgmath']

  # Add any paths that contain templates here, relative to this directory.
  templates_path = ['_templates']

-- Markus --

Re: [PATCH RFC 0/2] docs: Deal with some Sphinx deprecation warnings

[Jonathan Corbet]

On Wed, 22 May 2019 15:25:36 +0200
Markus Heiser <markus.heiser@darmarit.de> wrote:

> Lets use 1.7 :
> 
> - no need for Use_SSI wrapper
> - new log should work with 1.7 [1] --> no need for kernellog.py and
>    additional imports, instead include on top of python modules ::
> 
>      from sphinx.util import logging
>      logger = logging.getLogger('kerneldoc')

I think we're going to have to drag things forward at some point in the
not-too-distant future, but I think I'd rather not do that quite yet.  The
cost of supporting older sphinx for a few releases while we warn people is
not all that high.  So I think we should:

 - Put in (a future version of) my hacks for now, plus whatever else might
   be needed to make 2.0 work right.

 - Fix the fallout with regard to out-of-toctree .rst files so that we can
   actually build again with current sphinx.

 - Update Documentation/sphinx/requirements.txt to ask for something a wee
   bit more recent than 1.4.9.

 - Add a warning when building with an older version that (say) 1.7 will
   be required as of (say) 5.5.

Does this make sense?

Thanks,

jon

Re: [PATCH RFC 0/2] docs: Deal with some Sphinx deprecation warnings

[Mauro Carvalho Chehab]

Em Wed, 22 May 2019 09:45:59 -0600
Jonathan Corbet <corbet@lwn.net> escreveu:

> On Wed, 22 May 2019 15:25:36 +0200
> Markus Heiser <markus.heiser@darmarit.de> wrote:
> 
> > Lets use 1.7 :
> > 
> > - no need for Use_SSI wrapper
> > - new log should work with 1.7 [1] --> no need for kernellog.py and
> >    additional imports, instead include on top of python modules ::
> > 
> >      from sphinx.util import logging
> >      logger = logging.getLogger('kerneldoc')  
> 
> I think we're going to have to drag things forward at some point in the
> not-too-distant future, but I think I'd rather not do that quite yet.  The
> cost of supporting older sphinx for a few releases while we warn people is
> not all that high.  So I think we should:
> 
>  - Put in (a future version of) my hacks for now, plus whatever else might
>    be needed to make 2.0 work right.
> 
>  - Fix the fallout with regard to out-of-toctree .rst files so that we can
>    actually build again with current sphinx.
> 
>  - Update Documentation/sphinx/requirements.txt to ask for something a wee
>    bit more recent than 1.4.9.

You should remember to also update conf.py (with currently points to 1.3):

	# If your documentation needs a minimal Sphinx version, state it here.
	needs_sphinx = '1.3'

Also, if you touch there, you should also touch:

	./scripts/sphinx-pre-install

The change there won't be as trivial as just changing this line:

	$virtenv_dir = "sphinx_1.4";

as the script should now run sphinx-build --version, in order to check
if the version is lower than the new minimal version. It probably makes
sense to make it grep the version from needs_sphinx at conf.py.

>  - Add a warning when building with an older version that (say) 1.7 will
>    be required as of (say) 5.5.

It probably makes sense to add such check at the pre-install script,
and add a:

	SPHINXOPTS="-jauto"

somewhere if version is 1.7 or upper.

> 
> Does this make sense?

It makes sense to me.

Thanks,
Mauro

Re: [PATCH RFC 0/2] docs: Deal with some Sphinx deprecation warnings

[Mauro Carvalho Chehab]

Em Wed, 22 May 2019 13:04:08 -0300
Mauro Carvalho Chehab <mchehab@kernel.org> escreveu:

> Em Wed, 22 May 2019 09:45:59 -0600
> Jonathan Corbet <corbet@lwn.net> escreveu:
> 
> > On Wed, 22 May 2019 15:25:36 +0200
> > Markus Heiser <markus.heiser@darmarit.de> wrote:
> >   
> > > Lets use 1.7 :
> > > 
> > > - no need for Use_SSI wrapper
> > > - new log should work with 1.7 [1] --> no need for kernellog.py and
> > >    additional imports, instead include on top of python modules ::
> > > 
> > >      from sphinx.util import logging
> > >      logger = logging.getLogger('kerneldoc')    
> > 
> > I think we're going to have to drag things forward at some point in the
> > not-too-distant future, but I think I'd rather not do that quite yet.  The
> > cost of supporting older sphinx for a few releases while we warn people is
> > not all that high.  So I think we should:
> > 
> >  - Put in (a future version of) my hacks for now, plus whatever else might
> >    be needed to make 2.0 work right.
> > 
> >  - Fix the fallout with regard to out-of-toctree .rst files so that we can
> >    actually build again with current sphinx.
> > 
> >  - Update Documentation/sphinx/requirements.txt to ask for something a wee
> >    bit more recent than 1.4.9.  
> 
> You should remember to also update conf.py (with currently points to 1.3):
> 
> 	# If your documentation needs a minimal Sphinx version, state it here.
> 	needs_sphinx = '1.3'
> 
> Also, if you touch there, you should also touch:
> 
> 	./scripts/sphinx-pre-install
> 
> The change there won't be as trivial as just changing this line:
> 
> 	$virtenv_dir = "sphinx_1.4";
> 
> as the script should now run sphinx-build --version, in order to check
> if the version is lower than the new minimal version. It probably makes
> sense to make it grep the version from needs_sphinx at conf.py.
> 
> >  - Add a warning when building with an older version that (say) 1.7 will
> >    be required as of (say) 5.5.  
> 
> It probably makes sense to add such check at the pre-install script,
> and add a:
> 
> 	SPHINXOPTS="-jauto"
> 
> somewhere if version is 1.7 or upper.
> 

I'm meaning something like the enclosed patch.

(PS.: I'm still working at the patch)



Thanks,
Mauro

[RFC PATCH] scripts/sphinx-pre-install: make it handle Sphinx versions

As we want to switch to a newer Sphinx version in the future,
add some version detected logic.

Signed-off-by: Mauro Carvalho Chehab <mchehab+samsung@kernel.org>

diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install
index f6a5c0bae31e..8835aede4c61 100755
--- a/scripts/sphinx-pre-install
+++ b/scripts/sphinx-pre-install
@@ -13,7 +13,7 @@ use strict;
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 
-my $virtenv_dir = "sphinx_1.4";
+my $conf = "Documentation/conf.py";
 my $requirement_file = "Documentation/sphinx/requirements.txt";
 
 #
@@ -27,6 +27,10 @@ my $optional = 0;
 my $need_symlink = 0;
 my $need_sphinx = 0;
 my $install = "";
+my $min_version;
+my $rec_version;
+my $cur_version;
+my $virtenv_dir = "sphinx_";
 
 #
 # Command line arguments
@@ -76,6 +80,52 @@ my %texlive = (
 # Subroutines that checks if a feature exists
 #
 
+sub handle_sphinx_version()
+{
+	open IN, $conf;
+	while (<IN>) {
+		if (m/^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]/) {
+			$min_version=$1;
+			last;
+		}
+	}
+	close IN;
+
+	die "Can't get needs_sphinx version from $conf" if (!$min_version);
+
+	open IN, $requirement_file;
+	while (<IN>) {
+		if (m/^\s*Sphinx\s*==\s*([\d\.]+)$/) {
+			$rec_version=$1;
+			last;
+		}
+	}
+	close IN;
+
+	open IN, "sphinx-build --version 2>&1 |";
+	while (<IN>) {
+		if (m/^\s*sphinx-build\s+([\d\.]+)$/) {
+			$cur_version=$1;
+			last;
+		}
+	}
+	close IN;
+
+	$virtenv_dir .= $rec_version;
+
+	# Sphinx is not installed
+	return if (!$cur_version);
+
+	if ($cur_version lt $min_version) {
+		print "Sphinx version older than $min_version! We recommend at least $rec_version";
+		exit -1;
+	}
+
+	if ($cur_version lt $rec_version) {
+		print "Warning: we recommend at least Sphinx version $rec_version";
+	}
+}
+
 sub check_missing(%)
 {
 	my %map = %{$_[0]};
@@ -587,6 +637,8 @@ while (@ARGV) {
 	}
 }
 
+handle_sphinx_version();
+
 #
 # Determine the system type. There's no standard unique way that would
 # work with all distros with a minimal package install. So, several