From 9ca876f96323934b8ace04b123531f05e3bed029 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 30 Oct 2020 08:40:31 +0100 Subject: docs: kernel_abi.py: add a script to parse ABI documentation The ABI documentation is special: it is not plain text files, but, instead, files with an strict format, as specified by Documentation/ABI/README. Add a parser for it. Acked-by: Jonathan Corbet Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/48abf1a410237e63f85354a8cd7027fdf25657bf.1604042072.git.mchehab+huawei@kernel.org Signed-off-by: Greg Kroah-Hartman --- Documentation/sphinx/kernel_abi.py | 157 +++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 Documentation/sphinx/kernel_abi.py (limited to 'Documentation/sphinx/kernel_abi.py') diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py new file mode 100644 index 000000000000..f8e7a02066c2 --- /dev/null +++ b/Documentation/sphinx/kernel_abi.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8; mode: python -*- +# SPDX-License-Identifier: GPL-2.0 +u""" + kernel-abi + ~~~~~~~~~~ + + Implementation of the ``kernel-abi`` reST-directive. + + :copyright: Copyright (C) 2016 Markus Heiser + :copyright: Copyright (C) 2016-2020 Mauro Carvalho Chehab + :maintained-by: Mauro Carvalho Chehab + :license: GPL Version 2, June 1991 see Linux/COPYING for details. + + The ``kernel-abi`` (:py:class:`KernelCmd`) directive calls the + scripts/get_abi.pl script to parse the Kernel ABI files. + + Overview of directive's argument and options. + + .. code-block:: rst + + .. kernel-abi:: + :debug: + + The argument ```` is required. It contains the + location of the ABI files to be parsed. + + ``debug`` + Inserts a code-block with the *raw* reST. Sometimes it is helpful to see + what reST is generated. + +""" + +import sys +import os +from os import path +import subprocess + +from sphinx.ext.autodoc import AutodocReporter + +from docutils import nodes +from docutils.parsers.rst import Directive, directives +from docutils.statemachine import ViewList +from docutils.utils.error_reporting import ErrorString + + +__version__ = '1.0' + +# We can't assume that six is installed +PY3 = sys.version_info[0] == 3 +PY2 = sys.version_info[0] == 2 +if PY3: + # pylint: disable=C0103, W0622 + unicode = str + basestring = str + +def setup(app): + + app.add_directive("kernel-abi", KernelCmd) + return dict( + version = __version__ + , parallel_read_safe = True + , parallel_write_safe = True + ) + +class KernelCmd(Directive): + + u"""KernelABI (``kernel-abi``) directive""" + + required_arguments = 1 + optional_arguments = 0 + has_content = False + final_argument_whitespace = True + + option_spec = { + "debug" : directives.flag + } + + def warn(self, message, **replace): + replace["fname"] = self.state.document.current_source + replace["line_no"] = replace.get("line_no", self.lineno) + message = ("%(fname)s:%(line_no)s: [kernel-abi WARN] : " + message) % replace + self.state.document.settings.env.app.warn(message, prefix="") + + def run(self): + + doc = self.state.document + if not doc.settings.file_insertion_enabled: + raise self.warning("docutils: file insertion disabled") + + env = doc.settings.env + cwd = path.dirname(doc.current_source) + cmd = "get_abi.pl rest --dir " + cmd += self.arguments[0] + + srctree = path.abspath(os.environ["srctree"]) + + fname = cmd + + # extend PATH with $(srctree)/scripts + path_env = os.pathsep.join([ + srctree + os.sep + "scripts", + os.environ["PATH"] + ]) + shell_env = os.environ.copy() + shell_env["PATH"] = path_env + shell_env["srctree"] = srctree + + lines = self.runCmd(cmd, shell=True, cwd=cwd, env=shell_env) + nodeList = self.nestedParse(lines, fname) + return nodeList + + def runCmd(self, cmd, **kwargs): + u"""Run command ``cmd`` and return it's stdout as unicode.""" + + try: + proc = subprocess.Popen( + cmd + , stdout = subprocess.PIPE + , stderr = subprocess.PIPE + , universal_newlines = True + , **kwargs + ) + out, err = proc.communicate() + if err: + self.warn(err) + if proc.returncode != 0: + raise self.severe( + u"command '%s' failed with return code %d" + % (cmd, proc.returncode) + ) + except OSError as exc: + raise self.severe(u"problems with '%s' directive: %s." + % (self.name, ErrorString(exc))) + return unicode(out) + + def nestedParse(self, lines, fname): + content = ViewList() + node = nodes.section() + + if "debug" in self.options: + code_block = "\n\n.. code-block:: rst\n :linenos:\n" + for l in lines.split("\n"): + code_block += "\n " + l + lines = code_block + "\n\n" + + for c, l in enumerate(lines.split("\n")): + content.append(l, fname, c) + + buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter + self.state.memo.title_styles = [] + self.state.memo.section_level = 0 + self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter) + try: + self.state.nested_parse(content, 0, node, match_titles=1) + finally: + self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf + return node.children -- cgit v1.2.3 From 823830d4065bfb115b948c9f21e713a15d8d95ef Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 30 Oct 2020 08:40:32 +0100 Subject: docs: kernel_abi.py: fix UTF-8 support The parser breaks with UTF-8 characters with Sphinx 1.4. Acked-by: Jonathan Corbet Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/9e7c8e3b0efaa1ae0536da6493ab438bd3f9fe58.1604042072.git.mchehab+huawei@kernel.org Signed-off-by: Greg Kroah-Hartman --- Documentation/sphinx/kernel_abi.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) (limited to 'Documentation/sphinx/kernel_abi.py') diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py index f8e7a02066c2..fe69c213716d 100644 --- a/Documentation/sphinx/kernel_abi.py +++ b/Documentation/sphinx/kernel_abi.py @@ -1,5 +1,7 @@ # -*- coding: utf-8; mode: python -*- +# coding=utf-8 # SPDX-License-Identifier: GPL-2.0 +# u""" kernel-abi ~~~~~~~~~~ @@ -30,6 +32,7 @@ u""" """ +import codecs import sys import os from os import path @@ -45,14 +48,6 @@ from docutils.utils.error_reporting import ErrorString __version__ = '1.0' -# We can't assume that six is installed -PY3 = sys.version_info[0] == 3 -PY2 = sys.version_info[0] == 2 -if PY3: - # pylint: disable=C0103, W0622 - unicode = str - basestring = str - def setup(app): app.add_directive("kernel-abi", KernelCmd) @@ -117,12 +112,12 @@ class KernelCmd(Directive): cmd , stdout = subprocess.PIPE , stderr = subprocess.PIPE - , universal_newlines = True , **kwargs ) out, err = proc.communicate() - if err: - self.warn(err) + + out, err = codecs.decode(out, 'utf-8'), codecs.decode(err, 'utf-8') + if proc.returncode != 0: raise self.severe( u"command '%s' failed with return code %d" @@ -131,7 +126,7 @@ class KernelCmd(Directive): except OSError as exc: raise self.severe(u"problems with '%s' directive: %s." % (self.name, ErrorString(exc))) - return unicode(out) + return out def nestedParse(self, lines, fname): content = ViewList() -- cgit v1.2.3 From c830fa9a4d363caacc4ba3f63c06b9e967ad13a4 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 30 Oct 2020 08:40:33 +0100 Subject: docs: kernel_abi.py: make it compatible with Sphinx 1.7+ The same way kerneldoc.py needed changes to work with newer Sphinx, this script needs the same changes. While here, reorganize the include order to match kerneldoc.py. Acked-by: Jonathan Corbet Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/f2b25caef5db7738629773a03463908d3b39b83a.1604042072.git.mchehab+huawei@kernel.org Signed-off-by: Greg Kroah-Hartman --- Documentation/sphinx/kernel_abi.py | 39 ++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) (limited to 'Documentation/sphinx/kernel_abi.py') diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py index fe69c213716d..8601a3b75a28 100644 --- a/Documentation/sphinx/kernel_abi.py +++ b/Documentation/sphinx/kernel_abi.py @@ -33,18 +33,27 @@ u""" """ import codecs -import sys import os -from os import path import subprocess +import sys -from sphinx.ext.autodoc import AutodocReporter +from os import path -from docutils import nodes -from docutils.parsers.rst import Directive, directives +from docutils import nodes, statemachine from docutils.statemachine import ViewList +from docutils.parsers.rst import directives, Directive from docutils.utils.error_reporting import ErrorString +# +# AutodocReporter is only good up to Sphinx 1.7 +# +import sphinx + +Use_SSI = sphinx.__version__[:3] >= '1.7' +if Use_SSI: + from sphinx.util.docutils import switch_source_input +else: + from sphinx.ext.autodoc import AutodocReporter __version__ = '1.0' @@ -142,11 +151,17 @@ class KernelCmd(Directive): content.append(l, fname, c) buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter - self.state.memo.title_styles = [] - self.state.memo.section_level = 0 - self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter) - try: - self.state.nested_parse(content, 0, node, match_titles=1) - finally: - self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf + + if Use_SSI: + with switch_source_input(self.state, content): + self.state.nested_parse(content, 0, node, match_titles=1) + else: + self.state.memo.title_styles = [] + self.state.memo.section_level = 0 + self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter) + try: + self.state.nested_parse(content, 0, node, match_titles=1) + finally: + self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf + return node.children -- cgit v1.2.3 From 997b7c8b4a95681dc75e6b720ad6f0ea705368c3 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 30 Oct 2020 08:40:34 +0100 Subject: docs: kernel_abi.py: use --enable-lineno for get_abi.pl Just like kernel-doc extension, we need to be able to identify what part of an imported document has issues, as reporting them as: get_abi.pl rest --dir $srctree/Documentation/ABI/obsolete --rst-source:1689: ERROR: Unexpected indentation. Makes a lot harder for someone to fix. It should be noticed that it the line which will be reported is the line where the "What:" definition is, and not the line with actually has an error. Acked-by: Jonathan Corbet Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/d6155ab16fb7631f2fa8e7a770eae72f24bf7cc5.1604042072.git.mchehab+huawei@kernel.org Signed-off-by: Greg Kroah-Hartman --- Documentation/sphinx/kernel_abi.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'Documentation/sphinx/kernel_abi.py') diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py index 8601a3b75a28..096dec482e96 100644 --- a/Documentation/sphinx/kernel_abi.py +++ b/Documentation/sphinx/kernel_abi.py @@ -36,6 +36,7 @@ import codecs import os import subprocess import sys +import re from os import path @@ -93,7 +94,7 @@ class KernelCmd(Directive): env = doc.settings.env cwd = path.dirname(doc.current_source) - cmd = "get_abi.pl rest --dir " + cmd = "get_abi.pl rest --enable-lineno --dir " cmd += self.arguments[0] srctree = path.abspath(os.environ["srctree"]) @@ -137,7 +138,7 @@ class KernelCmd(Directive): % (self.name, ErrorString(exc))) return out - def nestedParse(self, lines, fname): + def nestedParse(self, lines, f): content = ViewList() node = nodes.section() @@ -147,8 +148,17 @@ class KernelCmd(Directive): code_block += "\n " + l lines = code_block + "\n\n" - for c, l in enumerate(lines.split("\n")): - content.append(l, fname, c) + line_regex = re.compile("^#define LINENO (\S+)\#([0-9]+)$") + ln = 0 + + for line in lines.split("\n"): + match = line_regex.search(line) + if match: + f = match.group(1) + # sphinx counts lines from 0 + ln = int(match.group(2)) - 1 + else: + content.append(line, f, ln) buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter -- cgit v1.2.3 From 3c543d29891a42b23c1f9a3a4788ae37fb94cce2 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 30 Oct 2020 08:40:35 +0100 Subject: docs: kernel_abi.py: Handle with a lazy Sphinx parser The Sphinx docutils parser is lazy: if the content is bigger than a certain number of lines, it silenlty stops parsing it, producing an incomplete content. This seems to be worse on newer Sphinx versions, like 2.0. So, change the logic to parse the contents per input file. Acked-by: Jonathan Corbet Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/4659b60795739308e34d2d00c57ee0742a9cd2ab.1604042072.git.mchehab+huawei@kernel.org Signed-off-by: Greg Kroah-Hartman --- Documentation/sphinx/kernel_abi.py | 39 +++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) (limited to 'Documentation/sphinx/kernel_abi.py') diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py index 096dec482e96..ce5f3b0ae811 100644 --- a/Documentation/sphinx/kernel_abi.py +++ b/Documentation/sphinx/kernel_abi.py @@ -37,6 +37,7 @@ import os import subprocess import sys import re +import kernellog from os import path @@ -80,12 +81,6 @@ class KernelCmd(Directive): "debug" : directives.flag } - def warn(self, message, **replace): - replace["fname"] = self.state.document.current_source - replace["line_no"] = replace.get("line_no", self.lineno) - message = ("%(fname)s:%(line_no)s: [kernel-abi WARN] : " + message) % replace - self.state.document.settings.env.app.warn(message, prefix="") - def run(self): doc = self.state.document @@ -111,7 +106,7 @@ class KernelCmd(Directive): shell_env["srctree"] = srctree lines = self.runCmd(cmd, shell=True, cwd=cwd, env=shell_env) - nodeList = self.nestedParse(lines, fname) + nodeList = self.nestedParse(lines, self.arguments[0]) return nodeList def runCmd(self, cmd, **kwargs): @@ -138,9 +133,9 @@ class KernelCmd(Directive): % (self.name, ErrorString(exc))) return out - def nestedParse(self, lines, f): + def nestedParse(self, lines, fname): content = ViewList() - node = nodes.section() + node = nodes.section() if "debug" in self.options: code_block = "\n\n.. code-block:: rst\n :linenos:\n" @@ -150,22 +145,42 @@ class KernelCmd(Directive): line_regex = re.compile("^#define LINENO (\S+)\#([0-9]+)$") ln = 0 + n = 0 + f = fname for line in lines.split("\n"): + n = n + 1 match = line_regex.search(line) if match: - f = match.group(1) + new_f = match.group(1) + + # Sphinx parser is lazy: it stops parsing contents in the + # middle, if it is too big. So, handle it per input file + if new_f != f and content: + self.do_parse(content, node) + content = ViewList() + + f = new_f + # sphinx counts lines from 0 ln = int(match.group(2)) - 1 else: content.append(line, f, ln) - buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter + kernellog.info(self.state.document.settings.env.app, "%s: parsed %i lines" % (fname, n)) + + if content: + self.do_parse(content, node) + return node.children + + def do_parse(self, content, node): if Use_SSI: with switch_source_input(self.state, content): self.state.nested_parse(content, 0, node, match_titles=1) else: + buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter + self.state.memo.title_styles = [] self.state.memo.section_level = 0 self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter) @@ -173,5 +188,3 @@ class KernelCmd(Directive): self.state.nested_parse(content, 0, node, match_titles=1) finally: self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf - - return node.children -- cgit v1.2.3 From 642514df1bc1c6a9318b537670fdb55a6ee38e45 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 30 Oct 2020 08:40:41 +0100 Subject: docs: ABI: make it parse ABI/stable as ReST-compatible files Now that the stable ABI files are compatible with ReST, parse them without converting complex descriptions as literal blocks nor escaping special characters. Please notice that escaping special characters will probably be needed at descriptions, at least for the asterisk character. Acked-by: Jonathan Corbet Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/59ccbaa75ff05f23e701dd9a0bbe118e9343a553.1604042072.git.mchehab+huawei@kernel.org Signed-off-by: Greg Kroah-Hartman --- Documentation/admin-guide/abi-stable.rst | 1 + Documentation/sphinx/kernel_abi.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'Documentation/sphinx/kernel_abi.py') diff --git a/Documentation/admin-guide/abi-stable.rst b/Documentation/admin-guide/abi-stable.rst index 7495d7a35048..70490736e0d3 100644 --- a/Documentation/admin-guide/abi-stable.rst +++ b/Documentation/admin-guide/abi-stable.rst @@ -11,3 +11,4 @@ Most interfaces (like syscalls) are expected to never change and always be available. .. kernel-abi:: $srctree/Documentation/ABI/stable + :rst: diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py index ce5f3b0ae811..f3da859c9878 100644 --- a/Documentation/sphinx/kernel_abi.py +++ b/Documentation/sphinx/kernel_abi.py @@ -73,12 +73,13 @@ class KernelCmd(Directive): u"""KernelABI (``kernel-abi``) directive""" required_arguments = 1 - optional_arguments = 0 + optional_arguments = 2 has_content = False final_argument_whitespace = True option_spec = { - "debug" : directives.flag + "debug" : directives.flag, + "rst" : directives.unchanged } def run(self): @@ -92,6 +93,9 @@ class KernelCmd(Directive): cmd = "get_abi.pl rest --enable-lineno --dir " cmd += self.arguments[0] + if 'rst' in self.options: + cmd += " --rst-source" + srctree = path.abspath(os.environ["srctree"]) fname = cmd -- cgit v1.2.3