1# -*- coding: utf-8; mode: python -*-
2# pylint: disable=C0103, R0903, R0912, R0915
3u"""
4    scalable figure and image handling
5    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
6
7    Sphinx extension which implements scalable image handling.
8
9    :copyright:  Copyright (C) 2016  Markus Heiser
10    :license:    GPL Version 2, June 1991 see Linux/COPYING for details.
11
12    The build for image formats depend on image's source format and output's
13    destination format. This extension implement methods to simplify image
14    handling from the author's POV. Directives like ``kernel-figure`` implement
15    methods *to* always get the best output-format even if some tools are not
16    installed. For more details take a look at ``convert_image(...)`` which is
17    the core of all conversions.
18
19    * ``.. kernel-image``: for image handling / a ``.. image::`` replacement
20
21    * ``.. kernel-figure``: for figure handling / a ``.. figure::`` replacement
22
23    * ``.. kernel-render``: for render markup / a concept to embed *render*
24      markups (or languages). Supported markups (see ``RENDER_MARKUP_EXT``)
25
26      - ``DOT``: render embedded Graphviz's **DOC**
27      - ``SVG``: render embedded Scalable Vector Graphics (**SVG**)
28      - ... *developable*
29
30    Used tools:
31
32    * ``dot(1)``: Graphviz (https://www.graphviz.org). If Graphviz is not
33      available, the DOT language is inserted as literal-block.
34      For conversion to PDF, ``rsvg-convert(1)`` of librsvg
35      (https://gitlab.gnome.org/GNOME/librsvg) is used when available.
36
37    * SVG to PDF: To generate PDF, you need at least one of this tools:
38
39      - ``convert(1)``: ImageMagick (https://www.imagemagick.org)
40      - ``inkscape(1)``: Inkscape (https://inkscape.org/)
41
42    List of customizations:
43
44    * generate PDF from SVG / used by PDF (LaTeX) builder
45
46    * generate SVG (html-builder) and PDF (latex-builder) from DOT files.
47      DOT: see https://www.graphviz.org/content/dot-language
48
49    """
50
51import os
52from os import path
53import subprocess
54from hashlib import sha1
55import re
56from docutils import nodes
57from docutils.statemachine import ViewList
58from docutils.parsers.rst import directives
59from docutils.parsers.rst.directives import images
60import sphinx
61from sphinx.util.nodes import clean_astext
62import kernellog
63
64Figure = images.Figure
65
66__version__  = '1.0.0'
67
68# simple helper
69# -------------
70
71def which(cmd):
72    """Searches the ``cmd`` in the ``PATH`` environment.
73
74    This *which* searches the PATH for executable ``cmd`` . First match is
75    returned, if nothing is found, ``None` is returned.
76    """
77    envpath = os.environ.get('PATH', None) or os.defpath
78    for folder in envpath.split(os.pathsep):
79        fname = folder + os.sep + cmd
80        if path.isfile(fname):
81            return fname
82
83def mkdir(folder, mode=0o775):
84    if not path.isdir(folder):
85        os.makedirs(folder, mode)
86
87def file2literal(fname):
88    with open(fname, "r") as src:
89        data = src.read()
90        node = nodes.literal_block(data, data)
91    return node
92
93def isNewer(path1, path2):
94    """Returns True if ``path1`` is newer than ``path2``
95
96    If ``path1`` exists and is newer than ``path2`` the function returns
97    ``True`` is returned otherwise ``False``
98    """
99    return (path.exists(path1)
100            and os.stat(path1).st_ctime > os.stat(path2).st_ctime)
101
102def pass_handle(self, node):           # pylint: disable=W0613
103    pass
104
105# setup conversion tools and sphinx extension
106# -------------------------------------------
107
108# Graphviz's dot(1) support
109dot_cmd = None
110# dot(1) -Tpdf should be used
111dot_Tpdf = False
112
113# ImageMagick' convert(1) support
114convert_cmd = None
115
116# librsvg's rsvg-convert(1) support
117rsvg_convert_cmd = None
118
119# Inkscape's inkscape(1) support
120inkscape_cmd = None
121# Inkscape prior to 1.0 uses different command options
122inkscape_ver_one = False
123
124
125def setup(app):
126    # check toolchain first
127    app.connect('builder-inited', setupTools)
128
129    # image handling
130    app.add_directive("kernel-image",  KernelImage)
131    app.add_node(kernel_image,
132                 html    = (visit_kernel_image, pass_handle),
133                 latex   = (visit_kernel_image, pass_handle),
134                 texinfo = (visit_kernel_image, pass_handle),
135                 text    = (visit_kernel_image, pass_handle),
136                 man     = (visit_kernel_image, pass_handle), )
137
138    # figure handling
139    app.add_directive("kernel-figure", KernelFigure)
140    app.add_node(kernel_figure,
141                 html    = (visit_kernel_figure, pass_handle),
142                 latex   = (visit_kernel_figure, pass_handle),
143                 texinfo = (visit_kernel_figure, pass_handle),
144                 text    = (visit_kernel_figure, pass_handle),
145                 man     = (visit_kernel_figure, pass_handle), )
146
147    # render handling
148    app.add_directive('kernel-render', KernelRender)
149    app.add_node(kernel_render,
150                 html    = (visit_kernel_render, pass_handle),
151                 latex   = (visit_kernel_render, pass_handle),
152                 texinfo = (visit_kernel_render, pass_handle),
153                 text    = (visit_kernel_render, pass_handle),
154                 man     = (visit_kernel_render, pass_handle), )
155
156    app.connect('doctree-read', add_kernel_figure_to_std_domain)
157
158    return dict(
159        version = __version__,
160        parallel_read_safe = True,
161        parallel_write_safe = True
162    )
163
164
165def setupTools(app):
166    u"""
167    Check available build tools and log some *verbose* messages.
168
169    This function is called once, when the builder is initiated.
170    """
171    global dot_cmd, dot_Tpdf, convert_cmd, rsvg_convert_cmd   # pylint: disable=W0603
172    global inkscape_cmd, inkscape_ver_one  # pylint: disable=W0603
173    kernellog.verbose(app, "kfigure: check installed tools ...")
174
175    dot_cmd = which('dot')
176    convert_cmd = which('convert')
177    rsvg_convert_cmd = which('rsvg-convert')
178    inkscape_cmd = which('inkscape')
179
180    if dot_cmd:
181        kernellog.verbose(app, "use dot(1) from: " + dot_cmd)
182
183        try:
184            dot_Thelp_list = subprocess.check_output([dot_cmd, '-Thelp'],
185                                    stderr=subprocess.STDOUT)
186        except subprocess.CalledProcessError as err:
187            dot_Thelp_list = err.output
188            pass
189
190        dot_Tpdf_ptn = b'pdf'
191        dot_Tpdf = re.search(dot_Tpdf_ptn, dot_Thelp_list)
192    else:
193        kernellog.warn(app, "dot(1) not found, for better output quality install "
194                       "graphviz from https://www.graphviz.org")
195    if inkscape_cmd:
196        kernellog.verbose(app, "use inkscape(1) from: " + inkscape_cmd)
197        inkscape_ver = subprocess.check_output([inkscape_cmd, '--version'],
198                                               stderr=subprocess.DEVNULL)
199        ver_one_ptn = b'Inkscape 1'
200        inkscape_ver_one = re.search(ver_one_ptn, inkscape_ver)
201        convert_cmd = None
202        rsvg_convert_cmd = None
203        dot_Tpdf = False
204
205    else:
206        if convert_cmd:
207            kernellog.verbose(app, "use convert(1) from: " + convert_cmd)
208        else:
209            kernellog.verbose(app,
210                "Neither inkscape(1) nor convert(1) found.\n"
211                "For SVG to PDF conversion, "
212                "install either Inkscape (https://inkscape.org/) (preferred) or\n"
213                "ImageMagick (https://www.imagemagick.org)")
214
215        if rsvg_convert_cmd:
216            kernellog.verbose(app, "use rsvg-convert(1) from: " + rsvg_convert_cmd)
217            kernellog.verbose(app, "use 'dot -Tsvg' and rsvg-convert(1) for DOT -> PDF conversion")
218            dot_Tpdf = False
219        else:
220            kernellog.verbose(app,
221                "rsvg-convert(1) not found.\n"
222                "  SVG rendering of convert(1) is done by ImageMagick-native renderer.")
223            if dot_Tpdf:
224                kernellog.verbose(app, "use 'dot -Tpdf' for DOT -> PDF conversion")
225            else:
226                kernellog.verbose(app, "use 'dot -Tsvg' and convert(1) for DOT -> PDF conversion")
227
228
229# integrate conversion tools
230# --------------------------
231
232RENDER_MARKUP_EXT = {
233    # The '.ext' must be handled by convert_image(..) function's *in_ext* input.
234    # <name> : <.ext>
235    'DOT' : '.dot',
236    'SVG' : '.svg'
237}
238
239def convert_image(img_node, translator, src_fname=None):
240    """Convert a image node for the builder.
241
242    Different builder prefer different image formats, e.g. *latex* builder
243    prefer PDF while *html* builder prefer SVG format for images.
244
245    This function handles output image formats in dependence of source the
246    format (of the image) and the translator's output format.
247    """
248    app = translator.builder.app
249
250    fname, in_ext = path.splitext(path.basename(img_node['uri']))
251    if src_fname is None:
252        src_fname = path.join(translator.builder.srcdir, img_node['uri'])
253        if not path.exists(src_fname):
254            src_fname = path.join(translator.builder.outdir, img_node['uri'])
255
256    dst_fname = None
257
258    # in kernel builds, use 'make SPHINXOPTS=-v' to see verbose messages
259
260    kernellog.verbose(app, 'assert best format for: ' + img_node['uri'])
261
262    if in_ext == '.dot':
263
264        if not dot_cmd:
265            kernellog.verbose(app,
266                              "dot from graphviz not available / include DOT raw.")
267            img_node.replace_self(file2literal(src_fname))
268
269        elif translator.builder.format == 'latex':
270            dst_fname = path.join(translator.builder.outdir, fname + '.pdf')
271            img_node['uri'] = fname + '.pdf'
272            img_node['candidates'] = {'*': fname + '.pdf'}
273
274
275        elif translator.builder.format == 'html':
276            dst_fname = path.join(
277                translator.builder.outdir,
278                translator.builder.imagedir,
279                fname + '.svg')
280            img_node['uri'] = path.join(
281                translator.builder.imgpath, fname + '.svg')
282            img_node['candidates'] = {
283                '*': path.join(translator.builder.imgpath, fname + '.svg')}
284
285        else:
286            # all other builder formats will include DOT as raw
287            img_node.replace_self(file2literal(src_fname))
288
289    elif in_ext == '.svg':
290
291        if translator.builder.format == 'latex':
292            if not inkscape_cmd and convert_cmd is None:
293                kernellog.warn(app,
294                                  "no SVG to PDF conversion available / include SVG raw."
295                                  "\nIncluding large raw SVGs can cause xelatex error."
296                                  "\nInstall Inkscape (preferred) or ImageMagick.")
297                img_node.replace_self(file2literal(src_fname))
298            else:
299                dst_fname = path.join(translator.builder.outdir, fname + '.pdf')
300                img_node['uri'] = fname + '.pdf'
301                img_node['candidates'] = {'*': fname + '.pdf'}
302
303    if dst_fname:
304        # the builder needs not to copy one more time, so pop it if exists.
305        translator.builder.images.pop(img_node['uri'], None)
306        _name = dst_fname[len(str(translator.builder.outdir)) + 1:]
307
308        if isNewer(dst_fname, src_fname):
309            kernellog.verbose(app,
310                              "convert: {out}/%s already exists and is newer" % _name)
311
312        else:
313            ok = False
314            mkdir(path.dirname(dst_fname))
315
316            if in_ext == '.dot':
317                kernellog.verbose(app, 'convert DOT to: {out}/' + _name)
318                if translator.builder.format == 'latex' and not dot_Tpdf:
319                    svg_fname = path.join(translator.builder.outdir, fname + '.svg')
320                    ok1 = dot2format(app, src_fname, svg_fname)
321                    ok2 = svg2pdf_by_rsvg(app, svg_fname, dst_fname)
322                    ok = ok1 and ok2
323
324                else:
325                    ok = dot2format(app, src_fname, dst_fname)
326
327            elif in_ext == '.svg':
328                kernellog.verbose(app, 'convert SVG to: {out}/' + _name)
329                ok = svg2pdf(app, src_fname, dst_fname)
330
331            if not ok:
332                img_node.replace_self(file2literal(src_fname))
333
334
335def dot2format(app, dot_fname, out_fname):
336    """Converts DOT file to ``out_fname`` using ``dot(1)``.
337
338    * ``dot_fname`` pathname of the input DOT file, including extension ``.dot``
339    * ``out_fname`` pathname of the output file, including format extension
340
341    The *format extension* depends on the ``dot`` command (see ``man dot``
342    option ``-Txxx``). Normally you will use one of the following extensions:
343
344    - ``.ps`` for PostScript,
345    - ``.svg`` or ``svgz`` for Structured Vector Graphics,
346    - ``.fig`` for XFIG graphics and
347    - ``.png`` or ``gif`` for common bitmap graphics.
348
349    """
350    out_format = path.splitext(out_fname)[1][1:]
351    cmd = [dot_cmd, '-T%s' % out_format, dot_fname]
352    exit_code = 42
353
354    with open(out_fname, "w") as out:
355        exit_code = subprocess.call(cmd, stdout = out)
356        if exit_code != 0:
357            kernellog.warn(app,
358                          "Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
359    return bool(exit_code == 0)
360
361def svg2pdf(app, svg_fname, pdf_fname):
362    """Converts SVG to PDF with ``inkscape(1)`` or ``convert(1)`` command.
363
364    Uses ``inkscape(1)`` from Inkscape (https://inkscape.org/) or ``convert(1)``
365    from ImageMagick (https://www.imagemagick.org) for conversion.
366    Returns ``True`` on success and ``False`` if an error occurred.
367
368    * ``svg_fname`` pathname of the input SVG file with extension (``.svg``)
369    * ``pdf_name``  pathname of the output PDF file with extension (``.pdf``)
370
371    """
372    cmd = [convert_cmd, svg_fname, pdf_fname]
373    cmd_name = 'convert(1)'
374
375    if inkscape_cmd:
376        cmd_name = 'inkscape(1)'
377        if inkscape_ver_one:
378            cmd = [inkscape_cmd, '-o', pdf_fname, svg_fname]
379        else:
380            cmd = [inkscape_cmd, '-z', '--export-pdf=%s' % pdf_fname, svg_fname]
381
382    try:
383        warning_msg = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
384        exit_code = 0
385    except subprocess.CalledProcessError as err:
386        warning_msg = err.output
387        exit_code = err.returncode
388        pass
389
390    if exit_code != 0:
391        kernellog.warn(app, "Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
392        if warning_msg:
393            kernellog.warn(app, "Warning msg from %s: %s"
394                           % (cmd_name, str(warning_msg, 'utf-8')))
395    elif warning_msg:
396        kernellog.verbose(app, "Warning msg from %s (likely harmless):\n%s"
397                          % (cmd_name, str(warning_msg, 'utf-8')))
398
399    return bool(exit_code == 0)
400
401def svg2pdf_by_rsvg(app, svg_fname, pdf_fname):
402    """Convert SVG to PDF with ``rsvg-convert(1)`` command.
403
404    * ``svg_fname`` pathname of input SVG file, including extension ``.svg``
405    * ``pdf_fname`` pathname of output PDF file, including extension ``.pdf``
406
407    Input SVG file should be the one generated by ``dot2format()``.
408    SVG -> PDF conversion is done by ``rsvg-convert(1)``.
409
410    If ``rsvg-convert(1)`` is unavailable, fall back to ``svg2pdf()``.
411
412    """
413
414    if rsvg_convert_cmd is None:
415        ok = svg2pdf(app, svg_fname, pdf_fname)
416    else:
417        cmd = [rsvg_convert_cmd, '--format=pdf', '-o', pdf_fname, svg_fname]
418        # use stdout and stderr from parent
419        exit_code = subprocess.call(cmd)
420        if exit_code != 0:
421            kernellog.warn(app, "Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
422        ok = bool(exit_code == 0)
423
424    return ok
425
426
427# image handling
428# ---------------------
429
430def visit_kernel_image(self, node):    # pylint: disable=W0613
431    """Visitor of the ``kernel_image`` Node.
432
433    Handles the ``image`` child-node with the ``convert_image(...)``.
434    """
435    img_node = node[0]
436    convert_image(img_node, self)
437
438class kernel_image(nodes.image):
439    """Node for ``kernel-image`` directive."""
440    pass
441
442class KernelImage(images.Image):
443    u"""KernelImage directive
444
445    Earns everything from ``.. image::`` directive, except *remote URI* and
446    *glob* pattern. The KernelImage wraps a image node into a
447    kernel_image node. See ``visit_kernel_image``.
448    """
449
450    def run(self):
451        uri = self.arguments[0]
452        if uri.endswith('.*') or uri.find('://') != -1:
453            raise self.severe(
454                'Error in "%s: %s": glob pattern and remote images are not allowed'
455                % (self.name, uri))
456        result = images.Image.run(self)
457        if len(result) == 2 or isinstance(result[0], nodes.system_message):
458            return result
459        (image_node,) = result
460        # wrap image node into a kernel_image node / see visitors
461        node = kernel_image('', image_node)
462        return [node]
463
464# figure handling
465# ---------------------
466
467def visit_kernel_figure(self, node):   # pylint: disable=W0613
468    """Visitor of the ``kernel_figure`` Node.
469
470    Handles the ``image`` child-node with the ``convert_image(...)``.
471    """
472    img_node = node[0][0]
473    convert_image(img_node, self)
474
475class kernel_figure(nodes.figure):
476    """Node for ``kernel-figure`` directive."""
477
478class KernelFigure(Figure):
479    u"""KernelImage directive
480
481    Earns everything from ``.. figure::`` directive, except *remote URI* and
482    *glob* pattern.  The KernelFigure wraps a figure node into a kernel_figure
483    node. See ``visit_kernel_figure``.
484    """
485
486    def run(self):
487        uri = self.arguments[0]
488        if uri.endswith('.*') or uri.find('://') != -1:
489            raise self.severe(
490                'Error in "%s: %s":'
491                ' glob pattern and remote images are not allowed'
492                % (self.name, uri))
493        result = Figure.run(self)
494        if len(result) == 2 or isinstance(result[0], nodes.system_message):
495            return result
496        (figure_node,) = result
497        # wrap figure node into a kernel_figure node / see visitors
498        node = kernel_figure('', figure_node)
499        return [node]
500
501
502# render handling
503# ---------------------
504
505def visit_kernel_render(self, node):
506    """Visitor of the ``kernel_render`` Node.
507
508    If rendering tools available, save the markup of the ``literal_block`` child
509    node into a file and replace the ``literal_block`` node with a new created
510    ``image`` node, pointing to the saved markup file. Afterwards, handle the
511    image child-node with the ``convert_image(...)``.
512    """
513    app = self.builder.app
514    srclang = node.get('srclang')
515
516    kernellog.verbose(app, 'visit kernel-render node lang: "%s"' % (srclang))
517
518    tmp_ext = RENDER_MARKUP_EXT.get(srclang, None)
519    if tmp_ext is None:
520        kernellog.warn(app, 'kernel-render: "%s" unknown / include raw.' % (srclang))
521        return
522
523    if not dot_cmd and tmp_ext == '.dot':
524        kernellog.verbose(app, "dot from graphviz not available / include raw.")
525        return
526
527    literal_block = node[0]
528
529    code      = literal_block.astext()
530    hashobj   = code.encode('utf-8') #  str(node.attributes)
531    fname     = path.join('%s-%s' % (srclang, sha1(hashobj).hexdigest()))
532
533    tmp_fname = path.join(
534        self.builder.outdir, self.builder.imagedir, fname + tmp_ext)
535
536    if not path.isfile(tmp_fname):
537        mkdir(path.dirname(tmp_fname))
538        with open(tmp_fname, "w") as out:
539            out.write(code)
540
541    img_node = nodes.image(node.rawsource, **node.attributes)
542    img_node['uri'] = path.join(self.builder.imgpath, fname + tmp_ext)
543    img_node['candidates'] = {
544        '*': path.join(self.builder.imgpath, fname + tmp_ext)}
545
546    literal_block.replace_self(img_node)
547    convert_image(img_node, self, tmp_fname)
548
549
550class kernel_render(nodes.General, nodes.Inline, nodes.Element):
551    """Node for ``kernel-render`` directive."""
552    pass
553
554class KernelRender(Figure):
555    u"""KernelRender directive
556
557    Render content by external tool.  Has all the options known from the
558    *figure*  directive, plus option ``caption``.  If ``caption`` has a
559    value, a figure node with the *caption* is inserted. If not, a image node is
560    inserted.
561
562    The KernelRender directive wraps the text of the directive into a
563    literal_block node and wraps it into a kernel_render node. See
564    ``visit_kernel_render``.
565    """
566    has_content = True
567    required_arguments = 1
568    optional_arguments = 0
569    final_argument_whitespace = False
570
571    # earn options from 'figure'
572    option_spec = Figure.option_spec.copy()
573    option_spec['caption'] = directives.unchanged
574
575    def run(self):
576        return [self.build_node()]
577
578    def build_node(self):
579
580        srclang = self.arguments[0].strip()
581        if srclang not in RENDER_MARKUP_EXT.keys():
582            return [self.state_machine.reporter.warning(
583                'Unknown source language "%s", use one of: %s.' % (
584                    srclang, ",".join(RENDER_MARKUP_EXT.keys())),
585                line=self.lineno)]
586
587        code = '\n'.join(self.content)
588        if not code.strip():
589            return [self.state_machine.reporter.warning(
590                'Ignoring "%s" directive without content.' % (
591                    self.name),
592                line=self.lineno)]
593
594        node = kernel_render()
595        node['alt'] = self.options.get('alt','')
596        node['srclang'] = srclang
597        literal_node = nodes.literal_block(code, code)
598        node += literal_node
599
600        caption = self.options.get('caption')
601        if caption:
602            # parse caption's content
603            parsed = nodes.Element()
604            self.state.nested_parse(
605                ViewList([caption], source=''), self.content_offset, parsed)
606            caption_node = nodes.caption(
607                parsed[0].rawsource, '', *parsed[0].children)
608            caption_node.source = parsed[0].source
609            caption_node.line = parsed[0].line
610
611            figure_node = nodes.figure('', node)
612            for k,v in self.options.items():
613                figure_node[k] = v
614            figure_node += caption_node
615
616            node = figure_node
617
618        return node
619
620def add_kernel_figure_to_std_domain(app, doctree):
621    """Add kernel-figure anchors to 'std' domain.
622
623    The ``StandardDomain.process_doc(..)`` method does not know how to resolve
624    the caption (label) of ``kernel-figure`` directive (it only knows about
625    standard nodes, e.g. table, figure etc.). Without any additional handling
626    this will result in a 'undefined label' for kernel-figures.
627
628    This handle adds labels of kernel-figure to the 'std' domain labels.
629    """
630
631    std = app.env.domains["std"]
632    docname = app.env.docname
633    labels = std.data["labels"]
634
635    for name, explicit in doctree.nametypes.items():
636        if not explicit:
637            continue
638        labelid = doctree.nameids[name]
639        if labelid is None:
640            continue
641        node = doctree.ids[labelid]
642
643        if node.tagname == 'kernel_figure':
644            for n in node.next_node():
645                if n.tagname == 'caption':
646                    sectname = clean_astext(n)
647                    # add label to std domain
648                    labels[name] = docname, labelid, sectname
649                    break
650