Added automatic generation of examples for documentation purposes Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
diff --git a/docs/Makefile b/docs/Makefile index 1475281..b175e17 100644 --- a/docs/Makefile +++ b/docs/Makefile
@@ -43,6 +43,7 @@ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) livehtml: + rm -rf examples @$(SPHINXAUTOBUILD) -b html --host ${LIVEHTML_HOST} --port ${LIVEHTML_PORT} --ignore \*.swp --ignore \*~ $(SPHINXOPTS) "$(SOURCEDIR)" "$(BUILDDIR)/html" .PHONY: help livehtml Makefile @@ -63,32 +64,7 @@ .PHONY: env -# Update fuzzer / minitest markdown links. -tests-links: - @mkdir -p $(MAKEDIR)/examples - @cd $(MAKEDIR)/examples && find -name '*.md' -delete && find -type d -empty -delete - @cd $(MAKEDIR)/examples && \ - for I in $$(cd ../../tests/ ; find -name '*.md' | sort); do \ - F=$$(dirname $$I); D=$$(dirname $$F); N=$$(basename $$F); \ - S=../../tests/$$I; O=$$D/$$N.md; \ - if [ ! -d $$D ]; then \ - mkdir -p $$D; \ - fi; \ - ln -sf $$(realpath $$S --relative-to=$$D) $$O; \ - done - @cd $(MAKEDIR)/examples && \ - for I in $$(cd ../../tests/ ; find -name '*.svg' -o -name '*.png' | sort); do \ - F=$$(dirname $$I); D=$$(dirname $$F); N=$$(basename $$I); \ - S=../../tests/$$I; O=$$D/$$N; \ - ln -sf $$(realpath $$S --relative-to=$$D) $$O; \ - done - -links: tests-links - @true - -.PHONY: tests-links links - # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile links +%: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/collect_examples.py b/docs/collect_examples.py new file mode 100644 index 0000000..3bbc10c --- /dev/null +++ b/docs/collect_examples.py
@@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2020 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +import os +from shutil import copy, copytree, ignore_patterns + +from make_xml_for_docs import make_xml_for_docs + +# ============================================================================= + + +README_NAME = "README.rst" +TEMPLATE_NAME = "examples.rst.template" +INDEX_NAME = "examples.rst" + + +def process_verilogs(src_dir, dst_dir): + """ + Looks for Verilog models for V2X and runs make_xml_for_docs on them. + + If at least one Verilog model is found at a directory level then + the function does not look deeper. Otherwise it searches for models + recursively. + """ + + got_verilog = False + + # Find and process all verilog files at this directory level + for f in os.listdir(src_dir): + f_name = os.path.join(src_dir, f) + if os.path.isfile(f_name) and f_name.endswith(".sim.v"): + got_verilog = True + + make_xml_for_docs(f_name, dst_dir) + + # Don't go deeper if one was found + if got_verilog is True: + return + + # Look into subdirectories + for d in os.listdir(src_dir): + d_name = os.path.join(src_dir, d) + if os.path.isdir(d_name): + + process_verilogs( + os.path.join(src_dir, d), + os.path.join(dst_dir, d) + ) + + +def collect_examples(): + """ + Collects test designs that have readme files and converts them for + documentation examples. + """ + + # Base directory for tests + tests_dir = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "../tests" + ) + ) + + # Base directory for examples + examples_dir = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "examples" + ) + ) + + # Skip if the directory already exist + if os.path.isdir(examples_dir): + return + + print("Collecting test designs and including them in \"examples\" section") + + # Create the directory + os.makedirs(examples_dir, exist_ok=True) + + # Look for all subdirectories that have "readme.rst" file inside + tests = [] + for d in os.listdir(tests_dir): + + # Only directories + d_name = os.path.join(tests_dir, d) + if os.path.isdir(d_name): + + # Must contain the readme file + f_name = os.path.join(d_name, README_NAME) + if os.path.isfile(f_name): + tests.append((d, d_name,)) + + # Process each test + check_ignore = ignore_patterns("*.v", "*.xml") + for test_name, test_src in tests: + + test_rel = os.path.relpath(test_src, tests_dir) + test_dst = os.path.join(examples_dir, test_rel) + + print("", test_name) + + # Copy files + copytree(test_src, test_dst, ignore=check_ignore) + + # Build XMLs for verilogs + process_verilogs(test_src, test_dst) + + # Build examples.rst + tname = os.path.join(os.path.dirname(__file__), TEMPLATE_NAME) + fname = os.path.join(examples_dir, INDEX_NAME) + + with open(tname, "r") as fsrc, open(fname, "w") as fdst: + + # Copy + for line in fsrc: + fdst.write(line) + + # Append included tests + tests = sorted(tests, key=lambda t:t[0]) + for test_name, _ in tests: + fdst.write(" {}/{}\n".format(test_name, README_NAME)) + + +if __name__ == "__main__": + collect_examples() +
diff --git a/docs/conf.py b/docs/conf.py index ec05997..358f647 100644 --- a/docs/conf.py +++ b/docs/conf.py
@@ -38,6 +38,8 @@ from sphinx.highlighting import lexers from pygments.lexers.hdl import VerilogLexer +from collect_examples import collect_examples + lexers['verilog'] = VerilogLexer(tabsize=2) # -- General configuration ------------------------------------------------ @@ -98,7 +100,6 @@ import subprocess subprocess.call('git fetch origin --unshallow', cwd=docs_dir, shell=True) subprocess.check_call('git fetch origin --tags', cwd=docs_dir, shell=True) - subprocess.check_call('make links', cwd=docs_dir, shell=True) # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -252,6 +253,10 @@ def setup(app): + + # Collect tests to form examples + collect_examples() + github_code_repo = 'https://github.com/SymbiFlow/python-symbiflow-v2x/' github_code_branch = 'blob/master/'
diff --git a/docs/examples.rst b/docs/examples.rst.template similarity index 78% rename from docs/examples.rst rename to docs/examples.rst.template index 158bdd3..68a49e4 100644 --- a/docs/examples.rst +++ b/docs/examples.rst.template
@@ -5,6 +5,3 @@ .. toctree:: - examples/clocks.rst - examples/dsp.rst - examples/vtr-examples.rst \ No newline at end of file
diff --git a/docs/index.rst b/docs/index.rst index 6c144c0..2ed4c79 100644 --- a/docs/index.rst +++ b/docs/index.rst
@@ -5,4 +5,4 @@ :glob: :hidden: - examples.rst + examples/examples.rst
diff --git a/docs/make_xml_for_docs.py b/docs/make_xml_for_docs.py new file mode 100644 index 0000000..7f20257 --- /dev/null +++ b/docs/make_xml_for_docs.py
@@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2020 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +import argparse +import os +import shlex + +from v2x import vlog_to_model +from v2x import vlog_to_pbtype +from v2x.xmlinc import xmlinc + +from vtr_xml_utils.convert import vtr_stylize_xml + +# ============================================================================= + + +def make_xml_for_docs(vlog_src, output_dir): + """ + Recursively runs V2X (model and pb_type) on the given Verilog file and + all its includes. Output XMLs are stylized using vtr-xml-utils. + + The Verilog file is copied to the output_dir. All include paths are + updated so that they reflect the module hierarchy. It is assumed that + a Verilog file is named <module>.sim.v. Next V2X is run followed by XML + stylization using vtr-xml-utils. + + The process continues recursively (depth-first). + """ + + # Make absolute paths + vlog_src = os.path.abspath(vlog_src) + output_dir = os.path.abspath(output_dir) + + # Make the output directory + os.makedirs(output_dir, exist_ok=True) + + # Copy the verilog file, rearrange includes + vlog_title = os.path.basename(vlog_src) + vlog_dir = os.path.dirname(vlog_src) + vlog_dst = os.path.join(output_dir, vlog_title) + + # Scan it for dependencies (includes), rewrite them + vlog_deps = [] + + with open(vlog_src, "r") as fsrc, open(vlog_dst, "w") as fdst: + for src_line in fsrc: + + # Modify included file path + line = src_line.strip() + if line.startswith("`include"): + fields = shlex.split(line) + assert len(fields) == 2, fields + + dep_src = fields[1] + dep_mod = os.path.basename(dep_src).split(".")[0] + dep_dst = "./{f}/{f}.sim.v".format(f=dep_mod) + + vlog_deps.append((dep_src, dep_dst,)) + + dst_line = "`include \"{}\"\n".format(dep_dst) + + # Pass unchanges + else: + dst_line = src_line + + # Write the line + fdst.write(dst_line) + + # Process dependencies recursively first + for dep_src, dep_dst in vlog_deps: + dep_file = os.path.join(vlog_dir, dep_src) + dep_dir = os.path.join(output_dir, os.path.dirname(dep_dst)) + + make_xml_for_docs(dep_file, dep_dir) + + # Run V2X + vlog_mod = vlog_title.split(".")[0] + + model_file = os.path.join(output_dir, vlog_mod + ".model.xml") + pbtype_file = os.path.join(output_dir, vlog_mod + ".pb_type.xml") + + xml = vlog_to_model.vlog_to_model([vlog_dst], None, None, model_file) + with open(model_file, "w") as fp: + fp.write(xml) + + xml = vtr_stylize_xml(model_file) + with open(model_file, "w") as fp: + fp.write(xml) + + xml = vlog_to_pbtype.vlog_to_pbtype([vlog_dst], pbtype_file, None) + with open(pbtype_file, "w") as fp: + fp.write(xml) + + xml = vtr_stylize_xml(pbtype_file) + with open(pbtype_file, "w") as fp: + fp.write(xml) + + +# ============================================================================= + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + + parser.add_argument( + "verilog", + help="Input Verilog file" + ) + parser.add_argument( + "path", + help="Output path" + ) + + args = parser.parse_args() + make_xml_for_docs(args.verilog, args.path)
diff --git a/docs/requirements.txt b/docs/requirements.txt index 7ed74ef..fb3652c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt
@@ -18,3 +18,6 @@ pycairo # vext.gi + +# V2X +-e ../