Merge pull request #1846 from antmicro/use-gh-runners
gh: initial use of gh action custom runners
diff --git a/.github/kokoro/steps/hostsetup.sh b/.github/kokoro/steps/hostsetup.sh
index dc267b1..91cc230 100755
--- a/.github/kokoro/steps/hostsetup.sh
+++ b/.github/kokoro/steps/hostsetup.sh
@@ -87,14 +87,14 @@
lsb \
nodejs \
psmisc \
- python3.8 \
- python3.8-dev \
- python3.8-venv
+ python3 \
+ python3-dev \
+ python3-venv
echo "========================================"
echo "Enter virtual env for python 3.8"
echo "----------------------------------------"
-python3.8 -mvenv startup_python
+python3 -mvenv startup_python
source startup_python/bin/activate
which python
python --version
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..6122381
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,100 @@
+on:
+ pull_request:
+ push:
+ schedule:
+ - cron: '0 0 * * *'
+
+name: CI Build
+
+jobs:
+
+ Tests:
+ container: ubuntu:bionic
+
+ runs-on: [self-hosted, Linux, X64]
+
+ env:
+ ALLOW_ROOT: true
+
+ steps:
+
+ - uses: actions/checkout@v2
+ with:
+ submodules: recursive
+
+ - name: Install
+ run: |
+ apt update
+ apt install -y \
+ bash bison build-essential ca-certificates clang-format cmake psmisc \
+ colordiff coreutils git flex python3 python3-dev python3-venv xsltproc
+
+ - name: Build
+ run: make build --output-sync=target --warn-undefined-variables -j$(nproc)
+
+ - name: Environment
+ run: make env --output-sync=target --warn-undefined-variables
+
+ - name: Run Test
+ run: make test --output-sync=target --warn-undefined-variables
+
+ - uses: actions/upload-artifact@v2
+ if: ${{ always() }}
+ with:
+ path: |
+ **/results*.gz
+ **/plot_*.svg
+
+ BuildDatabase:
+ container: ubuntu:bionic
+
+ runs-on: [self-hosted, Linux, X64]
+
+ strategy:
+ fail-fast: false
+ matrix:
+ family: ['artix7', 'zynq7', 'kintex7', 'spartan7']
+
+ env:
+ ALLOW_ROOT: true
+ GHA_EXTERNAL_DISK: "tools"
+ XILINX_LOCAL_USER_DATA: "no"
+
+ steps:
+
+ - uses: actions/checkout@v2
+ with:
+ submodules: recursive
+
+ - name: Install
+ run: |
+ apt update
+ apt install -y \
+ bash bison build-essential ca-certificates clang-format cmake psmisc \
+ colordiff coreutils git flex python3 python3-dev python3-venv xsltproc
+
+ - name: Build
+ run: make build --output-sync=target --warn-undefined-variables -j$(nproc)
+
+ - name: Environment
+ run: make env --output-sync=target --warn-undefined-variables
+
+ - name: Run Test
+ run: .github/workflows/scripts/db.sh
+ env:
+ XRAY_SETTINGS: ${{ matrix.family }}
+
+ - uses: actions/upload-artifact@v2
+ if: ${{ always() }}
+ with:
+ name: ${{ matrix.family }}
+ path: |
+ **/results*.gz
+ **/plot_*.svg
+ **/diff.html
+ **/diff.json
+ **/diff.patch
+ **/*sponge_log.xml
+ **/fuzzers/*.tgz
+ **/database/${{ matrix.family }}/**"
+
diff --git a/.github/workflows/scripts/db.sh b/.github/workflows/scripts/db.sh
new file mode 100755
index 0000000..1b69807
--- /dev/null
+++ b/.github/workflows/scripts/db.sh
@@ -0,0 +1,219 @@
+#!/bin/bash
+# Copyright (C) 2017-2020 The Project X-Ray Authors.
+#
+# Use of this source code is governed by a ISC-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/ISC
+#
+# SPDX-License-Identifier: ISC
+
+set -e
+
+source $(dirname "$0")/hostinfo.sh
+
+echo
+echo "======================================="
+echo "Creating Vivado Symbolic Link"
+echo "---------------------------------------"
+ln -s /mnt/aux/Xilinx /opt/Xilinx
+ls /opt/Xilinx/Vivado
+source /opt/Xilinx/Vivado/2017.2/settings64.sh
+vivado -version
+
+echo
+echo "========================================"
+echo "Downloading current database"
+echo "----------------------------------------"
+(
+ script --return --flush --command "./download-latest-db.sh" -
+)
+echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo "Preparing database"
+echo "----------------------------------------"
+(
+ make db-prepare-${XRAY_SETTINGS}
+)
+echo "----------------------------------------"
+
+source settings/$XRAY_SETTINGS.sh
+
+echo
+echo "========================================"
+echo "Cleaning out current database"
+echo "----------------------------------------"
+(
+ cd database
+ make clean-${XRAY_SETTINGS}-db
+)
+echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo "Running Database build"
+echo "----------------------------------------"
+(
+ # Output which fuzzers we are going to run
+ echo "make --dry-run"
+ make --dry-run db-${XRAY_SETTINGS}-all
+ echo "----------------------------------------"
+
+ # Run the fuzzers
+ set -x +e
+ tmp=$(mktemp)
+ script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$MAX_VIVADO_PROCESS db-${XRAY_SETTINGS}-all" $tmp
+ DATABASE_RET=$?
+ set +x -e
+
+ if [[ $DATABASE_RET != 0 ]] ; then
+ # Collect the Vivado logs into one tgz archive
+ echo "Packing failing test cases"
+ # Looking for the failing directories and packing them
+ # example of line from which the failing fuzzer directory gets extracted:
+ # - Makefile:87: recipe for target '000-db-init/000-init-db/run.xc7a100tfgg676-1.ok' failed --> fuzzers/000-db-init
+ grep -Po "recipe for target '\K(.*)(?=\/run\..*\.ok')" $tmp | sed -e 's/^/fuzzers\//' | xargs tar -zcf fuzzers/fails.tgz
+ echo "----------------------------------------"
+ echo "A failure occurred during Database build."
+ echo "----------------------------------------"
+ rm $tmp
+
+ echo "========================================"
+ echo " Disk space in failure path"
+ echo "----------------------------------------"
+ du -sh
+
+ exit $DATABASE_RET
+ fi
+)
+echo "----------------------------------------"
+
+# Format the database
+make db-format-${XRAY_SETTINGS}
+# Update the database/Info.md file
+make db-info
+
+# Output if the database has differences
+echo
+echo "========================================"
+echo " Database Differences"
+echo "----------------------------------------"
+(
+ cd database
+ # Update the index with any new files
+ git add \
+ --verbose \
+ --all \
+ --ignore-errors \
+ .
+
+ # Output what git status
+ echo
+ echo "----------------------------------------"
+ echo " Database Status"
+ echo "----------------------------------------"
+ git status
+ echo "----------------------------------------"
+
+
+ # Output a summary of how the files have changed
+ echo
+ echo "----------------------------------------"
+ echo " Database Diff Summary"
+ echo "----------------------------------------"
+ git diff --stat --irreversible-delete --find-renames --find-copies --ignore-all-space origin/master
+
+ # Save the diff to be uploaded as an artifact
+ echo
+ echo "----------------------------------------"
+ echo " Saving diff output"
+ echo "----------------------------------------"
+ # Patch file
+ git diff \
+ --patch-with-stat --no-color --irreversible-delete --find-renames --find-copies origin/master \
+ > diff.patch
+
+ MAX_DIFF_LINES=50000
+ DIFF_LINES="$(wc -l diff.patch | sed -e's/ .*$//')"
+ if [ $DIFF_LINES -gt $MAX_DIFF_LINES ]; then
+ echo
+ echo "----------------------------------------"
+ echo " Database Diff"
+ echo "----------------------------------------"
+ echo "diff has $DIFF_LINES lines which is too large to display!"
+
+ echo
+ echo "----------------------------------------"
+ echo " Generating pretty diff output"
+ echo "----------------------------------------"
+ echo "diff has $DIFF_LINES lines which is too large for HTML output!"
+ else
+ # Output the actually diff
+ echo
+ echo "----------------------------------------"
+ echo " Database Diff"
+ echo "----------------------------------------"
+ git diff --color --irreversible-delete --find-renames --find-copies --ignore-all-space origin/master
+
+ echo
+ echo "----------------------------------------"
+ echo " Generating pretty diff output"
+ echo "----------------------------------------"
+ (
+ # Allow the diff2html to fail.
+ set +e
+
+ # Pretty HTML file version
+ diff2html --summary=open --file diff.html --format html \
+ -- \
+ --irreversible-delete --find-renames --find-copies \
+ --ignore-all-space origin/master || true
+
+ # Programmatic JSON version
+ diff2html --file diff.json --format json \
+ -- \
+ --irreversible-delete --find-renames --find-copies \
+ --ignore-all-space origin/master || true
+ ) || true
+ fi
+)
+echo "----------------------------------------"
+
+# Check the database and fail if it is broken.
+set -x +e
+make db-check-${XRAY_SETTINGS}
+CHECK_RET=$?
+set +x -e
+
+echo
+echo "========================================"
+echo " Testing HTML generation"
+echo "----------------------------------------"
+(
+ cd htmlgen
+ source htmlgen.sh $XRAY_SETTINGS
+)
+
+# If we get here, then all the fuzzers completed fine. Hence we are
+# going to assume we don't want to keep all the build / logs / etc (as
+# they are quite large). Thus do a clean to get rid of them.
+echo
+echo "========================================"
+echo " Cleaning up after success"
+echo "----------------------------------------"
+(
+ cd fuzzers
+ echo
+ echo "Cleaning up so CI doesn't save all the excess data."
+ make clean_fuzzers
+ make clean_piplists
+)
+echo "----------------------------------------"
+
+echo "========================================"
+echo " Final disk space after cleanup"
+echo "----------------------------------------"
+du -sh
+
+exit $CHECK_RET
diff --git a/.github/workflows/scripts/hostinfo.sh b/.github/workflows/scripts/hostinfo.sh
new file mode 100755
index 0000000..577577c
--- /dev/null
+++ b/.github/workflows/scripts/hostinfo.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# Copyright (C) 2017-2020 The Project X-Ray Authors.
+#
+# Use of this source code is governed by a ISC-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/ISC
+#
+# SPDX-License-Identifier: ISC
+
+set -e
+
+echo
+echo "========================================"
+echo "Host Environment"
+echo "----------------------------------------"
+export
+echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo "Host CPU"
+echo "----------------------------------------"
+export CORES=$(nproc --all)
+echo "Cores: $CORES"
+echo
+echo "Memory"
+echo "----------------------------------------"
+cat /proc/meminfo
+echo "----------------------------------------"
+export MEM_GB=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo)/(1024*1024)))
+echo "Total Memory (GB): $MEM_GB"
+
+# Approx memory per grid process
+export MEM_PER_RUN=8
+export MAX_GRID_CPU=$(($MEM_GB/$MEM_PER_RUN))
+export MAX_VIVADO_PROCESS=$(($MEM_GB/$MEM_PER_RUN))
diff --git a/Makefile b/Makefile
index 53c3878..acb7f42 100644
--- a/Makefile
+++ b/Makefile
@@ -10,10 +10,14 @@
INSTALL_DIR ?=
+# Skip this check if the ALLOW_ROOT var is defined
+# E.g. when running in GH action custom runners CI
+ifndef ALLOW_ROOT
# Check if root
ifeq ($(shell id -u),0)
$(error ERROR: Running as ID 0)
endif
+endif
# Tools + Environment
IN_ENV = if [ -e env/bin/activate ]; then . env/bin/activate; fi; source utils/environment.python.sh;
@@ -275,4 +279,4 @@
$(MAKE) -C fuzzers clean
rm -rf build
-.PHONY: clean
\ No newline at end of file
+.PHONY: clean
diff --git a/fuzzers/065-gtp-common-pips/Makefile b/fuzzers/065-gtp-common-pips/Makefile
index 5458067..3a7ab88 100644
--- a/fuzzers/065-gtp-common-pips/Makefile
+++ b/fuzzers/065-gtp-common-pips/Makefile
@@ -54,7 +54,7 @@
cd $(BUILD_DIR)/ && ${XRAY_VIVADO} -mode batch -source ${FUZDIR}/output_cmt.tcl
clean:
- rm -rf build_* run.*.ok
+ rm -rf ${BUILD_DIR} ${RUN_OK}
pushdb: database
${XRAY_MERGEDB} gtp_common_mid_left $(BUILD_DIR)/segbits_gtp_common.db
diff --git a/fuzzers/065b-gtp-common-pips/Makefile b/fuzzers/065b-gtp-common-pips/Makefile
index 7338091..c806e18 100644
--- a/fuzzers/065b-gtp-common-pips/Makefile
+++ b/fuzzers/065b-gtp-common-pips/Makefile
@@ -5,6 +5,7 @@
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC
+
export FUZDIR=$(shell pwd)
PIP_TYPE?=gtp_common_mid_${XRAY_PART}
PIP_FILE?=gtp_common_mid_ck_mux
@@ -24,7 +25,7 @@
SPECIMENS_DEPS=$(BUILD_DIR)/cmt_regions.csv
A_PIPLIST=gtp_common_mid_ck_mux.txt
-CHECK_ARGS= --zero-entries --timeout-iters 5 --todo-dir $(BUILD_DIR)/todo
+CHECK_ARGS= --zero-entries --timeout-iters 10 --todo-dir $(BUILD_DIR)/todo
include ../pip_loop.mk
@@ -53,7 +54,7 @@
cd $(BUILD_DIR)/ && ${XRAY_VIVADO} -mode batch -source ${FUZDIR}/output_cmt.tcl
clean:
- rm -rf build_* run.*.ok
+ rm -rf ${BUILD_DIR} ${RUN_OK}
pushdb: database
${XRAY_MERGEDB} gtp_common_mid_left $(BUILD_DIR)/segbits_gtp_common.db
diff --git a/fuzzers/072-ordered_wires/Makefile b/fuzzers/072-ordered_wires/Makefile
index 1905e5e..92ab82e 100644
--- a/fuzzers/072-ordered_wires/Makefile
+++ b/fuzzers/072-ordered_wires/Makefile
@@ -7,7 +7,10 @@
# SPDX-License-Identifier: ISC
N := 1
+
BUILD_DIR = build_${XRAY_PART}
+RUN_OK = run.${XRAY_PART}.ok
+
SPECIMENS := $(addprefix $(BUILD_DIR)/specimen_,$(shell seq -f '%03.0f' $(N)))
SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS))
MAX_VIVADO_PROCESS ?= 4
@@ -24,13 +27,13 @@
touch $@
run:
- rm -rf $(BUILD_DIR) run.${XRAY_PART}.ok
+ rm -rf $(BUILD_DIR) $(RUN_OK)
$(MAKE) database
$(MAKE) pushdb
- touch run.${XRAY_PART}.ok
+ touch $(RUN_OK)
clean:
- rm -rf build_* run.*.ok
+ rm -rf ${BUILD_DIR} ${RUN_OK}
.PHONY: database pushdb run clean
diff --git a/fuzzers/073-get_counts/Makefile b/fuzzers/073-get_counts/Makefile
index 4cf0c02..28a9e54 100644
--- a/fuzzers/073-get_counts/Makefile
+++ b/fuzzers/073-get_counts/Makefile
@@ -7,7 +7,10 @@
# SPDX-License-Identifier: ISC
N := 1
+
BUILD_DIR = build_${XRAY_PART}
+RUN_OK = run.${XRAY_PART}.ok
+
SPECIMENS := $(addprefix $(BUILD_DIR)/specimen_,$(shell seq -f '%03.0f' $(N)))
SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS))
@@ -22,13 +25,13 @@
touch $@
run:
- rm -rf $(BUILD_DIR) run.${XRAY_PART}.ok
+ rm -rf $(BUILD_DIR) $(RUN_OK)
$(MAKE) database
$(MAKE) pushdb
- touch run.${XRAY_PART}.ok
+ touch $(RUN_OK)
clean:
- rm -rf build_* run.*.ok
+ rm -rf ${BUILD_DIR} ${RUN_OK}
.PHONY: database pushdb run clean
diff --git a/fuzzers/074-dump_all/Makefile b/fuzzers/074-dump_all/Makefile
index d2dcabd..2e8106d 100644
--- a/fuzzers/074-dump_all/Makefile
+++ b/fuzzers/074-dump_all/Makefile
@@ -7,7 +7,10 @@
# SPDX-License-Identifier: ISC
N := 1
+
BUILD_DIR = build_${XRAY_PART}
+RUN_OK = run.${XRAY_PART}.ok
+
SPECIMENS := $(addprefix $(BUILD_DIR)/specimen_,$(shell seq -f '%03.0f' $(N)))
SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS))
MAX_VIVADO_PROCESS ?= 4
@@ -34,10 +37,10 @@
$(MAKE) pushdb
# Clean up intermediate files after successful pushdb.
find $(BUILD_DIR) -name "*.json5" -delete
- touch run.${XRAY_PART}.ok
+ touch ${RUN_OK}
clean:
- rm -rf build_* run.*.ok
+ rm -rf ${BUILD_DIR} ${RUN_OK}
.PHONY: database pushdb run clean
diff --git a/fuzzers/075-pins/Makefile b/fuzzers/075-pins/Makefile
index 153c803..1b167c3 100644
--- a/fuzzers/075-pins/Makefile
+++ b/fuzzers/075-pins/Makefile
@@ -7,6 +7,8 @@
# SPDX-License-Identifier: ISC
N := 1
BUILD_DIR = build_${XRAY_PART}
+RUN_OK = run.${XRAY_PART}.ok
+
SPECIMENS := $(addprefix $(BUILD_DIR)/specimen_,$(shell seq -f '%03.0f' $(N)))
SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS))
@@ -21,12 +23,12 @@
touch $@
run:
- rm -rf $(BUILD_DIR) run.${XRAY_PART}.ok
+ rm -rf $(BUILD_DIR) $(RUN_OK)
$(MAKE) database
$(MAKE) pushdb
- touch run.$(XRAY_PART).ok
+ touch $(RUN_OK)
clean:
- rm -rf build_* run.*.ok
+ rm -rf ${BUILD_DIR} ${RUN_OK}
.PHONY: database pushdb run clean
diff --git a/fuzzers/Makefile b/fuzzers/Makefile
index 7f8e945..ee43c95 100644
--- a/fuzzers/Makefile
+++ b/fuzzers/Makefile
@@ -116,7 +116,11 @@
$(eval $(call fuzzer,031-cmt-mmcm,005-tilegrid,all))
$(eval $(call fuzzer,032-cmt-pll,005-tilegrid,all))
$(eval $(call fuzzer,034-cmt-pll-pips,005-tilegrid 071-ppips,all))
+ifneq ($(XRAY_DATABASE),kintex7)
+# FIXME: 034b fuzzer is generating conflicting bits around the FREQ_BB[N] bits.
+# The fuzzer can be re-enabled once the conflicting bits are not generated anymore
$(eval $(call fuzzer,034b-cmt-mmcm-pips,005-tilegrid 071-ppips,all))
+endif
$(eval $(call fuzzer,035-iob-ilogic,005-tilegrid,all))
$(eval $(call fuzzer,035a-iob-idelay,005-tilegrid,all))
$(eval $(call fuzzer,035b-iob-iserdes,005-tilegrid,all))
diff --git a/fuzzers/run_fuzzer.py b/fuzzers/run_fuzzer.py
index 7ea75a1..d8ca8aa 100755
--- a/fuzzers/run_fuzzer.py
+++ b/fuzzers/run_fuzzer.py
@@ -512,7 +512,7 @@
)
log(running_msg)
- log_suffix = ".{}.log".format(time_start.isoformat())
+ log_suffix = ".{}.log".format(time_start.isoformat()).replace(":", "-")
fuzzer_stdout = os.path.join(fuzzer_logdir, "stdout" + log_suffix)
fuzzer_stderr = os.path.join(fuzzer_logdir, "stderr" + log_suffix)
diff --git a/prjxray/tile_segbits.py b/prjxray/tile_segbits.py
index b9f940b..8fd6b15 100644
--- a/prjxray/tile_segbits.py
+++ b/prjxray/tile_segbits.py
@@ -10,6 +10,7 @@
# SPDX-License-Identifier: ISC
from collections import namedtuple
from prjxray import bitstream
+from prjxray import util
from prjxray.grid_types import BlockType
import enum
@@ -84,15 +85,21 @@
if tile_db.ppips is not None:
with open(tile_db.ppips) as f:
+ util.lock_file(f, 10)
self.ppips = read_ppips(f)
+ util.unlock_file(f)
if tile_db.segbits is not None:
with open(tile_db.segbits) as f:
+ util.lock_file(f, 10)
self.segbits[BlockType.CLB_IO_CLK] = read_segbits(f)
+ util.unlock_file(f)
if tile_db.block_ram_segbits is not None:
with open(tile_db.block_ram_segbits) as f:
+ util.lock_file(f, 10)
self.segbits[BlockType.BLOCK_RAM] = read_segbits(f)
+ util.unlock_file(f)
for block_type in self.segbits:
for feature in self.segbits[block_type]:
diff --git a/prjxray/util.py b/prjxray/util.py
index 6c28579..5ce4550 100644
--- a/prjxray/util.py
+++ b/prjxray/util.py
@@ -8,10 +8,12 @@
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC
+import fcntl
import math
import os
import random
import re
+import signal
import yaml
from .roi import Roi
@@ -254,8 +256,10 @@
def parse_db_lines(fn):
with open(fn, "r") as f:
+ lock_file(f, 10)
for line in f:
yield line, parse_db_line(line)
+ unlock_file(f)
def write_db_lines(fn, entries, track_origin=False):
@@ -269,8 +273,10 @@
new_lines.append(new_line)
with open(fn, "w") as f:
+ lock_file(f, 10)
for line in sorted(new_lines):
print(line, file=f)
+ unlock_file(f)
def parse_tagbit(x):
@@ -402,3 +408,22 @@
yes_arg, dest=dest, action='store_true', default=default, **kwargs)
parser.add_argument(
'--no-' + dashed, dest=dest, action='store_false', **kwargs)
+
+
+def timeout_handler(signum, frame):
+ raise Exception("ERROR: could not lock file!")
+
+
+def lock_file(fd, timeout):
+ try:
+ signal.signal(signal.SIGALRM, timeout_handler)
+ signal.alarm(timeout)
+ fcntl.flock(fd.fileno(), fcntl.LOCK_EX)
+ signal.alarm(0)
+ except Exception as e:
+ print(e)
+ exit(1)
+
+
+def unlock_file(fd):
+ fcntl.flock(fd.fileno(), fcntl.LOCK_UN)
diff --git a/requirements.txt b/requirements.txt
index e10243a..5421081 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,3 @@
--e third_party/fasm
--e third_party/python-sdf-timing
--e .
intervaltree
junit-xml
numpy
@@ -10,9 +7,15 @@
progressbar2
pyjson5
pytest
+pytest-runner
pyyaml
scipy>=1.2.1
simplejson
sympy
textx
yapf==0.24.0
+
+# Third party
+-e third_party/fasm
+-e third_party/python-sdf-timing
+-e .
diff --git a/utils/roi_all.py b/utils/roi_all.py
index 660f952..d3d3cea 100755
--- a/utils/roi_all.py
+++ b/utils/roi_all.py
@@ -9,13 +9,26 @@
#
# SPDX-License-Identifier: ISC
import argparse
-import yaml
-import subprocess
+import multiprocessing as mp
import os
import re
+import subprocess
+import yaml
+
from prjxray import util
+def worker(arglist):
+ part, cwd = arglist
+ cmd = "make roi_only"
+
+ env = os.environ.copy()
+ env['XRAY_PART'] = part
+
+ print("running subprocess")
+ subprocess.run(cmd.split(' '), check=True, env=env, cwd=cwd)
+
+
def main():
"""Rois all parts for a family by calling "make roi_only" over all parts
with the same device as XRAY_PART.
@@ -36,12 +49,17 @@
if device['fabric'] == information['device']:
valid_devices.append(name)
+ tasks = []
+
for part, data in util.get_parts(db_root).items():
if data['device'] in valid_devices:
- command = "make roi_only"
- env['XRAY_PART'] = part
cwd = os.getenv('XRAY_FUZZERS_DIR')
- subprocess.run(command.split(' '), check=True, env=env, cwd=cwd)
+
+ tasks.append((part, cwd))
+
+ with mp.Pool() as pool:
+ for _ in pool.imap_unordered(worker, tasks):
+ pass
if __name__ == '__main__':
