Merge pull request #2589 from hzeller/feature-20260829-hash
Fix smoke test projects at a particular hash
diff --git a/.github/bin/smoke-projects.hashes b/.github/bin/smoke-projects.hashes
new file mode 100644
index 0000000..5298441
--- /dev/null
+++ b/.github/bin/smoke-projects.hashes
@@ -0,0 +1,20 @@
+a12778b40ae905f82eb2c23ea4ac037f98099fae https://github.com/jamieiles/80x86
+4da15979747f326bde2f9869c64e587ce599772c https://github.com/pulp-platform/axi
+b48037e28544425839dbd617d45b1a82631bc1a9 https://github.com/bespoke-silicon-group/basejump_stl
+f91010f654a5dfd00f83dbe25dbda482218d540b https://github.com/black-parrot/black-parrot
+86d09682492176a1f8fd1732e469bc2d309453a3 https://github.com/chipsalliance/caliptra-rtl
+f9fbd09655ec6ee457438140c13a168390ebd043 https://github.com/chipsalliance/Cores-VeeR-EH2
+3ac6208ed0210ff66a557dd7dc4e79e34f1f2f4a https://github.com/openhwgroup/cva6
+0602ee4627b10f301298f2673d826cdd6baa9327 https://github.com/ijor/fx68k
+8b8ee086aef72e0833b7f0493d9d33f1e4d3c8e2 https://github.com/lowRISC/ibex
+a19e629a1879801ffcc6f2e6256ca435c20570f3 https://github.com/steveicarus/ivtest
+8e83643c22a3ba7612a9bb9cec93292dad618ab5 https://github.com/trivialmips/nontrivial-mips
+00981352642fb65ef2b6c47143a0f6bb31fd4a97 https://github.com/lowRISC/opentitan
+491a4dc9bb25c33e0183caaf683102b9ce273bc1 https://github.com/gtaylormb/opl3_fpga
+7b65f6ba0bce58d4d859082660123b7100aae975 https://github.com/rsd-devel/rsd
+ebb5e3551a9d93c0ee95f0b767dd878b8927e702 https://github.com/syntacore/scr1
+f200eb2ed7b69ac1c6b8eddd47654522aeee5ce8 https://github.com/olofk/serv
+c4229f3bd5220e6d3ba8f390e5d09c87e462e9c7 https://github.com/chipsalliance/sv-tests
+7e9841ff775ea22fba3f222f7bdfe35668345e00 https://github.com/taichi-ishitani/tnoc
+5d72b6618acfddece7f09382a032ccbc05862fdc https://github.com/SymbiFlow/uvm
+1c8e05fd1e9a79ceb8b996a0996674122eed086f https://github.com/SymbiFlow/XilinxUnisimLibrary
diff --git a/.github/bin/smoke-test.sh b/.github/bin/smoke-test.sh
index 4ce05d2..6e4c080 100755
--- a/.github/bin/smoke-test.sh
+++ b/.github/bin/smoke-test.sh
@@ -39,6 +39,7 @@
TMPDIR="${TMPDIR:-/tmp}"
readonly BASE_TEST_DIR="${TMPDIR}/test/verible-smoke-test"
+readonly DEFAULT_HASH_FILE="$(dirname $0)/smoke-projects.hashes"
# Write log files to this directory
readonly SMOKE_LOGGING_DIR="${SMOKE_LOGGING_DIR:-$BASE_TEST_DIR/error-logs}"
@@ -81,29 +82,12 @@
#
# There are some known issues which are all recorded in the associative
# array below, mapping them to Verible issue tracker numbers.
-# TODO(hzeller): there should be a configuration file that contains two
-# columns: URL + hash, so that we can fetch a particular known version not
-# a moving target.
-readonly TEST_GIT_PROJECTS="https://github.com/lowRISC/ibex \
- https://github.com/lowRISC/opentitan \
- https://github.com/chipsalliance/sv-tests \
- https://github.com/chipsalliance/Cores-VeeR-EH2 \
- https://github.com/chipsalliance/caliptra-rtl \
- https://github.com/openhwgroup/cva6 \
- https://github.com/SymbiFlow/uvm \
- https://github.com/taichi-ishitani/tnoc \
- https://github.com/ijor/fx68k \
- https://github.com/jamieiles/80x86 \
- https://github.com/SymbiFlow/XilinxUnisimLibrary \
- https://github.com/black-parrot/black-parrot
- https://github.com/steveicarus/ivtest \
- https://github.com/trivialmips/nontrivial-mips \
- https://github.com/pulp-platform/axi \
- https://github.com/rsd-devel/rsd \
- https://github.com/syntacore/scr1 \
- https://github.com/olofk/serv \
- https://github.com/bespoke-silicon-group/basejump_stl \
- https://github.com/gtaylormb/opl3_fpga"
+readonly PROJECT_HASHES_FILE="${1:-${DEFAULT_HASH_FILE}}"
+
+if [ ! -f "${PROJECT_HASHES_FILE}" ]; then
+ echo "Project hashes file not found: ${PROJECT_HASHES_FILE}"
+ exit 1
+fi
##
# Some of the files in the projects will have issues.
@@ -237,11 +221,10 @@
expected_count_key="${TOOL_SHORT_NAME}:${PROJECT_NAME}"
if [[ -v ExpectedFailCount[${expected_count_key}] ]]; then
expected_count=${ExpectedFailCount[${expected_count_key}]}
- # We allow some 5% deviation from expected values before we complain loudly
- local GRACE_VALUE=$(( ${expected_count} / 20 ))
- if [ ${GRACE_VALUE} -lt 2 ]; then
- GRACE_VALUE=2
- fi
+
+ # Since we have a fixed set of projects and hashes, we don't allow any
+ # derivation from the expected value (used to be 5%)
+ local GRACE_VALUE=0
if [ ${OBSERVED_NONZERO_COUNT} -gt ${expected_count} ] ; then
local ALLOWED_UP_TO=$((${expected_count} + ${GRACE_VALUE}))
if [ ${OBSERVED_NONZERO_COUNT} -gt ${ALLOWED_UP_TO} ]; then
@@ -254,6 +237,7 @@
elif [ ${OBSERVED_NONZERO_COUNT} -lt ${expected_count} ] ; then
echo "::notice:: 🎉 Yay, reduced non-zero exit count ${expected_count} -> ${OBSERVED_NONZERO_COUNT}"
echo "Set ExpectedFailCount[${TOOL_SHORT_NAME}:${PROJECT_NAME}]=${OBSERVED_NONZERO_COUNT}"
+ return 1 # we still want the exact value to be recorded so actively fail
fi
else
if [ ${OBSERVED_NONZERO_COUNT} -gt 0 ] ; then
@@ -270,11 +254,14 @@
#
# First parameter : project name
# Second parameter: name of file containing a list of {System}Verilog files
+# Third parameter : git URL
+# Fourth parameter: git hash (optional, defaults to master)
function run_smoke_test() {
local PROJECT_FILE_LIST=${TMPDIR}/filelist.$$.list
local PROJECT_NAME=$1
local FILELIST=$2
local GIT_URL=$3
+ local GIT_HASH=${4:-master}
local NUM_FILES=$(wc -l < ${FILELIST})
local result=0
@@ -355,7 +342,7 @@
else
# This is an so far unknown issue
echo "::error:: 😱 ${single_file}: crash exit code $EXIT_CODE for $tool"
- echo "Input File URL: ${GIT_URL}/blob/master/$(echo $single_file | cut -d/ -f6-)"
+ echo "Input File URL: ${GIT_URL}/blob/${GIT_HASH}/$(echo $single_file | cut -d/ -f6-)"
head -15 ${PROJECT_FILE_TOOL_OUT} # Might be useful in this case
result=$((${result} + 1))
fi
@@ -388,18 +375,20 @@
bazel build ${BAZEL_BUILD_OPTIONS} :install-binaries &
# While compiling, run potentially slow network ops
-for git_project in ${TEST_GIT_PROJECTS} ; do
- PROJECT_NAME="$(basename $git_project)"
+while read -r git_hash git_project _; do
+ [[ -z "${git_hash}" || "${git_hash}" =~ ^# ]] && continue
+ PROJECT_NAME="$(basename "${git_project}")"
PROJECT_DIR="${BASE_TEST_DIR}/${PROJECT_NAME}"
- git clone ${git_project} ${PROJECT_DIR} 2>/dev/null &
-done
+ ( git clone "${git_project}" "${PROJECT_DIR}" && git -C "${PROJECT_DIR}" checkout -q "${git_hash}" ) 2>/dev/null &
+done < "${PROJECT_HASHES_FILE}"
echo "base test dir ${BASE_TEST_DIR}; writing logs to ${SMOKE_LOGGING_DIR}"
echo "Waiting... for compilation and project download finished"
wait
-for git_project in ${TEST_GIT_PROJECTS} ; do
- PROJECT_NAME="$(basename $git_project)"
+while read -r git_hash git_project _; do
+ [[ -z "${git_hash}" || "${git_hash}" =~ ^# ]] && continue
+ PROJECT_NAME="$(basename "${git_project}")"
PROJECT_DIR="${BASE_TEST_DIR}/${PROJECT_NAME}"
# Already cloned above
@@ -412,14 +401,14 @@
FILELIST="${PROJECT_DIR}/verible.filelist"
find "${PROJECT_DIR}" -name "*.sv" -o -name "*.svh" -o -name "*.v" | sort > ${FILELIST}
- run_smoke_test "${PROJECT_NAME}" "${FILELIST}" "${git_project}"
+ run_smoke_test "${PROJECT_NAME}" "${FILELIST}" "${git_project}" "${git_hash}"
status_sum=$((${status_sum} + $?))
echo
-done
+done < "${PROJECT_HASHES_FILE}"
echo "::endgroup::"
-echo "There were a total of ${status_sum} new, undocumented issues."
+echo "There were a total of ${status_sum} mismatches"
# Let's see if there are any issues that are fixed in the meantime.
if [ "${#KnownIssue[@]}" -ne 0 ]; then
