.github/bin/smoke-test.sh

#!/usr/bin/env bash
# -*- mode: sh; sh-basic-offset: 2; indent-tabs-mode: nil; -*-
# Copyright 2021 The Verible Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

###
# Simple smoke test by running all Verible tools over various code-bases
# that can be fetched by git and making sure it does not crash, letting
# out the magic smoke.
#
# We explicitly do _not_ look if the tools fail with some other exit
# code, as we don't care if e.g. there are lint issues in the projects
# at hand, just that Verible is not crashing on input.
#
# Once we are able to parse full of system verilog, we should check
# that verible-verilog-syntax always return 0.
# For now, even  if there are syntax errors reported: as long as it doesn't
# crash Verible, we're good.
###

set -u   # Be strict: only allow using a variable after it is assigned

BAZEL_BUILD_OPTIONS="-c opt"

TMPDIR="${TMPDIR:-/tmp}"
readonly BASE_TEST_DIR="${TMPDIR}/test/verible-smoke-test"
readonly PROJECT_HASHES_FILE="$(dirname $0)/smoke-projects.hashes"

# Write log files to this directory
readonly SMOKE_LOGGING_DIR="${SMOKE_LOGGING_DIR:-$BASE_TEST_DIR/error-logs}"

# Make the directory for the logs
if [ ! -d "${SMOKE_LOGGING_DIR}" ]; then
  mkdir -p "${SMOKE_LOGGING_DIR}"
fi

# Some terminal codes to highlight
readonly TERM_RED=$'\033[1;31m'
readonly TERM_BOLD=$'\033[1m'
readonly TERM_RESET=$'\033[0m'

readonly BINARY_BASE_DIR=bazel-bin/verible/verilog/tools

readonly ISSUE_PREFIX="https://github.com/chipsalliance/verible/issues"

# In case the binaries are run with ASAN:
# By default, failing asan binaries exit with exit code 1.
# Let's change it to something that we can distinguish from 'normal operation'
export ASAN_OPTIONS="exitcode=140"

readonly VERIBLE_TOOLS_TO_RUN="syntax/verible-verilog-syntax \
                               lint/verible-verilog-lint \
                               formatter/verible-verilog-format \
                               project/verible-verilog-project \
                               preprocessor/verible-verilog-preprocessor \
                               kythe/verible-verilog-kythe-extractor"

# A few projects that can be fetched from git and represent a good
# cross-section of different styles.
# Always in search for more Open Source github Verilog and SystemVerilog
# projects to add here.
#
# We run all Verible tools on these.
#
# Not included: https://github.com/google/skywater-pdk (the recursive
# checkout is gigantic.)
#
# There are some known issues which are all recorded in the associative
# array below, mapping them to Verible issue tracker numbers.

if [ ! -f "${PROJECT_HASHES_FILE}" ]; then
  echo "Project hashes file not found: ${PROJECT_HASHES_FILE}"
  exit 1
fi

##
# Some of the files in the projects will have issues.
# We record each of them with the associated bug number in this set so that
# we can locally waive the issue and continue with more files,
# but also have it documented which are affected.
#
# Any new issues that arise should be recorded in the verible bug-tracker to
# be fixed.
# Goal: The following list shall be empty :)
# Format: <tool-name>:<basename of file-name in project>
declare -A KnownIssue

# At the moment, the list is empty

#--- Too many to mention manually, so here we do the 'waive all' approach
# Format: <tool-name>:<project>
declare -A KnownProjectToolIssue
KnownProjectToolIssue[project:caliptra-rtl]=1946

##
# Some tools still not fully process all files and return a non-zero exit
# count.
# This is to record the expected non-zero exits, so that we can report if
# things improve or regress.
# TODO: get these numbers to zero. If zero, the corresponding line can be
# removed.
declare -A ExpectedFailCount

ExpectedFailCount[syntax:ibex]=13
ExpectedFailCount[lint:ibex]=13
ExpectedFailCount[project:ibex]=224
ExpectedFailCount[preprocessor:ibex]=403

ExpectedFailCount[syntax:opentitan]=88
ExpectedFailCount[lint:opentitan]=88
ExpectedFailCount[project:opentitan]=1077
ExpectedFailCount[formatter:opentitan]=0
ExpectedFailCount[preprocessor:opentitan]=3017

ExpectedFailCount[syntax:sv-tests]=74
ExpectedFailCount[lint:sv-tests]=73
ExpectedFailCount[project:sv-tests]=176
ExpectedFailCount[preprocessor:sv-tests]=129

ExpectedFailCount[syntax:caliptra-rtl]=41
ExpectedFailCount[lint:caliptra-rtl]=40
ExpectedFailCount[project:caliptra-rtl]=474
ExpectedFailCount[preprocessor:caliptra-rtl]=993

ExpectedFailCount[syntax:Cores-VeeR-EH2]=2
ExpectedFailCount[lint:Cores-VeeR-EH2]=2
ExpectedFailCount[project:Cores-VeeR-EH2]=42
ExpectedFailCount[preprocessor:Cores-VeeR-EH2]=43

ExpectedFailCount[syntax:cva6]=7
ExpectedFailCount[lint:cva6]=7
ExpectedFailCount[project:cva6]=84
ExpectedFailCount[preprocessor:cva6]=140

ExpectedFailCount[syntax:uvm]=1
ExpectedFailCount[lint:uvm]=1
ExpectedFailCount[project:uvm]=41
ExpectedFailCount[preprocessor:uvm]=128

ExpectedFailCount[syntax:tnoc]=3
ExpectedFailCount[lint:tnoc]=3
ExpectedFailCount[project:tnoc]=21
ExpectedFailCount[preprocessor:tnoc]=57

ExpectedFailCount[project:80x86]=2
ExpectedFailCount[preprocessor:80x86]=7

ExpectedFailCount[syntax:XilinxUnisimLibrary]=4
ExpectedFailCount[lint:XilinxUnisimLibrary]=4
ExpectedFailCount[project:XilinxUnisimLibrary]=22
ExpectedFailCount[preprocessor:XilinxUnisimLibrary]=96

ExpectedFailCount[syntax:black-parrot]=155
ExpectedFailCount[lint:black-parrot]=155
ExpectedFailCount[project:black-parrot]=172
ExpectedFailCount[preprocessor:black-parrot]=173

ExpectedFailCount[syntax:ivtest]=116
ExpectedFailCount[lint:ivtest]=116
ExpectedFailCount[project:ivtest]=145
ExpectedFailCount[preprocessor:ivtest]=26

ExpectedFailCount[syntax:nontrivial-mips]=2
ExpectedFailCount[lint:nontrivial-mips]=2
ExpectedFailCount[project:nontrivial-mips]=79
ExpectedFailCount[preprocessor:nontrivial-mips]=78

ExpectedFailCount[project:axi]=79
ExpectedFailCount[preprocessor:axi]=79

ExpectedFailCount[syntax:rsd]=5
ExpectedFailCount[lint:rsd]=5
ExpectedFailCount[project:rsd]=52
ExpectedFailCount[preprocessor:rsd]=49

ExpectedFailCount[project:scr1]=44
ExpectedFailCount[preprocessor:scr1]=46

ExpectedFailCount[project:serv]=1
ExpectedFailCount[preprocessor:serv]=1

ExpectedFailCount[syntax:basejump_stl]=498
ExpectedFailCount[lint:basejump_stl]=498
ExpectedFailCount[project:basejump_stl]=619
ExpectedFailCount[formatter:basejump_stl]=1
ExpectedFailCount[preprocessor:basejump_stl]=659

ExpectedFailCount[syntax:opl3_fpga]=2
ExpectedFailCount[lint:opl3_fpga]=2
ExpectedFailCount[project:opl3_fpga]=4
ExpectedFailCount[preprocessor:opl3_fpga]=4

# Ideally, we expect all tools to process all files with a zero exit code.
# However, that is not always the case, so we document the current
# state, so that we can see regressions.
# Currently, either way will not change the output.
function verify_expected_non_zero_exit_count() {
  local TOOL_SHORT_NAME=$1
  local PROJECT_NAME=$2
  local FILE_COUNT=$3
  local OBSERVED_NONZERO_COUNT=$4

  echo "  -> Non zero exit code ${OBSERVED_NONZERO_COUNT}/${NUM_FILES}"

  expected_count_key="${TOOL_SHORT_NAME}:${PROJECT_NAME}"
  if [[ -v ExpectedFailCount[${expected_count_key}] ]]; then
    expected_count=${ExpectedFailCount[${expected_count_key}]}

    # Since we have a fixed set of projects and hashes, we don't allow any
    # derivation from the expected value (used to be 5%)
    local GRACE_VALUE=0
    if [ ${OBSERVED_NONZERO_COUNT} -gt ${expected_count} ] ; then
      local ALLOWED_UP_TO=$((${expected_count} + ${GRACE_VALUE}))
      if [ ${OBSERVED_NONZERO_COUNT} -gt ${ALLOWED_UP_TO} ]; then
        echo "::error:: 😱 ${expected_count_key} Expected failures ${expected_count}, got ${OBSERVED_NONZERO_COUNT}"
        return 1
      else
        echo "::warning:: 😱 ${expected_count_key} Expected failures ${expected_count}, got ${OBSERVED_NONZERO_COUNT} (will be an error once ${GRACE_VALUE} more)"
        return 0  # Still within grace range.
      fi
    elif [ ${OBSERVED_NONZERO_COUNT} -lt ${expected_count} ] ; then
      echo "::notice:: 🎉 Yay, reduced non-zero exit count ${expected_count} -> ${OBSERVED_NONZERO_COUNT}"
      echo "Set ExpectedFailCount[${TOOL_SHORT_NAME}:${PROJECT_NAME}]=${OBSERVED_NONZERO_COUNT}"
      return 1  # we still want the exact value to be recorded so actively fail
    fi
  else
    if [ ${OBSERVED_NONZERO_COUNT} -gt 0 ] ; then
      echo "::error:: ********************* Not mentioned in ExpectedFailCount *******"
      echo "Add  ExpectedFailCount[${TOOL_SHORT_NAME}:${PROJECT_NAME}]=${OBSERVED_NONZERO_COUNT}"
      return 1
    fi
  fi
  return 0
}

# Run smoke test on provided files for project.
# Returns 0 if all tools finished without crashing.
#
# First parameter : project name
# Second parameter: name of file containing a list of {System}Verilog files
# Third parameter : git URL
# Fourth parameter: git hash (optional, defaults to master)
function run_smoke_test() {
  local PROJECT_FILE_LIST=${TMPDIR}/filelist.$$.list
  local PROJECT_NAME=$1
  local FILELIST=$2
  local GIT_URL=$3
  local GIT_HASH=${4:-master}
  local NUM_FILES=$(wc -l < ${FILELIST})
  local result=0

  local PROJECT_LOG_BASE="${SMOKE_LOGGING_DIR}/${PROJECT_NAME}"
  mkdir -p ${PROJECT_LOG_BASE}

  echo "::group::== Running verible on ${TERM_BOLD}${PROJECT_NAME}${TERM_RESET} with ${NUM_FILES} files =="

  for tool in $VERIBLE_TOOLS_TO_RUN ; do
    printf "%-20s %-32s\n" ${PROJECT_NAME} ${tool}
    local short_tool_name=$(dirname ${tool})
    local non_zero_exit_code=0
    local PROJECT_TOOL_LOG_BASE="${PROJECT_LOG_BASE}/${short_tool_name}"
    mkdir -p "${PROJECT_TOOL_LOG_BASE}"

    while read single_file; do
      # TODO(hzeller) the project tool and kythe extractor are meant to run
      # on a bunch of files not individual files. Create a complete file-list
      if [[ $tool == *-project ]]; then
        EXTRA_PARAM="symbol-table-defs --file_list_root=/ --file_list_path"
        # a <(echo $single_file) does not work, so use actual file.
        echo ${single_file} > ${PROJECT_FILE_LIST}
        file_param="${PROJECT_FILE_LIST}"
      elif [[ $tool == *-extractor ]]; then
        EXTRA_PARAM="--file_list_root=/ --file_list_path"
        # a <(echo $single_file) does not work, so use actual file.
        echo ${single_file} > ${PROJECT_FILE_LIST}
        file_param="${PROJECT_FILE_LIST}"
      elif [[ $tool == *-lint ]]; then
        EXTRA_PARAM="--lint_fatal=false"
	      file_param=${single_file}
      elif [[ $tool == *-preprocessor ]]; then
        EXTRA_PARAM="preprocess"
	      file_param=${single_file}
      else
        EXTRA_PARAM=""
        file_param=${single_file}
      fi
      local FNAME=$(basename ${file_param})  # make hash ?
      local PROJECT_FILE_TOOL_OUT="${PROJECT_TOOL_LOG_BASE}/${FNAME}"

      ${BINARY_BASE_DIR}/${tool} ${EXTRA_PARAM} ${file_param} > ${PROJECT_FILE_TOOL_OUT} 2>&1
      local EXIT_CODE=$?

      if [ $EXIT_CODE -eq 0 ]; then
        rm -f "${PROJECT_FILE_TOOL_OUT}"   # not interested keeping logfiles of successful runs
      else
        # Even though we don't fail globally, let's at least count how many times
        # our tools exit with non-zero. Long term, we'd like to have them succeed
        # on all files
        non_zero_exit_code=$[non_zero_exit_code + 1]

        # The error-log-analyzer.py requires the files in a particular format
        local ANALYZE_DIR="${SMOKE_LOGGING_DIR}/${PROJECT_NAME}-nonzeros"
        mkdir -p "${ANALYZE_DIR}"
        cp "${PROJECT_FILE_TOOL_OUT}" "${ANALYZE_DIR}/${EXIT_CODE}-${FNAME}_${short_tool_name}"
      fi

      # A regular error exit code we accept as normal operation of the tool if
      # it encountered a syntax error. Here, we are only interested in not
      # receiving a signal such as an abort or segmentation fault.
      # So we check for 126, 127 (command not found or executable)
      # and >= 128 (128 + signal-number)
      # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
      if [ $EXIT_CODE -ge 126 ]; then
        if [[ $tool == *-project ]]; then
          file_param="<(echo $single_file)"   # make easy to reproduce
        fi
        echo "${TERM_RED}${BINARY_BASE_DIR}/${tool} ${EXTRA_PARAM} ${file_param} ${TERM_RESET}"
        waive_file_key="${short_tool_name}:$(basename ${single_file})"
        waive_project_key="${short_tool_name}:${PROJECT_NAME}"
        if [[ -v KnownIssue[${waive_file_key}] ]]; then
          bug_number=${KnownIssue[${waive_file_key}]}
          echo "  --> Known issue: 🐞 ${ISSUE_PREFIX}/${bug_number}"
          unset KnownIssue[${waive_file_key}]
        elif [[ -v KnownProjectToolIssue[${waive_project_key}] ]]; then
          echo "  --> Known issue 🐞🐞 possibly one of ${ISSUE_PREFIX}/${KnownProjectToolIssue[${waive_project_key}]}"
        else
          # This is an so far unknown issue
          echo "::error:: 😱 ${single_file}: crash exit code $EXIT_CODE for $tool"
          echo "Input File URL: ${GIT_URL}/blob/${GIT_HASH}/$(echo $single_file | cut -d/ -f6-)"
          head -15 ${PROJECT_FILE_TOOL_OUT}   # Might be useful in this case
          result=$((${result} + 1))
        fi
      fi
    done < ${FILELIST}

    # Let's see if number of non-zero exit codes match what we expect
    verify_expected_non_zero_exit_count ${short_tool_name} ${PROJECT_NAME} ${NUM_FILES} ${non_zero_exit_code}
    result=$((${result} + $?))

  done  # for tool

  rm -f ${PROJECT_FILE_LIST}
  return ${result}
}

# --- main

KEEP_LOGS=0
VERBOSE=0
PROJECT_FILTER=""

while [[ $# -gt 0 ]]; do
  case "$1" in
    --keep-logs)
      KEEP_LOGS=1
      shift
      ;;
    --verbose|-v)
      VERBOSE=1
      shift
      ;;
    --filter=*)
      PROJECT_FILTER="${1#*=}"
      shift
      ;;
    --filter|-f)
      if [[ $# -lt 2 ]]; then
        echo "Error: $1 requires an argument." >&2
        exit 1
      fi
      PROJECT_FILTER="$2"
      shift 2
      ;;
    -h|--help)
      echo "Usage: $0 [--keep-logs] [--verbose] [--filter=<name>]"
      exit 0
      ;;
    *)
      echo "Unknown option: $1" >&2
      echo "Usage: $0 [--keep-logs] [--verbose] [--filter=<name>]" >&2
      exit 1
      ;;
  esac
done

if [ ${VERBOSE} -eq 0 ]; then
  # Suppress '... aborted' messages bash would print when a tool crashes.
  exec 2>/dev/null
fi

mkdir -p "${BASE_TEST_DIR}"
if [ ${KEEP_LOGS} -eq 0 ]; then
  trap 'rm -rf -- "${BASE_TEST_DIR}"' EXIT
fi

status_sum=0

#-- Prepare stuff in parallel
#  - build the binaries
#  - git fetch the projects.

# Build all the installable binaries that we're going to use below.
# TODO: Consider running with
#  * address sanitizer (best result with libc++). CAVE: slow
#  * running with -D_GLIBCXX_DEBUG (also see #1056)
bazel build ${BAZEL_BUILD_OPTIONS} :install-binaries &

# While compiling, run potentially slow network ops
while read -r git_hash git_project _; do
  [[ -z "${git_hash}" || "${git_hash}" =~ ^# ]] && continue
  PROJECT_NAME="$(basename "${git_project}")"
  if [[ -n "${PROJECT_FILTER}" && "${PROJECT_NAME}" != *"${PROJECT_FILTER}"* ]]; then
    continue
  fi
  PROJECT_DIR="${BASE_TEST_DIR}/${PROJECT_NAME}"
  ( git clone "${git_project}" "${PROJECT_DIR}" && git -C "${PROJECT_DIR}" checkout -q "${git_hash}" ) 2>/dev/null &
done < "${PROJECT_HASHES_FILE}"

echo "base test dir ${BASE_TEST_DIR}; writing logs to ${SMOKE_LOGGING_DIR}"
echo "Waiting... for compilation and project download finished"
wait

while read -r git_hash git_project _; do
  [[ -z "${git_hash}" || "${git_hash}" =~ ^# ]] && continue
  PROJECT_NAME="$(basename "${git_project}")"
  if [[ -n "${PROJECT_FILTER}" && "${PROJECT_NAME}" != *"${PROJECT_FILTER}"* ]]; then
    continue
  fi
  PROJECT_DIR="${BASE_TEST_DIR}/${PROJECT_NAME}"
  # Already cloned above

  if [ ! -d "${PROJECT_DIR}" ]; then
    echo "Didn't see ${PROJECT_DIR}. Network connection flaky ? Skipping."
    continue
  fi

  # Just collect everything that looks like Verilog/SystemVerilog
  FILELIST="${PROJECT_DIR}/verible.filelist"
  find "${PROJECT_DIR}" -name "*.sv" -o -name "*.svh" -o -name "*.v" | sort > ${FILELIST}

  run_smoke_test "${PROJECT_NAME}" "${FILELIST}" "${git_project}" "${git_hash}"
  status_sum=$((${status_sum} + $?))
  echo
done < "${PROJECT_HASHES_FILE}"

echo "::endgroup::"

echo "There were a total of ${status_sum} mismatches"

# Let's see if there are any issues that are fixed in the meantime.
if [ -z "${PROJECT_FILTER}" ] && [ "${#KnownIssue[@]}" -ne 0 ]; then
  echo "::warning ::There are ${#KnownIssue[@]} tool/file combinations, that no longer fail"
  declare -A DistinctIssues
  for key in "${!KnownIssue[@]}"; do
    echo " 🎉 ${key}"
    DistinctIssues[${KnownIssue[${key}]}]=1
  done
  echo
  echo "::notice ::These were referencing the following issues. Maybe they are fixed now ?"
  for issue_id in "${!DistinctIssues[@]}"; do
    echo " 🐞 ${ISSUE_PREFIX}/${issue_id}"
  done
fi

if [ ${KEEP_LOGS} -ne 0 ]; then
  echo "Logs and project files kept in ${BASE_TEST_DIR} (error logs: ${SMOKE_LOGGING_DIR})"
fi

exit ${status_sum}