#!/usr/bin/env bash
# -*- mode: sh; sh-basic-offset: 2; indent-tabs-mode: nil; -*-
# Copyright 2021 The Verible Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

###
# Simple smoke test by running all Verible tools over various code-bases
# that can be fetched by git and making sure it does not crash, letting
# out the magic smoke.
#
# We explicitly do _not_ look if the tools fail with some other exit
# code, as we don't care if e.g. there are lint issues in the projects
# at hand, just that Verible is not crashing on input.
#
# Once we are able to parse full of system verilog, we should check
# that verible-verilog-syntax always return 0.
# For now, even  if there are syntax errors reported: as long as it doesn't
# crash Verible, we're good.
###

# Suppress '... aborted' messages bash would print when a tool crashes.
# Comment out to see syntax errors in bash while working on script.
exec 2>/dev/null

set -u   # Be strict: only allow using a variable after it is assigned

BAZEL_BUILD_OPTIONS="-c opt"

TMPDIR="${TMPDIR:-/tmp}"
readonly BASE_TEST_DIR=${TMPDIR}/test/verible-smoke-test

# Some terminal codes to highlight
readonly TERM_RED=$'\033[1;31m'
readonly TERM_BOLD=$'\033[1m'
readonly TERM_RESET=$'\033[0m'

readonly BINARY_BASE_DIR=bazel-bin/verilog/tools

# In case the binaries are run with ASAN:
# By default, failing asan binaries exit with exit code 1.
# Let's change it to something that we can distinguish from 'normal operation'
export ASAN_OPTIONS="exitcode=140"

readonly VERIBLE_TOOLS_TO_RUN="syntax/verible-verilog-syntax \
                               lint/verible-verilog-lint \
                               formatter/verible-verilog-format \
                               project/verible-verilog-project \
                               kythe/verible-verilog-kythe-extractor"

# A few projects that can be fetched from git and represent a good
# cross-section of different styles.
# Always in search for more Open Source github Verilog and SystemVerilog
# projects to add here.
#
# We run all Verible tools on these.
#
# Not included: https://github.com/google/skywater-pdk (the recursive
# checkout is gigantic.)
#
# There are some known issues which are all recorded in the associative
# array below, mapping them to Verible issue tracker numbers.
readonly TEST_GIT_PROJECTS="https://github.com/lowRISC/ibex \
         https://github.com/lowRISC/opentitan \
         https://github.com/chipsalliance/Cores-VeeR-EH2 \
         https://github.com/openhwgroup/cva6 \
         https://github.com/SymbiFlow/uvm \
         https://github.com/taichi-ishitani/tnoc \
         https://github.com/ijor/fx68k \
         https://github.com/jamieiles/80x86 \
         https://github.com/SymbiFlow/XilinxUnisimLibrary \
         https://github.com/black-parrot/black-parrot
         https://github.com/steveicarus/ivtest \
         https://github.com/krevanth/ZAP \
         https://github.com/trivialmips/nontrivial-mips \
         https://github.com/pulp-platform/axi \
         https://github.com/rsd-devel/rsd \
         https://github.com/syntacore/scr1 \
         https://github.com/bespoke-silicon-group/basejump_stl"

##
# Some of the files in the projects will have issues.
# We record each of them with the associated bug number in this set so that
# we can locally waive the issue and continue with more files,
# but also have it documented which are affected.
#
# Any new issues that arise should be recorded in the verible bug-tracker to
# be fixed.
# Goal: The following list shall be empty :)
declare -A KnownIssue

#--- Opentitan
# Original issue - 1006 has been resolved
# There is also bug 1008 which only shows up if compiled with asan

#--- ivtest
KnownIssue[formatter:$BASE_TEST_DIR/ivtest/ivltests/pr2202846c.v]=1015

#--- nontrivial-mips
KnownIssue[formatter:$BASE_TEST_DIR/nontrivial-mips/src/cpu/decode/decoder.sv]=984
KnownIssue[formatter:$BASE_TEST_DIR/nontrivial-mips/src/cpu/exec/multi_cycle_exec.sv]=984

#--- rsd
KnownIssue[formatter:$BASE_TEST_DIR/rsd/Processor/Src/Privileged/CSR_Unit.sv]=984

#--- scr1
KnownIssue[formatter:$BASE_TEST_DIR/scr1/src/core/scr1_tapc.sv]=1015
KnownIssue[formatter:$BASE_TEST_DIR/scr1/src/tb/scr1_memory_tb_ahb.sv]=1015

#--- Too many to mention manually, so here we do the 'waive all' approach
declare -A KnownProjectToolIssue

##
# Some tools still not fully process all files and return a non-zero exit
# count.
# This is to record the expected non-zero exits, so that we can report if
# things improve or regress.
# TODO: get these numbers to zero. If zero, the corresponding line can be
# removed.
declare -A ExpectedFailCount

ExpectedFailCount[syntax:ibex]=14
ExpectedFailCount[lint:ibex]=14
ExpectedFailCount[project:ibex]=192

ExpectedFailCount[syntax:opentitan]=35
ExpectedFailCount[lint:opentitan]=35
ExpectedFailCount[project:opentitan]=727

ExpectedFailCount[syntax:Cores-VeeR-EH2]=2
ExpectedFailCount[lint:Cores-VeeR-EH2]=2
ExpectedFailCount[project:Cores-VeeR-EH2]=42

ExpectedFailCount[syntax:cva6]=4
ExpectedFailCount[lint:cva6]=4
ExpectedFailCount[project:cva6]=70

ExpectedFailCount[syntax:uvm]=1
ExpectedFailCount[lint:uvm]=1
ExpectedFailCount[project:uvm]=43

ExpectedFailCount[syntax:tnoc]=3
ExpectedFailCount[lint:tnoc]=3
ExpectedFailCount[project:tnoc]=24

ExpectedFailCount[project:80x86]=2

ExpectedFailCount[syntax:XilinxUnisimLibrary]=9
ExpectedFailCount[lint:XilinxUnisimLibrary]=9
ExpectedFailCount[project:XilinxUnisimLibrary]=27

ExpectedFailCount[syntax:black-parrot]=165
ExpectedFailCount[lint:black-parrot]=165
ExpectedFailCount[project:black-parrot]=178

ExpectedFailCount[syntax:ivtest]=188
ExpectedFailCount[lint:ivtest]=188
ExpectedFailCount[formatter:ivtest]=2
ExpectedFailCount[project:ivtest]=217

ExpectedFailCount[project:ZAP]=29

ExpectedFailCount[syntax:nontrivial-mips]=2
ExpectedFailCount[lint:nontrivial-mips]=2
ExpectedFailCount[formatter:nontrivial-mips]=2
ExpectedFailCount[project:nontrivial-mips]=81

ExpectedFailCount[project:axi]=69

ExpectedFailCount[syntax:rsd]=1
ExpectedFailCount[lint:rsd]=1
ExpectedFailCount[formatter:rsd]=1
ExpectedFailCount[project:rsd]=43

ExpectedFailCount[formatter:scr1]=2
ExpectedFailCount[project:scr1]=45

ExpectedFailCount[syntax:basejump_stl]=462
ExpectedFailCount[lint:basejump_stl]=462
ExpectedFailCount[project:basejump_stl]=569

# Ideally, we expect all tools to process all files with a zero exit code.
# However, that is not always the case, so we document the current
# state, so that we can see regressions.
# Currently, either way will not change the output.
function verify_expected_non_zero_exit_count() {
  local TOOL_SHORT_NAME=$1
  local PROJECT_NAME=$2
  local FILE_COUNT=$3
  local OBSERVED_NONZERO_COUNT=$4

  echo "  -> Non zero exit code ${OBSERVED_NONZERO_COUNT}/${NUM_FILES}"

  expected_count_key="${TOOL_SHORT_NAME}:${PROJECT_NAME}"
  if [[ -v ExpectedFailCount[${expected_count_key}] ]]; then
    expected_count=${ExpectedFailCount[${expected_count_key}]}
    # We allow some 5% deviation from expected values before we complain loudly
    local GRACE_VALUE=$(( ${expected_count} / 20 ))
    if [ ${GRACE_VALUE} -lt 2 ]; then
      GRACE_VALUE=2
    fi
    if [ ${OBSERVED_NONZERO_COUNT} -gt ${expected_count} ] ; then
      local ALLOWED_UP_TO=$((${expected_count} + ${GRACE_VALUE}))
      if [ ${OBSERVED_NONZERO_COUNT} -gt ${ALLOWED_UP_TO} ]; then
        echo "::error:: 😱 ${expected_count_key} Expected failures ${expected_count}, got ${OBSERVED_NONZERO_COUNT}"
        return 1
      else
        echo "::warning:: 😱 ${expected_count_key} Expected failures ${expected_count}, got ${OBSERVED_NONZERO_COUNT} (will be an error once ${GRACE_VALUE} more)"
        return 0  # Still within grace range.
      fi
    elif [ ${OBSERVED_NONZERO_COUNT} -lt ${expected_count} ] ; then
      echo "::notice:: 🎉 Yay, reduced non-zero exit count ${expected_count} -> ${OBSERVED_NONZERO_COUNT}"
      echo "Set ExpectedFailCount[${TOOL_SHORT_NAME}:${PROJECT_NAME}]=${OBSERVED_NONZERO_COUNT}"
    fi
  else
    if [ ${OBSERVED_NONZERO_COUNT} -gt 0 ] ; then
      echo "::error:: ********************* Not mentioned in ExpectedFailCount *******"
      echo "Add  ExpectedFailCount[${TOOL_SHORT_NAME}:${PROJECT_NAME}]=${OBSERVED_NONZERO_COUNT}"
      return 1
    fi
  fi
  return 0
}

# Run smoke test on provided files for project.
# Returns 0 if all tools finished without crashing.
#
# First parameter : project name
# Second parameter: name of file containing a list of {System}Verilog files
function run_smoke_test() {
  local PROJECT_FILE_LIST=${TMPDIR}/filelist.$$.list
  local TOOL_OUT=${TMPDIR}/tool.$$.out
  local PROJECT_NAME=$1
  local FILELIST=$2
  local GIT_URL=$3
  local NUM_FILES=$(wc -l < ${FILELIST})
  local result=0

  echo "::group::== Running verible on ${TERM_BOLD}${PROJECT_NAME}${TERM_RESET} with ${NUM_FILES} files =="

  for tool in $VERIBLE_TOOLS_TO_RUN ; do
    printf "%-20s %-32s\n" ${PROJECT_NAME} ${tool}
    local short_tool_name=$(dirname ${tool})
    local non_zero_exit_code=0

    while read single_file; do
      # TODO(hzeller) the project tool and kythe extractor are meant to run
      # on a bunch of files not individual files. Create a complete file-list
      if [[ $tool == *-project ]]; then
        EXTRA_PARAM="symbol-table-defs --file_list_root=/ --file_list_path"
        # a <(echo $single_file) does not work, so use actual file.
        echo ${single_file} > ${PROJECT_FILE_LIST}
        file_param="${PROJECT_FILE_LIST}"
       elif [[ $tool == *-extractor ]]; then
        EXTRA_PARAM="--file_list_root=/ --file_list_path"
        # a <(echo $single_file) does not work, so use actual file.
        echo ${single_file} > ${PROJECT_FILE_LIST}
        file_param="${PROJECT_FILE_LIST}"
       elif [[ $tool == *-lint ]]; then
        EXTRA_PARAM="--lint_fatal=false"
	file_param=${single_file}
       else
        EXTRA_PARAM=""
        file_param=${single_file}
      fi

      ${BINARY_BASE_DIR}/${tool} ${EXTRA_PARAM} ${file_param} > ${TOOL_OUT} 2>&1
      local EXIT_CODE=$?

      # Even though we don't fail globally, let's at least count how many times
      # our tools exit with non-zero. Long term, we'd like to have them succeed
      # on all files
      if [ $EXIT_CODE -ne 0 ]; then
        non_zero_exit_code=$[non_zero_exit_code + 1]
      fi

      # A regular error exit code we accept as normal operation of the tool if
      # it encountered a syntax error. Here, we are only interested in not
      # receiving a signal such as an abort or segmentation fault.
      # So we check for 126, 127 (command not found or executable)
      # and >= 128 (128 + signal-number)
      # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
      if [ $EXIT_CODE -ge 126 ]; then
        if [[ $tool == *-project ]]; then
          file_param="<(echo $single_file)"   # make easy to reproduce
        fi
        echo "${TERM_RED}${BINARY_BASE_DIR}/${tool} ${EXTRA_PARAM} ${file_param} ${TERM_RESET}"
        waive_file_key="${short_tool_name}:${single_file}"
        waive_project_key="${short_tool_name}:${PROJECT_NAME}"
        if [[ -v KnownIssue[${waive_file_key}] ]]; then
          bug_number=${KnownIssue[${waive_file_key}]}
          echo "  --> Known issue: 🐞 https://github.com/chipsalliance/verible/issues/$bug_number"
          unset KnownIssue[${waive_file_key}]
        elif [[ -v KnownProjectToolIssue[${waive_project_key}] ]]; then
          echo "  --> Known issue 🐞🐞 possibly one of ${KnownProjectToolIssue[${waive_project_key}]}"
        else
          # This is an so far unknown issue
          echo "::error:: 😱 ${single_file}: crash exit code $EXIT_CODE for $tool"
          echo "Input File URL: ${GIT_URL}/blob/master/$(echo $single_file | cut -d/ -f6-)"
          head -15 ${TOOL_OUT}   # Might be useful in this case
          result=$((${result} + 1))
        fi
      fi
    done < ${FILELIST}

    # Let's see if number of non-zero exit codes match what we expect
    verify_expected_non_zero_exit_count ${short_tool_name} ${PROJECT_NAME} ${NUM_FILES} ${non_zero_exit_code}
    result=$((${result} + $?))

  done  # for tool

  rm -f ${PROJECT_FILE_LIST} ${TOOL_OUT}
  return ${result}
}

mkdir -p $BASE_TEST_DIR
trap 'rm -rf -- "$BASE_TEST_DIR"' EXIT

status_sum=0

#-- Prepare stuff in parallel
#  - build the binaries
#  - git fetch the projects.

# Build all the installable binaries that we're going to use below.
# TODO: Consider running with
#  * address sanitizer (best result with libc++). CAVE: slow
#  * running with -D_GLIBCXX_DEBUG (also see #1056)
bazel build ${BAZEL_BUILD_OPTIONS} :install-binaries &

# While compiling, run potentially slow network ops
for git_project in ${TEST_GIT_PROJECTS} ; do
  PROJECT_NAME=$(basename $git_project)
  PROJECT_DIR=${BASE_TEST_DIR}/${PROJECT_NAME}
  git clone ${git_project} ${PROJECT_DIR} 2>/dev/null &
done

echo "Waiting... for compilation and project download finished"
wait

for git_project in ${TEST_GIT_PROJECTS} ; do
  PROJECT_NAME=$(basename $git_project)
  PROJECT_DIR=${BASE_TEST_DIR}/${PROJECT_NAME}
  # Already cloned above

  # Just collect everything that looks like Verilog/SystemVerilog
  FILELIST=${PROJECT_DIR}/verible.filelist
  find ${PROJECT_DIR} -name "*.sv" -o -name "*.svh" -o -name "*.v" | sort > ${FILELIST}

  run_smoke_test ${PROJECT_NAME} ${FILELIST} ${git_project}
  status_sum=$((${status_sum} + $?))
  echo
done

echo "::endgroup::"

echo "There were a total of ${status_sum} new, undocumented issues."

# Let's see if there are any issues that are fixed in the meantime.
if [ ${#KnownIssue[@]} -ne 0 ]; then
  echo "::warning ::There are ${#KnownIssue[@]} tool/file combinations, that no longer fail"
  declare -A DistinctIssues
  for key in "${!KnownIssue[@]}"; do
    echo " 🎉 ${key}"
    DistinctIssues[${KnownIssue[${key}]}]=1
  done
  echo
  echo "::notice ::These were referencing the following issues. Maybe they are fixed now ?"
  for issue_id in "${!DistinctIssues[@]}"; do
    echo " 🐞 https://github.com/chipsalliance/verible/issues/$issue_id"
  done
  echo
fi

exit ${status_sum}