Multi-process parsing initial support Signed-off-by: Alain <alainmarcel@yahoo.com>
diff --git a/.travis.yml b/.travis.yml index 865cd7d..e4c319f 100644 --- a/.travis.yml +++ b/.travis.yml
@@ -37,4 +37,5 @@ - make release - sudo make install - make test_install + - make test
diff --git a/Makefile b/Makefile index 9341abb..f5e2649 100644 --- a/Makefile +++ b/Makefile
@@ -6,7 +6,7 @@ mkdir -p dist; cd build; cmake ../ -DCMAKE_BUILD_TYPE=Release $(MAKE) -C build - cd build; ../tests/regression.tcl mt=0 show_diff + debug: mkdir -p dbuild/tests; mkdir -p dbuild/dist; @@ -14,10 +14,9 @@ cd dbuild; cmake ../ -DCMAKE_BUILD_TYPE=Debug $(MAKE) -C dbuild - test: mkdir -p build/tests; - cd build; ../tests/regression.tcl mt=0 + cd build; ../tests/regression.tcl mt=0 show_diff clean: rm -rf dist;
diff --git a/README.md b/README.md index 0462eac..a96121b 100644 --- a/README.md +++ b/README.md
@@ -122,6 +122,8 @@ * A simple example of creating a new error message and generating errors can be found here: [`python_listener.py`](src/API/python_listener.py) + * A simple example for design-level data model exploration can be found here: [`myscriptPerDesign.py`](tests/UnitPython/myscriptPerDesign.py) + * The complete Python API is described in the following files: [`SLAPI.h`](src/API/SLAPI.h) [`vobjecttypes`](src/API/vobjecttypes.py) * Waivers can be installed in slwaivers.py files in the execution directory or install directory /usr/local/lib/surelog/python
diff --git a/src/Cache/Cache.cpp b/src/Cache/Cache.cpp index 0b3fb5f..34d0d89 100644 --- a/src/Cache/Cache.cpp +++ b/src/Cache/Cache.cpp
@@ -67,7 +67,7 @@ std::string cacheFileName) { /* Schema version */ if (schemaVersion != header->m_flb_version()->c_str()) { - return false; + return false; } /* Tool version */ @@ -87,11 +87,16 @@ time_t ct = get_mtime(cacheFileName.c_str()); std::string fileName = header->m_file()->c_str(); time_t ft = get_mtime(fileName.c_str()); - if (ft == -1) return false; - if (ct == -1) return false; - if (ct < ft) return false; + if (ft == -1) { + return false; + } + if (ct == -1) { + return false; + } + if (ct < ft) { + return false; + } } - return true; }
diff --git a/src/Cache/ParseCache.cpp b/src/Cache/ParseCache.cpp index e76f71e..26b97d0 100644 --- a/src/Cache/ParseCache.cpp +++ b/src/Cache/ParseCache.cpp
@@ -177,7 +177,9 @@ bool ParseCache::checkCacheIsValid_(std::string cacheFileName) { uint8_t* buffer_pointer = openFlatBuffers(cacheFileName); - if (buffer_pointer == NULL) return false; + if (buffer_pointer == NULL) { + return false; + } if (!PARSECACHE::ParseCacheBufferHasIdentifier(buffer_pointer)) { delete[] buffer_pointer; return false;
diff --git a/src/CommandLine/CommandLineParser.cpp b/src/CommandLine/CommandLineParser.cpp index a507994..87dd533 100644 --- a/src/CommandLine/CommandLineParser.cpp +++ b/src/CommandLine/CommandLineParser.cpp
@@ -27,6 +27,8 @@ #include <sstream> #include <sys/types.h> #include <sys/stat.h> +#include <sys/param.h> +#include <unistd.h> #include "CommandLine/CommandLineParser.h" #include "Utils/StringUtils.h" #include "Utils/FileUtils.h" @@ -108,6 +110,7 @@ " if \"max\" is given, the program will use one " "thread", " per core on the host", + " -mp <mb_max_process> 0 up to 512 max processes, 0 or 1 being single process", " -split <line number> Split files or modules larger than specified line " "number for multi thread compilation", " -timescale=<timescale> Specifies the overall timescale", @@ -230,6 +233,7 @@ m_help(false), m_cacheAllowed(true), m_nbMaxTreads(0), + m_nbMaxProcesses(0), m_fullCompileDir(0), m_cacheDirId(0), m_note(true), @@ -359,6 +363,7 @@ int CommandLineParser::parseCommandLine(int argc, const char** argv) { std::string exe_name = argv[0]; + m_exePath = exe_name; std::string exe_path = FileUtils::getPathName(exe_name); m_precompiledDirId = m_symbolTable->registerSymbol(exe_path + "pkg/"); if (!FileUtils::fileExists(exe_path + "pkg/")) { @@ -522,7 +527,8 @@ } i++; m_nbLinesForFileSplitting = atoi(all_arguments[i].c_str()); - } else if (all_arguments[i] == "-mt") { + } else if (all_arguments[i] == "-mt" || all_arguments[i] == "-mp") { + bool mt = (all_arguments[i] == "-mt"); if (i == all_arguments.size() - 1) { Location loc(getSymbolTable()->registerSymbol(all_arguments[i])); Error err(ErrorDefinition::CMD_MT_MISSING_LEVEL, loc); @@ -551,13 +557,21 @@ maxMT = (concurentThreadsSupported / 2); } - if (maxMT == 0) - m_nbMaxTreads = maxMT; - else { - m_nbMaxTreads = maxMT; - if (m_nbMaxTreads < 2) m_nbMaxTreads = 2; + if (maxMT == 0) { + if (mt) + m_nbMaxTreads = maxMT; + else + m_nbMaxProcesses = maxMT; + } else { + if (mt) { + m_nbMaxTreads = maxMT; + if (m_nbMaxTreads < 2) m_nbMaxTreads = 2; + } else { + m_nbMaxProcesses = maxMT; + if (m_nbMaxProcesses < 2) m_nbMaxProcesses = 2; + } Location loc( - getSymbolTable()->registerSymbol(std::to_string(m_nbMaxTreads))); + getSymbolTable()->registerSymbol(std::to_string(maxMT))); Error err(ErrorDefinition::CMD_NUMBER_THREADS, loc); m_errors->addError(err); } @@ -685,6 +699,16 @@ m_parse = true; m_compile = true; m_elaborate = true; + } else if (all_arguments[i] == "-parseonly") { + m_writePpOutput = true; + m_parse = true; + m_compile = false; + m_elaborate = false; + m_parseOnly = true; + int ret = chdir(".."); + if (ret < 0) { + std::cout << "Could not change directory to ../\n" << std::endl; + } } else if (all_arguments[i] == "-nocomp") { m_compile = false; m_elaborate = false;
diff --git a/src/CommandLine/CommandLineParser.h b/src/CommandLine/CommandLineParser.h index 6253976..e18729f 100644 --- a/src/CommandLine/CommandLineParser.h +++ b/src/CommandLine/CommandLineParser.h
@@ -59,7 +59,7 @@ return m_defineList; } bool fileunit() { return m_fileunit; } // File or all compilation semantic - + void setFileUnit() { m_fileunit = true; } /* PP Output file/dir options */ SymbolId writePpOutputFileId() { return m_writePpOutputFileId; } SymbolId getOutputDir() { return m_outputDir; } @@ -72,21 +72,28 @@ SymbolId getLogFileId() { return m_logFileId; } SymbolId getDefaultLogFileId() { return m_defaultLogFileId; } bool writePpOutput() { return m_writePpOutput; } + void setwritePpOutput(bool value) { m_writePpOutput = value; } bool cacheAllowed() { return m_cacheAllowed; } + void setCacheAllowed(bool val) { m_cacheAllowed = val; } bool lineOffsetsAsComments() { return m_lineOffsetsAsComments; } SymbolId getCacheDir() { return m_cacheDirId; } SymbolId getPrecompiledDir() { return m_precompiledDirId; } bool usePPOutoutFileLocation() { return m_ppOutputFileLocation; } /* PP Output content generation options */ bool filterFileLine() { return m_filterFileLine; } + void setFilterFileLine(bool val) { m_filterFileLine = val; } bool filterSimpleDirectives() { return m_filterSimpleDirectives; } bool filterProtectedRegions() { return m_filterProtectedRegions; } bool filterComments() { return m_filterComments; } bool filterInfo() { return !m_info; } bool filterNote() { return !m_note; } bool filterWarning() { return !m_warning; } + void setFilterInfo() { m_info = false; } + void setFilterNote() { m_note = false; } + void setFilterWarning() { m_warning = false; } /* Debug/traces options */ bool muteStdout() { return m_muteStdout; } + void setMuteStdout() { m_muteStdout = true; } bool verbose() { return m_verbose; } bool profile() { return m_profile; } int getDebugLevel() { return m_debugLevel; } @@ -100,8 +107,13 @@ static std::string getVersionNumber() { return m_versionNumber; } /* Core functions options */ bool parse() { return m_parse; } + bool parseOnly() { return m_parseOnly; } bool compile() { return m_compile; } bool elaborate() { return m_elaborate; } + void setParse(bool val) { m_parse = val; } + void setParseOnly(bool val) { m_parseOnly = val; } + void setCompile(bool val) { m_compile = val; } + void setElaborate(bool val) { m_elaborate = val; } bool pythonListener() { return m_pythonListener && m_pythonAllowed; } bool pythonAllowed() { return m_pythonAllowed; } void noPython() { m_pythonAllowed = false; } @@ -116,6 +128,9 @@ ErrorContainer* getErrorContainer() { return m_errors; } SymbolTable* getSymbolTable() { return m_symbolTable; } unsigned short int getNbMaxTreads() { return m_nbMaxTreads; } + unsigned short int getNbMaxProcesses() { return m_nbMaxProcesses; } + void setNbMaxTreads(unsigned short int max) { m_nbMaxTreads = max; } + void setNbMaxProcesses(unsigned short int max) { m_nbMaxProcesses = max; } unsigned int getNbLinesForFileSpliting() { return m_nbLinesForFileSplitting; } bool useTbb() { return m_useTbb; } std::string getTimeScale() { return m_timescale; } @@ -123,6 +138,7 @@ const std::string currentDateTime(); bool parseBuiltIn(); std::string getBuiltInPath() { return m_builtinPath; } + std::string getExePath() { return m_exePath; } private: bool plus_arguments_(const std::string& s); void processArgs_(std::vector<std::string>& args, @@ -162,12 +178,14 @@ bool m_filterProtectedRegions; bool m_filterComments; bool m_parse; + bool m_parseOnly; bool m_compile; bool m_elaborate; bool m_diff_comp_mode; bool m_help; bool m_cacheAllowed; unsigned short int m_nbMaxTreads; + unsigned short int m_nbMaxProcesses; SymbolId m_compileUnitDirectory; SymbolId m_compileAllDirectory; SymbolId m_outputDir; @@ -200,6 +218,7 @@ bool m_ppOutputFileLocation; bool m_logFileSpecified; std::string m_builtinPath; + std::string m_exePath; }; }; // namespace SURELOG
diff --git a/src/ErrorReporting/ErrorContainer.cpp b/src/ErrorReporting/ErrorContainer.cpp index 56423f8..da63beb 100644 --- a/src/ErrorReporting/ErrorContainer.cpp +++ b/src/ErrorReporting/ErrorContainer.cpp
@@ -44,15 +44,20 @@ /* Do nothing here */ } +#include <unistd.h> +#include <stdio.h> + void ErrorContainer::init() { if (ErrorDefinition::init()) { const std::string& logFileName = m_clp->getSymbolTable()->getSymbol(m_clp->getLogFileId()); std::ofstream ofs; ofs.open(logFileName, std::fstream::out); + char cwd[1024]; + getcwd(cwd, sizeof(cwd)); if (!ofs.good()) { std::cerr << "[FATAL:LG0001] Cannot create log file \"" << logFileName - << "\"" << std::endl; + << "\"" << "dir: " << cwd << std::endl; return; } ofs.close();
diff --git a/src/SourceCompile/CompileSourceFile.cpp b/src/SourceCompile/CompileSourceFile.cpp index c1e45a0..5b1aa52 100644 --- a/src/SourceCompile/CompileSourceFile.cpp +++ b/src/SourceCompile/CompileSourceFile.cpp
@@ -251,10 +251,14 @@ } bool CompileSourceFile::postPreprocess_() { + SymbolTable* symbolTable = getCompiler()->getSymbolTable(); + if (m_commandLineParser->parseOnly()) { + m_ppResultFileId = m_symbolTable->registerSymbol(symbolTable->getSymbol(m_fileId)); + return true; + } std::string m_pp_result = m_pp->getPreProcessedFileContent(); if (m_commandLineParser->writePpOutput() || (m_commandLineParser->writePpOutputFileId() != 0)) { - SymbolTable* symbolTable = getCompiler()->getSymbolTable(); const std::string& directory = symbolTable->getSymbol(m_commandLineParser->getFullCompileDir()); std::string fileName = symbolTable->getSymbol(m_fileId);
diff --git a/src/SourceCompile/Compiler.cpp b/src/SourceCompile/Compiler.cpp index b47783e..a40b864 100644 --- a/src/SourceCompile/Compiler.cpp +++ b/src/SourceCompile/Compiler.cpp
@@ -21,7 +21,7 @@ * Created on March 4, 2017, 5:16 PM */ #include <stdint.h> - +#include <cstdlib> #include "CommandLine/CommandLineParser.h" #include "ErrorReporting/ErrorContainer.h" #include "SourceCompile/SymbolTable.h" @@ -256,7 +256,82 @@ return true; } - +bool Compiler::createMultiProcess_() { + unsigned int nbProcesses = m_commandLineParser->getNbMaxProcesses(); + if (nbProcesses == 0) + return true; + // Create CMakeLists.txt + if (m_commandLineParser->writePpOutput() || + (m_commandLineParser->writePpOutputFileId() != 0)) { + SymbolTable* symbolTable = getSymbolTable(); + const std::string& directory = + symbolTable->getSymbol(m_commandLineParser->getFullCompileDir()); + std::ofstream ofs; + std::string fileList = directory + "/CMakeLists.txt"; + ofs.open(fileList); + if (ofs.good()) { + ofs << "cmake_minimum_required (VERSION 3.0)" << std::endl; + ofs << "# Auto generated by Surelog" << std::endl; + unsigned int size = m_compilers.size(); + std::string full_exe_path = m_commandLineParser->getExePath(); + full_exe_path = FileUtils::getFullPath(full_exe_path); + full_exe_path = StringUtils::eliminateRelativePath(full_exe_path); + std::string fileUnit = ""; + if (m_commandLineParser->fileunit()) + fileUnit = " -fileunit "; + int modulo = 0; + if (size > nbProcesses) { + modulo = size / (nbProcesses); + if (modulo == 0) + modulo = 1; + } + int index = 0; + std::string fileList; + for (unsigned int i = 0; i < size; i++) { + std::string fileName = m_compilers[i]->getSymbolTable()->getSymbol( + m_compilers[i]->getPpOutputFileId()); + if (strstr(fileName.c_str(), "builtin.sv")) continue; + index++; + fileList += " " + fileName; + if ((index == modulo) || (i == size -1)) { + std::string targetname = std::to_string(i) + "_" + FileUtils::fileName(fileName); + ofs <<"add_custom_command(OUTPUT " << targetname << std::endl; + ofs <<" COMMAND " << full_exe_path << fileUnit << + " -parseonly -mt 0 -mp 0 -nostdout -nobuiltin -l " + << directory + targetname + ".log" << " " << fileList << std::endl; + ofs << ")" << std::endl; + fileList = ""; + index = 0; + } + } + ofs << "add_custom_target(Parse ALL DEPENDS" << std::endl; + index = 0; + for (unsigned int i = 0; i < size; i++) { + std::string fileName = m_compilers[i]->getSymbolTable()->getSymbol( + m_compilers[i]->getPpOutputFileId()); + if (strstr(fileName.c_str(), "builtin.sv")) continue; + index++; + if ((index == modulo) || (i == size -1)) { + ofs << std::to_string(i) << "_" << FileUtils::fileName(fileName) << std::endl; + index = 0; + } + } + ofs << ")" << std::endl; + ofs << std::flush; + ofs.close(); + std::string command = "cd " + directory + ";" + "cmake .; make -j " + std::to_string(nbProcesses); + std::cout << "Running: " << command << std::endl << std::flush; + int result = system(command.c_str()); + std::cout << "cmake result: " << result << std::endl; + } else { + std::cout << "Could not create CMakeLists.txt: " << fileList << std::endl; + } + } + + + return true; +} + bool Compiler::parseinit_() { Precompiled* prec = Precompiled::getSingleton(); // Single out the large files. @@ -290,11 +365,14 @@ nbThreads = 0; } int effectiveNbThreads = 0; - if (nbThreads > 4) - effectiveNbThreads = (int)(log(((float)nbThreads + 1.0) / 4.0) * 10.0); - else - effectiveNbThreads = nbThreads; - + if (m_commandLineParser->getNbMaxProcesses()) { + effectiveNbThreads = m_commandLineParser->getNbMaxProcesses(); + } else { + if (nbThreads > 4) + effectiveNbThreads = (int)(log(((float)nbThreads + 1.0) / 4.0) * 10.0); + else + effectiveNbThreads = nbThreads; + } AnalyzeFile* fileAnalyzer = new AnalyzeFile( m_commandLineParser, m_design, fileName, origFile, effectiveNbThreads); fileAnalyzer->analyze(); @@ -596,19 +674,23 @@ profile += msg; tmr.reset(); } - createFileList_(); + // Parse bool parserInitialized = false; if (m_commandLineParser->parse() || m_commandLineParser->pythonListener() || m_commandLineParser->pythonEvalScriptPerFile() || m_commandLineParser->pythonEvalScript()) { parseinit_(); + createFileList_(); + createMultiProcess_(); parserInitialized = true; if (!compileFileSet_(CompileSourceFile::Parse, true, m_compilers)) return false; // Small files and large file chunks if (!compileFileSet_(CompileSourceFile::Parse, true, m_compilersParentFiles)) return false; // Recombine chunks + } else { + createFileList_(); } if (m_commandLineParser->profile()) {
diff --git a/src/SourceCompile/Compiler.h b/src/SourceCompile/Compiler.h index 3e3cbd8..175f7dd 100644 --- a/src/SourceCompile/Compiler.h +++ b/src/SourceCompile/Compiler.h
@@ -75,6 +75,7 @@ bool ppinit_(); bool createFileList_(); + bool createMultiProcess_(); bool parseinit_(); bool pythoninit_(); bool compileFileSet_(CompileSourceFile::Action action, bool allowMultithread,
diff --git a/src/SourceCompile/PreprocessFile.cpp b/src/SourceCompile/PreprocessFile.cpp index 2b1ab9d..163898a 100644 --- a/src/SourceCompile/PreprocessFile.cpp +++ b/src/SourceCompile/PreprocessFile.cpp
@@ -279,6 +279,8 @@ } bool PreprocessFile::preprocess() { + if (getCompileSourceFile()->getCommandLineParser()->parseOnly()) + return true; Timer tmr; PPCache cache(this); if (cache.restore()) { @@ -1068,6 +1070,8 @@ } void PreprocessFile::saveCache() { + if (getCompileSourceFile()->getCommandLineParser()->parseOnly()) + return; if (m_macroBody == "") { if (!m_usingCachedVersion) { PPCache cache(this);
diff --git a/src/Utils/FileUtils.cpp b/src/Utils/FileUtils.cpp index dc4310f..530e044 100644 --- a/src/Utils/FileUtils.cpp +++ b/src/Utils/FileUtils.cpp
@@ -313,3 +313,12 @@ } return "FAILED_TO_LOAD_CONTENT"; } + +std::string FileUtils::fileName(std::string str) { + char c = '/'; + auto it1 = std::find_if(str.rbegin(), str.rend(), + [c](char ch) { return (ch == c); }); + if (it1 != str.rend()) + str.erase(str.begin(), it1.base()); + return str; +} \ No newline at end of file
diff --git a/src/Utils/FileUtils.h b/src/Utils/FileUtils.h index ce65776..c34f643 100644 --- a/src/Utils/FileUtils.h +++ b/src/Utils/FileUtils.h
@@ -37,6 +37,7 @@ static int mkDir(const char* path); static std::string getFullPath(const std::string path); static std::string getPathName(const std::string path); + static std::string fileName(std::string str); static unsigned long fileSize(const std::string name); static std::vector<SymbolId> collectFiles(const std::string dirPath, const std::string extension,
diff --git a/src/Utils/StringUtils.cpp b/src/Utils/StringUtils.cpp index 69f9ede..d0cfbe6 100644 --- a/src/Utils/StringUtils.cpp +++ b/src/Utils/StringUtils.cpp
@@ -267,13 +267,12 @@ } std::string StringUtils::leaf(std::string str) { - std::string result; char c = '.'; auto it1 = std::find_if(str.rbegin(), str.rend(), [c](char ch) { return (ch == c); }); - if (it1 != str.rend()) str.erase(str.begin(), it1.base()); + if (it1 != str.rend()) + str.erase(str.begin(), it1.base()); return str; - return result; } std::string& StringUtils::getRootFileName(std::string& str) {
diff --git a/src/main.cpp b/src/main.cpp index 1f8f794..28b8c20 100644 --- a/src/main.cpp +++ b/src/main.cpp
@@ -37,6 +37,7 @@ SURELOG::ErrorContainer* errors = new SURELOG::ErrorContainer (symbolTable); SURELOG::CommandLineParser* clp = new SURELOG::CommandLineParser (errors, symbolTable, diff_comp_mode, fileunit); success = clp->parseCommandLine (argc, argv); + bool parseOnly = clp->parseOnly(); errors->printMessages (clp->muteStdout ()); if (success && (!clp->help())) { @@ -75,7 +76,10 @@ delete errors; if ((!noFatalErrors) || (!success)) codedReturn |= 1; - return codedReturn; + if (parseOnly) + return 0; + else + return codedReturn; } int