Multi-process parsing initial support
Signed-off-by: Alain <alainmarcel@yahoo.com>
diff --git a/.travis.yml b/.travis.yml
index 865cd7d..e4c319f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -37,4 +37,5 @@
- make release
- sudo make install
- make test_install
+ - make test
diff --git a/Makefile b/Makefile
index 9341abb..f5e2649 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@
mkdir -p dist;
cd build; cmake ../ -DCMAKE_BUILD_TYPE=Release
$(MAKE) -C build
- cd build; ../tests/regression.tcl mt=0 show_diff
+
debug:
mkdir -p dbuild/tests;
mkdir -p dbuild/dist;
@@ -14,10 +14,9 @@
cd dbuild; cmake ../ -DCMAKE_BUILD_TYPE=Debug
$(MAKE) -C dbuild
-
test:
mkdir -p build/tests;
- cd build; ../tests/regression.tcl mt=0
+ cd build; ../tests/regression.tcl mt=0 show_diff
clean:
rm -rf dist;
diff --git a/README.md b/README.md
index 0462eac..a96121b 100644
--- a/README.md
+++ b/README.md
@@ -122,6 +122,8 @@
* A simple example of creating a new error message and generating errors can be found here: [`python_listener.py`](src/API/python_listener.py)
+ * A simple example for design-level data model exploration can be found here: [`myscriptPerDesign.py`](tests/UnitPython/myscriptPerDesign.py)
+
* The complete Python API is described in the following files: [`SLAPI.h`](src/API/SLAPI.h) [`vobjecttypes`](src/API/vobjecttypes.py)
* Waivers can be installed in slwaivers.py files in the execution directory or install directory /usr/local/lib/surelog/python
diff --git a/src/Cache/Cache.cpp b/src/Cache/Cache.cpp
index 0b3fb5f..34d0d89 100644
--- a/src/Cache/Cache.cpp
+++ b/src/Cache/Cache.cpp
@@ -67,7 +67,7 @@
std::string cacheFileName) {
/* Schema version */
if (schemaVersion != header->m_flb_version()->c_str()) {
- return false;
+ return false;
}
/* Tool version */
@@ -87,11 +87,16 @@
time_t ct = get_mtime(cacheFileName.c_str());
std::string fileName = header->m_file()->c_str();
time_t ft = get_mtime(fileName.c_str());
- if (ft == -1) return false;
- if (ct == -1) return false;
- if (ct < ft) return false;
+ if (ft == -1) {
+ return false;
+ }
+ if (ct == -1) {
+ return false;
+ }
+ if (ct < ft) {
+ return false;
+ }
}
-
return true;
}
diff --git a/src/Cache/ParseCache.cpp b/src/Cache/ParseCache.cpp
index e76f71e..26b97d0 100644
--- a/src/Cache/ParseCache.cpp
+++ b/src/Cache/ParseCache.cpp
@@ -177,7 +177,9 @@
bool ParseCache::checkCacheIsValid_(std::string cacheFileName) {
uint8_t* buffer_pointer = openFlatBuffers(cacheFileName);
- if (buffer_pointer == NULL) return false;
+ if (buffer_pointer == NULL) {
+ return false;
+ }
if (!PARSECACHE::ParseCacheBufferHasIdentifier(buffer_pointer)) {
delete[] buffer_pointer;
return false;
diff --git a/src/CommandLine/CommandLineParser.cpp b/src/CommandLine/CommandLineParser.cpp
index a507994..87dd533 100644
--- a/src/CommandLine/CommandLineParser.cpp
+++ b/src/CommandLine/CommandLineParser.cpp
@@ -27,6 +27,8 @@
#include <sstream>
#include <sys/types.h>
#include <sys/stat.h>
+#include <sys/param.h>
+#include <unistd.h>
#include "CommandLine/CommandLineParser.h"
#include "Utils/StringUtils.h"
#include "Utils/FileUtils.h"
@@ -108,6 +110,7 @@
" if \"max\" is given, the program will use one "
"thread",
" per core on the host",
+ " -mp <mb_max_process> 0 up to 512 max processes, 0 or 1 being single process",
" -split <line number> Split files or modules larger than specified line "
"number for multi thread compilation",
" -timescale=<timescale> Specifies the overall timescale",
@@ -230,6 +233,7 @@
m_help(false),
m_cacheAllowed(true),
m_nbMaxTreads(0),
+ m_nbMaxProcesses(0),
m_fullCompileDir(0),
m_cacheDirId(0),
m_note(true),
@@ -359,6 +363,7 @@
int CommandLineParser::parseCommandLine(int argc, const char** argv) {
std::string exe_name = argv[0];
+ m_exePath = exe_name;
std::string exe_path = FileUtils::getPathName(exe_name);
m_precompiledDirId = m_symbolTable->registerSymbol(exe_path + "pkg/");
if (!FileUtils::fileExists(exe_path + "pkg/")) {
@@ -522,7 +527,8 @@
}
i++;
m_nbLinesForFileSplitting = atoi(all_arguments[i].c_str());
- } else if (all_arguments[i] == "-mt") {
+ } else if (all_arguments[i] == "-mt" || all_arguments[i] == "-mp") {
+ bool mt = (all_arguments[i] == "-mt");
if (i == all_arguments.size() - 1) {
Location loc(getSymbolTable()->registerSymbol(all_arguments[i]));
Error err(ErrorDefinition::CMD_MT_MISSING_LEVEL, loc);
@@ -551,13 +557,21 @@
maxMT = (concurentThreadsSupported / 2);
}
- if (maxMT == 0)
- m_nbMaxTreads = maxMT;
- else {
- m_nbMaxTreads = maxMT;
- if (m_nbMaxTreads < 2) m_nbMaxTreads = 2;
+ if (maxMT == 0) {
+ if (mt)
+ m_nbMaxTreads = maxMT;
+ else
+ m_nbMaxProcesses = maxMT;
+ } else {
+ if (mt) {
+ m_nbMaxTreads = maxMT;
+ if (m_nbMaxTreads < 2) m_nbMaxTreads = 2;
+ } else {
+ m_nbMaxProcesses = maxMT;
+ if (m_nbMaxProcesses < 2) m_nbMaxProcesses = 2;
+ }
Location loc(
- getSymbolTable()->registerSymbol(std::to_string(m_nbMaxTreads)));
+ getSymbolTable()->registerSymbol(std::to_string(maxMT)));
Error err(ErrorDefinition::CMD_NUMBER_THREADS, loc);
m_errors->addError(err);
}
@@ -685,6 +699,16 @@
m_parse = true;
m_compile = true;
m_elaborate = true;
+ } else if (all_arguments[i] == "-parseonly") {
+ m_writePpOutput = true;
+ m_parse = true;
+ m_compile = false;
+ m_elaborate = false;
+ m_parseOnly = true;
+ int ret = chdir("..");
+ if (ret < 0) {
+ std::cout << "Could not change directory to ../\n" << std::endl;
+ }
} else if (all_arguments[i] == "-nocomp") {
m_compile = false;
m_elaborate = false;
diff --git a/src/CommandLine/CommandLineParser.h b/src/CommandLine/CommandLineParser.h
index 6253976..e18729f 100644
--- a/src/CommandLine/CommandLineParser.h
+++ b/src/CommandLine/CommandLineParser.h
@@ -59,7 +59,7 @@
return m_defineList;
}
bool fileunit() { return m_fileunit; } // File or all compilation semantic
-
+ void setFileUnit() { m_fileunit = true; }
/* PP Output file/dir options */
SymbolId writePpOutputFileId() { return m_writePpOutputFileId; }
SymbolId getOutputDir() { return m_outputDir; }
@@ -72,21 +72,28 @@
SymbolId getLogFileId() { return m_logFileId; }
SymbolId getDefaultLogFileId() { return m_defaultLogFileId; }
bool writePpOutput() { return m_writePpOutput; }
+ void setwritePpOutput(bool value) { m_writePpOutput = value; }
bool cacheAllowed() { return m_cacheAllowed; }
+ void setCacheAllowed(bool val) { m_cacheAllowed = val; }
bool lineOffsetsAsComments() { return m_lineOffsetsAsComments; }
SymbolId getCacheDir() { return m_cacheDirId; }
SymbolId getPrecompiledDir() { return m_precompiledDirId; }
bool usePPOutoutFileLocation() { return m_ppOutputFileLocation; }
/* PP Output content generation options */
bool filterFileLine() { return m_filterFileLine; }
+ void setFilterFileLine(bool val) { m_filterFileLine = val; }
bool filterSimpleDirectives() { return m_filterSimpleDirectives; }
bool filterProtectedRegions() { return m_filterProtectedRegions; }
bool filterComments() { return m_filterComments; }
bool filterInfo() { return !m_info; }
bool filterNote() { return !m_note; }
bool filterWarning() { return !m_warning; }
+ void setFilterInfo() { m_info = false; }
+ void setFilterNote() { m_note = false; }
+ void setFilterWarning() { m_warning = false; }
/* Debug/traces options */
bool muteStdout() { return m_muteStdout; }
+ void setMuteStdout() { m_muteStdout = true; }
bool verbose() { return m_verbose; }
bool profile() { return m_profile; }
int getDebugLevel() { return m_debugLevel; }
@@ -100,8 +107,13 @@
static std::string getVersionNumber() { return m_versionNumber; }
/* Core functions options */
bool parse() { return m_parse; }
+ bool parseOnly() { return m_parseOnly; }
bool compile() { return m_compile; }
bool elaborate() { return m_elaborate; }
+ void setParse(bool val) { m_parse = val; }
+ void setParseOnly(bool val) { m_parseOnly = val; }
+ void setCompile(bool val) { m_compile = val; }
+ void setElaborate(bool val) { m_elaborate = val; }
bool pythonListener() { return m_pythonListener && m_pythonAllowed; }
bool pythonAllowed() { return m_pythonAllowed; }
void noPython() { m_pythonAllowed = false; }
@@ -116,6 +128,9 @@
ErrorContainer* getErrorContainer() { return m_errors; }
SymbolTable* getSymbolTable() { return m_symbolTable; }
unsigned short int getNbMaxTreads() { return m_nbMaxTreads; }
+ unsigned short int getNbMaxProcesses() { return m_nbMaxProcesses; }
+ void setNbMaxTreads(unsigned short int max) { m_nbMaxTreads = max; }
+ void setNbMaxProcesses(unsigned short int max) { m_nbMaxProcesses = max; }
unsigned int getNbLinesForFileSpliting() { return m_nbLinesForFileSplitting; }
bool useTbb() { return m_useTbb; }
std::string getTimeScale() { return m_timescale; }
@@ -123,6 +138,7 @@
const std::string currentDateTime();
bool parseBuiltIn();
std::string getBuiltInPath() { return m_builtinPath; }
+ std::string getExePath() { return m_exePath; }
private:
bool plus_arguments_(const std::string& s);
void processArgs_(std::vector<std::string>& args,
@@ -162,12 +178,14 @@
bool m_filterProtectedRegions;
bool m_filterComments;
bool m_parse;
+ bool m_parseOnly;
bool m_compile;
bool m_elaborate;
bool m_diff_comp_mode;
bool m_help;
bool m_cacheAllowed;
unsigned short int m_nbMaxTreads;
+ unsigned short int m_nbMaxProcesses;
SymbolId m_compileUnitDirectory;
SymbolId m_compileAllDirectory;
SymbolId m_outputDir;
@@ -200,6 +218,7 @@
bool m_ppOutputFileLocation;
bool m_logFileSpecified;
std::string m_builtinPath;
+ std::string m_exePath;
};
}; // namespace SURELOG
diff --git a/src/ErrorReporting/ErrorContainer.cpp b/src/ErrorReporting/ErrorContainer.cpp
index 56423f8..da63beb 100644
--- a/src/ErrorReporting/ErrorContainer.cpp
+++ b/src/ErrorReporting/ErrorContainer.cpp
@@ -44,15 +44,20 @@
/* Do nothing here */
}
+#include <unistd.h>
+#include <stdio.h>
+
void ErrorContainer::init() {
if (ErrorDefinition::init()) {
const std::string& logFileName =
m_clp->getSymbolTable()->getSymbol(m_clp->getLogFileId());
std::ofstream ofs;
ofs.open(logFileName, std::fstream::out);
+ char cwd[1024];
+ getcwd(cwd, sizeof(cwd));
if (!ofs.good()) {
std::cerr << "[FATAL:LG0001] Cannot create log file \"" << logFileName
- << "\"" << std::endl;
+ << "\"" << "dir: " << cwd << std::endl;
return;
}
ofs.close();
diff --git a/src/SourceCompile/CompileSourceFile.cpp b/src/SourceCompile/CompileSourceFile.cpp
index c1e45a0..5b1aa52 100644
--- a/src/SourceCompile/CompileSourceFile.cpp
+++ b/src/SourceCompile/CompileSourceFile.cpp
@@ -251,10 +251,14 @@
}
bool CompileSourceFile::postPreprocess_() {
+ SymbolTable* symbolTable = getCompiler()->getSymbolTable();
+ if (m_commandLineParser->parseOnly()) {
+ m_ppResultFileId = m_symbolTable->registerSymbol(symbolTable->getSymbol(m_fileId));
+ return true;
+ }
std::string m_pp_result = m_pp->getPreProcessedFileContent();
if (m_commandLineParser->writePpOutput() ||
(m_commandLineParser->writePpOutputFileId() != 0)) {
- SymbolTable* symbolTable = getCompiler()->getSymbolTable();
const std::string& directory =
symbolTable->getSymbol(m_commandLineParser->getFullCompileDir());
std::string fileName = symbolTable->getSymbol(m_fileId);
diff --git a/src/SourceCompile/Compiler.cpp b/src/SourceCompile/Compiler.cpp
index b47783e..a40b864 100644
--- a/src/SourceCompile/Compiler.cpp
+++ b/src/SourceCompile/Compiler.cpp
@@ -21,7 +21,7 @@
* Created on March 4, 2017, 5:16 PM
*/
#include <stdint.h>
-
+#include <cstdlib>
#include "CommandLine/CommandLineParser.h"
#include "ErrorReporting/ErrorContainer.h"
#include "SourceCompile/SymbolTable.h"
@@ -256,7 +256,82 @@
return true;
}
-
+bool Compiler::createMultiProcess_() {
+ unsigned int nbProcesses = m_commandLineParser->getNbMaxProcesses();
+ if (nbProcesses == 0)
+ return true;
+ // Create CMakeLists.txt
+ if (m_commandLineParser->writePpOutput() ||
+ (m_commandLineParser->writePpOutputFileId() != 0)) {
+ SymbolTable* symbolTable = getSymbolTable();
+ const std::string& directory =
+ symbolTable->getSymbol(m_commandLineParser->getFullCompileDir());
+ std::ofstream ofs;
+ std::string fileList = directory + "/CMakeLists.txt";
+ ofs.open(fileList);
+ if (ofs.good()) {
+ ofs << "cmake_minimum_required (VERSION 3.0)" << std::endl;
+ ofs << "# Auto generated by Surelog" << std::endl;
+ unsigned int size = m_compilers.size();
+ std::string full_exe_path = m_commandLineParser->getExePath();
+ full_exe_path = FileUtils::getFullPath(full_exe_path);
+ full_exe_path = StringUtils::eliminateRelativePath(full_exe_path);
+ std::string fileUnit = "";
+ if (m_commandLineParser->fileunit())
+ fileUnit = " -fileunit ";
+ int modulo = 0;
+ if (size > nbProcesses) {
+ modulo = size / (nbProcesses);
+ if (modulo == 0)
+ modulo = 1;
+ }
+ int index = 0;
+ std::string fileList;
+ for (unsigned int i = 0; i < size; i++) {
+ std::string fileName = m_compilers[i]->getSymbolTable()->getSymbol(
+ m_compilers[i]->getPpOutputFileId());
+ if (strstr(fileName.c_str(), "builtin.sv")) continue;
+ index++;
+ fileList += " " + fileName;
+ if ((index == modulo) || (i == size -1)) {
+ std::string targetname = std::to_string(i) + "_" + FileUtils::fileName(fileName);
+ ofs <<"add_custom_command(OUTPUT " << targetname << std::endl;
+ ofs <<" COMMAND " << full_exe_path << fileUnit <<
+ " -parseonly -mt 0 -mp 0 -nostdout -nobuiltin -l "
+ << directory + targetname + ".log" << " " << fileList << std::endl;
+ ofs << ")" << std::endl;
+ fileList = "";
+ index = 0;
+ }
+ }
+ ofs << "add_custom_target(Parse ALL DEPENDS" << std::endl;
+ index = 0;
+ for (unsigned int i = 0; i < size; i++) {
+ std::string fileName = m_compilers[i]->getSymbolTable()->getSymbol(
+ m_compilers[i]->getPpOutputFileId());
+ if (strstr(fileName.c_str(), "builtin.sv")) continue;
+ index++;
+ if ((index == modulo) || (i == size -1)) {
+ ofs << std::to_string(i) << "_" << FileUtils::fileName(fileName) << std::endl;
+ index = 0;
+ }
+ }
+ ofs << ")" << std::endl;
+ ofs << std::flush;
+ ofs.close();
+ std::string command = "cd " + directory + ";" + "cmake .; make -j " + std::to_string(nbProcesses);
+ std::cout << "Running: " << command << std::endl << std::flush;
+ int result = system(command.c_str());
+ std::cout << "cmake result: " << result << std::endl;
+ } else {
+ std::cout << "Could not create CMakeLists.txt: " << fileList << std::endl;
+ }
+ }
+
+
+ return true;
+}
+
bool Compiler::parseinit_() {
Precompiled* prec = Precompiled::getSingleton();
// Single out the large files.
@@ -290,11 +365,14 @@
nbThreads = 0;
}
int effectiveNbThreads = 0;
- if (nbThreads > 4)
- effectiveNbThreads = (int)(log(((float)nbThreads + 1.0) / 4.0) * 10.0);
- else
- effectiveNbThreads = nbThreads;
-
+ if (m_commandLineParser->getNbMaxProcesses()) {
+ effectiveNbThreads = m_commandLineParser->getNbMaxProcesses();
+ } else {
+ if (nbThreads > 4)
+ effectiveNbThreads = (int)(log(((float)nbThreads + 1.0) / 4.0) * 10.0);
+ else
+ effectiveNbThreads = nbThreads;
+ }
AnalyzeFile* fileAnalyzer = new AnalyzeFile(
m_commandLineParser, m_design, fileName, origFile, effectiveNbThreads);
fileAnalyzer->analyze();
@@ -596,19 +674,23 @@
profile += msg;
tmr.reset();
}
- createFileList_();
+
// Parse
bool parserInitialized = false;
if (m_commandLineParser->parse() || m_commandLineParser->pythonListener() ||
m_commandLineParser->pythonEvalScriptPerFile() ||
m_commandLineParser->pythonEvalScript()) {
parseinit_();
+ createFileList_();
+ createMultiProcess_();
parserInitialized = true;
if (!compileFileSet_(CompileSourceFile::Parse, true, m_compilers))
return false; // Small files and large file chunks
if (!compileFileSet_(CompileSourceFile::Parse, true,
m_compilersParentFiles))
return false; // Recombine chunks
+ } else {
+ createFileList_();
}
if (m_commandLineParser->profile()) {
diff --git a/src/SourceCompile/Compiler.h b/src/SourceCompile/Compiler.h
index 3e3cbd8..175f7dd 100644
--- a/src/SourceCompile/Compiler.h
+++ b/src/SourceCompile/Compiler.h
@@ -75,6 +75,7 @@
bool ppinit_();
bool createFileList_();
+ bool createMultiProcess_();
bool parseinit_();
bool pythoninit_();
bool compileFileSet_(CompileSourceFile::Action action, bool allowMultithread,
diff --git a/src/SourceCompile/PreprocessFile.cpp b/src/SourceCompile/PreprocessFile.cpp
index 2b1ab9d..163898a 100644
--- a/src/SourceCompile/PreprocessFile.cpp
+++ b/src/SourceCompile/PreprocessFile.cpp
@@ -279,6 +279,8 @@
}
bool PreprocessFile::preprocess() {
+ if (getCompileSourceFile()->getCommandLineParser()->parseOnly())
+ return true;
Timer tmr;
PPCache cache(this);
if (cache.restore()) {
@@ -1068,6 +1070,8 @@
}
void PreprocessFile::saveCache() {
+ if (getCompileSourceFile()->getCommandLineParser()->parseOnly())
+ return;
if (m_macroBody == "") {
if (!m_usingCachedVersion) {
PPCache cache(this);
diff --git a/src/Utils/FileUtils.cpp b/src/Utils/FileUtils.cpp
index dc4310f..530e044 100644
--- a/src/Utils/FileUtils.cpp
+++ b/src/Utils/FileUtils.cpp
@@ -313,3 +313,12 @@
}
return "FAILED_TO_LOAD_CONTENT";
}
+
+std::string FileUtils::fileName(std::string str) {
+ char c = '/';
+ auto it1 = std::find_if(str.rbegin(), str.rend(),
+ [c](char ch) { return (ch == c); });
+ if (it1 != str.rend())
+ str.erase(str.begin(), it1.base());
+ return str;
+}
\ No newline at end of file
diff --git a/src/Utils/FileUtils.h b/src/Utils/FileUtils.h
index ce65776..c34f643 100644
--- a/src/Utils/FileUtils.h
+++ b/src/Utils/FileUtils.h
@@ -37,6 +37,7 @@
static int mkDir(const char* path);
static std::string getFullPath(const std::string path);
static std::string getPathName(const std::string path);
+ static std::string fileName(std::string str);
static unsigned long fileSize(const std::string name);
static std::vector<SymbolId> collectFiles(const std::string dirPath,
const std::string extension,
diff --git a/src/Utils/StringUtils.cpp b/src/Utils/StringUtils.cpp
index 69f9ede..d0cfbe6 100644
--- a/src/Utils/StringUtils.cpp
+++ b/src/Utils/StringUtils.cpp
@@ -267,13 +267,12 @@
}
std::string StringUtils::leaf(std::string str) {
- std::string result;
char c = '.';
auto it1 = std::find_if(str.rbegin(), str.rend(),
[c](char ch) { return (ch == c); });
- if (it1 != str.rend()) str.erase(str.begin(), it1.base());
+ if (it1 != str.rend())
+ str.erase(str.begin(), it1.base());
return str;
- return result;
}
std::string& StringUtils::getRootFileName(std::string& str) {
diff --git a/src/main.cpp b/src/main.cpp
index 1f8f794..28b8c20 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -37,6 +37,7 @@
SURELOG::ErrorContainer* errors = new SURELOG::ErrorContainer (symbolTable);
SURELOG::CommandLineParser* clp = new SURELOG::CommandLineParser (errors, symbolTable, diff_comp_mode, fileunit);
success = clp->parseCommandLine (argc, argv);
+ bool parseOnly = clp->parseOnly();
errors->printMessages (clp->muteStdout ());
if (success && (!clp->help()))
{
@@ -75,7 +76,10 @@
delete errors;
if ((!noFatalErrors) || (!success))
codedReturn |= 1;
- return codedReturn;
+ if (parseOnly)
+ return 0;
+ else
+ return codedReturn;
}
int
