/*
 Copyright 2019 Alain Dargelas

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */

/*
 * File:   Compiler.cpp
 * Author: alain
 *
 * Created on March 4, 2017, 5:16 PM
 */
#include <stdint.h>
#include <cstdlib>
#include "CommandLine/CommandLineParser.h"
#include "ErrorReporting/ErrorContainer.h"
#include "SourceCompile/SymbolTable.h"
#include "SourceCompile/CompilationUnit.h"
#include "SourceCompile/PreprocessFile.h"
#include "SourceCompile/CompilationUnit.h"
#include "SourceCompile/CompileSourceFile.h"
#include "SourceCompile/Compiler.h"
#include "SourceCompile/CheckCompile.h"
#include "antlr4-runtime.h"
#include "DesignCompile/CompileDesign.h"
#include "SourceCompile/AnalyzeFile.h"
#include "Library/ParseLibraryDef.h"
#include "Utils/FileUtils.h"
#include "Package/Precompiled.h"
#include "Utils/StringUtils.h"
#include "Utils/Timer.h"
#include <math.h>
using namespace antlr4;

#include "API/PythonAPI.h"
#include "SourceCompile/CheckCompile.h"
#include <mutex>
#include <thread>
#include <vector>
#include <iostream>
#include <fstream>
using namespace SURELOG;

Compiler::Compiler(CommandLineParser* commandLineParser, ErrorContainer* errors,
                   SymbolTable* symbolTable)
    : m_commandLineParser(commandLineParser),
      m_errors(errors),
      m_symbolTable(symbolTable),
      m_commonCompilationUnit(NULL) {
  m_design = NULL;
#ifdef USETBB
  if (getCommandLineParser()->useTbb() &&
      (getCommandLineParser()->getNbMaxTreads() > 0))
    tbb::task_scheduler_init init(getCommandLineParser()->getNbMaxTreads());
#endif
}

Compiler::Compiler(const Compiler& orig) {}

Compiler::~Compiler() {
  std::map<SymbolId, PreprocessFile::AntlrParserHandler*>::iterator itr;
  for (itr = m_antlrPpMap.begin(); itr != m_antlrPpMap.end(); itr++) {
    delete (*itr).second;
  }
  if (m_commonCompilationUnit) {
    delete m_commonCompilationUnit;
  }
  cleanup_();
}

struct FunctorCompileOneFile {
  FunctorCompileOneFile(CompileSourceFile* compileSource,
                        CompileSourceFile::Action action)
      : m_compileSourceFile(compileSource), m_action(action) {}

  int operator()() const {
    if (m_compileSourceFile->getCommandLineParser()->pythonListener() ||
        m_compileSourceFile->getCommandLineParser()
            ->pythonEvalScriptPerFile()) {
      PyThreadState* interpState = PythonAPI::initNewInterp();
      m_compileSourceFile->setPythonInterp(interpState);
    }
    return m_compileSourceFile->compile(m_action);
  }

 private:
  CompileSourceFile* m_compileSourceFile;
  CompileSourceFile::Action m_action;
};

bool Compiler::compileOneFile_(CompileSourceFile* compiler,
                               CompileSourceFile::Action action) {
  bool status = compiler->compile(action);
  return status;
}

bool Compiler::isLibraryFile(SymbolId id) {
  if (m_libraryFiles.find(id) == m_libraryFiles.end()) {
    return false;
  } else {
    return true;
  }
}

bool Compiler::ppinit_() {
  if (!m_commandLineParser->fileunit()) {
    m_commonCompilationUnit = new CompilationUnit(false);
  }

  CompilationUnit* comp_unit = m_commonCompilationUnit;

  // Source files (.v, .sv on the command line)
  std::set<SymbolId> sourceFiles;
  unsigned int size = m_commandLineParser->getSourceFiles().size();
  for (unsigned int i = 0; i < size; i++) {
    SymbolTable* symbols = m_symbolTable;
    if (m_commandLineParser->fileunit()) {
      comp_unit = new CompilationUnit(true);
      m_compilationUnits.push_back(comp_unit);
      symbols = new SymbolTable(*m_commandLineParser->getSymbolTable());
      m_symbolTables.push_back(symbols);
    }
    ErrorContainer* errors = new ErrorContainer(symbols);
    m_errorContainers.push_back(errors);
    errors->regiterCmdLine(m_commandLineParser);

    SymbolId id = m_commandLineParser->getSourceFiles()[i];

    std::string fileName = m_commandLineParser->getSymbolTable()->getSymbol(id);
    std::string fullPath = FileUtils::getFullPath(fileName);
    SymbolId fullPathId =
        m_commandLineParser->getSymbolTable()->registerSymbol(fullPath);
    Library* library = m_librarySet->getLibrary(fullPathId);
    sourceFiles.insert(fullPathId);

    CompileSourceFile* compiler = new CompileSourceFile(
        m_commandLineParser->getSourceFiles()[i], m_commandLineParser, errors,
        this, symbols, comp_unit, library);
    m_compilers.push_back(compiler);
  }

  // Library files
  std::set<SymbolId> libFiles;
  // (-v <file>)
  size = m_commandLineParser->getLibraryFiles().size();
  for (unsigned int i = 0; i < size; i++) {
    SymbolId id = m_commandLineParser->getLibraryFiles()[i];
    libFiles.insert(id);
  }
  // (-y <path> +libext+<ext>)
  for (auto path : m_commandLineParser->getLibraryPaths()) {
    for (auto ext : m_commandLineParser->getLibraryExtensions()) {
      auto files = FileUtils::collectFiles(
          path, ext, m_commandLineParser->getSymbolTable());
      for (auto file : files) {
        libFiles.insert(file);
      }
    }
  }
  for (auto id : libFiles) {
    SymbolTable* symbols = m_symbolTable;
    if (m_commandLineParser->fileunit()) {
      comp_unit = new CompilationUnit(true);
      m_compilationUnits.push_back(comp_unit);
      symbols = new SymbolTable(*m_commandLineParser->getSymbolTable());
      m_symbolTables.push_back(symbols);
    }
    ErrorContainer* errors = new ErrorContainer(symbols);
    m_errorContainers.push_back(errors);
    errors->regiterCmdLine(m_commandLineParser);

    std::string fullPath = FileUtils::getFullPath(
        m_commandLineParser->getSymbolTable()->getSymbol(id));

    // This line registers the file in the "work" library:
    SymbolId fullPathId =
        m_commandLineParser->getSymbolTable()->registerSymbol(fullPath);
    /*Library* library  = */ m_librarySet->getLibrary(fullPathId);
    m_libraryFiles.insert(id);
    m_libraryFiles.insert(fullPathId);
    // No need to register a compiler
    // CompileSourceFile* compiler = new CompileSourceFile
    // (m_commandLineParser->getLibraryFiles ()[i],
    //                                                     m_commandLineParser,
    //                                                     errors, this,
    //                                                     symbols, comp_unit,
    //                                                     library);
    // m_compilers.push_back (compiler);
  }

  // Libraries (.map)
  for (auto& lib : m_librarySet->getLibraries()) {
    for (auto id : lib.getFiles()) {
      std::string fileName = lib.getSymbols()->getSymbol(id);
      if (sourceFiles.find(id) != sourceFiles.end()) {
        // These files are already included in the command line
        continue;
      }

      if (strstr(fileName.c_str(), ".map")) {
        // .map files are not parsed with the regular parser
        continue;
      }
      SymbolTable* symbols = m_symbolTable;
      if (m_commandLineParser->fileunit()) {
        comp_unit = new CompilationUnit(true);
        m_compilationUnits.push_back(comp_unit);
        symbols = new SymbolTable(*m_commandLineParser->getSymbolTable());
        m_symbolTables.push_back(symbols);
      }
      ErrorContainer* errors = new ErrorContainer(symbols);
      m_errorContainers.push_back(errors);
      errors->regiterCmdLine(m_commandLineParser);

      CompileSourceFile* compiler = new CompileSourceFile(
          id, m_commandLineParser, errors, this, symbols, comp_unit, &lib);
      m_compilers.push_back(compiler);
    }
  }
  return true;
}

bool Compiler::createFileList_()
{
  if (m_commandLineParser->writePpOutput() ||
          (m_commandLineParser->writePpOutputFileId() != 0)) {
    SymbolTable* symbolTable = getSymbolTable();
    const std::string& directory =
            symbolTable->getSymbol(m_commandLineParser->getFullCompileDir());
    std::ofstream ofs;
    std::string fileList = directory + "/file.lst";
    ofs.open(fileList);
    if (ofs.good()) {
      unsigned int size = m_compilers.size();
      for (unsigned int i = 0; i < size; i++) {
        std::string fileName = m_compilers[i]->getSymbolTable()->getSymbol(
                m_compilers[i]->getPpOutputFileId());
        fileName = StringUtils::replaceAll(fileName, "//", "/");
        ofs << fileName << std::flush << std::endl;
      }
      ofs.close();
    } else {
      std::cout << "Could not create filelist: " << fileList << std::endl;
    }
  }
  return true;
}
 
bool Compiler::createMultiProcess_() {
  unsigned int nbProcesses = m_commandLineParser->getNbMaxProcesses();
  if (nbProcesses == 0)
    return true;
  // Create CMakeLists.txt
  if (m_commandLineParser->writePpOutput() ||
          (m_commandLineParser->writePpOutputFileId() != 0)) {
    SymbolTable* symbolTable = getSymbolTable();
    const std::string& directory =
            symbolTable->getSymbol(m_commandLineParser->getFullCompileDir());
    std::ofstream ofs;
    std::string fileList = directory + "/CMakeLists.txt";
    ofs.open(fileList);
    if (ofs.good()) {
      ofs << "cmake_minimum_required (VERSION 3.0)" << std::endl;
      ofs << "# Auto generated by Surelog" << std::endl;

      // Optimize the load balance, try to even out the work in each thread by the
      // size of the files
      std::vector<std::vector < CompileSourceFile*>> jobArray(nbProcesses);
      std::vector<unsigned long> jobSize(nbProcesses);
      unsigned int largestJob = 0;
      for (unsigned int i = 0; i < m_compilers.size(); i++) {
        unsigned int size = m_compilers[i]->getJobSize(CompileSourceFile::Action::Parse);
        if (size > largestJob) {
          largestJob = size;
        }
      }
      std::cout << "LARGEST JOB SIZE: " << largestJob << std::endl;
      unsigned int bigJobThreashold = (largestJob / nbProcesses) * 3;
      std::cout << "LARGE JOB THREASHOLD: " << bigJobThreashold << std::endl;
      std::vector<CompileSourceFile*> bigJobs;
      for (unsigned short i = 0; i < nbProcesses; i++) jobSize[i] = 0;
      Precompiled* prec = Precompiled::getSingleton();
      for (unsigned int i = 0; i < m_compilers.size(); i++) {
        std::string root = m_compilers[i]->getSymbolTable()->getSymbol(m_compilers[i]->getFileId());
        root = StringUtils::getRootFileName(root);
        if (prec->isFilePrecompiled(root)) {
          continue;
        }
        unsigned int size = m_compilers[i]->getJobSize(CompileSourceFile::Action::Parse);
        if (size > bigJobThreashold) {
          bigJobs.push_back(m_compilers[i]);
          continue;
        }
        unsigned int newJobIndex = 0;
        uint64_t minJobQueue = ULLONG_MAX;
        for (unsigned short ii = 0; ii < nbProcesses; ii++) {
          if (jobSize[ii] < minJobQueue) {
            newJobIndex = ii;
            minJobQueue = jobSize[ii];
          }
        }
        jobSize[newJobIndex] += size;
        jobArray[newJobIndex].push_back(m_compilers[i]);
      }
      
      std::string full_exe_path = m_commandLineParser->getExePath();
      full_exe_path = FileUtils::getFullPath(full_exe_path);
      full_exe_path = StringUtils::eliminateRelativePath(full_exe_path);
      if (m_commandLineParser->profile()) {
        full_exe_path += " -profile";
      }
        
      std::string fileUnit = "";
      if (m_commandLineParser->fileunit())
        fileUnit = " -fileunit ";
     
      std::vector<std::string > targets;
      // Small jobs batch in clump processes
      int absoluteIndex = 0;
      for (unsigned short i = 0; i < nbProcesses; i++) {  
        std::string fileList; 
        std::string lastFile;
        absoluteIndex++;
        for (unsigned int j = 0; j < jobArray[i].size(); j++) {
          CompileSourceFile* compiler = jobArray[i][j];
          std::string fileName = compiler->getSymbolTable()->getSymbol(
                                        compiler->getPpOutputFileId());
          fileList += " " + fileName;
          lastFile = fileName;
        } 
        if (fileList != "") {
          std::string targetname = std::to_string(absoluteIndex) + "_" + FileUtils::fileName(lastFile);
          targets.push_back(targetname);
          ofs << "add_custom_command(OUTPUT " << targetname << std::endl;
          ofs << "  COMMAND " << full_exe_path << fileUnit <<
                  " -parseonly -mt 0 -mp 0 -nostdout -nobuiltin -l "
                  << directory + FileUtils::fileName(targetname) + ".log" << " " << fileList << std::endl;
          ofs << ")" << std::endl; 
        }
      }
      // Big jobs
      for (CompileSourceFile* compiler : bigJobs) {
        absoluteIndex++;
        std::string fileName = compiler->getSymbolTable()->getSymbol(
                                        compiler->getPpOutputFileId());
        std::string targetname = std::to_string(absoluteIndex) + "_"  + FileUtils::fileName(fileName);
        targets.push_back(targetname);
        ofs <<"add_custom_command(OUTPUT " << targetname << std::endl;
        ofs <<"  COMMAND " << full_exe_path << fileUnit <<
                            " -parseonly -mt 0 -mp 0 -nostdout -nobuiltin -l "
                           <<  directory + FileUtils::fileName(targetname) + ".log" << " " << fileName << std::endl;
        ofs << ")" << std::endl;  
      }
            
      ofs << "add_custom_target(Parse ALL DEPENDS" << std::endl; 
      for (auto target : targets) {
        ofs << target << std::endl;  
      }
      ofs << ")" << std::endl; 
      ofs << std::flush;      
      ofs.close();
      std::string command = "cd " + directory + ";" + "cmake .; make -j " + std::to_string(nbProcesses);
      std::cout << "Running: " << command << std::endl << std::flush;
      int result = system(command.c_str());
      std::cout << "cmake result: " << result << std::endl;
    } else {
      std::cout << "Could not create CMakeLists.txt: " << fileList << std::endl;
    }
  }
  
  
  return true;
}
  
bool Compiler::parseinit_() {
  Precompiled* prec = Precompiled::getSingleton();
  // Single out the large files.
  // Small files are going to be scheduled in multiple threads based on size.
  // Large files are going to be compiled in a different batch in multithread
  
  if (!m_commandLineParser->fileunit()) {
    unsigned int size = m_symbolTables.size();
    for (unsigned int i = 0; i < size; i++) {
      delete m_symbolTables[i];
    }
    size = m_errorContainers.size();
    for (unsigned int i = 0; i < size; i++) {
      delete m_errorContainers[i];
    }
    m_symbolTables.clear();
    m_errorContainers.clear();
  }

  std::vector<CompileSourceFile*> tmp_compilers;
  unsigned int size = m_compilers.size();
  for (unsigned int i = 0; i < size; i++) {
    std::string fileName = m_compilers[i]->getSymbolTable()->getSymbol(
        m_compilers[i]->getPpOutputFileId());
    std::string origFile = m_compilers[i]->getSymbolTable()->getSymbol(
        m_compilers[i]->getFileId());
    unsigned int nbThreads = m_commandLineParser->getNbMaxTreads();
    std::string root = fileName;
    root = StringUtils::getRootFileName(root);
    if (prec->isFilePrecompiled(root)) {
      nbThreads = 0;
    }
    int effectiveNbThreads = 0;

    if (nbThreads > 4)
      effectiveNbThreads = (int) (log(((float) nbThreads + 1.0) / 4.0) * 10.0);
    else
      effectiveNbThreads = nbThreads;
    
    AnalyzeFile* fileAnalyzer = new AnalyzeFile(
        m_commandLineParser, m_design, fileName, origFile, effectiveNbThreads);
    fileAnalyzer->analyze();
    m_compilers[i]->setFileAnalyzer(fileAnalyzer);
    if (fileAnalyzer->getSplitFiles().size() > 1) {
      // Schedule parent
      m_compilersParentFiles.push_back(m_compilers[i]);
      m_compilers[i]->initParser();

      if (!m_commandLineParser->fileunit()) {
        SymbolTable* symbols =
            new SymbolTable(*m_commandLineParser->getSymbolTable());
        m_symbolTables.push_back(symbols);
        m_compilers[i]->setSymbolTable(symbols);
        // fileContent->setSymbolTable(symbols);
        ErrorContainer* errors = new ErrorContainer(symbols);
        m_errorContainers.push_back(errors);
        errors->regiterCmdLine(m_commandLineParser);
        m_compilers[i]->setErrorContainer(errors);
      }

      FileContent* fileContent =
          new FileContent(m_compilers[i]->getParser()->getFileId(0),
                          m_compilers[i]->getParser()->getLibrary(),
                          m_compilers[i]->getSymbolTable(),
                          m_compilers[i]->getErrorContainer(), NULL, 0);
      m_compilers[i]->getParser()->setFileContent(fileContent);

      int j = 0;
      for (auto chunk : fileAnalyzer->getSplitFiles()) {
        SymbolTable* symbols =
            new SymbolTable(*m_commandLineParser->getSymbolTable());
        m_symbolTables.push_back(symbols);
        SymbolId ppId = symbols->registerSymbol(chunk);
        symbols->registerSymbol(
            m_compilers[i]->getParser()->getFileName(LINE1));
        CompileSourceFile* chunkCompiler = new CompileSourceFile(
            m_compilers[i], ppId, fileAnalyzer->getLineOffsets()[j]);
        // Schedule chunk
        tmp_compilers.push_back(chunkCompiler);

        chunkCompiler->setSymbolTable(symbols);
        ErrorContainer* errors = new ErrorContainer(symbols);
        m_errorContainers.push_back(errors);
        errors->regiterCmdLine(m_commandLineParser);
        chunkCompiler->setErrorContainer(errors);
        // chunkCompiler->getParser ()->setFileContent (fileContent);

        FileContent* fileContent =
            new FileContent(m_compilers[i]->getParser()->getFileId(0),
                            m_compilers[i]->getParser()->getLibrary(), symbols,
                            errors, NULL, ppId);
        chunkCompiler->getParser()->setFileContent(fileContent);
        getDesign()->addFileContent(m_compilers[i]->getParser()->getFileId(0),
                                    fileContent);

        j++;
      }
    } else {
      if (!m_commandLineParser->fileunit()) {
        SymbolTable* symbols =
            new SymbolTable(*m_commandLineParser->getSymbolTable());
        m_symbolTables.push_back(symbols);
        m_compilers[i]->setSymbolTable(symbols);
        ErrorContainer* errors = new ErrorContainer(symbols);
        m_errorContainers.push_back(errors);
        errors->regiterCmdLine(m_commandLineParser);
        m_compilers[i]->setErrorContainer(errors);
      }

      tmp_compilers.push_back(m_compilers[i]);
    }
  }
  m_compilers = tmp_compilers;

  return true;
}

bool Compiler::pythoninit_() { return parseinit_(); }

ErrorContainer::Stats Compiler::getErrorStats() {
  ErrorContainer::Stats stats;
  unsigned int size = m_errorContainers.size();
  for (unsigned int i = 0; i < size; i++) {
    ErrorContainer::Stats tmp;
    tmp = m_errorContainers[i]->getErrorStats();
    stats += tmp;
  }

  return stats;
}

bool Compiler::cleanup_() {
  unsigned int size = m_compilers.size();
  for (unsigned int i = 0; i < size; i++) {
    delete m_compilers[i];
  }
  size = m_compilationUnits.size();
  for (unsigned int i = 0; i < size; i++) {
    delete m_compilationUnits[i];
  }
  size = m_symbolTables.size();
  for (unsigned int i = 0; i < size; i++) {
    delete m_symbolTables[i];
  }
  size = m_errorContainers.size();
  for (unsigned int i = 0; i < size; i++) {
    delete m_errorContainers[i];
  }
  return true;
}

bool Compiler::compileFileSet_(CompileSourceFile::Action action,
                               bool allowMultithread,
                               std::vector<CompileSourceFile*>& container) {
  unsigned short maxThreadCount = m_commandLineParser->getNbMaxTreads();
  if (allowMultithread == false) {
    maxThreadCount = 0;
  }

  if (maxThreadCount == 0) {
    // Single thread
    unsigned int size = container.size();
    for (unsigned int i = 0; i < size; i++) {
      container[i]->setPythonInterp(PythonAPI::getMainInterp());
      bool status = compileOneFile_(container[i], action);
      m_errors->appendErrors(*container[i]->getErrorContainer());
      m_errors->printMessages(m_commandLineParser->muteStdout());
      if ((!status) || container[i]->getErrorContainer()->hasFatalErrors())
        return false;
    }
  } else if (getCommandLineParser()->useTbb() &&
             (action != CompileSourceFile::Action::Parse)) {
#ifdef USETBB
    // TBB Thread management
    int maxThreadCount = m_commandLineParser->getNbMaxTreads();
    if (allowMultithread == false) {
      maxThreadCount = 0;
    }
    unsigned int size = container.size();
    if (maxThreadCount) {
      for (unsigned int i = 0; i < size; i++) {
        m_taskGroup.run(FunctorCompileOneFile(container[i], action));
      }
      m_taskGroup.wait();
      bool fatalErrors = false;
      for (unsigned int i = 0; i < size; i++) {
        // Promote report to master error container
        m_errors->appendErrors(*container[i]->getErrorContainer());
        if (container[i]->getErrorContainer()->hasFatalErrors()) {
          fatalErrors = true;
        }
        if (getCommandLineParser()->pythonListener()) {
          container[i]->shutdownPythonInterp();
        }
        m_errors->printMessages(m_commandLineParser->muteStdout());
      }
      if (fatalErrors) return false;
    } else {
      for (unsigned int i = 0; i < size; i++) {
        container[i]->setPythonInterp(PythonAPI::getMainInterp());
        bool status = compileOneFile_(container[i], action);
        m_errors->appendErrors(*container[i]->getErrorContainer());
        m_errors->printMessages(m_commandLineParser->muteStdout());
        if ((!status) || container[i]->getErrorContainer()->hasFatalErrors())
          return false;
      }
    }
#endif
  } else {
    // Custom Thread management

    // Optimize the load balance, try to even out the work in each thread by the
    // size of the files
    std::vector<std::vector<CompileSourceFile*>> jobArray(maxThreadCount);

    std::vector<unsigned long> jobSize(maxThreadCount);

    for (unsigned short i = 0; i < maxThreadCount; i++) jobSize[i] = 0;

    for (unsigned int i = 0; i < container.size(); i++) {
      unsigned int size = container[i]->getJobSize(action);
      unsigned int newJobIndex = 0;
      uint64_t minJobQueue = ULLONG_MAX;
      for (unsigned short ii = 0; ii < maxThreadCount; ii++) {
        if (jobSize[ii] < minJobQueue) {
          newJobIndex = ii;
          minJobQueue = jobSize[ii];
        }
      }

      jobSize[newJobIndex] += size;
      jobArray[newJobIndex].push_back(container[i]);
    }

    if (getCommandLineParser()->profile()) {
      if (action == CompileSourceFile::Preprocess)
        std::cout << "Preprocessing task\n";
      else if (action == CompileSourceFile::Parse)
        std::cout << "Parsing task\n";
      else
        std::cout << "Misc Task\n";
      for (unsigned short i = 0; i < maxThreadCount; i++) {
        std::cout << "Thread " << i << " : \n";
        int sum = 0;
        for (unsigned int j = 0; j < jobArray[i].size(); j++) {
          std::string fileName;
          if (jobArray[i][j]->getPreprocessor())
            fileName = jobArray[i][j]->getPreprocessor()->getFileName(0);
          if (jobArray[i][j]->getParser())
            fileName = jobArray[i][j]->getParser()->getFileName(0);
          sum += jobArray[i][j]->getJobSize(action);
          std::cout << jobArray[i][j]->getJobSize(action) << " " << fileName
                    << "\n";
        }
        std::cout << ", Total: " << sum << std::endl << std::flush;
      }
    }

    // Create the threads with their respective workloads
    std::vector<std::thread*> threads;
    for (unsigned short i = 0; i < maxThreadCount; i++) {
      std::thread* th = new std::thread([=] {
        for (unsigned int j = 0; j < jobArray[i].size(); j++) {
          if (getCommandLineParser()->pythonListener() ||
              getCommandLineParser()->pythonEvalScriptPerFile()) {
            PyThreadState* interpState = PythonAPI::initNewInterp();
            jobArray[i][j]->setPythonInterp(interpState);
          }

          jobArray[i][j]->compile(action);

          if (getCommandLineParser()->pythonListener() ||
              getCommandLineParser()->pythonEvalScriptPerFile()) {
            jobArray[i][j]->shutdownPythonInterp();
          }
        }
      });
      threads.push_back(th);
    }

    // Sync the threads
    for (unsigned int th = 0; th < threads.size(); th++) {
      threads[th]->join();
    }

    // Delete the threads
    for (unsigned int th = 0; th < threads.size(); th++) {
      delete threads[th];
    }

    // Promote report to master error container
    bool fatalErrors = false;
    for (unsigned int j = 0; j < container.size(); j++) {
      m_errors->appendErrors(*container[j]->getErrorContainer());
      if (container[j]->getErrorContainer()->hasFatalErrors())
        fatalErrors = true;
    }
    m_errors->printMessages(m_commandLineParser->muteStdout());

    if (fatalErrors) return false;
  }
  return true;
}

bool Compiler::compile() {
  std::string profile;
  Timer tmr;
  Timer tmrTotal;
  // Scan the libraries definition
  if (!parseLibrariesDef_()) return false;

  if (m_commandLineParser->profile()) {
    std::string msg = "Scan libraries took " +
                      StringUtils::to_string(tmr.elapsed_rounded()) + "s\n";
    std::cout << msg << std::endl;
    profile += msg;
    tmr.reset();
  }

  // Preprocess
  ppinit_();
  if (!compileFileSet_(CompileSourceFile::Preprocess,
                       m_commandLineParser->fileunit(), m_compilers))
    return false;

  // Single thread post Preprocess
  if (!compileFileSet_(CompileSourceFile::PostPreprocess, false, m_compilers))
    return false;

  if (m_commandLineParser->profile()) {
    std::string msg = "Preprocessing took " +
                      StringUtils::to_string(tmr.elapsed_rounded()) + "s\n";
    std::cout << msg << std::endl;
    for (unsigned int i = 0; i < m_compilers.size(); i++) {
      msg += m_compilers[i]->getPreprocessor()->getProfileInfo();
    }
    std::cout << msg << std::endl;
    profile += msg;
    tmr.reset();
  }

  // Parse
  bool parserInitialized = false;
  if (m_commandLineParser->parse() || m_commandLineParser->pythonListener() ||
      m_commandLineParser->pythonEvalScriptPerFile() ||
      m_commandLineParser->pythonEvalScript()) {
    parseinit_();
    createFileList_();
    createMultiProcess_();
    parserInitialized = true;
    if (!compileFileSet_(CompileSourceFile::Parse, true, m_compilers))
      return false;  // Small files and large file chunks
    if (!compileFileSet_(CompileSourceFile::Parse, true,
                         m_compilersParentFiles))
      return false;  // Recombine chunks
  } else {
    createFileList_();
  }

  if (m_commandLineParser->profile()) {
    std::string msg =
        "Parsing took " + StringUtils::to_string(tmr.elapsed_rounded()) + "s\n";
    for (unsigned int i = 0; i < m_compilersParentFiles.size(); i++) {
      msg += m_compilersParentFiles[i]->getParser()->getProfileInfo();
    }
    for (unsigned int i = 0; i < m_compilers.size(); i++) {
      msg += m_compilers[i]->getParser()->getProfileInfo();
    }

    std::cout << msg << std::endl;
    profile += msg;
    tmr.reset();
  }

  // Check Parsing
  CheckCompile* checkComp = new CheckCompile(this);
  checkComp->check();
  delete checkComp;
  m_errors->printMessages(m_commandLineParser->muteStdout());

  // Python Listener
  if (m_commandLineParser->pythonListener() ||
      m_commandLineParser->pythonEvalScriptPerFile()) {
    if (!parserInitialized) pythoninit_();
    if (!compileFileSet_(CompileSourceFile::PythonAPI, true, m_compilers))
      return false;
    if (!compileFileSet_(CompileSourceFile::PythonAPI, true,
                         m_compilersParentFiles))
      return false;

    if (m_commandLineParser->profile()) {
      std::string msg = "Python file processing took " +
                        StringUtils::to_string(tmr.elapsed_rounded()) + "s\n";
      std::cout << msg << std::endl;
      profile += msg;
      tmr.reset();
    }
  }

  if (m_commandLineParser->compile()) {
    // Compile Design, has its own thread management
    CompileDesign* compileDesign = new CompileDesign(this);
    compileDesign->compile();
    m_errors->printMessages(m_commandLineParser->muteStdout());

    if (m_commandLineParser->profile()) {
      std::string msg = "Compilation took " +
                        StringUtils::to_string(tmr.elapsed_rounded()) + "s\n";
      std::cout << msg << std::endl;
      profile += msg;
      tmr.reset();
    }

    if (m_commandLineParser->elaborate()) {
      compileDesign->elaborate();
      m_errors->printMessages(m_commandLineParser->muteStdout());

      if (m_commandLineParser->profile()) {
        std::string msg = "Elaboration took " +
                          StringUtils::to_string(tmr.elapsed_rounded()) + "s\n";
        std::cout << msg << std::endl;
        profile += msg;
        tmr.reset();
      }

      if (m_commandLineParser->pythonEvalScript()) {
        PythonAPI::evalScript(m_commandLineParser->getSymbolTable()->getSymbol(
                                  m_commandLineParser->pythonEvalScriptId()),
                              m_design);
        if (m_commandLineParser->profile()) {
          std::string msg = "Python design processing took " +
                            StringUtils::to_string(tmr.elapsed_rounded()) +
                            "s\n";
          profile += msg;
          std::cout << msg << std::endl;
          tmr.reset();
        }
      }
      m_errors->printMessages(m_commandLineParser->muteStdout());
    }

    delete compileDesign;
  }
  if (m_commandLineParser->profile()) {
    std::string msg = "Total time " +
                      StringUtils::to_string(tmrTotal.elapsed_rounded()) +
                      "s\n";
    profile += msg;
    profile = std::string("==============\n") + "PROFILE\n" +
              std::string("==============\n") + profile + "==============\n";
    std::cout << profile << std::endl;
    m_errors->printToLogFile(profile);
  }
  return true;
}

void Compiler::registerAntlrPpHandlerForId(
    SymbolId id, PreprocessFile::AntlrParserHandler* pp) {
  std::map<SymbolId, PreprocessFile::AntlrParserHandler*>::iterator itr =
      m_antlrPpMap.find(id);
  if (itr != m_antlrPpMap.end()) {
    delete (*itr).second;
    m_antlrPpMap.erase(itr);
    m_antlrPpMap.insert(std::make_pair(id, pp));
    return;
  }
  m_antlrPpMap.insert(std::make_pair(id, pp));
}

PreprocessFile::AntlrParserHandler* Compiler::getAntlrPpHandlerForId(
    SymbolId id) {
  std::map<SymbolId, PreprocessFile::AntlrParserHandler*>::iterator itr =
      m_antlrPpMap.find(id);
  if (itr != m_antlrPpMap.end()) {
    PreprocessFile::AntlrParserHandler* ptr = (*itr).second;
    return ptr;
  }
  return NULL;
}

bool Compiler::parseLibrariesDef_() {
  m_librarySet = new LibrarySet();
  m_configSet = new ConfigSet();
  m_design = new Design(getErrorContainer(), m_librarySet, m_configSet);
  ParseLibraryDef* libParser = new ParseLibraryDef(
      m_commandLineParser, m_errors, m_symbolTable, m_librarySet, m_configSet);
  return libParser->parseLibrariesDefinition();
}