PR #443: Package Extractor Kythe facts extraction for Packages. Convers: - package name (also package name after `endpackage`). - package body. - package imports: `import pkg::*` and `import pkg::my_var` Fixes: #392 GitHub PR https://github.com/google/verible/pull/443 Copybara import of the project: - 9c9a6298174a99542431d8cc052024db53447517 Moving from PackageExtractor to this branch because of co... by MinaToma <minatoma1999@gmail.com> Closes #443 PiperOrigin-RevId: 331161160
diff --git a/verilog/CST/BUILD b/verilog/CST/BUILD index 5e4e1bc..d8eba2d 100644 --- a/verilog/CST/BUILD +++ b/verilog/CST/BUILD
@@ -561,6 +561,7 @@ name = "package_test", srcs = ["package_test.cc"], deps = [ + ":match_test_utils", ":package", "//common/analysis:syntax_tree_search", "//common/analysis:syntax_tree_search_test_utils",
diff --git a/verilog/CST/module.cc b/verilog/CST/module.cc index 235434a..2d3000f 100644 --- a/verilog/CST/module.cc +++ b/verilog/CST/module.cc
@@ -82,7 +82,7 @@ } const TokenInfo* GetModuleEndLabel(const verible::Symbol& s) { - const auto& label_node = + const auto* label_node = verible::GetSubtreeAsSymbol(s, NodeEnum::kModuleDeclaration, 3); if (label_node == nullptr) { return nullptr;
diff --git a/verilog/CST/package.cc b/verilog/CST/package.cc index c1934c6..7baa6c4 100644 --- a/verilog/CST/package.cc +++ b/verilog/CST/package.cc
@@ -33,6 +33,11 @@ return SearchSyntaxTree(root, NodekPackageDeclaration()); } +std::vector<verible::TreeSearchMatch> FindAllPackageImportItems( + const verible::Symbol& root) { + return SearchSyntaxTree(root, NodekPackageImportItem()); +} + const verible::TokenInfo& GetPackageNameToken(const verible::Symbol& s) { const auto& name_node = GetPackageNameLeaf(s); return name_node.get(); @@ -42,4 +47,48 @@ return verible::GetSubtreeAsLeaf(s, NodeEnum::kPackageDeclaration, 2); } +const verible::SyntaxTreeLeaf* GetPackageNameEndLabel( + const verible::Symbol& package_declaration) { + const auto* label_node = verible::GetSubtreeAsSymbol( + package_declaration, NodeEnum::kPackageDeclaration, 6); + if (label_node == nullptr) { + return nullptr; + } + const auto& package_name = verible::GetSubtreeAsLeaf( + verible::SymbolCastToNode(*label_node), NodeEnum::kLabel, 1); + return &package_name; +} + +const verible::Symbol* GetPackageItemList( + const verible::Symbol& package_declaration) { + return verible::GetSubtreeAsSymbol(package_declaration, + NodeEnum::kPackageDeclaration, 4); +} + +const verible::SyntaxTreeNode& GetScopePrefixFromPackageImportItem( + const verible::Symbol& package_import_item) { + return verible::GetSubtreeAsNode(package_import_item, + NodeEnum::kPackageImportItem, 0, + NodeEnum::kScopePrefix); +} + +const verible::SyntaxTreeLeaf& GetImportedPackageName( + const verible::Symbol& package_import_item) { + return verible::GetSubtreeAsLeaf( + GetScopePrefixFromPackageImportItem(package_import_item), + NodeEnum::kScopePrefix, 0); +} + +const verible::SyntaxTreeLeaf* GeImportedItemNameFromPackageImportItem( + const verible::Symbol& package_import_item) { + const auto& imported_item = verible::GetSubtreeAsLeaf( + package_import_item, NodeEnum::kPackageImportItem, 1); + + if (imported_item.get().token_enum() != verilog_tokentype::SymbolIdentifier) { + return nullptr; + } + + return &imported_item; +} + } // namespace verilog
diff --git a/verilog/CST/package.h b/verilog/CST/package.h index 027ca31..a1ae638 100644 --- a/verilog/CST/package.h +++ b/verilog/CST/package.h
@@ -30,10 +30,42 @@ std::vector<verible::TreeSearchMatch> FindAllPackageDeclarations( const verible::Symbol&); +// Find all package imports items. +std::vector<verible::TreeSearchMatch> FindAllPackageImportItems( + const verible::Symbol& root); + // Extract the subnode of a package declaration that is the package name. const verible::TokenInfo& GetPackageNameToken(const verible::Symbol&); +// Return the node spanning the name of the package. const verible::SyntaxTreeLeaf& GetPackageNameLeaf(const verible::Symbol& s); + +// Extracts the node that spans the name of the package after endpackage if +// exists. +const verible::SyntaxTreeLeaf* GetPackageNameEndLabel( + const verible::Symbol& package_declaration); + +// Extracts the node that spans the body of the package. +const verible::Symbol* GetPackageItemList( + const verible::Symbol& package_declaration); + +// Extracts the node spanning the ScopePrefix node within PackageImportItem +// node. +const verible::SyntaxTreeNode& GetScopePrefixFromPackageImportItem( + const verible::Symbol& package_import_item); + +// Extracts package name for package import (node tagged with +// kPackageImportItem). +// e.g import pkg::my_integer return the node spanning "pkg". +const verible::SyntaxTreeLeaf& GetImportedPackageName( + const verible::Symbol& package_import_item); + +// Extracts the symbol identifier from PackageImportItem if exits. +// e.g import pkg::my_integer return the node spanning "my_integer". +// return nullptr in case of import pkg::*. +const verible::SyntaxTreeLeaf* GeImportedItemNameFromPackageImportItem( + const verible::Symbol& package_import_item); + } // namespace verilog #endif // VERIBLE_VERILOG_CST_PACKAGE_H_
diff --git a/verilog/CST/package_test.cc b/verilog/CST/package_test.cc index 40be7fb..be33cc3 100644 --- a/verilog/CST/package_test.cc +++ b/verilog/CST/package_test.cc
@@ -36,6 +36,7 @@ #include "common/text/token_info.h" #include "common/util/casts.h" #include "common/util/logging.h" +#include "verilog/CST/match_test_utils.h" #include "verilog/analysis/verilog_analyzer.h" #undef EXPECT_OK @@ -50,6 +51,7 @@ using verible::down_cast; using verible::SyntaxTreeNode; using verible::SyntaxTreeSearchTestCase; +using verible::TextStructureView; using verible::TreeSearchMatch; TEST(FindAllPackageDeclarationsTest, VariousTests) { @@ -345,5 +347,154 @@ EXPECT_EQ(token.text(), "foo"); } +TEST(GetPackageNameTest, GetPackageEndLabelName) { + constexpr int kTag = 1; + const SyntaxTreeSearchTestCase testcases[] = { + {""}, + {"package foo;\n endpackage"}, + {"package foo;\n endpackage: ", {kTag, "foo"}}, + {"package foo;\n function int f();\n return 10;\n endfunction: f\n class " + "c; endclass: c\n " + "endpackage: ", + {kTag, "foo"}}, + }; + + for (const auto& test : testcases) { + const absl::string_view code(test.code); + VerilogAnalyzer analyzer(code, "test-file"); + const auto code_copy = analyzer.Data().Contents(); + ASSERT_OK(analyzer.Analyze()) << "failed on:\n" << code; + const auto& root = analyzer.Data().SyntaxTree(); + + const auto declarations = + FindAllPackageDeclarations(*ABSL_DIE_IF_NULL(root)); + + std::vector<TreeSearchMatch> names; + for (const auto& decl : declarations) { + const auto* package_name = GetPackageNameEndLabel(*decl.match); + if (package_name == nullptr) continue; + names.push_back(TreeSearchMatch{package_name, {}}); + } + + std::ostringstream diffs; + EXPECT_TRUE(test.ExactMatchFindings(names, code_copy, &diffs)) + << "failed on:\n" + << code << "\ndiffs:\n" + << diffs.str(); + } +} + +TEST(GetPackageBodyTest, GetPackageItemList) { + constexpr int kTag = 1; + const SyntaxTreeSearchTestCase testcases[] = { + {""}, + {"package foo;\n endpackage"}, + {"package foo;\n endpackage: foo"}, + {"package foo;\n", + {kTag, + "function int f();\n return 10;\n endfunction: f\n class " + "c; endclass: c"}, + "\n ", + "endpackage: foo"}, + }; + + for (const auto& test : testcases) { + const absl::string_view code(test.code); + VerilogAnalyzer analyzer(code, "test-file"); + const auto code_copy = analyzer.Data().Contents(); + ASSERT_OK(analyzer.Analyze()) << "failed on:\n" << code; + const auto& root = analyzer.Data().SyntaxTree(); + + const auto declarations = + FindAllPackageDeclarations(*ABSL_DIE_IF_NULL(root)); + + std::vector<TreeSearchMatch> lists; + for (const auto& decl : declarations) { + const auto* package_item_list = GetPackageItemList(*decl.match); + if (package_item_list == nullptr) continue; + lists.push_back(TreeSearchMatch{package_item_list, {}}); + } + + std::ostringstream diffs; + EXPECT_TRUE(test.ExactMatchFindings(lists, code_copy, &diffs)) + << "failed on:\n" + << code << "\ndiffs:\n" + << diffs.str(); + } +} + +TEST(PackageImportTest, GetImportedPackageName) { + constexpr int kTag = 1; // value doesn't matter + const SyntaxTreeSearchTestCase kTestCases[] = { + {""}, + {"module m; endmodule\n"}, + {"package pkg; endpackage\nmodule m();\n import ", + {kTag, "pkg"}, + "::*;\nendmodule"}, + {"package pkg;\n int my_int;\nendpackage\nmodule m\nimport ", + {kTag, "pkg"}, + "::*;\nimport ", + {kTag, "pkg"}, + "::my_int;\n();\nendmodule"}, + }; + for (const auto& test : kTestCases) { + TestVerilogSyntaxRangeMatches( + __FUNCTION__, test, [](const TextStructureView& text_structure) { + const auto& root = text_structure.SyntaxTree(); + const auto decls = FindAllPackageImportItems(*ABSL_DIE_IF_NULL(root)); + + std::vector<TreeSearchMatch> names; + for (const auto& decl : decls) { + const auto& name = GetImportedPackageName(*decl.match); + names.emplace_back(TreeSearchMatch{&name, {/* ignored context */}}); + } + return names; + }); + } +} + +TEST(PackageImportTest, GetImportedItemName) { + constexpr int kTag = 1; // value doesn't matter + const SyntaxTreeSearchTestCase kTestCases[] = { + {""}, + {"module m; endmodule\n"}, + {"package pkg; endpackage\nmodule m();\n import pkg::*;\nendmodule"}, + {"package pkg;\n int my_int;\nendpackage\nmodule m();\n import " + "pkg::*;\nimport pkg::", + {kTag, "my_int"}, + ";\nendmodule"}, + {"package pkg;\n int my_int;\nclass " + "my_class;\nendclass\nendpackage\nmodule m();\n import " + "pkg::*;\nimport pkg::", + {kTag, "my_int"}, + ";\nimport pkg::", + {kTag, "my_class"}, + ";\nendmodule"}, + {"package pkg;\n int my_int;\nclass " + "my_class;\nendclass\nendpackage\nmodule m\n import " + "pkg::*;\nimport pkg::", + {kTag, "my_int"}, + ";\nimport pkg::", + {kTag, "my_class"}, + ";\n();\n\nendmodule"}, + }; + for (const auto& test : kTestCases) { + TestVerilogSyntaxRangeMatches( + __FUNCTION__, test, [](const TextStructureView& text_structure) { + const auto& root = text_structure.SyntaxTree(); + const auto decls = FindAllPackageImportItems(*ABSL_DIE_IF_NULL(root)); + + std::vector<TreeSearchMatch> names; + for (const auto& decl : decls) { + const auto* name = + GeImportedItemNameFromPackageImportItem(*decl.match); + if (name == nullptr) continue; + names.emplace_back(TreeSearchMatch{name, {/* ignored context*/}}); + } + return names; + }); + } +} + } // namespace } // namespace verilog
diff --git a/verilog/tools/kythe/BUILD b/verilog/tools/kythe/BUILD index 60ef26e..38ef6d1 100644 --- a/verilog/tools/kythe/BUILD +++ b/verilog/tools/kythe/BUILD
@@ -116,6 +116,7 @@ "//verilog/CST:macro", "//verilog/CST:module", "//verilog/CST:net", + "//verilog/CST:package", "//verilog/CST:port", "//verilog/CST:tasks", "//verilog/CST:verilog_matchers",
diff --git a/verilog/tools/kythe/indexing_facts_tree_extractor.cc b/verilog/tools/kythe/indexing_facts_tree_extractor.cc index 715f7ce..9e9afbc 100644 --- a/verilog/tools/kythe/indexing_facts_tree_extractor.cc +++ b/verilog/tools/kythe/indexing_facts_tree_extractor.cc
@@ -26,6 +26,7 @@ #include "verilog/CST/macro.h" #include "verilog/CST/module.h" #include "verilog/CST/net.h" +#include "verilog/CST/package.h" #include "verilog/CST/port.h" #include "verilog/CST/tasks.h" #include "verilog/CST/verilog_matchers.h" @@ -37,6 +38,7 @@ namespace { +using verible::SyntaxTreeLeaf; using verible::SyntaxTreeNode; using verible::TreeSearchMatch; @@ -130,6 +132,10 @@ ExtractNetDeclaration(node); break; } + case NodeEnum::kPackageDeclaration: { + ExtractPackageDeclaration(node); + break; + } case NodeEnum::kPreprocessorDefine: { ExtractMacroDefinition(node); break; @@ -154,6 +160,10 @@ ExtractClassDeclaration(node); break; } + case NodeEnum::kPackageImportItem: { + ExtractPackageImport(node); + break; + } default: { TreeContextVisitor::Visit(node); } @@ -211,7 +221,7 @@ } void IndexingFactsTreeExtractor::ExtractModulePort( - const verible::SyntaxTreeNode& module_port_node) { + const SyntaxTreeNode& module_port_node) { const auto tag = static_cast<verilog::NodeEnum>(module_port_node.Tag().tag); // TODO(minatoma): Fix case like: @@ -221,7 +231,7 @@ // For extracting cases like: // module m(input a, input b); if (tag == NodeEnum::kPortDeclaration) { - const verible::SyntaxTreeLeaf* leaf = + const SyntaxTreeLeaf* leaf = GetIdentifierFromModulePortDeclaration(module_port_node); facts_tree_context_.top().NewChild( @@ -230,7 +240,7 @@ } else { // For extracting Non-ANSI style ports: // module m(a, b); - const verible::SyntaxTreeLeaf* leaf = + const SyntaxTreeLeaf* leaf = GetIdentifierFromPortReference(module_port_node); facts_tree_context_.top().NewChild( @@ -240,8 +250,8 @@ } void IndexingFactsTreeExtractor::ExtractInputOutputDeclaration( - const verible::SyntaxTreeNode& identifier_unpacked_dimension) { - const verible::SyntaxTreeLeaf* port_name_leaf = + const SyntaxTreeNode& identifier_unpacked_dimension) { + const SyntaxTreeLeaf* port_name_leaf = GetSymbolIdentifierFromIdentifierUnpackedDimensions( identifier_unpacked_dimension); @@ -290,7 +300,7 @@ // Module ports are treated as anchors in instantiations. for (const TreeSearchMatch& port : port_names) { - const verible::SyntaxTreeLeaf* leaf = GetIdentifier(*port.match); + const SyntaxTreeLeaf* leaf = GetIdentifier(*port.match); const Anchor port_name_anchor(leaf->get(), context_.base); indexing_node_data.AppendAnchor(port_name_anchor); @@ -317,6 +327,40 @@ } } +void IndexingFactsTreeExtractor::ExtractPackageDeclaration( + const SyntaxTreeNode& package_declaration_node) { + IndexingNodeData package_node_data(IndexingFactType::kPackage); + IndexingFactNode package_node(package_node_data); + + { + const IndexingFactsTreeContext::AutoPop p(&facts_tree_context_, + &package_node); + // Extract package name. + const SyntaxTreeLeaf& package_name_leaf = + GetPackageNameLeaf(package_declaration_node); + const Anchor class_name_anchor(package_name_leaf.get(), context_.base); + facts_tree_context_.top().Value().AppendAnchor(class_name_anchor); + + // Extract package name after endpackage if exists. + const SyntaxTreeLeaf* package_end_name = + GetPackageNameEndLabel(package_declaration_node); + + if (package_end_name != nullptr) { + const Anchor package_end_anchor(package_end_name->get(), context_.base); + facts_tree_context_.top().Value().AppendAnchor(package_end_anchor); + } + + // Visit package body it exists. + const verible::Symbol* package_item_list = + GetPackageItemList(package_declaration_node); + if (package_item_list != nullptr) { + Visit(verible::SymbolCastToNode(*package_item_list)); + } + } + + facts_tree_context_.top().NewChild(package_node); +} + void IndexingFactsTreeExtractor::ExtractMacroDefinition( const verible::SyntaxTreeNode& preprocessor_definition) { const verible::SyntaxTreeLeaf& macro_name = @@ -367,7 +411,7 @@ } void IndexingFactsTreeExtractor::ExtractFunctionDeclaration( - const verible::SyntaxTreeNode& function_declaration_node) { + const SyntaxTreeNode& function_declaration_node) { IndexingNodeData function_node_data(IndexingFactType::kFunctionOrTask); IndexingFactNode function_node(function_node_data); @@ -383,7 +427,7 @@ ExtractFunctionTaskPort(function_declaration_node); // Extract function body. - const verible::SyntaxTreeNode& function_body = + const SyntaxTreeNode& function_body = GetFunctionBlockStatementList(function_declaration_node); Visit(function_body); } @@ -392,7 +436,7 @@ } void IndexingFactsTreeExtractor::ExtractTaskDeclaration( - const verible::SyntaxTreeNode& task_declaration_node) { + const SyntaxTreeNode& task_declaration_node) { IndexingNodeData task_node_data(IndexingFactType::kFunctionOrTask); IndexingFactNode task_node(task_node_data); @@ -408,7 +452,7 @@ ExtractFunctionTaskPort(task_declaration_node); // Extract task body. - const verible::SyntaxTreeNode& task_body = + const SyntaxTreeNode& task_body = GetTaskStatementList(task_declaration_node); Visit(task_body); } @@ -417,12 +461,12 @@ } void IndexingFactsTreeExtractor::ExtractFunctionTaskPort( - const verible::SyntaxTreeNode& function_declaration_node) { - const std::vector<verible::TreeSearchMatch> ports = + const SyntaxTreeNode& function_declaration_node) { + const std::vector<TreeSearchMatch> ports = FindAllTaskFunctionPortDeclarations(function_declaration_node); - for (const verible::TreeSearchMatch& port : ports) { - const verible::SyntaxTreeLeaf* leaf = + for (const TreeSearchMatch& port : ports) { + const SyntaxTreeLeaf* leaf = GetIdentifierFromTaskFunctionPortItem(*port.match); // TODO(minatoma): Consider using kPorts or kParam for ports and params @@ -434,7 +478,7 @@ } void IndexingFactsTreeExtractor::ExtractFunctionOrTaskCall( - const verible::SyntaxTreeNode& function_call_node) { + const SyntaxTreeNode& function_call_node) { IndexingNodeData function_node_data(IndexingFactType::kFunctionCall); IndexingFactNode function_node(function_node_data); @@ -463,14 +507,12 @@ const IndexingFactsTreeContext::AutoPop p(&facts_tree_context_, &class_node); // Extract class name. - const verible::SyntaxTreeLeaf& class_name_leaf = - GetClassName(class_declaration); + const SyntaxTreeLeaf& class_name_leaf = GetClassName(class_declaration); const Anchor class_name_anchor(class_name_leaf.get(), context_.base); facts_tree_context_.top().Value().AppendAnchor(class_name_anchor); // Extract class name after endclass. - const verible::SyntaxTreeLeaf* class_end_name = - GetClassEndLabel(class_declaration); + const SyntaxTreeLeaf* class_end_name = GetClassEndLabel(class_declaration); if (class_end_name != nullptr) { const Anchor class_end_anchor(class_end_name->get(), context_.base); @@ -512,5 +554,24 @@ } } +void IndexingFactsTreeExtractor::ExtractPackageImport( + const SyntaxTreeNode& package_import_item) { + IndexingNodeData package_import_data(IndexingFactType::kPackageImport); + + const SyntaxTreeLeaf& package_name = + GetImportedPackageName(package_import_item); + + package_import_data.AppendAnchor(Anchor(package_name.get(), context_.base)); + + const SyntaxTreeLeaf* imported_item = + GeImportedItemNameFromPackageImportItem(package_import_item); + if (imported_item != nullptr) { + package_import_data.AppendAnchor( + Anchor(imported_item->get(), context_.base)); + } + + facts_tree_context_.top().NewChild(package_import_data); +} + } // namespace kythe } // namespace verilog
diff --git a/verilog/tools/kythe/indexing_facts_tree_extractor.h b/verilog/tools/kythe/indexing_facts_tree_extractor.h index 0fce56a..3184a1f 100644 --- a/verilog/tools/kythe/indexing_facts_tree_extractor.h +++ b/verilog/tools/kythe/indexing_facts_tree_extractor.h
@@ -69,6 +69,10 @@ void ExtractNetDeclaration( const verible::SyntaxTreeNode& net_declaration_node); + // Extract package declarations and creates its corresponding facts tree. + void ExtractPackageDeclaration( + const verible::SyntaxTreeNode& package_declaration_node); + // Extract macro definitions and explores its arguments and creates its // corresponding facts tree. void ExtractMacroDefinition( @@ -107,6 +111,9 @@ const verible::SyntaxTreeNode& data_declaration, const std::vector<verible::TreeSearchMatch>& register_variables); + // Extracts package imports and creates its corresponding fact tree. + void ExtractPackageImport(const verible::SyntaxTreeNode& package_import_item); + // The Root of the constructed tree IndexingFactNode root_{IndexingNodeData(IndexingFactType::kFile)};
diff --git a/verilog/tools/kythe/indexing_facts_tree_extractor_test.cc b/verilog/tools/kythe/indexing_facts_tree_extractor_test.cc index ea0b84f..df96da6 100644 --- a/verilog/tools/kythe/indexing_facts_tree_extractor_test.cc +++ b/verilog/tools/kythe/indexing_facts_tree_extractor_test.cc
@@ -1387,6 +1387,113 @@ EXPECT_EQ(result_pair.right, nullptr) << *result_pair.right; } +TEST(PackageImportTest, PackageAndImportedItemName) { + constexpr int kTag = 1; // value doesn't matter + const verible::SyntaxTreeSearchTestCase kTestCase = { + {"package ", + {kTag, "pkg1"}, + ";\nendpackage\npackage ", + {kTag, "pkg"}, + ";\nclass ", + {kTag, "my_class"}, + ";\nendclass\nfunction ", + {kTag, "my_function"}, + "();\nendfunction\nendpackage\nmodule ", + {kTag, "m"}, + "();\nimport ", + {kTag, "pkg1"}, + "::*;\nimport ", + {kTag, "pkg"}, + "::", + {kTag, "my_function"}, + ";\nimport ", + {kTag, "pkg"}, + "::", + {kTag, "my_class"}, + ";\nendmodule"}, + }; + + constexpr absl::string_view file_name = "verilog.v"; + int exit_status = 0; + bool parse_ok = false; + + const IndexingFactNode expected( + { + { + Anchor(file_name, 0, kTestCase.code.size()), + Anchor(kTestCase.code, 0, kTestCase.code.size()), + }, + IndexingFactType ::kFile, + }, + // refers to package pkg1. + T({ + { + Anchor(kTestCase.expected_tokens[1], kTestCase.code), + }, + IndexingFactType::kPackage, + }), + // refers to package pkg. + T( + { + { + Anchor(kTestCase.expected_tokens[3], kTestCase.code), + }, + IndexingFactType ::kPackage, + }, + // refers to class my_class. + T({ + { + Anchor(kTestCase.expected_tokens[5], kTestCase.code), + }, + IndexingFactType ::kClass, + }), + // refers to function my_function. + T({ + { + Anchor(kTestCase.expected_tokens[7], kTestCase.code), + }, + IndexingFactType ::kFunctionOrTask, + })), + // refers to module m.. + T( + { + { + Anchor(kTestCase.expected_tokens[9], kTestCase.code), + }, + IndexingFactType ::kModule, + }, + // refers to import pkg1::*. + T({ + { + Anchor(kTestCase.expected_tokens[11], kTestCase.code), + }, + IndexingFactType ::kPackageImport, + }), + // refers to import pkg::my_function. + T({ + { + Anchor(kTestCase.expected_tokens[13], kTestCase.code), + Anchor(kTestCase.expected_tokens[15], kTestCase.code), + }, + IndexingFactType ::kPackageImport, + }), + // refers to import pkg::my_class. + T({ + { + Anchor(kTestCase.expected_tokens[17], kTestCase.code), + Anchor(kTestCase.expected_tokens[19], kTestCase.code), + }, + IndexingFactType ::kPackageImport, + }))); + + const auto facts_tree = + ExtractOneFile(kTestCase.code, file_name, exit_status, parse_ok); + + const auto result_pair = DeepEqual(facts_tree, expected); + EXPECT_EQ(result_pair.left, nullptr) << *result_pair.left; + EXPECT_EQ(result_pair.right, nullptr) << *result_pair.right; +} + } // namespace } // namespace kythe } // namespace verilog
diff --git a/verilog/tools/kythe/kythe_facts_extractor.cc b/verilog/tools/kythe/kythe_facts_extractor.cc index 75dd69e..3ec8761 100644 --- a/verilog/tools/kythe/kythe_facts_extractor.cc +++ b/verilog/tools/kythe/kythe_facts_extractor.cc
@@ -24,7 +24,59 @@ namespace verilog { namespace kythe { -void KytheFactsExtractor::Visit(const IndexingFactNode& node) { +// Creates the signature for package. +std::string CreatePackageSignature(absl::string_view package_name) { + return absl::StrCat(package_name, "#package"); +} + +// Creates the signature for modules. +std::string CreateModuleSignature(absl::string_view module_name) { + return absl::StrCat(module_name, "#module"); +} + +// Creates the signature for classes. +std::string CreateClassSignature(absl::string_view class_name) { + return absl::StrCat(class_name, "#class"); +} + +// Creates the signature for variables. +std::string CreateVariableSignature(absl::string_view variable_name) { + return absl::StrCat(variable_name, "#variable"); +} + +// Creates the signature for functions or tasks. +std::string CreateFunctionOrTaskSignature(absl::string_view function_name) { + return absl::StrCat(function_name, "#function"); +} + +std::string CreateMacroSignature(absl::string_view macro_name) { + return absl::StrCat(macro_name, "#macro"); +} + +void KytheFactsExtractor::ExtractKytheFacts(const IndexingFactNode& root) { + CreatePackageScopes(root); + IndexingFactNodeTagResolver(root); +} + +void KytheFactsExtractor::CreatePackageScopes(const IndexingFactNode& root) { + // Searches for packages as a first pass and saves their scope to be used for + // imports. + for (const IndexingFactNode& child : root.Children()) { + if (child.Value().GetIndexingFactType() != IndexingFactType::kPackage) { + continue; + } + + VName package_vname = ExtractPackageDeclaration(child); + std::vector<VName> current_scope; + Visit(child, package_vname, current_scope); + + // Save the scope and the members of each package. + package_scope_context_[package_vname.signature] = current_scope; + } +} + +void KytheFactsExtractor::IndexingFactNodeTagResolver( + const IndexingFactNode& node) { const auto tag = node.Value().GetIndexingFactType(); VName vname(""); @@ -78,43 +130,70 @@ vname = ExtractFunctionOrTaskCall(node); break; } + case IndexingFactType::kPackageImport: { + vname = ExtractPackageImport(node); + break; + } case IndexingFactType::kMacroCall: { vname = ExtractMacroCall(node); break; } + case IndexingFactType::kPackage: { + // Return as packages should be extracted at the first pass. + return; + } + default: { + break; + } } - if (tag != IndexingFactType::kFile) { - GenerateEdgeString(vname, kEdgeChildOf, vnames_context_.top()); + // Determines whether or not to create the childof relation with the parent. + switch (tag) { + case IndexingFactType::kFile: + case IndexingFactType::kPackageImport: { + break; + } + default: { + if (!vnames_context_.empty()) { + GenerateEdgeString(vname, kEdgeChildOf, vnames_context_.top()); + } + break; + } } std::vector<VName> current_scope; - const ScopeContext::AutoPop scope_auto_pop(&scope_context_, ¤t_scope); // Determines whether or not to add the current node as a scope in vnames // context. switch (tag) { - case IndexingFactType::kFile: case IndexingFactType::kModule: case IndexingFactType::kModuleInstance: case IndexingFactType::kVariableDefinition: case IndexingFactType::kFunctionOrTask: { - const VNameContext::AutoPop vnames_auto_pop(&vnames_context_, &vname); - for (const verible::VectorTree<IndexingNodeData>& child : - node.Children()) { - Visit(child); - } + Visit(node, vname, current_scope); break; } default: { - for (const verible::VectorTree<IndexingNodeData>& child : - node.Children()) { - Visit(child); - } + Visit(node, current_scope); } } } +void KytheFactsExtractor::Visit(const IndexingFactNode& node, + const VName& vname, + std::vector<VName>& current_scope) { + const VNameContext::AutoPop vnames_auto_pop(&vnames_context_, &vname); + Visit(node, current_scope); +} + +void KytheFactsExtractor::Visit(const IndexingFactNode& node, + std::vector<VName>& current_scope) { + const ScopeContext::AutoPop scope_auto_pop(&scope_context_, ¤t_scope); + for (const IndexingFactNode& child : node.Children()) { + IndexingFactNodeTagResolver(child); + } +} + VName KytheFactsExtractor::ExtractFileFact( const IndexingFactNode& file_fact_node) { const VName file_vname(file_path_, "", "", ""); @@ -231,6 +310,28 @@ } } +VName KytheFactsExtractor::ExtractPackageDeclaration( + const IndexingFactNode& package_declaration_node) { + const auto& anchors = package_declaration_node.Value().Anchors(); + const Anchor& package_name = anchors[0]; + + const VName package_vname(file_path_, + CreateScopeRelativeSignature( + CreatePackageSignature(package_name.Value()))); + const VName package_name_anchor = PrintAnchorVName(package_name); + + GenerateFactString(package_vname, kFactNodeKind, kNodePackage); + GenerateEdgeString(package_name_anchor, kEdgeDefinesBinding, package_vname); + + if (anchors.size() > 1) { + const Anchor& package_end_label = anchors[1]; + const VName package_end_label_anchor = PrintAnchorVName(package_end_label); + GenerateEdgeString(package_end_label_anchor, kEdgeRef, package_vname); + } + + return package_vname; +} + VName KytheFactsExtractor::ExtractMacroDefinition( const IndexingFactNode& macro_definition_node) { const Anchor& macro_name = macro_definition_node.Value().Anchors()[0]; @@ -347,6 +448,61 @@ return class_instance_vname; } +VName KytheFactsExtractor::ExtractPackageImport( + const IndexingFactNode& import_fact_node) { + const auto& anchors = import_fact_node.Value().Anchors(); + const Anchor& package_name = anchors[0]; + + const VName package_vname(file_path_, + CreatePackageSignature(package_name.Value())); + const VName package_anchor = PrintAnchorVName(package_name); + + GenerateEdgeString(package_anchor, kEdgeRefImports, package_vname); + + // case of import pkg::my_variable. + if (anchors.size() > 1) { + const Anchor& imported_item_name = anchors[1]; + const VName& defintion_vname = + *ABSL_DIE_IF_NULL(SearchForDefinitionVNameInPackage( + package_name.Value(), imported_item_name.Value())); + + const VName imported_item_anchor = PrintAnchorVName(imported_item_name); + GenerateEdgeString(imported_item_anchor, kEdgeRef, defintion_vname); + + // Add the found definition to the current scope as if it was declared in + // our scope so that it can be captured without "::". + scope_context_.top().push_back(defintion_vname); + } else { + // case of import pkg::*. + // Add all the definitions in that package to the current scope as if it was + // declared in our scope so that it can be captured without "::". + const auto current_package_scope = package_scope_context_.find( + CreatePackageSignature(package_name.Value())); + for (const VName& vname : current_package_scope->second) { + scope_context_.top().push_back(vname); + } + } + + return package_vname; +} + +const VName* KytheFactsExtractor::SearchForDefinitionVNameInPackage( + absl::string_view package_name, absl::string_view reference_name) const { + const auto package_scope = + package_scope_context_.find(CreatePackageSignature(package_name)); + if (package_scope == package_scope_context_.end()) { + return nullptr; + } + + for (const VName& vname : package_scope->second) { + if (absl::StartsWith(vname.signature, reference_name)) { + return &vname; + } + } + + return nullptr; +} + VName KytheFactsExtractor::PrintAnchorVName(const Anchor& anchor) { const VName anchor_vname(file_path_, absl::Substitute(R"(@$0:$1)", anchor.StartLocation(), @@ -363,7 +519,9 @@ std::string KytheFactsExtractor::CreateScopeRelativeSignature( absl::string_view signature) const { - return absl::StrCat(signature, "#", vnames_context_.top().signature); + return vnames_context_.empty() + ? std::string(signature) + : absl::StrCat(signature, "#", vnames_context_.top().signature); } void KytheFactsExtractor::GenerateFactString( @@ -382,33 +540,13 @@ source_node.ToString(), edge_name, target_node.ToString()); } -std::string CreateModuleSignature(absl::string_view module_name) { - return absl::StrCat(module_name, "#module"); -} - -std::string CreateClassSignature(absl::string_view class_name) { - return absl::StrCat(class_name, "#class"); -} - -std::string CreateVariableSignature(absl::string_view variable_name) { - return absl::StrCat(variable_name, "#variable"); -} - -std::string CreateMacroSignature(absl::string_view macro_name) { - return absl::StrCat(macro_name, "#macro"); -} - -std::string CreateFunctionOrTaskSignature(absl::string_view function_name) { - return absl::StrCat(function_name, "#function"); -} - std::string GetFilePathFromRoot(const IndexingFactNode& root) { return root.Value().Anchors()[0].Value(); } std::ostream& KytheFactsPrinter::Print(std::ostream& stream) const { KytheFactsExtractor kythe_extractor(GetFilePathFromRoot(root_), &stream); - kythe_extractor.Visit(root_); + kythe_extractor.ExtractKytheFacts(root_); return stream; }
diff --git a/verilog/tools/kythe/kythe_facts_extractor.h b/verilog/tools/kythe/kythe_facts_extractor.h index 62dde02..5c09a9e 100644 --- a/verilog/tools/kythe/kythe_facts_extractor.h +++ b/verilog/tools/kythe/kythe_facts_extractor.h
@@ -15,6 +15,7 @@ #ifndef VERIBLE_VERILOG_TOOLS_KYTHE_KYTHE_FACTS_EXTRACTOR_H_ #define VERIBLE_VERILOG_TOOLS_KYTHE_KYTHE_FACTS_EXTRACTOR_H_ +#include <map> #include <string> #include <utility> @@ -48,8 +49,8 @@ std::ostream* stream) : file_path_(file_path), stream_(stream) {} - // Traverses the current tree and explore its children. - void Visit(const IndexingFactNode&); + // Extracts kythe facts from the given IndexingFactsTree root. + void ExtractKytheFacts(const IndexingFactNode&); private: // Container with a stack of VNames to hold context of VNames during traversal @@ -129,6 +130,28 @@ } }; + // Searches for a definition suitable for the given reference within package + // scope context of the given package name. + const VName* SearchForDefinitionVNameInPackage( + absl::string_view package_name, absl::string_view reference_name) const; + + // Resolves the tag of the given node and directs the flow to the appropriate + // function to extract kythe facts for that node. + void IndexingFactNodeTagResolver(const IndexingFactNode&); + + // Add the given VName to vnames_context (to be used in scope relative + // signatures) and visits the children of the given node. + void Visit(const IndexingFactNode& node, const VName& vname, + std::vector<VName>& current_scope); + + // Directs the flow to the children of the given node creating new scope for + // that node. + void Visit(const IndexingFactNode& node, std::vector<VName>& current_scope); + + // Extracts Packages and saves its scope to package_scope_context to be used + // for definition searching. + void CreatePackageScopes(const IndexingFactNode&); + // Extracts kythe facts from file node and returns it VName. VName ExtractFileFact(const IndexingFactNode&); @@ -159,6 +182,12 @@ VName ExtractFunctionOrTaskCall( const IndexingFactNode& function_call_fact_node); + // Extracts kythe facts from a package declaration node and returns its VName. + VName ExtractPackageDeclaration(const IndexingFactNode& node); + + // Extracts kythe facts from package import node and returns its VName. + VName ExtractPackageImport(const IndexingFactNode& node); + // Extracts kythe facts from a macro definition node and returns its VName. VName ExtractMacroDefinition(const IndexingFactNode& macro_definition_node); @@ -195,26 +224,32 @@ // the visitor traverses the facts tree. ScopeContext scope_context_; + // Saved packages signatures alongside with their inner members. + // This is used for resolving references to some variables after using import + // pkg::*. + // e.g + // package pkg1; + // function my_fun(); endfunction + // class my_class; endclass + // endpackage + // + // package pkg2; + // function my_fun(); endfunction + // class my_class; endclass + // endpackage + // + // Creates the following: + // { + // "pkg1": ["my_fun", "my_class"], + // "pkg2": ["my_fun", "my_class"] + // } + std::map<std::string, std::vector<VName>> package_scope_context_; + // Output stream for capturing, redirecting, testing and verifying the // output. std::ostream* stream_; }; -// Creates the signature for module names. -std::string CreateModuleSignature(absl::string_view); - -// Creates the signature for Class names. -std::string CreateClassSignature(absl::string_view); - -// Creates the signature for module instantiations. -std::string CreateVariableSignature(absl::string_view); - -// Creates the signature for macro. -std::string CreateMacroSignature(absl::string_view); - -// Creates the signature for functions or tasks. -std::string CreateFunctionOrTaskSignature(absl::string_view); - std::ostream& operator<<(std::ostream&, const KytheFactsPrinter&); } // namespace kythe
diff --git a/verilog/tools/kythe/kythe_schema_constants.h b/verilog/tools/kythe/kythe_schema_constants.h index ba177dd..067df7f 100644 --- a/verilog/tools/kythe/kythe_schema_constants.h +++ b/verilog/tools/kythe/kythe_schema_constants.h
@@ -26,6 +26,7 @@ // Kythe Nodes. constexpr absl::string_view kNodeAnchor = "anchor"; constexpr absl::string_view kNodeRecord = "record"; +constexpr absl::string_view kNodePackage = "package"; constexpr absl::string_view kNodeMacro = "macro"; constexpr absl::string_view kNodeFile = "file"; constexpr absl::string_view kNodeBuiltin = "tbuiltin"; @@ -48,6 +49,7 @@ constexpr absl::string_view kEdgeRef = "/kythe/edge/ref"; constexpr absl::string_view kEdgeRefExpands = "/kythe/edge/ref/expands"; constexpr absl::string_view kEdgeRefCall = "/kythe/edge/ref/call"; +constexpr absl::string_view kEdgeRefImports = "/kythe/edge/ref/imports"; constexpr absl::string_view kEdgeTyped = "/kythe/edge/typed"; } // namespace kythe
diff --git a/verilog/tools/kythe/verilog_extractor_indexing_fact_type.h b/verilog/tools/kythe/verilog_extractor_indexing_fact_type.h index 8331a3e..f01d0c3 100644 --- a/verilog/tools/kythe/verilog_extractor_indexing_fact_type.h +++ b/verilog/tools/kythe/verilog_extractor_indexing_fact_type.h
@@ -27,6 +27,7 @@ kFile, kMacro, kModule, + kPackage, kMacroCall, kClass, kClassInstance, @@ -35,6 +36,7 @@ kVariableReference, kFunctionOrTask, kFunctionCall, + kPackageImport, // END GENERATE -- do not delete };