Merge origin/master into fix/2579 Keep the thin formatter_test.cc from the split. Move the #2547 LongMacroSumLocalparamConverges regression from master's megafile into formatter_issue_regression_test.cc.
diff --git a/verible/verilog/analysis/verilog-equivalence.cc b/verible/verilog/analysis/verilog-equivalence.cc index 68c8c98..d5c12bea 100644 --- a/verible/verilog/analysis/verilog-equivalence.cc +++ b/verible/verilog/analysis/verilog-equivalence.cc
@@ -89,11 +89,17 @@ return IsUnlexed(verilog_tokentype(token.token_enum())); } -// MacroIdentifier vs MacroIdItem depends only on whether the macro ends the -// line (see POST_MACRO_ID in verilog.lex). Spelling-equal macros are -// format-equivalent across that reclassification. -static bool AreSpellingEqualLineEndingMacros(const TokenInfo &left, - const TokenInfo &right) { +// True when left/right differ only because surrounding whitespace changed +// token classification (e.g. MacroCallCloseToEndLine vs ')', or +// MacroIdentifier vs MacroIdItem with unchanged spelling). +static bool TokensAreWhitespaceDependentFormatEquivalent( + const TokenInfo &left, const TokenInfo &right) { + if ((left.token_enum() == verilog_tokentype::MacroCallCloseToEndLine && + right.text() == ")") || + (right.token_enum() == verilog_tokentype::MacroCallCloseToEndLine && + left.text() == ")")) { + return true; + } const auto is_line_ending_macro = [](int token_enum) { return token_enum == verilog_tokentype::MacroIdentifier || token_enum == verilog_tokentype::MacroIdItem; @@ -184,14 +190,8 @@ // Some token enums differ only by surrounding whitespace (e.g. whether a // macro or ')' ends a line). Treat those pairs as matching enums when the // spelling is unchanged so FormatEquivalent tolerates re-wrapping. - const bool whitespace_dependent_macro_enum_match = - ((l->token_enum() == verilog_tokentype::MacroCallCloseToEndLine && - r->text() == ")") || - (r->token_enum() == verilog_tokentype::MacroCallCloseToEndLine && - l->text() == ")") || - AreSpellingEqualLineEndingMacros(*l, *r)); if (l->token_enum() != r->token_enum() && - !whitespace_dependent_macro_enum_match) { + !TokensAreWhitespaceDependentFormatEquivalent(*l, *r)) { if (errstream != nullptr) { *errstream << "Mismatched token enums. got: "; token_printer(*l, *errstream); @@ -270,15 +270,7 @@ return IsWhitespace(verilog_tokentype(t.token_enum())); }, [=](const TokenInfo &l, const TokenInfo &r) { - // MacroCallCloseToEndLine should be considered equivalent to ')', as - // they are whitespace dependant - if (((r.token_enum() == verilog_tokentype::MacroCallCloseToEndLine) && - (l.text() == ")")) || - ((l.token_enum() == verilog_tokentype::MacroCallCloseToEndLine) && - (r.text() == ")"))) { - return true; - } - if (AreSpellingEqualLineEndingMacros(l, r)) { + if (TokensAreWhitespaceDependentFormatEquivalent(l, r)) { return true; } return l.EquivalentWithoutLocation(r);
diff --git a/verible/verilog/formatting/formatter_issue_regression_test.cc b/verible/verilog/formatting/formatter_issue_regression_test.cc index b813c86..3c98d4e 100644 --- a/verible/verilog/formatting/formatter_issue_regression_test.cc +++ b/verible/verilog/formatting/formatter_issue_regression_test.cc
@@ -31,6 +31,50 @@ using testing::HasSubstr; +// Regression for https://github.com/chipsalliance/verible/issues/2547: +// A localparam initialized to a sum of long macros must converge: infix `+` +// stays with the following operand so re-format does not oscillate between +// `+\n`MACRO` and `+ `MACRO`. +TEST(FormatterEndToEndTest, LongMacroSumLocalparamConverges) { + static constexpr FormatterTestCase kTestCases[] = { + {"module m;\n" + " localparam N =\n" + " `MACRO_GEN3_SCRAMBLE_LFSR_REGOUT\n" + " + `MACRO_GEN3_SCRAMBLE_REGIN\n" + " + `MACRO_GEN3_SCRAMBLE_REGOUT\n" + " ;\n" + "endmodule\n", + "module m;\n" + " localparam N =\n" + " `MACRO_GEN3_SCRAMBLE_LFSR_REGOUT\n" + " + `MACRO_GEN3_SCRAMBLE_REGIN\n" + " + `MACRO_GEN3_SCRAMBLE_REGOUT;\n" + "endmodule\n"}, + // Already in pass-1 form must stay stable. + {"module m;\n" + " localparam N =\n" + " `MACRO_GEN3_SCRAMBLE_LFSR_REGOUT\n" + " + `MACRO_GEN3_SCRAMBLE_REGIN\n" + " + `MACRO_GEN3_SCRAMBLE_REGOUT;\n" + "endmodule\n", + "module m;\n" + " localparam N =\n" + " `MACRO_GEN3_SCRAMBLE_LFSR_REGOUT\n" + " + `MACRO_GEN3_SCRAMBLE_REGIN\n" + " + `MACRO_GEN3_SCRAMBLE_REGOUT;\n" + "endmodule\n"}, + }; + FormatStyle style; // default column_limit (100) + for (const auto &test_case : kTestCases) { + VLOG(1) << "code-to-format:\n" << test_case.input << "<EOF>"; + std::ostringstream stream; + const auto status = + FormatVerilog(test_case.input, "<filename>", style, stream); + EXPECT_OK(status) << status.message(); + EXPECT_EQ(stream.str(), test_case.expected) << "code:\n" << test_case.input; + } +} + // Regression for https://github.com/chipsalliance/verible/issues/2544: // Wrapping a $bits(...)'(...) cast may leave `MACRO at EOL, reclassifying // MacroIdentifier as MacroIdItem. FormatEquivalent must accept that, and
diff --git a/verible/verilog/formatting/tree-unwrapper.cc b/verible/verilog/formatting/tree-unwrapper.cc index 2286f8c..09fa8c0 100644 --- a/verible/verilog/formatting/tree-unwrapper.cc +++ b/verible/verilog/formatting/tree-unwrapper.cc
@@ -1641,6 +1641,53 @@ } } +// True when a leaf partition contains only a binary/infix operator (and +// optional comments/attributes). Used to normalize wrapping of expressions +// like `A + B + C` so operators always stay with the following operand. +static bool PartitionIsInfixOperatorOnly(const TokenPartitionTree &partition) { + if (!is_leaf(partition)) return false; + const auto tokens = partition.Value().TokensRange(); + if (tokens.empty()) return false; + + const verible::PreFormatToken *op = nullptr; + for (const auto &token : tokens) { + switch (token.TokenEnum()) { + case verilog_tokentype::TK_COMMENT_BLOCK: + case verilog_tokentype::TK_EOL_COMMENT: + case verilog_tokentype::TK_ATTRIBUTE: + break; + default: + if (GetFormatTokenType(static_cast<verilog_tokentype>( + token.TokenEnum())) != FormatTokenType::binary_operator || + op != nullptr) { + return false; + } + op = &token; + break; + } + } + return op != nullptr; +} + +// Always attach infix-operator-only partitions to the following operand. +// Attachment based on original newlines is unstable for macro sums: +// `A\n+\n`B vs `A\n+ `B produce different partition shapes and oscillate +// under re-format (GitHub issue 2547). +static void AttachInfixOperatorsToFollowingOperands( + TokenPartitionTree *partition) { + // Iterate by index; merges invalidate sibling pointers. + for (int i = 0; i < static_cast<int>(partition->Children().size()); ++i) { + auto &child = partition->Children()[i]; + if (!PartitionIsInfixOperatorOnly(child)) continue; + if (NextLeaf(child) == nullptr) continue; + VLOG(4) << "Attaching infix operator partition to following operand:\n" + << child; + verible::MergeLeafIntoNextLeaf(&child); + // Children shifted; re-check current index. + --i; + } +} + static void AttachTrailingSemicolonToPreviousPartition( TokenPartitionTree *partition) { // TODO(mglb): Replace this function with @@ -3137,6 +3184,7 @@ case NodeEnum::kParamDeclaration: { AttachTrailingSemicolonToPreviousPartition(&partition); AttachOpeningBraceToDeclarationsAssignmentOperator(&partition); + AttachInfixOperatorsToFollowingOperands(&partition); break; }
diff --git a/verible/verilog/preprocessor/verilog-preprocess.cc b/verible/verilog/preprocessor/verilog-preprocess.cc index 1d847fb..39e816a 100644 --- a/verible/verilog/preprocessor/verilog-preprocess.cc +++ b/verible/verilog/preprocessor/verilog-preprocess.cc
@@ -272,6 +272,16 @@ if ((*token_iter)->text() == ")") { break; } + // Any other token -- in particular the EOF token from an unterminated + // macro call -- would otherwise leave token_iter and parameters_size + // unchanged and spin this loop forever. An early ')' (handled above) is the + // legal way to pass fewer arguments than parameters; anything else here is + // a malformed call, so reject it rather than silently accepting it. The + // caller records the returned message as a preprocessor error. + return absl::InvalidArgumentError(absl::StrCat( + "unexpected token while scanning arguments of macro call `", + macro_name_str, + ": expected ',' or ')', but got: ", (*token_iter)->ToString())); } if (parameters_size > 0) { while (parameters_size--) { @@ -305,8 +315,15 @@ if (config_.expand_macros) { verible::MacroCall macro_call; - RETURN_IF_ERROR( - ConsumeAndParseMacroCall(iter, generator, ¯o_call, *found)); + absl::Status call_status = + ConsumeAndParseMacroCall(iter, generator, ¯o_call, *found); + if (!call_status.ok()) { + // Surface a malformed macro call (e.g. an unterminated argument list) as + // a preprocessor error at the call site instead of silently accepting it. + std::string message(call_status.message()); + preprocess_data_.errors.emplace_back(**iter, message); + return call_status; + } RETURN_IF_ERROR(ExpandMacro(macro_call, found)); } auto &lexed = preprocess_data_.lexed_macros_backup.back();
diff --git a/verible/verilog/preprocessor/verilog-preprocess_test.cc b/verible/verilog/preprocessor/verilog-preprocess_test.cc index a8e41d9..c5d8eaa 100644 --- a/verible/verilog/preprocessor/verilog-preprocess_test.cc +++ b/verible/verilog/preprocessor/verilog-preprocess_test.cc
@@ -139,6 +139,17 @@ } } +// A function-like macro invoked with no closing ')' (EOF reached mid-call) must +// not spin ConsumeAndParseMacroCall forever. It is a malformed call, so the +// preprocessor rejects it with an error rather than silently accepting it. +// (Completing the test at all also proves the scan terminated instead of +// hanging.) +TEST(VerilogPreprocessTest, UnterminatedMacroCallIsRejected) { + PreprocessorTester tester("`define FOO(a, b) a\n`FOO(x", + VerilogPreprocess::Config({.expand_macros = true})); + EXPECT_FALSE(tester.PreprocessorData().errors.empty()); +} + #define EXPECT_PARSE_OK() \ do { \ EXPECT_TRUE(tester.Status().ok()) << "Unexpected analyzer failure."; \