From 5237aeac06141215da25698d8ca4709d167b14e6 Mon Sep 17 00:00:00 2001 From: Jeff Trull Date: Sat, 1 Jan 2022 22:03:19 -0800 Subject: [PATCH 1/3] Fix line directives for ifdef and ifndef when default hooks are used At some point in the past the handling for #if and #ifdef diverged. The code that handles emitting a line directive when a conditional section is skipped worked for #if but not ifdef/ifndef. This problem was not observable when the eat_whitespace hooks were used instead of the default_preprocessing hooks, because the former signals skipped newlines through the may_skip_whitespace hook, hiding the problem. Furthermore, the majority of Wave tests use the eat_whitespace hooks, so it wasn't visible there. This change restores ifdef/ifndef to the same section as #if, so any changes to conditional handling will happen uniformly. Also, a test case is added to cover the default hooks and this particular case. --- include/boost/wave/util/cpp_iterator.hpp | 102 +++++++---------- test/build/Jamfile.v2 | 9 ++ test/testwave/default_hooks.cpp | 137 +++++++++++++++++++++++ 3 files changed, 186 insertions(+), 62 deletions(-) create mode 100644 test/testwave/default_hooks.cpp diff --git a/include/boost/wave/util/cpp_iterator.hpp b/include/boost/wave/util/cpp_iterator.hpp index dfaa0127e..26214f616 100644 --- a/include/boost/wave/util/cpp_iterator.hpp +++ b/include/boost/wave/util/cpp_iterator.hpp @@ -327,10 +327,12 @@ class pp_iterator_functor { void on_define(parse_node_type const &node); void on_undefine(lexer_type const &it); - void on_ifdef(result_type const& found_directive, lexer_type const &it); -// typename parse_tree_type::const_iterator const &end); - void on_ifndef(result_type const& found_directive, lexer_type const& it); -// typename parse_tree_type::const_iterator const &end); + void on_ifdef(result_type const& found_directive, + typename parse_tree_type::const_iterator const &begin, + typename parse_tree_type::const_iterator const &end); + void on_ifndef(result_type const& found_directive, + typename parse_tree_type::const_iterator const &begin, + typename parse_tree_type::const_iterator const &end); void on_else(); void on_endif(); void on_illformed(typename result_type::string_type s); @@ -946,7 +948,7 @@ namespace impl { util::impl::call_skipped_token_hook(ctx, *it); for (++it; it != end; ++it) { - token_id id = token_id(*it); + token_id id = token_id(*it); if (T_CPPCOMMENT == id || T_NEWLINE == id || context_policies::util::ccomment_has_newline(*it)) @@ -957,14 +959,15 @@ namespace impl { return true; // no more significant tokens on this line } - if (!IS_CATEGORY(id, WhiteSpaceTokenType)) + if (!IS_CATEGORY(id, WhiteSpaceTokenType)) // ccomment, cppcomment, space, space2, placeholder break; // this token gets skipped if (call_hook) util::impl::call_skipped_token_hook(ctx, *it); } - return need_no_newline_at_end_of_file(ctx.get_language()); + return need_no_newline_at_end_of_file(ctx.get_language()) && + ((it == end) || (T_EOF == token_id(*it))); } /////////////////////////////////////////////////////////////////////////// @@ -1045,8 +1048,7 @@ pp_iterator_functor::ensure_is_last_on_line(IteratorT& it, bool call_h // enable error recovery (start over with the next line) impl::skip_to_eol(ctx, it, iter_ctx->last); - string_type str(util::impl::as_string( - iter_ctx->first, it)); + string_type str(util::impl::as_string(iter_ctx->first, it)); seen_newline = true; iter_ctx->first = it; @@ -1254,24 +1256,6 @@ pp_iterator_functor::handle_pp_directive(IteratorT &it) call_hook_in_skip = false; break; - case T_PP_IFDEF: // #ifdef - if (!impl::call_found_directive_hook(ctx, *it) && - extract_identifier(it)) - { - on_ifdef(directive, it); - } - call_hook_in_skip = false; - break; - - case T_PP_IFNDEF: // #ifndef - if (!impl::call_found_directive_hook(ctx, *it) && - extract_identifier(it)) - { - on_ifndef(directive, it); - } - call_hook_in_skip = false; - break; - #if BOOST_WAVE_SUPPORT_MS_EXTENSIONS != 0 // case T_MSEXT_PP_REGION: // #region ... // break; @@ -1466,13 +1450,13 @@ pp_iterator_functor::dispatch_directive( // on_undefine(*nodeval.begin()); // break; // -// case T_PP_IFDEF: // #ifdef -// on_ifdef(found_directive, begin_child_it, end_child_it); -// break; -// -// case T_PP_IFNDEF: // #ifndef -// on_ifndef(found_directive, begin_child_it, end_child_it); -// break; + case T_PP_IFDEF: // #ifdef + on_ifdef(found_directive, begin_child_it, end_child_it); + break; + + case T_PP_IFNDEF: // #ifndef + on_ifndef(found_directive, begin_child_it, end_child_it); + break; case T_PP_IF: // #if on_if(found_directive, begin_child_it, end_child_it); @@ -1860,26 +1844,23 @@ pp_iterator_functor::on_undefine (lexer_type const &it) template inline void pp_iterator_functor::on_ifdef( - result_type const& found_directive, lexer_type const &it) -// typename parse_tree_type::const_iterator const &it) -// typename parse_tree_type::const_iterator const &end) + result_type const& found_directive, + typename parse_tree_type::const_iterator const &begin, + typename parse_tree_type::const_iterator const &end) { - // get_token_value get_value; - // token_sequence_type toexpand; - // - // std::copy(make_ref_transform_iterator((*begin).children.begin(), get_value), - // make_ref_transform_iterator((*begin).children.end(), get_value), - // std::inserter(toexpand, toexpand.end())); + get_token_value get_value; + token_sequence_type toexpand; - bool is_defined = false; - token_sequence_type directive; + std::copy(make_ref_transform_iterator((*begin).children.begin(), get_value), + make_ref_transform_iterator((*begin).children.end(), get_value), + std::inserter(toexpand, toexpand.end())); - directive.insert(directive.end(), *it); + bool is_defined = false; do { - is_defined = ctx.is_defined_macro((*it).get_value()); // toexpand.begin(), toexpand.end()); + is_defined = ctx.is_defined_macro(toexpand.begin(), toexpand.end()); } while (ctx.get_hooks().evaluated_conditional_expression(ctx.derived(), - found_directive, directive, is_defined)); + found_directive, toexpand, is_defined)); ctx.enter_if_block(is_defined); } @@ -1891,26 +1872,23 @@ pp_iterator_functor::on_ifdef( template inline void pp_iterator_functor::on_ifndef( - result_type const& found_directive, lexer_type const &it) -// typename parse_tree_type::const_iterator const &it) -// typename parse_tree_type::const_iterator const &end) + result_type const& found_directive, + typename parse_tree_type::const_iterator const &begin, + typename parse_tree_type::const_iterator const &end) { - // get_token_value get_value; - // token_sequence_type toexpand; - // - // std::copy(make_ref_transform_iterator((*begin).children.begin(), get_value), - // make_ref_transform_iterator((*begin).children.end(), get_value), - // std::inserter(toexpand, toexpand.end())); + get_token_value get_value; + token_sequence_type toexpand; - bool is_defined = false; - token_sequence_type directive; + std::copy(make_ref_transform_iterator((*begin).children.begin(), get_value), + make_ref_transform_iterator((*begin).children.end(), get_value), + std::inserter(toexpand, toexpand.end())); - directive.insert(directive.end(), *it); + bool is_defined = false; do { - is_defined = ctx.is_defined_macro((*it).get_value()); // toexpand.begin(), toexpand.end()); + is_defined = ctx.is_defined_macro(toexpand.begin(), toexpand.end()); } while (ctx.get_hooks().evaluated_conditional_expression(ctx.derived(), - found_directive, directive, is_defined)); + found_directive, toexpand, is_defined)); ctx.enter_if_block(!is_defined); } diff --git a/test/build/Jamfile.v2 b/test/build/Jamfile.v2 index 0e935bb4e..9ba2a6050 100644 --- a/test/build/Jamfile.v2 +++ b/test/build/Jamfile.v2 @@ -224,5 +224,14 @@ test-suite wave /boost/thread//boost_thread /boost/filesystem//boost_filesystem ] + + [ + run + # sources + ../testwave/default_hooks.cpp + /boost/wave//boost_wave + /boost/thread//boost_thread + /boost/filesystem//boost_filesystem + ] ; diff --git a/test/testwave/default_hooks.cpp b/test/testwave/default_hooks.cpp new file mode 100644 index 000000000..28a8c915d --- /dev/null +++ b/test/testwave/default_hooks.cpp @@ -0,0 +1,137 @@ + +// Copyright 2018 Peter Dimov. +// Copyrigth 2022 Jeff Trull. +// +// Distributed under the Boost Software License, Version 1.0. +// +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt + +// Most test coverage of Wave happens with the "eat_whitespace" hooks in effect +// This adds some coverage for Wave with the (minimal) default hooks + +#include +#include +#include +#include +#include +#include +#include +#include + +int main() +{ + std::string input( + // check LINE directives after conditional (false) sections + "#if 0\n" + "int aa;\n" + "#endif\n" + "char c1;\n" // #LINE 4 + + "#ifdef FOO\n" + "int bb;\n" + "#endif\n" + "char c2;\n" // #LINE 8 + + "#ifndef __FILE__\n" + "int cc;\n" + "#endif\n" + "char c3;\n" // #LINE 13 + ); + + using namespace boost::wave; + + typedef std::pair tokdata_type; + typedef std::vector tokdata_list_type; + tokdata_list_type expected{ + {T_PP_LINE, "#line"}, + {T_SPACE, " "}, + {T_INTLIT, "4"}, + {T_SPACE, " "}, + {T_STRINGLIT, nullptr}, // exact filepath is not interesting + {T_NEWLINE, "\n"}, + {T_CHAR, "char"}, + {T_SPACE, " "}, + {T_IDENTIFIER, "c1"}, + {T_SEMICOLON, ";"}, + {T_NEWLINE, "\n"}, + + {T_PP_LINE, "#line"}, + {T_SPACE, " "}, + {T_INTLIT, "8"}, + {T_SPACE, " "}, + {T_STRINGLIT, nullptr}, // exact filepath is not interesting + {T_NEWLINE, "\n"}, + {T_CHAR, "char"}, + {T_SPACE, " "}, + {T_IDENTIFIER, "c2"}, + {T_SEMICOLON, ";"}, + {T_NEWLINE, "\n"}, + + {T_PP_LINE, "#line"}, + {T_SPACE, " "}, + {T_INTLIT, "12"}, + {T_SPACE, " "}, + {T_STRINGLIT, nullptr}, // exact filepath is not interesting + {T_NEWLINE, "\n"}, + {T_CHAR, "char"}, + {T_SPACE, " "}, + {T_IDENTIFIER, "c3"}, + {T_SEMICOLON, ";"}, + {T_NEWLINE, "\n"}, + + {T_EOF, nullptr} +}; + + try + { + typedef cpplexer::lex_token<> token_type; + typedef cpplexer::lex_iterator lex_iterator_type; + typedef context context_type; + + context_type ctx( input.begin(), input.end(), "input.cpp" ); + + tokdata_list_type::const_iterator first_expect = expected.begin(); + tokdata_list_type::const_iterator last_expect = expected.end(); + for( context_type::iterator_type first = ctx.begin(), last = ctx.end(); first != last; ++first, ++first_expect ) + { + if (first_expect == last_expect) + { + std::cerr << "more tokens were produced than expected\n"; + return 3; + } + + if (token_id(*first) != first_expect->first) + { + std::cerr << "expected " << get_token_name(first_expect->first) << ", got " << get_token_name(token_id(*first)) << "\n"; + return 3; + } + + if ((first_expect->second != nullptr) && (first_expect->second != first->get_value())) + { + std::cerr << "expected token value " << first_expect->second << ", got " << first->get_value() << "\n"; + return 3; + } + + std::cout << first->get_value(); + } + + if (first_expect != last_expect) + { + std::cerr << "fewer tokens were produced than expected\n"; + return 3; + } + + return 0; + } + catch( cpp_exception const & x ) + { + std::cerr << x.file_name() << "(" << x.line_no() << "): " << x.description() << std::endl; + return 1; + } + catch( std::exception const & x ) + { + std::cerr << "Exception: " << x.what() << std::endl; + return 2; + } +} From 6828d37b0861166ba1735e505ca45f33e9060230 Mon Sep 17 00:00:00 2001 From: Jeff Trull Date: Sat, 1 Jan 2022 22:17:55 -0800 Subject: [PATCH 2/3] Remove some accidentally included changes --- include/boost/wave/util/cpp_iterator.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/boost/wave/util/cpp_iterator.hpp b/include/boost/wave/util/cpp_iterator.hpp index 26214f616..432d12203 100644 --- a/include/boost/wave/util/cpp_iterator.hpp +++ b/include/boost/wave/util/cpp_iterator.hpp @@ -959,15 +959,14 @@ namespace impl { return true; // no more significant tokens on this line } - if (!IS_CATEGORY(id, WhiteSpaceTokenType)) // ccomment, cppcomment, space, space2, placeholder + if (!IS_CATEGORY(id, WhiteSpaceTokenType)) break; // this token gets skipped if (call_hook) util::impl::call_skipped_token_hook(ctx, *it); } - return need_no_newline_at_end_of_file(ctx.get_language()) && - ((it == end) || (T_EOF == token_id(*it))); + return need_no_newline_at_end_of_file(ctx.get_language()); } /////////////////////////////////////////////////////////////////////////// From 431c7cdf38a60c76558b7d7992b1c6b72153eebe Mon Sep 17 00:00:00 2001 From: Jeff Trull Date: Sat, 1 Jan 2022 22:23:13 -0800 Subject: [PATCH 3/3] Clean up some comments --- test/testwave/default_hooks.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/testwave/default_hooks.cpp b/test/testwave/default_hooks.cpp index 28a8c915d..16e5678bf 100644 --- a/test/testwave/default_hooks.cpp +++ b/test/testwave/default_hooks.cpp @@ -36,7 +36,7 @@ int main() "#ifndef __FILE__\n" "int cc;\n" "#endif\n" - "char c3;\n" // #LINE 13 + "char c3;\n" // #LINE 12 ); using namespace boost::wave; @@ -60,7 +60,7 @@ int main() {T_SPACE, " "}, {T_INTLIT, "8"}, {T_SPACE, " "}, - {T_STRINGLIT, nullptr}, // exact filepath is not interesting + {T_STRINGLIT, nullptr}, {T_NEWLINE, "\n"}, {T_CHAR, "char"}, {T_SPACE, " "}, @@ -72,7 +72,7 @@ int main() {T_SPACE, " "}, {T_INTLIT, "12"}, {T_SPACE, " "}, - {T_STRINGLIT, nullptr}, // exact filepath is not interesting + {T_STRINGLIT, nullptr}, {T_NEWLINE, "\n"}, {T_CHAR, "char"}, {T_SPACE, " "},