Allow reexpansion of currently expanding macros during argument evaluation

CMakeLists.txt

@@ -5,6 +5,11 @@ option(DISABLE_CPP03_SYNTAX_CHECK "Disable the C++03 syntax check." OFF)
 
 include(CheckCXXCompilerFlag)
 
+if (WIN32)
+    # prevent simplifyPath_cppcheck() from wasting time on looking for a hypothetical network host
+    add_definitions(-DUNCHOST=$ENV{COMPUTERNAME})
+endif()
+
 function(add_compile_options_safe FLAG)
     string(MAKE_C_IDENTIFIER "HAS_CXX_FLAG${FLAG}" mangled_flag)
     check_cxx_compiler_flag(${FLAG} ${mangled_flag})

run-tests.py

@@ -71,7 +71,6 @@ def cleanup(out):
          'c99-6_10_3_4_p6.c',
          'expr_usual_conversions.c', # condition is true: 4U - 30 >= 0
          'stdint.c',
-         'stringize_misc.c',
 
          # GCC..
          'diagnostic-pragma-1.c',

simplecpp.cpp

@@ -102,8 +102,6 @@ static const simplecpp::TokenString ONCE("once");
 
 static const simplecpp::TokenString HAS_INCLUDE("__has_include");
 
-static const simplecpp::TokenString INNER_COMMA(",,");
-
 template<class T> static std::string toString(T t)
 {
     // NOLINTNEXTLINE(misc-const-correctness) - false positive
@@ -888,7 +886,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename,
             }
 
             if (prefix.empty())
-                push_back(new Token(s, location)); // push string without newlines
+                push_back(new Token(s, location, isspace(stream.peekChar()))); // push string without newlines
             else
                 back()->setstr(prefix + s);
 
@@ -918,7 +916,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename,
             }
         }
 
-        push_back(new Token(currentToken, location));
+        push_back(new Token(currentToken, location, isspace(stream.peekChar())));
 
         if (multiline)
             location.col += currentToken.size();
@@ -1548,9 +1546,9 @@ namespace simplecpp {
                 // Copy macro call to a new tokenlist with no linebreaks
                 const Token * const rawtok1 = rawtok;
                 TokenList rawtokens2(inputFiles);
-                rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location));
+                rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead));
                 rawtok = rawtok->next;
-                rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location));
+                rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead));
                 rawtok = rawtok->next;
                 int par = 1;
                 while (rawtok && par > 0) {
@@ -1560,13 +1558,10 @@ namespace simplecpp {
                         --par;
                     else if (rawtok->op == '#' && !sameline(rawtok->previous, rawtok))
                         throw Error(rawtok->location, "it is invalid to use a preprocessor directive as macro parameter");
-                    rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location));
+                    rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead));
                     rawtok = rawtok->next;
                 }
-                bool first = true;
-                if (valueToken && valueToken->str() == rawtok1->str())
-                    first = false;
-                if (expand(&output2, rawtok1->location, rawtokens2.cfront(), macros, expandedmacros, first))
+                if (expand(&output2, rawtok1->location, rawtokens2.cfront(), macros, expandedmacros))
                     rawtok = rawtok1->next;
             } else {
                 rawtok = expand(&output2, rawtok->location, rawtok, macros, expandedmacros);
@@ -1622,10 +1617,6 @@ namespace simplecpp {
                 rawtok = rawtok2->next;
             }
             output->takeTokens(output2);
-            for (Token* tok = output->front(); tok; tok = tok->next) {
-                if (tok->str() == INNER_COMMA)
-                    tok->setstr(",");
-            }
             return rawtok;
         }
 
@@ -1792,28 +1783,12 @@ namespace simplecpp {
                     // A##B => AB
                     tok = expandHashHash(tokens, rawloc, tok, macros, expandedmacros, parametertokens);
                 } else if (tok->op == '#' && sameline(tok, tok->next) && tok->next->op != '#') {
-                    tok = expandHash(tokens, rawloc, tok, macros, expandedmacros, parametertokens);
+                    tok = expandHash(tokens, rawloc, tok, expandedmacros, parametertokens);
                 } else {
                     if (!expandArg(tokens, tok, rawloc, macros, expandedmacros, parametertokens)) {
-                        bool expanded = false;
-                        const MacroMap::const_iterator it = macros.find(tok->str());
-                        if (it != macros.end() && expandedmacros.find(tok->str()) == expandedmacros.end()) {
-                            const Macro &m = it->second;
-                            if (!m.functionLike()) {
-                                Token* mtok = tokens->back();
-                                m.expand(tokens, rawloc, tok, macros, expandedmacros);
-                                for (mtok = mtok->next; mtok; mtok = mtok->next) {
-                                    if (mtok->op == ',')
-                                        mtok->setstr(INNER_COMMA);
-                                }
-                                expanded = true;
-                            }
-                        }
-                        if (!expanded) {
-                            tokens->push_back(new Token(*tok));
-                            if (tok->macro.empty() && (par > 0 || tok->str() != "("))
-                                tokens->back()->macro = name();
-                        }
+                        tokens->push_back(new Token(*tok));
+                        if (tok->macro.empty() && (par > 0 || tok->str() != "("))
+                            tokens->back()->macro = name();
                     }
 
                     if (tok->op == '(')
@@ -1831,10 +1806,8 @@ namespace simplecpp {
             return sameline(lpar,tok) ? tok : nullptr;
         }
 
-        const Token * expand(TokenList * const output, const Location &loc, const Token * const nameTokInst, const MacroMap &macros, std::set<TokenString> expandedmacros, bool first=false) const {
-
-            if (!first)
-                expandedmacros.insert(nameTokInst->str());
+        const Token * expand(TokenList * const output, const Location &loc, const Token * const nameTokInst, const MacroMap &macros, std::set<TokenString> expandedmacros) const {
+            expandedmacros.insert(nameTokInst->str());
 
             usageList.push_back(loc);
 
@@ -1917,6 +1890,14 @@ namespace simplecpp {
                     if (sameline(tok, tok->next) && tok->next && tok->next->op == '#' && tok->next->next && tok->next->next->op == '#') {
                         if (!sameline(tok, tok->next->next->next))
                             throw invalidHashHash::unexpectedNewline(tok->location, name());
+                        if (variadic && tok->op == ',' && tok->next->next->next->str() == args.back()) {
+                            Token *const comma = newMacroToken(tok->str(), loc, isReplaced(expandedmacros), tok);
+                            output->push_back(comma);
+                            tok = expandToken(output, loc, tok->next->next->next, macros, expandedmacros, parametertokens2);
+                            if (output->back() == comma)
+                                output->deleteToken(comma);
+                            continue;
+                        }
                         TokenList new_output(files);
                         if (!expandArg(&new_output, tok, parametertokens2))
                             output->push_back(newMacroToken(tok->str(), loc, isReplaced(expandedmacros), tok));
@@ -1961,7 +1942,7 @@ namespace simplecpp {
                     tok = expandHashHash(output, loc, tok->previous, macros, expandedmacros, parametertokens2);
                 } else {
                     // #123 => "123"
-                    tok = expandHash(output, loc, tok->previous, macros, expandedmacros, parametertokens2);
+                    tok = expandHash(output, loc, tok->previous, expandedmacros, parametertokens2);
                 }
             }
 
@@ -2138,14 +2119,17 @@ namespace simplecpp {
                 return true;
             for (const Token *partok = parametertokens[argnr]->next; partok != parametertokens[argnr + 1U];) {
                 const MacroMap::const_iterator it = macros.find(partok->str());
-                if (it != macros.end() && !partok->isExpandedFrom(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end()))
-                    partok = it->second.expand(output, loc, partok, macros, expandedmacros);
-                else {
+                if (it != macros.end() && !partok->isExpandedFrom(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) {
+                    const std::set<TokenString> expandedmacros2; // temporary amnesia to allow reexpansion of currently expanding macros during argument evaluation
+                    partok = it->second.expand(output, loc, partok, macros, expandedmacros2);
+                } else {
                     output->push_back(newMacroToken(partok->str(), loc, isReplaced(expandedmacros), partok));
                     output->back()->macro = partok->macro;
                     partok = partok->next;
                 }
             }
+            if (tok->whitespaceahead && output->back())
+                output->back()->whitespaceahead = true;
             return true;
         }
 
@@ -2154,18 +2138,22 @@ namespace simplecpp {
          * @param output  destination tokenlist
          * @param loc     location for expanded token
          * @param tok     The # token
-         * @param macros  all macros
          * @param expandedmacros   set with expanded macros, with this macro
          * @param parametertokens  parameters given when expanding this macro
          * @return token after the X
          */
-        const Token *expandHash(TokenList *output, const Location &loc, const Token *tok, const MacroMap &macros, const std::set<TokenString> &expandedmacros, const std::vector<const Token*> &parametertokens) const {
+        const Token *expandHash(TokenList *output, const Location &loc, const Token *tok, const std::set<TokenString> &expandedmacros, const std::vector<const Token*> &parametertokens) const {
             TokenList tokenListHash(files);
-            tok = expandToken(&tokenListHash, loc, tok->next, macros, expandedmacros, parametertokens);
+            const MacroMap macros2; // temporarily bypass macro expansion
+            tok = expandToken(&tokenListHash, loc, tok->next, macros2, expandedmacros, parametertokens);
             std::ostringstream ostr;
             ostr << '\"';
-            for (const Token *hashtok = tokenListHash.cfront(); hashtok; hashtok = hashtok->next)
+            for (const Token *hashtok = tokenListHash.cfront(), *next; hashtok; hashtok = next) {
+                next = hashtok->next;
                 ostr << hashtok->str();
+                if (next && hashtok->whitespaceahead)
+                    ostr << ' ';
+            }
             ostr << '\"';
             output->push_back(newMacroToken(escapeString(ostr.str()), loc, isReplaced(expandedmacros)));
             return tok;

simplecpp.h

@@ -94,13 +94,13 @@ namespace simplecpp {
      */
     class SIMPLECPP_LIB Token {
     public:
-        Token(const TokenString &s, const Location &loc) :
-            location(loc), previous(nullptr), next(nullptr), string(s) {
+        Token(const TokenString &s, const Location &loc, bool wsahead = false) :
+            whitespaceahead(wsahead), location(loc), previous(nullptr), next(nullptr), string(s) {
             flags();
         }
 
         Token(const Token &tok) :
-            macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), location(tok.location), previous(nullptr), next(nullptr), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {
+            macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), previous(nullptr), next(nullptr), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {
         }
 
         void flags() {
@@ -132,6 +132,7 @@ namespace simplecpp {
         bool comment;
         bool name;
         bool number;
+        bool whitespaceahead;
         Location location;
         Token *previous;
         Token *next;
@@ -153,6 +154,8 @@ namespace simplecpp {
         void setExpandedFrom(const Token *tok, const Macro* m) {
             mExpandedFrom = tok->mExpandedFrom;
             mExpandedFrom.insert(m);
+            if (tok->whitespaceahead)
+                whitespaceahead = true;
         }
         bool isExpandedFrom(const Macro* m) const {
             return mExpandedFrom.find(m) != mExpandedFrom.end();

test.cpp

@@ -16,6 +16,9 @@
 #include <string>
 #include <vector>
 
+#define STRINGIZE_(x) #x
+#define STRINGIZE(x) STRINGIZE_(x)
+
 static int numberOfFailedAssertions = 0;
 
 #define ASSERT_EQUALS(expected, actual)  (assertEquals((expected), (actual), __LINE__))
@@ -44,7 +47,7 @@ static int assertEquals(const std::string &expected, const std::string &actual,
     return (expected == actual);
 }
 
-static int assertEquals(const unsigned int &expected, const unsigned int &actual, int line)
+static int assertEquals(const long long &expected, const long long &actual, int line)
 {
     return assertEquals(std::to_string(expected), std::to_string(actual), line);
 }
@@ -717,6 +720,17 @@ static void define_define_11()
     ASSERT_EQUALS("\n\n\n\nP2DIR ;", preprocess(code));
 }
 
+static void define_define_11a()
+{
+    const char code[] = "#define A_B_C              0x1\n"
+                        "#define A_ADDRESS          0x00001000U\n"
+                        "#define A                  ((uint32_t ) A_ADDRESS)\n"
+                        "#define CONCAT(x, y, z)    x ## _ ## y ## _ ## z\n"
+                        "#define TEST_MACRO         CONCAT(A, B, C)\n"
+                        "TEST_MACRO\n";
+    ASSERT_EQUALS("\n\n\n\n\n0x1", preprocess(code));
+}
+
 static void define_define_12()
 {
     const char code[] = "#define XY(Z)  Z\n"
@@ -1019,6 +1033,17 @@ static void hash()
                   preprocess("#define A(x)  (x)\n"
                              "#define B(x)  A(#x)\n"
                              "B(123)"));
+
+    ASSERT_EQUALS("\n\nprintf ( \"bar(3)\" \"\\n\" ) ;",
+                  preprocess("#define bar(x) x % 2\n"
+                             "#define foo(x) printf(#x \"\\n\")\n"
+                             "foo(bar(3));"));
+
+    ASSERT_EQUALS("\n\n\n\"Y Y\"",
+                  preprocess("#define X(x,y)    x y\n"
+                             "#define STR_(x)   #x\n"
+                             "#define STR(x)    STR_(x)\n"
+                             "STR(X(Y,Y))"));
 }
 
 static void hashhash1()   // #4703
@@ -1058,6 +1083,16 @@ static void hashhash4()    // nonstandard gcc/clang extension for empty varargs
     ASSERT_EQUALS("\n\na ( 1 ) ;", preprocess(code));
 }
 
+static void hashhash4a()
+{
+    const char code[] = "#define GETMYID(a) ((a))+1\n"
+                        "#define FIGHT_FOO(c, ...) foo(c, ##__VA_ARGS__)\n"
+                        "#define FIGHT_BAR(c, args...) bar(c, ##args)\n"
+                        "FIGHT_FOO(1, GETMYID(a));\n"
+                        "FIGHT_BAR(1, GETMYID(b));";
+    ASSERT_EQUALS("\n\n\nfoo ( 1 , ( ( a ) ) + 1 ) ;\nbar ( 1 , ( ( b ) ) + 1 ) ;", preprocess(code));
+}
+
 static void hashhash5()
 {
     ASSERT_EQUALS("x1", preprocess("x##__LINE__"));
@@ -2567,8 +2602,8 @@ static void simplifyPath_cppcheck()
     ASSERT_EQUALS("src/", simplecpp::simplifyPath("src/abc/../"));
 
     // Handling of UNC paths on Windows
-    ASSERT_EQUALS("//src/test.cpp", simplecpp::simplifyPath("//src/test.cpp"));
-    ASSERT_EQUALS("//src/test.cpp", simplecpp::simplifyPath("///src/test.cpp"));
+    ASSERT_EQUALS("//" STRINGIZE(UNCHOST) "/test.cpp", simplecpp::simplifyPath("//" STRINGIZE(UNCHOST) "/test.cpp"));
+    ASSERT_EQUALS("//" STRINGIZE(UNCHOST) "/test.cpp", simplecpp::simplifyPath("///" STRINGIZE(UNCHOST) "/test.cpp"));
 }
 
 static void simplifyPath_New()
@@ -2855,6 +2890,7 @@ int main(int argc, char **argv)
     TEST_CASE(define_define_9); // line break in nested macro call
     TEST_CASE(define_define_10);
     TEST_CASE(define_define_11);
+    TEST_CASE(define_define_11a);
     TEST_CASE(define_define_12); // expand result of ##
     TEST_CASE(define_define_13);
     TEST_CASE(define_define_14);
@@ -2892,6 +2928,7 @@ int main(int argc, char **argv)
     TEST_CASE(hashhash2);
     TEST_CASE(hashhash3);
     TEST_CASE(hashhash4);
+    TEST_CASE(hashhash4a); // #66, #130
     TEST_CASE(hashhash5);
     TEST_CASE(hashhash6);
     TEST_CASE(hashhash7); // # ## #  (C standard; 6.10.3.3.p4)