make sure the language for a TokenList/Token is always determined [skip ci]

firewave · firewave · commit 5f9bf441afa5 · 2024-02-09T00:36:38.000+01:00
diff --git a/lib/clangimport.cpp b/lib/clangimport.cpp
@@ -330,8 +330,12 @@ namespace clangimport {
         void dumpAst(int num = 0, int indent = 0) const;
         void createTokens1(TokenList &tokenList) {
             //dumpAst();
-            if (!tokenList.back())
+            if (!tokenList.back()) {
                 setLocations(tokenList, 0, 1, 1);
+                // FIXME: treat as C++ if no filename (i.e. no lang) is specified for now
+                if (tokenList.getSourceFilePath().empty())
+                    tokenList.setLang(Standards::Language::CPP);
+            }
             else
                 setLocations(tokenList, tokenList.back()->fileIndex(), tokenList.back()->linenr(), 1);
             createTokens(tokenList);
@@ -626,6 +630,7 @@ void clangimport::AstNode::setValueType(Token *tok)
             continue;
 
         TokenList decl(nullptr);
+        decl.setLang(tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
         addTypeTokens(decl, type, tok->scope());
         if (!decl.front())
             break;
diff --git a/lib/cppcheck.cpp b/lib/cppcheck.cpp
@@ -635,9 +635,12 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
 
     try {
         if (mSettings.library.markupFile(filename)) {
+            // this is not a real source file - we just want to tokenize it. treat it as C anyways as the language needs to be determined.
             Tokenizer tokenizer(mSettings, this);
-            if (fileStream)
+            tokenizer.list.setLang(Standards::Language::C);
+            if (fileStream) {
                 tokenizer.list.createTokens(*fileStream, filename);
+            }
             else {
                 std::ifstream in(filename);
                 tokenizer.list.createTokens(in, filename);
@@ -804,7 +807,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
             }
             Tokenizer tokenizer2(mSettings, this);
             std::istringstream istr2(code);
-            tokenizer2.list.createTokens(istr2);
+            tokenizer2.list.createTokens(istr2, Path::identify(*files.begin()));
             executeRules("define", tokenizer2);
         }
 #endif
diff --git a/lib/importproject.cpp b/lib/importproject.cpp
@@ -560,11 +560,11 @@ namespace {
             replaceAll(c, "$(Configuration)", p.configuration);
             replaceAll(c, "$(Platform)", p.platformStr);
 
-            // TODO : Better evaluation
-            Settings s;
-            std::istringstream istr(c);
+            // TODO: evaluate without using the Tokenizer
+            const Settings s;
             Tokenizer tokenizer(s);
-            tokenizer.tokenize(istr,"vcxproj");
+            std::istringstream istr(c);
+            tokenizer.tokenize(istr,"vcxproj.c");
             for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next()) {
                 if (tok->str() == "(" && tok->astOperand1() && tok->astOperand2()) {
                     if (tok->astOperand1()->expressionString() == "Configuration.Contains")
diff --git a/lib/library.cpp b/lib/library.cpp
@@ -53,10 +53,10 @@ static std::vector<std::string> getnames(const char *names)
     return ret;
 }
 
-static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList)
+static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList)
 {
     std::istringstream istr(valid + ',');
-    tokenList.createTokens(istr);
+    tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C);
     for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (Token::Match(tok,"- %num%")) {
             tok->str("-" + tok->strAt(1));
@@ -929,7 +929,7 @@ bool Library::isIntArgValid(const Token *ftok, int argnr, const MathLib::bigint
     if (ac->valid.find('.') != std::string::npos)
         return isFloatArgValid(ftok, argnr, argvalue);
     TokenList tokenList(nullptr);
-    gettokenlistfromvalid(ac->valid, tokenList);
+    gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
     for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (tok->isNumber() && argvalue == MathLib::toBigNumber(tok->str()))
             return true;
@@ -949,7 +949,7 @@ bool Library::isFloatArgValid(const Token *ftok, int argnr, double argvalue) con
     if (!ac || ac->valid.empty())
         return true;
     TokenList tokenList(nullptr);
-    gettokenlistfromvalid(ac->valid, tokenList);
+    gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
     for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (Token::Match(tok, "%num% : %num%") && argvalue >= MathLib::toDoubleNumber(tok->str()) && argvalue <= MathLib::toDoubleNumber(tok->strAt(2)))
             return true;
@@ -1738,13 +1738,14 @@ bool Library::hasAnyTypeCheck(const std::string& typeName) const
 
 std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
                                                  const Settings* settings,
+                                                 bool cpp,
                                                  std::unordered_map<nonneg int, const Token*>* lookupVarId)
 {
     std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(settings);
     {
         const std::string code = "return " + returnValue + ";";
         std::istringstream istr(code);
-        if (!tokenList->createTokens(istr))
+        if (!tokenList->createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
             return nullptr;
     }
 
diff --git a/lib/library.h b/lib/library.h
@@ -607,6 +607,7 @@ CPPCHECKLIB const Library::Container * getLibraryContainer(const Token * tok);
 
 std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
                                                  const Settings* settings,
+                                                 bool cpp,
                                                  std::unordered_map<nonneg int, const Token*>* lookupVarId = nullptr);
 
 /// @}
diff --git a/lib/programmemory.cpp b/lib/programmemory.cpp
@@ -1596,7 +1596,7 @@ namespace {
                                     arg_map[argn] = v;
                                 argn++;
                             }
-                            return evaluateLibraryFunction(arg_map, returnValue, settings);
+                            return evaluateLibraryFunction(arg_map, returnValue, settings, ftok->isCpp());
                         }
                     }
                 }
@@ -1752,15 +1752,16 @@ std::vector<ValueFlow::Value> execute(const Scope* scope, ProgramMemory& pm, con
 
 ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
                                          const std::string& returnValue,
-                                         const Settings* settings)
+                                         const Settings* settings,
+                                         bool cpp)
 {
     thread_local static std::unordered_map<std::string,
                                            std::function<ValueFlow::Value(const std::unordered_map<nonneg int, ValueFlow::Value>& arg)>>
     functions = {};
     if (functions.count(returnValue) == 0) {
 
         std::unordered_map<nonneg int, const Token*> lookupVarId;
-        std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, &lookupVarId);
+        std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, cpp, &lookupVarId);
 
         functions[returnValue] =
             [lookupVarId, expr, settings](const std::unordered_map<nonneg int, ValueFlow::Value>& xargs) {
diff --git a/lib/programmemory.h b/lib/programmemory.h
@@ -211,7 +211,8 @@ ProgramMemory getProgramMemory(const Token* tok, const Token* expr, const ValueF
 
 ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
                                          const std::string& returnValue,
-                                         const Settings* settings);
+                                         const Settings* settings,
+                                         bool cpp);
 
 #endif
 
diff --git a/lib/symboldatabase.cpp b/lib/symboldatabase.cpp
@@ -7491,7 +7491,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                     ValueType valuetype;
                     TokenList tokenList(&mSettings);
                     std::istringstream istr(typestr+";");
-                    tokenList.createTokens(istr);
+                    tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
                     tokenList.simplifyStdType();
                     if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings, mIsCpp)) {
                         valuetype.originalTypeName = typestr;
@@ -7581,7 +7581,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                 }
                 TokenList tokenList(&mSettings);
                 std::istringstream istr(typestr+";");
-                if (tokenList.createTokens(istr)) {
+                if (tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C)) {
                     ValueType vt;
                     tokenList.simplifyPlatformTypes();
                     tokenList.simplifyStdType();
diff --git a/lib/token.cpp b/lib/token.cpp
@@ -2752,5 +2752,6 @@ bool Token::isCpp() const
     if (mTokensFrontBack) {
         return mTokensFrontBack->list.isCPP();
     }
+    assert(false);
     return true; // assume C++ by default
 }
diff --git a/lib/token.h b/lib/token.h
@@ -154,7 +154,7 @@ class CPPCHECKLIB Token {
     // for usage in TestToken only
     Token();
 
-    TokensFrontBack* mTokensFrontBack{};
+    TokensFrontBack* const mTokensFrontBack{};
 
 public:
     Token(const Token &) = delete;
diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp
@@ -17,6 +17,9 @@
  */
 
 //---------------------------------------------------------------------------
+
+#define ASSERT_LANG
+
 #include "tokenlist.h"
 
 #include "astutils.h"
@@ -30,6 +33,9 @@
 #include "standards.h"
 #include "token.h"
 
+#ifdef ASSERT_LANG
+#include <cassert>
+#endif
 #include <cctype>
 #include <exception>
 #include <functional>
@@ -83,7 +89,7 @@ void TokenList::deallocateTokens()
 
 void TokenList::determineCppC()
 {
-    // only try to determine it if it wasn't enforced
+    // only try to determine if it wasn't enforced
     if (mLang == Standards::Language::None) {
         mLang = Path::identify(getSourceFilePath());
     }
@@ -317,8 +323,37 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
 
 bool TokenList::createTokens(std::istream &code, const std::string& file0)
 {
+#ifdef ASSERT_LANG
+    assert(!file0.empty());
+#endif
+
     appendFileIfNew(file0);
 
+    return createTokensInternal(code, file0);
+}
+
+//---------------------------------------------------------------------------
+
+bool TokenList::createTokens(std::istream &code, Standards::Language lang)
+{
+#ifdef ASSERT_LANG
+    assert(lang != Standards::Language::None);
+#endif
+    if (mLang == Standards::Language::None) {
+        mLang = lang;
+    } else {
+#ifdef ASSERT_LANG
+        assert(lang == mLang);
+#endif
+    }
+
+    return createTokensInternal(code, "");
+}
+
+//---------------------------------------------------------------------------
+
+bool TokenList::createTokensInternal(std::istream &code, const std::string& file0)
+{
     simplecpp::OutputList outputList;
     simplecpp::TokenList tokens(code, mFiles, file0, &outputList);
 
@@ -2096,3 +2131,31 @@ bool TokenList::isKeyword(const std::string &str) const
     static const auto& latest_c_keywords = Keywords::getAll(Standards::cstd_t::CLatest);
     return latest_c_keywords.find(str) != latest_c_keywords.end();
 }
+
+bool TokenList::isC() const
+{
+#ifdef ASSERT_LANG
+    assert(mLang != Standards::Language::None);
+#endif
+
+    return mLang == Standards::Language::C;
+}
+
+bool TokenList::isCPP() const
+{
+#ifdef ASSERT_LANG
+    assert(mLang != Standards::Language::None);
+#endif
+
+    return mLang == Standards::Language::CPP;
+}
+
+void TokenList::setLang(Standards::Language lang)
+{
+#ifdef ASSERT_LANG
+    assert(lang != Standards::Language::None);
+    assert(mLang == Standards::Language::None);
+#endif
+
+    mLang = lang;
+}
diff --git a/lib/tokenlist.h b/lib/tokenlist.h
@@ -62,14 +62,12 @@ class CPPCHECKLIB TokenList {
     const std::string& getSourceFilePath() const;
 
     /** Is the code C. Used for bailouts */
-    bool isC() const {
-        return mLang == Standards::Language::C;
-    }
+    bool isC() const;
 
     /** Is the code CPP. Used for bailouts */
-    bool isCPP() const {
-        return mLang == Standards::Language::CPP;
-    }
+    bool isCPP() const;
+
+    void setLang(Standards::Language lang);
 
     /**
      * Delete all tokens in given token list
@@ -105,7 +103,8 @@ class CPPCHECKLIB TokenList {
      * @param code input stream for code
      * @param file0 source file name
      */
-    bool createTokens(std::istream &code, const std::string& file0 = emptyString);
+    bool createTokens(std::istream &code, const std::string& file0);
+    bool createTokens(std::istream &code, Standards::Language lang);
 
     void createTokens(simplecpp::TokenList&& tokenList);
 
@@ -203,6 +202,8 @@ class CPPCHECKLIB TokenList {
 private:
     void determineCppC();
 
+    bool createTokensInternal(std::istream &code, const std::string& file0);
+
     /** Token list */
     TokensFrontBack mTokensFrontBack;
 
diff --git a/lib/valueflow.cpp b/lib/valueflow.cpp
@@ -3808,20 +3808,20 @@ static bool isNotEqual(std::pair<const Token*, const Token*> x, std::pair<const
     start2 = skipCVRefs(start2, y.second);
     return !(start1 == x.second && start2 == y.second);
 }
-static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y)
+static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y, bool cpp)
 {
     TokenList tokenList(nullptr);
     std::istringstream istr(y);
-    tokenList.createTokens(istr);
+    tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C);
     return isNotEqual(x, std::make_pair(tokenList.front(), tokenList.back()));
 }
-static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y)
+static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y, bool cpp)
 {
     if (y == nullptr)
         return false;
     if (y->originalTypeName.empty())
         return false;
-    return isNotEqual(x, y->originalTypeName);
+    return isNotEqual(x, y->originalTypeName, cpp);
 }
 
 static bool isDifferentType(const Token* src, const Token* dst)
@@ -3836,9 +3836,9 @@ static bool isDifferentType(const Token* src, const Token* dst)
         std::pair<const Token*, const Token*> parentdecl = Token::typeDecl(dst);
         if (isNotEqual(decl, parentdecl))
             return true;
-        if (isNotEqual(decl, dst->valueType()))
+        if (isNotEqual(decl, dst->valueType(), dst->isCpp()))
             return true;
-        if (isNotEqual(parentdecl, src->valueType()))
+        if (isNotEqual(parentdecl, src->valueType(), src->isCpp()))
             return true;
     }
     return false;
@@ -7668,7 +7668,7 @@ static void valueFlowLibraryFunction(Token *tok, const std::string &returnValue,
     if (returnValue.find("arg") != std::string::npos && argValues.empty())
         return;
     productParams(settings, argValues, [&](const std::unordered_map<nonneg int, ValueFlow::Value>& arg) {
-        ValueFlow::Value value = evaluateLibraryFunction(arg, returnValue, &settings);
+        ValueFlow::Value value = evaluateLibraryFunction(arg, returnValue, &settings, tok->isCpp());
         if (value.isUninitValue())
             return;
         ValueFlow::Value::ValueKind kind = ValueFlow::Value::ValueKind::Known;
@@ -9162,11 +9162,11 @@ static bool getMinMaxValues(const ValueType *vt, const Platform &platform, MathL
     return true;
 }
 
-static bool getMinMaxValues(const std::string &typestr, const Settings &settings, MathLib::bigint &minvalue, MathLib::bigint &maxvalue)
+static bool getMinMaxValues(const std::string &typestr, const Settings &settings, bool cpp, MathLib::bigint &minvalue, MathLib::bigint &maxvalue)
 {
     TokenList typeTokens(&settings);
     std::istringstream istr(typestr+";");
-    if (!typeTokens.createTokens(istr))
+    if (!typeTokens.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
         return false;
     typeTokens.simplifyPlatformTypes();
     typeTokens.simplifyStdType();
@@ -9288,7 +9288,7 @@ static void valueFlowUnknownFunctionReturn(TokenList &tokenlist, const Settings
         // Get min/max values for return type
         const std::string &typestr = settings.library.returnValueType(tok->previous());
         MathLib::bigint minvalue, maxvalue;
-        if (!getMinMaxValues(typestr, settings, minvalue, maxvalue))
+        if (!getMinMaxValues(typestr, settings, tok->isCpp(), minvalue, maxvalue))
             continue;
 
         for (MathLib::bigint value : unknownValues) {
diff --git a/test/testlibrary.cpp b/test/testlibrary.cpp
diff --git a/test/testsimplifytypedef.cpp b/test/testsimplifytypedef.cpp
diff --git a/test/testtoken.cpp b/test/testtoken.cpp

Original file line number	Diff line number	Diff line change
`@@ -1596,7 +1596,7 @@ namespace {`
`1596`	`1596`	`arg_map[argn] = v;`
`1597`	`1597`	`argn++;`
`1598`	`1598`	`}`
`1599`		`- return evaluateLibraryFunction(arg_map, returnValue, settings);`
	`1599`	`+ return evaluateLibraryFunction(arg_map, returnValue, settings, ftok->isCpp());`
`1600`	`1600`	`}`
`1601`	`1601`	`}`
`1602`	`1602`	`}`
`@@ -1752,15 +1752,16 @@ std::vector<ValueFlow::Value> execute(const Scope* scope, ProgramMemory& pm, con`
`1752`	`1752`
`1753`	`1753`	`ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,`
`1754`	`1754`	`const std::string& returnValue,`
`1755`		`- const Settings* settings)`
	`1755`	`+ const Settings* settings,`
	`1756`	`+ bool cpp)`
`1756`	`1757`	`{`
`1757`	`1758`	`thread_local static std::unordered_map<std::string,`
`1758`	`1759`	`std::function<ValueFlow::Value(const std::unordered_map<nonneg int, ValueFlow::Value>& arg)>>`
`1759`	`1760`	`functions = {};`
`1760`	`1761`	`if (functions.count(returnValue) == 0) {`
`1761`	`1762`
`1762`	`1763`	`std::unordered_map<nonneg int, const Token*> lookupVarId;`
`1763`		`- std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, &lookupVarId);`
	`1764`	`+ std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, cpp, &lookupVarId);`
`1764`	`1765`
`1765`	`1766`	`functions[returnValue] =`
`1766`	`1767`	`[lookupVarId, expr, settings](const std::unordered_map<nonneg int, ValueFlow::Value>& xargs) {`
Original file line number	Diff line number	Diff line change
`@@ -2752,5 +2752,6 @@ bool Token::isCpp() const`
`2752`	`2752`	`if (mTokensFrontBack) {`
`2753`	`2753`	`return mTokensFrontBack->list.isCPP();`
`2754`	`2754`	`}`
	`2755`	`+ assert(false);`
`2755`	`2756`	`return true; // assume C++ by default`
`2756`	`2757`	`}`