Skip to content

Commit 7bfcce6

Browse files
firewavefirewave
committed
make sure the language for a TokenList/Token is always determined
1 parent 0c82876 commit 7bfcce6

15 files changed

+134
-58
lines changed

‎lib/clangimport.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,8 +330,12 @@ namespace clangimport {
330330
void dumpAst(int num = 0, int indent = 0) const;
331331
void createTokens1(TokenList &tokenList) {
332332
//dumpAst();
333-
if (!tokenList.back())
333+
if (!tokenList.back()) {
334334
setLocations(tokenList, 0, 1, 1);
335+
// FIXME: treat as C++ if no filename (i.e. no lang) is specified for now
336+
if (tokenList.getSourceFilePath().empty())
337+
tokenList.setLang(Standards::Language::CPP);
338+
}
335339
else
336340
setLocations(tokenList, tokenList.back()->fileIndex(), tokenList.back()->linenr(), 1);
337341
createTokens(tokenList);
@@ -626,6 +630,7 @@ void clangimport::AstNode::setValueType(Token *tok)
626630
continue;
627631

628632
TokenList decl(nullptr);
633+
decl.setLang(tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
629634
addTypeTokens(decl, type, tok->scope());
630635
if (!decl.front())
631636
break;

‎lib/cppcheck.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -636,9 +636,12 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
636636
if (mSettings.checks.isEnabled(Checks::unusedFunction) &&
637637
mSettings.useSingleJob() &&
638638
mSettings.buildDir.empty()) {
639+
// this is not a real source file - we just want to tokenize it. treat it as C anyways as the language needs to be determined.
639640
Tokenizer tokenizer(mSettings, this);
640-
if (fileStream)
641+
tokenizer.list.setLang(Standards::Language::C);
642+
if (fileStream) {
641643
tokenizer.list.createTokens(*fileStream, filename);
644+
}
642645
else {
643646
std::ifstream in(filename);
644647
tokenizer.list.createTokens(in, filename);
@@ -806,7 +809,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
806809
}
807810
Tokenizer tokenizer2(mSettings, this);
808811
std::istringstream istr2(code);
809-
tokenizer2.list.createTokens(istr2);
812+
tokenizer2.list.createTokens(istr2, Path::identify(*files.begin()));
810813
executeRules("define", tokenizer2);
811814
}
812815
#endif

‎lib/importproject.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -560,11 +560,11 @@ namespace {
560560
replaceAll(c, "$(Configuration)", p.configuration);
561561
replaceAll(c, "$(Platform)", p.platformStr);
562562

563-
// TODO : Better evaluation
564-
Settings s;
565-
std::istringstream istr(c);
563+
// TODO: evaluate without using the Tokenizer
564+
const Settings s;
566565
Tokenizer tokenizer(s);
567-
tokenizer.tokenize(istr,"vcxproj");
566+
std::istringstream istr(c);
567+
tokenizer.tokenize(istr,"vcxproj.c");
568568
for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next()) {
569569
if (tok->str() == "(" && tok->astOperand1() && tok->astOperand2()) {
570570
if (tok->astOperand1()->expressionString() == "Configuration.Contains")

‎lib/library.cpp‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,10 @@ static std::vector<std::string> getnames(const char *names)
5353
return ret;
5454
}
5555

56-
static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList)
56+
static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList)
5757
{
5858
std::istringstream istr(valid + ',');
59-
tokenList.createTokens(istr);
59+
tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C);
6060
for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
6161
if (Token::Match(tok,"- %num%")) {
6262
tok->str("-" + tok->strAt(1));
@@ -929,7 +929,7 @@ bool Library::isIntArgValid(const Token *ftok, int argnr, const MathLib::bigint
929929
if (ac->valid.find('.') != std::string::npos)
930930
return isFloatArgValid(ftok, argnr, argvalue);
931931
TokenList tokenList(nullptr);
932-
gettokenlistfromvalid(ac->valid, tokenList);
932+
gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
933933
for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
934934
if (tok->isNumber() && argvalue == MathLib::toBigNumber(tok->str()))
935935
return true;
@@ -949,7 +949,7 @@ bool Library::isFloatArgValid(const Token *ftok, int argnr, double argvalue) con
949949
if (!ac || ac->valid.empty())
950950
return true;
951951
TokenList tokenList(nullptr);
952-
gettokenlistfromvalid(ac->valid, tokenList);
952+
gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
953953
for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
954954
if (Token::Match(tok, "%num% : %num%") && argvalue >= MathLib::toDoubleNumber(tok->str()) && argvalue <= MathLib::toDoubleNumber(tok->strAt(2)))
955955
return true;
@@ -1738,13 +1738,14 @@ bool Library::hasAnyTypeCheck(const std::string& typeName) const
17381738

17391739
std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
17401740
const Settings* settings,
1741+
bool cpp,
17411742
std::unordered_map<nonneg int, const Token*>* lookupVarId)
17421743
{
17431744
std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(settings);
17441745
{
17451746
const std::string code = "return " + returnValue + ";";
17461747
std::istringstream istr(code);
1747-
if (!tokenList->createTokens(istr))
1748+
if (!tokenList->createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
17481749
return nullptr;
17491750
}
17501751

‎lib/library.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -607,6 +607,7 @@ CPPCHECKLIB const Library::Container * getLibraryContainer(const Token * tok);
607607

608608
std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
609609
const Settings* settings,
610+
bool cpp,
610611
std::unordered_map<nonneg int, const Token*>* lookupVarId = nullptr);
611612

612613
/// @}

‎lib/programmemory.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1596,7 +1596,7 @@ namespace {
15961596
arg_map[argn] = v;
15971597
argn++;
15981598
}
1599-
return evaluateLibraryFunction(arg_map, returnValue, settings);
1599+
return evaluateLibraryFunction(arg_map, returnValue, settings, ftok->isCpp());
16001600
}
16011601
}
16021602
}
@@ -1752,15 +1752,16 @@ std::vector<ValueFlow::Value> execute(const Scope* scope, ProgramMemory& pm, con
17521752

17531753
ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
17541754
const std::string& returnValue,
1755-
const Settings* settings)
1755+
const Settings* settings,
1756+
bool cpp)
17561757
{
17571758
thread_local static std::unordered_map<std::string,
17581759
std::function<ValueFlow::Value(const std::unordered_map<nonneg int, ValueFlow::Value>& arg)>>
17591760
functions = {};
17601761
if (functions.count(returnValue) == 0) {
17611762

17621763
std::unordered_map<nonneg int, const Token*> lookupVarId;
1763-
std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, &lookupVarId);
1764+
std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, cpp, &lookupVarId);
17641765

17651766
functions[returnValue] =
17661767
[lookupVarId, expr, settings](const std::unordered_map<nonneg int, ValueFlow::Value>& xargs) {

‎lib/programmemory.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,8 @@ ProgramMemory getProgramMemory(const Token* tok, const Token* expr, const ValueF
211211

212212
ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
213213
const std::string& returnValue,
214-
const Settings* settings);
214+
const Settings* settings,
215+
bool cpp);
215216

216217
#endif
217218

‎lib/symboldatabase.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7573,7 +7573,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
75737573
ValueType valuetype;
75747574
TokenList tokenList(&mSettings);
75757575
std::istringstream istr(typestr+";");
7576-
tokenList.createTokens(istr);
7576+
tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
75777577
tokenList.simplifyStdType();
75787578
if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings, mIsCpp)) {
75797579
valuetype.originalTypeName = typestr;
@@ -7663,7 +7663,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
76637663
}
76647664
TokenList tokenList(&mSettings);
76657665
std::istringstream istr(typestr+";");
7666-
if (tokenList.createTokens(istr)) {
7666+
if (tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C)) {
76677667
ValueType vt;
76687668
tokenList.simplifyPlatformTypes();
76697669
tokenList.simplifyStdType();

‎lib/token.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1479,7 +1479,6 @@ class CPPCHECKLIB Token {
14791479
mImpl->mDebug = td;
14801480
}
14811481

1482-
/** defaults to C++ if it cannot be determined */
14831482
bool isCpp() const;
14841483
};
14851484

‎lib/tokenlist.cpp‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
*/
1818

1919
//---------------------------------------------------------------------------
20+
21+
//#define N_ASSERT_LANG
22+
2023
#include "tokenlist.h"
2124

2225
#include "astutils.h"
@@ -30,6 +33,12 @@
3033
#include "standards.h"
3134
#include "token.h"
3235

36+
#ifndef N_ASSERT_LANG
37+
#include <cassert>
38+
#define ASSERT_LANG(x) assert(x)
39+
#else
40+
#define ASSERT_LANG(x)
41+
#endif
3342
#include <cctype>
3443
#include <exception>
3544
#include <functional>
@@ -83,7 +92,7 @@ void TokenList::deallocateTokens()
8392

8493
void TokenList::determineCppC()
8594
{
86-
// only try to determine it if it wasn't enforced
95+
// only try to determine if it wasn't enforced
8796
if (mLang == Standards::Language::None) {
8897
mLang = Path::identify(getSourceFilePath());
8998
}
@@ -317,8 +326,31 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
317326

318327
bool TokenList::createTokens(std::istream &code, const std::string& file0)
319328
{
329+
ASSERT_LANG(!file0.empty());
330+
320331
appendFileIfNew(file0);
321332

333+
return createTokensInternal(code, file0);
334+
}
335+
336+
//---------------------------------------------------------------------------
337+
338+
bool TokenList::createTokens(std::istream &code, Standards::Language lang)
339+
{
340+
ASSERT_LANG(lang != Standards::Language::None);
341+
if (mLang == Standards::Language::None) {
342+
mLang = lang;
343+
} else {
344+
ASSERT_LANG(lang == mLang);
345+
}
346+
347+
return createTokensInternal(code, "");
348+
}
349+
350+
//---------------------------------------------------------------------------
351+
352+
bool TokenList::createTokensInternal(std::istream &code, const std::string& file0)
353+
{
322354
simplecpp::OutputList outputList;
323355
simplecpp::TokenList tokens(code, mFiles, file0, &outputList);
324356

@@ -2096,3 +2128,33 @@ bool TokenList::isKeyword(const std::string &str) const
20962128
static const auto& latest_c_keywords = Keywords::getAll(Standards::cstd_t::CLatest);
20972129
return latest_c_keywords.find(str) != latest_c_keywords.end();
20982130
}
2131+
2132+
bool TokenList::isC() const
2133+
{
2134+
ASSERT_LANG(mLang != Standards::Language::None);
2135+
2136+
// TODO: remove the fallback
2137+
if (mLang == Standards::Language::None)
2138+
return false; // treat as C++ by default
2139+
2140+
return mLang == Standards::Language::C;
2141+
}
2142+
2143+
bool TokenList::isCPP() const
2144+
{
2145+
ASSERT_LANG(mLang != Standards::Language::None);
2146+
2147+
// TODO: remove the fallback
2148+
if (mLang == Standards::Language::None)
2149+
return true; // treat as C++ by default
2150+
2151+
return mLang == Standards::Language::CPP;
2152+
}
2153+
2154+
void TokenList::setLang(Standards::Language lang)
2155+
{
2156+
ASSERT_LANG(lang != Standards::Language::None);
2157+
ASSERT_LANG(mLang == Standards::Language::None);
2158+
2159+
mLang = lang;
2160+
}

0 commit comments

Comments
 (0)