Skip to content

Commit 7abd2f3

Browse files
firewavefirewave
committed
make sure the language for a TokenList/Token is always determined [skip ci]
1 parent 1cf85c0 commit 7abd2f3

15 files changed

+137
-60
lines changed

‎lib/clangimport.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,8 +330,12 @@ namespace clangimport {
330330
void dumpAst(int num = 0, int indent = 0) const;
331331
void createTokens1(TokenList &tokenList) {
332332
//dumpAst();
333-
if (!tokenList.back())
333+
if (!tokenList.back()) {
334334
setLocations(tokenList, 0, 1, 1);
335+
// FIXME: treat as C++ if no filename (i.e. no lang) is specified for now
336+
if (tokenList.getSourceFilePath().empty())
337+
tokenList.setLang(Standards::Language::CPP);
338+
}
335339
else
336340
setLocations(tokenList, tokenList.back()->fileIndex(), tokenList.back()->linenr(), 1);
337341
createTokens(tokenList);
@@ -626,6 +630,7 @@ void clangimport::AstNode::setValueType(Token *tok)
626630
continue;
627631

628632
TokenList decl(nullptr);
633+
decl.setLang(tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
629634
addTypeTokens(decl, type, tok->scope());
630635
if (!decl.front())
631636
break;

‎lib/cppcheck.cpp‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -633,10 +633,14 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
633633

634634
try {
635635
if (mSettings.library.markupFile(filename)) {
636-
if (mSettings.isUnusedFunctionCheckEnabled() && mSettings.buildDir.empty()) {
636+
if (mSettings.isUnusedFunctionCheckEnabled() &&
637+
mSettings.buildDir.empty()) {
638+
// this is not a real source file - we just want to tokenize it. treat it as C anyways as the language needs to be determined.
637639
Tokenizer tokenizer(mSettings, this);
638-
if (fileStream)
640+
tokenizer.list.setLang(Standards::Language::C);
641+
if (fileStream) {
639642
tokenizer.list.createTokens(*fileStream, filename);
643+
}
640644
else {
641645
std::ifstream in(filename);
642646
tokenizer.list.createTokens(in, filename);
@@ -804,7 +808,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
804808
}
805809
Tokenizer tokenizer2(mSettings, this);
806810
std::istringstream istr2(code);
807-
tokenizer2.list.createTokens(istr2);
811+
tokenizer2.list.createTokens(istr2, Path::identify(*files.begin()));
808812
executeRules("define", tokenizer2);
809813
}
810814
#endif

‎lib/importproject.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -560,11 +560,11 @@ namespace {
560560
replaceAll(c, "$(Configuration)", p.configuration);
561561
replaceAll(c, "$(Platform)", p.platformStr);
562562

563-
// TODO : Better evaluation
564-
Settings s;
565-
std::istringstream istr(c);
563+
// TODO: evaluate without using the Tokenizer
564+
const Settings s;
566565
Tokenizer tokenizer(s);
567-
tokenizer.tokenize(istr,"vcxproj");
566+
std::istringstream istr(c);
567+
tokenizer.tokenize(istr,"vcxproj.c");
568568
for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next()) {
569569
if (tok->str() == "(" && tok->astOperand1() && tok->astOperand2()) {
570570
if (tok->astOperand1()->expressionString() == "Configuration.Contains")

‎lib/library.cpp‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,10 @@ static std::vector<std::string> getnames(const char *names)
5353
return ret;
5454
}
5555

56-
static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList)
56+
static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList)
5757
{
5858
std::istringstream istr(valid + ',');
59-
tokenList.createTokens(istr);
59+
tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C);
6060
for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
6161
if (Token::Match(tok,"- %num%")) {
6262
tok->str("-" + tok->strAt(1));
@@ -929,7 +929,7 @@ bool Library::isIntArgValid(const Token *ftok, int argnr, const MathLib::bigint
929929
if (ac->valid.find('.') != std::string::npos)
930930
return isFloatArgValid(ftok, argnr, argvalue);
931931
TokenList tokenList(nullptr);
932-
gettokenlistfromvalid(ac->valid, tokenList);
932+
gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
933933
for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
934934
if (tok->isNumber() && argvalue == MathLib::toBigNumber(tok->str()))
935935
return true;
@@ -949,7 +949,7 @@ bool Library::isFloatArgValid(const Token *ftok, int argnr, double argvalue) con
949949
if (!ac || ac->valid.empty())
950950
return true;
951951
TokenList tokenList(nullptr);
952-
gettokenlistfromvalid(ac->valid, tokenList);
952+
gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
953953
for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
954954
if (Token::Match(tok, "%num% : %num%") && argvalue >= MathLib::toDoubleNumber(tok->str()) && argvalue <= MathLib::toDoubleNumber(tok->strAt(2)))
955955
return true;
@@ -1738,13 +1738,14 @@ bool Library::hasAnyTypeCheck(const std::string& typeName) const
17381738

17391739
std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
17401740
const Settings* settings,
1741+
bool cpp,
17411742
std::unordered_map<nonneg int, const Token*>* lookupVarId)
17421743
{
17431744
std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(settings);
17441745
{
17451746
const std::string code = "return " + returnValue + ";";
17461747
std::istringstream istr(code);
1747-
if (!tokenList->createTokens(istr))
1748+
if (!tokenList->createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
17481749
return nullptr;
17491750
}
17501751

‎lib/library.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -607,6 +607,7 @@ CPPCHECKLIB const Library::Container * getLibraryContainer(const Token * tok);
607607

608608
std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
609609
const Settings* settings,
610+
bool cpp,
610611
std::unordered_map<nonneg int, const Token*>* lookupVarId = nullptr);
611612

612613
/// @}

‎lib/programmemory.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1596,7 +1596,7 @@ namespace {
15961596
arg_map[argn] = v;
15971597
argn++;
15981598
}
1599-
return evaluateLibraryFunction(arg_map, returnValue, settings);
1599+
return evaluateLibraryFunction(arg_map, returnValue, settings, ftok->isCpp());
16001600
}
16011601
}
16021602
}
@@ -1752,15 +1752,16 @@ std::vector<ValueFlow::Value> execute(const Scope* scope, ProgramMemory& pm, con
17521752

17531753
ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
17541754
const std::string& returnValue,
1755-
const Settings* settings)
1755+
const Settings* settings,
1756+
bool cpp)
17561757
{
17571758
thread_local static std::unordered_map<std::string,
17581759
std::function<ValueFlow::Value(const std::unordered_map<nonneg int, ValueFlow::Value>& arg)>>
17591760
functions = {};
17601761
if (functions.count(returnValue) == 0) {
17611762

17621763
std::unordered_map<nonneg int, const Token*> lookupVarId;
1763-
std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, &lookupVarId);
1764+
std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, cpp, &lookupVarId);
17641765

17651766
functions[returnValue] =
17661767
[lookupVarId, expr, settings](const std::unordered_map<nonneg int, ValueFlow::Value>& xargs) {

‎lib/programmemory.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,8 @@ ProgramMemory getProgramMemory(const Token* tok, const Token* expr, const ValueF
211211

212212
ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
213213
const std::string& returnValue,
214-
const Settings* settings);
214+
const Settings* settings,
215+
bool cpp);
215216

216217
#endif
217218

‎lib/symboldatabase.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7585,7 +7585,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
75857585
ValueType valuetype;
75867586
TokenList tokenList(&mSettings);
75877587
std::istringstream istr(typestr+";");
7588-
tokenList.createTokens(istr);
7588+
tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
75897589
tokenList.simplifyStdType();
75907590
if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings, mIsCpp)) {
75917591
valuetype.originalTypeName = typestr;
@@ -7675,7 +7675,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
76757675
}
76767676
TokenList tokenList(&mSettings);
76777677
std::istringstream istr(typestr+";");
7678-
if (tokenList.createTokens(istr)) {
7678+
if (tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C)) {
76797679
ValueType vt;
76807680
tokenList.simplifyPlatformTypes();
76817681
tokenList.simplifyStdType();

‎lib/token.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1479,10 +1479,8 @@ class CPPCHECKLIB Token {
14791479
mImpl->mDebug = td;
14801480
}
14811481

1482-
/** defaults to true if it cannot be determined */
14831482
bool isCpp() const;
14841483

1485-
/** defaults to false if it cannot be determined */
14861484
bool isC() const;
14871485
};
14881486

‎lib/tokenlist.cpp‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
*/
1818

1919
//---------------------------------------------------------------------------
20+
2021
#include "tokenlist.h"
2122

2223
#include "astutils.h"
@@ -40,6 +41,15 @@
4041

4142
#include <simplecpp.h>
4243

44+
//#define N_ASSERT_LANG
45+
46+
#ifndef N_ASSERT_LANG
47+
#include <cassert>
48+
#define ASSERT_LANG(x) assert(x)
49+
#else
50+
#define ASSERT_LANG(x)
51+
#endif
52+
4353
// How many compileExpression recursions are allowed?
4454
// For practical code this could be endless. But in some special torture test
4555
// there needs to be a limit.
@@ -83,7 +93,7 @@ void TokenList::deallocateTokens()
8393

8494
void TokenList::determineCppC()
8595
{
86-
// only try to determine it if it wasn't enforced
96+
// only try to determine if it wasn't enforced
8797
if (mLang == Standards::Language::None) {
8898
mLang = Path::identify(getSourceFilePath());
8999
}
@@ -317,8 +327,31 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
317327

318328
bool TokenList::createTokens(std::istream &code, const std::string& file0)
319329
{
330+
ASSERT_LANG(!file0.empty());
331+
320332
appendFileIfNew(file0);
321333

334+
return createTokensInternal(code, file0);
335+
}
336+
337+
//---------------------------------------------------------------------------
338+
339+
bool TokenList::createTokens(std::istream &code, Standards::Language lang)
340+
{
341+
ASSERT_LANG(lang != Standards::Language::None);
342+
if (mLang == Standards::Language::None) {
343+
mLang = lang;
344+
} else {
345+
ASSERT_LANG(lang == mLang);
346+
}
347+
348+
return createTokensInternal(code, "");
349+
}
350+
351+
//---------------------------------------------------------------------------
352+
353+
bool TokenList::createTokensInternal(std::istream &code, const std::string& file0)
354+
{
322355
simplecpp::OutputList outputList;
323356
simplecpp::TokenList tokens(code, mFiles, file0, &outputList);
324357

@@ -2096,3 +2129,33 @@ bool TokenList::isKeyword(const std::string &str) const
20962129
static const auto& latest_c_keywords = Keywords::getAll(Standards::cstd_t::CLatest);
20972130
return latest_c_keywords.find(str) != latest_c_keywords.end();
20982131
}
2132+
2133+
bool TokenList::isC() const
2134+
{
2135+
ASSERT_LANG(mLang != Standards::Language::None);
2136+
2137+
// TODO: remove the fallback
2138+
if (mLang == Standards::Language::None)
2139+
return false; // treat as C++ by default
2140+
2141+
return mLang == Standards::Language::C;
2142+
}
2143+
2144+
bool TokenList::isCPP() const
2145+
{
2146+
ASSERT_LANG(mLang != Standards::Language::None);
2147+
2148+
// TODO: remove the fallback
2149+
if (mLang == Standards::Language::None)
2150+
return true; // treat as C++ by default
2151+
2152+
return mLang == Standards::Language::CPP;
2153+
}
2154+
2155+
void TokenList::setLang(Standards::Language lang)
2156+
{
2157+
ASSERT_LANG(lang != Standards::Language::None);
2158+
ASSERT_LANG(mLang == Standards::Language::None);
2159+
2160+
mLang = lang;
2161+
}

0 commit comments

Comments
 (0)