Skip to content

Commit 81700b4

Browse files
firewavefirewave
authored
some TokenList cleanups (#5848)
1 parent 328f98b commit 81700b4

File tree

5 files changed

+36
-29
lines changed

5 files changed

+36
-29
lines changed

‎Makefile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -491,7 +491,7 @@ $(libcppdir)/checkautovariables.o: lib/checkautovariables.cpp lib/addoninfo.h li
491491
$(libcppdir)/checkbool.o: lib/checkbool.cpp lib/addoninfo.h lib/astutils.h lib/check.h lib/checkbool.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
492492
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/checkbool.cpp
493493

494-
$(libcppdir)/checkboost.o: lib/checkboost.cpp lib/check.h lib/checkboost.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
494+
$(libcppdir)/checkboost.o: lib/checkboost.cpp lib/addoninfo.h lib/check.h lib/checkboost.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
495495
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/checkboost.cpp
496496

497497
$(libcppdir)/checkbufferoverrun.o: lib/checkbufferoverrun.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/astutils.h lib/check.h lib/checkbufferoverrun.h lib/color.h lib/config.h lib/ctu.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h
@@ -593,7 +593,7 @@ $(libcppdir)/infer.o: lib/infer.cpp lib/calculate.h lib/config.h lib/errortypes.
593593
$(libcppdir)/keywords.o: lib/keywords.cpp lib/config.h lib/keywords.h lib/standards.h lib/utils.h
594594
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/keywords.cpp
595595

596-
$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h
596+
$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h
597597
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/library.cpp
598598

599599
$(libcppdir)/mathlib.o: lib/mathlib.cpp externals/simplecpp/simplecpp.h lib/config.h lib/errortypes.h lib/mathlib.h lib/utils.h
@@ -626,7 +626,7 @@ $(libcppdir)/settings.o: lib/settings.cpp externals/picojson/picojson.h lib/addo
626626
$(libcppdir)/summaries.o: lib/summaries.cpp lib/addoninfo.h lib/analyzerinfo.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/summaries.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
627627
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/summaries.cpp
628628

629-
$(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/mathlib.h lib/path.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h
629+
$(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h
630630
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/suppressions.cpp
631631

632632
$(libcppdir)/templatesimplifier.o: lib/templatesimplifier.cpp lib/addoninfo.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h

‎lib/tokenize.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3361,10 +3361,10 @@ bool Tokenizer::simplifyTokens1(const std::string &configuration)
33613361
if (mTimerResults) {
33623362
Timer t("Tokenizer::simplifyTokens1::createAst", mSettings->showtime, mTimerResults);
33633363
list.createAst();
3364-
list.validateAst();
3364+
list.validateAst(mSettings->debugnormal);
33653365
} else {
33663366
list.createAst();
3367-
list.validateAst();
3367+
list.validateAst(mSettings->debugnormal);
33683368
}
33693369

33703370
if (mTimerResults) {

‎lib/tokenlist.cpp‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ TokenList::TokenList(const Settings* settings) :
5050
mSettings(settings)
5151
{
5252
mTokensFrontBack.list = this;
53+
if (mSettings && (mSettings->enforcedLang != Settings::Language::None)) {
54+
mLang = mSettings->enforcedLang;
55+
}
5356
}
5457

5558
TokenList::~TokenList()
@@ -80,12 +83,12 @@ void TokenList::deallocateTokens()
8083

8184
void TokenList::determineCppC()
8285
{
83-
if (!mSettings) {
84-
mIsC = Path::isC(getSourceFilePath());
85-
mIsCpp = Path::isCPP(getSourceFilePath());
86-
} else {
87-
mIsC = mSettings->enforcedLang == Settings::Language::C || (mSettings->enforcedLang == Settings::Language::None && Path::isC(getSourceFilePath()));
88-
mIsCpp = mSettings->enforcedLang == Settings::Language::CPP || (mSettings->enforcedLang == Settings::Language::None && Path::isCPP(getSourceFilePath()));
86+
// only try to determine it if it wasn't enforced
87+
if (mLang == Settings::Language::None) {
88+
if (Path::isC(getSourceFilePath()))
89+
mLang = Settings::Language::C;
90+
else if (Path::isCPP(getSourceFilePath()))
91+
mLang = Settings::Language::CPP;
8992
}
9093
}
9194

@@ -1722,10 +1725,10 @@ namespace {
17221725
};
17231726
}
17241727

1725-
void TokenList::validateAst() const
1728+
void TokenList::validateAst(bool print) const
17261729
{
17271730
OnException oe{[&] {
1728-
if (mSettings && mSettings->debugnormal)
1731+
if (print)
17291732
mTokensFrontBack.front->printOut();
17301733
}};
17311734
// Check for some known issues in AST to avoid crash/hang later on
@@ -1854,7 +1857,7 @@ void TokenList::simplifyPlatformTypes()
18541857
if (!mSettings)
18551858
return;
18561859

1857-
const bool isCPP11 = mSettings->standards.cpp >= Standards::CPP11;
1860+
const bool isCPP11 = isCPP() && (mSettings->standards.cpp >= Standards::CPP11);
18581861

18591862
enum { isLongLong, isLong, isInt } type;
18601863

@@ -1996,7 +1999,7 @@ void TokenList::simplifyStdType()
19961999
continue;
19972000
}
19982001

1999-
if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (mSettings->standards.c >= Standards::C99 && Token::Match(tok, "complex|_Complex"))) {
2002+
if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (isC() && (!mSettings || (mSettings->standards.c >= Standards::C99)) && Token::Match(tok, "complex|_Complex"))) {
20002003
bool isFloat= false;
20012004
bool isSigned = false;
20022005
bool isUnsigned = false;
@@ -2019,7 +2022,7 @@ void TokenList::simplifyStdType()
20192022
else if (Token::Match(tok2, "float|double")) {
20202023
isFloat = true;
20212024
typeSpec = tok2;
2022-
} else if (mSettings->standards.c >= Standards::C99 && Token::Match(tok2, "complex|_Complex"))
2025+
} else if (isC() && (!mSettings || (mSettings->standards.c >= Standards::C99)) && Token::Match(tok2, "complex|_Complex"))
20232026
isComplex = !isFloat || tok2->str() == "_Complex" || Token::Match(tok2->next(), "*|&|%name%"); // Ensure that "complex" is not the variables name
20242027
else if (Token::Match(tok2, "char|int")) {
20252028
if (!typeSpec)
@@ -2057,7 +2060,7 @@ void TokenList::simplifyStdType()
20572060

20582061
bool TokenList::isKeyword(const std::string &str) const
20592062
{
2060-
if (mIsCpp) {
2063+
if (isCPP()) {
20612064
// TODO: integrate into keywords?
20622065
// types and literals are not handled as keywords
20632066
static const std::unordered_set<std::string> cpp_types = {"bool", "false", "true"};

‎lib/tokenlist.h‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,13 @@
2222
//---------------------------------------------------------------------------
2323

2424
#include "config.h"
25+
#include "settings.h"
2526

2627
#include <cstddef>
2728
#include <iosfwd>
2829
#include <string>
2930
#include <vector>
3031

31-
class Settings;
3232
class Token;
3333
class TokenList;
3434

@@ -61,12 +61,12 @@ class CPPCHECKLIB TokenList {
6161

6262
/** Is the code C. Used for bailouts */
6363
bool isC() const {
64-
return mIsC;
64+
return mLang == Settings::Language::C;
6565
}
6666

6767
/** Is the code CPP. Used for bailouts */
6868
bool isCPP() const {
69-
return mIsCpp;
69+
return mLang == Settings::Language::CPP;
7070
}
7171

7272
/**
@@ -171,7 +171,7 @@ class CPPCHECKLIB TokenList {
171171
* Check abstract syntax tree.
172172
* Throws InternalError on failure
173173
*/
174-
void validateAst() const;
174+
void validateAst(bool print) const;
175175

176176
/**
177177
* Verify that the given token is an element of the tokenlist.
@@ -214,8 +214,7 @@ class CPPCHECKLIB TokenList {
214214
const Settings* const mSettings{};
215215

216216
/** File is known to be C/C++ code */
217-
bool mIsC{};
218-
bool mIsCpp{};
217+
Settings::Language mLang{Settings::Language::None};
219218
};
220219

221220
/// @}

‎test/testtokenize.cpp‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2873,28 +2873,33 @@ class TestTokenizer : public TestFixture {
28732873
}
28742874
{
28752875
const char code[] = "float complex x;";
2876-
const char expected[] = "_Complex float x ;";
2876+
const char expected[] = "float complex x ;";
28772877
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
28782878
}
2879+
{
2880+
const char code[] = "float complex x;";
2881+
const char expected[] = "_Complex float x ;";
2882+
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
2883+
}
28792884
{
28802885
const char code[] = "complex float x;";
28812886
const char expected[] = "_Complex float x ;";
2882-
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
2887+
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
28832888
}
28842889
{
28852890
const char code[] = "complex long double x;";
28862891
const char expected[] = "_Complex long double x ;";
2887-
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
2892+
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
28882893
}
28892894
{
28902895
const char code[] = "long double complex x;";
28912896
const char expected[] = "_Complex long double x ;";
2892-
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
2897+
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
28932898
}
28942899
{
28952900
const char code[] = "double complex;";
28962901
const char expected[] = "double complex ;";
2897-
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
2902+
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
28982903
}
28992904
}
29002905

@@ -5975,7 +5980,7 @@ class TestTokenizer : public TestFixture {
59755980
tokenList.prepareTernaryOpForAST();
59765981
tokenList.list.createAst();
59775982

5978-
tokenList.list.validateAst();
5983+
tokenList.list.validateAst(false);
59795984

59805985
// Basic AST validation
59815986
for (const Token *tok = tokenList.list.front(); tok; tok = tok->next()) {

0 commit comments

Comments
 (0)