diff --git a/src/tokenizer.cc b/src/tokenizer.cc index f24655c75..54bf88c05 100644 --- a/src/tokenizer.cc +++ b/src/tokenizer.cc @@ -99,10 +99,15 @@ std::unique_ptr Tokenizer::NextToken() { data_.substr(current_position_, end_pos - current_position_); current_position_ = end_pos; + // Check for "NaN" explicitly. + bool is_nan = + (tok_str.size() == 3 && std::tolower(tok_str[0]) == 'n' && + std::tolower(tok_str[1]) == 'a' && std::tolower(tok_str[2]) == 'n'); + // Starts with an alpha is a string. - if (!std::isdigit(tok_str[0]) && - !(tok_str[0] == '-' && std::isdigit(tok_str[1])) && - !(tok_str[0] == '.' && std::isdigit(tok_str[1]))) { + if (!is_nan && !std::isdigit(tok_str[0]) && + !(tok_str[0] == '-' && tok_str.size() >= 2 && std::isdigit(tok_str[1])) && + !(tok_str[0] == '.' && tok_str.size() >= 2 && std::isdigit(tok_str[1]))) { // If we've got a continuation, skip over the end of line and get the next // token. if (tok_str == "\\") { @@ -126,17 +131,21 @@ std::unique_ptr Tokenizer::NextToken() { } // Handle hex strings - if (tok_str.size() > 2 && tok_str[0] == '0' && tok_str[1] == 'x') { + if (!is_nan && tok_str.size() > 2 && tok_str[0] == '0' && tok_str[1] == 'x') { auto tok = MakeUnique(TokenType::kHex); tok->SetStringValue(tok_str); return tok; } bool is_double = false; - for (const char ch : tok_str) { - if (ch == '.') { - is_double = true; - break; + if (is_nan) { + is_double = true; + } else { + for (const char ch : tok_str) { + if (ch == '.') { + is_double = true; + break; + } } } diff --git a/src/tokenizer_test.cc b/src/tokenizer_test.cc index d50ec467c..fd7dba48d 100644 --- a/src/tokenizer_test.cc +++ b/src/tokenizer_test.cc @@ -14,6 +14,7 @@ #include "src/tokenizer.h" +#include #include #include "gtest/gtest.h" @@ -77,6 +78,33 @@ TEST_F(TokenizerTest, ProcessDouble) { EXPECT_TRUE(next->IsEOS()); } +namespace { + +void TestNaN(const std::string& nan_str) { + Tokenizer t(nan_str); + auto next = t.NextToken(); + ASSERT_TRUE(next != nullptr); + EXPECT_TRUE(next->IsDouble()); + EXPECT_TRUE(std::isnan(next->AsDouble())); + + next = t.NextToken(); + ASSERT_TRUE(next != nullptr); + EXPECT_TRUE(next->IsEOS()); +} + +} // namespace + +TEST_F(TokenizerTest, ProcessNaN) { + TestNaN("nan"); + TestNaN("naN"); + TestNaN("nAn"); + TestNaN("nAN"); + TestNaN("Nan"); + TestNaN("NaN"); + TestNaN("NAn"); + TestNaN("NAN"); +} + TEST_F(TokenizerTest, ProcessNegativeDouble) { Tokenizer t("-123.456"); auto next = t.NextToken();