diff --git a/DEVELOPER_GUIDE.rst b/DEVELOPER_GUIDE.rst index 7f34e7bab31..f7798c9e921 100644 --- a/DEVELOPER_GUIDE.rst +++ b/DEVELOPER_GUIDE.rst @@ -231,6 +231,8 @@ Most of the time you just need to run ./gradlew build which will make sure you p - Run all unit tests. * - ./gradlew :integ-test:integTest - Run all integration test (this takes time). + * - ./gradlew :integ-test:yamlRestTest + - Run rest integration test. * - ./gradlew :doctest:doctest - Run doctests * - ./gradlew build diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java index 18e6541514e..a5702b4f77a 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java @@ -109,13 +109,6 @@ public class DateTimeFormatters { public static final DateTimeFormatter SQL_LITERAL_DATE_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - public static final DateTimeFormatter DATE_TIME_FORMATTER = - new DateTimeFormatterBuilder() - .appendOptional(SQL_LITERAL_DATE_TIME_FORMAT) - .appendOptional(STRICT_DATE_OPTIONAL_TIME_FORMATTER) - .appendOptional(STRICT_HOUR_MINUTE_SECOND_FORMATTER) - .toFormatter(); - /** todo. only support timestamp in format yyyy-MM-dd HH:mm:ss. */ public static final DateTimeFormatter DATE_TIME_FORMATTER_WITHOUT_NANO = SQL_LITERAL_DATE_TIME_FORMAT; diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 33f2e120bc2..7b5bade79ce 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -40,6 +40,7 @@ plugins { apply plugin: 'opensearch.build' apply plugin: 'opensearch.rest-test' +apply plugin: 'opensearch.yaml-rest-test' apply plugin: 'java' apply plugin: 'io.freefair.lombok' apply plugin: 'com.wiredforcode.spawn' @@ -258,6 +259,13 @@ testClusters { plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" } + yamlRestTest { + testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) + plugin(getGeoSpatialPlugin()) + plugin ":opensearch-sql-plugin" + setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" + } remoteCluster { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) @@ -411,6 +419,10 @@ task integTestWithSecurity(type: RestIntegTestTask) { } } +yamlRestTest { + systemProperty 'tests.security.manager', 'false' +} + // Run PPL ITs and new, legacy and comparison SQL ITs with new SQL engine enabled integTest { useCluster testClusters.remoteCluster diff --git a/integ-test/src/yamlRestTest/java/org/opensearch/sql/rest/RestHandlerClientYamlTestSuiteIT.java b/integ-test/src/yamlRestTest/java/org/opensearch/sql/rest/RestHandlerClientYamlTestSuiteIT.java new file mode 100644 index 00000000000..3f30cf8dafa --- /dev/null +++ b/integ-test/src/yamlRestTest/java/org/opensearch/sql/rest/RestHandlerClientYamlTestSuiteIT.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.rest; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.opensearch.test.rest.yaml.ClientYamlTestCandidate; +import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; + +public class RestHandlerClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { + + public RestHandlerClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return OpenSearchClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/api/ppl.json b/integ-test/src/yamlRestTest/resources/rest-api-spec/api/ppl.json new file mode 100644 index 00000000000..6b6e56307c4 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/api/ppl.json @@ -0,0 +1,22 @@ +{ + "ppl": { + "documentation": { + "url": "https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/index.rst", + "description": "OpenSearch PPL Reference Manual" + }, + "stability" : "stable", + "url": { + "paths": [ + { + "path" : "/_plugins/_ppl", + "methods" : ["POST"] + } + ] + }, + "params": {}, + "body": { + "description": "PPL Query", + "required":true + } + } +} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/api/query.settings.json b/integ-test/src/yamlRestTest/resources/rest-api-spec/api/query.settings.json new file mode 100644 index 00000000000..82829d4e40b --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/api/query.settings.json @@ -0,0 +1,22 @@ +{ + "query.settings": { + "documentation": { + "url": "https://github.com/opensearch-project/sql/blob/main/docs/user/admin/settings.rst", + "description": "Query Settings" + }, + "stability" : "stable", + "url": { + "paths": [ + { + "path" : "/_plugins/_query/settings", + "methods" : ["PUT"] + } + ] + }, + "params": {}, + "body": { + "description": "Query Settings", + "required":true + } + } +} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/2489.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/2489.yml new file mode 100644 index 00000000000..58b1b204d48 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/2489.yml @@ -0,0 +1,50 @@ +setup: + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + plugins.calcite.fallback.allowed : false + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + plugins.calcite.fallback.allowed : true + +--- +"Handle epoch field in string format": + - skip: + features: + - headers + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{"timestamp": "1705642934886"}' + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=test | fields timestamp' + - match: {"total": 1} + - match: {"schema": [{"name": "timestamp", "type": "timestamp"}]} + - match: {"datarows": [["2024-01-19 05:42:14.886"]]} + diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 68c6fda617f..21551f086cb 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -17,7 +17,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; -import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER; import static org.opensearch.sql.utils.DateTimeFormatters.STRICT_HOUR_MINUTE_SECOND_FORMATTER; import static org.opensearch.sql.utils.DateTimeFormatters.STRICT_YEAR_MONTH_DAY_FORMATTER; @@ -44,6 +43,7 @@ import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateFormatters; import org.opensearch.common.time.FormatNames; +import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; import org.opensearch.sql.data.model.ExprCollectionValue; @@ -262,9 +262,10 @@ private static ExprValue parseDateTimeString(String value, OpenSearchDateType da DateFormatters.from(STRICT_YEAR_MONTH_DAY_FORMATTER.parse(value)).toLocalDate()); default: return new ExprTimestampValue( - DateFormatters.from(DATE_TIME_FORMATTER.parse(value)).toInstant()); + DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(value)) + .toInstant()); } - } catch (DateTimeParseException ignored) { + } catch (DateTimeParseException | IllegalArgumentException ignored) { // ignored } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index 89dfd4dbdb2..42281044c82 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -78,6 +78,7 @@ class OpenSearchExprValueFactoryTest { .put("dateV", OpenSearchDateType.of(DATE)) .put("timeV", OpenSearchDateType.of(TIME)) .put("timestampV", OpenSearchDateType.of(TIMESTAMP)) + .put("timeonlyV", OpenSearchDateType.of("HH:mm:ss")) .put("datetimeDefaultV", OpenSearchDateType.of()) .put("dateStringV", OpenSearchDateType.of("date")) .put("timeStringV", OpenSearchDateType.of("time")) @@ -312,10 +313,6 @@ public void constructDatetime() { assertEquals( new ExprTimestampValue("2015-01-01 12:10:30"), tupleValue("{\"timestampV\":\"2015-01-01T12:10:30\"}").get("timestampV")), - () -> - assertEquals( - new ExprTimestampValue("2015-01-01 12:10:30"), - tupleValue("{\"timestampV\":\"2015-01-01 12:10:30\"}").get("timestampV")), () -> assertEquals( new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), @@ -348,10 +345,6 @@ public void constructDatetime() { assertEquals( new ExprTimestampValue("1984-05-10 20:30:40"), tupleValue("{ \"dateTimeCustomV\" : 19840510203040 }").get("dateTimeCustomV")), - () -> - assertEquals( - new ExprTimestampValue("2015-01-01 12:10:30"), - constructFromObject("timestampV", "2015-01-01 12:10:30")), () -> assertEquals( new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), @@ -361,7 +354,7 @@ public void constructDatetime() { () -> assertEquals( new ExprTimeValue("19:36:22"), - tupleValue("{\"timestampV\":\"19:36:22\"}").get("timestampV")), + tupleValue("{\"timeonlyV\":\"19:36:22\"}").get("timeonlyV")), // case: timestamp-formatted field, but it only gets a date: should match a date () -> @@ -384,10 +377,6 @@ public void constructDatetime_fromCustomFormat() { "Construct TIMESTAMP from \"2015-01-01 12-10-30\" failed, unsupported format.", exception.getMessage()); - assertEquals( - new ExprTimestampValue("2015-01-01 12:10:30"), - constructFromObject("customAndEpochMillisV", "2015-01-01 12:10:30")); - assertEquals( new ExprTimestampValue("2015-01-01 12:10:30"), constructFromObject("customAndEpochMillisV", "2015-01-01-12-10-30")); @@ -700,7 +689,7 @@ public void constructArrayOfCustomEpochMillisReturnsAll() { List.of( new ExprTimestampValue("2015-01-01 12:10:30"), new ExprTimestampValue("1999-11-09 01:09:44"))), - tupleValue("{\"customAndEpochMillisV\":[\"2015-01-01 12:10:30\",\"1999-11-09 01:09:44\"]}") + tupleValue("{\"customAndEpochMillisV\":[\"2015-01-01-12-10-30\",\"1999-11-09-01-09-44\"]}") .get("customAndEpochMillisV")); }