Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import org.apache.paimon.utils.StringUtils;

import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;

import okhttp3.Dispatcher;
import okhttp3.Headers;
Expand All @@ -45,12 +44,12 @@
import static okhttp3.ConnectionSpec.CLEARTEXT;
import static okhttp3.ConnectionSpec.COMPATIBLE_TLS;
import static okhttp3.ConnectionSpec.MODERN_TLS;
import static org.apache.paimon.rest.RESTObjectMapper.OBJECT_MAPPER;
import static org.apache.paimon.utils.ThreadPoolUtils.createCachedThreadPool;

/** HTTP client for REST catalog. */
public class HttpClient implements RESTClient {

private static final ObjectMapper OBJECT_MAPPER = RESTObjectMapper.create();
private static final String THREAD_NAME = "REST-CATALOG-HTTP-CLIENT-THREAD-POOL";
private static final MediaType MEDIA_TYPE = MediaType.parse("application/json");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
import java.util.concurrent.ScheduledExecutorService;

import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE;
import static org.apache.paimon.CoreOptions.PATH;
import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch;
import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase;
import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable;
Expand Down Expand Up @@ -471,14 +472,17 @@ private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistEx
throw new TableNoPermissionException(identifier, e);
}

TableSchema schema = TableSchema.create(response.getSchemaId(), response.getSchema());
FileStoreTable table =
FileStoreTableFactory.create(
fileIO(),
new Path(response.getPath()),
TableSchema.create(response.getSchemaId(), response.getSchema()),
// TODO add uuid from server
new Path(schema.options().get(PATH.key())),
schema,
new CatalogEnvironment(
identifier, null, Lock.emptyFactory(), catalogLoader()));
identifier,
response.getId(),
Lock.emptyFactory(),
catalogLoader()));
CoreOptions options = table.coreOptions();
if (options.type() == TableType.OBJECT_TABLE) {
String objectLocation = options.objectLocation();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@
/** Object mapper for REST request and response. */
public class RESTObjectMapper {

public static ObjectMapper create() {
public static final ObjectMapper OBJECT_MAPPER = RESTObjectMapper.create();

private static ObjectMapper create() {
ObjectMapper mapper = new ObjectMapper();
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
Expand All @@ -43,7 +45,7 @@ public static ObjectMapper create() {
return mapper;
}

public static Module createPaimonRestJacksonModule() {
private static Module createPaimonRestJacksonModule() {
SimpleModule module = new SimpleModule("Paimon_REST");
registerJsonObjects(
module,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,13 @@
@JsonIgnoreProperties(ignoreUnknown = true)
public class GetDatabaseResponse implements RESTResponse, Database {

private static final String FIELD_ID = "id";
private static final String FIELD_NAME = "name";
private static final String FIELD_OPTIONS = "options";

@JsonProperty(FIELD_ID)
private final String id;

@JsonProperty(FIELD_NAME)
private final String name;

Expand All @@ -46,12 +50,19 @@ public class GetDatabaseResponse implements RESTResponse, Database {

@JsonCreator
public GetDatabaseResponse(
@JsonProperty(FIELD_ID) String id,
@JsonProperty(FIELD_NAME) String name,
@JsonProperty(FIELD_OPTIONS) Map<String, String> options) {
this.id = id;
this.name = name;
this.options = options;
}

@JsonGetter(FIELD_ID)
public String getId() {
return id;
}

@JsonGetter(FIELD_NAME)
public String getName() {
return name;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,16 @@
@JsonIgnoreProperties(ignoreUnknown = true)
public class GetTableResponse implements RESTResponse {

private static final String FIELD_PATH = "path";
private static final String FIELD_ID = "id";
private static final String FIELD_NAME = "name";
private static final String FIELD_SCHEMA_ID = "schemaId";
private static final String FIELD_SCHEMA = "schema";

@JsonProperty(FIELD_PATH)
private final String path;
@JsonProperty(FIELD_ID)
private final String id;

@JsonProperty(FIELD_NAME)
private final String name;

@JsonProperty(FIELD_SCHEMA_ID)
private final long schemaId;
Expand All @@ -45,17 +49,24 @@ public class GetTableResponse implements RESTResponse {

@JsonCreator
public GetTableResponse(
@JsonProperty(FIELD_PATH) String path,
@JsonProperty(FIELD_ID) String id,
@JsonProperty(FIELD_NAME) String name,
@JsonProperty(FIELD_SCHEMA_ID) long schemaId,
@JsonProperty(FIELD_SCHEMA) Schema schema) {
this.path = path;
this.id = id;
this.name = name;
this.schemaId = schemaId;
this.schema = schema;
}

@JsonGetter(FIELD_PATH)
public String getPath() {
return this.path;
@JsonGetter(FIELD_ID)
public String getId() {
return this.id;
}

@JsonGetter(FIELD_NAME)
public String getName() {
return this.name;
}

@JsonGetter(FIELD_SCHEMA_ID)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

package org.apache.paimon.rest;

import org.apache.paimon.CoreOptions;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.partition.Partition;
import org.apache.paimon.rest.requests.AlterDatabaseRequest;
Expand Down Expand Up @@ -55,6 +54,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;

import static org.apache.paimon.rest.RESTCatalogInternalOptions.DATABASE_COMMENT;

Expand All @@ -81,7 +81,7 @@ public static GetDatabaseResponse getDatabaseResponse(String name) {
Map<String, String> options = new HashMap<>();
options.put("a", "b");
options.put(DATABASE_COMMENT.key(), "comment");
return new GetDatabaseResponse(name, options);
return new GetDatabaseResponse(UUID.randomUUID().toString(), name, options);
}

public static ListDatabasesResponse listDatabasesResponse(String name) {
Expand Down Expand Up @@ -226,18 +226,11 @@ public static List<SchemaChange> getChanges() {
return schemaChanges;
}

public static GetTableResponse getTableResponseEnablePartition() {
Map<String, String> options = new HashMap<>();
options.put("option-1", "value-1");
options.put(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true");
return new GetTableResponse("/tmp/2", 1, schema(options));
}

public static GetTableResponse getTableResponse() {
Map<String, String> options = new HashMap<>();
options.put("option-1", "value-1");
options.put("option-2", "value-2");
return new GetTableResponse("/tmp/1", 1, schema(options));
return new GetTableResponse(UUID.randomUUID().toString(), "", 1, schema(options));
}

public static MockResponse mockResponse(String body, int httpCode) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

package org.apache.paimon.rest;

import org.apache.paimon.catalog.AbstractCatalog;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.CatalogFactory;
Expand All @@ -42,7 +41,6 @@
import org.apache.paimon.table.FileStoreTable;

import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;

import okhttp3.mockwebserver.Dispatcher;
import okhttp3.mockwebserver.MockResponse;
Expand All @@ -51,11 +49,13 @@

import java.io.IOException;
import java.util.List;
import java.util.UUID;

import static org.apache.paimon.rest.RESTObjectMapper.OBJECT_MAPPER;

/** Mock REST server for testing. */
public class RESTCatalogServer {

private static final ObjectMapper OBJECT_MAPPER = RESTObjectMapper.create();
private static final String PREFIX = "paimon";
private static final String DATABASE_URI = String.format("/v1/%s/databases", PREFIX);

Expand Down Expand Up @@ -219,9 +219,8 @@ private static MockResponse renameTableApiHandler(
FileStoreTable table = (FileStoreTable) catalog.getTable(requestBody.getNewIdentifier());
RESTResponse response =
new GetTableResponse(
AbstractCatalog.newTableLocation(
catalog.warehouse(), requestBody.getNewIdentifier())
.toString(),
UUID.randomUUID().toString(),
tableName,
table.schema().id(),
table.schema().toSchema());
return mockResponse(response, 200);
Expand Down Expand Up @@ -251,7 +250,9 @@ private static MockResponse databaseApiHandler(
RESTResponse response;
if (request.getMethod().equals("GET")) {
Database database = catalog.getDatabase(databaseName);
response = new GetDatabaseResponse(database.name(), database.options());
response =
new GetDatabaseResponse(
UUID.randomUUID().toString(), database.name(), database.options());
return mockResponse(response, 200);
} else if (request.getMethod().equals("DELETE")) {
catalog.dropDatabase(databaseName, false, true);
Expand All @@ -267,7 +268,12 @@ private static MockResponse tablesApiHandler(
CreateTableRequest requestBody =
OBJECT_MAPPER.readValue(request.getBody().readUtf8(), CreateTableRequest.class);
catalog.createTable(requestBody.getIdentifier(), requestBody.getSchema(), false);
response = new GetTableResponse("", 1L, requestBody.getSchema());
response =
new GetTableResponse(
UUID.randomUUID().toString(),
requestBody.getIdentifier().getTableName(),
1L,
requestBody.getSchema());
return mockResponse(response, 200);
} else if (request.getMethod().equals("GET")) {
catalog.listTables(databaseName);
Expand All @@ -286,8 +292,8 @@ private static MockResponse tableApiHandler(
FileStoreTable table = (FileStoreTable) catalog.getTable(identifier);
response =
new GetTableResponse(
AbstractCatalog.newTableLocation(catalog.warehouse(), identifier)
.toString(),
UUID.randomUUID().toString(),
tableName,
table.schema().id(),
table.schema().toSchema());
return mockResponse(response, 200);
Expand All @@ -297,7 +303,12 @@ private static MockResponse tableApiHandler(
OBJECT_MAPPER.readValue(request.getBody().readUtf8(), AlterTableRequest.class);
catalog.alterTable(identifier, requestBody.getChanges(), false);
FileStoreTable table = (FileStoreTable) catalog.getTable(identifier);
response = new GetTableResponse("", table.schema().id(), table.schema().toSchema());
response =
new GetTableResponse(
UUID.randomUUID().toString(),
tableName,
table.schema().id(),
table.schema().toSchema());
return mockResponse(response, 200);
} else if (request.getMethod().equals("DELETE")) {
Identifier identifier = Identifier.create(databaseName, tableName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,4 +116,9 @@ private void createTable(
""),
true);
}

// TODO implement this
@Override
@Test
public void testTableUUID() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,22 +40,20 @@
import org.apache.paimon.types.IntType;

import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;

import org.junit.Test;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;

import static org.apache.paimon.rest.RESTObjectMapper.OBJECT_MAPPER;
import static org.junit.Assert.assertEquals;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;

/** Test for {@link RESTObjectMapper}. */
public class RESTObjectMapperTest {

private static final ObjectMapper OBJECT_MAPPER = RESTObjectMapper.create();

@Test
public void configResponseParseTest() throws Exception {
String confKey = "a";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
package org.apache.paimon.rest;

import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;

import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer;
Expand All @@ -28,11 +27,12 @@
import java.io.IOException;
import java.util.concurrent.TimeUnit;

import static org.apache.paimon.rest.RESTObjectMapper.OBJECT_MAPPER;

/** Mock a http web service locally. */
public class TestHttpWebServer {

private MockWebServer mockWebServer;
private final ObjectMapper objectMapper = RESTObjectMapper.create();
private String baseUrl;
private final String path;

Expand Down Expand Up @@ -63,19 +63,10 @@ public void enqueueResponse(MockResponse mockResponseObj) {
mockWebServer.enqueue(mockResponseObj);
}

public void enqueueResponse(RESTResponse response, Integer code)
throws JsonProcessingException {
enqueueResponse(createResponseBody(response), code);
}

public String getBaseUrl() {
return baseUrl;
}

public ObjectMapper getObjectMapper() {
return objectMapper;
}

public MockResponse generateMockResponse(String data, Integer code) {
return new MockResponse()
.setResponseCode(code)
Expand All @@ -84,15 +75,10 @@ public MockResponse generateMockResponse(String data, Integer code) {
}

public String createResponseBody(RESTResponse response) throws JsonProcessingException {
return objectMapper.writeValueAsString(response);
return OBJECT_MAPPER.writeValueAsString(response);
}

public <T> T readRequestBody(String body, Class<T> requestType) throws JsonProcessingException {
return objectMapper.readValue(body, requestType);
}

public <T> T readResponseBody(String body, Class<T> responseType)
throws JsonProcessingException {
return objectMapper.readValue(body, responseType);
return OBJECT_MAPPER.readValue(body, requestType);
}
}
Loading
Loading