From 74793edf5e23be52fd88fdb540165287e6dc6fc3 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 12 Mar 2019 19:55:08 +0800 Subject: [PATCH 01/10] Support PostgreSQL and CockroachDB backends implemented: #441 Change-Id: I979affd9f8b2916e6696672fc6e3fda75f9b76da --- .travis.yml | 4 + hugegraph-dist/pom.xml | 5 + .../assembly/static/conf/hugegraph.properties | 1 + .../src/assembly/travis/install-backend.sh | 2 + .../src/assembly/travis/install-postgresql.sh | 17 + .../src/assembly/travis/start-server.sh | 16 +- .../baidu/hugegraph/dist/RegisterUtil.java | 15 + .../src/main/resources/backend.properties | 2 +- .../backend/store/mysql/MysqlSessions.java | 7 +- .../backend/store/mysql/MysqlStore.java | 5 +- .../backend/store/mysql/MysqlTable.java | 2 +- .../backend/store/mysql/MysqlTables.java | 4 +- hugegraph-postgresql/pom.xml | 31 ++ .../store/postgresql/PostgresqlFeatures.java | 24 + .../store/postgresql/PostgresqlOptions.java | 82 ++++ .../postgresql/PostgresqlSerializer.java | 24 + .../store/postgresql/PostgresqlSessions.java | 36 ++ .../store/postgresql/PostgresqlStore.java | 40 ++ .../postgresql/PostgresqlStoreProvider.java | 139 ++++++ .../store/postgresql/PostgresqlTable.java | 341 ++++++++++++++ .../store/postgresql/PostgresqlTables.java | 422 ++++++++++++++++++ hugegraph-test/pom.xml | 15 + .../baidu/hugegraph/api/MetricsApiTest.java | 1 + .../com/baidu/hugegraph/api/TaskApiTest.java | 2 +- .../baidu/hugegraph/core/EdgeCoreTest.java | 13 +- .../baidu/hugegraph/core/VertexCoreTest.java | 9 +- .../src/main/resources/hugegraph.properties | 1 + pom.xml | 1 + 28 files changed, 1243 insertions(+), 18 deletions(-) create mode 100755 hugegraph-dist/src/assembly/travis/install-postgresql.sh create mode 100644 hugegraph-postgresql/pom.xml create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlFeatures.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStore.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStoreProvider.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java diff --git a/.travis.yml b/.travis.yml index 4f7be19fc7..b6dd5ac5ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,9 @@ jdk: sudo: required +addons: + postgresql: "9.5" + cache: directories: - $HOME/.m2 @@ -80,6 +83,7 @@ env: - BACKEND=mysql - BACKEND=hbase - BACKEND=rocksdb + - BACKEND=postgresql global: - RELEASE_BRANCH=^release-.*$ - RELEASE_TAG=^v[0-9]\..*$ diff --git a/hugegraph-dist/pom.xml b/hugegraph-dist/pom.xml index eb08e872ec..00d583277b 100644 --- a/hugegraph-dist/pom.xml +++ b/hugegraph-dist/pom.xml @@ -59,6 +59,11 @@ hugegraph-hbase ${project.version} + + com.baidu.hugegraph + hugegraph-postgresql + ${project.version} + io.airlift airline diff --git a/hugegraph-dist/src/assembly/static/conf/hugegraph.properties b/hugegraph-dist/src/assembly/static/conf/hugegraph.properties index fe0729034c..2973ffd51d 100644 --- a/hugegraph-dist/src/assembly/static/conf/hugegraph.properties +++ b/hugegraph-dist/src/assembly/static/conf/hugegraph.properties @@ -33,6 +33,7 @@ cassandra.password= # mysql backend config +#jdbc.driver=com.mysql.jdbc.Driver #jdbc.url=jdbc:mysql://127.0.0.1:3306 #jdbc.username=root #jdbc.password= diff --git a/hugegraph-dist/src/assembly/travis/install-backend.sh b/hugegraph-dist/src/assembly/travis/install-backend.sh index 7349b07649..6fdd97ce9f 100755 --- a/hugegraph-dist/src/assembly/travis/install-backend.sh +++ b/hugegraph-dist/src/assembly/travis/install-backend.sh @@ -10,4 +10,6 @@ elif [ "$BACKEND" == "hbase" ]; then $TRAVIS_DIR/install-hbase.sh elif [ "$BACKEND" == "mysql" ]; then $TRAVIS_DIR/install-mysql.sh +elif [ "$BACKEND" == "postgresql" ]; then + $TRAVIS_DIR/install-postgresql.sh fi diff --git a/hugegraph-dist/src/assembly/travis/install-postgresql.sh b/hugegraph-dist/src/assembly/travis/install-postgresql.sh new file mode 100755 index 0000000000..191cff1d6d --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/install-postgresql.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -ev + +TRAVIS_DIR=`dirname $0` +CONF=$TRAVIS_DIR/../../../../hugegraph-test/src/main/resources/hugegraph.properties + +POSTGRESQL_DRIVER=org.postgresql.Driver +POSTGRESQL_URL=jdbc:postgresql://localhost:5432/ +POSTGRESQL_USERNAME=postgres + +# Set PostgreSQL configurations +sed -i "s/jdbc.driver=.*/jdbc.driver=$POSTGRESQL_DRIVER/" $CONF +sed -i "s?jdbc.url=.*?jdbc.url=$POSTGRESQL_URL?" $CONF +sed -i "s/jdbc.username=.*/jdbc.username=$POSTGRESQL_USERNAME/" $CONF + +sudo service postgresql restart diff --git a/hugegraph-dist/src/assembly/travis/start-server.sh b/hugegraph-dist/src/assembly/travis/start-server.sh index 873151d722..6bf0fd8cfe 100755 --- a/hugegraph-dist/src/assembly/travis/start-server.sh +++ b/hugegraph-dist/src/assembly/travis/start-server.sh @@ -7,15 +7,29 @@ BASE_DIR=hugegraph-$VERSION BIN=$BASE_DIR/bin CONF=$BASE_DIR/conf/hugegraph.properties +# PostgreSQL configurations +POSTGRESQL_DRIVER=org.postgresql.Driver +POSTGRESQL_URL=jdbc:postgresql://localhost:5432/ +POSTGRESQL_USERNAME=postgres + declare -A backend_serializer_map=(["memory"]="text" ["cassandra"]="cassandra" \ ["scylladb"]="scylladb" ["mysql"]="mysql" \ - ["hbase"]="hbase" ["rocksdb"]="binary") + ["hbase"]="hbase" ["rocksdb"]="binary" \ + ["postgresql"]="postgresql") SERIALIZER=${backend_serializer_map[$BACKEND]} +# Set backend and serializer sed -i "s/backend=.*/backend=$BACKEND/" $CONF sed -i "s/serializer=.*/serializer=$SERIALIZER/" $CONF +# Set PostgreSQL configurations if needed +if [ "$BACKEND" == "postgresql" ]; then + sed -i "s/#jdbc.driver=.*/jdbc.driver=$POSTGRESQL_DRIVER/" $CONF + sed -i "s?#jdbc.url=.*?jdbc.url=$POSTGRESQL_URL?" $CONF + sed -i "s/#jdbc.username=.*/jdbc.username=$POSTGRESQL_USERNAME/" $CONF +fi + # Append schema.sync_deletion=true to config file echo "schema.sync_deletion=true" >> $CONF diff --git a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java index a93746b829..7d19f1638a 100644 --- a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java +++ b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java @@ -89,6 +89,9 @@ private static void registerBackend(String backend) { case "palo": registerPalo(); break; + case "postgresql": + registerPostgresql(); + break; default: throw new HugeException("Unsupported backend type '%s'", backend); } @@ -165,6 +168,18 @@ public static void registerPalo() { "com.baidu.hugegraph.backend.store.palo.PaloStoreProvider"); } + public static void registerPostgresql() { + // Register config + OptionSpace.register("postgresql", + "com.baidu.hugegraph.backend.store.postgresql.PostgresqlOptions"); + // Register serializer + SerializerFactory.register("postgresql", + "com.baidu.hugegraph.backend.store.postgresql.PostgresqlSerializer"); + // Register backend + BackendProviderFactory.register("postgresql", + "com.baidu.hugegraph.backend.store.postgresql.PostgresqlStoreProvider"); + } + public static void registerServer() { OptionSpace.register("server", "com.baidu.hugegraph.config.ServerOptions"); } diff --git a/hugegraph-dist/src/main/resources/backend.properties b/hugegraph-dist/src/main/resources/backend.properties index 98c446a84d..45f95d2f1c 100644 --- a/hugegraph-dist/src/main/resources/backend.properties +++ b/hugegraph-dist/src/main/resources/backend.properties @@ -1 +1 @@ -backends=[cassandra, scylladb, rocksdb, mysql, palo, hbase] +backends=[cassandra, scylladb, rocksdb, mysql, palo, hbase, postgresql] diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java index 6749596efa..4f29a9c979 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java @@ -147,8 +147,11 @@ public void createDatabase() { try (Connection conn = this.openWithoutDB(0)) { conn.createStatement().execute(sql); } catch (SQLException e) { - throw new BackendException("Failed to create database '%s'", - this.database); + if (!e.getMessage().endsWith("already exists")) { + throw new BackendException("Failed to create database '%s'", e, + this.database); + } + // Ignore exception if database already exists } } diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java index da6e62867c..1e8fc05d96 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java @@ -57,7 +57,7 @@ public abstract class MysqlStore extends AbstractBackendStore { private final Map tables; - private MysqlSessions sessions; + protected MysqlSessions sessions; public MysqlStore(final BackendStoreProvider provider, final String database, final String store) { @@ -114,7 +114,8 @@ public synchronized void open(HugeConfig config) { try { this.sessions.open(config); } catch (Exception e) { - if (!e.getMessage().startsWith("Unknown database")) { + if (!e.getMessage().startsWith("Unknown database") && + !e.getMessage().endsWith("does not exist")) { throw new BackendException("Failed connect with mysql, " + "please ensure it's ok", e); } diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java index 716e71d3a2..f675bc9c4d 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java @@ -521,7 +521,7 @@ protected void wrapOffset(StringBuilder select, Query query) { select.append(";"); } - private static Object serializeValue(Object value) { + protected static Object serializeValue(Object value) { if (value instanceof Id) { value = ((Id) value).asObject(); } diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java index c929bd4c4a..594aae3803 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java @@ -106,8 +106,8 @@ public void increaseCounter(Session session, HugeType type, long increment) { String update = String.format( "INSERT INTO %s VALUES ('%s', %s) " + - "ON DUPLICATE KEY UPDATE " + - "ID = ID + 1;", TABLE, type.name(), increment); + "ON DUPLICATE KEY UPDATE ID = ID + %s;", + TABLE, type.name(), increment, increment); try { session.execute(update); } catch (SQLException e) { diff --git a/hugegraph-postgresql/pom.xml b/hugegraph-postgresql/pom.xml new file mode 100644 index 0000000000..fa3181582f --- /dev/null +++ b/hugegraph-postgresql/pom.xml @@ -0,0 +1,31 @@ + + + + hugegraph + com.baidu.hugegraph + 0.9.2 + + 4.0.0 + + hugegraph-postgresql + + + + com.baidu.hugegraph + hugegraph-core + ${project.version} + + + com.baidu.hugegraph + hugegraph-mysql + ${project.version} + + + org.postgresql + postgresql + 42.1.4 + + + diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlFeatures.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlFeatures.java new file mode 100644 index 0000000000..8f033288a8 --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlFeatures.java @@ -0,0 +1,24 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import com.baidu.hugegraph.backend.store.mysql.MysqlFeatures; + +public class PostgresqlFeatures extends MysqlFeatures {} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java new file mode 100644 index 0000000000..bd68cf4889 --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java @@ -0,0 +1,82 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import com.baidu.hugegraph.config.ConfigOption; +import com.baidu.hugegraph.config.OptionHolder; + +import static com.baidu.hugegraph.config.OptionChecker.*; + +public class PostgresqlOptions extends OptionHolder { + + private PostgresqlOptions() { + super(); + } + + private static volatile PostgresqlOptions instance; + + public static synchronized PostgresqlOptions instance() { + if (instance == null) { + instance = new PostgresqlOptions(); + instance.registerOptions(); + } + return instance; + } + + public static final ConfigOption JDBC_DRIVER = + new ConfigOption<>( + "jdbc.driver", + "The JDBC driver class to connect database.", + disallowEmpty(), + "org.postgresql.Driver" + ); + + public static final ConfigOption JDBC_URL = + new ConfigOption<>( + "jdbc.url", + "The url of database in JDBC format.", + disallowEmpty(), + "jdbc:postgresql://127.0.0.1:5432" + ); + + public static final ConfigOption SSL_MODE = + new ConfigOption<>( + "jdbc.ssl_mode", + "The url of database in JDBC format.", + disallowEmpty(), + "disable" + ); + + public static final ConfigOption JDBC_USERNAME = + new ConfigOption<>( + "jdbc.username", + "The username to login database.", + disallowEmpty(), + "root" + ); + + public static final ConfigOption JDBC_PASSWORD = + new ConfigOption<>( + "jdbc.password", + "The password corresponding to jdbc.username.", + null, + "" + ); +} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java new file mode 100644 index 0000000000..adb465743a --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java @@ -0,0 +1,24 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import com.baidu.hugegraph.backend.store.mysql.MysqlSerializer; + +public class PostgresqlSerializer extends MysqlSerializer {} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java new file mode 100644 index 0000000000..204e889ea3 --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java @@ -0,0 +1,36 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import com.baidu.hugegraph.backend.store.mysql.MysqlSessions; +import com.baidu.hugegraph.config.HugeConfig; + +public class PostgresqlSessions extends MysqlSessions { + + public PostgresqlSessions(HugeConfig config, String database, String store) { + super(config, database, store); + } + + @Override + protected String buildCreateDatabase(String database) { + return String.format("CREATE DATABASE %s ENCODING='UTF-8' " + + "TEMPLATE=template0 LC_COLLATE='C' LC_CTYPE='C';", database); + } +} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStore.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStore.java new file mode 100644 index 0000000000..165f6d64eb --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStore.java @@ -0,0 +1,40 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import com.baidu.hugegraph.backend.store.BackendFeatures; +import com.baidu.hugegraph.backend.store.BackendStoreProvider; +import com.baidu.hugegraph.backend.store.mysql.MysqlStore; +import com.baidu.hugegraph.config.HugeConfig; + +public abstract class PostgresqlStore extends MysqlStore { + + private static final BackendFeatures FEATURES = new PostgresqlFeatures(); + + public PostgresqlStore(BackendStoreProvider provider, + String database, String name) { + super(provider, database, name); + } + + @Override + protected PostgresqlSessions openSessionPool(HugeConfig config) { + return new PostgresqlSessions(config, this.database(), this.store()); + } +} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStoreProvider.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStoreProvider.java new file mode 100644 index 0000000000..52521518f6 --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStoreProvider.java @@ -0,0 +1,139 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.store.BackendStore; +import com.baidu.hugegraph.backend.store.BackendStoreProvider; +import com.baidu.hugegraph.backend.store.mysql.MysqlSessions; +import com.baidu.hugegraph.backend.store.mysql.MysqlStoreProvider; +import com.baidu.hugegraph.backend.store.mysql.MysqlTable; +import com.baidu.hugegraph.type.HugeType; +import com.baidu.hugegraph.type.define.Directions; + +public class PostgresqlStoreProvider extends MysqlStoreProvider { + + @Override + protected BackendStore newSchemaStore(String store) { + return new PostgresqlSchemaStore(this, this.database(), store); + } + + @Override + protected BackendStore newGraphStore(String store) { + return new PostgresqlGraphStore(this, this.database(), store); + } + + @Override + public String type() { + return "postgresql"; + } + + @Override + public String version() { + return "1.0"; + } + + public static class PostgresqlSchemaStore extends PostgresqlStore { + + private final PostgresqlTables.Counters counters; + + public PostgresqlSchemaStore(BackendStoreProvider provider, + String database, String store) { + super(provider, database, store); + + this.counters = new PostgresqlTables.Counters(); + + registerTableManager(HugeType.VERTEX_LABEL, + new PostgresqlTables.VertexLabel()); + registerTableManager(HugeType.EDGE_LABEL, + new PostgresqlTables.EdgeLabel()); + registerTableManager(HugeType.PROPERTY_KEY, + new PostgresqlTables.PropertyKey()); + registerTableManager(HugeType.INDEX_LABEL, + new PostgresqlTables.IndexLabel()); + } + + @Override + protected Collection tables() { + List tables = new ArrayList<>(super.tables()); + tables.add(this.counters); + return tables; + } + + @Override + public void increaseCounter(HugeType type, long increment) { + this.checkSessionConnected(); + MysqlSessions.Session session = super.sessions.session(); + this.counters.increaseCounter(session, type, increment); + } + + @Override + public long getCounter(HugeType type) { + this.checkSessionConnected(); + MysqlSessions.Session session = super.sessions.session(); + return this.counters.getCounter(session, type); + } + } + + public static class PostgresqlGraphStore extends PostgresqlStore { + + public PostgresqlGraphStore(BackendStoreProvider provider, + String database, String store) { + super(provider, database, store); + + registerTableManager(HugeType.VERTEX, + new PostgresqlTables.Vertex(store)); + registerTableManager(HugeType.EDGE_OUT, + new PostgresqlTables.Edge(store, + Directions.OUT)); + registerTableManager(HugeType.EDGE_IN, + new PostgresqlTables.Edge(store, + Directions.IN)); + registerTableManager(HugeType.SECONDARY_INDEX, + new PostgresqlTables.SecondaryIndex(store)); + registerTableManager(HugeType.RANGE_INDEX, + new PostgresqlTables.RangeIndex(store)); + registerTableManager(HugeType.SEARCH_INDEX, + new PostgresqlTables.SearchIndex(store)); + } + + @Override + public Id nextId(HugeType type) { + throw new UnsupportedOperationException( + "PostgresqlGraphStore.nextId()"); + } + + @Override + public void increaseCounter(HugeType type, long increment) { + throw new UnsupportedOperationException( + "PostgresqlGraphStore.increaseCounter()"); + } + + @Override + public long getCounter(HugeType type) { + throw new UnsupportedOperationException( + "PostgresqlGraphStore.getCounter()"); + } + } +} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java new file mode 100644 index 0000000000..c222ca3766 --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java @@ -0,0 +1,341 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.postgresql.core.Utils; +import org.slf4j.Logger; + +import com.baidu.hugegraph.HugeException; +import com.baidu.hugegraph.backend.BackendException; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.query.Condition; +import com.baidu.hugegraph.backend.query.Query; +import com.baidu.hugegraph.backend.store.TableDefine; +import com.baidu.hugegraph.backend.store.mysql.MysqlBackendEntry; +import com.baidu.hugegraph.backend.store.mysql.MysqlEntryIterator; +import com.baidu.hugegraph.backend.store.mysql.MysqlSessions.Session; +import com.baidu.hugegraph.backend.store.mysql.MysqlTable; +import com.baidu.hugegraph.backend.store.mysql.WhereBuilder; +import com.baidu.hugegraph.type.define.HugeKeys; +import com.baidu.hugegraph.util.Log; + +public abstract class PostgresqlTable extends MysqlTable { + + private static final Logger LOG = Log.logger(PostgresqlStore.class); + + private String insertTemplate; + + public PostgresqlTable(String table) { + super(table); + } + + @Override + protected void createTable(Session session, TableDefine tableDefine) { + StringBuilder sql = new StringBuilder(); + sql.append("CREATE TABLE IF NOT EXISTS "); + sql.append(this.table()).append(" ("); + // Add columns + for (Map.Entry entry : + tableDefine.columns().entrySet()) { + sql.append(formatKey(entry.getKey())); + sql.append(" "); + sql.append(entry.getValue()); + sql.append(", "); + } + // Specified primary keys + sql.append(" PRIMARY KEY ("); + int i = 0; + int size = tableDefine.keys().size(); + for (HugeKeys key : tableDefine.keys()) { + sql.append(formatKey(key)); + if (++i != size) { + sql.append(", "); + } + } + sql.append("))"); + + LOG.debug("Create table: {}", sql); + try { + session.execute(sql.toString()); + } catch (SQLException e) { + throw new BackendException("Failed to create table with '%s'", + e, sql); + } + } + + @Override + protected void dropTable(Session session) { + LOG.debug("Drop table: {}", this.table()); + String sql = String.format("DROP TABLE IF EXISTS %s CASCADE;", + this.table()); + try { + session.execute(sql); + } catch (SQLException e) { + throw new BackendException("Failed to drop table with '%s'", + e, sql); + } + } + + @Override + protected void truncateTable(Session session) { + LOG.debug("Truncate table: {}", this.table()); + String sql = String.format("TRUNCATE TABLE %s CASCADE;", this.table()); + try { + session.execute(sql); + } catch (SQLException e) { + throw new BackendException("Failed to truncate table with '%s'", + e, sql); + } + } + + /** + * Insert an entire row + */ + @Override + public void insert(Session session, MysqlBackendEntry.Row entry) { + String template = this.buildInsertTemplate(entry); + + PreparedStatement insertStmt; + try { + // Create or get insert prepare statement + insertStmt = session.prepareStatement(template); + int i = 1; + int size = entry.columns().size(); + for (Object object : entry.columns().values()) { + if (object.equals("\u0000")) { + object = ""; + } + insertStmt.setObject(i, object); + insertStmt.setObject(size + i++, object); + } + } catch (SQLException e) { + throw new BackendException("Failed to prepare statement '%s'" + + "for entry: %s", template, entry); + } + session.add(insertStmt); + } + + @Override + public void delete(Session session, MysqlBackendEntry.Row entry) { + List idNames = this.idColumnName(); + String template = this.buildDeleteTemplate(idNames); + + PreparedStatement deleteStmt; + try { + deleteStmt = session.prepareStatement(template); + if (entry.columns().isEmpty()) { + // Delete just by id + List idValues = this.idColumnValue(entry); + assert idNames.size() == idValues.size(); + + for (int i = 0, n = idNames.size(); i < n; i++) { + deleteStmt.setObject(i + 1, idValues.get(i)); + } + } else { + // Delete just by column keys(must be id columns) + for (int i = 0, n = idNames.size(); i < n; i++) { + HugeKeys key = idNames.get(i); + Object value = entry.column(key); + if (value != null && value.equals("\u0000")) { + value = "\'\'"; + } + + deleteStmt.setObject(i + 1, value); + } + } + } catch (SQLException e) { + throw new BackendException("Failed to prepare statement '%s'" + + "with entry columns %s", + template, entry.columns().values()); + } + session.add(deleteStmt); + } + + protected String buildInsertTemplate(MysqlBackendEntry.Row entry) { + if (this.insertTemplate != null) { + return this.insertTemplate; + } + + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO ").append(this.table()).append(" ("); + + int i = 0; + int size = entry.columns().size(); + for (HugeKeys key : entry.columns().keySet()) { + insert.append(formatKey(key)); + if (++i != size) { + insert.append(", "); + } + } + insert.append(") VALUES ("); + + for (i = 0; i < size; i++) { + insert.append("?"); + if (i != size - 1) { + insert.append(", "); + } + } + insert.append(")"); + + i = 0; + size = this.tableDefine().keys().size(); + insert.append(" ON CONFLICT ("); + for (HugeKeys key : this.tableDefine().keys()) { + insert.append(formatKey(key)); + if (++i != size) { + insert.append(", "); + } + } + insert.append(")"); + + i = 0; + size = entry.columns().keySet().size(); + insert.append(" DO UPDATE SET "); + for (HugeKeys key : entry.columns().keySet()) { + insert.append(formatKey(key)).append(" = ?"); + if (++i != size) { + insert.append(", "); + } + } + + this.insertTemplate = insert.toString(); + return this.insertTemplate; + } + + protected void wrapPage(StringBuilder select, Query query) { + String page = query.page(); + // It's the first time if page is empty + if (!page.isEmpty()) { + MysqlEntryIterator.PageState + pageState = MysqlEntryIterator.PageState.fromString(page); + Map columns = pageState.columns(); + + List idColumnNames = this.idColumnName(); + List values = new ArrayList<>(idColumnNames.size()); + for (HugeKeys key : idColumnNames) { + values.add(columns.get(key)); + } + + // Need add `where` to `select` when query is IdQuery + boolean startWithWhere = query.conditions().isEmpty(); + WhereBuilder where = new WhereBuilder(startWithWhere); + where.gte(formatKeys(idColumnNames), values); + if (!startWithWhere) { + select.append(" AND"); + } + select.append(where.build()); + } + + int i = 0; + int size = this.tableDefine().keys().size(); + + // Set order-by to keep results order consistence for result + select.append(" ORDER BY "); + for (HugeKeys hugeKey : this.tableDefine().keys()) { + String key = formatKey(hugeKey); + select.append(key).append(" "); + select.append("ASC "); + if (++i != size) { + select.append(", "); + } + } + + assert query.limit() != Query.NO_LIMIT; + // Fetch `limit + 1` records for judging whether reached the last page + select.append(" limit "); + select.append(query.limit() + 1); + select.append(";"); + } + + @Override + protected StringBuilder relation2Sql(Condition.Relation relation) { + String key = relation.serialKey().toString(); + Object value = relation.serialValue(); + + value = serializeValue(value); + + StringBuilder sql = new StringBuilder(32); + sql.append(key); + switch (relation.relation()) { + case EQ: + sql.append(" = ").append(value); + break; + case NEQ: + sql.append(" != ").append(value); + break; + case GT: + sql.append(" > ").append(value); + break; + case GTE: + sql.append(" >= ").append(value); + break; + case LT: + sql.append(" < ").append(value); + break; + case LTE: + sql.append(" <= ").append(value); + break; + case IN: + sql.append(" IN ("); + List values = (List) value; + for (int i = 0, n = values.size(); i < n; i++) { + sql.append(serializeValue(values.get(i))); + if (i != n - 1) { + sql.append(", "); + } + } + sql.append(")"); + break; + case CONTAINS: + case CONTAINS_KEY: + case SCAN: + default: + throw new AssertionError("Unsupported relation: " + relation); + } + return sql; + } + + protected static Object serializeValue(Object value) { + if (value instanceof Id) { + value = ((Id) value).asObject(); + } + if (value instanceof String) { + if (value == "\u0000") { + return "\'\'"; + } + StringBuilder builder = new StringBuilder(32); + builder.append('\''); + try { + Utils.escapeLiteral(builder, (String) value, false); + } catch (SQLException e) { + throw new HugeException("Failed to escape '%s'", e, value); + } + builder.append('\''); + value = builder.toString(); + } + return value; + } +} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java new file mode 100644 index 0000000000..794c3e9182 --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java @@ -0,0 +1,422 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.baidu.hugegraph.backend.BackendException; +import com.baidu.hugegraph.backend.id.EdgeId; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.id.IdGenerator; +import com.baidu.hugegraph.backend.id.IdUtil; +import com.baidu.hugegraph.backend.id.SplicingIdGenerator; +import com.baidu.hugegraph.backend.store.BackendEntry; +import com.baidu.hugegraph.backend.store.TableDefine; +import com.baidu.hugegraph.backend.store.mysql.MysqlBackendEntry; +import com.baidu.hugegraph.backend.store.mysql.MysqlSessions.Session; +import com.baidu.hugegraph.backend.store.mysql.MysqlTables; +import com.baidu.hugegraph.type.HugeType; +import com.baidu.hugegraph.type.define.Directions; +import com.baidu.hugegraph.type.define.HugeKeys; +import com.baidu.hugegraph.util.E; + +public class PostgresqlTables { + + private static final String NOT_NULL = "NOT NULL"; + private static final String DEFAULT_EMPTY = "DEFAULT ''"; + + private static final String DATATYPE_PK = "INT"; + private static final String DATATYPE_SL = "INT"; // VL/EL + private static final String DATATYPE_IL = "INT"; + + private static final String INT = "INT"; + private static final String FLOAT = "FLOAT"; + private static final String VARCHAR = "VARCHAR(255)"; + private static final String TEXT = "VARCHAR(65533)"; + private static final String BOOL = "BOOL"; + + public static class PostgresqlTableTemplate extends PostgresqlTable { + + protected TableDefine define; + + public PostgresqlTableTemplate(String table) { + super(table); + } + + @Override + public TableDefine tableDefine() { + return this.define; + } + } + + public static class Counters extends PostgresqlTableTemplate { + + public static final String TABLE = "counters"; + + public Counters() { + super(TABLE); + + this.define = new TableDefine(); + this.define.column(HugeKeys.SCHEMA_TYPE, VARCHAR); + this.define.column(HugeKeys.ID, INT); + this.define.keys(HugeKeys.SCHEMA_TYPE); + } + + public long getCounter(Session session, HugeType type) { + String schemaCol = formatKey(HugeKeys.SCHEMA_TYPE); + String idCol = formatKey(HugeKeys.ID); + + String select = String.format("SELECT ID FROM %s WHERE %s = '%s';", + TABLE, schemaCol, type.name()); + try { + ResultSet resultSet = session.select(select); + if (resultSet.next()) { + return resultSet.getLong(idCol); + } else { + return 0L; + } + } catch (SQLException e) { + throw new BackendException( + "Failed to get id from counters with type '%s'", + e, type); + } + } + + public void increaseCounter(Session session, HugeType type, + long increment) { + String update = String.format("INSERT INTO %s (%s, %s) VALUES " + + "('%s', %s) ON CONFLICT (%s) " + + "DO UPDATE SET ID = %s.ID + %s;", + TABLE, + formatKey(HugeKeys.SCHEMA_TYPE), + formatKey(HugeKeys.ID), + type.name(), increment, + formatKey(HugeKeys.SCHEMA_TYPE), + TABLE, increment); + try { + session.execute(update); + } catch (SQLException e) { + throw new BackendException( + "Failed to update counters with type '%s'", e, type); + } + } + } + + public static class VertexLabel extends PostgresqlTableTemplate { + + public static final String TABLE = "vertex_labels"; + + public VertexLabel() { + super(TABLE); + this.define = new TableDefine(); + this.define.column(HugeKeys.ID, DATATYPE_SL, NOT_NULL); + this.define.column(HugeKeys.NAME, VARCHAR, NOT_NULL); + this.define.column(HugeKeys.ID_STRATEGY, INT, NOT_NULL); + this.define.column(HugeKeys.PRIMARY_KEYS, VARCHAR, DEFAULT_EMPTY); + this.define.column(HugeKeys.PROPERTIES, VARCHAR, DEFAULT_EMPTY); + this.define.column(HugeKeys.NULLABLE_KEYS, VARCHAR, DEFAULT_EMPTY); + this.define.column(HugeKeys.INDEX_LABELS, VARCHAR, DEFAULT_EMPTY); + this.define.column(HugeKeys.ENABLE_LABEL_INDEX, BOOL, NOT_NULL); + this.define.column(HugeKeys.USER_DATA, VARCHAR, DEFAULT_EMPTY); + this.define.column(HugeKeys.STATUS, INT, NOT_NULL); + // Unique keys/hash keys + this.define.keys(HugeKeys.ID); + } + } + + public static class EdgeLabel extends PostgresqlTableTemplate { + + public static final String TABLE = "edge_labels"; + + public EdgeLabel() { + super(TABLE); + this.define = new TableDefine(); + this.define.column(HugeKeys.ID, DATATYPE_SL, NOT_NULL); + this.define.column(HugeKeys.NAME, VARCHAR, NOT_NULL); + this.define.column(HugeKeys.FREQUENCY, INT, NOT_NULL); + this.define.column(HugeKeys.SOURCE_LABEL, INT, NOT_NULL); + this.define.column(HugeKeys.TARGET_LABEL, INT, NOT_NULL); + this.define.column(HugeKeys.SORT_KEYS, VARCHAR, DEFAULT_EMPTY); + this.define.column(HugeKeys.PROPERTIES, VARCHAR, DEFAULT_EMPTY); + this.define.column(HugeKeys.NULLABLE_KEYS, VARCHAR, DEFAULT_EMPTY); + this.define.column(HugeKeys.INDEX_LABELS, VARCHAR, DEFAULT_EMPTY); + this.define.column(HugeKeys.ENABLE_LABEL_INDEX, BOOL, NOT_NULL); + this.define.column(HugeKeys.USER_DATA, VARCHAR, DEFAULT_EMPTY); + this.define.column(HugeKeys.STATUS, INT, NOT_NULL); + // Unique keys/hash keys + this.define.keys(HugeKeys.ID); + } + } + + public static class PropertyKey extends PostgresqlTableTemplate { + + public static final String TABLE = "property_keys"; + + public PropertyKey() { + super(TABLE); + this.define = new TableDefine(); + this.define.column(HugeKeys.ID, DATATYPE_PK, NOT_NULL); + this.define.column(HugeKeys.NAME, VARCHAR, NOT_NULL); + this.define.column(HugeKeys.DATA_TYPE, INT, NOT_NULL); + this.define.column(HugeKeys.CARDINALITY, INT, NOT_NULL); + this.define.column(HugeKeys.PROPERTIES, VARCHAR, DEFAULT_EMPTY); + this.define.column(HugeKeys.USER_DATA, VARCHAR, DEFAULT_EMPTY); + this.define.column(HugeKeys.STATUS, INT, NOT_NULL); + // Unique keys/hash keys + this.define.keys(HugeKeys.ID); + } + } + + public static class IndexLabel extends PostgresqlTableTemplate { + + public static final String TABLE = "index_labels"; + + public IndexLabel() { + super(TABLE); + this.define = new TableDefine(); + this.define.column(HugeKeys.ID, DATATYPE_IL, NOT_NULL); + this.define.column(HugeKeys.NAME, VARCHAR, NOT_NULL); + this.define.column(HugeKeys.BASE_TYPE, INT, NOT_NULL); + this.define.column(HugeKeys.BASE_VALUE, INT, NOT_NULL); + this.define.column(HugeKeys.INDEX_TYPE, INT, NOT_NULL); + this.define.column(HugeKeys.FIELDS, VARCHAR, NOT_NULL); + this.define.column(HugeKeys.STATUS, INT, NOT_NULL); + // Unique keys/hash keys + this.define.keys(HugeKeys.ID); + } + } + + public static class Vertex extends PostgresqlTableTemplate { + + public static final String TABLE = "vertices"; + + public Vertex(String store) { + super(joinTableName(store, TABLE)); + + this.define = new TableDefine(); + this.define.column(HugeKeys.ID, VARCHAR, NOT_NULL); + this.define.column(HugeKeys.LABEL, INT, NOT_NULL); + this.define.column(HugeKeys.PROPERTIES, TEXT, DEFAULT_EMPTY); + // Unique keys/hash keys + this.define.keys(HugeKeys.ID); + } + } + + public static class Edge extends PostgresqlTableTemplate { + + private final Directions direction; + private final String delByLabelTemplate; + + public Edge(String store, Directions direction) { + super(joinTableName(store, MysqlTables.Edge.table(direction))); + + this.direction = direction; + this.delByLabelTemplate = String.format( + "DELETE FROM %s WHERE %s = ?;", + this.table(), formatKey(HugeKeys.LABEL)); + + this.define = new TableDefine(); + this.define.column(HugeKeys.OWNER_VERTEX, VARCHAR, NOT_NULL); + this.define.column(HugeKeys.DIRECTION, INT, NOT_NULL); + this.define.column(HugeKeys.LABEL, INT, NOT_NULL); + this.define.column(HugeKeys.SORT_VALUES, VARCHAR, NOT_NULL, + DEFAULT_EMPTY); + this.define.column(HugeKeys.OTHER_VERTEX, VARCHAR, NOT_NULL); + this.define.column(HugeKeys.PROPERTIES, TEXT, DEFAULT_EMPTY); + // Unique keys/hash keys + this.define.keys(HugeKeys.OWNER_VERTEX, HugeKeys.DIRECTION, + HugeKeys.LABEL, HugeKeys.SORT_VALUES, + HugeKeys.OTHER_VERTEX); + } + + @Override + protected List idColumnValue(Id id) { + EdgeId edgeId; + if (!(id instanceof EdgeId)) { + String[] idParts = EdgeId.split(id); + if (idParts.length == 1) { + // Delete edge by label + return Arrays.asList((Object[]) idParts); + } + id = IdUtil.readString(id.asString()); + edgeId = EdgeId.parse(id.asString()); + } else { + edgeId = (EdgeId) id; + } + + E.checkState(edgeId.direction() == this.direction, + "Can't query %s edges from %s edges table", + edgeId.direction(), this.direction); + + List list = new ArrayList<>(5); + list.add(IdUtil.writeString(edgeId.ownerVertexId())); + list.add(edgeId.direction().code()); + list.add(edgeId.edgeLabelId().asLong()); + list.add(edgeId.sortValues()); + list.add(IdUtil.writeString(edgeId.otherVertexId())); + return list; + } + + @Override + public void delete(Session session, MysqlBackendEntry.Row entry) { + // Let super class do delete if not deleting edge by label + List idParts = this.idColumnValue(entry.id()); + if (idParts.size() > 1 || entry.columns().size() > 0) { + super.delete(session, entry); + return; + } + + // The only element is label + this.deleteEdgesByLabel(session, entry.id()); + } + + private void deleteEdgesByLabel(Session session, Id label) { + PreparedStatement deleteStmt; + try { + // Create or get delete prepare statement + deleteStmt = session.prepareStatement(this.delByLabelTemplate); + // Delete edges + deleteStmt.setObject(1, label.asLong()); + } catch (SQLException e) { + throw new BackendException("Failed to prepare statement '%s'", + this.delByLabelTemplate); + } + session.add(deleteStmt); + } + + @Override + protected BackendEntry mergeEntries(BackendEntry e1, BackendEntry e2) { + // Merge edges into vertex + // TODO: merge rows before calling row2Entry() + + MysqlBackendEntry current = (MysqlBackendEntry) e1; + MysqlBackendEntry next = (MysqlBackendEntry) e2; + + E.checkState(current == null || current.type().isVertex(), + "The current entry must be null or VERTEX"); + E.checkState(next != null && next.type().isEdge(), + "The next entry must be EDGE"); + + if (current != null) { + Id nextVertexId = IdGenerator.of( + next.column(HugeKeys.OWNER_VERTEX)); + if (current.id().equals(nextVertexId)) { + current.subRow(next.row()); + return current; + } + } + + return this.wrapByVertex(next); + } + + private MysqlBackendEntry wrapByVertex(MysqlBackendEntry edge) { + assert edge.type().isEdge(); + String ownerVertex = edge.column(HugeKeys.OWNER_VERTEX); + E.checkState(ownerVertex != null, "Invalid backend entry"); + Id vertexId = IdGenerator.of(ownerVertex); + MysqlBackendEntry vertex = new MysqlBackendEntry(HugeType.VERTEX, + vertexId); + + vertex.column(HugeKeys.ID, ownerVertex); + vertex.column(HugeKeys.PROPERTIES, ""); + + vertex.subRow(edge.row()); + return vertex; + } + } + + public abstract static class Index extends PostgresqlTableTemplate { + + public Index(String table) { + super(table); + } + + protected abstract String entryId(MysqlBackendEntry entry); + } + + public static class SecondaryIndex extends Index { + + public static final String TABLE = "secondary_indexes"; + + public SecondaryIndex(String store) { + this(store, TABLE); + } + + protected SecondaryIndex(String store, String table) { + super(joinTableName(store, table)); + + this.define = new TableDefine(); + this.define.column(HugeKeys.FIELD_VALUES, VARCHAR, NOT_NULL); + this.define.column(HugeKeys.INDEX_LABEL_ID, INT, NOT_NULL); + this.define.column(HugeKeys.ELEMENT_IDS, VARCHAR, NOT_NULL); + // Unique keys/hash keys + this.define.keys(HugeKeys.FIELD_VALUES, + HugeKeys.INDEX_LABEL_ID, + HugeKeys.ELEMENT_IDS); + } + + @Override + protected final String entryId(MysqlBackendEntry entry) { + String fieldValues = entry.column(HugeKeys.FIELD_VALUES); + Long labelId = entry.column(HugeKeys.INDEX_LABEL_ID); + return SplicingIdGenerator.concat(fieldValues, labelId.toString()); + } + } + + public static class SearchIndex extends SecondaryIndex { + + public static final String TABLE = "search_indexes"; + + public SearchIndex(String store) { + super(store, TABLE); + } + } + + public static class RangeIndex extends Index { + + public static final String TABLE = "range_indexes"; + + public RangeIndex(String store) { + super(joinTableName(store, TABLE)); + + this.define = new TableDefine(); + this.define.column(HugeKeys.INDEX_LABEL_ID, INT, NOT_NULL); + this.define.column(HugeKeys.FIELD_VALUES, FLOAT, NOT_NULL); + this.define.column(HugeKeys.ELEMENT_IDS, VARCHAR, NOT_NULL); + // Unique keys/hash keys + this.define.keys(HugeKeys.INDEX_LABEL_ID, + HugeKeys.FIELD_VALUES, + HugeKeys.ELEMENT_IDS); + } + + @Override + protected final String entryId(MysqlBackendEntry entry) { + Double fieldValue = entry.column(HugeKeys.FIELD_VALUES); + Long labelId = entry.column(HugeKeys.INDEX_LABEL_ID); + return SplicingIdGenerator.concat(labelId.toString(), + fieldValue.toString()); + } + } +} diff --git a/hugegraph-test/pom.xml b/hugegraph-test/pom.xml index fdc9d0e4d4..ae5d845772 100644 --- a/hugegraph-test/pom.xml +++ b/hugegraph-test/pom.xml @@ -47,6 +47,11 @@ hugegraph-mysql ${project.version} + + com.baidu.hugegraph + hugegraph-postgresql + ${project.version} + com.baidu.hugegraph hugegraph-dist @@ -293,5 +298,15 @@ hbase + + postgresql + + false + + + postgresql + postgresql + + diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/MetricsApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/MetricsApiTest.java index bff59662df..7175785e85 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/MetricsApiTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/MetricsApiTest.java @@ -68,6 +68,7 @@ public void testMetricsBackend() { case "memory": case "mysql": case "hbase": + case "postgresql": String except = (String) assertMapContains(graph, "exception"); Assert.assertTrue(except, except.contains(notSupport)); break; diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/TaskApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/TaskApiTest.java index 040c22c0de..75663669ff 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/TaskApiTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/TaskApiTest.java @@ -56,7 +56,7 @@ public void prepareSchema() { public void testList() { int taskId = this.rebuild(); - Response r = client().get(path); + Response r = client().get(path, ImmutableMap.of("limit", -1)); String content = assertResponseStatus(200, r); List> tasks = assertJsonContains(content, "tasks"); assertArrayContains(tasks, "id", taskId); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java index 21bb51ccc3..f855ad10a8 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java @@ -2394,7 +2394,7 @@ public void testQueryEdgeByPropertyWithEmptyString() { Assert.assertEquals("", edge.value("tool")); edge = graph.traversal().E().has("tool", "").has("place", "park") - .has("reason", "jeer").next(); + .has("reason", "jeer").next(); Assert.assertEquals(1, (int) edge.value("id")); } @@ -2578,6 +2578,7 @@ public void testQueryEdgeByPage() { Assert.assertEquals(90, edges.size()); } + @SuppressWarnings("unchecked") @Test public void testQueryEdgeByPageResultsMatched() { Assume.assumeTrue("Not support paging", @@ -2593,17 +2594,19 @@ public void testQueryEdgeByPageResultsMatched() { String page = PageState.PAGE_NONE; int size = 20; + Set pageAll = new HashSet<>(); for (int i = 0; i < 100 / size; i++) { iter = graph.traversal().E() .has("~page", page).limit(size); - List vertexes = IteratorUtils.asList(iter); - Assert.assertEquals(size, vertexes.size()); + List edges = IteratorUtils.asList(iter); + Assert.assertEquals(size, edges.size()); - List expected = all.subList(i * size, (i + 1) * size); - Assert.assertEquals(expected, vertexes); + pageAll.addAll(edges); page = TraversalUtil.page(iter); } + Assert.assertEquals(100, pageAll.size()); + Assert.assertTrue(all.containsAll(pageAll)); Assert.assertNull(page); } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java index 98c1822bb5..8dbf46d3ff 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java @@ -3265,6 +3265,7 @@ public void testQueryByPage() { Assert.assertEquals(90, vertexes.size()); } + @SuppressWarnings("unchecked") @Test public void testQueryByPageResultsMatched() { Assume.assumeTrue("Not support paging", @@ -3280,17 +3281,19 @@ public void testQueryByPageResultsMatched() { String page = PageState.PAGE_NONE; int size = 20; + Set pageAll = new HashSet<>(); for (int i = 0; i < 100 / size; i++) { iter = graph.traversal().V() .has("~page", page).limit(size); - List vertexes = IteratorUtils.asList(iter); + List vertexes = IteratorUtils.asList(iter); Assert.assertEquals(size, vertexes.size()); - List expected = all.subList(i * size, (i + 1) * size); - Assert.assertEquals(expected, vertexes); + pageAll.addAll(vertexes); page = TraversalUtil.page(iter); } + Assert.assertEquals(100, pageAll.size()); + Assert.assertTrue(all.containsAll(pageAll)); Assert.assertNull(page); } diff --git a/hugegraph-test/src/main/resources/hugegraph.properties b/hugegraph-test/src/main/resources/hugegraph.properties index 479584a48c..9dbc8ecc97 100644 --- a/hugegraph-test/src/main/resources/hugegraph.properties +++ b/hugegraph-test/src/main/resources/hugegraph.properties @@ -37,6 +37,7 @@ jdbc.username=root jdbc.password= jdbc.reconnect_max_times=3 jdbc.reconnect_interval=3 +jdbc.sslmode=disable # palo backend config palo.host=localhost diff --git a/pom.xml b/pom.xml index 0c2a9bf326..4c2ac6ef61 100644 --- a/pom.xml +++ b/pom.xml @@ -124,6 +124,7 @@ hugegraph-mysql hugegraph-palo hugegraph-hbase + hugegraph-postgresql From ebf79e81cdee0fddec1b56f32e600a5b415484af Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Mon, 29 Apr 2019 17:32:34 +0800 Subject: [PATCH 02/10] adjust table define Change-Id: I155d9f121dc47309c0aab3181cb82916bf62d91f --- .travis.yml | 3 - .../hugegraph/backend/store/TableDefine.java | 17 +- .../assembly/static/conf/hugegraph.properties | 2 +- .../src/assembly/travis/install-postgresql.sh | 5 +- .../backend/store/mysql/MysqlOptions.java | 10 +- .../backend/store/mysql/MysqlTable.java | 41 ++- .../backend/store/mysql/MysqlTables.java | 124 +++++-- .../store/postgresql/PostgresqlOptions.java | 24 +- .../postgresql/PostgresqlSerializer.java | 33 +- .../store/postgresql/PostgresqlTable.java | 223 +----------- .../store/postgresql/PostgresqlTables.java | 334 ++++-------------- .../baidu/hugegraph/core/EdgeCoreTest.java | 2 +- .../baidu/hugegraph/core/VertexCoreTest.java | 2 +- 13 files changed, 258 insertions(+), 562 deletions(-) diff --git a/.travis.yml b/.travis.yml index b6dd5ac5ca..a2c687549b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,9 +5,6 @@ jdk: sudo: required -addons: - postgresql: "9.5" - cache: directories: - $HOME/.m2 diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java index 3dd4358e22..9bc6af4f89 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java @@ -29,20 +29,35 @@ import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.InsertionOrderUtil; +import jersey.repackaged.com.google.common.collect.ImmutableMap; + public class TableDefine { private final Map columns; private final List keys; + private final Map typesMapping; public TableDefine() { this.columns = InsertionOrderUtil.newMap(); this.keys = InsertionOrderUtil.newList(); + this.typesMapping = ImmutableMap.of(); + } + + public TableDefine(Map typesMapping) { + this.columns = InsertionOrderUtil.newMap(); + this.keys = InsertionOrderUtil.newList(); + this.typesMapping = typesMapping; } public TableDefine column(HugeKeys key, String... desc) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < desc.length; i++) { - sb.append(desc[i]); + String type = desc[i]; + if (i == 0 && this.typesMapping.containsKey(type)) { + type = this.typesMapping.get(type); + } + assert type != null; + sb.append(type); if (i != desc.length - 1) { sb.append(" "); } diff --git a/hugegraph-dist/src/assembly/static/conf/hugegraph.properties b/hugegraph-dist/src/assembly/static/conf/hugegraph.properties index 2973ffd51d..5a83ffd30b 100644 --- a/hugegraph-dist/src/assembly/static/conf/hugegraph.properties +++ b/hugegraph-dist/src/assembly/static/conf/hugegraph.properties @@ -39,7 +39,7 @@ cassandra.password= #jdbc.password= #jdbc.reconnect_max_times=3 #jdbc.reconnect_interval=3 - +#jdbc.sslmode=disable # palo backend config #palo.host=127.0.0.1 diff --git a/hugegraph-dist/src/assembly/travis/install-postgresql.sh b/hugegraph-dist/src/assembly/travis/install-postgresql.sh index 191cff1d6d..73f318e97a 100755 --- a/hugegraph-dist/src/assembly/travis/install-postgresql.sh +++ b/hugegraph-dist/src/assembly/travis/install-postgresql.sh @@ -3,7 +3,7 @@ set -ev TRAVIS_DIR=`dirname $0` -CONF=$TRAVIS_DIR/../../../../hugegraph-test/src/main/resources/hugegraph.properties +CONF=hugegraph-test/src/main/resources/hugegraph.properties POSTGRESQL_DRIVER=org.postgresql.Driver POSTGRESQL_URL=jdbc:postgresql://localhost:5432/ @@ -14,4 +14,5 @@ sed -i "s/jdbc.driver=.*/jdbc.driver=$POSTGRESQL_DRIVER/" $CONF sed -i "s?jdbc.url=.*?jdbc.url=$POSTGRESQL_URL?" $CONF sed -i "s/jdbc.username=.*/jdbc.username=$POSTGRESQL_USERNAME/" $CONF -sudo service postgresql restart +sudo service postgresql stop 9.2 +sudo service postgresql start 9.5 diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java index c6bd854864..db9fc53a43 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java @@ -27,7 +27,7 @@ public class MysqlOptions extends OptionHolder { - private MysqlOptions() { + protected MysqlOptions() { super(); } @@ -89,4 +89,12 @@ public static synchronized MysqlOptions instance() { rangeInt(1, 10), 3 ); + + public static final ConfigOption SSL_MODE = + new ConfigOption<>( + "jdbc.ssl_mode", + "The url of database in JDBC format.", + disallowEmpty(), + "disable" + ); } diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java index f675bc9c4d..6df2e7a39b 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Set; +import org.apache.logging.log4j.util.Strings; import org.slf4j.Logger; import com.baidu.hugegraph.backend.BackendException; @@ -50,6 +51,8 @@ public abstract class MysqlTable private static final Logger LOG = Log.logger(MysqlStore.class); + protected String dropTableTemplate = "DROP TABLE IF EXISTS %s;"; + protected String truncateTableTemplate = "TRUNCATE TABLE %s;"; // The template for insert and delete statements private String insertTemplate; private String deleteTemplate; @@ -91,14 +94,16 @@ protected void createTable(Session session, TableDefine tableDefine) { // Specified primary keys sql.append(" PRIMARY KEY ("); int i = 0; + int size = tableDefine.keys().size(); for (HugeKeys key : tableDefine.keys()) { - sql.append(key); - if (++i != tableDefine.keys().size()) { + sql.append(formatKey(key)); + if (++i != size) { sql.append(", "); } } - - sql.append(")) ENGINE=InnoDB;"); + sql.append("))"); + sql.append(this.engine()); + sql.append(";"); LOG.debug("Create table: {}", sql); try { @@ -109,9 +114,13 @@ protected void createTable(Session session, TableDefine tableDefine) { } } + protected String engine() { + return " ENGINE=InnoDB"; + } + protected void dropTable(Session session) { LOG.debug("Drop table: {}", this.table()); - String sql = String.format("DROP TABLE IF EXISTS %s;", this.table()); + String sql = String.format(this.dropTableTemplate, this.table()); try { session.execute(sql); } catch (SQLException e) { @@ -122,7 +131,7 @@ protected void dropTable(Session session) { protected void truncateTable(Session session) { LOG.debug("Truncate table: {}", this.table()); - String sql = String.format("TRUNCATE TABLE %s;", this.table()); + String sql = String.format(this.truncateTableTemplate, this.table()); try { session.execute(sql); } catch (SQLException e) { @@ -202,7 +211,7 @@ public void insert(Session session, MysqlBackendEntry.Row entry) { // Create or get insert prepare statement insertStmt = session.prepareStatement(template); int i = 1; - for (Object object : entry.columns().values()) { + for (Object object : this.insertTemplateObjects(entry)) { insertStmt.setObject(i++, object); } } catch (SQLException e) { @@ -413,7 +422,7 @@ protected StringBuilder relation2Sql(Condition.Relation relation) { Object value = relation.serialValue(); // Serialize value (TODO: should move to Serializer) - value = serializeValue(value); + value = this.serializeValue(value); StringBuilder sql = new StringBuilder(32); sql.append(key); @@ -503,6 +512,8 @@ protected void wrapPage(StringBuilder select, Query query) { select.append(where.build()); } + select.append(this.orderByKeys()); + assert query.limit() != Query.NO_LIMIT; // Fetch `limit + 1` records for judging whether reached the last page select.append(" limit "); @@ -510,6 +521,10 @@ protected void wrapPage(StringBuilder select, Query query) { select.append(";"); } + protected String orderByKeys() { + return Strings.EMPTY; + } + protected void wrapOffset(StringBuilder select, Query query) { assert query.limit() >= 0; assert query.offset() >= 0; @@ -521,7 +536,7 @@ protected void wrapOffset(StringBuilder select, Query query) { select.append(";"); } - protected static Object serializeValue(Object value) { + protected Object serializeValue(Object value) { if (value instanceof Id) { value = ((Id) value).asObject(); } @@ -545,6 +560,14 @@ protected void appendPartition(StringBuilder delete) { // pass } + protected List insertTemplateObjects(MysqlBackendEntry.Row entry) { + List objects = new ArrayList<>(); + for (Object key : entry.columns().keySet()) { + objects.add(entry.columns().get(key)); + } + return objects; + } + public static String formatKey(HugeKeys key) { return key.name(); } diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java index 594aae3803..faf083a0c4 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.backend.id.EdgeId; @@ -40,20 +41,30 @@ import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.E; +import jersey.repackaged.com.google.common.collect.ImmutableMap; + public class MysqlTables { - private static final String INT = "INT"; + public static final String BOOLEAN = "BOOLEAN"; + public static final String TINYINT = "TINYINT"; + public static final String INT = "INT"; + public static final String DOUBLE = "DOUBLE"; + public static final String SMALL_TEXT = "SMALL_TEXT"; + public static final String MID_TEXT = "MID_TEXT"; + public static final String LARGE_TEXT = "LARGE_TEXT"; + + private static final String DATATYPE_PK = INT; + private static final String DATATYPE_SL = INT; // VL/EL + private static final String DATATYPE_IL = INT; - private static final String DATATYPE_PK = "INT"; - private static final String DATATYPE_SL = "INT"; // VL/EL - private static final String DATATYPE_IL = "INT"; + private static final String SMALL_JSON = MID_TEXT; + private static final String LARGE_JSON = LARGE_TEXT; - private static final String BOOLEAN = "BOOLEAN"; - private static final String TINYINT = "TINYINT"; - private static final String DOUBLE = "DOUBLE"; - private static final String VARCHAR = "VARCHAR(255)"; - private static final String SMALL_JSON = "VARCHAR(1024)"; - private static final String LARGE_JSON = "TEXT"; + private static final Map TYPES_MAPPING = ImmutableMap.of( + SMALL_TEXT, "VARCHAR(255)", + MID_TEXT, "VARCHAR(1024)", + LARGE_TEXT, "TEXT" + ); public static class MysqlTableTemplate extends MysqlTable { @@ -74,10 +85,14 @@ public static class Counters extends MysqlTableTemplate { public static final String TABLE = "counters"; public Counters() { + this(TYPES_MAPPING); + } + + public Counters(Map typesMapping) { super(TABLE); - this.define = new TableDefine(); - this.define.column(HugeKeys.SCHEMA_TYPE, VARCHAR); + this.define = new TableDefine(typesMapping); + this.define.column(HugeKeys.SCHEMA_TYPE, SMALL_TEXT); this.define.column(HugeKeys.ID, INT); this.define.keys(HugeKeys.SCHEMA_TYPE); } @@ -122,11 +137,15 @@ public static class VertexLabel extends MysqlTableTemplate { public static final String TABLE = "vertex_labels"; public VertexLabel() { + this(TYPES_MAPPING); + } + + public VertexLabel(Map typesMapping) { super(TABLE); - this.define = new TableDefine(); + this.define = new TableDefine(typesMapping); this.define.column(HugeKeys.ID, DATATYPE_SL); - this.define.column(HugeKeys.NAME, VARCHAR); + this.define.column(HugeKeys.NAME, SMALL_TEXT); this.define.column(HugeKeys.ID_STRATEGY, TINYINT); this.define.column(HugeKeys.PRIMARY_KEYS, SMALL_JSON); this.define.column(HugeKeys.PROPERTIES, SMALL_JSON); @@ -144,11 +163,15 @@ public static class EdgeLabel extends MysqlTableTemplate { public static final String TABLE = "edge_labels"; public EdgeLabel() { + this(TYPES_MAPPING); + } + + public EdgeLabel(Map typesMapping) { super(TABLE); - this.define = new TableDefine(); + this.define = new TableDefine(typesMapping); this.define.column(HugeKeys.ID, DATATYPE_SL); - this.define.column(HugeKeys.NAME, VARCHAR); + this.define.column(HugeKeys.NAME, SMALL_TEXT); this.define.column(HugeKeys.FREQUENCY, TINYINT); this.define.column(HugeKeys.SOURCE_LABEL, DATATYPE_SL); this.define.column(HugeKeys.TARGET_LABEL, DATATYPE_SL); @@ -168,11 +191,15 @@ public static class PropertyKey extends MysqlTableTemplate { public static final String TABLE = "property_keys"; public PropertyKey() { + this(TYPES_MAPPING); + } + + public PropertyKey(Map typesMapping) { super(TABLE); - this.define = new TableDefine(); + this.define = new TableDefine(typesMapping); this.define.column(HugeKeys.ID, DATATYPE_PK); - this.define.column(HugeKeys.NAME, VARCHAR); + this.define.column(HugeKeys.NAME, SMALL_TEXT); this.define.column(HugeKeys.DATA_TYPE, TINYINT); this.define.column(HugeKeys.CARDINALITY, TINYINT); this.define.column(HugeKeys.PROPERTIES, SMALL_JSON); @@ -187,11 +214,15 @@ public static class IndexLabel extends MysqlTableTemplate { public static final String TABLE = "index_labels"; public IndexLabel() { + this(TYPES_MAPPING); + } + + public IndexLabel(Map typesMapping) { super(TABLE); - this.define = new TableDefine(); + this.define = new TableDefine(typesMapping); this.define.column(HugeKeys.ID, DATATYPE_IL); - this.define.column(HugeKeys.NAME, VARCHAR); + this.define.column(HugeKeys.NAME, SMALL_TEXT); this.define.column(HugeKeys.BASE_TYPE, TINYINT); this.define.column(HugeKeys.BASE_VALUE, DATATYPE_SL); this.define.column(HugeKeys.INDEX_TYPE, TINYINT); @@ -206,10 +237,14 @@ public static class Vertex extends MysqlTableTemplate { public static final String TABLE = "vertices"; public Vertex(String store) { + this(store, TYPES_MAPPING); + } + + public Vertex(String store, Map typesMapping) { super(joinTableName(store, TABLE)); - this.define = new TableDefine(); - this.define.column(HugeKeys.ID, VARCHAR); + this.define = new TableDefine(typesMapping); + this.define.column(HugeKeys.ID, SMALL_TEXT); this.define.column(HugeKeys.LABEL, DATATYPE_SL); this.define.column(HugeKeys.PROPERTIES, LARGE_JSON); this.define.keys(HugeKeys.ID); @@ -223,7 +258,12 @@ public static class Edge extends MysqlTableTemplate { private final Directions direction; private final String delByLabelTemplate; - protected Edge(String store, Directions direction) { + public Edge(String store, Directions direction) { + this(store, direction, TYPES_MAPPING); + } + + public Edge(String store, Directions direction, + Map typesMapping) { super(joinTableName(store, table(direction))); this.direction = direction; @@ -231,12 +271,12 @@ protected Edge(String store, Directions direction) { "DELETE FROM %s WHERE %s = ?;", this.table(), formatKey(HugeKeys.LABEL)); - this.define = new TableDefine(); - this.define.column(HugeKeys.OWNER_VERTEX, VARCHAR); + this.define = new TableDefine(typesMapping); + this.define.column(HugeKeys.OWNER_VERTEX, SMALL_TEXT); this.define.column(HugeKeys.DIRECTION, TINYINT); this.define.column(HugeKeys.LABEL, DATATYPE_SL); - this.define.column(HugeKeys.SORT_VALUES, VARCHAR); - this.define.column(HugeKeys.OTHER_VERTEX, VARCHAR); + this.define.column(HugeKeys.SORT_VALUES, SMALL_TEXT); + this.define.column(HugeKeys.OTHER_VERTEX, SMALL_TEXT); this.define.column(HugeKeys.PROPERTIES, LARGE_JSON); this.define.keys(HugeKeys.OWNER_VERTEX, HugeKeys.DIRECTION, HugeKeys.LABEL, HugeKeys.SORT_VALUES, @@ -244,7 +284,7 @@ protected Edge(String store, Directions direction) { } @Override - protected List idColumnValue(Id id) { + public List idColumnValue(Id id) { EdgeId edgeId; if (!(id instanceof EdgeId)) { String[] idParts = EdgeId.split(id); @@ -299,7 +339,7 @@ private void deleteEdgesByLabel(Session session, Id label) { } @Override - protected BackendEntry mergeEntries(BackendEntry e1, BackendEntry e2) { + public BackendEntry mergeEntries(BackendEntry e1, BackendEntry e2) { // Merge edges into vertex // TODO: merge rows before calling row2Entry() @@ -366,22 +406,24 @@ public static class SecondaryIndex extends Index { public static final String TABLE = "secondary_indexes"; public SecondaryIndex(String store) { - this(store, TABLE); + this(store, TABLE, TYPES_MAPPING); } - public SecondaryIndex(String store, String table) { + + public SecondaryIndex(String store, String table, + Map typesMapping) { super(joinTableName(store, table)); - this.define = new TableDefine(); - this.define.column(HugeKeys.FIELD_VALUES, VARCHAR); + this.define = new TableDefine(typesMapping); + this.define.column(HugeKeys.FIELD_VALUES, SMALL_TEXT); this.define.column(HugeKeys.INDEX_LABEL_ID, DATATYPE_IL); - this.define.column(HugeKeys.ELEMENT_IDS, VARCHAR); + this.define.column(HugeKeys.ELEMENT_IDS, SMALL_TEXT); this.define.keys(HugeKeys.FIELD_VALUES, HugeKeys.INDEX_LABEL_ID, HugeKeys.ELEMENT_IDS); } @Override - protected final String entryId(MysqlBackendEntry entry) { + public final String entryId(MysqlBackendEntry entry) { String fieldValues = entry.column(HugeKeys.FIELD_VALUES); Integer labelId = entry.column(HugeKeys.INDEX_LABEL_ID); return SplicingIdGenerator.concat(fieldValues, labelId.toString()); @@ -393,7 +435,7 @@ public static class SearchIndex extends SecondaryIndex { public static final String TABLE = "search_indexes"; public SearchIndex(String store) { - super(store, TABLE); + super(store, TABLE, TYPES_MAPPING); } } @@ -402,19 +444,23 @@ public static class RangeIndex extends Index { public static final String TABLE = "range_indexes"; public RangeIndex(String store) { + this(store, TYPES_MAPPING); + } + + public RangeIndex(String store, Map typesMapping) { super(joinTableName(store, TABLE)); - this.define = new TableDefine(); + this.define = new TableDefine(typesMapping); this.define.column(HugeKeys.INDEX_LABEL_ID, DATATYPE_IL); this.define.column(HugeKeys.FIELD_VALUES, DOUBLE); - this.define.column(HugeKeys.ELEMENT_IDS, VARCHAR); + this.define.column(HugeKeys.ELEMENT_IDS, SMALL_TEXT); this.define.keys(HugeKeys.INDEX_LABEL_ID, HugeKeys.FIELD_VALUES, HugeKeys.ELEMENT_IDS); } @Override - protected final String entryId(MysqlBackendEntry entry) { + public final String entryId(MysqlBackendEntry entry) { Double fieldValue = entry.column(HugeKeys.FIELD_VALUES); Integer labelId = entry.column(HugeKeys.INDEX_LABEL_ID); return SplicingIdGenerator.concat(labelId.toString(), diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java index bd68cf4889..75ccc843cd 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java @@ -19,12 +19,12 @@ package com.baidu.hugegraph.backend.store.postgresql; +import com.baidu.hugegraph.backend.store.mysql.MysqlOptions; import com.baidu.hugegraph.config.ConfigOption; -import com.baidu.hugegraph.config.OptionHolder; -import static com.baidu.hugegraph.config.OptionChecker.*; +import static com.baidu.hugegraph.config.OptionChecker.disallowEmpty; -public class PostgresqlOptions extends OptionHolder { +public class PostgresqlOptions extends MysqlOptions { private PostgresqlOptions() { super(); @@ -56,27 +56,11 @@ public static synchronized PostgresqlOptions instance() { "jdbc:postgresql://127.0.0.1:5432" ); - public static final ConfigOption SSL_MODE = - new ConfigOption<>( - "jdbc.ssl_mode", - "The url of database in JDBC format.", - disallowEmpty(), - "disable" - ); - public static final ConfigOption JDBC_USERNAME = new ConfigOption<>( "jdbc.username", "The username to login database.", disallowEmpty(), - "root" - ); - - public static final ConfigOption JDBC_PASSWORD = - new ConfigOption<>( - "jdbc.password", - "The password corresponding to jdbc.username.", - null, - "" + "postgres" ); } diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java index adb465743a..edd276c4ef 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java @@ -19,6 +19,37 @@ package com.baidu.hugegraph.backend.store.postgresql; +import org.apache.logging.log4j.util.Strings; + +import com.baidu.hugegraph.backend.id.IdUtil; +import com.baidu.hugegraph.backend.serializer.TableBackendEntry; +import com.baidu.hugegraph.backend.store.BackendEntry; import com.baidu.hugegraph.backend.store.mysql.MysqlSerializer; +import com.baidu.hugegraph.structure.HugeIndex; +import com.baidu.hugegraph.type.define.HugeKeys; + +public class PostgresqlSerializer extends MysqlSerializer { -public class PostgresqlSerializer extends MysqlSerializer {} + @Override + public BackendEntry writeIndex(HugeIndex index) { + TableBackendEntry entry = newBackendEntry(index); + /* + * When field-values is null and elementIds size is 0, it is + * meaningful for deletion of index data in secondary/range index. + */ + if (index.fieldValues() == null && index.elementIds().size() == 0) { + entry.column(HugeKeys.INDEX_LABEL_ID, index.indexLabel().asLong()); + } else { + Object value = index.fieldValues(); + if (value != null && value.equals("\u0000")) { + value = Strings.EMPTY; + } + entry.column(HugeKeys.FIELD_VALUES, value); + entry.column(HugeKeys.INDEX_LABEL_ID, index.indexLabel().asLong()); + entry.column(HugeKeys.ELEMENT_IDS, + IdUtil.writeString(index.elementId())); + entry.subId(index.elementId()); + } + return entry; + } +} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java index c222ca3766..abc54de0ea 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java @@ -19,161 +19,44 @@ package com.baidu.hugegraph.backend.store.postgresql; -import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.Map; +import org.apache.logging.log4j.util.Strings; import org.postgresql.core.Utils; -import org.slf4j.Logger; import com.baidu.hugegraph.HugeException; -import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.backend.query.Condition; -import com.baidu.hugegraph.backend.query.Query; -import com.baidu.hugegraph.backend.store.TableDefine; import com.baidu.hugegraph.backend.store.mysql.MysqlBackendEntry; -import com.baidu.hugegraph.backend.store.mysql.MysqlEntryIterator; -import com.baidu.hugegraph.backend.store.mysql.MysqlSessions.Session; import com.baidu.hugegraph.backend.store.mysql.MysqlTable; -import com.baidu.hugegraph.backend.store.mysql.WhereBuilder; import com.baidu.hugegraph.type.define.HugeKeys; -import com.baidu.hugegraph.util.Log; public abstract class PostgresqlTable extends MysqlTable { - private static final Logger LOG = Log.logger(PostgresqlStore.class); - private String insertTemplate; public PostgresqlTable(String table) { super(table); + this.dropTableTemplate = "DROP TABLE IF EXISTS %s CASCADE;"; + this.truncateTableTemplate = "TRUNCATE TABLE %s CASCADE;"; } @Override - protected void createTable(Session session, TableDefine tableDefine) { - StringBuilder sql = new StringBuilder(); - sql.append("CREATE TABLE IF NOT EXISTS "); - sql.append(this.table()).append(" ("); - // Add columns - for (Map.Entry entry : - tableDefine.columns().entrySet()) { - sql.append(formatKey(entry.getKey())); - sql.append(" "); - sql.append(entry.getValue()); - sql.append(", "); - } - // Specified primary keys - sql.append(" PRIMARY KEY ("); - int i = 0; - int size = tableDefine.keys().size(); - for (HugeKeys key : tableDefine.keys()) { - sql.append(formatKey(key)); - if (++i != size) { - sql.append(", "); - } - } - sql.append("))"); - - LOG.debug("Create table: {}", sql); - try { - session.execute(sql.toString()); - } catch (SQLException e) { - throw new BackendException("Failed to create table with '%s'", - e, sql); - } - } - - @Override - protected void dropTable(Session session) { - LOG.debug("Drop table: {}", this.table()); - String sql = String.format("DROP TABLE IF EXISTS %s CASCADE;", - this.table()); - try { - session.execute(sql); - } catch (SQLException e) { - throw new BackendException("Failed to drop table with '%s'", - e, sql); - } + protected String engine() { + return Strings.EMPTY; } - @Override - protected void truncateTable(Session session) { - LOG.debug("Truncate table: {}", this.table()); - String sql = String.format("TRUNCATE TABLE %s CASCADE;", this.table()); - try { - session.execute(sql); - } catch (SQLException e) { - throw new BackendException("Failed to truncate table with '%s'", - e, sql); - } - } - /** - * Insert an entire row - */ @Override - public void insert(Session session, MysqlBackendEntry.Row entry) { - String template = this.buildInsertTemplate(entry); - - PreparedStatement insertStmt; - try { - // Create or get insert prepare statement - insertStmt = session.prepareStatement(template); - int i = 1; - int size = entry.columns().size(); - for (Object object : entry.columns().values()) { - if (object.equals("\u0000")) { - object = ""; - } - insertStmt.setObject(i, object); - insertStmt.setObject(size + i++, object); - } - } catch (SQLException e) { - throw new BackendException("Failed to prepare statement '%s'" + - "for entry: %s", template, entry); - } - session.add(insertStmt); + protected List insertTemplateObjects(MysqlBackendEntry.Row entry) { + List objects = new ArrayList<>(); + objects.addAll(super.insertTemplateObjects(entry)); + objects.addAll(super.insertTemplateObjects(entry)); + return objects; } @Override - public void delete(Session session, MysqlBackendEntry.Row entry) { - List idNames = this.idColumnName(); - String template = this.buildDeleteTemplate(idNames); - - PreparedStatement deleteStmt; - try { - deleteStmt = session.prepareStatement(template); - if (entry.columns().isEmpty()) { - // Delete just by id - List idValues = this.idColumnValue(entry); - assert idNames.size() == idValues.size(); - - for (int i = 0, n = idNames.size(); i < n; i++) { - deleteStmt.setObject(i + 1, idValues.get(i)); - } - } else { - // Delete just by column keys(must be id columns) - for (int i = 0, n = idNames.size(); i < n; i++) { - HugeKeys key = idNames.get(i); - Object value = entry.column(key); - if (value != null && value.equals("\u0000")) { - value = "\'\'"; - } - - deleteStmt.setObject(i + 1, value); - } - } - } catch (SQLException e) { - throw new BackendException("Failed to prepare statement '%s'" + - "with entry columns %s", - template, entry.columns().values()); - } - session.add(deleteStmt); - } - protected String buildInsertTemplate(MysqlBackendEntry.Row entry) { if (this.insertTemplate != null) { return this.insertTemplate; @@ -225,34 +108,11 @@ protected String buildInsertTemplate(MysqlBackendEntry.Row entry) { return this.insertTemplate; } - protected void wrapPage(StringBuilder select, Query query) { - String page = query.page(); - // It's the first time if page is empty - if (!page.isEmpty()) { - MysqlEntryIterator.PageState - pageState = MysqlEntryIterator.PageState.fromString(page); - Map columns = pageState.columns(); - - List idColumnNames = this.idColumnName(); - List values = new ArrayList<>(idColumnNames.size()); - for (HugeKeys key : idColumnNames) { - values.add(columns.get(key)); - } - - // Need add `where` to `select` when query is IdQuery - boolean startWithWhere = query.conditions().isEmpty(); - WhereBuilder where = new WhereBuilder(startWithWhere); - where.gte(formatKeys(idColumnNames), values); - if (!startWithWhere) { - select.append(" AND"); - } - select.append(where.build()); - } - + // Set order-by to keep results order consistence for PostgreSQL result + protected String orderByKeys() { int i = 0; int size = this.tableDefine().keys().size(); - - // Set order-by to keep results order consistence for result + StringBuilder select = new StringBuilder(); select.append(" ORDER BY "); for (HugeKeys hugeKey : this.tableDefine().keys()) { String key = formatKey(hugeKey); @@ -262,63 +122,10 @@ protected void wrapPage(StringBuilder select, Query query) { select.append(", "); } } - - assert query.limit() != Query.NO_LIMIT; - // Fetch `limit + 1` records for judging whether reached the last page - select.append(" limit "); - select.append(query.limit() + 1); - select.append(";"); - } - - @Override - protected StringBuilder relation2Sql(Condition.Relation relation) { - String key = relation.serialKey().toString(); - Object value = relation.serialValue(); - - value = serializeValue(value); - - StringBuilder sql = new StringBuilder(32); - sql.append(key); - switch (relation.relation()) { - case EQ: - sql.append(" = ").append(value); - break; - case NEQ: - sql.append(" != ").append(value); - break; - case GT: - sql.append(" > ").append(value); - break; - case GTE: - sql.append(" >= ").append(value); - break; - case LT: - sql.append(" < ").append(value); - break; - case LTE: - sql.append(" <= ").append(value); - break; - case IN: - sql.append(" IN ("); - List values = (List) value; - for (int i = 0, n = values.size(); i < n; i++) { - sql.append(serializeValue(values.get(i))); - if (i != n - 1) { - sql.append(", "); - } - } - sql.append(")"); - break; - case CONTAINS: - case CONTAINS_KEY: - case SCAN: - default: - throw new AssertionError("Unsupported relation: " + relation); - } - return sql; + return select.toString(); } - protected static Object serializeValue(Object value) { + protected Object serializeValue(Object value) { if (value instanceof Id) { value = ((Id) value).asObject(); } diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java index 794c3e9182..a48d8b7525 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java @@ -19,102 +19,79 @@ package com.baidu.hugegraph.backend.store.postgresql; -import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; +import java.util.Map; import com.baidu.hugegraph.backend.BackendException; -import com.baidu.hugegraph.backend.id.EdgeId; import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.backend.id.IdGenerator; -import com.baidu.hugegraph.backend.id.IdUtil; -import com.baidu.hugegraph.backend.id.SplicingIdGenerator; import com.baidu.hugegraph.backend.store.BackendEntry; import com.baidu.hugegraph.backend.store.TableDefine; import com.baidu.hugegraph.backend.store.mysql.MysqlBackendEntry; import com.baidu.hugegraph.backend.store.mysql.MysqlSessions.Session; import com.baidu.hugegraph.backend.store.mysql.MysqlTables; +import com.baidu.hugegraph.backend.store.mysql.MysqlTables.MysqlTableTemplate; import com.baidu.hugegraph.type.HugeType; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.type.define.HugeKeys; -import com.baidu.hugegraph.util.E; -public class PostgresqlTables { +import jersey.repackaged.com.google.common.collect.ImmutableMap; - private static final String NOT_NULL = "NOT NULL"; - private static final String DEFAULT_EMPTY = "DEFAULT ''"; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.BOOLEAN; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.DOUBLE; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.INT; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.LARGE_TEXT; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.MID_TEXT; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.SMALL_TEXT; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.TINYINT; - private static final String DATATYPE_PK = "INT"; - private static final String DATATYPE_SL = "INT"; // VL/EL - private static final String DATATYPE_IL = "INT"; +public class PostgresqlTables { - private static final String INT = "INT"; - private static final String FLOAT = "FLOAT"; - private static final String VARCHAR = "VARCHAR(255)"; - private static final String TEXT = "VARCHAR(65533)"; - private static final String BOOL = "BOOL"; + private static final Map TYPES_MAPPING = + ImmutableMap.builder() + .put(BOOLEAN, "BOOL") + .put(TINYINT, INT) + .put(DOUBLE, "FLOAT") + .put(SMALL_TEXT, "VARCHAR(255)") + .put(MID_TEXT, "VARCHAR(1024)") + .put(LARGE_TEXT, "VARCHAR(65533)") + .build(); public static class PostgresqlTableTemplate extends PostgresqlTable { - protected TableDefine define; + protected MysqlTableTemplate template; - public PostgresqlTableTemplate(String table) { - super(table); + public PostgresqlTableTemplate(MysqlTableTemplate template) { + super(template.table()); + this.template = template; } @Override public TableDefine tableDefine() { - return this.define; + return this.template.tableDefine(); } } public static class Counters extends PostgresqlTableTemplate { - public static final String TABLE = "counters"; - public Counters() { - super(TABLE); - - this.define = new TableDefine(); - this.define.column(HugeKeys.SCHEMA_TYPE, VARCHAR); - this.define.column(HugeKeys.ID, INT); - this.define.keys(HugeKeys.SCHEMA_TYPE); + super(new MysqlTables.Counters(TYPES_MAPPING)); } public long getCounter(Session session, HugeType type) { - String schemaCol = formatKey(HugeKeys.SCHEMA_TYPE); - String idCol = formatKey(HugeKeys.ID); - - String select = String.format("SELECT ID FROM %s WHERE %s = '%s';", - TABLE, schemaCol, type.name()); - try { - ResultSet resultSet = session.select(select); - if (resultSet.next()) { - return resultSet.getLong(idCol); - } else { - return 0L; - } - } catch (SQLException e) { - throw new BackendException( - "Failed to get id from counters with type '%s'", - e, type); - } + MysqlTables.Counters table = (MysqlTables.Counters) this.template; + return table.getCounter(session, type); } public void increaseCounter(Session session, HugeType type, long increment) { - String update = String.format("INSERT INTO %s (%s, %s) VALUES " + - "('%s', %s) ON CONFLICT (%s) " + - "DO UPDATE SET ID = %s.ID + %s;", - TABLE, - formatKey(HugeKeys.SCHEMA_TYPE), - formatKey(HugeKeys.ID), - type.name(), increment, - formatKey(HugeKeys.SCHEMA_TYPE), - TABLE, increment); + String update = String.format( + "INSERT INTO %s (%s, %s) VALUES ('%s', %s) " + + "ON CONFLICT (%s) DO UPDATE SET ID = %s.ID + %s;", + this.table(), formatKey(HugeKeys.SCHEMA_TYPE), + formatKey(HugeKeys.ID), type.name(), increment, + formatKey(HugeKeys.SCHEMA_TYPE), + this.table(), increment); try { session.execute(update); } catch (SQLException e) { @@ -126,85 +103,29 @@ public void increaseCounter(Session session, HugeType type, public static class VertexLabel extends PostgresqlTableTemplate { - public static final String TABLE = "vertex_labels"; - public VertexLabel() { - super(TABLE); - this.define = new TableDefine(); - this.define.column(HugeKeys.ID, DATATYPE_SL, NOT_NULL); - this.define.column(HugeKeys.NAME, VARCHAR, NOT_NULL); - this.define.column(HugeKeys.ID_STRATEGY, INT, NOT_NULL); - this.define.column(HugeKeys.PRIMARY_KEYS, VARCHAR, DEFAULT_EMPTY); - this.define.column(HugeKeys.PROPERTIES, VARCHAR, DEFAULT_EMPTY); - this.define.column(HugeKeys.NULLABLE_KEYS, VARCHAR, DEFAULT_EMPTY); - this.define.column(HugeKeys.INDEX_LABELS, VARCHAR, DEFAULT_EMPTY); - this.define.column(HugeKeys.ENABLE_LABEL_INDEX, BOOL, NOT_NULL); - this.define.column(HugeKeys.USER_DATA, VARCHAR, DEFAULT_EMPTY); - this.define.column(HugeKeys.STATUS, INT, NOT_NULL); - // Unique keys/hash keys - this.define.keys(HugeKeys.ID); + super(new MysqlTables.VertexLabel(TYPES_MAPPING)); } } public static class EdgeLabel extends PostgresqlTableTemplate { - public static final String TABLE = "edge_labels"; - public EdgeLabel() { - super(TABLE); - this.define = new TableDefine(); - this.define.column(HugeKeys.ID, DATATYPE_SL, NOT_NULL); - this.define.column(HugeKeys.NAME, VARCHAR, NOT_NULL); - this.define.column(HugeKeys.FREQUENCY, INT, NOT_NULL); - this.define.column(HugeKeys.SOURCE_LABEL, INT, NOT_NULL); - this.define.column(HugeKeys.TARGET_LABEL, INT, NOT_NULL); - this.define.column(HugeKeys.SORT_KEYS, VARCHAR, DEFAULT_EMPTY); - this.define.column(HugeKeys.PROPERTIES, VARCHAR, DEFAULT_EMPTY); - this.define.column(HugeKeys.NULLABLE_KEYS, VARCHAR, DEFAULT_EMPTY); - this.define.column(HugeKeys.INDEX_LABELS, VARCHAR, DEFAULT_EMPTY); - this.define.column(HugeKeys.ENABLE_LABEL_INDEX, BOOL, NOT_NULL); - this.define.column(HugeKeys.USER_DATA, VARCHAR, DEFAULT_EMPTY); - this.define.column(HugeKeys.STATUS, INT, NOT_NULL); - // Unique keys/hash keys - this.define.keys(HugeKeys.ID); + super(new MysqlTables.EdgeLabel(TYPES_MAPPING)); } } public static class PropertyKey extends PostgresqlTableTemplate { - public static final String TABLE = "property_keys"; - public PropertyKey() { - super(TABLE); - this.define = new TableDefine(); - this.define.column(HugeKeys.ID, DATATYPE_PK, NOT_NULL); - this.define.column(HugeKeys.NAME, VARCHAR, NOT_NULL); - this.define.column(HugeKeys.DATA_TYPE, INT, NOT_NULL); - this.define.column(HugeKeys.CARDINALITY, INT, NOT_NULL); - this.define.column(HugeKeys.PROPERTIES, VARCHAR, DEFAULT_EMPTY); - this.define.column(HugeKeys.USER_DATA, VARCHAR, DEFAULT_EMPTY); - this.define.column(HugeKeys.STATUS, INT, NOT_NULL); - // Unique keys/hash keys - this.define.keys(HugeKeys.ID); + super(new MysqlTables.PropertyKey(TYPES_MAPPING)); } } public static class IndexLabel extends PostgresqlTableTemplate { - public static final String TABLE = "index_labels"; - public IndexLabel() { - super(TABLE); - this.define = new TableDefine(); - this.define.column(HugeKeys.ID, DATATYPE_IL, NOT_NULL); - this.define.column(HugeKeys.NAME, VARCHAR, NOT_NULL); - this.define.column(HugeKeys.BASE_TYPE, INT, NOT_NULL); - this.define.column(HugeKeys.BASE_VALUE, INT, NOT_NULL); - this.define.column(HugeKeys.INDEX_TYPE, INT, NOT_NULL); - this.define.column(HugeKeys.FIELDS, VARCHAR, NOT_NULL); - this.define.column(HugeKeys.STATUS, INT, NOT_NULL); - // Unique keys/hash keys - this.define.keys(HugeKeys.ID); + super(new MysqlTables.IndexLabel(TYPES_MAPPING)); } } @@ -213,175 +134,51 @@ public static class Vertex extends PostgresqlTableTemplate { public static final String TABLE = "vertices"; public Vertex(String store) { - super(joinTableName(store, TABLE)); - - this.define = new TableDefine(); - this.define.column(HugeKeys.ID, VARCHAR, NOT_NULL); - this.define.column(HugeKeys.LABEL, INT, NOT_NULL); - this.define.column(HugeKeys.PROPERTIES, TEXT, DEFAULT_EMPTY); - // Unique keys/hash keys - this.define.keys(HugeKeys.ID); + super(new MysqlTables.Vertex(store, TYPES_MAPPING)); } } public static class Edge extends PostgresqlTableTemplate { - private final Directions direction; - private final String delByLabelTemplate; - public Edge(String store, Directions direction) { - super(joinTableName(store, MysqlTables.Edge.table(direction))); - - this.direction = direction; - this.delByLabelTemplate = String.format( - "DELETE FROM %s WHERE %s = ?;", - this.table(), formatKey(HugeKeys.LABEL)); - - this.define = new TableDefine(); - this.define.column(HugeKeys.OWNER_VERTEX, VARCHAR, NOT_NULL); - this.define.column(HugeKeys.DIRECTION, INT, NOT_NULL); - this.define.column(HugeKeys.LABEL, INT, NOT_NULL); - this.define.column(HugeKeys.SORT_VALUES, VARCHAR, NOT_NULL, - DEFAULT_EMPTY); - this.define.column(HugeKeys.OTHER_VERTEX, VARCHAR, NOT_NULL); - this.define.column(HugeKeys.PROPERTIES, TEXT, DEFAULT_EMPTY); - // Unique keys/hash keys - this.define.keys(HugeKeys.OWNER_VERTEX, HugeKeys.DIRECTION, - HugeKeys.LABEL, HugeKeys.SORT_VALUES, - HugeKeys.OTHER_VERTEX); + super(new MysqlTables.Edge(store, direction, TYPES_MAPPING)); } @Override protected List idColumnValue(Id id) { - EdgeId edgeId; - if (!(id instanceof EdgeId)) { - String[] idParts = EdgeId.split(id); - if (idParts.length == 1) { - // Delete edge by label - return Arrays.asList((Object[]) idParts); - } - id = IdUtil.readString(id.asString()); - edgeId = EdgeId.parse(id.asString()); - } else { - edgeId = (EdgeId) id; - } - - E.checkState(edgeId.direction() == this.direction, - "Can't query %s edges from %s edges table", - edgeId.direction(), this.direction); - - List list = new ArrayList<>(5); - list.add(IdUtil.writeString(edgeId.ownerVertexId())); - list.add(edgeId.direction().code()); - list.add(edgeId.edgeLabelId().asLong()); - list.add(edgeId.sortValues()); - list.add(IdUtil.writeString(edgeId.otherVertexId())); - return list; + MysqlTables.Edge table = (MysqlTables.Edge) this.template; + return table.idColumnValue(id); } @Override public void delete(Session session, MysqlBackendEntry.Row entry) { - // Let super class do delete if not deleting edge by label - List idParts = this.idColumnValue(entry.id()); - if (idParts.size() > 1 || entry.columns().size() > 0) { - super.delete(session, entry); - return; - } - - // The only element is label - this.deleteEdgesByLabel(session, entry.id()); - } - - private void deleteEdgesByLabel(Session session, Id label) { - PreparedStatement deleteStmt; - try { - // Create or get delete prepare statement - deleteStmt = session.prepareStatement(this.delByLabelTemplate); - // Delete edges - deleteStmt.setObject(1, label.asLong()); - } catch (SQLException e) { - throw new BackendException("Failed to prepare statement '%s'", - this.delByLabelTemplate); - } - session.add(deleteStmt); + MysqlTables.Edge table = (MysqlTables.Edge) this.template; + table.delete(session, entry); } @Override protected BackendEntry mergeEntries(BackendEntry e1, BackendEntry e2) { - // Merge edges into vertex - // TODO: merge rows before calling row2Entry() - - MysqlBackendEntry current = (MysqlBackendEntry) e1; - MysqlBackendEntry next = (MysqlBackendEntry) e2; - - E.checkState(current == null || current.type().isVertex(), - "The current entry must be null or VERTEX"); - E.checkState(next != null && next.type().isEdge(), - "The next entry must be EDGE"); - - if (current != null) { - Id nextVertexId = IdGenerator.of( - next.column(HugeKeys.OWNER_VERTEX)); - if (current.id().equals(nextVertexId)) { - current.subRow(next.row()); - return current; - } - } - - return this.wrapByVertex(next); - } - - private MysqlBackendEntry wrapByVertex(MysqlBackendEntry edge) { - assert edge.type().isEdge(); - String ownerVertex = edge.column(HugeKeys.OWNER_VERTEX); - E.checkState(ownerVertex != null, "Invalid backend entry"); - Id vertexId = IdGenerator.of(ownerVertex); - MysqlBackendEntry vertex = new MysqlBackendEntry(HugeType.VERTEX, - vertexId); - - vertex.column(HugeKeys.ID, ownerVertex); - vertex.column(HugeKeys.PROPERTIES, ""); - - vertex.subRow(edge.row()); - return vertex; + MysqlTables.Edge table = (MysqlTables.Edge) this.template; + return table.mergeEntries(e1, e2); } } - public abstract static class Index extends PostgresqlTableTemplate { - - public Index(String table) { - super(table); - } - - protected abstract String entryId(MysqlBackendEntry entry); - } - - public static class SecondaryIndex extends Index { + public static class SecondaryIndex extends PostgresqlTableTemplate { public static final String TABLE = "secondary_indexes"; public SecondaryIndex(String store) { - this(store, TABLE); + super(new MysqlTables.SecondaryIndex(store, TABLE, TYPES_MAPPING)); } - protected SecondaryIndex(String store, String table) { - super(joinTableName(store, table)); - - this.define = new TableDefine(); - this.define.column(HugeKeys.FIELD_VALUES, VARCHAR, NOT_NULL); - this.define.column(HugeKeys.INDEX_LABEL_ID, INT, NOT_NULL); - this.define.column(HugeKeys.ELEMENT_IDS, VARCHAR, NOT_NULL); - // Unique keys/hash keys - this.define.keys(HugeKeys.FIELD_VALUES, - HugeKeys.INDEX_LABEL_ID, - HugeKeys.ELEMENT_IDS); + public SecondaryIndex(String store, String table) { + super(new MysqlTables.SecondaryIndex(store, table, TYPES_MAPPING)); } - @Override protected final String entryId(MysqlBackendEntry entry) { - String fieldValues = entry.column(HugeKeys.FIELD_VALUES); - Long labelId = entry.column(HugeKeys.INDEX_LABEL_ID); - return SplicingIdGenerator.concat(fieldValues, labelId.toString()); + MysqlTables.SecondaryIndex table = + (MysqlTables.SecondaryIndex) this.template; + return table.entryId(entry); } } @@ -394,29 +191,16 @@ public SearchIndex(String store) { } } - public static class RangeIndex extends Index { - - public static final String TABLE = "range_indexes"; + public static class RangeIndex extends PostgresqlTableTemplate { public RangeIndex(String store) { - super(joinTableName(store, TABLE)); - - this.define = new TableDefine(); - this.define.column(HugeKeys.INDEX_LABEL_ID, INT, NOT_NULL); - this.define.column(HugeKeys.FIELD_VALUES, FLOAT, NOT_NULL); - this.define.column(HugeKeys.ELEMENT_IDS, VARCHAR, NOT_NULL); - // Unique keys/hash keys - this.define.keys(HugeKeys.INDEX_LABEL_ID, - HugeKeys.FIELD_VALUES, - HugeKeys.ELEMENT_IDS); + super(new MysqlTables.RangeIndex(store, TYPES_MAPPING)); } - @Override protected final String entryId(MysqlBackendEntry entry) { - Double fieldValue = entry.column(HugeKeys.FIELD_VALUES); - Long labelId = entry.column(HugeKeys.INDEX_LABEL_ID); - return SplicingIdGenerator.concat(labelId.toString(), - fieldValue.toString()); + MysqlTables.RangeIndex table = + (MysqlTables.RangeIndex) this.template; + return table.entryId(entry); } } } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java index f855ad10a8..95fbd270d2 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java @@ -2578,7 +2578,6 @@ public void testQueryEdgeByPage() { Assert.assertEquals(90, edges.size()); } - @SuppressWarnings("unchecked") @Test public void testQueryEdgeByPageResultsMatched() { Assume.assumeTrue("Not support paging", @@ -2598,6 +2597,7 @@ public void testQueryEdgeByPageResultsMatched() { for (int i = 0; i < 100 / size; i++) { iter = graph.traversal().E() .has("~page", page).limit(size); + @SuppressWarnings("unchecked") List edges = IteratorUtils.asList(iter); Assert.assertEquals(size, edges.size()); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java index 8dbf46d3ff..4d06ad20f3 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java @@ -3265,7 +3265,6 @@ public void testQueryByPage() { Assert.assertEquals(90, vertexes.size()); } - @SuppressWarnings("unchecked") @Test public void testQueryByPageResultsMatched() { Assume.assumeTrue("Not support paging", @@ -3285,6 +3284,7 @@ public void testQueryByPageResultsMatched() { for (int i = 0; i < 100 / size; i++) { iter = graph.traversal().V() .has("~page", page).limit(size); + @SuppressWarnings("unchecked") List vertexes = IteratorUtils.asList(iter); Assert.assertEquals(size, vertexes.size()); From b3c37686e67b16b3fb663aabb8a9e22bf32419ce Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 14 May 2019 18:22:12 +0800 Subject: [PATCH 03/10] tiny improve Change-Id: I8fdd4f60325d39975ef2406ff3e085d71f91814b --- .../com/baidu/hugegraph/backend/store/TableDefine.java | 2 ++ hugegraph-postgresql/pom.xml | 2 +- .../backend/store/postgresql/PostgresqlTables.java | 8 ++------ 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java index 9bc6af4f89..27e5645686 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java @@ -53,6 +53,8 @@ public TableDefine column(HugeKeys key, String... desc) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < desc.length; i++) { String type = desc[i]; + // The first element of 'desc' is column data type, which may be + // mapped to actual data type supported by backend store if (i == 0 && this.typesMapping.containsKey(type)) { type = this.typesMapping.get(type); } diff --git a/hugegraph-postgresql/pom.xml b/hugegraph-postgresql/pom.xml index fa3181582f..c9680e49f6 100644 --- a/hugegraph-postgresql/pom.xml +++ b/hugegraph-postgresql/pom.xml @@ -5,7 +5,7 @@ hugegraph com.baidu.hugegraph - 0.9.2 + 0.10.0 4.0.0 diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java index a48d8b7525..d9a676e511 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java @@ -176,9 +176,7 @@ public SecondaryIndex(String store, String table) { } protected final String entryId(MysqlBackendEntry entry) { - MysqlTables.SecondaryIndex table = - (MysqlTables.SecondaryIndex) this.template; - return table.entryId(entry); + return ((MysqlTables.SecondaryIndex) this.template).entryId(entry); } } @@ -198,9 +196,7 @@ public RangeIndex(String store) { } protected final String entryId(MysqlBackendEntry entry) { - MysqlTables.RangeIndex table = - (MysqlTables.RangeIndex) this.template; - return table.entryId(entry); + return ((MysqlTables.RangeIndex) this.template).entryId(entry); } } } From 851553fe1933c9095cc37f65ee40efe9b371c525 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 16 May 2019 17:20:50 +0800 Subject: [PATCH 04/10] tiny improve Change-Id: I069984fccceaab6eef630bbfe5ee722c66cb925b --- .../com/baidu/hugegraph/dist/RegisterUtil.java | 12 +++++++----- .../backend/store/mysql/MysqlTable.java | 16 ++++++++++++---- .../store/postgresql/PostgresqlOptions.java | 2 +- .../store/postgresql/PostgresqlSessions.java | 3 +-- .../store/postgresql/PostgresqlTable.java | 7 +++---- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java index 7d19f1638a..c789fa7c1c 100644 --- a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java +++ b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java @@ -52,7 +52,8 @@ public static void registerBackends() { String confFile = "/backend.properties"; InputStream input = RegisterUtil.class.getClass() .getResourceAsStream(confFile); - E.checkState(input != null, "Can't read file '%s' as stream", confFile); + E.checkState(input != null, + "Can't read file '%s' as stream", confFile); PropertiesConfiguration props = new PropertiesConfiguration(); props.setDelimiterParsingDisabled(true); @@ -93,7 +94,8 @@ private static void registerBackend(String backend) { registerPostgresql(); break; default: - throw new HugeException("Unsupported backend type '%s'", backend); + throw new HugeException("Unsupported backend type '%s'", + backend); } } @@ -171,13 +173,13 @@ public static void registerPalo() { public static void registerPostgresql() { // Register config OptionSpace.register("postgresql", - "com.baidu.hugegraph.backend.store.postgresql.PostgresqlOptions"); + "com.baidu.hugegraph.backend.store.postgresql.PostgresqlOptions"); // Register serializer SerializerFactory.register("postgresql", - "com.baidu.hugegraph.backend.store.postgresql.PostgresqlSerializer"); + "com.baidu.hugegraph.backend.store.postgresql.PostgresqlSerializer"); // Register backend BackendProviderFactory.register("postgresql", - "com.baidu.hugegraph.backend.store.postgresql.PostgresqlStoreProvider"); + "com.baidu.hugegraph.backend.store.postgresql.PostgresqlStoreProvider"); } public static void registerServer() { diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java index 6df2e7a39b..9058118a2a 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java @@ -120,7 +120,7 @@ protected String engine() { protected void dropTable(Session session) { LOG.debug("Drop table: {}", this.table()); - String sql = String.format(this.dropTableTemplate, this.table()); + String sql = this.buildDropTemplate(); try { session.execute(sql); } catch (SQLException e) { @@ -131,7 +131,7 @@ protected void dropTable(Session session) { protected void truncateTable(Session session) { LOG.debug("Truncate table: {}", this.table()); - String sql = String.format(this.truncateTableTemplate, this.table()); + String sql = this.buildTruncateTemplate(); try { session.execute(sql); } catch (SQLException e) { @@ -199,6 +199,14 @@ protected String buildDeleteTemplate(List idNames) { return this.deleteTemplate; } + protected String buildDropTemplate() { + return String.format(this.dropTableTemplate, this.table()); + } + + protected String buildTruncateTemplate() { + return String.format(this.truncateTableTemplate, this.table()); + } + /** * Insert an entire row */ @@ -211,7 +219,7 @@ public void insert(Session session, MysqlBackendEntry.Row entry) { // Create or get insert prepare statement insertStmt = session.prepareStatement(template); int i = 1; - for (Object object : this.insertTemplateObjects(entry)) { + for (Object object : this.buildInsertObjects(entry)) { insertStmt.setObject(i++, object); } } catch (SQLException e) { @@ -560,7 +568,7 @@ protected void appendPartition(StringBuilder delete) { // pass } - protected List insertTemplateObjects(MysqlBackendEntry.Row entry) { + protected List buildInsertObjects(MysqlBackendEntry.Row entry) { List objects = new ArrayList<>(); for (Object key : entry.columns().keySet()) { objects.add(entry.columns().get(key)); diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java index 75ccc843cd..6d2d6b57dd 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java @@ -53,7 +53,7 @@ public static synchronized PostgresqlOptions instance() { "jdbc.url", "The url of database in JDBC format.", disallowEmpty(), - "jdbc:postgresql://127.0.0.1:5432" + "jdbc:postgresql://127.0.0.1:5432/" ); public static final ConfigOption JDBC_USERNAME = diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java index 204e889ea3..a5ea77c4f6 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java @@ -30,7 +30,6 @@ public PostgresqlSessions(HugeConfig config, String database, String store) { @Override protected String buildCreateDatabase(String database) { - return String.format("CREATE DATABASE %s ENCODING='UTF-8' " + - "TEMPLATE=template0 LC_COLLATE='C' LC_CTYPE='C';", database); + return String.format("CREATE DATABASE %s ENCODING='UTF-8'", database); } } diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java index abc54de0ea..4775f86075 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java @@ -47,12 +47,11 @@ protected String engine() { return Strings.EMPTY; } - @Override - protected List insertTemplateObjects(MysqlBackendEntry.Row entry) { + protected List buildInsertObjects(MysqlBackendEntry.Row entry) { List objects = new ArrayList<>(); - objects.addAll(super.insertTemplateObjects(entry)); - objects.addAll(super.insertTemplateObjects(entry)); + objects.addAll(super.buildInsertObjects(entry)); + objects.addAll(super.buildInsertObjects(entry)); return objects; } From d1deeb6f6f0c992c019575ee74026d524196c685 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Mon, 20 May 2019 13:10:58 +0800 Subject: [PATCH 05/10] cockroach not support template args of CREATE DATABASE Change-Id: Ic284bf0cbd49683ce12e88d8178e953df84034cc --- .../backend/store/mysql/MysqlSessions.java | 2 +- .../backend/store/mysql/MysqlTable.java | 6 +-- .../store/postgresql/PostgresqlSessions.java | 41 ++++++++++++++++++- .../store/postgresql/PostgresqlTable.java | 10 ++++- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java index 4f29a9c979..6a87c9eb9d 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java @@ -197,7 +197,7 @@ public boolean existsDatabase() { /** * Connect DB without specified database */ - private Connection openWithoutDB(int timeout) { + protected Connection openWithoutDB(int timeout) { String jdbcUrl = this.config.get(MysqlOptions.JDBC_URL); String url = new URIBuilder().setPath(jdbcUrl) .setParameter("socketTimeout", diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java index 9058118a2a..7107a481a3 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java @@ -51,8 +51,6 @@ public abstract class MysqlTable private static final Logger LOG = Log.logger(MysqlStore.class); - protected String dropTableTemplate = "DROP TABLE IF EXISTS %s;"; - protected String truncateTableTemplate = "TRUNCATE TABLE %s;"; // The template for insert and delete statements private String insertTemplate; private String deleteTemplate; @@ -200,11 +198,11 @@ protected String buildDeleteTemplate(List idNames) { } protected String buildDropTemplate() { - return String.format(this.dropTableTemplate, this.table()); + return String.format("DROP TABLE IF EXISTS %s;", this.table()); } protected String buildTruncateTemplate() { - return String.format(this.truncateTableTemplate, this.table()); + return String.format("TRUNCATE TABLE %s;", this.table()); } /** diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java index a5ea77c4f6..638d63c061 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java @@ -19,17 +19,54 @@ package com.baidu.hugegraph.backend.store.postgresql; +import java.sql.Connection; +import java.sql.SQLException; + +import org.postgresql.util.PSQLException; +import org.slf4j.Logger; + +import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.backend.store.mysql.MysqlSessions; +import com.baidu.hugegraph.backend.store.mysql.MysqlStore; import com.baidu.hugegraph.config.HugeConfig; +import com.baidu.hugegraph.util.Log; public class PostgresqlSessions extends MysqlSessions { + private static final Logger LOG = Log.logger(MysqlStore.class); + + private static final String COCKROACH_DB_CREATE = + "CREATE DATABASE %s ENCODING='UTF-8'"; + private static final String POSTGRESQL_DB_CREATE = COCKROACH_DB_CREATE + + " TEMPLATE=template0 LC_COLLATE='C' LC_CTYPE='C';"; + public PostgresqlSessions(HugeConfig config, String database, String store) { super(config, database, store); } @Override - protected String buildCreateDatabase(String database) { - return String.format("CREATE DATABASE %s ENCODING='UTF-8'", database); + public void createDatabase() { + // Create database with non-database-session + LOG.debug("Create database: {}", this.database()); + + String sql = String.format(POSTGRESQL_DB_CREATE, this.database()); + try (Connection conn = this.openWithoutDB(0)) { + try { + conn.createStatement().execute(sql); + } catch (PSQLException e) { + // CockroackDB not support 'template' args of CREATE DATABASE + if (e.getMessage().contains("syntax error at or near " + + "\"template\"")) { + sql = String.format(COCKROACH_DB_CREATE, this.database()); + conn.createStatement().execute(sql); + } + } + } catch (SQLException e) { + if (!e.getMessage().endsWith("already exists")) { + throw new BackendException("Failed to create database '%s'", e, + this.database()); + } + // Ignore exception if database already exists + } } } diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java index 4775f86075..1a4ae005cc 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java @@ -38,8 +38,14 @@ public abstract class PostgresqlTable extends MysqlTable { public PostgresqlTable(String table) { super(table); - this.dropTableTemplate = "DROP TABLE IF EXISTS %s CASCADE;"; - this.truncateTableTemplate = "TRUNCATE TABLE %s CASCADE;"; + } + + protected String buildDropTemplate() { + return String.format("DROP TABLE IF EXISTS %s CASCADE;", this.table()); + } + + protected String buildTruncateTemplate() { + return String.format("TRUNCATE TABLE %s CASCADE;", this.table()); } @Override From f9f2d57e24fc61e71c4f494f9e0917d41c810ba0 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Mon, 20 May 2019 16:43:08 +0800 Subject: [PATCH 06/10] move serializeValue() to serializer for table serializer Change-Id: If9ef9b126e56d3ad368e87741995f6b4887e0d54 --- .../store/cassandra/CassandraTable.java | 18 +------- .../backend/serializer/TableSerializer.java | 41 +++++++++++++++---- .../backend/store/mysql/MysqlSerializer.java | 5 +++ .../backend/store/mysql/MysqlTable.java | 22 +--------- .../postgresql/PostgresqlSerializer.java | 20 +++++++++ .../store/postgresql/PostgresqlTable.java | 25 ----------- 6 files changed, 62 insertions(+), 69 deletions(-) diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java index 646eb6b449..7aadff7bd1 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java @@ -266,9 +266,6 @@ protected Clause relation2Cql(Relation relation) { String key = relation.serialKey().toString(); Object value = relation.serialValue(); - // Serialize value (TODO: should move to Serializer) - value = serializeValue(value); - switch (relation.relation()) { case EQ: return QueryBuilder.eq(key, value); @@ -281,12 +278,7 @@ protected Clause relation2Cql(Relation relation) { case LTE: return QueryBuilder.lte(key, value); case IN: - List values = (List) value; - List serializedValues = new ArrayList<>(values.size()); - for (Object v : values) { - serializedValues.add(serializeValue(v)); - } - return QueryBuilder.in(key, serializedValues); + return QueryBuilder.in(key, value); case CONTAINS: return QueryBuilder.contains(key, value); case CONTAINS_KEY: @@ -331,14 +323,6 @@ protected static Select cloneSelect(Select select, String table) { return CopyUtil.copy(select, QueryBuilder.select().from(table)); } - protected static Object serializeValue(Object value) { - // Serialize value (TODO: should move to Serializer) - if (value instanceof Id) { - value = ((Id) value).asObject(); - } - return value; - } - protected Iterator results2Entries(Query q, ResultSet r) { return new CassandraEntryIterator(r, q, (e1, row) -> { CassandraBackendEntry e2 = row2Entry(q.resultType(), row); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java index 63a5047b12..5a439c1c05 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java @@ -19,7 +19,9 @@ package com.baidu.hugegraph.backend.serializer; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Set; import com.baidu.hugegraph.HugeGraph; @@ -353,16 +355,27 @@ protected Query writeQueryEdgeCondition(Query query) { @Override protected Query writeQueryCondition(Query query) { - if (query.resultType().isGraph()) { - ConditionQuery result = (ConditionQuery) query; - // No user-prop when serialize - assert result.allSysprop(); - for (Condition.Relation r : result.relations()) { - if (r.relation() == Condition.RelationType.CONTAINS) { - r.serialValue(JsonUtil.toJson(r.value())); + ConditionQuery result = (ConditionQuery) query; + // No user-prop when serialize + assert result.allSysprop(); + for (Condition.Relation r : result.relations()) { + if (r.relation() == Condition.RelationType.IN) { + List values = (List) r.value(); + List serializedValues = new ArrayList<>(values.size()); + for (Object v : values) { + serializedValues.add(this.serializeValue(v)); } + r.serialValue(serializedValues); + } else { + r.serialValue(this.serializeValue(r.value())); + } + + if (query.resultType().isGraph() && + r.relation() == Condition.RelationType.CONTAINS) { + r.serialValue(JsonUtil.toJson(r.serialValue())); } } + return query; } @@ -577,6 +590,20 @@ protected abstract void formatProperties(HugeElement element, protected abstract void parseProperties(HugeElement element, TableBackendEntry.Row row); + protected Object serializeValue(Object value) { + if (value instanceof Id) { + value = ((Id) value).asObject(); + } + if (value instanceof String) { + value = escapeStrings((String) value); + } + return value; + } + + protected String escapeStrings(String value) { + return value; + } + protected void writeEnableLabelIndex(SchemaLabel schema, TableBackendEntry entry) { entry.column(HugeKeys.ENABLE_LABEL_INDEX, schema.enableLabelIndex()); diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSerializer.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSerializer.java index 6ad3012d6e..e57e26dc99 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSerializer.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSerializer.java @@ -162,4 +162,9 @@ protected void readUserdata(SchemaElement schema, schema.userdata(e.getKey(), e.getValue()); } } + + @Override + protected String escapeStrings(String value) { + return MysqlUtil.escapeString(value); + } } diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java index 7107a481a3..9eb5629d5d 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java @@ -427,9 +427,6 @@ protected StringBuilder relation2Sql(Condition.Relation relation) { String key = relation.serialKey().toString(); Object value = relation.serialValue(); - // Serialize value (TODO: should move to Serializer) - value = this.serializeValue(value); - StringBuilder sql = new StringBuilder(32); sql.append(key); switch (relation.relation()) { @@ -453,13 +450,8 @@ protected StringBuilder relation2Sql(Condition.Relation relation) { break; case IN: sql.append(" IN ("); - List values = (List) value; - for (int i = 0, n = values.size(); i < n; i++) { - sql.append(serializeValue(values.get(i))); - if (i != n - 1) { - sql.append(", "); - } - } + String values = Strings.join((List) value, ','); + sql.append(values); sql.append(")"); break; case CONTAINS: @@ -542,16 +534,6 @@ protected void wrapOffset(StringBuilder select, Query query) { select.append(";"); } - protected Object serializeValue(Object value) { - if (value instanceof Id) { - value = ((Id) value).asObject(); - } - if (value instanceof String) { - value = MysqlUtil.escapeString((String) value); - } - return value; - } - protected Iterator results2Entries(Query query, ResultSet results) { return new MysqlEntryIterator(results, query, this::mergeEntries); diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java index edd276c4ef..49fbef4789 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java @@ -19,8 +19,12 @@ package com.baidu.hugegraph.backend.store.postgresql; +import java.sql.SQLException; + import org.apache.logging.log4j.util.Strings; +import org.postgresql.core.Utils; +import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.backend.id.IdUtil; import com.baidu.hugegraph.backend.serializer.TableBackendEntry; import com.baidu.hugegraph.backend.store.BackendEntry; @@ -52,4 +56,20 @@ public BackendEntry writeIndex(HugeIndex index) { } return entry; } + + @Override + protected String escapeStrings(String value) { + if (value.equals("\u0000")) { + return "\'\'"; + } + StringBuilder builder = new StringBuilder(32); + builder.append('\''); + try { + Utils.escapeLiteral(builder, value, false); + } catch (SQLException e) { + throw new HugeException("Failed to escape '%s'", e, value); + } + builder.append('\''); + return builder.toString(); + } } diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java index 1a4ae005cc..5c8025302d 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java @@ -19,15 +19,11 @@ package com.baidu.hugegraph.backend.store.postgresql; -import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import org.apache.logging.log4j.util.Strings; -import org.postgresql.core.Utils; -import com.baidu.hugegraph.HugeException; -import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.store.mysql.MysqlBackendEntry; import com.baidu.hugegraph.backend.store.mysql.MysqlTable; import com.baidu.hugegraph.type.define.HugeKeys; @@ -129,25 +125,4 @@ protected String orderByKeys() { } return select.toString(); } - - protected Object serializeValue(Object value) { - if (value instanceof Id) { - value = ((Id) value).asObject(); - } - if (value instanceof String) { - if (value == "\u0000") { - return "\'\'"; - } - StringBuilder builder = new StringBuilder(32); - builder.append('\''); - try { - Utils.escapeLiteral(builder, (String) value, false); - } catch (SQLException e) { - throw new HugeException("Failed to escape '%s'", e, value); - } - builder.append('\''); - value = builder.toString(); - } - return value; - } } From dcb9e62a40414db8478be718e341a1d8a54e6ced Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Mon, 20 May 2019 18:08:47 +0800 Subject: [PATCH 07/10] tiny improve Change-Id: I24d95890295768e150621e07c57d7b9e8217589e --- .../backend/serializer/TableSerializer.java | 10 +++++--- .../hugegraph/backend/store/TableDefine.java | 3 +-- .../backend/store/mysql/MysqlSerializer.java | 2 +- .../backend/store/mysql/MysqlTable.java | 1 + .../store/postgresql/PostgresqlFeatures.java | 24 ------------------- .../postgresql/PostgresqlSerializer.java | 4 ++-- .../store/postgresql/PostgresqlStore.java | 3 --- .../store/postgresql/PostgresqlTable.java | 12 ++++++---- .../store/postgresql/PostgresqlTables.java | 7 +++--- 9 files changed, 24 insertions(+), 42 deletions(-) delete mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlFeatures.java diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java index 5a439c1c05..0115a91465 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java @@ -341,7 +341,8 @@ protected Query writeQueryEdgeCondition(Query query) { if (r.key() == HugeKeys.OWNER_VERTEX || r.key() == HugeKeys.OTHER_VERTEX) { // Serialize vertex id - r.serialValue(IdUtil.writeString((Id) value)); + String id = IdUtil.writeString((Id) value); + r.serialValue(this.escapeString(id)); } else { // Serialize label id r.serialValue(((Id) value).asObject()); @@ -359,6 +360,9 @@ protected Query writeQueryCondition(Query query) { // No user-prop when serialize assert result.allSysprop(); for (Condition.Relation r : result.relations()) { + if (!(r.value().equals(r.serialValue()))) { + continue; + } if (r.relation() == Condition.RelationType.IN) { List values = (List) r.value(); List serializedValues = new ArrayList<>(values.size()); @@ -595,12 +599,12 @@ protected Object serializeValue(Object value) { value = ((Id) value).asObject(); } if (value instanceof String) { - value = escapeStrings((String) value); + value = this.escapeString((String) value); } return value; } - protected String escapeStrings(String value) { + protected String escapeString(String value) { return value; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java index 27e5645686..ee952bfa64 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java @@ -28,8 +28,7 @@ import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.InsertionOrderUtil; - -import jersey.repackaged.com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap; public class TableDefine { diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSerializer.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSerializer.java index e57e26dc99..cda08f86e0 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSerializer.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSerializer.java @@ -164,7 +164,7 @@ protected void readUserdata(SchemaElement schema, } @Override - protected String escapeStrings(String value) { + protected String escapeString(String value) { return MysqlUtil.escapeString(value); } } diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java index 9eb5629d5d..fcc93857f1 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java @@ -427,6 +427,7 @@ protected StringBuilder relation2Sql(Condition.Relation relation) { String key = relation.serialKey().toString(); Object value = relation.serialValue(); + StringBuilder sql = new StringBuilder(32); sql.append(key); switch (relation.relation()) { diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlFeatures.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlFeatures.java deleted file mode 100644 index 8f033288a8..0000000000 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlFeatures.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2017 HugeGraph Authors - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.baidu.hugegraph.backend.store.postgresql; - -import com.baidu.hugegraph.backend.store.mysql.MysqlFeatures; - -public class PostgresqlFeatures extends MysqlFeatures {} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java index 49fbef4789..4a787e35c9 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java @@ -58,11 +58,11 @@ public BackendEntry writeIndex(HugeIndex index) { } @Override - protected String escapeStrings(String value) { + protected String escapeString(String value) { if (value.equals("\u0000")) { return "\'\'"; } - StringBuilder builder = new StringBuilder(32); + StringBuilder builder = new StringBuilder(8 + value.length()); builder.append('\''); try { Utils.escapeLiteral(builder, value, false); diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStore.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStore.java index 165f6d64eb..5bd11786a8 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStore.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStore.java @@ -19,15 +19,12 @@ package com.baidu.hugegraph.backend.store.postgresql; -import com.baidu.hugegraph.backend.store.BackendFeatures; import com.baidu.hugegraph.backend.store.BackendStoreProvider; import com.baidu.hugegraph.backend.store.mysql.MysqlStore; import com.baidu.hugegraph.config.HugeConfig; public abstract class PostgresqlStore extends MysqlStore { - private static final BackendFeatures FEATURES = new PostgresqlFeatures(); - public PostgresqlStore(BackendStoreProvider provider, String database, String name) { super(provider, database, name); diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java index 5c8025302d..112036085c 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java @@ -30,7 +30,8 @@ public abstract class PostgresqlTable extends MysqlTable { - private String insertTemplate; + private String insertTemplate = null; + private String orderByKeys = null; public PostgresqlTable(String table) { super(table); @@ -111,10 +112,12 @@ protected String buildInsertTemplate(MysqlBackendEntry.Row entry) { // Set order-by to keep results order consistence for PostgreSQL result protected String orderByKeys() { + if (this.orderByKeys != null) { + return this.orderByKeys; + } int i = 0; int size = this.tableDefine().keys().size(); - StringBuilder select = new StringBuilder(); - select.append(" ORDER BY "); + StringBuilder select = new StringBuilder(" ORDER BY "); for (HugeKeys hugeKey : this.tableDefine().keys()) { String key = formatKey(hugeKey); select.append(key).append(" "); @@ -123,6 +126,7 @@ protected String orderByKeys() { select.append(", "); } } - return select.toString(); + this.orderByKeys = select.toString(); + return this.orderByKeys; } } diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java index d9a676e511..b8514c330e 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java @@ -50,7 +50,8 @@ public class PostgresqlTables { private static final Map TYPES_MAPPING = ImmutableMap.builder() .put(BOOLEAN, "BOOL") - .put(TINYINT, INT) + .put(TINYINT, "INT") + .put(INT, "INT") .put(DOUBLE, "FLOAT") .put(SMALL_TEXT, "VARCHAR(255)") .put(MID_TEXT, "VARCHAR(1024)") @@ -165,7 +166,7 @@ protected BackendEntry mergeEntries(BackendEntry e1, BackendEntry e2) { public static class SecondaryIndex extends PostgresqlTableTemplate { - public static final String TABLE = "secondary_indexes"; + public static final String TABLE = MysqlTables.SecondaryIndex.TABLE; public SecondaryIndex(String store) { super(new MysqlTables.SecondaryIndex(store, TABLE, TYPES_MAPPING)); @@ -182,7 +183,7 @@ protected final String entryId(MysqlBackendEntry entry) { public static class SearchIndex extends SecondaryIndex { - public static final String TABLE = "search_indexes"; + public static final String TABLE = MysqlTables.SearchIndex.TABLE; public SearchIndex(String store) { super(store, TABLE); From 6ec35265aefd0ccc0112eb0db17e71f880ab054e Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 21 May 2019 10:48:26 +0800 Subject: [PATCH 08/10] tiny improve Change-Id: Ifbdfcee163a201919c8b7eabb89e5d478d5ba427 --- .../java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java index fcc93857f1..9eb5629d5d 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java @@ -427,7 +427,6 @@ protected StringBuilder relation2Sql(Condition.Relation relation) { String key = relation.serialKey().toString(); Object value = relation.serialValue(); - StringBuilder sql = new StringBuilder(32); sql.append(key); switch (relation.relation()) { From ebcaa422001a206347140e22f1593b91bcadd08a Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Tue, 21 May 2019 16:06:12 +0800 Subject: [PATCH 09/10] tiny improve Change-Id: I698c30fd29d28cc6d6f83dcccd04571df466f9de --- .../com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java index db9fc53a43..240b7e7deb 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java @@ -93,7 +93,7 @@ public static synchronized MysqlOptions instance() { public static final ConfigOption SSL_MODE = new ConfigOption<>( "jdbc.ssl_mode", - "The url of database in JDBC format.", + "The SSL mode of connections with database.", disallowEmpty(), "disable" ); From 5d352a196f4b6afba02357d2ed8ac2d2de16bfa7 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Wed, 22 May 2019 14:43:47 +0800 Subject: [PATCH 10/10] tiny improve Change-Id: I02658e1173c8c23e44d1dc53d2a03d33d02e4338 --- .../com/baidu/hugegraph/backend/store/mysql/MysqlStore.java | 2 +- .../backend/store/postgresql/PostgresqlStoreProvider.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java index 1e8fc05d96..3c39ddbbc6 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java @@ -57,7 +57,7 @@ public abstract class MysqlStore extends AbstractBackendStore { private final Map tables; - protected MysqlSessions sessions; + private MysqlSessions sessions; public MysqlStore(final BackendStoreProvider provider, final String database, final String store) { diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStoreProvider.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStoreProvider.java index 52521518f6..85f68e022e 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStoreProvider.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStoreProvider.java @@ -84,14 +84,14 @@ protected Collection tables() { @Override public void increaseCounter(HugeType type, long increment) { this.checkSessionConnected(); - MysqlSessions.Session session = super.sessions.session(); + MysqlSessions.Session session = this.session(type); this.counters.increaseCounter(session, type, increment); } @Override public long getCounter(HugeType type) { this.checkSessionConnected(); - MysqlSessions.Session session = super.sessions.session(); + MysqlSessions.Session session = this.session(type); return this.counters.getCounter(session, type); } }