diff --git a/hugegraph-client/pom.xml b/hugegraph-client/pom.xml index ad96edea8..99505558b 100644 --- a/hugegraph-client/pom.xml +++ b/hugegraph-client/pom.xml @@ -38,7 +38,11 @@ hugegraph-common ${hugegraph.common.version} - + + org.lz4 + lz4-java + 1.4.0 + org.glassfish.jersey.containers jersey-container-servlet diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/example/BatchExample.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/example/BatchExample.java index 94f59e5e2..8f2da31fc 100644 --- a/hugegraph-client/src/main/java/com/baidu/hugegraph/example/BatchExample.java +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/example/BatchExample.java @@ -31,7 +31,7 @@ public class BatchExample { public static void main(String[] args) { - // If connect failed will throw a exception. + // If connect failed will throw an exception. HugeClient hugeClient = HugeClient.builder("http://localhost:8080", "hugegraph").build(); diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/HBaseSerializer.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/HBaseSerializer.java new file mode 100644 index 000000000..024618fa6 --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/HBaseSerializer.java @@ -0,0 +1,124 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct; + +import com.baidu.hugegraph.driver.HugeClient; +import com.baidu.hugegraph.serializer.direct.struct.HugeType; +import com.baidu.hugegraph.serializer.direct.util.BytesBuffer; +import com.baidu.hugegraph.serializer.direct.util.GraphSchema; +import com.baidu.hugegraph.serializer.direct.util.Id; +import com.baidu.hugegraph.serializer.direct.util.IdGenerator; +import com.baidu.hugegraph.structure.GraphElement; +import com.baidu.hugegraph.structure.graph.Edge; +import com.baidu.hugegraph.structure.schema.PropertyKey; + +import java.util.Arrays; +import java.util.Map; + +/** + * TODO: review later + */ +public class HBaseSerializer { + + private int edgeLogicPartitions = 30; + private int vertexLogicPartitions = 10; + private HugeClient client; + private GraphSchema graphSchema; + + + public HBaseSerializer(HugeClient client, int vertexPartitions, int edgePartitions){ + this.client = client; + this.graphSchema = new GraphSchema(client); + this.edgeLogicPartitions = edgePartitions; + this.vertexLogicPartitions = vertexPartitions; + } + + public byte[] getKeyBytes(GraphElement e) { + byte[] array = null; + if(e.type() == "vertex" && e.id() != null){ + BytesBuffer buffer = BytesBuffer.allocate(2 + 1 + e.id().toString().length()); + buffer.writeShort(getPartition(HugeType.VERTEX, IdGenerator.of(e.id()))); + buffer.writeId(IdGenerator.of(e.id())); + array = buffer.bytes(); + }else if ( e.type() == "edge" ){ + BytesBuffer buffer = BytesBuffer.allocate(BytesBuffer.BUF_EDGE_ID); + Edge edge = (Edge)e; + buffer.writeShort(getPartition(HugeType.EDGE, IdGenerator.of(edge.sourceId()))); + buffer.writeId(IdGenerator.of(edge.sourceId())); + buffer.write(HugeType.EDGE_OUT.code()); + buffer.writeId(IdGenerator.of(graphSchema.getEdgeLabel(e.label()).id())); //出现错误 + buffer.writeStringWithEnding(""); + buffer.writeId(IdGenerator.of(edge.targetId())); + array = buffer.bytes(); + } + return array; + } + + public byte[] getValueBytes(GraphElement e) { + byte[] array = null; + if(e.type() == "vertex"){ + int propsCount = e.properties().size() ; //vertex.sizeOfProperties(); + BytesBuffer buffer = BytesBuffer.allocate(8 + 16 * propsCount); + buffer.writeId(IdGenerator.of(graphSchema.getVertexLabel(e.label()).id())); + buffer.writeVInt(propsCount); + for(Map.Entry entry : e.properties().entrySet()){ + PropertyKey propertyKey = graphSchema.getPropertyKey(entry.getKey()); + buffer.writeVInt(propertyKey.id().intValue()); + buffer.writeProperty(propertyKey.dataType(),entry.getValue()); + } + array = buffer.bytes(); + } else if ( e.type() == "edge" ){ + int propsCount = e.properties().size(); + BytesBuffer buffer = BytesBuffer.allocate(4 + 16 * propsCount); + buffer.writeVInt(propsCount); + for(Map.Entry entry : e.properties().entrySet()){ + PropertyKey propertyKey = graphSchema.getPropertyKey(entry.getKey()); + buffer.writeVInt(propertyKey.id().intValue()); + buffer.writeProperty(propertyKey.dataType(),entry.getValue()); + } + array = buffer.bytes(); + } + + return array; + } + + public short getPartition(HugeType type, Id id) { + int hashcode = Arrays.hashCode(id.asBytes()); + short partition = 1; + if (type.isEdge()) { + partition = (short) (hashcode % edgeLogicPartitions); + } else if (type.isVertex()) { + partition = (short) (hashcode % vertexLogicPartitions); + } + return partition > 0 ? partition : (short) -partition; + } + + public int getEdgeLogicPartitions(){ + return this.edgeLogicPartitions; + } + + public int getVertexLogicPartitions(){ + return this.vertexLogicPartitions; + } + + public void close(){ + this.client.close(); + } +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/RocksDBSerializer.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/RocksDBSerializer.java new file mode 100644 index 000000000..de964f3cb --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/RocksDBSerializer.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct; + +/** + * TODO: review later + * + * In this serializer, we only support normal type now: + * - number + * - string + * And they will be transferred to bytes directly + **/ +public class RocksDBSerializer { + // TODO: Support write RocksDB directly +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/reuse/BytesDemo.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/reuse/BytesDemo.java new file mode 100644 index 000000000..dc3d99678 --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/reuse/BytesDemo.java @@ -0,0 +1,194 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct.reuse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import com.baidu.hugegraph.driver.GraphManager; +import com.baidu.hugegraph.driver.HugeClient; +import com.baidu.hugegraph.driver.SchemaManager; +import com.baidu.hugegraph.serializer.direct.HBaseSerializer; +import com.baidu.hugegraph.serializer.direct.RocksDBSerializer; +import com.baidu.hugegraph.structure.graph.Edge; +import com.baidu.hugegraph.structure.graph.Vertex; + +/** + * @author jin + * This class is a demo for rocksdb/HBase put(rowkey, values) which use Client-Side's graph struct + * And we don't need to construct the graph element, just use it and transfer them to bytes array + * instead of json format + */ +public class BytesDemo { + + static HugeClient client; + boolean bypassServer = true; + RocksDBSerializer ser; + HBaseSerializer HBaseSer; + + public static void main(String[] args) { + BytesDemo ins = new BytesDemo(); + ins.initGraph(); + } + + void initGraph() { + int edgeLogicPartitions = 16; + int vertexLogicPartitions = 8; + // If connect failed will throw an exception. + client = HugeClient.builder("http://localhost:8081", "hugegraph").build(); + + SchemaManager schema = client.schema(); + + + schema.propertyKey("name").asText().ifNotExist().create(); + schema.propertyKey("age").asInt().ifNotExist().create(); + schema.propertyKey("lang").asText().ifNotExist().create(); + schema.propertyKey("date").asText().ifNotExist().create(); + schema.propertyKey("price").asText().ifNotExist().create(); + + schema.vertexLabel("person") + .properties("name", "age") + .useCustomizeStringId() + .enableLabelIndex(false) + .ifNotExist() + .create(); + + schema.vertexLabel("personB") + .properties("price") + .nullableKeys("price") + .useCustomizeNumberId() + .enableLabelIndex(false) + .ifNotExist() + .create(); + + schema.vertexLabel("software") + .properties("name", "lang", "price") + .useCustomizeStringId() + .enableLabelIndex(false) + .ifNotExist() + .create(); + + schema.edgeLabel("knows") + .link("person", "person") + .properties("date") + .enableLabelIndex(false) + .ifNotExist() + .create(); + + schema.edgeLabel("created") + .link("person", "software") + .properties("date") + .enableLabelIndex(false) + .ifNotExist() + .create(); + + HBaseSer = new HBaseSerializer(client, vertexLogicPartitions, edgeLogicPartitions); + writeGraphElements(); + + client.close(); + } + + private void writeGraphElements() { + GraphManager graph = client.graph(); + // construct some vertexes & edges + Vertex peter = new Vertex("person"); + peter.property("name", "peter"); + peter.property("age", 35); + peter.id("peter"); + + Vertex lop = new Vertex("software"); + lop.property("name", "lop"); + lop.property("lang", "java"); + lop.property("price", "328"); + lop.id("lop"); + + Vertex vadasB = new Vertex("personB"); + vadasB.property("price", "120"); + vadasB.id(12345); + + Edge peterCreateLop = new Edge("created").source(peter).target(lop).property("date", "2017-03-24"); + + List vertices = new ArrayList(){{ + add(peter);add(lop);add(vadasB); + }}; + + + List edges = new ArrayList(){{ + add(peterCreateLop); + }}; + + // Old way: encode to json then send to server + if (bypassServer) { + writeDirectly(vertices, edges); + } else { + writeByServer(graph, vertices, edges); + } + } + + /* we transfer the vertex & edge into bytes array + * TODO: use a batch and send them together + * */ + void writeDirectly(List vertices, List edges) { + for (Vertex vertex : vertices) { + byte[] rowkey = HBaseSer.getKeyBytes(vertex); + byte[] values = HBaseSer.getValueBytes(vertex); + sendRpcToHBase("vertex", rowkey, values); + } + + for (Edge edge: edges) { + byte[] rowkey = HBaseSer.getKeyBytes(edge); + byte[] values = HBaseSer.getValueBytes(edge); + sendRpcToHBase("edge", rowkey, values); + } + } + + boolean sendRpcToRocksDB(byte[] rowkey, byte[] values) { + // here we call the rpc + boolean flag = false; + //flag = put(rowkey, values); + return flag; + } + + void writeByServer(GraphManager graph, List vertices, List edges) { + vertices = graph.addVertices(vertices); + vertices.forEach(System.out::println); + + edges = graph.addEdges(edges, false); + edges.forEach(System.out::println); + } + + boolean sendRpcToHBase(String type, byte[] rowkey, byte[] values) { + boolean flag = false; + try { + flag = put(type, rowkey, values); + } catch (IOException e) { + e.printStackTrace(); + } + return flag; + } + + + boolean put (String type, byte[] rowkey, byte[] values) throws IOException { + // TODO: put to HBase + return true; + } + +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/struct/DataType.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/struct/DataType.java new file mode 100644 index 000000000..a12aa4607 --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/struct/DataType.java @@ -0,0 +1,205 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct.struct; + +import java.util.Date; +import java.util.UUID; + +import com.baidu.hugegraph.serializer.direct.util.HugeException; +import com.baidu.hugegraph.serializer.direct.util.StringEncoding; +import com.baidu.hugegraph.util.CollectionUtil; +import com.baidu.hugegraph.util.DateUtil; +import com.baidu.hugegraph.util.E; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Table; + +public enum DataType { + + UNKNOWN(0, "unknown", Object.class), + OBJECT(1, "object", Object.class), + BOOLEAN(2, "boolean", Boolean.class), + BYTE(3, "byte", Byte.class), + INT(4, "int", Integer.class), + LONG(5, "long", Long.class), + FLOAT(6, "float", Float.class), + DOUBLE(7, "double", Double.class), + TEXT(8, "text", String.class), + //BLOB(9, "blob", Blob.class), + DATE(10, "date", Date.class), + UUID(11, "uuid", UUID.class); + + private final byte code; + private final String name; + private final Class clazz; + + static { + register(DataType.class); + } + + static Table, Byte, DataType> TABLE = HashBasedTable.create(); + + static void register(Class clazz) { + Object enums; + try { + enums = clazz.getMethod("values").invoke(null); + } catch (Exception e) { + throw new HugeException("DataType invalid", e); + } + for (DataType e : CollectionUtil.toList(enums)) { + TABLE.put(clazz, e.code(), e); + } + } + + static T fromCode(Class clazz, byte code) { + @SuppressWarnings("unchecked") + T value = (T) TABLE.get(clazz, code); + if (value == null) { + E.checkArgument(false, "Can't construct %s from code %s", + clazz.getSimpleName(), code); + } + return value; + } + + DataType(int code, String name, Class clazz) { + assert code < 256; + this.code = (byte) code; + this.name = name; + this.clazz = clazz; + } + + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public Class clazz() { + return this.clazz; + } + + public boolean isText() { + return this == DataType.TEXT; + } + + public boolean isNumber() { + return this == BYTE || this == INT || this == LONG || + this == FLOAT || this == DOUBLE; + } + + public boolean isNumber4() { + // Store index value of Byte using 4 bytes + return this == BYTE || this == INT || this == FLOAT; + } + + public boolean isNumber8() { + return this == LONG || this == DOUBLE; + } + + //public boolean isBlob() { + // return this == DataType.BLOB; + //} + + public boolean isDate() { + return this == DataType.DATE; + } + + public boolean isUUID() { + return this == DataType.UUID; + } + + public Number valueToNumber(V value) { + if (!(this.isNumber() && value instanceof Number)) { + return null; + } + if (this.clazz.isInstance(value)) { + return (Number) value; + } + + Number number; + try { + switch (this) { + case BYTE: + number = Byte.valueOf(value.toString()); + break; + case INT: + number = Integer.valueOf(value.toString()); + break; + case LONG: + number = Long.valueOf(value.toString()); + break; + case FLOAT: + number = Float.valueOf(value.toString()); + break; + case DOUBLE: + number = Double.valueOf(value.toString()); + break; + default: + throw new AssertionError(String.format( + "Number type only contains Byte, Integer, " + + "Long, Float, Double, but got %s", this.clazz())); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "Can't read '%s' as %s: %s", + value, this.name, e.getMessage())); + } + return number; + } + + public Date valueToDate(V value) { + if (!this.isDate()) { + return null; + } + if (value instanceof Date) { + return (Date) value; + } else if (value instanceof Integer) { + return new Date(((Number) value).intValue()); + } else if (value instanceof Long) { + return new Date(((Number) value).longValue()); + } else if (value instanceof String) { + return DateUtil.parse((String) value); + } + return null; + } + + public UUID valueToUUID(V value) { + if (!this.isUUID()) { + return null; + } + if (value instanceof UUID) { + return (UUID) value; + } else if (value instanceof String) { + return StringEncoding.uuid((String) value); + } + return null; + } + + + public static DataType fromClass(Class clazz) { + for (DataType type : DataType.values()) { + if (type.clazz() == clazz) { + return type; + } + } + throw new HugeException("Unknown clazz '%s' for DataType", clazz); + } +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/struct/HugeType.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/struct/HugeType.java new file mode 100644 index 000000000..7c88567a5 --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/struct/HugeType.java @@ -0,0 +1,190 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct.struct; + +import java.util.HashMap; +import java.util.Map; + +public enum HugeType { + + UNKNOWN(0, "UNKNOWN"), + + /* Schema types */ + VERTEX_LABEL(1, "VL"), + EDGE_LABEL(2, "EL"), + PROPERTY_KEY(3, "PK"), + INDEX_LABEL(4, "IL"), + + META(40, "M"), + COUNTER(50, "C"), + + /* Data types */ + VERTEX(101, "V"), + // System meta + SYS_PROPERTY(102, "S"), + // Property + PROPERTY(103, "U"), + // Vertex aggregate property + AGGR_PROPERTY_V(104, "VP"), + // Edge aggregate property + AGGR_PROPERTY_E(105, "EP"), + // Olap property + OLAP(106, "AP"), + // Edge + EDGE(120, "E"), + // Edge's direction is OUT for the specified vertex + EDGE_OUT(130, "O"), + // Edge's direction is IN for the specified vertex + EDGE_IN(140, "I"), + + SECONDARY_INDEX(150, "SI"), + VERTEX_LABEL_INDEX(151, "VI"), + EDGE_LABEL_INDEX(152, "EI"), + RANGE_INT_INDEX(160, "II"), + RANGE_FLOAT_INDEX(161, "FI"), + RANGE_LONG_INDEX(162, "LI"), + RANGE_DOUBLE_INDEX(163, "DI"), + SEARCH_INDEX(170, "AI"), + SHARD_INDEX(175, "HI"), + UNIQUE_INDEX(178, "UI"), + + TASK(180, "T"), + + // System schema + SYS_SCHEMA(250, "SS"), + + MAX_TYPE(255, "~"); + + private byte type = 0; + private String name; + + private static final Map ALL_NAME; + + static { + ALL_NAME = new HashMap<>(); + for (HugeType type : values()) { + ALL_NAME.put(type.name, type); + } + } + + HugeType(int type, String name) { + assert type < 256; + this.type = (byte) type; + this.name = name; + } + + public byte code() { + return this.type; + } + + public String string() { + return this.name; + } + + public String readableName() { + return this.name().replace('_', ' ').toLowerCase(); + } + + public boolean isSchema() { + return this == HugeType.VERTEX_LABEL || + this == HugeType.EDGE_LABEL || + this == HugeType.PROPERTY_KEY || + this == HugeType.INDEX_LABEL; + } + + public boolean isGraph() { + return this.isVertex() || this.isEdge(); + } + + public boolean isVertex() { + return this == HugeType.VERTEX; + } + + public boolean isEdge() { + return this == EDGE || this == EDGE_OUT || this == EDGE_IN; + } + + public boolean isIndex() { + return this == VERTEX_LABEL_INDEX || this == EDGE_LABEL_INDEX || + this == SECONDARY_INDEX || this == SEARCH_INDEX || + this == RANGE_INT_INDEX || this == RANGE_FLOAT_INDEX || + this == RANGE_LONG_INDEX || this == RANGE_DOUBLE_INDEX || + this == SHARD_INDEX || this == UNIQUE_INDEX; + } + + public boolean isStringIndex() { + return this == VERTEX_LABEL_INDEX || this == EDGE_LABEL_INDEX || + this == SECONDARY_INDEX || this == SEARCH_INDEX || + this == SHARD_INDEX || this == UNIQUE_INDEX; + } + + public boolean isNumericIndex() { + return this == RANGE_INT_INDEX || this == RANGE_FLOAT_INDEX || + this == RANGE_LONG_INDEX || this == RANGE_DOUBLE_INDEX || + this == SHARD_INDEX; + } + + public boolean isSecondaryIndex() { + return this == VERTEX_LABEL_INDEX || this == EDGE_LABEL_INDEX || + this == SECONDARY_INDEX; + } + + public boolean isSearchIndex() { + return this == SEARCH_INDEX; + } + + public boolean isRangeIndex() { + return this == RANGE_INT_INDEX || this == RANGE_FLOAT_INDEX || + this == RANGE_LONG_INDEX || this == RANGE_DOUBLE_INDEX; + } + + public boolean isRange4Index() { + return this == RANGE_INT_INDEX || this == RANGE_FLOAT_INDEX; + } + + public boolean isRange8Index() { + return this == RANGE_LONG_INDEX || this == RANGE_DOUBLE_INDEX; + } + + public boolean isShardIndex() { + return this == SHARD_INDEX; + } + + public boolean isUniqueIndex() { + return this == UNIQUE_INDEX; + } + + public boolean isVertexAggregateProperty() { + return this == AGGR_PROPERTY_V; + } + + public boolean isEdgeAggregateProperty() { + return this == AGGR_PROPERTY_E; + } + + public boolean isAggregateProperty() { + return this.isVertexAggregateProperty() || + this.isEdgeAggregateProperty(); + } + + public static HugeType fromString(String type) { + return ALL_NAME.get(type); + } +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/BytesBuffer.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/BytesBuffer.java new file mode 100644 index 000000000..81320e865 --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/BytesBuffer.java @@ -0,0 +1,759 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct.util; + +import com.baidu.hugegraph.serializer.direct.struct.HugeType; +import com.baidu.hugegraph.util.*; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Date; +import java.util.UUID; + +/** + * class BytesBuffer is a util for read/write binary + */ +public final class BytesBuffer extends OutputStream { + + public static final int BYTE_LEN = Byte.BYTES; + public static final int SHORT_LEN = Short.BYTES; + public static final int INT_LEN = Integer.BYTES; + public static final int LONG_LEN = Long.BYTES; + public static final int CHAR_LEN = Character.BYTES; + public static final int FLOAT_LEN = Float.BYTES; + public static final int DOUBLE_LEN = Double.BYTES; + public static final int BLOB_LEN = 4; + + public static final int UINT8_MAX = ((byte) -1) & 0xff; + public static final int UINT16_MAX = ((short) -1) & 0xffff; + public static final long UINT32_MAX = (-1) & 0xffffffffL; + + // NOTE: +1 to let code 0 represent length 1 + public static final int ID_LEN_MASK = 0x7f; + public static final int ID_LEN_MAX = 0x7f + 1; // 128 + public static final int BIG_ID_LEN_MAX = 0x7fff + 1; // 32768 + + public static final byte STRING_ENDING_BYTE = (byte) 0x00; + public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff; + public static final int STRING_LEN_MAX = UINT16_MAX; + public static final long BLOB_LEN_MAX = 1 * Bytes.GB; + + // The value must be in range [8, ID_LEN_MAX] + public static final int INDEX_HASH_ID_THRESHOLD = 32; + + public static final int DEFAULT_CAPACITY = 64; + public static final int MAX_BUFFER_CAPACITY = 128 * 1024 * 1024; // 128M + + public static final int BUF_EDGE_ID = 128; + public static final int BUF_PROPERTY = 64; + + private ByteBuffer buffer; + private final boolean resize; + + public BytesBuffer() { + this(DEFAULT_CAPACITY); + } + + public BytesBuffer(int capacity) { + E.checkArgument(capacity <= MAX_BUFFER_CAPACITY, + "Capacity exceeds max buffer capacity: %s", + MAX_BUFFER_CAPACITY); + this.buffer = ByteBuffer.allocate(capacity); + this.resize = true; + } + + public BytesBuffer(ByteBuffer buffer) { + E.checkNotNull(buffer, "buffer"); + this.buffer = buffer; + this.resize = false; + } + + public static BytesBuffer allocate(int capacity) { + return new BytesBuffer(capacity); + } + + public static BytesBuffer wrap(ByteBuffer buffer) { + return new BytesBuffer(buffer); + } + + public static BytesBuffer wrap(byte[] array) { + return new BytesBuffer(ByteBuffer.wrap(array)); + } + + public static BytesBuffer wrap(byte[] array, int offset, int length) { + return new BytesBuffer(ByteBuffer.wrap(array, offset, length)); + } + + public ByteBuffer asByteBuffer() { + return this.buffer; + } + + public BytesBuffer forReadWritten() { + this.buffer.flip(); + return this; + } + + public BytesBuffer forReadAll() { + this.buffer.position(this.buffer.limit()); + return this; + } + + public byte[] array() { + return this.buffer.array(); + } + + public byte[] bytes() { + byte[] bytes = this.buffer.array(); + int position = this.buffer.position(); + if (position == bytes.length) { + return bytes; + } else { + return Arrays.copyOf(bytes, position); + } + } + + public int position() { + return this.buffer.position(); + } + + public BytesBuffer copyFrom(BytesBuffer other) { + this.write(other.bytes()); + return this; + } + + public int remaining() { + return this.buffer.remaining(); + } + + private void require(int size) { + // Does need to resize? + if (this.buffer.limit() - this.buffer.position() >= size) { + return; + } + // Can't resize for wrapped buffer since will change the origin ref + E.checkState(this.resize, "Can't resize for wrapped buffer"); + + // Extra capacity as buffer + int newcapacity = size + this.buffer.limit() + DEFAULT_CAPACITY; + E.checkArgument(newcapacity <= MAX_BUFFER_CAPACITY, + "Capacity exceeds max buffer capacity: %s", + MAX_BUFFER_CAPACITY); + ByteBuffer newBuffer = ByteBuffer.allocate(newcapacity); + this.buffer.flip(); + newBuffer.put(this.buffer); + this.buffer = newBuffer; + } + + public BytesBuffer write(byte val) { + require(BYTE_LEN); + this.buffer.put(val); + return this; + } + + @Override + public void write(int val) { + assert val <= UINT8_MAX; + require(BYTE_LEN); + this.buffer.put((byte) val); + } + + @Override + public void write(byte[] val) { + require(BYTE_LEN * val.length); + this.buffer.put(val); + } + + @Override + public void write(byte[] val, int offset, int length) { + require(BYTE_LEN * length); + this.buffer.put(val, offset, length); + } + + public BytesBuffer writeBoolean(boolean val) { + this.write(val ? 1 : 0); + return this; + } + + public BytesBuffer writeChar(char val) { + require(CHAR_LEN); + this.buffer.putChar(val); + return this; + } + + public BytesBuffer writeShort(short val) { + require(SHORT_LEN); + this.buffer.putShort(val); + return this; + } + + public BytesBuffer writeInt(int val) { + require(INT_LEN); + this.buffer.putInt(val); + return this; + } + + public BytesBuffer writeLong(long val) { + require(LONG_LEN); + this.buffer.putLong(val); + return this; + } + + public BytesBuffer writeFloat(float val) { + require(FLOAT_LEN); + this.buffer.putFloat(val); + return this; + } + + public BytesBuffer writeDouble(double val) { + require(DOUBLE_LEN); + this.buffer.putDouble(val); + return this; + } + + public byte peek() { + return this.buffer.get(this.buffer.position()); + } + + public byte peekLast() { + return this.buffer.get(this.buffer.capacity() - 1); + } + + public byte read() { + return this.buffer.get(); + } + + public byte[] read(int length) { + byte[] bytes = new byte[length]; + this.buffer.get(bytes); + return bytes; + } + + public boolean readBoolean() { + return this.buffer.get() != 0; + } + + public char readChar() { + return this.buffer.getChar(); + } + + public short readShort() { + return this.buffer.getShort(); + } + + public int readInt() { + return this.buffer.getInt(); + } + + public long readLong() { + return this.buffer.getLong(); + } + + public float readFloat() { + return this.buffer.getFloat(); + } + + public double readDouble() { + return this.buffer.getDouble(); + } + + public BytesBuffer writeBytes(byte[] bytes) { + E.checkArgument(bytes.length <= UINT16_MAX, + "The max length of bytes is %s, but got %s", + UINT16_MAX, bytes.length); + require(SHORT_LEN + bytes.length); + this.writeVInt(bytes.length); + this.write(bytes); + return this; + } + + public byte[] readBytes() { + int length = this.readVInt(); + assert length >= 0; + byte[] bytes = this.read(length); + return bytes; + } + + public BytesBuffer writeBigBytes(byte[] bytes) { + E.checkArgument(bytes.length <= BLOB_LEN_MAX, + "The max length of bytes is %s, but got %s", + BLOB_LEN_MAX, bytes.length); + require(BLOB_LEN + bytes.length); + this.writeVInt(bytes.length); + this.write(bytes); + return this; + } + + public byte[] readBigBytes() { + int length = this.readVInt(); + assert length >= 0; + byte[] bytes = this.read(length); + return bytes; + } + + public BytesBuffer writeStringRaw(String val) { + this.write(StringEncoding.encode(val)); + return this; + } + + public BytesBuffer writeString(String val) { + byte[] bytes = StringEncoding.encode(val); + this.writeBytes(bytes); + return this; + } + + public String readString() { + return StringEncoding.decode(this.readBytes()); + } + + public BytesBuffer writeStringWithEnding(String value) { + if (!value.isEmpty()) { + byte[] bytes = StringEncoding.encode(value); + /* + * assert '0x00'/'0xFF' not exist in string index id + * NOTE: + * 0x00 is NULL in UTF8(or ASCII) bytes + * 0xFF is not a valid byte in UTF8 bytes + */ + assert !Bytes.contains(bytes, STRING_ENDING_BYTE_FF) : + "Invalid UTF8 bytes: " + value; + if (Bytes.contains(bytes, STRING_ENDING_BYTE)) { + E.checkArgument(false, + "Can't contains byte '0x00' in string: '%s'", + value); + } + this.write(bytes); + } + /* + * Choose 0x00 as ending symbol (see #1057) + * The following is out of date: + * A reasonable ending symbol should be 0x00(to ensure order), but + * considering that some backends like PG do not support 0x00 string, + * so choose 0xFF currently. + */ + this.write(STRING_ENDING_BYTE); + return this; + } + + public String readStringWithEnding() { + return StringEncoding.decode(this.readBytesWithEnding()); + } + + public BytesBuffer writeStringToRemaining(String value) { + byte[] bytes = StringEncoding.encode(value); + this.write(bytes); + return this; + } + + public String readStringFromRemaining() { + byte[] bytes = new byte[this.buffer.remaining()]; + this.buffer.get(bytes); + return StringEncoding.decode(bytes); + } + + public BytesBuffer writeUInt8(int val) { + assert val <= UINT8_MAX; + this.write(val); + return this; + } + + public int readUInt8() { + return this.read() & 0x000000ff; + } + + public BytesBuffer writeUInt16(int val) { + assert val <= UINT16_MAX; + this.writeShort((short) val); + return this; + } + + public int readUInt16() { + return this.readShort() & 0x0000ffff; + } + + public BytesBuffer writeUInt32(long val) { + assert val <= UINT32_MAX; + this.writeInt((int) val); + return this; + } + + public long readUInt32() { + return this.readInt() & 0xffffffffL; + } + + public BytesBuffer writeVInt(int value) { + // NOTE: negative numbers are not compressed + if (value > 0x0fffffff || value < 0) { + this.write(0x80 | ((value >>> 28) & 0x7f)); + } + if (value > 0x1fffff || value < 0) { + this.write(0x80 | ((value >>> 21) & 0x7f)); + } + if (value > 0x3fff || value < 0) { + this.write(0x80 | ((value >>> 14) & 0x7f)); + } + if (value > 0x7f || value < 0) { + this.write(0x80 | ((value >>> 7) & 0x7f)); + } + this.write(value & 0x7f); + + return this; + } + + public int readVInt() { + byte leading = this.read(); + E.checkArgument(leading != 0x80, + "Unexpected varint with leading byte '0x%s'", + Bytes.toHex(leading)); + int value = leading & 0x7f; + if (leading >= 0) { + assert (leading & 0x80) == 0; + return value; + } + + int i = 1; + for (; i < 5; i++) { + byte b = this.read(); + if (b >= 0) { + value = b | (value << 7); + break; + } else { + value = (b & 0x7f) | (value << 7); + } + } + + E.checkArgument(i < 5, + "Unexpected varint %s with too many bytes(%s)", + value, i + 1); + E.checkArgument(i < 4 || (leading & 0x70) == 0, + "Unexpected varint %s with leading byte '0x%s'", + value, Bytes.toHex(leading)); + return value; + } + + public BytesBuffer writeVLong(long value) { + if (value < 0) { + this.write((byte) 0x81); + } + if (value > 0xffffffffffffffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 56) & 0x7f)); + } + if (value > 0x1ffffffffffffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 49) & 0x7f)); + } + if (value > 0x3ffffffffffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 42) & 0x7f)); + } + if (value > 0x7ffffffffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 35) & 0x7f)); + } + if (value > 0xfffffffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 28) & 0x7f)); + } + if (value > 0x1fffffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 21) & 0x7f)); + } + if (value > 0x3fffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 14) & 0x7f)); + } + if (value > 0x7fL || value < 0L) { + this.write(0x80 | ((int) (value >>> 7) & 0x7f)); + } + this.write((int) value & 0x7f); + + return this; + } + + public long readVLong() { + byte leading = this.read(); + E.checkArgument(leading != 0x80, + "Unexpected varlong with leading byte '0x%s'", + Bytes.toHex(leading)); + long value = leading & 0x7fL; + if (leading >= 0) { + assert (leading & 0x80) == 0; + return value; + } + + int i = 1; + for (; i < 10; i++) { + byte b = this.read(); + if (b >= 0) { + value = b | (value << 7); + break; + } else { + value = (b & 0x7f) | (value << 7); + } + } + + E.checkArgument(i < 10, + "Unexpected varlong %s with too many bytes(%s)", + value, i + 1); + E.checkArgument(i < 9 || (leading & 0x7e) == 0, + "Unexpected varlong %s with leading byte '0x%s'", + value, Bytes.toHex(leading)); + return value; + } + + public BytesBuffer writeId(Id id) { + return this.writeId(id, false); + } + + public BytesBuffer writeId(Id id, boolean big) { + switch (id.type()) { + case LONG: + // Number Id + long value = id.asLong(); + this.writeNumber(value); + break; + case UUID: + // UUID Id + byte[] bytes = id.asBytes(); + assert bytes.length == Id.UUID_LENGTH; + this.writeUInt8(0x7f); // 0b01111111 means UUID + this.write(bytes); + break; + default: + // String Id + bytes = id.asBytes(); + int len = bytes.length; + E.checkArgument(len > 0, "Can't write empty id"); + if (!big) { + E.checkArgument(len <= ID_LEN_MAX, + "Id max length is %s, but got %s {%s}", + ID_LEN_MAX, len, id); + len -= 1; // mapping [1, 128] to [0, 127] + this.writeUInt8(len | 0x80); + } else { + E.checkArgument(len <= BIG_ID_LEN_MAX, + "Big id max length is %s, but got %s {%s}", + BIG_ID_LEN_MAX, len, id); + len -= 1; + int high = len >> 8; + int low = len & 0xff; + this.writeUInt8(high | 0x80); + this.writeUInt8(low); + } + this.write(bytes); + break; + } + return this; + } + + public BytesBuffer writeIndexId(Id id, HugeType type) { + return this.writeIndexId(id, type, true); + } + + public BytesBuffer writeIndexId(Id id, HugeType type, boolean withEnding) { + byte[] bytes = id.asBytes(); + int len = bytes.length; + E.checkArgument(len > 0, "Can't write empty id"); + + this.write(bytes); + if (type.isStringIndex()) { + if (Bytes.contains(bytes, STRING_ENDING_BYTE)) { + // Not allow STRING_ENDING_BYTE exist in string index id + E.checkArgument(false, + "The %s type index id can't contains " + + "byte '0x%s', but got: 0x%s", type, + Bytes.toHex(STRING_ENDING_BYTE), + Bytes.toHex(bytes)); + } + if (withEnding) { + this.writeStringWithEnding(""); + } + } + return this; + } + + private void writeNumber(long val) { + /* + * 8 kinds of number, 2 ~ 9 bytes number: + * 0b 0kkksxxx X... + * 0(1 bit) + kind(3 bits) + signed(1 bit) + number(n bits) + * + * 2 byte : 0b 0000 1xxx X(8 bits) [0, 2047] + * 0b 0000 0xxx X(8 bits) [-2048, -1] + * 3 bytes: 0b 0001 1xxx X X [0, 524287] + * 0b 0001 0xxx X X [-524288, -1] + * 4 bytes: 0b 0010 1xxx X X X [0, 134217727] + * 0b 0010 0xxx X X X [-134217728, -1] + * 5 bytes: 0b 0011 1xxx X X X X [0, 2^35 - 1] + * 0b 0011 0xxx X X X X [-2^35, -1] + * 6 bytes: 0b 0100 1xxx X X X X X [0, 2^43 - 1] + * 0b 0100 0xxx X X X X X [-2^43, -1] + * 7 bytes: 0b 0101 1xxx X X X X X X [0, 2^51 - 1] + * 0b 0101 0xxx X X X X X X [-2^51, -1] + * 8 bytes: 0b 0110 1xxx X X X X X X X [0, 2^59 - 1] + * 0b 0110 0xxx X X X X X X X [-2^59, -1] + * 9 bytes: 0b 0111 1000 X X X X X X X X [0, 2^64 - 1] + * 0b 0111 0000 X X X X X X X X [-2^64, -1] + * + * NOTE: 0b 0111 1111 is used by 128 bits UUID + * 0b 0111 1110 is used by EdgeId + */ + int positive = val >= 0 ? 0x08 : 0x00; + if (~0x7ffL <= val && val <= 0x7ffL) { + int high3bits = (int) (val >> 8) & 0x07; + this.writeUInt8(0x00 | positive | high3bits); + this.writeUInt8((byte) val); + } else if (~0x7ffffL <= val && val <= 0x7ffffL) { + int high3bits = (int) (val >> 16) & 0x07; + this.writeUInt8(0x10 | positive | high3bits); + this.writeShort((short) val); + } else if (~0x7ffffffL <= val && val <= 0x7ffffffL) { + int high3bits = (int) (val >> 24 & 0x07); + this.writeUInt8(0x20 | positive | high3bits); + this.write((byte) (val >> 16)); + this.writeShort((short) val); + } else if (~0x7ffffffffL <= val && val <= 0x7ffffffffL) { + int high3bits = (int) (val >> 32) & 0x07; + this.writeUInt8(0x30 | positive | high3bits); + this.writeInt((int) val); + } else if (~0x7ffffffffffL <= val && val <= 0x7ffffffffffL) { + int high3bits = (int) (val >> 40) & 0x07; + this.writeUInt8(0x40 | positive | high3bits); + this.write((byte) (val >> 32)); + this.writeInt((int) val); + } else if (~0x7ffffffffffffL <= val && val <= 0x7ffffffffffffL) { + int high3bits = (int) (val >> 48) & 0x07; + this.writeUInt8(0x50 | positive | high3bits); + this.writeShort((short) (val >> 32)); + this.writeInt((int) val); + } else if (~0x7ffffffffffffffL <= val && val <= 0x7ffffffffffffffL) { + int high3bits = (int) (val >> 56) & 0x07; + this.writeUInt8(0x60 | positive | high3bits); + this.write((byte) (val >> 48)); + this.writeShort((short) (val >> 32)); + this.writeInt((int) val); + } else { + // high3bits is always 0b000 for 9 bytes number + this.writeUInt8(0x70 | positive); + this.writeLong(val); + } + } + + private long readNumber(byte b) { + E.checkArgument((b & 0x80) == 0, + "Not a number type with prefix byte '0x%s'", + Bytes.toHex(b)); + // Parse the kind from byte 0kkksxxx + int kind = b >>> 4; + boolean positive = (b & 0x08) > 0; + long high3bits = b & 0x07; + long value = high3bits << ((kind + 1) * 8); + switch (kind) { + case 0: + value |= this.readUInt8(); + break; + case 1: + value |= this.readUInt16(); + break; + case 2: + value |= this.readUInt8() << 16 | this.readUInt16(); + break; + case 3: + value |= this.readUInt32(); + break; + case 4: + value |= (long) this.readUInt8() << 32 | this.readUInt32(); + break; + case 5: + value |= (long) this.readUInt16() << 32 | this.readUInt32(); + break; + case 6: + value |= (long) this.readUInt8() << 48 | + (long) this.readUInt16() << 32 | + this.readUInt32(); + break; + case 7: + assert high3bits == 0L; + value |= this.readLong(); + break; + default: + throw new AssertionError("Invalid length of number: " + kind); + } + if (!positive && kind < 7) { + // Restore the bits of the original negative number + long mask = Long.MIN_VALUE >> (52 - kind * 8); + value |= mask; + } + return value; + } + + private byte[] readBytesWithEnding() { + int start = this.buffer.position(); + boolean foundEnding = false; + while (this.remaining() > 0) { + byte current = this.read(); + if (current == STRING_ENDING_BYTE) { + foundEnding = true; + break; + } + } + E.checkArgument(foundEnding, "Not found ending '0x%s'", + Bytes.toHex(STRING_ENDING_BYTE)); + int end = this.buffer.position() - 1; + int len = end - start; + byte[] bytes = new byte[len]; + System.arraycopy(this.array(), start, bytes, 0, len); + return bytes; + } + + public void writeProperty(com.baidu.hugegraph.structure.constant.DataType dataType, Object value) { + switch (dataType) { + case BOOLEAN: + this.writeVInt(((Boolean) value) ? 1 : 0); + break; + case BYTE: + this.writeVInt((Byte) value); + break; + case INT: + this.writeVInt((Integer) value); + break; + case FLOAT: + this.writeFloat((Float) value); + break; + case LONG: + this.writeVLong((Long) value); + break; + case DATE: + this.writeVLong(((Date) value).getTime()); + break; + case DOUBLE: + this.writeDouble((Double) value); + break; + case TEXT: + this.writeString((String) value); + break; + + case UUID: + UUID uuid = (UUID) value; + // Generally writeVLong(uuid) can't save space + this.writeLong(uuid.getMostSignificantBits()); + this.writeLong(uuid.getLeastSignificantBits()); + break; + default: + //this.writeBytes(KryoUtil.toKryoWithType(value)); + break; + } + + } + +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/GraphSchema.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/GraphSchema.java new file mode 100644 index 000000000..af432f271 --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/GraphSchema.java @@ -0,0 +1,105 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct.util; + +import java.util.HashMap; +import java.util.Map; + +import com.baidu.hugegraph.driver.HugeClient; +import com.baidu.hugegraph.exception.ServerException; +import com.baidu.hugegraph.structure.schema.EdgeLabel; +import com.baidu.hugegraph.structure.schema.PropertyKey; +import com.baidu.hugegraph.structure.schema.VertexLabel; + +/** + * We could get all graph schema from server and cache/update it in client(subset of SchemaManager) + */ +public class GraphSchema { + + private final HugeClient client; + private final Map propertyKeys; + private final Map vertexLabels; + private final Map edgeLabels; + + public GraphSchema(HugeClient client) { + this.client = client; + this.propertyKeys = new HashMap<>(); + this.vertexLabels = new HashMap<>(); + this.edgeLabels = new HashMap<>(); + // init all schema + this.updateAll(); + } + + public void updateAll() { + this.propertyKeys.clear(); + this.vertexLabels.clear(); + this.edgeLabels.clear(); + + client.schema().getPropertyKeys().forEach(pk -> this.propertyKeys.put(pk.name(), pk)); + client.schema().getVertexLabels().forEach(vl -> this.vertexLabels.put(vl.name(), vl)); + client.schema().getEdgeLabels().forEach(el -> this.edgeLabels.put(el.name(), el)); + } + + public PropertyKey getPropertyKey(String name) { + PropertyKey propertyKey = this.propertyKeys.get(name); + if (propertyKey == null) { + try { + propertyKey = this.client.schema().getPropertyKey(name); + } catch (ServerException e) { + throw new HugeException("The property key '%s' doesn't exist", name); + } + } + return propertyKey; + } + + public VertexLabel getVertexLabel(String name) { + VertexLabel vertexLabel = this.vertexLabels.get(name); + if (vertexLabel == null) { + try { + vertexLabel = this.client.schema().getVertexLabel(name); + } catch (ServerException e) { + throw new HugeException("The vertex label '%s' doesn't exist", name); + } + } + return vertexLabel; + } + + public EdgeLabel getEdgeLabel(String name) { + EdgeLabel edgeLabel = this.edgeLabels.get(name); + if (edgeLabel == null) { + try { + edgeLabel = this.client.schema().getEdgeLabel(name); + } catch (ServerException e) { + throw new HugeException("The edge label '%s' doesn't exist", name); + } + } + return edgeLabel; + } + + public EdgeLabel getEdgeLabel(int id) { + for (EdgeLabel label : edgeLabels.values()) { + if (label.id() == id) { + return label; + } + } + + throw new HugeException("The edge label id '%s' doesn't exist", id); + } +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/HugeException.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/HugeException.java new file mode 100644 index 000000000..143e5ae27 --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/HugeException.java @@ -0,0 +1,56 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct.util; + +/** + * Used for fix Exception problems, simplify from server + */ +public class HugeException extends RuntimeException { + + private static final long serialVersionUID = -8711375282196157051L; + + public HugeException(String message) { + super(message); + } + + public HugeException(String message, Throwable cause) { + super(message, cause); + } + + public HugeException(String message, Object... args) { + super(String.format(message, args)); + } + + public HugeException(String message, Throwable cause, Object... args) { + super(String.format(message, args), cause); + } + + public Throwable rootCause() { + return rootCause(this); + } + + public static Throwable rootCause(Throwable e) { + Throwable cause = e; + while (cause.getCause() != null) { + cause = cause.getCause(); + } + return cause; + } +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/Id.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/Id.java new file mode 100644 index 000000000..0aee0d4f4 --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/Id.java @@ -0,0 +1,88 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct.util; + +import com.baidu.hugegraph.util.E; + +public interface Id extends Comparable { + + public static final int UUID_LENGTH = 16; + + public Object asObject(); + + public String asString(); + + public long asLong(); + + public byte[] asBytes(); + + public int length(); + + public IdType type(); + + public default boolean number() { + return this.type() == IdType.LONG; + } + + public default boolean uuid() { + return this.type() == IdType.UUID; + } + + public default boolean string() { + return this.type() == IdType.STRING; + } + + public default boolean edge() { + return this.type() == IdType.EDGE; + } + + public enum IdType { + + UNKNOWN, + LONG, + UUID, + STRING, + EDGE; + + public char prefix() { + if (this == UNKNOWN) { + return 'N'; + } + return this.name().charAt(0); + } + + public static IdType valueOfPrefix(String id) { + E.checkArgument(id != null && id.length() > 0, + "Invalid id '%s'", id); + switch (id.charAt(0)) { + case 'L': + return IdType.LONG; + case 'U': + return IdType.UUID; + case 'S': + return IdType.STRING; + case 'E': + return IdType.EDGE; + default: + return IdType.UNKNOWN; + } + } + } +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/IdGenerator.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/IdGenerator.java new file mode 100644 index 000000000..3e6c1e966 --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/IdGenerator.java @@ -0,0 +1,440 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct.util; + + +import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.LongEncoding; +import com.baidu.hugegraph.util.NumericUtil; +import com.baidu.hugegraph.serializer.direct.util.Id.IdType; + +import java.util.Objects; +import java.util.UUID; + +public abstract class IdGenerator { + + public static final Id ZERO = IdGenerator.of(0L); + + public final static Id of(String id) { + return new StringId(id); + } + + public final static Id of(UUID id) { + return new UuidId(id); + } + + public final static Id of(String id, boolean uuid) { + return uuid ? new UuidId(id) : new StringId(id); + } + + public final static Id of(long id) { + return new LongId(id); + } + + public static Id of(Object id) { + if (id instanceof Id) { + return (Id) id; + } else if (id instanceof String) { + return of((String) id); + } else if (id instanceof Number) { + return of(((Number) id).longValue()); + } else if (id instanceof UUID) { + return of((UUID) id); + } + return new ObjectId(id); + } + + public final static Id of(byte[] bytes, IdType type) { + switch (type) { + case LONG: + return new LongId(bytes); + case UUID: + return new UuidId(bytes); + case STRING: + return new StringId(bytes); + default: + throw new AssertionError("Invalid id type " + type); + } + } + + public final static Id ofStoredString(String id, IdType type) { + switch (type) { + case LONG: + return of(LongEncoding.decodeSignedB64(id)); + case UUID: + byte[] bytes = StringEncoding.decodeBase64(id); + return of(bytes, IdType.UUID); + case STRING: + return of(id); + default: + throw new AssertionError("Invalid id type " + type); + } + } + + public final static String asStoredString(Id id) { + switch (id.type()) { + case LONG: + return LongEncoding.encodeSignedB64(id.asLong()); + case UUID: + return StringEncoding.encodeBase64(id.asBytes()); + case STRING: + return id.asString(); + default: + throw new AssertionError("Invalid id type " + id.type()); + } + } + + public final static IdType idType(Id id) { + if (id instanceof LongId) { + return IdType.LONG; + } + if (id instanceof UuidId) { + return IdType.UUID; + } + if (id instanceof StringId) { + return IdType.STRING; + } + + return IdType.UNKNOWN; + } + + private final static int compareType(Id id1, Id id2) { + return idType(id1).ordinal() - idType(id2).ordinal(); + } + + /****************************** id defines ******************************/ + + public static final class StringId implements Id { + + private final String id; + + public StringId(String id) { + E.checkArgument(!id.isEmpty(), "The id can't be empty"); + this.id = id; + } + + public StringId(byte[] bytes) { + this.id = StringEncoding.decode(bytes); + } + + @Override + public IdType type() { + return IdType.STRING; + } + + @Override + public Object asObject() { + return this.id; + } + + @Override + public String asString() { + return this.id; + } + + @Override + public long asLong() { + return Long.parseLong(this.id); + } + + @Override + public byte[] asBytes() { + return StringEncoding.encode(this.id); + } + + @Override + public int length() { + return this.id.length(); + } + + @Override + public int compareTo(Id other) { + int cmp = compareType(this, other); + if (cmp != 0) { + return cmp; + } + return this.id.compareTo(other.asString()); + } + + @Override + public int hashCode() { + return this.id.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof StringId)) { + return false; + } + return this.id.equals(((StringId) other).id); + } + + @Override + public String toString() { + return this.id; + } + } + + public static final class LongId extends Number implements Id { + + private static final long serialVersionUID = -7732461469037400190L; + + private final long id; + + public LongId(long id) { + this.id = id; + } + + public LongId(byte[] bytes) { + this.id = NumericUtil.bytesToLong(bytes); + } + + @Override + public IdType type() { + return IdType.LONG; + } + + @Override + public Object asObject() { + return this.id; + } + + @Override + public String asString() { + // TODO: encode with base64 + return Long.toString(this.id); + } + + @Override + public long asLong() { + return this.id; + } + + @Override + public byte[] asBytes() { + return NumericUtil.longToBytes(this.id); + } + + @Override + public int length() { + return Long.BYTES; + } + + @Override + public int compareTo(Id other) { + int cmp = compareType(this, other); + if (cmp != 0) { + return cmp; + } + return Long.compare(this.id, other.asLong()); + } + + @Override + public int hashCode() { + return Long.hashCode(this.id); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Number)) { + return false; + } + return this.id == ((Number) other).longValue(); + } + + @Override + public String toString() { + return String.valueOf(this.id); + } + + @Override + public int intValue() { + return (int) this.id; + } + + @Override + public long longValue() { + return this.id; + } + + @Override + public float floatValue() { + return this.id; + } + + @Override + public double doubleValue() { + return this.id; + } + } + + public static final class UuidId implements Id { + + private final UUID uuid; + + public UuidId(String string) { + this(StringEncoding.uuid(string)); + } + + public UuidId(byte[] bytes) { + this(fromBytes(bytes)); + } + + public UuidId(UUID uuid) { + E.checkArgument(uuid != null, "The uuid can't be null"); + this.uuid = uuid; + } + + @Override + public IdType type() { + return IdType.UUID; + } + + @Override + public Object asObject() { + return this.uuid; + } + + @Override + public String asString() { + return this.uuid.toString(); + } + + @Override + public long asLong() { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] asBytes() { + BytesBuffer buffer = BytesBuffer.allocate(16); + buffer.writeLong(this.uuid.getMostSignificantBits()); + buffer.writeLong(this.uuid.getLeastSignificantBits()); + return buffer.bytes(); + } + + private static UUID fromBytes(byte[] bytes) { + E.checkArgument(bytes != null, "The UUID can't be null"); + BytesBuffer buffer = BytesBuffer.wrap(bytes); + long high = buffer.readLong(); + long low = buffer.readLong(); + return new UUID(high, low); + } + + @Override + public int length() { + return UUID_LENGTH; + } + + @Override + public int compareTo(Id other) { + E.checkNotNull(other, "compare id"); + int cmp = compareType(this, other); + if (cmp != 0) { + return cmp; + } + return this.uuid.compareTo(((UuidId) other).uuid); + } + + @Override + public int hashCode() { + return this.uuid.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof UuidId)) { + return false; + } + return this.uuid.equals(((UuidId) other).uuid); + } + + @Override + public String toString() { + return this.uuid.toString(); + } + } + + /** + * This class is just used by backend store for wrapper object as Id + */ + private static final class ObjectId implements Id { + + private final Object object; + + public ObjectId(Object object) { + E.checkNotNull(object, "object"); + this.object = object; + } + + @Override + public IdType type() { + return IdType.UNKNOWN; + } + + @Override + public Object asObject() { + return this.object; + } + + @Override + public String asString() { + throw new UnsupportedOperationException(); + } + + @Override + public long asLong() { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] asBytes() { + throw new UnsupportedOperationException(); + } + + @Override + public int length() { + throw new UnsupportedOperationException(); + } + + @Override + public int compareTo(Id o) { + throw new UnsupportedOperationException(); + } + + @Override + public int hashCode() { + return this.object.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ObjectId)) { + return false; + } + return Objects.equals(this.object, ((ObjectId) other).object); + } + + @Override + public String toString() { + return this.object.toString(); + } + } +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/LZ4Util.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/LZ4Util.java new file mode 100644 index 000000000..32684369c --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/LZ4Util.java @@ -0,0 +1,93 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct.util; + + +import net.jpountz.lz4.LZ4BlockOutputStream; +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; +import net.jpountz.lz4.LZ4BlockInputStream; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +public class LZ4Util { + + protected static final float DEFAULT_BUFFER_RATIO = 1.5f; + + public static BytesBuffer compress(byte[] bytes, int blockSize) { + return compress(bytes, blockSize, DEFAULT_BUFFER_RATIO); + } + + public static BytesBuffer compress(byte[] bytes, int blockSize, + float bufferRatio) { + float ratio = bufferRatio <= 0.0F ? DEFAULT_BUFFER_RATIO : bufferRatio; + LZ4Factory factory = LZ4Factory.fastestInstance(); + LZ4Compressor compressor = factory.fastCompressor(); + int initBufferSize = Math.round(bytes.length / ratio); + BytesBuffer buf = new BytesBuffer(initBufferSize); + LZ4BlockOutputStream lz4Output = new LZ4BlockOutputStream( + buf, blockSize, compressor); + try { + lz4Output.write(bytes); + lz4Output.close(); + } catch (IOException e) { + throw new HugeException("Failed to compress", e); + } + /* + * If need to perform reading outside the method, + * remember to call forReadWritten() + */ + return buf; + } + + public static BytesBuffer decompress(byte[] bytes, int blockSize) { + return decompress(bytes, blockSize, DEFAULT_BUFFER_RATIO); + } + + public static BytesBuffer decompress(byte[] bytes, int blockSize, + float bufferRatio) { + float ratio = bufferRatio <= 0.0F ? DEFAULT_BUFFER_RATIO : bufferRatio; + LZ4Factory factory = LZ4Factory.fastestInstance(); + LZ4FastDecompressor decompressor = factory.fastDecompressor(); + ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + int initBufferSize = Math.min(Math.round(bytes.length * ratio), + BytesBuffer.MAX_BUFFER_CAPACITY); + BytesBuffer buf = new BytesBuffer(initBufferSize); + LZ4BlockInputStream lzInput = new LZ4BlockInputStream(bais, + decompressor); + int count; + byte[] buffer = new byte[blockSize]; + try { + while ((count = lzInput.read(buffer)) != -1) { + buf.write(buffer, 0, count); + } + lzInput.close(); + } catch (IOException e) { + throw new HugeException("Failed to decompress", e); + } + /* + * If need to perform reading outside the method, + * remember to call forReadWritten() + */ + return buf; + } +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/SplicingIdGenerator.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/SplicingIdGenerator.java new file mode 100644 index 000000000..ab504fb15 --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/SplicingIdGenerator.java @@ -0,0 +1,132 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct.util; + +import java.util.Arrays; +import java.util.List; + +import com.baidu.hugegraph.util.IdUtil; + +/** + * This class is used for merge / parse id in primaryKey mode, simplify from server + **/ +public class SplicingIdGenerator { + + private static volatile SplicingIdGenerator instance; + + public static SplicingIdGenerator instance() { + if (instance == null) { + synchronized (SplicingIdGenerator.class) { + if (instance == null) { + instance = new SplicingIdGenerator(); + } + } + } + return instance; + } + + /* + * The following defines can't be java regex special characters: "\^$.|?*+()[{" + */ + private static final char ESCAPE = '`'; + private static final char IDS_SPLITOR = '>'; + private static final char ID_SPLITOR = ':'; + private static final char NAME_SPLITOR = '!'; + + public static final String ESCAPE_STR = String.valueOf(ESCAPE); + public static final String IDS_SPLITOR_STR = String.valueOf(IDS_SPLITOR); + public static final String ID_SPLITOR_STR = String.valueOf(ID_SPLITOR); + + /** + * Generate a string id of HugeVertex from Vertex name + */ +// public Id generate(HugeVertex vertex) { +// /* +// * Hash for row-key which will be evenly distributed. +// * We can also use LongEncoding.encode() to encode the int/long hash +// * if needed. +// * id = String.format("%s%s%s", HashUtil.hash(id), ID_SPLITOR, id); +// */ +// // TODO: use binary Id with binary fields instead of string id +// return splicing(vertex.schemaLabel().id().asString(), vertex.name()); +// } + + /** + * Concat multiple ids into one composite id with IDS_SPLITOR + * @param ids the string id values to be contacted + * @return contacted string value + */ + public static String concat(String... ids) { + // NOTE: must support string id when using this method + return IdUtil.escape(IDS_SPLITOR, ESCAPE, ids); + } + + /** + * Split a composite id into multiple ids with IDS_SPLITOR + * @param ids the string id value to be split + * @return split string values + */ + public static String[] split(String ids) { + return IdUtil.unescape(ids, IDS_SPLITOR_STR, ESCAPE_STR); + } + + /** + * Concat property values with NAME_SPLITOR + * @param values the property values to be concatted + * @return concatted string value + */ + public static String concatValues(List values) { + // Convert the object list to string array + int valuesSize = values.size(); + String[] parts = new String[valuesSize]; + for (int i = 0; i < valuesSize; i++) { + parts[i] = values.get(i).toString(); + } + return IdUtil.escape(NAME_SPLITOR, ESCAPE, parts); + } + + /** + * Concat property values with NAME_SPLITOR + * @param values the property values to be contacted + * @return contacted string value + */ + public static String concatValues(Object... values) { + return concatValues(Arrays.asList(values)); + } + + /** + * Concat multiple parts into a single id with ID_SPLITOR + * @param parts the string id values to be spliced + * @return spliced id object + */ + public static Id splicing(String... parts) { + String escaped = IdUtil.escape(ID_SPLITOR, ESCAPE, parts); + return IdGenerator.of(escaped); + } + + /** + * Parse a single id into multiple parts with ID_SPLITOR + * @param id the id object to be parsed + * @return parsed string id parts + */ + public static String[] parse(Id id) { + return IdUtil.unescape(id.asString(), ID_SPLITOR_STR, ESCAPE_STR); + } +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/StringEncoding.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/StringEncoding.java new file mode 100644 index 000000000..5668f14af --- /dev/null +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/serializer/direct/util/StringEncoding.java @@ -0,0 +1,153 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.serializer.direct.util; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.UUID; + +import com.baidu.hugegraph.util.Bytes; +import com.baidu.hugegraph.util.E; +import com.google.common.base.CharMatcher; + +/** + * Used for encode / decode string values, simplify from server + */ +public final class StringEncoding { + + private static final MessageDigest DIGEST; + private static final byte[] BYTES_EMPTY = new byte[0]; + private static final String STRING_EMPTY = ""; + private static final int BLOCK_SIZE = 4096; + + static { + final String ALG = "SHA-256"; + try { + DIGEST = MessageDigest.getInstance(ALG); + } catch (NoSuchAlgorithmException e) { + throw new HugeException("Failed to load algorithm %s", e, ALG); + } + } + + private static final Base64.Encoder BASE64_ENCODER = Base64.getEncoder(); + private static final Base64.Decoder BASE64_DECODER = Base64.getDecoder(); + + // Similar to {@link StringSerializer} + public static int writeAsciiString(byte[] array, int offset, String value) { + E.checkArgument(CharMatcher.ascii().matchesAllOf(value), + "'%s' must be ASCII string", value); + int len = value.length(); + if (len == 0) { + array[offset++] = (byte) 0x80; + return offset; + } + + int i = 0; + do { + int c = value.charAt(i); + assert c <= 127; + byte b = (byte) c; + if (++i == len) { + b |= 0x80; // End marker + } + array[offset++] = b; + } while (i < len); + + return offset; + } + + public static String readAsciiString(byte[] array, int offset) { + StringBuilder sb = new StringBuilder(); + int c = 0; + do { + c = 0xFF & array[offset++]; + if (c != 0x80) { + sb.append((char) (c & 0x7F)); + } + } while ((c & 0x80) <= 0); + return sb.toString(); + } + + public static int getAsciiByteLength(String value) { + E.checkArgument(CharMatcher.ascii().matchesAllOf(value), + "'%s' must be ASCII string", value); + return value.isEmpty() ? 1 : value.length(); + } + + public static byte[] encode(String value) { + return value.getBytes(UTF_8); + } + + public static String decode(byte[] bytes) { + if (bytes.length == 0) { + return STRING_EMPTY; + } + return new String(bytes, UTF_8); + } + + public static String decode(byte[] bytes, int offset, int length) { + if (length == 0) { + return STRING_EMPTY; + } + return new String(bytes, offset, length, UTF_8); + } + + public static String encodeBase64(byte[] bytes) { + return BASE64_ENCODER.encodeToString(bytes); + } + + public static byte[] decodeBase64(String value) { + if (value.isEmpty()) { + return BYTES_EMPTY; + } + return BASE64_DECODER.decode(value); + } + + public static String sha256(String string) { + byte[] stringBytes = encode(string); + DIGEST.reset(); + return StringEncoding.encodeBase64(DIGEST.digest(stringBytes)); + } + + public static String format(byte[] bytes) { + return String.format("%s[0x%s]", decode(bytes), Bytes.toHex(bytes)); + } + + public static UUID uuid(String value) { + E.checkArgument(value != null, "The UUID can't be null"); + try { + if (value.contains("-") && value.length() == 36) { + return UUID.fromString(value); + } + // UUID represented by hex string + E.checkArgument(value.length() == 32, + "Invalid UUID string: %s", value); + String high = value.substring(0, 16); + String low = value.substring(16); + return new UUID(Long.parseUnsignedLong(high, 16), + Long.parseUnsignedLong(low, 16)); + } catch (NumberFormatException ignored) { + throw new IllegalArgumentException("Invalid UUID string: " + value); + } + } +} diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/GraphElement.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/GraphElement.java index 042882c2b..c4363b887 100644 --- a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/GraphElement.java +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/GraphElement.java @@ -93,4 +93,8 @@ public Map properties() { protected abstract GraphElement setProperty(String key, Object value); public abstract GraphElement removeProperty(String key); + + public int sizeOfProperties() { + return properties.size(); + } } diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/Direction.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/Direction.java index 7d93c211a..676bd8785 100644 --- a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/Direction.java +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/Direction.java @@ -27,8 +27,8 @@ public enum Direction { BOTH(3, "both"); - private byte code = 0; - private String name = null; + private byte code; + private String name; Direction(int code, String name) { assert code < 256; diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/HugeType.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/HugeType.java index cdfb1818d..fe9de078b 100644 --- a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/HugeType.java +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/constant/HugeType.java @@ -65,8 +65,8 @@ public enum HugeType { // Metrics METRICS(240, "metrics"); - private int code; - private String name = null; + private final int code; + private final String name; HugeType(int code, String name) { assert code < 256; diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Edge.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Edge.java index 6d6c0dfa0..baddda378 100644 --- a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Edge.java +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Edge.java @@ -56,6 +56,7 @@ public Edge(@JsonProperty("label") String label) { this.name = null; } + @Override public String id() { return this.id; } diff --git a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Vertex.java b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Vertex.java index 0322410f6..f0ce717f0 100644 --- a/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Vertex.java +++ b/hugegraph-client/src/main/java/com/baidu/hugegraph/structure/graph/Vertex.java @@ -38,6 +38,7 @@ public Vertex(@JsonProperty("label") String label) { this.type = "vertex"; } + @Override public Object id() { return this.id; } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java index 899a6ea02..864e4f746 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java @@ -45,7 +45,7 @@ public class EdgeBuilder extends ElementBuilder { private final VertexLabel sourceLabel; private final VertexLabel targetLabel; private final Collection nonNullKeys; - // Used to optimize access performace + // Used to optimize access performance private VertexIdsIndex vertexIdsIndex; private String[] lastNames; @@ -57,7 +57,7 @@ public EdgeBuilder(LoadContext context, InputStruct struct, this.sourceLabel = this.getVertexLabel(this.edgeLabel.sourceLabel()); this.targetLabel = this.getVertexLabel(this.edgeLabel.targetLabel()); this.nonNullKeys = this.nonNullableKeys(this.edgeLabel); - // Ensure that the source/target id fileds are matched with id strategy + // Ensure that the source/target id fields are matched with id strategy this.checkIdFields(this.sourceLabel, this.mapping.sourceFields()); this.checkIdFields(this.targetLabel, this.mapping.targetFields()); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java index edc503fde..36b5dc236 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java @@ -19,7 +19,6 @@ package com.baidu.hugegraph.loader.flink; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -129,7 +128,7 @@ private synchronized void flushAll() { } @Override - public synchronized void writeRecord(T row) throws IOException { + public synchronized void writeRecord(T row) { for (Map.Entry> builder : this.builders.entrySet()) { ElementMapping elementMapping = builder.getKey().mapping(); @@ -164,7 +163,7 @@ private Tuple2> buildGraphData(ElementBuilder element return Tuple2.of(op, elementBuilder.build(fields, values)); } - private void flush(ElementBuilder elementBuilder, List rows) { + private void flush(ElementBuilder elementBuilder, List rows) { GraphManager g = this.loadContext.client().graph(); ElementMapping elementMapping = elementBuilder.mapping(); for (String row : rows) { diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/DataTypeUtil.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/DataTypeUtil.java index 87ff516bb..6f4e4dbac 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/DataTypeUtil.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/DataTypeUtil.java @@ -94,7 +94,7 @@ public static long parseNumber(String key, Object rawValue) { if (rawValue instanceof Number) { return ((Number) rawValue).longValue(); } else if (rawValue instanceof String) { - // trim() is a little time consuming + // trim() is a little time-consuming return parseLong(((String) rawValue).trim()); } throw new IllegalArgumentException(String.format(