From ee9b42a94f697857d80f0d62cad3f3f071943826 Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Wed, 9 Dec 2015 15:21:28 +0200 Subject: [PATCH 01/52] TTB Initial commit --- pom.xml | 5 + .../com/basho/riak/client/core/RiakNode.java | 18 +- .../core/netty/RiakTTBChannelInitializer.java | 26 +++ .../riak/client/core/netty/RiakTTBCodec.java | 164 ++++++++++++++++++ .../riak/client/core/netty/ITestTTBNode.java | 72 ++++++++ 5 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java create mode 100644 src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java create mode 100644 src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java diff --git a/pom.xml b/pom.xml index d2382f368..8fec6a338 100755 --- a/pom.xml +++ b/pom.xml @@ -361,5 +361,10 @@ riak-pb 2.1.1.0 + + org.erlang.otp + jinterface + 1.5.6 + diff --git a/src/main/java/com/basho/riak/client/core/RiakNode.java b/src/main/java/com/basho/riak/client/core/RiakNode.java index 51928f780..b413b8fc3 100644 --- a/src/main/java/com/basho/riak/client/core/RiakNode.java +++ b/src/main/java/com/basho/riak/client/core/RiakNode.java @@ -88,6 +88,7 @@ public enum State private volatile boolean blockOnMaxConnections; private HealthCheckFactory healthCheckFactory; + private boolean useTTB; private final ChannelFutureListener writeListener = new ChannelFutureListener() @@ -186,6 +187,7 @@ private RiakNode(Builder builder) throws UnknownHostException this.keyStore = builder.keyStore; this.keyPassword = builder.keyPassword; this.healthCheckFactory = builder.healthCheckFactory; + this.useTTB = builder.useTTB; if (builder.bootstrap != null) { @@ -252,7 +254,14 @@ public synchronized RiakNode start() throws UnknownHostException throw new UnknownHostException("RiakNode:start - Failed resolving host " + remoteAddress); } - bootstrap.handler(new RiakChannelInitializer(this)).remoteAddress(socketAddress); + final RiakChannelInitializer initialize; + if (useTTB) { + initialize = new RiakTTBChannelInitializer(this); + } else { + initialize = new RiakChannelInitializer(this); + } + + bootstrap.handler(initialize).remoteAddress(socketAddress); if (connectionTimeout > 0) { @@ -1243,6 +1252,7 @@ public static class Builder private int idleTimeout = DEFAULT_IDLE_TIMEOUT; private int connectionTimeout = DEFAULT_CONNECTION_TIMEOUT; private HealthCheckFactory healthCheckFactory = DEFAULT_HEALTHCHECK_FACTORY; + private boolean useTTB = false; private Bootstrap bootstrap; private ScheduledExecutorService executor; private boolean blockOnMaxConnections; @@ -1489,6 +1499,12 @@ public Builder withHealthCheck(HealthCheckFactory factory) return this; } + public Builder useTTB() + { + this.useTTB = true; + return this; + } + /** * Builds a RiakNode. * If a Netty {@code Bootstrap} and/or a {@code ScheduledExecutorService} has not been provided they diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java new file mode 100644 index 000000000..ce06fa804 --- /dev/null +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java @@ -0,0 +1,26 @@ +package com.basho.riak.client.core.netty; + +import com.basho.riak.client.core.RiakResponseListener; +import com.basho.riak.client.core.util.Constants; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; + +/** + * Created by srg on 12/9/15. + */ +public class RiakTTBChannelInitializer extends RiakChannelInitializer { + public RiakTTBChannelInitializer(RiakResponseListener listener) { + super(listener); + } + + @Override + public void initChannel(SocketChannel ch) throws Exception + { + ChannelPipeline p = ch.pipeline(); + p.addLast(new LoggingHandler(LogLevel.WARN)); + super.initChannel(ch); + p.replace(Constants.MESSAGE_CODEC, Constants.MESSAGE_CODEC, new RiakTTBCodec()); + } +} diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java new file mode 100644 index 000000000..c7f977a5d --- /dev/null +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java @@ -0,0 +1,164 @@ +package com.basho.riak.client.core.netty; + +import com.basho.riak.client.core.RiakMessage; +import com.basho.riak.protobuf.RiakMessageCodes; +import com.basho.riak.protobuf.RiakTsPB; +import com.ericsson.otp.erlang.*; +import com.google.protobuf.InvalidProtocolBufferException; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageCodec; + +import java.util.List; +import java.util.concurrent.Exchanger; + +/** + * Created by srg on 12/9/15. + */ +public class RiakTTBCodec extends ByteToMessageCodec { + private static final OtpErlangAtom undefined = new OtpErlangAtom("undefined"); + private static final OtpErlangAtom tscell = new OtpErlangAtom("tscell"); + private static final OtpErlangAtom tsrow = new OtpErlangAtom("tsrow"); + private static final OtpErlangAtom tsputreq = new OtpErlangAtom("tsputreq"); + private static final OtpErlangString geoCheckin = //new OtpErlangBinary( + new OtpErlangString("GeoCheckin"); //); + + private final OtpErlangBinary family1 = new OtpErlangBinary(new OtpErlangString("family1")); + + + @Override + protected void encode(ChannelHandlerContext ctx, RiakMessage msg, ByteBuf out) throws Exception { + switch (msg.getCode()) { + case RiakMessageCodes.MSG_TsPutReq: + encodeTsPut(msg, out); + break; + default: + throw new IllegalStateException("Unsupported message with code " + msg.getCode()); + } + } + + private static OtpErlangTuple pbCellToTtb(RiakTsPB.TsCell c) { + final OtpErlangObject o[]; + + if (c.hasVarcharValue()) { + o = new OtpErlangObject[] + {tscell, + new OtpErlangBinary(new OtpErlangString(c.getVarcharValue().toStringUtf8())), + undefined, undefined, undefined, undefined}; + } else if (c.hasSint64Value()) { + o = new OtpErlangObject[] + {tscell, + undefined, + new OtpErlangLong(c.getSint64Value()), undefined, undefined, undefined}; + } else if (c.hasTimestampValue()) { + o = new OtpErlangObject[] + {tscell, + undefined, undefined, + new OtpErlangLong(c.getTimestampValue()), + undefined, undefined}; + } else if (c.hasBooleanValue()) { + o = new OtpErlangObject[] + {tscell, + undefined, undefined, undefined, + new OtpErlangBoolean(c.getBooleanValue()), + undefined}; + } else if (c.hasDoubleValue()) { + o = new OtpErlangObject[] + {tscell, + undefined, undefined, undefined, undefined, + new OtpErlangDouble(c.getDoubleValue())}; + } else { + throw new IllegalStateException("Unsupported type"); + } + + return new OtpErlangTuple(o); + } + + private static OtpErlangTuple pbRowToTtb(RiakTsPB.TsRow r) { + final OtpErlangObject cells[] = new OtpErlangObject[r.getCellsCount()]; + for (int i = 0; i < cells.length; ++i) { + final RiakTsPB.TsCell c = r.getCells(i); + cells[i] = pbCellToTtb(c); + } + + final OtpErlangList l = new OtpErlangList(cells); + return new OtpErlangTuple(new OtpErlangObject[] {tsrow, l}); + } + + private static OtpErlangTuple encodeTsPut(RiakMessage msg, ByteBuf out) throws InvalidProtocolBufferException { + final RiakTsPB.TsPutReq req = RiakTsPB.TsPutReq.parseFrom(msg.getData()); + assert req != null; + + + final OtpErlangObject rows[] = new OtpErlangObject[req.getRowsCount()]; + for (int i = 0; i < rows.length; ++i) { + final RiakTsPB.TsRow r = req.getRows(i); + rows[i] = pbRowToTtb(r); + } + + OtpErlangObject[] elems = new OtpErlangObject[] + {tsputreq, new OtpErlangString(req.getTable().toStringUtf8()), + new OtpErlangList(), + new OtpErlangList(rows)}; + + final OtpErlangTuple request = new OtpErlangTuple(elems); + + + OtpOutputStream os = new OtpOutputStream(request); + byte data[] = os.toByteArray(); + + int length = data.length; + out.writeInt(length); + out.writeBytes(data); + + return request; + } + + @Override + protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { + // Make sure we have 4 bytes + if (in.readableBytes() >= 4) { + in.markReaderIndex(); + + int i = in.readableBytes(); + + int length = in.readInt(); + // See if we have the full frame. + if (in.readableBytes() < length) { + in.capacity(length); + in.resetReaderIndex(); + return; + } + + + final byte[] array = new byte[length]; + in.readBytes(array); + + final OtpErlangObject o; + try { + OtpInputStream is = new OtpInputStream(array); + o = is.read_any(); + }catch (Exception ex){ + in.resetReaderIndex(); + return; + } + + + + + +// // See if we have the full frame. +// if (in.readableBytes() < length) { +// in.resetReaderIndex(); +// return; +// } else { +// byte code = in.readByte(); +// byte[] array = new byte[length - 1]; +// in.readBytes(array); +// out.add(new RiakMessage(code, array)); +// } + + assert o != null; + } + } +} diff --git a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java new file mode 100644 index 000000000..fc586b2b6 --- /dev/null +++ b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java @@ -0,0 +1,72 @@ +package com.basho.riak.client.core.netty; + +import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.RiakNode; +import com.basho.riak.client.core.operations.ts.StoreOperation; +import com.basho.riak.client.core.query.timeseries.Cell; +import com.basho.riak.client.core.query.timeseries.Row; +import com.basho.riak.client.core.util.BinaryValue; +import junit.framework.Assert; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; + +/** + * Created by srg on 12/9/15. + */ +public class ITestTTBNode { + + protected final static String tableName = "GeoCheckin"; + + protected final static long now = 1443806900000l; // "now" + protected final static long fiveMinsInMS = 5l * 60l * 1000l; + protected final static long fiveMinsAgo = now - fiveMinsInMS; + protected final static long tenMinsAgo = fiveMinsAgo - fiveMinsInMS; + protected final static long fifteenMinsAgo = tenMinsAgo - fiveMinsInMS; + protected final static long fifteenMinsInFuture = now + (fiveMinsInMS * 3l); + + private RiakNode riakNode; + + + @Before + public void setup() throws UnknownHostException { + riakNode = new RiakNode.Builder() + .useTTB() + .withRemotePort(10017) + .build(); + + riakNode.start(); + } + + + @After + public void shutdonw(){ + if (riakNode != null) { + riakNode.shutdown(); + } + } + + @Test + public void test() throws InterruptedException, ExecutionException { + final List rows = Arrays.asList( + // "Normal" Data + new Row(new Cell("hash1"), new Cell("user1"), Cell.newTimestamp(fifteenMinsAgo), new Cell("cloudy"), new Cell(79.0), new Cell(1), new Cell(true)) +// new Row(new Cell("hash1")) + ); + + final StoreOperation store = new StoreOperation.Builder(BinaryValue.create(tableName + 112)).withRows(rows).build(); + + if (!riakNode.execute(store)) { + Assert.fail("Cant store data"); + } + + store.get(); + assert store != null; + + } +} From f43e2502dbe40c445d0a1b10d069f214cfa19ca5 Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Wed, 9 Dec 2015 17:54:11 +0200 Subject: [PATCH 02/52] Added proper handling of: TsPutReq (encoding), tsputresp(encoding), rpberrorresp(encoding). --- .../core/netty/RiakTTBChannelInitializer.java | 2 +- .../riak/client/core/netty/RiakTTBCodec.java | 89 ++++++++++++------- .../riak/client/core/netty/ITestTTBNode.java | 39 +++++--- 3 files changed, 85 insertions(+), 45 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java index ce06fa804..c243f70b1 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java @@ -8,7 +8,7 @@ import io.netty.handler.logging.LoggingHandler; /** - * Created by srg on 12/9/15. + * @author Sergey Galkin */ public class RiakTTBChannelInitializer extends RiakChannelInitializer { public RiakTTBChannelInitializer(RiakResponseListener listener) { diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java index c7f977a5d..42333eaf9 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java @@ -2,18 +2,20 @@ import com.basho.riak.client.core.RiakMessage; import com.basho.riak.protobuf.RiakMessageCodes; +import com.basho.riak.protobuf.RiakPB; import com.basho.riak.protobuf.RiakTsPB; import com.ericsson.otp.erlang.*; +import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageCodec; +import java.io.UnsupportedEncodingException; import java.util.List; -import java.util.concurrent.Exchanger; /** - * Created by srg on 12/9/15. + * @author Sergey Galkin */ public class RiakTTBCodec extends ByteToMessageCodec { private static final OtpErlangAtom undefined = new OtpErlangAtom("undefined"); @@ -28,13 +30,33 @@ public class RiakTTBCodec extends ByteToMessageCodec { @Override protected void encode(ChannelHandlerContext ctx, RiakMessage msg, ByteBuf out) throws Exception { + final OtpErlangTuple t; + switch (msg.getCode()) { case RiakMessageCodes.MSG_TsPutReq: - encodeTsPut(msg, out); + t = encodeTsPut(msg); break; default: - throw new IllegalStateException("Unsupported message with code " + msg.getCode()); + throw new IllegalStateException("Can't encode TTB request, unsupported message with code " + msg.getCode()); } + + final OtpOutputStream os = new OtpOutputStream(t); + byte data[] = os.toByteArray(); + + int length = data.length+1; + out.writeInt(length); + + /** + * DO A TOP SECRET HACK + * + * It seems that the message is missing the 131 tag required by term_to_binary. + * As is evident from the Java output, this tag is not being added by jinterface encode. + * If I simply add 131 to the beginning of the binary it decodes correctly. + * + * http://stackoverflow.com/questions/15189447/jinterface-to-create-external-erlang-term + */ + out.writeByte(131); + out.writeBytes(data); } private static OtpErlangTuple pbCellToTtb(RiakTsPB.TsCell c) { @@ -85,7 +107,11 @@ private static OtpErlangTuple pbRowToTtb(RiakTsPB.TsRow r) { return new OtpErlangTuple(new OtpErlangObject[] {tsrow, l}); } - private static OtpErlangTuple encodeTsPut(RiakMessage msg, ByteBuf out) throws InvalidProtocolBufferException { + private static OtpErlangBinary pbStrToTtb(ByteString bs) { + return new OtpErlangBinary(bs.toByteArray()); + } + + private static OtpErlangTuple encodeTsPut(RiakMessage msg) throws InvalidProtocolBufferException, UnsupportedEncodingException { final RiakTsPB.TsPutReq req = RiakTsPB.TsPutReq.parseFrom(msg.getData()); assert req != null; @@ -97,21 +123,11 @@ private static OtpErlangTuple encodeTsPut(RiakMessage msg, ByteBuf out) throws I } OtpErlangObject[] elems = new OtpErlangObject[] - {tsputreq, new OtpErlangString(req.getTable().toStringUtf8()), + {tsputreq, pbStrToTtb(req.getTable()), new OtpErlangList(), new OtpErlangList(rows)}; - final OtpErlangTuple request = new OtpErlangTuple(elems); - - - OtpOutputStream os = new OtpOutputStream(request); - byte data[] = os.toByteArray(); - - int length = data.length; - out.writeInt(length); - out.writeBytes(data); - - return request; + return new OtpErlangTuple(elems); } @Override @@ -120,8 +136,6 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t if (in.readableBytes() >= 4) { in.markReaderIndex(); - int i = in.readableBytes(); - int length = in.readInt(); // See if we have the full frame. if (in.readableBytes() < length) { @@ -143,22 +157,29 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t return; } - - - - -// // See if we have the full frame. -// if (in.readableBytes() < length) { -// in.resetReaderIndex(); -// return; -// } else { -// byte code = in.readByte(); -// byte[] array = new byte[length - 1]; -// in.readBytes(array); -// out.add(new RiakMessage(code, array)); -// } - assert o != null; + if (o instanceof OtpErlangTuple && ((OtpErlangTuple)o).elementAt(0) instanceof OtpErlangAtom) { + final OtpErlangTuple t = ((OtpErlangTuple)o); + final OtpErlangAtom resp = (OtpErlangAtom) ((OtpErlangTuple)o).elementAt(0); + final String v = resp.atomValue(); + + if ("tsputresp".equals(v)) { + final RiakTsPB.TsPutResp r = RiakTsPB.TsPutResp.newBuilder().build(); + out.add(new RiakMessage(RiakMessageCodes.MSG_TsPutResp, r.toByteArray())); + } else if ("rpberrorresp".equals(v)) { + final OtpErlangString errMsg = (OtpErlangString) t.elementAt(1); + final OtpErlangLong errCode = (OtpErlangLong) t.elementAt(2); + final RiakPB.RpbErrorResp r = RiakPB.RpbErrorResp.newBuilder() + .setErrcode(errCode.intValue()) + .setErrmsg(ByteString.copyFromUtf8(errMsg.stringValue())) + .build(); + out.add(new RiakMessage(RiakMessageCodes.MSG_ErrorResp, r.toByteArray())); + } else { + throw new IllegalStateException("Can't decode TTB response, unsupported atom '"+ v + "'"); + } + } else { + throw new IllegalStateException("Can't decode TTB response"); + } } } } diff --git a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java index fc586b2b6..601032915 100644 --- a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java +++ b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java @@ -1,6 +1,5 @@ package com.basho.riak.client.core.netty; -import com.basho.riak.client.core.RiakFuture; import com.basho.riak.client.core.RiakNode; import com.basho.riak.client.core.operations.ts.StoreOperation; import com.basho.riak.client.core.query.timeseries.Cell; @@ -14,10 +13,11 @@ import java.net.UnknownHostException; import java.util.Arrays; import java.util.List; +import java.util.UUID; import java.util.concurrent.ExecutionException; /** - * Created by srg on 12/9/15. + * @author Sergey Galkin */ public class ITestTTBNode { @@ -31,6 +31,10 @@ public class ITestTTBNode { protected final static long fifteenMinsInFuture = now + (fiveMinsInMS * 3l); private RiakNode riakNode; + private final List rows = Arrays.asList( + new Row(new Cell("hash1"), new Cell("user1"), Cell.newTimestamp(fifteenMinsAgo), + new Cell("cloudy"), new Cell(79.0), new Cell(1), new Cell(true)) + ); @Before @@ -52,14 +56,8 @@ public void shutdonw(){ } @Test - public void test() throws InterruptedException, ExecutionException { - final List rows = Arrays.asList( - // "Normal" Data - new Row(new Cell("hash1"), new Cell("user1"), Cell.newTimestamp(fifteenMinsAgo), new Cell("cloudy"), new Cell(79.0), new Cell(1), new Cell(true)) -// new Row(new Cell("hash1")) - ); - - final StoreOperation store = new StoreOperation.Builder(BinaryValue.create(tableName + 112)).withRows(rows).build(); + public void storeData() throws InterruptedException, ExecutionException { + final StoreOperation store = new StoreOperation.Builder(BinaryValue.create(tableName)).withRows(rows).build(); if (!riakNode.execute(store)) { Assert.fail("Cant store data"); @@ -67,6 +65,27 @@ public void test() throws InterruptedException, ExecutionException { store.get(); assert store != null; + } + + @Test + public void storeDatatoNoneExistentTable() throws InterruptedException, ExecutionException { + final StoreOperation store = new StoreOperation.Builder(BinaryValue.create(UUID.randomUUID().toString())) + .withRows(rows).build(); + if (!riakNode.execute(store)) { + Assert.fail("Cant store data"); + } + + try { + store.get(); + } catch (ExecutionException e) { + if (e.getCause() instanceof RiakResponseException + && e.getCause().getMessage().startsWith("Bucket type") + && e.getCause().getMessage().endsWith("is missing.")) { + // It's OK + return; + } + throw e; + } } } From c28724e9e4682865bcbe0d09f0fb8b7d57463bb3 Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Wed, 9 Dec 2015 17:56:53 +0200 Subject: [PATCH 03/52] RighTTBCodec is forced to be used by default --- src/main/java/com/basho/riak/client/core/RiakNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/basho/riak/client/core/RiakNode.java b/src/main/java/com/basho/riak/client/core/RiakNode.java index b413b8fc3..1108e4d73 100644 --- a/src/main/java/com/basho/riak/client/core/RiakNode.java +++ b/src/main/java/com/basho/riak/client/core/RiakNode.java @@ -1252,7 +1252,7 @@ public static class Builder private int idleTimeout = DEFAULT_IDLE_TIMEOUT; private int connectionTimeout = DEFAULT_CONNECTION_TIMEOUT; private HealthCheckFactory healthCheckFactory = DEFAULT_HEALTHCHECK_FACTORY; - private boolean useTTB = false; + private boolean useTTB = true; private Bootstrap bootstrap; private ScheduledExecutorService executor; private boolean blockOnMaxConnections; From 01ef4301c567e2b5e538a633666e421a635884f3 Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Thu, 10 Dec 2015 06:19:29 +0200 Subject: [PATCH 04/52] Netty Logger has been removed --- .../basho/riak/client/core/netty/RiakTTBChannelInitializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java index c243f70b1..5927549a9 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java @@ -19,7 +19,7 @@ public RiakTTBChannelInitializer(RiakResponseListener listener) { public void initChannel(SocketChannel ch) throws Exception { ChannelPipeline p = ch.pipeline(); - p.addLast(new LoggingHandler(LogLevel.WARN)); + //p.addLast(new LoggingHandler(LogLevel.WARN)); super.initChannel(ch); p.replace(Constants.MESSAGE_CODEC, Constants.MESSAGE_CODEC, new RiakTTBCodec()); } From f43f0fa600c98a6565d776d80ab912ee0252ed05 Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Thu, 10 Dec 2015 22:45:17 +0200 Subject: [PATCH 05/52] TsGetReq Added TsGetReq and TsGetResp --- .../riak/client/core/netty/RiakTTBCodec.java | 143 ++++++++++++++++-- .../riak/client/core/netty/ITestTTBNode.java | 83 +++++++--- 2 files changed, 195 insertions(+), 31 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java index 42333eaf9..2194893ac 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java @@ -12,6 +12,7 @@ import io.netty.handler.codec.ByteToMessageCodec; import java.io.UnsupportedEncodingException; +import java.util.Arrays; import java.util.List; /** @@ -22,11 +23,14 @@ public class RiakTTBCodec extends ByteToMessageCodec { private static final OtpErlangAtom tscell = new OtpErlangAtom("tscell"); private static final OtpErlangAtom tsrow = new OtpErlangAtom("tsrow"); private static final OtpErlangAtom tsputreq = new OtpErlangAtom("tsputreq"); - private static final OtpErlangString geoCheckin = //new OtpErlangBinary( - new OtpErlangString("GeoCheckin"); //); - - private final OtpErlangBinary family1 = new OtpErlangBinary(new OtpErlangString("family1")); + private static final OtpErlangAtom tsqueryreq = new OtpErlangAtom("tsqueryreq"); + private static final OtpErlangAtom tsinterpolation = new OtpErlangAtom("tsinterpolation"); + private static final OtpErlangAtom tsgetreq = new OtpErlangAtom("tsgetreq"); + private static final OtpErlangAtom _false = new OtpErlangAtom("false"); + private static final OtpErlangList EMPTY_ERLANG_LIST = new OtpErlangList(); + private static final OtpErlangTuple EMPTY_ERLANG_TSCELL = new OtpErlangTuple(new OtpErlangObject[]{ + tscell, undefined, undefined, undefined, undefined, undefined}); @Override protected void encode(ChannelHandlerContext ctx, RiakMessage msg, ByteBuf out) throws Exception { @@ -36,6 +40,12 @@ protected void encode(ChannelHandlerContext ctx, RiakMessage msg, ByteBuf out) t case RiakMessageCodes.MSG_TsPutReq: t = encodeTsPut(msg); break; + case RiakMessageCodes.MSG_TsGetReq: + t = encodeTsGet(msg); + break; +// case RiakMessageCodes.MSG_TsQueryReq: +// t = encodeTsQuery(msg); +// break; default: throw new IllegalStateException("Can't encode TTB request, unsupported message with code " + msg.getCode()); } @@ -65,7 +75,7 @@ private static OtpErlangTuple pbCellToTtb(RiakTsPB.TsCell c) { if (c.hasVarcharValue()) { o = new OtpErlangObject[] {tscell, - new OtpErlangBinary(new OtpErlangString(c.getVarcharValue().toStringUtf8())), + pbStrToTtb(c.getVarcharValue()), undefined, undefined, undefined, undefined}; } else if (c.hasSint64Value()) { o = new OtpErlangObject[] @@ -90,27 +100,34 @@ private static OtpErlangTuple pbCellToTtb(RiakTsPB.TsCell c) { undefined, undefined, undefined, undefined, new OtpErlangDouble(c.getDoubleValue())}; } else { - throw new IllegalStateException("Unsupported type"); + return EMPTY_ERLANG_TSCELL; } return new OtpErlangTuple(o); } - private static OtpErlangTuple pbRowToTtb(RiakTsPB.TsRow r) { - final OtpErlangObject cells[] = new OtpErlangObject[r.getCellsCount()]; + private static OtpErlangList pbCellsToTtb(List pbCells) { + final OtpErlangObject cells[] = new OtpErlangObject[pbCells.size()]; for (int i = 0; i < cells.length; ++i) { - final RiakTsPB.TsCell c = r.getCells(i); + final RiakTsPB.TsCell c = pbCells.get(i); cells[i] = pbCellToTtb(c); } - final OtpErlangList l = new OtpErlangList(cells); - return new OtpErlangTuple(new OtpErlangObject[] {tsrow, l}); + return new OtpErlangList(cells); + } + + private static OtpErlangTuple pbRowToTtb(RiakTsPB.TsRow r) { + return new OtpErlangTuple(new OtpErlangObject[] {tsrow, pbCellsToTtb(r.getCellsList())}); } private static OtpErlangBinary pbStrToTtb(ByteString bs) { return new OtpErlangBinary(bs.toByteArray()); } + private static OtpErlangTuple pbInterpolationToTtb(RiakTsPB.TsInterpolation interpolation) { + return new OtpErlangTuple(new OtpErlangObject[] {tsinterpolation, pbStrToTtb(interpolation.getBase()), EMPTY_ERLANG_LIST}); + } + private static OtpErlangTuple encodeTsPut(RiakMessage msg) throws InvalidProtocolBufferException, UnsupportedEncodingException { final RiakTsPB.TsPutReq req = RiakTsPB.TsPutReq.parseFrom(msg.getData()); assert req != null; @@ -122,7 +139,7 @@ private static OtpErlangTuple encodeTsPut(RiakMessage msg) throws InvalidProtoco rows[i] = pbRowToTtb(r); } - OtpErlangObject[] elems = new OtpErlangObject[] + final OtpErlangObject[] elems = new OtpErlangObject[] {tsputreq, pbStrToTtb(req.getTable()), new OtpErlangList(), new OtpErlangList(rows)}; @@ -130,6 +147,95 @@ private static OtpErlangTuple encodeTsPut(RiakMessage msg) throws InvalidProtoco return new OtpErlangTuple(elems); } + private OtpErlangTuple encodeTsQuery(RiakMessage msg) throws InvalidProtocolBufferException { + final RiakTsPB.TsQueryReq req = RiakTsPB.TsQueryReq.parseFrom(msg.getData()); + + final OtpErlangObject[] elems = new OtpErlangObject[] + {tsqueryreq, pbInterpolationToTtb(req.getQuery()), _false}; + + return new OtpErlangTuple(elems); + } + + private static OtpErlangTuple encodeTsGet(RiakMessage msg) throws InvalidProtocolBufferException { + final RiakTsPB.TsGetReq req = RiakTsPB.TsGetReq.parseFrom(msg.getData()); + + final OtpErlangObject[] elems = new OtpErlangObject[] + {tsgetreq, pbStrToTtb(req.getTable()), + pbCellsToTtb(req.getKeyList()), + undefined}; + + return new OtpErlangTuple(elems); + } + + private static ByteString ttbBinaryToPb(OtpErlangBinary b) { + return ByteString.copyFromUtf8(new String(b.binaryValue())); + } + + private static List ttbRowsToPb(OtpErlangList ttbRows) { + final RiakTsPB.TsRow rows[] = new RiakTsPB.TsRow[ttbRows.arity()]; + + for (int i=0; i ttbColumnDescriptionToPb(OtpErlangList ttbDescrs) { + final RiakTsPB.TsColumnDescription descrs[] = new RiakTsPB.TsColumnDescription[ttbDescrs.arity()]; + for (int i=0; i out) throws Exception { // Make sure we have 4 bytes @@ -160,7 +266,7 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t assert o != null; if (o instanceof OtpErlangTuple && ((OtpErlangTuple)o).elementAt(0) instanceof OtpErlangAtom) { final OtpErlangTuple t = ((OtpErlangTuple)o); - final OtpErlangAtom resp = (OtpErlangAtom) ((OtpErlangTuple)o).elementAt(0); + final OtpErlangAtom resp = (OtpErlangAtom)t.elementAt(0); final String v = resp.atomValue(); if ("tsputresp".equals(v)) { @@ -174,6 +280,17 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t .setErrmsg(ByteString.copyFromUtf8(errMsg.stringValue())) .build(); out.add(new RiakMessage(RiakMessageCodes.MSG_ErrorResp, r.toByteArray())); + } else if ("tsgetresp".equals(v)) { + final RiakTsPB.TsGetResp r = RiakTsPB.TsGetResp.newBuilder() + .addAllColumns(ttbColumnDescriptionToPb((OtpErlangList)t.elementAt(1))) + .addAllRows(ttbRowsToPb((OtpErlangList)t.elementAt(2))) + .build(); + + out.add(new RiakMessage(RiakMessageCodes.MSG_TsGetResp, r.toByteArray())); + +// } else if ("tsqueryresp".equals(v)) { +// final RiakTsPB.TsQueryResp r = tsqueryrespToPB(); +// out.add(new RiakMessage(RiakMessageCodes.MSG_TsQueryResp, r.toByteArray())); } else { throw new IllegalStateException("Can't decode TTB response, unsupported atom '"+ v + "'"); } diff --git a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java index 601032915..25f7983b2 100644 --- a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java +++ b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java @@ -1,13 +1,17 @@ package com.basho.riak.client.core.netty; +import com.basho.riak.client.core.FutureOperation; import com.basho.riak.client.core.RiakNode; +import com.basho.riak.client.core.operations.ts.FetchOperation; +import com.basho.riak.client.core.operations.ts.QueryOperation; import com.basho.riak.client.core.operations.ts.StoreOperation; import com.basho.riak.client.core.query.timeseries.Cell; +import com.basho.riak.client.core.query.timeseries.QueryResult; import com.basho.riak.client.core.query.timeseries.Row; import com.basho.riak.client.core.util.BinaryValue; -import junit.framework.Assert; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.net.UnknownHostException; @@ -16,6 +20,9 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + /** * @author Sergey Galkin */ @@ -31,10 +38,19 @@ public class ITestTTBNode { protected final static long fifteenMinsInFuture = now + (fiveMinsInMS * 3l); private RiakNode riakNode; - private final List rows = Arrays.asList( - new Row(new Cell("hash1"), new Cell("user1"), Cell.newTimestamp(fifteenMinsAgo), - new Cell("cloudy"), new Cell(79.0), new Cell(1), new Cell(true)) - ); + protected final static List rows = Arrays.asList( + // "Normal" Data + new Row(new Cell("hash1"), new Cell("user1"), Cell.newTimestamp(fifteenMinsAgo), new Cell("cloudy"), new Cell(79.0), new Cell(1), new Cell(true)), + new Row(new Cell("hash1"), new Cell("user1"), Cell.newTimestamp(fiveMinsAgo), new Cell("sunny"), new Cell(80.5), new Cell(2), new Cell(true)), + new Row(new Cell("hash1"), new Cell("user1"), Cell.newTimestamp(now), new Cell("sunny"), new Cell(81.0), new Cell(10), new Cell(false)), + + // Null Cell row + new Row(new Cell("hash1"), new Cell("user2"), Cell.newTimestamp(fiveMinsAgo), new Cell("cloudy"), null, null, new Cell(true)), + + // Data for single reads / deletes + new Row(new Cell("hash2"), new Cell("user4"), Cell.newTimestamp(fifteenMinsAgo), new Cell("rain"), new Cell(79.0), new Cell(2), new Cell(false)), + new Row(new Cell("hash2"), new Cell("user4"), Cell.newTimestamp(fiveMinsAgo), new Cell("wind"), new Cell(50.5), new Cell(3), new Cell(true)), + new Row(new Cell("hash2"), new Cell("user4"), Cell.newTimestamp(now), new Cell("snow"), new Cell(20.0), new Cell(11), new Cell(true))); @Before @@ -57,27 +73,47 @@ public void shutdonw(){ @Test public void storeData() throws InterruptedException, ExecutionException { - final StoreOperation store = new StoreOperation.Builder(BinaryValue.create(tableName)).withRows(rows).build(); + storeDataInternal(tableName, rows.subList(0,1)); + } - if (!riakNode.execute(store)) { - Assert.fail("Cant store data"); - } + @Test + public void getData() throws ExecutionException, InterruptedException { + storeDataInternal(tableName, rows); - store.get(); - assert store != null; + for (Row r: rows){ + final FetchOperation fetch = new FetchOperation.Builder(BinaryValue.create(tableName), + r.getCells().subList(0,3) // use the only PK values + ).build(); + + QueryResult queryResult = execute(fetch); + + assertEquals(1, queryResult.getRows().size()); + //assertArrayEquals(r.getCells().toArray(), queryResult.getRows().get(0).getCells().toArray()); + } } + @Ignore @Test - public void storeDatatoNoneExistentTable() throws InterruptedException, ExecutionException { - final StoreOperation store = new StoreOperation.Builder(BinaryValue.create(UUID.randomUUID().toString())) - .withRows(rows).build(); + public void queryData() throws InterruptedException, ExecutionException { + storeDataInternal(tableName, rows); - if (!riakNode.execute(store)) { - Assert.fail("Cant store data"); - } + final String queryText = "select * from GeoCheckin " + + "where user = 'user1' and " + + "geohash = 'hash1' and " + + "(time = " + tenMinsAgo +" and " + + "(time < "+ now + ")) "; + final QueryOperation query = new QueryOperation.Builder(BinaryValue.create(queryText)).build(); + final QueryResult queryResult = execute(query); + + assertEquals(7, queryResult.getColumnDescriptions().size()); + assertEquals(1, queryResult.getRows().size()); + } + + @Test + public void storeDatatoNoneExistentTable() throws InterruptedException, ExecutionException { try { - store.get(); + storeDataInternal(UUID.randomUUID().toString(), rows); } catch (ExecutionException e) { if (e.getCause() instanceof RiakResponseException && e.getCause().getMessage().startsWith("Bucket type") @@ -88,4 +124,15 @@ public void storeDatatoNoneExistentTable() throws InterruptedException, Executio throw e; } } + + private > T execute(O operation) throws ExecutionException, InterruptedException { + riakNode.execute((FutureOperation) operation); + return (T)((FutureOperation) operation).get(); + } + + private void storeDataInternal(String table, List rows) throws ExecutionException, InterruptedException { + final StoreOperation store = new StoreOperation.Builder(BinaryValue.create(table)).withRows(rows).build(); + + execute(store); + } } From 48dc9bdb535c7ddaec5ad32ab2c91dcaa6cc3626 Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Tue, 12 Jan 2016 22:19:05 +0200 Subject: [PATCH 06/52] =?UTF-8?q?Added=20ToggleTTBEncodingOperation=20and?= =?UTF-8?q?=20smart=20TTB/PB=20=20=20encoding/decoding=D1=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../riak/client/core/DefaultNodeManager.java | 17 ++++ .../riak/client/core/netty/RiakTTBCodec.java | 84 +++++++++++++++++-- .../ToggleTTBEncodingOperation.java | 47 +++++++++++ .../ts/ITestToggleTTBEncodingOperation.java | 27 ++++++ 4 files changed, 166 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/basho/riak/client/core/operations/ToggleTTBEncodingOperation.java create mode 100644 src/test/java/com/basho/riak/client/core/operations/itest/ts/ITestToggleTTBEncodingOperation.java diff --git a/src/main/java/com/basho/riak/client/core/DefaultNodeManager.java b/src/main/java/com/basho/riak/client/core/DefaultNodeManager.java index 454020873..f85364843 100644 --- a/src/main/java/com/basho/riak/client/core/DefaultNodeManager.java +++ b/src/main/java/com/basho/riak/client/core/DefaultNodeManager.java @@ -20,6 +20,8 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; + +import com.basho.riak.client.core.operations.ToggleTTBEncodingOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,6 +109,21 @@ public void nodeStateChanged(RiakNode node, State state) try { lock.writeLock().lock(); + // TODO: REFACTOR BEFORE MERGE THIS TO DEVELOP + final ToggleTTBEncodingOperation op = new ToggleTTBEncodingOperation(true); + try { + boolean switchedToTTB = node.execute(op); + if (!switchedToTTB) { + throw new Exception("RiakNode '" + node.getRemoteAddress() + "' failed to switch to use Native(TTB) encoding"); + } + + logger.warn("RiakNode was switched to use Native encoding: {}:{}", + node.getRemoteAddress(), node.getPort()); + + } catch ( Exception e) { + throw new RuntimeException("Can't switch RiakNode to use Native (TTB) encoding"); + } + if (unhealthy.remove(node)) { healthy.add(node); diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java index 2194893ac..454bad0ab 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java @@ -8,17 +8,21 @@ import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageCodec; import java.io.UnsupportedEncodingException; import java.util.Arrays; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; /** * @author Sergey Galkin */ public class RiakTTBCodec extends ByteToMessageCodec { + private RiakMessageCodec pbCodec = new RiakMessageCodec(); + private static final OtpErlangAtom undefined = new OtpErlangAtom("undefined"); private static final OtpErlangAtom tscell = new OtpErlangAtom("tscell"); private static final OtpErlangAtom tsrow = new OtpErlangAtom("tsrow"); @@ -32,22 +36,46 @@ public class RiakTTBCodec extends ByteToMessageCodec { private static final OtpErlangTuple EMPTY_ERLANG_TSCELL = new OtpErlangTuple(new OtpErlangObject[]{ tscell, undefined, undefined, undefined, undefined, undefined}); + // TODO: REFACTOR BEFORE MERGE + private static ConcurrentHashMap ttbPerNode = new ConcurrentHashMap(); + + private boolean isTTBMessage(byte code) { + switch (code) { + case RiakMessageCodes.MSG_TsPutReq: + case RiakMessageCodes.MSG_TsGetReq: + return true; + default: + return false; + } + } + + private static boolean isTTBOnForChanel(Channel channel) { + return Boolean.TRUE.equals(ttbPerNode.get(channel.remoteAddress().toString())); + } + @Override protected void encode(ChannelHandlerContext ctx, RiakMessage msg, ByteBuf out) throws Exception { final OtpErlangTuple t; - switch (msg.getCode()) { - case RiakMessageCodes.MSG_TsPutReq: - t = encodeTsPut(msg); - break; - case RiakMessageCodes.MSG_TsGetReq: - t = encodeTsGet(msg); - break; + final boolean isTtbOn = isTTBOnForChanel(ctx.channel()); + + if (isTtbOn && isTTBMessage(msg.getCode())) { + switch (msg.getCode()) { + case RiakMessageCodes.MSG_TsPutReq: + t = encodeTsPut(msg); + break; + case RiakMessageCodes.MSG_TsGetReq: + t = encodeTsGet(msg); + break; // case RiakMessageCodes.MSG_TsQueryReq: // t = encodeTsQuery(msg); // break; - default: - throw new IllegalStateException("Can't encode TTB request, unsupported message with code " + msg.getCode()); + default: + throw new IllegalStateException("Can't encode TTB request, unsupported message with code " + msg.getCode()); + } + } else { + pbCodec.encode(ctx, msg, out); + return; } final OtpOutputStream os = new OtpOutputStream(t); @@ -238,6 +266,9 @@ private static RiakTsPB.TsCell ttbCellToPb(OtpErlangTuple c) { @Override protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { + + final boolean isTTBOn = isTTBOnForChanel(ctx.channel()); + // Make sure we have 4 bytes if (in.readableBytes() >= 4) { in.markReaderIndex(); @@ -250,6 +281,23 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t return; } + if (!isTTBOn) { + // Decode as PB + + in.resetReaderIndex(); + pbCodec.decode(null, in, out); + + if (out.size() == 1) { + RiakMessage m = (RiakMessage) out.get(0); + switch ( m.getCode()) { + case RiakMessageCodes.MSG_ToggleEncodingResp: + final RiakPB.RpbToggleEncodingResp r = RiakPB.RpbToggleEncodingResp.PARSER.parseFrom(m.getData()); + ttbPerNode.put(ctx.channel().remoteAddress().toString(), r.getUseNative()); + break; + } + } + return; + } final byte[] array = new byte[length]; in.readBytes(array); @@ -260,6 +308,24 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t o = is.read_any(); }catch (Exception ex){ in.resetReaderIndex(); + in.markReaderIndex(); + + // skip length header + in.readInt(); + + // read msg code + byte code = in.readByte(); + + if (isTTBMessage(code)) { + throw ex; + } + + in.resetReaderIndex(); + + /** + * As we failed with TTB encoding, it worth to try with PB + */ + pbCodec.decode(null, in, out); return; } diff --git a/src/main/java/com/basho/riak/client/core/operations/ToggleTTBEncodingOperation.java b/src/main/java/com/basho/riak/client/core/operations/ToggleTTBEncodingOperation.java new file mode 100644 index 000000000..0490c9729 --- /dev/null +++ b/src/main/java/com/basho/riak/client/core/operations/ToggleTTBEncodingOperation.java @@ -0,0 +1,47 @@ +package com.basho.riak.client.core.operations; + +import com.basho.riak.protobuf.RiakMessageCodes; +import com.basho.riak.protobuf.RiakPB; + +import java.util.List; + +/** + * Created by srg on 1/12/16. + */ +public class ToggleTTBEncodingOperation extends PBFutureOperation +{ + + public ToggleTTBEncodingOperation(boolean useTTBEncoding) { + super(RiakMessageCodes.MSG_ToggleEncodingReq, + RiakMessageCodes.MSG_ToggleEncodingResp, + RiakPB.RpbToggleEncodingResp.newBuilder().setUseNative(useTTBEncoding), + RiakPB.RpbToggleEncodingResp.PARSER); + } + + @Override + protected Response convert(List responses) { + // This is not a streaming op, there will only be one response + checkAndGetSingleResponse(responses); + return new ToggleTTBEncodingOperation.Response(responses.get(0).getUseNative()); + } + + @Override + public String getQueryInfo() { + return "SwitchToUSETTTBOperation"; + } + + /** + * Response returned from a SwitchToUSETTTBOperation + */ + public static class Response { + private final boolean useNativeEncoding; + + public Response(boolean useNativeEncoding) { + this.useNativeEncoding = useNativeEncoding; + } + + public boolean isUseNativeEncoding() { + return useNativeEncoding; + } + } +} diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ts/ITestToggleTTBEncodingOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ts/ITestToggleTTBEncodingOperation.java new file mode 100644 index 000000000..508cfe531 --- /dev/null +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ts/ITestToggleTTBEncodingOperation.java @@ -0,0 +1,27 @@ +package com.basho.riak.client.core.operations.itest.ts; + +import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.operations.ToggleTTBEncodingOperation; +import com.basho.riak.client.core.operations.itest.ITestBase; +import org.junit.Test; + +import java.util.concurrent.ExecutionException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Created by srg on 1/12/16. + */ +public class ITestToggleTTBEncodingOperation extends ITestBase { + @Test + public void testSingleDelete() throws ExecutionException, InterruptedException + { + final boolean useTTB = false; + final ToggleTTBEncodingOperation op = new ToggleTTBEncodingOperation(useTTB); + final RiakFuture future = cluster.execute(op); + + final ToggleTTBEncodingOperation.Response response = future.get(); + assertEquals(useTTB, response.isUseNativeEncoding()); + } +} From b46e1b159b2d0923829e5f479e83f878a5bdbfda Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Fri, 15 Jan 2016 06:44:43 +0200 Subject: [PATCH 07/52] Fix broken tests --- .../riak/client/core/netty/ITestTTBNode.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java index 25f7983b2..e43711e91 100644 --- a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java +++ b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java @@ -81,13 +81,13 @@ public void getData() throws ExecutionException, InterruptedException { storeDataInternal(tableName, rows); for (Row r: rows){ - final FetchOperation fetch = new FetchOperation.Builder(BinaryValue.create(tableName), - r.getCells().subList(0,3) // use the only PK values + final FetchOperation fetch = new FetchOperation.Builder(tableName, + r.getCellsCopy().subList(0,3) // use the only PK values ).build(); QueryResult queryResult = execute(fetch); - assertEquals(1, queryResult.getRows().size()); + assertEquals(1, queryResult.getRowsCount()); //assertArrayEquals(r.getCells().toArray(), queryResult.getRows().get(0).getCells().toArray()); } } @@ -103,11 +103,11 @@ public void queryData() throws InterruptedException, ExecutionException { "(time = " + tenMinsAgo +" and " + "(time < "+ now + ")) "; - final QueryOperation query = new QueryOperation.Builder(BinaryValue.create(queryText)).build(); + final QueryOperation query = new QueryOperation.Builder(queryText).build(); final QueryResult queryResult = execute(query); - assertEquals(7, queryResult.getColumnDescriptions().size()); - assertEquals(1, queryResult.getRows().size()); + assertEquals(7, queryResult.getColumnDescriptionsCopy().size()); + assertEquals(1, queryResult.getRowsCount()); } @Test @@ -116,8 +116,8 @@ public void storeDatatoNoneExistentTable() throws InterruptedException, Executio storeDataInternal(UUID.randomUUID().toString(), rows); } catch (ExecutionException e) { if (e.getCause() instanceof RiakResponseException - && e.getCause().getMessage().startsWith("Bucket type") - && e.getCause().getMessage().endsWith("is missing.")) { + && e.getCause().getMessage().startsWith("Time Series table") + && e.getCause().getMessage().endsWith("does not exist.")) { // It's OK return; } @@ -131,7 +131,7 @@ private > T execute(O operation) throw } private void storeDataInternal(String table, List rows) throws ExecutionException, InterruptedException { - final StoreOperation store = new StoreOperation.Builder(BinaryValue.create(table)).withRows(rows).build(); + final StoreOperation store = new StoreOperation.Builder(table).withRows(rows).build(); execute(store); } From 83e509f1d4b2a6927292daa8eb724b0d66e2c66f Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Fri, 15 Jan 2016 14:32:03 +0200 Subject: [PATCH 08/52] TTB codec adjusted to RiakTS 1.1 changes --- .../riak/client/core/netty/RiakTTBCodec.java | 32 ++++++++++++++----- .../riak/client/core/netty/ITestTTBNode.java | 8 ++++- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java index 454bad0ab..6a66cce65 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java @@ -81,9 +81,12 @@ protected void encode(ChannelHandlerContext ctx, RiakMessage msg, ByteBuf out) t final OtpOutputStream os = new OtpOutputStream(t); byte data[] = os.toByteArray(); - int length = data.length+1; + int length = data.length+2; out.writeInt(length); + + out.writeByte(msg.getCode()); + /** * DO A TOP SECRET HACK * @@ -299,9 +302,17 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t return; } - final byte[] array = new byte[length]; + final byte[] array = new byte[length-1]; + final int msgCode = in.readByte(); in.readBytes(array); + switch (msgCode) { + case RiakMessageCodes.MSG_TsPutResp: + final RiakTsPB.TsPutResp r = RiakTsPB.TsPutResp.newBuilder().build(); + out.add(new RiakMessage(RiakMessageCodes.MSG_TsPutResp, r.toByteArray())); + return; + } + final OtpErlangObject o; try { OtpInputStream is = new OtpInputStream(array); @@ -335,15 +346,20 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t final OtpErlangAtom resp = (OtpErlangAtom)t.elementAt(0); final String v = resp.atomValue(); - if ("tsputresp".equals(v)) { - final RiakTsPB.TsPutResp r = RiakTsPB.TsPutResp.newBuilder().build(); - out.add(new RiakMessage(RiakMessageCodes.MSG_TsPutResp, r.toByteArray())); - } else if ("rpberrorresp".equals(v)) { - final OtpErlangString errMsg = (OtpErlangString) t.elementAt(1); + if ("rpberrorresp".equals(v)) { + final String errMsg; + if (t.elementAt(1) instanceof OtpErlangString) { + errMsg = ((OtpErlangString) t.elementAt(1)).stringValue(); + } else if (t.elementAt(1) instanceof OtpErlangBinary) { + errMsg = new String(((OtpErlangBinary) t.elementAt(1)).binaryValue()); + } else { + throw new IllegalStateException(); + } + final OtpErlangLong errCode = (OtpErlangLong) t.elementAt(2); final RiakPB.RpbErrorResp r = RiakPB.RpbErrorResp.newBuilder() .setErrcode(errCode.intValue()) - .setErrmsg(ByteString.copyFromUtf8(errMsg.stringValue())) + .setErrmsg(ByteString.copyFromUtf8(errMsg)) .build(); out.add(new RiakMessage(RiakMessageCodes.MSG_ErrorResp, r.toByteArray())); } else if ("tsgetresp".equals(v)) { diff --git a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java index e43711e91..aab77c88f 100644 --- a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java +++ b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java @@ -2,6 +2,7 @@ import com.basho.riak.client.core.FutureOperation; import com.basho.riak.client.core.RiakNode; +import com.basho.riak.client.core.operations.ToggleTTBEncodingOperation; import com.basho.riak.client.core.operations.ts.FetchOperation; import com.basho.riak.client.core.operations.ts.QueryOperation; import com.basho.riak.client.core.operations.ts.StoreOperation; @@ -22,6 +23,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author Sergey Galkin @@ -54,13 +56,17 @@ public class ITestTTBNode { @Before - public void setup() throws UnknownHostException { + public void setup() throws UnknownHostException, ExecutionException, InterruptedException { riakNode = new RiakNode.Builder() .useTTB() .withRemotePort(10017) .build(); riakNode.start(); + final ToggleTTBEncodingOperation op = new ToggleTTBEncodingOperation(true); + riakNode.execute(op); + + assertTrue(op.get().isUseNativeEncoding()); } From 65f2a92a18e3163ca1b62251c6c0423ea61f8766 Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Wed, 20 Jan 2016 14:41:07 +0200 Subject: [PATCH 09/52] Deferred encoding operation support was introduces for TS Store and Fetch operations to avoid double conversion from PB->TTB --- .../basho/riak/client/core/RiakMessage.java | 47 ++++++++++++++++++- .../riak/client/core/netty/RiakTTBCodec.java | 22 +++++++-- .../core/operations/PBFutureOperation.java | 2 +- .../ts/DeferredEncodingOperation.java | 25 ++++++++++ .../core/operations/ts/FetchOperation.java | 2 +- .../core/operations/ts/StoreOperation.java | 2 +- 6 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/basho/riak/client/core/operations/ts/DeferredEncodingOperation.java diff --git a/src/main/java/com/basho/riak/client/core/RiakMessage.java b/src/main/java/com/basho/riak/client/core/RiakMessage.java index d24e3f526..4392a1749 100644 --- a/src/main/java/com/basho/riak/client/core/RiakMessage.java +++ b/src/main/java/com/basho/riak/client/core/RiakMessage.java @@ -18,19 +18,35 @@ /** * Encapsulates the raw bytes sent to or received from Riak. * @author Brian Roach + * @author Sergey Galkin * @since 2.0 */ public final class RiakMessage { private final byte code; - private final byte[] data; + private byte[] data; + private final Object dataObject; public RiakMessage(byte code, byte[] data) { this.code = code; this.data = data; + this.dataObject = null; } - + + /** + * Creates message for deferred encoding. + * + * @author Sergey Galkin + * @since 2.4 + */ + public RiakMessage(byte code, Object dataObj) + { + this.code = code; + this.data = null; + this.dataObject = dataObj; + } + public byte getCode() { return code; @@ -40,4 +56,31 @@ public byte[] getData() { return data; } + + /** + * @author Sergey Galkin + * @since 2.4 + */ + public boolean isEncoded() + { + return data != null; + } + + /** + * @author Sergey Galkin + * @since 2.4 + */ + public void setData(byte[] data) + { + this.data = data; + } + + /** + * @author Sergey Galkin + * @since 2.4 + */ + public T getDataObject() + { + return (T) dataObject; + } } diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java index 6a66cce65..b9520a6b0 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java @@ -6,6 +6,7 @@ import com.basho.riak.protobuf.RiakTsPB; import com.ericsson.otp.erlang.*; import com.google.protobuf.ByteString; +import com.google.protobuf.GeneratedMessage; import com.google.protobuf.InvalidProtocolBufferException; import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; @@ -74,6 +75,10 @@ protected void encode(ChannelHandlerContext ctx, RiakMessage msg, ByteBuf out) t throw new IllegalStateException("Can't encode TTB request, unsupported message with code " + msg.getCode()); } } else { + if (!msg.isEncoded()) { + final GeneratedMessage.Builder reqBuilder = msg.getDataObject(); + msg.setData(reqBuilder.build().toByteArray()); + } pbCodec.encode(ctx, msg, out); return; } @@ -160,9 +165,12 @@ private static OtpErlangTuple pbInterpolationToTtb(RiakTsPB.TsInterpolation inte } private static OtpErlangTuple encodeTsPut(RiakMessage msg) throws InvalidProtocolBufferException, UnsupportedEncodingException { - final RiakTsPB.TsPutReq req = RiakTsPB.TsPutReq.parseFrom(msg.getData()); - assert req != null; - + final RiakTsPB.TsPutReqOrBuilder req; + if (msg.isEncoded()) { + req = RiakTsPB.TsPutReq.parseFrom(msg.getData()); + }else { + req = msg.getDataObject(); + } final OtpErlangObject rows[] = new OtpErlangObject[req.getRowsCount()]; for (int i = 0; i < rows.length; ++i) { @@ -188,7 +196,13 @@ private OtpErlangTuple encodeTsQuery(RiakMessage msg) throws InvalidProtocolBuff } private static OtpErlangTuple encodeTsGet(RiakMessage msg) throws InvalidProtocolBufferException { - final RiakTsPB.TsGetReq req = RiakTsPB.TsGetReq.parseFrom(msg.getData()); + final RiakTsPB.TsGetReqOrBuilder req; + if (msg.isEncoded()) { + req = RiakTsPB.TsGetReq.parseFrom(msg.getData()); + + } else { + req = msg.getDataObject(); + } final OtpErlangObject[] elems = new OtpErlangObject[] {tsgetreq, pbStrToTtb(req.getTable()), diff --git a/src/main/java/com/basho/riak/client/core/operations/PBFutureOperation.java b/src/main/java/com/basho/riak/client/core/operations/PBFutureOperation.java index f2b862eaf..d6e8ffa58 100644 --- a/src/main/java/com/basho/riak/client/core/operations/PBFutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/PBFutureOperation.java @@ -23,7 +23,7 @@ public abstract class PBFutureOperation extends FutureOperation { protected final Builder reqBuilder; private final com.google.protobuf.Parser respParser; - private final byte reqMessageCode; + protected final byte reqMessageCode; private final byte respMessageCode; diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/DeferredEncodingOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/DeferredEncodingOperation.java new file mode 100644 index 000000000..75d78a6a0 --- /dev/null +++ b/src/main/java/com/basho/riak/client/core/operations/ts/DeferredEncodingOperation.java @@ -0,0 +1,25 @@ +package com.basho.riak.client.core.operations.ts; + +import com.basho.riak.client.core.RiakMessage; +import com.basho.riak.client.core.operations.PBFutureOperation; +import com.google.protobuf.GeneratedMessage; + + +/** + * @author Sergey Galkin + */ +public abstract class DeferredEncodingOperation extends PBFutureOperation { + + protected DeferredEncodingOperation(final byte reqMessageCode, + final byte respMessageCode, + final GeneratedMessage.Builder reqBuilder, + com.google.protobuf.Parser respParser) + { + super(reqMessageCode, respMessageCode, reqBuilder, respParser); + } + + @Override + protected RiakMessage createChannelMessage() { + return new RiakMessage(reqMessageCode, reqBuilder); + } +} diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java index 61795773b..e72d32fd9 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java @@ -15,7 +15,7 @@ * @author Sergey Galkin * @since 2.0.3 */ -public class FetchOperation extends PBFutureOperation +public class FetchOperation extends DeferredEncodingOperation { private final Builder builder; private String queryInfoMessage; diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java index dcc3f57a7..cd2b5931c 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java @@ -19,7 +19,7 @@ * @author Sergey Galkin * @since 2.0.3 */ -public class StoreOperation extends PBFutureOperation +public class StoreOperation extends DeferredEncodingOperation { private final String tableName; private final int rowCount; From 970735cbc5ee091b377cf90205f30650a7171b1d Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Thu, 21 Jan 2016 08:15:33 +0200 Subject: [PATCH 10/52] Added proper handling of com.basho.riak.pbcport --- .../com/basho/riak/client/core/netty/ITestTTBNode.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java index aab77c88f..d69756c16 100644 --- a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java +++ b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java @@ -57,9 +57,17 @@ public class ITestTTBNode { @Before public void setup() throws UnknownHostException, ExecutionException, InterruptedException { + /** + * Riak PBC port + * + * In case you want/need to use a custom PBC port you may pass it by using the following system property + */ + final int testRiakPort = Integer.getInteger("com.basho.riak.pbcport", RiakNode.Builder.DEFAULT_REMOTE_PORT); + + riakNode = new RiakNode.Builder() .useTTB() - .withRemotePort(10017) + .withRemotePort(testRiakPort) .build(); riakNode.start(); From a59d9f3b881973f209018084a8d68112d032294c Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Fri, 22 Jan 2016 21:01:46 +0200 Subject: [PATCH 11/52] rework preventive toggle the Native/TTB encoding to be toggled on per-connection/channel basis --- .../riak/client/core/DefaultNodeManager.java | 15 --------- .../com/basho/riak/client/core/RiakNode.java | 26 ++++++++++++++- .../riak/client/core/netty/RiakTTBCodec.java | 33 +++++++++++++------ .../riak/client/core/netty/ITestTTBNode.java | 15 +++++---- 4 files changed, 56 insertions(+), 33 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/DefaultNodeManager.java b/src/main/java/com/basho/riak/client/core/DefaultNodeManager.java index f85364843..0dc159c4f 100644 --- a/src/main/java/com/basho/riak/client/core/DefaultNodeManager.java +++ b/src/main/java/com/basho/riak/client/core/DefaultNodeManager.java @@ -109,21 +109,6 @@ public void nodeStateChanged(RiakNode node, State state) try { lock.writeLock().lock(); - // TODO: REFACTOR BEFORE MERGE THIS TO DEVELOP - final ToggleTTBEncodingOperation op = new ToggleTTBEncodingOperation(true); - try { - boolean switchedToTTB = node.execute(op); - if (!switchedToTTB) { - throw new Exception("RiakNode '" + node.getRemoteAddress() + "' failed to switch to use Native(TTB) encoding"); - } - - logger.warn("RiakNode was switched to use Native encoding: {}:{}", - node.getRemoteAddress(), node.getPort()); - - } catch ( Exception e) { - throw new RuntimeException("Can't switch RiakNode to use Native (TTB) encoding"); - } - if (unhealthy.remove(node)) { healthy.add(node); diff --git a/src/main/java/com/basho/riak/client/core/RiakNode.java b/src/main/java/com/basho/riak/client/core/RiakNode.java index 1108e4d73..98394ccd0 100644 --- a/src/main/java/com/basho/riak/client/core/RiakNode.java +++ b/src/main/java/com/basho/riak/client/core/RiakNode.java @@ -16,8 +16,10 @@ package com.basho.riak.client.core; import com.basho.riak.client.core.netty.*; +import com.basho.riak.client.core.operations.ToggleTTBEncodingOperation; import com.basho.riak.client.core.util.Constants; import com.basho.riak.client.core.util.HostAndPort; +import com.basho.riak.protobuf.RiakMessageCodes; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; @@ -776,6 +778,23 @@ else if (protocols.contains("TLSv1.1")) } + // TODO: REFACTOR BEFORE MERGE THIS TO DEVELOP (make it to be configurable) + // Toggling the Native/TTB encoding + final ToggleTTBEncodingOperation operation = new ToggleTTBEncodingOperation(true); + try { + inProgressMap.put(c, operation); + c.writeAndFlush(operation); + + ToggleTTBEncodingOperation.Response r = operation.get(); + boolean switchedToTTB = r.isUseNativeEncoding(); + if (!switchedToTTB) { + throw new RuntimeException("RiakNode '" + remoteAddress + ":" + port + "' failed to switch to use Native(TTB) encoding"); + } + + logger.warn("Channel '{}' was switched to use Native encoding: {}", c.hashCode(), c); + } catch ( Exception e) { + throw new RuntimeException("Can't switch channel to use Native (TTB) encoding"); + } return c; } @@ -849,7 +868,12 @@ public void onSuccess(Channel channel, final RiakMessage response) if (inProgress.isDone()) { inProgressMap.remove(channel); - returnConnection(channel); // return permit + + // TODO: REALLY DIRTY HACK to prevent channel from being returned back to the pool + // as a result of successful response on preventive toggling TTB usage + if (response.getCode() != RiakMessageCodes.MSG_ToggleEncodingResp) { + returnConnection(channel); // return permit + } } } } diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java index b9520a6b0..7bffaaa64 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java @@ -12,16 +12,19 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageCodec; +import io.netty.util.AttributeKey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.UnsupportedEncodingException; import java.util.Arrays; import java.util.List; -import java.util.concurrent.ConcurrentHashMap; /** * @author Sergey Galkin */ public class RiakTTBCodec extends ByteToMessageCodec { + private final static Logger logger = LoggerFactory.getLogger(RiakTTBCodec.class); private RiakMessageCodec pbCodec = new RiakMessageCodec(); private static final OtpErlangAtom undefined = new OtpErlangAtom("undefined"); @@ -37,9 +40,6 @@ public class RiakTTBCodec extends ByteToMessageCodec { private static final OtpErlangTuple EMPTY_ERLANG_TSCELL = new OtpErlangTuple(new OtpErlangObject[]{ tscell, undefined, undefined, undefined, undefined, undefined}); - // TODO: REFACTOR BEFORE MERGE - private static ConcurrentHashMap ttbPerNode = new ConcurrentHashMap(); - private boolean isTTBMessage(byte code) { switch (code) { case RiakMessageCodes.MSG_TsPutReq: @@ -50,17 +50,14 @@ private boolean isTTBMessage(byte code) { } } - private static boolean isTTBOnForChanel(Channel channel) { - return Boolean.TRUE.equals(ttbPerNode.get(channel.remoteAddress().toString())); - } - @Override protected void encode(ChannelHandlerContext ctx, RiakMessage msg, ByteBuf out) throws Exception { final OtpErlangTuple t; - final boolean isTtbOn = isTTBOnForChanel(ctx.channel()); if (isTtbOn && isTTBMessage(msg.getCode())) { + logger.trace("Encoding message {}, encoding '{}', is deferred: {}, channel '{}' {}", + msg.getCode(), "TTB", !msg.isEncoded(), ctx.channel().hashCode(), ctx.channel()); switch (msg.getCode()) { case RiakMessageCodes.MSG_TsPutReq: t = encodeTsPut(msg); @@ -75,6 +72,9 @@ protected void encode(ChannelHandlerContext ctx, RiakMessage msg, ByteBuf out) t throw new IllegalStateException("Can't encode TTB request, unsupported message with code " + msg.getCode()); } } else { + logger.trace("Encoding message {}, encoding '{}', is deferred: {}, channel '{}' {}", + msg.getCode(), "PB", !msg.isEncoded(), ctx.channel().hashCode(), ctx.channel()); + if (!msg.isEncoded()) { final GeneratedMessage.Builder reqBuilder = msg.getDataObject(); msg.setData(reqBuilder.build().toByteArray()); @@ -309,7 +309,8 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t switch ( m.getCode()) { case RiakMessageCodes.MSG_ToggleEncodingResp: final RiakPB.RpbToggleEncodingResp r = RiakPB.RpbToggleEncodingResp.PARSER.parseFrom(m.getData()); - ttbPerNode.put(ctx.channel().remoteAddress().toString(), r.getUseNative()); + updateTTBUsageForChannel(ctx.channel(), r.getUseNative()); + logger.debug("Native/TTB encoding for channel '{}' is set to {}: {}", ctx.channel().hashCode(), r.getUseNative(), ctx.channel()); break; } } @@ -395,4 +396,16 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t } } } + + private static final AttributeKey USE_TTB = AttributeKey.valueOf("useTTB"); + + private static boolean isTTBOnForChanel(Channel channel) { + final Boolean b = channel.attr(USE_TTB).get(); + return Boolean.TRUE.equals(b); + } + + private static boolean updateTTBUsageForChannel(Channel channel, boolean useTTB) { + channel.attr(USE_TTB).getAndSet(Boolean.valueOf(useTTB)); + return useTTB; + } } diff --git a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java index d69756c16..ab8ce1d85 100644 --- a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java +++ b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java @@ -2,14 +2,12 @@ import com.basho.riak.client.core.FutureOperation; import com.basho.riak.client.core.RiakNode; -import com.basho.riak.client.core.operations.ToggleTTBEncodingOperation; import com.basho.riak.client.core.operations.ts.FetchOperation; import com.basho.riak.client.core.operations.ts.QueryOperation; import com.basho.riak.client.core.operations.ts.StoreOperation; import com.basho.riak.client.core.query.timeseries.Cell; import com.basho.riak.client.core.query.timeseries.QueryResult; import com.basho.riak.client.core.query.timeseries.Row; -import com.basho.riak.client.core.util.BinaryValue; import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -21,9 +19,7 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * @author Sergey Galkin @@ -68,13 +64,18 @@ public void setup() throws UnknownHostException, ExecutionException, Interrupted riakNode = new RiakNode.Builder() .useTTB() .withRemotePort(testRiakPort) +// .withMaxConnections(1) +// .withMinConnections(1) .build(); riakNode.start(); - final ToggleTTBEncodingOperation op = new ToggleTTBEncodingOperation(true); - riakNode.execute(op); - assertTrue(op.get().isUseNativeEncoding()); + /* + Since TTB is toggled in preventive manner for each conection there is no needs to toggle it manually + */ + //final ToggleTTBEncodingOperation op = new ToggleTTBEncodingOperation(true); + //riakNode.execute(op); + //assertTrue(op.get().isUseNativeEncoding()); } From 914c2153cf9061b0e9b0c9fdadf909a7cc6d27f0 Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Mon, 25 Jan 2016 19:41:11 +0200 Subject: [PATCH 12/52] getting rid of creation TsPutReq.Builder and TsGetReq.Builder --- .../riak/client/core/netty/RiakTTBCodec.java | 94 ++++++++++++++----- .../core/operations/ts/FetchOperation.java | 29 ++++-- .../core/operations/ts/StoreOperation.java | 50 ++++++---- .../client/core/query/timeseries/Cell.java | 2 +- .../client/core/query/timeseries/Row.java | 2 +- 5 files changed, 129 insertions(+), 48 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java index 7bffaaa64..f507c67ce 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java @@ -1,6 +1,10 @@ package com.basho.riak.client.core.netty; import com.basho.riak.client.core.RiakMessage; +import com.basho.riak.client.core.operations.ts.FetchOperation; +import com.basho.riak.client.core.operations.ts.StoreOperation; +import com.basho.riak.client.core.query.timeseries.Cell; +import com.basho.riak.client.core.query.timeseries.Row; import com.basho.riak.protobuf.RiakMessageCodes; import com.basho.riak.protobuf.RiakPB; import com.basho.riak.protobuf.RiakTsPB; @@ -17,8 +21,7 @@ import org.slf4j.LoggerFactory; import java.io.UnsupportedEncodingException; -import java.util.Arrays; -import java.util.List; +import java.util.*; /** * @author Sergey Galkin @@ -142,6 +145,30 @@ private static OtpErlangTuple pbCellToTtb(RiakTsPB.TsCell c) { return new OtpErlangTuple(o); } + private static int sizeOfIterable(Iterable values) { + if (values instanceof Collection) { + return ((Collection)values).size(); + } + + int size = 0; + for(Object value : values) { + ++size; + } + return size; + } + + private static OtpErlangList tsCellsToTtb(Iterable tsCells) { + final int size = sizeOfIterable(tsCells); + final OtpErlangObject cells[] = new OtpErlangObject[size]; + int idx =-1; + + for (Cell c: tsCells) { + cells[++idx] = pbCellToTtb(c.getPbCell()); + } + + return new OtpErlangList(cells); + } + private static OtpErlangList pbCellsToTtb(List pbCells) { final OtpErlangObject cells[] = new OtpErlangObject[pbCells.size()]; for (int i = 0; i < cells.length; ++i) { @@ -156,6 +183,10 @@ private static OtpErlangTuple pbRowToTtb(RiakTsPB.TsRow r) { return new OtpErlangTuple(new OtpErlangObject[] {tsrow, pbCellsToTtb(r.getCellsList())}); } + private static OtpErlangBinary strToTtb(String s) { + return new OtpErlangBinary(s.getBytes()); + } + private static OtpErlangBinary pbStrToTtb(ByteString bs) { return new OtpErlangBinary(bs.toByteArray()); } @@ -165,23 +196,36 @@ private static OtpErlangTuple pbInterpolationToTtb(RiakTsPB.TsInterpolation inte } private static OtpErlangTuple encodeTsPut(RiakMessage msg) throws InvalidProtocolBufferException, UnsupportedEncodingException { - final RiakTsPB.TsPutReqOrBuilder req; + final OtpErlangObject[] elems; if (msg.isEncoded()) { + final RiakTsPB.TsPutReqOrBuilder req; req = RiakTsPB.TsPutReq.parseFrom(msg.getData()); - }else { - req = msg.getDataObject(); - } + final OtpErlangObject rows[] = new OtpErlangObject[req.getRowsCount()]; + for (int i = 0; i < rows.length; ++i) { + final RiakTsPB.TsRow r = req.getRows(i); + rows[i] = pbRowToTtb(r); + } - final OtpErlangObject rows[] = new OtpErlangObject[req.getRowsCount()]; - for (int i = 0; i < rows.length; ++i) { - final RiakTsPB.TsRow r = req.getRows(i); - rows[i] = pbRowToTtb(r); - } + elems = new OtpErlangObject[] + {tsputreq, pbStrToTtb(req.getTable()), + new OtpErlangList(), + new OtpErlangList(rows)}; - final OtpErlangObject[] elems = new OtpErlangObject[] - {tsputreq, pbStrToTtb(req.getTable()), - new OtpErlangList(), - new OtpErlangList(rows)}; + } else { + final StoreOperation.Builder builder = msg.getDataObject(); + final int size = sizeOfIterable(builder.getRows()); + final OtpErlangObject rows[] = new OtpErlangObject[size]; + int idx =-1; + + for (Row r: builder.getRows()) { + rows[++idx] = pbRowToTtb(r.getPbRow()); + } + + elems = new OtpErlangObject[] + {tsputreq, strToTtb(builder.getTableName()), + new OtpErlangList(), + new OtpErlangList(rows)}; + } return new OtpErlangTuple(elems); } @@ -196,18 +240,24 @@ private OtpErlangTuple encodeTsQuery(RiakMessage msg) throws InvalidProtocolBuff } private static OtpErlangTuple encodeTsGet(RiakMessage msg) throws InvalidProtocolBufferException { - final RiakTsPB.TsGetReqOrBuilder req; + final OtpErlangObject[] elems; if (msg.isEncoded()) { + final RiakTsPB.TsGetReqOrBuilder req; req = RiakTsPB.TsGetReq.parseFrom(msg.getData()); + elems = new OtpErlangObject[] + {tsgetreq, pbStrToTtb(req.getTable()), + pbCellsToTtb(req.getKeyList()), + undefined}; + } else { - req = msg.getDataObject(); - } + final FetchOperation.Builder builder = msg.getDataObject(); - final OtpErlangObject[] elems = new OtpErlangObject[] - {tsgetreq, pbStrToTtb(req.getTable()), - pbCellsToTtb(req.getKeyList()), - undefined}; + elems = new OtpErlangObject[] + {tsgetreq, strToTtb(builder.getTableName()), + tsCellsToTtb(builder.getKeyValues()), + undefined}; + } return new OtpErlangTuple(elems); } diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java index e72d32fd9..184886c6e 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java @@ -1,10 +1,9 @@ package com.basho.riak.client.core.operations.ts; -import com.basho.riak.client.core.operations.PBFutureOperation; +import com.basho.riak.client.core.RiakMessage; import com.basho.riak.client.core.query.timeseries.*; import com.basho.riak.protobuf.RiakTsPB; import com.basho.riak.protobuf.RiakMessageCodes; -import com.google.protobuf.ByteString; import java.util.List; @@ -24,10 +23,16 @@ private FetchOperation(Builder builder) { super(RiakMessageCodes.MSG_TsGetReq, RiakMessageCodes.MSG_TsGetResp, - builder.reqBuilder, +// builder.reqBuilder, + null, RiakTsPB.TsGetResp.PARSER); - this.builder = builder; + this.builder = builder; + } + + @Override + protected RiakMessage createChannelMessage() { + return new RiakMessage(RiakMessageCodes.MSG_TsGetReq, builder); } @Override @@ -72,7 +77,7 @@ public static class Builder private final String tableName; private final Iterable keyValues; - private final RiakTsPB.TsGetReq.Builder reqBuilder = RiakTsPB.TsGetReq.newBuilder(); + //private final RiakTsPB.TsGetReq.Builder reqBuilder = RiakTsPB.TsGetReq.newBuilder(); public Builder(String tableName, Iterable keyValues) { @@ -86,8 +91,8 @@ public Builder(String tableName, Iterable keyValues) throw new IllegalArgumentException("Key Values cannot be null or an empty."); } - this.reqBuilder.setTable(ByteString.copyFromUtf8(tableName)); - this.reqBuilder.addAllKey(ConvertibleIterable.asIterablePbCell(keyValues)); +// this.reqBuilder.setTable(ByteString.copyFromUtf8(tableName)); +// this.reqBuilder.addAllKey(ConvertibleIterable.asIterablePbCell(keyValues)); this.tableName = tableName; this.keyValues = keyValues; @@ -95,7 +100,7 @@ public Builder(String tableName, Iterable keyValues) public Builder withTimeout(int timeout) { - this.reqBuilder.setTimeout(timeout); +// this.reqBuilder.setTimeout(timeout); return this; } @@ -103,5 +108,13 @@ public FetchOperation build() { return new FetchOperation(this); } + + public String getTableName() { + return tableName; + } + + public Iterable getKeyValues() { + return keyValues; + } } } diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java index cd2b5931c..9bb93c840 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java @@ -1,13 +1,10 @@ package com.basho.riak.client.core.operations.ts; -import com.basho.riak.client.core.operations.PBFutureOperation; -import com.basho.riak.client.core.query.timeseries.CollectionConverters; +import com.basho.riak.client.core.RiakMessage; import com.basho.riak.client.core.query.timeseries.ColumnDescription; -import com.basho.riak.client.core.query.timeseries.ConvertibleIterable; import com.basho.riak.client.core.query.timeseries.Row; import com.basho.riak.protobuf.RiakMessageCodes; import com.basho.riak.protobuf.RiakTsPB; -import com.google.protobuf.ByteString; import java.util.Collection; import java.util.List; @@ -21,8 +18,7 @@ */ public class StoreOperation extends DeferredEncodingOperation { - private final String tableName; - private final int rowCount; + private final Builder builder; private String queryInfoMessage; @@ -30,11 +26,19 @@ private StoreOperation(Builder builder) { super(RiakMessageCodes.MSG_TsPutReq, RiakMessageCodes.MSG_TsPutResp, - builder.reqBuilder, +// builder.reqBuilder, + null, RiakTsPB.TsPutResp.PARSER); - this.tableName = builder.reqBuilder.getTable().toStringUtf8(); - this.rowCount = builder.reqBuilder.getRowsCount(); + this.builder = builder; + +// this.tableName = builder.reqBuilder.getTable().toStringUtf8(); +// this.rowCount = builder.reqBuilder.getRowsCount(); + } + + @Override + protected RiakMessage createChannelMessage() { + return new RiakMessage(RiakMessageCodes.MSG_TsPutReq, builder); } @Override @@ -59,12 +63,15 @@ public String getQueryInfo() private String createQueryInfoMessage() { - return "INSERT <" + this.rowCount + " rows> into " + this.tableName; + return "INSERT into " + builder.tableName; } public static class Builder { - private final RiakTsPB.TsPutReq.Builder reqBuilder; + private final String tableName; + private Iterable rows; + +// private final RiakTsPB.TsPutReq.Builder reqBuilder; public Builder(String tableName) { @@ -73,22 +80,33 @@ public Builder(String tableName) throw new IllegalArgumentException("TableName can not be null or empty"); } - this.reqBuilder = RiakTsPB.TsPutReq.newBuilder(); - this.reqBuilder.setTable(ByteString.copyFromUtf8(tableName)); + this.tableName = tableName; +// this.reqBuilder = RiakTsPB.TsPutReq.newBuilder(); +// this.reqBuilder.setTable(ByteString.copyFromUtf8(tableName)); } public Builder withColumns(Collection columns) { - this.reqBuilder.addAllColumns(CollectionConverters.convertColumnDescriptionsToPb(columns)); - return this; +// this.reqBuilder.addAllColumns(CollectionConverters.convertColumnDescriptionsToPb(columns)); +// return this; + throw new UnsupportedOperationException(); } public Builder withRows(Iterable rows) { - this.reqBuilder.addAllRows(ConvertibleIterable.asIterablePbRow(rows)); +// this.reqBuilder.addAllRows(ConvertibleIterable.asIterablePbRow(rows)); + this.rows = rows; return this; } + public String getTableName() { + return tableName; + } + + public Iterable getRows() { + return rows; + } + public StoreOperation build() { return new StoreOperation(this); diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java b/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java index ccdaf7dd9..0d30dc6b7 100644 --- a/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java +++ b/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java @@ -194,7 +194,7 @@ public boolean getBoolean() return pbCell.getBooleanValue(); } - RiakTsPB.TsCell getPbCell() + public RiakTsPB.TsCell getPbCell() { return pbCell; } diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/Row.java b/src/main/java/com/basho/riak/client/core/query/timeseries/Row.java index 8a015c6e3..7a6c60b31 100644 --- a/src/main/java/com/basho/riak/client/core/query/timeseries/Row.java +++ b/src/main/java/com/basho/riak/client/core/query/timeseries/Row.java @@ -70,7 +70,7 @@ public List getCellsCopy() return cells; } - RiakTsPB.TsRow getPbRow() + public RiakTsPB.TsRow getPbRow() { return pbRow; } From fcc01e0c17ee7902680c99d2f48b8bef6dd20c81 Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Wed, 27 Jan 2016 16:29:27 +0200 Subject: [PATCH 13/52] Fix error propogation in case of Channel fails to switch to use Native(TTB) encoding --- src/main/java/com/basho/riak/client/core/RiakNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/basho/riak/client/core/RiakNode.java b/src/main/java/com/basho/riak/client/core/RiakNode.java index 98394ccd0..0f4a82c4d 100644 --- a/src/main/java/com/basho/riak/client/core/RiakNode.java +++ b/src/main/java/com/basho/riak/client/core/RiakNode.java @@ -793,7 +793,7 @@ else if (protocols.contains("TLSv1.1")) logger.warn("Channel '{}' was switched to use Native encoding: {}", c.hashCode(), c); } catch ( Exception e) { - throw new RuntimeException("Can't switch channel to use Native (TTB) encoding"); + throw new RuntimeException("Can't switch channel to use Native (TTB) encoding", e); } return c; From c4e5e5c3964ff5f790b0e4759cbb1d819ec844af Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 12 Apr 2016 11:00:05 -0400 Subject: [PATCH 14/52] Pull in change from another branch by @lehoff --- .../basho/riak/client/core/netty/RiakTTBChannelInitializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java index 5927549a9..fdf0b3fa3 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java @@ -19,7 +19,7 @@ public RiakTTBChannelInitializer(RiakResponseListener listener) { public void initChannel(SocketChannel ch) throws Exception { ChannelPipeline p = ch.pipeline(); - //p.addLast(new LoggingHandler(LogLevel.WARN)); + p.addLast(new LoggingHandler(LogLevel.ERROR)); super.initChannel(ch); p.replace(Constants.MESSAGE_CODEC, Constants.MESSAGE_CODEC, new RiakTTBCodec()); } From 5c09a183fcc79285b882173e0a51441f74e6270f Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Thu, 14 Apr 2016 11:42:42 -0400 Subject: [PATCH 15/52] Remove TTB Switch code, update PB dependency --- pom.xml | 2 +- .../riak/client/core/DefaultNodeManager.java | 39 ++++++++------- .../basho/riak/client/core/RiakMessage.java | 5 +- .../core/netty/RiakTTBChannelInitializer.java | 26 ---------- .../riak/client/core/netty/RiakTTBCodec.java | 41 ++-------------- .../ToggleTTBEncodingOperation.java | 47 ------------------- .../riak/client/core/netty/ITestTTBNode.java | 4 +- .../ts/ITestToggleTTBEncodingOperation.java | 27 ----------- 8 files changed, 28 insertions(+), 163 deletions(-) delete mode 100644 src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java delete mode 100644 src/main/java/com/basho/riak/client/core/operations/ToggleTTBEncodingOperation.java delete mode 100644 src/test/java/com/basho/riak/client/core/operations/itest/ts/ITestToggleTTBEncodingOperation.java diff --git a/pom.xml b/pom.xml index 00ca5cb7f..76cbc6eaa 100755 --- a/pom.xml +++ b/pom.xml @@ -368,7 +368,7 @@ com.basho.riak.protobuf riak-pb - 2.1.1.0 + 2.1.1.1-SNAPSHOT org.erlang.otp diff --git a/src/main/java/com/basho/riak/client/core/DefaultNodeManager.java b/src/main/java/com/basho/riak/client/core/DefaultNodeManager.java index 0dc159c4f..d9d45a634 100644 --- a/src/main/java/com/basho/riak/client/core/DefaultNodeManager.java +++ b/src/main/java/com/basho/riak/client/core/DefaultNodeManager.java @@ -21,24 +21,23 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; -import com.basho.riak.client.core.operations.ToggleTTBEncodingOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * - * The default {@link NodeManager} used by {@link RiakCluster} if none is + * The default {@link NodeManager} used by {@link RiakCluster} if none is * specified. - * - * This NodeManager round-robins through a list of {@link RiakNode}s and attempts - * to execute the operation passed to it. If a node reports that it is - * health checking it is removed from the list until it sends an update that it - * is again running. If the selected node cannot accept the operation because all - * connections are in use or it unable to make a new connection, the next node in - * the list is tried until either the operation is accepted or all nodes have - * been tried. If no nodes are able to accept the operation its setException() + * + * This NodeManager round-robins through a list of {@link RiakNode}s and attempts + * to execute the operation passed to it. If a node reports that it is + * health checking it is removed from the list until it sends an update that it + * is again running. If the selected node cannot accept the operation because all + * connections are in use or it unable to make a new connection, the next node in + * the list is tried until either the operation is accepted or all nodes have + * been tried. If no nodes are able to accept the operation its setException() * method is called with a {@link NoNodesAvailableException}. - * + * * @author Brian Roach * @since 2.0 */ @@ -49,7 +48,7 @@ public class DefaultNodeManager implements NodeManager, NodeStateListener private final AtomicInteger index = new AtomicInteger(); private final Logger logger = LoggerFactory.getLogger(DefaultNodeManager.class); private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); - + @Override public void init(List nodes) { @@ -75,7 +74,7 @@ public boolean executeOnNode(FutureOperation operation, RiakNode previousNode) { int startIndex = index.getAndIncrement(); int currentIndex = startIndex; - + do { if (healthy.get(Math.abs(currentIndex % healthy.size())).execute(operation)) @@ -91,7 +90,7 @@ else if (healthy.size() == 1) { executed = healthy.get(0).execute(operation); } - + return executed; } finally @@ -99,7 +98,7 @@ else if (healthy.size() == 1) lock.readLock().unlock(); } } - + @Override public void nodeStateChanged(RiakNode node, State state) { @@ -112,7 +111,7 @@ public void nodeStateChanged(RiakNode node, State state) if (unhealthy.remove(node)) { healthy.add(node); - logger.info("NodeManager moved node to healthy list; {}:{}", + logger.info("NodeManager moved node to healthy list; {}:{}", node.getRemoteAddress(), node.getPort()); } } @@ -128,7 +127,7 @@ public void nodeStateChanged(RiakNode node, State state) if (healthy.remove(node)) { unhealthy.add(node); - logger.info("NodeManager moved node to unhealthy list; {}:{}", + logger.info("NodeManager moved node to unhealthy list; {}:{}", node.getRemoteAddress(), node.getPort()); } } @@ -176,7 +175,7 @@ public void addNode(RiakNode newNode) { lock.writeLock().unlock(); } - + } @Override @@ -196,12 +195,12 @@ public boolean removeNode(RiakNode node) { lock.writeLock().unlock(); } - + if (removed) { node.removeStateListener(this); node.shutdown(); - logger.info("NodeManager removed and shutdown node; {}:{}", + logger.info("NodeManager removed and shutdown node; {}:{}", node.getRemoteAddress(), node.getPort()); } return removed; diff --git a/src/main/java/com/basho/riak/client/core/RiakMessage.java b/src/main/java/com/basho/riak/client/core/RiakMessage.java index 4392a1749..5160dcf88 100644 --- a/src/main/java/com/basho/riak/client/core/RiakMessage.java +++ b/src/main/java/com/basho/riak/client/core/RiakMessage.java @@ -26,7 +26,7 @@ public final class RiakMessage private final byte code; private byte[] data; private final Object dataObject; - + public RiakMessage(byte code, byte[] data) { this.code = code; @@ -51,7 +51,7 @@ public byte getCode() { return code; } - + public byte[] getData() { return data; @@ -79,6 +79,7 @@ public void setData(byte[] data) * @author Sergey Galkin * @since 2.4 */ + @SuppressWarnings("unchecked") public T getDataObject() { return (T) dataObject; diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java deleted file mode 100644 index fdf0b3fa3..000000000 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBChannelInitializer.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.basho.riak.client.core.netty; - -import com.basho.riak.client.core.RiakResponseListener; -import com.basho.riak.client.core.util.Constants; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.logging.LoggingHandler; - -/** - * @author Sergey Galkin - */ -public class RiakTTBChannelInitializer extends RiakChannelInitializer { - public RiakTTBChannelInitializer(RiakResponseListener listener) { - super(listener); - } - - @Override - public void initChannel(SocketChannel ch) throws Exception - { - ChannelPipeline p = ch.pipeline(); - p.addLast(new LoggingHandler(LogLevel.ERROR)); - super.initChannel(ch); - p.replace(Constants.MESSAGE_CODEC, Constants.MESSAGE_CODEC, new RiakTTBCodec()); - } -} diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java index f507c67ce..8bda93dad 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java @@ -45,8 +45,7 @@ public class RiakTTBCodec extends ByteToMessageCodec { private boolean isTTBMessage(byte code) { switch (code) { - case RiakMessageCodes.MSG_TsPutReq: - case RiakMessageCodes.MSG_TsGetReq: + case RiakMessageCodes.MSG_TsTtbMsg: return true; default: return false; @@ -56,9 +55,8 @@ private boolean isTTBMessage(byte code) { @Override protected void encode(ChannelHandlerContext ctx, RiakMessage msg, ByteBuf out) throws Exception { final OtpErlangTuple t; - final boolean isTtbOn = isTTBOnForChanel(ctx.channel()); - if (isTtbOn && isTTBMessage(msg.getCode())) { + if (isTTBMessage(msg.getCode())) { logger.trace("Encoding message {}, encoding '{}', is deferred: {}, channel '{}' {}", msg.getCode(), "TTB", !msg.isEncoded(), ctx.channel().hashCode(), ctx.channel()); switch (msg.getCode()) { @@ -151,7 +149,7 @@ private static int sizeOfIterable(Iterable values) { } int size = 0; - for(Object value : values) { + for(Object ignored : values) { ++size; } return size; @@ -334,8 +332,6 @@ private static RiakTsPB.TsCell ttbCellToPb(OtpErlangTuple c) { @Override protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { - final boolean isTTBOn = isTTBOnForChanel(ctx.channel()); - // Make sure we have 4 bytes if (in.readableBytes() >= 4) { in.markReaderIndex(); @@ -348,25 +344,6 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t return; } - if (!isTTBOn) { - // Decode as PB - - in.resetReaderIndex(); - pbCodec.decode(null, in, out); - - if (out.size() == 1) { - RiakMessage m = (RiakMessage) out.get(0); - switch ( m.getCode()) { - case RiakMessageCodes.MSG_ToggleEncodingResp: - final RiakPB.RpbToggleEncodingResp r = RiakPB.RpbToggleEncodingResp.PARSER.parseFrom(m.getData()); - updateTTBUsageForChannel(ctx.channel(), r.getUseNative()); - logger.debug("Native/TTB encoding for channel '{}' is set to {}: {}", ctx.channel().hashCode(), r.getUseNative(), ctx.channel()); - break; - } - } - return; - } - final byte[] array = new byte[length-1]; final int msgCode = in.readByte(); in.readBytes(array); @@ -446,16 +423,4 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t } } } - - private static final AttributeKey USE_TTB = AttributeKey.valueOf("useTTB"); - - private static boolean isTTBOnForChanel(Channel channel) { - final Boolean b = channel.attr(USE_TTB).get(); - return Boolean.TRUE.equals(b); - } - - private static boolean updateTTBUsageForChannel(Channel channel, boolean useTTB) { - channel.attr(USE_TTB).getAndSet(Boolean.valueOf(useTTB)); - return useTTB; - } } diff --git a/src/main/java/com/basho/riak/client/core/operations/ToggleTTBEncodingOperation.java b/src/main/java/com/basho/riak/client/core/operations/ToggleTTBEncodingOperation.java deleted file mode 100644 index 0490c9729..000000000 --- a/src/main/java/com/basho/riak/client/core/operations/ToggleTTBEncodingOperation.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.basho.riak.client.core.operations; - -import com.basho.riak.protobuf.RiakMessageCodes; -import com.basho.riak.protobuf.RiakPB; - -import java.util.List; - -/** - * Created by srg on 1/12/16. - */ -public class ToggleTTBEncodingOperation extends PBFutureOperation -{ - - public ToggleTTBEncodingOperation(boolean useTTBEncoding) { - super(RiakMessageCodes.MSG_ToggleEncodingReq, - RiakMessageCodes.MSG_ToggleEncodingResp, - RiakPB.RpbToggleEncodingResp.newBuilder().setUseNative(useTTBEncoding), - RiakPB.RpbToggleEncodingResp.PARSER); - } - - @Override - protected Response convert(List responses) { - // This is not a streaming op, there will only be one response - checkAndGetSingleResponse(responses); - return new ToggleTTBEncodingOperation.Response(responses.get(0).getUseNative()); - } - - @Override - public String getQueryInfo() { - return "SwitchToUSETTTBOperation"; - } - - /** - * Response returned from a SwitchToUSETTTBOperation - */ - public static class Response { - private final boolean useNativeEncoding; - - public Response(boolean useNativeEncoding) { - this.useNativeEncoding = useNativeEncoding; - } - - public boolean isUseNativeEncoding() { - return useNativeEncoding; - } - } -} diff --git a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java index ab8ce1d85..7dc4607ff 100644 --- a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java +++ b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java @@ -62,7 +62,6 @@ public void setup() throws UnknownHostException, ExecutionException, Interrupted riakNode = new RiakNode.Builder() - .useTTB() .withRemotePort(testRiakPort) // .withMaxConnections(1) // .withMinConnections(1) @@ -140,7 +139,8 @@ public void storeDatatoNoneExistentTable() throws InterruptedException, Executio } } - private > T execute(O operation) throws ExecutionException, InterruptedException { + private > T execute(O operation) throws ExecutionException, InterruptedException + { riakNode.execute((FutureOperation) operation); return (T)((FutureOperation) operation).get(); } diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ts/ITestToggleTTBEncodingOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ts/ITestToggleTTBEncodingOperation.java deleted file mode 100644 index 508cfe531..000000000 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ts/ITestToggleTTBEncodingOperation.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.basho.riak.client.core.operations.itest.ts; - -import com.basho.riak.client.core.RiakFuture; -import com.basho.riak.client.core.operations.ToggleTTBEncodingOperation; -import com.basho.riak.client.core.operations.itest.ITestBase; -import org.junit.Test; - -import java.util.concurrent.ExecutionException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Created by srg on 1/12/16. - */ -public class ITestToggleTTBEncodingOperation extends ITestBase { - @Test - public void testSingleDelete() throws ExecutionException, InterruptedException - { - final boolean useTTB = false; - final ToggleTTBEncodingOperation op = new ToggleTTBEncodingOperation(useTTB); - final RiakFuture future = cluster.execute(op); - - final ToggleTTBEncodingOperation.Response response = future.get(); - assertEquals(useTTB, response.isUseNativeEncoding()); - } -} From f743ff35f65f385d312b3b41478e055ab7c99789 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Mon, 18 Apr 2016 12:03:46 -0400 Subject: [PATCH 16/52] Add new TTB codec boilerplate --- .../client/core/codec/TermToBinaryCodec.java | 123 ++++++++++++++++++ .../core/operations/TTBFutureOperation.java | 61 +++++++++ 2 files changed, 184 insertions(+) create mode 100644 src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java create mode 100644 src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java new file mode 100644 index 000000000..0032615d0 --- /dev/null +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -0,0 +1,123 @@ +package com.basho.riak.client.core.codec; + + +import com.basho.riak.client.core.query.timeseries.Cell; +import com.basho.riak.client.core.query.timeseries.QueryResult; +import com.basho.riak.client.core.query.timeseries.Row; +import com.ericsson.otp.erlang.OtpErlangAtom; +import com.ericsson.otp.erlang.OtpErlangList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TermToBinaryCodec +{ + private final static Logger logger = LoggerFactory.getLogger(TermToBinaryCodec.class); + + private static final OtpErlangAtom undefined = new OtpErlangAtom("undefined"); + private static final OtpErlangAtom tsinterpolation = new OtpErlangAtom("tsinterpolation"); + + private static final OtpErlangAtom _false = new OtpErlangAtom("false"); + private static final OtpErlangList EMPTY_ERLANG_LIST = new OtpErlangList(); + + private static class Messages + { + public static final OtpErlangAtom tsGetReq = new OtpErlangAtom("tsgetreq"); + public static final OtpErlangAtom tsGetResp = new OtpErlangAtom("tsgetresp"); + + public static final OtpErlangAtom tsQueryReq = new OtpErlangAtom("tsqueryreq"); + public static final OtpErlangAtom tsQueryResp = new OtpErlangAtom("tsqueryresp"); + + public static final OtpErlangAtom tsPutReq = new OtpErlangAtom("tsputreq"); + public static final OtpErlangAtom tsPutResp = new OtpErlangAtom("tsputresp"); + + public static final OtpErlangAtom tsDelReq = new OtpErlangAtom("tsdelreq"); + public static final OtpErlangAtom tsDelResp = new OtpErlangAtom("tsdelresp"); + } + + /* + Get + + Msg0 = #tsgetreq{table = iolist_to_binary(Table), + key = Key}; + Msg1 = Msg0#tsgetreq{timeout = proplists:get_value(timeout, Options)}, + Msg = {Msg1, {msgopts, Options}}, + + */ + + public static byte[] encodeTsGetRequest(String tableName, Iterable keyValues, Long timeout) + { + // fill me in + return null; + } + + public static QueryResult decodeTsGetResponse(byte[] response) + { + // fill me in + return null; + } + + /* + Query + + Content = #tsinterpolation{ + base = iolist_to_binary(QueryText), + interpolations = serialize_interpolations(Interpolations)}, + Msg0 = #tsqueryreq{query = Content}, + Msg1 = Msg0#tsqueryreq{cover_context = Cover}, + Msg = {Msg1, {msgopts, Options}}, + */ + + public static byte[] encodeTsQueryRequest(String queryText) + { + // fill me in + return null; + } + + public static QueryResult decodeTsQueryResponse(byte[] response) + { + // fill me in + return null; + } + + /* + Put + Measurements = [{...},{...}]. + Message = #tsputreq{table = iolist_to_binary(TableName), + columns = [], + rows = Measurements}; + Msg = {Message, {msgopts, Options}}, + */ + + public static byte[] encodeTsPutRequest(String tableName, Iterable rows) + { + // fill me in + return null; + } + + public static Void decodeTsPutResponse(byte[] response) + { + // Do we return anything in TTB? + return null; + } + + /* + Delete + + Message = #tsdelreq{table = iolist_to_binary(Table), + key = riak_pb_ts_codec:encode_cells_non_strict(Key), + vclock = proplists:get_value(vclock, Options), + timeout = proplists:get_value(timeout, Options)}, + */ + + public static byte[] encodeTsDeleteRequest(String tableName, Iterable keyValues, byte[] vclock, Long timeout) + { + // fill me in + return null; + } + + public static Void decodeTsDeleteResponse(byte[] response) + { + // Do we return anything in TTB? + return null; + } +} diff --git a/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java b/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java new file mode 100644 index 000000000..0f1ea5ce1 --- /dev/null +++ b/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java @@ -0,0 +1,61 @@ +package com.basho.riak.client.core.operations; + +import com.basho.riak.client.core.FutureOperation; +import com.basho.riak.client.core.RiakMessage; +import com.basho.riak.protobuf.RiakMessageCodes; + +/** + * An abstract TTB operation that introduces generic encoding/decoding + * + * @author Alex Moore + * @param The type the operation returns + * @param The protocol type returned + * @param Query info type + + * @since 2.0.6 + */ + +public abstract class TTBFutureOperation extends FutureOperation +{ + protected final byte reqMessageCode = RiakMessageCodes.MSG_TsTtbMsg; + protected final byte respMessageCode = RiakMessageCodes.MSG_TsTtbMsg; + private TTBEncoder requestBuilder; + private TTBParser responseParser; + + protected TTBFutureOperation(TTBEncoder requestBuilder, TTBParser responseParser) + { + this.requestBuilder = requestBuilder; + this.responseParser = responseParser; + } + + @Override + protected RiakMessage createChannelMessage() + { + return new RiakMessage(reqMessageCode, requestBuilder.build()); + } + + @Override + protected U decode(RiakMessage rawMessage) + { + Operations.checkMessageType(rawMessage, respMessageCode); + + byte[] data = rawMessage.getData(); + + if (data.length == 0) // not found + { + return null; + } + + return responseParser.parseFrom(data); + } + + public interface TTBEncoder + { + byte[] build(); + } + + public interface TTBParser + { + U parseFrom(byte[] data); + } +} From 6aa4237e5c5a2de7676cb09ed7e730d5836c2b92 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Mon, 18 Apr 2016 14:56:52 -0400 Subject: [PATCH 17/52] Stubbing out Store operation --- .../riak/client/core/FutureOperation.java | 10 +++++ .../core/operations/PBFutureOperation.java | 11 ----- .../core/operations/TTBFutureOperation.java | 8 ++-- .../core/operations/ts/StoreOperation.java | 40 +++++-------------- .../core/operations/ts/TTBConverters.java | 38 ++++++++++++++++++ 5 files changed, 62 insertions(+), 45 deletions(-) create mode 100644 src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java diff --git a/src/main/java/com/basho/riak/client/core/FutureOperation.java b/src/main/java/com/basho/riak/client/core/FutureOperation.java index a3c21919f..95528d4f0 100644 --- a/src/main/java/com/basho/riak/client/core/FutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/FutureOperation.java @@ -373,6 +373,16 @@ public final void await(long timeout, TimeUnit unit) throws InterruptedException latch.await(timeout, unit); } + protected U checkAndGetSingleResponse(List responses) + { + if (responses.size() > 1) + { + LoggerFactory.getLogger(this.getClass()).error("Received {} responses when only one was expected.", + responses.size()); + } + + return responses.get(0); + } private void stateCheck(State... allowedStates) { diff --git a/src/main/java/com/basho/riak/client/core/operations/PBFutureOperation.java b/src/main/java/com/basho/riak/client/core/operations/PBFutureOperation.java index d6e8ffa58..d19acca69 100644 --- a/src/main/java/com/basho/riak/client/core/operations/PBFutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/PBFutureOperation.java @@ -65,15 +65,4 @@ protected U decode(RiakMessage rawMessage) { throw new IllegalArgumentException("Invalid message received", e); } } - - protected U checkAndGetSingleResponse(List responses) - { - if (responses.size() > 1) - { - LoggerFactory.getLogger(this.getClass()).error("Received {} responses when only one was expected.", - responses.size()); - } - - return responses.get(0); - } } diff --git a/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java b/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java index 0f1ea5ce1..cdd2444c5 100644 --- a/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java @@ -2,6 +2,7 @@ import com.basho.riak.client.core.FutureOperation; import com.basho.riak.client.core.RiakMessage; +import com.basho.riak.client.core.operations.ts.*; import com.basho.riak.protobuf.RiakMessageCodes; /** @@ -19,8 +20,8 @@ public abstract class TTBFutureOperation extends FutureOperation responseParser; + protected final TTBEncoder requestBuilder; + protected final TTBParser responseParser; protected TTBFutureOperation(TTBEncoder requestBuilder, TTBParser responseParser) { @@ -49,8 +50,9 @@ protected U decode(RiakMessage rawMessage) return responseParser.parseFrom(data); } - public interface TTBEncoder + public interface TTBEncoder { + byte[] build(); } diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java index 9bb93c840..71df9dc40 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java @@ -1,10 +1,8 @@ package com.basho.riak.client.core.operations.ts; -import com.basho.riak.client.core.RiakMessage; +import com.basho.riak.client.core.operations.TTBFutureOperation; import com.basho.riak.client.core.query.timeseries.ColumnDescription; import com.basho.riak.client.core.query.timeseries.Row; -import com.basho.riak.protobuf.RiakMessageCodes; -import com.basho.riak.protobuf.RiakTsPB; import java.util.Collection; import java.util.List; @@ -16,37 +14,22 @@ * @author Sergey Galkin * @since 2.0.3 */ -public class StoreOperation extends DeferredEncodingOperation +public class StoreOperation extends TTBFutureOperation { private final Builder builder; private String queryInfoMessage; - - private StoreOperation(Builder builder) + private StoreOperation(final Builder builder) { - super(RiakMessageCodes.MSG_TsPutReq, - RiakMessageCodes.MSG_TsPutResp, -// builder.reqBuilder, - null, - RiakTsPB.TsPutResp.PARSER); - + super(new TTBConverters.StoreEncoder(builder), new TTBConverters.VoidDecoder()); this.builder = builder; - -// this.tableName = builder.reqBuilder.getTable().toStringUtf8(); -// this.rowCount = builder.reqBuilder.getRowsCount(); - } - - @Override - protected RiakMessage createChannelMessage() { - return new RiakMessage(RiakMessageCodes.MSG_TsPutReq, builder); } @Override - protected Void convert(List responses) + protected Void convert(List responses) { // This is not a streaming op, there will only be one response checkAndGetSingleResponse(responses); - return null; } @@ -71,8 +54,6 @@ public static class Builder private final String tableName; private Iterable rows; -// private final RiakTsPB.TsPutReq.Builder reqBuilder; - public Builder(String tableName) { if (tableName == null || tableName.length() == 0) @@ -81,29 +62,26 @@ public Builder(String tableName) } this.tableName = tableName; -// this.reqBuilder = RiakTsPB.TsPutReq.newBuilder(); -// this.reqBuilder.setTable(ByteString.copyFromUtf8(tableName)); } public Builder withColumns(Collection columns) { -// this.reqBuilder.addAllColumns(CollectionConverters.convertColumnDescriptionsToPb(columns)); -// return this; throw new UnsupportedOperationException(); } public Builder withRows(Iterable rows) { -// this.reqBuilder.addAllRows(ConvertibleIterable.asIterablePbRow(rows)); this.rows = rows; return this; } - public String getTableName() { + public String getTableName() + { return tableName; } - public Iterable getRows() { + public Iterable getRows() + { return rows; } diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java new file mode 100644 index 000000000..95b982bd6 --- /dev/null +++ b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java @@ -0,0 +1,38 @@ +package com.basho.riak.client.core.operations.ts; + +import com.basho.riak.client.core.codec.TermToBinaryCodec; +import com.basho.riak.client.core.operations.TTBFutureOperation; + +class TTBConverters +{ + static class StoreEncoder implements TTBFutureOperation.TTBEncoder + { + private byte[] message = null; + private final StoreOperation.Builder builder; + + StoreEncoder(StoreOperation.Builder builder) + { + this.builder = builder; + } + + @Override + public byte[] build() + { + if (message == null) + { + message = TermToBinaryCodec.encodeTsPutRequest(builder.getTableName(), builder.getRows()); + } + return message; + } + } + + static class VoidDecoder implements TTBFutureOperation.TTBParser + { + @Override + public Void parseFrom(byte[] data) + { + return null; + } + } + +} From d3a2d938389732dc7a6004b79f3f7748c69671d9 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Mon, 18 Apr 2016 12:50:16 -0700 Subject: [PATCH 18/52] Encoding of tsgetreq to byte[] is complete. --- .../client/core/codec/TermToBinaryCodec.java | 34 ++++++++++++------- .../client/core/query/timeseries/Cell.java | 25 ++++++++++++++ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index 0032615d0..b432e3890 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -1,11 +1,13 @@ package com.basho.riak.client.core.codec; - import com.basho.riak.client.core.query.timeseries.Cell; import com.basho.riak.client.core.query.timeseries.QueryResult; import com.basho.riak.client.core.query.timeseries.Row; import com.ericsson.otp.erlang.OtpErlangAtom; import com.ericsson.otp.erlang.OtpErlangList; +import com.ericsson.otp.erlang.OtpExternal; +import com.ericsson.otp.erlang.OtpOutputStream; +import java.util.Collection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,22 +34,30 @@ private static class Messages public static final OtpErlangAtom tsDelReq = new OtpErlangAtom("tsdelreq"); public static final OtpErlangAtom tsDelResp = new OtpErlangAtom("tsdelresp"); + + public static final OtpErlangAtom tsUndefined = new OtpErlangAtom("undefined"); } - /* - Get + public static byte[] encodeTsGetRequest(String tableName, Collection keyValues, Long timeout) + { + final OtpOutputStream os = new OtpOutputStream(); + os.write(OtpExternal.versionTag); - Msg0 = #tsgetreq{table = iolist_to_binary(Table), - key = Key}; - Msg1 = Msg0#tsgetreq{timeout = proplists:get_value(timeout, Options)}, - Msg = {Msg1, {msgopts, Options}}, + // NB: TsGetReq is a 4-tuple: tsgetreq, tableName, [key values], timeout + os.write_tuple_head(4); + os.write_any(Messages.tsGetReq); + os.write_string(tableName); - */ + os.write_list_head(keyValues.size()); + for (Cell k : keyValues) { + os.write_any(k.getErlangObject()); + } + os.write_nil(); // NB: finishes the list - public static byte[] encodeTsGetRequest(String tableName, Iterable keyValues, Long timeout) - { - // fill me in - return null; + // TODO GH-611 timeout? + os.write_any(Messages.tsUndefined); + + return os.toByteArray(); } public static QueryResult decodeTsGetResponse(byte[] response) diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java b/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java index 0d30dc6b7..80d08ac1f 100644 --- a/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java +++ b/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java @@ -2,6 +2,11 @@ import com.basho.riak.client.core.util.BinaryValue; import com.basho.riak.protobuf.RiakTsPB; +import com.ericsson.otp.erlang.OtpErlangBinary; +import com.ericsson.otp.erlang.OtpErlangBoolean; +import com.ericsson.otp.erlang.OtpErlangDouble; +import com.ericsson.otp.erlang.OtpErlangLong; +import com.ericsson.otp.erlang.OtpErlangObject; import com.google.protobuf.ByteString; import java.util.Calendar; @@ -199,6 +204,26 @@ public RiakTsPB.TsCell getPbCell() return pbCell; } + public OtpErlangObject getErlangObject() { + if (pbCell.hasVarcharValue()) { + return new OtpErlangBinary(pbCell.getVarcharValue().toByteArray()); + } + if (pbCell.hasSint64Value()) { + return new OtpErlangLong(pbCell.getSint64Value()); + } + if (pbCell.hasTimestampValue()) { + return new OtpErlangLong(pbCell.getTimestampValue()); + } + if (pbCell.hasBooleanValue()) { + return new OtpErlangBoolean(pbCell.getBooleanValue()); + } + if (pbCell.hasDoubleValue()) { + return new OtpErlangDouble(pbCell.getDoubleValue()); + } + // TODO GH-611 throw exception? + return null; + } + @Override public String toString() { From c061b2595c8942f6e2de3b599c9673a53e100a29 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Mon, 18 Apr 2016 14:19:19 -0700 Subject: [PATCH 19/52] Encode query and put TTB requests. --- .../client/core/codec/TermToBinaryCodec.java | 83 ++++++++++++------- .../core/operations/ts/StoreOperation.java | 10 +-- .../core/operations/ts/TTBConverters.java | 5 +- 3 files changed, 62 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index b432e3890..12db0c769 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -28,6 +28,7 @@ private static class Messages public static final OtpErlangAtom tsQueryReq = new OtpErlangAtom("tsqueryreq"); public static final OtpErlangAtom tsQueryResp = new OtpErlangAtom("tsqueryresp"); + public static final OtpErlangAtom tsInterpolation = new OtpErlangAtom("tsinterpolation"); public static final OtpErlangAtom tsPutReq = new OtpErlangAtom("tsputreq"); public static final OtpErlangAtom tsPutResp = new OtpErlangAtom("tsputresp"); @@ -38,10 +39,10 @@ private static class Messages public static final OtpErlangAtom tsUndefined = new OtpErlangAtom("undefined"); } - public static byte[] encodeTsGetRequest(String tableName, Collection keyValues, Long timeout) + public static OtpOutputStream encodeTsGetRequest(String tableName, Collection keyValues, Long timeout) { final OtpOutputStream os = new OtpOutputStream(); - os.write(OtpExternal.versionTag); + os.write(OtpExternal.versionTag); // NB: this is the reqired 0x83 (131) value // NB: TsGetReq is a 4-tuple: tsgetreq, tableName, [key values], timeout os.write_tuple_head(4); @@ -57,7 +58,7 @@ public static byte[] encodeTsGetRequest(String tableName, Collection keyVa // TODO GH-611 timeout? os.write_any(Messages.tsUndefined); - return os.toByteArray(); + return os; } public static QueryResult decodeTsGetResponse(byte[] response) @@ -66,21 +67,32 @@ public static QueryResult decodeTsGetResponse(byte[] response) return null; } - /* - Query - - Content = #tsinterpolation{ - base = iolist_to_binary(QueryText), - interpolations = serialize_interpolations(Interpolations)}, - Msg0 = #tsqueryreq{query = Content}, - Msg1 = Msg0#tsqueryreq{cover_context = Cover}, - Msg = {Msg1, {msgopts, Options}}, - */ - - public static byte[] encodeTsQueryRequest(String queryText) + public static OtpOutputStream encodeTsQueryRequest(String queryText) { - // fill me in - return null; + final OtpOutputStream os = new OtpOutputStream(); + os.write(OtpExternal.versionTag); // NB: this is the reqired 0x83 (131) value + + // TsQueryReq is a 4-tuple: {'tsqueryreq', TsInt, boolIsStreaming, bytesCoverContext} + os.write_tuple_head(4); + os.write_any(Messages.tsQueryReq); + + // TsInterpolation is a 3-tuple + // {'tsinterpolation', query, []} empty list is interpolations + os.write_tuple_head(3); + os.write_any(Messages.tsInterpolation); + os.write_string(queryText); + // interpolations is an empty list + os.write_list_head(0); + os.write_nil(); + + // streaming is false for now + os.write_boolean(false); + + // cover_context is an empty list + os.write_list_head(0); + os.write_nil(); + + return os; } public static QueryResult decodeTsQueryResponse(byte[] response) @@ -89,19 +101,32 @@ public static QueryResult decodeTsQueryResponse(byte[] response) return null; } - /* - Put - Measurements = [{...},{...}]. - Message = #tsputreq{table = iolist_to_binary(TableName), - columns = [], - rows = Measurements}; - Msg = {Message, {msgopts, Options}}, - */ - - public static byte[] encodeTsPutRequest(String tableName, Iterable rows) + public static OtpOutputStream encodeTsPutRequest(String tableName, Collection rows) { - // fill me in - return null; + final OtpOutputStream os = new OtpOutputStream(); + os.write(OtpExternal.versionTag); // NB: this is the reqired 0x83 (131) value + + // TsPutReq is a 4-tuple: {'tsputreq', tableName, [], [rows]} + // columns is empte + os.write_tuple_head(4); + os.write_any(Messages.tsPutReq); + os.write_string(tableName); + // columns is an empty list + os.write_list_head(0); + os.write_nil(); + + // write rows + // each row is a tuple of cells + os.write_list_head(rows.size()); + for (Row r : rows) { + os.write_tuple_head(r.getCellsCount()); + for (Cell c : r) { + os.write_any(c.getErlangObject()); + } + } + os.write_nil(); + + return os; } public static Void decodeTsPutResponse(byte[] response) diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java index 71df9dc40..83f3acc4f 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java @@ -52,7 +52,7 @@ private String createQueryInfoMessage() public static class Builder { private final String tableName; - private Iterable rows; + private Collection rows; public Builder(String tableName) { @@ -69,7 +69,7 @@ public Builder withColumns(Collection columns) throw new UnsupportedOperationException(); } - public Builder withRows(Iterable rows) + public Builder withRows(Collection rows) { this.rows = rows; return this; @@ -80,7 +80,7 @@ public String getTableName() return tableName; } - public Iterable getRows() + public Collection getRows() { return rows; } @@ -90,6 +90,4 @@ public StoreOperation build() return new StoreOperation(this); } } -} - - +} \ No newline at end of file diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java index 95b982bd6..b42c3fd69 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java @@ -2,6 +2,7 @@ import com.basho.riak.client.core.codec.TermToBinaryCodec; import com.basho.riak.client.core.operations.TTBFutureOperation; +import com.ericsson.otp.erlang.OtpOutputStream; class TTBConverters { @@ -20,7 +21,9 @@ public byte[] build() { if (message == null) { - message = TermToBinaryCodec.encodeTsPutRequest(builder.getTableName(), builder.getRows()); + OtpOutputStream os = TermToBinaryCodec.encodeTsPutRequest(builder.getTableName(), builder.getRows()); + // TODO GH-611 should the output stream or base type be returned? + message = os.toByteArray(); } return message; } From e01e3fecd2186bd6312c05e84b95f9f35132c4a6 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 19 Apr 2016 11:08:53 -0400 Subject: [PATCH 20/52] Stub out other TS operations to use TTB --- .../client/core/codec/TermToBinaryCodec.java | 4 +- .../core/operations/TTBFutureOperation.java | 19 +-- .../core/operations/ts/DeleteOperation.java | 33 ++-- .../core/operations/ts/FetchOperation.java | 38 +++-- .../core/operations/ts/QueryOperation.java | 22 +-- .../core/operations/ts/StoreOperation.java | 6 +- .../core/operations/ts/TTBConverters.java | 93 ++++++++++- .../riak/client/core/netty/ITestTTBNode.java | 153 ------------------ 8 files changed, 151 insertions(+), 217 deletions(-) delete mode 100644 src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index 12db0c769..c717dc234 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -39,7 +39,7 @@ private static class Messages public static final OtpErlangAtom tsUndefined = new OtpErlangAtom("undefined"); } - public static OtpOutputStream encodeTsGetRequest(String tableName, Collection keyValues, Long timeout) + public static OtpOutputStream encodeTsGetRequest(String tableName, Collection keyValues, int timeout) { final OtpOutputStream os = new OtpOutputStream(); os.write(OtpExternal.versionTag); // NB: this is the reqired 0x83 (131) value @@ -144,7 +144,7 @@ public static Void decodeTsPutResponse(byte[] response) timeout = proplists:get_value(timeout, Options)}, */ - public static byte[] encodeTsDeleteRequest(String tableName, Iterable keyValues, byte[] vclock, Long timeout) + public static OtpOutputStream encodeTsDeleteRequest(String tableName, Iterable keyValues, byte[] vclock, int timeout) { // fill me in return null; diff --git a/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java b/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java index cdd2444c5..34dc859e8 100644 --- a/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java @@ -2,7 +2,6 @@ import com.basho.riak.client.core.FutureOperation; import com.basho.riak.client.core.RiakMessage; -import com.basho.riak.client.core.operations.ts.*; import com.basho.riak.protobuf.RiakMessageCodes; /** @@ -10,20 +9,19 @@ * * @author Alex Moore * @param The type the operation returns - * @param The protocol type returned * @param Query info type * @since 2.0.6 */ -public abstract class TTBFutureOperation extends FutureOperation +public abstract class TTBFutureOperation extends FutureOperation { protected final byte reqMessageCode = RiakMessageCodes.MSG_TsTtbMsg; protected final byte respMessageCode = RiakMessageCodes.MSG_TsTtbMsg; protected final TTBEncoder requestBuilder; - protected final TTBParser responseParser; + protected final TTBParser responseParser; - protected TTBFutureOperation(TTBEncoder requestBuilder, TTBParser responseParser) + protected TTBFutureOperation(TTBEncoder requestBuilder, TTBParser responseParser) { this.requestBuilder = requestBuilder; this.responseParser = responseParser; @@ -36,7 +34,7 @@ protected RiakMessage createChannelMessage() } @Override - protected U decode(RiakMessage rawMessage) + protected byte[] decode(RiakMessage rawMessage) { Operations.checkMessageType(rawMessage, respMessageCode); @@ -47,17 +45,16 @@ protected U decode(RiakMessage rawMessage) return null; } - return responseParser.parseFrom(data); + return data; } - public interface TTBEncoder + public interface TTBEncoder { - byte[] build(); } - public interface TTBParser + public interface TTBParser { - U parseFrom(byte[] data); + T parseFrom(byte[] data); } } diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/DeleteOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/DeleteOperation.java index 4264610ee..7108d3bef 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/DeleteOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/DeleteOperation.java @@ -1,6 +1,7 @@ package com.basho.riak.client.core.operations.ts; import com.basho.riak.client.core.operations.PBFutureOperation; +import com.basho.riak.client.core.operations.TTBFutureOperation; import com.basho.riak.client.core.query.timeseries.Cell; import com.basho.riak.client.core.query.timeseries.ConvertibleIterable; import com.basho.riak.protobuf.RiakMessageCodes; @@ -16,23 +17,20 @@ * @author Sergey Galkin * @since 2.0.3 */ -public class DeleteOperation extends PBFutureOperation +public class DeleteOperation extends TTBFutureOperation { private final Builder builder; private String queryInfoMessage; private DeleteOperation(Builder builder) { - super(RiakMessageCodes.MSG_TsDelReq, - RiakMessageCodes.MSG_TsDelResp, - builder.reqBuilder, - RiakTsPB.TsDelResp.PARSER); + super(new TTBConverters.DeleteEncoder(builder), new TTBConverters.VoidDecoder()); this.builder = builder; } @Override - protected Void convert(List responses) + protected Void convert(List responses) { // This is not a streaming op, there will only be one response checkAndGetSingleResponse(responses); @@ -73,8 +71,7 @@ public static class Builder { private final String tableName; private final Iterable keyValues; - - private final RiakTsPB.TsDelReq.Builder reqBuilder = RiakTsPB.TsDelReq.newBuilder(); + private int timeout = 0; public Builder(String tableName, Iterable keyValues) { @@ -88,9 +85,6 @@ public Builder(String tableName, Iterable keyValues) throw new IllegalArgumentException("Key Values cannot be null or an empty."); } - this.reqBuilder.setTable(ByteString.copyFromUtf8(tableName)); - this.reqBuilder.addAllKey(ConvertibleIterable.asIterablePbCell(keyValues)); - this.tableName = tableName; this.keyValues = keyValues; } @@ -101,10 +95,25 @@ public Builder withTimeout(int timeout) { throw new IllegalArgumentException("Timeout must be positive, or 0 for no timeout."); } - this.reqBuilder.setTimeout(timeout); + this.timeout = timeout; return this; } + public String getTableName() + { + return tableName; + } + + public Iterable getKeyValues() + { + return keyValues; + } + + public int getTimeout() + { + return timeout; + } + public DeleteOperation build() { return new DeleteOperation(this); diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java index 184886c6e..9d4b260ae 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java @@ -1,6 +1,7 @@ package com.basho.riak.client.core.operations.ts; import com.basho.riak.client.core.RiakMessage; +import com.basho.riak.client.core.operations.TTBFutureOperation; import com.basho.riak.client.core.query.timeseries.*; import com.basho.riak.protobuf.RiakTsPB; import com.basho.riak.protobuf.RiakMessageCodes; @@ -14,20 +15,15 @@ * @author Sergey Galkin * @since 2.0.3 */ -public class FetchOperation extends DeferredEncodingOperation +public class FetchOperation extends TTBFutureOperation { private final Builder builder; private String queryInfoMessage; private FetchOperation(Builder builder) { - super(RiakMessageCodes.MSG_TsGetReq, - RiakMessageCodes.MSG_TsGetResp, -// builder.reqBuilder, - null, - RiakTsPB.TsGetResp.PARSER); - - this.builder = builder; + super(new TTBConverters.FetchEncoder(builder), new TTBConverters.QueryResultDecoder()); + this.builder = builder; } @Override @@ -36,12 +32,11 @@ protected RiakMessage createChannelMessage() { } @Override - protected QueryResult convert(List responses) + protected QueryResult convert(List responses) { // This is not a streaming op, there will only be one response - final RiakTsPB.TsGetResp response = checkAndGetSingleResponse(responses); - - return PbResultFactory.convertPbGetResp(response); + final byte[] bytes = checkAndGetSingleResponse(responses); + return this.responseParser.parseFrom(bytes); } @Override @@ -76,8 +71,7 @@ public static class Builder { private final String tableName; private final Iterable keyValues; - - //private final RiakTsPB.TsGetReq.Builder reqBuilder = RiakTsPB.TsGetReq.newBuilder(); + private int timeout = 0; public Builder(String tableName, Iterable keyValues) { @@ -91,16 +85,13 @@ public Builder(String tableName, Iterable keyValues) throw new IllegalArgumentException("Key Values cannot be null or an empty."); } -// this.reqBuilder.setTable(ByteString.copyFromUtf8(tableName)); -// this.reqBuilder.addAllKey(ConvertibleIterable.asIterablePbCell(keyValues)); - this.tableName = tableName; this.keyValues = keyValues; } public Builder withTimeout(int timeout) { -// this.reqBuilder.setTimeout(timeout); + this.timeout = timeout; return this; } @@ -109,12 +100,19 @@ public FetchOperation build() return new FetchOperation(this); } - public String getTableName() { + public String getTableName() + { return tableName; } - public Iterable getKeyValues() { + public Iterable getKeyValues() + { return keyValues; } + + public int getTimeout() + { + return timeout; + } } } diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/QueryOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/QueryOperation.java index 37406131f..9b83a8281 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/QueryOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/QueryOperation.java @@ -1,9 +1,7 @@ package com.basho.riak.client.core.operations.ts; -import com.basho.riak.client.core.operations.PBFutureOperation; -import com.basho.riak.client.core.query.timeseries.PbResultFactory; +import com.basho.riak.client.core.operations.TTBFutureOperation; import com.basho.riak.client.core.query.timeseries.QueryResult; -import com.basho.riak.protobuf.RiakMessageCodes; import com.basho.riak.protobuf.RiakTsPB; import com.google.protobuf.ByteString; @@ -16,26 +14,23 @@ * @author Sergey Galkin * @since 2.0.3 */ -public class QueryOperation extends PBFutureOperation +public class QueryOperation extends TTBFutureOperation { private final String queryText; private QueryOperation(Builder builder) { - super(RiakMessageCodes.MSG_TsQueryReq, - RiakMessageCodes.MSG_TsQueryResp, - RiakTsPB.TsQueryReq.newBuilder().setQuery(builder.interpolationBuilder), - RiakTsPB.TsQueryResp.PARSER); + super(new TTBConverters.QueryEncoder(builder), new TTBConverters.QueryResultDecoder()); this.queryText = builder.queryText; } @Override - protected QueryResult convert(List responses) + protected QueryResult convert(List responses) { // This is not a streaming op, there will only be one response - final RiakTsPB.TsQueryResp response = checkAndGetSingleResponse(responses); - return PbResultFactory.convertPbQueryResp(response); + final byte[] response = checkAndGetSingleResponse(responses); + return this.responseParser.parseFrom(response); } @Override @@ -59,6 +54,11 @@ public Builder(String queryText) this.interpolationBuilder.setBase(ByteString.copyFromUtf8(queryText)); } + public String getQueryText() + { + return queryText; + } + public QueryOperation build() { return new QueryOperation(this); diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java index 83f3acc4f..d5e9c410e 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/StoreOperation.java @@ -14,7 +14,7 @@ * @author Sergey Galkin * @since 2.0.3 */ -public class StoreOperation extends TTBFutureOperation +public class StoreOperation extends TTBFutureOperation { private final Builder builder; private String queryInfoMessage; @@ -26,7 +26,7 @@ private StoreOperation(final Builder builder) } @Override - protected Void convert(List responses) + protected Void convert(List responses) { // This is not a streaming op, there will only be one response checkAndGetSingleResponse(responses); @@ -90,4 +90,4 @@ public StoreOperation build() return new StoreOperation(this); } } -} \ No newline at end of file +} diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java index b42c3fd69..3c5f12878 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java @@ -1,27 +1,35 @@ package com.basho.riak.client.core.operations.ts; +import com.basho.riak.client.api.commands.timeseries.Query; import com.basho.riak.client.core.codec.TermToBinaryCodec; import com.basho.riak.client.core.operations.TTBFutureOperation; +import com.basho.riak.client.core.query.timeseries.Cell; +import com.basho.riak.client.core.query.timeseries.QueryResult; import com.ericsson.otp.erlang.OtpOutputStream; +import java.util.ArrayList; +import java.util.LinkedList; + class TTBConverters { - static class StoreEncoder implements TTBFutureOperation.TTBEncoder + private static abstract class MemoizingEncoder implements TTBFutureOperation.TTBEncoder { - private byte[] message = null; - private final StoreOperation.Builder builder; + protected byte[] message = null; + protected final T builder; - StoreEncoder(StoreOperation.Builder builder) + MemoizingEncoder(T builder) { this.builder = builder; } + abstract OtpOutputStream buildMessage(); + @Override public byte[] build() { if (message == null) { - OtpOutputStream os = TermToBinaryCodec.encodeTsPutRequest(builder.getTableName(), builder.getRows()); + OtpOutputStream os = buildMessage(); // TODO GH-611 should the output stream or base type be returned? message = os.toByteArray(); } @@ -29,6 +37,73 @@ public byte[] build() } } + static class StoreEncoder extends MemoizingEncoder + { + StoreEncoder(StoreOperation.Builder builder) + { + super(builder); + } + + @Override + OtpOutputStream buildMessage() + { + return TermToBinaryCodec.encodeTsPutRequest(builder.getTableName(), builder.getRows()); + } + } + + static class DeleteEncoder extends MemoizingEncoder + { + DeleteEncoder(DeleteOperation.Builder builder) + { + super(builder); + } + + @Override + OtpOutputStream buildMessage() + { + return TermToBinaryCodec.encodeTsDeleteRequest(builder.getTableName(), + builder.getKeyValues(), + null, + builder.getTimeout()); + } + } + + static class FetchEncoder extends MemoizingEncoder + { + FetchEncoder(FetchOperation.Builder builder) + { + super(builder); + } + + @Override + OtpOutputStream buildMessage() + { + // TODO: Remove this later + LinkedList list = new LinkedList<>(); + for (Cell c : builder.getKeyValues()) + { + list.add(c); + } + return TermToBinaryCodec.encodeTsGetRequest(builder.getTableName(), + list, + builder.getTimeout()); + } + } + + static class QueryEncoder extends MemoizingEncoder + { + QueryEncoder(QueryOperation.Builder builder) + { + super(builder); + } + + @Override + OtpOutputStream buildMessage() + { + return TermToBinaryCodec.encodeTsQueryRequest(builder.getQueryText()); + } + } + static class VoidDecoder implements TTBFutureOperation.TTBParser { @Override @@ -38,4 +113,12 @@ public Void parseFrom(byte[] data) } } + static class QueryResultDecoder implements TTBFutureOperation.TTBParser + { + @Override + public QueryResult parseFrom(byte[] data) + { + return TermToBinaryCodec.decodeTsGetResponse(data); + } + } } diff --git a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java b/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java deleted file mode 100644 index 7dc4607ff..000000000 --- a/src/test/java/com/basho/riak/client/core/netty/ITestTTBNode.java +++ /dev/null @@ -1,153 +0,0 @@ -package com.basho.riak.client.core.netty; - -import com.basho.riak.client.core.FutureOperation; -import com.basho.riak.client.core.RiakNode; -import com.basho.riak.client.core.operations.ts.FetchOperation; -import com.basho.riak.client.core.operations.ts.QueryOperation; -import com.basho.riak.client.core.operations.ts.StoreOperation; -import com.basho.riak.client.core.query.timeseries.Cell; -import com.basho.riak.client.core.query.timeseries.QueryResult; -import com.basho.riak.client.core.query.timeseries.Row; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import java.net.UnknownHostException; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutionException; - -import static org.junit.Assert.assertEquals; - -/** - * @author Sergey Galkin - */ -public class ITestTTBNode { - - protected final static String tableName = "GeoCheckin"; - - protected final static long now = 1443806900000l; // "now" - protected final static long fiveMinsInMS = 5l * 60l * 1000l; - protected final static long fiveMinsAgo = now - fiveMinsInMS; - protected final static long tenMinsAgo = fiveMinsAgo - fiveMinsInMS; - protected final static long fifteenMinsAgo = tenMinsAgo - fiveMinsInMS; - protected final static long fifteenMinsInFuture = now + (fiveMinsInMS * 3l); - - private RiakNode riakNode; - protected final static List rows = Arrays.asList( - // "Normal" Data - new Row(new Cell("hash1"), new Cell("user1"), Cell.newTimestamp(fifteenMinsAgo), new Cell("cloudy"), new Cell(79.0), new Cell(1), new Cell(true)), - new Row(new Cell("hash1"), new Cell("user1"), Cell.newTimestamp(fiveMinsAgo), new Cell("sunny"), new Cell(80.5), new Cell(2), new Cell(true)), - new Row(new Cell("hash1"), new Cell("user1"), Cell.newTimestamp(now), new Cell("sunny"), new Cell(81.0), new Cell(10), new Cell(false)), - - // Null Cell row - new Row(new Cell("hash1"), new Cell("user2"), Cell.newTimestamp(fiveMinsAgo), new Cell("cloudy"), null, null, new Cell(true)), - - // Data for single reads / deletes - new Row(new Cell("hash2"), new Cell("user4"), Cell.newTimestamp(fifteenMinsAgo), new Cell("rain"), new Cell(79.0), new Cell(2), new Cell(false)), - new Row(new Cell("hash2"), new Cell("user4"), Cell.newTimestamp(fiveMinsAgo), new Cell("wind"), new Cell(50.5), new Cell(3), new Cell(true)), - new Row(new Cell("hash2"), new Cell("user4"), Cell.newTimestamp(now), new Cell("snow"), new Cell(20.0), new Cell(11), new Cell(true))); - - - @Before - public void setup() throws UnknownHostException, ExecutionException, InterruptedException { - /** - * Riak PBC port - * - * In case you want/need to use a custom PBC port you may pass it by using the following system property - */ - final int testRiakPort = Integer.getInteger("com.basho.riak.pbcport", RiakNode.Builder.DEFAULT_REMOTE_PORT); - - - riakNode = new RiakNode.Builder() - .withRemotePort(testRiakPort) -// .withMaxConnections(1) -// .withMinConnections(1) - .build(); - - riakNode.start(); - - /* - Since TTB is toggled in preventive manner for each conection there is no needs to toggle it manually - */ - //final ToggleTTBEncodingOperation op = new ToggleTTBEncodingOperation(true); - //riakNode.execute(op); - //assertTrue(op.get().isUseNativeEncoding()); - } - - - @After - public void shutdonw(){ - if (riakNode != null) { - riakNode.shutdown(); - } - } - - @Test - public void storeData() throws InterruptedException, ExecutionException { - storeDataInternal(tableName, rows.subList(0,1)); - } - - @Test - public void getData() throws ExecutionException, InterruptedException { - storeDataInternal(tableName, rows); - - for (Row r: rows){ - final FetchOperation fetch = new FetchOperation.Builder(tableName, - r.getCellsCopy().subList(0,3) // use the only PK values - ).build(); - - QueryResult queryResult = execute(fetch); - - assertEquals(1, queryResult.getRowsCount()); - //assertArrayEquals(r.getCells().toArray(), queryResult.getRows().get(0).getCells().toArray()); - } - } - - @Ignore - @Test - public void queryData() throws InterruptedException, ExecutionException { - storeDataInternal(tableName, rows); - - final String queryText = "select * from GeoCheckin " + - "where user = 'user1' and " + - "geohash = 'hash1' and " + - "(time = " + tenMinsAgo +" and " + - "(time < "+ now + ")) "; - - final QueryOperation query = new QueryOperation.Builder(queryText).build(); - final QueryResult queryResult = execute(query); - - assertEquals(7, queryResult.getColumnDescriptionsCopy().size()); - assertEquals(1, queryResult.getRowsCount()); - } - - @Test - public void storeDatatoNoneExistentTable() throws InterruptedException, ExecutionException { - try { - storeDataInternal(UUID.randomUUID().toString(), rows); - } catch (ExecutionException e) { - if (e.getCause() instanceof RiakResponseException - && e.getCause().getMessage().startsWith("Time Series table") - && e.getCause().getMessage().endsWith("does not exist.")) { - // It's OK - return; - } - throw e; - } - } - - private > T execute(O operation) throws ExecutionException, InterruptedException - { - riakNode.execute((FutureOperation) operation); - return (T)((FutureOperation) operation).get(); - } - - private void storeDataInternal(String table, List rows) throws ExecutionException, InterruptedException { - final StoreOperation store = new StoreOperation.Builder(table).withRows(rows).build(); - - execute(store); - } -} From 05df99197babab1bccf97ff5ff547d943230b217 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 19 Apr 2016 08:51:47 -0700 Subject: [PATCH 21/52] Remove TTB methods for TsDelReq - not supported yet. --- .../client/core/codec/TermToBinaryCodec.java | 28 ++----------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index c717dc234..ea5677de7 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -55,8 +55,7 @@ public static OtpOutputStream encodeTsGetRequest(String tableName, Collection keyValues, byte[] vclock, int timeout) - { - // fill me in - return null; - } - - public static Void decodeTsDeleteResponse(byte[] response) - { - // Do we return anything in TTB? - return null; - } -} +} \ No newline at end of file From 15834e684e72503cf051406143e3002b53f79838 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 19 Apr 2016 15:54:31 -0400 Subject: [PATCH 22/52] Revert Delete to use PB --- .../core/operations/ts/DeleteOperation.java | 33 +++++++------------ .../core/operations/ts/TTBConverters.java | 19 ----------- 2 files changed, 12 insertions(+), 40 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/DeleteOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/DeleteOperation.java index 7108d3bef..4264610ee 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/DeleteOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/DeleteOperation.java @@ -1,7 +1,6 @@ package com.basho.riak.client.core.operations.ts; import com.basho.riak.client.core.operations.PBFutureOperation; -import com.basho.riak.client.core.operations.TTBFutureOperation; import com.basho.riak.client.core.query.timeseries.Cell; import com.basho.riak.client.core.query.timeseries.ConvertibleIterable; import com.basho.riak.protobuf.RiakMessageCodes; @@ -17,20 +16,23 @@ * @author Sergey Galkin * @since 2.0.3 */ -public class DeleteOperation extends TTBFutureOperation +public class DeleteOperation extends PBFutureOperation { private final Builder builder; private String queryInfoMessage; private DeleteOperation(Builder builder) { - super(new TTBConverters.DeleteEncoder(builder), new TTBConverters.VoidDecoder()); + super(RiakMessageCodes.MSG_TsDelReq, + RiakMessageCodes.MSG_TsDelResp, + builder.reqBuilder, + RiakTsPB.TsDelResp.PARSER); this.builder = builder; } @Override - protected Void convert(List responses) + protected Void convert(List responses) { // This is not a streaming op, there will only be one response checkAndGetSingleResponse(responses); @@ -71,7 +73,8 @@ public static class Builder { private final String tableName; private final Iterable keyValues; - private int timeout = 0; + + private final RiakTsPB.TsDelReq.Builder reqBuilder = RiakTsPB.TsDelReq.newBuilder(); public Builder(String tableName, Iterable keyValues) { @@ -85,6 +88,9 @@ public Builder(String tableName, Iterable keyValues) throw new IllegalArgumentException("Key Values cannot be null or an empty."); } + this.reqBuilder.setTable(ByteString.copyFromUtf8(tableName)); + this.reqBuilder.addAllKey(ConvertibleIterable.asIterablePbCell(keyValues)); + this.tableName = tableName; this.keyValues = keyValues; } @@ -95,25 +101,10 @@ public Builder withTimeout(int timeout) { throw new IllegalArgumentException("Timeout must be positive, or 0 for no timeout."); } - this.timeout = timeout; + this.reqBuilder.setTimeout(timeout); return this; } - public String getTableName() - { - return tableName; - } - - public Iterable getKeyValues() - { - return keyValues; - } - - public int getTimeout() - { - return timeout; - } - public DeleteOperation build() { return new DeleteOperation(this); diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java index 3c5f12878..c1cdece55 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java @@ -1,13 +1,11 @@ package com.basho.riak.client.core.operations.ts; -import com.basho.riak.client.api.commands.timeseries.Query; import com.basho.riak.client.core.codec.TermToBinaryCodec; import com.basho.riak.client.core.operations.TTBFutureOperation; import com.basho.riak.client.core.query.timeseries.Cell; import com.basho.riak.client.core.query.timeseries.QueryResult; import com.ericsson.otp.erlang.OtpOutputStream; -import java.util.ArrayList; import java.util.LinkedList; class TTBConverters @@ -51,23 +49,6 @@ OtpOutputStream buildMessage() } } - static class DeleteEncoder extends MemoizingEncoder - { - DeleteEncoder(DeleteOperation.Builder builder) - { - super(builder); - } - - @Override - OtpOutputStream buildMessage() - { - return TermToBinaryCodec.encodeTsDeleteRequest(builder.getTableName(), - builder.getKeyValues(), - null, - builder.getTimeout()); - } - } - static class FetchEncoder extends MemoizingEncoder { FetchEncoder(FetchOperation.Builder builder) From 51d60cc8215c56a53050b6be540568956acc01b0 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 19 Apr 2016 16:27:02 -0700 Subject: [PATCH 23/52] Decode TTB Get/Query response Build entire message with msg length and code 141 --- .../client/core/codec/TermToBinaryCodec.java | 109 ++++++++++++++++-- .../core/operations/ts/TTBConverters.java | 16 ++- .../core/query/timeseries/QueryResult.java | 29 ++++- 3 files changed, 140 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index ea5677de7..b03f99a69 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -4,9 +4,17 @@ import com.basho.riak.client.core.query.timeseries.QueryResult; import com.basho.riak.client.core.query.timeseries.Row; import com.ericsson.otp.erlang.OtpErlangAtom; +import com.ericsson.otp.erlang.OtpErlangBinary; +import com.ericsson.otp.erlang.OtpErlangBoolean; +import com.ericsson.otp.erlang.OtpErlangDecodeException; +import com.ericsson.otp.erlang.OtpErlangDouble; import com.ericsson.otp.erlang.OtpErlangList; +import com.ericsson.otp.erlang.OtpErlangLong; +import com.ericsson.otp.erlang.OtpErlangObject; import com.ericsson.otp.erlang.OtpExternal; +import com.ericsson.otp.erlang.OtpInputStream; import com.ericsson.otp.erlang.OtpOutputStream; +import java.util.Calendar; import java.util.Collection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,7 +44,7 @@ private static class Messages public static final OtpErlangAtom tsDelReq = new OtpErlangAtom("tsdelreq"); public static final OtpErlangAtom tsDelResp = new OtpErlangAtom("tsdelresp"); - public static final OtpErlangAtom tsUndefined = new OtpErlangAtom("undefined"); + public static final OtpErlangAtom rpbErrorResp = new OtpErlangAtom("rpberrorresp"); } public static OtpOutputStream encodeTsGetRequest(String tableName, Collection keyValues, int timeout) @@ -60,10 +68,9 @@ public static OtpOutputStream encodeTsGetRequest(String tableName, Collection rows) @@ -133,4 +139,93 @@ public static Void decodeTsPutResponse(byte[] response) // Do we return anything in TTB? return null; } + + private static QueryResult decodeTsResponse(byte[] response) throws OtpErlangDecodeException + { + QueryResult result = null; + + OtpInputStream is = new OtpInputStream(response); + final int msgArity = is.read_tuple_head(); + // Response is: + // {'rpberrorresp', ErrMsg, ErrCode} + // {'tsgetresp', {ColNames, ColTypes, Rows}} + // {'tsqueryresp', {ColNames, ColTypes, Rows}} + final String respAtom = is.read_atom(); + switch (respAtom) { + case "rpberrorresp": + // TODO process error + assert(msgArity == 3); + break; + case "tsgetresp": + case "tsqueryresp": + assert(msgArity == 2); + + final int dataArity = is.read_tuple_head(); + assert(dataArity == 3); + + final int colNameCount = is.read_list_head(); + String[] columnNames = new String[colNameCount]; + for (int i = 0; i < colNameCount; i++) { + String colName = is.read_string(); + columnNames[i] = colName; + } + + final int colTypeCount = is.read_list_head(); + assert(colNameCount == colTypeCount); + String[] columnTypes = new String[colTypeCount]; + for (int i = 0; i < colTypeCount; i++) { + String colType = is.read_string(); + columnTypes[i] = colType; + } + + final int rowCount = is.read_list_head(); + Row[] rows = new Row[rowCount]; + for (int i = 0; i < rowCount; i++) { + final int rowDataCount = is.read_tuple_head(); + assert(colNameCount == rowDataCount); + + Cell[] cells = new Cell[rowDataCount]; + for (int j = 0; i < rowDataCount; j++) { + OtpErlangObject cell = is.read_any(); + if (cell instanceof OtpErlangBinary) { + OtpErlangBinary v = (OtpErlangBinary)cell; + // TODO GH-611 this may not be correct encoding + String s = new String(v.binaryValue()); + cells[j] = new Cell(s); + } + else if (cell instanceof OtpErlangBoolean) { + OtpErlangBoolean v = (OtpErlangBoolean)cell; + cells[j] = new Cell(v.booleanValue()); + } + else if (cell instanceof OtpErlangLong) { + OtpErlangLong v = (OtpErlangLong)cell; + if ("timestamp".equals(columnTypes[j])) { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(v.longValue()); + cells[j] = new Cell(cal); + } else { + cells[j] = new Cell(v.longValue()); + } + } + else if (cell instanceof OtpErlangDouble) { + OtpErlangDouble v = (OtpErlangDouble)cell; + cells[j] = new Cell(v.doubleValue()); + } + else { + // TODO GH-611 throw exception? + } + } + + rows[i] = new Row(cells); + } + + result = new QueryResult(rows); + + break; + default: + // TODO GH-611 throw exception? + } + + return result; + } } \ No newline at end of file diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java index c1cdece55..7038d5d14 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java @@ -4,7 +4,10 @@ import com.basho.riak.client.core.operations.TTBFutureOperation; import com.basho.riak.client.core.query.timeseries.Cell; import com.basho.riak.client.core.query.timeseries.QueryResult; +import com.basho.riak.protobuf.RiakMessageCodes; import com.ericsson.otp.erlang.OtpOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; import java.util.LinkedList; @@ -12,6 +15,7 @@ class TTBConverters { private static abstract class MemoizingEncoder implements TTBFutureOperation.TTBEncoder { + // private final byte reqMessageCode = RiakMessageCodes.MSG_TsTtbMsg; protected byte[] message = null; protected final T builder; @@ -28,8 +32,18 @@ public byte[] build() if (message == null) { OtpOutputStream os = buildMessage(); + os.flush(); // TODO GH-611 should the output stream or base type be returned? - message = os.toByteArray(); + // TODO GH-611 we still need to add msgCode 141 and message length + final int msgLen = os.length() + 5; // Add 4-byte msg length and 1-byte msgCode + ByteArrayOutputStream bs = new ByteArrayOutputStream(msgLen); + DataOutputStream ds = new DataOutputStream(bs); + ds.writeInt(msgLen); + ds.writeByte(RiakMessageCodes.MSG_TsTtbMsg); + ds.flush(); + os.writeTo(bs); + bs.flush(); + message = bs.toByteArray(); } return message; } diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/QueryResult.java b/src/main/java/com/basho/riak/client/core/query/timeseries/QueryResult.java index 5b69addc0..d9fa76d06 100644 --- a/src/main/java/com/basho/riak/client/core/query/timeseries/QueryResult.java +++ b/src/main/java/com/basho/riak/client/core/query/timeseries/QueryResult.java @@ -3,6 +3,8 @@ import com.basho.riak.protobuf.RiakTsPB; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -18,14 +20,16 @@ public class QueryResult implements Iterable { public static final QueryResult EMPTY = new QueryResult(); private final Iterable pbRows; - private final int pbRowsCount; + private final Row[] rows; + private final int rowCount; private final List pbColumnDescriptions; private QueryResult() { this.pbRows = Collections.emptyList(); - this.pbRowsCount = 0; + this.rowCount = 0; this.pbColumnDescriptions = Collections.emptyList(); + this.rows = null; } public QueryResult(List tsRows) @@ -37,14 +41,23 @@ public QueryResult(Iterable tsRowsIterator, int rowCount) { this.pbColumnDescriptions = null; this.pbRows = tsRowsIterator; - this.pbRowsCount = rowCount; + this.rowCount = rowCount; + this.rows = null; } public QueryResult(List columnsList, List rowsList) { this.pbColumnDescriptions = columnsList; this.pbRows = rowsList; - this.pbRowsCount = rowsList.size(); + this.rowCount = rowsList.size(); + this.rows = null; + } + + public QueryResult(Row[] rows) { + this.rows = rows; + this.rowCount = rows.length; + this.pbRows = Collections.emptyList(); + this.pbColumnDescriptions = Collections.emptyList(); } /** @@ -62,7 +75,11 @@ public List getColumnDescriptionsCopy() */ public Iterator iterator() { - return ConvertibleIterator.iterateAsRow(this.pbRows.iterator()); + if (this.rows != null) { + return Arrays.asList(this.rows).iterator(); + } else { + return ConvertibleIterator.iterateAsRow(this.pbRows.iterator()); + } } /** @@ -71,7 +88,7 @@ public Iterator iterator() */ public int getRowsCount() { - return this.pbRowsCount; + return this.rowCount; } /** From 519cea2b20f1120db1f01bd54a8d8c31699b6341 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Wed, 20 Apr 2016 11:19:53 -0400 Subject: [PATCH 24/52] stub out test file --- .../core/operations/codec/TermToBinaryCodecTest.java | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/test/java/com/basho/riak/client/core/operations/codec/TermToBinaryCodecTest.java diff --git a/src/test/java/com/basho/riak/client/core/operations/codec/TermToBinaryCodecTest.java b/src/test/java/com/basho/riak/client/core/operations/codec/TermToBinaryCodecTest.java new file mode 100644 index 000000000..337f2e06b --- /dev/null +++ b/src/test/java/com/basho/riak/client/core/operations/codec/TermToBinaryCodecTest.java @@ -0,0 +1,7 @@ +package com.basho.riak.client.core.operations.codec; + + +public class TermToBinaryCodecTest +{ + +} From e568a4ca97f313b9a1c92a3bb5deb4b719536c6a Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Wed, 20 Apr 2016 13:35:21 -0700 Subject: [PATCH 25/52] Move error decoding into RiakMessage for pbuf and ttb messages. Add TODOs --- .../basho/riak/client/core/RiakMessage.java | 145 +++--- .../client/core/codec/TermToBinaryCodec.java | 2 - .../core/netty/RiakResponseHandler.java | 37 +- .../riak/client/core/netty/RiakTTBCodec.java | 426 ------------------ .../ts/DeferredEncodingOperation.java | 25 - .../core/operations/ts/FetchOperation.java | 6 +- .../core/operations/ts/TTBConverters.java | 50 +- .../core/operations/OperationsTest.java | 9 +- 8 files changed, 137 insertions(+), 563 deletions(-) delete mode 100644 src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java delete mode 100644 src/main/java/com/basho/riak/client/core/operations/ts/DeferredEncodingOperation.java diff --git a/src/main/java/com/basho/riak/client/core/RiakMessage.java b/src/main/java/com/basho/riak/client/core/RiakMessage.java index 5160dcf88..cb9cb0afd 100644 --- a/src/main/java/com/basho/riak/client/core/RiakMessage.java +++ b/src/main/java/com/basho/riak/client/core/RiakMessage.java @@ -1,20 +1,14 @@ -/* - * Copyright 2013 Basho Technologies Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.basho.riak.client.core; +import com.basho.riak.client.core.netty.RiakResponseException; +import com.basho.riak.protobuf.RiakMessageCodes; +import com.basho.riak.protobuf.RiakPB; +import com.ericsson.otp.erlang.OtpErlangDecodeException; +import com.ericsson.otp.erlang.OtpInputStream; +import com.google.protobuf.InvalidProtocolBufferException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Encapsulates the raw bytes sent to or received from Riak. * @author Brian Roach @@ -23,65 +17,102 @@ */ public final class RiakMessage { + private static final Logger logger = LoggerFactory.getLogger(RiakMessage.class); private final byte code; - private byte[] data; - private final Object dataObject; + private final byte[] data; + private final OtpInputStream ttbInputStream; + private final RiakResponseException riakError; public RiakMessage(byte code, byte[] data) { this.code = code; this.data = data; - this.dataObject = null; - } - /** - * Creates message for deferred encoding. - * - * @author Sergey Galkin - * @since 2.4 - */ - public RiakMessage(byte code, Object dataObj) - { - this.code = code; - this.data = null; - this.dataObject = dataObj; + switch(this.code) { + case RiakMessageCodes.MSG_ErrorResp: + this.ttbInputStream = null; + this.riakError = getRiakErrorFromPbuf(this.data); + break; + case RiakMessageCodes.MSG_TsTtbMsg: + this.ttbInputStream = new OtpInputStream(data); + this.riakError = getRiakErrorFromTtb(this.ttbInputStream); + // Set stream back to the beginning for codec's use + this.ttbInputStream.setPos(0); + break; + default: + this.riakError = null; + this.ttbInputStream = null; + } } - public byte getCode() - { + public byte getCode() { return code; } - public byte[] getData() - { + public byte[] getData() { return data; } - /** - * @author Sergey Galkin - * @since 2.4 - */ - public boolean isEncoded() - { - return data != null; + public boolean isTtbMessage() { + return this.ttbInputStream != null; } - /** - * @author Sergey Galkin - * @since 2.4 - */ - public void setData(byte[] data) - { - this.data = data; + public boolean isRiakError() { + return this.riakError != null; } - /** - * @author Sergey Galkin - * @since 2.4 - */ - @SuppressWarnings("unchecked") - public T getDataObject() - { - return (T) dataObject; + public OtpInputStream getTtbStream() { + return this.ttbInputStream; + } + + public RiakResponseException getRiakError() { + return this.riakError; + } + + private static RiakResponseException getRiakErrorFromPbuf(byte[] data) { + try { + RiakPB.RpbErrorResp err = RiakPB.RpbErrorResp.parseFrom(data); + return new RiakResponseException(err.getErrcode(), err.getErrmsg().toStringUtf8()); + } catch (InvalidProtocolBufferException ex) { + // TODO GH-611 + logger.error("exception", ex); + return new RiakResponseException(0, "could not parse protocol buffers error"); + } + } + + private RiakResponseException getRiakErrorFromTtb(OtpInputStream ttbInputStream) { + int ttbMsgArity = 0; + + try { + ttbMsgArity = ttbInputStream.read_tuple_head(); + } catch (OtpErlangDecodeException ex) { + // TODO GH-611 + logger.error("exception", ex); + } + + if (ttbMsgArity == 3) { + // NB: may be an error response + // TODO GH-611 message atom constants? + String atom = "unknown"; + try { + atom = ttbInputStream.read_atom(); + } catch (OtpErlangDecodeException ex) { + // TODO GH-611 + logger.error("exception", ex); + } + + if ("rpberrorresp".equals(atom)) { + try { + String errMsg = ttbInputStream.read_string(); // TODO GH-611 encoding? + int errCode = ttbInputStream.read_int(); // TODO GH-611 is errcode an int? + return new RiakResponseException(errCode, errMsg); + } catch (OtpErlangDecodeException ex) { + // TODO GH-611 + logger.error("exception", ex); + } + } + } + + return null; } -} +} \ No newline at end of file diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index b03f99a69..440eed3a7 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -21,8 +21,6 @@ public class TermToBinaryCodec { - private final static Logger logger = LoggerFactory.getLogger(TermToBinaryCodec.class); - private static final OtpErlangAtom undefined = new OtpErlangAtom("undefined"); private static final OtpErlangAtom tsinterpolation = new OtpErlangAtom("tsinterpolation"); diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakResponseHandler.java b/src/main/java/com/basho/riak/client/core/netty/RiakResponseHandler.java index f98f7edf6..6b295d470 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakResponseHandler.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakResponseHandler.java @@ -1,28 +1,9 @@ -/* - * Copyright 2013 Basho Technologies Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.basho.riak.client.core.netty; import com.basho.riak.client.core.RiakMessage; import com.basho.riak.client.core.RiakResponseListener; -import com.basho.riak.protobuf.RiakMessageCodes; -import com.basho.riak.protobuf.RiakPB; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @@ -31,9 +12,7 @@ */ public class RiakResponseHandler extends ChannelInboundHandlerAdapter { - private RiakResponseListener listener; - private final Logger logger = LoggerFactory.getLogger(RiakResponseHandler.class); public RiakResponseHandler(RiakResponseListener listener) { @@ -45,16 +24,9 @@ public RiakResponseHandler(RiakResponseListener listener) public void channelRead(ChannelHandlerContext chc, Object message) throws Exception { RiakMessage riakMessage = (RiakMessage) message; - if (riakMessage.getCode() == RiakMessageCodes.MSG_ErrorResp) - { - RiakPB.RpbErrorResp error = RiakPB.RpbErrorResp.parseFrom(riakMessage.getData()); - - listener.onRiakErrorResponse(chc.channel(), - new RiakResponseException(error.getErrcode(), - error.getErrmsg().toStringUtf8())); - } - else - { + if (riakMessage.isRiakError()) { + listener.onRiakErrorResponse(chc.channel(), riakMessage.getRiakError()); + } else { listener.onSuccess(chc.channel(), riakMessage); } } @@ -68,5 +40,4 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) listener.onException(ctx.channel(), cause); ctx.close(); } - -} +} \ No newline at end of file diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java b/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java deleted file mode 100644 index 8bda93dad..000000000 --- a/src/main/java/com/basho/riak/client/core/netty/RiakTTBCodec.java +++ /dev/null @@ -1,426 +0,0 @@ -package com.basho.riak.client.core.netty; - -import com.basho.riak.client.core.RiakMessage; -import com.basho.riak.client.core.operations.ts.FetchOperation; -import com.basho.riak.client.core.operations.ts.StoreOperation; -import com.basho.riak.client.core.query.timeseries.Cell; -import com.basho.riak.client.core.query.timeseries.Row; -import com.basho.riak.protobuf.RiakMessageCodes; -import com.basho.riak.protobuf.RiakPB; -import com.basho.riak.protobuf.RiakTsPB; -import com.ericsson.otp.erlang.*; -import com.google.protobuf.ByteString; -import com.google.protobuf.GeneratedMessage; -import com.google.protobuf.InvalidProtocolBufferException; -import io.netty.buffer.ByteBuf; -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ByteToMessageCodec; -import io.netty.util.AttributeKey; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.UnsupportedEncodingException; -import java.util.*; - -/** - * @author Sergey Galkin - */ -public class RiakTTBCodec extends ByteToMessageCodec { - private final static Logger logger = LoggerFactory.getLogger(RiakTTBCodec.class); - private RiakMessageCodec pbCodec = new RiakMessageCodec(); - - private static final OtpErlangAtom undefined = new OtpErlangAtom("undefined"); - private static final OtpErlangAtom tscell = new OtpErlangAtom("tscell"); - private static final OtpErlangAtom tsrow = new OtpErlangAtom("tsrow"); - private static final OtpErlangAtom tsputreq = new OtpErlangAtom("tsputreq"); - private static final OtpErlangAtom tsqueryreq = new OtpErlangAtom("tsqueryreq"); - private static final OtpErlangAtom tsinterpolation = new OtpErlangAtom("tsinterpolation"); - private static final OtpErlangAtom tsgetreq = new OtpErlangAtom("tsgetreq"); - - private static final OtpErlangAtom _false = new OtpErlangAtom("false"); - private static final OtpErlangList EMPTY_ERLANG_LIST = new OtpErlangList(); - private static final OtpErlangTuple EMPTY_ERLANG_TSCELL = new OtpErlangTuple(new OtpErlangObject[]{ - tscell, undefined, undefined, undefined, undefined, undefined}); - - private boolean isTTBMessage(byte code) { - switch (code) { - case RiakMessageCodes.MSG_TsTtbMsg: - return true; - default: - return false; - } - } - - @Override - protected void encode(ChannelHandlerContext ctx, RiakMessage msg, ByteBuf out) throws Exception { - final OtpErlangTuple t; - - if (isTTBMessage(msg.getCode())) { - logger.trace("Encoding message {}, encoding '{}', is deferred: {}, channel '{}' {}", - msg.getCode(), "TTB", !msg.isEncoded(), ctx.channel().hashCode(), ctx.channel()); - switch (msg.getCode()) { - case RiakMessageCodes.MSG_TsPutReq: - t = encodeTsPut(msg); - break; - case RiakMessageCodes.MSG_TsGetReq: - t = encodeTsGet(msg); - break; -// case RiakMessageCodes.MSG_TsQueryReq: -// t = encodeTsQuery(msg); -// break; - default: - throw new IllegalStateException("Can't encode TTB request, unsupported message with code " + msg.getCode()); - } - } else { - logger.trace("Encoding message {}, encoding '{}', is deferred: {}, channel '{}' {}", - msg.getCode(), "PB", !msg.isEncoded(), ctx.channel().hashCode(), ctx.channel()); - - if (!msg.isEncoded()) { - final GeneratedMessage.Builder reqBuilder = msg.getDataObject(); - msg.setData(reqBuilder.build().toByteArray()); - } - pbCodec.encode(ctx, msg, out); - return; - } - - final OtpOutputStream os = new OtpOutputStream(t); - byte data[] = os.toByteArray(); - - int length = data.length+2; - out.writeInt(length); - - - out.writeByte(msg.getCode()); - - /** - * DO A TOP SECRET HACK - * - * It seems that the message is missing the 131 tag required by term_to_binary. - * As is evident from the Java output, this tag is not being added by jinterface encode. - * If I simply add 131 to the beginning of the binary it decodes correctly. - * - * http://stackoverflow.com/questions/15189447/jinterface-to-create-external-erlang-term - */ - out.writeByte(131); - out.writeBytes(data); - } - - private static OtpErlangTuple pbCellToTtb(RiakTsPB.TsCell c) { - final OtpErlangObject o[]; - - if (c.hasVarcharValue()) { - o = new OtpErlangObject[] - {tscell, - pbStrToTtb(c.getVarcharValue()), - undefined, undefined, undefined, undefined}; - } else if (c.hasSint64Value()) { - o = new OtpErlangObject[] - {tscell, - undefined, - new OtpErlangLong(c.getSint64Value()), undefined, undefined, undefined}; - } else if (c.hasTimestampValue()) { - o = new OtpErlangObject[] - {tscell, - undefined, undefined, - new OtpErlangLong(c.getTimestampValue()), - undefined, undefined}; - } else if (c.hasBooleanValue()) { - o = new OtpErlangObject[] - {tscell, - undefined, undefined, undefined, - new OtpErlangBoolean(c.getBooleanValue()), - undefined}; - } else if (c.hasDoubleValue()) { - o = new OtpErlangObject[] - {tscell, - undefined, undefined, undefined, undefined, - new OtpErlangDouble(c.getDoubleValue())}; - } else { - return EMPTY_ERLANG_TSCELL; - } - - return new OtpErlangTuple(o); - } - - private static int sizeOfIterable(Iterable values) { - if (values instanceof Collection) { - return ((Collection)values).size(); - } - - int size = 0; - for(Object ignored : values) { - ++size; - } - return size; - } - - private static OtpErlangList tsCellsToTtb(Iterable tsCells) { - final int size = sizeOfIterable(tsCells); - final OtpErlangObject cells[] = new OtpErlangObject[size]; - int idx =-1; - - for (Cell c: tsCells) { - cells[++idx] = pbCellToTtb(c.getPbCell()); - } - - return new OtpErlangList(cells); - } - - private static OtpErlangList pbCellsToTtb(List pbCells) { - final OtpErlangObject cells[] = new OtpErlangObject[pbCells.size()]; - for (int i = 0; i < cells.length; ++i) { - final RiakTsPB.TsCell c = pbCells.get(i); - cells[i] = pbCellToTtb(c); - } - - return new OtpErlangList(cells); - } - - private static OtpErlangTuple pbRowToTtb(RiakTsPB.TsRow r) { - return new OtpErlangTuple(new OtpErlangObject[] {tsrow, pbCellsToTtb(r.getCellsList())}); - } - - private static OtpErlangBinary strToTtb(String s) { - return new OtpErlangBinary(s.getBytes()); - } - - private static OtpErlangBinary pbStrToTtb(ByteString bs) { - return new OtpErlangBinary(bs.toByteArray()); - } - - private static OtpErlangTuple pbInterpolationToTtb(RiakTsPB.TsInterpolation interpolation) { - return new OtpErlangTuple(new OtpErlangObject[] {tsinterpolation, pbStrToTtb(interpolation.getBase()), EMPTY_ERLANG_LIST}); - } - - private static OtpErlangTuple encodeTsPut(RiakMessage msg) throws InvalidProtocolBufferException, UnsupportedEncodingException { - final OtpErlangObject[] elems; - if (msg.isEncoded()) { - final RiakTsPB.TsPutReqOrBuilder req; - req = RiakTsPB.TsPutReq.parseFrom(msg.getData()); - final OtpErlangObject rows[] = new OtpErlangObject[req.getRowsCount()]; - for (int i = 0; i < rows.length; ++i) { - final RiakTsPB.TsRow r = req.getRows(i); - rows[i] = pbRowToTtb(r); - } - - elems = new OtpErlangObject[] - {tsputreq, pbStrToTtb(req.getTable()), - new OtpErlangList(), - new OtpErlangList(rows)}; - - } else { - final StoreOperation.Builder builder = msg.getDataObject(); - final int size = sizeOfIterable(builder.getRows()); - final OtpErlangObject rows[] = new OtpErlangObject[size]; - int idx =-1; - - for (Row r: builder.getRows()) { - rows[++idx] = pbRowToTtb(r.getPbRow()); - } - - elems = new OtpErlangObject[] - {tsputreq, strToTtb(builder.getTableName()), - new OtpErlangList(), - new OtpErlangList(rows)}; - } - - return new OtpErlangTuple(elems); - } - - private OtpErlangTuple encodeTsQuery(RiakMessage msg) throws InvalidProtocolBufferException { - final RiakTsPB.TsQueryReq req = RiakTsPB.TsQueryReq.parseFrom(msg.getData()); - - final OtpErlangObject[] elems = new OtpErlangObject[] - {tsqueryreq, pbInterpolationToTtb(req.getQuery()), _false}; - - return new OtpErlangTuple(elems); - } - - private static OtpErlangTuple encodeTsGet(RiakMessage msg) throws InvalidProtocolBufferException { - final OtpErlangObject[] elems; - if (msg.isEncoded()) { - final RiakTsPB.TsGetReqOrBuilder req; - req = RiakTsPB.TsGetReq.parseFrom(msg.getData()); - - elems = new OtpErlangObject[] - {tsgetreq, pbStrToTtb(req.getTable()), - pbCellsToTtb(req.getKeyList()), - undefined}; - - } else { - final FetchOperation.Builder builder = msg.getDataObject(); - - elems = new OtpErlangObject[] - {tsgetreq, strToTtb(builder.getTableName()), - tsCellsToTtb(builder.getKeyValues()), - undefined}; - } - - return new OtpErlangTuple(elems); - } - - private static ByteString ttbBinaryToPb(OtpErlangBinary b) { - return ByteString.copyFromUtf8(new String(b.binaryValue())); - } - - private static List ttbRowsToPb(OtpErlangList ttbRows) { - final RiakTsPB.TsRow rows[] = new RiakTsPB.TsRow[ttbRows.arity()]; - - for (int i=0; i ttbColumnDescriptionToPb(OtpErlangList ttbDescrs) { - final RiakTsPB.TsColumnDescription descrs[] = new RiakTsPB.TsColumnDescription[ttbDescrs.arity()]; - for (int i=0; i out) throws Exception { - - // Make sure we have 4 bytes - if (in.readableBytes() >= 4) { - in.markReaderIndex(); - - int length = in.readInt(); - // See if we have the full frame. - if (in.readableBytes() < length) { - in.capacity(length); - in.resetReaderIndex(); - return; - } - - final byte[] array = new byte[length-1]; - final int msgCode = in.readByte(); - in.readBytes(array); - - switch (msgCode) { - case RiakMessageCodes.MSG_TsPutResp: - final RiakTsPB.TsPutResp r = RiakTsPB.TsPutResp.newBuilder().build(); - out.add(new RiakMessage(RiakMessageCodes.MSG_TsPutResp, r.toByteArray())); - return; - } - - final OtpErlangObject o; - try { - OtpInputStream is = new OtpInputStream(array); - o = is.read_any(); - }catch (Exception ex){ - in.resetReaderIndex(); - in.markReaderIndex(); - - // skip length header - in.readInt(); - - // read msg code - byte code = in.readByte(); - - if (isTTBMessage(code)) { - throw ex; - } - - in.resetReaderIndex(); - - /** - * As we failed with TTB encoding, it worth to try with PB - */ - pbCodec.decode(null, in, out); - return; - } - - assert o != null; - if (o instanceof OtpErlangTuple && ((OtpErlangTuple)o).elementAt(0) instanceof OtpErlangAtom) { - final OtpErlangTuple t = ((OtpErlangTuple)o); - final OtpErlangAtom resp = (OtpErlangAtom)t.elementAt(0); - final String v = resp.atomValue(); - - if ("rpberrorresp".equals(v)) { - final String errMsg; - if (t.elementAt(1) instanceof OtpErlangString) { - errMsg = ((OtpErlangString) t.elementAt(1)).stringValue(); - } else if (t.elementAt(1) instanceof OtpErlangBinary) { - errMsg = new String(((OtpErlangBinary) t.elementAt(1)).binaryValue()); - } else { - throw new IllegalStateException(); - } - - final OtpErlangLong errCode = (OtpErlangLong) t.elementAt(2); - final RiakPB.RpbErrorResp r = RiakPB.RpbErrorResp.newBuilder() - .setErrcode(errCode.intValue()) - .setErrmsg(ByteString.copyFromUtf8(errMsg)) - .build(); - out.add(new RiakMessage(RiakMessageCodes.MSG_ErrorResp, r.toByteArray())); - } else if ("tsgetresp".equals(v)) { - final RiakTsPB.TsGetResp r = RiakTsPB.TsGetResp.newBuilder() - .addAllColumns(ttbColumnDescriptionToPb((OtpErlangList)t.elementAt(1))) - .addAllRows(ttbRowsToPb((OtpErlangList)t.elementAt(2))) - .build(); - - out.add(new RiakMessage(RiakMessageCodes.MSG_TsGetResp, r.toByteArray())); - -// } else if ("tsqueryresp".equals(v)) { -// final RiakTsPB.TsQueryResp r = tsqueryrespToPB(); -// out.add(new RiakMessage(RiakMessageCodes.MSG_TsQueryResp, r.toByteArray())); - } else { - throw new IllegalStateException("Can't decode TTB response, unsupported atom '"+ v + "'"); - } - } else { - throw new IllegalStateException("Can't decode TTB response"); - } - } - } -} diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/DeferredEncodingOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/DeferredEncodingOperation.java deleted file mode 100644 index 75d78a6a0..000000000 --- a/src/main/java/com/basho/riak/client/core/operations/ts/DeferredEncodingOperation.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.basho.riak.client.core.operations.ts; - -import com.basho.riak.client.core.RiakMessage; -import com.basho.riak.client.core.operations.PBFutureOperation; -import com.google.protobuf.GeneratedMessage; - - -/** - * @author Sergey Galkin - */ -public abstract class DeferredEncodingOperation extends PBFutureOperation { - - protected DeferredEncodingOperation(final byte reqMessageCode, - final byte respMessageCode, - final GeneratedMessage.Builder reqBuilder, - com.google.protobuf.Parser respParser) - { - super(reqMessageCode, respMessageCode, reqBuilder, respParser); - } - - @Override - protected RiakMessage createChannelMessage() { - return new RiakMessage(reqMessageCode, reqBuilder); - } -} diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java index 9d4b260ae..47b754b39 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java @@ -15,6 +15,7 @@ * @author Sergey Galkin * @since 2.0.3 */ +// TODO GH-611 We have the ability to send TsGetReq / Resp TTB messages public class FetchOperation extends TTBFutureOperation { private final Builder builder; @@ -26,11 +27,6 @@ private FetchOperation(Builder builder) this.builder = builder; } - @Override - protected RiakMessage createChannelMessage() { - return new RiakMessage(RiakMessageCodes.MSG_TsGetReq, builder); - } - @Override protected QueryResult convert(List responses) { diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java index 7038d5d14..e91453838 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java @@ -5,11 +5,15 @@ import com.basho.riak.client.core.query.timeseries.Cell; import com.basho.riak.client.core.query.timeseries.QueryResult; import com.basho.riak.protobuf.RiakMessageCodes; +import com.ericsson.otp.erlang.OtpErlangDecodeException; import com.ericsson.otp.erlang.OtpOutputStream; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; +import java.io.IOException; import java.util.LinkedList; +import java.util.logging.Level; +import java.util.logging.Logger; class TTBConverters { @@ -31,20 +35,29 @@ public byte[] build() { if (message == null) { - OtpOutputStream os = buildMessage(); - os.flush(); - // TODO GH-611 should the output stream or base type be returned? - // TODO GH-611 we still need to add msgCode 141 and message length - final int msgLen = os.length() + 5; // Add 4-byte msg length and 1-byte msgCode - ByteArrayOutputStream bs = new ByteArrayOutputStream(msgLen); - DataOutputStream ds = new DataOutputStream(bs); - ds.writeInt(msgLen); - ds.writeByte(RiakMessageCodes.MSG_TsTtbMsg); - ds.flush(); - os.writeTo(bs); - bs.flush(); - message = bs.toByteArray(); + try { + OtpOutputStream os = buildMessage(); + os.flush(); + + // TODO GH-611 should the output stream or base type be returned? + // TODO GH-611 we still need to add msgCode 141 and message length + final int msgLen = os.length() + 5; // Add 4-byte msg length and 1-byte msgCode + ByteArrayOutputStream bs = new ByteArrayOutputStream(msgLen); + + DataOutputStream ds = new DataOutputStream(bs); + ds.writeInt(msgLen); + ds.writeByte(RiakMessageCodes.MSG_TsTtbMsg); + ds.flush(); + + os.writeTo(bs); + bs.flush(); + message = bs.toByteArray(); + } catch (IOException ex) { + // TODO GH-611 + Logger.getLogger(TTBConverters.class.getName()).log(Level.SEVERE, null, ex); + } } + return message; } } @@ -113,7 +126,16 @@ static class QueryResultDecoder implements TTBFutureOperation.TTBParser Date: Wed, 20 Apr 2016 14:38:36 -0700 Subject: [PATCH 26/52] Added tests for RiakMessage error decoding. --- .../basho/riak/client/core/RiakMessage.java | 4 +- .../client/core/codec/TermToBinaryCodec.java | 3 +- .../riak/client/core/RiakMessageTest.java | 40 +++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/basho/riak/client/core/RiakMessageTest.java diff --git a/src/main/java/com/basho/riak/client/core/RiakMessage.java b/src/main/java/com/basho/riak/client/core/RiakMessage.java index cb9cb0afd..067c38af7 100644 --- a/src/main/java/com/basho/riak/client/core/RiakMessage.java +++ b/src/main/java/com/basho/riak/client/core/RiakMessage.java @@ -5,7 +5,9 @@ import com.basho.riak.protobuf.RiakPB; import com.ericsson.otp.erlang.OtpErlangDecodeException; import com.ericsson.otp.erlang.OtpInputStream; +import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; +import java.nio.charset.StandardCharsets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,7 +105,7 @@ private RiakResponseException getRiakErrorFromTtb(OtpInputStream ttbInputStream) if ("rpberrorresp".equals(atom)) { try { - String errMsg = ttbInputStream.read_string(); // TODO GH-611 encoding? + String errMsg = new String(ttbInputStream.read_binary(), StandardCharsets.UTF_8); int errCode = ttbInputStream.read_int(); // TODO GH-611 is errcode an int? return new RiakResponseException(errCode, errMsg); } catch (OtpErlangDecodeException ex) { diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index 440eed3a7..af2fd54ca 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -14,6 +14,7 @@ import com.ericsson.otp.erlang.OtpExternal; import com.ericsson.otp.erlang.OtpInputStream; import com.ericsson.otp.erlang.OtpOutputStream; +import java.nio.charset.StandardCharsets; import java.util.Calendar; import java.util.Collection; import org.slf4j.Logger; @@ -188,7 +189,7 @@ private static QueryResult decodeTsResponse(byte[] response) throws OtpErlangDec if (cell instanceof OtpErlangBinary) { OtpErlangBinary v = (OtpErlangBinary)cell; // TODO GH-611 this may not be correct encoding - String s = new String(v.binaryValue()); + String s = new String(v.binaryValue(), StandardCharsets.UTF_8); cells[j] = new Cell(s); } else if (cell instanceof OtpErlangBoolean) { diff --git a/src/test/java/com/basho/riak/client/core/RiakMessageTest.java b/src/test/java/com/basho/riak/client/core/RiakMessageTest.java new file mode 100644 index 000000000..9416732bc --- /dev/null +++ b/src/test/java/com/basho/riak/client/core/RiakMessageTest.java @@ -0,0 +1,40 @@ +package com.basho.riak.client.core; + +import com.basho.riak.client.core.netty.RiakResponseException; +import com.basho.riak.protobuf.RiakMessageCodes; +import com.basho.riak.protobuf.RiakPB; +import com.google.protobuf.ByteString; +import org.junit.Assert; +import org.junit.Test; + +/** + * + * @author Luke Bakken + */ +public class RiakMessageTest +{ + @Test + public void parsesPbufErrorCorrectly() { + RiakPB.RpbErrorResp.Builder b = RiakPB.RpbErrorResp.newBuilder(); + b.setErrcode(1234); + b.setErrmsg(ByteString.copyFromUtf8("this is an error")); + RiakPB.RpbErrorResp rpbErrorResp = b.build(); + + RiakMessage msg = new RiakMessage(RiakMessageCodes.MSG_ErrorResp, rpbErrorResp.toByteArray()); + RiakResponseException err = msg.getRiakError(); + Assert.assertEquals("this is an error", err.getMessage()); + Assert.assertEquals(1234, err.getCode()); + } + + @Test + public void parsesTtbErrorCorrectly() { + final byte[] TTB_ERROR = {(byte)131, 104, 3, 100, 0, 12, 114, 112, 98, 101, 114, + 114, 111, 114, 114, 101, 115, 112, 109, 0, 0, 0, 16, 116, 104, 105, 115, 32, 105, 115, + 32, 97, 110, 32, 101, 114, 114, 111, 114, 98, 0, 0, 4, (byte)210}; + + RiakMessage msg = new RiakMessage(RiakMessageCodes.MSG_TsTtbMsg, TTB_ERROR); + RiakResponseException err = msg.getRiakError(); + Assert.assertEquals("this is an error", err.getMessage()); + Assert.assertEquals(1234, err.getCode()); + } +} \ No newline at end of file From 572a0312c4860196177ecdbcef21cd14b41a751f Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Wed, 20 Apr 2016 16:42:49 -0700 Subject: [PATCH 27/52] Add test for TTB encoding TsGetReq --- .../client/core/codec/TermToBinaryCodec.java | 8 ++-- .../core/codec/TermToBinaryCodecTest.java | 39 +++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index af2fd54ca..8759fc66f 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -17,8 +17,6 @@ import java.nio.charset.StandardCharsets; import java.util.Calendar; import java.util.Collection; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class TermToBinaryCodec { @@ -54,7 +52,7 @@ public static OtpOutputStream encodeTsGetRequest(String tableName, Collection + */ +public class TermToBinaryCodecTest +{ + private static final String TABLE_NAME = "test_table"; + + @Test + public void encodesGetRequestCorrectly() { + final byte[] exp = {(byte)131, 104, 4, 100, 0, 8, 116, 115, 103, 101, 116, 114, 101, 113, 109, 0, + 0, 0, 10, 116, 101, 115, 116, 95, 116, 97, 98, 108, 101, 108, 0, 0, 0, 3, + 109, 0, 0, 0, 6, 115, 101, 114, 105, 101, 115, 109, 0, 0, 0, 6, 102, 97, + 109, 105, 108, 121, 98, 0, (byte)188, 97, 78, 106, 98, 0, 0, 19, (byte)136}; + + Cell k1 = new Cell("series"); + Cell k2 = new Cell("family"); + Cell k3 = new Cell(12345678); + Cell[] key = {k1, k2, k3}; + + try { + OtpOutputStream os = TermToBinaryCodec.encodeTsGetRequest(TABLE_NAME, Arrays.asList(key), 5000); + os.flush(); + byte[] msg = os.toByteArray(); + Assert.assertArrayEquals(exp, msg); + } catch (IOException ex) { + Assert.fail(ex.getMessage()); + } + } +} \ No newline at end of file From b84619d7a9a4c96f3c4cc27336c98eb30f0f156a Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Wed, 20 Apr 2016 19:18:05 -0700 Subject: [PATCH 28/52] Add test for TTB encoding TsQueryReq --- .../client/core/codec/TermToBinaryCodec.java | 3 --- .../core/codec/TermToBinaryCodecTest.java | 21 +++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index 8759fc66f..37ab009c1 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -85,14 +85,12 @@ public static OtpOutputStream encodeTsQueryRequest(String queryText) os.write_any(Messages.tsInterpolation); os.write_binary(queryText.getBytes(StandardCharsets.UTF_8)); // interpolations is an empty list - os.write_list_head(0); os.write_nil(); // streaming is false for now os.write_boolean(false); // cover_context is an empty list - os.write_list_head(0); os.write_nil(); return os; @@ -114,7 +112,6 @@ public static OtpOutputStream encodeTsPutRequest(String tableName, Collection>, [<<"series">>, <<"family">>, 12345678], 5000} final byte[] exp = {(byte)131, 104, 4, 100, 0, 8, 116, 115, 103, 101, 116, 114, 101, 113, 109, 0, 0, 0, 10, 116, 101, 115, 116, 95, 116, 97, 98, 108, 101, 108, 0, 0, 0, 3, 109, 0, 0, 0, 6, 115, 101, 114, 105, 101, 115, 109, 0, 0, 0, 6, 102, 97, @@ -36,4 +38,23 @@ public void encodesGetRequestCorrectly() { Assert.fail(ex.getMessage()); } } + + @Test + public void encodesQueryRequestCorrectly() { + // {tsqueryreq, {tsinterpolation, <<"SELECT * FROM FRAZZLE">>, []}, false, []} + final byte[] exp = {(byte)131, 104, 4, 100, 0, 10, 116, 115, 113, 117, 101, 114, 121, 114, 101, + 113, 104, 3, 100, 0, 15, 116, 115, 105, 110, 116, 101, 114, 112, 111, + 108, 97, 116, 105, 111, 110, 109, 0, 0, 0, 21, 83, 69, 76, 69, 67, 84, + 32, 42, 32, 70, 82, 79, 77, 32, 70, 82, 65, 90, 90, 76, 69, 106, 100, 0, + 5, 102, 97, 108, 115, 101, 106}; + + try { + OtpOutputStream os = TermToBinaryCodec.encodeTsQueryRequest(QUERY); + os.flush(); + byte[] msg = os.toByteArray(); + Assert.assertArrayEquals(exp, msg); + } catch (IOException ex) { + Assert.fail(ex.getMessage()); + } + } } \ No newline at end of file From d55c4efa9ed322a32f45ebfece2f1f85db9cdf4c Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Thu, 21 Apr 2016 11:28:20 -0400 Subject: [PATCH 29/52] Add null cell handling to TTB PUT encoding, fix tuple/list mixup, add test --- .../client/core/codec/TermToBinaryCodec.java | 16 ++++++-- .../client/core/netty/RiakMessageCodec.java | 8 ++-- .../core/codec/TermToBinaryCodecTest.java | 40 ++++++++++++++++++- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index 37ab009c1..f84591f26 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -118,10 +118,20 @@ public static OtpOutputStream encodeTsPutRequest(String tableName, Collection out) t { in.markReaderIndex(); int length = in.readInt(); - + // See if we have the full frame. if (in.readableBytes() < length) { @@ -58,8 +58,8 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t in.readBytes(array); out.add(new RiakMessage(code,array)); } - + } } - + } diff --git a/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java b/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java index 49b26899e..92f929598 100644 --- a/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java +++ b/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java @@ -1,9 +1,13 @@ package com.basho.riak.client.core.codec; import com.basho.riak.client.core.query.timeseries.Cell; +import com.basho.riak.client.core.query.timeseries.Row; import com.ericsson.otp.erlang.OtpOutputStream; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; + import org.junit.Assert; import org.junit.Test; @@ -57,4 +61,38 @@ public void encodesQueryRequestCorrectly() { Assert.fail(ex.getMessage()); } } -} \ No newline at end of file + + @Test + public void encodesPutRequestCorrectly() { + // What erlang generates + // A = riakc_ts_put_operator:serialize(<<"test_table">>,[[<<"series">>, <<"family">>, 12345678, 1, true, 34.3, []]], true). + // A = {tsputreq,<<"test_table">>,[],[[<<"series">>,<<"family">>,12345678,1,true,34.3,[]]]} + // rp(term_to_binary(A)). => exp array below + + // # What we generate + // B = <<-125, 104, 4, 100, 0, 8, 116, 115, 112, 117, 116, 114, 101, 113, 109, 0, 0, 0, 10, 116, 101, 115, 116, 95, 116, 97, 98, 108, 101, 106, 108, 0, 0, 0, 1, 108, 0, 0, 0, 7, 109, 0, 0, 0, 6, 115, 101, 114, 105, 101, 115, 109, 0, 0, 0, 6, 102, 97, 109, 105, 108, 121, 98, 0, -68, 97, 78, 97, 1, 100, 0, 4, 116, 114, 117, 101, 70, 64, 65, 38, 102, 102, 102, 102, 102, 106, 106, 106, 106>>. + // B2 = binary_to_term(B). + // B2 = {tsputreq,<<"test_table">>,[],[[<<"series">>,<<"family">>,12345678,1,true,34.3,[]]]} + // A = B2. #true (wtf) (magic) + + final byte[] exp = {(byte)131,104,4,100,0,8,116,115,112,117,116,114,101,113,109,0, + 0,0,7,70,82,65,90,90,76,69,106,108,0,0,0,1,108,0,0,0,7, + 109,0,0,0,6,115,101,114,105,101,115,109,0,0,0,6,102,97, + 109,105,108,121,98,0,(byte)188,97,78,97,1,100,0,4,116,114,117, + 101,99,51,46,52,50,57,57,57,57,57,57,57,57,57,57,57,57, + 57,55,49,53,55,56,101,43,48,49,0,0,0,0,0,106,106,106}; + + final ArrayList rows = new ArrayList<>(1); + rows.add(new Row(new Cell("series"), new Cell("family"), Cell.newTimestamp(12345678), + new Cell(1l), new Cell(true), new Cell(34.3), null)); + + try { + OtpOutputStream os = TermToBinaryCodec.encodeTsPutRequest(TABLE_NAME, rows); + os.flush(); + byte[] msg = os.toByteArray(); + Assert.assertArrayEquals(exp, msg); + } catch (IOException ex) { + Assert.fail(ex.getMessage()); + } + } +} From ae6bbfe2dd6685333dbb0f54b0f9f6dc647379f4 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Thu, 21 Apr 2016 11:39:32 -0400 Subject: [PATCH 30/52] fix example request tuple --- .../client/core/codec/TermToBinaryCodec.java | 3 +-- .../core/codec/TermToBinaryCodecTest.java | 25 ++++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index f84591f26..d2c42c43e 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -118,7 +118,7 @@ public static OtpOutputStream encodeTsPutRequest(String tableName, Collection>,[[<<"series">>, <<"family">>, 12345678, 1, true, 34.3, []]], true). - // A = {tsputreq,<<"test_table">>,[],[[<<"series">>,<<"family">>,12345678,1,true,34.3,[]]]} + // A = riakc_ts_put_operator:serialize(<<"test_table">>,[{<<"series">>, <<"family">>, 12345678, 1, true, 34.3, []}], true). + // A = {tsputreq,<<"test_table">>,[],[{<<"series">>,<<"family">>,12345678,1,true,34.3,[]}]} // rp(term_to_binary(A)). => exp array below // # What we generate - // B = <<-125, 104, 4, 100, 0, 8, 116, 115, 112, 117, 116, 114, 101, 113, 109, 0, 0, 0, 10, 116, 101, 115, 116, 95, 116, 97, 98, 108, 101, 106, 108, 0, 0, 0, 1, 108, 0, 0, 0, 7, 109, 0, 0, 0, 6, 115, 101, 114, 105, 101, 115, 109, 0, 0, 0, 6, 102, 97, 109, 105, 108, 121, 98, 0, -68, 97, 78, 97, 1, 100, 0, 4, 116, 114, 117, 101, 70, 64, 65, 38, 102, 102, 102, 102, 102, 106, 106, 106, 106>>. - // B2 = binary_to_term(B). - // B2 = {tsputreq,<<"test_table">>,[],[[<<"series">>,<<"family">>,12345678,1,true,34.3,[]]]} - // A = B2. #true (wtf) (magic) + // B = <<-125, 104, 4, 100, 0, 8, 116, 115, 112, 117, 116, 114, 101, 113, 109, 0, 0, 0, 10, 116, 101, 115, 116, 95, 116, 97, 98, 108, 101, 106, 108, 0, 0, 0, 1, 104, 7, 109, 0, 0, 0, 6, 115, 101, 114, 105, 101, 115, 109, 0, 0, 0, 6, 102, 97, 109, 105, 108, 121, 98, 0, -68, 97, 78, 97, 1, 100, 0, 4, 116, 114, 117, 101, 70, 64, 65, 38, 102, 102, 102, 102, 102, 106, 106, 106>>. + // C = binary_to_term(B). + // C = {tsputreq,<<"test_table">>,[],[{<<"series">>,<<"family">>,12345678,1,true,34.3,[]}]} + // A = C. #true (wtf) (magic) final byte[] exp = {(byte)131,104,4,100,0,8,116,115,112,117,116,114,101,113,109,0, - 0,0,7,70,82,65,90,90,76,69,106,108,0,0,0,1,108,0,0,0,7, - 109,0,0,0,6,115,101,114,105,101,115,109,0,0,0,6,102,97, - 109,105,108,121,98,0,(byte)188,97,78,97,1,100,0,4,116,114,117, - 101,99,51,46,52,50,57,57,57,57,57,57,57,57,57,57,57,57, - 57,55,49,53,55,56,101,43,48,49,0,0,0,0,0,106,106,106}; + 0,0,10,116,101,115,116,95,116,97,98,108,101,106,108,0,0, + 0,1,104,7,109,0,0,0,6,115,101,114,105,101,115,109,0,0,0, + 6,102,97,109,105,108,121,98,0,(byte)188,97,78,97,1,100,0,4, + 116,114,117,101,99,51,46,52,50,57,57,57,57,57,57,57,57, + 57,57,57,57,57,55,49,53,55,56,101,43,48,49,0,0,0,0,0, + 106,106}; final ArrayList rows = new ArrayList<>(1); rows.add(new Row(new Cell("series"), new Cell("family"), Cell.newTimestamp(12345678), - new Cell(1l), new Cell(true), new Cell(34.3), null)); + new Cell(1L), new Cell(true), new Cell(34.3), null)); try { OtpOutputStream os = TermToBinaryCodec.encodeTsPutRequest(TABLE_NAME, rows); From 30a676f4b95ac9b24849d5982ca621c390093454 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 21 Apr 2016 08:54:22 -0700 Subject: [PATCH 31/52] Add a TsPutReq test --- .../core/codec/TermToBinaryCodecTest.java | 105 ++++++++++++------ 1 file changed, 73 insertions(+), 32 deletions(-) diff --git a/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java b/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java index 9572d6bec..ee12d26a1 100644 --- a/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java +++ b/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java @@ -6,8 +6,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.List; - +import java.util.Calendar; import org.junit.Assert; import org.junit.Test; @@ -21,39 +20,39 @@ public class TermToBinaryCodecTest private static final String QUERY = "SELECT * FROM FRAZZLE"; @Test - public void encodesGetRequestCorrectly() { - // {tsgetreq, <<"test_table">>, [<<"series">>, <<"family">>, 12345678], 5000} - final byte[] exp = {(byte)131, 104, 4, 100, 0, 8, 116, 115, 103, 101, 116, 114, 101, 113, 109, 0, - 0, 0, 10, 116, 101, 115, 116, 95, 116, 97, 98, 108, 101, 108, 0, 0, 0, 3, - 109, 0, 0, 0, 6, 115, 101, 114, 105, 101, 115, 109, 0, 0, 0, 6, 102, 97, - 109, 105, 108, 121, 98, 0, (byte)188, 97, 78, 106, 98, 0, 0, 19, (byte)136}; - - Cell k1 = new Cell("series"); - Cell k2 = new Cell("family"); - Cell k3 = new Cell(12345678); - Cell[] key = {k1, k2, k3}; + public void encodesPutRequestCorrectly_1() { + // {tsputreq, <<"test_table">>, [], [{<<"varchar">>, 12345678, 12.34, true, 12345}, {<<"string">>, 8765432, 43.21, false, 543321}]} + final byte[] exp = {(byte)131, 104, 4, 100, 0, 8, 116, 115, 112, 117, 116, 114, 101, 113, 109, 0, + 0, 0, 10, 116, 101, 115, 116, 95, 116, 97, 98, 108, 101, 106, 108, 0, 0, + 0, 2, 104, 5, 109, 0, 0, 0, 7, 118, 97, 114, 99, 104, 97, 114, 98, 0, (byte)188, + 97, 78, 99, 49, 46, 50, 51, 51, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, + 57, 57, 57, 56, 53, 55, 57, 101, 43, 48, 49, 0, 0, 0, 0, 0, 100, 0, 4, 116, + 114, 117, 101, 98, 0, 0, 48, 57, 104, 5, 109, 0, 0, 0, 6, 115, 116, 114, + 105, 110, 103, 98, 0, (byte)133, (byte)191, (byte)248, 99, 52, 46, 51, 50, 49, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 56, 53, 50, 55, 101, 43, 48, 49, + 0, 0, 0, 0, 0, 100, 0, 5, 102, 97, 108, 115, 101, 98, 0, 8, 74, 89, 106}; - try { - OtpOutputStream os = TermToBinaryCodec.encodeTsGetRequest(TABLE_NAME, Arrays.asList(key), 5000); - os.flush(); - byte[] msg = os.toByteArray(); - Assert.assertArrayEquals(exp, msg); - } catch (IOException ex) { - Assert.fail(ex.getMessage()); - } - } + Cell c1 = new Cell("varchar"); + Cell c2 = new Cell(12345678L); + Cell c3 = new Cell(12.34); + Cell c4 = new Cell(true); + Calendar cal1 = Calendar.getInstance(); + cal1.setTimeInMillis(12345); + Cell c5 = new Cell(cal1); + Row r1 = new Row(c1, c2, c3, c4, c5); - @Test - public void encodesQueryRequestCorrectly() { - // {tsqueryreq, {tsinterpolation, <<"SELECT * FROM FRAZZLE">>, []}, false, []} - final byte[] exp = {(byte)131, 104, 4, 100, 0, 10, 116, 115, 113, 117, 101, 114, 121, 114, 101, - 113, 104, 3, 100, 0, 15, 116, 115, 105, 110, 116, 101, 114, 112, 111, - 108, 97, 116, 105, 111, 110, 109, 0, 0, 0, 21, 83, 69, 76, 69, 67, 84, - 32, 42, 32, 70, 82, 79, 77, 32, 70, 82, 65, 90, 90, 76, 69, 106, 100, 0, - 5, 102, 97, 108, 115, 101, 106}; + Cell c6 = new Cell("string"); + Cell c7 = new Cell(8765432L); + Cell c8 = new Cell(43.21); + Cell c9 = new Cell(false); + Calendar cal2 = Calendar.getInstance(); + cal2.setTimeInMillis(543321); + Cell c10 = new Cell(cal2); + Row r2 = new Row(c6, c7, c8, c9, c10); + Row[] rows = { r1, r2 }; try { - OtpOutputStream os = TermToBinaryCodec.encodeTsQueryRequest(QUERY); + OtpOutputStream os = TermToBinaryCodec.encodeTsPutRequest(TABLE_NAME, Arrays.asList(rows)); os.flush(); byte[] msg = os.toByteArray(); Assert.assertArrayEquals(exp, msg); @@ -63,7 +62,7 @@ public void encodesQueryRequestCorrectly() { } @Test - public void encodesPutRequestCorrectly() { + public void encodesPutRequestCorrectly_2() { // What erlang generates // A = riakc_ts_put_operator:serialize(<<"test_table">>,[{<<"series">>, <<"family">>, 12345678, 1, true, 34.3, []}], true). // A = {tsputreq,<<"test_table">>,[],[{<<"series">>,<<"family">>,12345678,1,true,34.3,[]}]} @@ -96,4 +95,46 @@ public void encodesPutRequestCorrectly() { Assert.fail(ex.getMessage()); } } + + @Test + public void encodesGetRequestCorrectly() { + // {tsgetreq, <<"test_table">>, [<<"series">>, <<"family">>, 12345678], 5000} + final byte[] exp = {(byte)131, 104, 4, 100, 0, 8, 116, 115, 103, 101, 116, 114, 101, 113, 109, 0, + 0, 0, 10, 116, 101, 115, 116, 95, 116, 97, 98, 108, 101, 108, 0, 0, 0, 3, + 109, 0, 0, 0, 6, 115, 101, 114, 105, 101, 115, 109, 0, 0, 0, 6, 102, 97, + 109, 105, 108, 121, 98, 0, (byte)188, 97, 78, 106, 98, 0, 0, 19, (byte)136}; + + Cell k1 = new Cell("series"); + Cell k2 = new Cell("family"); + Cell k3 = new Cell(12345678); + Cell[] key = {k1, k2, k3}; + + try { + OtpOutputStream os = TermToBinaryCodec.encodeTsGetRequest(TABLE_NAME, Arrays.asList(key), 5000); + os.flush(); + byte[] msg = os.toByteArray(); + Assert.assertArrayEquals(exp, msg); + } catch (IOException ex) { + Assert.fail(ex.getMessage()); + } + } + + @Test + public void encodesQueryRequestCorrectly() { + // {tsqueryreq, {tsinterpolation, <<"SELECT * FROM FRAZZLE">>, []}, false, []} + final byte[] exp = {(byte)131, 104, 4, 100, 0, 10, 116, 115, 113, 117, 101, 114, 121, 114, 101, + 113, 104, 3, 100, 0, 15, 116, 115, 105, 110, 116, 101, 114, 112, 111, + 108, 97, 116, 105, 111, 110, 109, 0, 0, 0, 21, 83, 69, 76, 69, 67, 84, + 32, 42, 32, 70, 82, 79, 77, 32, 70, 82, 65, 90, 90, 76, 69, 106, 100, 0, + 5, 102, 97, 108, 115, 101, 106}; + + try { + OtpOutputStream os = TermToBinaryCodec.encodeTsQueryRequest(QUERY); + os.flush(); + byte[] msg = os.toByteArray(); + Assert.assertArrayEquals(exp, msg); + } catch (IOException ex) { + Assert.fail(ex.getMessage()); + } + } } From e5e650c6315a150dae925086efdf994e3c254c62 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 21 Apr 2016 10:09:26 -0700 Subject: [PATCH 32/52] Fix unit tests for TTB codec. --- .../client/core/codec/TermToBinaryCodec.java | 10 ++-- .../core/codec/TermToBinaryCodecTest.java | 48 ++++++++++--------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index d2c42c43e..dac3a718b 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -120,14 +120,10 @@ public static OtpOutputStream encodeTsPutRequest(String tableName, Collection>,[{<<"series">>, <<"family">>, 12345678, 1, true, 34.3, []}], true). // A = {tsputreq,<<"test_table">>,[],[{<<"series">>,<<"family">>,12345678,1,true,34.3,[]}]} - // rp(term_to_binary(A)). => exp array below - - // # What we generate - // B = <<-125, 104, 4, 100, 0, 8, 116, 115, 112, 117, 116, 114, 101, 113, 109, 0, 0, 0, 10, 116, 101, 115, 116, 95, 116, 97, 98, 108, 101, 106, 108, 0, 0, 0, 1, 104, 7, 109, 0, 0, 0, 6, 115, 101, 114, 105, 101, 115, 109, 0, 0, 0, 6, 102, 97, 109, 105, 108, 121, 98, 0, -68, 97, 78, 97, 1, 100, 0, 4, 116, 114, 117, 101, 70, 64, 65, 38, 102, 102, 102, 102, 102, 106, 106, 106>>. - // C = binary_to_term(B). - // C = {tsputreq,<<"test_table">>,[],[{<<"series">>,<<"family">>,12345678,1,true,34.3,[]}]} - // A = C. #true (wtf) (magic) - - final byte[] exp = {(byte)131,104,4,100,0,8,116,115,112,117,116,114,101,113,109,0, - 0,0,10,116,101,115,116,95,116,97,98,108,101,106,108,0,0, - 0,1,104,7,109,0,0,0,6,115,101,114,105,101,115,109,0,0,0, - 6,102,97,109,105,108,121,98,0,(byte)188,97,78,97,1,100,0,4, - 116,114,117,101,99,51,46,52,50,57,57,57,57,57,57,57,57, - 57,57,57,57,57,55,49,53,55,56,101,43,48,49,0,0,0,0,0, - 106,106}; + final byte[] exp = {(byte)131,104,4, // outer tuple arity 4 + 100,0,8,116,115,112,117,116,114,101,113, // tsputreq atom + 109,0,0,0,10,116,101,115,116,95,116,97,98,108,101, // table name binary + 106, // empty list + 108,0,0,0,1, // list start arity 1 + 104,7, // row tuple arity 7 + 109,0,0,0,6,115,101,114,105,101,115, // series binary + 109,0,0,0,6,102,97,109,105,108,121, // family binary + 98,0,(byte)188,97,78, // integer + 97,1, // small integer + 100,0,4,116,114,117,101, // true atom + // NB: this is what Erlang generates, an old-style float + // 99,51,46,52,50,57,57,57,57,57,57,57,57,57,57,57,57,57,55,49,53,55,56,101,43,48,49,0,0,0,0,0, // float_ext len 31 + // NB: this is what JInterface generates, a new-style float + 70, 64, 65, 38, 102, 102, 102, 102, 102, + 106, // null cell empty list + 106}; // list arity 1 end final ArrayList rows = new ArrayList<>(1); rows.add(new Row(new Cell("series"), new Cell("family"), Cell.newTimestamp(12345678), From 92e31eafe61b6e3d1d53d8c15b7c10999c7c7b02 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 21 Apr 2016 11:17:00 -0700 Subject: [PATCH 33/52] Remove the double-header from TTB data. --- .../core/operations/ts/TTBConverters.java | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java index e91453838..d55f6a2b7 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java @@ -4,13 +4,9 @@ import com.basho.riak.client.core.operations.TTBFutureOperation; import com.basho.riak.client.core.query.timeseries.Cell; import com.basho.riak.client.core.query.timeseries.QueryResult; -import com.basho.riak.protobuf.RiakMessageCodes; import com.ericsson.otp.erlang.OtpErlangDecodeException; import com.ericsson.otp.erlang.OtpOutputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; import java.io.IOException; - import java.util.LinkedList; import java.util.logging.Level; import java.util.logging.Logger; @@ -19,7 +15,6 @@ class TTBConverters { private static abstract class MemoizingEncoder implements TTBFutureOperation.TTBEncoder { - // private final byte reqMessageCode = RiakMessageCodes.MSG_TsTtbMsg; protected byte[] message = null; protected final T builder; @@ -38,20 +33,7 @@ public byte[] build() try { OtpOutputStream os = buildMessage(); os.flush(); - - // TODO GH-611 should the output stream or base type be returned? - // TODO GH-611 we still need to add msgCode 141 and message length - final int msgLen = os.length() + 5; // Add 4-byte msg length and 1-byte msgCode - ByteArrayOutputStream bs = new ByteArrayOutputStream(msgLen); - - DataOutputStream ds = new DataOutputStream(bs); - ds.writeInt(msgLen); - ds.writeByte(RiakMessageCodes.MSG_TsTtbMsg); - ds.flush(); - - os.writeTo(bs); - bs.flush(); - message = bs.toByteArray(); + message = os.toByteArray(); } catch (IOException ex) { // TODO GH-611 Logger.getLogger(TTBConverters.class.getName()).log(Level.SEVERE, null, ex); From ac3101775d583f88c595231409c4ce4110c83d71 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Fri, 22 Apr 2016 10:25:26 -0400 Subject: [PATCH 34/52] Fix a few encoding/decoding issues --- .../basho/riak/client/core/RiakMessage.java | 5 ++-- .../client/core/codec/TermToBinaryCodec.java | 24 ++++++++++++------- .../core/codec/TermToBinaryCodecTest.java | 11 +++++---- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/RiakMessage.java b/src/main/java/com/basho/riak/client/core/RiakMessage.java index 067c38af7..982d2442d 100644 --- a/src/main/java/com/basho/riak/client/core/RiakMessage.java +++ b/src/main/java/com/basho/riak/client/core/RiakMessage.java @@ -23,6 +23,7 @@ public final class RiakMessage private final byte code; private final byte[] data; private final OtpInputStream ttbInputStream; + // TODO offer output stream? private final RiakResponseException riakError; public RiakMessage(byte code, byte[] data) @@ -113,8 +114,8 @@ private RiakResponseException getRiakErrorFromTtb(OtpInputStream ttbInputStream) logger.error("exception", ex); } } - } + } return null; } -} \ No newline at end of file +} diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index dac3a718b..54af650a2 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -90,8 +90,8 @@ public static OtpOutputStream encodeTsQueryRequest(String queryText) // streaming is false for now os.write_boolean(false); - // cover_context is an empty list - os.write_nil(); + // cover_context is an undefined atom + os.write_any(undefined); return os; } @@ -163,29 +163,29 @@ private static QueryResult decodeTsResponse(byte[] response) throws OtpErlangDec assert(dataArity == 3); final int colNameCount = is.read_list_head(); - String[] columnNames = new String[colNameCount]; + final String[] columnNames = new String[colNameCount]; for (int i = 0; i < colNameCount; i++) { - String colName = is.read_string(); + final String colName = new String(is.read_binary(), StandardCharsets.UTF_8); columnNames[i] = colName; } final int colTypeCount = is.read_list_head(); assert(colNameCount == colTypeCount); - String[] columnTypes = new String[colTypeCount]; + final String[] columnTypes = new String[colTypeCount]; for (int i = 0; i < colTypeCount; i++) { - String colType = is.read_string(); + final String colType = new String(is.read_binary(), StandardCharsets.UTF_8); columnTypes[i] = colType; } final int rowCount = is.read_list_head(); - Row[] rows = new Row[rowCount]; + final Row[] rows = new Row[rowCount]; for (int i = 0; i < rowCount; i++) { final int rowDataCount = is.read_tuple_head(); assert(colNameCount == rowDataCount); - Cell[] cells = new Cell[rowDataCount]; + final Cell[] cells = new Cell[rowDataCount]; for (int j = 0; i < rowDataCount; j++) { - OtpErlangObject cell = is.read_any(); + final OtpErlangObject cell = is.read_any(); if (cell instanceof OtpErlangBinary) { OtpErlangBinary v = (OtpErlangBinary)cell; // TODO GH-611 this may not be correct encoding @@ -210,6 +210,12 @@ else if (cell instanceof OtpErlangDouble) { OtpErlangDouble v = (OtpErlangDouble)cell; cells[j] = new Cell(v.doubleValue()); } + else if (cell instanceof OtpErlangList) + { + final OtpErlangList l = (OtpErlangList)cell; + assert(l.arity() == 0); + cells[j] = null; + } else { // TODO GH-611 throw exception? } diff --git a/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java b/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java index 23489dd2c..1b62af90c 100644 --- a/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java +++ b/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java @@ -124,11 +124,12 @@ public void encodesGetRequestCorrectly() { @Test public void encodesQueryRequestCorrectly() { // {tsqueryreq, {tsinterpolation, <<"SELECT * FROM FRAZZLE">>, []}, false, []} - final byte[] exp = {(byte)131, 104, 4, 100, 0, 10, 116, 115, 113, 117, 101, 114, 121, 114, 101, - 113, 104, 3, 100, 0, 15, 116, 115, 105, 110, 116, 101, 114, 112, 111, - 108, 97, 116, 105, 111, 110, 109, 0, 0, 0, 21, 83, 69, 76, 69, 67, 84, - 32, 42, 32, 70, 82, 79, 77, 32, 70, 82, 65, 90, 90, 76, 69, 106, 100, 0, - 5, 102, 97, 108, 115, 101, 106}; + final byte[] exp = {(byte)131,104,4,100,0,10,116,115,113,117,101,114,121,114,101, + 113,104,3,100,0,15,116,115,105,110,116,101,114,112,111, + 108,97,116,105,111,110,109,0,0,0,21,83,69,76,69,67,84, + 32,42,32,70,82,79,77,32,70,82,65,90,90,76,69,106,100,0, + 5,102,97,108,115,101,100,0,9,117,110,100,101,102,105, + 110,101,100}; try { OtpOutputStream os = TermToBinaryCodec.encodeTsQueryRequest(QUERY); From e90065ea37e0c2908530612545ab558881b730a2 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Fri, 22 Apr 2016 15:59:32 -0400 Subject: [PATCH 35/52] Fix codec to parse QueryResponses correctly, add some error handling around parsing bad TTB --- .../riak/client/core/FutureOperation.java | 39 ++- .../codec/InvalidTermToBinaryException.java | 18 ++ .../client/core/codec/TermToBinaryCodec.java | 254 +++++++++++------- .../core/operations/ts/TTBConverters.java | 9 +- .../core/codec/TermToBinaryCodecTest.java | 78 +++++- 5 files changed, 285 insertions(+), 113 deletions(-) create mode 100644 src/main/java/com/basho/riak/client/core/codec/InvalidTermToBinaryException.java diff --git a/src/main/java/com/basho/riak/client/core/FutureOperation.java b/src/main/java/com/basho/riak/client/core/FutureOperation.java index 95528d4f0..f7bbd1622 100644 --- a/src/main/java/com/basho/riak/client/core/FutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/FutureOperation.java @@ -16,6 +16,8 @@ package com.basho.riak.client.core; +import com.basho.riak.client.api.RiakException; +import com.sun.javaws.exceptions.InvalidArgumentException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -309,17 +311,35 @@ public final T get() throws InterruptedException, ExecutionException { latch.await(); + throwExceptionIfSet(); + + if (null == converted) + { + tryConvertResponse(); + } + + return converted; + } + + private void throwExceptionIfSet() throws ExecutionException + { if (exception != null) { throw new ExecutionException(exception); } - else if (null == converted) + } + + private void tryConvertResponse() throws ExecutionException + { + try { converted = convert(rawResponse); - } - - return converted; + catch(IllegalArgumentException ex) + { + exception = ex; + throwExceptionIfSet(); + } } @Override @@ -331,13 +351,12 @@ public final T get(long timeout, TimeUnit unit) throws InterruptedException, Exe { throw new TimeoutException(); } - else if (exception != null) - { - throw new ExecutionException(exception); - } - else if (null == converted) + + throwExceptionIfSet(); + + if (null == converted) { - converted = convert(rawResponse); + tryConvertResponse(); } return converted; diff --git a/src/main/java/com/basho/riak/client/core/codec/InvalidTermToBinaryException.java b/src/main/java/com/basho/riak/client/core/codec/InvalidTermToBinaryException.java new file mode 100644 index 000000000..97146d42b --- /dev/null +++ b/src/main/java/com/basho/riak/client/core/codec/InvalidTermToBinaryException.java @@ -0,0 +1,18 @@ +package com.basho.riak.client.core.codec; + +import com.ericsson.otp.erlang.OtpErlangDecodeException; + +import java.io.IOException; + +public class InvalidTermToBinaryException extends IOException +{ + public InvalidTermToBinaryException(String errorMessage, OtpErlangDecodeException cause) + { + super(errorMessage, cause); + } + + public InvalidTermToBinaryException(String errorMessage) + { + super(errorMessage); + } +} diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index 54af650a2..519e4e1c1 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -1,32 +1,22 @@ package com.basho.riak.client.core.codec; +import com.basho.riak.client.api.RiakException; import com.basho.riak.client.core.query.timeseries.Cell; import com.basho.riak.client.core.query.timeseries.QueryResult; import com.basho.riak.client.core.query.timeseries.Row; -import com.ericsson.otp.erlang.OtpErlangAtom; -import com.ericsson.otp.erlang.OtpErlangBinary; -import com.ericsson.otp.erlang.OtpErlangBoolean; -import com.ericsson.otp.erlang.OtpErlangDecodeException; -import com.ericsson.otp.erlang.OtpErlangDouble; -import com.ericsson.otp.erlang.OtpErlangList; -import com.ericsson.otp.erlang.OtpErlangLong; -import com.ericsson.otp.erlang.OtpErlangObject; -import com.ericsson.otp.erlang.OtpExternal; -import com.ericsson.otp.erlang.OtpInputStream; -import com.ericsson.otp.erlang.OtpOutputStream; +import com.basho.riak.protobuf.RiakTsPB; +import com.ericsson.otp.erlang.*; +import com.google.protobuf.ByteString; + import java.nio.charset.StandardCharsets; -import java.util.Calendar; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; +import java.util.Locale; public class TermToBinaryCodec { - private static final OtpErlangAtom undefined = new OtpErlangAtom("undefined"); - private static final OtpErlangAtom tsinterpolation = new OtpErlangAtom("tsinterpolation"); - - private static final OtpErlangAtom _false = new OtpErlangAtom("false"); - private static final OtpErlangList EMPTY_ERLANG_LIST = new OtpErlangList(); - - private static class Messages + private static final class Messages { public static final OtpErlangAtom tsGetReq = new OtpErlangAtom("tsgetreq"); public static final OtpErlangAtom tsGetResp = new OtpErlangAtom("tsgetresp"); @@ -44,6 +34,8 @@ private static class Messages public static final OtpErlangAtom rpbErrorResp = new OtpErlangAtom("rpberrorresp"); } + private static final OtpErlangAtom undefined = new OtpErlangAtom("undefined"); + public static OtpOutputStream encodeTsGetRequest(String tableName, Collection keyValues, int timeout) { final OtpOutputStream os = new OtpOutputStream(); @@ -55,7 +47,8 @@ public static OtpOutputStream encodeTsGetRequest(String tableName, Collection columnDescriptions = parseColumnDescriptions(is); - final int rowCount = is.read_list_head(); - final Row[] rows = new Row[rowCount]; - for (int i = 0; i < rowCount; i++) { - final int rowDataCount = is.read_tuple_head(); - assert(colNameCount == rowDataCount); - - final Cell[] cells = new Cell[rowDataCount]; - for (int j = 0; i < rowDataCount; j++) { - final OtpErlangObject cell = is.read_any(); - if (cell instanceof OtpErlangBinary) { - OtpErlangBinary v = (OtpErlangBinary)cell; - // TODO GH-611 this may not be correct encoding - String s = new String(v.binaryValue(), StandardCharsets.UTF_8); - cells[j] = new Cell(s); - } - else if (cell instanceof OtpErlangBoolean) { - OtpErlangBoolean v = (OtpErlangBoolean)cell; - cells[j] = new Cell(v.booleanValue()); - } - else if (cell instanceof OtpErlangLong) { - OtpErlangLong v = (OtpErlangLong)cell; - if ("timestamp".equals(columnTypes[j])) { - Calendar cal = Calendar.getInstance(); - cal.setTimeInMillis(v.longValue()); - cells[j] = new Cell(cal); - } else { - cells[j] = new Cell(v.longValue()); - } - } - else if (cell instanceof OtpErlangDouble) { - OtpErlangDouble v = (OtpErlangDouble)cell; - cells[j] = new Cell(v.doubleValue()); - } - else if (cell instanceof OtpErlangList) - { - final OtpErlangList l = (OtpErlangList)cell; - assert(l.arity() == 0); - cells[j] = null; - } - else { - // TODO GH-611 throw exception? - } - } - - rows[i] = new Row(cells); - } + final ArrayList rows = parseRows(is, columnDescriptions); - result = new QueryResult(rows); + result = new QueryResult(columnDescriptions, rows); break; default: @@ -233,4 +168,127 @@ else if (cell instanceof OtpErlangList) return result; } + + private static ArrayList parseColumnDescriptions(OtpInputStream is) + throws OtpErlangDecodeException + { + final int colNameCount = is.read_list_head(); + final String[] columnNames = new String[colNameCount]; + for (int colNameIdx = 0; colNameIdx < colNameCount; colNameIdx++) + { + final String colName = new String(is.read_binary(), StandardCharsets.UTF_8); + columnNames[colNameIdx] = colName; + + final boolean isLastRow = colNameIdx + 1 == colNameCount; + if (isLastRow) + { + is.read_nil(); + } + } + + + final int colTypeCount = is.read_list_head(); + assert (colNameCount == colTypeCount); + final String[] columnTypes = new String[colTypeCount]; + + for (int colTypeIdx = 0; colTypeIdx < colTypeCount; colTypeIdx++) + { + final String colType = is.read_atom(); + columnTypes[colTypeIdx] = colType; + + final boolean isLastRow = colTypeIdx + 1 == colNameCount; + if (isLastRow) + { + is.read_nil(); + } + } + + final ArrayList columnDescriptions = new ArrayList<>(colNameCount); + for (int colDescIdx = 0; colDescIdx < colNameCount; colDescIdx++) + { + + final RiakTsPB.TsColumnDescription desc = RiakTsPB.TsColumnDescription.newBuilder().setName( + ByteString.copyFromUtf8(columnNames[colDescIdx])).setType(RiakTsPB.TsColumnType.valueOf( + columnTypes[colDescIdx].toUpperCase(Locale.US))).build(); + columnDescriptions.add(desc); + } + return columnDescriptions; + } + + private static ArrayList parseRows(OtpInputStream is, List columnDescriptions) + throws OtpErlangDecodeException, InvalidTermToBinaryException + { + final int rowCount = is.read_list_head(); + final ArrayList rows = new ArrayList<>(rowCount); + + for (int rowIdx = 0; rowIdx < rowCount; rowIdx++) + { + final boolean isLastRow = rowIdx + 1 == columnDescriptions.size(); + rows.add(parseRow(is, columnDescriptions, isLastRow)); + } + return rows; + } + + private static RiakTsPB.TsRow parseRow(OtpInputStream is, List columnDescriptions, Boolean isLastRow) + throws OtpErlangDecodeException, InvalidTermToBinaryException + { + final int rowDataCount = is.read_tuple_head(); + assert (columnDescriptions.size() == rowDataCount); + + final Cell[] cells = new Cell[rowDataCount]; + for (int j = 0; j < rowDataCount; j++) + { + final OtpErlangObject cell = is.read_any(); + cells[j] = parseCell(columnDescriptions, j, cell); + } + + if (isLastRow) + { + is.read_nil(); + } + + return new Row(cells).getPbRow(); + } + + private static Cell parseCell(List columnDescriptions, int j, OtpErlangObject cell) throws InvalidTermToBinaryException + { + if (cell instanceof OtpErlangBinary) + { + OtpErlangBinary v = (OtpErlangBinary) cell; + String s = new String(v.binaryValue(), StandardCharsets.UTF_8); + return new Cell(s); + } + else if (cell instanceof OtpErlangLong) + { + OtpErlangLong v = (OtpErlangLong) cell; + if (columnDescriptions.get(j).getType() == RiakTsPB.TsColumnType.TIMESTAMP) + { + return Cell.newTimestamp(v.longValue()); + } + else + { + return new Cell(v.longValue()); + } + } + else if (cell instanceof OtpErlangDouble) + { + OtpErlangDouble v = (OtpErlangDouble) cell; + return new Cell(v.doubleValue()); + } + else if (cell instanceof OtpErlangAtom) + { + OtpErlangAtom v = (OtpErlangAtom) cell; + return new Cell(v.booleanValue()); + } + else if (cell instanceof OtpErlangList) + { + final OtpErlangList l = (OtpErlangList) cell; + assert (l.arity() == 0); + return null; + } + else + { + throw new InvalidTermToBinaryException("Unknown cell type encountered: " + cell.toString() + ", unable to continue parsing."); + } + } } diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java index d55f6a2b7..a275a55c4 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java @@ -1,5 +1,7 @@ package com.basho.riak.client.core.operations.ts; +import com.basho.riak.client.api.RiakException; +import com.basho.riak.client.core.codec.InvalidTermToBinaryException; import com.basho.riak.client.core.codec.TermToBinaryCodec; import com.basho.riak.client.core.operations.TTBFutureOperation; import com.basho.riak.client.core.query.timeseries.Cell; @@ -108,13 +110,14 @@ static class QueryResultDecoder implements TTBFutureOperation.TTBParser>,<<"user">>,<<"time">>,<<"weather">>,<<"temperature">>,<<"uv_index">>,<<"observed">>], + [varchar,varchar,timestamp,varchar,double,sint64,boolean], + [ + {<<"hash1">>,<<"user2">>,1443806600000,<<"cloudy">>,[],[],true} + ] + } + } + */ + + final byte[] input = + {-125, 104, 2, 100, 0, 11, 116, 115, 113, 117, 101, 114, 121, 114, 101, 115, 112, 104, 3, 108, 0, 0, + 0, 7, 109, 0, 0, 0, 7, 103, 101, 111, 104, 97, 115, 104, 109, 0, 0, 0, 4, 117, 115, 101, 114, 109, + 0, 0, 0, 4, 116, 105, 109, 101, 109, 0, 0, 0, 7, 119, 101, 97, 116, 104, 101, 114, 109, 0, 0, 0, 11, + 116, 101, 109, 112, 101, 114, 97, 116, 117, 114, 101, 109, 0, 0, 0, 8, 117, 118, 95, 105, 110, 100, + 101, 120, 109, 0, 0, 0, 8, 111, 98, 115, 101, 114, 118, 101, 100, 106, 108, 0, 0, 0, 7, 100, 0, 7, + 118, 97, 114, 99, 104, 97, 114, 100, 0, 7, 118, 97, 114, 99, 104, 97, 114, 100, 0, 9, 116, 105, 109, + 101, 115, 116, 97, 109, 112, 100, 0, 7, 118, 97, 114, 99, 104, 97, 114, 100, 0, 6, 100, 111, 117, + 98, 108, 101, 100, 0, 6, 115, 105, 110, 116, 54, 52, 100, 0, 7, 98, 111, 111, 108, 101, 97, 110, + 106, 108, 0, 0, 0, 1, 104, 7, 109, 0, 0, 0, 5, 104, 97, 115, 104, 49, 109, 0, 0, 0, 5, 117, 115, + 101, 114, 50, 110, 6, 0, 64, 91, -108, 41, 80, 1, 109, 0, 0, 0, 6, 99, 108, 111, 117, 100, 121, 106, + 106, 100, 0, 4, 116, 114, 117, 101, 106}; + + final ColumnDescription[] expectedColumnDescriptions = new ColumnDescription[7]; + expectedColumnDescriptions[0] = new ColumnDescription("geohash", ColumnDescription.ColumnType.VARCHAR); + expectedColumnDescriptions[1] = new ColumnDescription("user", ColumnDescription.ColumnType.VARCHAR); + expectedColumnDescriptions[2] = new ColumnDescription("time", ColumnDescription.ColumnType.TIMESTAMP); + expectedColumnDescriptions[3] = new ColumnDescription("weather", ColumnDescription.ColumnType.VARCHAR); + expectedColumnDescriptions[4] = new ColumnDescription("temperature", ColumnDescription.ColumnType.DOUBLE); + expectedColumnDescriptions[5] = new ColumnDescription("uv_index", ColumnDescription.ColumnType.SINT64); + expectedColumnDescriptions[6] = new ColumnDescription("observed", ColumnDescription.ColumnType.BOOLEAN); + + final Row row = new Row(new Cell("hash1"), new Cell("user2"), Cell.newTimestamp(1443806600000L), new Cell("cloudy"), null, null, new Cell(true)); + final Row[] expectedRows = new Row[1]; + expectedRows[0] = (row); + + try + { + final QueryResult actual = TermToBinaryCodec.decodeTsQueryResponse(input); + + final List actualColumnDescriptions = actual.getColumnDescriptionsCopy(); + final List actualRows = actual.getRowsCopy(); + + Assert.assertArrayEquals(expectedColumnDescriptions, + actualColumnDescriptions.toArray(new ColumnDescription[actualColumnDescriptions.size()])); + + Assert.assertArrayEquals(expectedRows, actualRows.toArray(new Row[actualRows.size()])); + } + catch (InvalidTermToBinaryException ex) + { + Assert.fail(ex.getMessage()); + } + + } } From fe88d2b6386b208fa7273809fe316c120f46514e Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Fri, 22 Apr 2016 21:58:56 -0400 Subject: [PATCH 36/52] Fix a ttb row read bug, change a test to match current behavior --- .../riak/client/core/codec/TermToBinaryCodec.java | 11 ++--------- .../client/api/commands/itest/ITestTimeSeries.java | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index 519e4e1c1..a549c8fbf 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -1,6 +1,5 @@ package com.basho.riak.client.core.codec; -import com.basho.riak.client.api.RiakException; import com.basho.riak.client.core.query.timeseries.Cell; import com.basho.riak.client.core.query.timeseries.QueryResult; import com.basho.riak.client.core.query.timeseries.Row; @@ -223,13 +222,12 @@ private static ArrayList parseRows(OtpInputStream is, List columnDescriptions, Boolean isLastRow) + private static RiakTsPB.TsRow parseRow(OtpInputStream is, List columnDescriptions) throws OtpErlangDecodeException, InvalidTermToBinaryException { final int rowDataCount = is.read_tuple_head(); @@ -242,11 +240,6 @@ private static RiakTsPB.TsRow parseRow(OtpInputStream is, List GetCreatedTableFullDescriptions() From e4ef4758e3585e3a3812ce8d8d956a1e81194360 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Sun, 24 Apr 2016 18:49:00 -0700 Subject: [PATCH 37/52] Simplify detection of when to read end of list. --- .../riak/client/core/FutureOperation.java | 7 ++----- .../client/core/codec/TermToBinaryCodec.java | 20 ++++++++----------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/FutureOperation.java b/src/main/java/com/basho/riak/client/core/FutureOperation.java index f7bbd1622..0e0c2aa64 100644 --- a/src/main/java/com/basho/riak/client/core/FutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/FutureOperation.java @@ -16,11 +16,6 @@ package com.basho.riak.client.core; -import com.basho.riak.client.api.RiakException; -import com.sun.javaws.exceptions.InvalidArgumentException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.Arrays; import java.util.HashSet; import java.util.LinkedList; @@ -30,6 +25,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.locks.ReentrantLock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author Brian Roach diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index a549c8fbf..5399b8cfa 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -6,7 +6,6 @@ import com.basho.riak.protobuf.RiakTsPB; import com.ericsson.otp.erlang.*; import com.google.protobuf.ByteString; - import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; @@ -177,29 +176,26 @@ private static ArrayList parseColumnDescriptions(O { final String colName = new String(is.read_binary(), StandardCharsets.UTF_8); columnNames[colNameIdx] = colName; + } - final boolean isLastRow = colNameIdx + 1 == colNameCount; - if (isLastRow) - { - is.read_nil(); - } + if (colNameCount > 0) + { + is.read_nil(); } final int colTypeCount = is.read_list_head(); assert (colNameCount == colTypeCount); final String[] columnTypes = new String[colTypeCount]; - for (int colTypeIdx = 0; colTypeIdx < colTypeCount; colTypeIdx++) { final String colType = is.read_atom(); columnTypes[colTypeIdx] = colType; + } - final boolean isLastRow = colTypeIdx + 1 == colNameCount; - if (isLastRow) - { - is.read_nil(); - } + if (colTypeCount > 0) + { + is.read_nil(); } final ArrayList columnDescriptions = new ArrayList<>(colNameCount); From 8834f0736058e9ddc8670a76ed59a3ef2d56f2a5 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Mon, 25 Apr 2016 17:00:06 -0400 Subject: [PATCH 38/52] Add port option + TS testing to makefile --- Makefile | 7 +- pom.xml | 5 + pom.xml.orig | 446 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 457 insertions(+), 1 deletion(-) create mode 100755 pom.xml.orig diff --git a/Makefile b/Makefile index 0f2255718..e2cddbeb8 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ .PHONY: compile test unit-test integration-test protogen +PORT = 8087 + compile: mvn clean compile @@ -9,7 +11,10 @@ unit-test: mvn test integration-test: - mvn -Pitest,default -Dcom.basho.riak.2i=true -Dcom.basho.riak.yokozuna=true -Dcom.basho.riak.buckettype=true -Dcom.basho.riak.crdt=true -Dcom.basho.riak.lifecycle=true verify + mvn -Pitest,default -Dcom.basho.riak.2i=true -Dcom.basho.riak.yokozuna=true -Dcom.basho.riak.buckettype=true -Dcom.basho.riak.crdt=true -Dcom.basho.riak.lifecycle=true -Dcom.basho.riak.pbcport=$(PORT)verify + +integration-test-timeseries: + mvn -Pitest,default -Dcom.basho.riak.buckettype=true -Dcom.basho.riak.crdt=true -Dcom.basho.riak.lifecycle=true -Dcom.basho.riak.timeseries=true -Dcom.basho.riak.pbcport=$(PORT) verify protogen: mvn -Pprotobuf-generate generate-sources diff --git a/pom.xml b/pom.xml index 1e4d96bdb..e8fe8d21a 100755 --- a/pom.xml +++ b/pom.xml @@ -431,5 +431,10 @@ antlr-complete 3.5.2 + + org.erlang.otp + jinterface + 1.5.6 + diff --git a/pom.xml.orig b/pom.xml.orig new file mode 100755 index 000000000..e479ea5c9 --- /dev/null +++ b/pom.xml.orig @@ -0,0 +1,446 @@ + + 4.0.0 + com.basho.riak + riak-client + jar + 2.0.6-SNAPSHOT + Riak Client for Java + Java client for Riak 2.0 + https://github.com/basho/riak-java-client + + 3.0 + + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + Brian Roach + roach@basho.com + Basho Technologies, Inc + http://www.basho.com + + + David Rusek + drusek@basho.com + Basho Technologies, Inc + http://www.basho.com + + + Alex Moore + amoore@basho.com + Basho Technologies, Inc + http://www.basho.com + + + Chris Mancini + cmancini@basho.com + Basho Technologies, Inc + http://www.basho.com + + + Luke Bakken + lbakken@basho.com + Basho Technologies, Inc + http://www.basho.com + + + + + scm:git:ssh://git@github.com/basho/riak-java-client.git + scm:git:ssh://git@github.com/basho/riak-java-client.git + http://github.com/basho/riak-java-client + HEAD + + + + UTF-8 + UTF-8 + 1.6.4 + + + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots + + + sonatype-nexus-staging + Nexus Release Repository + https://oss.sonatype.org/service/local/staging/deploy/maven2 + + + + + + default + + true + + + + org.slf4j + slf4j-nop + 1.7.5 + test + + + + + test-debug-logging + + + org.slf4j + slf4j-simple + 1.7.5 + test + + + + + itest + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.19.1 + + + + integration-test + verify + + + + + + + + + jar-with-dependencies + + + bundle.fat + true + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.6 + + + src/main/assembly/jar-with-dependencies.xml + + + + + + + make-assembly + package + + single + + + + + + + + + release + + + performRelease + true + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + + + + + + + doclint-java8-disable + + [1.8,) + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + -Xdoclint:none + com.basho.riak.protobuf.util + + + + + + + protobuf-generate + + + protobuf-generate + true + + + + + + com.webguys + string-template-maven-plugin + 1.1 + + + + + + + + generate-sources + + render + + + + + + org.twdata.maven + mojo-executor + 2.2.0 + + + + + com.github.igor-petruk.protobuf + protobuf-maven-plugin + 0.6.5 + + false + + ${basedir}/riak_pb/src + + ${project.build.sourceDirectory} + + + + + run + + + + + + + + + versions-maven + + + + org.codehaus.mojo + versions-maven-plugin + 2.2 + + + + + + + + + maven-release-plugin + 2.5.2 + + forked-path + + + + org.jacoco + jacoco-maven-plugin + 0.7.4.201502262128 + + + jacoco-initialize + + prepare-agent + + + + jacoco-site + package + + report + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + 1.7 + 1.7 + + -Werror + -Xlint:all + + + + + + org.codehaus.mojo + animal-sniffer-maven-plugin + 1.14 + + + org.codehaus.mojo.signature + java17 + 1.0 + + + + + test + + check + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.14 + + + debug + true + yyyy-MM-dd HH:mm:ss + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.10.3 + + com.basho.riak.protobuf.util + + + + + + + org.slf4j + slf4j-api + 1.7.21 + + + org.mockito + mockito-core + 1.10.19 + test + + + org.powermock + powermock-api-mockito + ${powermock.version} + test + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + org.hamcrest + hamcrest-core + 1.3 + test + + + com.jayway.awaitility + awaitility + 1.3.5 + test + + + com.fasterxml.jackson.core + jackson-databind + 2.7.3 + + + com.fasterxml.jackson.datatype + jackson-datatype-joda + 2.7.3 + jar + + + io.netty + netty-all + 4.0.33.Final + + +<<<<<<< HEAD + com.basho.riak.protobuf + riak-pb + 2.1.1.1-SNAPSHOT + + + org.erlang.otp + jinterface + 1.5.6 +======= + com.google.protobuf + protobuf-java + 2.6.1 + + + org.antlr + antlr-complete + 3.5.2 +>>>>>>> develop + + + From 7243f389a790f258a107803ccf27e3987021c744 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Mon, 25 Apr 2016 16:02:04 -0700 Subject: [PATCH 39/52] Small Makefile change to only set RIAK_PORT if it isn't set in the env. Remove git artifact (.orig file) --- Makefile | 8 +- pom.xml.orig | 446 --------------------------------------------------- 2 files changed, 4 insertions(+), 450 deletions(-) delete mode 100755 pom.xml.orig diff --git a/Makefile b/Makefile index e2cddbeb8..e727eed1f 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -.PHONY: compile test unit-test integration-test protogen +.PHONY: compile test unit-test integration-test integration-test-timeseries protogen -PORT = 8087 +RIAK_PORT ?= 8087 compile: mvn clean compile @@ -11,10 +11,10 @@ unit-test: mvn test integration-test: - mvn -Pitest,default -Dcom.basho.riak.2i=true -Dcom.basho.riak.yokozuna=true -Dcom.basho.riak.buckettype=true -Dcom.basho.riak.crdt=true -Dcom.basho.riak.lifecycle=true -Dcom.basho.riak.pbcport=$(PORT)verify + mvn -Pitest,default -Dcom.basho.riak.2i=true -Dcom.basho.riak.yokozuna=true -Dcom.basho.riak.buckettype=true -Dcom.basho.riak.crdt=true -Dcom.basho.riak.lifecycle=true -Dcom.basho.riak.pbcport=$(RIAK_PORT) verify integration-test-timeseries: - mvn -Pitest,default -Dcom.basho.riak.buckettype=true -Dcom.basho.riak.crdt=true -Dcom.basho.riak.lifecycle=true -Dcom.basho.riak.timeseries=true -Dcom.basho.riak.pbcport=$(PORT) verify + mvn -Pitest,default -Dcom.basho.riak.buckettype=true -Dcom.basho.riak.crdt=true -Dcom.basho.riak.lifecycle=true -Dcom.basho.riak.timeseries=true -Dcom.basho.riak.pbcport=$(RIAK_PORT) verify protogen: mvn -Pprotobuf-generate generate-sources diff --git a/pom.xml.orig b/pom.xml.orig deleted file mode 100755 index e479ea5c9..000000000 --- a/pom.xml.orig +++ /dev/null @@ -1,446 +0,0 @@ - - 4.0.0 - com.basho.riak - riak-client - jar - 2.0.6-SNAPSHOT - Riak Client for Java - Java client for Riak 2.0 - https://github.com/basho/riak-java-client - - 3.0 - - - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - A business-friendly OSS license - - - - - - Brian Roach - roach@basho.com - Basho Technologies, Inc - http://www.basho.com - - - David Rusek - drusek@basho.com - Basho Technologies, Inc - http://www.basho.com - - - Alex Moore - amoore@basho.com - Basho Technologies, Inc - http://www.basho.com - - - Chris Mancini - cmancini@basho.com - Basho Technologies, Inc - http://www.basho.com - - - Luke Bakken - lbakken@basho.com - Basho Technologies, Inc - http://www.basho.com - - - - - scm:git:ssh://git@github.com/basho/riak-java-client.git - scm:git:ssh://git@github.com/basho/riak-java-client.git - http://github.com/basho/riak-java-client - HEAD - - - - UTF-8 - UTF-8 - 1.6.4 - - - - - sonatype-nexus-snapshots - Sonatype Nexus Snapshots - https://oss.sonatype.org/content/repositories/snapshots - - - sonatype-nexus-staging - Nexus Release Repository - https://oss.sonatype.org/service/local/staging/deploy/maven2 - - - - - - default - - true - - - - org.slf4j - slf4j-nop - 1.7.5 - test - - - - - test-debug-logging - - - org.slf4j - slf4j-simple - 1.7.5 - test - - - - - itest - - - - org.apache.maven.plugins - maven-failsafe-plugin - 2.19.1 - - - - integration-test - verify - - - - - - - - - jar-with-dependencies - - - bundle.fat - true - - - - - - org.apache.maven.plugins - maven-assembly-plugin - 2.6 - - - src/main/assembly/jar-with-dependencies.xml - - - - - - - make-assembly - package - - single - - - - - - - - - release - - - performRelease - true - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - - sign-artifacts - verify - - sign - - - - - - - - - - doclint-java8-disable - - [1.8,) - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - -Xdoclint:none - com.basho.riak.protobuf.util - - - - - - - protobuf-generate - - - protobuf-generate - true - - - - - - com.webguys - string-template-maven-plugin - 1.1 - - - - - - - - generate-sources - - render - - - - - - org.twdata.maven - mojo-executor - 2.2.0 - - - - - com.github.igor-petruk.protobuf - protobuf-maven-plugin - 0.6.5 - - false - - ${basedir}/riak_pb/src - - ${project.build.sourceDirectory} - - - - - run - - - - - - - - - versions-maven - - - - org.codehaus.mojo - versions-maven-plugin - 2.2 - - - - - - - - - maven-release-plugin - 2.5.2 - - forked-path - - - - org.jacoco - jacoco-maven-plugin - 0.7.4.201502262128 - - - jacoco-initialize - - prepare-agent - - - - jacoco-site - package - - report - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.5.1 - - 1.7 - 1.7 - - -Werror - -Xlint:all - - - - - - org.codehaus.mojo - animal-sniffer-maven-plugin - 1.14 - - - org.codehaus.mojo.signature - java17 - 1.0 - - - - - test - - check - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.14 - - - debug - true - yyyy-MM-dd HH:mm:ss - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.3 - - com.basho.riak.protobuf.util - - - - - - - org.slf4j - slf4j-api - 1.7.21 - - - org.mockito - mockito-core - 1.10.19 - test - - - org.powermock - powermock-api-mockito - ${powermock.version} - test - - - org.powermock - powermock-module-junit4 - ${powermock.version} - test - - - org.hamcrest - hamcrest-core - 1.3 - test - - - com.jayway.awaitility - awaitility - 1.3.5 - test - - - com.fasterxml.jackson.core - jackson-databind - 2.7.3 - - - com.fasterxml.jackson.datatype - jackson-datatype-joda - 2.7.3 - jar - - - io.netty - netty-all - 4.0.33.Final - - -<<<<<<< HEAD - com.basho.riak.protobuf - riak-pb - 2.1.1.1-SNAPSHOT - - - org.erlang.otp - jinterface - 1.5.6 -======= - com.google.protobuf - protobuf-java - 2.6.1 - - - org.antlr - antlr-complete - 3.5.2 ->>>>>>> develop - - - From f497262ef0452e285c7ebbc4f86972fff44f1507 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 26 Apr 2016 11:44:04 -0400 Subject: [PATCH 40/52] Remainder of TODOs for 611 --- .../basho/riak/client/core/RiakMessage.java | 102 ++++++++++------- .../client/core/codec/TermToBinaryCodec.java | 8 +- .../core/operations/DeleteOperation.java | 8 +- .../core/operations/DtFetchOperation.java | 8 +- .../core/operations/DtUpdateOperation.java | 52 ++++----- .../operations/FetchBucketPropsOperation.java | 22 ++-- .../core/operations/FetchOperation.java | 83 +++++++------- .../core/operations/ListBucketsOperation.java | 32 +++--- .../core/operations/ListKeysOperation.java | 2 +- .../core/operations/MapReduceOperation.java | 38 +++---- .../client/core/operations/Operations.java | 4 +- .../core/operations/PBFutureOperation.java | 4 +- .../operations/ResetBucketPropsOperation.java | 22 ++-- .../core/operations/SearchOperation.java | 12 +- .../SecondaryIndexQueryOperation.java | 107 +++++++++--------- .../operations/StoreBucketPropsOperation.java | 10 +- .../core/operations/StoreOperation.java | 78 ++++++------- .../core/operations/TTBFutureOperation.java | 2 +- .../operations/YzDeleteIndexOperation.java | 16 +-- .../operations/YzFetchIndexOperation.java | 34 +++--- .../core/operations/YzGetSchemaOperation.java | 26 ++--- .../core/operations/YzPutIndexOperation.java | 12 +- .../core/operations/YzPutSchemaOperation.java | 16 +-- .../core/operations/ts/FetchOperation.java | 1 - .../core/operations/ts/TTBConverters.java | 35 +++--- .../client/core/query/timeseries/Cell.java | 8 +- .../core/operations/OperationsTest.java | 16 +-- .../codec/TermToBinaryCodecTest.java | 7 -- 28 files changed, 397 insertions(+), 368 deletions(-) delete mode 100644 src/test/java/com/basho/riak/client/core/operations/codec/TermToBinaryCodecTest.java diff --git a/src/main/java/com/basho/riak/client/core/RiakMessage.java b/src/main/java/com/basho/riak/client/core/RiakMessage.java index 982d2442d..828396189 100644 --- a/src/main/java/com/basho/riak/client/core/RiakMessage.java +++ b/src/main/java/com/basho/riak/client/core/RiakMessage.java @@ -5,14 +5,16 @@ import com.basho.riak.protobuf.RiakPB; import com.ericsson.otp.erlang.OtpErlangDecodeException; import com.ericsson.otp.erlang.OtpInputStream; -import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import java.nio.charset.StandardCharsets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.charset.StandardCharsets; +import java.util.logging.Level; + /** * Encapsulates the raw bytes sent to or received from Riak. + * * @author Brian Roach * @author Sergey Galkin * @since 2.0 @@ -31,7 +33,8 @@ public RiakMessage(byte code, byte[] data) this.code = code; this.data = data; - switch(this.code) { + switch (this.code) + { case RiakMessageCodes.MSG_ErrorResp: this.ttbInputStream = null; this.riakError = getRiakErrorFromPbuf(this.data); @@ -48,70 +51,91 @@ public RiakMessage(byte code, byte[] data) } } - public byte getCode() { + private static RiakResponseException getRiakErrorFromPbuf(byte[] data) + { + try + { + RiakPB.RpbErrorResp err = RiakPB.RpbErrorResp.parseFrom(data); + return new RiakResponseException(err.getErrcode(), err.getErrmsg().toStringUtf8()); + } + catch (InvalidProtocolBufferException ex) + { + logger.error("exception", ex); + return new RiakResponseException(0, "Could not parse protocol buffers error"); + } + } + + public byte getCode() + { return code; } - public byte[] getData() { + public byte[] getData() + { return data; } - public boolean isTtbMessage() { + public boolean isTtbMessage() + { return this.ttbInputStream != null; } - public boolean isRiakError() { + public boolean isRiakError() + { return this.riakError != null; } - public OtpInputStream getTtbStream() { + public OtpInputStream getTtbStream() + { return this.ttbInputStream; } - public RiakResponseException getRiakError() { + public RiakResponseException getRiakError() + { return this.riakError; } - private static RiakResponseException getRiakErrorFromPbuf(byte[] data) { - try { - RiakPB.RpbErrorResp err = RiakPB.RpbErrorResp.parseFrom(data); - return new RiakResponseException(err.getErrcode(), err.getErrmsg().toStringUtf8()); - } catch (InvalidProtocolBufferException ex) { - // TODO GH-611 - logger.error("exception", ex); - return new RiakResponseException(0, "could not parse protocol buffers error"); - } - } - - private RiakResponseException getRiakErrorFromTtb(OtpInputStream ttbInputStream) { - int ttbMsgArity = 0; + private RiakResponseException getRiakErrorFromTtb(OtpInputStream ttbInputStream) + { + final String decodeErrorMsg = "Error decoding Riak TTB Response, unexpected format."; + int ttbMsgArity; - try { + try + { ttbMsgArity = ttbInputStream.read_tuple_head(); - } catch (OtpErlangDecodeException ex) { - // TODO GH-611 - logger.error("exception", ex); + } + catch (OtpErlangDecodeException ex) + { + logger.error(decodeErrorMsg + " Was expecting a tuple head.", ex); + throw new IllegalArgumentException(decodeErrorMsg, ex); } - if (ttbMsgArity == 3) { + if (ttbMsgArity == 3) + { // NB: may be an error response - // TODO GH-611 message atom constants? - String atom = "unknown"; - try { + String atom; + try + { atom = ttbInputStream.read_atom(); - } catch (OtpErlangDecodeException ex) { - // TODO GH-611 - logger.error("exception", ex); + } + catch (OtpErlangDecodeException ex) + { + logger.error(decodeErrorMsg + " Was expecting an atom.", ex); + throw new IllegalArgumentException(decodeErrorMsg, ex); } - if ("rpberrorresp".equals(atom)) { - try { + if ("rpberrorresp".equals(atom)) + { + try + { String errMsg = new String(ttbInputStream.read_binary(), StandardCharsets.UTF_8); - int errCode = ttbInputStream.read_int(); // TODO GH-611 is errcode an int? + int errCode = ttbInputStream.read_int(); return new RiakResponseException(errCode, errMsg); - } catch (OtpErlangDecodeException ex) { - // TODO GH-611 - logger.error("exception", ex); + } + catch (OtpErlangDecodeException ex) + { + logger.error(decodeErrorMsg, ex); + throw new IllegalArgumentException(decodeErrorMsg, ex); } } } diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index 5399b8cfa..dbd642ace 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -6,6 +6,9 @@ import com.basho.riak.protobuf.RiakTsPB; import com.ericsson.otp.erlang.*; import com.google.protobuf.ByteString; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; @@ -14,6 +17,7 @@ public class TermToBinaryCodec { + private static Logger logger = LoggerFactory.getLogger(TermToBinaryCodec.class); private static final class Messages { public static final OtpErlangAtom tsGetReq = new OtpErlangAtom("tsgetreq"); @@ -161,7 +165,9 @@ private static QueryResult decodeTsResponse(byte[] response) throws OtpErlangDec break; default: - // TODO GH-611 throw exception? + final String errorMsg = "Unsupported response message received: " + respAtom; + logger.error(errorMsg); + throw new IllegalArgumentException(errorMsg); } return result; diff --git a/src/main/java/com/basho/riak/client/core/operations/DeleteOperation.java b/src/main/java/com/basho/riak/client/core/operations/DeleteOperation.java index 61b1350c5..9e018da3d 100644 --- a/src/main/java/com/basho/riak/client/core/operations/DeleteOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/DeleteOperation.java @@ -24,7 +24,7 @@ import java.util.List; -import static com.basho.riak.client.core.operations.Operations.checkMessageType; +import static com.basho.riak.client.core.operations.Operations.checkPBMessageType; import com.basho.riak.client.core.query.Location; /** @@ -46,7 +46,7 @@ private DeleteOperation(Builder builder) } @Override - protected Void convert(List rawResponse) + protected Void convert(List rawResponse) { return null; } @@ -54,7 +54,7 @@ protected Void convert(List rawResponse) @Override protected Void decode(RiakMessage rawResponse) { - checkMessageType(rawResponse, RiakMessageCodes.MSG_DelResp); + checkPBMessageType(rawResponse, RiakMessageCodes.MSG_DelResp); return null; } @@ -85,7 +85,7 @@ public Builder(Location location) { throw new IllegalArgumentException("Location can not be null"); } - + reqBuilder.setBucket(ByteString.copyFrom(location.getNamespace().getBucketName().unsafeGetValue())); reqBuilder.setKey(ByteString.copyFrom(location.getKey().unsafeGetValue())); reqBuilder.setType(ByteString.copyFrom(location.getNamespace().getBucketType().unsafeGetValue())); diff --git a/src/main/java/com/basho/riak/client/core/operations/DtFetchOperation.java b/src/main/java/com/basho/riak/client/core/operations/DtFetchOperation.java index 0cd46a6f6..6e5a7849c 100644 --- a/src/main/java/com/basho/riak/client/core/operations/DtFetchOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/DtFetchOperation.java @@ -74,7 +74,7 @@ protected RiakMessage createChannelMessage() @Override protected RiakDtPB.DtFetchResp decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_DtFetchResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_DtFetchResp); try { return RiakDtPB.DtFetchResp.parseFrom(rawMessage.getData()); @@ -106,7 +106,7 @@ public Builder(Location location) { throw new IllegalArgumentException("Location can not be null"); } - + reqBuilder.setBucket(ByteString.copyFrom(location.getNamespace().getBucketName().unsafeGetValue())); reqBuilder.setKey(ByteString.copyFrom(location.getKey().unsafeGetValue())); reqBuilder.setType(ByteString.copyFrom(location.getNamespace().getBucketType().unsafeGetValue())); @@ -238,7 +238,7 @@ public DtFetchOperation build() } - public static class Response + public static class Response { private final BinaryValue context; private final RiakDatatype crdtElement; @@ -276,7 +276,7 @@ protected static abstract class Init> protected abstract T self(); protected abstract Response build(); - + T withContext(BinaryValue context) { if (context != null) diff --git a/src/main/java/com/basho/riak/client/core/operations/DtUpdateOperation.java b/src/main/java/com/basho/riak/client/core/operations/DtUpdateOperation.java index ecc767cbc..cf13ac748 100644 --- a/src/main/java/com/basho/riak/client/core/operations/DtUpdateOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/DtUpdateOperation.java @@ -47,7 +47,7 @@ private DtUpdateOperation(Builder builder) } @Override - protected Response convert(List rawResponse) + protected Response convert(List rawResponse) { if (rawResponse.size() != 1) { @@ -57,8 +57,8 @@ protected Response convert(List rawResponse) RiakDtPB.DtUpdateResp response = rawResponse.iterator().next(); CrdtResponseConverter converter = new CrdtResponseConverter(); RiakDatatype element = converter.convert(response); - - Response.Builder responseBuilder = + + Response.Builder responseBuilder = new Response.Builder().withCrdtElement(element); if (response.hasKey()) @@ -66,7 +66,7 @@ protected Response convert(List rawResponse) BinaryValue key = BinaryValue.unsafeCreate(response.getKey().toByteArray()); responseBuilder.withGeneratedKey(key); } - + if (response.hasContext()) { BinaryValue context = BinaryValue.unsafeCreate(response.getContext().toByteArray()); @@ -86,7 +86,7 @@ protected RiakMessage createChannelMessage() @Override protected RiakDtPB.DtUpdateResp decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_DtUpdateResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_DtUpdateResp); try { RiakDtPB.DtUpdateResp resp = RiakDtPB.DtUpdateResp.parseFrom(rawMessage.getData()); @@ -109,7 +109,7 @@ public static class Builder private final RiakDtPB.DtUpdateReq.Builder reqBuilder = RiakDtPB.DtUpdateReq.newBuilder(); private final Location location; private boolean removeOpPresent = false; - + /** * Construct a builder for a DtUpdateOperation. * @param location The location of the object in Riak. @@ -124,11 +124,11 @@ else if (location.getNamespace().getBucketTypeAsString().equals(Namespace.DEFAUL { throw new IllegalArgumentException("Default bucket type does not accept CRDTs"); } - + reqBuilder.setBucket(ByteString.copyFrom(location.getNamespace().getBucketName().unsafeGetValue())); reqBuilder.setType(ByteString.copyFrom(location.getNamespace().getBucketType().unsafeGetValue())); reqBuilder.setKey(ByteString.copyFrom(location.getKey().unsafeGetValue())); - + this.location = location; } @@ -142,17 +142,17 @@ else if (namespace.getBucketTypeAsString().equals(Namespace.DEFAULT_BUCKET_TYPE) { throw new IllegalArgumentException("Default bucket type does not accept CRDTs"); } - + // This is simply for the returned query info Location loc = new Location(namespace, "RIAK_GENERATED"); - + reqBuilder.setBucket(ByteString.copyFrom(loc.getNamespace().getBucketName().unsafeGetValue())); reqBuilder.setType(ByteString.copyFrom(loc.getNamespace().getBucketType().unsafeGetValue())); - + this.location = loc; - + } - + /** * Set the context for this operation. * @@ -269,7 +269,7 @@ public DtUpdateOperation build() { throw new IllegalStateException("Remove operations cannot be performed without a context."); } - + return new DtUpdateOperation(this); } @@ -293,7 +293,7 @@ RiakDtPB.SetOp getSetOp(SetOp op) { setOpBuilder.addRemoves(ByteString.copyFrom(element.unsafeGetValue())); } - + if (setOpBuilder.getRemovesCount() > 0) { removeOpPresent = true; @@ -348,7 +348,7 @@ RiakDtPB.MapOp getMapOp(MapOp op) { mapOpBuilder.addRemoves(getMapField(field)); } - + if (mapOpBuilder.getRemovesCount() > 0) { removeOpPresent = true; @@ -437,38 +437,38 @@ private Builder withOp(SetOp op) } } - + public static class Response extends DtFetchOperation.Response { private final BinaryValue generatedKey; - + private Response(Init builder) { super(builder); this.generatedKey = builder.generatedKey; } - + public boolean hasGeneratedKey() { return generatedKey != null; } - + public BinaryValue getGeneratedKey() { return generatedKey; } - + protected static abstract class Init> extends DtFetchOperation.Response.Init { private BinaryValue generatedKey; - + T withGeneratedKey(BinaryValue generatedKey) { this.generatedKey = generatedKey; return self(); } } - + static class Builder extends Init { @Override @@ -476,14 +476,14 @@ protected Builder self() { return this; } - + @Override protected Response build() { return new Response(this); } - + } } - + } diff --git a/src/main/java/com/basho/riak/client/core/operations/FetchBucketPropsOperation.java b/src/main/java/com/basho/riak/client/core/operations/FetchBucketPropsOperation.java index dff694e43..c119f7fd1 100644 --- a/src/main/java/com/basho/riak/client/core/operations/FetchBucketPropsOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/FetchBucketPropsOperation.java @@ -35,17 +35,17 @@ public class FetchBucketPropsOperation extends FutureOperation rawResponse) + protected Response convert(List rawResponse) { - // This isn't streaming, there will only be one response. + // This isn't streaming, there will only be one response. RiakPB.RpbBucketProps pbProps = rawResponse.get(0).getProps(); return new Response(BucketPropertiesConverter.convert(pbProps)); } @@ -60,7 +60,7 @@ protected RiakMessage createChannelMessage() @Override protected RiakPB.RpbGetBucketResp decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_GetBucketResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_GetBucketResp); try { return RiakPB.RpbGetBucketResp.parseFrom(rawMessage.getData()); @@ -76,13 +76,13 @@ public Namespace getQueryInfo() { return namespace; } - + public static class Builder { - private final RiakPB.RpbGetBucketReq.Builder reqBuilder = + private final RiakPB.RpbGetBucketReq.Builder reqBuilder = RiakPB.RpbGetBucketReq.newBuilder(); private final Namespace namespace; - + /** * Construct a builder for a FetchBucketPropsOperation. * @param namespace The namespace for the bucket. @@ -97,13 +97,13 @@ public Builder(Namespace namespace) reqBuilder.setType(ByteString.copyFrom(namespace.getBucketType().unsafeGetValue())); this.namespace = namespace; } - + public FetchBucketPropsOperation build() { return new FetchBucketPropsOperation(this); } } - + /** * Response from Fetching a bucket's properties. */ @@ -114,7 +114,7 @@ private Response(BucketProperties props) { this.props = props; } - + /** * Returns the fetched BucketProperties. * @return the BucketProperties. diff --git a/src/main/java/com/basho/riak/client/core/operations/FetchOperation.java b/src/main/java/com/basho/riak/client/core/operations/FetchOperation.java index 09e940de4..bc0784be6 100644 --- a/src/main/java/com/basho/riak/client/core/operations/FetchOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/FetchOperation.java @@ -16,7 +16,6 @@ package com.basho.riak.client.core.operations; import com.basho.riak.client.api.cap.BasicVClock; -import com.basho.riak.client.api.cap.VClock; import com.basho.riak.client.core.FutureOperation; import com.basho.riak.client.core.RiakMessage; import com.basho.riak.client.core.converters.RiakObjectConverter; @@ -41,7 +40,7 @@ public class FetchOperation extends FutureOperation responses) + protected FetchOperation.Response convert(List responses) { // This is not a streaming op, there will only be one response if (responses.size() > 1) { logger.error("Received {} responses when only one was expected.", responses.size()); } - + RiakKvPB.RpbGetResp response = responses.get(0); - - FetchOperation.Response.Builder responseBuilder = + + FetchOperation.Response.Builder responseBuilder = new FetchOperation.Response.Builder(); - - // If the response is null ... it means not found. Riak only sends + + // If the response is null ... it means not found. Riak only sends // a message code and zero bytes when that's the case. (See: decode() ) // Because that makes sense! if (null == response) { responseBuilder.withNotFound(true); } - else + else { // To unify the behavior of having just a tombstone vs. siblings // that include a tombstone, we create an empty object and mark @@ -104,7 +103,7 @@ protected FetchOperation.Response convert(List responses) RiakObject ro = new RiakObject() .setDeleted(true) .setVClock(new BasicVClock(response.getVclock().toByteArray())); - + responseBuilder.addObject(ro); } else @@ -115,7 +114,7 @@ protected FetchOperation.Response convert(List responses) responseBuilder.withUnchanged(response.hasUnchanged() ? response.getUnchanged() : false); } - + return responseBuilder.build(); } @@ -131,13 +130,13 @@ public Location getQueryInfo() { return location; } - + public static class Builder { - private final RiakKvPB.RpbGetReq.Builder reqBuilder = + private final RiakKvPB.RpbGetReq.Builder reqBuilder = RiakKvPB.RpbGetReq.newBuilder(); private final Location location; - + /** * Construct a FetchOperation that will retrieve an object from Riak stored * at the provided Location. @@ -149,14 +148,14 @@ public Builder(Location location) { throw new IllegalArgumentException("Location can not be null."); } - + reqBuilder.setKey(ByteString.copyFrom(location.getKey().unsafeGetValue())); reqBuilder.setBucket(ByteString.copyFrom(location.getNamespace().getBucketName().unsafeGetValue())); reqBuilder.setType(ByteString.copyFrom(location.getNamespace().getBucketType().unsafeGetValue())); this.location = location; - + } - + /** * Set the R value for this FetchOperation. * If not asSet the bucket default is used. @@ -203,7 +202,7 @@ public Builder withNotFoundOK(boolean notFoundOk) * Set the basic_quorum value. *

* The parameter controls whether a read request should return early in - * some fail cases. + * some fail cases. * E.g. If a quorum of nodes has already * returned notfound/error, don't wait around for the rest. *

@@ -232,7 +231,7 @@ public Builder withReturnDeletedVClock(boolean returnDeletedVClock) *

* Causes Riak to only return the metadata for the object. The value * will be asSet to null. - * @param headOnly true to return only metadata. + * @param headOnly true to return only metadata. * @return a reference to this object. */ public Builder withHeadOnly(boolean headOnly) @@ -242,9 +241,9 @@ public Builder withHeadOnly(boolean headOnly) } /** - * Do not return the object if the supplied vclock matches. + * Do not return the object if the supplied vclock matches. * @param vclock the vclock to match on - * @return a refrence to this object. + * @return a refrence to this object. */ public Builder withIfNotModified(byte[] vclock) { @@ -290,42 +289,42 @@ public Builder withSloppyQuorum(boolean sloppyQuorum) reqBuilder.setSloppyQuorum(sloppyQuorum); return this; } - + public FetchOperation build() { return new FetchOperation(this); } - - + + } - + protected static abstract class KvResponseBase { private final List objectList; - + protected KvResponseBase(Init builder) { this.objectList = builder.objectList; } - + public List getObjectList() { return objectList; } - - protected static abstract class Init> + + protected static abstract class Init> { private final List objectList = new LinkedList(); protected abstract T self(); protected abstract KvResponseBase build(); - + T addObject(RiakObject object) { objectList.add(object); return self(); } - + T addObjects(List objects) { objectList.addAll(objects); @@ -333,48 +332,48 @@ T addObjects(List objects) } } } - - + + public static class Response extends KvResponseBase { private final boolean notFound; private final boolean unchanged; - + private Response(Init builder) { super(builder); this.notFound = builder.notFound; this.unchanged = builder.unchanged; } - + public boolean isNotFound() { return notFound; } - + public boolean isUnchanged() { return unchanged; } - + protected static abstract class Init> extends KvResponseBase.Init { private boolean notFound; private boolean unchanged; - + T withNotFound(boolean notFound) { this.notFound = notFound; return self(); } - + T withUnchanged(boolean unchanged) { this.unchanged = unchanged; return self(); } } - + static class Builder extends Init { @Override diff --git a/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java index 2baf3585e..6592832f6 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java @@ -31,7 +31,7 @@ public class ListBucketsOperation extends FutureOperation rawResponse) + protected ListBucketsOperation.Response convert(List rawResponse) { List buckets = new ArrayList(rawResponse.size()); for (RiakKvPB.RpbListBucketsResp resp : rawResponse) @@ -69,7 +69,7 @@ protected RiakKvPB.RpbListBucketsResp decode(RiakMessage rawMessage) { try { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_ListBucketsResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_ListBucketsResp); return RiakKvPB.RpbListBucketsResp.parseFrom(rawMessage.getData()); } catch (InvalidProtocolBufferException e) @@ -84,19 +84,19 @@ public BinaryValue getQueryInfo() { return bucketType; } - + public static class Builder { - private final RiakKvPB.RpbListBucketsReq.Builder reqBuilder = + private final RiakKvPB.RpbListBucketsReq.Builder reqBuilder = RiakKvPB.RpbListBucketsReq.newBuilder().setStream(true); private BinaryValue bucketType = BinaryValue.create(Namespace.DEFAULT_BUCKET_TYPE); - + /** * Create a Builder for a ListBucketsOperation. */ public Builder() {} - + /** * Provide a timeout for this operation. * @param timeout value in milliseconds @@ -111,10 +111,10 @@ public Builder withTimeout(int timeout) reqBuilder.setTimeout(timeout); return this; } - + /** * Set the bucket type. - * If unset {@link Namespace#DEFAULT_BUCKET_TYPE} is used. + * If unset {@link Namespace#DEFAULT_BUCKET_TYPE} is used. * @param bucketType the bucket type to use * @return A reference to this object. */ @@ -128,34 +128,34 @@ public Builder withBucketType(BinaryValue bucketType) this.bucketType = bucketType; return this; } - + public ListBucketsOperation build() { return new ListBucketsOperation(this); } - + } - + public static class Response { private final BinaryValue bucketType; private final List buckets; - + Response(BinaryValue bucketType, List buckets) { this.bucketType = bucketType; this.buckets = buckets; } - + public BinaryValue getBucketType() { return bucketType; } - + public List getBuckets() { return buckets; } } - + } diff --git a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java index f8cc7f561..2ebd99904 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java @@ -66,7 +66,7 @@ protected RiakKvPB.RpbListKeysResp decode(RiakMessage rawMessage) { try { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_ListKeysResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_ListKeysResp); return RiakKvPB.RpbListKeysResp.parseFrom(rawMessage.getData()); } catch (InvalidProtocolBufferException e) diff --git a/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java b/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java index 0ee1fb182..388364eed 100644 --- a/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java @@ -43,7 +43,7 @@ public class MapReduceOperation extends FutureOperation rawResponse) { - // Riak streams the result back. Each message from Riak contains a int + // Riak streams the result back. Each message from Riak contains a int // that tells you what phase the result is from. The result from a phase // can span multiple messages. Each result chunk is a JSON array. - + final JsonNodeFactory factory = JsonNodeFactory.instance; final ObjectMapper mapper = new ObjectMapper(); final Map resultMap = new LinkedHashMap(); - + int phase = 0; - + for (RiakKvPB.RpbMapRedResp response : rawResponse) { if (response.hasPhase()) @@ -76,12 +76,12 @@ protected Response convert(List rawResponse) { jsonArray = resultMap.get(phase); } - else + else { jsonArray = factory.arrayNode(); resultMap.put(phase, jsonArray); } - + JsonNode responseJson; try { @@ -92,7 +92,7 @@ protected Response convert(List rawResponse) logger.error("Mapreduce job returned non-JSON; {}",response.getResponse().toStringUtf8()); throw new RuntimeException("Non-JSON response from MR job", ex); } - + if (responseJson.isArray()) { jsonArray.addAll((ArrayNode)responseJson); @@ -116,7 +116,7 @@ protected RiakMessage createChannelMessage() @Override protected RiakKvPB.RpbMapRedResp decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_MapRedResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_MapRedResp); try { return RiakKvPB.RpbMapRedResp.parseFrom(rawMessage.getData()); @@ -138,13 +138,13 @@ public BinaryValue getQueryInfo() { return mapReduce; } - + public static class Builder { private final RiakKvPB.RpbMapRedReq.Builder reqBuilder = RiakKvPB.RpbMapRedReq.newBuilder(); private final BinaryValue mapReduce; - + /** * Create a MapReduce operation builder with the given function. * @@ -156,33 +156,33 @@ public Builder(BinaryValue mapReduce) { throw new IllegalArgumentException("MapReduce can not be null or empty."); } - - + + reqBuilder.setRequest(ByteString.copyFrom(mapReduce.unsafeGetValue())) .setContentType(ByteString.copyFromUtf8("application/json")); this.mapReduce = mapReduce; - + } - + public MapReduceOperation build() { return new MapReduceOperation(this); } } - + public static class Response { private final Map resultMap; - + Response(Map results) { this.resultMap = results; } - + public Map getResults() { return resultMap; } - + } } diff --git a/src/main/java/com/basho/riak/client/core/operations/Operations.java b/src/main/java/com/basho/riak/client/core/operations/Operations.java index 2a5f77607..fb72f9d77 100644 --- a/src/main/java/com/basho/riak/client/core/operations/Operations.java +++ b/src/main/java/com/basho/riak/client/core/operations/Operations.java @@ -21,7 +21,7 @@ public class Operations { - public static void checkMessageType(RiakMessage msg, byte expected) + public static void checkPBMessageType(RiakMessage msg, byte expected) { byte pbMessageCode = msg.getCode(); if (pbMessageCode != expected) @@ -47,7 +47,7 @@ public static short getUnsignedByteValue(byte b) /** * Convert a Java signed int to unsigned. - * Returns the unsigned value as a (signed) long. + * Returns the unsigned value as a (signed) long. * @param i a java signed int * @return a long containing the converted value. */ diff --git a/src/main/java/com/basho/riak/client/core/operations/PBFutureOperation.java b/src/main/java/com/basho/riak/client/core/operations/PBFutureOperation.java index d19acca69..c111be6f5 100644 --- a/src/main/java/com/basho/riak/client/core/operations/PBFutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/PBFutureOperation.java @@ -7,8 +7,6 @@ import com.google.protobuf.InvalidProtocolBufferException; import org.slf4j.LoggerFactory; -import java.util.List; - /** * An abstract PB operation that introduces generic encoding/decoding * @@ -45,7 +43,7 @@ protected RiakMessage createChannelMessage() { @Override protected U decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, respMessageCode); + Operations.checkPBMessageType(rawMessage, respMessageCode); try { byte[] data = rawMessage.getData(); diff --git a/src/main/java/com/basho/riak/client/core/operations/ResetBucketPropsOperation.java b/src/main/java/com/basho/riak/client/core/operations/ResetBucketPropsOperation.java index 6c4fec32d..f12590ce0 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ResetBucketPropsOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ResetBucketPropsOperation.java @@ -32,15 +32,15 @@ public class ResetBucketPropsOperation extends FutureOperation rawResponse) + protected Void convert(List rawResponse) { return null; } @@ -48,16 +48,16 @@ protected Void convert(List rawResponse) @Override protected RiakMessage createChannelMessage() { - RiakPB.RpbResetBucketReq req = + RiakPB.RpbResetBucketReq req = reqBuilder.build(); - + return new RiakMessage(RiakMessageCodes.MSG_ResetBucketReq, req.toByteArray()); } @Override protected Void decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_ResetBucketResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_ResetBucketResp); return null; } @@ -66,15 +66,15 @@ public Namespace getQueryInfo() { return namespace; } - + public static class Builder { - private final RiakPB.RpbResetBucketReq.Builder reqBuilder = + private final RiakPB.RpbResetBucketReq.Builder reqBuilder = RiakPB.RpbResetBucketReq.newBuilder(); private final Namespace namespace; - + /** - * Construct a builder for a ResetBucketPropsOperation. + * Construct a builder for a ResetBucketPropsOperation. * @param namespace The namespace in Riak. */ public Builder(Namespace namespace) @@ -87,7 +87,7 @@ public Builder(Namespace namespace) reqBuilder.setType(ByteString.copyFrom(namespace.getBucketType().unsafeGetValue())); this.namespace = namespace; } - + public ResetBucketPropsOperation build() { return new ResetBucketPropsOperation(this); diff --git a/src/main/java/com/basho/riak/client/core/operations/SearchOperation.java b/src/main/java/com/basho/riak/client/core/operations/SearchOperation.java index 65b2facf5..6f5f29a06 100644 --- a/src/main/java/com/basho/riak/client/core/operations/SearchOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/SearchOperation.java @@ -57,7 +57,7 @@ private SearchOperation(Builder builder) } @Override - protected SearchOperation.Response convert(List rawResponse) + protected SearchOperation.Response convert(List rawResponse) { // This isn't a streaming op, there will only be one protobuf RiakSearchPB.RpbSearchQueryResp resp = rawResponse.get(0); @@ -92,7 +92,7 @@ protected RiakMessage createChannelMessage() @Override protected RiakSearchPB.RpbSearchQueryResp decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_SearchQueryResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_SearchQueryResp); try { return RiakSearchPB.RpbSearchQueryResp.parseFrom(rawMessage.getData()); @@ -274,7 +274,7 @@ public static class Response implements Iterable private final List>> results; private final float maxScore; private final int numResults; - + Response(List>> results, float maxScore, int numResults) { this.results = results; @@ -308,13 +308,13 @@ public int numResults() /** * Returns the entire list of results from the search query. - * @return a list containing all the result sets. + * @return a list containing all the result sets. */ public List>> getAllResults() { return results; } - + } - + } diff --git a/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java b/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java index f19961a6c..49fe22ca0 100644 --- a/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java @@ -17,7 +17,6 @@ import com.basho.riak.client.core.FutureOperation; import com.basho.riak.client.core.RiakMessage; -import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.util.BinaryValue; import com.basho.riak.protobuf.RiakMessageCodes; @@ -38,7 +37,7 @@ public class SecondaryIndexQueryOperation extends FutureOperation rawResponse) { - SecondaryIndexQueryOperation.Response.Builder responseBuilder = + SecondaryIndexQueryOperation.Response.Builder responseBuilder = new SecondaryIndexQueryOperation.Response.Builder(); - + for (RiakKvPB.RpbIndexResp pbEntry : rawResponse) { /** - * The 2i API is inconsistent on the Riak side. If it's not - * a range query, return_terms is ignored it only returns the + * The 2i API is inconsistent on the Riak side. If it's not + * a range query, return_terms is ignored it only returns the * list of object keys and you have to have * preserved the index key if you want to return it to the user - * with the results. - * + * with the results. + * * Also, the $key index queries just ignore return_terms altogether. */ - + if (pbReq.getReturnTerms() && !query.indexName.toString().equalsIgnoreCase("$key")) { if (pbReq.hasRangeMin()) { for (RpbPair pair : pbEntry.getResultsList()) { - responseBuilder.addEntry(new Response.Entry(BinaryValue.unsafeCreate(pair.getKey().toByteArray()), + responseBuilder.addEntry(new Response.Entry(BinaryValue.unsafeCreate(pair.getKey().toByteArray()), BinaryValue.unsafeCreate(pair.getValue().toByteArray()))); } } @@ -94,7 +93,7 @@ protected SecondaryIndexQueryOperation.Response convert(List @@ -445,9 +444,9 @@ public Builder withContinuation(BinaryValue continuation) *

*

* Note that this is not recommended for queries that could return a large - * result set; the overhead in Riak is substantial. + * result set; the overhead in Riak is substantial. *

- * + * * @param orderByKey true to sort the results, false to return as-is. * @return a reference to this object. */ @@ -467,7 +466,7 @@ public Builder withRegexTermFilter(BinaryValue filter) this.termFilter = filter; return this; } - + /** * Set the timeout for the query. *

@@ -481,7 +480,7 @@ public Builder withTimeout(int timeout) this.timeout = timeout; return this; } - + public Query build() { // sanity checks @@ -511,7 +510,7 @@ else if (termFilter != null && indexName.toStringUtf8().endsWith("_int")) } } } - + public static class Response { private final BinaryValue continuation; @@ -522,7 +521,7 @@ private Response(Builder builder) this.continuation = builder.continuation; this.entryList = builder.entryList; } - + public boolean hasContinuation() { return continuation != null; @@ -532,12 +531,12 @@ public BinaryValue getContinuation() { return continuation; } - + public List getEntryList() { return entryList; } - + public static class Entry { private final BinaryValue indexKey; @@ -569,31 +568,31 @@ public BinaryValue getObjectKey() return objectKey; } } - + static class Builder { private BinaryValue continuation; - private List entryList = + private List entryList = new ArrayList(); - + Builder withContinuation(BinaryValue continuation) { this.continuation = continuation; return this; } - + Builder addEntry(Response.Entry entry) { entryList.add(entry); return this; } - + Response build() { return new Response(this); } } - + } - + } diff --git a/src/main/java/com/basho/riak/client/core/operations/StoreBucketPropsOperation.java b/src/main/java/com/basho/riak/client/core/operations/StoreBucketPropsOperation.java index 9f35c32f6..3f738316b 100644 --- a/src/main/java/com/basho/riak/client/core/operations/StoreBucketPropsOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/StoreBucketPropsOperation.java @@ -42,7 +42,7 @@ private StoreBucketPropsOperation(Builder builder) } @Override - protected Void convert(List rawResponse) + protected Void convert(List rawResponse) { return null; } @@ -57,7 +57,7 @@ protected RiakMessage createChannelMessage() @Override protected Void decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_SetBucketResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_SetBucketResp); return null; } @@ -437,7 +437,7 @@ public static class Builder extends PropsBuilder private final RiakPB.RpbSetBucketReq.Builder reqBuilder = RiakPB.RpbSetBucketReq.newBuilder(); private final Namespace namespace; - + /** * Constructs a builder for a StoreBucketPropsOperation. * @param namespace The namespace in Riak. @@ -452,13 +452,13 @@ public Builder(Namespace namespace) reqBuilder.setType(ByteString.copyFrom(namespace.getBucketType().unsafeGetValue())); this.namespace = namespace; } - + @Override protected Builder self() { return this; } - + public StoreBucketPropsOperation build() { reqBuilder.setProps(propsBuilder); diff --git a/src/main/java/com/basho/riak/client/core/operations/StoreOperation.java b/src/main/java/com/basho/riak/client/core/operations/StoreOperation.java index b9b6b2dcb..d12e27742 100644 --- a/src/main/java/com/basho/riak/client/core/operations/StoreOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/StoreOperation.java @@ -27,7 +27,7 @@ import java.util.List; -import static com.basho.riak.client.core.operations.Operations.checkMessageType; +import static com.basho.riak.client.core.operations.Operations.checkPBMessageType; import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; @@ -42,7 +42,7 @@ public class StoreOperation extends FutureOperation responses) + protected Response convert(List responses) { // There should only be one response message from Riak. if (responses.size() != 1) @@ -59,29 +59,29 @@ protected Response convert(List responses) } RiakKvPB.RpbPutResp response = responses.get(0); - - StoreOperation.Response.Builder responseBuilder = + + StoreOperation.Response.Builder responseBuilder = new StoreOperation.Response.Builder(); - + // This only exists if no key was specified in the put request if (response.hasKey()) { responseBuilder.withGeneratedKey(BinaryValue.unsafeCreate(response.getKey().toByteArray())); } - + // Is there a case where this isn't true? Can a delete interleve? if (response.getContentCount() > 0) { responseBuilder.addObjects(RiakObjectConverter.convert(response.getContentList(), response.getVclock())); } - + return responseBuilder.build(); } @Override protected RiakKvPB.RpbPutResp decode(RiakMessage rawMessage) { - checkMessageType(rawMessage, RiakMessageCodes.MSG_PutResp); + checkPBMessageType(rawMessage, RiakMessageCodes.MSG_PutResp); try { return RiakKvPB.RpbPutResp.parseFrom(rawMessage.getData()); @@ -110,7 +110,7 @@ public static class Builder { private final RiakKvPB.RpbPutReq.Builder reqBuilder = RiakKvPB.RpbPutReq.newBuilder(); private final Location location; - + /** * Constructs a builder for a StoreOperation * @param location The location in Riak at which to store. @@ -121,15 +121,15 @@ public Builder(Location location) { throw new IllegalArgumentException("Location cannot be null"); } - + reqBuilder.setType(ByteString.copyFrom(location.getNamespace().getBucketType().unsafeGetValue())); reqBuilder.setBucket(ByteString.copyFrom(location.getNamespace().getBucketName().unsafeGetValue())); reqBuilder.setKey(ByteString.copyFrom(location.getKey().unsafeGetValue())); - + this.location = location; - + } - + /** * Constructs a builder for a StoreOperation. *

@@ -145,28 +145,28 @@ public Builder(Namespace namespace) } reqBuilder.setType(ByteString.copyFrom(namespace.getBucketType().unsafeGetValue())); reqBuilder.setBucket(ByteString.copyFrom(namespace.getBucketName().unsafeGetValue())); - + this.location = new Location(namespace, "RIAK_GENERATED"); - + } - - + + public Builder withContent(RiakObject content) { if (null == content) { throw new IllegalArgumentException("Object cannot be null."); } - + reqBuilder.setContent(RiakObjectConverter.convert(content)); - + if (content.getVClock() != null) { reqBuilder.setVclock(ByteString.copyFrom(content.getVClock().getBytes())); } return this; } - + /** * Set the W value for this StoreOperation. * If not asSet the bucket default is used. @@ -205,7 +205,7 @@ public Builder withPw(int pw) /** * Return the object after storing (including any siblings). - * @param returnBody true to return the object. + * @param returnBody true to return the object. * @return a reference to this object. */ public Builder withReturnBody(boolean returnBody) @@ -219,7 +219,7 @@ public Builder withReturnBody(boolean returnBody) *

* Causes Riak to only return the metadata for the object. The value * will be asSet to null. - * @param returnHead true to return only metadata. + * @param returnHead true to return only metadata. * @return a reference to this object. */ public Builder withReturnHead(boolean returnHead) @@ -231,7 +231,7 @@ public Builder withReturnHead(boolean returnHead) /** * Set the if_not_modified flag for this StoreOperation. *

- * Setting this to true means to update the value only if the + * Setting this to true means to update the value only if the * vclock in the supplied object matches the one in the database. *

*

@@ -250,13 +250,13 @@ public Builder withIfNotModified(boolean ifNotModified) /** * Set the if_none_match flag value for this StoreOperation. *

- * Setting this to true means store the value only if this - * bucket/key combination are not already defined. + * Setting this to true means store the value only if this + * bucket/key combination are not already defined. *

- * Be aware that there are several cases where + * Be aware that there are several cases where * this may not actually happen. Use of this option is discouraged. *

- * + * * @param ifNoneMatch the if_non-match value. * @return a reference to this object. */ @@ -318,49 +318,49 @@ public Builder withSloppyQuorum(boolean sloppyQuorum) reqBuilder.setSloppyQuorum(sloppyQuorum); return this; } - + public StoreOperation build() { return new StoreOperation(this); } - + } - + /** * Response returned from a StoreOperation */ public static class Response extends FetchOperation.KvResponseBase { private final BinaryValue generatedKey; - + private Response(Init builder) { super(builder); this.generatedKey = builder.generatedKey; } - + public boolean hasGeneratedKey() { return generatedKey != null; } - + public BinaryValue getGeneratedKey() { return generatedKey; } - + protected static abstract class Init> extends FetchOperation.KvResponseBase.Init { private BinaryValue generatedKey; - + T withGeneratedKey(BinaryValue key) { this.generatedKey = key; return self(); } - + } - + static class Builder extends Init { @Override @@ -368,7 +368,7 @@ protected Builder self() { return this; } - + @Override protected Response build() { diff --git a/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java b/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java index 34dc859e8..2932e7544 100644 --- a/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java @@ -36,7 +36,7 @@ protected RiakMessage createChannelMessage() @Override protected byte[] decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, respMessageCode); + Operations.checkPBMessageType(rawMessage, respMessageCode); byte[] data = rawMessage.getData(); diff --git a/src/main/java/com/basho/riak/client/core/operations/YzDeleteIndexOperation.java b/src/main/java/com/basho/riak/client/core/operations/YzDeleteIndexOperation.java index a3af69ccc..165ffd0e1 100644 --- a/src/main/java/com/basho/riak/client/core/operations/YzDeleteIndexOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/YzDeleteIndexOperation.java @@ -30,15 +30,15 @@ public class YzDeleteIndexOperation extends FutureOperation { private final RiakYokozunaPB.RpbYokozunaIndexDeleteReq.Builder reqBuilder; private final String indexName; - + private YzDeleteIndexOperation(Builder builder) { this.reqBuilder = builder.reqBuilder; this.indexName = builder.indexName; } - + @Override - protected Void convert(List rawResponse) + protected Void convert(List rawResponse) { return null; } @@ -48,13 +48,13 @@ protected RiakMessage createChannelMessage() { RiakYokozunaPB.RpbYokozunaIndexDeleteReq req = reqBuilder.build(); return new RiakMessage(RiakMessageCodes.MSG_YokozunaIndexDeleteReq, req.toByteArray()); - + } @Override protected Void decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_DelResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_DelResp); return null; } @@ -63,13 +63,13 @@ public String getQueryInfo() { return indexName; } - + public static class Builder { private RiakYokozunaPB.RpbYokozunaIndexDeleteReq.Builder reqBuilder = RiakYokozunaPB.RpbYokozunaIndexDeleteReq.newBuilder(); private final String indexName; - + public Builder(String indexName) { if (null == indexName || indexName.length() == 0) @@ -79,7 +79,7 @@ public Builder(String indexName) reqBuilder.setName(ByteString.copyFromUtf8(indexName)); this.indexName = indexName; } - + public YzDeleteIndexOperation build() { return new YzDeleteIndexOperation(this); diff --git a/src/main/java/com/basho/riak/client/core/operations/YzFetchIndexOperation.java b/src/main/java/com/basho/riak/client/core/operations/YzFetchIndexOperation.java index 77191d0a9..1872d90a2 100644 --- a/src/main/java/com/basho/riak/client/core/operations/YzFetchIndexOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/YzFetchIndexOperation.java @@ -29,30 +29,30 @@ * @author Brian Roach */ public class YzFetchIndexOperation extends FutureOperation -{ +{ private final RiakYokozunaPB.RpbYokozunaIndexGetReq.Builder reqBuilder; private final String indexName; - + private YzFetchIndexOperation(Builder builder) { this.reqBuilder = builder.reqBuilder; this.indexName = builder.indexName; } - + @Override protected Response convert(List rawResponse) { // This isn't a streaming op, so there's only one protobuf in the list RiakYokozunaPB.RpbYokozunaIndexGetResp response = rawResponse.get(0); List indexList = new ArrayList(response.getIndexCount()); - + for (RiakYokozunaPB.RpbYokozunaIndex pbIndex : response.getIndexList()) { indexList.add(new YokozunaIndex(pbIndex.getName().toStringUtf8(), pbIndex.getSchema().toStringUtf8()) .withNVal(pbIndex.getNVal())); } - + return new Response(indexList); } @@ -61,13 +61,13 @@ protected RiakMessage createChannelMessage() { RiakYokozunaPB.RpbYokozunaIndexGetReq req = reqBuilder.build(); return new RiakMessage(RiakMessageCodes.MSG_YokozunaIndexGetReq, req.toByteArray()); - + } @Override protected RiakYokozunaPB.RpbYokozunaIndexGetResp decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_YokozunaIndexGetResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_YokozunaIndexGetResp); try { return RiakYokozunaPB.RpbYokozunaIndexGetResp.parseFrom(rawMessage.getData()); @@ -76,7 +76,7 @@ protected RiakYokozunaPB.RpbYokozunaIndexGetResp decode(RiakMessage rawMessage) { throw new IllegalArgumentException("Invalid message received", ex); } - + } @Override @@ -84,17 +84,17 @@ public String getQueryInfo() { return indexName; } - + public static class Builder { - private final RiakYokozunaPB.RpbYokozunaIndexGetReq.Builder reqBuilder = + private final RiakYokozunaPB.RpbYokozunaIndexGetReq.Builder reqBuilder = RiakYokozunaPB.RpbYokozunaIndexGetReq.newBuilder(); private String indexName = "All Indexes"; - + public Builder() {} - - public Builder withIndexName(String indexName) + + public Builder withIndexName(String indexName) { if (null == indexName || indexName.length() == 0) { @@ -104,22 +104,22 @@ public Builder withIndexName(String indexName) this.indexName = indexName; return this; } - + public YzFetchIndexOperation build() { return new YzFetchIndexOperation(this); } } - + public static class Response { private final List indexList; - + Response(List indexList) { this.indexList = indexList; } - + public List getIndexes() { return this.indexList; diff --git a/src/main/java/com/basho/riak/client/core/operations/YzGetSchemaOperation.java b/src/main/java/com/basho/riak/client/core/operations/YzGetSchemaOperation.java index 57eb53118..f725086ea 100644 --- a/src/main/java/com/basho/riak/client/core/operations/YzGetSchemaOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/YzGetSchemaOperation.java @@ -38,16 +38,16 @@ private YzGetSchemaOperation(Builder builder) this.reqBuilder = builder.reqBuilder; this.schemaName = builder.schemaName; } - + @Override protected YzGetSchemaOperation.Response convert(List rawResponse) { // This isn't a streaming op, so there's only one protobuf in the list RiakYokozunaPB.RpbYokozunaSchemaGetResp response = rawResponse.get(0); - + return new Response(new YokozunaSchema(response.getSchema().getName().toStringUtf8(), response.getSchema().getContent().toStringUtf8())); - + } @Override @@ -55,13 +55,13 @@ protected RiakMessage createChannelMessage() { RiakYokozunaPB.RpbYokozunaSchemaGetReq req = reqBuilder.build(); return new RiakMessage(RiakMessageCodes.MSG_YokozunaSchemaGetReq, req.toByteArray()); - + } @Override protected RiakYokozunaPB.RpbYokozunaSchemaGetResp decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_YokozunaSchemaGetResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_YokozunaSchemaGetResp); try { return RiakYokozunaPB.RpbYokozunaSchemaGetResp.parseFrom(rawMessage.getData()); @@ -70,7 +70,7 @@ protected RiakYokozunaPB.RpbYokozunaSchemaGetResp decode(RiakMessage rawMessage) { throw new IllegalArgumentException("Invalid message received", ex); } - + } @Override @@ -78,13 +78,13 @@ public String getQueryInfo() { return schemaName; } - + public static class Builder { - private final RiakYokozunaPB.RpbYokozunaSchemaGetReq.Builder reqBuilder = + private final RiakYokozunaPB.RpbYokozunaSchemaGetReq.Builder reqBuilder = RiakYokozunaPB.RpbYokozunaSchemaGetReq.newBuilder(); private final String schemaName; - + public Builder(String schemaName) { if (null == schemaName || schemaName.length() == 0) @@ -94,22 +94,22 @@ public Builder(String schemaName) reqBuilder.setName(ByteString.copyFromUtf8(schemaName)); this.schemaName = schemaName; } - + public YzGetSchemaOperation build() { return new YzGetSchemaOperation(this); } } - + public static class Response { private final YokozunaSchema schema; - + Response(YokozunaSchema schema) { this.schema = schema; } - + public YokozunaSchema getSchema() { return schema; diff --git a/src/main/java/com/basho/riak/client/core/operations/YzPutIndexOperation.java b/src/main/java/com/basho/riak/client/core/operations/YzPutIndexOperation.java index e50859733..1dd16ae04 100644 --- a/src/main/java/com/basho/riak/client/core/operations/YzPutIndexOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/YzPutIndexOperation.java @@ -31,13 +31,13 @@ public class YzPutIndexOperation extends FutureOperation rawResponse) { @@ -49,13 +49,13 @@ protected RiakMessage createChannelMessage() { RiakYokozunaPB.RpbYokozunaIndexPutReq req = reqBuilder.build(); return new RiakMessage(RiakMessageCodes.MSG_YokozunaIndexPutReq, req.toByteArray()); - + } @Override protected Void decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_PutResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_PutResp); return null; } @@ -64,7 +64,7 @@ public YokozunaIndex getQueryInfo() { return index; } - + public static class Builder { private final RiakYokozunaPB.RpbYokozunaIndexPutReq.Builder reqBuilder = @@ -82,7 +82,7 @@ public Builder(YokozunaIndex index) final RiakYokozunaPB.RpbYokozunaIndex.Builder indexBuilder = RiakYokozunaPB.RpbYokozunaIndex.newBuilder(); indexBuilder.setName(ByteString.copyFromUtf8(index.getName())); - // A null schema is valid; the default will be used + // A null schema is valid; the default will be used if (index.getSchema() != null) { indexBuilder.setSchema(ByteString.copyFromUtf8(index.getSchema())); diff --git a/src/main/java/com/basho/riak/client/core/operations/YzPutSchemaOperation.java b/src/main/java/com/basho/riak/client/core/operations/YzPutSchemaOperation.java index c19f9efd4..8e4c5da65 100644 --- a/src/main/java/com/basho/riak/client/core/operations/YzPutSchemaOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/YzPutSchemaOperation.java @@ -31,13 +31,13 @@ public class YzPutSchemaOperation extends FutureOperation rawResponse) { @@ -54,7 +54,7 @@ protected RiakMessage createChannelMessage() @Override protected Void decode(RiakMessage rawMessage) { - Operations.checkMessageType(rawMessage, RiakMessageCodes.MSG_PutResp); + Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_PutResp); return null; } @@ -63,24 +63,24 @@ public YokozunaSchema getQueryInfo() { return schema; } - + public static class Builder { private final RiakYokozunaPB.RpbYokozunaSchemaPutReq.Builder reqBuilder = RiakYokozunaPB.RpbYokozunaSchemaPutReq.newBuilder(); private final YokozunaSchema schema; - + public Builder(YokozunaSchema schema) { - RiakYokozunaPB.RpbYokozunaSchema.Builder schemaBuilder = + RiakYokozunaPB.RpbYokozunaSchema.Builder schemaBuilder = RiakYokozunaPB.RpbYokozunaSchema.newBuilder(); - + schemaBuilder.setName(ByteString.copyFromUtf8(schema.getName())); schemaBuilder.setContent(ByteString.copyFromUtf8(schema.getContent())); reqBuilder.setSchema(schemaBuilder); this.schema = schema; } - + public YzPutSchemaOperation build() { return new YzPutSchemaOperation(this); diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java b/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java index 47b754b39..6da295f4c 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/FetchOperation.java @@ -15,7 +15,6 @@ * @author Sergey Galkin * @since 2.0.3 */ -// TODO GH-611 We have the ability to send TsGetReq / Resp TTB messages public class FetchOperation extends TTBFutureOperation { private final Builder builder; diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java index a275a55c4..735297f07 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java @@ -1,6 +1,5 @@ package com.basho.riak.client.core.operations.ts; -import com.basho.riak.client.api.RiakException; import com.basho.riak.client.core.codec.InvalidTermToBinaryException; import com.basho.riak.client.core.codec.TermToBinaryCodec; import com.basho.riak.client.core.operations.TTBFutureOperation; @@ -8,17 +7,20 @@ import com.basho.riak.client.core.query.timeseries.QueryResult; import com.ericsson.otp.erlang.OtpErlangDecodeException; import com.ericsson.otp.erlang.OtpOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; import java.util.LinkedList; -import java.util.logging.Level; -import java.util.logging.Logger; class TTBConverters { + private static Logger logger = LoggerFactory.getLogger(TTBConverters.class); + private static abstract class MemoizingEncoder implements TTBFutureOperation.TTBEncoder { - protected byte[] message = null; protected final T builder; + protected byte[] message = null; MemoizingEncoder(T builder) { @@ -32,13 +34,15 @@ public byte[] build() { if (message == null) { - try { + try + { OtpOutputStream os = buildMessage(); os.flush(); message = os.toByteArray(); - } catch (IOException ex) { - // TODO GH-611 - Logger.getLogger(TTBConverters.class.getName()).log(Level.SEVERE, null, ex); + } + catch (IOException ex) + { + logger.error("Error creating term to binary message.", ex); } } @@ -76,9 +80,7 @@ OtpOutputStream buildMessage() { list.add(c); } - return TermToBinaryCodec.encodeTsGetRequest(builder.getTableName(), - list, - builder.getTimeout()); + return TermToBinaryCodec.encodeTsGetRequest(builder.getTableName(), list, builder.getTimeout()); } } @@ -112,12 +114,15 @@ public QueryResult parseFrom(byte[] data) { QueryResult rv; - try { + try + { rv = TermToBinaryCodec.decodeTsGetResponse(data); - } catch (OtpErlangDecodeException | InvalidTermToBinaryException ex) + } + catch (OtpErlangDecodeException | InvalidTermToBinaryException ex) { - Logger.getLogger(TTBConverters.class.getName()).log(Level.SEVERE, null, ex); - throw new IllegalArgumentException("Error decoding Riak TTB Response", ex); + final String errorMsg = "Error decoding Riak TTB response"; + logger.error(errorMsg, ex); + throw new IllegalArgumentException(errorMsg, ex); } return rv; diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java b/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java index 80d08ac1f..2f037a6cd 100644 --- a/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java +++ b/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java @@ -8,6 +8,8 @@ import com.ericsson.otp.erlang.OtpErlangLong; import com.ericsson.otp.erlang.OtpErlangObject; import com.google.protobuf.ByteString; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Calendar; import java.util.Date; @@ -31,6 +33,7 @@ public class Cell { + private static Logger logger = LoggerFactory.getLogger(Cell.class); static final Cell NullCell = new Cell(RiakTsPB.TsCell.newBuilder().build()); private final RiakTsPB.TsCell pbCell; @@ -220,8 +223,9 @@ public OtpErlangObject getErlangObject() { if (pbCell.hasDoubleValue()) { return new OtpErlangDouble(pbCell.getDoubleValue()); } - // TODO GH-611 throw exception? - return null; + + logger.error("Unknown TS cell type encountered."); + throw new IllegalArgumentException("Unknown TS cell type encountered."); } @Override diff --git a/src/test/java/com/basho/riak/client/core/operations/OperationsTest.java b/src/test/java/com/basho/riak/client/core/operations/OperationsTest.java index 9bdedfc90..d5656f57f 100644 --- a/src/test/java/com/basho/riak/client/core/operations/OperationsTest.java +++ b/src/test/java/com/basho/riak/client/core/operations/OperationsTest.java @@ -9,7 +9,7 @@ /** * Operations Class Unit Tests - + * * @author Alex Moore * @since 2.0.3 */ @@ -21,16 +21,18 @@ public void testThatMessageCodesAreWrittenCorrectlyInErrorMessages() // MSG_StartTls = 255 RiakMessage msg = null; - try { + try + { msg = new RiakMessage(RiakMessageCodes.MSG_StartTls, new byte[0]); - } catch (Exception ex) { - // TODO GH-611 + } + catch (Exception ex) + { assertTrue("unexpected exception", false); } try { - Operations.checkMessageType(msg, RiakMessageCodes.MSG_GetReq); + Operations.checkPBMessageType(msg, RiakMessageCodes.MSG_GetReq); } catch (IllegalStateException ex) { @@ -41,8 +43,8 @@ public void testThatMessageCodesAreWrittenCorrectlyInErrorMessages() @Test public void testThatSingedToUnsignedConversionIsCorrect() { - assertEquals(Operations.getUnsignedByteValue((byte)0x00), 0); - assertEquals(Operations.getUnsignedByteValue((byte)0xFF), 255); + assertEquals(Operations.getUnsignedByteValue((byte) 0x00), 0); + assertEquals(Operations.getUnsignedByteValue((byte) 0xFF), 255); assertEquals(Operations.getUnsignedIntValue(0x00000000), 0l); assertEquals(Operations.getUnsignedIntValue(0xffffffff), 4294967295l); } diff --git a/src/test/java/com/basho/riak/client/core/operations/codec/TermToBinaryCodecTest.java b/src/test/java/com/basho/riak/client/core/operations/codec/TermToBinaryCodecTest.java deleted file mode 100644 index 337f2e06b..000000000 --- a/src/test/java/com/basho/riak/client/core/operations/codec/TermToBinaryCodecTest.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.basho.riak.client.core.operations.codec; - - -public class TermToBinaryCodecTest -{ - -} From aa0d7a462df2639d5ca2114190c4b63e0df256ea Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 26 Apr 2016 15:39:40 -0400 Subject: [PATCH 41/52] Code cleanup --- .../client/core/codec/TermToBinaryCodec.java | 77 +++++++++---------- .../core/operations/ts/TTBConverters.java | 2 +- .../core/codec/TermToBinaryCodecTest.java | 2 +- 3 files changed, 38 insertions(+), 43 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index dbd642ace..fe4847c5e 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -17,26 +17,20 @@ public class TermToBinaryCodec { + private static final String TS_GET_REQ = "tsgetreq"; + private static final String TS_GET_RESP = "tsgetresp"; + private static final String TS_QUERY_REQ = "tsqueryreq"; + private static final String TS_QUERY_RESP = "tsqueryresp"; + private static final String TS_INTERPOLATION = "tsinterpolation"; + private static final String TS_PUT_REQ = "tsputreq"; + private static final String ERROR_RESP = "rpberrorresp"; + private static final String UNDEFINED = "undefined"; + private static final OtpErlangAtom TS_GET_REQ_ATOM = new OtpErlangAtom(TS_GET_REQ); + private static final OtpErlangAtom TS_QUERY_REQ_ATOM = new OtpErlangAtom(TS_QUERY_REQ); + private static final OtpErlangAtom TS_INTERPOLATION_ATOM = new OtpErlangAtom(TS_INTERPOLATION); + private static final OtpErlangAtom TS_PUT_REQ_ATOM = new OtpErlangAtom(TS_PUT_REQ); + private static final OtpErlangAtom UNDEFINED_ATOM = new OtpErlangAtom(UNDEFINED); private static Logger logger = LoggerFactory.getLogger(TermToBinaryCodec.class); - private static final class Messages - { - public static final OtpErlangAtom tsGetReq = new OtpErlangAtom("tsgetreq"); - public static final OtpErlangAtom tsGetResp = new OtpErlangAtom("tsgetresp"); - - public static final OtpErlangAtom tsQueryReq = new OtpErlangAtom("tsqueryreq"); - public static final OtpErlangAtom tsQueryResp = new OtpErlangAtom("tsqueryresp"); - public static final OtpErlangAtom tsInterpolation = new OtpErlangAtom("tsinterpolation"); - - public static final OtpErlangAtom tsPutReq = new OtpErlangAtom("tsputreq"); - public static final OtpErlangAtom tsPutResp = new OtpErlangAtom("tsputresp"); - - public static final OtpErlangAtom tsDelReq = new OtpErlangAtom("tsdelreq"); - public static final OtpErlangAtom tsDelResp = new OtpErlangAtom("tsdelresp"); - - public static final OtpErlangAtom rpbErrorResp = new OtpErlangAtom("rpberrorresp"); - } - - private static final OtpErlangAtom undefined = new OtpErlangAtom("undefined"); public static OtpOutputStream encodeTsGetRequest(String tableName, Collection keyValues, int timeout) { @@ -45,7 +39,7 @@ public static OtpOutputStream encodeTsGetRequest(String tableName, Collection rows) { final OtpOutputStream os = new OtpOutputStream(); @@ -104,7 +94,7 @@ public static OtpOutputStream encodeTsPutRequest(String tableName, Collection parseColumnDescriptions(O final ArrayList columnDescriptions = new ArrayList<>(colNameCount); for (int colDescIdx = 0; colDescIdx < colNameCount; colDescIdx++) { + final RiakTsPB.TsColumnDescription.Builder descBuilder = RiakTsPB.TsColumnDescription.newBuilder(); + + descBuilder.setName(ByteString.copyFromUtf8(columnNames[colDescIdx])); + descBuilder.setType(RiakTsPB.TsColumnType.valueOf(columnTypes[colDescIdx].toUpperCase(Locale.US))); - final RiakTsPB.TsColumnDescription desc = RiakTsPB.TsColumnDescription.newBuilder().setName( - ByteString.copyFromUtf8(columnNames[colDescIdx])).setType(RiakTsPB.TsColumnType.valueOf( - columnTypes[colDescIdx].toUpperCase(Locale.US))).build(); - columnDescriptions.add(desc); + columnDescriptions.add(descBuilder.build()); } return columnDescriptions; } - private static ArrayList parseRows(OtpInputStream is, List columnDescriptions) + private static ArrayList parseRows(OtpInputStream is, + List columnDescriptions) throws OtpErlangDecodeException, InvalidTermToBinaryException { final int rowCount = is.read_list_head(); @@ -245,7 +238,8 @@ private static RiakTsPB.TsRow parseRow(OtpInputStream is, List columnDescriptions, int j, OtpErlangObject cell) throws InvalidTermToBinaryException + private static Cell parseCell(List columnDescriptions, int j, OtpErlangObject cell) + throws InvalidTermToBinaryException { if (cell instanceof OtpErlangBinary) { @@ -283,7 +277,8 @@ else if (cell instanceof OtpErlangList) } else { - throw new InvalidTermToBinaryException("Unknown cell type encountered: " + cell.toString() + ", unable to continue parsing."); + throw new InvalidTermToBinaryException("Unknown cell type encountered: " + cell.toString() + ", unable to" + + " continue parsing."); } } } diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java index 735297f07..9c4895325 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java @@ -116,7 +116,7 @@ public QueryResult parseFrom(byte[] data) try { - rv = TermToBinaryCodec.decodeTsGetResponse(data); + rv = TermToBinaryCodec.decodeTsResultResponse(data); } catch (OtpErlangDecodeException | InvalidTermToBinaryException ex) { diff --git a/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java b/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java index 32d83a81f..9742f0822 100644 --- a/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java +++ b/src/test/java/com/basho/riak/client/core/codec/TermToBinaryCodecTest.java @@ -198,7 +198,7 @@ public void decodesQueryResultCorrectly() throws OtpErlangDecodeException try { - final QueryResult actual = TermToBinaryCodec.decodeTsQueryResponse(input); + final QueryResult actual = TermToBinaryCodec.decodeTsResultResponse(input); final List actualColumnDescriptions = actual.getColumnDescriptionsCopy(); final List actualRows = actual.getRowsCopy(); From 2bf1d8d11b97906daa6cffb24f3897d95e1d0c5b Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 26 Apr 2016 15:52:48 -0400 Subject: [PATCH 42/52] Remove unused case branch --- .../com/basho/riak/client/core/codec/TermToBinaryCodec.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index fe4847c5e..dc38c4603 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -137,10 +137,6 @@ private static QueryResult decodeTsResponse(byte[] response) final String respAtom = is.read_atom(); switch (respAtom) { - case ERROR_RESP: - // TODO process error - assert (msgArity == 3); - break; case TS_GET_RESP: case TS_QUERY_RESP: assert (msgArity == 2); From 2edbd44f345fa7bc1c3edfdfc3922a95ff171805 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 26 Apr 2016 14:02:55 -0700 Subject: [PATCH 43/52] No need to use OtpErlangAtom when write_atom() exists. --- .../basho/riak/client/core/RiakMessage.java | 8 +++---- .../client/core/codec/TermToBinaryCodec.java | 23 +++++++------------ 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/RiakMessage.java b/src/main/java/com/basho/riak/client/core/RiakMessage.java index 828396189..f0d42778f 100644 --- a/src/main/java/com/basho/riak/client/core/RiakMessage.java +++ b/src/main/java/com/basho/riak/client/core/RiakMessage.java @@ -6,12 +6,10 @@ import com.ericsson.otp.erlang.OtpErlangDecodeException; import com.ericsson.otp.erlang.OtpInputStream; import com.google.protobuf.InvalidProtocolBufferException; +import java.nio.charset.StandardCharsets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.charset.StandardCharsets; -import java.util.logging.Level; - /** * Encapsulates the raw bytes sent to or received from Riak. * @@ -25,8 +23,8 @@ public final class RiakMessage private final byte code; private final byte[] data; private final OtpInputStream ttbInputStream; - // TODO offer output stream? private final RiakResponseException riakError; + private static final String ERROR_RESP = "rpberrorresp"; public RiakMessage(byte code, byte[] data) { @@ -124,7 +122,7 @@ private RiakResponseException getRiakErrorFromTtb(OtpInputStream ttbInputStream) throw new IllegalArgumentException(decodeErrorMsg, ex); } - if ("rpberrorresp".equals(atom)) + if (ERROR_RESP.equals(atom)) { try { diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index dc38c4603..1f1e23905 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -6,14 +6,13 @@ import com.basho.riak.protobuf.RiakTsPB; import com.ericsson.otp.erlang.*; import com.google.protobuf.ByteString; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Locale; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TermToBinaryCodec { @@ -23,14 +22,8 @@ public class TermToBinaryCodec private static final String TS_QUERY_RESP = "tsqueryresp"; private static final String TS_INTERPOLATION = "tsinterpolation"; private static final String TS_PUT_REQ = "tsputreq"; - private static final String ERROR_RESP = "rpberrorresp"; private static final String UNDEFINED = "undefined"; - private static final OtpErlangAtom TS_GET_REQ_ATOM = new OtpErlangAtom(TS_GET_REQ); - private static final OtpErlangAtom TS_QUERY_REQ_ATOM = new OtpErlangAtom(TS_QUERY_REQ); - private static final OtpErlangAtom TS_INTERPOLATION_ATOM = new OtpErlangAtom(TS_INTERPOLATION); - private static final OtpErlangAtom TS_PUT_REQ_ATOM = new OtpErlangAtom(TS_PUT_REQ); - private static final OtpErlangAtom UNDEFINED_ATOM = new OtpErlangAtom(UNDEFINED); - private static Logger logger = LoggerFactory.getLogger(TermToBinaryCodec.class); + private static final Logger logger = LoggerFactory.getLogger(TermToBinaryCodec.class); public static OtpOutputStream encodeTsGetRequest(String tableName, Collection keyValues, int timeout) { @@ -39,7 +32,7 @@ public static OtpOutputStream encodeTsGetRequest(String tableName, Collection Date: Thu, 28 Apr 2016 09:22:46 -0400 Subject: [PATCH 44/52] Add security test option to makefile --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index e727eed1f..35f973165 100644 --- a/Makefile +++ b/Makefile @@ -16,5 +16,8 @@ integration-test: integration-test-timeseries: mvn -Pitest,default -Dcom.basho.riak.buckettype=true -Dcom.basho.riak.crdt=true -Dcom.basho.riak.lifecycle=true -Dcom.basho.riak.timeseries=true -Dcom.basho.riak.pbcport=$(RIAK_PORT) verify +integration-test-security: + mvn -Pitest,default -Dcom.basho.riak.security=true -Dcom.basho.riak.security.clientcert=true -Dcom.basho.riak.pbcport=$(RIAK_PORT) test-compile failsafe:integration-test + protogen: mvn -Pprotobuf-generate generate-sources From 5b092d20cd9c3907720d3c851d1a0217813a343b Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Fri, 29 Apr 2016 14:53:39 -0400 Subject: [PATCH 45/52] Remove PB objects entirely from TTB TS put path --- .../client/core/codec/TermToBinaryCodec.java | 44 +++- .../client/core/query/timeseries/Cell.java | 195 +++++++++++++----- .../query/timeseries/ConvertibleIterator.java | 5 +- 3 files changed, 187 insertions(+), 57 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index 1f1e23905..fdbe5c991 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -3,6 +3,7 @@ import com.basho.riak.client.core.query.timeseries.Cell; import com.basho.riak.client.core.query.timeseries.QueryResult; import com.basho.riak.client.core.query.timeseries.Row; +import com.basho.riak.client.core.util.CharsetUtils; import com.basho.riak.protobuf.RiakTsPB; import com.ericsson.otp.erlang.*; import com.google.protobuf.ByteString; @@ -36,9 +37,9 @@ public static OtpOutputStream encodeTsGetRequest(String tableName, Collection implements Iterator { + private static RiakTsPB.TsCell NullTSCell = RiakTsPB.TsCell.newBuilder().build(); private final Iterator iterator; public ConvertibleIterator(Iterator iterator) @@ -48,7 +49,7 @@ public ImmutablePBCellIterator(Iterator iterator) { protected RiakTsPB.TsCell convert(Cell cell) { if (cell == null) { - return Cell.NullCell.getPbCell(); + return NullTSCell; } return cell.getPbCell(); @@ -74,7 +75,7 @@ public ImmutableCellIterator(Iterator iterator) { @Override protected Cell convert(RiakTsPB.TsCell pbCell) { - if (pbCell.equals(Cell.NullCell.getPbCell())) + if (pbCell.equals(NullTSCell)) { return null; } From 9ccf8e70f6f96062dfdf500aedff0bb0d8b437e8 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 3 May 2016 10:47:55 -0400 Subject: [PATCH 46/52] Some small optimizations --- .../core/operations/ts/TTBConverters.java | 17 +- .../client/core/query/timeseries/Cell.java | 171 ++++++++---------- 2 files changed, 80 insertions(+), 108 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java index 9c4895325..cd2c9b997 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java @@ -20,7 +20,6 @@ class TTBConverters private static abstract class MemoizingEncoder implements TTBFutureOperation.TTBEncoder { protected final T builder; - protected byte[] message = null; MemoizingEncoder(T builder) { @@ -32,21 +31,7 @@ private static abstract class MemoizingEncoder implements TTBFutureOperation. @Override public byte[] build() { - if (message == null) - { - try - { - OtpOutputStream os = buildMessage(); - os.flush(); - message = os.toByteArray(); - } - catch (IOException ex) - { - logger.error("Error creating term to binary message.", ex); - } - } - - return message; + return buildMessage().toByteArray(); } } diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java b/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java index 20156146b..33d66deb3 100644 --- a/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java +++ b/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java @@ -3,7 +3,6 @@ import com.basho.riak.client.core.util.BinaryValue; import com.basho.riak.client.core.util.CharsetUtils; import com.basho.riak.protobuf.RiakTsPB; -import com.ericsson.otp.erlang.*; import com.google.protobuf.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,12 +29,12 @@ public class Cell { - private static Logger logger = LoggerFactory.getLogger(Cell.class); - private final String varcharValue; - private final Long sint64Value; - private final Double doubleValue; - private final Long timestampValue; - private final Boolean booleanValue; + private String varcharValue = ""; + private long sint64Value = 0L; + private double doubleValue = 0.0; + private long timestampValue = 0L; + private boolean booleanValue = false; + private int typeBitfield = 0; /** * Creates a new "Varchar" Cell, based on the UTF8 binary encoding of the provided String. @@ -49,11 +48,7 @@ public Cell(String varcharValue) throw new IllegalArgumentException("String value cannot be NULL."); } - this.varcharValue = varcharValue; - this.sint64Value = null; - this.doubleValue = null; - this.timestampValue = null; - this.booleanValue = null; + initVarchar(varcharValue); } /** @@ -68,11 +63,8 @@ public Cell(BinaryValue varcharValue) throw new IllegalArgumentException("BinaryValue value cannot be NULL."); } - this.varcharValue = varcharValue.toStringUtf8(); - this.sint64Value = null; - this.doubleValue = null; - this.timestampValue = null; - this.booleanValue = null; + initVarchar(varcharValue.toStringUtf8()); + } /** @@ -82,11 +74,7 @@ public Cell(BinaryValue varcharValue) */ public Cell(long sint64Value) { - this.varcharValue = null; - this.sint64Value = sint64Value; - this.doubleValue = null; - this.timestampValue = null; - this.booleanValue = null; + initSInt64(sint64Value); } /** @@ -96,11 +84,7 @@ public Cell(long sint64Value) */ public Cell(double doubleValue) { - this.varcharValue = null; - this.sint64Value = null; - this.doubleValue = doubleValue; - this.timestampValue = null; - this.booleanValue = null; + initDouble(doubleValue); } /** @@ -110,11 +94,7 @@ public Cell(double doubleValue) */ public Cell(boolean booleanValue) { - this.varcharValue = null; - this.sint64Value = null; - this.doubleValue = null; - this.timestampValue = null; - this.booleanValue = booleanValue; + initBoolean(booleanValue); } /** @@ -129,11 +109,7 @@ public Cell(Calendar timestampValue) throw new IllegalArgumentException("Calendar object for timestamp value cannot be NULL."); } - this.varcharValue = null; - this.sint64Value = null; - this.doubleValue = null; - this.timestampValue = timestampValue.getTimeInMillis(); - this.booleanValue = null; + initTimestamp(timestampValue.getTimeInMillis()); } /** @@ -148,54 +124,30 @@ public Cell(Date timestampValue) throw new IllegalArgumentException("Date object for timestamp value cannot be NULL."); } - this.varcharValue = null; - this.sint64Value = null; - this.doubleValue = null; - this.timestampValue = timestampValue.getTime(); - this.booleanValue = null; + initTimestamp(timestampValue.getTime()); } Cell(RiakTsPB.TsCell pbCell) { if (pbCell.hasBooleanValue()) { - this.varcharValue = null; - this.sint64Value = null; - this.doubleValue = null; - this.timestampValue = null; - this.booleanValue = pbCell.getBooleanValue(); + initBoolean(pbCell.getBooleanValue()); } else if (pbCell.hasDoubleValue()) { - this.varcharValue = null; - this.sint64Value = null; - this.doubleValue = pbCell.getDoubleValue(); - this.timestampValue = null; - this.booleanValue = null; + initDouble(pbCell.getDoubleValue()); } else if (pbCell.hasSint64Value()) { - this.varcharValue = null; - this.sint64Value = pbCell.getSint64Value(); - this.doubleValue = null; - this.timestampValue = null; - this.booleanValue = null; + initSInt64(pbCell.getSint64Value()); } else if (pbCell.hasTimestampValue()) { - this.varcharValue = null; - this.sint64Value = null; - this.doubleValue = null; - this.timestampValue = pbCell.getTimestampValue(); - this.booleanValue = null; + initTimestamp(pbCell.getTimestampValue()); } else if(pbCell.hasVarcharValue()) { - this.varcharValue = pbCell.getVarcharValue().toStringUtf8(); - this.sint64Value = null; - this.doubleValue = null; - this.timestampValue = null; - this.booleanValue = null; + initVarchar(pbCell.getVarcharValue().toStringUtf8()); } else { @@ -203,13 +155,38 @@ else if(pbCell.hasVarcharValue()) } } - private Cell(long rawTimestampValue, boolean isTimestamp) + private Cell() + { + } + + private void initBoolean(boolean booleanValue) { - this.varcharValue = null; - this.sint64Value = null; - this.doubleValue = null; - this.timestampValue = rawTimestampValue; - this.booleanValue = null; + typeBitfield |= 0x00000010; + this.booleanValue = booleanValue; + } + + private void initTimestamp(long timestampValue) + { + typeBitfield |= 0x00000008; + this.timestampValue = timestampValue; + } + + private void initDouble(double doubleValue) + { + typeBitfield |= 0x00000004; + this.doubleValue = doubleValue; + } + + private void initSInt64(long longValue) + { + typeBitfield |= 0x00000002; + this.sint64Value = longValue; + } + + private void initVarchar(String stringValue) + { + typeBitfield |= 0x00000001; + this.varcharValue = stringValue; } /** @@ -220,32 +197,34 @@ private Cell(long rawTimestampValue, boolean isTimestamp) */ public static Cell newTimestamp(long rawTimestampValue) { - return new Cell(rawTimestampValue, true); + final Cell cell = new Cell(); + cell.initTimestamp(rawTimestampValue); + return cell; } public boolean hasVarcharValue() { - return varcharValue != null; + return ((typeBitfield & 0x00000001) == 0x00000001); } public boolean hasLong() { - return sint64Value != null; + return ((typeBitfield & 0x00000002) == 0x00000002); } - public boolean hasTimestamp() + public boolean hasDouble() { - return timestampValue != null; + return ((typeBitfield & 0x00000004) == 0x00000004); } - public boolean hasBoolean() + public boolean hasTimestamp() { - return booleanValue != null; + return ((typeBitfield & 0x00000008) == 0x00000008); } - public boolean hasDouble() + public boolean hasBoolean() { - return doubleValue != null; + return ((typeBitfield & 0x00000010) == 0x00000010); } public String getVarcharAsUTF8String() @@ -359,34 +338,42 @@ public boolean equals(Object o) Cell cell = (Cell) o; - if (varcharValue != null ? !varcharValue.equals(cell.varcharValue) : cell.varcharValue != null) + if (sint64Value != cell.sint64Value) + { + return false; + } + if (Double.compare(cell.doubleValue, doubleValue) != 0) { return false; } - if (sint64Value != null ? !sint64Value.equals(cell.sint64Value) : cell.sint64Value != null) + if (timestampValue != cell.timestampValue) { return false; } - if (doubleValue != null ? !doubleValue.equals(cell.doubleValue) : cell.doubleValue != null) + if (booleanValue != cell.booleanValue) { return false; } - if (timestampValue != null ? !timestampValue.equals(cell.timestampValue) : cell.timestampValue != null) + if (typeBitfield != cell.typeBitfield) { return false; } - return booleanValue != null ? booleanValue.equals(cell.booleanValue) : cell.booleanValue == null; + return varcharValue.equals(cell.varcharValue); } @Override public int hashCode() { - int result = varcharValue != null ? varcharValue.hashCode() : 0; - result = 31 * result + (sint64Value != null ? sint64Value.hashCode() : 0); - result = 31 * result + (doubleValue != null ? doubleValue.hashCode() : 0); - result = 31 * result + (timestampValue != null ? timestampValue.hashCode() : 0); - result = 31 * result + (booleanValue != null ? booleanValue.hashCode() : 0); + int result; + long temp; + result = varcharValue.hashCode(); + result = 31 * result + (int) (sint64Value ^ (sint64Value >>> 32)); + temp = Double.doubleToLongBits(doubleValue); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + result = 31 * result + (int) (timestampValue ^ (timestampValue >>> 32)); + result = 31 * result + (booleanValue ? 1 : 0); + result = 31 * result + typeBitfield; return result; } } From 1f353b48e5104eb7df9d77fa8030d61fee5f2842 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Wed, 4 May 2016 15:01:34 -0400 Subject: [PATCH 47/52] Add test for PB error handling while using TTB operation --- .../api/commands/itest/ITestTimeSeries.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/test/java/com/basho/riak/client/api/commands/itest/ITestTimeSeries.java b/src/test/java/com/basho/riak/client/api/commands/itest/ITestTimeSeries.java index 32d3b2d9e..2a69115a3 100644 --- a/src/test/java/com/basho/riak/client/api/commands/itest/ITestTimeSeries.java +++ b/src/test/java/com/basho/riak/client/api/commands/itest/ITestTimeSeries.java @@ -4,13 +4,18 @@ import com.basho.riak.client.api.commands.buckets.FetchBucketProperties; import com.basho.riak.client.api.commands.buckets.StoreBucketProperties; import com.basho.riak.client.api.commands.timeseries.*; +import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.RiakNode; import com.basho.riak.client.core.operations.FetchBucketPropsOperation; +import com.basho.riak.client.core.operations.itest.ITestBase; import com.basho.riak.client.core.operations.itest.ts.ITestTsBase; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.query.timeseries.*; import org.junit.FixMethodOrder; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.runners.MethodSorters; import java.util.*; @@ -18,6 +23,7 @@ import java.util.concurrent.TimeUnit; import static org.junit.Assert.*; +import static org.junit.Assume.assumeTrue; /** * Time Series Commands Integration Tests @@ -70,6 +76,9 @@ private RiakFuture createTableAsync(final RiakClient client, Strin return client.executeAsync(cmd); } + @Rule + public ExpectedException thrown= ExpectedException.none(); + @Test public void test_a_TestCreateTableAndChangeNVal() throws InterruptedException, ExecutionException { @@ -384,6 +393,25 @@ public void test_r_TestDescribeTableCommandForNonExistingTable() throws Interrup assertTrue(message.toLowerCase().contains("not an active table")); } + @Test + public void test_z_TestPBCErrorsReturnWhenSecurityIsOn() throws InterruptedException, ExecutionException + { + assumeTrue(security); + + thrown.expect(ExecutionException.class); + thrown.expectMessage("Security is enabled, please STARTTLS first"); + + // Build connection WITHOUT security + final RiakNode node = new RiakNode.Builder().withRemoteAddress(hostname).withRemotePort(pbcPort).build(); + final RiakCluster cluster = new RiakCluster.Builder(node).build(); + cluster.start(); + final RiakClient client = new RiakClient(cluster); + + Query query = new Query.Builder("DESCRIBE " + tableName).build(); + + final QueryResult result = client.execute(query); + } + private static List GetCreatedTableFullDescriptions() { return Arrays.asList(new FullColumnDescription("geohash", ColumnDescription.ColumnType.VARCHAR, false, 1), From a0beb8e916ebb77f59d96f358086390e20c29790 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Wed, 4 May 2016 16:00:30 -0400 Subject: [PATCH 48/52] A few PR cleanup items --- .../core/operations/ts/TTBConverters.java | 11 ++- .../client/core/query/timeseries/Cell.java | 72 +++++++++++-------- 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java index cd2c9b997..30de0ba28 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java +++ b/src/main/java/com/basho/riak/client/core/operations/ts/TTBConverters.java @@ -10,18 +10,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.LinkedList; class TTBConverters { private static Logger logger = LoggerFactory.getLogger(TTBConverters.class); - private static abstract class MemoizingEncoder implements TTBFutureOperation.TTBEncoder + private static abstract class BuilderTTBEncoder implements TTBFutureOperation.TTBEncoder { protected final T builder; - MemoizingEncoder(T builder) + BuilderTTBEncoder(T builder) { this.builder = builder; } @@ -35,7 +34,7 @@ public byte[] build() } } - static class StoreEncoder extends MemoizingEncoder + static class StoreEncoder extends BuilderTTBEncoder { StoreEncoder(StoreOperation.Builder builder) { @@ -49,7 +48,7 @@ OtpOutputStream buildMessage() } } - static class FetchEncoder extends MemoizingEncoder + static class FetchEncoder extends BuilderTTBEncoder { FetchEncoder(FetchOperation.Builder builder) { @@ -69,7 +68,7 @@ OtpOutputStream buildMessage() } } - static class QueryEncoder extends MemoizingEncoder + static class QueryEncoder extends BuilderTTBEncoder { QueryEncoder(QueryOperation.Builder builder) { diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java b/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java index 33d66deb3..1f2bf05a3 100644 --- a/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java +++ b/src/main/java/com/basho/riak/client/core/query/timeseries/Cell.java @@ -4,8 +4,6 @@ import com.basho.riak.client.core.util.CharsetUtils; import com.basho.riak.protobuf.RiakTsPB; import com.google.protobuf.ByteString; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Calendar; import java.util.Date; @@ -29,12 +27,18 @@ public class Cell { + private static final int VARCHAR_MASK = 0x00000001; + private static final int SINT64_MASK = 0x00000002; + private static final int DOUBLE_MASK = 0x00000004; + private static final int TIMESTAMP_MASK = 0x00000008; + private static final int BOOLEAN_MASK = 0x00000010; + private int typeBitfield = 0x0; + private String varcharValue = ""; private long sint64Value = 0L; private double doubleValue = 0.0; private long timestampValue = 0L; private boolean booleanValue = false; - private int typeBitfield = 0; /** * Creates a new "Varchar" Cell, based on the UTF8 binary encoding of the provided String. @@ -145,7 +149,7 @@ else if (pbCell.hasTimestampValue()) { initTimestamp(pbCell.getTimestampValue()); } - else if(pbCell.hasVarcharValue()) + else if (pbCell.hasVarcharValue()) { initVarchar(pbCell.getVarcharValue().toStringUtf8()); } @@ -159,72 +163,82 @@ private Cell() { } + /** + * Creates a new "Timestamp" cell from the provided raw value. + * + * @param rawTimestampValue The epoch timestamp, including milliseconds. + * @return The new timestamp Cell. + */ + public static Cell newTimestamp(long rawTimestampValue) + { + final Cell cell = new Cell(); + cell.initTimestamp(rawTimestampValue); + return cell; + } + private void initBoolean(boolean booleanValue) { - typeBitfield |= 0x00000010; + setBitfieldType(BOOLEAN_MASK); this.booleanValue = booleanValue; } private void initTimestamp(long timestampValue) { - typeBitfield |= 0x00000008; + setBitfieldType(TIMESTAMP_MASK); this.timestampValue = timestampValue; } private void initDouble(double doubleValue) { - typeBitfield |= 0x00000004; + setBitfieldType(DOUBLE_MASK); this.doubleValue = doubleValue; } private void initSInt64(long longValue) { - typeBitfield |= 0x00000002; + setBitfieldType(SINT64_MASK); this.sint64Value = longValue; } private void initVarchar(String stringValue) { - typeBitfield |= 0x00000001; + setBitfieldType(VARCHAR_MASK); this.varcharValue = stringValue; } - /** - * Creates a new "Timestamp" cell from the provided raw value. - * - * @param rawTimestampValue The epoch timestamp, including milliseconds. - * @return The new timestamp Cell. - */ - public static Cell newTimestamp(long rawTimestampValue) + private void setBitfieldType(int mask) { - final Cell cell = new Cell(); - cell.initTimestamp(rawTimestampValue); - return cell; + typeBitfield |= mask; + } + + private boolean bitfieldHasType(int mask) + { + return (typeBitfield & mask) == mask; } public boolean hasVarcharValue() { - return ((typeBitfield & 0x00000001) == 0x00000001); + return bitfieldHasType(VARCHAR_MASK); } public boolean hasLong() { - return ((typeBitfield & 0x00000002) == 0x00000002); + return bitfieldHasType(SINT64_MASK); } public boolean hasDouble() { - return ((typeBitfield & 0x00000004) == 0x00000004); + return bitfieldHasType(DOUBLE_MASK); } public boolean hasTimestamp() { - return ((typeBitfield & 0x00000008) == 0x00000008); + return bitfieldHasType(TIMESTAMP_MASK); } public boolean hasBoolean() { - return ((typeBitfield & 0x00000010) == 0x00000010); + return bitfieldHasType(BOOLEAN_MASK); } public String getVarcharAsUTF8String() @@ -257,7 +271,7 @@ public boolean getBoolean() return booleanValue; } - public RiakTsPB.TsCell getPbCell() + RiakTsPB.TsCell getPbCell() { final RiakTsPB.TsCell.Builder builder = RiakTsPB.TsCell.newBuilder(); @@ -265,19 +279,19 @@ public RiakTsPB.TsCell getPbCell() { builder.setVarcharValue(ByteString.copyFromUtf8(varcharValue)); } - if (hasLong()) + else if (hasLong()) { builder.setSint64Value(sint64Value); } - if (hasTimestamp()) + else if (hasTimestamp()) { builder.setTimestampValue(timestampValue); } - if (hasBoolean()) + else if (hasBoolean()) { builder.setBooleanValue(booleanValue); } - if (hasDouble()) + else if (hasDouble()) { builder.setDoubleValue(doubleValue); } From 3d7cd4d3d2e7300eac353356e6f9b2312a31a5df Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Wed, 4 May 2016 16:07:28 -0400 Subject: [PATCH 49/52] The final final --- .../riak/client/core/query/timeseries/ConvertibleIterator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIterator.java b/src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIterator.java index f74001b0b..358a175e1 100644 --- a/src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIterator.java +++ b/src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIterator.java @@ -11,7 +11,7 @@ */ public abstract class ConvertibleIterator implements Iterator { - private static RiakTsPB.TsCell NullTSCell = RiakTsPB.TsCell.newBuilder().build(); + private static final RiakTsPB.TsCell NullTSCell = RiakTsPB.TsCell.newBuilder().build(); private final Iterator iterator; public ConvertibleIterator(Iterator iterator) From 450a95e2235d9f633f09003a8727047ef587d1ad Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Thu, 5 May 2016 15:16:09 -0400 Subject: [PATCH 50/52] Dont do error checking while writing TTB messages --- .../basho/riak/client/core/RiakMessage.java | 47 +++++++++---------- .../core/operations/TTBFutureOperation.java | 2 +- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/RiakMessage.java b/src/main/java/com/basho/riak/client/core/RiakMessage.java index f0d42778f..c1272a59a 100644 --- a/src/main/java/com/basho/riak/client/core/RiakMessage.java +++ b/src/main/java/com/basho/riak/client/core/RiakMessage.java @@ -22,30 +22,37 @@ public final class RiakMessage private static final Logger logger = LoggerFactory.getLogger(RiakMessage.class); private final byte code; private final byte[] data; - private final OtpInputStream ttbInputStream; private final RiakResponseException riakError; private static final String ERROR_RESP = "rpberrorresp"; public RiakMessage(byte code, byte[] data) + { + this(code, data, true); + } + + public RiakMessage(byte code, byte[] data, boolean doErrorCheck) { this.code = code; this.data = data; - switch (this.code) + if(doErrorCheck) + { + switch (this.code) + { + case RiakMessageCodes.MSG_ErrorResp: + this.riakError = getRiakErrorFromPbuf(this.data); + break; + case RiakMessageCodes.MSG_TsTtbMsg: + OtpInputStream ttbInputStream = new OtpInputStream(data); + this.riakError = getRiakErrorFromTtb(ttbInputStream); + break; + default: + this.riakError = null; + } + } + else { - case RiakMessageCodes.MSG_ErrorResp: - this.ttbInputStream = null; - this.riakError = getRiakErrorFromPbuf(this.data); - break; - case RiakMessageCodes.MSG_TsTtbMsg: - this.ttbInputStream = new OtpInputStream(data); - this.riakError = getRiakErrorFromTtb(this.ttbInputStream); - // Set stream back to the beginning for codec's use - this.ttbInputStream.setPos(0); - break; - default: - this.riakError = null; - this.ttbInputStream = null; + this.riakError = null; } } @@ -73,21 +80,11 @@ public byte[] getData() return data; } - public boolean isTtbMessage() - { - return this.ttbInputStream != null; - } - public boolean isRiakError() { return this.riakError != null; } - public OtpInputStream getTtbStream() - { - return this.ttbInputStream; - } - public RiakResponseException getRiakError() { return this.riakError; diff --git a/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java b/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java index 2932e7544..1d757a868 100644 --- a/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/TTBFutureOperation.java @@ -30,7 +30,7 @@ protected TTBFutureOperation(TTBEncoder requestBuilder, TTBParser responsePar @Override protected RiakMessage createChannelMessage() { - return new RiakMessage(reqMessageCode, requestBuilder.build()); + return new RiakMessage(reqMessageCode, requestBuilder.build(), false); } @Override From 8918c5b01e554f79bedd5c853b76f0d881d07dda Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Thu, 5 May 2016 16:47:02 -0400 Subject: [PATCH 51/52] Dont convert to PB and back when creating a row --- .../riak/client/core/FutureOperation.java | 2 +- .../core/netty/RiakOperationEncoder.java | 2 +- .../client/core/query/timeseries/Row.java | 56 ++++++++++++++----- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/FutureOperation.java b/src/main/java/com/basho/riak/client/core/FutureOperation.java index 0e0c2aa64..49d112b9d 100644 --- a/src/main/java/com/basho/riak/client/core/FutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/FutureOperation.java @@ -261,7 +261,7 @@ synchronized final void setException(Throwable t) public synchronized final Object channelMessage() { - Object message = createChannelMessage(); + final Object message = createChannelMessage(); state = State.WRITTEN; return message; } diff --git a/src/main/java/com/basho/riak/client/core/netty/RiakOperationEncoder.java b/src/main/java/com/basho/riak/client/core/netty/RiakOperationEncoder.java index 619c98b1d..35b5eeec2 100644 --- a/src/main/java/com/basho/riak/client/core/netty/RiakOperationEncoder.java +++ b/src/main/java/com/basho/riak/client/core/netty/RiakOperationEncoder.java @@ -33,5 +33,5 @@ protected void encode(ChannelHandlerContext ctx, FutureOperation operation, List { out.add(operation.channelMessage()); } - + } diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/Row.java b/src/main/java/com/basho/riak/client/core/query/timeseries/Row.java index 7a6c60b31..0a5790c45 100644 --- a/src/main/java/com/basho/riak/client/core/query/timeseries/Row.java +++ b/src/main/java/com/basho/riak/client/core/query/timeseries/Row.java @@ -14,6 +14,8 @@ public class Row implements Iterable { private final RiakTsPB.TsRow pbRow; + private final Iterable cells; + private final int cellCount; /** * Create a new row. @@ -21,11 +23,16 @@ public class Row implements Iterable */ public Row(Iterable cells) { - final RiakTsPB.TsRow.Builder builder = RiakTsPB.TsRow.newBuilder(); + pbRow = null; + this.cells = cells; + int cellCount = 0; - builder.addAllCells(ConvertibleIterable.asIterablePbCell(cells)); + for (Cell ignored : this.cells) + { + cellCount++; + } - this.pbRow = builder.build(); + this.cellCount = cellCount; } /** @@ -34,16 +41,16 @@ public Row(Iterable cells) */ public Row(Cell... cells) { - final RiakTsPB.TsRow.Builder builder = RiakTsPB.TsRow.newBuilder(); - // TODO: consider avoiding ArrayList creation under the hood of Arrays.asList - builder.addAllCells(ConvertibleIterable.asIterablePbCell(Arrays.asList(cells))); - - this.pbRow = builder.build(); + pbRow = null; + this.cells = Arrays.asList(cells); + cellCount = cells.length; } Row(RiakTsPB.TsRow pbRow) { this.pbRow = pbRow; + cells = null; + cellCount = pbRow.getCellsCount(); } /** @@ -52,7 +59,7 @@ public Row(Cell... cells) */ public int getCellsCount() { - return pbRow.getCellsCount(); + return cellCount; } /** @@ -72,7 +79,14 @@ public List getCellsCopy() public RiakTsPB.TsRow getPbRow() { - return pbRow; + if(pbRow != null) + { + return pbRow; + } + + RiakTsPB.TsRow.Builder builder = RiakTsPB.TsRow.newBuilder(); + builder.addAllCells(ConvertibleIterable.asIterablePbCell(cells)); + return builder.build(); } /** @@ -82,7 +96,14 @@ public RiakTsPB.TsRow getPbRow() @Override public Iterator iterator() { - return ConvertibleIterator.iterateAsCell(pbRow.getCellsList().iterator()); + if(cells != null) + { + return cells.iterator(); + } + else // if (pbRow != null) + { + return ConvertibleIterator.iterateAsCell(pbRow.getCellsList().iterator()); + } } @Override @@ -97,15 +118,22 @@ public boolean equals(Object o) return false; } - Row cells = (Row) o; + Row cells1 = (Row) o; - return !(pbRow != null ? !pbRow.equals(cells.pbRow) : cells.pbRow != null); + if (cellCount != cells1.cellCount) + { + return false; + } + return getCellsCopy().equals(cells1.getCellsCopy()); } @Override public int hashCode() { - return pbRow != null ? pbRow.hashCode() : 0; + int result = pbRow != null ? pbRow.hashCode() : 0; + result = 31 * result + (cells != null ? cells.hashCode() : 0); + result = 31 * result + cellCount; + return result; } } From 09ee18fbd111f49f47399a065b750a8fd2d204ed Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Thu, 5 May 2016 21:42:40 -0400 Subject: [PATCH 52/52] Adding single atom response handling --- .../basho/riak/client/core/RiakMessage.java | 10 +++++ .../client/core/codec/TermToBinaryCodec.java | 38 ++++++++++++++++--- .../itest/ts/ITestQueryOperation.java | 19 ++++++++++ 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/RiakMessage.java b/src/main/java/com/basho/riak/client/core/RiakMessage.java index c1272a59a..af27d1901 100644 --- a/src/main/java/com/basho/riak/client/core/RiakMessage.java +++ b/src/main/java/com/basho/riak/client/core/RiakMessage.java @@ -4,6 +4,7 @@ import com.basho.riak.protobuf.RiakMessageCodes; import com.basho.riak.protobuf.RiakPB; import com.ericsson.otp.erlang.OtpErlangDecodeException; +import com.ericsson.otp.erlang.OtpExternal; import com.ericsson.otp.erlang.OtpInputStream; import com.google.protobuf.InvalidProtocolBufferException; import java.nio.charset.StandardCharsets; @@ -97,6 +98,15 @@ private RiakResponseException getRiakErrorFromTtb(OtpInputStream ttbInputStream) try { + int firstByte = ttbInputStream.read1skip_version(); + + if(firstByte != OtpExternal.smallTupleTag && firstByte != OtpExternal.largeTupleTag) + { + return null; + } + + ttbInputStream.reset(); + ttbMsgArity = ttbInputStream.read_tuple_head(); } catch (OtpErlangDecodeException ex) diff --git a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java index fdbe5c991..d477733ad 100644 --- a/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java +++ b/src/main/java/com/basho/riak/client/core/codec/TermToBinaryCodec.java @@ -8,10 +8,8 @@ import com.ericsson.otp.erlang.*; import com.google.protobuf.ByteString; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Locale; +import java.util.*; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -152,6 +150,37 @@ private static QueryResult decodeTsResponse(byte[] response) QueryResult result = null; OtpInputStream is = new OtpInputStream(response); + + int firstByte = is.read1skip_version(); + is.reset(); + + if(firstByte != OtpExternal.smallTupleTag && firstByte != OtpExternal.largeTupleTag) + { + return parseAtomResult(is); + } + + return parseTupleResult(is); + } + + private static QueryResult parseAtomResult(OtpInputStream is) + throws OtpErlangDecodeException, InvalidTermToBinaryException + { + final String responseAtom = is.read_atom(); + + if(Objects.equals(responseAtom, TS_QUERY_RESP)) + { + return QueryResult.EMPTY; + } + + throw new InvalidTermToBinaryException("Invalid Response atom encountered: " + + responseAtom + ". Was expecting tsqueryresp"); + + } + + private static QueryResult parseTupleResult(OtpInputStream is) + throws OtpErlangDecodeException, InvalidTermToBinaryException + { + QueryResult result; final int msgArity = is.read_tuple_head(); // Response is: // {'rpberrorresp', ErrMsg, ErrCode} @@ -179,7 +208,6 @@ private static QueryResult decodeTsResponse(byte[] response) logger.error(errorMsg); throw new IllegalArgumentException(errorMsg); } - return result; } diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ts/ITestQueryOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ts/ITestQueryOperation.java index 7c0056724..f77d54d51 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ts/ITestQueryOperation.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ts/ITestQueryOperation.java @@ -7,6 +7,7 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.util.Random; import java.util.concurrent.ExecutionException; import static org.junit.Assert.*; @@ -67,4 +68,22 @@ public void querySomeMatches() throws ExecutionException, InterruptedException assertEquals(7, queryResult.getColumnDescriptionsCopy().size()); assertEquals(1, queryResult.getRowsCount()); } + + @Test + public void queryCreateTable() throws ExecutionException, InterruptedException + { + final String queryText = "CREATE TABLE BobbyTables" + Integer.toString(new Random().nextInt()) + + " (" + + " k1 varchar not null," + + " k2 varchar not null," + + " k3 timestamp not null, " + + " PRIMARY KEY((k1, k2, quantum(k3, 15, 'm')),k1, k2, k3)" + + " )"; + + final QueryResult queryResult = executeQuery(new QueryOperation.Builder(queryText)); + + assertNotNull(queryResult); + assertEquals(0, queryResult.getColumnDescriptionsCopy().size()); + assertEquals(0, queryResult.getRowsCount()); + } }