diff --git a/pom.xml b/pom.xml index 5f03a49bc..7e298c564 100644 --- a/pom.xml +++ b/pom.xml @@ -249,6 +249,18 @@ ${hyracks.version} + + org.apache.hyracks + hyracks-http + ${hyracks.fullstack.version} + + + + io.netty + netty-all + 4.1.6.Final + + org.apache.hyracks algebricks-compiler @@ -670,6 +682,8 @@ UTF-8 UTF-8 + 0.2.17-incubating + 0.3.1 0.3.0 0.11 @@ -680,5 +694,6 @@ vxquery-cli vxquery-xtest vxquery-benchmark + vxquery-rest diff --git a/src/site/apt/user_query.apt b/src/site/apt/user_query.apt index c5132c3f4..f4d0dd5ab 100644 --- a/src/site/apt/user_query.apt +++ b/src/site/apt/user_query.apt @@ -35,21 +35,26 @@ vxq "path-to"\test.xq Command line options for all systems. ---------------------------------------- --O N : Optimization Level. Default: Full Optimization --available-processors N : Number of available processors. (default java's available processors) --client-net-ip-address VAL : IP Address of the ClusterController --client-net-port N : Port of the ClusterController (default 1098) --compileonly : Compile the query and stop --frame-size N : Frame size in bytes. (default 65536) --local-node-controllers N : Number of local node controllers (default 1) --repeatexec N : Number of times to repeat execution --showast : Show abstract syntax tree --showoet : Show optimized expression tree --showquery : Show query string --showrp : Show Runtime plan --showtet : Show translated expression tree --timing : Produce timing information --hdfs-conf VAL : The folder containing the HDFS configuration files + -O N : Optimization Level. (default: Full Optimization) + -available-processors N : Number of available processors. (default: java's available processors) + -buffer-size N : Disk read buffer size in bytes. + -compileonly : Compile the query and stop. + -frame-size N : Frame size in bytes. (default: 65,536) + -hdfs-conf VAL : Directory path to Hadoop configuration files + -join-hash-size N : Join hash size in bytes. (default: 67,108,864) + -local-node-controllers N : Number of local node controllers. (default: 1) + -maximum-data-size N : Maximum possible data size in bytes. (default: 150,323,855,000) + -repeatexec N : Number of times to repeat execution. + -rest-ip-address VAL : IP Address of the REST Server. + -rest-port N : Port of REST Server. + -result-file VAL : File path to save the query result. + -showast : Show abstract syntax tree. + -showoet : Show optimized expression tree. + -showquery : Show query string. + -showrp : Show Runtime plan. + -showtet : Show translated expression tree. + -timing : Produce timing information. + -timing-ignore-queries N : Ignore the first X number of quereies. ---------------------------------------- * Java Options diff --git a/vxquery-cli/pom.xml b/vxquery-cli/pom.xml index 52c32ad9d..e7ad16a45 100644 --- a/vxquery-cli/pom.xml +++ b/vxquery-cli/pom.xml @@ -14,7 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. --> - + 4.0.0 @@ -61,12 +62,6 @@ - maven-antrun-plugin @@ -82,64 +77,11 @@ org.apache.vxquery - apache-vxquery-core + apache-vxquery-rest 0.7-SNAPSHOT - compile - - - - org.apache.hyracks - hyracks-api - - - - org.apache.hyracks - hyracks-client - - - - org.apache.hyracks - hyracks-control-cc - - - - org.apache.hyracks - hyracks-control-nc - - - - org.apache.hyracks - algebricks-common - - - - org.apache.hyracks - algebricks-core - - - - org.apache.hyracks - hyracks-control-common - - - - org.apache.hyracks - hyracks-dataflow-std - - - - org.apache.hyracks - hyracks-hdfs-core - - - - org.apache.hyracks - hyracks-hdfs-2.x - - diff --git a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java index 25ff9c47d..23d5eaaf9 100644 --- a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java +++ b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java @@ -14,73 +14,57 @@ */ package org.apache.vxquery.cli; -import java.io.ByteArrayOutputStream; +import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON; + import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintWriter; -import java.io.StringReader; -import java.net.InetAddress; -import java.nio.file.Files; +import java.io.PrintStream; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Date; -import java.util.EnumSet; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.logging.LogManager; + +import javax.xml.bind.JAXBException; import org.apache.commons.io.FileUtils; -import org.apache.hyracks.api.client.HyracksConnection; -import org.apache.hyracks.api.client.IHyracksClientConnection; -import org.apache.hyracks.api.client.NodeControllerInfo; -import org.apache.hyracks.api.comm.IFrame; -import org.apache.hyracks.api.comm.IFrameTupleAccessor; -import org.apache.hyracks.api.comm.VSizeFrame; -import org.apache.hyracks.api.dataset.IHyracksDataset; -import org.apache.hyracks.api.dataset.IHyracksDatasetReader; -import org.apache.hyracks.api.dataset.ResultSetId; -import org.apache.hyracks.api.job.JobFlag; -import org.apache.hyracks.api.job.JobId; -import org.apache.hyracks.api.job.JobSpecification; -import org.apache.hyracks.client.dataset.HyracksDataset; -import org.apache.hyracks.control.cc.ClusterControllerService; -import org.apache.hyracks.control.common.controllers.CCConfig; -import org.apache.hyracks.control.common.controllers.NCConfig; -import org.apache.hyracks.control.nc.NodeControllerService; -import org.apache.hyracks.control.nc.resources.memory.FrameManager; -import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor; -import org.apache.vxquery.compiler.CompilerControlBlock; -import org.apache.vxquery.compiler.algebricks.VXQueryGlobalDataFactory; -import org.apache.vxquery.context.DynamicContext; -import org.apache.vxquery.context.DynamicContextImpl; -import org.apache.vxquery.context.RootStaticContextImpl; -import org.apache.vxquery.context.StaticContextImpl; -import org.apache.vxquery.exceptions.SystemException; -import org.apache.vxquery.result.ResultUtils; -import org.apache.vxquery.xmlquery.query.Module; -import org.apache.vxquery.xmlquery.query.VXQueryCompilationListener; -import org.apache.vxquery.xmlquery.query.XMLQueryCompiler; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.HttpClientUtils; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.vxquery.app.util.LocalClusterUtil; +import org.apache.vxquery.app.util.RestUtils; +import org.apache.vxquery.rest.request.QueryRequest; +import org.apache.vxquery.rest.response.AsyncQueryResponse; +import org.apache.vxquery.rest.response.Error; +import org.apache.vxquery.rest.response.ErrorResponse; +import org.apache.vxquery.rest.response.Metrics; +import org.apache.vxquery.rest.response.SyncQueryResponse; +import org.apache.vxquery.rest.service.VXQueryConfig; import org.kohsuke.args4j.Argument; import org.kohsuke.args4j.CmdLineParser; import org.kohsuke.args4j.Option; +/** + * CLI for VXQuery. This class is using the REST API to execute statements given by the user. + * + * @author Erandi Ganepola + */ public class VXQuery { + private final CmdLineOptions opts; - private final CmdLineOptions indexOpts; - - private ClusterControllerService cc; - private NodeControllerService[] ncs; - private IHyracksClientConnection hcc; - private IHyracksDataset hds; - private List collectionList; - private ResultSetId resultSetId; - private static List timingMessages = new ArrayList<>(); - private static long sumTiming; - private static long sumSquaredTiming; - private static long minTiming = Long.MAX_VALUE; - private static long maxTiming = Long.MIN_VALUE; + + private static LocalClusterUtil localClusterUtil; + private String restIpAddress; + private int restPort; + + private static List metricsList = new ArrayList<>(); + private int executionIteration; /** * Constructor to use command line options passed. @@ -90,26 +74,17 @@ public class VXQuery { */ public VXQuery(CmdLineOptions opts) { this.opts = opts; - // The index query returns only the result, without any other information. - this.indexOpts = opts; - indexOpts.showAST = false; - indexOpts.showOET = false; - indexOpts.showQuery = false; - indexOpts.showRP = false; - indexOpts.showTET = false; - indexOpts.timing = false; - indexOpts.compileOnly = false; - this.collectionList = new ArrayList(); } /** * Main method to get command line options and execute query process. * * @param args - * @throws Exception + * command line arguments */ - public static void main(String[] args) throws Exception { - Date start = new Date(); + public static void main(String[] args) { + LogManager.getLogManager().reset(); + final CmdLineOptions opts = new CmdLineOptions(); CmdLineParser parser = new CmdLineParser(opts); @@ -120,243 +95,249 @@ public static void main(String[] args) throws Exception { parser.printUsage(System.err); return; } - if (opts.arguments.isEmpty()) { + + if (opts.xqFiles.isEmpty()) { parser.printUsage(System.err); return; } - VXQuery vxq = new VXQuery(opts); - vxq.execute(); - // if -timing argument passed, show the starting and ending times - if (opts.timing) { - Date end = new Date(); - timingMessage("Execution time: " + (end.getTime() - start.getTime()) + " ms"); - if (opts.repeatExec > opts.timingIgnoreQueries) { - Double mean = (double) (sumTiming) / (opts.repeatExec - opts.timingIgnoreQueries); - double sd = Math.sqrt(sumSquaredTiming / (opts.repeatExec - opts.timingIgnoreQueries) - mean * mean); - timingMessage("Average execution time: " + mean + " ms"); - timingMessage("Standard deviation: " + String.format("%.4f", sd)); - timingMessage("Coefficient of variation: " + String.format("%.4f", sd / mean)); - timingMessage("Minimum execution time: " + minTiming + " ms"); - timingMessage("Maximum execution time: " + maxTiming + " ms"); - } - System.out.println("Timing Summary:"); - for (String time : timingMessages) { - System.out.println(" " + time); - } - } + VXQuery vxq = new VXQuery(opts); + vxq.execute(opts.xqFiles); } - /** - * Creates a new Hyracks connection with: the client IP address and port provided, if IP address is provided in command line. Otherwise create a new virtual - * cluster with Hyracks nodes. Queries passed are run either way. After running queries, if a virtual cluster has been created, it is shut down. - * - * @throws Exception - */ - private void execute() throws Exception { - System.setProperty("vxquery.buffer_size", Integer.toString(opts.bufferSize)); + private void execute(List xqFiles) { + if (opts.restIpAddress == null) { + System.out.println("No REST Ip address given. Creating a local hyracks cluster"); - if (opts.clientNetIpAddress != null) { - hcc = new HyracksConnection(opts.clientNetIpAddress, opts.clientNetPort); - runQueries(); - } else { - if (!opts.compileOnly) { - startLocalHyracks(); + VXQueryConfig vxqConfig = new VXQueryConfig(); + vxqConfig.setAvailableProcessors(opts.availableProcessors); + vxqConfig.setFrameSize(opts.frameSize); + vxqConfig.setHdfsConf(opts.hdfsConf); + vxqConfig.setJoinHashSize(opts.joinHashSize); + vxqConfig.setMaximumDataSize(opts.maximumDataSize); + + localClusterUtil = new LocalClusterUtil(); + try { + localClusterUtil.init(vxqConfig); + restIpAddress = localClusterUtil.getIpAddress(); + restPort = localClusterUtil.getRestPort(); + } catch (Exception e) { + System.err.println("Unable to start local hyracks cluster due to: " + e.getMessage()); + e.printStackTrace(); + return; } + } else { + restIpAddress = opts.restIpAddress; + restPort = opts.restPort; + } + + System.out.println("Running queries given in: " + Arrays.toString(xqFiles.toArray())); + runQueries(xqFiles); + + if (localClusterUtil != null) { try { - runQueries(); - } finally { - if (!opts.compileOnly) { - stopLocalHyracks(); - } + localClusterUtil.deinit(); + } catch (Exception e) { + System.err.println("Error occurred when stopping local hyracks: " + e.getMessage()); } } } - /** - * Reads the contents of the files passed in the list of arguments to a string. If -showquery argument is passed, output the query as string. Run the query - * for the string. - * - * @throws IOException - * @throws SystemException - * @throws Exception - */ + public void runQueries(List xqFiles) { + for (String xqFile : xqFiles) { + String query; + try { + query = slurp(xqFile); + } catch (IOException e) { + System.err.println(String.format("Error occurred when reading XQuery file %s with message: %s", xqFile, + e.getMessage())); + continue; + } - private void runQueries() throws Exception { - List queries = opts.arguments; - // Run the showIndexes query before executing any target query, to store the index metadata - List queriesIndex = new ArrayList(); - queriesIndex.add("vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndexes.xq"); - OutputStream resultStream = new ByteArrayOutputStream(); - executeQuery(queriesIndex.get(0), 1, resultStream, indexOpts); - ByteArrayOutputStream bos = (ByteArrayOutputStream) resultStream; - String result = new String(bos.toByteArray()); - String[] collections = result.split("\n"); - this.collectionList = Arrays.asList(collections); - executeQueries(queries); - } + System.out.println(); + System.out.println("===================================================="); + System.out.println("\tQuery - '" + xqFile + "'"); + System.out.println("===================================================="); - public void executeQueries(List queries) throws Exception { - for (String query : queries) { - OutputStream resultStream = System.out; - if (opts.resultFile != null) { - resultStream = new FileOutputStream(new File(opts.resultFile)); + QueryRequest request = createQueryRequest(opts, query); + metricsList.clear(); + + for (int i = 0; i < opts.repeatExec; i++) { + System.out.println("**** Repetition : " + (i + 1) + " ****"); + + executionIteration = i; + sendQueryRequest(xqFile, request, this); + } + + if (opts.repeatExec > 1) { + showTimingSummary(); } - executeQuery(query, opts.repeatExec, resultStream, opts); } } - public void executeQuery(String query, int repeatedExecution, OutputStream resultStream, CmdLineOptions options) - throws Exception { - PrintWriter writer = new PrintWriter(resultStream, true); - String qStr = slurp(query); - if (opts.showQuery) { - writer.println(qStr); + private void onSuccess(String xqFile, QueryRequest request, SyncQueryResponse response) { + if (response == null) { + System.err.println(String.format("Unable to execute query %s", request.getStatement())); + return; } - VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET, opts.showOET, - opts.showRP); - Date start = opts.timing ? new Date() : null; + if (opts.showQuery) { + printField("Query", response.getStatement()); + } - Map nodeControllerInfos = null; - if (hcc != null) { - nodeControllerInfos = hcc.getNodeControllerInfos(); + if (request.isShowMetrics()) { + String metrics = String.format("Compile Time:\t%d\nElapsed Time:\t%d", + response.getMetrics().getCompileTime(), response.getMetrics().getElapsedTime()); + printField("Metrics", metrics); } - XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize, - opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf); - resultSetId = createResultSetId(); - CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE), - resultSetId, null); - compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel, this.collectionList); - // if -timing argument passed, show the starting and ending times - Date end = opts.timing ? new Date() : null; - if (opts.timing) { - timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms"); + + if (request.isShowAbstractSyntaxTree()) { + printField("Abstract Syntax Tree", response.getAbstractSyntaxTree()); } - if (opts.compileOnly) { - return; + + if (request.isShowTranslatedExpressionTree()) { + printField("Translated Expression Tree", response.getTranslatedExpressionTree()); } - Module module = compiler.getModule(); - JobSpecification js = module.getHyracksJobSpecification(); - - DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext()); - js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory())); - - // Repeat execution for number of times provided in -repeatexec argument - for (int i = 0; i < repeatedExecution; ++i) { - start = opts.timing ? new Date() : null; - runJob(js, writer); - // if -timing argument passed, show the starting and ending times - if (opts.timing) { - end = new Date(); - long currentRun = end.getTime() - start.getTime(); - if ((i + 1) > opts.timingIgnoreQueries) { - sumTiming += currentRun; - sumSquaredTiming += currentRun * currentRun; - if (currentRun < minTiming) { - minTiming = currentRun; - } - if (maxTiming < currentRun) { - maxTiming = currentRun; - } - } - timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms"); - } + if (request.isShowOptimizedExpressionTree()) { + printField("Optimized Expression Tree", response.getOptimizedExpressionTree()); } - } - /** - * Creates a Hyracks dataset, if not already existing with the job frame size, and 1 reader. Allocates a new buffer of size specified in the frame of Hyracks - * node. Creates new dataset reader with the current job ID and result set ID. Outputs the string in buffer for each frame. - * - * @param spec - * JobSpecification object, containing frame size. Current specified job. - * @param writer - * Writer for output of job. - * @throws Exception - */ - private void runJob(JobSpecification spec, PrintWriter writer) throws Exception { - int nReaders = 1; - if (hds == null) { - hds = new HyracksDataset(hcc, spec.getFrameSize(), nReaders); + if (request.isShowRuntimePlan()) { + printField("Runtime Plan", response.getRuntimePlan()); } - JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME)); + printField("Results", response.getResults()); - FrameManager resultDisplayFrameMgr = new FrameManager(spec.getFrameSize()); - IFrame frame = new VSizeFrame(resultDisplayFrameMgr); - IHyracksDatasetReader reader = hds.createReader(jobId, resultSetId); - IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor(); + if (executionIteration >= opts.timingIgnoreQueries) { + metricsList.add(response.getMetrics()); + } + } - while (reader.read(frame) > 0) { - writer.print(ResultUtils.getStringFromBuffer(frame.getBuffer(), frameTupleAccessor)); - writer.flush(); - frame.getBuffer().clear(); + private void onFailure(String xqFile, ErrorResponse response) { + if (response == null) { + System.err.println(String.format("Unable to execute query in %s", xqFile)); + return; } - hcc.waitForCompletion(jobId); - } + System.err.println(); + System.err.println("------------------------ Errors ---------------------"); - /** - * Create a unique result set id to get the correct query back from the cluster. - * - * @return Result Set id generated with current system time. - */ - protected ResultSetId createResultSetId() { - return new ResultSetId(System.nanoTime()); + Error error = response.getError(); + String errorMsg = String.format("Code:\t %d\nMessage:\t %s", error.getCode(), error.getMessage()); + printField(System.err, String.format("Errors for '%s'", xqFile), errorMsg); } /** - * Start local virtual cluster with cluster controller node and node controller nodes. IP address provided for node controller is localhost. Unassigned ports - * 39000 and 39001 are used for client and cluster port respectively. Creates a new Hyracks connection with the IP address and client ports. + * Submits a query to be executed by the REST API. Will call {@link #onFailure(String, ErrorResponse)} if any error + * occurs when submitting the query. Else will call {@link #onSuccess(String, QueryRequest, SyncQueryResponse)} with + * the {@link AsyncQueryResponse} * - * @throws Exception + * @param xqFile + * .xq file with the query to be executed + * @param request + * {@link QueryRequest} instance to be submitted to REST API + * @param cli + * cli class instance */ - public void startLocalHyracks() throws Exception { - String localAddress = InetAddress.getLocalHost().getHostAddress(); - CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = localAddress; - ccConfig.clientNetPort = 39000; - ccConfig.clusterNetIpAddress = localAddress; - ccConfig.clusterNetPort = 39001; - ccConfig.httpPort = 39002; - ccConfig.profileDumpPeriod = 10000; - cc = new ClusterControllerService(ccConfig); - cc.start(); - - ncs = new NodeControllerService[opts.localNodeControllers]; - for (int i = 0; i < ncs.length; i++) { - NCConfig ncConfig = new NCConfig(); - ncConfig.ccHost = "localhost"; - ncConfig.ccPort = 39001; - ncConfig.clusterNetIPAddress = localAddress; - ncConfig.dataIPAddress = localAddress; - ncConfig.resultIPAddress = localAddress; - ncConfig.nodeId = "nc" + (i + 1); - //TODO: enable index folder as a cli option for on-the-fly indexing queries - ncConfig.ioDevices = Files.createTempDirectory(ncConfig.nodeId).toString(); - ncs[i] = new NodeControllerService(ncConfig); - ncs[i].start(); + private static void sendQueryRequest(String xqFile, QueryRequest request, VXQuery cli) { + URI uri = null; + try { + uri = RestUtils.buildQueryURI(request, cli.restIpAddress, cli.restPort); + } catch (URISyntaxException e) { + System.err.println( + String.format("Unable to build URI to call REST API for query: %s", request.getStatement())); + cli.onFailure(xqFile, null); } - hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); + CloseableHttpClient httpClient = HttpClients.custom().build(); + try { + HttpGet httpGet = new HttpGet(uri); + httpGet.setHeader(HttpHeaders.ACCEPT, CONTENT_TYPE_JSON); + + try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) { + HttpEntity entity = httpResponse.getEntity(); + + String response = RestUtils.readEntity(entity); + if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + cli.onSuccess(xqFile, request, + RestUtils.mapEntity(response, SyncQueryResponse.class, CONTENT_TYPE_JSON)); + } else { + cli.onFailure(xqFile, RestUtils.mapEntity(response, ErrorResponse.class, CONTENT_TYPE_JSON)); + } + } catch (IOException e) { + System.err.println("Error occurred when reading entity: " + e.getMessage()); + cli.onFailure(xqFile, null); + } catch (JAXBException e) { + System.err.println("Error occurred when mapping query response: " + e.getMessage()); + cli.onFailure(xqFile, null); + } + } finally { + HttpClientUtils.closeQuietly(httpClient); + } } /** - * Shuts down the virtual cluster, along with all nodes and node execution, network and queue managers. - * - * @throws Exception + * Once the query in a given .xq file has been executed (with repeated executions as well), this method calculates + * mean, standard deviation, minimum and maximum execution times. */ - public void stopLocalHyracks() throws Exception { - for (int i = 0; i < ncs.length; i++) { - ncs[i].stop(); + private void showTimingSummary() { + double sumTime = 0; + double sumSquaredTime = 0; + long minTime = Long.MAX_VALUE; + long maxTime = Long.MIN_VALUE; + + for (int i = 0; i < metricsList.size(); i++) { + Metrics metrics = metricsList.get(i); + long totalTime = metrics.getCompileTime() + metrics.getElapsedTime(); + + sumTime += totalTime; + sumSquaredTime += totalTime * totalTime; + + if (totalTime < minTime) { + minTime = totalTime; + } + + if (totalTime > maxTime) { + maxTime = totalTime; + } } - cc.stop(); + + double mean = sumTime / (opts.repeatExec - opts.timingIgnoreQueries); + double sd = Math.sqrt(sumSquaredTime / (opts.repeatExec - opts.timingIgnoreQueries) - mean * mean); + + System.out.println(); + System.out.println("\t**** Timing Summary ****"); + System.out.println("----------------------------------------------------"); + System.out.println(String.format("Repetitions:\t%d, Timing Ignored Iterations:\t%d", opts.repeatExec, + opts.timingIgnoreQueries)); + System.out.println("Average execution time:\t" + mean + " ms"); + System.out.println("Standard deviation:\t" + String.format("%.4f", sd)); + System.out.println("Coefficient of variation:\t" + String.format("%.4f", sd / mean)); + System.out.println("Minimum execution time:\t" + minTime + " ms"); + System.out.println("Maximum execution time:\t" + maxTime + " ms"); + System.out.println(); + } + + private static QueryRequest createQueryRequest(CmdLineOptions opts, String query) { + QueryRequest request = new QueryRequest(query); + request.setCompileOnly(opts.compileOnly); + request.setOptimization(opts.optimizationLevel); + request.setFrameSize(opts.frameSize); + request.setRepeatExecutions(opts.repeatExec); + request.setShowMetrics(opts.timing); + request.setShowAbstractSyntaxTree(opts.showAST); + request.setShowTranslatedExpressionTree(opts.showTET); + request.setShowOptimizedExpressionTree(opts.showOET); + request.setShowRuntimePlan(opts.showRP); + request.setAsync(false); + + return request; } /** - * Reads the contents of file given in query into a String. The file is always closed. For XML files UTF-8 encoding is used. + * Reads the contents of file given in query into a String. The file is always closed. For XML files UTF-8 encoding + * is used. * * @param query * The query with filename to be processed @@ -367,46 +348,49 @@ private static String slurp(String query) throws IOException { return FileUtils.readFileToString(new File(query), "UTF-8"); } - /** - * Save and print out the timing message. - * - * @param message - */ - private static void timingMessage(String message) { - System.out.println(message); - timingMessages.add(message); + private static void printField(PrintStream out, String field, String value) { + out.println(); + field = field + ":"; + out.print(field); + + String[] lines = value.split("\n"); + for (int i = 0; i < lines.length; i++) { + int margin = 4; + if (i != 0) { + margin += field.length(); + } + System.out.print(String.format("%1$" + margin + "s%2$s\n", "", lines[i])); + } + } + + private static void printField(String field, String value) { + printField(System.out, field, value); } /** * Helper class with fields and methods to handle all command line options */ private static class CmdLineOptions { - @Option(name = "-available-processors", usage = "Number of available processors. (default: java's available processors)") - private int availableProcessors = -1; + @Option(name = "-rest-ip-address", usage = "IP Address of the REST Server") + private String restIpAddress = null; - @Option(name = "-client-net-ip-address", usage = "IP Address of the ClusterController.") - private String clientNetIpAddress = null; + @Option(name = "-rest-port", usage = "Port of REST Server") + private int restPort = 8085; - @Option(name = "-client-net-port", usage = "Port of the ClusterController. (default: 1098)") - private int clientNetPort = 1098; + @Option(name = "-compileonly", usage = "Compile the query and stop.") + private boolean compileOnly; - @Option(name = "-local-node-controllers", usage = "Number of local node controllers. (default: 1)") - private int localNodeControllers = 1; + @Option(name = "-O", usage = "Optimization Level. (default: Full Optimization)") + private int optimizationLevel = Integer.MAX_VALUE; @Option(name = "-frame-size", usage = "Frame size in bytes. (default: 65,536)") private int frameSize = 65536; - @Option(name = "-join-hash-size", usage = "Join hash size in bytes. (default: 67,108,864)") - private long joinHashSize = -1; - - @Option(name = "-maximum-data-size", usage = "Maximum possible data size in bytes. (default: 150,323,855,000)") - private long maximumDataSize = -1; - - @Option(name = "-buffer-size", usage = "Disk read buffer size in bytes.") - private int bufferSize = -1; + @Option(name = "-repeatexec", usage = "Number of times to repeat execution.") + private int repeatExec = 1; - @Option(name = "-O", usage = "Optimization Level. (default: Full Optimization)") - private int optimizationLevel = Integer.MAX_VALUE; + @Option(name = "-timing", usage = "Produce timing information.") + private boolean timing; @Option(name = "-showquery", usage = "Show query string.") private boolean showQuery; @@ -423,29 +407,33 @@ private static class CmdLineOptions { @Option(name = "-showrp", usage = "Show Runtime plan.") private boolean showRP; - @Option(name = "-compileonly", usage = "Compile the query and stop.") - private boolean compileOnly; + // Optional (Not supported by REST API) parameters. Only used for creating a + // local hyracks cluster + @Option(name = "-join-hash-size", usage = "Join hash size in bytes. (default: 67,108,864)") + private long joinHashSize = -1; - @Option(name = "-repeatexec", usage = "Number of times to repeat execution.") - private int repeatExec = 1; + @Option(name = "-maximum-data-size", usage = "Maximum possible data size in bytes. (default: 150,323,855,000)") + private long maximumDataSize = -1; + + @Option(name = "-buffer-size", usage = "Disk read buffer size in bytes.") + private int bufferSize = -1; @Option(name = "-result-file", usage = "File path to save the query result.") private String resultFile = null; - @Option(name = "-timing", usage = "Produce timing information.") - private boolean timing; - @Option(name = "-timing-ignore-queries", usage = "Ignore the first X number of quereies.") - private int timingIgnoreQueries = 2; - - @Option(name = "-x", usage = "Bind an external variable") - private Map bindings = new HashMap<>(); + private int timingIgnoreQueries = 0; @Option(name = "-hdfs-conf", usage = "Directory path to Hadoop configuration files") private String hdfsConf = null; + @Option(name = "-available-processors", usage = "Number of available processors. (default: java's available processors)") + private int availableProcessors = -1; + + @Option(name = "-local-node-controllers", usage = "Number of local node controllers. (default: 1)") + private int localNodeControllers = 1; + @Argument - private List arguments = new ArrayList<>(); + private List xqFiles = new ArrayList<>(); } - -} +} \ No newline at end of file diff --git a/vxquery-cli/src/site/markdown/index.md b/vxquery-cli/src/site/markdown/index.md new file mode 100644 index 000000000..758c947f1 --- /dev/null +++ b/vxquery-cli/src/site/markdown/index.md @@ -0,0 +1,116 @@ + +# VXQuery CLI + +VXQuery CLI is the command line utility which can be used to execute XQueries +with ease. No pre-configuration needs to be done in order to execute an XQuery. + +--- + +## Quick Start + +*** + +### Requirements + +- Apache VXQuery™ source archive (apache-vxquery-X.Y-source-release.zip) +- JDK >= 1.8 +- Apache Maven >= 3.2 + +*** + +### Installing + +VXQuery CLI comes bundled with the VXQuery source distribution +(apache-vxquery-X.Y-source-release.zip). + +First, run `mvn package`. + +``` +$ unzip apache-vxquery-X.Y-source-release.zip +$ cd apache-vxquery-X.Y +$ mvn package -DskipTests +``` + +**vxquery-cli** binaries are located at `vxquery-cli/target/appassembler/bin`. +There are 2 files in this directory, **vxq** which is the bash executable for unix +based systems and **vxq.bat** for windows systems. Depending on the platform, +suitable executable needs to be selected. + +*** + +### Executing a Query + +#### Put the query into a file + +VXQuery CLI takes a file location as the argument where this file includes the +query(statement) to be executed. Suppose the following query needs to be executed. + +``` +for $x in doc("books.xml")/bookstore/book +where $x/price>30 +order by $x/title +return $x/title +``` +This statement is querying for the book titles in **books.xml** where price of +the book is greater than 30. Also this query asks for the results to be ordered by +*title* as well. Now, create a file (say **test.xq**) and put the above query as +the content. + +**NOTE:** You can replace **books.xml** with any XML file that you have and want +to run a query on. + +#### Execute the query + +We need to invoke the matching executable according to your platform (unix/windows) +inside `vxquery-cli/target/appassembler/bin` directory. To execute the query, run: + +``` +sh ./apache-vxquery-X.Y/vxquery-cli/target/appassembler/bin/vxq path/to/test.xq +``` + +*** + +## Command Line Options + +``` + -O N : Optimization Level. (default: Full Optimization) + -available-processors N : Number of available processors. (default: java's available processors) + -buffer-size N : Disk read buffer size in bytes. + -compileonly : Compile the query and stop. + -frame-size N : Frame size in bytes. (default: 65,536) + -hdfs-conf VAL : Directory path to Hadoop configuration files + -join-hash-size N : Join hash size in bytes. (default: 67,108,864) + -local-node-controllers N : Number of local node controllers. (default: 1) + -maximum-data-size N : Maximum possible data size in bytes. (default: 150,323,855,000) + -repeatexec N : Number of times to repeat execution. + -rest-ip-address VAL : IP Address of the REST Server. + -rest-port N : Port of REST Server. + -result-file VAL : File path to save the query result. + -showast : Show abstract syntax tree. + -showoet : Show optimized expression tree. + -showquery : Show query string. + -showrp : Show Runtime plan. + -showtet : Show translated expression tree. + -timing : Produce timing information. + -timing-ignore-queries N : Ignore the first X number of quereies. +``` + +**NOTE:** Normally, CLI starts a local VXQuery Server to execute the query. But, +if you already have a VXQuery Server running, you can send the query to the +inbuilt *REST Server* running in that server by specifying the **port** and **ip address** +of the REST Server through options `-rest-ip-address` and `-rest-port`. diff --git a/vxquery-cli/src/site/site.xml b/vxquery-cli/src/site/site.xml index 4d35a0fee..8b3198d5f 100644 --- a/vxquery-cli/src/site/site.xml +++ b/vxquery-cli/src/site/site.xml @@ -15,34 +15,39 @@ See the License for the specific language governing permissions and limitations under the License. --> - - VXQuery - ../images/VXQuery.png - ../index.html - + + VXQuery + ../images/VXQuery.png + ../index.html + - - Apache Software Foundation - ../images/asf_logo_wide.png - http://www.apache.org/ - + + Apache Software Foundation + ../images/asf_logo_wide.png + http://www.apache.org/ + - - org.apache.maven.skins - maven-fluido-skin - 1.5 - + + org.apache.maven.skins + maven-fluido-skin + 1.5 + - - -
Apache VXQuery, VXQuery, Apache, the Apache - feather logo, and the Apache VXQuery project logo are either - registered trademarks or trademarks of The Apache Software - Foundation in the United States and other countries. - All other marks mentioned may be trademarks or registered - trademarks of their respective owners. - ]]>
- + + + + + + +
Apache VXQuery, VXQuery, Apache, the Apache + feather logo, and the Apache VXQuery project logo are either + registered trademarks or trademarks of The Apache Software + Foundation in the United States and other countries. + All other marks mentioned may be trademarks or registered + trademarks of their respective owners. + ]]> +
+ diff --git a/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryRuntimeException.java b/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryRuntimeException.java new file mode 100644 index 000000000..7f748b72d --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryRuntimeException.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.exceptions; + +/** + * A runtime exception to be thrown by the VXQuery and related classes of the rest server + * + * @author Erandi Ganepola + */ +public class VXQueryRuntimeException extends RuntimeException { + + public VXQueryRuntimeException(String message) { + super(message); + } + + public VXQueryRuntimeException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryServletRuntimeException.java b/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryServletRuntimeException.java new file mode 100644 index 000000000..f05cd6f2c --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryServletRuntimeException.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.exceptions; + +/** + * A runtime exception class to be used to be thrown when runtime errors occur within servlets. + * + * @author Erandi Ganepola + */ +public class VXQueryServletRuntimeException extends VXQueryRuntimeException { + + public VXQueryServletRuntimeException(String message) { + super(message); + } + + public VXQueryServletRuntimeException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java index 5bb9d1adf..15270594c 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java @@ -285,5 +285,4 @@ public boolean hasIndex(ArrayList collections) { } return indexExists; } - } diff --git a/vxquery-rest/pom.xml b/vxquery-rest/pom.xml new file mode 100644 index 000000000..052132fd3 --- /dev/null +++ b/vxquery-rest/pom.xml @@ -0,0 +1,56 @@ + + + + + + apache-vxquery + org.apache.vxquery + 0.7-SNAPSHOT + + + 4.0.0 + jar + VXQuery REST Server + Apache VXQuery REST Server + + apache-vxquery-rest + + + + org.apache.vxquery + apache-vxquery-core + 0.7-SNAPSHOT + + + org.apache.hyracks + hyracks-client + + + org.apache.hyracks + hyracks-http + + + junit + junit + test + + + + \ No newline at end of file diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/app/VXQueryApplication.java b/vxquery-rest/src/main/java/org/apache/vxquery/app/VXQueryApplication.java new file mode 100644 index 000000000..f5e01659f --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/app/VXQueryApplication.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.app; + +import static org.apache.vxquery.rest.Constants.Properties.AVAILABLE_PROCESSORS; +import static org.apache.vxquery.rest.Constants.Properties.HDFS_CONFIG; +import static org.apache.vxquery.rest.Constants.Properties.JOIN_HASH_SIZE; +import static org.apache.vxquery.rest.Constants.Properties.MAXIMUM_DATA_SIZE; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.hyracks.api.application.ICCApplicationContext; +import org.apache.hyracks.api.application.ICCApplicationEntryPoint; +import org.apache.hyracks.api.client.ClusterControllerInfo; +import org.apache.vxquery.exceptions.VXQueryRuntimeException; +import org.apache.vxquery.rest.RestServer; +import org.apache.vxquery.rest.service.VXQueryConfig; +import org.apache.vxquery.rest.service.VXQueryService; +import org.kohsuke.args4j.CmdLineParser; +import org.kohsuke.args4j.Option; + +/** + * Main class responsible for starting the {@link RestServer} and + * {@link VXQueryService} classes. + * + * @author Erandi Ganepola + */ +public class VXQueryApplication implements ICCApplicationEntryPoint { + + private static final Logger LOGGER = Logger.getLogger(VXQueryApplication.class.getName()); + + private VXQueryService vxQueryService; + private RestServer restServer; + + @Override + public void start(ICCApplicationContext ccAppCtx, String[] args) throws Exception { + AppArgs appArgs = new AppArgs(); + if (args != null) { + CmdLineParser parser = new CmdLineParser(appArgs); + try { + parser.parseArgument(args); + } catch (Exception e) { + parser.printUsage(System.err); + throw new VXQueryRuntimeException("Unable to parse app arguments", e); + } + } + + VXQueryConfig config = + loadConfiguration(ccAppCtx.getCCContext().getClusterControllerInfo(), appArgs.getVxqueryConfig()); + vxQueryService = new VXQueryService(config); + restServer = new RestServer(vxQueryService, appArgs.getRestPort()); + } + + public synchronized void stop() { + try { + LOGGER.log(Level.INFO, "Stopping REST server"); + restServer.stop(); + + LOGGER.log(Level.INFO, "Stopping VXQueryService"); + vxQueryService.stop(); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Error occurred when stopping the application", e); + } + } + + @Override + public void startupCompleted() throws Exception { + try { + LOGGER.log(Level.INFO, "Starting VXQueryService"); + vxQueryService.start(); + LOGGER.log(Level.INFO, "VXQueryService started successfully"); + + LOGGER.log(Level.INFO, "Starting REST server"); + restServer.start(); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Error occurred when starting application", e); + stop(); + throw new VXQueryRuntimeException("Error occurred when starting application", e); + } + } + + /** + * Loads properties from + * + * 
+     * -appConfig foo/bar.properties
+     *
+ * + * file if specified in the app arguments. + * + * @param clusterControllerInfo + * cluster controller information + * @param propertiesFile + * vxquery configuration properties file, given by + * + * 
+     *            -appConfig
+     *
+ * + * option in app argument + * @return A new {@link VXQueryConfig} instance with either default properties + * or properties loaded from the properties file given. + */ + private VXQueryConfig loadConfiguration(ClusterControllerInfo clusterControllerInfo, String propertiesFile) { + VXQueryConfig vxqConfig = new VXQueryConfig(); + if (propertiesFile != null) { + try (InputStream in = new FileInputStream(propertiesFile)) { + System.getProperties().load(in); + } catch (IOException e) { + LOGGER.log(Level.SEVERE, + String.format("Error occurred when loading properties file %s", propertiesFile), e); + } + } + + vxqConfig.setAvailableProcessors(Integer.getInteger(AVAILABLE_PROCESSORS, 1)); + vxqConfig.setJoinHashSize(Long.getLong(JOIN_HASH_SIZE, -1)); + vxqConfig.setHdfsConf(System.getProperty(HDFS_CONFIG)); + vxqConfig.setMaximumDataSize(Long.getLong(MAXIMUM_DATA_SIZE, -1)); + + vxqConfig.setHyracksClientIp(clusterControllerInfo.getClientNetAddress()); + vxqConfig.setHyracksClientPort(clusterControllerInfo.getClientNetPort()); + + return vxqConfig; + } + + public VXQueryService getVxQueryService() { + return vxQueryService; + } + + public RestServer getRestServer() { + return restServer; + } + + /** + * Application Arguments bean class + */ + private class AppArgs { + @Option(name = "-restPort", usage = "The port on which REST server starts") + private int restPort = 8080; + + @Option(name = "-appConfig", usage = "Properties file location which includes VXQueryService Application additional configuration") + private String vxqueryConfig = null; + + public String getVxqueryConfig() { + return vxqueryConfig; + } + + public void setVxqueryConfig(String vxqueryConfig) { + this.vxqueryConfig = vxqueryConfig; + } + + public int getRestPort() { + return restPort; + } + + public void setRestPort(int restPort) { + this.restPort = restPort; + } + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/app/util/LocalClusterUtil.java b/vxquery-rest/src/main/java/org/apache/vxquery/app/util/LocalClusterUtil.java new file mode 100644 index 000000000..9b18745b3 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/app/util/LocalClusterUtil.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.vxquery.app.util; + +import static org.apache.vxquery.rest.Constants.Properties.AVAILABLE_PROCESSORS; +import static org.apache.vxquery.rest.Constants.Properties.HDFS_CONFIG; +import static org.apache.vxquery.rest.Constants.Properties.JOIN_HASH_SIZE; +import static org.apache.vxquery.rest.Constants.Properties.MAXIMUM_DATA_SIZE; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.util.Arrays; + +import org.apache.hyracks.api.client.HyracksConnection; +import org.apache.hyracks.api.client.IHyracksClientConnection; +import org.apache.hyracks.api.dataset.IHyracksDataset; +import org.apache.hyracks.client.dataset.HyracksDataset; +import org.apache.hyracks.control.cc.ClusterControllerService; +import org.apache.hyracks.control.common.controllers.CCConfig; +import org.apache.hyracks.control.common.controllers.NCConfig; +import org.apache.hyracks.control.nc.NodeControllerService; +import org.apache.vxquery.app.VXQueryApplication; +import org.apache.vxquery.rest.service.VXQueryConfig; +import org.apache.vxquery.rest.service.VXQueryService; + +/** + * A utility class to start the a local hyracks cluster. + * + * @author Preston Carman + */ +public class LocalClusterUtil { + /* + * Start local virtual cluster with cluster controller node and node controller + * nodes. IP address provided for node controller is localhost. Unassigned ports + * 39000 and 39001 are used for client and cluster port respectively. + */ + public static final int DEFAULT_HYRACKS_CC_CLIENT_PORT = 39000; + public static final int DEFAULT_HYRACKS_CC_CLUSTER_PORT = 39001; + public static final int DEFAULT_HYRACKS_CC_HTTP_PORT = 39002; + public static final int DEFAULT_VXQUERY_REST_PORT = 39003; + + // TODO review variable scope after XTest is updated to use the REST service. + public ClusterControllerService clusterControllerService; + public NodeControllerService nodeControllerSerivce; + public IHyracksClientConnection hcc; + public IHyracksDataset hds; + public VXQueryService vxQueryService; + + public void init(VXQueryConfig config) throws Exception { + // Following properties are needed by the app to setup + System.setProperty(AVAILABLE_PROCESSORS, String.valueOf(config.getAvailableProcessors())); + System.setProperty(JOIN_HASH_SIZE, String.valueOf(config.getJoinHashSize())); + System.setProperty(MAXIMUM_DATA_SIZE, String.valueOf(config.getMaximumDataSize())); + if (config.getHdfsConf() != null) { + System.setProperty(HDFS_CONFIG, config.getHdfsConf()); + } + + // Cluster controller + CCConfig ccConfig = createCCConfig(); + clusterControllerService = new ClusterControllerService(ccConfig); + clusterControllerService.start(); + + hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); + hds = new HyracksDataset(hcc, config.getFrameSize(), config.getAvailableProcessors()); + + // Node controller + NCConfig ncConfig = createNCConfig(); + nodeControllerSerivce = new NodeControllerService(ncConfig); + nodeControllerSerivce.start(); + + hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); + + // REST controller + config.setHyracksClientIp(ccConfig.clientNetIpAddress); + config.setHyracksClientPort(ccConfig.clientNetPort); + vxQueryService = new VXQueryService(config); + vxQueryService.start(); + } + + protected CCConfig createCCConfig() throws IOException { + String localAddress = getIpAddress(); + CCConfig ccConfig = new CCConfig(); + ccConfig.clientNetIpAddress = localAddress; + ccConfig.clientNetPort = DEFAULT_HYRACKS_CC_CLIENT_PORT; + ccConfig.clusterNetIpAddress = localAddress; + ccConfig.clusterNetPort = DEFAULT_HYRACKS_CC_CLUSTER_PORT; + ccConfig.httpPort = DEFAULT_HYRACKS_CC_HTTP_PORT; + ccConfig.profileDumpPeriod = 10000; + ccConfig.appCCMainClass = VXQueryApplication.class.getName(); + ccConfig.appArgs = Arrays.asList("-restPort", String.valueOf(DEFAULT_VXQUERY_REST_PORT)); + + return ccConfig; + } + + protected NCConfig createNCConfig() throws IOException { + String localAddress = getIpAddress(); + NCConfig ncConfig = new NCConfig(); + ncConfig.ccHost = "localhost"; + ncConfig.ccPort = DEFAULT_HYRACKS_CC_CLUSTER_PORT; + ncConfig.clusterNetIPAddress = localAddress; + ncConfig.dataIPAddress = localAddress; + ncConfig.resultIPAddress = localAddress; + ncConfig.nodeId = "test_node"; + ncConfig.ioDevices = Files.createTempDirectory(ncConfig.nodeId).toString(); + return ncConfig; + } + + public IHyracksClientConnection getHyracksClientConnection() { + return hcc; + } + + public VXQueryService getVxQueryService() { + return vxQueryService; + } + + public void deinit() throws Exception { + vxQueryService.stop(); + nodeControllerSerivce.stop(); + clusterControllerService.stop(); + } + + public static void main(String[] args) { + LocalClusterUtil localClusterUtil = new LocalClusterUtil(); + VXQueryConfig config = new VXQueryConfig(); + run(localClusterUtil, config); + } + + protected static void run(final LocalClusterUtil localClusterUtil, VXQueryConfig config) { + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + try { + localClusterUtil.deinit(); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + + try { + localClusterUtil.init(config); + while (true) { + Thread.sleep(10000); + } + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } + } + + public String getIpAddress() throws UnknownHostException { + return InetAddress.getLocalHost().getHostAddress(); + } + + public int getRestPort() { + return DEFAULT_VXQUERY_REST_PORT; + } + + @Deprecated + public IHyracksClientConnection getConnection() { + return hcc; + } + + @Deprecated + public IHyracksDataset getDataset() { + return hds; + } + +} \ No newline at end of file diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/app/util/RestUtils.java b/vxquery-rest/src/main/java/org/apache/vxquery/app/util/RestUtils.java new file mode 100644 index 000000000..fe918363a --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/app/util/RestUtils.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.app.util; + +import static org.apache.vxquery.rest.Constants.MODE_ASYNC; +import static org.apache.vxquery.rest.Constants.MODE_SYNC; +import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON; +import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML; +import static org.apache.vxquery.rest.Constants.Parameters.COMPILE_ONLY; +import static org.apache.vxquery.rest.Constants.Parameters.FRAME_SIZE; +import static org.apache.vxquery.rest.Constants.Parameters.METRICS; +import static org.apache.vxquery.rest.Constants.Parameters.MODE; +import static org.apache.vxquery.rest.Constants.Parameters.OPTIMIZATION; +import static org.apache.vxquery.rest.Constants.Parameters.REPEAT_EXECUTIONS; +import static org.apache.vxquery.rest.Constants.Parameters.SHOW_AST; +import static org.apache.vxquery.rest.Constants.Parameters.SHOW_OET; +import static org.apache.vxquery.rest.Constants.Parameters.SHOW_RP; +import static org.apache.vxquery.rest.Constants.Parameters.SHOW_TET; +import static org.apache.vxquery.rest.Constants.Parameters.STATEMENT; +import static org.apache.vxquery.rest.Constants.URLs.QUERY_ENDPOINT; +import static org.apache.vxquery.rest.Constants.URLs.QUERY_RESULT_ENDPOINT; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.net.URI; +import java.net.URISyntaxException; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Unmarshaller; + +import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpEntity; +import org.apache.http.client.utils.URIBuilder; +import org.apache.vxquery.rest.request.QueryRequest; +import org.apache.vxquery.rest.request.QueryResultRequest; + +/** + * A set of utility methods used by the REST related tasks + * + * @author Erandi Ganepola + */ +public class RestUtils { + + private RestUtils() { + } + + /** + * Builds the {@link URI} once the {@link QueryRequest} is given. Only the + * parameters given (different from the default values) are put in the + * {@link URI} + * + * @param request + * {@link QueryRequest} to be converted to a {@link URI} + * @param restIpAddress + * Ip address of the REST server + * @param restPort + * port of the REST server + * @return generated {@link URI} + * @throws URISyntaxException + */ + public static URI buildQueryURI(QueryRequest request, String restIpAddress, int restPort) + throws URISyntaxException { + URIBuilder builder = + new URIBuilder().setScheme("http").setHost(restIpAddress).setPort(restPort).setPath(QUERY_ENDPOINT); + + if (request.getStatement() != null) { + builder.addParameter(STATEMENT, request.getStatement()); + } + if (request.isCompileOnly()) { + builder.addParameter(COMPILE_ONLY, String.valueOf(request.isCompileOnly())); + } + if (request.getOptimization() != QueryRequest.DEFAULT_OPTIMIZATION) { + builder.addParameter(OPTIMIZATION, String.valueOf(request.getOptimization())); + } + if (request.getFrameSize() != QueryRequest.DEFAULT_FRAMESIZE) { + builder.addParameter(FRAME_SIZE, String.valueOf(request.getFrameSize())); + } + if (request.getRepeatExecutions() != 1) { + builder.addParameter(REPEAT_EXECUTIONS, String.valueOf(request.getRepeatExecutions())); + } + if (request.isShowMetrics()) { + builder.addParameter(METRICS, String.valueOf(request.isShowMetrics())); + } + if (request.isShowAbstractSyntaxTree()) { + builder.addParameter(SHOW_AST, String.valueOf(request.isShowAbstractSyntaxTree())); + } + if (request.isShowTranslatedExpressionTree()) { + builder.addParameter(SHOW_TET, String.valueOf(request.isShowTranslatedExpressionTree())); + } + if (request.isShowOptimizedExpressionTree()) { + builder.addParameter(SHOW_OET, String.valueOf(request.isShowOptimizedExpressionTree())); + } + if (request.isShowRuntimePlan()) { + builder.addParameter(SHOW_RP, String.valueOf(request.isShowRuntimePlan())); + } + if (!request.isAsync()) { + builder.addParameter(MODE, request.isAsync() ? MODE_ASYNC : MODE_SYNC); + } + + return builder.build(); + } + + /** + * Builds the query result {@link URI} given the {@link QueryResultRequest} + * + * @param resultRequest + * result request + * @param restIpAddress + * rest server's ip + * @param restPort + * port of the rest server + * @return generated {@link URI} + * @throws URISyntaxException + */ + public static URI buildQueryResultURI(QueryResultRequest resultRequest, String restIpAddress, int restPort) + throws URISyntaxException { + URIBuilder builder = new URIBuilder().setScheme("http").setHost(restIpAddress).setPort(restPort) + .setPath(QUERY_RESULT_ENDPOINT.replace("*", String.valueOf(resultRequest.getResultId()))); + + if (resultRequest.isShowMetrics()) { + builder.setParameter(METRICS, String.valueOf(resultRequest.isShowMetrics())); + } + + return builder.build(); + } + + /** + * Reads the entity from an {@link HttpEntity} + * + * @param entity + * entity instance to be read + * @return entity read by this method as a string + * @throws IOException + */ + public static String readEntity(HttpEntity entity) throws IOException { + StringBuilder responseBody = new StringBuilder(); + + try (InputStream in = entity.getContent()) { + BufferedReader reader = new BufferedReader(new InputStreamReader(in)); + String line; + while ((line = reader.readLine()) != null) { + responseBody.append(line); + } + } + return responseBody.toString(); + } + + /** + * Maps the object in the string representation to a java object. To map json + * entities, this method use {@link ObjectMapper}. For XML this method use + * {@link Unmarshaller}. + * + * @param entity + * string representation of the object + * @param type + * the class to which the string needs to be mapped to + * @param contentType + * json or XML + * @param + * content's class type + * @return mapped object + * @throws IOException + * @throws JAXBException + */ + public static T mapEntity(String entity, Class type, String contentType) throws IOException, JAXBException { + if (contentType == null) { + contentType = CONTENT_TYPE_JSON; + } + + switch (contentType) { + case CONTENT_TYPE_XML: + JAXBContext jaxbContext = JAXBContext.newInstance(type); + Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); + return type.cast(unmarshaller.unmarshal(new StringReader(entity))); + case CONTENT_TYPE_JSON: + default: + ObjectMapper jsonMapper = new ObjectMapper(); + return jsonMapper.readValue(entity, type); + } + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/Constants.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/Constants.java new file mode 100644 index 000000000..4ba79ec8c --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/Constants.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest; + +public class Constants { + + private Constants() { + } + + public class Parameters { + public static final String STATEMENT = "statement"; + public static final String RESULT_ID = "resultId "; + public static final String COMPILE_ONLY = "compileOnly"; + public static final String OPTIMIZATION = "optimization"; + public static final String FRAME_SIZE = "frameSize"; + public static final String REPEAT_EXECUTIONS = "repeatExecutions"; + public static final String METRICS = "metrics"; + public static final String SHOW_AST = "showAbstractSyntaxTree"; + public static final String SHOW_TET = "showTranslatedExpressionTree"; + public static final String SHOW_OET = "showOptimizedExpressionTree"; + public static final String SHOW_RP = "showRuntimePlan"; + public static final String MODE = "mode"; + } + + public class URLs { + public static final String BASE_PATH = "/vxquery"; + + public static final String QUERY_ENDPOINT = BASE_PATH + "/query"; + public static final String QUERY_RESULT_ENDPOINT = BASE_PATH + "/query/result/*"; + } + + public class Properties { + public static final String AVAILABLE_PROCESSORS = "org.apache.vxquery.available_processors"; + public static final String LOCAL_NODE_CONTROLLERS = "org.apache.vxquery.local_nc"; + public static final String JOIN_HASH_SIZE = "org.apache.vxquery.join_hash"; + public static final String MAXIMUM_DATA_SIZE = "org.apache.vxquery.data_size"; + public static final String HDFS_CONFIG = "org.apache.vxquery.hdfs_config"; + } + + public class HttpHeaderValues { + public static final String CONTENT_TYPE_JSON = "application/json"; + public static final String CONTENT_TYPE_XML = "application/xml"; + } + + public class ErrorCodes { + public static final int PROBLEM_WITH_QUERY = 400; + public static final int UNFORSEEN_PROBLEM = 500; + public static final int INVALID_INPUT = 405; + public static final int NOT_FOUND = 404; + } + + public static final String RESULT_URL_PREFIX = "/vxquery/query/result/"; + + public static final String MODE_ASYNC = "async"; + public static final String MODE_SYNC = "sync"; +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/RestServer.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/RestServer.java new file mode 100644 index 000000000..5b59c0c9a --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/RestServer.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest; + +import static org.apache.vxquery.rest.Constants.URLs.QUERY_ENDPOINT; +import static org.apache.vxquery.rest.Constants.URLs.QUERY_RESULT_ENDPOINT; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.hyracks.http.server.HttpServer; +import org.apache.hyracks.http.server.WebManager; +import org.apache.vxquery.exceptions.VXQueryRuntimeException; +import org.apache.vxquery.rest.service.VXQueryService; +import org.apache.vxquery.rest.servlet.QueryAPIServlet; +import org.apache.vxquery.rest.servlet.QueryResultAPIServlet; + +/** + * REST Server class responsible for starting a new server on a given port. + * + * @author Erandi Ganepola + */ +public class RestServer { + + public static final Logger LOGGER = Logger.getLogger(RestServer.class.getName()); + + private WebManager webManager; + private int port; + + public RestServer(VXQueryService vxQueryService, int port) { + if (port == 0) { + throw new IllegalArgumentException("REST Server port cannot be 0"); + } + + this.port = port; + + webManager = new WebManager(); + HttpServer restServer = new HttpServer(webManager.getBosses(), webManager.getWorkers(), this.port); + restServer.addServlet(new QueryAPIServlet(vxQueryService, restServer.ctx(), QUERY_ENDPOINT)); + restServer.addServlet(new QueryResultAPIServlet(vxQueryService, restServer.ctx(), QUERY_RESULT_ENDPOINT)); + webManager.add(restServer); + } + + public void start() { + try { + LOGGER.log(Level.FINE, "Starting rest server on port: " + port); + webManager.start(); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Error occurred when starting rest server", e); + throw new VXQueryRuntimeException("Unable to start REST server", e); + } + LOGGER.log(Level.INFO, "Rest server started on port: " + port); + } + + public void stop() { + try { + LOGGER.log(Level.FINE, "Stopping rest server"); + webManager.stop(); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Error occurred when stopping VXQueryService", e); + throw new VXQueryRuntimeException("Error occurred when stopping rest server", e); + } + LOGGER.log(Level.INFO, "Rest server stopped"); + } + + public int getPort() { + return port; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryRequest.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryRequest.java new file mode 100644 index 000000000..a88ae1ccd --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryRequest.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.rest.request; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.apache.vxquery.rest.RestServer; + +/** + * Request to represent a query request coming to the {@link RestServer} + * + * @author Erandi Ganepola + */ +public class QueryRequest { + + public static final int DEFAULT_FRAMESIZE = 65536; + public static final int DEFAULT_OPTIMIZATION = 0; + + private String statement; + private boolean async = true; + private boolean compileOnly; + private int optimization = DEFAULT_OPTIMIZATION; + /** Frame size in bytes. (default: 65,536) */ + private int frameSize = DEFAULT_FRAMESIZE; + private int repeatExecutions = 1; + private boolean showMetrics = false; + private boolean showAbstractSyntaxTree = false; + private boolean showTranslatedExpressionTree = false; + private boolean showOptimizedExpressionTree = false; + private boolean showRuntimePlan = false; + /** A unique UUID to uniquely identify a given request */ + private String requestId; + + /** An optional map of source files. Required for XTests */ + private Map sourceFileMap = new HashMap<>(); + + public QueryRequest(String statement) { + this(null, statement); + } + + public QueryRequest(String requestId, String statement) { + if (statement == null) { + throw new IllegalArgumentException("Statement cannot be null"); + } + + this.statement = statement; + this.requestId = requestId; + } + + public String getStatement() { + return statement; + } + + public boolean isCompileOnly() { + return compileOnly; + } + + public void setCompileOnly(boolean compileOnly) { + this.compileOnly = compileOnly; + } + + public int getOptimization() { + return optimization; + } + + public void setOptimization(int optimization) { + this.optimization = optimization; + } + + public int getFrameSize() { + return frameSize; + } + + public void setFrameSize(int frameSize) { + this.frameSize = frameSize; + } + + public int getRepeatExecutions() { + return repeatExecutions; + } + + public void setRepeatExecutions(int repeatExecutions) { + this.repeatExecutions = repeatExecutions; + } + + public boolean isShowAbstractSyntaxTree() { + return showAbstractSyntaxTree; + } + + public void setShowAbstractSyntaxTree(boolean showAbstractSyntaxTree) { + this.showAbstractSyntaxTree = showAbstractSyntaxTree; + } + + public boolean isShowTranslatedExpressionTree() { + return showTranslatedExpressionTree; + } + + public void setShowTranslatedExpressionTree(boolean showTranslatedExpressionTree) { + this.showTranslatedExpressionTree = showTranslatedExpressionTree; + } + + public boolean isShowOptimizedExpressionTree() { + return showOptimizedExpressionTree; + } + + public void setShowOptimizedExpressionTree(boolean showOptimizedExpressionTree) { + this.showOptimizedExpressionTree = showOptimizedExpressionTree; + } + + public boolean isShowRuntimePlan() { + return showRuntimePlan; + } + + public void setShowRuntimePlan(boolean showRuntimePlan) { + this.showRuntimePlan = showRuntimePlan; + } + + public boolean isShowMetrics() { + return showMetrics; + } + + public void setShowMetrics(boolean showMetrics) { + this.showMetrics = showMetrics; + } + + public String toString() { + return String.format("{ statement : %s }", statement); + } + + public String getRequestId() { + return requestId; + } + + public boolean isAsync() { + return async; + } + + public void setAsync(boolean async) { + this.async = async; + } + + public Map getSourceFileMap() { + return sourceFileMap; + } + + public void setSourceFileMap(Map sourceFileMap) { + this.sourceFileMap = sourceFileMap; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryResultRequest.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryResultRequest.java new file mode 100644 index 000000000..5e431814d --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryResultRequest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.request; + +import org.apache.vxquery.rest.RestServer; + +/** + * Request to represent a query request coming to the {@link RestServer} + * + * @author Erandi Ganepola + */ +public class QueryResultRequest { + + private long resultId; + private boolean showMetrics = false; + private String requestId; + + public QueryResultRequest(long resultId) { + this(resultId, null); + } + + public QueryResultRequest(long resultId, String requestId) { + this.resultId = resultId; + this.requestId = requestId; + } + + public long getResultId() { + return resultId; + } + + public boolean isShowMetrics() { + return showMetrics; + } + + public void setShowMetrics(boolean showMetrics) { + this.showMetrics = showMetrics; + } + + public String getRequestId() { + return requestId; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/APIResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/APIResponse.java new file mode 100644 index 000000000..57b67b700 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/APIResponse.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.response; + +import static org.apache.vxquery.rest.Constants.RESULT_URL_PREFIX; + +import org.apache.hyracks.api.dataset.ResultSetId; +import org.apache.vxquery.rest.request.QueryRequest; +import org.apache.vxquery.rest.service.Status; + +/** + * Base class for any type of response which can be sent by the REST server. + * These responses can be query responses, error responses or query result + * responses. + * + * @author Erandi Ganepola + */ +public class APIResponse { + + private String status; + private String requestId; + + public APIResponse() { + status = Status.SUCCESS.toString(); + } + + public APIResponse(String status) { + this.status = status; + } + + public String getStatus() { + return status; + } + + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + + public static ErrorResponse newErrorResponse(String requestId, Error error) { + ErrorResponse response = new ErrorResponse(); + response.setRequestId(requestId); + response.setError(error); + return response; + } + + public static QueryResponse newQueryResponse(QueryRequest request, ResultSetId resultSetId) { + QueryResponse response; + if (request.isAsync()) { + AsyncQueryResponse asyncQueryResponse = new AsyncQueryResponse(); + if (!request.isCompileOnly()) { + asyncQueryResponse.setResultId(resultSetId.getId()); + asyncQueryResponse.setResultUrl(RESULT_URL_PREFIX + resultSetId.getId()); + } + response = asyncQueryResponse; + } else { + response = new SyncQueryResponse(); + } + + response.setRequestId(request.getRequestId()); + return response; + } + + public static QueryResultResponse newQueryResultResponse(String requestId) { + QueryResultResponse response = new QueryResultResponse(); + response.setRequestId(requestId); + return response; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/AsyncQueryResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/AsyncQueryResponse.java new file mode 100644 index 000000000..3216dce1a --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/AsyncQueryResponse.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.rest.response; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Resource class to represent a response to a given user query + * + * @author Erandi Ganepola + */ +@XmlRootElement +public class AsyncQueryResponse extends QueryResponse { + + private long resultId; + private String resultUrl; + + public long getResultId() { + return resultId; + } + + public void setResultId(long resultId) { + this.resultId = resultId; + } + + public String getResultUrl() { + return resultUrl; + } + + public void setResultUrl(String resultUrl) { + this.resultUrl = resultUrl; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Error.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Error.java new file mode 100644 index 000000000..96a709d50 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Error.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.response; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Represents the + * + * 
+ * error
+ *
+ * + * part of an {@link ErrorResponse}. + * + * 
+ *     {@code
+ *     
+ *         405
+ *         Invalid Input
+ *     }
+ *
+ * + * @author Erandi Ganepola + */ +@XmlRootElement +public class Error { + + private int code; + private String message; + + public Error() { + } + + public Error(int code, String message) { + this.code = code; + this.message = message; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private int code = -1; + private String message = null; + + public Builder withCode(int code) { + this.code = code; + return this; + } + + public Builder withMessage(String message) { + this.message = message; + return this; + } + + public Error build() { + if (code == -1) { + code = 500; + } + + if (message == null) { + message = "unexpected Error"; + } + + return new Error(code, message); + } + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/ErrorResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/ErrorResponse.java new file mode 100644 index 000000000..2de2b2511 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/ErrorResponse.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.response; + +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.vxquery.rest.service.Status; + +/** + * 
+ *     {@code
+ *     
+ *         FATAL
+ *         jabsa-jkk77j-hbah45-jknasj-jjlas
+ *         
+ *             405
+ *             Invalid Input
+ *         
+ *     
+ *     }
+ *
+ * + * @author Erandi Ganepola + */ +@XmlRootElement +public class ErrorResponse extends APIResponse { + + private Error error; + + public ErrorResponse() { + super(Status.FATAL.toString()); + } + + public Error getError() { + return error; + } + + public void setError(Error error) { + this.error = error; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Metrics.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Metrics.java new file mode 100644 index 000000000..e34e1c679 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Metrics.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.response; + +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +public class Metrics { + private long compileTime; + private long elapsedTime; + + public long getCompileTime() { + return compileTime; + } + + public void setCompileTime(long compileTime) { + this.compileTime = compileTime; + } + + public long getElapsedTime() { + return elapsedTime; + } + + public void setElapsedTime(long elapsedTime) { + this.elapsedTime = elapsedTime; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResponse.java new file mode 100644 index 000000000..4916ffebc --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResponse.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.response; + +import org.apache.vxquery.rest.service.Status; + +/** + * The base class of the query response (the response returned when a query is + * sent for execution) + * + * @author Erandi Ganepola + */ +public class QueryResponse extends APIResponse { + + private String statement; + private String abstractSyntaxTree; + private String translatedExpressionTree; + private String optimizedExpressionTree; + private String runtimePlan; + private Metrics metrics = new Metrics(); + + public QueryResponse() { + super(Status.SUCCESS.toString()); + } + + public String getStatement() { + return statement; + } + + public void setStatement(String statement) { + this.statement = statement; + } + + public String getAbstractSyntaxTree() { + return abstractSyntaxTree; + } + + public void setAbstractSyntaxTree(String abstractSyntaxTree) { + this.abstractSyntaxTree = abstractSyntaxTree; + } + + public String getTranslatedExpressionTree() { + return translatedExpressionTree; + } + + public void setTranslatedExpressionTree(String translatedExpressionTree) { + this.translatedExpressionTree = translatedExpressionTree; + } + + public String getOptimizedExpressionTree() { + return optimizedExpressionTree; + } + + public void setOptimizedExpressionTree(String optimizedExpressionTree) { + this.optimizedExpressionTree = optimizedExpressionTree; + } + + public String getRuntimePlan() { + return runtimePlan; + } + + public void setRuntimePlan(String runtimePlan) { + this.runtimePlan = runtimePlan; + } + + public Metrics getMetrics() { + return metrics; + } + + public void setMetrics(Metrics metrics) { + this.metrics = metrics; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResultResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResultResponse.java new file mode 100644 index 000000000..2f7486557 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResultResponse.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.response; + +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.vxquery.rest.service.Status; + +@XmlRootElement +public class QueryResultResponse extends APIResponse { + + private String results; + private Metrics metrics = new Metrics(); + + public QueryResultResponse() { + super(Status.SUCCESS.toString()); + } + + public String getResults() { + return results; + } + + public void setResults(String results) { + this.results = results; + } + + public Metrics getMetrics() { + return metrics; + } + + public void setMetrics(Metrics metrics) { + this.metrics = metrics; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/SyncQueryResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/SyncQueryResponse.java new file mode 100644 index 000000000..b42a912c0 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/SyncQueryResponse.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.response; + +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +public class SyncQueryResponse extends QueryResponse { + + private String results; + + public String getResults() { + return results; + } + + public void setResults(String results) { + this.results = results; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/HyracksJobContext.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/HyracksJobContext.java new file mode 100644 index 000000000..9dc967f36 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/HyracksJobContext.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.service; + +import org.apache.hyracks.api.dataset.ResultSetId; +import org.apache.hyracks.api.job.JobId; + +/** + * A class to map {@link ResultSetId} with {@link JobId} when a job is submitted + * to hyracks. This mapping will later be used to determine the {@link JobId} + * instance of the corresponding {@link ResultSetId} + * + * @author Erandi Ganepola + */ +public class HyracksJobContext { + + private JobId jobId; + private int frameSize; + private ResultSetId resultSetId; + + public HyracksJobContext(JobId jobId, int frameSize, ResultSetId resultSetId) { + this.jobId = jobId; + this.frameSize = frameSize; + this.resultSetId = resultSetId; + } + + public JobId getJobId() { + return jobId; + } + + public int getFrameSize() { + return frameSize; + } + + public ResultSetId getResultSetId() { + return resultSetId; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/State.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/State.java new file mode 100644 index 000000000..d6b4b3cd5 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/State.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.service; + +/** + * An enum to represent states of {@link VXQueryService} class + * + * @author Erandi Ganepola + */ +public enum State { + STARTING, + STARTED, + STOPPING, + STOPPED +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/Status.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/Status.java new file mode 100644 index 000000000..5757ae764 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/Status.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.service; + +/** + * There can only 2 states for a response from the + * {@link org.apache.vxquery.rest.RestServer}. They are, + * + * 
+ * SUCCESS
+ *
+ * + * and + * + * 
+ * FATAL
+ *
+ * + * . This enum represents those two types. + * + * @author Erandi Ganepola + */ +public enum Status { + SUCCESS("success"), + FATAL("fatal"); + + private final String name; + + Status(String name) { + this.name = name; + } + + public String toString() { + return name; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryConfig.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryConfig.java new file mode 100644 index 000000000..4f20c6bc4 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryConfig.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.service; + +/** + * A class to store default/user specified configurations required at runtime by + * the {@link VXQueryService} class. These configuration will be loaded through + * a properties file. + * + * @author Erandi Ganepola + */ +public class VXQueryConfig { + + /** Number of available processors. (default: java's available processors) */ + private int availableProcessors = Runtime.getRuntime().availableProcessors(); + /** Setting frame size. (default: 65,536) */ + private int frameSize = 65536; + /** Join hash size in bytes. (default: 67,108,864) */ + private long joinHashSize = -1; + /** Maximum possible data size in bytes. (default: 150,323,855,000) */ + private long maximumDataSize = -1; + /** Directory path to Hadoop configuration files */ + private String hdfsConf = null; + + private String hyracksClientIp; + private int hyracksClientPort; + + public int getAvailableProcessors() { + return availableProcessors; + } + + public void setAvailableProcessors(int availableProcessors) { + if (availableProcessors > 0) { + this.availableProcessors = availableProcessors; + } + } + + public long getJoinHashSize() { + return joinHashSize; + } + + public void setJoinHashSize(long joinHashSize) { + this.joinHashSize = joinHashSize; + } + + public long getMaximumDataSize() { + return maximumDataSize; + } + + public void setMaximumDataSize(long maximumDataSize) { + this.maximumDataSize = maximumDataSize; + } + + public String getHdfsConf() { + return hdfsConf; + } + + public void setHdfsConf(String hdfsConf) { + this.hdfsConf = hdfsConf; + } + + public int getHyracksClientPort() { + return hyracksClientPort; + } + + public void setHyracksClientPort(int hyracksClientPort) { + this.hyracksClientPort = hyracksClientPort; + } + + public String getHyracksClientIp() { + return hyracksClientIp; + } + + public void setHyracksClientIp(String hyracksClientIp) { + this.hyracksClientIp = hyracksClientIp; + } + + public int getFrameSize() { + return frameSize; + } + + public void setFrameSize(int frameSize) { + this.frameSize = frameSize; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryService.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryService.java new file mode 100644 index 000000000..1d51b6af7 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryService.java @@ -0,0 +1,482 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.rest.service; + +import static java.util.logging.Level.SEVERE; +import static org.apache.vxquery.rest.Constants.ErrorCodes.NOT_FOUND; +import static org.apache.vxquery.rest.Constants.ErrorCodes.PROBLEM_WITH_QUERY; +import static org.apache.vxquery.rest.Constants.ErrorCodes.UNFORSEEN_PROBLEM; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable; +import org.apache.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor; +import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter; +import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor; +import org.apache.hyracks.api.client.HyracksConnection; +import org.apache.hyracks.api.client.IHyracksClientConnection; +import org.apache.hyracks.api.client.NodeControllerInfo; +import org.apache.hyracks.api.comm.IFrame; +import org.apache.hyracks.api.comm.IFrameTupleAccessor; +import org.apache.hyracks.api.comm.VSizeFrame; +import org.apache.hyracks.api.dataset.DatasetJobRecord; +import org.apache.hyracks.api.dataset.IHyracksDatasetReader; +import org.apache.hyracks.api.dataset.ResultSetId; +import org.apache.hyracks.api.exceptions.HyracksException; +import org.apache.hyracks.api.job.JobFlag; +import org.apache.hyracks.api.job.JobId; +import org.apache.hyracks.api.job.JobSpecification; +import org.apache.hyracks.client.dataset.HyracksDataset; +import org.apache.hyracks.control.nc.resources.memory.FrameManager; +import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor; +import org.apache.vxquery.compiler.CompilerControlBlock; +import org.apache.vxquery.compiler.algebricks.VXQueryGlobalDataFactory; +import org.apache.vxquery.compiler.algebricks.prettyprint.VXQueryLogicalExpressionPrettyPrintVisitor; +import org.apache.vxquery.context.DynamicContext; +import org.apache.vxquery.context.DynamicContextImpl; +import org.apache.vxquery.context.RootStaticContextImpl; +import org.apache.vxquery.context.StaticContextImpl; +import org.apache.vxquery.exceptions.ErrorCode; +import org.apache.vxquery.exceptions.SystemException; +import org.apache.vxquery.exceptions.VXQueryRuntimeException; +import org.apache.vxquery.rest.request.QueryRequest; +import org.apache.vxquery.rest.request.QueryResultRequest; +import org.apache.vxquery.rest.response.APIResponse; +import org.apache.vxquery.rest.response.Error; +import org.apache.vxquery.rest.response.QueryResponse; +import org.apache.vxquery.rest.response.QueryResultResponse; +import org.apache.vxquery.rest.response.SyncQueryResponse; +import org.apache.vxquery.result.ResultUtils; +import org.apache.vxquery.xmlquery.ast.ModuleNode; +import org.apache.vxquery.xmlquery.query.Module; +import org.apache.vxquery.xmlquery.query.XMLQueryCompiler; +import org.apache.vxquery.xmlquery.query.XQueryCompilationListener; + +import com.thoughtworks.xstream.XStream; +import com.thoughtworks.xstream.io.xml.DomDriver; + +/** + * Main class responsible for handling query requests. This class will first + * compile, then submit query to hyracks and finally fetch results for a given + * query. + * + * @author Erandi Ganepola + */ +public class VXQueryService { + + private static final Logger LOGGER = Logger.getLogger(VXQueryService.class.getName()); + + private static final Pattern EMBEDDED_SYSERROR_PATTERN = Pattern.compile("(\\p{javaUpperCase}{4}\\d{4})"); + + private volatile State state = State.STOPPED; + private VXQueryConfig vxQueryConfig; + private AtomicLong atomicLong = new AtomicLong(0); + private Map jobContexts = new ConcurrentHashMap<>(); + private IHyracksClientConnection hyracksClientConnection; + private HyracksDataset hyracksDataset; + + public VXQueryService(VXQueryConfig config) { + vxQueryConfig = config; + } + + /** + * Starts VXQueryService class by creating a {@link IHyracksClientConnection} + * which will later be used to submit and retrieve queries and results to/from + * hyracks. + */ + public synchronized void start() { + if (!State.STOPPED.equals(state)) { + throw new IllegalStateException("VXQueryService is at state : " + state); + } + + if (vxQueryConfig.getHyracksClientIp() == null) { + throw new IllegalArgumentException("hyracksClientIp is required to connect to hyracks"); + } + + setState(State.STARTING); + + try { + hyracksClientConnection = + new HyracksConnection(vxQueryConfig.getHyracksClientIp(), vxQueryConfig.getHyracksClientPort()); + } catch (Exception e) { + LOGGER.log(SEVERE, String.format("Unable to create a hyracks client connection to %s:%d", + vxQueryConfig.getHyracksClientIp(), vxQueryConfig.getHyracksClientPort())); + throw new VXQueryRuntimeException("Unable to create a hyracks client connection", e); + } + + LOGGER.log(Level.FINE, String.format("Using hyracks connection to %s:%d", vxQueryConfig.getHyracksClientIp(), + vxQueryConfig.getHyracksClientPort())); + + setState(State.STARTED); + LOGGER.log(Level.INFO, "VXQueryService started successfully"); + } + + private synchronized void setState(State newState) { + state = newState; + } + + /** + * Submits a query to hyracks to be run after compiling. Required intermediate + * results and metrics are also calculated according to the + * {@link QueryRequest}. Checks if this class has started before moving further. + * + * @param request + * {@link QueryRequest} containing information about the query to be + * executed and the merics required along with the results + * @return AsyncQueryResponse if no error occurs | ErrorResponse else + */ + public APIResponse execute(final QueryRequest request) { + QueryRequest indexingRequest = new QueryRequest("show-indexes()"); + indexingRequest.setAsync(false); + SyncQueryResponse indexingResponse = (SyncQueryResponse) execute(indexingRequest, new ArrayList<>()); + LOGGER.log(Level.FINE, String.format("Found indexes: %s", indexingResponse.getResults())); + + List collections = Arrays.asList(indexingResponse.getResults().split("\n")); + return execute(request, collections); + } + + private APIResponse execute(final QueryRequest request, List collections) { + if (!State.STARTED.equals(state)) { + throw new IllegalStateException("VXQueryService is at state : " + state); + } + + String query = request.getStatement(); + final ResultSetId resultSetId = createResultSetId(); + + QueryResponse response = APIResponse.newQueryResponse(request, resultSetId); + response.setStatement(query); + + // Obtaining the node controller information from hyracks client connection + Map nodeControllerInfos = null; + try { + nodeControllerInfos = hyracksClientConnection.getNodeControllerInfos(); + } catch (HyracksException e) { + LOGGER.log(Level.SEVERE, String.format("Error occurred when obtaining NC info: '%s'", e.getMessage())); + return APIResponse.newErrorResponse(request.getRequestId(), Error.builder().withCode(UNFORSEEN_PROBLEM) + .withMessage("Hyracks connection problem: " + e.getMessage()).build()); + } + + // Adding a query compilation listener + VXQueryCompilationListener listener = new VXQueryCompilationListener(response, + request.isShowAbstractSyntaxTree(), request.isShowTranslatedExpressionTree(), + request.isShowOptimizedExpressionTree(), request.isShowRuntimePlan()); + + Date start = new Date(); + // Compiling the XQuery given + final XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, request.getFrameSize(), + vxQueryConfig.getAvailableProcessors(), vxQueryConfig.getJoinHashSize(), + vxQueryConfig.getMaximumDataSize(), vxQueryConfig.getHdfsConf()); + CompilerControlBlock compilerControlBlock = new CompilerControlBlock( + new StaticContextImpl(RootStaticContextImpl.INSTANCE), resultSetId, request.getSourceFileMap()); + try { + compiler.compile(null, new StringReader(query), compilerControlBlock, request.getOptimization(), + collections); + } catch (AlgebricksException e) { + LOGGER.log(Level.SEVERE, String.format("Error occurred when compiling query: '%s' with message: '%s'", + query, e.getMessage())); + return APIResponse.newErrorResponse(request.getRequestId(), Error.builder().withCode(PROBLEM_WITH_QUERY) + .withMessage("Query compilation failure: " + e.getMessage()).build()); + } catch (SystemException e) { + LOGGER.log(Level.SEVERE, String.format("Error occurred when compiling query: '%s' with message: '%s'", + query, e.getMessage())); + return APIResponse.newErrorResponse(request.getRequestId(), + new Error(PROBLEM_WITH_QUERY, "Query compilation failure: " + e.getCode())); + } + + if (request.isShowMetrics()) { + response.getMetrics().setCompileTime(new Date().getTime() - start.getTime()); + } + + if (request.isCompileOnly()) { + return response; + } + + Module module = compiler.getModule(); + JobSpecification js = module.getHyracksJobSpecification(); + DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext()); + js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory())); + + HyracksJobContext hyracksJobContext; + start = new Date(); + if (!request.isAsync()) { + for (int i = 0; i < request.getRepeatExecutions(); i++) { + try { + hyracksJobContext = executeJob(js, resultSetId, request); + + } catch (Exception e) { + LOGGER.log(SEVERE, "Error occurred when submitting job to hyracks for query: " + query, e); + return APIResponse.newErrorResponse(request.getRequestId(), + Error.builder().withCode(UNFORSEEN_PROBLEM) + .withMessage("Error occurred when starting hyracks job").build()); + } + try { + String results = readResults(hyracksJobContext); + ((SyncQueryResponse) response).setResults(results); + } catch (HyracksException e) { + LOGGER.log(Level.SEVERE, "Error occurred when reading results", e); + SystemException se = getSystemException(e); + return APIResponse.newErrorResponse(request.getRequestId(), new Error(UNFORSEEN_PROBLEM, + String.format("Error occurred when reading results: %s", se != null ? se.getCode() : ""))); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Error occurred when reading results", e); + return APIResponse.newErrorResponse(request.getRequestId(), + new Error(UNFORSEEN_PROBLEM, "Error occurred when reading results: " + e.getMessage())); + } + } + } else { + try { + hyracksJobContext = executeJob(js, resultSetId, request); + } catch (Exception e) { + LOGGER.log(SEVERE, "Error occurred when submitting job to hyracks for query: " + query, e); + return APIResponse.newErrorResponse(request.getRequestId(), Error.builder().withCode(UNFORSEEN_PROBLEM) + .withMessage("Error occurred when starting hyracks job").build()); + } + jobContexts.put(resultSetId.getId(), hyracksJobContext); + } + + if (request.isShowMetrics()) { + response.getMetrics().setElapsedTime(new Date().getTime() - start.getTime()); + } + + return response; + } + + private HyracksJobContext executeJob(JobSpecification js, ResultSetId resultSetId, QueryRequest request) + throws Exception { + HyracksJobContext hyracksJobContext; + JobId jobId = hyracksClientConnection.startJob(js, EnumSet.of(JobFlag.PROFILE_RUNTIME)); + hyracksJobContext = new HyracksJobContext(jobId, js.getFrameSize(), resultSetId); + + return hyracksJobContext; + } + + private static SystemException getSystemException(HyracksException e) { + Throwable t = e; + Throwable candidate = t instanceof SystemException ? t : null; + while (t.getCause() != null) { + t = t.getCause(); + if (t instanceof SystemException) { + candidate = t; + } + } + + t = candidate == null ? t : candidate; + final String message = t.getMessage(); + if (message != null) { + Matcher m = EMBEDDED_SYSERROR_PATTERN.matcher(message); + if (m.find()) { + String eCode = m.group(1); + return new SystemException(ErrorCode.valueOf(eCode), e); + } + } + return null; + } + + /** + * Returns the query results for a given result set id. + * + * @param request + * {@link QueryResultRequest} with result ID required + * @return Either a {@link QueryResultResponse} if no error occurred | + * {@link org.apache.vxquery.rest.response.ErrorResponse} else. + */ + public APIResponse getResult(QueryResultRequest request) { + if (jobContexts.containsKey(request.getResultId())) { + QueryResultResponse resultResponse = APIResponse.newQueryResultResponse(request.getRequestId()); + Date start = new Date(); + try { + String results = readResults(jobContexts.get(request.getResultId())); + resultResponse.setResults(results); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Error occurred when reading results for id : " + request.getResultId()); + return APIResponse.newErrorResponse(request.getRequestId(), new Error(UNFORSEEN_PROBLEM, + "Error occurred when reading results for: " + request.getResultId())); + } + + if (request.isShowMetrics()) { + resultResponse.getMetrics().setElapsedTime(new Date().getTime() - start.getTime()); + } + + return resultResponse; + } else { + return APIResponse.newErrorResponse(request.getRequestId(), Error.builder().withCode(NOT_FOUND) + .withMessage("No query found for result ID : " + request.getResultId()).build()); + } + } + + /** + * Reads results from hyracks given the {@link HyracksJobContext} containing + * {@link ResultSetId} and {@link JobId} mapping. + * + * @param jobContext + * mapoing between the {@link ResultSetId} and corresponding hyracks + * {@link JobId} + * @return Results of the given query + * @throws Exception + * IOErrors and etc + */ + private String readResults(HyracksJobContext jobContext) throws Exception { + int nReaders = 1; + + if (hyracksDataset == null) { + hyracksDataset = new HyracksDataset(hyracksClientConnection, jobContext.getFrameSize(), nReaders); + } + + FrameManager resultDisplayFrameMgr = new FrameManager(jobContext.getFrameSize()); + IFrame frame = new VSizeFrame(resultDisplayFrameMgr); + IHyracksDatasetReader reader = hyracksDataset.createReader(jobContext.getJobId(), jobContext.getResultSetId()); + OutputStream resultStream = new ByteArrayOutputStream(); + + // This loop is required for XTests to reliably identify the error code of + // SystemException. + while (reader.getResultStatus() == DatasetJobRecord.Status.RUNNING) { + Thread.sleep(100); + } + + IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor(); + try (PrintWriter writer = new PrintWriter(resultStream, true)) { + while (reader.read(frame) > 0) { + writer.print(ResultUtils.getStringFromBuffer(frame.getBuffer(), frameTupleAccessor)); + writer.flush(); + frame.getBuffer().clear(); + } + } + + hyracksClientConnection.waitForCompletion(jobContext.getJobId()); + LOGGER.log(Level.FINE, String.format("Result for resultId %d completed", jobContext.getResultSetId().getId())); + return resultStream.toString(); + } + + /** + * Create a unique result set id to get the correct query back from the cluster. + * + * @return Result Set id generated with current system time. + */ + protected ResultSetId createResultSetId() { + long resultSetId = atomicLong.incrementAndGet(); + LOGGER.log(Level.FINE, String.format("Creating result set with ID : %d", resultSetId)); + return new ResultSetId(resultSetId); + } + + public synchronized void stop() { + if (!State.STOPPED.equals(state)) { + setState(State.STOPPING); + LOGGER.log(Level.FINE, "Stooping VXQueryService"); + setState(State.STOPPED); + LOGGER.log(Level.INFO, "VXQueryService stopped successfully"); + } else { + LOGGER.log(Level.INFO, "VXQueryService is already in state : " + state); + } + } + + public State getState() { + return state; + } + + /** + * A {@link XQueryCompilationListener} implementation to be used to add + * AbstractSyntaxTree, RuntimePlan and etc to the {@link QueryResponse} if + * requested by the user. + */ + private class VXQueryCompilationListener implements XQueryCompilationListener { + private QueryResponse response; + private boolean showAbstractSyntaxTree; + private boolean showTranslatedExpressionTree; + private boolean showOptimizedExpressionTree; + private boolean showRuntimePlan; + + public VXQueryCompilationListener(QueryResponse response, boolean showAbstractSyntaxTree, + boolean showTranslatedExpressionTree, boolean showOptimizedExpressionTree, boolean showRuntimePlan) { + this.response = response; + this.showAbstractSyntaxTree = showAbstractSyntaxTree; + this.showTranslatedExpressionTree = showTranslatedExpressionTree; + this.showOptimizedExpressionTree = showOptimizedExpressionTree; + this.showRuntimePlan = showRuntimePlan; + } + + @Override + public void notifyParseResult(ModuleNode moduleNode) { + if (showAbstractSyntaxTree) { + response.setAbstractSyntaxTree(new XStream(new DomDriver()).toXML(moduleNode)); + } + } + + @Override + public void notifyTranslationResult(Module module) { + if (showTranslatedExpressionTree) { + response.setTranslatedExpressionTree(appendPrettyPlan(new StringBuilder(), module).toString()); + } + } + + @Override + public void notifyTypecheckResult(Module module) { + } + + @Override + public void notifyCodegenResult(Module module) { + if (showRuntimePlan) { + JobSpecification jobSpec = module.getHyracksJobSpecification(); + try { + response.setRuntimePlan(jobSpec.toJSON().toString()); + } catch (IOException e) { + LOGGER.log(SEVERE, + "Error occurred when obtaining runtime plan from job specification : " + jobSpec.toString(), + e); + } + } + } + + @Override + public void notifyOptimizedResult(Module module) { + if (showOptimizedExpressionTree) { + response.setOptimizedExpressionTree(appendPrettyPlan(new StringBuilder(), module).toString()); + } + } + + @SuppressWarnings("Duplicates") + private StringBuilder appendPrettyPlan(StringBuilder sb, Module module) { + try { + ILogicalExpressionVisitor ev = + new VXQueryLogicalExpressionPrettyPrintVisitor(module.getModuleContext()); + AlgebricksAppendable buffer = new AlgebricksAppendable(); + LogicalOperatorPrettyPrintVisitor v = new LogicalOperatorPrettyPrintVisitor(buffer, ev); + PlanPrettyPrinter.printPlan(module.getBody(), v, 0); + sb.append(buffer.toString()); + } catch (AlgebricksException e) { + LOGGER.log(SEVERE, "Error occurred when pretty printing expression : " + e.getMessage()); + } + return sb; + } + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryAPIServlet.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryAPIServlet.java new file mode 100644 index 000000000..45ef9105f --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryAPIServlet.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.rest.servlet; + +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE; +import static org.apache.vxquery.rest.Constants.MODE_ASYNC; +import static org.apache.vxquery.rest.Constants.MODE_SYNC; +import static org.apache.vxquery.rest.Constants.Parameters.COMPILE_ONLY; +import static org.apache.vxquery.rest.Constants.Parameters.FRAME_SIZE; +import static org.apache.vxquery.rest.Constants.Parameters.METRICS; +import static org.apache.vxquery.rest.Constants.Parameters.MODE; +import static org.apache.vxquery.rest.Constants.Parameters.OPTIMIZATION; +import static org.apache.vxquery.rest.Constants.Parameters.REPEAT_EXECUTIONS; +import static org.apache.vxquery.rest.Constants.Parameters.SHOW_AST; +import static org.apache.vxquery.rest.Constants.Parameters.SHOW_OET; +import static org.apache.vxquery.rest.Constants.Parameters.SHOW_RP; +import static org.apache.vxquery.rest.Constants.Parameters.SHOW_TET; +import static org.apache.vxquery.rest.Constants.Parameters.STATEMENT; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.AbstractMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Level; +import java.util.stream.Collectors; + +import javax.xml.bind.JAXBException; + +import org.apache.hyracks.http.api.IServletRequest; +import org.apache.vxquery.app.util.RestUtils; +import org.apache.vxquery.rest.Constants; +import org.apache.vxquery.rest.request.QueryRequest; +import org.apache.vxquery.rest.response.APIResponse; +import org.apache.vxquery.rest.response.Error; +import org.apache.vxquery.rest.service.VXQueryService; + +/** + * Servlet to handle query requests. + * + * @author Erandi Ganepola + */ +public class QueryAPIServlet extends RestAPIServlet { + + private VXQueryService vxQueryService; + + public QueryAPIServlet(VXQueryService vxQueryService, ConcurrentMap ctx, String... paths) { + super(ctx, paths); + this.vxQueryService = vxQueryService; + } + + @Override + protected APIResponse doHandle(IServletRequest request) { + LOGGER.log(Level.INFO, + String.format("Received a query request with query : %s", request.getParameter("statement"))); + + QueryRequest queryRequest; + try { + queryRequest = getQueryRequest(request); + } catch (Exception e) { + return APIResponse.newErrorResponse(null, + Error.builder().withCode(Constants.ErrorCodes.INVALID_INPUT).withMessage("Invalid input").build()); + } + + try { + return vxQueryService.execute(queryRequest); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Error occurred when trying to execute query : " + queryRequest.getStatement(), e); + return APIResponse.newErrorResponse(queryRequest.getRequestId(), Error.builder() + .withCode(Constants.ErrorCodes.UNFORSEEN_PROBLEM).withMessage(e.getMessage()).build()); + } + } + + private QueryRequest getQueryRequest(IServletRequest request) throws IOException, JAXBException { + if (request.getParameter(STATEMENT) == null || request.getParameter(STATEMENT).trim().isEmpty()) { + throw new IllegalArgumentException("Parameter 'statement' is required to handle the request"); + } + + QueryRequest queryRequest = new QueryRequest(UUID.randomUUID().toString(), request.getParameter(STATEMENT)); + queryRequest.setCompileOnly(Boolean.parseBoolean(request.getParameter(COMPILE_ONLY))); + queryRequest.setShowMetrics(Boolean.parseBoolean(request.getParameter(METRICS))); + + queryRequest.setShowAbstractSyntaxTree(Boolean.parseBoolean(request.getParameter(SHOW_AST))); + queryRequest.setShowTranslatedExpressionTree(Boolean.parseBoolean(request.getParameter(SHOW_TET))); + queryRequest.setShowOptimizedExpressionTree(Boolean.parseBoolean(request.getParameter(SHOW_OET))); + queryRequest.setShowRuntimePlan(Boolean.parseBoolean(request.getParameter(SHOW_RP))); + + if (request.getParameter(OPTIMIZATION) != null) { + queryRequest.setOptimization(Integer.parseInt(request.getParameter(OPTIMIZATION))); + } + if (request.getParameter(FRAME_SIZE) != null) { + queryRequest.setFrameSize(Integer.parseInt(request.getParameter(FRAME_SIZE))); + } + if (request.getParameter(REPEAT_EXECUTIONS) != null) { + queryRequest.setRepeatExecutions(Integer.parseInt(request.getParameter(REPEAT_EXECUTIONS))); + } + + String sourceFileMap = request.getHttpRequest().content().toString(StandardCharsets.UTF_8); + if (sourceFileMap != null && !sourceFileMap.isEmpty()) { + Map map = (Map) RestUtils.mapEntity(sourceFileMap, Map.class, + request.getHeader(CONTENT_TYPE)); + LOGGER.log(Level.FINE, "Found source file map"); + Map fileMap = map.entrySet().stream() + .map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey(), new File(entry.getValue()))) + .collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue)); + queryRequest.setSourceFileMap(fileMap); + } + + if (request.getParameter(MODE) != null) { + switch (request.getParameter(MODE)) { + case MODE_SYNC: + queryRequest.setAsync(false); + break; + case MODE_ASYNC: + default: + queryRequest.setAsync(true); + break; + } + } + + return queryRequest; + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryResultAPIServlet.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryResultAPIServlet.java new file mode 100644 index 000000000..84b0187f3 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryResultAPIServlet.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.servlet; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Level; + +import org.apache.hyracks.http.api.IServletRequest; +import org.apache.vxquery.rest.Constants; +import org.apache.vxquery.rest.request.QueryResultRequest; +import org.apache.vxquery.rest.response.APIResponse; +import org.apache.vxquery.rest.response.Error; +import org.apache.vxquery.rest.service.VXQueryService; + +import io.netty.handler.codec.http.HttpResponseStatus; + +/** + * Servlet to handle query results requests. + * + * @author Erandi Ganepola + */ +public class QueryResultAPIServlet extends RestAPIServlet { + + private VXQueryService vxQueryService; + + public QueryResultAPIServlet(VXQueryService vxQueryService, ConcurrentMap ctx, String... paths) { + super(ctx, paths); + this.vxQueryService = vxQueryService; + } + + @Override + protected APIResponse doHandle(IServletRequest request) { + String uri = request.getHttpRequest().uri(); + long resultId; + try { + String pathParam = uri.substring(uri.lastIndexOf("/") + 1); + pathParam = pathParam.contains("?") ? pathParam.split("\\?")[0] : pathParam; + resultId = Long.parseLong(pathParam); + } catch (NumberFormatException e) { + LOGGER.log(Level.SEVERE, "Result ID could not be retrieved from URL"); + return APIResponse.newErrorResponse(null, Error.builder().withCode(HttpResponseStatus.BAD_REQUEST.code()) + .withMessage("Result ID couldn't be retrieved from URL").build()); + } + + QueryResultRequest resultRequest = new QueryResultRequest(resultId, UUID.randomUUID().toString()); + resultRequest.setShowMetrics(Boolean.parseBoolean(request.getParameter(Constants.Parameters.METRICS))); + LOGGER.log(Level.INFO, + String.format("Received a result request with resultId : %d", resultRequest.getResultId())); + return vxQueryService.getResult(resultRequest); + } +} diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/RestAPIServlet.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/RestAPIServlet.java new file mode 100644 index 000000000..bc93dfc89 --- /dev/null +++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/RestAPIServlet.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest.servlet; + +import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON; +import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; + +import org.apache.htrace.fasterxml.jackson.core.JsonProcessingException; +import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hyracks.http.api.IServletRequest; +import org.apache.hyracks.http.api.IServletResponse; +import org.apache.hyracks.http.server.AbstractServlet; +import org.apache.hyracks.http.server.utils.HttpUtil; +import org.apache.vxquery.exceptions.VXQueryRuntimeException; +import org.apache.vxquery.exceptions.VXQueryServletRuntimeException; +import org.apache.vxquery.rest.response.APIResponse; +import org.apache.vxquery.rest.response.AsyncQueryResponse; +import org.apache.vxquery.rest.response.ErrorResponse; +import org.apache.vxquery.rest.response.QueryResultResponse; +import org.apache.vxquery.rest.response.SyncQueryResponse; +import org.apache.vxquery.rest.service.Status; + +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpResponseStatus; + +/** + * Abstract servlet to handle REST API requests. + * + * @author Erandi Ganepola + */ +public abstract class RestAPIServlet extends AbstractServlet { + + protected final Logger LOGGER; + + private JAXBContext jaxbContext; + + public RestAPIServlet(ConcurrentMap ctx, String... paths) { + super(ctx, paths); + LOGGER = Logger.getLogger(this.getClass().getName()); + try { + jaxbContext = JAXBContext.newInstance(QueryResultResponse.class, AsyncQueryResponse.class, + SyncQueryResponse.class, ErrorResponse.class); + } catch (JAXBException e) { + LOGGER.log(Level.SEVERE, "Error occurred when creating JAXB context", e); + throw new VXQueryRuntimeException("Unable to load JAXBContext", e); + } + } + + @Override + protected final void post(IServletRequest request, IServletResponse response) { + getOrPost(request, response); + } + + @Override + protected final void get(IServletRequest request, IServletResponse response) { + getOrPost(request, response); + } + + private void getOrPost(IServletRequest request, IServletResponse response) { + try { + initResponse(request, response); + APIResponse entity = doHandle(request); + if (entity == null) { + LOGGER.log(Level.WARNING, "No entity found for request : " + request); + response.setStatus(HttpResponseStatus.BAD_REQUEST); + } else { + // Important to set Status OK before setting the entity because the response + // (chunked) checks it before + // writing the response to channel. + setResponseStatus(response, entity); + setEntity(request, response, entity); + } + } catch (IOException e) { + response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR); + LOGGER.log(Level.SEVERE, "Error occurred when setting content type", e); + } + } + + private void initResponse(IServletRequest request, IServletResponse response) throws IOException { + // enable cross-origin resource sharing + response.setHeader("Access-Control-Allow-Origin", "*"); + response.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + + HttpUtil.setContentType(response, "text/plain"); + } + + private void setEntity(IServletRequest request, IServletResponse response, APIResponse entity) throws IOException { + String accept = request.getHeader(HttpHeaderNames.ACCEPT, ""); + String entityString; + switch (accept) { + case CONTENT_TYPE_XML: + try { + HttpUtil.setContentType(response, CONTENT_TYPE_XML); + + Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); + jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + StringWriter sw = new StringWriter(); + jaxbMarshaller.marshal(entity, sw); + entityString = sw.toString(); + } catch (JAXBException e) { + LOGGER.log(Level.SEVERE, "Error occurred when mapping java object into xml", e); + throw new VXQueryServletRuntimeException("Error occurred when marshalling entity", e); + } + break; + case CONTENT_TYPE_JSON: + default: + try { + HttpUtil.setContentType(response, CONTENT_TYPE_JSON); + ObjectMapper jsonMapper = new ObjectMapper(); + entityString = jsonMapper.writeValueAsString(entity); + } catch (JsonProcessingException e) { + LOGGER.log(Level.SEVERE, "Error occurred when mapping java object into JSON", e); + throw new VXQueryServletRuntimeException("Error occurred when mapping entity", e); + } + break; + } + + response.writer().print(entityString); + } + + private void setResponseStatus(IServletResponse response, APIResponse entity) { + if (Status.SUCCESS.toString().equals(entity.getStatus())) { + response.setStatus(HttpResponseStatus.OK); + } else if (Status.FATAL.toString().equals(entity.getStatus())) { + HttpResponseStatus status = HttpResponseStatus.INTERNAL_SERVER_ERROR; + if (entity instanceof ErrorResponse) { + status = HttpResponseStatus.valueOf(((ErrorResponse) entity).getError().getCode()); + } + response.setStatus(status); + } + } + + /** + * This abstract method is supposed to return an object which will be the entity + * of the response being sent to the client. Implementing classes doesn't have + * to worry about the content type of the request. + * + * @param request + * {@link IServletRequest} received + * @return Object to be set as the entity of the response + */ + protected abstract APIResponse doHandle(IServletRequest request); +} diff --git a/vxquery-rest/src/site/markdown/index.md b/vxquery-rest/src/site/markdown/index.md new file mode 100644 index 000000000..703df6566 --- /dev/null +++ b/vxquery-rest/src/site/markdown/index.md @@ -0,0 +1,249 @@ + +# VXQuery REST Server + +VXQuery REST Server allows users to submit queries and receive results either synchronously or +asynchronously through the exposed REST API. Along with the statement to be executed, few other parameters can be given as +well. Complete REST API specification can be found at [REST API Specification](specification.html). + +## Installation + +No additional steps needed to be taken to get the REST Server up and running. That is, setting up a VXQuery cluster will +automatically start the REST Server on port `8080`. Please see [VXQuery Cluster Setup](../user_cluster_installation.html) +to understand how a VXQuery cluster is setup. + +## Getting Started + +Suppose we want to execute a very simple XQuery like: + +``` +for $x in (1, 2.0, 3) return $x +``` + +### Async (Default Mode) Example + +If we want to send this, following will be the plain HTTP request. + +``` +GET http://127.0.1.1:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x HTTP/1.1 +``` + +Note the query parameter `statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x` in which the above mentioned statement +has been encoded. If we send this request using **cURL**, it will look like follows. + +#### Accept: application/json + +``` +curl -i -H "Accept: application/json" -X GET "http://localhost:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x" +``` + +and the response is, + +``` +HTTP/1.1 200 OK +transfer-encoding: chunked +connection: keep-alive +Access-Control-Allow-Origin: * +Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept +content-type: application/json +content-length: 320 + +{ + "status": "success", + "requestId": "b0cbe06f-3454-4422-ba23-59150e1c1400", + "statement": "for $x in (1, 2.0, 3) return $x", + "abstractSyntaxTree": null, + "translatedExpressionTree": null, + "optimizedExpressionTree": null, + "runtimePlan": null, + "metrics": { + "compileTime": 0, + "elapsedTime": 0 + }, + "resultId": 6, + "resultUrl": "/vxquery/query/result/6" +} +``` + +#### Accept: application/xml + +``` +curl -i -H "Accept: application/xml" -X GET "http://localhost:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x" +``` + +and the response is, + +``` +HTTP/1.1 200 OK +transfer-encoding: chunked +connection: keep-alive +Access-Control-Allow-Origin: * +Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept +content-type: application/xml +content-length: 403 + + + + d0c2c0ef-2e46-4153-9d4b-1ef4593184e7 + + 0 + 0 + + for $x in (1, 2.0, 3) return $x + 8 + /vxquery/query/result/8 + +``` + +#### Result Fetching + +Since we have used the default mode (**async**), we only got the **resultId**. Now we have to send another request asking +for the actual results. Send a cURL request to `/vxquery/query/result/8` to fetch results for result ID 8. + +##### Accept: application/json + +``` +curl -i -H "Accept: application/json" -X GET "http://localhost:39003/vxquery/query/result/8" +``` + +and the response is, + +``` +HTTP/1.1 200 OK +transfer-encoding: chunked +connection: keep-alive +Access-Control-Allow-Origin: * +Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept +content-type: application/json +content-length: 137 + +{ + "status": "success", + "requestId": "d0c2c0ef-2e46-4153-9d4b-1ef4593184e7", + "results": "1\n2\n3\n", + "metrics": { + "compileTime": 0, + "elapsedTime": 0 + } +} +``` + +Note the *results* in the JSON content in the response. + +##### Accept: application/xml + +``` +curl -i -H "Accept: application/xml" -X GET "http://localhost:39003/vxquery/query/result/8" +``` + +and the response is, + +``` +HTTP/1.1 200 OK +transfer-encoding: chunked +connection: keep-alive +Access-Control-Allow-Origin: * +Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept +content-type: application/xml +content-length: 298 + + + + d0c2c0ef-2e46-4153-9d4b-1ef4593184e7 + + 0 + 0 + + 1 +2 +3 + + +``` + +Note the ** in the XML content in the response. + +### Sync (Synchronous Mode) Example + +Similarly to what we did under async requests, we can send the query requests here as well, but with the added query parameter +`mode=sync` which is to indicate that the response should be a synchronous one. That is, we wait for the query to be +executed and the response to arrive. + +``` +curl -i -H "Accept: application/xml" -X GET \ +"http://localhost:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x&mode=sync" +``` + +and the response now contains **results** instead of the **resultId** we received previously. + +``` +HTTP/1.1 200 OK +transfer-encoding: chunked +connection: keep-alive +Access-Control-Allow-Origin: * +Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept +content-type: application/xml +content-length: 353 + + + + 93b67f50-4f14-4304-a9b2-f75b4a736df3 + + 0 + 0 + + for $x in (1, 2.0, 3) return $x + 1 +2 +3 + + +``` + +Similarly with `accept:application/json`, + +``` +curl -i -H "Accept: application/json" -X GET \ +"http://localhost:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x&mode=sync" +``` + +and the response is, + +``` +HTTP/1.1 200 OK +transfer-encoding: chunked +connection: keep-alive +Access-Control-Allow-Origin: * +Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept +content-type: application/json +content-length: 291 + +{ + "status": "success", + "requestId": "8010a699-a6f2-423c-91e1-8ac17cd5c5cd", + "statement": "for $x in (1, 2.0, 3) return $x", + "abstractSyntaxTree": null, + "translatedExpressionTree": null, + "optimizedExpressionTree": null, + "runtimePlan": null, + "metrics": { + "compileTime": 0, + "elapsedTime": 0 + }, + "results": "1\n2\n3\n" +} +``` diff --git a/vxquery-rest/src/site/markdown/specification.md b/vxquery-rest/src/site/markdown/specification.md new file mode 100644 index 000000000..397743922 --- /dev/null +++ b/vxquery-rest/src/site/markdown/specification.md @@ -0,0 +1,102 @@ + +# REST API Specification + +Swagger configuration of the REST API can be found +[here](https://cwiki.apache.org/confluence/display/VXQUERY/SwaggerIO+Configuration). + +**NOTE:** This REST API supports both **content types**, `application/json` and `application/xml`. Based on the `accept` header +of your query request, REST API will return the results wither in *json* form or *XML* form. Returned content type +defaults to `application/json` if no `accept` header is present. + +Base Path **${host}/vxquery** + +## Query Request + +Request of this type should be submitted for a given *query* to be executed. Depending on the value of the parameter +`mode`, a **synchronous** response or an **asynchronous** response will be returned + +`*` required + +| Path | Method |Parameters | Type | Description | +| ------ | ------ | ------ | ----- |----- | +| /query | GET |statement* | string | Statement to be executed | +| | |mode | string | `sync` or `async`. **async** will return an asynchronous response **(default: async)** | +| | |compileOnly | boolean | If `true`, statement will be compiled, but won't be executed (default: false) | +| | |optimization | integer | Optimization level (0 - 2,147,483,647). (Default: Full optimization) | +| | |frameSize | integer | Frame size in bytes (default: 65536) | +| | |repeatExecutions|integer | Number of times to repeat execution (default: 1) | +| | |metrics | boolean | If `true`, returns metrics (compile and execution time) with the response (default: false) | +| | |showAbstractSyntaxTree | boolean | Shows abstract syntax tree if `true` (default: false) | +| | |showTranslatedExpressionTree | boolean | Shows translated expression tree if `true` (default: false) | +| | |showOptimizedExpressionTree | boolean | Shows optimized expression tree if `true` (default: false) | +| | |showRuntimePlan| boolean | Shows runtime plan if set to `true` (default: false) | + +### Synchronous Query Response + +Received only when `mode` is set to `sync` in the query request above. + +| Attribute | Type | Description | +| ------ | ------ | ------ | +|statement* |string | Statement submitted to be executed | +|status* |string | `success` to indicate that the query execution was successful | +|requestId* |string | A unique ID assigned to the request sent earlier | +|abstractSyntaxTree |string | Abstract Syntax Tree if requested in the query request. Else `null` | +|translatedExpressionTree |string | Translated Expression Tree if requested in the query request. Else `null` | +|optimizedExpressionTree |string | Optimized Expression Tree if requested in the query request. Else `null` | +|runtimePlan |string | Runtime plan if requested in the query request. Else `null` | +|metrics |metrics | Metrics (`compileTime` and `elapsedTime`) if requested in the query request | +|results* |string | Results of the query/statement submitted for execution | + +### Asynchronous Query Response + +Received only when `mode` is set to `async` (which is the default value) in the query request above. + +| Attribute | Type | Description | +| ------ | ------ | ------ | +|statement* |string | Statement submitted to be executed | +|status* |string | `success` to indicate that the query execution was successful | +|requestId* |string | A unique ID assigned to the request sent earlier | +|abstractSyntaxTree |string | Abstract Syntax Tree if requested in the query request. Else `null` | +|translatedExpressionTree |string | Translated Expression Tree if requested in the query request. Else `null` | +|optimizedExpressionTree |string | Optimized Expression Tree if requested in the query request. Else `null` | +|runtimePlan |string | Runtime plan if requested in the query request. Else `null` | +|metrics |metrics | Metrics (`compileTime` and `elapsedTime`) if requested in the query request | +|resultId* |string | Result ID of the query submitted for execution. This ID is required later for result fetching | +|resultUrl* |string | URL from which the results of the submitted query can be retrieved | + +### Result fetching (After an Asynchronous Query Response) + +The **resultId** received in the asynchronous query response needs to be submitted as a +**path parameter** (`/query/result/${resultId}`) to the REST API in order to retrieve the corresponding results. + +| Path | Method |Parameters | Type | Description | +| ------ | ------ | ------ | ----- |----- | +| /query/result/${resultId} | GET | metrics | boolean | If `true`, returns metrics (compile and execution time) with the response (default: false) | + +*** + +## Error Response + +In any of the above scenarios, if an error occurred while processing, REST API will return an *Error Response* as +specified below. + +| Attribute | Type | Description | +| ------ | ------ | ------ | +|status* |string | `fatal` to indicate that the query execution was failed at some point | +|requestId* |string | A unique ID assigned to the request sent | +|error* |Error | An error object which include an error code with an error message. {code: xxx, message: "error message"} | diff --git a/vxquery-rest/src/site/site.xml b/vxquery-rest/src/site/site.xml new file mode 100644 index 000000000..bd9b9abd7 --- /dev/null +++ b/vxquery-rest/src/site/site.xml @@ -0,0 +1,53 @@ + + + + VXQuery + ../images/VXQuery.png + ../index.html + + + + Apache Software Foundation + ../images/asf_logo_wide.png + http://www.apache.org/ + + + + org.apache.maven.skins + maven-fluido-skin + 1.5 + + + + + + + + + +
Apache VXQuery, VXQuery, Apache, the Apache + feather logo, and the Apache VXQuery project logo are either + registered trademarks or trademarks of The Apache Software + Foundation in the United States and other countries. + All other marks mentioned may be trademarks or registered + trademarks of their respective owners. + ]]> +
+ + \ No newline at end of file diff --git a/vxquery-rest/src/test/java/org/apache/vxquery/rest/AbstractRestServerTest.java b/vxquery-rest/src/test/java/org/apache/vxquery/rest/AbstractRestServerTest.java new file mode 100644 index 000000000..3d4b1240f --- /dev/null +++ b/vxquery-rest/src/test/java/org/apache/vxquery/rest/AbstractRestServerTest.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest; + +import java.net.URI; +import java.util.concurrent.TimeUnit; + +import javax.ws.rs.HttpMethod; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.utils.HttpClientUtils; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.vxquery.app.VXQueryApplication; +import org.apache.vxquery.app.util.LocalClusterUtil; +import org.apache.vxquery.app.util.RestUtils; +import org.apache.vxquery.rest.request.QueryRequest; +import org.apache.vxquery.rest.request.QueryResultRequest; +import org.apache.vxquery.rest.response.AsyncQueryResponse; +import org.apache.vxquery.rest.response.QueryResponse; +import org.apache.vxquery.rest.response.QueryResultResponse; +import org.apache.vxquery.rest.response.SyncQueryResponse; +import org.apache.vxquery.rest.service.VXQueryConfig; +import org.apache.vxquery.rest.service.VXQueryService; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; + +import io.netty.handler.codec.http.HttpResponseStatus; + +/** + * Abstract test class to be used for {@link VXQueryApplication} related tests. + * These tests are expected to use the REST API for querying and fetching + * results + * + * @author Erandi Ganepola + */ +public class AbstractRestServerTest { + + protected static LocalClusterUtil vxqueryLocalCluster = new LocalClusterUtil(); + protected static String restIpAddress; + protected static int restPort; + protected static VXQueryService vxQueryService; + + @BeforeClass + public static void setUp() throws Exception { + vxqueryLocalCluster.init(new VXQueryConfig()); + vxQueryService = vxqueryLocalCluster.getVxQueryService(); + restIpAddress = vxqueryLocalCluster.getIpAddress(); + restPort = vxqueryLocalCluster.getRestPort(); + } + + @AfterClass + public static void tearDown() throws Exception { + vxqueryLocalCluster.deinit(); + } + + protected static String normalize(String string) { + if (string == null) { + return null; + } + + return string.replace("\r\n", "").replace("\n", "").replace("\r", ""); + } + + protected static void checkMetrics(QueryResponse response, boolean showMetrics) { + if (showMetrics) { + Assert.assertTrue(response.getMetrics().getCompileTime() > 0); + Assert.assertTrue(response.getMetrics().getElapsedTime() > 0); + } else { + Assert.assertTrue(response.getMetrics().getCompileTime() == 0); + Assert.assertTrue(response.getMetrics().getElapsedTime() == 0); + } + } + + protected static void checkResults(AsyncQueryResponse response, boolean compileOnly) { + if (compileOnly) { + Assert.assertNull(response.getResultUrl()); + Assert.assertEquals(0, response.getResultId()); + } else { + Assert.assertTrue(response.getResultUrl().startsWith(Constants.RESULT_URL_PREFIX)); + Assert.assertNotEquals(0, response.getResultId()); + } + } + + protected static void checkResults(SyncQueryResponse response, boolean compileOnly) { + if (compileOnly) { + Assert.assertNull(response.getResults()); + } else { + Assert.assertNotNull(response.getResults()); + Assert.assertFalse(response.getResults().isEmpty()); + } + } + + /** + * Submit a {@link QueryRequest} and fetth the resulting + * {@link AsyncQueryResponse} + * + * @param uri + * uri of the GET request + * @param accepts + * application/json | application/xml + * @param method + * Http Method to be used to send the request + * @return Response received for the query request + * @throws Exception + */ + protected static T getQuerySuccessResponse(URI uri, String accepts, Class type, String method) + throws Exception { + CloseableHttpClient httpClient = HttpClients.custom().setConnectionTimeToLive(20, TimeUnit.SECONDS).build(); + + try { + HttpUriRequest request = getRequest(uri, method); + + if (accepts != null) { + request.setHeader(HttpHeaders.ACCEPT, accepts); + } + + try (CloseableHttpResponse httpResponse = httpClient.execute(request)) { + Assert.assertEquals(HttpResponseStatus.OK.code(), httpResponse.getStatusLine().getStatusCode()); + if (accepts != null) { + Assert.assertEquals(accepts, httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue()); + } + + HttpEntity entity = httpResponse.getEntity(); + Assert.assertNotNull(entity); + + String response = RestUtils.readEntity(entity); + return RestUtils.mapEntity(response, type, accepts); + } + } finally { + HttpClientUtils.closeQuietly(httpClient); + } + } + + /** + * Fetch the {@link QueryResultResponse} from query result endpoint once the + * corresponding {@link QueryResultRequest} is given. + * + * @param resultRequest + * {@link QueryResultRequest} + * @param accepts + * expected + * + * 
+     *            Accepts
+     *
+ * + * header in responses + * @param method + * Http Method to be used to send the request + * @return query result response received + * @throws Exception + */ + protected static QueryResultResponse getQueryResultResponse(QueryResultRequest resultRequest, String accepts, + String method) throws Exception { + URI uri = RestUtils.buildQueryResultURI(resultRequest, restIpAddress, restPort); + CloseableHttpClient httpClient = HttpClients.custom().setConnectionTimeToLive(20, TimeUnit.SECONDS).build(); + try { + HttpUriRequest request = getRequest(uri, method); + + if (accepts != null) { + request.setHeader(HttpHeaders.ACCEPT, accepts); + } + + try (CloseableHttpResponse httpResponse = httpClient.execute(request)) { + if (accepts != null) { + Assert.assertEquals(accepts, httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue()); + } + Assert.assertEquals(httpResponse.getStatusLine().getStatusCode(), HttpResponseStatus.OK.code()); + + HttpEntity entity = httpResponse.getEntity(); + Assert.assertNotNull(entity); + + String response = RestUtils.readEntity(entity); + return RestUtils.mapEntity(response, QueryResultResponse.class, accepts); + } + } finally { + HttpClientUtils.closeQuietly(httpClient); + } + } + + /** + * Creates a POST or GET request accordingly from the given {@link URI} + * + * @param uri + * URI to which the request us to be sent + * @param method + * Http method- GET or POST + * @return request + */ + protected static HttpUriRequest getRequest(URI uri, String method) { + HttpUriRequest request; + switch (method) { + case HttpMethod.POST: + request = new HttpPost(uri); + break; + case HttpMethod.GET: + default: + request = new HttpGet(uri); + } + + return request; + } +} diff --git a/vxquery-rest/src/test/java/org/apache/vxquery/rest/ErrorResponseTest.java b/vxquery-rest/src/test/java/org/apache/vxquery/rest/ErrorResponseTest.java new file mode 100644 index 000000000..12b61f31c --- /dev/null +++ b/vxquery-rest/src/test/java/org/apache/vxquery/rest/ErrorResponseTest.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest; + +import static org.apache.vxquery.app.util.RestUtils.buildQueryResultURI; +import static org.apache.vxquery.app.util.RestUtils.buildQueryURI; +import static org.apache.vxquery.rest.Constants.ErrorCodes.INVALID_INPUT; +import static org.apache.vxquery.rest.Constants.ErrorCodes.NOT_FOUND; +import static org.apache.vxquery.rest.Constants.ErrorCodes.PROBLEM_WITH_QUERY; +import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON; +import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML; + +import java.net.URI; +import java.util.concurrent.TimeUnit; + +import javax.ws.rs.HttpMethod; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.utils.HttpClientUtils; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.vxquery.app.util.RestUtils; +import org.apache.vxquery.rest.request.QueryRequest; +import org.apache.vxquery.rest.request.QueryResultRequest; +import org.apache.vxquery.rest.response.ErrorResponse; +import org.junit.Assert; +import org.junit.Test; + +/** + * Tests error codes of the possible error responses that can be received for + * erroneous queries. + * + * @author Erandi Ganepola + */ +public class ErrorResponseTest extends AbstractRestServerTest { + + @Test + public void testInvalidInput01() throws Exception { + QueryRequest request = new QueryRequest(" "); + runTest(buildQueryURI(request, restIpAddress, restPort), null, INVALID_INPUT); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, INVALID_INPUT); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, INVALID_INPUT); + } + + @Test + public void testInvalidInput02() throws Exception { + QueryRequest request = new QueryRequest(""); + runTest(buildQueryURI(request, restIpAddress, restPort), null, 405); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, 405); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, 405); + } + + @Test + public void testInvalidQuery01() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1,2,3) return $y"); + runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY); + } + + @Test + public void testInvalidQuery02() throws Exception { + QueryRequest request = new QueryRequest("for x in (1,2,3) return $x"); + runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY); + } + + @Test + public void testInvalidQuery03() throws Exception { + QueryRequest request = new QueryRequest("insert nodes into doc(\"abcd.xml\")/books"); + runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY); + } + + @Test + public void testInvalidQuery04() throws Exception { + QueryRequest request = new QueryRequest("delete nodes /a/b//node()"); + runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY); + } + + @Test + public void testInvalidResultId() throws Exception { + QueryResultRequest request = new QueryResultRequest(1000); + runTest(buildQueryResultURI(request, restIpAddress, restPort), null, NOT_FOUND); + runTest(buildQueryResultURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, NOT_FOUND); + runTest(buildQueryResultURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, NOT_FOUND); + } + + @Test + public void testSyncInvalidInput01() throws Exception { + QueryRequest request = new QueryRequest(" "); + request.setAsync(false); + runTest(buildQueryURI(request, restIpAddress, restPort), null, INVALID_INPUT); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, INVALID_INPUT); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, INVALID_INPUT); + } + + @Test + public void testSyncInvalidInput02() throws Exception { + QueryRequest request = new QueryRequest(""); + request.setAsync(false); + runTest(buildQueryURI(request, restIpAddress, restPort), null, 405); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, 405); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, 405); + } + + @Test + public void testSyncInvalidQuery01() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1,2,3) return $y"); + request.setAsync(false); + runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY); + } + + @Test + public void testSyncInvalidQuery02() throws Exception { + QueryRequest request = new QueryRequest("for x in (1,2,3) return $x"); + request.setAsync(false); + runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY); + } + + @Test + public void testSyncInvalidQuery03() throws Exception { + QueryRequest request = new QueryRequest("insert nodes into doc(\"abcd.xml\")/books"); + request.setAsync(false); + runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY); + } + + @Test + public void testSyncInvalidQuery04() throws Exception { + QueryRequest request = new QueryRequest("delete nodes /a/b//node()"); + request.setAsync(false); + runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY); + runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY); + } + + private void runTest(URI uri, String accepts, int expectedStatusCode) throws Exception { + runTest(uri, accepts, expectedStatusCode, HttpMethod.GET); + runTest(uri, accepts, expectedStatusCode, HttpMethod.POST); + } + + private void runTest(URI uri, String accepts, int expectedStatusCode, String httpMethod) throws Exception { + CloseableHttpClient httpClient = HttpClients.custom().setConnectionTimeToLive(20, TimeUnit.SECONDS).build(); + + ErrorResponse errorResponse; + try { + HttpUriRequest request = getRequest(uri, httpMethod); + if (accepts != null) { + request.setHeader(HttpHeaders.ACCEPT, accepts); + } + + try (CloseableHttpResponse httpResponse = httpClient.execute(request)) { + Assert.assertEquals(expectedStatusCode, httpResponse.getStatusLine().getStatusCode()); + if (accepts != null) { + Assert.assertEquals(accepts, httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue()); + } + + HttpEntity entity = httpResponse.getEntity(); + Assert.assertNotNull(entity); + + String response = RestUtils.readEntity(entity); + errorResponse = RestUtils.mapEntity(response, ErrorResponse.class, accepts); + } + } finally { + HttpClientUtils.closeQuietly(httpClient); + } + + Assert.assertNotNull(errorResponse); + Assert.assertNotNull(errorResponse.getError().getMessage()); + Assert.assertEquals(errorResponse.getError().getCode(), expectedStatusCode); + } +} diff --git a/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessAsyncResponseTest.java b/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessAsyncResponseTest.java new file mode 100644 index 000000000..d1385c8fe --- /dev/null +++ b/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessAsyncResponseTest.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest; + +import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON; +import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML; + +import java.net.URI; + +import javax.ws.rs.HttpMethod; + +import org.apache.vxquery.app.util.RestUtils; +import org.apache.vxquery.rest.request.QueryRequest; +import org.apache.vxquery.rest.request.QueryResultRequest; +import org.apache.vxquery.rest.response.APIResponse; +import org.apache.vxquery.rest.response.AsyncQueryResponse; +import org.apache.vxquery.rest.response.ErrorResponse; +import org.apache.vxquery.rest.response.QueryResultResponse; +import org.apache.vxquery.rest.service.Status; +import org.junit.Assert; +import org.junit.Test; + +/** + * This class tests the success responses received for XQueries submitted. i.e + * we are submitting correct queries which are expected to return a predictable + * result. All the parameters that are expected to be sent with query requests + * are subjected to test in this test class + * + * @author Erandi Ganepola + */ +public class SuccessAsyncResponseTest extends AbstractRestServerTest { + + @Test + public void testSimpleQuery001() throws Exception { + QueryRequest request = new QueryRequest("1+1"); + request.setShowAbstractSyntaxTree(true); + request.setShowOptimizedExpressionTree(true); + request.setShowRuntimePlan(true); + request.setShowTranslatedExpressionTree(true); + request.setShowMetrics(false); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSimpleQuery002() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setShowAbstractSyntaxTree(true); + request.setShowOptimizedExpressionTree(true); + request.setShowRuntimePlan(true); + request.setShowTranslatedExpressionTree(true); + request.setShowMetrics(true); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSimpleQuery003() throws Exception { + QueryRequest request = new QueryRequest("1+2+3"); + request.setShowAbstractSyntaxTree(false); + request.setShowOptimizedExpressionTree(false); + request.setShowRuntimePlan(false); + request.setShowTranslatedExpressionTree(false); + request.setShowMetrics(false); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSimpleQuery004() throws Exception { + QueryRequest request = new QueryRequest("fn:true()"); + request.setShowAbstractSyntaxTree(false); + request.setShowOptimizedExpressionTree(false); + request.setShowRuntimePlan(true); + request.setShowTranslatedExpressionTree(false); + request.setShowMetrics(false); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSingleParameterNone() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSingleParameterMetrics() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setShowMetrics(true); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSingleParameterAST() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setShowAbstractSyntaxTree(true); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSingleParameterOptimization() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setOptimization(10000); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSingleParameterFrameSize() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setFrameSize((int) Math.pow(2, 12)); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSingleParameterCompileOnly() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setCompileOnly(true); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSingleParameterOET() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setShowOptimizedExpressionTree(true); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSingleParameterTET() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setShowTranslatedExpressionTree(true); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSingleParameterRP() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setShowRuntimePlan(true); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + private void runTest(String contentType, QueryRequest request) throws Exception { + runTest(contentType, request, HttpMethod.GET); + runTest(contentType, request, HttpMethod.POST); + } + + private void runTest(String contentType, QueryRequest request, String httpMethod) throws Exception { + URI queryEndpointUri = RestUtils.buildQueryURI(request, restIpAddress, restPort); + + /* + * ========== Query Response Testing ========== + */ + // Testing the accuracy of VXQueryService class + AsyncQueryResponse expectedAsyncQueryResponse = (AsyncQueryResponse) vxQueryService.execute(request); + + Assert.assertEquals(Status.SUCCESS.toString(), expectedAsyncQueryResponse.getStatus()); + Assert.assertEquals(request.getStatement(), expectedAsyncQueryResponse.getStatement()); + checkResults(expectedAsyncQueryResponse, request.isCompileOnly()); + checkMetrics(expectedAsyncQueryResponse, request.isShowMetrics()); + if (request.isShowMetrics()) { + Assert.assertTrue(expectedAsyncQueryResponse.getMetrics().getCompileTime() > 0); + } else { + Assert.assertTrue(expectedAsyncQueryResponse.getMetrics().getCompileTime() == 0); + } + if (request.isShowAbstractSyntaxTree()) { + Assert.assertNotNull(expectedAsyncQueryResponse.getAbstractSyntaxTree()); + } else { + Assert.assertNull(expectedAsyncQueryResponse.getAbstractSyntaxTree()); + } + if (request.isShowTranslatedExpressionTree()) { + Assert.assertNotNull(expectedAsyncQueryResponse.getTranslatedExpressionTree()); + } else { + Assert.assertNull(expectedAsyncQueryResponse.getTranslatedExpressionTree()); + } + if (request.isShowOptimizedExpressionTree()) { + Assert.assertNotNull(expectedAsyncQueryResponse.getOptimizedExpressionTree()); + } else { + Assert.assertNull(expectedAsyncQueryResponse.getOptimizedExpressionTree()); + } + if (request.isShowRuntimePlan()) { + Assert.assertNotNull(expectedAsyncQueryResponse.getRuntimePlan()); + } else { + Assert.assertNull(expectedAsyncQueryResponse.getRuntimePlan()); + } + + // Testing the accuracy of REST server and servlets + AsyncQueryResponse actualAsyncQueryResponse = + getQuerySuccessResponse(queryEndpointUri, contentType, AsyncQueryResponse.class, httpMethod); + + Assert.assertNotNull(actualAsyncQueryResponse.getRequestId()); + Assert.assertEquals(request.getStatement(), actualAsyncQueryResponse.getStatement()); + Assert.assertEquals(Status.SUCCESS.toString(), actualAsyncQueryResponse.getStatus()); + checkResults(actualAsyncQueryResponse, request.isCompileOnly()); + checkMetrics(actualAsyncQueryResponse, request.isShowMetrics()); + // Cannot check this because Runtime plan include some object IDs which differ + // Assert.assertEquals(expectedAsyncQueryResponse.getRuntimePlan(), + // actualAsyncQueryResponse.getRuntimePlan()); + if (request.isShowRuntimePlan()) { + Assert.assertNotNull(actualAsyncQueryResponse.getRuntimePlan()); + } else { + Assert.assertNull(actualAsyncQueryResponse.getRuntimePlan()); + } + Assert.assertEquals(normalize(expectedAsyncQueryResponse.getOptimizedExpressionTree()), + normalize(actualAsyncQueryResponse.getOptimizedExpressionTree())); + Assert.assertEquals(normalize(expectedAsyncQueryResponse.getTranslatedExpressionTree()), + normalize(actualAsyncQueryResponse.getTranslatedExpressionTree())); + Assert.assertEquals(normalize(expectedAsyncQueryResponse.getAbstractSyntaxTree()), + normalize(actualAsyncQueryResponse.getAbstractSyntaxTree())); + + /* + * ========== Query Result Response Testing ======== + */ + QueryResultRequest resultRequest = new QueryResultRequest(actualAsyncQueryResponse.getResultId()); + resultRequest.setShowMetrics(true); + + if (request.isCompileOnly()) { + APIResponse resultResponse = vxQueryService.getResult(resultRequest); + Assert.assertTrue(resultResponse instanceof ErrorResponse); + } else { + QueryResultResponse expectedResultResponse = (QueryResultResponse) vxQueryService.getResult(resultRequest); + Assert.assertEquals(expectedResultResponse.getStatus(), Status.SUCCESS.toString()); + Assert.assertNotNull(expectedResultResponse.getResults()); + + QueryResultResponse actualResultResponse = getQueryResultResponse(resultRequest, contentType, httpMethod); + Assert.assertEquals(actualResultResponse.getStatus(), Status.SUCCESS.toString()); + Assert.assertNotNull(actualResultResponse.getResults()); + Assert.assertNotNull(actualResultResponse.getRequestId()); + Assert.assertEquals(normalize(expectedResultResponse.getResults()), + normalize(actualResultResponse.getResults())); + if (resultRequest.isShowMetrics()) { + Assert.assertTrue(actualResultResponse.getMetrics().getElapsedTime() > 0); + } else { + Assert.assertTrue(actualResultResponse.getMetrics().getElapsedTime() == 0); + } + + } + } +} diff --git a/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessSyncResponseTest.java b/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessSyncResponseTest.java new file mode 100644 index 000000000..31e016290 --- /dev/null +++ b/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessSyncResponseTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.vxquery.rest; + +import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON; +import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML; + +import java.net.URI; + +import javax.ws.rs.HttpMethod; + +import org.apache.vxquery.app.util.RestUtils; +import org.apache.vxquery.rest.request.QueryRequest; +import org.apache.vxquery.rest.response.SyncQueryResponse; +import org.apache.vxquery.rest.service.Status; +import org.junit.Assert; +import org.junit.Test; + +/** + * This class tests the success responses received for XQueries submitted. i.e + * we are submitting correct queries which are expected to return a predictable + * result. All the parameters that are expected to be sent with query requests + * are subjected to test in this test class + * + * @author Erandi Ganepola + */ +public class SuccessSyncResponseTest extends AbstractRestServerTest { + + @Test + public void testSimpleQuery001() throws Exception { + QueryRequest request = new QueryRequest("1+1"); + request.setShowAbstractSyntaxTree(true); + request.setShowOptimizedExpressionTree(true); + request.setShowRuntimePlan(true); + request.setShowTranslatedExpressionTree(true); + request.setShowMetrics(false); + request.setAsync(false); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSimpleQuery002() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setShowAbstractSyntaxTree(true); + request.setShowOptimizedExpressionTree(true); + request.setShowRuntimePlan(true); + request.setShowTranslatedExpressionTree(true); + request.setShowMetrics(true); + request.setAsync(false); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSimpleQuery003() throws Exception { + QueryRequest request = new QueryRequest("1+2+3"); + request.setShowAbstractSyntaxTree(false); + request.setShowOptimizedExpressionTree(false); + request.setShowRuntimePlan(false); + request.setShowTranslatedExpressionTree(false); + request.setShowMetrics(false); + request.setAsync(false); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSimpleQuery004() throws Exception { + QueryRequest request = new QueryRequest("fn:true()"); + request.setShowAbstractSyntaxTree(false); + request.setShowOptimizedExpressionTree(false); + request.setShowRuntimePlan(true); + request.setShowTranslatedExpressionTree(false); + request.setShowMetrics(false); + request.setAsync(false); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSingleParameterNone() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setAsync(false); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSingleParameterCompileOnly() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setCompileOnly(true); + request.setAsync(false); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + @Test + public void testSingleParameterRepeatExecutions() throws Exception { + QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x"); + request.setRepeatExecutions(5); + request.setAsync(false); + + runTest(null, request); + runTest(CONTENT_TYPE_JSON, request); + runTest(CONTENT_TYPE_XML, request); + } + + private void runTest(String contentType, QueryRequest request) throws Exception { + runTest(contentType, request, HttpMethod.GET); + runTest(contentType, request, HttpMethod.POST); + } + + private void runTest(String contentType, QueryRequest request, String httpMethod) throws Exception { + URI queryEndpointUri = RestUtils.buildQueryURI(request, restIpAddress, restPort); + + /* + * ========== Query Response Testing ========== + */ + // Testing the accuracy of VXQueryService class + SyncQueryResponse expectedSyncQueryResponse = (SyncQueryResponse) vxQueryService.execute(request); + + Assert.assertEquals(Status.SUCCESS.toString(), expectedSyncQueryResponse.getStatus()); + Assert.assertEquals(request.getStatement(), expectedSyncQueryResponse.getStatement()); + checkResults(expectedSyncQueryResponse, request.isCompileOnly()); + checkMetrics(expectedSyncQueryResponse, request.isShowMetrics()); + if (request.isShowMetrics()) { + Assert.assertTrue(expectedSyncQueryResponse.getMetrics().getCompileTime() > 0); + } else { + Assert.assertTrue(expectedSyncQueryResponse.getMetrics().getCompileTime() == 0); + } + if (request.isShowAbstractSyntaxTree()) { + Assert.assertNotNull(expectedSyncQueryResponse.getAbstractSyntaxTree()); + } else { + Assert.assertNull(expectedSyncQueryResponse.getAbstractSyntaxTree()); + } + if (request.isShowTranslatedExpressionTree()) { + Assert.assertNotNull(expectedSyncQueryResponse.getTranslatedExpressionTree()); + } else { + Assert.assertNull(expectedSyncQueryResponse.getTranslatedExpressionTree()); + } + if (request.isShowOptimizedExpressionTree()) { + Assert.assertNotNull(expectedSyncQueryResponse.getOptimizedExpressionTree()); + } else { + Assert.assertNull(expectedSyncQueryResponse.getOptimizedExpressionTree()); + } + if (request.isShowRuntimePlan()) { + Assert.assertNotNull(expectedSyncQueryResponse.getRuntimePlan()); + } else { + Assert.assertNull(expectedSyncQueryResponse.getRuntimePlan()); + } + + // Testing the accuracy of REST server and servlets + SyncQueryResponse actualSyncQueryResponse = + getQuerySuccessResponse(queryEndpointUri, contentType, SyncQueryResponse.class, httpMethod); + + Assert.assertNotNull(actualSyncQueryResponse.getRequestId()); + Assert.assertEquals(request.getStatement(), actualSyncQueryResponse.getStatement()); + Assert.assertEquals(Status.SUCCESS.toString(), actualSyncQueryResponse.getStatus()); + checkMetrics(actualSyncQueryResponse, request.isShowMetrics()); + checkResults(actualSyncQueryResponse, request.isCompileOnly()); + // Cannot check this because Runtime plan include some object IDs which differ + // Assert.assertEquals(expectedSyncQueryResponse.getRuntimePlan(), + // actualSyncQueryResponse.getRuntimePlan()); + if (request.isShowRuntimePlan()) { + Assert.assertNotNull(actualSyncQueryResponse.getRuntimePlan()); + } else { + Assert.assertNull(actualSyncQueryResponse.getRuntimePlan()); + } + Assert.assertEquals(normalize(expectedSyncQueryResponse.getOptimizedExpressionTree()), + normalize(actualSyncQueryResponse.getOptimizedExpressionTree())); + Assert.assertEquals(normalize(expectedSyncQueryResponse.getTranslatedExpressionTree()), + normalize(actualSyncQueryResponse.getTranslatedExpressionTree())); + Assert.assertEquals(normalize(expectedSyncQueryResponse.getAbstractSyntaxTree()), + normalize(actualSyncQueryResponse.getAbstractSyntaxTree())); + + /* + * ========== Query Result Response Testing ======== + */ + String expectedResults = expectedSyncQueryResponse.getResults(); + String actualResults = actualSyncQueryResponse.getResults(); + if (!request.isCompileOnly()) { + Assert.assertNotNull(expectedResults); + Assert.assertNotNull(actualResults); + } + Assert.assertEquals(normalize(expectedResults), normalize(actualResults)); + } +} diff --git a/vxquery-rest/src/test/resources/vxquery.properties b/vxquery-rest/src/test/resources/vxquery.properties new file mode 100644 index 000000000..8870f7972 --- /dev/null +++ b/vxquery-rest/src/test/resources/vxquery.properties @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Number of processors to be used for query processing +#org.apache.vxquery.available_processors=-1 + +# Number of local node controllers to be created when creating a local hyracks cluster +org.apache.vxquery.local_nc=1 + +# Join hash size +#org.apache.vxquery.join_hash=-1 + +# Maximum Data Size +#org.apache.vxquery.data_size=-1 + +# hdfs config directory +#org.apache.vxquery.hdfs_config=foo/bar diff --git a/vxquery-server/pom.xml b/vxquery-server/pom.xml index e572c3605..7331fde13 100644 --- a/vxquery-server/pom.xml +++ b/vxquery-server/pom.xml @@ -41,7 +41,7 @@ org.codehaus.mojo appassembler-maven-plugin - 1.1.1 + 2.0.0 @@ -53,6 +53,10 @@ org.apache.hyracks.control.cc.CCDriver vxquerycc + + -app-cc-main-class + org.apache.vxquery.app.VXQueryApplication + org.apache.hyracks.control.nc.NCDriver @@ -139,15 +143,13 @@ org.apache.vxquery - apache-vxquery-core + apache-vxquery-rest 0.7-SNAPSHOT - org.apache.hyracks hyracks-control-cc - org.apache.hyracks hyracks-control-nc diff --git a/vxquery-xtest/pom.xml b/vxquery-xtest/pom.xml index a00bec201..a32d913b4 100644 --- a/vxquery-xtest/pom.xml +++ b/vxquery-xtest/pom.xml @@ -143,6 +143,12 @@ 0.7-SNAPSHOT + + org.apache.vxquery + apache-vxquery-rest + 0.7-SNAPSHOT + + org.apache.hyracks hyracks-api diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java index 0e5b48105..4d2ae8a38 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java @@ -19,81 +19,45 @@ import org.apache.hyracks.api.client.HyracksConnection; import org.apache.hyracks.client.dataset.HyracksDataset; -import org.apache.hyracks.control.cc.ClusterControllerService; -import org.apache.hyracks.control.common.controllers.CCConfig; -import org.apache.hyracks.control.common.controllers.NCConfig; -import org.apache.hyracks.control.nc.NodeControllerService; +import org.apache.vxquery.app.util.LocalClusterUtil; +import org.apache.vxquery.rest.service.VXQueryConfig; -import java.io.File; import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; public class TestClusterUtil { - private static final int CLIENT_NET_PORT = 39000; - private static final int CLUSTER_NET_PORT = 39001; - private static final int PROFILE_DUMP_PERIOD = 10000; - private static final String CC_HOST = "localhost"; - private static final String NODE_ID = "nc1"; - private static final String IO_DEVICES = "target/tmp/indexFolder"; - private static HyracksConnection hcc; private static HyracksDataset hds; + public static final LocalClusterUtil localClusterUtil = new LocalClusterUtil(); + private TestClusterUtil() { } - public static CCConfig createCCConfig() throws UnknownHostException { - String publicAddress = InetAddress.getLocalHost().getHostAddress(); - CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = publicAddress; - ccConfig.clientNetPort = CLIENT_NET_PORT; - ccConfig.clusterNetIpAddress = publicAddress; - ccConfig.clusterNetPort = CLUSTER_NET_PORT; - ccConfig.profileDumpPeriod = PROFILE_DUMP_PERIOD; - return ccConfig; - } + private static VXQueryConfig loadConfiguration(XTestOptions opts) { + VXQueryConfig vxqConfig = new VXQueryConfig(); - public static NCConfig createNCConfig() throws UnknownHostException { - String publicAddress = InetAddress.getLocalHost().getHostAddress(); - NCConfig ncConfig1 = new NCConfig(); - ncConfig1.ccHost = CC_HOST; - ncConfig1.ccPort = CLUSTER_NET_PORT; - ncConfig1.clusterNetIPAddress = publicAddress; - ncConfig1.dataIPAddress = publicAddress; - ncConfig1.resultIPAddress = publicAddress; - ncConfig1.nodeId = NODE_ID; - ncConfig1.ioDevices = IO_DEVICES; - return ncConfig1; + vxqConfig.setAvailableProcessors(opts.threads); + vxqConfig.setFrameSize(opts.frameSize); + vxqConfig.setHdfsConf(opts.hdfsConf); + + return vxqConfig; } - public static ClusterControllerService startCC(XTestOptions opts) throws IOException { - CCConfig ccConfig = createCCConfig(); - File outDir = new File("target/ClusterController"); - outDir.mkdirs(); - File ccRoot = File.createTempFile(TestRunner.class.getName(), ".data", outDir); - ccRoot.delete(); - ccRoot.mkdir(); - ccConfig.ccRoot = ccRoot.getAbsolutePath(); + public static void startCluster(XTestOptions opts, LocalClusterUtil localClusterUtil) throws IOException { try { - ClusterControllerService cc = new ClusterControllerService(ccConfig); - cc.start(); - hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); - hds = new HyracksDataset(hcc, opts.frameSize, opts.threads); - return cc; + VXQueryConfig config = loadConfiguration(opts); + localClusterUtil.init(config); + hcc = (HyracksConnection) localClusterUtil.getConnection(); + hds = (HyracksDataset) localClusterUtil.getDataset(); } catch (Exception e) { throw new IOException(e); } - } - public static NodeControllerService startNC() throws IOException { - NCConfig ncConfig = createNCConfig(); + public static void stopCluster(LocalClusterUtil localClusterUtil) throws IOException { try { - NodeControllerService nc = new NodeControllerService(ncConfig); - nc.start(); - return nc; + localClusterUtil.deinit(); } catch (Exception e) { throw new IOException(e); } @@ -107,13 +71,4 @@ public static HyracksDataset getDataset() { return hds; } - public static void stopCluster(ClusterControllerService cc, NodeControllerService nc) throws IOException { - try { - nc.stop(); - cc.stop(); - } catch (Exception e) { - throw new IOException(e); - } - } - } diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java index fa0a90034..8ef942624 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java @@ -14,101 +14,54 @@ */ package org.apache.vxquery.xtest; +import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON; + import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; +import java.net.URI; +import java.net.URISyntaxException; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.EnumSet; -import java.util.List; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.hyracks.api.client.IHyracksClientConnection; -import org.apache.hyracks.api.client.NodeControllerInfo; -import org.apache.hyracks.api.comm.IFrame; -import org.apache.hyracks.api.comm.IFrameTupleAccessor; -import org.apache.hyracks.api.comm.VSizeFrame; -import org.apache.hyracks.api.dataset.DatasetJobRecord; -import org.apache.hyracks.api.dataset.IHyracksDataset; -import org.apache.hyracks.api.dataset.IHyracksDatasetReader; -import org.apache.hyracks.api.dataset.ResultSetId; -import org.apache.hyracks.api.exceptions.HyracksException; -import org.apache.hyracks.api.job.JobFlag; -import org.apache.hyracks.api.job.JobId; -import org.apache.hyracks.api.job.JobSpecification; -import org.apache.hyracks.control.nc.resources.memory.FrameManager; -import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor; -import org.apache.vxquery.compiler.CompilerControlBlock; -import org.apache.vxquery.compiler.algebricks.VXQueryGlobalDataFactory; -import org.apache.vxquery.context.DynamicContext; -import org.apache.vxquery.context.DynamicContextImpl; -import org.apache.vxquery.context.RootStaticContextImpl; -import org.apache.vxquery.context.StaticContextImpl; +import javax.xml.bind.JAXBException; + +import org.apache.commons.io.FileUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.HttpClientUtils; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.vxquery.app.util.RestUtils; import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; -import org.apache.vxquery.result.ResultUtils; -import org.apache.vxquery.xmlquery.query.VXQueryCompilationListener; -import org.apache.vxquery.xmlquery.query.XMLQueryCompiler; +import org.apache.vxquery.rest.request.QueryRequest; +import org.apache.vxquery.rest.response.APIResponse; +import org.apache.vxquery.rest.response.ErrorResponse; +import org.apache.vxquery.rest.response.SyncQueryResponse; +import org.codehaus.jackson.map.ObjectMapper; public class TestRunner { + private static final Pattern EMBEDDED_SYSERROR_PATTERN = Pattern.compile("(\\p{javaUpperCase}{4}\\d{4})"); - private List collectionList; + private XTestOptions opts; - private IHyracksClientConnection hcc; - private IHyracksDataset hds; public TestRunner(XTestOptions opts) throws UnknownHostException { this.opts = opts; - this.collectionList = new ArrayList(); } public void open() throws Exception { - hcc = TestClusterUtil.getConnection(); - hds = TestClusterUtil.getDataset(); - } - - protected static TestConfiguration getIndexConfiguration(TestCase testCase) { - XTestOptions opts = new XTestOptions(); - opts.verbose = false; - opts.threads = 1; - opts.showQuery = true; - opts.showResult = true; - opts.hdfsConf = "src/test/resources/hadoop/conf"; - opts.catalog = StringUtils.join(new String[] { "src", "test", "resources", "VXQueryCatalog.xml" }, - File.separator); - TestConfiguration indexConf = new TestConfiguration(); - indexConf.options = opts; - String baseDir = new File(opts.catalog).getParent(); - try { - String root = new File(baseDir).getCanonicalPath(); - indexConf.testRoot = new File(root + "/./"); - indexConf.resultOffsetPath = new File(root + "/./ExpectedResults/"); - indexConf.sourceFileMap = testCase.getSourceFileMap(); - indexConf.xqueryFileExtension = ".xq"; - indexConf.xqueryxFileExtension = "xqx"; - indexConf.xqueryQueryOffsetPath = new File(root + "/./Queries/XQuery/"); - } catch (IOException e) { - e.printStackTrace(); - } - return indexConf; - } public TestCaseResult run(final TestCase testCase) { TestCaseResult res = new TestCaseResult(testCase); - TestCase testCaseIndex = new TestCase(getIndexConfiguration(testCase)); - testCaseIndex.setFolder("Indexing/Partition-1/"); - testCaseIndex.setName("showIndexes"); - runQuery(testCaseIndex, res); - String[] collections = res.result.split("\n"); - this.collectionList = Arrays.asList(collections); runQueries(testCase, res); return res; } @@ -121,83 +74,32 @@ public void runQuery(TestCase testCase, TestCaseResult res) { long start = System.currentTimeMillis(); try { - try { - if (opts.showQuery) { + String query = FileUtils.readFileToString(testCase.getXQueryFile(), "UTF-8"); - FileInputStream query = new FileInputStream(testCase.getXQueryFile()); - System.err.println("***Query for " + testCase.getXQueryDisplayName() + ": "); - System.err.println(IOUtils.toString(query, "UTF-8")); - query.close(); - } - - VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET, - opts.showOET, opts.showRP); - - Map nodeControllerInfos = null; - if (hcc != null) { - nodeControllerInfos = hcc.getNodeControllerInfos(); - } + if (opts.showQuery) { + System.err.println("***Query for " + testCase.getXQueryDisplayName() + ": "); + System.err.println(query); + } - XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize, - opts.hdfsConf); - Reader in = new InputStreamReader(new FileInputStream(testCase.getXQueryFile()), "UTF-8"); - CompilerControlBlock ccb = new CompilerControlBlock( - new StaticContextImpl(RootStaticContextImpl.INSTANCE), - new ResultSetId(testCase.getXQueryDisplayName().hashCode()), testCase.getSourceFileMap()); - compiler.compile(testCase.getXQueryDisplayName(), in, ccb, opts.optimizationLevel, collectionList); - JobSpecification spec = compiler.getModule().getHyracksJobSpecification(); - in.close(); - - DynamicContext dCtx = new DynamicContextImpl(compiler.getModule().getModuleContext()); - spec.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory())); - - spec.setMaxReattempts(0); - JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME)); - - FrameManager resultDisplayFrameMgr = new FrameManager(spec.getFrameSize()); - IFrame frame = new VSizeFrame(resultDisplayFrameMgr); - IHyracksDatasetReader reader = hds.createReader(jobId, ccb.getResultSetId()); - // TODO(tillw) remove this loop once the IHyracksDatasetReader reliably returns the correct exception - while (reader.getResultStatus() == DatasetJobRecord.Status.RUNNING) { - Thread.sleep(1); - } - IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor(); - res.result = ""; - while (reader.read(frame) > 0) { - res.result += ResultUtils.getStringFromBuffer(frame.getBuffer(), frameTupleAccessor); - frame.getBuffer().clear(); - } - res.result.trim(); - hcc.waitForCompletion(jobId); - } catch (HyracksException e) { - Throwable t = e; - while (t.getCause() != null) { - t = t.getCause(); - } - final String message = t.getMessage(); - if (message != null) { - Matcher m = EMBEDDED_SYSERROR_PATTERN.matcher(message); - if (m.find()) { - String eCode = m.group(1); - throw new SystemException(ErrorCode.valueOf(eCode), e); - } + QueryRequest request = createQueryRequest(opts, query); + APIResponse response = sendQueryRequest(request, testCase.getSourceFileMap()); + if (response instanceof SyncQueryResponse) { + res.result = ((SyncQueryResponse) response).getResults(); + } else { + System.err.println("Error response: Failure when running the query"); + ErrorResponse errorResponse = (ErrorResponse) response; + Matcher m = EMBEDDED_SYSERROR_PATTERN.matcher(errorResponse.getError().getMessage()); + + Exception e = new RuntimeException("Failed to run the query"); + if (m.find()) { + String eCode = m.group(1); + throw new SystemException(ErrorCode.valueOf(eCode), e); + } else { + throw e; } - throw e; } } catch (Throwable e) { - // Check for nested SystemExceptions. - Throwable error = e; - while (error != null) { - if (error instanceof SystemException) { - res.error = error; - break; - } - error = error.getCause(); - } - // Default - if (res.error == null) { - res.error = e; - } + res.error = e; } finally { try { res.compare(); @@ -208,6 +110,7 @@ public void runQuery(TestCase testCase, TestCaseResult res) { long end = System.currentTimeMillis(); res.time = end - start; } + if (opts.showResult) { if (res.result == null) { System.err.println("***Error: "); @@ -218,7 +121,55 @@ public void runQuery(TestCase testCase, TestCaseResult res) { System.err.println(res.result); } } + } + + private static QueryRequest createQueryRequest(XTestOptions opts, String query) { + QueryRequest request = new QueryRequest(query); + request.setCompileOnly(opts.compileOnly); + request.setOptimization(opts.optimizationLevel); + request.setFrameSize(opts.frameSize); + request.setShowAbstractSyntaxTree(opts.showAST); + request.setShowTranslatedExpressionTree(opts.showTET); + request.setShowOptimizedExpressionTree(opts.showOET); + request.setShowRuntimePlan(opts.showRP); + request.setAsync(false); + + return request; + } + + private static APIResponse sendQueryRequest(QueryRequest request, Map sourceFileMap) + throws IOException, URISyntaxException { + + URI uri = RestUtils.buildQueryURI(request, TestClusterUtil.localClusterUtil.getIpAddress(), + TestClusterUtil.localClusterUtil.getRestPort()); + CloseableHttpClient httpClient = HttpClients.custom().build(); + + try { + HttpPost httpRequest = new HttpPost(uri); + httpRequest.setHeader(HttpHeaders.ACCEPT, CONTENT_TYPE_JSON); + + ObjectMapper mapper = new ObjectMapper(); + String fileMap = mapper.writeValueAsString(sourceFileMap); + httpRequest.setEntity(new StringEntity(fileMap, StandardCharsets.UTF_8)); + + try (CloseableHttpResponse httpResponse = httpClient.execute(httpRequest)) { + HttpEntity entity = httpResponse.getEntity(); + String response = RestUtils.readEntity(entity); + if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + return RestUtils.mapEntity(response, SyncQueryResponse.class, CONTENT_TYPE_JSON); + } else { + return RestUtils.mapEntity(response, ErrorResponse.class, CONTENT_TYPE_JSON); + } + } catch (IOException e) { + System.err.println("Error occurred when reading entity: " + e.getMessage()); + } catch (JAXBException e) { + System.err.println("Error occurred when mapping query response: " + e.getMessage()); + } + } finally { + HttpClientUtils.closeQuietly(httpClient); + } + return null; } public void runQueries(TestCase testCase, TestCaseResult res) { diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java index 5aae69106..df7a71db7 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java @@ -16,10 +16,6 @@ */ package org.apache.vxquery.xtest; -import org.apache.hyracks.control.cc.ClusterControllerService; -import org.apache.hyracks.control.nc.NodeControllerService; -import org.mortbay.jetty.Server; - import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -28,6 +24,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.mortbay.jetty.Server; + public class XTest { private XTestOptions opts; private Server server; @@ -36,8 +34,6 @@ public class XTest { private TestRunnerFactory trf; private int count; private int finishCount; - private static NodeControllerService nc; - private static ClusterControllerService cc; XTest(XTestOptions opts) { this.opts = opts; @@ -81,8 +77,7 @@ public void reportResult(TestCaseResult result) { } } }); - cc = TestClusterUtil.startCC(opts); - nc = TestClusterUtil.startNC(); + TestClusterUtil.startCluster(opts, TestClusterUtil.localClusterUtil); trf = new TestRunnerFactory(opts); trf.registerReporters(reporters); TestCaseFactory tcf = new TestCaseFactory(trf, eSvc, opts); @@ -104,7 +99,7 @@ synchronized void waitForCompletion() throws InterruptedException { r.close(); } try { - TestClusterUtil.stopCluster(cc, nc); + TestClusterUtil.stopCluster(TestClusterUtil.localClusterUtil); } catch (IOException e) { e.printStackTrace(); } diff --git a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java index 1e2dcf623..8f77de4a5 100644 --- a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java +++ b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java @@ -22,8 +22,6 @@ import java.io.IOException; import org.apache.commons.io.FileUtils; -import org.apache.hyracks.control.cc.ClusterControllerService; -import org.apache.hyracks.control.nc.NodeControllerService; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -38,8 +36,6 @@ public abstract class AbstractXQueryTest { private TestRunner tr; private static MiniDFS dfs; private final static String TMP = "target/tmp"; - private static NodeControllerService nc; - private static ClusterControllerService cc; protected abstract XTestOptions getTestOptions(); @@ -92,8 +88,7 @@ public void afterTest() throws Exception { @BeforeClass public static void setup() throws IOException { - cc = TestClusterUtil.startCC(getDefaultTestOptions()); - nc = TestClusterUtil.startNC(); + TestClusterUtil.startCluster(getDefaultTestOptions(), TestClusterUtil.localClusterUtil); setupFS(); } @@ -116,7 +111,7 @@ public static void setupFS() throws IOException { @AfterClass public static void shutdown() throws IOException { removeFS(); - TestClusterUtil.stopCluster(cc, nc); + TestClusterUtil.stopCluster(TestClusterUtil.localClusterUtil); } public static void removeFS() throws IOException { diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-1.txt new file mode 100644 index 000000000..b1db9739f --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-1.txt @@ -0,0 +1,3 @@ +33 +32 +31 \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-2.txt new file mode 100644 index 000000000..d93567eff --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-2.txt @@ -0,0 +1,3 @@ +33 +31 +32 \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-3.txt new file mode 100644 index 000000000..2ab87649b --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-3.txt @@ -0,0 +1,3 @@ +32 +33 +31 \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-4.txt new file mode 100644 index 000000000..d1d6bb751 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-4.txt @@ -0,0 +1,3 @@ +32 +31 +33 \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-5.txt new file mode 100644 index 000000000..2044b18be --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-5.txt @@ -0,0 +1,3 @@ +31 +33 +32 \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-1.txt new file mode 100644 index 000000000..b1db9739f --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-1.txt @@ -0,0 +1,3 @@ +33 +32 +31 \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-2.txt new file mode 100644 index 000000000..d93567eff --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-2.txt @@ -0,0 +1,3 @@ +33 +31 +32 \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-3.txt new file mode 100644 index 000000000..2ab87649b --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-3.txt @@ -0,0 +1,3 @@ +32 +33 +31 \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-4.txt new file mode 100644 index 000000000..d1d6bb751 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-4.txt @@ -0,0 +1,3 @@ +32 +31 +33 \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-5.txt new file mode 100644 index 000000000..2044b18be --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-5.txt @@ -0,0 +1,3 @@ +31 +33 +32 \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-1.txt new file mode 100644 index 000000000..ea5a19e96 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-1.txt @@ -0,0 +1,3 @@ +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-2.txt new file mode 100644 index 000000000..46985a9f9 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-2.txt @@ -0,0 +1,3 @@ +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-3.txt new file mode 100644 index 000000000..b97b85081 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-3.txt @@ -0,0 +1,3 @@ +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-4.txt new file mode 100644 index 000000000..0bacd2b4d --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-4.txt @@ -0,0 +1,3 @@ +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-5.txt new file mode 100644 index 000000000..0e79fdcbd --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-5.txt @@ -0,0 +1,3 @@ +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-1.txt new file mode 100644 index 000000000..ea5a19e96 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-1.txt @@ -0,0 +1,3 @@ +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-2.txt new file mode 100644 index 000000000..46985a9f9 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-2.txt @@ -0,0 +1,3 @@ +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-3.txt new file mode 100644 index 000000000..b97b85081 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-3.txt @@ -0,0 +1,3 @@ +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-4.txt new file mode 100644 index 000000000..0bacd2b4d --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-4.txt @@ -0,0 +1,3 @@ +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-5.txt new file mode 100644 index 000000000..0e79fdcbd --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-5.txt @@ -0,0 +1,3 @@ +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-1.txt new file mode 100644 index 000000000..ea5a19e96 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-1.txt @@ -0,0 +1,3 @@ +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-2.txt new file mode 100644 index 000000000..46985a9f9 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-2.txt @@ -0,0 +1,3 @@ +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-3.txt new file mode 100644 index 000000000..b97b85081 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-3.txt @@ -0,0 +1,3 @@ +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-4.txt new file mode 100644 index 000000000..0bacd2b4d --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-4.txt @@ -0,0 +1,3 @@ +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-5.txt new file mode 100644 index 000000000..0e79fdcbd --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-5.txt @@ -0,0 +1,3 @@ +{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75} +{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5} +{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25} diff --git a/vxquery-xtest/src/test/resources/cat/JsonParserQueries.xml b/vxquery-xtest/src/test/resources/cat/JsonParserQueries.xml index 1287225d2..9f07e2b0e 100644 --- a/vxquery-xtest/src/test/resources/cat/JsonParserQueries.xml +++ b/vxquery-xtest/src/test/resources/cat/JsonParserQueries.xml @@ -89,16 +89,31 @@ Parsing a collection of json files. q15_parser.txt + q15_parser-1.txt + q15_parser-2.txt + q15_parser-3.txt + q15_parser-4.txt + q15_parser-5.txt Parsing a collection of json files. - q15_parser.txt + q15_parser.txt + q15_parser-1.txt + q15_parser-2.txt + q15_parser-3.txt + q15_parser-4.txt + q15_parser-5.txt Parsing a collection of json files. - q15_parser.txt + q15_parser.txt + q15_parser-1.txt + q15_parser-2.txt + q15_parser-3.txt + q15_parser-4.txt + q15_parser-5.txt Parsing a collection of json files.