diff --git a/beeline/pom.xml b/beeline/pom.xml index b500c3e56e10..d2667328601c 100644 --- a/beeline/pom.xml +++ b/beeline/pom.xml @@ -81,7 +81,7 @@ jackson-core - jline + org.jline jline @@ -107,6 +107,11 @@ + + org.apache.hadoop + hadoop-hdfs + test + org.apache.thrift libthrift diff --git a/beeline/src/java/org/apache/hive/beeline/AbstractCommandHandler.java b/beeline/src/java/org/apache/hive/beeline/AbstractCommandHandler.java index 7fc3f958f5dc..579df9f79b62 100644 --- a/beeline/src/java/org/apache/hive/beeline/AbstractCommandHandler.java +++ b/beeline/src/java/org/apache/hive/beeline/AbstractCommandHandler.java @@ -26,8 +26,8 @@ import java.util.LinkedList; import java.util.List; -import jline.console.completer.Completer; -import jline.console.completer.NullCompleter; +import org.jline.reader.Completer; +import org.jline.reader.impl.completer.NullCompleter; /** * An abstract implementation of CommandHandler. diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 49b5fad11be7..ef27eb9bb415 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -28,6 +28,7 @@ import java.io.EOFException; import java.io.File; import java.io.FileInputStream; +import java.io.IOError; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -100,6 +101,7 @@ import org.apache.hive.beeline.hs2connection.HS2ConnectionFileUtils; import org.apache.hive.beeline.hs2connection.HiveSiteHS2ConnectionFileParser; import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; +import org.apache.hive.common.util.MatchingStringsCompleter; import org.apache.hive.common.util.ShutdownHookManager; import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.jdbc.HiveConnection; @@ -110,11 +112,16 @@ import com.google.common.annotations.VisibleForTesting; -import jline.console.ConsoleReader; -import jline.console.completer.Completer; -import jline.console.completer.FileNameCompleter; -import jline.console.completer.StringsCompleter; -import jline.console.history.FileHistory; +import org.jline.reader.Completer; +import org.jline.reader.History; +import org.jline.reader.LineReader; +import org.jline.reader.LineReaderBuilder; +import org.jline.reader.impl.LineReaderImpl; +import org.jline.reader.impl.history.DefaultHistory; +import org.jline.terminal.TerminalBuilder; + +import static org.jline.builtins.Completers.FileNameCompleter; + /** * A console SQL shell with command completion. @@ -152,13 +159,13 @@ public class BeeLine implements Closeable { private PrintStream outputStream = new PrintStream(System.out, true); private PrintStream errorStream = new PrintStream(System.err, true); private InputStream inputStream = System.in; - private ConsoleReader consoleReader; + private LineReader lineReader; private List batch = null; private final Reflector reflector = new Reflector(this); private String dbName = null; private String currentDatabase = null; - private FileHistory history; + private History history; // Indicates if this instance of beeline is running in compatibility mode, or beeline mode private boolean isBeeLine = true; @@ -204,7 +211,7 @@ public class BeeLine implements Closeable { new ReflectiveCommandHandler(this, new String[] {"quit", "done", "exit"}, null), new ReflectiveCommandHandler(this, new String[] {"connect", "open"}, - new Completer[] {new StringsCompleter(getConnectionURLExamples())}), + new Completer[] {new MatchingStringsCompleter(getConnectionURLExamples())}), new ReflectiveCommandHandler(this, new String[] {"describe"}, new Completer[] {new TableNameCompletor(this)}), new ReflectiveCommandHandler(this, new String[] {"indexes"}, @@ -233,7 +240,7 @@ public class BeeLine implements Closeable { null), new ReflectiveCommandHandler(this, new String[] {"metadata"}, new Completer[] { - new StringsCompleter(getMetadataMethodNames())}), + new MatchingStringsCompleter(getMetadataMethodNames())}), new ReflectiveCommandHandler(this, new String[] {"nativesql"}, null), new ReflectiveCommandHandler(this, new String[] {"dbinfo"}, @@ -263,9 +270,9 @@ public class BeeLine implements Closeable { new ReflectiveCommandHandler(this, new String[] {"closeall"}, null), new ReflectiveCommandHandler(this, new String[] {"isolation"}, - new Completer[] {new StringsCompleter(getIsolationLevels())}), + new Completer[] {new MatchingStringsCompleter(getIsolationLevels())}), new ReflectiveCommandHandler(this, new String[] {"outputformat"}, - new Completer[] {new StringsCompleter( + new Completer[] {new MatchingStringsCompleter( formats.keySet().toArray(new String[0]))}), new ReflectiveCommandHandler(this, new String[] {"autocommit"}, null), @@ -309,9 +316,9 @@ public class BeeLine implements Closeable { static { try { - Class.forName("jline.console.ConsoleReader"); + Class.forName("org.jline.reader.LineReader"); } catch (Throwable t) { - throw new ExceptionInInitializerError("jline-missing"); + throw new ExceptionInInitializerError("jline3-missing"); } } @@ -400,7 +407,7 @@ public class BeeLine implements Closeable { .withLongOpt("help") .withDescription("Display this message") .create('h')); - + // -getUrlsFromBeelineSite options.addOption(OptionBuilder .withLongOpt("getUrlsFromBeelineSite") @@ -569,19 +576,15 @@ public BeeLine() { public BeeLine(boolean isBeeLine) { this.isBeeLine = isBeeLine; this.signalHandler = new SunSignalHandler(this); - this.shutdownHook = new Runnable() { - @Override - public void run() { - try { - if (history != null) { - history.setMaxSize(getOpts().getMaxHistoryRows()); - history.flush(); - } - } catch (IOException e) { - error(e); - } finally { - close(); + this.shutdownHook = () -> { + try { + if (history != null) { + history.save(); } + } catch (IOException e) { + error(e); + } finally { + close(); } }; } @@ -863,7 +866,7 @@ private boolean connectUsingArgs(BeelineParser beelineParser, CommandLine cl) { getOpts().setHelpAsked(true); return true; } - + if (cl.hasOption("getUrlsFromBeelineSite")) { printBeelineSiteUrls(); getOpts().setBeelineSiteUrlsAsked(true); @@ -937,8 +940,8 @@ private boolean connectUsingArgs(BeelineParser beelineParser, CommandLine cl) { String propertyFile = cl.getOptionValue("property-file"); if (propertyFile != null) { try { - this.consoleReader = new ConsoleReader(); - } catch (IOException e) { + this.lineReader = LineReaderBuilder.builder().build(); + } catch (IOError e) { handleException(e); } if (!dispatch("!properties " + propertyFile)) { @@ -980,7 +983,7 @@ private void printBeelineSiteUrls() { } } } - + private boolean isZkBasedUrl(String urlFromBeelineSite) { String zkJdbcUriParam = ("serviceDiscoveryMode=zooKeeper").toLowerCase(); if (urlFromBeelineSite.toLowerCase().contains(zkJdbcUriParam)) { @@ -1116,9 +1119,9 @@ public int begin(String[] args, InputStream inputStream, boolean keepHistory) th //add shutdown hook to cleanup the beeline for smooth exit addBeelineShutdownHook(); - //this method also initializes the consoleReader which is + //this method also initializes the lineReader which is //needed by initArgs for certain execution paths - ConsoleReader reader = initializeConsoleReader(inputStream); + initializeLineReader(inputStream); if (isBeeLine) { int code = initArgs(args); if (code != 0) { @@ -1146,7 +1149,7 @@ public int begin(String[] args, InputStream inputStream, boolean keepHistory) th } catch (Exception e) { // ignore } - return execute(reader, false); + return execute(lineReader, false); } /* @@ -1350,7 +1353,7 @@ private int executeFile(String fileName) { } fileStream = fs.open(path); } - return execute(initializeConsoleReader(fileStream), !getOpts().getForce()); + return execute(initializeLineReader(fileStream), !getOpts().getForce()); } catch (Throwable t) { handleException(t); return ERRNO_OTHER; @@ -1359,10 +1362,10 @@ private int executeFile(String fileName) { } } - private int execute(ConsoleReader reader, boolean exitOnError) { + private int execute(LineReader reader, boolean exitOnError) { int lastExecutionResult = ERRNO_OK; Character mask = (System.getProperty("jline.terminal", "").equals("jline.UnsupportedTerminal")) ? null - : ConsoleReader.NULL_MASK; + : LineReaderImpl.NULL_MASK; while (!exit) { try { @@ -1403,7 +1406,7 @@ private void setupHistory() throws IOException { return; } - this.history = new FileHistory(new File(getOpts().getHistoryFile())); + this.history = new DefaultHistory(); } private void addBeelineShutdownHook() throws IOException { @@ -1411,40 +1414,50 @@ private void addBeelineShutdownHook() throws IOException { ShutdownHookManager.addShutdownHook(getShutdownHook()); } - public ConsoleReader initializeConsoleReader(InputStream inputStream) throws IOException { + public LineReader initializeLineReader(InputStream inputStream) throws IOException { + final LineReaderBuilder builder = LineReaderBuilder.builder(); if (inputStream != null) { // ### NOTE: fix for sf.net bug 879425. // Working around an issue in jline-2.1.2, see https://github.com/jline/jline/issues/10 // by appending a newline to the end of inputstream InputStream inputStreamAppendedNewline = new SequenceInputStream(inputStream, new ByteArrayInputStream((new String("\n")).getBytes())); - consoleReader = new ConsoleReader(inputStreamAppendedNewline, getErrorStream()); - consoleReader.setCopyPasteDetection(true); // jline will detect if is regular character + builder.terminal(TerminalBuilder.builder() + .streams(inputStreamAppendedNewline, getErrorStream()).build()); } else { - consoleReader = new ConsoleReader(getInputStream(), getErrorStream()); + builder.terminal(TerminalBuilder.builder() + .streams(getInputStream(), getErrorStream()).build()); } - //disable the expandEvents for the purpose of backward compatibility - consoleReader.setExpandEvents(false); + if (inputStream instanceof FileInputStream || inputStream instanceof FSDataInputStream) { + // from script.. no need to load history and no need of completer, either + lineReader = builder.build(); + return lineReader; + } try { // now set the output for the history if (this.history != null) { - consoleReader.setHistory(this.history); - } else { - consoleReader.setHistoryEnabled(false); + builder.history(this.history); + builder.variable(LineReader.HISTORY_FILE, new File(getOpts().getHistoryFile())); + builder.variable(LineReader.HISTORY_FILE_SIZE, getOpts().getMaxHistoryRows()); + // in-memory keep more data, but at least 500 entries + builder.variable(LineReader.HISTORY_SIZE, Math.max(500, 3 * getOpts().getMaxHistoryRows())); } } catch (Exception e) { handleException(e); } - if (inputStream instanceof FileInputStream || inputStream instanceof FSDataInputStream) { - // from script.. no need to load history and no need of completer, either - return consoleReader; + builder.completer(new BeeLineCompleter(this)); + lineReader = builder.build(); + lineReader.unsetOpt(LineReader.Option.HISTORY_TIMESTAMPED); + + if (this.history != null) { + this.history.attach(lineReader); } - consoleReader.addCompleter(new BeeLineCompleter(this)); - return consoleReader; + return lineReader; + } void usage() { @@ -1495,13 +1508,12 @@ boolean dispatch(String line) { } line = HiveStringUtils.removeComments(line); + line = line.trim(); - if (line.trim().length() == 0) { + if (line.length() == 0) { return true; } - line = line.trim(); - // save it to the current script, if any if (scriptOutputFile != null) { scriptOutputFile.addLine(line); @@ -2497,12 +2509,12 @@ InputStream getInputStream() { return inputStream; } - ConsoleReader getConsoleReader() { - return consoleReader; + LineReader getLineReader() { + return lineReader; } - void setConsoleReader(ConsoleReader reader) { - this.consoleReader = reader; + void setLineReader(LineReader reader) { + this.lineReader = reader; } List getBatch() { diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineCommandCompleter.java b/beeline/src/java/org/apache/hive/beeline/BeeLineCommandCompleter.java index 44bfc9fa61b3..8583b998af7a 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLineCommandCompleter.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLineCommandCompleter.java @@ -21,10 +21,10 @@ import java.util.LinkedList; import java.util.List; -import jline.console.completer.AggregateCompleter; -import jline.console.completer.Completer; -import jline.console.completer.NullCompleter; -import jline.console.completer.StringsCompleter; +import org.apache.hive.common.util.MatchingStringsCompleter; +import org.jline.reader.Completer; +import org.jline.reader.impl.completer.AggregateCompleter; +import org.jline.reader.impl.completer.NullCompleter; class BeeLineCommandCompleter extends AggregateCompleter { public BeeLineCommandCompleter(Iterable handlers) { @@ -32,21 +32,21 @@ public BeeLineCommandCompleter(Iterable handlers) { } public static List getCompleters(Iterable handlers){ - List completers = new LinkedList(); + List completers = new LinkedList<>(); for (CommandHandler handler : handlers) { String[] commandNames = handler.getNames(); if (commandNames != null) { for (String commandName : commandNames) { - List compl = new LinkedList(); - compl.add(new StringsCompleter(BeeLine.COMMAND_PREFIX + commandName)); + List compl = new LinkedList<>(); + compl.add(new MatchingStringsCompleter(BeeLine.COMMAND_PREFIX + commandName)); compl.addAll(Arrays.asList(handler.getParameterCompleters())); compl.add(new NullCompleter()); // last param no complete - completers.add(new AggregateCompleter(compl.toArray(new Completer[compl.size()]))); + completers.add(new AggregateCompleter(compl.toArray(new Completer[0]))); } } } return completers; } -} \ No newline at end of file +} diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineCompleter.java b/beeline/src/java/org/apache/hive/beeline/BeeLineCompleter.java index 9213bcf76781..0f1e7971836e 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLineCompleter.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLineCompleter.java @@ -24,7 +24,10 @@ import java.util.List; -import jline.console.completer.Completer; +import org.jline.reader.Candidate; +import org.jline.reader.Completer; +import org.jline.reader.LineReader; +import org.jline.reader.ParsedLine; /** * Completor for BeeLine. It dispatches to sub-completors based on the @@ -42,17 +45,15 @@ class BeeLineCompleter implements Completer { } @Override - public int complete(String buf, int pos, List cand) { - if (buf != null && buf.startsWith(BeeLine.COMMAND_PREFIX) - && !buf.startsWith(BeeLine.COMMAND_PREFIX + "all") - && !buf.startsWith(BeeLine.COMMAND_PREFIX + "sql")) { - return beeLine.getCommandCompletor().complete(buf, pos, cand); + public void complete(LineReader reader, ParsedLine line, List candidates) { + if (line != null && line.line().startsWith(BeeLine.COMMAND_PREFIX) + && !line.line().startsWith(BeeLine.COMMAND_PREFIX + "all") + && !line.line().startsWith(BeeLine.COMMAND_PREFIX + "sql")) { + beeLine.getCommandCompletor().complete(reader, line, candidates); } else { if (beeLine.getDatabaseConnection() != null && beeLine.getDatabaseConnection().getSQLCompleter() != null) { - return beeLine.getDatabaseConnection().getSQLCompleter().complete(buf, pos, cand); - } else { - return -1; + beeLine.getDatabaseConnection().getSQLCompleter().complete(reader, line, candidates); } } } -} \ No newline at end of file +} diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index 17af4e2cd714..9808305aff32 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -22,6 +22,15 @@ */ package org.apache.hive.beeline; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.common.util.MatchingStringsCompleter; +import org.jline.reader.Candidate; +import org.jline.reader.Completer; +import org.jline.reader.LineReader; +import org.jline.reader.ParsedLine; +import org.jline.terminal.Terminal; +import org.jline.terminal.TerminalBuilder; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -41,13 +50,6 @@ import java.util.Set; import java.util.TreeSet; -import jline.Terminal; -import jline.TerminalFactory; -import jline.console.completer.Completer; -import jline.console.completer.StringsCompleter; -import jline.console.history.MemoryHistory; -import org.apache.hadoop.hive.conf.HiveConf; - public class BeeLineOpts implements Completer { public static final int DEFAULT_MAX_WIDTH = 80; public static final int DEFAULT_MAX_HEIGHT = 80; @@ -86,7 +88,7 @@ public class BeeLineOpts implements Completer { private boolean showElapsedTime = true; private boolean entireLineAsCommand = false; private String numberFormat = "default"; - private final Terminal terminal = TerminalFactory.get(); + private Terminal terminal; private int maxWidth = DEFAULT_MAX_WIDTH; private int maxHeight = DEFAULT_MAX_HEIGHT; private int maxColumnWidth = DEFAULT_MAX_COLUMN_WIDTH; @@ -106,7 +108,7 @@ public class BeeLineOpts implements Completer { private final File rcFile = new File(saveDir(), "beeline.properties"); private String historyFile = new File(saveDir(), "history").getAbsolutePath(); - private int maxHistoryRows = MemoryHistory.DEFAULT_MAX_SIZE; + private int maxHistoryRows = 500; // as in MemoryHistory of JLine 2 private String scriptFile = null; private String[] initFiles = null; @@ -152,6 +154,11 @@ public String get(String envVar) { public BeeLineOpts(BeeLine beeLine, Properties props) { this.beeLine = beeLine; + try { + terminal = TerminalBuilder.terminal(); + } catch(IOException e) { + // nothing to do + } if (terminal.getWidth() > 0) { maxWidth = terminal.getWidth(); } @@ -195,12 +202,11 @@ public File saveDir() { @Override - public int complete(String buf, int pos, List cand) { + public void complete(LineReader reader, ParsedLine line, List candidates) { try { - return new StringsCompleter(propertyNames()).complete(buf, pos, cand); + new MatchingStringsCompleter(propertyNames()).complete(reader, line, candidates); } catch (Exception e) { beeLine.handleException(e); - return -1; } } @@ -742,4 +748,3 @@ public static void setEnv(Env envToUse){ env = envToUse; } } - diff --git a/beeline/src/java/org/apache/hive/beeline/BooleanCompleter.java b/beeline/src/java/org/apache/hive/beeline/BooleanCompleter.java index 6ab007dd05e0..d2976695d1a4 100644 --- a/beeline/src/java/org/apache/hive/beeline/BooleanCompleter.java +++ b/beeline/src/java/org/apache/hive/beeline/BooleanCompleter.java @@ -17,15 +17,14 @@ */ package org.apache.hive.beeline; -import jline.console.completer.StringsCompleter; +import org.apache.hive.common.util.MatchingStringsCompleter; /** * JLine completor boolean value (true/false) */ -class BooleanCompleter extends StringsCompleter { +class BooleanCompleter extends MatchingStringsCompleter { public BooleanCompleter(){ - super(new String[] {"true", "false"}); + super("true", "false"); } - -} \ No newline at end of file +} diff --git a/beeline/src/java/org/apache/hive/beeline/CommandHandler.java b/beeline/src/java/org/apache/hive/beeline/CommandHandler.java index 18fcfc40b228..7751cfe04349 100644 --- a/beeline/src/java/org/apache/hive/beeline/CommandHandler.java +++ b/beeline/src/java/org/apache/hive/beeline/CommandHandler.java @@ -22,7 +22,7 @@ */ package org.apache.hive.beeline; -import jline.console.completer.Completer; +import org.jline.reader.Completer; /** * A generic command to be executed. Execution of the command @@ -80,4 +80,4 @@ interface CommandHandler { * @return */ public Throwable getLastException(); -} \ No newline at end of file +} diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index 42cc87c1bb60..143ed9e687e4 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -48,6 +48,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.ListIterator; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -67,6 +68,8 @@ import org.apache.hive.jdbc.Utils; import org.apache.hive.jdbc.Utils.JdbcConnectionParams; import org.apache.hive.jdbc.logs.InPlaceUpdateStream; +import org.jline.reader.History; +import org.jline.reader.impl.LineReaderImpl; public class Commands { @@ -186,13 +189,9 @@ public boolean addlocaldriverjar(String line) { } public boolean history(String line) { - Iterator hist = beeLine.getConsoleReader().getHistory().entries(); - String[] tmp; - while(hist.hasNext()){ - tmp = hist.next().toString().split(":", 2); - tmp[0] = Integer.toString(Integer.parseInt(tmp[0]) + 1); - beeLine.output(beeLine.getColorBuffer().pad(tmp[0], 6) - .append(":" + tmp[1])); + for (History.Entry entry : beeLine.getLineReader().getHistory()) { + beeLine.output(beeLine.getColorBuffer().pad(Integer.toString(entry.index() + 1), 6) + .append(": " + entry.line())); } return true; } @@ -291,7 +290,7 @@ public boolean dropall(String line) { return beeLine.error(beeLine.loc("no-current-connection")); } try { - if (!(beeLine.getConsoleReader().readLine(beeLine.loc("really-drop-all")).equals("y"))) { + if (!(beeLine.getLineReader().readLine(beeLine.loc("really-drop-all")).equals("y"))) { return beeLine.error("abort-drop-all"); } @@ -1090,7 +1089,7 @@ private boolean showReport() { public String handleMultiLineCmd(String line) throws IOException { line = HiveStringUtils.removeComments(line); Character mask = (System.getProperty("jline.terminal", "").equals("jline.UnsupportedTerminal")) ? null - : jline.console.ConsoleReader.NULL_MASK; + : LineReaderImpl.NULL_MASK; while (isMultiLine(line) && beeLine.getOpts().isAllowMultiLineCommand()) { StringBuilder prompt = new StringBuilder(beeLine.getPrompt()); @@ -1103,14 +1102,14 @@ public String handleMultiLineCmd(String line) throws IOException { } String extra; //avoid NPE below if for some reason -e argument has multi-line command - if (beeLine.getConsoleReader() == null) { + if (beeLine.getLineReader() == null) { throw new RuntimeException("Console reader not initialized. This could happen when there " + "is a multi-line command using -e option and which requires further reading from console"); } if (beeLine.getOpts().isSilent() && beeLine.getOpts().getScriptFile() != null) { - extra = beeLine.getConsoleReader().readLine(null, mask); + extra = beeLine.getLineReader().readLine(null, mask); } else { - extra = beeLine.getConsoleReader().readLine(prompt.toString()); + extra = beeLine.getLineReader().readLine(prompt.toString()); } if (extra == null) { //it happens when using -f and the line of cmds does not end with ; @@ -1663,11 +1662,11 @@ public boolean connect(Properties props) throws IOException { && !JdbcConnectionParams.AUTH_SSO_BROWSER_MODE.equals(auth)) { String urlForPrompt = url.substring(0, url.contains(";") ? url.indexOf(';') : url.length()); if (username == null) { - username = beeLine.getConsoleReader().readLine("Enter username for " + urlForPrompt + ": "); + username = beeLine.getLineReader().readLine("Enter username for " + urlForPrompt + ": "); } props.setProperty(JdbcConnectionParams.AUTH_USER, username); if (password == null) { - password = beeLine.getConsoleReader().readLine("Enter password for " + urlForPrompt + ": ", + password = beeLine.getLineReader().readLine("Enter password for " + urlForPrompt + ": ", new Character('*')); } props.setProperty(JdbcConnectionParams.AUTH_PASSWD, password); @@ -1963,7 +1962,7 @@ public boolean manual(String line) throws IOException { // silly little pager if (index % (beeLine.getOpts().getMaxHeight() - 1) == 0) { - String ret = beeLine.getConsoleReader().readLine(beeLine.loc("enter-for-more")); + String ret = beeLine.getLineReader().readLine(beeLine.loc("enter-for-more")); if (ret != null && ret.startsWith("q")) { break; } diff --git a/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java b/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java index 129fc2eb9cc9..1fe21fbd3caf 100644 --- a/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java +++ b/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java @@ -39,8 +39,8 @@ import org.apache.hive.jdbc.HiveConnection; -import jline.console.completer.ArgumentCompleter; -import jline.console.completer.Completer; +import org.jline.reader.Completer; +import org.jline.reader.impl.completer.ArgumentCompleter; class DatabaseConnection { private static final String HIVE_VAR_PREFIX = "hivevar:"; @@ -79,24 +79,7 @@ void setCompletions(boolean skipmeta) throws SQLException, IOException { : getDatabaseMetaData().getExtraNameCharacters(); // setup the completer for the database - sqlCompleter = new ArgumentCompleter( - new ArgumentCompleter.AbstractArgumentDelimiter() { - // delimiters for SQL statements are any - // non-letter-or-number characters, except - // underscore and characters that are specified - // by the database to be valid name identifiers. - @Override - public boolean isDelimiterChar(CharSequence buffer, int pos) { - char c = buffer.charAt(pos); - if (Character.isWhitespace(c)) { - return true; - } - return !(Character.isLetterOrDigit(c)) - && c != '_' - && extraNameCharacters.indexOf(c) == -1; - } - }, - new SQLCompleter(SQLCompleter.getSQLCompleters(beeLine, skipmeta))); + sqlCompleter = new ArgumentCompleter(new SQLCompleter(SQLCompleter.getSQLCompleters(beeLine, skipmeta))); // not all argument elements need to hold true ((ArgumentCompleter) sqlCompleter).setStrict(false); } diff --git a/beeline/src/java/org/apache/hive/beeline/ReflectiveCommandHandler.java b/beeline/src/java/org/apache/hive/beeline/ReflectiveCommandHandler.java index a37ee8914200..6fbda4c670dc 100644 --- a/beeline/src/java/org/apache/hive/beeline/ReflectiveCommandHandler.java +++ b/beeline/src/java/org/apache/hive/beeline/ReflectiveCommandHandler.java @@ -22,9 +22,9 @@ */ package org.apache.hive.beeline; -import jline.console.completer.Completer; - import org.apache.hadoop.fs.shell.Command; +import org.jline.reader.Completer; + /** * A {@link Command} implementation that uses reflection to diff --git a/beeline/src/java/org/apache/hive/beeline/SQLCompleter.java b/beeline/src/java/org/apache/hive/beeline/SQLCompleter.java index b40b3a50636a..bd5985ff3017 100644 --- a/beeline/src/java/org/apache/hive/beeline/SQLCompleter.java +++ b/beeline/src/java/org/apache/hive/beeline/SQLCompleter.java @@ -30,11 +30,11 @@ import java.util.StringTokenizer; import java.util.TreeSet; -import jline.console.completer.StringsCompleter; +import org.apache.hive.common.util.MatchingStringsCompleter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class SQLCompleter extends StringsCompleter { +class SQLCompleter extends MatchingStringsCompleter { private static final Logger LOG = LoggerFactory.getLogger(SQLCompleter.class.getName()); diff --git a/beeline/src/java/org/apache/hive/beeline/TableNameCompletor.java b/beeline/src/java/org/apache/hive/beeline/TableNameCompletor.java index 1eefe1788262..23b91f803729 100644 --- a/beeline/src/java/org/apache/hive/beeline/TableNameCompletor.java +++ b/beeline/src/java/org/apache/hive/beeline/TableNameCompletor.java @@ -24,8 +24,11 @@ import java.util.List; -import jline.console.completer.Completer; -import jline.console.completer.StringsCompleter; +import org.apache.hive.common.util.MatchingStringsCompleter; +import org.jline.reader.Candidate; +import org.jline.reader.Completer; +import org.jline.reader.LineReader; +import org.jline.reader.ParsedLine; class TableNameCompletor implements Completer { private final BeeLine beeLine; @@ -38,11 +41,11 @@ class TableNameCompletor implements Completer { } @Override - public int complete(String buf, int pos, List cand) { - if (beeLine.getDatabaseConnection() == null) { - return -1; + public void complete(LineReader reader, ParsedLine line, List candidates) { + final DatabaseConnection connection = beeLine.getDatabaseConnection(); + if (connection != null) { + new MatchingStringsCompleter(beeLine.getDatabaseConnection().getTableNames(true)) + .complete(reader, line, candidates); } - return new StringsCompleter(beeLine.getDatabaseConnection().getTableNames(true)) - .complete(buf, pos, cand); } -} \ No newline at end of file +} diff --git a/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java b/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java index c8f4d4e42e55..58677678d3cf 100644 --- a/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java +++ b/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java @@ -62,7 +62,7 @@ public void testNumHistories() throws Exception { Method method = beeline.getClass().getDeclaredMethod("setupHistory"); method.setAccessible(true); method.invoke(beeline); - beeline.initializeConsoleReader(null); + beeline.initializeLineReader(null); beeline.dispatch("!history"); String output = os.toString("UTF-8"); int numHistories = output.split("\n").length; @@ -80,7 +80,7 @@ public void testHistory() throws Exception { Method method = beeline.getClass().getDeclaredMethod("setupHistory"); method.setAccessible(true); method.invoke(beeline); - beeline.initializeConsoleReader(null); + beeline.initializeLineReader(null); beeline.dispatch("!history"); String output = os.toString("UTF-8"); String[] tmp = output.split("\n"); diff --git a/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java b/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java index 26a892fdcfc6..96397844e734 100644 --- a/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java +++ b/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java @@ -70,7 +70,7 @@ public TestBeelineArgParsing(String connectionString, String driverClazzName, St this.defaultSupported = defaultSupported; } - public class TestBeeline extends BeeLine { + public static class TestBeeline extends BeeLine { String connectArgs = null; List properties = new ArrayList(); diff --git a/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java b/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java index 5ea4d11b7abd..7f236b8fc7d0 100644 --- a/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java +++ b/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java @@ -36,6 +36,7 @@ import java.io.OutputStream; import java.io.PrintStream; import java.net.URISyntaxException; +import java.nio.charset.Charset; public class TestHiveCli { private static final Logger LOG = LoggerFactory.getLogger(TestHiveCli.class.getName()); @@ -245,7 +246,7 @@ private void executeCMD(String[] args, String input, int retCode) { int ret = 0; try { if (input != null) { - inputStream = IOUtils.toInputStream(input); + inputStream = IOUtils.toInputStream(input, Charset.defaultCharset()); } ret = cli.runWithArgs(args, inputStream); } catch (Throwable e) { @@ -291,8 +292,8 @@ public static void init(){ public void setup() throws IOException, URISyntaxException { System.setProperty("datanucleus.schema.autoCreateAll", "true"); cli = new HiveCli(); - initFromFile(); redirectOutputStream(); + initFromFile(); } private void redirectOutputStream() { diff --git a/cli/pom.xml b/cli/pom.xml index e6f5ce07a68c..2765e6f8d221 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -82,7 +82,7 @@ test - jline + org.jline jline @@ -113,6 +113,10 @@ hadoop-mapreduce-client-core true + + org.apache.logging.log4j + log4j-1.2-api + org.apache.hbase diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index 4d57b22d0afa..d16ac528b523 100644 --- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -18,25 +18,7 @@ package org.apache.hadoop.hive.cli; -import static org.apache.hadoop.util.StringUtils.stringifyException; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.PrintStream; -import java.io.UnsupportedEncodingException; -import java.nio.charset.StandardCharsets; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - +import com.google.common.base.Splitter; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -72,24 +54,42 @@ import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.IOUtils; import org.apache.hive.common.util.HiveStringUtils; +import org.apache.hive.common.util.MatchingStringsCompleter; import org.apache.hive.common.util.ShutdownHookManager; +import org.jline.reader.Candidate; +import org.jline.reader.Completer; +import org.jline.reader.History; +import org.jline.reader.LineReader; +import org.jline.reader.LineReaderBuilder; +import org.jline.reader.ParsedLine; +import org.jline.reader.impl.DefaultParser; +import org.jline.reader.impl.completer.ArgumentCompleter; +import org.jline.reader.impl.history.DefaultHistory; +import org.jline.terminal.Terminal; +import org.jline.terminal.Terminal.Signal; +import org.jline.terminal.Terminal.SignalHandler; +import org.jline.terminal.TerminalBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Splitter; - -import jline.console.ConsoleReader; -import jline.console.completer.ArgumentCompleter; -import jline.console.completer.ArgumentCompleter.AbstractArgumentDelimiter; -import jline.console.completer.ArgumentCompleter.ArgumentDelimiter; -import jline.console.completer.Completer; -import jline.console.completer.StringsCompleter; -import jline.console.history.FileHistory; -import jline.console.history.History; -import jline.console.history.PersistentHistory; -import sun.misc.Signal; -import sun.misc.SignalHandler; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import static org.apache.hadoop.util.StringUtils.stringifyException; /** * CliDriver. @@ -105,7 +105,7 @@ public class CliDriver { public static final String HIVERCFILE = ".hiverc"; private final LogHelper console; - protected ConsoleReader reader; + protected LineReader reader; private Configuration conf; public CliDriver() { @@ -369,8 +369,8 @@ public CommandProcessorResponse processLine(String line, boolean allowInterrupti if (allowInterrupting) { // Remember all threads that were running at the time we started line processing. // Hook up the custom Ctrl+C handler while processing this line - interruptSignal = new Signal("INT"); - oldSignal = Signal.handle(interruptSignal, new SignalHandler() { + interruptSignal = Terminal.Signal.INT; + oldSignal = reader.getTerminal().handle(interruptSignal, new SignalHandler() { private boolean interruptRequested; @Override @@ -432,8 +432,8 @@ public void handle(Signal signal) { return lastRet; } finally { // Once we are done processing the line, restore the old handler - if (oldSignal != null && interruptSignal != null) { - Signal.handle(interruptSignal, oldSignal); + if (oldSignal != null) { + reader.getTerminal().handle(interruptSignal, oldSignal); } } } @@ -585,7 +585,7 @@ public void processSelectDatabase(CliSessionState ss) throws IOException, Comman public static Completer[] getCommandCompleter() { // StringsCompleter matches against a pre-defined wordlist // We start with an empty wordlist and build it up - List candidateStrings = new ArrayList(); + List candidateStrings = new ArrayList<>(); // We add Hive function names // For functions that aren't infix operators, we add an open @@ -604,23 +604,11 @@ public static Completer[] getCommandCompleter() { candidateStrings.add(s.toLowerCase()); } - StringsCompleter strCompleter = new StringsCompleter(candidateStrings); - - // Because we use parentheses in addition to whitespace - // as a keyword delimiter, we need to define a new ArgumentDelimiter - // that recognizes parenthesis as a delimiter. - ArgumentDelimiter delim = new AbstractArgumentDelimiter() { - @Override - public boolean isDelimiterChar(CharSequence buffer, int pos) { - char c = buffer.charAt(pos); - return (Character.isWhitespace(c) || c == '(' || c == ')' || - c == '[' || c == ']'); - } - }; + Completer strCompleter = new MatchingStringsCompleter(candidateStrings); // The ArgumentCompletor allows us to match multiple tokens // in the same line. - final ArgumentCompleter argCompleter = new ArgumentCompleter(delim, strCompleter); + final ArgumentCompleter argCompleter = new ArgumentCompleter(strCompleter); // By default ArgumentCompletor is in "strict" mode meaning // a token is only auto-completed if all prior tokens // match. We don't want that since there are valid tokens @@ -632,19 +620,16 @@ public boolean isDelimiterChar(CharSequence buffer, int pos) { // the opening parenthesis is unnecessary (and uncommon) in Hive. // We stack a custom Completor on top of our ArgumentCompletor // to reverse this. - Completer customCompletor = new Completer () { - @Override - public int complete (String buffer, int offset, List completions) { - List comp = completions; - int ret = argCompleter.complete(buffer, offset, completions); - // ConsoleReader will do the substitution if and only if there - // is exactly one valid completion, so we ignore other cases. - if (completions.size() == 1) { - if (comp.get(0).endsWith("( ")) { - comp.set(0, comp.get(0).trim()); - } + Completer customCompletor = (reader, line, candidates) -> { + argCompleter.complete(reader, line, candidates); + candidates.forEach(System.out::println); + // ConsoleReader will do the substitution if and only if there + // is exactly one valid completion, so we ignore other cases. + if (candidates.size() == 1) { + String candidateStr = candidates.get(0).value(); + if (candidateStr.endsWith("( ")) { + candidates.set(0, new Candidate(candidateStr.trim())); } - return ret; } }; @@ -653,61 +638,55 @@ public int complete (String buffer, int offset, List completions) { vars.add(conf.varname); } - StringsCompleter confCompleter = new StringsCompleter(vars) { + Completer confCompleter = new MatchingStringsCompleter(vars) { @Override - public int complete(final String buffer, final int cursor, final List clist) { - int result = super.complete(buffer, cursor, clist); - if (clist.isEmpty() && cursor > 1 && buffer.charAt(cursor - 1) == '=') { - HiveConf.ConfVars var = HiveConf.getConfVars(buffer.substring(0, cursor - 1)); + public void complete(LineReader reader, ParsedLine line, List candidates) { + super.complete(reader, line, candidates); + final int cursor = line.cursor(); + if (candidates.isEmpty() && cursor > 1 && line.word().charAt(cursor - 1) == '=') { + HiveConf.ConfVars var = HiveConf.getConfVars(line.word().substring(0, cursor - 1)); if (var == null) { - return result; + return; } if (var.getValidator() instanceof Validator.StringSet) { Validator.StringSet validator = (Validator.StringSet)var.getValidator(); - clist.addAll(validator.getExpected()); + validator.getExpected().stream().map(Candidate::new).forEach(candidates::add); } else if (var.getValidator() != null) { - clist.addAll(Arrays.asList(var.getValidator().toDescription(), "")); + candidates.add(new Candidate(var.getValidator().toDescription())); } else { - clist.addAll(Arrays.asList("Expects " + var.typeString() + " type value", "")); + candidates.add(new Candidate("Expects " + var.typeString() + " type value")); } - return cursor; + return; } - if (clist.size() > DELIMITED_CANDIDATE_THRESHOLD) { - Set delimited = new LinkedHashSet(); - for (CharSequence candidate : clist) { + if (candidates.size() > DELIMITED_CANDIDATE_THRESHOLD) { + Set delimited = new LinkedHashSet<>(); + for (Candidate candidate : candidates) { Iterator it = Splitter.on(".").split( - candidate.subSequence(cursor, candidate.length())).iterator(); + candidate.value().subSequence(cursor, candidate.value().length())).iterator(); if (it.hasNext()) { String next = it.next(); if (next.isEmpty()) { next = "."; } - candidate = buffer != null ? buffer.substring(0, cursor) + next : next; + candidate = new Candidate(line.line() != null ? line.line().substring(0, cursor) + next : next); } delimited.add(candidate); } - clist.clear(); - clist.addAll(delimited); + candidates.clear(); + candidates.addAll(delimited); } - return result; } }; - StringsCompleter setCompleter = new StringsCompleter("set") { - @Override - public int complete(String buffer, int cursor, List clist) { - return buffer != null && buffer.equals("set") ? super.complete(buffer, cursor, clist) : -1; - } - }; + Completer setCompleter = new MatchingStringsCompleter("set"); ArgumentCompleter propCompleter = new ArgumentCompleter(setCompleter, confCompleter) { @Override - public int complete(String buffer, int offset, List completions) { - int ret = super.complete(buffer, offset, completions); - if (completions.size() == 1) { - completions.set(0, ((String)completions.get(0)).trim()); + public void complete(LineReader reader, ParsedLine line, List candidates) { + super.complete(reader, line, candidates); + if (candidates.size() == 1) { + candidates.set(0, new Candidate(candidates.get(0).value().trim())); } - return ret; } }; return new Completer[] {propCompleter, customCompletor}; @@ -846,7 +825,7 @@ private CommandProcessorResponse executeDriver(CliSessionState ss, HiveConf conf console.printInfo(HiveConf.generateMrDeprecationWarning()); } - setupConsoleReader(); + setupLineReader(); String line; CommandProcessorResponse response = new CommandProcessorResponse(); @@ -879,15 +858,12 @@ private CommandProcessorResponse executeDriver(CliSessionState ss, HiveConf conf return response; } - private void setupCmdHistory() { + private String setupCmdHistory() { final String HISTORYFILE = ".hivehistory"; String historyDirectory = System.getProperty("user.home"); - PersistentHistory history = null; try { if ((new File(historyDirectory)).exists()) { - String historyFile = historyDirectory + File.separator + HISTORYFILE; - history = new FileHistory(new File(historyFile)); - reader.setHistory(history); + return historyDirectory + File.separator + HISTORYFILE; } else { System.err.println("WARNING: Directory for Hive history file: " + historyDirectory + " does not exist. History will not be available during this session."); @@ -896,32 +872,48 @@ private void setupCmdHistory() { System.err.println("WARNING: Encountered an error while trying to initialize Hive's " + "history file. History will not be available during this session."); System.err.println(e.getMessage()); + return null; } // add shutdown hook to flush the history to history file - ShutdownHookManager.addShutdownHook(new Runnable() { - @Override - public void run() { - History h = reader.getHistory(); - if (h instanceof FileHistory) { - try { - ((FileHistory) h).flush(); - } catch (IOException e) { - System.err.println("WARNING: Failed to write command history file: " + e.getMessage()); - } - } + ShutdownHookManager.addShutdownHook(() -> { + History h = reader.getHistory(); + try { + h.save(); + } catch (IOException e) { + System.err.println("WARNING: Failed to write command history file: " + e.getMessage()); } }); + return null; } - protected void setupConsoleReader() throws IOException { - reader = new ConsoleReader(); - reader.setExpandEvents(false); - reader.setBellEnabled(false); - for (Completer completer : getCommandCompleter()) { - reader.addCompleter(completer); + protected void setupLineReader() throws IOException { + LineReaderBuilder builder = LineReaderBuilder.builder(); + builder.variable(LineReader.BELL_STYLE, "audible"); + Arrays.stream(getCommandCompleter()).forEach(builder::completer); + builder.terminal(TerminalBuilder.terminal()); + builder.parser(getDefaultParser()); + builder.history(new DefaultHistory()); + + String historyFile = setupCmdHistory(); + if (historyFile != null) { + builder.variable(LineReader.HISTORY_FILE, historyFile); } - setupCmdHistory(); + reader = builder.build(); + } + + static DefaultParser getDefaultParser() { + return new DefaultParser() { + @Override + public boolean isDelimiterChar(CharSequence buffer, int pos) { + // Because we use parentheses in addition to whitespace + // as a keyword delimiter, we need to define a new ArgumentDelimiter + // that recognizes parenthesis as a delimiter. + final char c = buffer.charAt(pos); + return (Character.isWhitespace(c) || c == '(' || c == ')' || + c == '[' || c == ']'); + } + }; } /** @@ -961,5 +953,4 @@ private static String spacesForString(String s) { public void setHiveVariables(Map hiveVariables) { SessionState.get().setHiveVariables(hiveVariables); } - } diff --git a/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java b/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java index 37448fee2138..b59bc034c8e6 100644 --- a/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java +++ b/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java @@ -29,7 +29,6 @@ import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.io.PrintStream; import java.lang.reflect.Field; @@ -40,11 +39,6 @@ import java.util.List; import java.util.Map; -import jline.console.ConsoleReader; -import jline.console.completer.ArgumentCompleter; -import jline.console.completer.Completer; - - import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.io.SessionStream; @@ -56,6 +50,9 @@ import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.processors.CommandProcessorException; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.jline.reader.Candidate; +import org.jline.reader.Completer; +import org.jline.reader.impl.completer.ArgumentCompleter; import org.junit.Test; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNotNull; @@ -142,7 +139,7 @@ public void testThatCliDriverDoesNotStripComments() throws Exception { CliDriver cliDriver = new CliDriver(); // issue a command with bad options cliDriver.processCmd("!ls --abcdefghijklmnopqrstuvwxyz123456789"); - assertTrue("Comments with '--; should not have been stripped, so command should fail", false); + fail("Comments with '--; should not have been stripped, so command should fail"); } catch (CommandProcessorException e) { // this is expected to happen } finally { @@ -165,8 +162,6 @@ public void testThatCliDriverDoesNotStripComments() throws Exception { * Schema to throw against test * @return Output that would have been sent to the user * @throws CommandProcessorException - * @throws CommandNeedRetryException - * won't actually be thrown */ private PrintStream headerPrintingTestDriver(Schema mockSchema) throws CommandProcessorException { CliDriver cliDriver = new CliDriver(); @@ -203,21 +198,25 @@ public void testGetCommandCompletor() { Completer[] completors = CliDriver.getCommandCompleter(); assertEquals(2, completors.length); assertTrue(completors[0] instanceof ArgumentCompleter); - assertTrue(completors[1] instanceof Completer); + assertNotNull(completors[1]); - List testList = Arrays.asList(")"); - completors[1].complete("fdsdfsdf", 0, testList); - assertEquals(")", testList.get(0)); - testList=new ArrayList(); - completors[1].complete("len", 0, testList); - assertTrue(testList.get(0).toString().endsWith("length(")); + final List candidates1 = new ArrayList<>(); + candidates1.add(new Candidate(")")); + completors[1].complete(null, CliDriver.getDefaultParser().parse("fdsdfsdf", 0), candidates1); + assertEquals(")", candidates1.get(0).value()); - testList=new ArrayList(); - completors[0].complete("set f", 0, testList); - assertEquals("set", testList.get(0)); + final List candidates2 = new ArrayList<>(); + completors[1].complete(null, CliDriver.getDefaultParser().parse("length", 0), candidates2); + System.out.printf("--- --> %s%n", candidates2.get(0).value()); + assertTrue(candidates2.get(0).value().endsWith("length(")); + final List candidates3 = new ArrayList<>(); + completors[0].complete(null, CliDriver.getDefaultParser().parse("set f", 0), candidates3); + assertEquals("set", candidates3.get(0).value()); } + + @Test public void testRun() throws Exception { // clean history @@ -417,59 +416,9 @@ private static void setEnv(String key, String value) throws Exception { } private static class FakeCliDriver extends CliDriver { - - @Override - protected void setupConsoleReader() throws IOException { - reader = new FakeConsoleReader(); - } - - } - - private static class FakeConsoleReader extends ConsoleReader { - private int counter = 0; - File temp = null; - - public FakeConsoleReader() throws IOException { - super(); - - } - @Override - public String readLine(String prompt) throws IOException { - FileWriter writer; - switch (counter++) { - case 0: - return "!echo test message;"; - case 1: - temp = File.createTempFile("hive", "test"); - temp.deleteOnExit(); - return "source " + temp.getAbsolutePath() + ";"; - case 2: - temp = File.createTempFile("hive", "test"); - temp.deleteOnExit(); - writer = new FileWriter(temp); - writer.write("bla bla bla"); - writer.close(); - return "list file file://" + temp.getAbsolutePath() + ";"; - case 3: - return "!echo "; - case 4: - return "test message;"; - case 5: - return "source fakeFile;"; - case 6: - temp = File.createTempFile("hive", "test"); - temp.deleteOnExit(); - writer = new FileWriter(temp); - writer.write("source fakeFile;"); - writer.close(); - return "list file file://" + temp.getAbsolutePath() + ";"; - - - // drop table over10k; - default: - return null; - } + protected void setupLineReader() throws IOException { + reader = null; } } diff --git a/common/pom.xml b/common/pom.xml index c0fdaa70f0b3..e4e4a36d41ad 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -71,7 +71,7 @@ orc-core - jline + org.jline jline @@ -99,6 +99,11 @@ org.eclipse.jetty jetty-webapp + + org.fusesource.jansi + jansi + 1.18 + joda-time joda-time diff --git a/common/src/java/org/apache/hive/common/util/MatchingStringsCompleter.java b/common/src/java/org/apache/hive/common/util/MatchingStringsCompleter.java new file mode 100644 index 000000000000..3aece062b9fb --- /dev/null +++ b/common/src/java/org/apache/hive/common/util/MatchingStringsCompleter.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import org.jline.reader.Candidate; +import org.jline.reader.Completer; +import org.jline.reader.LineReader; +import org.jline.reader.ParsedLine; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; + +/** + * A matching string Completer based on JLine2's StringCompleter + */ +public class MatchingStringsCompleter implements Completer { + protected SortedSet candidateStrings = new TreeSet<>(); + + public MatchingStringsCompleter() { + // empty + } + + public MatchingStringsCompleter(String... strings) { + this(Arrays.asList(strings)); + } + + public MatchingStringsCompleter(Iterable strings) { + strings.forEach(candidateStrings::add); + } + + public MatchingStringsCompleter(Candidate... candidates) { + Arrays.stream(candidates).map(Candidate::value).forEach(candidateStrings::add); + } + + public Collection getStrings() { + return candidateStrings; + } + + @Override + public void complete(LineReader reader, ParsedLine line, List candidates) { + assert candidates != null; + + if (line == null) { + candidateStrings.stream().map(Candidate::new).forEach(candidates::add); + } else { + for (String match : this.candidateStrings.tailSet(line.word())) { + if (!match.startsWith(line.word())) { + break; + } + candidates.add(new Candidate(match)); + } + } + } +} diff --git a/hcatalog/hcatalog-pig-adapter/pom.xml b/hcatalog/hcatalog-pig-adapter/pom.xml index fce976308285..3563083f91c3 100644 --- a/hcatalog/hcatalog-pig-adapter/pom.xml +++ b/hcatalog/hcatalog-pig-adapter/pom.xml @@ -30,6 +30,7 @@ Hive HCatalog Pig Adapter ../.. + 3.21.0 @@ -107,9 +108,9 @@ test - jline + org.jline jline - 0.9.94 + ${jline.version} test diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceCommandLine.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceCommandLine.java index af5874db2985..aca398ed89e5 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceCommandLine.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceCommandLine.java @@ -21,8 +21,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; -import jline.TerminalFactory; - import java.util.Arrays; import java.util.Properties; import java.util.Set; @@ -36,6 +34,7 @@ import org.apache.commons.cli.ParseException; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.log.LogHelpers; +import org.jline.terminal.TerminalBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; @@ -401,7 +400,7 @@ private static void printUsage() { HelpFormatter hf = new HelpFormatter(); try { int width = hf.getWidth(); - int jlineWidth = TerminalFactory.get().getWidth(); + int jlineWidth = TerminalBuilder.terminal().getWidth(); width = Math.min(160, Math.max(jlineWidth, width)); hf.setWidth(width); } catch (Throwable t) { // Ignore diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/status/LlapStatusServiceCommandLine.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/status/LlapStatusServiceCommandLine.java index bee50791a3a6..25f10b4a8fd0 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/status/LlapStatusServiceCommandLine.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/status/LlapStatusServiceCommandLine.java @@ -21,8 +21,6 @@ import java.util.Arrays; import java.util.Properties; -import jline.TerminalFactory; - import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; @@ -30,6 +28,7 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.jline.terminal.TerminalBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -238,7 +237,7 @@ private static void printUsage() { HelpFormatter hf = new HelpFormatter(); try { int width = hf.getWidth(); - int jlineWidth = TerminalFactory.get().getWidth(); + int jlineWidth = TerminalBuilder.terminal().getWidth(); width = Math.min(160, Math.max(jlineWidth, width)); hf.setWidth(width); } catch (Throwable t) { // Ignore diff --git a/pom.xml b/pom.xml index 978a6e6a05ff..5b68e956d978 100644 --- a/pom.xml +++ b/pom.xml @@ -152,7 +152,7 @@ 1.1 9.3.27.v20190418 1.19 - 2.14.6 + 3.21.0 2.0.2 2.9.9 6.0.0 @@ -430,7 +430,7 @@ ${javolution.version} - jline + org.jline jline ${jline.version} diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml index 8ef5116694b0..7f2fc294ff7f 100644 --- a/standalone-metastore/metastore-server/pom.xml +++ b/standalone-metastore/metastore-server/pom.xml @@ -241,7 +241,7 @@ sqlline - jline + org.jline jline diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaToolCommandLine.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaToolCommandLine.java index ce43a8c4f57a..a2dc5a5d7409 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaToolCommandLine.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaToolCommandLine.java @@ -27,10 +27,10 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.jline.terminal.TerminalBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jline.TerminalFactory; class HiveMetaToolCommandLine { private static final Logger LOGGER = LoggerFactory.getLogger(HiveMetaToolCommandLine.class.getName()); @@ -199,7 +199,7 @@ private static void printUsage() { HelpFormatter hf = new HelpFormatter(); try { int width = hf.getWidth(); - int jlineWidth = TerminalFactory.get().getWidth(); + int jlineWidth = TerminalBuilder.terminal().getWidth(); width = Math.min(160, Math.max(jlineWidth, width)); hf.setWidth(width); } catch (Throwable t) { // Ignore diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 1599907c63c7..b7f90dad8591 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -91,8 +91,8 @@ com.google.protobuf 2.6.1 2.6.1 - 1.9.0 - 2.14.6 + 1.12.0 + 3.21.0 4.0.4 4.0.0-alpha-2-SNAPSHOT 1.9.4 @@ -297,7 +297,7 @@ ${sqlline.version} - jline + org.jline jline ${jline.version}