TAJO-2091: Error or progress update should use stderr instead of stdout.

tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java

@@ -64,6 +64,7 @@ public class TajoCli implements Closeable {
   private final ConsoleReader reader;
   private final InputStream sin;
   private final PrintWriter sout;
+  private final PrintWriter serr;
   private TajoFileHistory history;
 
   private final boolean reconnect;  // reconnect on invalid session
@@ -132,6 +133,10 @@ public PrintWriter getOutput() {
       return sout;
     }
 
+    public PrintWriter getError() {
+      return serr;
+    }
+
     public TajoConf getConf() {
       return conf;
     }
@@ -177,7 +182,7 @@ public Map<String, TajoShellCommand> getCommands() {
     }
   }
 
-  public TajoCli(TajoConf c, String [] args, @Nullable Properties clientParams, InputStream in, OutputStream out)
+  public TajoCli(TajoConf c, String [] args, @Nullable Properties clientParams, InputStream in, OutputStream out, OutputStream err)
       throws Exception {
 
     CommandLineParser parser = new PosixParser();
@@ -194,6 +199,7 @@ public TajoCli(TajoConf c, String [] args, @Nullable Properties clientParams, In
 
     this.reader.setExpandEvents(false);
     this.sout = new PrintWriter(reader.getOutput());
+    this.serr = new PrintWriter(new OutputStreamWriter(err, "UTF-8"));
     initFormatter();
 
     if (cmd.hasOption("help")) {
@@ -264,6 +270,7 @@ public TajoCli(TajoConf c, String [] args, @Nullable Properties clientParams, In
         displayFormatter.setScriptMode();
         int exitCode = executeScript(cmd.getOptionValue("c"));
         sout.flush();
+        serr.flush();
         System.exit(exitCode);
       }
       if (cmd.hasOption("f")) {
@@ -275,6 +282,7 @@ public TajoCli(TajoConf c, String [] args, @Nullable Properties clientParams, In
           script = replaceParam(script, cmd.getOptionValues("param"));
           int exitCode = executeScript(script);
           sout.flush();
+          serr.flush();
           System.exit(exitCode);
         } else {
           System.err.println(ERROR_PREFIX + "No such a file \"" + cmd.getOptionValue("f") + "\"");
@@ -493,6 +501,7 @@ public int executeMetaCommand(String line) {
         onError(t);
         return -1;
       } finally {
+        context.getError().flush();
         context.getOutput().flush();
       }
 
@@ -614,11 +623,11 @@ private void waitForQueryCompleted(QueryId queryId) {
         }
 
         if (TajoClientUtil.isQueryRunning(status.getState())) {
-          displayFormatter.printProgress(sout, status);
+          displayFormatter.printProgress(serr, status);
         }
 
         if (TajoClientUtil.isQueryComplete(status.getState()) && status.getState() != QueryState.QUERY_KILL_WAIT) {
-          displayFormatter.printProgress(sout, status);
+          displayFormatter.printProgress(serr, status);
           break;
         } else {
           Thread.sleep(Math.min(200 * progressRetries, 1000));
@@ -627,10 +636,10 @@ private void waitForQueryCompleted(QueryId queryId) {
       }
 
       if (status.getState() == QueryState.QUERY_ERROR || status.getState() == QueryState.QUERY_FAILED) {
-        displayFormatter.printErrorMessage(sout, status);
+        displayFormatter.printErrorMessage(serr, status);
         wasError = true;
       } else if (status.getState() == QueryState.QUERY_KILLED) {
-        displayFormatter.printKilledMessage(sout, queryId);
+        displayFormatter.printKilledMessage(serr, queryId);
         wasError = true;
       } else {
         if (status.getState() == QueryState.QUERY_SUCCEEDED) {
@@ -671,18 +680,18 @@ public int executeScript(String script) throws Exception {
 
   private void printUsage() {
     HelpFormatter formatter = new HelpFormatter();
-    formatter.printHelp("tsql [options] [database]", options);
+    formatter.printUsage(this.serr, 80, "tsql [options] [database]", options);
   }
 
   private void printInvalidCommand(String command) {
-    sout.println("Invalid command " + command + ". Try \\? for help.");
+    serr.println("Invalid command " + command + ". Try \\? for help.");
   }
 
   private void onError(Throwable t) {
     Preconditions.checkNotNull(t);
 
     wasError = true;
-    displayFormatter.printErrorMessage(sout, t.getMessage());
+    displayFormatter.printErrorMessage(serr, t.getMessage());
 
     if (reconnect && (t instanceof InvalidClientSessionException)) {
       try {
@@ -707,7 +716,7 @@ public void close() {
 
   public static void main(String [] args) throws Exception {
     TajoConf conf = new TajoConf();
-    TajoCli shell = new TajoCli(conf, args, new Properties(), System.in, System.out);
+    TajoCli shell = new TajoCli(conf, args, new Properties(), System.in, System.out, System.err);
     System.out.println();
     System.exit(shell.runShell());
   }

tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/ConnectDatabaseCommand.java

@@ -65,7 +65,7 @@ public void invoke(String[] cmd) throws Exception {
         );
 
       } catch (TajoException se) {
-        context.getOutput().write(String.format("ERROR: %s%n", se.getMessage()));
+        context.getError().write(String.format("ERROR: %s%n", se.getMessage()));
       }
     }
   }

tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java

@@ -58,7 +58,7 @@ public void invoke(String[] cmd) throws TajoException {
       String tableName = tableNameMaker.toString().replace("\"", "");
       TableDesc desc = client.getTableDesc(tableName);
       if (desc == null) {
-        context.getOutput().println("Did not find any relation named \"" + tableName + "\"");
+        context.getError().println("Did not find any relation named \"" + tableName + "\"");
       } else {
         context.getOutput().println(toFormattedString(desc));
         // If there exists any indexes for the table, print index information
@@ -81,7 +81,7 @@ public void invoke(String[] cmd) throws TajoException {
     } else if (cmd.length == 1) {
       List<String> tableList = client.getTableList(null);
       if (tableList.size() == 0) {
-        context.getOutput().println("No Relation Found");
+        context.getError().println("No Relation Found");
       }
       for (String table : tableList) {
         context.getOutput().println(table);

tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/ExecExternalShellCommand.java

@@ -53,12 +53,13 @@ public void invoke(String[] command) throws Exception {
     execCommand[2] = builtCommand;
 
     PrintWriter sout = context.getOutput();
+    PrintWriter serr = context.getError();
 
     CountDownLatch latch = new CountDownLatch(2);
     Process process = Runtime.getRuntime().exec(execCommand);
     try {
       InputStreamConsoleWriter inWriter = new InputStreamConsoleWriter(process.getInputStream(), sout, "", latch);
-      InputStreamConsoleWriter errWriter = new InputStreamConsoleWriter(process.getErrorStream(), sout, "ERROR: ", latch);
+      InputStreamConsoleWriter errWriter = new InputStreamConsoleWriter(process.getErrorStream(), serr, "ERROR: ", latch);
 
       inWriter.start();
       errWriter.start();

tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/ExitCommand.java

@@ -36,8 +36,8 @@ public String getCommand() {
 
   @Override
   public void invoke(String[] cmd) throws Exception {
-    context.getOutput().println("bye!");
-    context.getOutput().close();
+    context.getError().println("bye!");
+    context.getError().flush();
     System.exit(0);
   }
 

tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/HdfsCommand.java

@@ -42,7 +42,7 @@ public void invoke(String[] command) throws Exception {
 
       fsShell.run(dfsCommands);
     } catch (Exception e) {
-      context.getOutput().println("ERROR: " + e.getMessage());
+      context.getError().println("ERROR: " + e.getMessage());
     }
   }
 

tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/HelpCommand.java

@@ -96,7 +96,7 @@ public void invoke(String[] cmd) throws Exception {
       if (context.getCommands().containsKey(slashCommand)) {
         context.getCommands().get(slashCommand).printHelp();
       } else {
-        context.getOutput().println("Command not found: " + cmd[1]);
+        context.getError().println("Command not found: " + cmd[1]);
       }
     }
   }

tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java

@@ -71,7 +71,7 @@ public void set(String key, String val) throws NoSuchSessionVariableException {
       }
 
       if (SessionVars.isDeprecated(key)) {
-        context.getOutput().println("Warning: deprecated to directly use config key in TajoConf.ConfVars. " +
+        context.getError().println("Warning: deprecated to directly use config key in TajoConf.ConfVars. " +
             "Please execute '\\help set'.");
       }
     } else {

tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/TajoAdminCommand.java

@@ -42,7 +42,7 @@ public void invoke(String[] command) throws Exception {
 
       admin.runCommand(dfsCommands);
     } catch (Exception e) {
-      context.getOutput().println("ERROR: " + e.getMessage());
+      context.getError().println("ERROR: " + e.getMessage());
     }
   }
 

tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/TajoGetConfCommand.java

@@ -42,7 +42,7 @@ public void invoke(String[] command) throws Exception {
 
       getconf.runCommand(getConfCommands);
     } catch (Exception e) {
-      context.getOutput().println("ERROR: " + e.getMessage());
+      context.getError().println("ERROR: " + e.getMessage());
     }
   }
 

tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/TajoHAAdminCommand.java

@@ -42,7 +42,7 @@ public void invoke(String[] command) throws Exception {
 
       haAdmin.runCommand(haAdminCommands);
     } catch (Exception e) {
-      context.getOutput().println("ERROR: " + e.getMessage());
+      context.getError().println("ERROR: " + e.getMessage());
     }
   }
 

tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java

@@ -66,7 +66,7 @@ public class TestDefaultCliOutputFormatter {
   public void setUp() throws Exception {
     conf = cluster.getConfiguration();
     ByteArrayOutputStream out = new ByteArrayOutputStream();
-    tajoCli = new TajoCli(conf, new String[]{}, null, System.in, out);
+    tajoCli = new TajoCli(conf, new String[]{}, null, System.in, out, out);
     cliContext = tajoCli.getContext();
   }
 

tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java

@@ -69,6 +69,7 @@ public class TestTajoCli {
   private TajoCli tajoCli;
   private Path currentResultPath;
   private ByteArrayOutputStream out;
+  private ByteArrayOutputStream err;
 
   @Rule
   public TestName name = new TestName();
@@ -81,14 +82,16 @@ public TestTajoCli() {
   @Before
   public void setUp() throws Exception {
     out = new ByteArrayOutputStream();
+    err = new ByteArrayOutputStream();
     Properties connParams = new Properties();
     connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, "3");
-    tajoCli = new TajoCli(cluster.getConfiguration(), new String[]{}, connParams, System.in, out);
+    tajoCli = new TajoCli(cluster.getConfiguration(), new String[]{}, connParams, System.in, out, err);
   }
 
   @After
   public void tearDown() throws IOException {
     out.close();
+    err.close();
     if (tajoCli != null) {
       tajoCli.close();
     }
@@ -106,17 +109,33 @@ private void assertOutputResult(String actual) throws Exception {
     assertOutputResult(name.getMethodName() + ".result", actual);
   }
 
+  private void assertErrorResult(String actual, boolean required) throws Exception {
+    String fileName = name.getMethodName() + ".err";
+    if (required) {
+      assertOutputResult(fileName, actual);
+    }
+  }
+
   private void assertOutputResult(String expectedResultFile, String actual) throws Exception {
     assertOutputResult(expectedResultFile, actual, null, null);
   }
 
+  private boolean existsFile(String fileName) throws IOException {
+    FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
+    Path filePath = StorageUtil.concatPath(currentResultPath, fileName);
+    return fs.exists(filePath);
+  }
+
+  private Path getAbsolutePath(String fileName) {
+    return StorageUtil.concatPath(currentResultPath, fileName);
+  }
+
   private void assertOutputResult(String expectedResultFile, String actual, String[] paramKeys, String[] paramValues)
     throws Exception {
-    FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
-    Path resultFile = StorageUtil.concatPath(currentResultPath, expectedResultFile);
-    assertTrue(resultFile.toString() + " existence check", fs.exists(resultFile));
+    Path path = getAbsolutePath(expectedResultFile);
+    assertTrue(path.toString() + " existence check", existsFile(expectedResultFile));
 
-    String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
+    String expectedResult = FileUtil.readTextFile(new File(path.toUri()));
 
     if (paramKeys != null) {
       for (int i = 0; i < paramKeys.length; i++) {
@@ -164,7 +183,7 @@ public void testParseConf() throws Exception {
     assertEquals("tajo.executor.join.inner.in-memory-table-num=256", confValues[1]);
 
     TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
-    try (TajoCli testCli = new TajoCli(tajoConf, args, null, System.in, System.out)) {
+    try (TajoCli testCli = new TajoCli(tajoConf, args, null, System.in, System.out, err)) {
       assertEquals("false", testCli.getContext().get(SessionVars.CLI_PAGING_ENABLED));
       assertEquals("256", testCli.getContext().getConf().get("tajo.executor.join.inner.in-memory-table-num"));
     }
@@ -310,8 +329,10 @@ private void verifySelectResultWithNullTrue() throws Exception {
 
     tajoCli.executeScript(sql);
 
-    String consoleResult = new String(out.toByteArray());
-    assertOutputResult(consoleResult);
+    String stdoutResult = new String(out.toByteArray());
+    assertOutputResult(stdoutResult);
+    String stdErrResult = new String(err.toByteArray());
+    assertErrorResult(stdErrResult, false);
   }
 
   @Test
@@ -345,7 +366,8 @@ public void testGetConf() throws Exception {
     setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
 
     try (ByteArrayOutputStream out = new ByteArrayOutputStream();
-         TajoCli tajoCli = new TajoCli(tajoConf, new String[]{}, null, System.in, out)) {
+         ByteArrayOutputStream err = new ByteArrayOutputStream();
+         TajoCli tajoCli = new TajoCli(tajoConf, new String[]{}, null, System.in, out, err)) {
       tajoCli.executeMetaCommand("\\getconf tajo.rootdir");
 
       String consoleResult = new String(out.toByteArray());
@@ -359,7 +381,7 @@ public void testShowMasters() throws Exception {
     setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
 
     ByteArrayOutputStream out = new ByteArrayOutputStream();
-    TajoCli tajoCli = new TajoCli(tajoConf, new String[]{}, null, System.in, out);
+    TajoCli tajoCli = new TajoCli(tajoConf, new String[]{}, null, System.in, out, err);
     tajoCli.executeMetaCommand("\\admin -showmasters");
 
     String consoleResult = new String(out.toByteArray());
@@ -395,7 +417,7 @@ public void run() {
           setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
           Properties connParams = new Properties();
           connParams.setProperty(ClientParameters.RETRY, "3");
-          TajoCli tc = new TajoCli(tajoConf, new String[]{}, connParams, is, out);
+          TajoCli tc = new TajoCli(tajoConf, new String[]{}, connParams, is, out, err);
 
           tc.executeMetaCommand("\\set ON_ERROR_STOP false");
           assertSessionVar(tc, SessionVars.ON_ERROR_STOP.keyname(), "false");
@@ -489,7 +511,7 @@ public void testNonForwardQueryPause() throws Exception {
     assertEquals(0L, tableDesc.getStats().getNumRows().longValue());
 
     try (InputStream testInput = new ByteArrayInputStream(new byte[]{(byte) DefaultTajoCliOutputFormatter.QUIT_COMMAND});
-         TajoCli cli = new TajoCli(cluster.getConfiguration(), new String[]{}, null, testInput, out)) {
+         TajoCli cli = new TajoCli(cluster.getConfiguration(), new String[]{}, null, testInput, out, err)) {
       setVar(cli, SessionVars.CLI_PAGE_ROWS, "2");
       setVar(cli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
 
@@ -504,7 +526,7 @@ public void testNonForwardQueryPause() throws Exception {
   @Test
   public void testResultRowNumWhenSelectingOnPartitionedTable() throws Exception {
     try (TajoCli cli2 = new TajoCli(cluster.getConfiguration(), new String[]{}, null, System.in,
-        new NullOutputStream())) {
+        new NullOutputStream(), new NullOutputStream())) {
       cli2.executeScript("create table region_part (r_regionkey int8, r_name text) " +
           "partition by column (r_comment text) as select * from region");