From 8e8d6d3d4dfb8e3bf6f33c3bbb4096c263c61077 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 28 Nov 2016 23:38:59 -0500 Subject: [PATCH 1/5] Implement simple C++ pretty printer for record batches. Debugging efforts Change-Id: Id413c1295c85f1b6d8954e0f7bd4dc6eed958a52 --- cpp/CMakeLists.txt | 1 + cpp/src/arrow/CMakeLists.txt | 2 + cpp/src/arrow/ipc/ipc-json-test.cc | 20 -- cpp/src/arrow/ipc/json-integration-test.cc | 7 + cpp/src/arrow/ipc/json-internal.cc | 2 +- cpp/src/arrow/pretty_print-test.cc | 87 ++++++++ cpp/src/arrow/pretty_print.cc | 207 ++++++++++++++++++ cpp/src/arrow/pretty_print.h | 35 +++ cpp/src/arrow/test-util.h | 23 ++ cpp/src/arrow/type_traits.h | 6 + integration/data/simple.json | 2 +- .../org/apache/arrow/tools/Integration.java | 13 ++ 12 files changed, 383 insertions(+), 22 deletions(-) create mode 100644 cpp/src/arrow/pretty_print-test.cc create mode 100644 cpp/src/arrow/pretty_print.cc create mode 100644 cpp/src/arrow/pretty_print.h diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 1a970081234..798d75fe556 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -745,6 +745,7 @@ set(ARROW_SRCS src/arrow/array.cc src/arrow/builder.cc src/arrow/column.cc + src/arrow/pretty_print.cc src/arrow/schema.cc src/arrow/table.cc src/arrow/type.cc diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 81851bc5b3e..6c0dea20ba7 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -21,6 +21,7 @@ install(FILES array.h column.h builder.h + pretty_print.h schema.h table.h type.h @@ -37,6 +38,7 @@ set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS}) ADD_ARROW_TEST(array-test) ADD_ARROW_TEST(column-test) +ADD_ARROW_TEST(pretty_print-test) ADD_ARROW_TEST(schema-test) ADD_ARROW_TEST(table-test) diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc index e5c3a081fca..ba4d9ca9828 100644 --- a/cpp/src/arrow/ipc/ipc-json-test.cc +++ b/cpp/src/arrow/ipc/ipc-json-test.cc @@ -96,26 +96,6 @@ void CheckPrimitive(const std::shared_ptr& type, TestArrayRoundTrip(*array.get()); } -template -void MakeArray(const std::shared_ptr& type, const std::vector& is_valid, - const std::vector& values, std::shared_ptr* out) { - std::shared_ptr values_buffer; - std::shared_ptr values_bitmap; - - ASSERT_OK(test::CopyBufferFromVector(values, &values_buffer)); - ASSERT_OK(test::GetBitmapFromBoolVector(is_valid, &values_bitmap)); - - using ArrayType = typename TypeTraits::ArrayType; - - int32_t null_count = 0; - for (bool val : is_valid) { - if (!val) { ++null_count; } - } - - *out = std::make_shared(type, static_cast(values.size()), - values_buffer, null_count, values_bitmap); -} - TEST(TestJsonSchemaWriter, FlatTypes) { std::vector> fields = {field("f0", int8()), field("f1", int16(), false), field("f2", int32()), field("f3", int64(), false), diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc index 7a313f791e6..c4e68472a19 100644 --- a/cpp/src/arrow/ipc/json-integration-test.cc +++ b/cpp/src/arrow/ipc/json-integration-test.cc @@ -31,6 +31,7 @@ #include "arrow/io/file.h" #include "arrow/ipc/file.h" #include "arrow/ipc/json.h" +#include "arrow/pretty_print.h" #include "arrow/schema.h" #include "arrow/table.h" #include "arrow/test-util.h" @@ -171,6 +172,12 @@ static Status ValidateArrowVsJson( if (!json_batch->Equals(*arrow_batch.get())) { std::stringstream ss; ss << "Record batch " << i << " did not match"; + + ss << "\nJSON: \n "; + RETURN_NOT_OK(PrettyPrint(*json_batch.get(), &ss)); + + ss << "\nArrow: \n "; + RETURN_NOT_OK(PrettyPrint(*arrow_batch.get(), &ss)); return Status::Invalid(ss.str()); } } diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc index e56bcb32b94..50f5b0cb1bd 100644 --- a/cpp/src/arrow/ipc/json-internal.cc +++ b/cpp/src/arrow/ipc/json-internal.cc @@ -343,7 +343,7 @@ class JsonSchemaWriter : public TypeVisitor { class JsonArrayWriter : public ArrayVisitor { public: - explicit JsonArrayWriter(const std::string& name, const Array& array, RjWriter* writer) + JsonArrayWriter(const std::string& name, const Array& array, RjWriter* writer) : name_(name), array_(array), writer_(writer) {} Status Write() { return VisitArray(name_, array_); } diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc new file mode 100644 index 00000000000..979dd6c353e --- /dev/null +++ b/cpp/src/arrow/pretty_print-test.cc @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" + +#include "arrow/array.h" +#include "arrow/pretty_print.h" +#include "arrow/test-util.h" +#include "arrow/type.h" +#include "arrow/types/list.h" +#include "arrow/types/primitive.h" +#include "arrow/types/string.h" +#include "arrow/types/struct.h" +#include "arrow/type_traits.h" + +namespace arrow { + +class TestArrayPrinter : public ::testing::Test { + public: + void SetUp() {} + + void Print(const Array& array) {} + + private: + std::ostringstream sink_; +}; + +template +void CheckPrimitive(const std::vector& is_valid, + const std::vector& values, const char* expected) { + std::ostringstream sink; + + MemoryPool* pool = default_memory_pool(); + typename TypeTraits::BuilderType builder(pool, std::make_shared()); + + for (size_t i = 0; i < values.size(); ++i) { + if (is_valid[i]) { + ASSERT_OK(builder.Append(values[i])); + } else { + ASSERT_OK(builder.AppendNull()); + } + } + + std::shared_ptr array; + ASSERT_OK(builder.Finish(&array)); + + ASSERT_OK(PrettyPrint(*array.get(), &sink)); + + std::string result = sink.str(); + ASSERT_EQ(std::string(expected, strlen(expected)), result); +} + +TEST_F(TestArrayPrinter, PrimitiveType) { + std::vector is_valid = {true, true, false, true, false}; + + std::vector values = {0, 1, 2, 3, 4}; + static const char* expected = R"expected([0, 1, null, 3, null])expected"; + CheckPrimitive(is_valid, values, expected); + + std::vector values2 = {"foo", "bar", "", "baz", ""}; + static const char* ex2 = R"expected(["foo", "bar", null, "baz", null])expected"; + CheckPrimitive(is_valid, values2, ex2); +} + +} // namespace arrow diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc new file mode 100644 index 00000000000..8c3f693b607 --- /dev/null +++ b/cpp/src/arrow/pretty_print.cc @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "arrow/array.h" +#include "arrow/table.h" +#include "arrow/type.h" +#include "arrow/type_traits.h" +#include "arrow/types/list.h" +#include "arrow/types/string.h" +#include "arrow/types/struct.h" +#include "arrow/pretty_print.h" + +namespace arrow { + +class ArrayPrinter : public ArrayVisitor { + public: + ArrayPrinter(const Array& array, std::ostream* sink) + : array_(array), sink_(sink) {} + + Status Print() { + return VisitArray(array_); + } + + Status VisitArray(const Array& array) { + return array.Accept(this); + } + + template + typename std::enable_if::value, void>::type WriteDataValues( + const T& array) { + const auto data = array.raw_data(); + for (int i = 0; i < array.length(); ++i) { + if (i > 0) { + (*sink_) << ", "; + } + if (array.IsNull(i)) { + (*sink_) << "null"; + } else { + (*sink_) << data[i]; + } + } + } + + // String (Utf8), Binary + template + typename std::enable_if::value, void>::type + WriteDataValues(const T& array) { + int32_t length; + for (int i = 0; i < array.length(); ++i) { + if (i > 0) { + (*sink_) << ", "; + } + if (array.IsNull(i)) { + (*sink_) << "null"; + } else { + const char* buf = reinterpret_cast(array.GetValue(i, &length)); + (*sink_) << "\"" << std::string(buf, length) << "\""; + } + } + } + + template + typename std::enable_if::value, void>::type + WriteDataValues(const T& array) { + for (int i = 0; i < array.length(); ++i) { + if (i > 0) { + (*sink_) << ", "; + } + if (array.IsNull(i)) { + (*sink_) << "null"; + } else { + (*sink_) << (array.Value(i) ? "true" : "false"); + } + } + } + + void OpenArray() { + (*sink_) << "["; + } + + void CloseArray() { + (*sink_) << "]"; + } + + template + Status WritePrimitive(const T& array) { + OpenArray(); + WriteDataValues(array); + CloseArray(); + return Status::OK(); + } + + template + Status WriteVarBytes(const T& array) { + OpenArray(); + WriteDataValues(array); + CloseArray(); + return Status::OK(); + } + + Status Visit(const NullArray& array) override { + return Status::OK(); + } + + Status Visit(const BooleanArray& array) override { return WritePrimitive(array); } + + Status Visit(const Int8Array& array) override { return WritePrimitive(array); } + + Status Visit(const Int16Array& array) override { return WritePrimitive(array); } + + Status Visit(const Int32Array& array) override { return WritePrimitive(array); } + + Status Visit(const Int64Array& array) override { return WritePrimitive(array); } + + Status Visit(const UInt8Array& array) override { return WritePrimitive(array); } + + Status Visit(const UInt16Array& array) override { return WritePrimitive(array); } + + Status Visit(const UInt32Array& array) override { return WritePrimitive(array); } + + Status Visit(const UInt64Array& array) override { return WritePrimitive(array); } + + Status Visit(const HalfFloatArray& array) override { return WritePrimitive(array); } + + Status Visit(const FloatArray& array) override { return WritePrimitive(array); } + + Status Visit(const DoubleArray& array) override { return WritePrimitive(array); } + + Status Visit(const StringArray& array) override { return WriteVarBytes(array); } + + Status Visit(const BinaryArray& array) override { return WriteVarBytes(array); } + + Status Visit(const DateArray& array) override { return Status::NotImplemented("date"); } + + Status Visit(const TimeArray& array) override { return Status::NotImplemented("time"); } + + Status Visit(const TimestampArray& array) override { + return Status::NotImplemented("timestamp"); + } + + Status Visit(const IntervalArray& array) override { + return Status::NotImplemented("interval"); + } + + Status Visit(const DecimalArray& array) override { + return Status::NotImplemented("decimal"); + } + + Status Visit(const ListArray& array) override { + // auto type = static_cast(array.type().get()); + // for (size_t i = 0; i < fields.size(); ++i) { + // RETURN_NOT_OK(VisitArray(fields[i]->name, *arrays[i].get())); + // } + // return WriteChildren(type->children(), {array.values()}); + return Status::OK(); + } + + Status Visit(const StructArray& array) override { + // auto type = static_cast(array.type().get()); + // for (size_t i = 0; i < fields.size(); ++i) { + // RETURN_NOT_OK(VisitArray(fields[i]->name, *arrays[i].get())); + // } + // return WriteChildren(type->children(), array.fields()); + return Status::OK(); + } + + Status Visit(const UnionArray& array) override { + return Status::NotImplemented("union"); + } + + private: + const Array& array_; + std::ostream* sink_; +}; + +Status PrettyPrint(const Array& arr, std::ostream* sink) { + ArrayPrinter printer(arr, sink); + return printer.Print(); +} + +Status PrettyPrint(const RecordBatch& batch, std::ostream* sink) { + for (int i = 0; i < batch.num_columns(); ++i) { + const std::string& name = batch.column_name(i); + (*sink) << name << ": "; + RETURN_NOT_OK(PrettyPrint(*batch.column(i).get(), sink)); + (*sink) << "\n"; + } + return Status::OK(); +} + +} // namespace arrow diff --git a/cpp/src/arrow/pretty_print.h b/cpp/src/arrow/pretty_print.h new file mode 100644 index 00000000000..c7cdbd4085d --- /dev/null +++ b/cpp/src/arrow/pretty_print.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef ARROW_PRETTY_PRINT_H +#define ARROW_PRETTY_PRINT_H + +#include + +#include "arrow/util/visibility.h" +#include "arrow/type_fwd.h" + +namespace arrow { + +class Status; + +Status ARROW_EXPORT PrettyPrint(const RecordBatch& batch, std::ostream* sink); +Status ARROW_EXPORT PrettyPrint(const Array& arr, std::ostream* sink); + +} // namespace arrow + +#endif // ARROW_PRETTY_PRINT_H diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h index 63c2166a573..78997d1bc58 100644 --- a/cpp/src/arrow/test-util.h +++ b/cpp/src/arrow/test-util.h @@ -32,6 +32,7 @@ #include "arrow/schema.h" #include "arrow/table.h" #include "arrow/type.h" +#include "arrow/type_traits.h" #include "arrow/util/bit-util.h" #include "arrow/util/buffer.h" #include "arrow/util/logging.h" @@ -250,6 +251,28 @@ Status MakeRandomBytePoolBuffer(int32_t length, MemoryPool* pool, } } // namespace test + +template +void MakeArray(const std::shared_ptr& type, const std::vector& is_valid, + const std::vector& values, std::shared_ptr* out) { + std::shared_ptr values_buffer; + std::shared_ptr values_bitmap; + + ASSERT_OK(test::CopyBufferFromVector(values, &values_buffer)); + ASSERT_OK(test::GetBitmapFromBoolVector(is_valid, &values_bitmap)); + + using ArrayType = typename TypeTraits::ArrayType; + + int32_t null_count = 0; + for (bool val : is_valid) { + if (!val) { ++null_count; } + } + + *out = std::make_shared(type, static_cast(values.size()), + values_buffer, null_count, values_bitmap); +} + + } // namespace arrow #endif // ARROW_TEST_UTIL_H_ diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h index bbb807488e3..c21c5002035 100644 --- a/cpp/src/arrow/type_traits.h +++ b/cpp/src/arrow/type_traits.h @@ -192,6 +192,12 @@ struct IsFloatingPoint { static constexpr bool value = std::is_floating_point::value; }; +template +struct IsNumeric { + PRIMITIVE_TRAITS(T); + static constexpr bool value = std::is_arithmetic::value; +}; + } // namespace arrow #endif // ARROW_TYPE_TRAITS_H diff --git a/integration/data/simple.json b/integration/data/simple.json index a91b405d4f0..fb903e7ac4b 100644 --- a/integration/data/simple.json +++ b/integration/data/simple.json @@ -31,7 +31,7 @@ "vectors": [ {"type": "VALIDITY", "typeBitWidth": 1}, {"type": "OFFSET", "typeBitWidth": 32}, - {"type": "DATA", "typeBitWidth": 64} + {"type": "DATA", "typeBitWidth": 8} ] } } diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java index 85af30da1e8..63b4d7d3965 100644 --- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java +++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java @@ -29,6 +29,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorLoader; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.VectorUnloader; @@ -114,6 +115,7 @@ public void execute(File arrowFile, File jsonFile) throws IOException { // initialize vectors VectorSchemaRoot root; while ((root = reader.read()) != null) { + printVectors(root.getFieldVectors()); VectorUnloader vectorUnloader = new VectorUnloader(root); try (ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();) { arrowWriter.writeRecordBatch(recordBatch); @@ -255,6 +257,17 @@ private static void compare(VectorSchemaRoot arrowRoot, VectorSchemaRoot jsonRoo } } + public static void printVectors(List vectors) { + for (FieldVector vector : vectors) { + LOGGER.debug(vector.getField().getName()); + ValueVector.Accessor accessor = vector.getAccessor(); + int valueCount = accessor.getValueCount(); + for (int i = 0; i < valueCount; i++) { + LOGGER.debug(String.valueOf(accessor.getObject(i))); + } + } + } + private static void compareSchemas(Schema jsonSchema, Schema arrowSchema) { if (!arrowSchema.equals(jsonSchema)) { throw new IllegalArgumentException("Different schemas:\n" + arrowSchema + "\n" + jsonSchema); From e33bed3b8345e62b465b7d142bed2b197f72b3f3 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 30 Nov 2016 11:45:30 -0500 Subject: [PATCH 2/5] clang-format, add all-OK message to integration_test.py Change-Id: Ieba9835aab71dbd1fb280d9b29cb8a7cefbccb6b --- cpp/src/arrow/pretty_print-test.cc | 8 +++---- cpp/src/arrow/pretty_print.cc | 37 ++++++++---------------------- cpp/src/arrow/pretty_print.h | 2 +- cpp/src/arrow/test-util.h | 1 - integration/integration_test.py | 2 +- 5 files changed, 16 insertions(+), 34 deletions(-) diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc index 979dd6c353e..10af41d16af 100644 --- a/cpp/src/arrow/pretty_print-test.cc +++ b/cpp/src/arrow/pretty_print-test.cc @@ -16,8 +16,8 @@ // under the License. #include -#include #include +#include #include #include #include @@ -29,11 +29,11 @@ #include "arrow/pretty_print.h" #include "arrow/test-util.h" #include "arrow/type.h" +#include "arrow/type_traits.h" #include "arrow/types/list.h" #include "arrow/types/primitive.h" #include "arrow/types/string.h" #include "arrow/types/struct.h" -#include "arrow/type_traits.h" namespace arrow { @@ -48,8 +48,8 @@ class TestArrayPrinter : public ::testing::Test { }; template -void CheckPrimitive(const std::vector& is_valid, - const std::vector& values, const char* expected) { +void CheckPrimitive(const std::vector& is_valid, const std::vector& values, + const char* expected) { std::ostringstream sink; MemoryPool* pool = default_memory_pool(); diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc index 8c3f693b607..4521df2d307 100644 --- a/cpp/src/arrow/pretty_print.cc +++ b/cpp/src/arrow/pretty_print.cc @@ -18,37 +18,30 @@ #include #include "arrow/array.h" +#include "arrow/pretty_print.h" #include "arrow/table.h" #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/types/list.h" #include "arrow/types/string.h" #include "arrow/types/struct.h" -#include "arrow/pretty_print.h" namespace arrow { class ArrayPrinter : public ArrayVisitor { public: - ArrayPrinter(const Array& array, std::ostream* sink) - : array_(array), sink_(sink) {} + ArrayPrinter(const Array& array, std::ostream* sink) : array_(array), sink_(sink) {} - Status Print() { - return VisitArray(array_); - } + Status Print() { return VisitArray(array_); } - Status VisitArray(const Array& array) { - return array.Accept(this); - } + Status VisitArray(const Array& array) { return array.Accept(this); } template typename std::enable_if::value, void>::type WriteDataValues( const T& array) { const auto data = array.raw_data(); for (int i = 0; i < array.length(); ++i) { - if (i > 0) { - (*sink_) << ", "; - } + if (i > 0) { (*sink_) << ", "; } if (array.IsNull(i)) { (*sink_) << "null"; } else { @@ -63,9 +56,7 @@ class ArrayPrinter : public ArrayVisitor { WriteDataValues(const T& array) { int32_t length; for (int i = 0; i < array.length(); ++i) { - if (i > 0) { - (*sink_) << ", "; - } + if (i > 0) { (*sink_) << ", "; } if (array.IsNull(i)) { (*sink_) << "null"; } else { @@ -79,9 +70,7 @@ class ArrayPrinter : public ArrayVisitor { typename std::enable_if::value, void>::type WriteDataValues(const T& array) { for (int i = 0; i < array.length(); ++i) { - if (i > 0) { - (*sink_) << ", "; - } + if (i > 0) { (*sink_) << ", "; } if (array.IsNull(i)) { (*sink_) << "null"; } else { @@ -90,13 +79,9 @@ class ArrayPrinter : public ArrayVisitor { } } - void OpenArray() { - (*sink_) << "["; - } + void OpenArray() { (*sink_) << "["; } - void CloseArray() { - (*sink_) << "]"; - } + void CloseArray() { (*sink_) << "]"; } template Status WritePrimitive(const T& array) { @@ -114,9 +99,7 @@ class ArrayPrinter : public ArrayVisitor { return Status::OK(); } - Status Visit(const NullArray& array) override { - return Status::OK(); - } + Status Visit(const NullArray& array) override { return Status::OK(); } Status Visit(const BooleanArray& array) override { return WritePrimitive(array); } diff --git a/cpp/src/arrow/pretty_print.h b/cpp/src/arrow/pretty_print.h index c7cdbd4085d..dcb236d7269 100644 --- a/cpp/src/arrow/pretty_print.h +++ b/cpp/src/arrow/pretty_print.h @@ -20,8 +20,8 @@ #include -#include "arrow/util/visibility.h" #include "arrow/type_fwd.h" +#include "arrow/util/visibility.h" namespace arrow { diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h index 78997d1bc58..b86a1809cd0 100644 --- a/cpp/src/arrow/test-util.h +++ b/cpp/src/arrow/test-util.h @@ -272,7 +272,6 @@ void MakeArray(const std::shared_ptr& type, const std::vector& i values_buffer, null_count, values_bitmap); } - } // namespace arrow #endif // ARROW_TEST_UTIL_H_ diff --git a/integration/integration_test.py b/integration/integration_test.py index 6ea634d7795..88dc3ad7971 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -165,7 +165,7 @@ def run_all_tests(debug=False): runner = IntegrationRunner(json_files, testers, debug=debug) runner.run() - + print('-- All tests passed!') if __name__ == '__main__': parser = argparse.ArgumentParser(description='Arrow integration test CLI') From 56a1c41e6ac5eba8475733a2cd92f5a51179ccb3 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 30 Nov 2016 11:50:14 -0500 Subject: [PATCH 3/5] We are only padding to 8 byte boundaries Change-Id: Idc4d9004399b0f9445a781ec6bad1c72b3e0e4af --- format/IPC.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/format/IPC.md b/format/IPC.md index a55dcdff481..d386e6048cf 100644 --- a/format/IPC.md +++ b/format/IPC.md @@ -24,7 +24,7 @@ In general, the file looks like: ``` - + ... @@ -49,7 +49,7 @@ appropriate alignment and padding): ``` - + ``` From 57e4926bcd526a3cc492a09f36254a9ad965170f Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 30 Nov 2016 12:06:52 -0500 Subject: [PATCH 4/5] cpplint Change-Id: I1a115d50104b284972a159c4287c4157b49a1c3a --- cpp/src/arrow/pretty_print.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc index 4521df2d307..c0b4b08274a 100644 --- a/cpp/src/arrow/pretty_print.cc +++ b/cpp/src/arrow/pretty_print.cc @@ -16,6 +16,7 @@ // under the License. #include +#include #include "arrow/array.h" #include "arrow/pretty_print.h" @@ -25,6 +26,7 @@ #include "arrow/types/list.h" #include "arrow/types/string.h" #include "arrow/types/struct.h" +#include "arrow/util/status.h" namespace arrow { From 1efeaedfdc349ba9a40bdc518834c8c631089e8c Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 30 Nov 2016 13:23:44 -0500 Subject: [PATCH 5/5] Remove debug printing from Java Change-Id: Ibec20eb0efaa768131cc7c56892b84261da844e5 --- .../java/org/apache/arrow/tools/Integration.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java index 63b4d7d3965..85af30da1e8 100644 --- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java +++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java @@ -29,7 +29,6 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorLoader; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.VectorUnloader; @@ -115,7 +114,6 @@ public void execute(File arrowFile, File jsonFile) throws IOException { // initialize vectors VectorSchemaRoot root; while ((root = reader.read()) != null) { - printVectors(root.getFieldVectors()); VectorUnloader vectorUnloader = new VectorUnloader(root); try (ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch();) { arrowWriter.writeRecordBatch(recordBatch); @@ -257,17 +255,6 @@ private static void compare(VectorSchemaRoot arrowRoot, VectorSchemaRoot jsonRoo } } - public static void printVectors(List vectors) { - for (FieldVector vector : vectors) { - LOGGER.debug(vector.getField().getName()); - ValueVector.Accessor accessor = vector.getAccessor(); - int valueCount = accessor.getValueCount(); - for (int i = 0; i < valueCount; i++) { - LOGGER.debug(String.valueOf(accessor.getObject(i))); - } - } - } - private static void compareSchemas(Schema jsonSchema, Schema arrowSchema) { if (!arrowSchema.equals(jsonSchema)) { throw new IllegalArgumentException("Different schemas:\n" + arrowSchema + "\n" + jsonSchema);