GH-35627: [C++][Format][Integration] Add string view to the arrow format

LICENSE.txt

@@ -1894,7 +1894,7 @@ This project includes code from the autobrew project.
 The following files are based on code from the autobrew project:
 * r/tools/autobrew
 * dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
-* dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb 
+* dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb
 
 Copyright (c) 2019, Jeroen Ooms
 License: MIT
@@ -1976,6 +1976,20 @@ License: http://www.apache.org/licenses/LICENSE-2.0
 
 --------------------------------------------------------------------------------
 
+This project includes code from Velox.
+
+ * cpp/src/arrow/util/string_header.h
+
+is based on Velox's
+
+ * velox/type/StringView.h
+
+Copyright: Copyright (c) Facebook, Inc. and its affiliates.
+Home page: https://github.com/facebookincubator/velox
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
 The file cpp/src/arrow/vendored/musl/strptime.c has the following license
 
 Copyright © 2005-2020 Rich Felker, et al.

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -4593,6 +4593,7 @@ macro(build_opentelemetry)
        -DWITH_OTLP=ON
        -DWITH_OTLP_HTTP=ON
        -DWITH_OTLP_GRPC=OFF
+       -DWITH_STL=ON
        "-DProtobuf_INCLUDE_DIR=${OPENTELEMETRY_PROTOBUF_INCLUDE_DIR}"
        "-DProtobuf_LIBRARY=${OPENTELEMETRY_PROTOBUF_INCLUDE_DIR}"
        "-DProtobuf_PROTOC_EXECUTABLE=${OPENTELEMETRY_PROTOC_EXECUTABLE}")

cpp/src/arrow/CMakeLists.txt

@@ -226,6 +226,7 @@ set(ARROW_SRCS
     util/ree_util.cc
     util/string.cc
     util/string_builder.cc
+    util/string_header.cc
     util/task_group.cc
     util/tdigest.cc
     util/thread_pool.cc

cpp/src/arrow/array/array_base.cc

@@ -87,6 +87,10 @@ struct ScalarFromArraySlotImpl {
     return Finish(a.GetString(index_));
   }
 
+  Status Visit(const BinaryViewArray& a) {
+    return Finish(std::string{a.GetView(index_)});
+  }
+
   Status Visit(const FixedSizeBinaryArray& a) { return Finish(a.GetString(index_)); }
 
   Status Visit(const DayTimeIntervalArray& a) { return Finish(a.Value(index_)); }

cpp/src/arrow/array/array_binary.cc

@@ -89,6 +89,28 @@ LargeStringArray::LargeStringArray(int64_t length,
 
 Status LargeStringArray::ValidateUTF8() const { return internal::ValidateUTF8(*data_); }
 
+BinaryViewArray::BinaryViewArray(const std::shared_ptr<ArrayData>& data) {
+  ARROW_CHECK_EQ(data->type->id(), Type::BINARY_VIEW);
+  SetData(data);
+}
+
+BinaryViewArray::BinaryViewArray(int64_t length, std::shared_ptr<Buffer> headers,
+                                 BufferVector char_buffers,
+                                 std::shared_ptr<Buffer> null_bitmap, int64_t null_count,
+                                 int64_t offset)
+    : PrimitiveArray(binary_view(), length, std::move(headers), std::move(null_bitmap),
+                     null_count, offset) {
+  data_->buffers.resize(char_buffers.size() + 2);
+  std::move(char_buffers.begin(), char_buffers.end(), data_->buffers.begin() + 2);
+}
+
+StringViewArray::StringViewArray(const std::shared_ptr<ArrayData>& data) {
+  ARROW_CHECK_EQ(data->type->id(), Type::STRING_VIEW);
+  SetData(data);
+}
+
+Status StringViewArray::ValidateUTF8() const { return internal::ValidateUTF8(*data_); }
+
 FixedSizeBinaryArray::FixedSizeBinaryArray(const std::shared_ptr<ArrayData>& data) {
   SetData(data);
 }

cpp/src/arrow/array/array_binary.h

@@ -22,6 +22,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <optional>
 #include <string>
 #include <string_view>
 #include <vector>
@@ -217,6 +218,84 @@ class ARROW_EXPORT LargeStringArray : public LargeBinaryArray {
   Status ValidateUTF8() const;
 };
 
+// ----------------------------------------------------------------------
+// BinaryView and StringView
+
+/// Concrete Array class for variable-size binary view data using the
+/// StringHeader struct to reference in-line or out-of-line string values
+class ARROW_EXPORT BinaryViewArray : public PrimitiveArray {
+ public:
+  using TypeClass = BinaryViewType;
+  using IteratorType = stl::ArrayIterator<BinaryViewArray>;
+
+  explicit BinaryViewArray(const std::shared_ptr<ArrayData>& data);
+
+  BinaryViewArray(int64_t length, std::shared_ptr<Buffer> headers,
+                  BufferVector char_buffers,
+                  std::shared_ptr<Buffer> null_bitmap = NULLPTR,
+                  int64_t null_count = kUnknownNullCount, int64_t offset = 0);
+
+  const StringHeader* raw_values() const {
+    return reinterpret_cast<const StringHeader*>(raw_values_) + data_->offset;
+  }
+
+  // For API compatibility with BinaryArray etc.
+  std::string_view GetView(int64_t i) const {
+    const auto& s = raw_values()[i];
+    if (raw_pointers_) {
+      return std::string_view{s};
+    }
+    if (s.IsInline()) {
+      return {s.GetInlineData(), s.size()};
+    }
+    auto* char_buffers = data_->buffers.data() + 2;
+    return {char_buffers[s.GetBufferIndex()]->data_as<char>() + s.GetBufferOffset(),
+            s.size()};
+  }
+
+  std::optional<std::string_view> operator[](int64_t i) const {
+    return *IteratorType(*this, i);
+  }
+
+  IteratorType begin() const { return IteratorType(*this); }
+  IteratorType end() const { return IteratorType(*this, length()); }
+
+  bool has_raw_pointers() const { return raw_pointers_; }
+
+ protected:
+  using PrimitiveArray::PrimitiveArray;
+
+  void SetData(const std::shared_ptr<ArrayData>& data) {
+    PrimitiveArray::SetData(data);
+    raw_pointers_ =
+        internal::checked_cast<const BinaryViewType&>(*type()).has_raw_pointers();
+  }
+
+  bool raw_pointers_ = false;
+};
+
+/// Concrete Array class for variable-size string view (utf-8) data using
+/// StringHeader to reference in-line or out-of-line string values
+class ARROW_EXPORT StringViewArray : public BinaryViewArray {
+ public:
+  using TypeClass = StringViewType;
+
+  explicit StringViewArray(const std::shared_ptr<ArrayData>& data);
+
+  StringViewArray(int64_t length, std::shared_ptr<Buffer> data, BufferVector char_buffers,
+                  std::shared_ptr<Buffer> null_bitmap = NULLPTR,
+                  int64_t null_count = kUnknownNullCount, int64_t offset = 0)
+      : BinaryViewArray(length, std::move(data), std::move(char_buffers),
+                        std::move(null_bitmap), null_count, offset) {
+    data_->type = utf8_view();
+  }
+
+  /// \brief Validate that this array contains only valid UTF8 entries
+  ///
+  /// This check is also implied by ValidateFull()
+  Status ValidateUTF8() const;
+};
+
 // ----------------------------------------------------------------------
 // Fixed width binary