diff --git a/appveyor.yml b/appveyor.yml index 7c83b7ea869..afc2649d258 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -34,6 +34,7 @@ only_commits: cache: - C:\Users\Appveyor\clcache - C:\Users\Appveyor\.cargo\registry + - '%LocalAppData%\\go-build' matrix: fast_finish: true @@ -79,13 +80,15 @@ environment: APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 USE_CLCACHE: false - JOB: "Go" - TARGET: x86_64-pc-windows-msvc + MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 + MINGW_PREFIX: c:\msys64\mingw64 + MSYSTEM: MINGW64 + USE_CLCACHE: false GOROOT: 'c:\go111' GOPATH: c:\gopath GO111MODULE: on GOTOOLDIR: '%GOROOT%\pkg\tool\windows_amd64' PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%' - USE_CLCACHE: false MSVC_DEFAULT_OPTIONS: ON APPVEYOR_SAVE_CACHE_ON_ERROR: true diff --git a/ci/go-build-main.bat b/ci/go-build-main.bat index 4b4dc63a777..9c298089b6e 100644 --- a/ci/go-build-main.bat +++ b/ci/go-build-main.bat @@ -21,6 +21,6 @@ pushd go\arrow %GOROOT%\bin\go version %GOROOT%\bin\go get -v ./... || exit /B -%GOROOT%\bin\go test -race ./... || exit /B +%GOROOT%\bin\go test ./... || exit /B popd diff --git a/go/arrow/array/boolean_test.go b/go/arrow/array/boolean_test.go index e6f4b9bf2bc..e2d58050b69 100644 --- a/go/arrow/array/boolean_test.go +++ b/go/arrow/array/boolean_test.go @@ -17,7 +17,9 @@ package array_test import ( + "fmt" "reflect" + "strings" "testing" "github.com/apache/arrow/go/arrow/array" @@ -258,3 +260,29 @@ func TestBooleanSliceOutOfBounds(t *testing.T) { }) } } + +func TestBooleanStringer(t *testing.T) { + pool := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer pool.AssertSize(t, 0) + + var ( + values = []bool{true, false, true, false, true, false, true, false, true, false} + valids = []bool{true, true, false, true, true, true, false, true, true, true} + ) + + b := array.NewBooleanBuilder(pool) + defer b.Release() + + b.AppendValues(values, valids) + + arr := b.NewArray().(*array.Boolean) + defer arr.Release() + + out := new(strings.Builder) + fmt.Fprintf(out, "%v", arr) + + const want = "[true false (null) false true false (null) false true false]" + if got := out.String(); got != want { + t.Fatalf("invalid stringer:\ngot= %q\nwant=%q", got, want) + } +} diff --git a/go/arrow/doc.go b/go/arrow/doc.go index ebd9c8ebf12..a91e62c02b6 100644 --- a/go/arrow/doc.go +++ b/go/arrow/doc.go @@ -33,6 +33,7 @@ package arrow //go:generate go run _tools/tmpl/main.go -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl //go:generate go run _tools/tmpl/main.go -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl tensor/numeric.gen.go.tmpl tensor/numeric.gen_test.go.tmpl +//go:generate go run ./gen-flatbuffers.go // stringer //go:generate stringer -type=Type diff --git a/go/arrow/gen-flatbuffers.go b/go/arrow/gen-flatbuffers.go new file mode 100644 index 00000000000..265d49ff6a8 --- /dev/null +++ b/go/arrow/gen-flatbuffers.go @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build ignore + +package main + +import ( + "bytes" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" +) + +func main() { + dir, err := ioutil.TempDir("", "go-arrow-") + if err != nil { + log.Fatalf("could not create top-level temporary directory: %v", err) + } + defer os.RemoveAll(dir) + + genFormat(dir) + genFeather(dir) +} + +func genFormat(dir string) { + args := []string{"--go", "-o", filepath.Join(dir, "format")} + fnames, err := filepath.Glob("../../format/*.fbs") + if err != nil || len(fnames) == 0 { + log.Fatalf("could not retrieve list of format FlatBuffers files: files=%d err=%v", + len(fnames), err, + ) + } + args = append(args, fnames...) + + gen := exec.Command("flatc", args...) + gen.Stdout = os.Stdout + gen.Stderr = os.Stderr + + err = gen.Run() + if err != nil { + log.Fatal(err) + } + + err = os.MkdirAll("./internal/flatbuf", 0755) + if err != nil { + log.Fatalf("could not create ./internal/flatbuf directory: %v", err) + } + + base := filepath.Join(dir, "format", "org", "apache", "arrow", "flatbuf") + fnames, err = filepath.Glob(filepath.Join(base, "*.go")) + if err != nil { + log.Fatalf("could not glob %v/*.go: %v", base, err) + } + + for _, fname := range fnames { + dst := filepath.Join(".", "internal", "flatbuf", filepath.Base(fname)) + process(dst, fname) + } +} + +func genFeather(dir string) { + gen := exec.Command("flatc", + "--go", "-o", filepath.Join(dir, "feather"), + "../../cpp/src/arrow/ipc/feather.fbs", + ) + + gen.Stdout = os.Stdout + gen.Stderr = os.Stderr + + err := gen.Run() + if err != nil { + log.Fatal(err) + } + + err = os.MkdirAll("./ipc/feather/fbs", 0755) + if err != nil { + log.Fatalf("could not create ./feather/fbs directory: %v", err) + } + + base := filepath.Join(dir, "feather", "arrow") + fnames, err := filepath.Glob(filepath.Join(base, "ipc", "feather", "*", "*.go")) + if err != nil { + log.Fatalf("could not glob ipc/feather/*.go: %v", err) + } + + for _, fname := range fnames { + dst := strings.Replace(fname, base, ".", -1) + process(dst, fname) + } +} + +func process(dst, fname string) { + raw, err := ioutil.ReadFile(fname) + if err != nil { + log.Fatal(err) + } + + f, err := os.Create(dst) + if err != nil { + log.Fatal(err) + } + defer f.Close() + + if !bytes.HasPrefix(raw, []byte(hdr)) { + _, err = f.Write([]byte(hdr)) + if err != nil { + log.Fatal(err) + } + } + + _, err = f.Write(raw) + if err != nil { + log.Fatal(err) + } + + err = f.Close() + if err != nil { + log.Fatal(err) + } +} + +const hdr = `// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +` diff --git a/go/arrow/go.mod b/go/arrow/go.mod index 451f68ed579..ac0e6d57a52 100644 --- a/go/arrow/go.mod +++ b/go/arrow/go.mod @@ -18,6 +18,7 @@ module github.com/apache/arrow/go/arrow require ( github.com/davecgh/go-spew v1.1.0 // indirect + github.com/google/flatbuffers v1.10.0 github.com/pkg/errors v0.8.1 github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/testify v1.2.0 diff --git a/go/arrow/internal/bitutil/bitutil.go b/go/arrow/internal/bitutil/bitutil.go index ef25a996a21..06a2a950331 100644 --- a/go/arrow/internal/bitutil/bitutil.go +++ b/go/arrow/internal/bitutil/bitutil.go @@ -27,6 +27,9 @@ var ( FlippedBitMask = [8]byte{254, 253, 251, 247, 239, 223, 191, 127} ) +// IsMultipleOf8 returns whether v is a multiple of 8. +func IsMultipleOf8(v int64) bool { return v&7 == 0 } + // NextPowerOf2 rounds x to the next power of two. func NextPowerOf2(x int) int { return 1 << uint(bits.Len(uint(x))) } diff --git a/go/arrow/internal/bitutil/bitutil_test.go b/go/arrow/internal/bitutil/bitutil_test.go index 108db9a1574..62134dc64d4 100644 --- a/go/arrow/internal/bitutil/bitutil_test.go +++ b/go/arrow/internal/bitutil/bitutil_test.go @@ -17,6 +17,7 @@ package bitutil_test import ( + "fmt" "math/rand" "testing" @@ -25,6 +26,35 @@ import ( "github.com/stretchr/testify/assert" ) +func TestIsMultipleOf8(t *testing.T) { + for _, tc := range []struct { + v int64 + want bool + }{ + {-16, true}, + {-9, false}, + {-8, true}, + {-7, false}, + {-4, false}, + {-1, false}, + {-0, true}, + {0, true}, + {1, false}, + {4, false}, + {7, false}, + {8, true}, + {9, false}, + {16, true}, + } { + t.Run(fmt.Sprintf("v=%d", tc.v), func(t *testing.T) { + got := bitutil.IsMultipleOf8(tc.v) + if got != tc.want { + t.Fatalf("IsMultipleOf8(%d): got=%v, want=%v", tc.v, got, tc.want) + } + }) + } +} + func TestCeilByte(t *testing.T) { tests := []struct { name string diff --git a/go/arrow/internal/flatbuf/Binary.go b/go/arrow/internal/flatbuf/Binary.go new file mode 100644 index 00000000000..25b617a6b28 --- /dev/null +++ b/go/arrow/internal/flatbuf/Binary.go @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Binary struct { + _tab flatbuffers.Table +} + +func GetRootAsBinary(buf []byte, offset flatbuffers.UOffsetT) *Binary { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Binary{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Binary) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Binary) Table() flatbuffers.Table { + return rcv._tab +} + +func BinaryStart(builder *flatbuffers.Builder) { + builder.StartObject(0) +} +func BinaryEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Block.go b/go/arrow/internal/flatbuf/Block.go new file mode 100644 index 00000000000..91727789828 --- /dev/null +++ b/go/arrow/internal/flatbuf/Block.go @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Block struct { + _tab flatbuffers.Struct +} + +func (rcv *Block) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Block) Table() flatbuffers.Table { + return rcv._tab.Table +} + +/// Index to the start of the RecordBlock (note this is past the Message header) +func (rcv *Block) Offset() int64 { + return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0)) +} +/// Index to the start of the RecordBlock (note this is past the Message header) +func (rcv *Block) MutateOffset(n int64) bool { + return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n) +} + +/// Length of the metadata +func (rcv *Block) MetaDataLength() int32 { + return rcv._tab.GetInt32(rcv._tab.Pos + flatbuffers.UOffsetT(8)) +} +/// Length of the metadata +func (rcv *Block) MutateMetaDataLength(n int32) bool { + return rcv._tab.MutateInt32(rcv._tab.Pos+flatbuffers.UOffsetT(8), n) +} + +/// Length of the data (this is aligned so there can be a gap between this and +/// the metatdata). +func (rcv *Block) BodyLength() int64 { + return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(16)) +} +/// Length of the data (this is aligned so there can be a gap between this and +/// the metatdata). +func (rcv *Block) MutateBodyLength(n int64) bool { + return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(16), n) +} + +func CreateBlock(builder *flatbuffers.Builder, offset int64, metaDataLength int32, bodyLength int64) flatbuffers.UOffsetT { + builder.Prep(8, 24) + builder.PrependInt64(bodyLength) + builder.Pad(4) + builder.PrependInt32(metaDataLength) + builder.PrependInt64(offset) + return builder.Offset() +} diff --git a/go/arrow/internal/flatbuf/Bool.go b/go/arrow/internal/flatbuf/Bool.go new file mode 100644 index 00000000000..6a4a9d26867 --- /dev/null +++ b/go/arrow/internal/flatbuf/Bool.go @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Bool struct { + _tab flatbuffers.Table +} + +func GetRootAsBool(buf []byte, offset flatbuffers.UOffsetT) *Bool { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Bool{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Bool) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Bool) Table() flatbuffers.Table { + return rcv._tab +} + +func BoolStart(builder *flatbuffers.Builder) { + builder.StartObject(0) +} +func BoolEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Buffer.go b/go/arrow/internal/flatbuf/Buffer.go new file mode 100644 index 00000000000..171366d6baa --- /dev/null +++ b/go/arrow/internal/flatbuf/Buffer.go @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// A Buffer represents a single contiguous memory segment +type Buffer struct { + _tab flatbuffers.Struct +} + +func (rcv *Buffer) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Buffer) Table() flatbuffers.Table { + return rcv._tab.Table +} + +/// The relative offset into the shared memory page where the bytes for this +/// buffer starts +func (rcv *Buffer) Offset() int64 { + return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0)) +} +/// The relative offset into the shared memory page where the bytes for this +/// buffer starts +func (rcv *Buffer) MutateOffset(n int64) bool { + return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n) +} + +/// The absolute length (in bytes) of the memory buffer. The memory is found +/// from offset (inclusive) to offset + length (non-inclusive). +func (rcv *Buffer) Length() int64 { + return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(8)) +} +/// The absolute length (in bytes) of the memory buffer. The memory is found +/// from offset (inclusive) to offset + length (non-inclusive). +func (rcv *Buffer) MutateLength(n int64) bool { + return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(8), n) +} + +func CreateBuffer(builder *flatbuffers.Builder, offset int64, length int64) flatbuffers.UOffsetT { + builder.Prep(8, 16) + builder.PrependInt64(length) + builder.PrependInt64(offset) + return builder.Offset() +} diff --git a/go/arrow/internal/flatbuf/Date.go b/go/arrow/internal/flatbuf/Date.go new file mode 100644 index 00000000000..d1fce940ae4 --- /dev/null +++ b/go/arrow/internal/flatbuf/Date.go @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// Date is either a 32-bit or 64-bit type representing elapsed time since UNIX +/// epoch (1970-01-01), stored in either of two units: +/// +/// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no +/// leap seconds), where the values are evenly divisible by 86400000 +/// * Days (32 bits) since the UNIX epoch +type Date struct { + _tab flatbuffers.Table +} + +func GetRootAsDate(buf []byte, offset flatbuffers.UOffsetT) *Date { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Date{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Date) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Date) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Date) Unit() int16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 1 +} + +func (rcv *Date) MutateUnit(n int16) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func DateStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func DateAddUnit(builder *flatbuffers.Builder, unit int16) { + builder.PrependInt16Slot(0, unit, 1) +} +func DateEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/DateUnit.go b/go/arrow/internal/flatbuf/DateUnit.go new file mode 100644 index 00000000000..bb49f287e68 --- /dev/null +++ b/go/arrow/internal/flatbuf/DateUnit.go @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type DateUnit = int16 +const ( + DateUnitDAY DateUnit = 0 + DateUnitMILLISECOND DateUnit = 1 +) + +var EnumNamesDateUnit = map[DateUnit]string{ + DateUnitDAY:"DAY", + DateUnitMILLISECOND:"MILLISECOND", +} + diff --git a/go/arrow/internal/flatbuf/Decimal.go b/go/arrow/internal/flatbuf/Decimal.go new file mode 100644 index 00000000000..a0a9b5019c5 --- /dev/null +++ b/go/arrow/internal/flatbuf/Decimal.go @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Decimal struct { + _tab flatbuffers.Table +} + +func GetRootAsDecimal(buf []byte, offset flatbuffers.UOffsetT) *Decimal { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Decimal{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Decimal) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Decimal) Table() flatbuffers.Table { + return rcv._tab +} + +/// Total number of decimal digits +func (rcv *Decimal) Precision() int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt32(o + rcv._tab.Pos) + } + return 0 +} + +/// Total number of decimal digits +func (rcv *Decimal) MutatePrecision(n int32) bool { + return rcv._tab.MutateInt32Slot(4, n) +} + +/// Number of digits after the decimal point "." +func (rcv *Decimal) Scale() int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.GetInt32(o + rcv._tab.Pos) + } + return 0 +} + +/// Number of digits after the decimal point "." +func (rcv *Decimal) MutateScale(n int32) bool { + return rcv._tab.MutateInt32Slot(6, n) +} + +func DecimalStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func DecimalAddPrecision(builder *flatbuffers.Builder, precision int32) { + builder.PrependInt32Slot(0, precision, 0) +} +func DecimalAddScale(builder *flatbuffers.Builder, scale int32) { + builder.PrependInt32Slot(1, scale, 0) +} +func DecimalEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/DictionaryBatch.go b/go/arrow/internal/flatbuf/DictionaryBatch.go new file mode 100644 index 00000000000..ad3bc1716a2 --- /dev/null +++ b/go/arrow/internal/flatbuf/DictionaryBatch.go @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// For sending dictionary encoding information. Any Field can be +/// dictionary-encoded, but in this case none of its children may be +/// dictionary-encoded. +/// There is one vector / column per dictionary, but that vector / column +/// may be spread across multiple dictionary batches by using the isDelta +/// flag +type DictionaryBatch struct { + _tab flatbuffers.Table +} + +func GetRootAsDictionaryBatch(buf []byte, offset flatbuffers.UOffsetT) *DictionaryBatch { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &DictionaryBatch{} + x.Init(buf, n+offset) + return x +} + +func (rcv *DictionaryBatch) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *DictionaryBatch) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *DictionaryBatch) Id() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *DictionaryBatch) MutateId(n int64) bool { + return rcv._tab.MutateInt64Slot(4, n) +} + +func (rcv *DictionaryBatch) Data(obj *RecordBatch) *RecordBatch { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + x := rcv._tab.Indirect(o + rcv._tab.Pos) + if obj == nil { + obj = new(RecordBatch) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// If isDelta is true the values in the dictionary are to be appended to a +/// dictionary with the indicated id +func (rcv *DictionaryBatch) IsDelta() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +/// If isDelta is true the values in the dictionary are to be appended to a +/// dictionary with the indicated id +func (rcv *DictionaryBatch) MutateIsDelta(n byte) bool { + return rcv._tab.MutateByteSlot(8, n) +} + +func DictionaryBatchStart(builder *flatbuffers.Builder) { + builder.StartObject(3) +} +func DictionaryBatchAddId(builder *flatbuffers.Builder, id int64) { + builder.PrependInt64Slot(0, id, 0) +} +func DictionaryBatchAddData(builder *flatbuffers.Builder, data flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(data), 0) +} +func DictionaryBatchAddIsDelta(builder *flatbuffers.Builder, isDelta byte) { + builder.PrependByteSlot(2, isDelta, 0) +} +func DictionaryBatchEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/DictionaryEncoding.go b/go/arrow/internal/flatbuf/DictionaryEncoding.go new file mode 100644 index 00000000000..1e7c12fe9b4 --- /dev/null +++ b/go/arrow/internal/flatbuf/DictionaryEncoding.go @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// Dictionary encoding metadata +type DictionaryEncoding struct { + _tab flatbuffers.Table +} + +func GetRootAsDictionaryEncoding(buf []byte, offset flatbuffers.UOffsetT) *DictionaryEncoding { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &DictionaryEncoding{} + x.Init(buf, n+offset) + return x +} + +func (rcv *DictionaryEncoding) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *DictionaryEncoding) Table() flatbuffers.Table { + return rcv._tab +} + +/// The known dictionary id in the application where this data is used. In +/// the file or streaming formats, the dictionary ids are found in the +/// DictionaryBatch messages +func (rcv *DictionaryEncoding) Id() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +/// The known dictionary id in the application where this data is used. In +/// the file or streaming formats, the dictionary ids are found in the +/// DictionaryBatch messages +func (rcv *DictionaryEncoding) MutateId(n int64) bool { + return rcv._tab.MutateInt64Slot(4, n) +} + +/// The dictionary indices are constrained to be positive integers. If this +/// field is null, the indices must be signed int32 +func (rcv *DictionaryEncoding) IndexType(obj *Int) *Int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + x := rcv._tab.Indirect(o + rcv._tab.Pos) + if obj == nil { + obj = new(Int) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// The dictionary indices are constrained to be positive integers. If this +/// field is null, the indices must be signed int32 +/// By default, dictionaries are not ordered, or the order does not have +/// semantic meaning. In some statistical, applications, dictionary-encoding +/// is used to represent ordered categorical data, and we provide a way to +/// preserve that metadata here +func (rcv *DictionaryEncoding) IsOrdered() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +/// By default, dictionaries are not ordered, or the order does not have +/// semantic meaning. In some statistical, applications, dictionary-encoding +/// is used to represent ordered categorical data, and we provide a way to +/// preserve that metadata here +func (rcv *DictionaryEncoding) MutateIsOrdered(n byte) bool { + return rcv._tab.MutateByteSlot(8, n) +} + +func DictionaryEncodingStart(builder *flatbuffers.Builder) { + builder.StartObject(3) +} +func DictionaryEncodingAddId(builder *flatbuffers.Builder, id int64) { + builder.PrependInt64Slot(0, id, 0) +} +func DictionaryEncodingAddIndexType(builder *flatbuffers.Builder, indexType flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(indexType), 0) +} +func DictionaryEncodingAddIsOrdered(builder *flatbuffers.Builder, isOrdered byte) { + builder.PrependByteSlot(2, isOrdered, 0) +} +func DictionaryEncodingEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Endianness.go b/go/arrow/internal/flatbuf/Endianness.go new file mode 100644 index 00000000000..22e349e2798 --- /dev/null +++ b/go/arrow/internal/flatbuf/Endianness.go @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +/// ---------------------------------------------------------------------- +/// Endianness of the platform producing the data +type Endianness = int16 +const ( + EndiannessLittle Endianness = 0 + EndiannessBig Endianness = 1 +) + +var EnumNamesEndianness = map[Endianness]string{ + EndiannessLittle:"Little", + EndiannessBig:"Big", +} + diff --git a/go/arrow/internal/flatbuf/Field.go b/go/arrow/internal/flatbuf/Field.go new file mode 100644 index 00000000000..1310b9db9c4 --- /dev/null +++ b/go/arrow/internal/flatbuf/Field.go @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// A field represents a named column in a record / row batch or child of a +/// nested type. +/// +/// - children is only for nested Arrow arrays +/// - For primitive types, children will have length 0 +/// - nullable should default to true in general +type Field struct { + _tab flatbuffers.Table +} + +func GetRootAsField(buf []byte, offset flatbuffers.UOffsetT) *Field { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Field{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Field) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Field) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Field) Name() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *Field) Nullable() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Field) MutateNullable(n byte) bool { + return rcv._tab.MutateByteSlot(6, n) +} + +func (rcv *Field) TypeType() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Field) MutateTypeType(n byte) bool { + return rcv._tab.MutateByteSlot(8, n) +} + +func (rcv *Field) Type(obj *flatbuffers.Table) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + rcv._tab.Union(obj, o) + return true + } + return false +} + +func (rcv *Field) Dictionary(obj *DictionaryEncoding) *DictionaryEncoding { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + x := rcv._tab.Indirect(o + rcv._tab.Pos) + if obj == nil { + obj = new(DictionaryEncoding) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +func (rcv *Field) Children(obj *Field, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Field) ChildrenLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func (rcv *Field) CustomMetadata(obj *KeyValue, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Field) CustomMetadataLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func FieldStart(builder *flatbuffers.Builder) { + builder.StartObject(7) +} +func FieldAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(name), 0) +} +func FieldAddNullable(builder *flatbuffers.Builder, nullable byte) { + builder.PrependByteSlot(1, nullable, 0) +} +func FieldAddTypeType(builder *flatbuffers.Builder, typeType byte) { + builder.PrependByteSlot(2, typeType, 0) +} +func FieldAddType(builder *flatbuffers.Builder, type_ flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(type_), 0) +} +func FieldAddDictionary(builder *flatbuffers.Builder, dictionary flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(dictionary), 0) +} +func FieldAddChildren(builder *flatbuffers.Builder, children flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(children), 0) +} +func FieldStartChildrenVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func FieldAddCustomMetadata(builder *flatbuffers.Builder, customMetadata flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(6, flatbuffers.UOffsetT(customMetadata), 0) +} +func FieldStartCustomMetadataVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func FieldEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/FieldNode.go b/go/arrow/internal/flatbuf/FieldNode.go new file mode 100644 index 00000000000..b8d711540ab --- /dev/null +++ b/go/arrow/internal/flatbuf/FieldNode.go @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// Data structures for describing a table row batch (a collection of +/// equal-length Arrow arrays) +/// Metadata about a field at some level of a nested type tree (but not +/// its children). +/// +/// For example, a List with values [[1, 2, 3], null, [4], [5, 6], null] +/// would have {length: 5, null_count: 2} for its List node, and {length: 6, +/// null_count: 0} for its Int16 node, as separate FieldNode structs +type FieldNode struct { + _tab flatbuffers.Struct +} + +func (rcv *FieldNode) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *FieldNode) Table() flatbuffers.Table { + return rcv._tab.Table +} + +/// The number of value slots in the Arrow array at this level of a nested +/// tree +func (rcv *FieldNode) Length() int64 { + return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0)) +} +/// The number of value slots in the Arrow array at this level of a nested +/// tree +func (rcv *FieldNode) MutateLength(n int64) bool { + return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n) +} + +/// The number of observed nulls. Fields with null_count == 0 may choose not +/// to write their physical validity bitmap out as a materialized buffer, +/// instead setting the length of the bitmap buffer to 0. +func (rcv *FieldNode) NullCount() int64 { + return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(8)) +} +/// The number of observed nulls. Fields with null_count == 0 may choose not +/// to write their physical validity bitmap out as a materialized buffer, +/// instead setting the length of the bitmap buffer to 0. +func (rcv *FieldNode) MutateNullCount(n int64) bool { + return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(8), n) +} + +func CreateFieldNode(builder *flatbuffers.Builder, length int64, nullCount int64) flatbuffers.UOffsetT { + builder.Prep(8, 16) + builder.PrependInt64(nullCount) + builder.PrependInt64(length) + return builder.Offset() +} diff --git a/go/arrow/internal/flatbuf/FixedSizeBinary.go b/go/arrow/internal/flatbuf/FixedSizeBinary.go new file mode 100644 index 00000000000..4e660d5077f --- /dev/null +++ b/go/arrow/internal/flatbuf/FixedSizeBinary.go @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type FixedSizeBinary struct { + _tab flatbuffers.Table +} + +func GetRootAsFixedSizeBinary(buf []byte, offset flatbuffers.UOffsetT) *FixedSizeBinary { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &FixedSizeBinary{} + x.Init(buf, n+offset) + return x +} + +func (rcv *FixedSizeBinary) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *FixedSizeBinary) Table() flatbuffers.Table { + return rcv._tab +} + +/// Number of bytes per value +func (rcv *FixedSizeBinary) ByteWidth() int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt32(o + rcv._tab.Pos) + } + return 0 +} + +/// Number of bytes per value +func (rcv *FixedSizeBinary) MutateByteWidth(n int32) bool { + return rcv._tab.MutateInt32Slot(4, n) +} + +func FixedSizeBinaryStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func FixedSizeBinaryAddByteWidth(builder *flatbuffers.Builder, byteWidth int32) { + builder.PrependInt32Slot(0, byteWidth, 0) +} +func FixedSizeBinaryEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/FixedSizeList.go b/go/arrow/internal/flatbuf/FixedSizeList.go new file mode 100644 index 00000000000..dabf5cc8581 --- /dev/null +++ b/go/arrow/internal/flatbuf/FixedSizeList.go @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type FixedSizeList struct { + _tab flatbuffers.Table +} + +func GetRootAsFixedSizeList(buf []byte, offset flatbuffers.UOffsetT) *FixedSizeList { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &FixedSizeList{} + x.Init(buf, n+offset) + return x +} + +func (rcv *FixedSizeList) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *FixedSizeList) Table() flatbuffers.Table { + return rcv._tab +} + +/// Number of list items per value +func (rcv *FixedSizeList) ListSize() int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt32(o + rcv._tab.Pos) + } + return 0 +} + +/// Number of list items per value +func (rcv *FixedSizeList) MutateListSize(n int32) bool { + return rcv._tab.MutateInt32Slot(4, n) +} + +func FixedSizeListStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func FixedSizeListAddListSize(builder *flatbuffers.Builder, listSize int32) { + builder.PrependInt32Slot(0, listSize, 0) +} +func FixedSizeListEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/FloatingPoint.go b/go/arrow/internal/flatbuf/FloatingPoint.go new file mode 100644 index 00000000000..d039f3462ec --- /dev/null +++ b/go/arrow/internal/flatbuf/FloatingPoint.go @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type FloatingPoint struct { + _tab flatbuffers.Table +} + +func GetRootAsFloatingPoint(buf []byte, offset flatbuffers.UOffsetT) *FloatingPoint { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &FloatingPoint{} + x.Init(buf, n+offset) + return x +} + +func (rcv *FloatingPoint) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *FloatingPoint) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *FloatingPoint) Precision() int16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *FloatingPoint) MutatePrecision(n int16) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func FloatingPointStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func FloatingPointAddPrecision(builder *flatbuffers.Builder, precision int16) { + builder.PrependInt16Slot(0, precision, 0) +} +func FloatingPointEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Footer.go b/go/arrow/internal/flatbuf/Footer.go new file mode 100644 index 00000000000..516e96079b0 --- /dev/null +++ b/go/arrow/internal/flatbuf/Footer.go @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// Arrow File metadata +/// +type Footer struct { + _tab flatbuffers.Table +} + +func GetRootAsFooter(buf []byte, offset flatbuffers.UOffsetT) *Footer { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Footer{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Footer) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Footer) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Footer) Version() int16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Footer) MutateVersion(n int16) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func (rcv *Footer) Schema(obj *Schema) *Schema { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + x := rcv._tab.Indirect(o + rcv._tab.Pos) + if obj == nil { + obj = new(Schema) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +func (rcv *Footer) Dictionaries(obj *Block, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 24 + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Footer) DictionariesLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func (rcv *Footer) RecordBatches(obj *Block, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 24 + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Footer) RecordBatchesLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func FooterStart(builder *flatbuffers.Builder) { + builder.StartObject(4) +} +func FooterAddVersion(builder *flatbuffers.Builder, version int16) { + builder.PrependInt16Slot(0, version, 0) +} +func FooterAddSchema(builder *flatbuffers.Builder, schema flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(schema), 0) +} +func FooterAddDictionaries(builder *flatbuffers.Builder, dictionaries flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(dictionaries), 0) +} +func FooterStartDictionariesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(24, numElems, 8) +} +func FooterAddRecordBatches(builder *flatbuffers.Builder, recordBatches flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(recordBatches), 0) +} +func FooterStartRecordBatchesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(24, numElems, 8) +} +func FooterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Int.go b/go/arrow/internal/flatbuf/Int.go new file mode 100644 index 00000000000..791f2fed792 --- /dev/null +++ b/go/arrow/internal/flatbuf/Int.go @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Int struct { + _tab flatbuffers.Table +} + +func GetRootAsInt(buf []byte, offset flatbuffers.UOffsetT) *Int { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Int{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Int) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Int) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Int) BitWidth() int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt32(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Int) MutateBitWidth(n int32) bool { + return rcv._tab.MutateInt32Slot(4, n) +} + +func (rcv *Int) IsSigned() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Int) MutateIsSigned(n byte) bool { + return rcv._tab.MutateByteSlot(6, n) +} + +func IntStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func IntAddBitWidth(builder *flatbuffers.Builder, bitWidth int32) { + builder.PrependInt32Slot(0, bitWidth, 0) +} +func IntAddIsSigned(builder *flatbuffers.Builder, isSigned byte) { + builder.PrependByteSlot(1, isSigned, 0) +} +func IntEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Interval.go b/go/arrow/internal/flatbuf/Interval.go new file mode 100644 index 00000000000..91023dd58f7 --- /dev/null +++ b/go/arrow/internal/flatbuf/Interval.go @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Interval struct { + _tab flatbuffers.Table +} + +func GetRootAsInterval(buf []byte, offset flatbuffers.UOffsetT) *Interval { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Interval{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Interval) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Interval) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Interval) Unit() int16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Interval) MutateUnit(n int16) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func IntervalStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func IntervalAddUnit(builder *flatbuffers.Builder, unit int16) { + builder.PrependInt16Slot(0, unit, 0) +} +func IntervalEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/IntervalUnit.go b/go/arrow/internal/flatbuf/IntervalUnit.go new file mode 100644 index 00000000000..62baea99117 --- /dev/null +++ b/go/arrow/internal/flatbuf/IntervalUnit.go @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type IntervalUnit = int16 +const ( + IntervalUnitYEAR_MONTH IntervalUnit = 0 + IntervalUnitDAY_TIME IntervalUnit = 1 +) + +var EnumNamesIntervalUnit = map[IntervalUnit]string{ + IntervalUnitYEAR_MONTH:"YEAR_MONTH", + IntervalUnitDAY_TIME:"DAY_TIME", +} + diff --git a/go/arrow/internal/flatbuf/KeyValue.go b/go/arrow/internal/flatbuf/KeyValue.go new file mode 100644 index 00000000000..c1b85318ecd --- /dev/null +++ b/go/arrow/internal/flatbuf/KeyValue.go @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// user defined key value pairs to add custom metadata to arrow +/// key namespacing is the responsibility of the user +type KeyValue struct { + _tab flatbuffers.Table +} + +func GetRootAsKeyValue(buf []byte, offset flatbuffers.UOffsetT) *KeyValue { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &KeyValue{} + x.Init(buf, n+offset) + return x +} + +func (rcv *KeyValue) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *KeyValue) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *KeyValue) Key() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *KeyValue) Value() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func KeyValueStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func KeyValueAddKey(builder *flatbuffers.Builder, key flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(key), 0) +} +func KeyValueAddValue(builder *flatbuffers.Builder, value flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(value), 0) +} +func KeyValueEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/List.go b/go/arrow/internal/flatbuf/List.go new file mode 100644 index 00000000000..ba84319d3f6 --- /dev/null +++ b/go/arrow/internal/flatbuf/List.go @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type List struct { + _tab flatbuffers.Table +} + +func GetRootAsList(buf []byte, offset flatbuffers.UOffsetT) *List { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &List{} + x.Init(buf, n+offset) + return x +} + +func (rcv *List) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *List) Table() flatbuffers.Table { + return rcv._tab +} + +func ListStart(builder *flatbuffers.Builder) { + builder.StartObject(0) +} +func ListEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Map.go b/go/arrow/internal/flatbuf/Map.go new file mode 100644 index 00000000000..e41f1e3a427 --- /dev/null +++ b/go/arrow/internal/flatbuf/Map.go @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// A Map is a logical nested type that is represented as +/// +/// List> +/// +/// In this layout, the keys and values are each respectively contiguous. We do +/// not constrain the key and value types, so the application is responsible +/// for ensuring that the keys are hashable and unique. Whether the keys are sorted +/// may be set in the metadata for this field +/// +/// In a Field with Map type, the Field has a child Struct field, which then +/// has two children: key type and the second the value type. The names of the +/// child fields may be respectively "entry", "key", and "value", but this is +/// not enforced +/// +/// Map +/// - child[0] entry: Struct +/// - child[0] key: K +/// - child[1] value: V +/// +/// Neither the "entry" field nor the "key" field may be nullable. +/// +/// The metadata is structured so that Arrow systems without special handling +/// for Map can make Map an alias for List. The "layout" attribute for the Map +/// field must have the same contents as a List. +type Map struct { + _tab flatbuffers.Table +} + +func GetRootAsMap(buf []byte, offset flatbuffers.UOffsetT) *Map { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Map{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Map) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Map) Table() flatbuffers.Table { + return rcv._tab +} + +/// Set to true if the keys within each value are sorted +func (rcv *Map) KeysSorted() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +/// Set to true if the keys within each value are sorted +func (rcv *Map) MutateKeysSorted(n byte) bool { + return rcv._tab.MutateByteSlot(4, n) +} + +func MapStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func MapAddKeysSorted(builder *flatbuffers.Builder, keysSorted byte) { + builder.PrependByteSlot(0, keysSorted, 0) +} +func MapEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Message.go b/go/arrow/internal/flatbuf/Message.go new file mode 100644 index 00000000000..f91ebf622a9 --- /dev/null +++ b/go/arrow/internal/flatbuf/Message.go @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Message struct { + _tab flatbuffers.Table +} + +func GetRootAsMessage(buf []byte, offset flatbuffers.UOffsetT) *Message { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Message{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Message) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Message) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Message) Version() int16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Message) MutateVersion(n int16) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func (rcv *Message) HeaderType() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Message) MutateHeaderType(n byte) bool { + return rcv._tab.MutateByteSlot(6, n) +} + +func (rcv *Message) Header(obj *flatbuffers.Table) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + rcv._tab.Union(obj, o) + return true + } + return false +} + +func (rcv *Message) BodyLength() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Message) MutateBodyLength(n int64) bool { + return rcv._tab.MutateInt64Slot(10, n) +} + +func (rcv *Message) CustomMetadata(obj *KeyValue, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Message) CustomMetadataLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func MessageStart(builder *flatbuffers.Builder) { + builder.StartObject(5) +} +func MessageAddVersion(builder *flatbuffers.Builder, version int16) { + builder.PrependInt16Slot(0, version, 0) +} +func MessageAddHeaderType(builder *flatbuffers.Builder, headerType byte) { + builder.PrependByteSlot(1, headerType, 0) +} +func MessageAddHeader(builder *flatbuffers.Builder, header flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(header), 0) +} +func MessageAddBodyLength(builder *flatbuffers.Builder, bodyLength int64) { + builder.PrependInt64Slot(3, bodyLength, 0) +} +func MessageAddCustomMetadata(builder *flatbuffers.Builder, customMetadata flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(customMetadata), 0) +} +func MessageStartCustomMetadataVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func MessageEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/MessageHeader.go b/go/arrow/internal/flatbuf/MessageHeader.go new file mode 100644 index 00000000000..18730bed59f --- /dev/null +++ b/go/arrow/internal/flatbuf/MessageHeader.go @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +/// ---------------------------------------------------------------------- +/// The root Message type +/// This union enables us to easily send different message types without +/// redundant storage, and in the future we can easily add new message types. +/// +/// Arrow implementations do not need to implement all of the message types, +/// which may include experimental metadata types. For maximum compatibility, +/// it is best to send data using RecordBatch +type MessageHeader = byte +const ( + MessageHeaderNONE MessageHeader = 0 + MessageHeaderSchema MessageHeader = 1 + MessageHeaderDictionaryBatch MessageHeader = 2 + MessageHeaderRecordBatch MessageHeader = 3 + MessageHeaderTensor MessageHeader = 4 + MessageHeaderSparseTensor MessageHeader = 5 +) + +var EnumNamesMessageHeader = map[MessageHeader]string{ + MessageHeaderNONE:"NONE", + MessageHeaderSchema:"Schema", + MessageHeaderDictionaryBatch:"DictionaryBatch", + MessageHeaderRecordBatch:"RecordBatch", + MessageHeaderTensor:"Tensor", + MessageHeaderSparseTensor:"SparseTensor", +} + diff --git a/go/arrow/internal/flatbuf/MetadataVersion.go b/go/arrow/internal/flatbuf/MetadataVersion.go new file mode 100644 index 00000000000..0094430c750 --- /dev/null +++ b/go/arrow/internal/flatbuf/MetadataVersion.go @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type MetadataVersion = int16 +const ( + /// 0.1.0 + MetadataVersionV1 MetadataVersion = 0 + /// 0.2.0 + MetadataVersionV2 MetadataVersion = 1 + /// 0.3.0 -> 0.7.1 + MetadataVersionV3 MetadataVersion = 2 + /// >= 0.8.0 + MetadataVersionV4 MetadataVersion = 3 +) + +var EnumNamesMetadataVersion = map[MetadataVersion]string{ + MetadataVersionV1:"V1", + MetadataVersionV2:"V2", + MetadataVersionV3:"V3", + MetadataVersionV4:"V4", +} + diff --git a/go/arrow/internal/flatbuf/Null.go b/go/arrow/internal/flatbuf/Null.go new file mode 100644 index 00000000000..3c3eb4bda36 --- /dev/null +++ b/go/arrow/internal/flatbuf/Null.go @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// These are stored in the flatbuffer in the Type union below +type Null struct { + _tab flatbuffers.Table +} + +func GetRootAsNull(buf []byte, offset flatbuffers.UOffsetT) *Null { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Null{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Null) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Null) Table() flatbuffers.Table { + return rcv._tab +} + +func NullStart(builder *flatbuffers.Builder) { + builder.StartObject(0) +} +func NullEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Precision.go b/go/arrow/internal/flatbuf/Precision.go new file mode 100644 index 00000000000..417d1e788e8 --- /dev/null +++ b/go/arrow/internal/flatbuf/Precision.go @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type Precision = int16 +const ( + PrecisionHALF Precision = 0 + PrecisionSINGLE Precision = 1 + PrecisionDOUBLE Precision = 2 +) + +var EnumNamesPrecision = map[Precision]string{ + PrecisionHALF:"HALF", + PrecisionSINGLE:"SINGLE", + PrecisionDOUBLE:"DOUBLE", +} + diff --git a/go/arrow/internal/flatbuf/RecordBatch.go b/go/arrow/internal/flatbuf/RecordBatch.go new file mode 100644 index 00000000000..b997bdf16c7 --- /dev/null +++ b/go/arrow/internal/flatbuf/RecordBatch.go @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// A data header describing the shared memory layout of a "record" or "row" +/// batch. Some systems call this a "row batch" internally and others a "record +/// batch". +type RecordBatch struct { + _tab flatbuffers.Table +} + +func GetRootAsRecordBatch(buf []byte, offset flatbuffers.UOffsetT) *RecordBatch { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &RecordBatch{} + x.Init(buf, n+offset) + return x +} + +func (rcv *RecordBatch) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *RecordBatch) Table() flatbuffers.Table { + return rcv._tab +} + +/// number of records / rows. The arrays in the batch should all have this +/// length +func (rcv *RecordBatch) Length() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +/// number of records / rows. The arrays in the batch should all have this +/// length +func (rcv *RecordBatch) MutateLength(n int64) bool { + return rcv._tab.MutateInt64Slot(4, n) +} + +/// Nodes correspond to the pre-ordered flattened logical schema +func (rcv *RecordBatch) Nodes(obj *FieldNode, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 16 + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *RecordBatch) NodesLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +/// Nodes correspond to the pre-ordered flattened logical schema +/// Buffers correspond to the pre-ordered flattened buffer tree +/// +/// The number of buffers appended to this list depends on the schema. For +/// example, most primitive arrays will have 2 buffers, 1 for the validity +/// bitmap and 1 for the values. For struct arrays, there will only be a +/// single buffer for the validity (nulls) bitmap +func (rcv *RecordBatch) Buffers(obj *Buffer, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 16 + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *RecordBatch) BuffersLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +/// Buffers correspond to the pre-ordered flattened buffer tree +/// +/// The number of buffers appended to this list depends on the schema. For +/// example, most primitive arrays will have 2 buffers, 1 for the validity +/// bitmap and 1 for the values. For struct arrays, there will only be a +/// single buffer for the validity (nulls) bitmap +func RecordBatchStart(builder *flatbuffers.Builder) { + builder.StartObject(3) +} +func RecordBatchAddLength(builder *flatbuffers.Builder, length int64) { + builder.PrependInt64Slot(0, length, 0) +} +func RecordBatchAddNodes(builder *flatbuffers.Builder, nodes flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(nodes), 0) +} +func RecordBatchStartNodesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(16, numElems, 8) +} +func RecordBatchAddBuffers(builder *flatbuffers.Builder, buffers flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(buffers), 0) +} +func RecordBatchStartBuffersVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(16, numElems, 8) +} +func RecordBatchEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Schema.go b/go/arrow/internal/flatbuf/Schema.go new file mode 100644 index 00000000000..a8011c52422 --- /dev/null +++ b/go/arrow/internal/flatbuf/Schema.go @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// A Schema describes the columns in a row batch +type Schema struct { + _tab flatbuffers.Table +} + +func GetRootAsSchema(buf []byte, offset flatbuffers.UOffsetT) *Schema { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Schema{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Schema) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Schema) Table() flatbuffers.Table { + return rcv._tab +} + +/// endianness of the buffer +/// it is Little Endian by default +/// if endianness doesn't match the underlying system then the vectors need to be converted +func (rcv *Schema) Endianness() int16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +/// endianness of the buffer +/// it is Little Endian by default +/// if endianness doesn't match the underlying system then the vectors need to be converted +func (rcv *Schema) MutateEndianness(n int16) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func (rcv *Schema) Fields(obj *Field, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Schema) FieldsLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func (rcv *Schema) CustomMetadata(obj *KeyValue, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Schema) CustomMetadataLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func SchemaStart(builder *flatbuffers.Builder) { + builder.StartObject(3) +} +func SchemaAddEndianness(builder *flatbuffers.Builder, endianness int16) { + builder.PrependInt16Slot(0, endianness, 0) +} +func SchemaAddFields(builder *flatbuffers.Builder, fields flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(fields), 0) +} +func SchemaStartFieldsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func SchemaAddCustomMetadata(builder *flatbuffers.Builder, customMetadata flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(customMetadata), 0) +} +func SchemaStartCustomMetadataVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func SchemaEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go b/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go new file mode 100644 index 00000000000..c09006c2906 --- /dev/null +++ b/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// Compressed Sparse Row format, that is matrix-specific. +type SparseMatrixIndexCSR struct { + _tab flatbuffers.Table +} + +func GetRootAsSparseMatrixIndexCSR(buf []byte, offset flatbuffers.UOffsetT) *SparseMatrixIndexCSR { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &SparseMatrixIndexCSR{} + x.Init(buf, n+offset) + return x +} + +func (rcv *SparseMatrixIndexCSR) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *SparseMatrixIndexCSR) Table() flatbuffers.Table { + return rcv._tab +} + +/// indptrBuffer stores the location and size of indptr array that +/// represents the range of the rows. +/// The i-th row spans from indptr[i] to indptr[i+1] in the data. +/// The length of this array is 1 + (the number of rows), and the type +/// of index value is long. +/// +/// For example, let X be the following 6x4 matrix: +/// +/// X := [[0, 1, 2, 0], +/// [0, 0, 3, 0], +/// [0, 4, 0, 5], +/// [0, 0, 0, 0], +/// [6, 0, 7, 8], +/// [0, 9, 0, 0]]. +/// +/// The array of non-zero values in X is: +/// +/// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. +/// +/// And the indptr of X is: +/// +/// indptr(X) = [0, 2, 3, 5, 5, 8, 10]. +func (rcv *SparseMatrixIndexCSR) IndptrBuffer(obj *Buffer) *Buffer { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + x := o + rcv._tab.Pos + if obj == nil { + obj = new(Buffer) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// indptrBuffer stores the location and size of indptr array that +/// represents the range of the rows. +/// The i-th row spans from indptr[i] to indptr[i+1] in the data. +/// The length of this array is 1 + (the number of rows), and the type +/// of index value is long. +/// +/// For example, let X be the following 6x4 matrix: +/// +/// X := [[0, 1, 2, 0], +/// [0, 0, 3, 0], +/// [0, 4, 0, 5], +/// [0, 0, 0, 0], +/// [6, 0, 7, 8], +/// [0, 9, 0, 0]]. +/// +/// The array of non-zero values in X is: +/// +/// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. +/// +/// And the indptr of X is: +/// +/// indptr(X) = [0, 2, 3, 5, 5, 8, 10]. +/// indicesBuffer stores the location and size of the array that +/// contains the column indices of the corresponding non-zero values. +/// The type of index value is long. +/// +/// For example, the indices of the above X is: +/// +/// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. +/// +/// Note that the indices are sorted in lexicographical order for each row. +func (rcv *SparseMatrixIndexCSR) IndicesBuffer(obj *Buffer) *Buffer { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + x := o + rcv._tab.Pos + if obj == nil { + obj = new(Buffer) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// indicesBuffer stores the location and size of the array that +/// contains the column indices of the corresponding non-zero values. +/// The type of index value is long. +/// +/// For example, the indices of the above X is: +/// +/// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. +/// +/// Note that the indices are sorted in lexicographical order for each row. +func SparseMatrixIndexCSRStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func SparseMatrixIndexCSRAddIndptrBuffer(builder *flatbuffers.Builder, indptrBuffer flatbuffers.UOffsetT) { + builder.PrependStructSlot(0, flatbuffers.UOffsetT(indptrBuffer), 0) +} +func SparseMatrixIndexCSRAddIndicesBuffer(builder *flatbuffers.Builder, indicesBuffer flatbuffers.UOffsetT) { + builder.PrependStructSlot(1, flatbuffers.UOffsetT(indicesBuffer), 0) +} +func SparseMatrixIndexCSREnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/SparseTensor.go b/go/arrow/internal/flatbuf/SparseTensor.go new file mode 100644 index 00000000000..389832b95a1 --- /dev/null +++ b/go/arrow/internal/flatbuf/SparseTensor.go @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type SparseTensor struct { + _tab flatbuffers.Table +} + +func GetRootAsSparseTensor(buf []byte, offset flatbuffers.UOffsetT) *SparseTensor { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &SparseTensor{} + x.Init(buf, n+offset) + return x +} + +func (rcv *SparseTensor) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *SparseTensor) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *SparseTensor) TypeType() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *SparseTensor) MutateTypeType(n byte) bool { + return rcv._tab.MutateByteSlot(4, n) +} + +/// The type of data contained in a value cell. +/// Currently only fixed-width value types are supported, +/// no strings or nested types. +func (rcv *SparseTensor) Type(obj *flatbuffers.Table) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + rcv._tab.Union(obj, o) + return true + } + return false +} + +/// The type of data contained in a value cell. +/// Currently only fixed-width value types are supported, +/// no strings or nested types. +/// The dimensions of the tensor, optionally named. +func (rcv *SparseTensor) Shape(obj *TensorDim, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *SparseTensor) ShapeLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +/// The dimensions of the tensor, optionally named. +/// The number of non-zero values in a sparse tensor. +func (rcv *SparseTensor) NonZeroLength() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +/// The number of non-zero values in a sparse tensor. +func (rcv *SparseTensor) MutateNonZeroLength(n int64) bool { + return rcv._tab.MutateInt64Slot(10, n) +} + +func (rcv *SparseTensor) SparseIndexType() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *SparseTensor) MutateSparseIndexType(n byte) bool { + return rcv._tab.MutateByteSlot(12, n) +} + +/// Sparse tensor index +func (rcv *SparseTensor) SparseIndex(obj *flatbuffers.Table) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + if o != 0 { + rcv._tab.Union(obj, o) + return true + } + return false +} + +/// Sparse tensor index +/// The location and size of the tensor's data +func (rcv *SparseTensor) Data(obj *Buffer) *Buffer { + o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) + if o != 0 { + x := o + rcv._tab.Pos + if obj == nil { + obj = new(Buffer) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// The location and size of the tensor's data +func SparseTensorStart(builder *flatbuffers.Builder) { + builder.StartObject(7) +} +func SparseTensorAddTypeType(builder *flatbuffers.Builder, typeType byte) { + builder.PrependByteSlot(0, typeType, 0) +} +func SparseTensorAddType(builder *flatbuffers.Builder, type_ flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(type_), 0) +} +func SparseTensorAddShape(builder *flatbuffers.Builder, shape flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(shape), 0) +} +func SparseTensorStartShapeVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func SparseTensorAddNonZeroLength(builder *flatbuffers.Builder, nonZeroLength int64) { + builder.PrependInt64Slot(3, nonZeroLength, 0) +} +func SparseTensorAddSparseIndexType(builder *flatbuffers.Builder, sparseIndexType byte) { + builder.PrependByteSlot(4, sparseIndexType, 0) +} +func SparseTensorAddSparseIndex(builder *flatbuffers.Builder, sparseIndex flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(sparseIndex), 0) +} +func SparseTensorAddData(builder *flatbuffers.Builder, data flatbuffers.UOffsetT) { + builder.PrependStructSlot(6, flatbuffers.UOffsetT(data), 0) +} +func SparseTensorEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/SparseTensorIndex.go b/go/arrow/internal/flatbuf/SparseTensorIndex.go new file mode 100644 index 00000000000..4b82de7efec --- /dev/null +++ b/go/arrow/internal/flatbuf/SparseTensorIndex.go @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type SparseTensorIndex = byte +const ( + SparseTensorIndexNONE SparseTensorIndex = 0 + SparseTensorIndexSparseTensorIndexCOO SparseTensorIndex = 1 + SparseTensorIndexSparseMatrixIndexCSR SparseTensorIndex = 2 +) + +var EnumNamesSparseTensorIndex = map[SparseTensorIndex]string{ + SparseTensorIndexNONE:"NONE", + SparseTensorIndexSparseTensorIndexCOO:"SparseTensorIndexCOO", + SparseTensorIndexSparseMatrixIndexCSR:"SparseMatrixIndexCSR", +} + diff --git a/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go b/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go new file mode 100644 index 00000000000..6d8a0e8fb79 --- /dev/null +++ b/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// EXPERIMENTAL: Data structures for sparse tensors +/// Coodinate format of sparse tensor index. +type SparseTensorIndexCOO struct { + _tab flatbuffers.Table +} + +func GetRootAsSparseTensorIndexCOO(buf []byte, offset flatbuffers.UOffsetT) *SparseTensorIndexCOO { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &SparseTensorIndexCOO{} + x.Init(buf, n+offset) + return x +} + +func (rcv *SparseTensorIndexCOO) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *SparseTensorIndexCOO) Table() flatbuffers.Table { + return rcv._tab +} + +/// COO's index list are represented as a NxM matrix, +/// where N is the number of non-zero values, +/// and M is the number of dimensions of a sparse tensor. +/// indicesBuffer stores the location and size of this index matrix. +/// The type of index value is long, so the stride for the index matrix is unnecessary. +/// +/// For example, let X be a 2x3x4x5 tensor, and it has the following 6 non-zero values: +/// +/// X[0, 1, 2, 0] := 1 +/// X[1, 1, 2, 3] := 2 +/// X[0, 2, 1, 0] := 3 +/// X[0, 1, 3, 0] := 4 +/// X[0, 1, 2, 1] := 5 +/// X[1, 2, 0, 4] := 6 +/// +/// In COO format, the index matrix of X is the following 4x6 matrix: +/// +/// [[0, 0, 0, 0, 1, 1], +/// [1, 1, 1, 2, 1, 2], +/// [2, 2, 3, 1, 2, 0], +/// [0, 1, 0, 0, 3, 4]] +/// +/// Note that the indices are sorted in lexicographical order. +func (rcv *SparseTensorIndexCOO) IndicesBuffer(obj *Buffer) *Buffer { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + x := o + rcv._tab.Pos + if obj == nil { + obj = new(Buffer) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// COO's index list are represented as a NxM matrix, +/// where N is the number of non-zero values, +/// and M is the number of dimensions of a sparse tensor. +/// indicesBuffer stores the location and size of this index matrix. +/// The type of index value is long, so the stride for the index matrix is unnecessary. +/// +/// For example, let X be a 2x3x4x5 tensor, and it has the following 6 non-zero values: +/// +/// X[0, 1, 2, 0] := 1 +/// X[1, 1, 2, 3] := 2 +/// X[0, 2, 1, 0] := 3 +/// X[0, 1, 3, 0] := 4 +/// X[0, 1, 2, 1] := 5 +/// X[1, 2, 0, 4] := 6 +/// +/// In COO format, the index matrix of X is the following 4x6 matrix: +/// +/// [[0, 0, 0, 0, 1, 1], +/// [1, 1, 1, 2, 1, 2], +/// [2, 2, 3, 1, 2, 0], +/// [0, 1, 0, 0, 3, 4]] +/// +/// Note that the indices are sorted in lexicographical order. +func SparseTensorIndexCOOStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func SparseTensorIndexCOOAddIndicesBuffer(builder *flatbuffers.Builder, indicesBuffer flatbuffers.UOffsetT) { + builder.PrependStructSlot(0, flatbuffers.UOffsetT(indicesBuffer), 0) +} +func SparseTensorIndexCOOEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Struct_.go b/go/arrow/internal/flatbuf/Struct_.go new file mode 100644 index 00000000000..427e7060382 --- /dev/null +++ b/go/arrow/internal/flatbuf/Struct_.go @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// A Struct_ in the flatbuffer metadata is the same as an Arrow Struct +/// (according to the physical memory layout). We used Struct_ here as +/// Struct is a reserved word in Flatbuffers +type Struct_ struct { + _tab flatbuffers.Table +} + +func GetRootAsStruct_(buf []byte, offset flatbuffers.UOffsetT) *Struct_ { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Struct_{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Struct_) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Struct_) Table() flatbuffers.Table { + return rcv._tab +} + +func Struct_Start(builder *flatbuffers.Builder) { + builder.StartObject(0) +} +func Struct_End(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Tensor.go b/go/arrow/internal/flatbuf/Tensor.go new file mode 100644 index 00000000000..73e50b55d59 --- /dev/null +++ b/go/arrow/internal/flatbuf/Tensor.go @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Tensor struct { + _tab flatbuffers.Table +} + +func GetRootAsTensor(buf []byte, offset flatbuffers.UOffsetT) *Tensor { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Tensor{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Tensor) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Tensor) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Tensor) TypeType() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Tensor) MutateTypeType(n byte) bool { + return rcv._tab.MutateByteSlot(4, n) +} + +/// The type of data contained in a value cell. Currently only fixed-width +/// value types are supported, no strings or nested types +func (rcv *Tensor) Type(obj *flatbuffers.Table) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + rcv._tab.Union(obj, o) + return true + } + return false +} + +/// The type of data contained in a value cell. Currently only fixed-width +/// value types are supported, no strings or nested types +/// The dimensions of the tensor, optionally named +func (rcv *Tensor) Shape(obj *TensorDim, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Tensor) ShapeLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +/// The dimensions of the tensor, optionally named +/// Non-negative byte offsets to advance one value cell along each dimension +func (rcv *Tensor) Strides(j int) int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.GetInt64(a + flatbuffers.UOffsetT(j*8)) + } + return 0 +} + +func (rcv *Tensor) StridesLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +/// Non-negative byte offsets to advance one value cell along each dimension +/// The location and size of the tensor's data +func (rcv *Tensor) Data(obj *Buffer) *Buffer { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + x := o + rcv._tab.Pos + if obj == nil { + obj = new(Buffer) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// The location and size of the tensor's data +func TensorStart(builder *flatbuffers.Builder) { + builder.StartObject(5) +} +func TensorAddTypeType(builder *flatbuffers.Builder, typeType byte) { + builder.PrependByteSlot(0, typeType, 0) +} +func TensorAddType(builder *flatbuffers.Builder, type_ flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(type_), 0) +} +func TensorAddShape(builder *flatbuffers.Builder, shape flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(shape), 0) +} +func TensorStartShapeVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func TensorAddStrides(builder *flatbuffers.Builder, strides flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(strides), 0) +} +func TensorStartStridesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(8, numElems, 8) +} +func TensorAddData(builder *flatbuffers.Builder, data flatbuffers.UOffsetT) { + builder.PrependStructSlot(4, flatbuffers.UOffsetT(data), 0) +} +func TensorEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/TensorDim.go b/go/arrow/internal/flatbuf/TensorDim.go new file mode 100644 index 00000000000..14b82120887 --- /dev/null +++ b/go/arrow/internal/flatbuf/TensorDim.go @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// Data structures for dense tensors +/// Shape data for a single axis in a tensor +type TensorDim struct { + _tab flatbuffers.Table +} + +func GetRootAsTensorDim(buf []byte, offset flatbuffers.UOffsetT) *TensorDim { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &TensorDim{} + x.Init(buf, n+offset) + return x +} + +func (rcv *TensorDim) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *TensorDim) Table() flatbuffers.Table { + return rcv._tab +} + +/// Length of dimension +func (rcv *TensorDim) Size() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +/// Length of dimension +func (rcv *TensorDim) MutateSize(n int64) bool { + return rcv._tab.MutateInt64Slot(4, n) +} + +/// Name of the dimension, optional +func (rcv *TensorDim) Name() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +/// Name of the dimension, optional +func TensorDimStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func TensorDimAddSize(builder *flatbuffers.Builder, size int64) { + builder.PrependInt64Slot(0, size, 0) +} +func TensorDimAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(name), 0) +} +func TensorDimEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Time.go b/go/arrow/internal/flatbuf/Time.go new file mode 100644 index 00000000000..3f3b7fcceee --- /dev/null +++ b/go/arrow/internal/flatbuf/Time.go @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// Time type. The physical storage type depends on the unit +/// - SECOND and MILLISECOND: 32 bits +/// - MICROSECOND and NANOSECOND: 64 bits +type Time struct { + _tab flatbuffers.Table +} + +func GetRootAsTime(buf []byte, offset flatbuffers.UOffsetT) *Time { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Time{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Time) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Time) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Time) Unit() int16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 1 +} + +func (rcv *Time) MutateUnit(n int16) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func (rcv *Time) BitWidth() int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.GetInt32(o + rcv._tab.Pos) + } + return 32 +} + +func (rcv *Time) MutateBitWidth(n int32) bool { + return rcv._tab.MutateInt32Slot(6, n) +} + +func TimeStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func TimeAddUnit(builder *flatbuffers.Builder, unit int16) { + builder.PrependInt16Slot(0, unit, 1) +} +func TimeAddBitWidth(builder *flatbuffers.Builder, bitWidth int32) { + builder.PrependInt32Slot(1, bitWidth, 32) +} +func TimeEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/TimeUnit.go b/go/arrow/internal/flatbuf/TimeUnit.go new file mode 100644 index 00000000000..cb0c7427d6c --- /dev/null +++ b/go/arrow/internal/flatbuf/TimeUnit.go @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type TimeUnit = int16 +const ( + TimeUnitSECOND TimeUnit = 0 + TimeUnitMILLISECOND TimeUnit = 1 + TimeUnitMICROSECOND TimeUnit = 2 + TimeUnitNANOSECOND TimeUnit = 3 +) + +var EnumNamesTimeUnit = map[TimeUnit]string{ + TimeUnitSECOND:"SECOND", + TimeUnitMILLISECOND:"MILLISECOND", + TimeUnitMICROSECOND:"MICROSECOND", + TimeUnitNANOSECOND:"NANOSECOND", +} + diff --git a/go/arrow/internal/flatbuf/Timestamp.go b/go/arrow/internal/flatbuf/Timestamp.go new file mode 100644 index 00000000000..ba966295d6d --- /dev/null +++ b/go/arrow/internal/flatbuf/Timestamp.go @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, excluding +/// leap seconds, as a 64-bit integer. Note that UNIX time does not include +/// leap seconds. +/// +/// The Timestamp metadata supports both "time zone naive" and "time zone +/// aware" timestamps. Read about the timezone attribute for more detail +type Timestamp struct { + _tab flatbuffers.Table +} + +func GetRootAsTimestamp(buf []byte, offset flatbuffers.UOffsetT) *Timestamp { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Timestamp{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Timestamp) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Timestamp) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Timestamp) Unit() int16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Timestamp) MutateUnit(n int16) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +/// The time zone is a string indicating the name of a time zone, one of: +/// +/// * As used in the Olson time zone database (the "tz database" or +/// "tzdata"), such as "America/New_York" +/// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30 +/// +/// Whether a timezone string is present indicates different semantics about +/// the data: +/// +/// * If the time zone is null or equal to an empty string, the data is "time +/// zone naive" and shall be displayed *as is* to the user, not localized +/// to the locale of the user. This data can be though of as UTC but +/// without having "UTC" as the time zone, it is not considered to be +/// localized to any time zone +/// +/// * If the time zone is set to a valid value, values can be displayed as +/// "localized" to that time zone, even though the underlying 64-bit +/// integers are identical to the same data stored in UTC. Converting +/// between time zones is a metadata-only operation and does not change the +/// underlying values +func (rcv *Timestamp) Timezone() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +/// The time zone is a string indicating the name of a time zone, one of: +/// +/// * As used in the Olson time zone database (the "tz database" or +/// "tzdata"), such as "America/New_York" +/// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30 +/// +/// Whether a timezone string is present indicates different semantics about +/// the data: +/// +/// * If the time zone is null or equal to an empty string, the data is "time +/// zone naive" and shall be displayed *as is* to the user, not localized +/// to the locale of the user. This data can be though of as UTC but +/// without having "UTC" as the time zone, it is not considered to be +/// localized to any time zone +/// +/// * If the time zone is set to a valid value, values can be displayed as +/// "localized" to that time zone, even though the underlying 64-bit +/// integers are identical to the same data stored in UTC. Converting +/// between time zones is a metadata-only operation and does not change the +/// underlying values +func TimestampStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func TimestampAddUnit(builder *flatbuffers.Builder, unit int16) { + builder.PrependInt16Slot(0, unit, 0) +} +func TimestampAddTimezone(builder *flatbuffers.Builder, timezone flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(timezone), 0) +} +func TimestampEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/Type.go b/go/arrow/internal/flatbuf/Type.go new file mode 100644 index 00000000000..c8bb7f849ab --- /dev/null +++ b/go/arrow/internal/flatbuf/Type.go @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +/// ---------------------------------------------------------------------- +/// Top-level Type value, enabling extensible type-specific metadata. We can +/// add new logical types to Type without breaking backwards compatibility +type Type = byte +const ( + TypeNONE Type = 0 + TypeNull Type = 1 + TypeInt Type = 2 + TypeFloatingPoint Type = 3 + TypeBinary Type = 4 + TypeUtf8 Type = 5 + TypeBool Type = 6 + TypeDecimal Type = 7 + TypeDate Type = 8 + TypeTime Type = 9 + TypeTimestamp Type = 10 + TypeInterval Type = 11 + TypeList Type = 12 + TypeStruct_ Type = 13 + TypeUnion Type = 14 + TypeFixedSizeBinary Type = 15 + TypeFixedSizeList Type = 16 + TypeMap Type = 17 +) + +var EnumNamesType = map[Type]string{ + TypeNONE:"NONE", + TypeNull:"Null", + TypeInt:"Int", + TypeFloatingPoint:"FloatingPoint", + TypeBinary:"Binary", + TypeUtf8:"Utf8", + TypeBool:"Bool", + TypeDecimal:"Decimal", + TypeDate:"Date", + TypeTime:"Time", + TypeTimestamp:"Timestamp", + TypeInterval:"Interval", + TypeList:"List", + TypeStruct_:"Struct_", + TypeUnion:"Union", + TypeFixedSizeBinary:"FixedSizeBinary", + TypeFixedSizeList:"FixedSizeList", + TypeMap:"Map", +} + diff --git a/go/arrow/internal/flatbuf/Union.go b/go/arrow/internal/flatbuf/Union.go new file mode 100644 index 00000000000..ccf988a5084 --- /dev/null +++ b/go/arrow/internal/flatbuf/Union.go @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// A union is a complex type with children in Field +/// By default ids in the type vector refer to the offsets in the children +/// optionally typeIds provides an indirection between the child offset and the type id +/// for each child typeIds[offset] is the id used in the type vector +type Union struct { + _tab flatbuffers.Table +} + +func GetRootAsUnion(buf []byte, offset flatbuffers.UOffsetT) *Union { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Union{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Union) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Union) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Union) Mode() int16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Union) MutateMode(n int16) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func (rcv *Union) TypeIds(j int) int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.GetInt32(a + flatbuffers.UOffsetT(j*4)) + } + return 0 +} + +func (rcv *Union) TypeIdsLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func UnionStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func UnionAddMode(builder *flatbuffers.Builder, mode int16) { + builder.PrependInt16Slot(0, mode, 0) +} +func UnionAddTypeIds(builder *flatbuffers.Builder, typeIds flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(typeIds), 0) +} +func UnionStartTypeIdsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func UnionEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/internal/flatbuf/UnionMode.go b/go/arrow/internal/flatbuf/UnionMode.go new file mode 100644 index 00000000000..4357682ad4d --- /dev/null +++ b/go/arrow/internal/flatbuf/UnionMode.go @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type UnionMode = int16 +const ( + UnionModeSparse UnionMode = 0 + UnionModeDense UnionMode = 1 +) + +var EnumNamesUnionMode = map[UnionMode]string{ + UnionModeSparse:"Sparse", + UnionModeDense:"Dense", +} + diff --git a/go/arrow/internal/flatbuf/Utf8.go b/go/arrow/internal/flatbuf/Utf8.go new file mode 100644 index 00000000000..4ff365a3750 --- /dev/null +++ b/go/arrow/internal/flatbuf/Utf8.go @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// Unicode with UTF-8 encoding +type Utf8 struct { + _tab flatbuffers.Table +} + +func GetRootAsUtf8(buf []byte, offset flatbuffers.UOffsetT) *Utf8 { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Utf8{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Utf8) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Utf8) Table() flatbuffers.Table { + return rcv._tab +} + +func Utf8Start(builder *flatbuffers.Builder) { + builder.StartObject(0) +} +func Utf8End(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/ipc/cmd/arrow-cat/main.go b/go/arrow/ipc/cmd/arrow-cat/main.go new file mode 100644 index 00000000000..cb4ff5a121d --- /dev/null +++ b/go/arrow/ipc/cmd/arrow-cat/main.go @@ -0,0 +1,210 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command arrow-cat displays the content of an Arrow stream or file. +// +// Examples: +// +// $> arrow-cat ./testdata/primitives.data +// version: V4 +// record 1/3... +// col[0] "bools": [true (null) (null) false true] +// col[1] "int8s": [-1 (null) (null) -4 -5] +// col[2] "int16s": [-1 (null) (null) -4 -5] +// col[3] "int32s": [-1 (null) (null) -4 -5] +// col[4] "int64s": [-1 (null) (null) -4 -5] +// col[5] "uint8s": [1 (null) (null) 4 5] +// col[6] "uint16s": [1 (null) (null) 4 5] +// col[7] "uint32s": [1 (null) (null) 4 5] +// col[8] "uint64s": [1 (null) (null) 4 5] +// col[9] "float32s": [1 (null) (null) 4 5] +// col[10] "float64s": [1 (null) (null) 4 5] +// record 2/3... +// col[0] "bools": [true (null) (null) false true] +// [...] +// +// $> gen-arrow-stream | arrow-cat +// record 1... +// col[0] "bools": [true (null) (null) false true] +// col[1] "int8s": [-1 (null) (null) -4 -5] +// col[2] "int16s": [-1 (null) (null) -4 -5] +// col[3] "int32s": [-1 (null) (null) -4 -5] +// col[4] "int64s": [-1 (null) (null) -4 -5] +// col[5] "uint8s": [1 (null) (null) 4 5] +// col[6] "uint16s": [1 (null) (null) 4 5] +// col[7] "uint32s": [1 (null) (null) 4 5] +// col[8] "uint64s": [1 (null) (null) 4 5] +// col[9] "float32s": [1 (null) (null) 4 5] +// col[10] "float64s": [1 (null) (null) 4 5] +// record 2... +// col[0] "bools": [true (null) (null) false true] +// [...] +package main // import "github.com/apache/arrow/go/arrow/ipc/cmd/arrow-cat" + +import ( + "bytes" + "flag" + "fmt" + "io" + "log" + "os" + + "github.com/apache/arrow/go/arrow/ipc" + "github.com/apache/arrow/go/arrow/memory" + "github.com/pkg/errors" +) + +func main() { + log.SetPrefix("arrow-cat: ") + log.SetFlags(0) + + flag.Parse() + + var err error + switch flag.NArg() { + case 0: + err = processStream(os.Stdout, os.Stdin) + default: + err = processFiles(os.Stdout, flag.Args()) + } + if err != nil { + log.Fatal(err) + } +} + +func processStream(w io.Writer, rin io.Reader) error { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(nil, 0) + + r, err := ipc.NewReader(rin, ipc.WithAllocator(mem)) + if err != nil { + return err + } + defer r.Release() + + n := 0 + for r.Next() { + n++ + fmt.Fprintf(w, "record %d...\n", n) + rec := r.Record() + for i, col := range rec.Columns() { + fmt.Fprintf(w, " col[%d] %q: %v\n", i, rec.ColumnName(i), col) + } + } + return nil +} + +func processFiles(w io.Writer, names []string) error { + for _, name := range names { + err := processFile(w, name) + if err != nil { + return err + } + } + return nil +} + +func processFile(w io.Writer, fname string) error { + + f, err := os.Open(fname) + if err != nil { + return err + } + defer f.Close() + + hdr := make([]byte, len(ipc.Magic)) + _, err = io.ReadFull(f, hdr) + if err != nil { + return errors.Errorf("could not read file header: %v", err) + } + f.Seek(0, io.SeekStart) + + if !bytes.Equal(hdr, ipc.Magic) { + // try as a stream. + return processStream(w, f) + } + + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(nil, 0) + + r, err := ipc.NewFileReader(f, ipc.WithAllocator(mem)) + if err != nil { + return err + } + defer r.Close() + + fmt.Fprintf(w, "version: %v\n", r.Version()) + for i := 0; i < r.NumRecords(); i++ { + fmt.Fprintf(w, "record %d/%d...\n", i+1, r.NumRecords()) + rec, err := r.Record(i) + if err != nil { + return err + } + defer rec.Release() + + for i, col := range rec.Columns() { + fmt.Fprintf(w, " col[%d] %q: %v\n", i, rec.ColumnName(i), col) + } + } + return nil +} + +func init() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, `Command arrow-cat displays the content of an Arrow stream or file. + +Usage: arrow-cat [OPTIONS] [FILE1 [FILE2 [...]]] + +Examples: + + $> arrow-cat ./testdata/primitives.data + version: V4 + record 1/3... + col[0] "bools": [true (null) (null) false true] + col[1] "int8s": [-1 (null) (null) -4 -5] + col[2] "int16s": [-1 (null) (null) -4 -5] + col[3] "int32s": [-1 (null) (null) -4 -5] + col[4] "int64s": [-1 (null) (null) -4 -5] + col[5] "uint8s": [1 (null) (null) 4 5] + col[6] "uint16s": [1 (null) (null) 4 5] + col[7] "uint32s": [1 (null) (null) 4 5] + col[8] "uint64s": [1 (null) (null) 4 5] + col[9] "float32s": [1 (null) (null) 4 5] + col[10] "float64s": [1 (null) (null) 4 5] + record 2/3... + col[0] "bools": [true (null) (null) false true] + [...] + + $> gen-arrow-stream | arrow-cat + record 1... + col[0] "bools": [true (null) (null) false true] + col[1] "int8s": [-1 (null) (null) -4 -5] + col[2] "int16s": [-1 (null) (null) -4 -5] + col[3] "int32s": [-1 (null) (null) -4 -5] + col[4] "int64s": [-1 (null) (null) -4 -5] + col[5] "uint8s": [1 (null) (null) 4 5] + col[6] "uint16s": [1 (null) (null) 4 5] + col[7] "uint32s": [1 (null) (null) 4 5] + col[8] "uint64s": [1 (null) (null) 4 5] + col[9] "float32s": [1 (null) (null) 4 5] + col[10] "float64s": [1 (null) (null) 4 5] + record 2... + col[0] "bools": [true (null) (null) false true] + [...] +`) + os.Exit(0) + } +} diff --git a/go/arrow/ipc/cmd/arrow-ls/main.go b/go/arrow/ipc/cmd/arrow-ls/main.go new file mode 100644 index 00000000000..b858bf5c5db --- /dev/null +++ b/go/arrow/ipc/cmd/arrow-ls/main.go @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command arrow-ls displays the listing of an Arrow file. +// +// Examples: +// +// $> arrow-ls ./testdata/primitives.data +// version: V4 +// schema: +// fields: 11 +// - bools: type=bool, nullable +// - int8s: type=int8, nullable +// - int16s: type=int16, nullable +// - int32s: type=int32, nullable +// - int64s: type=int64, nullable +// - uint8s: type=uint8, nullable +// - uint16s: type=uint16, nullable +// - uint32s: type=uint32, nullable +// - uint64s: type=uint64, nullable +// - float32s: type=float32, nullable +// - float64s: type=float64, nullable +// records: 3 +// +// $> gen-arrow-stream | arrow-ls +// schema: +// fields: 11 +// - bools: type=bool, nullable +// - int8s: type=int8, nullable +// - int16s: type=int16, nullable +// - int32s: type=int32, nullable +// - int64s: type=int64, nullable +// - uint8s: type=uint8, nullable +// - uint16s: type=uint16, nullable +// - uint32s: type=uint32, nullable +// - uint64s: type=uint64, nullable +// - float32s: type=float32, nullable +// - float64s: type=float64, nullable +// records: 3 +package main // import "github.com/apache/arrow/go/arrow/ipc/cmd/arrow-ls" + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "strings" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/ipc" + "github.com/apache/arrow/go/arrow/memory" +) + +func main() { + log.SetPrefix("arrow-ls: ") + log.SetFlags(0) + + flag.Parse() + + var err error + switch flag.NArg() { + case 0: + err = processStream(os.Stdout, os.Stdin) + default: + err = processFiles(os.Stdout, flag.Args()) + } + if err != nil { + log.Fatal(err) + } +} + +func processStream(w io.Writer, rin io.Reader) error { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(nil, 0) + + r, err := ipc.NewReader(rin, ipc.WithAllocator(mem)) + if err != nil { + return err + } + defer r.Release() + + fmt.Fprintf(w, "schema:\n%v", displaySchema(r.Schema())) + + nrecs := 0 + for r.Next() { + nrecs++ + } + fmt.Fprintf(w, "records: %d\n", nrecs) + return nil +} + +func processFiles(w io.Writer, names []string) error { + for _, name := range names { + err := processFile(w, name) + if err != nil { + return err + } + } + return nil +} + +func processFile(w io.Writer, fname string) error { + + f, err := os.Open(fname) + if err != nil { + return err + } + defer f.Close() + + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(nil, 0) + + r, err := ipc.NewFileReader(f, ipc.WithAllocator(mem)) + if err != nil { + return err + } + defer r.Close() + + fmt.Fprintf(w, "version: %v\n", r.Version()) + fmt.Fprintf(w, "schema:\n%v", displaySchema(r.Schema())) + fmt.Fprintf(w, "records: %d\n", r.NumRecords()) + return nil +} + +func displaySchema(s *arrow.Schema) string { + o := new(strings.Builder) + fmt.Fprintf(o, "%*.sfields: %d\n", 2, "", len(s.Fields())) + for _, f := range s.Fields() { + displayField(o, f, 4) + } + if meta := s.Metadata(); meta.Len() > 0 { + fmt.Fprintf(o, "metadata: %v\n", meta) + } + return o.String() +} + +func displayField(o io.Writer, field arrow.Field, inc int) { + nullable := "" + if field.Nullable { + nullable = ", nullable" + } + fmt.Fprintf(o, "%*.s- %s: type=%v%v\n", inc, "", field.Name, field.Type.Name(), nullable) + if field.HasMetadata() { + fmt.Fprintf(o, "%*.smetadata: %v\n", inc, "", field.Metadata) + } +} + +func init() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, `Command arrow-ls displays the listing of an Arrow file. + +Usage: arrow-ls [OPTIONS] [FILE1 [FILE2 [...]]] + +Examples: + + $> arrow-ls ./testdata/primitives.data + version: V4 + schema: + fields: 11 + - bools: type=bool, nullable + - int8s: type=int8, nullable + - int16s: type=int16, nullable + - int32s: type=int32, nullable + - int64s: type=int64, nullable + - uint8s: type=uint8, nullable + - uint16s: type=uint16, nullable + - uint32s: type=uint32, nullable + - uint64s: type=uint64, nullable + - float32s: type=float32, nullable + - float64s: type=float64, nullable + records: 3 + + $> gen-arrow-stream | arrow-ls + schema: + fields: 11 + - bools: type=bool, nullable + - int8s: type=int8, nullable + - int16s: type=int16, nullable + - int32s: type=int32, nullable + - int64s: type=int64, nullable + - uint8s: type=uint8, nullable + - uint16s: type=uint16, nullable + - uint32s: type=uint32, nullable + - uint64s: type=uint64, nullable + - float32s: type=float32, nullable + - float64s: type=float64, nullable + records: 3 +`) + os.Exit(0) + } +} diff --git a/go/arrow/ipc/dict.go b/go/arrow/ipc/dict.go new file mode 100644 index 00000000000..802a5acf157 --- /dev/null +++ b/go/arrow/ipc/dict.go @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/array" + "github.com/pkg/errors" +) + +type dictMap map[int64]array.Interface +type dictTypeMap map[int64]arrow.Field + +type dictMemo struct { + dict2id map[array.Interface]int64 + id2dict dictMap // map of dictionary ID to dictionary array +} + +func newMemo() dictMemo { + return dictMemo{ + dict2id: make(map[array.Interface]int64), + id2dict: make(dictMap), + } +} + +func (memo *dictMemo) Len() int { return len(memo.id2dict) } + +func (memo *dictMemo) delete() { + for id, v := range memo.id2dict { + delete(memo.id2dict, id) + delete(memo.dict2id, v) + v.Release() + } +} + +func (memo dictMemo) Dict(id int64) (array.Interface, bool) { + v, ok := memo.id2dict[id] + return v, ok +} + +func (memo *dictMemo) ID(v array.Interface) int64 { + id, ok := memo.dict2id[v] + if ok { + return id + } + + v.Retain() + id = int64(len(memo.dict2id)) + memo.dict2id[v] = id + memo.id2dict[id] = v + return id +} + +func (memo dictMemo) HasDict(v array.Interface) bool { + _, ok := memo.dict2id[v] + return ok +} + +func (memo dictMemo) HasID(id int64) bool { + _, ok := memo.id2dict[id] + return ok +} + +func (memo *dictMemo) Add(id int64, v array.Interface) { + if _, dup := memo.id2dict[id]; dup { + panic(errors.Errorf("arrow/ipc: duplicate id=%d", id)) + } + v.Retain() + memo.id2dict[id] = v + memo.dict2id[v] = id +} diff --git a/go/arrow/ipc/dict_test.go b/go/arrow/ipc/dict_test.go new file mode 100644 index 00000000000..ba511840fe0 --- /dev/null +++ b/go/arrow/ipc/dict_test.go @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "fmt" + "testing" + + "github.com/apache/arrow/go/arrow/array" + "github.com/apache/arrow/go/arrow/memory" +) + +func TestDictMemo(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + bldr := array.NewFloat64Builder(mem) + defer bldr.Release() + + bldr.AppendValues([]float64{1.0, 1.1, 1.2, 1.3}, nil) + f0 := bldr.NewFloat64Array() + defer f0.Release() + + bldr.AppendValues([]float64{11.0, 11.1, 11.2, 11.3}, nil) + f1 := bldr.NewFloat64Array() + defer f1.Release() + + bldr.AppendValues([]float64{11.0, 11.1, 11.2, 11.3}, nil) + f2 := bldr.NewFloat64Array() + defer f2.Release() + + memo := newMemo() + defer memo.delete() + + if got, want := memo.Len(), 0; got != want { + t.Fatalf("invalid length: got=%d, want=%d", got, want) + } + + memo.Add(0, f0) + memo.Add(1, f1) + + if !memo.HasID(0) { + t.Fatalf("could not find id=0") + } + + if !memo.HasID(1) { + t.Fatalf("could not find id=1") + } + + if got, want := memo.Len(), 2; got != want { + t.Fatalf("invalid length: got=%d, want=%d", got, want) + } + + var ff array.Interface + + ff = f0 + if !memo.HasDict(ff) { + t.Fatalf("failed to find f0 through interface") + } + + ff = f1 + if !memo.HasDict(ff) { + t.Fatalf("failed to find f1 through interface") + } + + ff = f2 + if memo.HasDict(ff) { + t.Fatalf("should not have found f2") + } + + fct := func(v array.Interface) array.Interface { + return v + } + + if !memo.HasDict(fct(f1)) { + t.Fatalf("failed to find dict through func through interface") + } + + if memo.HasDict(f2) { + t.Fatalf("should not have found f2") + } + + ff = f0 + for i, f := range []array.Interface{f0, f1, ff, fct(f0), fct(f1)} { + if !memo.HasDict(f) { + t.Fatalf("failed to find dict %d", i) + } + } + + v, ok := memo.Dict(0) + if !ok { + t.Fatalf("expected to find id=0") + } + if v != f0 { + t.Fatalf("expected fo find id=0 array") + } + + v, ok = memo.Dict(2) + if ok { + t.Fatalf("should not have found id=2") + } + v, ok = memo.Dict(-2) + if ok { + t.Fatalf("should not have found id=-2") + } + + if got, want := memo.ID(f0), int64(0); got != want { + t.Fatalf("found invalid id. got=%d, want=%d", got, want) + } + + if got, want := memo.ID(f2), int64(2); got != want { + t.Fatalf("found invalid id. got=%d, want=%d", got, want) + } + if !memo.HasDict(f2) { + t.Fatalf("should have found f2") + } + + // test we don't leak nor "double-delete" when adding an array multiple times. + memo.Add(42, f2) + if got, want := memo.ID(f2), int64(42); got != want { + t.Fatalf("found invalid id. got=%d, want=%d", got, want) + } + memo.Add(43, f2) + if got, want := memo.ID(f2), int64(43); got != want { + t.Fatalf("found invalid id. got=%d, want=%d", got, want) + } + if got, want := memo.Len(), 5; got != want { + t.Fatalf("invalid length. got=%d, want=%d", got, want) + } +} + +func TestDictMemoPanics(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + bldr := array.NewFloat64Builder(mem) + defer bldr.Release() + + bldr.AppendValues([]float64{1.0, 1.1, 1.2, 1.3}, nil) + f0 := bldr.NewFloat64Array() + defer f0.Release() + + bldr.AppendValues([]float64{11.0, 11.1, 11.2, 11.3}, nil) + f1 := bldr.NewFloat64Array() + defer f1.Release() + + for _, tc := range []struct { + vs []array.Interface + ids []int64 + }{ + { + vs: []array.Interface{f0, f1}, + ids: []int64{0, 0}, + }, + { + vs: []array.Interface{f0, f0}, + ids: []int64{0, 0}, + }, + } { + t.Run("", func(t *testing.T) { + defer func() { + e := recover() + if e == nil { + t.Fatalf("should have panicked!") + } + if got, want := e.(error), fmt.Errorf("arrow/ipc: duplicate id=%d", 0); got.Error() != want.Error() { + t.Fatalf("invalid panic message.\ngot= %q\nwant=%q", got, want) + } + }() + + memo := newMemo() + defer memo.delete() + + if got, want := memo.Len(), 0; got != want { + t.Fatalf("invalid length: got=%d, want=%d", got, want) + } + + memo.Add(tc.ids[0], tc.vs[0]) + memo.Add(tc.ids[1], tc.vs[1]) + }) + } +} diff --git a/go/arrow/ipc/feather/fbs/CTable.go b/go/arrow/ipc/feather/fbs/CTable.go new file mode 100644 index 00000000000..09d262fc76d --- /dev/null +++ b/go/arrow/ipc/feather/fbs/CTable.go @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package fbs + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type CTable struct { + _tab flatbuffers.Table +} + +func GetRootAsCTable(buf []byte, offset flatbuffers.UOffsetT) *CTable { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &CTable{} + x.Init(buf, n+offset) + return x +} + +func (rcv *CTable) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *CTable) Table() flatbuffers.Table { + return rcv._tab +} + +/// Some text (or a name) metadata about what the file is, optional +func (rcv *CTable) Description() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +/// Some text (or a name) metadata about what the file is, optional +func (rcv *CTable) NumRows() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *CTable) MutateNumRows(n int64) bool { + return rcv._tab.MutateInt64Slot(6, n) +} + +func (rcv *CTable) Columns(obj *Column, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *CTable) ColumnsLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +/// Version number of the Feather format +func (rcv *CTable) Version() int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.GetInt32(o + rcv._tab.Pos) + } + return 0 +} + +/// Version number of the Feather format +func (rcv *CTable) MutateVersion(n int32) bool { + return rcv._tab.MutateInt32Slot(10, n) +} + +/// Table metadata (likely JSON), not yet used +func (rcv *CTable) Metadata() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +/// Table metadata (likely JSON), not yet used +func CTableStart(builder *flatbuffers.Builder) { + builder.StartObject(5) +} +func CTableAddDescription(builder *flatbuffers.Builder, description flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(description), 0) +} +func CTableAddNumRows(builder *flatbuffers.Builder, numRows int64) { + builder.PrependInt64Slot(1, numRows, 0) +} +func CTableAddColumns(builder *flatbuffers.Builder, columns flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(columns), 0) +} +func CTableStartColumnsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func CTableAddVersion(builder *flatbuffers.Builder, version int32) { + builder.PrependInt32Slot(3, version, 0) +} +func CTableAddMetadata(builder *flatbuffers.Builder, metadata flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(metadata), 0) +} +func CTableEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/ipc/feather/fbs/CategoryMetadata.go b/go/arrow/ipc/feather/fbs/CategoryMetadata.go new file mode 100644 index 00000000000..e2b56f28c07 --- /dev/null +++ b/go/arrow/ipc/feather/fbs/CategoryMetadata.go @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package fbs + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type CategoryMetadata struct { + _tab flatbuffers.Table +} + +func GetRootAsCategoryMetadata(buf []byte, offset flatbuffers.UOffsetT) *CategoryMetadata { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &CategoryMetadata{} + x.Init(buf, n+offset) + return x +} + +func (rcv *CategoryMetadata) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *CategoryMetadata) Table() flatbuffers.Table { + return rcv._tab +} + +/// The category codes are presumed to be integers that are valid indexes into +/// the levels array +func (rcv *CategoryMetadata) Levels(obj *PrimitiveArray) *PrimitiveArray { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + x := rcv._tab.Indirect(o + rcv._tab.Pos) + if obj == nil { + obj = new(PrimitiveArray) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// The category codes are presumed to be integers that are valid indexes into +/// the levels array +func (rcv *CategoryMetadata) Ordered() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *CategoryMetadata) MutateOrdered(n byte) bool { + return rcv._tab.MutateByteSlot(6, n) +} + +func CategoryMetadataStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func CategoryMetadataAddLevels(builder *flatbuffers.Builder, levels flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(levels), 0) +} +func CategoryMetadataAddOrdered(builder *flatbuffers.Builder, ordered byte) { + builder.PrependByteSlot(1, ordered, 0) +} +func CategoryMetadataEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/ipc/feather/fbs/Column.go b/go/arrow/ipc/feather/fbs/Column.go new file mode 100644 index 00000000000..275ce4c4ec4 --- /dev/null +++ b/go/arrow/ipc/feather/fbs/Column.go @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package fbs + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Column struct { + _tab flatbuffers.Table +} + +func GetRootAsColumn(buf []byte, offset flatbuffers.UOffsetT) *Column { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Column{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Column) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Column) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Column) Name() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *Column) Values(obj *PrimitiveArray) *PrimitiveArray { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + x := rcv._tab.Indirect(o + rcv._tab.Pos) + if obj == nil { + obj = new(PrimitiveArray) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +func (rcv *Column) MetadataType() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Column) MutateMetadataType(n byte) bool { + return rcv._tab.MutateByteSlot(8, n) +} + +func (rcv *Column) Metadata(obj *flatbuffers.Table) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + rcv._tab.Union(obj, o) + return true + } + return false +} + +/// This should (probably) be JSON +func (rcv *Column) UserMetadata() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +/// This should (probably) be JSON +func ColumnStart(builder *flatbuffers.Builder) { + builder.StartObject(5) +} +func ColumnAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(name), 0) +} +func ColumnAddValues(builder *flatbuffers.Builder, values flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(values), 0) +} +func ColumnAddMetadataType(builder *flatbuffers.Builder, metadataType byte) { + builder.PrependByteSlot(2, metadataType, 0) +} +func ColumnAddMetadata(builder *flatbuffers.Builder, metadata flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(metadata), 0) +} +func ColumnAddUserMetadata(builder *flatbuffers.Builder, userMetadata flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(userMetadata), 0) +} +func ColumnEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/ipc/feather/fbs/DateMetadata.go b/go/arrow/ipc/feather/fbs/DateMetadata.go new file mode 100644 index 00000000000..bbe0523a06a --- /dev/null +++ b/go/arrow/ipc/feather/fbs/DateMetadata.go @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package fbs + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type DateMetadata struct { + _tab flatbuffers.Table +} + +func GetRootAsDateMetadata(buf []byte, offset flatbuffers.UOffsetT) *DateMetadata { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &DateMetadata{} + x.Init(buf, n+offset) + return x +} + +func (rcv *DateMetadata) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *DateMetadata) Table() flatbuffers.Table { + return rcv._tab +} + +func DateMetadataStart(builder *flatbuffers.Builder) { + builder.StartObject(0) +} +func DateMetadataEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/ipc/feather/fbs/Encoding.go b/go/arrow/ipc/feather/fbs/Encoding.go new file mode 100644 index 00000000000..7cf367962d4 --- /dev/null +++ b/go/arrow/ipc/feather/fbs/Encoding.go @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package fbs + +type Encoding = int8 +const ( + EncodingPLAIN Encoding = 0 + /// Data is stored dictionary-encoded + /// dictionary size: + /// dictionary data: + /// dictionary index: + /// + /// TODO: do we care about storing the index values in a smaller typeclass + EncodingDICTIONARY Encoding = 1 +) + +var EnumNamesEncoding = map[Encoding]string{ + EncodingPLAIN:"PLAIN", + EncodingDICTIONARY:"DICTIONARY", +} + diff --git a/go/arrow/ipc/feather/fbs/PrimitiveArray.go b/go/arrow/ipc/feather/fbs/PrimitiveArray.go new file mode 100644 index 00000000000..18e81b4c51b --- /dev/null +++ b/go/arrow/ipc/feather/fbs/PrimitiveArray.go @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package fbs + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type PrimitiveArray struct { + _tab flatbuffers.Table +} + +func GetRootAsPrimitiveArray(buf []byte, offset flatbuffers.UOffsetT) *PrimitiveArray { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &PrimitiveArray{} + x.Init(buf, n+offset) + return x +} + +func (rcv *PrimitiveArray) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *PrimitiveArray) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *PrimitiveArray) Type() int8 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt8(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *PrimitiveArray) MutateType(n int8) bool { + return rcv._tab.MutateInt8Slot(4, n) +} + +func (rcv *PrimitiveArray) Encoding() int8 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.GetInt8(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *PrimitiveArray) MutateEncoding(n int8) bool { + return rcv._tab.MutateInt8Slot(6, n) +} + +/// Relative memory offset of the start of the array data excluding the size +/// of the metadata +func (rcv *PrimitiveArray) Offset() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +/// Relative memory offset of the start of the array data excluding the size +/// of the metadata +func (rcv *PrimitiveArray) MutateOffset(n int64) bool { + return rcv._tab.MutateInt64Slot(8, n) +} + +/// The number of logical values in the array +func (rcv *PrimitiveArray) Length() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +/// The number of logical values in the array +func (rcv *PrimitiveArray) MutateLength(n int64) bool { + return rcv._tab.MutateInt64Slot(10, n) +} + +/// The number of observed nulls +func (rcv *PrimitiveArray) NullCount() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +/// The number of observed nulls +func (rcv *PrimitiveArray) MutateNullCount(n int64) bool { + return rcv._tab.MutateInt64Slot(12, n) +} + +/// The total size of the actual data in the file +func (rcv *PrimitiveArray) TotalBytes() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +/// The total size of the actual data in the file +func (rcv *PrimitiveArray) MutateTotalBytes(n int64) bool { + return rcv._tab.MutateInt64Slot(14, n) +} + +func PrimitiveArrayStart(builder *flatbuffers.Builder) { + builder.StartObject(6) +} +func PrimitiveArrayAddType(builder *flatbuffers.Builder, type_ int8) { + builder.PrependInt8Slot(0, type_, 0) +} +func PrimitiveArrayAddEncoding(builder *flatbuffers.Builder, encoding int8) { + builder.PrependInt8Slot(1, encoding, 0) +} +func PrimitiveArrayAddOffset(builder *flatbuffers.Builder, offset int64) { + builder.PrependInt64Slot(2, offset, 0) +} +func PrimitiveArrayAddLength(builder *flatbuffers.Builder, length int64) { + builder.PrependInt64Slot(3, length, 0) +} +func PrimitiveArrayAddNullCount(builder *flatbuffers.Builder, nullCount int64) { + builder.PrependInt64Slot(4, nullCount, 0) +} +func PrimitiveArrayAddTotalBytes(builder *flatbuffers.Builder, totalBytes int64) { + builder.PrependInt64Slot(5, totalBytes, 0) +} +func PrimitiveArrayEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/ipc/feather/fbs/TimeMetadata.go b/go/arrow/ipc/feather/fbs/TimeMetadata.go new file mode 100644 index 00000000000..63408ae576f --- /dev/null +++ b/go/arrow/ipc/feather/fbs/TimeMetadata.go @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package fbs + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type TimeMetadata struct { + _tab flatbuffers.Table +} + +func GetRootAsTimeMetadata(buf []byte, offset flatbuffers.UOffsetT) *TimeMetadata { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &TimeMetadata{} + x.Init(buf, n+offset) + return x +} + +func (rcv *TimeMetadata) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *TimeMetadata) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *TimeMetadata) Unit() int8 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt8(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *TimeMetadata) MutateUnit(n int8) bool { + return rcv._tab.MutateInt8Slot(4, n) +} + +func TimeMetadataStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func TimeMetadataAddUnit(builder *flatbuffers.Builder, unit int8) { + builder.PrependInt8Slot(0, unit, 0) +} +func TimeMetadataEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/ipc/feather/fbs/TimeUnit.go b/go/arrow/ipc/feather/fbs/TimeUnit.go new file mode 100644 index 00000000000..afd3876b272 --- /dev/null +++ b/go/arrow/ipc/feather/fbs/TimeUnit.go @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package fbs + +type TimeUnit = int8 +const ( + TimeUnitSECOND TimeUnit = 0 + TimeUnitMILLISECOND TimeUnit = 1 + TimeUnitMICROSECOND TimeUnit = 2 + TimeUnitNANOSECOND TimeUnit = 3 +) + +var EnumNamesTimeUnit = map[TimeUnit]string{ + TimeUnitSECOND:"SECOND", + TimeUnitMILLISECOND:"MILLISECOND", + TimeUnitMICROSECOND:"MICROSECOND", + TimeUnitNANOSECOND:"NANOSECOND", +} + diff --git a/go/arrow/ipc/feather/fbs/TimestampMetadata.go b/go/arrow/ipc/feather/fbs/TimestampMetadata.go new file mode 100644 index 00000000000..86ac29392d4 --- /dev/null +++ b/go/arrow/ipc/feather/fbs/TimestampMetadata.go @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package fbs + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type TimestampMetadata struct { + _tab flatbuffers.Table +} + +func GetRootAsTimestampMetadata(buf []byte, offset flatbuffers.UOffsetT) *TimestampMetadata { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &TimestampMetadata{} + x.Init(buf, n+offset) + return x +} + +func (rcv *TimestampMetadata) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *TimestampMetadata) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *TimestampMetadata) Unit() int8 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt8(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *TimestampMetadata) MutateUnit(n int8) bool { + return rcv._tab.MutateInt8Slot(4, n) +} + +/// Timestamp data is assumed to be UTC, but the time zone is stored here for +/// presentation as localized +func (rcv *TimestampMetadata) Timezone() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +/// Timestamp data is assumed to be UTC, but the time zone is stored here for +/// presentation as localized +func TimestampMetadataStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func TimestampMetadataAddUnit(builder *flatbuffers.Builder, unit int8) { + builder.PrependInt8Slot(0, unit, 0) +} +func TimestampMetadataAddTimezone(builder *flatbuffers.Builder, timezone flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(timezone), 0) +} +func TimestampMetadataEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/arrow/ipc/feather/fbs/Type.go b/go/arrow/ipc/feather/fbs/Type.go new file mode 100644 index 00000000000..f6a6771aaca --- /dev/null +++ b/go/arrow/ipc/feather/fbs/Type.go @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package fbs + +/// Feather is an experimental serialization format implemented using +/// techniques from Apache Arrow. It was created as a proof-of-concept of an +/// interoperable file format for storing data frames originating in Python or +/// R. It enabled the developers to sidestep some of the open design questions +/// in Arrow from early 2016 and instead create something simple and useful for +/// the intended use cases. +type Type = int8 +const ( + TypeBOOL Type = 0 + TypeINT8 Type = 1 + TypeINT16 Type = 2 + TypeINT32 Type = 3 + TypeINT64 Type = 4 + TypeUINT8 Type = 5 + TypeUINT16 Type = 6 + TypeUINT32 Type = 7 + TypeUINT64 Type = 8 + TypeFLOAT Type = 9 + TypeDOUBLE Type = 10 + TypeUTF8 Type = 11 + TypeBINARY Type = 12 + TypeCATEGORY Type = 13 + TypeTIMESTAMP Type = 14 + TypeDATE Type = 15 + TypeTIME Type = 16 +) + +var EnumNamesType = map[Type]string{ + TypeBOOL:"BOOL", + TypeINT8:"INT8", + TypeINT16:"INT16", + TypeINT32:"INT32", + TypeINT64:"INT64", + TypeUINT8:"UINT8", + TypeUINT16:"UINT16", + TypeUINT32:"UINT32", + TypeUINT64:"UINT64", + TypeFLOAT:"FLOAT", + TypeDOUBLE:"DOUBLE", + TypeUTF8:"UTF8", + TypeBINARY:"BINARY", + TypeCATEGORY:"CATEGORY", + TypeTIMESTAMP:"TIMESTAMP", + TypeDATE:"DATE", + TypeTIME:"TIME", +} + diff --git a/go/arrow/ipc/feather/fbs/TypeMetadata.go b/go/arrow/ipc/feather/fbs/TypeMetadata.go new file mode 100644 index 00000000000..b28e9f1fa02 --- /dev/null +++ b/go/arrow/ipc/feather/fbs/TypeMetadata.go @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package fbs + +type TypeMetadata = byte +const ( + TypeMetadataNONE TypeMetadata = 0 + TypeMetadataCategoryMetadata TypeMetadata = 1 + TypeMetadataTimestampMetadata TypeMetadata = 2 + TypeMetadataDateMetadata TypeMetadata = 3 + TypeMetadataTimeMetadata TypeMetadata = 4 +) + +var EnumNamesTypeMetadata = map[TypeMetadata]string{ + TypeMetadataNONE:"NONE", + TypeMetadataCategoryMetadata:"CategoryMetadata", + TypeMetadataTimestampMetadata:"TimestampMetadata", + TypeMetadataDateMetadata:"DateMetadata", + TypeMetadataTimeMetadata:"TimeMetadata", +} + diff --git a/go/arrow/ipc/file_reader.go b/go/arrow/ipc/file_reader.go new file mode 100644 index 00000000000..4f8646eb11c --- /dev/null +++ b/go/arrow/ipc/file_reader.go @@ -0,0 +1,453 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/array" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/flatbuf" + "github.com/apache/arrow/go/arrow/memory" + "github.com/pkg/errors" +) + +// FileReader is an Arrow file reader. +type FileReader struct { + r ReadAtSeeker + + footer struct { + offset int64 + buffer *memory.Buffer + data *flatbuf.Footer + } + + fields dictTypeMap + memo dictMemo + + schema *arrow.Schema + record array.Record +} + +// NewFileReader opens an Arrow file using the provided reader r. +func NewFileReader(r ReadAtSeeker, opts ...Option) (*FileReader, error) { + var ( + f = FileReader{ + r: r, + fields: make(dictTypeMap), + memo: newMemo(), + } + cfg = newConfig() + err error + ) + + for _, opt := range opts { + opt(cfg) + } + + if cfg.footer.offset <= 0 { + cfg.footer.offset, err = f.r.Seek(0, io.SeekEnd) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could retrieve footer offset") + } + } + f.footer.offset = cfg.footer.offset + + err = f.readFooter() + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not decode footer") + } + + err = f.readSchema() + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not decode schema") + } + + return &f, err +} + +func (f *FileReader) readFooter() error { + var err error + + if f.footer.offset <= int64(len(Magic)*2+4) { + return fmt.Errorf("arrow/ipc: file too small (%d)", f.footer.offset) + } + + eof := int64(len(Magic) + 4) + buf := make([]byte, eof) + n, err := f.r.ReadAt(buf, f.footer.offset-eof) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not read footer") + } + if n != len(buf) { + return errors.Errorf("arrow/ipc: could not read %d bytes from end of file", len(buf)) + } + + if !bytes.Equal(buf[4:], Magic) { + return errNotArrowFile + } + + size := int64(binary.LittleEndian.Uint32(buf[:4])) + if size <= 0 || size+int64(len(Magic)*2+4) > f.footer.offset { + return errInconsistentFileMetadata + } + + buf = make([]byte, size) + n, err = f.r.ReadAt(buf, f.footer.offset-size-eof) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not read footer data") + } + if n != len(buf) { + return errors.Errorf("arrow/ipc: could not read %d bytes from footer data", len(buf)) + } + + f.footer.buffer = memory.NewBufferBytes(buf) + f.footer.data = flatbuf.GetRootAsFooter(buf, 0) + return err +} + +func (f *FileReader) readSchema() error { + var err error + f.fields, err = dictTypesFromFB(f.footer.data.Schema(nil)) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not load dictionary types from file") + } + + for i := 0; i < f.NumDictionaries(); i++ { + blk, err := f.dict(i) + if err != nil { + return errors.Wrapf(err, "arrow/ipc: could read dictionary[%d]", i) + } + switch { + case !bitutil.IsMultipleOf8(blk.Offset): + return errors.Errorf("arrow/ipc: invalid file offset=%d for dictionary %d", blk.Offset, i) + case !bitutil.IsMultipleOf8(int64(blk.Meta)): + return errors.Errorf("arrow/ipc: invalid file metadata=%d position for dictionary %d", blk.Meta, i) + case !bitutil.IsMultipleOf8(blk.Body): + return errors.Errorf("arrow/ipc: invalid file body=%d position for dictionary %d", blk.Body, i) + } + + msg, err := blk.NewMessage() + if err != nil { + return err + } + defer msg.Release() + + id, dict, err := readDictionary(msg.meta, f.fields, f.r) + if err != nil { + return errors.Wrapf(err, "arrow/ipc: could not read dictionary %d from file", i) + } + f.memo.Add(id, dict) + dict.Release() // memo.Add increases ref-count of dict. + } + + schema := f.footer.data.Schema(nil) + if schema == nil { + return errors.New("arrow/ipc: could not load schema from flatbuffer data") + } + f.schema, err = schemaFromFB(schema, &f.memo) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not read schema") + } + + return err +} + +func (f *FileReader) block(i int) (fileBlock, error) { + var blk flatbuf.Block + if !f.footer.data.RecordBatches(&blk, i) { + return fileBlock{}, errors.Errorf("arrow/ipc: could not extract file block %d", i) + } + + return fileBlock{ + Offset: blk.Offset(), + Meta: blk.MetaDataLength(), + Body: blk.BodyLength(), + r: f.r, + }, nil +} + +func (f *FileReader) dict(i int) (fileBlock, error) { + var blk flatbuf.Block + if !f.footer.data.Dictionaries(&blk, i) { + return fileBlock{}, errors.Errorf("arrow/ipc: could not extract dictionary block %d", i) + } + + return fileBlock{ + Offset: blk.Offset(), + Meta: blk.MetaDataLength(), + Body: blk.BodyLength(), + r: f.r, + }, nil +} + +func (f *FileReader) Schema() *arrow.Schema { + return f.schema +} + +func (f *FileReader) NumDictionaries() int { + if f.footer.data == nil { + return 0 + } + return f.footer.data.DictionariesLength() +} + +func (f *FileReader) NumRecords() int { + return f.footer.data.RecordBatchesLength() +} + +func (f *FileReader) Version() MetadataVersion { + return MetadataVersion(f.footer.data.Version()) +} + +// Close cleans up resources used by the File. +// Close does not close the underlying reader. +func (f *FileReader) Close() error { + if f.footer.data != nil { + f.footer.data = nil + } + + if f.footer.buffer != nil { + f.footer.buffer.Release() + f.footer.buffer = nil + } + + if f.record != nil { + f.record.Release() + f.record = nil + } + return nil +} + +// Record returns the i-th record from the file. +// The returned value is valid until the next call to Record. +// Users need to call Retain on that Record to keep it valid for longer. +func (f *FileReader) Record(i int) (array.Record, error) { + if i < 0 || i > f.NumRecords() { + panic("arrow/ipc: record index out of bounds") + } + + blk, err := f.block(i) + if err != nil { + return nil, err + } + switch { + case !bitutil.IsMultipleOf8(blk.Offset): + return nil, errors.Errorf("arrow/ipc: invalid file offset=%d for record %d", blk.Offset, i) + case !bitutil.IsMultipleOf8(int64(blk.Meta)): + return nil, errors.Errorf("arrow/ipc: invalid file metadata=%d position for record %d", blk.Meta, i) + case !bitutil.IsMultipleOf8(blk.Body): + return nil, errors.Errorf("arrow/ipc: invalid file body=%d position for record %d", blk.Body, i) + } + + msg, err := blk.NewMessage() + if err != nil { + return nil, err + } + defer msg.Release() + + if msg.Type() != MessageRecordBatch { + return nil, errors.Errorf("arrow/ipc: message %d is not a Record", i) + } + + if f.record != nil { + f.record.Release() + } + + f.record = newRecord(f.schema, msg.meta, bytes.NewReader(msg.body.Bytes())) + return f.record, nil +} + +func newRecord(schema *arrow.Schema, meta *memory.Buffer, body ReadAtSeeker) array.Record { + var ( + msg = flatbuf.GetRootAsMessage(meta.Bytes(), 0) + md flatbuf.RecordBatch + ) + initFB(&md, msg.Header) + rows := md.Length() + + ctx := &arrayLoaderContext{ + src: ipcSource{ + meta: &md, + r: body, + }, + max: kMaxNestingDepth, + } + + cols := make([]array.Interface, len(schema.Fields())) + for i, field := range schema.Fields() { + cols[i] = ctx.loadArray(field.Type) + } + + return array.NewRecord(schema, cols, rows) +} + +type ipcSource struct { + meta *flatbuf.RecordBatch + r ReadAtSeeker +} + +func (src *ipcSource) buffer(i int) *memory.Buffer { + var buf flatbuf.Buffer + if !src.meta.Buffers(&buf, i) { + panic("buffer index out of bound") + } + if buf.Length() == 0 { + return memory.NewBufferBytes(nil) + } + + raw := make([]byte, buf.Length()) + _, err := src.r.ReadAt(raw, buf.Offset()) + if err != nil { + panic(err) + } + + return memory.NewBufferBytes(raw) +} + +func (src *ipcSource) fieldMetadata(i int) *flatbuf.FieldNode { + var node flatbuf.FieldNode + if !src.meta.Nodes(&node, i) { + panic("field metadata out of bound") + } + return &node +} + +type arrayLoaderContext struct { + src ipcSource + ifield int + ibuffer int + max int +} + +func (ctx *arrayLoaderContext) field() *flatbuf.FieldNode { + field := ctx.src.fieldMetadata(ctx.ifield) + ctx.ifield++ + return field +} + +func (ctx *arrayLoaderContext) buffer() *memory.Buffer { + buf := ctx.src.buffer(ctx.ibuffer) + ctx.ibuffer++ + return buf +} + +func (ctx *arrayLoaderContext) loadArray(dt arrow.DataType) array.Interface { + switch dt := dt.(type) { + case *arrow.NullType: + return ctx.loadNull() + + case *arrow.BooleanType, + *arrow.Int8Type, *arrow.Int16Type, *arrow.Int32Type, *arrow.Int64Type, + *arrow.Uint8Type, *arrow.Uint16Type, *arrow.Uint32Type, *arrow.Uint64Type, + *arrow.Float32Type, *arrow.Float64Type: + return ctx.loadPrimitive(dt) + + default: + panic(errors.Errorf("array type %T not handled yet", dt)) + } +} + +func (ctx *arrayLoaderContext) loadCommon(nbufs int) (*flatbuf.FieldNode, []*memory.Buffer) { + buffers := make([]*memory.Buffer, 0, nbufs) + field := ctx.field() + + var buf *memory.Buffer + switch field.NullCount() { + case 0: + ctx.ibuffer++ + default: + buf = ctx.buffer() + } + buffers = append(buffers, buf) + + return field, buffers +} + +func (ctx *arrayLoaderContext) loadNull() array.Interface { + field, buffers := ctx.loadCommon(1) + buffers = append(buffers, ctx.buffer()) + + data := array.NewData(arrow.Null, int(field.Length()), buffers, nil, int(field.NullCount()), 0) + defer data.Release() + + return array.MakeFromData(data) +} + +func (ctx *arrayLoaderContext) loadPrimitive(dt arrow.DataType) array.Interface { + field, buffers := ctx.loadCommon(2) + + switch field.Length() { + case 0: + buffers = append(buffers, nil) + ctx.ibuffer++ + default: + buffers = append(buffers, ctx.buffer()) + } + + data := array.NewData(dt, int(field.Length()), buffers, nil, int(field.NullCount()), 0) + defer data.Release() + + return array.MakeFromData(data) +} + +func readDictionary(meta *memory.Buffer, types dictTypeMap, r ReadAtSeeker) (int64, array.Interface, error) { + // msg := flatbuf.GetRootAsMessage(meta.Bytes(), 0) + // var dictBatch flatbuf.DictionaryBatch + // initFB(&dictBatch, msg.Header) + // + // id := dictBatch.Id() + // v, ok := types[id] + // if !ok { + // return id, nil, errors.Errorf("arrow/ipc: no type metadata for dictionary with ID=%d", id) + // } + // + // fields := []arrow.Field{v} + // + // // we need a schema for the record batch. + // schema := arrow.NewSchema(fields, nil) + // + // // the dictionary is embedded in a record batch with a single column. + // recBatch := dictBatch.Data(nil) + // + // var ( + // batchMeta *memory.Buffer + // body *memory.Buffer + // ) + // + // + // ctx := &arrayLoaderContext{ + // src: ipcSource{ + // meta: &md, + // r: bytes.NewReader(body.Bytes()), + // }, + // max: kMaxNestingDepth, + // } + // + // cols := make([]array.Interface, len(schema.Fields())) + // for i, field := range schema.Fields() { + // cols[i] = ctx.loadArray(field.Type) + // } + // + // batch := array.NewRecord(schema, cols, rows) + + panic("not implemented") +} diff --git a/go/arrow/ipc/ipc.go b/go/arrow/ipc/ipc.go new file mode 100644 index 00000000000..fc83e94609a --- /dev/null +++ b/go/arrow/ipc/ipc.go @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "io" + + "github.com/apache/arrow/go/arrow/memory" +) + +const ( + errNotArrowFile = errString("arrow/ipc: not an Arrow file") + errInconsistentFileMetadata = errString("arrow/ipc: file is smaller than indicated metadata size") +) + +type errString string + +func (s errString) Error() string { + return string(s) +} + +type ReadAtSeeker interface { + io.Reader + io.Seeker + io.ReaderAt +} + +type config struct { + alloc memory.Allocator + footer struct { + offset int64 + } +} + +func newConfig() *config { + return &config{ + alloc: memory.NewGoAllocator(), + } +} + +// Option is a functional option to configure opening or creating Arrow files +// and streams. +type Option func(*config) + +// WithFooterOffset specifies the Arrow footer position in bytes. +func WithFooterOffset(offset int64) Option { + return func(cfg *config) { + cfg.footer.offset = offset + } +} + +// WithAllocator specifies the Arrow memory allocator used while building records. +func WithAllocator(mem memory.Allocator) Option { + return func(cfg *config) { + cfg.alloc = mem + } +} diff --git a/go/arrow/ipc/message.go b/go/arrow/ipc/message.go new file mode 100644 index 00000000000..bb12dbbc518 --- /dev/null +++ b/go/arrow/ipc/message.go @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "encoding/binary" + "fmt" + "io" + "sync/atomic" + + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/internal/flatbuf" + "github.com/apache/arrow/go/arrow/memory" + "github.com/pkg/errors" +) + +// MetadataVersion represents the Arrow metadata version. +type MetadataVersion flatbuf.MetadataVersion + +const ( + MetadataV1 = MetadataVersion(flatbuf.MetadataVersionV1) // version for Arrow-0.1.0 + MetadataV2 = MetadataVersion(flatbuf.MetadataVersionV2) // version for Arrow-0.2.0 + MetadataV3 = MetadataVersion(flatbuf.MetadataVersionV3) // version for Arrow-0.3.0 to 0.7.1 + MetadataV4 = MetadataVersion(flatbuf.MetadataVersionV4) // version for >= Arrow-0.8.0 +) + +func (m MetadataVersion) String() string { + if v, ok := flatbuf.EnumNamesMetadataVersion[int16(m)]; ok { + return v + } + return fmt.Sprintf("MetadataVersion(%d)", int16(m)) +} + +// MessageType represents the type of Message in an Arrow format. +type MessageType flatbuf.MessageHeader + +const ( + MessageNone = MessageType(flatbuf.MessageHeaderNONE) + MessageSchema = MessageType(flatbuf.MessageHeaderSchema) + MessageDictionaryBatch = MessageType(flatbuf.MessageHeaderDictionaryBatch) + MessageRecordBatch = MessageType(flatbuf.MessageHeaderRecordBatch) + MessageTensor = MessageType(flatbuf.MessageHeaderTensor) + MessageSparseTensor = MessageType(flatbuf.MessageHeaderSparseTensor) +) + +func (m MessageType) String() string { + if v, ok := flatbuf.EnumNamesMessageHeader[byte(m)]; ok { + return v + } + return fmt.Sprintf("MessageType(%d)", int(m)) +} + +const ( + // maxNestingDepth is an arbitrary value to catch user mistakes. + // For deeply nested schemas, it is expected the user will indicate + // explicitly the maximum allowed recursion depth. + maxNestingDepth = 64 +) + +// Message is an IPC message, including metadata and body. +type Message struct { + refCount int64 + msg *flatbuf.Message + meta *memory.Buffer + body *memory.Buffer +} + +// NewMessage creates a new message from the metadata and body buffers. +// NewMessage panics if any of these buffers is nil. +func NewMessage(meta, body *memory.Buffer) *Message { + if meta == nil || body == nil { + panic("arrow/ipc: nil buffers") + } + meta.Retain() + body.Retain() + return &Message{ + refCount: 1, + msg: flatbuf.GetRootAsMessage(meta.Bytes(), 0), + meta: meta, + body: body, + } +} + +func newMessageFromFB(meta *flatbuf.Message, body *memory.Buffer) *Message { + if meta == nil || body == nil { + panic("arrow/ipc: nil buffers") + } + body.Retain() + return &Message{ + refCount: 1, + msg: meta, + meta: memory.NewBufferBytes(meta.Table().Bytes), + body: body, + } +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (msg *Message) Retain() { + atomic.AddInt64(&msg.refCount, 1) +} + +// Release decreases the reference count by 1. +// Release may be called simultaneously from multiple goroutines. +// When the reference count goes to zero, the memory is freed. +func (msg *Message) Release() { + debug.Assert(atomic.LoadInt64(&msg.refCount) > 0, "too many releases") + + if atomic.AddInt64(&msg.refCount, -1) == 0 { + msg.meta.Release() + msg.body.Release() + msg.msg = nil + msg.meta = nil + msg.body = nil + } +} + +func (msg *Message) Version() MetadataVersion { + return MetadataVersion(msg.msg.Version()) +} + +func (msg *Message) Type() MessageType { + return MessageType(msg.msg.HeaderType()) +} + +func (msg *Message) BodyLen() int64 { + return msg.msg.BodyLength() +} + +// MessageReader reads messages from an io.Reader. +type MessageReader struct { + r io.Reader + + refCount int64 + msg *Message +} + +// NewMessageReader returns a reader that reads messages from an input stream. +func NewMessageReader(r io.Reader) *MessageReader { + return &MessageReader{r: r, refCount: 1} +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (r *MessageReader) Retain() { + atomic.AddInt64(&r.refCount, 1) +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (r *MessageReader) Release() { + debug.Assert(atomic.LoadInt64(&r.refCount) > 0, "too many releases") + + if atomic.AddInt64(&r.refCount, -1) == 0 { + if r.msg != nil { + r.msg.Release() + r.msg = nil + } + } +} + +// Message returns the current message that has been extracted from the +// underlying stream. +// It is valid until the next call to Message. +func (r *MessageReader) Message() (*Message, error) { + var buf = make([]byte, 4) + _, err := io.ReadFull(r.r, buf) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not read message length") + } + msgLen := int32(binary.LittleEndian.Uint32(buf)) + if msgLen == 0 { + // optional 0 EOS control message + return nil, io.EOF // FIXME(sbinet): send nil instead? or a special EOS error? + } + + buf = make([]byte, msgLen) + _, err = io.ReadFull(r.r, buf) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not read message metadata") + } + + meta := flatbuf.GetRootAsMessage(buf, 0) + bodyLen := meta.BodyLength() + + buf = make([]byte, bodyLen) + _, err = io.ReadFull(r.r, buf) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not read message body") + } + body := memory.NewBufferBytes(buf) + + if r.msg != nil { + r.msg.Release() + r.msg = nil + } + r.msg = newMessageFromFB(meta, body) + + return r.msg, nil +} diff --git a/go/arrow/ipc/metadata.go b/go/arrow/ipc/metadata.go new file mode 100644 index 00000000000..f3ecc2e54fc --- /dev/null +++ b/go/arrow/ipc/metadata.go @@ -0,0 +1,412 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "fmt" + "io" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/flatbuf" + "github.com/apache/arrow/go/arrow/memory" + flatbuffers "github.com/google/flatbuffers/go" + "github.com/pkg/errors" +) + +// Magic string identifying an Apache Arrow file. +var Magic = []byte("ARROW1") + +const ( + currentMetadataVersion = MetadataV4 + minMetadataVersion = MetadataV4 + + kExtensionTypeKeyName = "arrow_extension_name" + kExtensionDataKeyName = "arrow_extension_data" + + // ARROW-109: We set this number arbitrarily to help catch user mistakes. For + // deeply nested schemas, it is expected the user will indicate explicitly the + // maximum allowed recursion depth + kMaxNestingDepth = 64 +) + +type fieldMetadata struct { + Len int64 + Nulls int64 + Offset int64 +} + +type bufferMetadata struct { + Offset int64 // relative offset into the memory page to the starting byte of the buffer + Len int64 // absolute length in bytes of the buffer +} + +type fileBlock struct { + Offset int64 + Meta int32 + Body int64 + + r io.ReaderAt +} + +func (blk fileBlock) NewMessage() (*Message, error) { + var ( + err error + buf []byte + r = blk.section() + ) + + buf = make([]byte, blk.Meta) + _, err = io.ReadFull(r, buf) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not read message metadata") + } + meta := memory.NewBufferBytes(buf[4:]) // drop buf-size already known from blk.Meta + + buf = make([]byte, blk.Body) + _, err = io.ReadFull(r, buf) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not read message body") + } + body := memory.NewBufferBytes(buf) + + return NewMessage(meta, body), nil +} + +func (blk fileBlock) section() io.Reader { + return io.NewSectionReader(blk.r, blk.Offset, int64(blk.Meta)+blk.Body) +} + +func nullableFromFB(v byte) bool { + return v != 0 +} + +// initFB is a helper function to handle flatbuffers' polymorphism. +func initFB(t interface { + Table() flatbuffers.Table + Init([]byte, flatbuffers.UOffsetT) +}, f func(tbl *flatbuffers.Table) bool) { + tbl := t.Table() + if !f(&tbl) { + panic(errors.Errorf("arrow/ipc: could not initialize %T from flatbuffer", t)) + } + t.Init(tbl.Bytes, tbl.Pos) +} + +func fieldFromFB(field *flatbuf.Field, memo *dictMemo) (arrow.Field, error) { + var ( + err error + o arrow.Field + ) + + o.Name = string(field.Name()) + o.Nullable = nullableFromFB(field.Nullable()) + o.Metadata, err = metadataFrom(field) + if err != nil { + return o, err + } + + encoding := field.Dictionary(nil) + switch encoding { + case nil: + n := field.ChildrenLength() + children := make([]arrow.Field, n) + for i := range children { + var childFB flatbuf.Field + if !field.Children(&childFB, i) { + return o, errors.Errorf("arrow/ipc: could not load field child %d", i) + } + child, err := fieldFromFB(&childFB, memo) + if err != nil { + return o, errors.Wrapf(err, "arrow/ipc: could not convert field child %d", i) + } + children[i] = child + } + + o.Type, err = typeFromFB(field, children, o.Metadata) + if err != nil { + return o, errors.Wrapf(err, "arrow/ipc: could not convert field type") + } + default: + // log.Printf("encoding: %v", encoding.Id()) + // n := field.ChildrenLength() + // log.Printf("children: %v", n) + panic("not implemented") // FIXME(sbinet) + } + + return o, nil +} + +func fieldFromFBDict(field *flatbuf.Field) (arrow.Field, error) { + var ( + o = arrow.Field{ + Name: string(field.Name()), + Nullable: nullableFromFB(field.Nullable()), + } + err error + memo = newMemo() + ) + + // any DictionaryEncoding set is ignored here. + + kids := make([]arrow.Field, field.ChildrenLength()) + for i := range kids { + var kid flatbuf.Field + if !field.Children(&kid, i) { + return o, errors.Errorf("arrow/ipc: could not load field child %d", i) + } + kids[i], err = fieldFromFB(&kid, &memo) + if err != nil { + return o, errors.Wrap(err, "arrow/ipc: field from dict") + } + } + + meta, err := metadataFrom(field) + if err != nil { + return o, errors.Wrap(err, "arrow/ipc: metadata for field from dict") + } + + o.Type, err = typeFromFB(field, kids, meta) + if err != nil { + return o, errors.Wrap(err, "arrow/ipc: type for field from dict") + } + + return o, nil +} + +func typeFromFB(field *flatbuf.Field, children []arrow.Field, md arrow.Metadata) (arrow.DataType, error) { + var data flatbuffers.Table + if !field.Type(&data) { + return nil, errors.Errorf("arrow/ipc: could not load field type data") + } + + dt, err := concreteTypeFromFB(field.TypeType(), data, children) + if err != nil { + return dt, err + } + + // look for extension metadata in custom metadata field. + if md.Len() > 0 { + i := md.FindKey(kExtensionTypeKeyName) + if i < 0 { + return dt, err + } + + panic("not implemented") // FIXME(sbinet) + } + + return dt, err +} + +func concreteTypeFromFB(typ flatbuf.Type, data flatbuffers.Table, children []arrow.Field) (arrow.DataType, error) { + var ( + dt arrow.DataType + err error + ) + + switch typ { + case flatbuf.TypeNONE: + return nil, errors.Errorf("arrow/ipc: Type metadata cannot be none") + + case flatbuf.TypeNull: + return arrow.Null, nil + + case flatbuf.TypeInt: + var dt flatbuf.Int + dt.Init(data.Bytes, data.Pos) + return intFromFB(dt) + + case flatbuf.TypeFloatingPoint: + var dt flatbuf.FloatingPoint + dt.Init(data.Bytes, data.Pos) + return floatFromFB(dt) + + case flatbuf.TypeBinary: + return arrow.BinaryTypes.Binary, nil + + case flatbuf.TypeFixedSizeBinary: + var dt flatbuf.FixedSizeBinary + dt.Init(data.Bytes, data.Pos) + return &arrow.FixedSizeBinaryType{ByteWidth: int(dt.ByteWidth())}, nil + + case flatbuf.TypeUtf8: + return arrow.BinaryTypes.String, nil + + case flatbuf.TypeBool: + return arrow.FixedWidthTypes.Boolean, nil + + default: + // FIXME(sbinet): implement all the other types. + panic(fmt.Errorf("arrow/ipc: type %v not implemented", flatbuf.EnumNamesType[typ])) + } + + return dt, err +} + +func intFromFB(data flatbuf.Int) (arrow.DataType, error) { + bw := data.BitWidth() + if bw > 64 { + return nil, errors.Errorf("arrow/ipc: integers with more than 64 bits not implemented (bits=%d)", bw) + } + if bw < 8 { + return nil, errors.Errorf("arrow/ipc: integers with less than 8 bits not implemented (bits=%d)", bw) + } + + switch bw { + case 8: + switch data.IsSigned() { + case 0: + return arrow.PrimitiveTypes.Uint8, nil + default: + return arrow.PrimitiveTypes.Int8, nil + } + + case 16: + switch data.IsSigned() { + case 0: + return arrow.PrimitiveTypes.Uint16, nil + default: + return arrow.PrimitiveTypes.Int16, nil + } + + case 32: + switch data.IsSigned() { + case 0: + return arrow.PrimitiveTypes.Uint32, nil + default: + return arrow.PrimitiveTypes.Int32, nil + } + + case 64: + switch data.IsSigned() { + case 0: + return arrow.PrimitiveTypes.Uint64, nil + default: + return arrow.PrimitiveTypes.Int64, nil + } + default: + return nil, errors.Errorf("arrow/ipc: integers not in cstdint are not implemented") + } +} + +func floatFromFB(data flatbuf.FloatingPoint) (arrow.DataType, error) { + switch p := data.Precision(); p { + case flatbuf.PrecisionHALF: + return nil, errors.Errorf("arrow/ipc: float16 not implemented") + case flatbuf.PrecisionSINGLE: + return arrow.PrimitiveTypes.Float32, nil + case flatbuf.PrecisionDOUBLE: + return arrow.PrimitiveTypes.Float64, nil + default: + return nil, errors.Errorf("arrow/ipc: floating point type with %d precision not implemented", p) + } +} + +type customMetadataer interface { + CustomMetadataLength() int + CustomMetadata(*flatbuf.KeyValue, int) bool +} + +func metadataFrom(md customMetadataer) (arrow.Metadata, error) { + var ( + keys = make([]string, md.CustomMetadataLength()) + vals = make([]string, md.CustomMetadataLength()) + ) + + for i := range keys { + var kv flatbuf.KeyValue + if !md.CustomMetadata(&kv, i) { + return arrow.Metadata{}, errors.Errorf("arrow/ipc: could not read key-value %d from flatbuffer", i) + } + keys[i] = string(kv.Key()) + vals[i] = string(kv.Value()) + } + + return arrow.NewMetadata(keys, vals), nil +} + +func schemaFromFB(schema *flatbuf.Schema, memo *dictMemo) (*arrow.Schema, error) { + var ( + err error + fields = make([]arrow.Field, schema.FieldsLength()) + ) + + for i := range fields { + var field flatbuf.Field + if !schema.Fields(&field, i) { + return nil, errors.Errorf("arrow/ipc: could not read field %d from schema", i) + } + + fields[i], err = fieldFromFB(&field, memo) + if err != nil { + return nil, errors.Wrapf(err, "arrow/ipc: could not convert field %d from flatbuf", i) + } + } + + md, err := metadataFrom(schema) + if err != nil { + return nil, errors.Wrapf(err, "arrow/ipc: could not convert schema metadata from flatbuf") + } + + return arrow.NewSchema(fields, &md), nil +} + +func dictTypesFromFB(schema *flatbuf.Schema) (dictTypeMap, error) { + var ( + err error + fields = make(dictTypeMap, schema.FieldsLength()) + ) + for i := 0; i < schema.FieldsLength(); i++ { + var field flatbuf.Field + if !schema.Fields(&field, i) { + return nil, errors.Errorf("arrow/ipc: could not load field %d from schema", i) + } + fields, err = visitField(&field, fields) + if err != nil { + return nil, errors.Wrapf(err, "arrow/ipc: could not visit field %d from schema", i) + } + } + return fields, err +} + +func visitField(field *flatbuf.Field, dict dictTypeMap) (dictTypeMap, error) { + var err error + meta := field.Dictionary(nil) + switch meta { + case nil: + // field is not dictionary encoded. + // => visit children. + for i := 0; i < field.ChildrenLength(); i++ { + var child flatbuf.Field + if !field.Children(&child, i) { + return nil, errors.Errorf("arrow/ipc: could not visit child %d from field", i) + } + dict, err = visitField(&child, dict) + if err != nil { + return nil, err + } + } + default: + // field is dictionary encoded. + // construct the data type for the dictionary: no descendants can be dict-encoded. + dfield, err := fieldFromFBDict(field) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not create data type for dictionary") + } + dict[meta.Id()] = dfield + } + return dict, err +} diff --git a/go/arrow/ipc/reader.go b/go/arrow/ipc/reader.go new file mode 100644 index 00000000000..dd98f7c2a71 --- /dev/null +++ b/go/arrow/ipc/reader.go @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "bytes" + "io" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/array" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/internal/flatbuf" + "github.com/apache/arrow/go/arrow/memory" + "github.com/pkg/errors" +) + +// Reader reads records from an io.Reader. +// Reader expects a schema (plus any dictionaries) as the first messages +// in the stream, followed by records. +type Reader struct { + r *MessageReader + schema *arrow.Schema + + refCount int64 + rec array.Record + err error + + types dictTypeMap + memo dictMemo + + mem memory.Allocator + + done bool +} + +// NewReader returns a reader that reads records from an input stream. +func NewReader(r io.Reader, opts ...Option) (*Reader, error) { + cfg := newConfig() + for _, opt := range opts { + opt(cfg) + } + + rr := &Reader{ + r: NewMessageReader(r), + types: make(dictTypeMap), + memo: newMemo(), + mem: cfg.alloc, + } + + err := rr.readSchema() + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not read schema from stream") + } + + return rr, nil +} + +// Err returns the last error encountered during the iteration over the +// underlying stream. +func (r *Reader) Err() error { return r.err } + +func (r *Reader) Schema() *arrow.Schema { return r.schema } + +func (r *Reader) readSchema() error { + msg, err := r.r.Message() + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not read message schema") + } + + if msg.Type() != MessageSchema { + return errors.Errorf("arrow/ipc: invalid message type (got=%v, want=%v)", msg.Type(), MessageSchema) + } + + // FIXME(sbinet) refactor msg-header handling. + var schemaFB flatbuf.Schema + initFB(&schemaFB, msg.msg.Header) + + r.types, err = dictTypesFromFB(&schemaFB) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could read dictionary types from message schema") + } + + // TODO(sbinet): in the future, we may want to reconcile IDs in the stream with + // those found in the schema. + for range r.types { + panic("not implemented") // FIXME(sbinet): ReadNextDictionary + } + + r.schema, err = schemaFromFB(&schemaFB, &r.memo) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not decode schema from message schema") + } + + return nil +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (r *Reader) Retain() { + atomic.AddInt64(&r.refCount, 1) +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (r *Reader) Release() { + debug.Assert(atomic.LoadInt64(&r.refCount) > 0, "too many releases") + + if atomic.AddInt64(&r.refCount, -1) == 0 { + if r.rec != nil { + r.rec.Release() + r.rec = nil + } + if r.r != nil { + r.r.Release() + r.r = nil + } + } +} + +// Next returns whether a Record could be extracted from the underlying stream. +func (r *Reader) Next() bool { + if r.rec != nil { + r.rec.Release() + r.rec = nil + } + + if r.err != nil || r.done { + return false + } + + return r.next() +} + +func (r *Reader) next() bool { + var msg *Message + msg, r.err = r.r.Message() + if r.err != nil { + r.done = true + if r.err == io.EOF { + r.err = nil + } + return false + } + + if got, want := msg.Type(), MessageRecordBatch; got != want { + r.err = errors.Errorf("arrow/ipc: invalid message type (got=%v, want=%v", got, want) + return false + } + + r.rec = newRecord(r.schema, msg.meta, bytes.NewReader(msg.body.Bytes())) + return true +} + +// Record returns the current record that has been extracted from the +// underlying stream. +// It is valid until the next call to Next. +func (r *Reader) Record() array.Record { + return r.rec +} + +var ( + _ array.RecordReader = (*Reader)(nil) +) diff --git a/go/arrow/schema.go b/go/arrow/schema.go index a0cccc2528b..0083707655e 100644 --- a/go/arrow/schema.go +++ b/go/arrow/schema.go @@ -19,6 +19,7 @@ package arrow import ( "fmt" "sort" + "strings" ) type Metadata struct { @@ -64,6 +65,30 @@ func (md Metadata) Len() int { return len(md.keys) } func (md Metadata) Keys() []string { return md.keys } func (md Metadata) Values() []string { return md.values } +func (md Metadata) String() string { + o := new(strings.Builder) + fmt.Fprintf(o, "[") + for i := range md.keys { + if i > 0 { + fmt.Fprintf(o, ", ") + } + fmt.Fprintf(o, "%q: %q", md.keys[i], md.values[i]) + } + fmt.Fprintf(o, "]") + return o.String() +} + +// FindKey returns the index of the key-value pair with the provided key name, +// or -1 if such a key does not exist. +func (md Metadata) FindKey(k string) int { + for i, v := range md.keys { + if v == k { + return i + } + } + return -1 +} + func (md Metadata) clone() Metadata { if len(md.keys) == 0 { return Metadata{} diff --git a/go/arrow/schema_test.go b/go/arrow/schema_test.go index 4ad36c0624f..0f2a0296dce 100644 --- a/go/arrow/schema_test.go +++ b/go/arrow/schema_test.go @@ -91,6 +91,34 @@ func TestMetadata(t *testing.T) { } }) } + + t.Run("find-key", func(t *testing.T) { + md := NewMetadata([]string{"k1", "k11"}, []string{"v1", "v11"}) + + if got, want := md.FindKey("k1"), 0; got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := md.FindKey(""), -1; got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := md.FindKey("k"), -1; got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := md.FindKey(" "), -1; got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := md.FindKey("k11"), 1; got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := md.FindKey("k11 "), -1; got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + }) } func TestSchema(t *testing.T) {