ARROW-5108: [Go] implement reading primitive arrays from Arrow file

appveyor.yml

@@ -34,6 +34,7 @@ only_commits:
 cache:
   - C:\Users\Appveyor\clcache
   - C:\Users\Appveyor\.cargo\registry
+  - '%LocalAppData%\\go-build'
 
 matrix:
   fast_finish: true
@@ -79,13 +80,15 @@ environment:
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
       USE_CLCACHE: false
     - JOB: "Go"
-      TARGET: x86_64-pc-windows-msvc
+      MINGW_PACKAGE_PREFIX: mingw-w64-x86_64
+      MINGW_PREFIX: c:\msys64\mingw64
+      MSYSTEM: MINGW64
+      USE_CLCACHE: false
       GOROOT: 'c:\go111'
       GOPATH: c:\gopath
       GO111MODULE: on
       GOTOOLDIR: '%GOROOT%\pkg\tool\windows_amd64'
       PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%'
-      USE_CLCACHE: false
 
   MSVC_DEFAULT_OPTIONS: ON
   APPVEYOR_SAVE_CACHE_ON_ERROR: true

ci/go-build-main.bat

@@ -21,6 +21,6 @@ pushd go\arrow
 
 %GOROOT%\bin\go version
 %GOROOT%\bin\go get -v ./... || exit /B
-%GOROOT%\bin\go test -race ./... || exit /B
+%GOROOT%\bin\go test   ./... || exit /B
 
 popd

go/arrow/array/boolean_test.go

@@ -17,7 +17,9 @@
 package array_test
 
 import (
+	"fmt"
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/apache/arrow/go/arrow/array"
@@ -258,3 +260,29 @@ func TestBooleanSliceOutOfBounds(t *testing.T) {
 		})
 	}
 }
+
+func TestBooleanStringer(t *testing.T) {
+	pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
+	defer pool.AssertSize(t, 0)
+
+	var (
+		values = []bool{true, false, true, false, true, false, true, false, true, false}
+		valids = []bool{true, true, false, true, true, true, false, true, true, true}
+	)
+
+	b := array.NewBooleanBuilder(pool)
+	defer b.Release()
+
+	b.AppendValues(values, valids)
+
+	arr := b.NewArray().(*array.Boolean)
+	defer arr.Release()
+
+	out := new(strings.Builder)
+	fmt.Fprintf(out, "%v", arr)
+
+	const want = "[true false (null) false true false (null) false true false]"
+	if got := out.String(); got != want {
+		t.Fatalf("invalid stringer:\ngot= %q\nwant=%q", got, want)
+	}
+}

go/arrow/doc.go

@@ -33,6 +33,7 @@ package arrow
 
 //go:generate go run _tools/tmpl/main.go -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl
 //go:generate go run _tools/tmpl/main.go -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl tensor/numeric.gen.go.tmpl tensor/numeric.gen_test.go.tmpl
+//go:generate go run ./gen-flatbuffers.go
 
 // stringer
 //go:generate stringer -type=Type

go/arrow/gen-flatbuffers.go

@@ -0,0 +1,155 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	dir, err := ioutil.TempDir("", "go-arrow-")
+	if err != nil {
+		log.Fatalf("could not create top-level temporary directory: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	genFormat(dir)
+	genFeather(dir)
+}
+
+func genFormat(dir string) {
+	args := []string{"--go", "-o", filepath.Join(dir, "format")}
+	fnames, err := filepath.Glob("../../format/*.fbs")
+	if err != nil || len(fnames) == 0 {
+		log.Fatalf("could not retrieve list of format FlatBuffers files: files=%d err=%v",
+			len(fnames), err,
+		)
+	}
+	args = append(args, fnames...)
+
+	gen := exec.Command("flatc", args...)
+	gen.Stdout = os.Stdout
+	gen.Stderr = os.Stderr
+
+	err = gen.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = os.MkdirAll("./internal/flatbuf", 0755)
+	if err != nil {
+		log.Fatalf("could not create ./internal/flatbuf directory: %v", err)
+	}
+
+	base := filepath.Join(dir, "format", "org", "apache", "arrow", "flatbuf")
+	fnames, err = filepath.Glob(filepath.Join(base, "*.go"))
+	if err != nil {
+		log.Fatalf("could not glob %v/*.go: %v", base, err)
+	}
+
+	for _, fname := range fnames {
+		dst := filepath.Join(".", "internal", "flatbuf", filepath.Base(fname))
+		process(dst, fname)
+	}
+}
+
+func genFeather(dir string) {
+	gen := exec.Command("flatc",
+		"--go", "-o", filepath.Join(dir, "feather"),
+		"../../cpp/src/arrow/ipc/feather.fbs",
+	)
+
+	gen.Stdout = os.Stdout
+	gen.Stderr = os.Stderr
+
+	err := gen.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = os.MkdirAll("./ipc/feather/fbs", 0755)
+	if err != nil {
+		log.Fatalf("could not create ./feather/fbs directory: %v", err)
+	}
+
+	base := filepath.Join(dir, "feather", "arrow")
+	fnames, err := filepath.Glob(filepath.Join(base, "ipc", "feather", "*", "*.go"))
+	if err != nil {
+		log.Fatalf("could not glob ipc/feather/*.go: %v", err)
+	}
+
+	for _, fname := range fnames {
+		dst := strings.Replace(fname, base, ".", -1)
+		process(dst, fname)
+	}
+}
+
+func process(dst, fname string) {
+	raw, err := ioutil.ReadFile(fname)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	f, err := os.Create(dst)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+
+	if !bytes.HasPrefix(raw, []byte(hdr)) {
+		_, err = f.Write([]byte(hdr))
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	_, err = f.Write(raw)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = f.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+const hdr = `// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+`

go/arrow/go.mod

@@ -18,6 +18,7 @@ module github.com/apache/arrow/go/arrow
 
 require (
 	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/google/flatbuffers v1.10.0
 	github.com/pkg/errors v0.8.1
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/stretchr/testify v1.2.0

go/arrow/internal/bitutil/bitutil.go

@@ -27,6 +27,9 @@ var (
 	FlippedBitMask = [8]byte{254, 253, 251, 247, 239, 223, 191, 127}
 )
 
+// IsMultipleOf8 returns whether v is a multiple of 8.
+func IsMultipleOf8(v int64) bool { return v&7 == 0 }
+
 // NextPowerOf2 rounds x to the next power of two.
 func NextPowerOf2(x int) int { return 1 << uint(bits.Len(uint(x))) }
 

go/arrow/internal/bitutil/bitutil_test.go

@@ -17,6 +17,7 @@
 package bitutil_test
 
 import (
+	"fmt"
 	"math/rand"
 	"testing"
 
@@ -25,6 +26,35 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestIsMultipleOf8(t *testing.T) {
+	for _, tc := range []struct {
+		v    int64
+		want bool
+	}{
+		{-16, true},
+		{-9, false},
+		{-8, true},
+		{-7, false},
+		{-4, false},
+		{-1, false},
+		{-0, true},
+		{0, true},
+		{1, false},
+		{4, false},
+		{7, false},
+		{8, true},
+		{9, false},
+		{16, true},
+	} {
+		t.Run(fmt.Sprintf("v=%d", tc.v), func(t *testing.T) {
+			got := bitutil.IsMultipleOf8(tc.v)
+			if got != tc.want {
+				t.Fatalf("IsMultipleOf8(%d): got=%v, want=%v", tc.v, got, tc.want)
+			}
+		})
+	}
+}
+
 func TestCeilByte(t *testing.T) {
 	tests := []struct {
 		name    string