From dc92b839d8fcc09c19f3ef5f47f7720f9a6df5b6 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 26 Feb 2018 06:31:41 -0800
Subject: [PATCH 01/51] fix typo

---
 js/src/vector.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/js/src/vector.ts b/js/src/vector.ts
index 6c2bbbb86a7..67a4f99be73 100644
--- a/js/src/vector.ts
+++ b/js/src/vector.ts
@@ -403,7 +403,7 @@ export class DictionaryVector<T extends DataType = DataType> extends Vector<Dict
             view = (view as any).view;
         }
         if (data instanceof DictionaryData && view instanceof DictionaryView) {
-            this.indices = view.indices;
+            this.indices = view.indicies;
             this.dictionary = data.dictionary;
         } else if (data instanceof ChunkedData && view instanceof ChunkedView) {
             const chunks = view.chunkVectors as DictionaryVector<T>[];

From ef1acc769e217b933488cf20bead1a1947681196 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 26 Feb 2018 06:31:53 -0800
Subject: [PATCH 02/51] read union buffers in the correct order

---
 js/src/ipc/reader/vector.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/js/src/ipc/reader/vector.ts b/js/src/ipc/reader/vector.ts
index b8c4871ebac..c4688f5e2b8 100644
--- a/js/src/ipc/reader/vector.ts
+++ b/js/src/ipc/reader/vector.ts
@@ -126,6 +126,6 @@ export abstract class TypeDataLoader extends TypeVisitor {
     protected visitUnionType(type: DenseUnion | SparseUnion, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
         return type.mode === UnionMode.Sparse ?
             new SparseUnionData(type as SparseUnion, length, this.readNullBitmap(type, nullCount), this.readTypeIds(type), this.visitFields(type.children), 0, nullCount) :
-            new DenseUnionData(type as DenseUnion, length, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.readTypeIds(type), this.visitFields(type.children), 0, nullCount);
+            new DenseUnionData(type as DenseUnion, length, this.readNullBitmap(type, nullCount), this.readTypeIds(type), this.readOffsets(type), this.visitFields(type.children), 0, nullCount);
     }
 }

From 6522cb006af2a5cba5606507c398b3ba2268f2ec Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 26 Feb 2018 06:32:10 -0800
Subject: [PATCH 03/51] fix Data generics for FixedSizeList

---
 js/src/data.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/js/src/data.ts b/js/src/data.ts
index 3bfb3209b69..963a6a476ba 100644
--- a/js/src/data.ts
+++ b/js/src/data.ts
@@ -17,8 +17,8 @@
 
 import { popcnt_bit_range } from './util/bit';
 import { VectorLike, Vector } from './vector';
+import { Int, Bool, FlatListType, List, Struct, Map_ } from './type';
 import { VectorType, TypedArray, TypedArrayConstructor, Dictionary } from './type';
-import { Int, Bool, FlatListType, List, FixedSizeList, Struct, Map_ } from './type';
 import { DataType, FlatType, ListType, NestedType, SingleNestedType, DenseUnion, SparseUnion } from './type';
 
 export function toTypedArray<T extends TypedArray>(ArrayType: TypedArrayConstructor<T>, values?: T | ArrayLike<number> | Iterable<number> | null): T {
@@ -46,7 +46,7 @@ export interface DataTypes<T extends DataType> {
 /*              [Type.Struct]*/ 13: NestedData<Struct>;
 /*               [Type.Union]*/ 14: UnionData;
 /*     [Type.FixedSizeBinary]*/ 15: FlatData<T>;
-/*       [Type.FixedSizeList]*/ 16: SingleNestedData<FixedSizeList<T>>;
+/*       [Type.FixedSizeList]*/ 16: SingleNestedData<any>;
 /*                 [Type.Map]*/ 17: NestedData<Map_>;
 /*  [Type.DenseUnion]*/ DenseUnion: DenseUnionData;
 /*[Type.SparseUnion]*/ SparseUnion: SparseUnionData;

From 43c671f7f201ca383e7d200a203475d35fbfd7dc Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 26 Feb 2018 06:32:49 -0800
Subject: [PATCH 04/51] [WIP] add Binary writer

---
 js/src/ipc/writer/binary.ts | 565 ++++++++++++++++++++++++++++++++++++
 1 file changed, 565 insertions(+)
 create mode 100644 js/src/ipc/writer/binary.ts

diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
new file mode 100644
index 00000000000..0b1cb26a361
--- /dev/null
+++ b/js/src/ipc/writer/binary.ts
@@ -0,0 +1,565 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { DenseUnionData } from '../../data';
+import { RecordBatch } from '../../recordbatch';
+import { VectorVisitor, TypeVisitor } from '../../visitor';
+import { align, getBool, packBools, iterateBits } from '../../util/bit';
+import { Vector, UnionVector, DictionaryVector, NestedVector } from '../../vector';
+import { BufferMetadata, FieldMetadata, Footer, FileBlock, Message, RecordBatchMetadata, DictionaryBatch } from '../metadata';
+import {
+    Schema, Field, TypedArray, MetadataVersion,
+    Dictionary,
+    Null, Int, Float,
+    Binary, Bool, Utf8, Decimal,
+    Date_, Time, Timestamp, Interval,
+    List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_,
+    UnionMode, SparseUnion, DenseUnion, FlatListType, DataType, FlatType, NestedType,
+} from '../../type';
+
+export class RecordBatchSerializer extends VectorVisitor {
+    protected byteLength = 0;
+    // @ts-ignore
+    protected buffers: TypedArray[];
+    // @ts-ignore
+    protected fieldNodes: FieldMetadata[];
+    // @ts-ignore
+    protected buffersMeta: BufferMetadata[];
+    public writeRecordBatch(recordBatch: RecordBatch) {
+        this.byteLength = 0;
+        this.buffers = [];
+        this.fieldNodes = [];
+        this.buffersMeta = [];
+        for (let vector: Vector, index = -1, numCols = recordBatch.numCols; ++index < numCols;) {
+            if (vector = recordBatch.getChildAt(index)!) {
+                this.visit(vector);
+            }
+        }
+        const b = new Builder();
+        _Message.finishMessageBuffer(
+            b, writeMessage(b, new RecordBatchMetadata(
+                MetadataVersion.V4, this.byteLength, this.fieldNodes, this.buffersMeta
+            ))
+        );
+        const metadataBytes = b.asUint8Array();
+        // 4 bytes for the metadata length + 4 bytes of padding + the length of the metadata buffer
+        const metadataBytesOffset = 8 + metadataBytes.byteLength;
+        // + the length of all the vector buffers
+        const recordBatchBytes = new Uint8Array(metadataBytesOffset + this.byteLength);
+        // Write the metadata length as the first 4 bytes
+        new DataView(recordBatchBytes.buffer).setInt32(0, metadataBytes.byteLength);
+        // Now write the buffers
+        const { buffers, buffersMeta } = this;
+        for (let bufferIndex = -1, buffersLen = buffers.length; ++bufferIndex < buffersLen;) {
+            const { buffer, byteLength } = buffers[bufferIndex];
+            const { offset: byteOffset } = buffersMeta[bufferIndex];
+            recordBatchBytes.set(
+                new Uint8Array(buffer, 0, byteLength),
+                metadataBytesOffset + byteOffset
+            );
+        }
+        return recordBatchBytes;
+    }
+    public visit<T extends DataType>(vector: Vector<T>) {
+        const { data, length, nullCount } = vector;
+        if (length > 2147483647) {
+            throw new RangeError('Cannot write arrays larger than 2^31 - 1 in length');
+        }
+        this.fieldNodes.push(new FieldMetadata(length, nullCount));
+        this.addBuffer(nullCount <= 0
+            ? new Uint8Array(0) // placeholder validity buffer
+            : this.getTruncatedBitmap(data.offset, length, data.nullBitmap!), 64);
+        return super.visit(vector);
+    }
+    public visitNull           (_vector: Vector<Null>)           { return this.addBuffer(null);              }
+    public visitBool           (vector: Vector<Bool>)            { return this.visitFlatVector(vector);      }
+    public visitInt            (vector: Vector<Int>)             { return this.visitFlatVector(vector);      }
+    public visitFloat          (vector: Vector<Float>)           { return this.visitFlatVector(vector);      }
+    public visitUtf8           (vector: Vector<Utf8>)            { return this.visitFlatListVector(vector);  }
+    public visitBinary         (vector: Vector<Binary>)          { return this.visitFlatListVector(vector);  }
+    public visitFixedSizeBinary(vector: Vector<FixedSizeBinary>) { return this.visitFlatVector(vector);      }
+    public visitDate           (vector: Vector<Date_>)           { return this.visitFlatVector(vector);      }
+    public visitTimestamp      (vector: Vector<Timestamp>)       { return this.visitFlatVector(vector);      }
+    public visitTime           (vector: Vector<Time>)            { return this.visitFlatVector(vector);      }
+    public visitDecimal        (vector: Vector<Decimal>)         { return this.visitFlatVector(vector);      }
+    public visitInterval       (vector: Vector<Interval>)        { return this.visitFlatVector(vector);      }
+    public visitStruct         (vector: Vector<Struct>)          { return this.visitNestedVector(vector);    }
+    public visitMap            (vector: Vector<Map_>)            { return this.visitNestedVector(vector);    }
+    public visitFixedSizeList  (vector: Vector<FixedSizeList>)   { return this.visitNestedVector(vector, 1); }
+    public visitList           (vector: Vector<List>)            { return this.visitNestedVector(vector, 1); }
+    public visitDictionary     (vector: DictionaryVector)        {
+        // Dictionary written out separately. Slice offset contained in the indices
+        return this.visit(vector.indices);
+    }
+    public visitUnion(vector: Vector<DenseUnion | SparseUnion>) {
+        const { data, type, length } = vector;
+        const { offset: sliceOffset, typeIds } = data;
+        // All Union Vectors have a typeIds buffer
+        this.addBuffer(typeIds);
+        // If this is a Sparse Union, treat it like all other Nested types
+        if (type.mode === UnionMode.Sparse) {
+            return this.visitNestedVector(vector);
+        } else if (type.mode === UnionMode.Dense) {
+            // If this is a Dense Union, add the valueOffsets buffer and potentially slice the children
+            const valueOffsets = (data as DenseUnionData).valueOffsets;
+            if (sliceOffset <= 0) {
+                // If the Vector hasn't been sliced, write the existing valueOffsets
+                this.addBuffer(valueOffsets);
+                // We can treat this like all other Nested types
+                return this.visitNestedVector(vector);
+            } else {
+                // A sliced Dense Union is an unpleasant case. Because the offsets are different for
+                // each child vector, we need to "rebase" the valueOffsets for each child
+                // Union typeIds are not necessary 0-indexed
+                const maxChildTypeId = Math.max(...type.typeIds);
+                const childLengths = new Int32Array(maxChildTypeId + 1);
+                // Set all to -1 to indicate that we haven't observed a first occurrence of a particular child yet
+                const childOffsets = new Int32Array(maxChildTypeId + 1).fill(-1);
+                const shiftedOffsets = new Int32Array(length);
+                const unshiftedOffsets = this.getZeroBasedValueOffsets(sliceOffset, length, valueOffsets);
+                for (let typeId, shift, index = -1; ++index < length;) {
+                    typeId = typeIds[index];
+                    // ~(-1) used to be faster than x === -1, so maybe worth benchmarking the difference of these two impls for large dense unions:
+                    // ~(shift = childOffsets[typeId]) || (shift = childOffsets[typeId] = unshiftedOffsets[index]);
+                    // Going with this form for now, as it's more readable
+                    if ((shift = childOffsets[typeId]) === -1) {
+                        shift = childOffsets[typeId] = unshiftedOffsets[typeId];
+                    }
+                    shiftedOffsets[index] = unshiftedOffsets[index] - shift;
+                    ++childLengths[typeId];
+                }
+                this.addBuffer(shiftedOffsets);
+                // Slice and visit children accordingly
+                for (let childIndex = -1, numChildren = type.children.length; ++childIndex < numChildren;) {
+                    const typeId = type.typeIds[childIndex];
+                    const child = (vector as UnionVector).getChildAt(childIndex)!;
+                    this.visit(child.slice(childOffsets[typeId], Math.min(length, childLengths[typeId])));
+                }
+            }
+        }
+        return this;
+    }
+    protected visitFlatVector<T extends FlatType>(vector: Vector<T>) {
+        // Use the TypedArray constructor defined by the Vector's DataType
+        const { ArrayType } = vector.type;
+        // Use `vector.toArray()` here, since the View currently applied to the Vector might iterate different values
+        // than are in the original Data's values buffer.
+        // An example is a TimestampVector[ns] that's been converted to an Int32Vector via `asEpochMilliseconds()`.
+        // If the Vector's View is a ValidityView, the null slots will be automatically coerced by the TypedArray
+        // constructor, e.g. `0` for an Int8Array, `NaN` for Float64Array, etc.
+        this.addBuffer(new ArrayType(vector.toArray()));
+        return this;
+    }
+    protected visitFlatListVector<T extends FlatListType>(vector: Vector<T>) {
+        const { data, length } = vector;
+        const sliceOffset = data.offset;
+        const { values, valueOffsets } = data;
+        const firstOffset = valueOffsets[0];
+        const lastOffset = valueOffsets[valueOffsets.length - 1];
+        const byteLength = Math.min(align(lastOffset - firstOffset, 8), values.byteLength - firstOffset);
+        // Push in the order of FlatList types
+        // valueOffsets buffer first
+        this.addBuffer(this.getZeroBasedValueOffsets(sliceOffset, length, valueOffsets));
+        // sliced values buffer second
+        this.addBuffer(values.subarray(firstOffset + sliceOffset, firstOffset + sliceOffset + byteLength));
+        return this;
+    }
+    protected visitNestedVector<T extends NestedType>(vector: Vector<T>, numChildren = vector.type.children.length) {
+        // Visit the children accordingly
+        for (let childIndex = -1; ++childIndex < numChildren;) {
+            this.visit((vector as NestedVector<T>).getChildAt(childIndex)!);
+        }
+        return this;
+    }
+    protected addBuffer(array?: TypedArray | null, alignment: number = 8) {
+        const values = array || new Uint8Array(0);
+        const alignedLength = align(values.byteLength, alignment);
+        this.buffers.push(values);
+        this.buffersMeta.push(new BufferMetadata(this.byteLength, alignedLength));
+        this.byteLength += alignedLength;
+    }
+    protected getTruncatedBitmap(sliceOffset: number, length: number, nullBitmap: Uint8Array) {
+        const alignedLength = align(length, 64);
+        if (sliceOffset > 0 || length < alignedLength) {
+            // With a sliced array / non-zero offset, we have to copy the bitmap
+            const bytes = new Uint8Array(alignedLength).fill(255);
+            bytes.set(
+                (sliceOffset % 8 === 0)
+                // If the sliceOffset is aligned to 1 byte, it's safe to slice the nullBitmap directly
+                ? nullBitmap.subarray(sliceOffset >> 3)
+                // iterate each bit starting from the sliceOffset, and repack into an aligned nullBitmap
+                : packBools(iterateBits(nullBitmap, sliceOffset, length, null, getBool))
+            );
+            return bytes;
+        }
+        return nullBitmap;
+    }
+    protected getZeroBasedValueOffsets(sliceOffset: number, length: number, valueOffsets: Int32Array) {
+        // If we have a non-zero offset, then the value offsets do not start at
+        // zero. We must a) create a new offsets array with shifted offsets and
+        // b) slice the values array accordingly
+        if (sliceOffset > 0 || valueOffsets[0] !== 0) {
+            const startOffset = valueOffsets[0];
+            const destOffsets = new Int32Array(length + 1);
+            for (let index = -1; ++index < length;) {
+                destOffsets[index] = valueOffsets[index] - startOffset;
+            }
+            // Final offset
+            destOffsets[length] = valueOffsets[length] - startOffset;
+            return destOffsets;
+        }
+        return valueOffsets;
+    }
+}
+
+import { flatbuffers } from 'flatbuffers';
+import Long = flatbuffers.Long;
+import Builder = flatbuffers.Builder;
+import * as File_ from '../../fb/File';
+import * as Schema_ from '../../fb/Schema';
+import * as Message_ from '../../fb/Message';
+
+import _Block = File_.org.apache.arrow.flatbuf.Block;
+import _Footer = File_.org.apache.arrow.flatbuf.Footer;
+import _Field = Schema_.org.apache.arrow.flatbuf.Field;
+import _Schema = Schema_.org.apache.arrow.flatbuf.Schema;
+import _Buffer = Schema_.org.apache.arrow.flatbuf.Buffer;
+import _Message = Message_.org.apache.arrow.flatbuf.Message;
+import _KeyValue = Schema_.org.apache.arrow.flatbuf.KeyValue;
+import _FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode;
+import _RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch;
+import _DictionaryBatch = Message_.org.apache.arrow.flatbuf.DictionaryBatch;
+import _DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding;
+import _Endianness = Schema_.org.apache.arrow.flatbuf.Endianness;
+
+import _Null = Schema_.org.apache.arrow.flatbuf.Null;
+import _Int = Schema_.org.apache.arrow.flatbuf.Int;
+import _FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint;
+import _Binary = Schema_.org.apache.arrow.flatbuf.Binary;
+import _Bool = Schema_.org.apache.arrow.flatbuf.Bool;
+import _Utf8 = Schema_.org.apache.arrow.flatbuf.Utf8;
+import _Decimal = Schema_.org.apache.arrow.flatbuf.Decimal;
+import _Date = Schema_.org.apache.arrow.flatbuf.Date;
+import _Time = Schema_.org.apache.arrow.flatbuf.Time;
+import _Timestamp = Schema_.org.apache.arrow.flatbuf.Timestamp;
+import _Interval = Schema_.org.apache.arrow.flatbuf.Interval;
+import _List = Schema_.org.apache.arrow.flatbuf.List;
+import _Struct = Schema_.org.apache.arrow.flatbuf.Struct_;
+import _Union = Schema_.org.apache.arrow.flatbuf.Union;
+import _FixedSizeBinary = Schema_.org.apache.arrow.flatbuf.FixedSizeBinary;
+import _FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList;
+import _Map = Schema_.org.apache.arrow.flatbuf.Map;
+
+export class TypeSerializer extends TypeVisitor {
+    constructor(protected builder: Builder) {
+        super();
+    }
+    public visitNull(_node: Null) {
+        const b = this.builder;
+        return (
+            _Null.startNull(b) ||
+            _Null.endNull(b)
+        );
+    }
+    public visitInt(node: Int) {
+        const b = this.builder;
+        return (
+            _Int.startInt(b) ||
+            _Int.addBitWidth(b, node.bitWidth) ||
+            _Int.addIsSigned(b, node.isSigned) ||
+            _Int.endInt(b)
+        );
+    }
+    public visitFloat(node: Float) {
+        const b = this.builder;
+        return (
+            _FloatingPoint.startFloatingPoint(b) ||
+            _FloatingPoint.addPrecision(b, node.precision) ||
+            _FloatingPoint.endFloatingPoint(b)
+        );
+    }
+    public visitBinary(_node: Binary) {
+        const b = this.builder;
+        return (
+            _Binary.startBinary(b) ||
+            _Binary.endBinary(b)
+        );
+    }
+    public visitBool(_node: Bool) {
+        const b = this.builder;
+        return (
+            _Bool.startBool(b) ||
+            _Bool.endBool(b)
+        );
+    }
+    public visitUtf8(_node: Utf8) {
+        const b = this.builder;
+        return (
+            _Utf8.startUtf8(b) ||
+            _Utf8.endUtf8(b)
+        );
+    }
+    public visitDecimal(node: Decimal) {
+        const b = this.builder;
+        return (
+            _Decimal.startDecimal(b) ||
+            _Decimal.addScale(b, node.scale) ||
+            _Decimal.addPrecision(b, node.precision) ||
+            _Decimal.endDecimal(b)
+        );
+    }
+    public visitDate(node: Date_) {
+        const b = this.builder;
+        return _Date.startDate(b) || _Date.addUnit(b, node.unit) || _Date.endDate(b);
+    }
+    public visitTime(node: Time) {
+        const b = this.builder;
+        return (
+            _Time.startTime(b) ||
+            _Time.addUnit(b, node.unit) ||
+            _Time.addBitWidth(b, node.bitWidth) ||
+            _Time.endTime(b)
+        );
+    }
+    public visitTimestamp(node: Timestamp) {
+        const b = this.builder;
+        const timezone = (node.timezone && b.createString(node.timezone)) || undefined;
+        return (
+            _Timestamp.startTimestamp(b) ||
+            _Timestamp.addUnit(b, node.unit) ||
+            (timezone !== undefined && _Timestamp.addTimezone(b, timezone)) ||
+            _Timestamp.endTimestamp(b)
+        );
+    }
+    public visitInterval(node: Interval) {
+        const b = this.builder;
+        return (
+            _Interval.startInterval(b) || _Interval.addUnit(b, node.unit) || _Interval.endInterval(b)
+        );
+    }
+    public visitList(_node: List) {
+        const b = this.builder;
+        return (
+            _List.startList(b) ||
+            _List.endList(b)
+        );
+    }
+    public visitStruct(_node: Struct) {
+        const b = this.builder;
+        return (
+            _Struct.startStruct_(b) ||
+            _Struct.endStruct_(b)
+        );
+    }
+    public visitUnion(node: Union) {
+        const b = this.builder;
+        const typeIds =
+            _Union.startTypeIdsVector(b, node.typeIds.length) ||
+            _Union.createTypeIdsVector(b, node.typeIds);
+        return (
+            _Union.startUnion(b) ||
+            _Union.addMode(b, node.mode) ||
+            _Union.addTypeIds(b, typeIds) ||
+            _Union.endUnion(b)
+        );
+    }
+    public visitDictionary(node: Dictionary) {
+        const b = this.builder;
+        const indexType = this.visit(node.indicies);
+        return (
+            _DictionaryEncoding.startDictionaryEncoding(b) ||
+            _DictionaryEncoding.addId(b, new Long(node.id, 0)) ||
+            _DictionaryEncoding.addIsOrdered(b, node.isOrdered) ||
+            (indexType !== undefined && _DictionaryEncoding.addIndexType(b, indexType)) ||
+            _DictionaryEncoding.endDictionaryEncoding(b)
+        );
+    }
+    public visitFixedSizeBinary(node: FixedSizeBinary) {
+        const b = this.builder;
+        return (
+            _FixedSizeBinary.startFixedSizeBinary(b) ||
+            _FixedSizeBinary.addByteWidth(b, node.byteWidth) ||
+            _FixedSizeBinary.endFixedSizeBinary(b)
+        );
+    }
+    public visitFixedSizeList(node: FixedSizeList) {
+        const b = this.builder;
+        return (
+            _FixedSizeList.startFixedSizeList(b) ||
+            _FixedSizeList.addListSize(b, node.listSize) ||
+            _FixedSizeList.endFixedSizeList(b)
+        );
+    }
+    public visitMap(node: Map_) {
+        const b = this.builder;
+        return (
+            _Map.startMap(b) ||
+            _Map.addKeysSorted(b, node.keysSorted) ||
+            _Map.endMap(b)
+        );
+    }
+}
+
+// @ts-ignore
+function writeFooter(b: Builder, node: Footer) {
+    let schemaOffset = writeSchema(b, node.schema);
+    let recordBatchesOffset: number | undefined = undefined;
+    let dictionaryBatchesOffset: number | undefined = undefined;
+    if (node.recordBatches && node.recordBatches.length) {
+        recordBatchesOffset =
+            _Footer.startRecordBatchesVector(b, node.recordBatches.length) ||
+            node.recordBatches.map((rb) => writeBlock(b, rb)) &&
+            b.endVector();
+    }
+    if (node.dictionaryBatches && node.dictionaryBatches.length) {
+        dictionaryBatchesOffset =
+            _Footer.startDictionariesVector(b, node.dictionaryBatches.length) ||
+            node.dictionaryBatches.map((db) => writeBlock(b, db)) &&
+            b.endVector();
+    }
+    return (
+        _Footer.startFooter(b) ||
+        _Footer.addSchema(b, schemaOffset) ||
+        _Footer.addVersion(b, node.schema.version) ||
+        (recordBatchesOffset !== undefined && _Footer.addRecordBatches(b, recordBatchesOffset)) ||
+        (dictionaryBatchesOffset !== undefined && _Footer.addDictionaries(b, dictionaryBatchesOffset)) ||
+        _Footer.endFooter(b)
+    );
+}
+
+function writeBlock(b: Builder, node: FileBlock) {
+    return _Block.createBlock(b, node.offset, node.metaDataLength, node.bodyLength);
+}
+
+function writeMessage(b: Builder, node: Message) {
+    let messageHeaderOffset = 0;
+    if (Message.isSchema(node)) {
+        messageHeaderOffset = writeSchema(b, node as Schema);
+    } else if (Message.isRecordBatch(node)) {
+        messageHeaderOffset = writeRecordBatch(b, node as RecordBatchMetadata);
+    } else if (Message.isDictionaryBatch(node)) {
+        messageHeaderOffset = writeDictionaryBatch(b, node as DictionaryBatch);
+    }
+    return (
+        _Message.startMessage(b) ||
+        _Message.addVersion(b, node.version) ||
+        _Message.addHeader(b, messageHeaderOffset) ||
+        _Message.addHeaderType(b, node.headerType) ||
+        _Message.addBodyLength(b, new Long(node.bodyLength, 0)) ||
+        _Message.endMessage(b)
+    );
+}
+
+function writeSchema(b: Builder, node: Schema) {
+    const fieldOffsets = node.fields.map((f) => writeField(b, f));
+    const fieldsOffset =
+        _Schema.startFieldsVector(b, fieldOffsets.length) ||
+        _Schema.createFieldsVector(b, fieldOffsets);
+    return (
+        _Schema.startSchema(b) ||
+        _Schema.addFields(b, fieldsOffset) ||
+        _Schema.addEndianness(b, _Endianness.Little) ||
+        _Schema.endSchema(b)
+    );
+}
+
+function writeRecordBatch(b: Builder, node: RecordBatchMetadata) {
+    let nodesOffset: number | undefined = undefined;
+    let buffersOffset: number | undefined = undefined;
+    if (node.nodes && node.nodes.length) {
+        nodesOffset =
+            _RecordBatch.startNodesVector(b, node.nodes.length) ||
+            node.nodes.map((n) => writeFieldNode(b, n)) &&
+            b.endVector();
+    }
+    if (node.buffers && node.buffers.length) {
+        buffersOffset =
+            _RecordBatch.startBuffersVector(b, node.buffers.length) ||
+            node.buffers.map((bm) => writeBuffer(b, bm)) &&
+            b.endVector();
+    }
+    return (
+        _RecordBatch.startRecordBatch(b) ||
+        _RecordBatch.addLength(b, new Long(node.length, 0)) ||
+        (nodesOffset !== undefined && _RecordBatch.addNodes(b, nodesOffset)) ||
+        (buffersOffset !== undefined && _RecordBatch.addBuffers(b, buffersOffset)) ||
+        _RecordBatch.endRecordBatch(b)
+    );
+}
+
+function writeDictionaryBatch(b: Builder, node: DictionaryBatch) {
+    const dataOffset = writeRecordBatch(b, node.data);
+    return (
+        _DictionaryBatch.startDictionaryBatch(b) ||
+        _DictionaryBatch.addId(b, new Long(node.id, 0)) ||
+        _DictionaryBatch.addIsDelta(b, node.isDelta) ||
+        _DictionaryBatch.addData(b, dataOffset) ||
+        _DictionaryBatch.endDictionaryBatch(b)
+    );
+}
+
+function writeBuffer(b: Builder, node: BufferMetadata) {
+    return _Buffer.createBuffer(b, new Long(node.offset, 0), new Long(node.length, 0));
+}
+
+function writeFieldNode(b: Builder, node: FieldMetadata) {
+    return _FieldNode.createFieldNode(b, new Long(node.length, 0), new Long(node.nullCount, 0));
+}
+
+function writeField(b: Builder, node: Field) {
+    let type = node.type;
+    let typeOffset = new TypeSerializer(b).visit(type);
+    let name: number | undefined = undefined;
+    let children: number | undefined = undefined;
+    let metadata: number | undefined = undefined;
+    let dictionary: number | undefined = undefined;
+    if (DataType.isDictionary(node.type)) {
+        dictionary = new TypeSerializer(b).visit(node.type.dictionary);
+    }
+    if (type.children && type.children.length) {
+        children = _Field.createChildrenVector(b, type.children.map((f) => writeField(b, f)));
+    }
+    if (node.metadata) {
+        metadata = _Field.createCustomMetadataVector(
+            b,
+            [...node.metadata].map(([k, v]) => {
+                const key = b.createString(k);
+                const val = b.createString(v);
+                return (
+                    _KeyValue.startKeyValue(b) ||
+                    _KeyValue.addKey(b, key) ||
+                    _KeyValue.addValue(b, val) ||
+                    _KeyValue.endKeyValue(b)
+                );
+            })
+        );
+    }
+    if (node.name) {
+        name = b.createString(node.name);
+    }
+    return (
+        _Field.startField(b) ||
+        _Field.addType(b, typeOffset) ||
+        _Field.addTypeType(b, node.typeId) ||
+        _Field.addNullable(b, !!node.nullable) ||
+        (name !== undefined && _Field.addName(b, name)) ||
+        (children !== undefined && _Field.addChildren(b, children)) ||
+        (dictionary !== undefined && _Field.addDictionary(b, dictionary)) ||
+        (metadata !== undefined && _Field.addCustomMetadata(b, metadata)) ||
+        _Field.endField(b)
+    );
+}

From c8ba1fe47a303653f0b704cad51b1bfd6b33904f Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 26 Feb 2018 07:27:14 -0800
Subject: [PATCH 05/51] don't write an empty buffer for NullVectors

---
 js/src/ipc/writer/binary.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
index 0b1cb26a361..4a053d1e935 100644
--- a/js/src/ipc/writer/binary.ts
+++ b/js/src/ipc/writer/binary.ts
@@ -85,7 +85,7 @@ export class RecordBatchSerializer extends VectorVisitor {
             : this.getTruncatedBitmap(data.offset, length, data.nullBitmap!), 64);
         return super.visit(vector);
     }
-    public visitNull           (_vector: Vector<Null>)           { return this.addBuffer(null);              }
+    public visitNull           (_vector: Vector<Null>)           { return this;                              }
     public visitBool           (vector: Vector<Bool>)            { return this.visitFlatVector(vector);      }
     public visitInt            (vector: Vector<Int>)             { return this.visitFlatVector(vector);      }
     public visitFloat          (vector: Vector<Float>)           { return this.visitFlatVector(vector);      }

From ae1f4819799ec487f539963e1d452e1e66de02d3 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 26 Feb 2018 07:32:53 -0800
Subject: [PATCH 06/51] align to 64-byte boundaries

---
 js/src/ipc/writer/binary.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
index 4a053d1e935..7b15fc86070 100644
--- a/js/src/ipc/writer/binary.ts
+++ b/js/src/ipc/writer/binary.ts
@@ -82,7 +82,7 @@ export class RecordBatchSerializer extends VectorVisitor {
         this.fieldNodes.push(new FieldMetadata(length, nullCount));
         this.addBuffer(nullCount <= 0
             ? new Uint8Array(0) // placeholder validity buffer
-            : this.getTruncatedBitmap(data.offset, length, data.nullBitmap!), 64);
+            : this.getTruncatedBitmap(data.offset, length, data.nullBitmap!));
         return super.visit(vector);
     }
     public visitNull           (_vector: Vector<Null>)           { return this;                              }
@@ -170,7 +170,7 @@ export class RecordBatchSerializer extends VectorVisitor {
         const { values, valueOffsets } = data;
         const firstOffset = valueOffsets[0];
         const lastOffset = valueOffsets[valueOffsets.length - 1];
-        const byteLength = Math.min(align(lastOffset - firstOffset, 8), values.byteLength - firstOffset);
+        const byteLength = Math.min(lastOffset - firstOffset, values.byteLength - firstOffset);
         // Push in the order of FlatList types
         // valueOffsets buffer first
         this.addBuffer(this.getZeroBasedValueOffsets(sliceOffset, length, valueOffsets));
@@ -185,7 +185,7 @@ export class RecordBatchSerializer extends VectorVisitor {
         }
         return this;
     }
-    protected addBuffer(array?: TypedArray | null, alignment: number = 8) {
+    protected addBuffer(array: TypedArray, alignment: number = 64) {
         const values = array || new Uint8Array(0);
         const alignedLength = align(values.byteLength, alignment);
         this.buffers.push(values);

From efb840f3b286603c7094b2f78a7575411d8ec9dc Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Tue, 27 Feb 2018 14:15:39 -0800
Subject: [PATCH 07/51] fix typo

---
 js/src/ipc/writer/binary.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
index 7b15fc86070..19f97d96df8 100644
--- a/js/src/ipc/writer/binary.ts
+++ b/js/src/ipc/writer/binary.ts
@@ -379,7 +379,7 @@ export class TypeSerializer extends TypeVisitor {
     }
     public visitDictionary(node: Dictionary) {
         const b = this.builder;
-        const indexType = this.visit(node.indicies);
+        const indexType = this.visit(node.indices);
         return (
             _DictionaryEncoding.startDictionaryEncoding(b) ||
             _DictionaryEncoding.addId(b, new Long(node.id, 0)) ||

From 18b9dd2a9f287f87d38745ef4171961ce64b646a Mon Sep 17 00:00:00 2001
From: lsb <leebutterman@gmail.com>
Date: Wed, 25 Apr 2018 19:08:52 -0700
Subject: [PATCH 08/51] Fix a typo

---
 js/DEVELOP.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/js/DEVELOP.md b/js/DEVELOP.md
index 1dd999a9efb..c7ffd5c46d0 100644
--- a/js/DEVELOP.md
+++ b/js/DEVELOP.md
@@ -64,7 +64,7 @@ This argument configuration also applies to `clean` and `test` scripts.
 
 * `npm run deploy`
 
-Uses [learna](https://github.com/lerna/lerna) to publish each build target to npm with [conventional](https://conventionalcommits.org/) [changelogs](https://github.com/conventional-changelog/conventional-changelog/tree/master/packages/conventional-changelog-cli).
+Uses [lerna](https://github.com/lerna/lerna) to publish each build target to npm with [conventional](https://conventionalcommits.org/) [changelogs](https://github.com/conventional-changelog/conventional-changelog/tree/master/packages/conventional-changelog-cli).
 
 # Updating the Arrow format flatbuffers generated code
 

From aaec76b34165974241768150b916f4306496aeef Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 17:12:39 -0700
Subject: [PATCH 09/51] fix @std/esm options for node10

---
 js/gulp/uglify-task.js | 19 ++++++++++++++++++-
 js/package.json        |  2 +-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/js/gulp/uglify-task.js b/js/gulp/uglify-task.js
index f8dc123cae7..ed0e7659441 100644
--- a/js/gulp/uglify-task.js
+++ b/js/gulp/uglify-task.js
@@ -30,7 +30,24 @@ const { memoizeTask } = require('./memoize-task');
 const { compileBinFiles } = require('./typescript-task');
 const { Observable, ReplaySubject } = require('rxjs');
 const UglifyJSPlugin = require(`uglifyjs-webpack-plugin`);
-const esmRequire = require(`@std/esm`)(module, { cjs: true, esm: `js`, warnings: false });
+const esmRequire = require(`@std/esm`)(module, {
+    mode: `js`,
+    warnings: false,
+    cjs: {
+        /* A boolean for storing ES modules in require.cache. */
+        cache: true,
+        /* A boolean for respecting require.extensions in ESM. */
+        extensions: true,
+        /* A boolean for __esModule interoperability. */
+        interop: true,
+        /* A boolean for importing named exports of CJS modules. */
+        namedExports: true,
+        /* A boolean for following CJS path rules in ESM. */
+        paths: true,
+        /* A boolean for __dirname, __filename, and require in ESM. */
+        vars: true,
+    }
+});
 
 const uglifyTask = ((cache, commonConfig) => memoizeTask(cache, function uglifyJS(target, format) {
 
diff --git a/js/package.json b/js/package.json
index 9bd9d13994e..9b2aeaba87c 100644
--- a/js/package.json
+++ b/js/package.json
@@ -63,7 +63,7 @@
     "tslib": "1.9.0"
   },
   "devDependencies": {
-    "@std/esm": "0.19.7",
+    "@std/esm": "0.26.0",
     "@types/glob": "5.0.35",
     "@types/jest": "22.1.0",
     "ast-types": "0.10.1",

From d98e178988c3857dc3c656bb2fdd244f8b8301a2 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 17:14:43 -0700
Subject: [PATCH 10/51] add option to run gulp cmds with `-t src` to run jest
 against the `src` folder direct

---
 js/gulp/argv.js         | 12 ++++++++----
 js/gulp/build-task.js   |  4 +++-
 js/gulp/package-task.js |  5 +++--
 js/gulp/test-task.js    | 11 ++++++++---
 js/gulp/util.js         | 19 ++++++++++++++-----
 js/gulpfile.js          |  7 +++----
 6 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/js/gulp/argv.js b/js/gulp/argv.js
index 8a83820c1fe..7dceb0f74c5 100644
--- a/js/gulp/argv.js
+++ b/js/gulp/argv.js
@@ -35,10 +35,14 @@ const argv = require(`command-line-args`)([
 
 const { targets, modules } = argv;
 
-argv.target && !targets.length && targets.push(argv.target);
-argv.module && !modules.length && modules.push(argv.module);
-(argv.all || !targets.length) && targets.push(`all`);
-(argv.all || !modules.length) && modules.push(`all`);
+if (argv.target === `src`) {
+    argv.target && !targets.length && targets.push(argv.target);
+} else {
+    argv.target && !targets.length && targets.push(argv.target);
+    argv.module && !modules.length && modules.push(argv.module);
+    (argv.all || !targets.length) && targets.push(`all`);
+    (argv.all || !modules.length) && modules.push(`all`);
+}
 
 if (argv.coverage && (!argv.json_files || !argv.json_files.length)) {
 
diff --git a/js/gulp/build-task.js b/js/gulp/build-task.js
index 01152e662fc..5eabb825c23 100644
--- a/js/gulp/build-task.js
+++ b/js/gulp/build-task.js
@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+const { Observable } = require('rxjs');
 const { npmPkgName } = require('./util');
 const { memoizeTask } = require('./memoize-task');
 
@@ -24,7 +25,8 @@ const typescriptTask = require('./typescript-task');
 const { arrowTask, arrowTSTask } = require('./arrow-task');
 
 const buildTask = ((cache) => memoizeTask(cache, function build(target, format, ...args) {
-    return target === npmPkgName               ? arrowTask(target, format, ...args)()
+    return target === `src`                    ? Observable.empty()
+         : target === npmPkgName               ? arrowTask(target, format, ...args)()
          : target === `ts`                     ? arrowTSTask(target, format, ...args)()
          : format === `umd` ? target === `es5` ? closureTask(target, format, ...args)()
                                                : uglifyTask(target, format, ...args)()
diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js
index 3390da01923..98812b379b5 100644
--- a/js/gulp/package-task.js
+++ b/js/gulp/package-task.js
@@ -27,6 +27,7 @@ const { Observable, ReplaySubject } = require('rxjs');
 const gulpJsonTransform = require('gulp-json-transform');
 
 const packageTask = ((cache) => memoizeTask(cache, function bundle(target, format) {
+    if (target === `src`) return Observable.empty();
     const out = targetDir(target, format);
     const jsonTransform = gulpJsonTransform(target === npmPkgName ? createMainPackageJson(target, format) :
                                             target === `ts`       ? createTypeScriptPackageJson(target, format)
@@ -48,7 +49,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
     types: `${mainExport}.d.ts`,
     module: `${mainExport}.mjs`,
     unpkg: `${mainExport}.es5.min.js`,
-    [`@std/esm`]: { esm: `mjs`, warnings: false, sourceMap: true }
+    [`@std/esm`]: { mode: `all`, warnings: false, sourceMap: true }
 });
   
 const createTypeScriptPackageJson = (target, format) => (orig) => ({
@@ -77,6 +78,6 @@ const createScopedPackageJSON = (target, format) => (({ name, ...orig }) =>
 const conditionallyAddStandardESMEntry = (target, format) => (packageJSON) => (
     format !== `esm` && format !== `cls`
         ?      packageJSON
-        : { ...packageJSON, [`@std/esm`]: { esm: `js`, warnings: false, sourceMap: true } }
+        : { ...packageJSON, [`@std/esm`]: { mode: `js`, warnings: false, sourceMap: true } }
 );
   
\ No newline at end of file
diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js
index 7f655548eb8..b0e34f8c944 100644
--- a/js/gulp/test-task.js
+++ b/js/gulp/test-task.js
@@ -50,7 +50,7 @@ const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function
     opts.env = { ...opts.env,
         TEST_TARGET: target,
         TEST_MODULE: format,
-        TEST_TS_SOURCE: !!argv.coverage,
+        TEST_TS_SOURCE: !!argv.coverage || (target === 'src') || (opts.env.TEST_TS_SOURCE === 'true'),
         JSON_PATHS: JSON.stringify(Array.isArray(argv.json_files) ? argv.json_files : [argv.json_files]),
         ARROW_PATHS: JSON.stringify(Array.isArray(argv.arrow_files) ? argv.arrow_files : [argv.arrow_files]),
     };
@@ -80,13 +80,18 @@ const javaFilesDir = path.join(testFilesDir, 'java');
 const jsonFilesDir = path.join(testFilesDir, 'json');
 
 async function cleanTestData() {
-    return await del([`${testFilesDir}/**`, `${snapshotsDir}/**`]);
+    return await del([
+        `${cppFilesDir}/**`,
+        `${javaFilesDir}/**`,
+        `${jsonFilesDir}/**`,
+        `${snapshotsDir}/**`
+    ]);
 }
 
 async function createTestJSON() {
     await mkdirp(jsonFilesDir);
     await exec(`shx cp ${ARROW_INTEGRATION_DIR}/data/*.json ${jsonFilesDir}`);
-    await exec(`python ${ARROW_INTEGRATION_DIR}/integration_test.py --write_generated_json ${jsonFilesDir}`);
+    await exec(`python3 ${ARROW_INTEGRATION_DIR}/integration_test.py --write_generated_json ${jsonFilesDir}`);
 }
 
 async function createTestData() {
diff --git a/js/gulp/util.js b/js/gulp/util.js
index f35a447e708..25bf2c23716 100644
--- a/js/gulp/util.js
+++ b/js/gulp/util.js
@@ -27,7 +27,10 @@ const npmOrgName = `@${npmPkgName}`;
 const releasesRootDir = `targets`;
 const knownTargets = [`es5`, `es2015`, `esnext`];
 const knownModules = [`cjs`, `esm`, `cls`, `umd`];
-const moduleFormatsToSkipCombosOf = { cls: { test: true, integration: true } };
+const tasksToSkipPerTargetOrFormat = {
+    src: { clean: true, build: true },
+    cls: { test: true, integration: true }
+};
 const packageJSONFields = [
   `version`, `license`, `description`,
   `author`, `homepage`, `repository`,
@@ -131,8 +134,14 @@ function* combinations(_targets, _modules) {
     const targets = known(knownTargets, _targets || [`all`]);
     const modules = known(knownModules, _modules || [`all`]);
 
-    if (_targets[0] === `all` && _modules[0] === `all`) {
+    if (_targets.indexOf(`src`) > -1) {
+        yield [`src`, ``];
+        return;
+    }
+
+    if (_targets.indexOf(`all`) > -1 && _modules.indexOf(`all`) > -1) {
         yield [`ts`, ``];
+        yield [`src`, ``];
         yield [npmPkgName, ``];
     }        
     
@@ -143,8 +152,8 @@ function* combinations(_targets, _modules) {
     }
 
     function known(known, values) {
-        return ~values.indexOf(`all`)
-            ? known
+        return ~values.indexOf(`all`) ? known
+            :  ~values.indexOf(`src`) ? [`src`]
             : Object.keys(
                 values.reduce((map, arg) => ((
                     (known.indexOf(arg) !== -1) &&
@@ -159,7 +168,7 @@ module.exports = {
 
     mainExport, npmPkgName, npmOrgName, metadataFiles, packageJSONFields,
 
-    knownTargets, knownModules, moduleFormatsToSkipCombosOf,
+    knownTargets, knownModules, tasksToSkipPerTargetOrFormat,
     ESKeywords, gCCLanguageNames, UMDSourceTargets, uglifyLanguageNames,
 
     taskName, packageName, tsconfigName, targetDir, combinations, observableFromStreams,
diff --git a/js/gulpfile.js b/js/gulpfile.js
index 891d6c79b38..dfebfaeb8bb 100644
--- a/js/gulpfile.js
+++ b/js/gulpfile.js
@@ -29,7 +29,7 @@ const {
     taskName, combinations,
     knownTargets, knownModules,
     npmPkgName, UMDSourceTargets,
-    moduleFormatsToSkipCombosOf
+    tasksToSkipPerTargetOrFormat
 } = require('./gulp/util');
 
 for (const [target, format] of combinations([`all`], [`all`])) {
@@ -99,9 +99,8 @@ function getTasks(name) {
     if (targets.indexOf(`ts`) !== -1) tasks.push(`${name}:ts`);
     if (targets.indexOf(npmPkgName) !== -1) tasks.push(`${name}:${npmPkgName}`);
     for (const [target, format] of combinations(targets, modules)) {
-        if (moduleFormatsToSkipCombosOf[format] && moduleFormatsToSkipCombosOf[format][name]) {
-            continue;
-        }
+        if (tasksToSkipPerTargetOrFormat[target] && tasksToSkipPerTargetOrFormat[target][name]) continue;
+        if (tasksToSkipPerTargetOrFormat[format] && tasksToSkipPerTargetOrFormat[format][name]) continue;
         tasks.push(`${name}:${taskName(target, format)}`);
     }
     return tasks.length && tasks || [(done) => done()];

From 7fff99ee448b4bb200b366670c9c9a4eb0f3d177 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 17:16:22 -0700
Subject: [PATCH 11/51] move IPC magic into its own module

---
 js/src/ipc/magic.ts         | 20 ++++++++++++++++++++
 js/src/ipc/reader/binary.ts | 21 +--------------------
 2 files changed, 21 insertions(+), 20 deletions(-)
 create mode 100644 js/src/ipc/magic.ts

diff --git a/js/src/ipc/magic.ts b/js/src/ipc/magic.ts
new file mode 100644
index 00000000000..c9e08a1be53
--- /dev/null
+++ b/js/src/ipc/magic.ts
@@ -0,0 +1,20 @@
+export const PADDING = 4;
+export const MAGIC_STR = 'ARROW1';
+export const MAGIC = new Uint8Array(MAGIC_STR.length);
+
+for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) {
+    MAGIC[i] = MAGIC_STR.charCodeAt(i);
+}
+
+export function checkForMagicArrowString(buffer: Uint8Array, index = 0) {
+    for (let i = -1, n = MAGIC.length; ++i < n;) {
+        if (MAGIC[i] !== buffer[index + i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+export const magicLength = MAGIC.length;
+export const magicAndPadding = magicLength + PADDING;
+export const magicX2AndPadding = magicLength * 2 + PADDING;
diff --git a/js/src/ipc/reader/binary.ts b/js/src/ipc/reader/binary.ts
index 26bc10bb377..d7d7cb690b7 100644
--- a/js/src/ipc/reader/binary.ts
+++ b/js/src/ipc/reader/binary.ts
@@ -18,6 +18,7 @@
 import { Vector } from '../../vector';
 import { flatbuffers } from 'flatbuffers';
 import { TypeDataLoader } from './vector';
+import { checkForMagicArrowString, PADDING, magicLength, magicAndPadding, magicX2AndPadding } from '../magic';
 import { Message, Footer, FileBlock, RecordBatchMetadata, DictionaryBatch, BufferMetadata, FieldMetadata, } from '../metadata';
 import {
     Schema, Field,
@@ -129,26 +130,6 @@ function readSchema(bb: ByteBuffer) {
     return { schema, readMessages };
 }
 
-const PADDING = 4;
-const MAGIC_STR = 'ARROW1';
-const MAGIC = new Uint8Array(MAGIC_STR.length);
-for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) {
-    MAGIC[i] = MAGIC_STR.charCodeAt(i);
-}
-
-function checkForMagicArrowString(buffer: Uint8Array, index = 0) {
-    for (let i = -1, n = MAGIC.length; ++i < n;) {
-        if (MAGIC[i] !== buffer[index + i]) {
-            return false;
-        }
-    }
-    return true;
-}
-
-const magicLength = MAGIC.length;
-const magicAndPadding = magicLength + PADDING;
-const magicX2AndPadding = magicLength * 2 + PADDING;
-
 function readStreamSchema(bb: ByteBuffer) {
     if (!checkForMagicArrowString(bb.bytes(), 0)) {
         for (const message of readMessages(bb)) {

From 4333e548f6c93ad21cc1fc73c60d5569b6c97959 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 17:17:46 -0700
Subject: [PATCH 12/51] FileBlock constructor should accept Long | number, have
 public number fields

---
 js/src/ipc/metadata.ts      | 7 ++++++-
 js/src/ipc/reader/binary.ts | 4 ++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/js/src/ipc/metadata.ts b/js/src/ipc/metadata.ts
index 88b7e52983b..9d2ea26b45e 100644
--- a/js/src/ipc/metadata.ts
+++ b/js/src/ipc/metadata.ts
@@ -25,7 +25,12 @@ export class Footer {
 }
 
 export class FileBlock {
-    constructor(public metaDataLength: number, public bodyLength: Long, public offset: Long) {}
+    public offset: number;
+    public bodyLength: number;
+    constructor(public metaDataLength: number, bodyLength: Long | number, offset: Long | number) {
+        this.offset = typeof offset === 'number' ? offset : offset.low;
+        this.bodyLength = typeof bodyLength === 'number' ? bodyLength : bodyLength.low;
+    }
 }
 
 export class Message {
diff --git a/js/src/ipc/reader/binary.ts b/js/src/ipc/reader/binary.ts
index d7d7cb690b7..e371831951d 100644
--- a/js/src/ipc/reader/binary.ts
+++ b/js/src/ipc/reader/binary.ts
@@ -172,11 +172,11 @@ function readFileSchema(bb: ByteBuffer) {
 function readFileMessages(footer: Footer) {
     return function* (bb: ByteBuffer) {
         for (let i = -1, batches = footer.dictionaryBatches, n = batches.length; ++i < n;) {
-            bb.setPosition(batches[i].offset.low);
+            bb.setPosition(batches[i].offset);
             yield readMessage(bb, bb.readInt32(bb.position())) as DictionaryBatch;
         }
         for (let i = -1, batches = footer.recordBatches, n = batches.length; ++i < n;) {
-            bb.setPosition(batches[i].offset.low);
+            bb.setPosition(batches[i].offset);
             yield readMessage(bb, bb.readInt32(bb.position())) as RecordBatchMetadata;
         }
     };

From 85eb7eeece00d84132dbd63b69353ac08239d9cd Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 17:18:18 -0700
Subject: [PATCH 13/51] fix erroneous footer length check in reader

---
 js/src/ipc/reader/binary.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/js/src/ipc/reader/binary.ts b/js/src/ipc/reader/binary.ts
index e371831951d..1f970df912f 100644
--- a/js/src/ipc/reader/binary.ts
+++ b/js/src/ipc/reader/binary.ts
@@ -162,7 +162,7 @@ function readFileSchema(bb: ByteBuffer) {
         (!checkForMagicArrowString(bb.bytes(), fileLength - magicLength) /* Missing magic end      */) ||
         (/*                                                    Invalid footer length  */
         (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 &&
-        (footerLength + magicX2AndPadding > fileLength))) {
+        (footerLength + footerOffset > fileLength))) {
         return null;
     }
     bb.setPosition(footerOffset - footerLength);

From a9d773d8fbeaa5979e22b08dce86b0c2fecbece6 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 17:19:35 -0700
Subject: [PATCH 14/51] move ValidityView into its own module, like ChunkedView
 is

---
 js/src/vector.ts          |  3 +-
 js/src/vector/flat.ts     | 49 +--------------------------------
 js/src/vector/validity.ts | 58 +++++++++++++++++++++++++++++++++++++++
 js/src/vector/view.ts     |  3 +-
 4 files changed, 63 insertions(+), 50 deletions(-)
 create mode 100644 js/src/vector/validity.ts

diff --git a/js/src/vector.ts b/js/src/vector.ts
index 67a4f99be73..8c1b9900ca8 100644
--- a/js/src/vector.ts
+++ b/js/src/vector.ts
@@ -180,10 +180,11 @@ import { Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, Float16, Floa
 import { Struct, Union, SparseUnion, DenseUnion, FixedSizeBinary, FixedSizeList, Map_, Dictionary } from './type';
 
 import { ChunkedView } from './vector/chunked';
+import { ValidityView } from './vector/validity';
 import { DictionaryView } from './vector/dictionary';
 import { ListView, FixedSizeListView, BinaryView, Utf8View } from './vector/list';
 import { UnionView, DenseUnionView, NestedView, StructView, MapView } from './vector/nested';
-import { FlatView, NullView, BoolView, ValidityView, PrimitiveView, FixedSizeView, Float16View } from './vector/flat';
+import { FlatView, NullView, BoolView, PrimitiveView, FixedSizeView, Float16View } from './vector/flat';
 import { DateDayView, DateMillisecondView, IntervalYearMonthView } from './vector/flat';
 import { TimestampDayView, TimestampSecondView, TimestampMillisecondView, TimestampMicrosecondView, TimestampNanosecondView } from './vector/flat';
 import { packBools } from './util/bit';
diff --git a/js/src/vector/flat.ts b/js/src/vector/flat.ts
index 06189c47d84..c16fd2b48ab 100644
--- a/js/src/vector/flat.ts
+++ b/js/src/vector/flat.ts
@@ -18,8 +18,8 @@
 import { Data } from '../data';
 import { View } from '../vector';
 import { getBool, setBool, iterateBits } from '../util/bit';
+import { FlatType, PrimitiveType, IterableArrayLike } from '../type';
 import { Bool, Float16, Date_, Interval, Null, Int32, Timestamp } from '../type';
-import { DataType, FlatType, PrimitiveType, IterableArrayLike } from '../type';
 
 export class FlatView<T extends FlatType> implements View<T> {
     public length: number;
@@ -103,53 +103,6 @@ export class BoolView extends FlatView<Bool> {
     }
 }
 
-export class ValidityView<T extends DataType> implements View<T> {
-    protected view: View<T>;
-    protected length: number;
-    protected offset: number;
-    protected nullBitmap: Uint8Array;
-    constructor(data: Data<T>, view: View<T>) {
-        this.view = view;
-        this.length = data.length;
-        this.offset = data.offset;
-        this.nullBitmap = data.nullBitmap!;
-    }
-    public clone(data: Data<T>): this {
-        return new ValidityView(data, this.view.clone(data)) as this;
-    }
-    public toArray(): IterableArrayLike<T['TValue'] | null> {
-        return [...this];
-    }
-    public indexOf(search: T['TValue']) {
-        let index = 0;
-        for (let value of this) {
-            if (value === search) { return index; }
-            ++index;
-        }
-
-        return -1;
-    }
-    public isValid(index: number): boolean {
-        const nullBitIndex = this.offset + index;
-        return getBool(null, index, this.nullBitmap[nullBitIndex >> 3], nullBitIndex % 8);
-    }
-    public get(index: number): T['TValue'] | null {
-        const nullBitIndex = this.offset + index;
-        return this.getNullable(this.view, index, this.nullBitmap[nullBitIndex >> 3], nullBitIndex % 8);
-    }
-    public set(index: number, value: T['TValue'] | null): void {
-        if (setBool(this.nullBitmap, this.offset + index, value != null)) {
-            this.view.set(index, value);
-        }
-    }
-    public [Symbol.iterator](): IterableIterator<T['TValue'] | null> {
-        return iterateBits<T['TValue'] | null>(this.nullBitmap, this.offset, this.length, this.view, this.getNullable);
-    }
-    protected getNullable(view: View<T>, index: number, byte: number, bit: number) {
-        return getBool(view, index, byte, bit) ? view.get(index) : null;
-    }
-}
-
 export class PrimitiveView<T extends PrimitiveType> extends FlatView<T> {
     public size: number;
     public ArrayType: T['ArrayType'];
diff --git a/js/src/vector/validity.ts b/js/src/vector/validity.ts
new file mode 100644
index 00000000000..a3e572847cb
--- /dev/null
+++ b/js/src/vector/validity.ts
@@ -0,0 +1,58 @@
+import { Data } from '../data';
+import { View, Vector } from '../vector';
+import { NestedView } from './nested';
+import { DataType, IterableArrayLike } from '../type';
+import { getBool, setBool, iterateBits } from '../util/bit';
+
+export class ValidityView<T extends DataType> implements View<T> {
+    protected view: View<T>;
+    protected length: number;
+    protected offset: number;
+    protected nullBitmap: Uint8Array;
+    constructor(data: Data<T>, view: View<T>) {
+        this.view = view;
+        this.length = data.length;
+        this.offset = data.offset;
+        this.nullBitmap = data.nullBitmap!;
+    }
+    public get size(): number {
+        return (this.view as any).size || 1;
+    }
+    public clone(data: Data<T>): this {
+        return new ValidityView(data, this.view.clone(data)) as this;
+    }
+    public toArray(): IterableArrayLike<T['TValue'] | null> {
+        return [...this];
+    }
+    public indexOf(search: T['TValue']) {
+        let index = 0;
+        for (let value of this) {
+            if (value === search) { return index; }
+            ++index;
+        }
+
+        return -1;
+    }
+    public isValid(index: number): boolean {
+        const nullBitIndex = this.offset + index;
+        return getBool(null, index, this.nullBitmap[nullBitIndex >> 3], nullBitIndex % 8);
+    }
+    public get(index: number): T['TValue'] | null {
+        const nullBitIndex = this.offset + index;
+        return this.getNullable(this.view, index, this.nullBitmap[nullBitIndex >> 3], nullBitIndex % 8);
+    }
+    public set(index: number, value: T['TValue'] | null): void {
+        if (setBool(this.nullBitmap, this.offset + index, value != null)) {
+            this.view.set(index, value);
+        }
+    }
+    public getChildAt<R extends DataType = DataType>(index: number): Vector<R> | null {
+        return (this.view as NestedView<any>).getChildAt<R>(index);
+    }
+    public [Symbol.iterator](): IterableIterator<T['TValue'] | null> {
+        return iterateBits<T['TValue'] | null>(this.nullBitmap, this.offset, this.length, this.view, this.getNullable);
+    }
+    protected getNullable(view: View<T>, index: number, byte: number, bit: number) {
+        return getBool(view, index, byte, bit) ? view.get(index) : null;
+    }
+}
diff --git a/js/src/vector/view.ts b/js/src/vector/view.ts
index c314a3180a5..36aeae71ee3 100644
--- a/js/src/vector/view.ts
+++ b/js/src/vector/view.ts
@@ -1,8 +1,9 @@
 export { ChunkedView } from './chunked';
+export { ValidityView } from './validity';
 export { DictionaryView } from './dictionary';
 export { ListView, FixedSizeListView, BinaryView, Utf8View } from './list';
 export { UnionView, DenseUnionView, NestedView, StructView, MapView } from './nested';
-export { FlatView, NullView, BoolView, ValidityView, PrimitiveView, FixedSizeView, Float16View } from './flat';
+export { FlatView, NullView, BoolView, PrimitiveView, FixedSizeView, Float16View } from './flat';
 export { DateDayView, DateMillisecondView } from './flat';
 export { IntervalYearMonthView, IntervalYearView, IntervalMonthView } from './flat';
 export { TimestampDayView, TimestampSecondView, TimestampMillisecondView, TimestampMicrosecondView, TimestampNanosecondView } from './flat';

From 508f4f8ea93127537c894772de468ac8c5b7090c Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 17:21:12 -0700
Subject: [PATCH 15/51] add getChildAt(n) methods to List and FixedSizeList
 Vectors to be more consistent with the other nested Vectors, make it easier
 to do the writer

---
 js/src/Arrow.externs.js |  8 ++++++++
 js/src/vector.ts        | 16 +++++++++++++---
 js/src/vector/list.ts   | 14 +++++++++++---
 3 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/js/src/Arrow.externs.js b/js/src/Arrow.externs.js
index b1ebd2d3c0b..51109a02ffe 100644
--- a/js/src/Arrow.externs.js
+++ b/js/src/Arrow.externs.js
@@ -539,7 +539,11 @@ var Utf8Vector = function() {};
 /** @type {?} */
 Utf8Vector.prototype.asBinary;
 var ListVector = function() {};
+/** @type {?} */
+ListVector.prototype.getChildAt;
 var FixedSizeListVector = function() {};
+/** @type {?} */
+FixedSizeListVector.prototype.getChildAt;
 var MapVector = function() {};
 /** @type {?} */
 MapVector.prototype.asStruct;
@@ -613,6 +617,10 @@ ValidityView.prototype.isValid;
 ValidityView.prototype.toArray;
 /** @type {?} */
 ValidityView.prototype.set;
+/** @type {?} */
+ValidityView.prototype.size;
+/** @type {?} */
+ValidityView.prototype.getChildAt;
 
 var DictionaryView = function() {};
 /** @type {?} */
diff --git a/js/src/vector.ts b/js/src/vector.ts
index 8c1b9900ca8..57101b59b05 100644
--- a/js/src/vector.ts
+++ b/js/src/vector.ts
@@ -358,15 +358,25 @@ export class Utf8Vector extends ListVectorBase<Utf8> {
 }
 
 export class ListVector<T extends DataType = DataType> extends ListVectorBase<List<T>> {
-    constructor(data: Data<List<T>>, view: View<List<T>> = new ListView(data)) {
+    // @ts-ignore
+    public readonly view: ListView<T>;
+    constructor(data: Data<T>, view: View<List<T>> = new ListView(data)) {
         super(data, view);
     }
+    public getChildAt(index: number): Vector<T> | null {
+        return this.view.getChildAt<T>(index);
+    }
 }
 
-export class FixedSizeListVector extends Vector<FixedSizeList> {
-    constructor(data: Data<FixedSizeList>, view: View<FixedSizeList> = new FixedSizeListView(data)) {
+export class FixedSizeListVector<T extends DataType = DataType> extends Vector<FixedSizeList<T>> {
+    // @ts-ignore
+    public readonly view: FixedSizeListView<T>;
+    constructor(data: Data<FixedSizeList<T>>, view: View<FixedSizeList<T>> = new FixedSizeListView(data)) {
         super(data, view);
     }
+    public getChildAt(index: number): Vector<T> | null {
+        return this.view.getChildAt<T>(index);
+    }
 }
 
 export class MapVector extends NestedVector<Map_> {
diff --git a/js/src/vector/list.ts b/js/src/vector/list.ts
index 8561c66ba4e..f1283f40cf9 100644
--- a/js/src/vector/list.ts
+++ b/js/src/vector/list.ts
@@ -19,7 +19,7 @@ import { Data } from '../data';
 import { View, Vector, createVector } from '../vector';
 import { TextEncoder, TextDecoder } from 'text-encoding-utf-8';
 import { List, Binary, Utf8, FixedSizeList, FlatListType } from '../type';
-import { ListType, DataType, IterableArrayLike } from '../type';
+import { ListType, SingleNestedType, DataType, IterableArrayLike } from '../type';
 
 export const encodeUtf8 = ((encoder) =>
     encoder.encode.bind(encoder) as (input?: string) => Uint8Array
@@ -29,7 +29,7 @@ export const decodeUtf8 = ((decoder) =>
     decoder.decode.bind(decoder) as (input?: ArrayBufferLike | ArrayBufferView) => string
 )(new TextDecoder('utf-8'));
 
-export abstract class ListViewBase<T extends (ListType | FlatListType | FixedSizeList)> implements View<T> {
+export abstract class ListViewBase<T extends (FlatListType | SingleNestedType)> implements View<T> {
     public length: number;
     public values: T['TArray'];
     public valueOffsets?: Int32Array;
@@ -81,10 +81,14 @@ export abstract class VariableListViewBase<T extends (ListType | FlatListType)>
 }
 
 export class ListView<T extends DataType> extends VariableListViewBase<List<T>> {
-    constructor(data: Data<List<T>>) {
+    public values: Vector<T>;
+    constructor(data: Data<T>) {
         super(data);
         this.values = createVector(data.values);
     }
+    public getChildAt<R extends T = T>(index: number): Vector<R> | null {
+        return index === 0 ? (this.values as Vector<R>) : null;
+    }
     protected getList(values: Vector<T>, index: number, valueOffsets: Int32Array) {
         return values.slice(valueOffsets[index], valueOffsets[index + 1]) as Vector<T>;
     }
@@ -100,11 +104,15 @@ export class ListView<T extends DataType> extends VariableListViewBase<List<T>>
 
 export class FixedSizeListView<T extends DataType> extends ListViewBase<FixedSizeList<T>> {
     public size: number;
+    public values: Vector<T>;
     constructor(data: Data<FixedSizeList<T>>) {
         super(data);
         this.size = data.type.listSize;
         this.values = createVector(data.values);
     }
+    public getChildAt<R extends T = T>(index: number): Vector<R> | null {
+        return index === 0 ? (this.values as Vector<R>) : null;
+    }
     protected getList(values: Vector<T>, index: number) {
         const size = this.size;
         return values.slice(index *= size, index + size) as Vector<T>;

From da0f45717f6ce1613a65a8768d225bde22d66b76 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 17:22:43 -0700
Subject: [PATCH 16/51] first pass at a working binary writer, only arrow
 stream format tested so far

---
 js/src/ipc/writer/arrow.ts  |  40 +++++
 js/src/ipc/writer/binary.ts | 312 +++++++++++++++++++++++++++---------
 2 files changed, 272 insertions(+), 80 deletions(-)
 create mode 100644 js/src/ipc/writer/arrow.ts

diff --git a/js/src/ipc/writer/arrow.ts b/js/src/ipc/writer/arrow.ts
new file mode 100644
index 00000000000..4ff82a61d2f
--- /dev/null
+++ b/js/src/ipc/writer/arrow.ts
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Table } from '../../table';
+import { serializeStream, serializeFile } from './binary';
+
+export function writeTableBinary(table: Table, stream = true) {
+    return concatBuffers(stream ? serializeStream(table) : serializeFile(table));
+}
+
+function concatBuffers(messages: Iterable<Uint8Array | Buffer>) {
+
+    let buffers = [], byteLength = 0;
+
+    for (const message of messages) {
+        buffers.push(message);
+        byteLength += message.byteLength;
+    }
+
+    const { buffer } = buffers.reduce(({ buffer, byteOffset }, bytes) => {
+        buffer.set(bytes, byteOffset);
+        return { buffer, byteOffset: byteOffset + bytes.byteLength };
+    }, { buffer: new Uint8Array(byteLength), byteOffset: 0 });
+
+    return buffer;
+}
diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
index 19f97d96df8..12686bdc3d2 100644
--- a/js/src/ipc/writer/binary.ts
+++ b/js/src/ipc/writer/binary.ts
@@ -15,31 +15,139 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import { Table } from '../../table';
 import { DenseUnionData } from '../../data';
 import { RecordBatch } from '../../recordbatch';
 import { VectorVisitor, TypeVisitor } from '../../visitor';
+import { MAGIC, PADDING, magicLength, magicAndPadding } from '../magic';
 import { align, getBool, packBools, iterateBits } from '../../util/bit';
-import { Vector, UnionVector, DictionaryVector, NestedVector } from '../../vector';
+import { Vector, UnionVector, DictionaryVector, NestedVector, ListVector } from '../../vector';
 import { BufferMetadata, FieldMetadata, Footer, FileBlock, Message, RecordBatchMetadata, DictionaryBatch } from '../metadata';
 import {
     Schema, Field, TypedArray, MetadataVersion,
+    DataType,
     Dictionary,
     Null, Int, Float,
     Binary, Bool, Utf8, Decimal,
     Date_, Time, Timestamp, Interval,
     List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_,
-    UnionMode, SparseUnion, DenseUnion, FlatListType, DataType, FlatType, NestedType,
+    FlatType, FlatListType, NestedType, UnionMode, SparseUnion, DenseUnion, SingleNestedType,
 } from '../../type';
 
-export class RecordBatchSerializer extends VectorVisitor {
-    protected byteLength = 0;
-    // @ts-ignore
-    protected buffers: TypedArray[];
-    // @ts-ignore
-    protected fieldNodes: FieldMetadata[];
-    // @ts-ignore
-    protected buffersMeta: BufferMetadata[];
-    public writeRecordBatch(recordBatch: RecordBatch) {
+export function* serializeStream(table: Table) {
+    yield serializeMessage(table.schema).buffer;
+    for (const [id, field] of table.schema.dictionaries) {
+        const vec = table.getColumn(field.name) as DictionaryVector;
+        if (vec && vec.dictionary) {
+            yield serializeDictionaryBatch(vec.dictionary, id).buffer;
+        }
+    }
+    for (const recordBatch of table.batches) {
+        yield serializeRecordBatch(recordBatch).buffer;
+    }
+}
+
+export function* serializeFile(table: Table) {
+
+    const recordBatches = [];
+    const dictionaryBatches = [];
+
+    // First yield the magic string (aligned)
+    let buffer = new Uint8Array(align(magicLength, 8));
+    let metadataLength, byteLength = buffer.byteLength;
+    buffer.set(MAGIC, 0);
+    yield buffer;
+
+    // Then yield the schema
+    ({ metadataLength, buffer } = serializeMessage(table.schema));
+    byteLength += buffer.byteLength;
+    yield buffer;
+
+    for (const [id, field] of table.schema.dictionaries) {
+        const vec = table.getColumn(field.name) as DictionaryVector;
+        if (vec && vec.dictionary) {
+            ({ metadataLength, buffer } = serializeDictionaryBatch(vec.dictionary, id));
+            dictionaryBatches.push(new FileBlock(metadataLength, buffer.byteLength, byteLength));
+            byteLength += buffer.byteLength;
+            yield buffer;
+        }
+    }
+    for (const recordBatch of table.batches) {
+        ({ metadataLength, buffer } = serializeRecordBatch(recordBatch));
+        recordBatches.push(new FileBlock(metadataLength, buffer.byteLength, byteLength));
+        byteLength += buffer.byteLength;
+        yield buffer;
+    }
+
+    // Then yield the footer metadata (not aligned)
+    ({ metadataLength, buffer } = serializeFooter(new Footer(dictionaryBatches, recordBatches, table.schema)));
+    yield buffer;
+    
+    // Last, yield the footer length + terminating magic arrow string (aligned)
+    buffer = new Uint8Array(align(magicAndPadding, 8));
+    new Uint32Array(buffer.buffer)[0] = metadataLength;
+    buffer.set(MAGIC, buffer.byteLength - PADDING);
+    yield buffer;
+}
+
+export function serializeRecordBatch(recordBatch: RecordBatch) {
+    const { byteLength, fieldNodes, length, buffers, buffersMeta } = new RecordBatchSerializer().visitRecordBatch(recordBatch);
+    const rbMeta = new RecordBatchMetadata(MetadataVersion.V4, length, fieldNodes, buffersMeta);
+    const rbData = concatBuffersWithMetadata(byteLength, buffers, buffersMeta);
+    return serializeMessage(rbMeta, rbData);
+}
+
+export function serializeDictionaryBatch(dictionary: Vector, id: Long | number, isDelta: boolean = false) {
+    const { byteLength, fieldNodes, length, buffers, buffersMeta } = new RecordBatchSerializer().visitRecordBatch(RecordBatch.from([dictionary]));
+    const rbMeta = new RecordBatchMetadata(MetadataVersion.V4, length, fieldNodes, buffersMeta);
+    const dbMeta = new DictionaryBatch(MetadataVersion.V4, rbMeta, id, isDelta);
+    const rbData = concatBuffersWithMetadata(byteLength, buffers, buffersMeta);
+    return serializeMessage(dbMeta, rbData);
+}
+
+export function serializeMessage(message: Message, data?: Uint8Array) {
+    const b = new Builder();
+    _Message.finishMessageBuffer(b, writeMessage(b, message));
+    // Slice out the buffer that contains the message metadata
+    const metadataBytes = b.asUint8Array();
+    // Reserve 4 bytes for writing the message size at the front.
+    // Metadata length includes the metadata byteLength + the 4
+    // bytes for the length, and rounded up to the nearest 8 bytes.
+    const metadataLength = align(4 + metadataBytes.byteLength, 8);
+    // + the length of the optional data buffer at the end, padded
+    const dataByteLength = align(data ? data.byteLength : 0, 8);
+    // since both metadataLength and dataByteLength are aligned to 8-byte
+    // boundaries, the entire message is also aligned.
+    const messageBytes = new Uint8Array(metadataLength + dataByteLength);
+    // Write the metadata length into the first 4 bytes, but subtract the
+    // bytes we use to hold the length itself.
+    new DataView(messageBytes.buffer).setInt32(0, metadataLength - 4, platformIsLittleEndian);
+    // Copy the metadata bytes into the message buffer
+    messageBytes.set(metadataBytes, 4);
+    // Copy the optional data buffer after the metadata bytes
+    (data && dataByteLength > 0) && messageBytes.set(data, metadataLength);
+    // if (messageBytes.byteLength % 8 !== 0) { debugger; }
+    // Return the metadata length because we need to write it into each FileBlock also
+    return { metadataLength, buffer: messageBytes };
+}
+
+export function serializeFooter(footer: Footer) {
+    const b = new Builder();
+    _Footer.finishFooterBuffer(b, writeFooter(b, footer));
+    // Slice out the buffer that contains the footer metadata
+    const footerBytes = b.asUint8Array();
+    const metadataLength = footerBytes.byteLength;
+    return { metadataLength, buffer: footerBytes };
+}
+
+class RecordBatchSerializer extends VectorVisitor {
+    public length = 0;
+    public byteLength = 0;
+    public buffers: TypedArray[] = [];
+    public fieldNodes: FieldMetadata[] = [];
+    public buffersMeta: BufferMetadata[] = [];
+    public visitRecordBatch(recordBatch: RecordBatch) {
+        this.length = 0;
         this.byteLength = 0;
         this.buffers = [];
         this.fieldNodes = [];
@@ -49,58 +157,38 @@ export class RecordBatchSerializer extends VectorVisitor {
                 this.visit(vector);
             }
         }
-        const b = new Builder();
-        _Message.finishMessageBuffer(
-            b, writeMessage(b, new RecordBatchMetadata(
-                MetadataVersion.V4, this.byteLength, this.fieldNodes, this.buffersMeta
-            ))
-        );
-        const metadataBytes = b.asUint8Array();
-        // 4 bytes for the metadata length + 4 bytes of padding + the length of the metadata buffer
-        const metadataBytesOffset = 8 + metadataBytes.byteLength;
-        // + the length of all the vector buffers
-        const recordBatchBytes = new Uint8Array(metadataBytesOffset + this.byteLength);
-        // Write the metadata length as the first 4 bytes
-        new DataView(recordBatchBytes.buffer).setInt32(0, metadataBytes.byteLength);
-        // Now write the buffers
-        const { buffers, buffersMeta } = this;
-        for (let bufferIndex = -1, buffersLen = buffers.length; ++bufferIndex < buffersLen;) {
-            const { buffer, byteLength } = buffers[bufferIndex];
-            const { offset: byteOffset } = buffersMeta[bufferIndex];
-            recordBatchBytes.set(
-                new Uint8Array(buffer, 0, byteLength),
-                metadataBytesOffset + byteOffset
-            );
-        }
-        return recordBatchBytes;
+        return this;
     }
     public visit<T extends DataType>(vector: Vector<T>) {
-        const { data, length, nullCount } = vector;
-        if (length > 2147483647) {
-            throw new RangeError('Cannot write arrays larger than 2^31 - 1 in length');
+        if (!DataType.isDictionary(vector.type)) {
+            const { data, length, nullCount } = vector;
+            this.length = Math.max(this.length, length);
+            if (length > 2147483647) {
+                throw new RangeError('Cannot write arrays larger than 2^31 - 1 in length');
+            }
+            this.fieldNodes.push(new FieldMetadata(length, nullCount));
+            this.addBuffer(nullCount <= 0
+                ? new Uint8Array(0) // placeholder validity buffer
+                : this.getTruncatedBitmap(data.offset, length, data.nullBitmap!));
         }
-        this.fieldNodes.push(new FieldMetadata(length, nullCount));
-        this.addBuffer(nullCount <= 0
-            ? new Uint8Array(0) // placeholder validity buffer
-            : this.getTruncatedBitmap(data.offset, length, data.nullBitmap!));
         return super.visit(vector);
     }
-    public visitNull           (_vector: Vector<Null>)           { return this;                              }
-    public visitBool           (vector: Vector<Bool>)            { return this.visitFlatVector(vector);      }
+    public visitNull           (_nullz: Vector<Null>)            { return this;                              }
+    public visitBool           (vector: Vector<Bool>)            { return this.visitBoolVector(vector);      }
     public visitInt            (vector: Vector<Int>)             { return this.visitFlatVector(vector);      }
     public visitFloat          (vector: Vector<Float>)           { return this.visitFlatVector(vector);      }
     public visitUtf8           (vector: Vector<Utf8>)            { return this.visitFlatListVector(vector);  }
     public visitBinary         (vector: Vector<Binary>)          { return this.visitFlatListVector(vector);  }
-    public visitFixedSizeBinary(vector: Vector<FixedSizeBinary>) { return this.visitFlatVector(vector);      }
     public visitDate           (vector: Vector<Date_>)           { return this.visitFlatVector(vector);      }
     public visitTimestamp      (vector: Vector<Timestamp>)       { return this.visitFlatVector(vector);      }
     public visitTime           (vector: Vector<Time>)            { return this.visitFlatVector(vector);      }
     public visitDecimal        (vector: Vector<Decimal>)         { return this.visitFlatVector(vector);      }
     public visitInterval       (vector: Vector<Interval>)        { return this.visitFlatVector(vector);      }
+    public visitList           (vector: Vector<List>)            { return this.visitListVector(vector);      }
     public visitStruct         (vector: Vector<Struct>)          { return this.visitNestedVector(vector);    }
+    public visitFixedSizeBinary(vector: Vector<FixedSizeBinary>) { return this.visitFlatVector(vector);      }
+    public visitFixedSizeList  (vector: Vector<FixedSizeList>)   { return this.visitListVector(vector);      }
     public visitMap            (vector: Vector<Map_>)            { return this.visitNestedVector(vector);    }
-    public visitFixedSizeList  (vector: Vector<FixedSizeList>)   { return this.visitNestedVector(vector, 1); }
-    public visitList           (vector: Vector<List>)            { return this.visitNestedVector(vector, 1); }
     public visitDictionary     (vector: DictionaryVector)        {
         // Dictionary written out separately. Slice offset contained in the indices
         return this.visit(vector.indices);
@@ -153,35 +241,65 @@ export class RecordBatchSerializer extends VectorVisitor {
         }
         return this;
     }
-    protected visitFlatVector<T extends FlatType>(vector: Vector<T>) {
-        // Use the TypedArray constructor defined by the Vector's DataType
-        const { ArrayType } = vector.type;
-        // Use `vector.toArray()` here, since the View currently applied to the Vector might iterate different values
-        // than are in the original Data's values buffer.
-        // An example is a TimestampVector[ns] that's been converted to an Int32Vector via `asEpochMilliseconds()`.
-        // If the Vector's View is a ValidityView, the null slots will be automatically coerced by the TypedArray
-        // constructor, e.g. `0` for an Int8Array, `NaN` for Float64Array, etc.
-        this.addBuffer(new ArrayType(vector.toArray()));
+    protected visitBoolVector(vector: Vector<Bool>) {
+        // Bool vector is a special case of FlatVector, as its data buffer needs to stay packed
+        let values, { data, length } = vector;
+        if (vector.nullCount >= length) {
+            // If all values are null, just insert a placeholder empty data buffer (fastest path)
+            this.addBuffer(new Uint8Array(0));
+        } else if (!((values = data.values) instanceof Uint8Array)) {
+            // Otherwise if the underlying data *isn't* a Uint8Array, enumerate
+            // the values as bools and re-pack them into a Uint8Array (slow path)
+            // todo (ptaylor): this assumes Uint8Array = already packed, but that may not always be true
+            this.addBuffer(packBools(vector));
+        } else {
+            // otherwise just slice the bitmap (fast path)
+            this.addBuffer(this.getTruncatedBitmap(data.offset, length, values));
+        }
         return this;
     }
+    protected visitFlatVector<T extends FlatType>(vector: Vector<T>) {
+        const { view, data } = vector;
+        const { offset, length, values } = data;
+        const scaledLength = length * ((view as any).size || 1);
+        return this.addBuffer(values.subarray(offset, scaledLength));
+    }
     protected visitFlatListVector<T extends FlatListType>(vector: Vector<T>) {
         const { data, length } = vector;
-        const sliceOffset = data.offset;
-        const { values, valueOffsets } = data;
+        const { offset, values, valueOffsets } = data;
         const firstOffset = valueOffsets[0];
         const lastOffset = valueOffsets[valueOffsets.length - 1];
         const byteLength = Math.min(lastOffset - firstOffset, values.byteLength - firstOffset);
-        // Push in the order of FlatList types
+        // Push in the order FlatList types read their buffers
         // valueOffsets buffer first
-        this.addBuffer(this.getZeroBasedValueOffsets(sliceOffset, length, valueOffsets));
+        this.addBuffer(this.getZeroBasedValueOffsets(offset, length, valueOffsets));
         // sliced values buffer second
-        this.addBuffer(values.subarray(firstOffset + sliceOffset, firstOffset + sliceOffset + byteLength));
+        this.addBuffer(values.subarray(firstOffset + offset, firstOffset + offset + byteLength));
+        return this;
+    }
+    protected visitListVector<T extends SingleNestedType>(vector: Vector<T>) {
+        const { data, length } = vector;
+        const { offset, valueOffsets } = <any> data;
+        // If we have valueOffsets (ListVector), push that buffer first
+        if (valueOffsets) {
+            this.addBuffer(this.getZeroBasedValueOffsets(offset, length, valueOffsets));
+        }
+        // Then insert the List's values child
+        const child = (vector as any as ListVector<T>).getChildAt(0);
+        if (child) {
+            child.length -= 1;
+            this.visit(child);
+            child.length += 1;
+        }
         return this;
     }
-    protected visitNestedVector<T extends NestedType>(vector: Vector<T>, numChildren = vector.type.children.length) {
+    protected visitNestedVector<T extends NestedType>(vector: Vector<T>) {
         // Visit the children accordingly
-        for (let childIndex = -1; ++childIndex < numChildren;) {
-            this.visit((vector as NestedVector<T>).getChildAt(childIndex)!);
+        const numChildren = (vector.type.children || []).length;
+        for (let child: Vector | null, childIndex = -1; ++childIndex < numChildren;) {
+            if (child = (vector as NestedVector<T>).getChildAt(childIndex)) {
+                this.visit(child);
+            }
         }
         return this;
     }
@@ -191,8 +309,9 @@ export class RecordBatchSerializer extends VectorVisitor {
         this.buffers.push(values);
         this.buffersMeta.push(new BufferMetadata(this.byteLength, alignedLength));
         this.byteLength += alignedLength;
+        return this;
     }
-    protected getTruncatedBitmap(sliceOffset: number, length: number, nullBitmap: Uint8Array) {
+    protected getTruncatedBitmap(sliceOffset: number, length: number, bitmap: Uint8Array) {
         const alignedLength = align(length, 64);
         if (sliceOffset > 0 || length < alignedLength) {
             // With a sliced array / non-zero offset, we have to copy the bitmap
@@ -200,13 +319,13 @@ export class RecordBatchSerializer extends VectorVisitor {
             bytes.set(
                 (sliceOffset % 8 === 0)
                 // If the sliceOffset is aligned to 1 byte, it's safe to slice the nullBitmap directly
-                ? nullBitmap.subarray(sliceOffset >> 3)
+                ? bitmap.subarray(sliceOffset >> 3)
                 // iterate each bit starting from the sliceOffset, and repack into an aligned nullBitmap
-                : packBools(iterateBits(nullBitmap, sliceOffset, length, null, getBool))
+                : packBools(iterateBits(bitmap, sliceOffset, length, null, getBool))
             );
             return bytes;
         }
-        return nullBitmap;
+        return bitmap;
     }
     protected getZeroBasedValueOffsets(sliceOffset: number, length: number, valueOffsets: Int32Array) {
         // If we have a non-zero offset, then the value offsets do not start at
@@ -414,7 +533,16 @@ export class TypeSerializer extends TypeVisitor {
     }
 }
 
-// @ts-ignore
+function concatBuffersWithMetadata(byteLength: number, buffers: Uint8Array[], buffersMeta: BufferMetadata[]) {
+    const data = new Uint8Array(byteLength);
+    for (let bufferIndex = -1, buffersLen = buffers.length; ++bufferIndex < buffersLen;) {
+        const { buffer, byteLength } = buffers[bufferIndex];
+        const { offset: byteOffset } = buffersMeta[bufferIndex];
+        data.set(new Uint8Array(buffer, 0, byteLength), byteOffset);
+    }
+    return data;
+}
+
 function writeFooter(b: Builder, node: Footer) {
     let schemaOffset = writeSchema(b, node.schema);
     let recordBatchesOffset: number | undefined = undefined;
@@ -422,13 +550,13 @@ function writeFooter(b: Builder, node: Footer) {
     if (node.recordBatches && node.recordBatches.length) {
         recordBatchesOffset =
             _Footer.startRecordBatchesVector(b, node.recordBatches.length) ||
-            node.recordBatches.map((rb) => writeBlock(b, rb)) &&
+            mapReverse(node.recordBatches, (rb) => writeBlock(b, rb)) &&
             b.endVector();
     }
     if (node.dictionaryBatches && node.dictionaryBatches.length) {
         dictionaryBatchesOffset =
             _Footer.startDictionariesVector(b, node.dictionaryBatches.length) ||
-            node.dictionaryBatches.map((db) => writeBlock(b, db)) &&
+            mapReverse(node.dictionaryBatches, (db) => writeBlock(b, db)) &&
             b.endVector();
     }
     return (
@@ -442,7 +570,11 @@ function writeFooter(b: Builder, node: Footer) {
 }
 
 function writeBlock(b: Builder, node: FileBlock) {
-    return _Block.createBlock(b, node.offset, node.metaDataLength, node.bodyLength);
+    return _Block.createBlock(b,
+        new Long(node.offset, 0),
+        node.metaDataLength,
+        new Long(node.bodyLength, 0)
+    );
 }
 
 function writeMessage(b: Builder, node: Message) {
@@ -472,7 +604,7 @@ function writeSchema(b: Builder, node: Schema) {
     return (
         _Schema.startSchema(b) ||
         _Schema.addFields(b, fieldsOffset) ||
-        _Schema.addEndianness(b, _Endianness.Little) ||
+        _Schema.addEndianness(b, platformIsLittleEndian ? _Endianness.Little : _Endianness.Big) ||
         _Schema.endSchema(b)
     );
 }
@@ -483,13 +615,13 @@ function writeRecordBatch(b: Builder, node: RecordBatchMetadata) {
     if (node.nodes && node.nodes.length) {
         nodesOffset =
             _RecordBatch.startNodesVector(b, node.nodes.length) ||
-            node.nodes.map((n) => writeFieldNode(b, n)) &&
+            mapReverse(node.nodes, (n) => writeFieldNode(b, n)) &&
             b.endVector();
     }
     if (node.buffers && node.buffers.length) {
         buffersOffset =
             _RecordBatch.startBuffersVector(b, node.buffers.length) ||
-            node.buffers.map((bm) => writeBuffer(b, bm)) &&
+            mapReverse(node.buffers, (b_) => writeBuffer(b, b_)) &&
             b.endVector();
     }
     return (
@@ -521,19 +653,24 @@ function writeFieldNode(b: Builder, node: FieldMetadata) {
 }
 
 function writeField(b: Builder, node: Field) {
+    let typeOffset = -1;
     let type = node.type;
-    let typeOffset = new TypeSerializer(b).visit(type);
+    let typeId = node.typeId;
     let name: number | undefined = undefined;
     let children: number | undefined = undefined;
     let metadata: number | undefined = undefined;
     let dictionary: number | undefined = undefined;
-    if (DataType.isDictionary(node.type)) {
-        dictionary = new TypeSerializer(b).visit(node.type.dictionary);
+    if (!DataType.isDictionary(type)) {
+        typeOffset = new TypeSerializer(b).visit(type);
+    } else {
+        typeId = type.dictionary.TType;
+        dictionary = new TypeSerializer(b).visit(type);
+        typeOffset = new TypeSerializer(b).visit(type.dictionary);
     }
     if (type.children && type.children.length) {
         children = _Field.createChildrenVector(b, type.children.map((f) => writeField(b, f)));
     }
-    if (node.metadata) {
+    if (node.metadata && node.metadata.size > 0) {
         metadata = _Field.createCustomMetadataVector(
             b,
             [...node.metadata].map(([k, v]) => {
@@ -554,7 +691,7 @@ function writeField(b: Builder, node: Field) {
     return (
         _Field.startField(b) ||
         _Field.addType(b, typeOffset) ||
-        _Field.addTypeType(b, node.typeId) ||
+        _Field.addTypeType(b, typeId) ||
         _Field.addNullable(b, !!node.nullable) ||
         (name !== undefined && _Field.addName(b, name)) ||
         (children !== undefined && _Field.addChildren(b, children)) ||
@@ -563,3 +700,18 @@ function writeField(b: Builder, node: Field) {
         _Field.endField(b)
     );
 }
+
+function mapReverse<T, U>(source: T[], callbackfn: (value: T, index: number, array: T[]) => U): U[] {
+    const result = new Array(source.length);
+    for (let i = -1, j = source.length; --j > -1;) {
+        result[i] = callbackfn(source[j], i, source);
+    }
+    return result;
+}
+
+const platformIsLittleEndian = (function() {
+    const buffer = new ArrayBuffer(2);
+    new DataView(buffer).setInt16(0, 256, true /* littleEndian */);
+    // Int16Array uses the platform's endianness.
+    return new Int16Array(buffer)[0] === 256;
+})();

From a242da8f480e64eafd2ad5d7cf94f309dce2099f Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 17:23:15 -0700
Subject: [PATCH 17/51] Add `Table.prototype.serialize` method to make
 ArrayBuffers from Tables

---
 js/src/Arrow.externs.js | 2 ++
 js/src/table.ts         | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/js/src/Arrow.externs.js b/js/src/Arrow.externs.js
index 51109a02ffe..4ccfe02f4d6 100644
--- a/js/src/Arrow.externs.js
+++ b/js/src/Arrow.externs.js
@@ -65,6 +65,8 @@ Table.prototype.batches;
 Table.prototype.countBy;
 /** @type {?} */
 Table.prototype.scan;
+/** @type {?} */
+Table.prototype.serialize;
 
 var CountByResult = function() {};
 /** @type {?} */
diff --git a/js/src/table.ts b/js/src/table.ts
index d0d699f366e..e398cbbb8e1 100644
--- a/js/src/table.ts
+++ b/js/src/table.ts
@@ -19,6 +19,7 @@ import { RecordBatch } from './recordbatch';
 import { Col, Predicate } from './predicate';
 import { Schema, Field, Struct } from './type';
 import { read, readAsync } from './ipc/reader/arrow';
+import { writeTableBinary } from './ipc/writer/arrow';
 import { isPromise, isAsyncIterable } from './util/compat';
 import { Vector, DictionaryVector, IntVector, StructVector } from './vector';
 import { ChunkedView } from './vector/chunked';
@@ -179,6 +180,10 @@ export class Table implements DataFrame {
         }
         return str;
     }
+    // @ts-ignore
+    public serialize(encoding = 'binary', stream = true) {
+        return writeTableBinary(this, stream);
+    }
     public rowsToString(separator = ' | '): TableToStringIterator {
         return new TableToStringIterator(tableRowsToString(this, separator));
     }

From db02c1c2a761faa6120c66e6add55fed10873f98 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 17:23:42 -0700
Subject: [PATCH 18/51] Add an integration test for binary writer

---
 js/test/integration/validate-tests.ts | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/js/test/integration/validate-tests.ts b/js/test/integration/validate-tests.ts
index f81634279c9..36220a68cdf 100644
--- a/js/test/integration/validate-tests.ts
+++ b/js/test/integration/validate-tests.ts
@@ -39,7 +39,7 @@ function resolvePathArgs(paths: string) {
             if (fs.existsSync(p)) {
                 return p;
             }
-            console.warn(`Could not find file "${p}"`);
+            console.error(`Could not find file "${p}"`);
             return undefined;
         });
 }
@@ -119,6 +119,7 @@ describe(`Integration`, () => {
         describe(path.join(dir, name), () => {
             testReaderIntegration(json, arrowBuffer);
             testTableFromBuffersIntegration(json, arrowBuffer);
+            testTableToBuffersIntegration(json, arrowBuffer);
         });
     }
 });
@@ -152,3 +153,19 @@ function testTableFromBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array)
         }
     });
 }
+
+function testTableToBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array) {
+    test(`serializing json to binary reports the same values as the original binary arrow table`, () => {
+        expect.hasAssertions();
+        const fromJSON = Table.from(jsonData);
+        const serialized = fromJSON.serialize();
+        const jsonTable = Table.from(serialized);
+        const binaryTable = Table.from(arrowBuffer);
+        expect(jsonTable.length).toEqual(binaryTable.length);
+        expect(jsonTable.numCols).toEqual(binaryTable.numCols);
+        for (let i = -1, n = jsonTable.numCols; ++i < n;) {
+            (jsonTable.getColumnAt(i) as any).name = jsonTable.schema.fields[i].name;
+            (expect(jsonTable.getColumnAt(i)) as any).toEqualVector(binaryTable.getColumnAt(i));
+        }
+    });
+}

From 402187e2f6051eb41abeb2df830cf5b5de6d685f Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 21:22:32 -0700
Subject: [PATCH 19/51] add apache license headers

---
 js/src/ipc/magic.ts       | 17 +++++++++++++++++
 js/src/vector/validity.ts | 17 +++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/js/src/ipc/magic.ts b/js/src/ipc/magic.ts
index c9e08a1be53..c6eb36a834e 100644
--- a/js/src/ipc/magic.ts
+++ b/js/src/ipc/magic.ts
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 export const PADDING = 4;
 export const MAGIC_STR = 'ARROW1';
 export const MAGIC = new Uint8Array(MAGIC_STR.length);
diff --git a/js/src/vector/validity.ts b/js/src/vector/validity.ts
index a3e572847cb..57e183793e2 100644
--- a/js/src/vector/validity.ts
+++ b/js/src/vector/validity.ts
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 import { Data } from '../data';
 import { View, Vector } from '../vector';
 import { NestedView } from './nested';

From 304e75d199c2bef0c26a8b28e04d2fd36dcb2064 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 21:51:36 -0700
Subject: [PATCH 20/51] fix magic string alignment in file reader, add file
 reader tests

---
 js/src/ipc/writer/binary.ts           |  8 +++----
 js/test/integration/validate-tests.ts | 33 +++++++++++++++------------
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
index 12686bdc3d2..3b0e21940a4 100644
--- a/js/src/ipc/writer/binary.ts
+++ b/js/src/ipc/writer/binary.ts
@@ -19,7 +19,7 @@ import { Table } from '../../table';
 import { DenseUnionData } from '../../data';
 import { RecordBatch } from '../../recordbatch';
 import { VectorVisitor, TypeVisitor } from '../../visitor';
-import { MAGIC, PADDING, magicLength, magicAndPadding } from '../magic';
+import { MAGIC, magicLength, magicAndPadding } from '../magic';
 import { align, getBool, packBools, iterateBits } from '../../util/bit';
 import { Vector, UnionVector, DictionaryVector, NestedVector, ListVector } from '../../vector';
 import { BufferMetadata, FieldMetadata, Footer, FileBlock, Message, RecordBatchMetadata, DictionaryBatch } from '../metadata';
@@ -84,9 +84,9 @@ export function* serializeFile(table: Table) {
     yield buffer;
     
     // Last, yield the footer length + terminating magic arrow string (aligned)
-    buffer = new Uint8Array(align(magicAndPadding, 8));
-    new Uint32Array(buffer.buffer)[0] = metadataLength;
-    buffer.set(MAGIC, buffer.byteLength - PADDING);
+    buffer = new Uint8Array(magicAndPadding);
+    new DataView(buffer.buffer).setInt32(0, metadataLength, platformIsLittleEndian);
+    buffer.set(MAGIC, buffer.byteLength - magicLength);
     yield buffer;
 }
 
diff --git a/js/test/integration/validate-tests.ts b/js/test/integration/validate-tests.ts
index 36220a68cdf..9bfab41d4ba 100644
--- a/js/test/integration/validate-tests.ts
+++ b/js/test/integration/validate-tests.ts
@@ -119,7 +119,8 @@ describe(`Integration`, () => {
         describe(path.join(dir, name), () => {
             testReaderIntegration(json, arrowBuffer);
             testTableFromBuffersIntegration(json, arrowBuffer);
-            testTableToBuffersIntegration(json, arrowBuffer);
+            testTableToBuffersIntegration('file')(json, arrowBuffer);
+            testTableToBuffersIntegration('stream')(json, arrowBuffer);
         });
     }
 });
@@ -154,18 +155,20 @@ function testTableFromBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array)
     });
 }
 
-function testTableToBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array) {
-    test(`serializing json to binary reports the same values as the original binary arrow table`, () => {
-        expect.hasAssertions();
-        const fromJSON = Table.from(jsonData);
-        const serialized = fromJSON.serialize();
-        const jsonTable = Table.from(serialized);
-        const binaryTable = Table.from(arrowBuffer);
-        expect(jsonTable.length).toEqual(binaryTable.length);
-        expect(jsonTable.numCols).toEqual(binaryTable.numCols);
-        for (let i = -1, n = jsonTable.numCols; ++i < n;) {
-            (jsonTable.getColumnAt(i) as any).name = jsonTable.schema.fields[i].name;
-            (expect(jsonTable.getColumnAt(i)) as any).toEqualVector(binaryTable.getColumnAt(i));
-        }
-    });
+function testTableToBuffersIntegration(arrowFormat: 'stream' | 'file') {
+    return function testTableToBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array) {
+        test(`serializing json to binary reports the same values as the original binary arrow table`, () => {
+            expect.hasAssertions();
+            const fromJSON = Table.from(jsonData);
+            const serialized = fromJSON.serialize('binary', arrowFormat === 'stream');
+            const jsonTable = Table.from(serialized);
+            const binaryTable = Table.from(arrowBuffer);
+            expect(jsonTable.length).toEqual(binaryTable.length);
+            expect(jsonTable.numCols).toEqual(binaryTable.numCols);
+            for (let i = -1, n = jsonTable.numCols; ++i < n;) {
+                (jsonTable.getColumnAt(i) as any).name = jsonTable.schema.fields[i].name;
+                (expect(jsonTable.getColumnAt(i)) as any).toEqualVector(binaryTable.getColumnAt(i));
+            }
+        });
+    }
 }

From 73a2fa9b86cb2ca1d5edaa2134af1afd63993630 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 10 May 2018 23:38:08 -0700
Subject: [PATCH 21/51] fix stream -> file, file -> stream, add tests

---
 js/src/ipc/writer/binary.ts           |  8 ++---
 js/test/integration/validate-tests.ts | 49 ++++++++++++++++-----------
 2 files changed, 34 insertions(+), 23 deletions(-)

diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
index 3b0e21940a4..cfcbc16e997 100644
--- a/js/src/ipc/writer/binary.ts
+++ b/js/src/ipc/writer/binary.ts
@@ -268,7 +268,7 @@ class RecordBatchSerializer extends VectorVisitor {
         const { data, length } = vector;
         const { offset, values, valueOffsets } = data;
         const firstOffset = valueOffsets[0];
-        const lastOffset = valueOffsets[valueOffsets.length - 1];
+        const lastOffset = valueOffsets[length];
         const byteLength = Math.min(lastOffset - firstOffset, values.byteLength - firstOffset);
         // Push in the order FlatList types read their buffers
         // valueOffsets buffer first
@@ -536,9 +536,9 @@ export class TypeSerializer extends TypeVisitor {
 function concatBuffersWithMetadata(byteLength: number, buffers: Uint8Array[], buffersMeta: BufferMetadata[]) {
     const data = new Uint8Array(byteLength);
     for (let bufferIndex = -1, buffersLen = buffers.length; ++bufferIndex < buffersLen;) {
-        const { buffer, byteLength } = buffers[bufferIndex];
-        const { offset: byteOffset } = buffersMeta[bufferIndex];
-        data.set(new Uint8Array(buffer, 0, byteLength), byteOffset);
+        const { offset } = buffersMeta[bufferIndex];
+        const { buffer, byteOffset, byteLength } = buffers[bufferIndex];
+        data.set(new Uint8Array(buffer, byteOffset, byteLength), offset);
     }
     return data;
 }
diff --git a/js/test/integration/validate-tests.ts b/js/test/integration/validate-tests.ts
index 9bfab41d4ba..0ab8694d398 100644
--- a/js/test/integration/validate-tests.ts
+++ b/js/test/integration/validate-tests.ts
@@ -55,7 +55,7 @@ const jsonAndArrowPaths = toArray(zip(
 .filter(([p1, p2]) => p1 !== undefined && p2 !== undefined) as [string, string][];
 
 expect.extend({
-    toEqualVector(v1: any, v2: any) {
+    toEqualVector([v1, format1, columnName]: [any, string, string], [v2, format2]: [any, string]) {
 
         const format = (x: any, y: any, msg= ' ') => `${
             this.utils.printExpected(x)}${
@@ -102,7 +102,7 @@ expect.extend({
         return {
             pass: allFailures.every(({ failures }) => failures.length === 0),
             message: () => [
-                `${v1.name}: (${format('json', 'arrow', ' !== ')})\n`,
+                `${columnName}: (${format(format1, format2, ' !== ')})\n`,
                 ...allFailures.map(({ failures, title }) =>
                     !failures.length ? `` : [`${title}:`, ...failures].join(`\n`))
             ].join('\n')
@@ -119,8 +119,10 @@ describe(`Integration`, () => {
         describe(path.join(dir, name), () => {
             testReaderIntegration(json, arrowBuffer);
             testTableFromBuffersIntegration(json, arrowBuffer);
-            testTableToBuffersIntegration('file')(json, arrowBuffer);
-            testTableToBuffersIntegration('stream')(json, arrowBuffer);
+            testTableToBuffersIntegration('json', 'file')(json, arrowBuffer);
+            testTableToBuffersIntegration('json', 'stream')(json, arrowBuffer);
+            testTableToBuffersIntegration('binary', 'file')(json, arrowBuffer);
+            testTableToBuffersIntegration('binary', 'stream')(json, arrowBuffer);
         });
     }
 });
@@ -134,8 +136,11 @@ function testReaderIntegration(jsonData: any, arrowBuffer: Uint8Array) {
             expect(jsonRecordBatch.length).toEqual(binaryRecordBatch.length);
             expect(jsonRecordBatch.numCols).toEqual(binaryRecordBatch.numCols);
             for (let i = -1, n = jsonRecordBatch.numCols; ++i < n;) {
-                (jsonRecordBatch.getChildAt(i) as any).name = jsonRecordBatch.schema.fields[i].name;
-                (expect(jsonRecordBatch.getChildAt(i)) as any).toEqualVector(binaryRecordBatch.getChildAt(i));
+                const v1 = jsonRecordBatch.getChildAt(i);
+                const v2 = binaryRecordBatch.getChildAt(i);
+                const name = jsonRecordBatch.schema.fields[i].name;
+                (expect([v1, `json`, name]) as any)
+                    .toEqualVector([v2, `binary`]);
             }
         }
     });
@@ -149,25 +154,31 @@ function testTableFromBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array)
         expect(jsonTable.length).toEqual(binaryTable.length);
         expect(jsonTable.numCols).toEqual(binaryTable.numCols);
         for (let i = -1, n = jsonTable.numCols; ++i < n;) {
-            (jsonTable.getColumnAt(i) as any).name = jsonTable.schema.fields[i].name;
-            (expect(jsonTable.getColumnAt(i)) as any).toEqualVector(binaryTable.getColumnAt(i));
+            const v1 = jsonTable.getColumnAt(i);
+            const v2 = binaryTable.getColumnAt(i);
+            const name = jsonTable.schema.fields[i].name;
+            (expect([v1, `json`, name]) as any)
+                .toEqualVector([v2, `binary`]);
         }
     });
 }
 
-function testTableToBuffersIntegration(arrowFormat: 'stream' | 'file') {
+function testTableToBuffersIntegration(srcFormat: 'json' | 'binary', arrowFormat: 'stream' | 'file') {
     return function testTableToBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array) {
-        test(`serializing json to binary reports the same values as the original binary arrow table`, () => {
+        test(`serializing ${srcFormat} to a ${arrowFormat} reports the same values as the alternate format`, () => {
             expect.hasAssertions();
-            const fromJSON = Table.from(jsonData);
-            const serialized = fromJSON.serialize('binary', arrowFormat === 'stream');
-            const jsonTable = Table.from(serialized);
-            const binaryTable = Table.from(arrowBuffer);
-            expect(jsonTable.length).toEqual(binaryTable.length);
-            expect(jsonTable.numCols).toEqual(binaryTable.numCols);
-            for (let i = -1, n = jsonTable.numCols; ++i < n;) {
-                (jsonTable.getColumnAt(i) as any).name = jsonTable.schema.fields[i].name;
-                (expect(jsonTable.getColumnAt(i)) as any).toEqualVector(binaryTable.getColumnAt(i));
+            const refFormat = srcFormat === 'json' ? `binary` : `json`;
+            const refTable = Table.from(refFormat === `json` ? jsonData : arrowBuffer);
+            const srcTable = Table.from(srcFormat === `json` ? jsonData : arrowBuffer);
+            const dstTable = Table.from(srcTable.serialize(`binary`, arrowFormat === `stream`));
+            expect(dstTable.length).toEqual(refTable.length);
+            expect(dstTable.numCols).toEqual(refTable.numCols);
+            for (let i = -1, n = dstTable.numCols; ++i < n;) {
+                const v1 = dstTable.getColumnAt(i);
+                const v2 = refTable.getColumnAt(i);
+                const name = dstTable.schema.fields[i].name;
+                (expect([v1, srcFormat, name]) as any)
+                    .toEqualVector([v2, refFormat]);
             }
         });
     }

From 4e80851388c786372f3998adb291225632ec1454 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sat, 12 May 2018 15:13:55 -0700
Subject: [PATCH 22/51] write correct recordBatch length

---
 js/src/ipc/writer/binary.ts | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
index cfcbc16e997..7e36a9382c1 100644
--- a/js/src/ipc/writer/binary.ts
+++ b/js/src/ipc/writer/binary.ts
@@ -91,15 +91,15 @@ export function* serializeFile(table: Table) {
 }
 
 export function serializeRecordBatch(recordBatch: RecordBatch) {
-    const { byteLength, fieldNodes, length, buffers, buffersMeta } = new RecordBatchSerializer().visitRecordBatch(recordBatch);
-    const rbMeta = new RecordBatchMetadata(MetadataVersion.V4, length, fieldNodes, buffersMeta);
+    const { byteLength, fieldNodes, buffers, buffersMeta } = new RecordBatchSerializer().visitRecordBatch(recordBatch);
+    const rbMeta = new RecordBatchMetadata(MetadataVersion.V4, recordBatch.length, fieldNodes, buffersMeta);
     const rbData = concatBuffersWithMetadata(byteLength, buffers, buffersMeta);
     return serializeMessage(rbMeta, rbData);
 }
 
 export function serializeDictionaryBatch(dictionary: Vector, id: Long | number, isDelta: boolean = false) {
-    const { byteLength, fieldNodes, length, buffers, buffersMeta } = new RecordBatchSerializer().visitRecordBatch(RecordBatch.from([dictionary]));
-    const rbMeta = new RecordBatchMetadata(MetadataVersion.V4, length, fieldNodes, buffersMeta);
+    const { byteLength, fieldNodes, buffers, buffersMeta } = new RecordBatchSerializer().visitRecordBatch(RecordBatch.from([dictionary]));
+    const rbMeta = new RecordBatchMetadata(MetadataVersion.V4, dictionary.length, fieldNodes, buffersMeta);
     const dbMeta = new DictionaryBatch(MetadataVersion.V4, rbMeta, id, isDelta);
     const rbData = concatBuffersWithMetadata(byteLength, buffers, buffersMeta);
     return serializeMessage(dbMeta, rbData);
@@ -141,15 +141,13 @@ export function serializeFooter(footer: Footer) {
 }
 
 class RecordBatchSerializer extends VectorVisitor {
-    public length = 0;
     public byteLength = 0;
     public buffers: TypedArray[] = [];
     public fieldNodes: FieldMetadata[] = [];
     public buffersMeta: BufferMetadata[] = [];
     public visitRecordBatch(recordBatch: RecordBatch) {
-        this.length = 0;
-        this.byteLength = 0;
         this.buffers = [];
+        this.byteLength = 0;
         this.fieldNodes = [];
         this.buffersMeta = [];
         for (let vector: Vector, index = -1, numCols = recordBatch.numCols; ++index < numCols;) {
@@ -162,7 +160,6 @@ class RecordBatchSerializer extends VectorVisitor {
     public visit<T extends DataType>(vector: Vector<T>) {
         if (!DataType.isDictionary(vector.type)) {
             const { data, length, nullCount } = vector;
-            this.length = Math.max(this.length, length);
             if (length > 2147483647) {
                 throw new RangeError('Cannot write arrays larger than 2^31 - 1 in length');
             }
@@ -285,13 +282,7 @@ class RecordBatchSerializer extends VectorVisitor {
             this.addBuffer(this.getZeroBasedValueOffsets(offset, length, valueOffsets));
         }
         // Then insert the List's values child
-        const child = (vector as any as ListVector<T>).getChildAt(0);
-        if (child) {
-            child.length -= 1;
-            this.visit(child);
-            child.length += 1;
-        }
-        return this;
+        return this.visit((vector as any as ListVector<T>).getChildAt(0)!);
     }
     protected visitNestedVector<T extends NestedType>(vector: Vector<T>) {
         // Visit the children accordingly
@@ -667,9 +658,7 @@ function writeField(b: Builder, node: Field) {
         dictionary = new TypeSerializer(b).visit(type);
         typeOffset = new TypeSerializer(b).visit(type.dictionary);
     }
-    if (type.children && type.children.length) {
-        children = _Field.createChildrenVector(b, type.children.map((f) => writeField(b, f)));
-    }
+    children = _Field.createChildrenVector(b, (type.children || []).map((f) => writeField(b, f)));
     if (node.metadata && node.metadata.size > 0) {
         metadata = _Field.createCustomMetadataVector(
             b,

From e75da1342aea129012625c64bef7eae21d8a1f27 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sat, 12 May 2018 15:16:24 -0700
Subject: [PATCH 23/51] add support for reading streaming format via node
 streams

---
 js/package.json            |  1 +
 js/src/Arrow.externs.js    |  8 +++-
 js/src/Arrow.ts            | 16 +++++++-
 js/src/ipc/reader/arrow.ts |  7 ++++
 js/src/ipc/reader/node.ts  | 65 +++++++++++++++++++++++++++++
 js/src/table.ts            | 29 ++-----------
 js/src/util/node.ts        | 84 ++++++++++++++++++++++++++++++++++++++
 7 files changed, 180 insertions(+), 30 deletions(-)
 create mode 100644 js/src/ipc/reader/node.ts
 create mode 100644 js/src/util/node.ts

diff --git a/js/package.json b/js/package.json
index 9b2aeaba87c..ee5ca3314a9 100644
--- a/js/package.json
+++ b/js/package.json
@@ -93,6 +93,7 @@
     "rxjs": "5.5.6",
     "shx": "0.2.2",
     "source-map-loader": "0.2.3",
+    "stream-to-iterator": "3.0.1",
     "trash": "4.2.1",
     "ts-jest": "22.0.1",
     "tslint": "5.9.1",
diff --git a/js/src/Arrow.externs.js b/js/src/Arrow.externs.js
index 4ccfe02f4d6..b22ee55d56a 100644
--- a/js/src/Arrow.externs.js
+++ b/js/src/Arrow.externs.js
@@ -124,9 +124,13 @@ Predicate.prototype.not;
 Predicate.prototype.ands;
 var Literal = function() {};
 
-var TableToStringIterator = function() {};
+var PipeIterator = function() {};
 /** @type {?} */
-TableToStringIterator.prototype.pipe;
+PipeIterator.prototype.pipe;
+
+var AsyncPipeIterator = function() {};
+/** @type {?} */
+AsyncPipeIterator.prototype.pipe;
 
 var RecordBatch = function() {};
 /** @type {?} */
diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts
index e4cf9753905..ad4a3c41399 100644
--- a/js/src/Arrow.ts
+++ b/js/src/Arrow.ts
@@ -20,6 +20,7 @@ import * as data_ from './data';
 import * as vector_ from './vector';
 import * as util_int_ from './util/int';
 import * as util_bit_ from './util/bit';
+import * as util_node from './util/node';
 import * as visitor_ from './visitor';
 import * as view_ from './vector/view';
 import * as predicate_ from './predicate';
@@ -27,7 +28,9 @@ import { Vector } from './vector';
 import { RecordBatch } from './recordbatch';
 import { Schema, Field, Type } from './type';
 import { Table, DataFrame, NextFunc, BindFunc, CountByResult } from './table';
-import { read, readAsync } from './ipc/reader/arrow';
+import { fromNodeStream } from './ipc/reader/node';
+import { read, readAsync, readNodeStream } from './ipc/reader/arrow';
+import { serializeFile, serializeStream } from './ipc/writer/binary';
 
 export import View = vector_.View;
 export import VectorLike = vector_.VectorLike;
@@ -36,7 +39,9 @@ export import IntBitWidth = type_.IntBitWidth;
 export import TimeBitWidth = type_.TimeBitWidth;
 export import TypedArrayConstructor = type_.TypedArrayConstructor;
 
-export { read, readAsync };
+export { fromNodeStream };
+export { read, readAsync, readNodeStream };
+export { serializeFile, serializeStream };
 export { Table, DataFrame, NextFunc, BindFunc, CountByResult };
 export { Field, Schema, RecordBatch, Vector, Type };
 
@@ -45,6 +50,8 @@ export namespace util {
     export import Int64 = util_int_.Int64;
     export import Int128 = util_int_.Int128;
     export import packBools = util_bit_.packBools;
+    export import PipeIterator = util_node.PipeIterator;
+    export import AsyncPipeIterator = util_node.AsyncPipeIterator;
 }
 
 export namespace data {
@@ -202,6 +209,11 @@ try {
 
         Arrow['read'] = read;
         Arrow['readAsync'] = readAsync;
+        Arrow['readNodeStream'] = readNodeStream;
+        Arrow['fromNodeStream'] = fromNodeStream;
+
+        Arrow['serializeFile'] = serializeFile;
+        Arrow['serializeStream'] = serializeStream;
 
         Arrow['Type'] = Type;
         Arrow['Field'] = Field;
diff --git a/js/src/ipc/reader/arrow.ts b/js/src/ipc/reader/arrow.ts
index af535900cbf..ad222a1054e 100644
--- a/js/src/ipc/reader/arrow.ts
+++ b/js/src/ipc/reader/arrow.ts
@@ -16,6 +16,7 @@
 // under the License.
 
 import { readJSON } from './json';
+import { fromNodeStream } from './node';
 import { RecordBatch } from '../../recordbatch';
 import { readBuffers, readBuffersAsync } from './binary';
 import { readRecordBatches, readRecordBatchesAsync, TypeDataLoader } from './vector';
@@ -46,3 +47,9 @@ export async function* readAsync(sources: AsyncIterable<Uint8Array | Buffer | st
         yield recordBatch;
     }
 }
+
+export async function* readNodeStream(stream: NodeJS.ReadableStream) {
+    for await (const recordBatch of readAsync(fromNodeStream(stream))) {
+        yield recordBatch as RecordBatch;
+    }
+}
diff --git a/js/src/ipc/reader/node.ts b/js/src/ipc/reader/node.ts
new file mode 100644
index 00000000000..ec7a1694b28
--- /dev/null
+++ b/js/src/ipc/reader/node.ts
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { PADDING } from '../magic';
+import { flatbuffers } from 'flatbuffers';
+import * as Message_ from '../../fb/Message';
+import ByteBuffer = flatbuffers.ByteBuffer;
+import _Message = Message_.org.apache.arrow.flatbuf.Message;
+
+export async function* fromNodeStream(stream: NodeJS.ReadableStream) {
+
+    let bb: ByteBuffer;
+    let bytesRead = 0, bytes = new Uint8Array(0);
+    let messageLength = 0, message: _Message | null = null;
+
+    for await (let chunk of (stream as any as AsyncIterable<Uint8Array | Buffer | string>)) {
+
+        const grown = new Uint8Array(bytes.length + chunk.length);
+
+        if (typeof chunk !== 'string') {
+            grown.set(bytes, 0) || grown.set(chunk, bytes.length);
+        } else {
+            for (let i = -1, j = bytes.length, n = chunk.length; ++i < n;) {
+                grown[i + j] = chunk.charCodeAt(i);
+            }
+        }
+
+        bytes = grown;
+
+        if (messageLength <= 0) {
+            messageLength = new DataView(bytes.buffer).getInt32(0, true);
+        }
+
+        while (messageLength < bytes.length) {
+            if (!message) {
+                (bb = new ByteBuffer(bytes)).setPosition(4);
+                if (message = _Message.getRootAsMessage(bb)) {
+                    messageLength += message.bodyLength().low;
+                    continue;
+                }
+                throw new Error(`Invalid message at position ${bytesRead}`);
+            }
+            bytesRead += messageLength + PADDING;
+            yield bytes.subarray(0, messageLength);
+            bytes = bytes.subarray(messageLength + PADDING);
+            messageLength = bytes.length <= 0 ? 0 :
+                new DataView(bytes.buffer).getInt32(bytes.byteOffset, true);
+            message = null;
+        }
+    }
+}
diff --git a/js/src/table.ts b/js/src/table.ts
index e398cbbb8e1..a365dbfe258 100644
--- a/js/src/table.ts
+++ b/js/src/table.ts
@@ -20,6 +20,7 @@ import { Col, Predicate } from './predicate';
 import { Schema, Field, Struct } from './type';
 import { read, readAsync } from './ipc/reader/arrow';
 import { writeTableBinary } from './ipc/writer/arrow';
+import { PipeIterator } from './util/node';
 import { isPromise, isAsyncIterable } from './util/compat';
 import { Vector, DictionaryVector, IntVector, StructVector } from './vector';
 import { ChunkedView } from './vector/chunked';
@@ -184,8 +185,8 @@ export class Table implements DataFrame {
     public serialize(encoding = 'binary', stream = true) {
         return writeTableBinary(this, stream);
     }
-    public rowsToString(separator = ' | '): TableToStringIterator {
-        return new TableToStringIterator(tableRowsToString(this, separator));
+    public rowsToString(separator = ' | ') {
+        return new PipeIterator(tableRowsToString(this, separator), 'utf8');
     }
 }
 
@@ -295,30 +296,6 @@ export class CountByResult extends Table implements DataFrame {
     }
 }
 
-export class TableToStringIterator implements IterableIterator<string> {
-    constructor(private iterator: IterableIterator<string>) {}
-    [Symbol.iterator]() { return this.iterator; }
-    next(value?: any) { return this.iterator.next(value); }
-    throw(error?: any) { return this.iterator.throw && this.iterator.throw(error) || { done: true, value: '' }; }
-    return(value?: any) { return this.iterator.return && this.iterator.return(value) || { done: true, value: '' }; }
-    pipe(stream: NodeJS.WritableStream) {
-        let res: IteratorResult<string>;
-        let write = () => {
-            if (stream['writable']) {
-                do {
-                    if ((res = this.next()).done) { break; }
-                } while (stream['write'](res.value + '\n', 'utf8'));
-            }
-            if (!res || !res.done) {
-                stream['once']('drain', write);
-            } else if (!(stream as any)['isTTY']) {
-                stream['end']('\n');
-            }
-        };
-        write();
-    }
-}
-
 function* tableRowsToString(table: Table, separator = ' | ') {
     const fields = table.schema.fields;
     const header = ['row_id', ...fields.map((f) => `${f}`)].map(stringify);
diff --git a/js/src/util/node.ts b/js/src/util/node.ts
new file mode 100644
index 00000000000..857765c1b65
--- /dev/null
+++ b/js/src/util/node.ts
@@ -0,0 +1,84 @@
+
+export class PipeIterator<T> implements IterableIterator<T> {
+    constructor(protected iterator: IterableIterator<T>, protected encoding?: any) {}
+    [Symbol.iterator]() { return this.iterator; }
+    next(value?: any) { return this.iterator.next(value); }
+    throw(error?: any) {
+        if (typeof this.iterator.throw === 'function') {
+            return this.iterator.throw(error);
+        }
+        return { done: true, value: null as any };
+    }
+    return(value?: any) {
+        if (typeof this.iterator.return === 'function') {
+            return this.iterator.return(value);
+        }
+        return { done: true, value: null as any };
+    }
+    pipe(stream: NodeJS.WritableStream) {
+        let { encoding } = this;
+        let res: IteratorResult<T>;
+        let write = (err?: any) => {
+            stream['removeListener']('error', write);
+            stream['removeListener']('drain', write);
+            if (err) return this.throw(err);
+            if (stream['writable']) {
+                do {
+                    if ((res = this.next()).done) break;
+                } while (emit(stream, encoding, res.value));
+            }
+            return wait(stream, encoding, res && res.done, write);
+        };
+        write();
+        return stream;
+    }
+}
+
+export class AsyncPipeIterator<T> implements AsyncIterableIterator<T> {
+    constructor(protected iterator: AsyncIterableIterator<T>, protected encoding?: any) {}
+    [Symbol.asyncIterator]() { return this.iterator; }
+    next(value?: any) { return this.iterator.next(value); }
+    async throw(error?: any) {
+        if (typeof this.iterator.throw === 'function') {
+            return this.iterator.throw(error);
+        }
+        return { done: true, value: null as any };
+    }
+    async return(value?: any) {
+        if (typeof this.iterator.return === 'function') {
+            return this.iterator.return(value);
+        }
+        return { done: true, value: null as any };
+    }
+    pipe(stream: NodeJS.WritableStream) {
+        let { encoding } = this;
+        let res: IteratorResult<T>;
+        let write = async (err?: any) => {
+            stream['removeListener']('error', write);
+            stream['removeListener']('drain', write);
+            if (err) return this.throw(err);
+            if (stream['writable']) {
+                do {
+                    if ((res = await this.next()).done) break;
+                } while (emit(stream, encoding, res.value));
+            }
+            return wait(stream, encoding, res && res.done, write);
+        };
+        write();
+        return stream;
+    }
+}
+
+function emit(stream: NodeJS.WritableStream, encoding: string, value: any) {
+    return stream['write']((encoding === 'utf8' ? value + '\n' : value) as any, encoding);
+}
+
+function wait(stream: NodeJS.WritableStream, encoding: string, done: boolean, write: (x?: any) => void) {
+    const p = eval('process'); // defeat closure compiler
+    if (!done) {
+        stream['once']('error', write);
+        stream['once']('drain', write);
+    } else if (!(!p || stream === p.stdout) && !(stream as any)['isTTY']) {
+        stream['end'](<any> (encoding === 'utf8' ? '\n' : new Uint8Array(0)));
+    }
+}

From 78cba3842ac029b87d9c6c41078b0457143b0ffb Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sat, 12 May 2018 15:18:02 -0700
Subject: [PATCH 24/51] arrow2csv: support reading arrow streams from stdin

---
 js/src/bin/arrow2csv.ts | 85 +++++++++++++++++++++++++----------------
 1 file changed, 53 insertions(+), 32 deletions(-)

diff --git a/js/src/bin/arrow2csv.ts b/js/src/bin/arrow2csv.ts
index 6d197c7b6b4..b39fda48d80 100644
--- a/js/src/bin/arrow2csv.ts
+++ b/js/src/bin/arrow2csv.ts
@@ -20,29 +20,63 @@
 /* tslint:disable */
 
 import * as fs from 'fs';
+import { promisify } from 'util';
 import * as Arrow from '../Arrow';
 
+const readFile = promisify(fs.readFile);
 const { parse } = require('json-bignum');
-const optionList = [
-    {
-        type: String,
-        name: 'schema', alias: 's',
-        optional: true, multiple: true,
-        typeLabel: '[underline]{columns}',
-        description: 'A space-delimited list of column names'
-    },
-    {
-        type: String,
-        name: 'file', alias: 'f',
-        optional: false, multiple: true,
-        description: 'The Arrow file to read'
+const argv = require(`command-line-args`)(cliOpts(), { partial: true });
+const files = [...(argv.file || []), ...(argv._unknown || [])].filter(Boolean);
+
+(async () => {
+    if (!files.length) {
+        let buffer = Buffer.from([]);
+        for await (let chunk of require('stream-to-iterator')(process.stdin as any)) {
+            chunk = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, 'binary');
+            buffer = Buffer.concat([buffer, chunk], buffer.length + chunk.length);
+        }
+        if (buffer.length === 0) {
+            return print_usage();
+        }
+        files.push(buffer);
+    }
+    for (let input of files) {
+        printTable(typeof input === 'string' ? await readFile(input) : input);
     }
-];
+})().catch((e) => { console.error(e); process.exit(1); });
 
-const argv = require(`command-line-args`)(optionList, { partial: true });
-const files = [...argv.file, ...(argv._unknown || [])].filter(Boolean);
+function printTable(input: any) {
+    let table: Arrow.Table;
+    try {
+        table = Arrow.Table.from(input);
+    } catch (e) {
+        table = Arrow.Table.from(parse(input + ''));
+    }
+    if (argv.schema && argv.schema.length) {
+        table = table.select(...argv.schema);
+    }
+    table.rowsToString().pipe(process.stdout);
+}
 
-if (!files.length) {
+function cliOpts() {
+    return [
+        {
+            type: String,
+            name: 'schema', alias: 's',
+            optional: true, multiple: true,
+            typeLabel: '[underline]{columns}',
+            description: 'A space-delimited list of column names'
+        },
+        {
+            type: String,
+            name: 'file', alias: 'f',
+            optional: false, multiple: true,
+            description: 'The Arrow file to read'
+        }
+    ];    
+}
+
+function print_usage() {
     console.log(require('command-line-usage')([
         {
             header: 'arrow2csv',
@@ -60,7 +94,7 @@ if (!files.length) {
         {
             header: 'Options',
             optionList: [
-                ...optionList,
+                ...cliOpts(),
                 {
                     name: 'help',
                     description: 'Print this usage guide.'
@@ -81,17 +115,4 @@ if (!files.length) {
         }
     ]));
     process.exit(1);
-}
-
-files.forEach((source) => {
-    let table: Arrow.Table, input = fs.readFileSync(source);
-    try {
-        table = Arrow.Table.from(input);
-    } catch (e) {
-        table = Arrow.Table.from(parse(input + ''));
-    }
-    if (argv.schema && argv.schema.length) {
-        table = table.select(...argv.schema);
-    }
-    table.rowsToString().pipe(process.stdout);
-});
+}
\ No newline at end of file

From 263d06d908e6cf187be5819cb86cbe145c7b19fa Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sat, 12 May 2018 15:18:39 -0700
Subject: [PATCH 25/51] clean up js integration script

---
 js/bin/integration.js | 126 ++++++++++++++++++++----------------------
 1 file changed, 59 insertions(+), 67 deletions(-)

diff --git a/js/bin/integration.js b/js/bin/integration.js
index 73162b6a8c3..6c064deac25 100755
--- a/js/bin/integration.js
+++ b/js/bin/integration.js
@@ -17,61 +17,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
-var fs = require('fs');
-var glob = require('glob');
-var path = require('path');
-var gulp = require.resolve(path.join(`..`, `node_modules/gulp/bin/gulp.js`));
-var child_process = require(`child_process`);
-var optionList = [
-    {
-        type: String,
-        name: 'mode',
-        description: 'The integration test to run'
-    },
-    {
-        type: String,
-        name: 'arrow', alias: 'a',
-        multiple: true, defaultValue: [],
-        description: 'The Arrow file[s] to read/write'
-    },
-    {
-        type: String,
-        name: 'json', alias: 'j',
-        multiple: true, defaultValue: [],
-        description: 'The JSON file[s] to read/write'
-    }
-];
-
-var argv = require(`command-line-args`)(optionList, { partial: true });
-
-function print_usage() {
-    console.log(require('command-line-usage')([
-        {
-            header: 'integration',
-            content: 'Script for running Arrow integration tests'
-        },
-        {
-            header: 'Synopsis',
-            content: [
-                '$ integration.js -j file.json -a file.arrow --mode validate'
-            ]
-        },
-        {
-            header: 'Options',
-            optionList: [
-                ...optionList,
-                {
-                    name: 'help',
-                    description: 'Print this usage guide.'
-                }
-            ]
-        },
-    ]));
-    process.exit(1);
-}
+const fs = require('fs');
+const glob = require('glob');
+const path = require('path');
+const child_process = require(`child_process`);
+const argv = require(`command-line-args`)(cliOpts(), { partial: true });
+const gulpPath = require.resolve(path.join(`..`, `node_modules/gulp/bin/gulp.js`));
 
-let jsonPaths = argv.json;
-let arrowPaths = argv.arrow;
+let jsonPaths = [...(argv.json || [])];
+let arrowPaths = [...(argv.arrow || [])];
 
 if (!argv.mode) {
     return print_usage();
@@ -89,12 +43,13 @@ if (mode === 'VALIDATE' && !jsonPaths.length) {
                     if (fs.existsSync(arrowPath)) {
                         jsonPaths.push(jsonPath);
                         arrowPaths.push(arrowPath);
-                        console.log('-j', jsonPath, '-a', arrowPath, '\\');
                     }
                 }
             }
             return [jsonPaths, arrowPaths];
         }, [[], []]);
+        console.log(`jsonPaths: [\n\t${jsonPaths.join('\n\t')}\n]`);
+        console.log(`arrowPaths: [\n\t${arrowPaths.join('\n\t')}\n]`);
     }
 } else if (!jsonPaths.length) {
     return print_usage();
@@ -107,24 +62,61 @@ switch (mode) {
             args.push('-j', p, '-a', arrowPaths[i]);
         });
         process.exitCode = child_process.spawnSync(
-            gulp, args,
+            gulpPath, args,
             {
                 cwd: path.resolve(__dirname, '..'),
                 stdio: ['ignore', 'inherit', 'inherit']
             }
         ).status || process.exitCode || 0;
-        // for (let i = -1, n = jsonPaths.length; ++i < n;) {
-        //     const jsonPath = jsonPaths[i];
-        //     const arrowPath = arrowPaths[i];
-        //     child_process.spawnSync(
-        //         gulp, args.concat(['-j', jsonPath, '-a', arrowPath]),
-        //         {
-        //             cwd: path.resolve(__dirname, '..'),
-        //             stdio: ['ignore', 'inherit', 'inherit']
-        //         }
-        //     );
-        // }
         break;
     default:
         print_usage();
 }
+
+function cliOpts() {
+    return [
+        {
+            type: String,
+            name: 'mode',
+            description: 'The integration test to run'
+        },
+        {
+            type: String,
+            name: 'arrow', alias: 'a',
+            multiple: true, defaultValue: [],
+            description: 'The Arrow file[s] to read/write'
+        },
+        {
+            type: String,
+            name: 'json', alias: 'j',
+            multiple: true, defaultValue: [],
+            description: 'The JSON file[s] to read/write'
+        }
+    ];
+}
+
+function print_usage() {
+    console.log(require('command-line-usage')([
+        {
+            header: 'integration',
+            content: 'Script for running Arrow integration tests'
+        },
+        {
+            header: 'Synopsis',
+            content: [
+                '$ integration.js -j file.json -a file.arrow --mode validate'
+            ]
+        },
+        {
+            header: 'Options',
+            optionList: [
+                ...cliOpts(),
+                {
+                    name: 'help',
+                    description: 'Print this usage guide.'
+                }
+            ]
+        },
+    ]));
+    process.exit(1);
+}

From 832cc301bf642bf8f76ca39c962456ffb8ee44bf Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sat, 12 May 2018 15:20:58 -0700
Subject: [PATCH 26/51] add more js integration scripts for creating/converting
 arrow formats

---
 js/bin/file-to-stream.js | 46 +++++++++++++++++++
 js/bin/json-to-arrow.js  | 97 ++++++++++++++++++++++++++++++++++++++++
 js/bin/stream-to-file.js | 37 +++++++++++++++
 js/index.js              |  1 +
 js/index.mjs             |  1 +
 js/index.ts              |  1 +
 js/package.json          |  4 +-
 js/tsconfig.json         |  2 +-
 8 files changed, 187 insertions(+), 2 deletions(-)
 create mode 100755 js/bin/file-to-stream.js
 create mode 100755 js/bin/json-to-arrow.js
 create mode 100755 js/bin/stream-to-file.js
 create mode 100644 js/index.js
 create mode 100644 js/index.mjs
 create mode 100644 js/index.ts

diff --git a/js/bin/file-to-stream.js b/js/bin/file-to-stream.js
new file mode 100755
index 00000000000..ffcb9f61727
--- /dev/null
+++ b/js/bin/file-to-stream.js
@@ -0,0 +1,46 @@
+#! /usr/bin/env node
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const fs = require('fs');
+const path = require('path');
+const streamToIterator = require('stream-to-iterator');
+
+const encoding = 'binary';
+const { util: { PipeIterator } } = require('../');
+const { Table, serializeStream } = require('../');
+
+(async () => {
+    // Todo (ptaylor): implement `serializeStreamAsync` that accepts an
+    // AsyncIterable<Buffer>, rather than aggregating into a Table first
+    const in_ = streamToIterator(process.argv.length < 3 ? process.stdin :
+        fs.createReadStream(path.resolve(process.argv[2]), { encoding }));
+    const out = process.argv.length < 4 ? process.stdout :
+        fs.createWriteStream(path.resolve(process.argv[3]), { encoding });
+    new PipeIterator(serializeStream(await Table.fromAsync(readArrowFile(in_))), encoding).pipe(out);
+
+})().catch((e) => { console.error(e); process.exit(1); });
+
+async function readArrowFile(stream) {
+    let buffer = Buffer.from([]);
+    for await (let chunk of stream) {
+        chunk = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding);
+        buffer = Buffer.concat([buffer, chunk], buffer.length + chunk.length);
+    }
+    return buffer;
+}
diff --git a/js/bin/json-to-arrow.js b/js/bin/json-to-arrow.js
new file mode 100755
index 00000000000..c36d594710b
--- /dev/null
+++ b/js/bin/json-to-arrow.js
@@ -0,0 +1,97 @@
+#! /usr/bin/env node
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const fs = require('fs');
+const glob = require('glob');
+const path = require('path');
+const { Table } = require('../index');
+const { promisify } = require('util');
+const { parse } = require('json-bignum');
+const argv = require(`command-line-args`)(cliOpts(), { partial: true });
+
+const encoding = 'binary';
+const stream = argv.format === 'stream';
+const jsonPaths = [...(argv.json || [])];
+const arrowPaths = [...(argv.arrow || [])];
+
+if (!jsonPaths.length || !arrowPaths.length || (jsonPaths.length !== arrowPaths.length)) {
+    return print_usage();
+}
+
+const readFile = callResolved(promisify(fs.readFile));
+const writeFile = callResolved(promisify(fs.writeFile));
+
+(async () => await Promise.all(jsonPaths.map(async (jPath, i) => {
+    const aPath = arrowPaths[i];
+    const arrowTable = Table.from(parse('' + (await readFile(jPath))));
+    await writeFile(aPath, arrowTable.serialize(encoding, stream), encoding);
+})))().catch((e) => { console.error(e); process.exit(1); });
+
+function callResolved(fn) {
+    return async (path_, ...xs) => await fn(path.resolve(path_), ...xs);
+}
+
+function cliOpts() {
+    return [
+        {
+            type: String,
+            name: 'format', alias: 'f',
+            multiple: false, defaultValue: 'file',
+            description: 'The Arrow format to write, either "file" or "stream"'
+        },
+        {
+            type: String,
+            name: 'arrow', alias: 'a',
+            multiple: true, defaultValue: [],
+            description: 'The Arrow file[s] to write'
+        },
+        {
+            type: String,
+            name: 'json', alias: 'j',
+            multiple: true, defaultValue: [],
+            description: 'The JSON file[s] to read'
+        }
+    ];
+}
+
+function print_usage() {
+    console.log(require('command-line-usage')([
+        {
+            header: 'json-to-arrow',
+            content: 'Script for converting an JSON Arrow file to a binary Arrow file'
+        },
+        {
+            header: 'Synopsis',
+            content: [
+                '$ json-to-arrow.js -j in.json -a out.arrow -f stream'
+            ]
+        },
+        {
+            header: 'Options',
+            optionList: [
+                ...cliOpts(),
+                {
+                    name: 'help',
+                    description: 'Print this usage guide.'
+                }
+            ]
+        },
+    ]));
+    process.exit(1);
+}
diff --git a/js/bin/stream-to-file.js b/js/bin/stream-to-file.js
new file mode 100755
index 00000000000..63d3850318e
--- /dev/null
+++ b/js/bin/stream-to-file.js
@@ -0,0 +1,37 @@
+#! /usr/bin/env node
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const fs = require('fs');
+const path = require('path');
+const streamToIterator = require('stream-to-iterator');
+
+const encoding = 'binary';
+const { util: { PipeIterator } } = require('../');
+const { Table, serializeFile, fromNodeStream } = require('../');
+
+(async () => {
+    // Todo (ptaylor): implement `serializeFileAsync` that accepts an
+    // AsyncIterable<Buffer>, rather than aggregating into a Table first
+    const in_ = streamToIterator(process.argv.length < 3 ? process.stdin :
+        fs.createReadStream(path.resolve(process.argv[2]), { encoding }));
+    const out = process.argv.length < 4 ? process.stdout :
+        fs.createWriteStream(path.resolve(process.argv[3]), { encoding });
+    new PipeIterator(serializeFile(await Table.fromAsync(fromNodeStream(in_))), encoding).pipe(out);
+
+})().catch((e) => { console.error(e); process.exit(1); });
diff --git a/js/index.js b/js/index.js
new file mode 100644
index 00000000000..1582b35ec83
--- /dev/null
+++ b/js/index.js
@@ -0,0 +1 @@
+module.exports = require('./targets/apache-arrow');
\ No newline at end of file
diff --git a/js/index.mjs b/js/index.mjs
new file mode 100644
index 00000000000..7ab8a28ee9b
--- /dev/null
+++ b/js/index.mjs
@@ -0,0 +1 @@
+export * from './targets/apache-arrow';
\ No newline at end of file
diff --git a/js/index.ts b/js/index.ts
new file mode 100644
index 00000000000..45c803f8a5b
--- /dev/null
+++ b/js/index.ts
@@ -0,0 +1 @@
+export * from './src/Arrow';
\ No newline at end of file
diff --git a/js/package.json b/js/package.json
index ee5ca3314a9..a5b48c40435 100644
--- a/js/package.json
+++ b/js/package.json
@@ -2,6 +2,7 @@
   "version": "0.3.0",
   "name": "apache-arrow",
   "description": "Apache Arrow columnar in-memory format",
+  "main": "./index",
   "bin": {
     "arrow2csv": "bin/arrow2csv.js"
   },
@@ -53,7 +54,7 @@
   ],
   "dependencies": {
     "@types/flatbuffers": "1.6.5",
-    "@types/node": "9.3.0",
+    "@types/node": "10.0.8",
     "@types/text-encoding-utf-8": "1.0.1",
     "command-line-args": "5.0.1",
     "command-line-usage": "4.1.0",
@@ -96,6 +97,7 @@
     "stream-to-iterator": "3.0.1",
     "trash": "4.2.1",
     "ts-jest": "22.0.1",
+    "ts-node": "6.0.3",
     "tslint": "5.9.1",
     "typedoc": "0.10.0",
     "typescript": "2.7.1",
diff --git a/js/tsconfig.json b/js/tsconfig.json
index a56166b886b..67296750dea 100644
--- a/js/tsconfig.json
+++ b/js/tsconfig.json
@@ -6,6 +6,6 @@
   },
   "compilerOptions": {
     "target": "ESNEXT",
-    "module": "es2015"
+    "module": "commonjs"
   }
 }

From de81ac1b8723e7b5d675ad6cb360302d6dc970de Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sat, 12 May 2018 15:21:41 -0700
Subject: [PATCH 27/51] Update JSTester to be an Arrow producer now too

---
 integration/integration_test.py | 34 +++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/integration/integration_test.py b/integration/integration_test.py
index 301017cac3d..20b8e48a5a0 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -1092,15 +1092,19 @@ def file_to_stream(self, file_path, stream_path):
         os.system(cmd)
 
 class JSTester(Tester):
-    PRODUCER = False
+    PRODUCER = True
     CONSUMER = True
 
-    INTEGRATION_EXE = os.path.join(ARROW_HOME, 'js/bin/integration.js')
+    EXE_PATH = os.path.join(ARROW_HOME, 'js/bin')
+    VALIDATE = os.path.join(EXE_PATH, 'integration.js')
+    JSON_TO_ARROW = os.path.join(EXE_PATH, 'json-to-arrow.js')
+    STREAM_TO_FILE = os.path.join(EXE_PATH, 'stream-to-file.js')
+    FILE_TO_STREAM = os.path.join(EXE_PATH, 'file-to-stream.js')
 
     name = 'JS'
 
-    def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
-        cmd = [self.INTEGRATION_EXE]
+    def _run(self, exe_cmd, arrow_path=None, json_path=None, command='VALIDATE'):
+        cmd = [exe_cmd]
 
         if arrow_path is not None:
             cmd.extend(['-a', arrow_path])
@@ -1111,16 +1115,30 @@ def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
         cmd.extend(['--mode', command])
 
         if self.debug:
+            cmd.extend(['-t', 'src'])
             print(' '.join(cmd))
 
         run_cmd(cmd)
 
     def validate(self, json_path, arrow_path):
-        return self._run(arrow_path, json_path, 'VALIDATE')
+        return self._run(self.VALIDATE, arrow_path, json_path, 'VALIDATE')
+
+    def json_to_file(self, json_path, arrow_path):
+        cmd = ['node --harmony-async-iteration', self.JSON_TO_ARROW, '-a', arrow_path, '-j', json_path]
+        cmd = ' '.join(cmd)
+        if self.debug:
+            print(cmd)
+        os.system(cmd)
 
     def stream_to_file(self, stream_path, file_path):
-        # Just copy stream to file, we can read the stream directly
-        cmd = ['cp', stream_path, file_path]
+        cmd = ['cat', stream_path, '|', 'node --harmony-async-iteration', self.STREAM_TO_FILE, '>', file_path]
+        cmd = ' '.join(cmd)
+        if self.debug:
+            print(cmd)
+        os.system(cmd)
+
+    def file_to_stream(self, file_path, stream_path):
+        cmd = ['cat', file_path, '|', 'node --harmony-async-iteration', self.FILE_TO_STREAM, '>', stream_path]
         cmd = ' '.join(cmd)
         if self.debug:
             print(cmd)
@@ -1133,7 +1151,7 @@ def get_static_json_files():
 
 
 def run_all_tests(debug=False):
-    testers = [CPPTester(debug=debug), JavaTester(debug=debug), JSTester(debug=debug)]
+    testers = [JSTester(debug=debug), CPPTester(debug=debug), JavaTester(debug=debug)]
     static_json_files = get_static_json_files()
     generated_json_files = get_generated_json_files()
     json_files = static_json_files + generated_json_files

From af9f4a8c5617ffcb108390ad4c9aa06286166244 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sat, 12 May 2018 21:03:38 -0700
Subject: [PATCH 28/51] run integration tests in node 10.1

---
 .travis.yml                     |  2 +-
 integration/integration_test.py |  8 ++++----
 js/bin/file-to-stream.js        | 25 ++++++++++++++-----------
 js/bin/json-to-arrow.js         |  4 +++-
 js/bin/stream-to-file.js        | 14 +++++++-------
 js/gulp/arrow-task.js           | 10 ++++++++--
 js/package.json                 | 12 ++++++------
 7 files changed, 43 insertions(+), 32 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index e91d7b166a9..68ab478652e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -145,7 +145,7 @@ matrix:
   - language: node_js
     os: linux
     node_js:
-    - '9.8'
+    - '10.1'
     before_script:
     - if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi
     - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 20b8e48a5a0..1f604be9f95 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -1124,21 +1124,21 @@ def validate(self, json_path, arrow_path):
         return self._run(self.VALIDATE, arrow_path, json_path, 'VALIDATE')
 
     def json_to_file(self, json_path, arrow_path):
-        cmd = ['node --harmony-async-iteration', self.JSON_TO_ARROW, '-a', arrow_path, '-j', json_path]
+        cmd = ['node', self.JSON_TO_ARROW, '-a', arrow_path, '-j', json_path]
         cmd = ' '.join(cmd)
         if self.debug:
             print(cmd)
         os.system(cmd)
 
     def stream_to_file(self, stream_path, file_path):
-        cmd = ['cat', stream_path, '|', 'node --harmony-async-iteration', self.STREAM_TO_FILE, '>', file_path]
+        cmd = ['cat', stream_path, '|', 'node', self.STREAM_TO_FILE, '>', file_path]
         cmd = ' '.join(cmd)
         if self.debug:
             print(cmd)
         os.system(cmd)
 
     def file_to_stream(self, file_path, stream_path):
-        cmd = ['cat', file_path, '|', 'node --harmony-async-iteration', self.FILE_TO_STREAM, '>', stream_path]
+        cmd = ['cat', file_path, '|', 'node', self.FILE_TO_STREAM, '>', stream_path]
         cmd = ' '.join(cmd)
         if self.debug:
             print(cmd)
@@ -1151,7 +1151,7 @@ def get_static_json_files():
 
 
 def run_all_tests(debug=False):
-    testers = [JSTester(debug=debug), CPPTester(debug=debug), JavaTester(debug=debug)]
+    testers = [CPPTester(debug=debug), JavaTester(debug=debug), JSTester(debug=debug)]
     static_json_files = get_static_json_files()
     generated_json_files = get_generated_json_files()
     json_files = static_json_files + generated_json_files
diff --git a/js/bin/file-to-stream.js b/js/bin/file-to-stream.js
index ffcb9f61727..3943785987c 100755
--- a/js/bin/file-to-stream.js
+++ b/js/bin/file-to-stream.js
@@ -19,28 +19,31 @@
 
 const fs = require('fs');
 const path = require('path');
-const streamToIterator = require('stream-to-iterator');
 
 const encoding = 'binary';
-const { util: { PipeIterator } } = require('../');
-const { Table, serializeStream } = require('../');
+const ext = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : '';
+const { util: { PipeIterator } } = require(`../index${ext}`);
+const { Table, serializeStream } = require(`../index${ext}`);
 
 (async () => {
     // Todo (ptaylor): implement `serializeStreamAsync` that accepts an
     // AsyncIterable<Buffer>, rather than aggregating into a Table first
-    const in_ = streamToIterator(process.argv.length < 3 ? process.stdin :
-        fs.createReadStream(path.resolve(process.argv[2]), { encoding }));
-    const out = process.argv.length < 4 ? process.stdout :
-        fs.createWriteStream(path.resolve(process.argv[3]), { encoding });
+    const in_ = process.argv.length < 3
+        ? process.stdin : fs.createReadStream(path.resolve(process.argv[2]));
+    const out = process.argv.length < 4
+        ? process.stdout : fs.createWriteStream(path.resolve(process.argv[3]));
     new PipeIterator(serializeStream(await Table.fromAsync(readArrowFile(in_))), encoding).pipe(out);
 
 })().catch((e) => { console.error(e); process.exit(1); });
 
 async function readArrowFile(stream) {
-    let buffer = Buffer.from([]);
+    let buffers = [], totalLength = 0;
     for await (let chunk of stream) {
-        chunk = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding);
-        buffer = Buffer.concat([buffer, chunk], buffer.length + chunk.length);
+        if (!Buffer.isBuffer(chunk)) {
+            chunk = Buffer.from(chunk, encoding);
+        }
+        buffers.push(chunk);
+        totalLength += chunk.byteLength;
     }
-    return buffer;
+    return Buffer.concat(buffers, totalLength);
 }
diff --git a/js/bin/json-to-arrow.js b/js/bin/json-to-arrow.js
index c36d594710b..f28b4145ffa 100755
--- a/js/bin/json-to-arrow.js
+++ b/js/bin/json-to-arrow.js
@@ -20,11 +20,13 @@
 const fs = require('fs');
 const glob = require('glob');
 const path = require('path');
-const { Table } = require('../index');
 const { promisify } = require('util');
 const { parse } = require('json-bignum');
 const argv = require(`command-line-args`)(cliOpts(), { partial: true });
 
+const ext = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : '';
+const { Table } = require(`../index${ext}`);
+
 const encoding = 'binary';
 const stream = argv.format === 'stream';
 const jsonPaths = [...(argv.json || [])];
diff --git a/js/bin/stream-to-file.js b/js/bin/stream-to-file.js
index 63d3850318e..cf32e274f33 100755
--- a/js/bin/stream-to-file.js
+++ b/js/bin/stream-to-file.js
@@ -19,19 +19,19 @@
 
 const fs = require('fs');
 const path = require('path');
-const streamToIterator = require('stream-to-iterator');
 
 const encoding = 'binary';
-const { util: { PipeIterator } } = require('../');
-const { Table, serializeFile, fromNodeStream } = require('../');
+const ext = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : '';
+const { util: { PipeIterator } } = require(`../index${ext}`);
+const { Table, serializeFile, fromNodeStream } = require(`../index${ext}`);
 
 (async () => {
     // Todo (ptaylor): implement `serializeFileAsync` that accepts an
     // AsyncIterable<Buffer>, rather than aggregating into a Table first
-    const in_ = streamToIterator(process.argv.length < 3 ? process.stdin :
-        fs.createReadStream(path.resolve(process.argv[2]), { encoding }));
-    const out = process.argv.length < 4 ? process.stdout :
-        fs.createWriteStream(path.resolve(process.argv[3]), { encoding });
+    const in_ = process.argv.length < 3
+        ? process.stdin : fs.createReadStream(path.resolve(process.argv[2]));
+    const out = process.argv.length < 4
+        ? process.stdout : fs.createWriteStream(path.resolve(process.argv[3]));
     new PipeIterator(serializeFile(await Table.fromAsync(fromNodeStream(in_))), encoding).pipe(out);
 
 })().catch((e) => { console.error(e); process.exit(1); });
diff --git a/js/gulp/arrow-task.js b/js/gulp/arrow-task.js
index eb83a6da310..95fc1eed0f8 100644
--- a/js/gulp/arrow-task.js
+++ b/js/gulp/arrow-task.js
@@ -20,10 +20,13 @@ const {
     targetDir, observableFromStreams
 } = require('./util');
 
+const del = require('del');
 const gulp = require('gulp');
 const path = require('path');
+const { promisify } = require('util');
 const gulpRename = require(`gulp-rename`);
 const { memoizeTask } = require('./memoize-task');
+const exec = promisify(require('child_process').exec);
 const { Observable, ReplaySubject } = require('rxjs');
 
 const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, format) {
@@ -48,8 +51,11 @@ const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, forma
     ).publish(new ReplaySubject()).refCount();
 }))({});
 
-const arrowTSTask = ((cache) => memoizeTask(cache, function copyTS(target, format) {
-    return observableFromStreams(gulp.src(`src/**/*.ts`), gulp.dest(targetDir(target, format)));
+const arrowTSTask = ((cache) => memoizeTask(cache, async function copyTS(target, format) {
+    const out = targetDir(target, format);
+    await exec(`mkdirp ${out}`);
+    await exec(`shx cp -r src/* ${out}`);
+    await del(`${out}/**/*.js`);
 }))({});
   
   
diff --git a/js/package.json b/js/package.json
index a5b48c40435..ba47cb96ed6 100644
--- a/js/package.json
+++ b/js/package.json
@@ -60,15 +60,15 @@
     "command-line-usage": "4.1.0",
     "flatbuffers": "trxcllnt/flatbuffers-esm",
     "json-bignum": "0.0.3",
-    "text-encoding-utf-8": "^1.0.2",
+    "text-encoding-utf-8": "trxcllnt/text-encoding-utf-8#6a959afc86b798a435bd7b97682a7eb1f735ee2e",
     "tslib": "1.9.0"
   },
   "devDependencies": {
     "@std/esm": "0.26.0",
     "@types/glob": "5.0.35",
-    "@types/jest": "22.1.0",
+    "@types/jest": "22.2.3",
     "ast-types": "0.10.1",
-    "babel-jest": "22.4.1",
+    "babel-jest": "22.4.3",
     "benchmark": "2.1.4",
     "coveralls": "3.0.0",
     "del": "3.0.0",
@@ -81,7 +81,7 @@
     "gulp-transform-js-ast": "1.0.2",
     "gulp-typescript": "3.2.4",
     "ix": "2.3.4",
-    "jest": "22.1.4",
+    "jest": "22.4.3",
     "jest-environment-node-debug": "2.0.0",
     "json": "9.0.6",
     "lerna": "2.7.1",
@@ -94,9 +94,8 @@
     "rxjs": "5.5.6",
     "shx": "0.2.2",
     "source-map-loader": "0.2.3",
-    "stream-to-iterator": "3.0.1",
     "trash": "4.2.1",
-    "ts-jest": "22.0.1",
+    "ts-jest": "22.4.6",
     "ts-node": "6.0.3",
     "tslint": "5.9.1",
     "typedoc": "0.10.0",
@@ -118,6 +117,7 @@
     "verbose": false,
     "globals": {
       "ts-jest": {
+        "skipBabel": true,
         "tsConfigFile": "test/tsconfig.json"
       }
     },

From 3187732158f16d58d33bfb59da25da035094dcd4 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sat, 12 May 2018 21:06:37 -0700
Subject: [PATCH 29/51] set bitmap alignment to 8 bytes if < 64 values

---
 js/src/ipc/writer/binary.ts | 78 ++++++++++++++++++++-----------------
 1 file changed, 42 insertions(+), 36 deletions(-)

diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
index 7e36a9382c1..699819d63a2 100644
--- a/js/src/ipc/writer/binary.ts
+++ b/js/src/ipc/writer/binary.ts
@@ -19,7 +19,7 @@ import { Table } from '../../table';
 import { DenseUnionData } from '../../data';
 import { RecordBatch } from '../../recordbatch';
 import { VectorVisitor, TypeVisitor } from '../../visitor';
-import { MAGIC, magicLength, magicAndPadding } from '../magic';
+import { MAGIC, magicLength, magicAndPadding, PADDING } from '../magic';
 import { align, getBool, packBools, iterateBits } from '../../util/bit';
 import { Vector, UnionVector, DictionaryVector, NestedVector, ListVector } from '../../vector';
 import { BufferMetadata, FieldMetadata, Footer, FileBlock, Message, RecordBatchMetadata, DictionaryBatch } from '../metadata';
@@ -113,17 +113,16 @@ export function serializeMessage(message: Message, data?: Uint8Array) {
     // Reserve 4 bytes for writing the message size at the front.
     // Metadata length includes the metadata byteLength + the 4
     // bytes for the length, and rounded up to the nearest 8 bytes.
-    const metadataLength = align(4 + metadataBytes.byteLength, 8);
+    const metadataLength = align(PADDING + metadataBytes.byteLength, 8);
     // + the length of the optional data buffer at the end, padded
-    const dataByteLength = align(data ? data.byteLength : 0, 8);
-    // since both metadataLength and dataByteLength are aligned to 8-byte
-    // boundaries, the entire message is also aligned.
-    const messageBytes = new Uint8Array(metadataLength + dataByteLength);
+    const dataByteLength = data ? data.byteLength : 0;
+    // ensure the entire message is aligned to an 8-byte boundary
+    const messageBytes = new Uint8Array(align(metadataLength + dataByteLength, 8));
     // Write the metadata length into the first 4 bytes, but subtract the
     // bytes we use to hold the length itself.
-    new DataView(messageBytes.buffer).setInt32(0, metadataLength - 4, platformIsLittleEndian);
+    new DataView(messageBytes.buffer).setInt32(0, metadataLength - PADDING, platformIsLittleEndian);
     // Copy the metadata bytes into the message buffer
-    messageBytes.set(metadataBytes, 4);
+    messageBytes.set(metadataBytes, PADDING);
     // Copy the optional data buffer after the metadata bytes
     (data && dataByteLength > 0) && messageBytes.set(data, metadataLength);
     // if (messageBytes.byteLength % 8 !== 0) { debugger; }
@@ -164,9 +163,12 @@ class RecordBatchSerializer extends VectorVisitor {
                 throw new RangeError('Cannot write arrays larger than 2^31 - 1 in length');
             }
             this.fieldNodes.push(new FieldMetadata(length, nullCount));
-            this.addBuffer(nullCount <= 0
+
+            const nullBitmapAlignment = length <= 64 ? 8 : 64;
+            const nullBitmap = nullCount <= 0
                 ? new Uint8Array(0) // placeholder validity buffer
-                : this.getTruncatedBitmap(data.offset, length, data.nullBitmap!));
+                : this.getTruncatedBitmap(data.offset, length, data.nullBitmap!);
+            this.addBuffer(nullBitmap, nullBitmapAlignment);
         }
         return super.visit(vector);
     }
@@ -240,19 +242,21 @@ class RecordBatchSerializer extends VectorVisitor {
     }
     protected visitBoolVector(vector: Vector<Bool>) {
         // Bool vector is a special case of FlatVector, as its data buffer needs to stay packed
+        let bitmap: Uint8Array;
         let values, { data, length } = vector;
+        let alignment = length <= 64 ? 8 : 64;
         if (vector.nullCount >= length) {
             // If all values are null, just insert a placeholder empty data buffer (fastest path)
-            this.addBuffer(new Uint8Array(0));
+            bitmap = new Uint8Array(0);
         } else if (!((values = data.values) instanceof Uint8Array)) {
             // Otherwise if the underlying data *isn't* a Uint8Array, enumerate
             // the values as bools and re-pack them into a Uint8Array (slow path)
-            // todo (ptaylor): this assumes Uint8Array = already packed, but that may not always be true
-            this.addBuffer(packBools(vector));
+            bitmap = packBools(vector);
         } else {
             // otherwise just slice the bitmap (fast path)
-            this.addBuffer(this.getTruncatedBitmap(data.offset, length, values));
+            bitmap = this.getTruncatedBitmap(data.offset, length, values);
         }
+        this.addBuffer(bitmap, alignment);
         return this;
     }
     protected visitFlatVector<T extends FlatType>(vector: Vector<T>) {
@@ -294,35 +298,34 @@ class RecordBatchSerializer extends VectorVisitor {
         }
         return this;
     }
-    protected addBuffer(array: TypedArray, alignment: number = 64) {
-        const values = array || new Uint8Array(0);
-        const alignedLength = align(values.byteLength, alignment);
+    protected addBuffer(values: TypedArray, alignment: number = 64) {
+        const length = align(values.byteLength, alignment);
         this.buffers.push(values);
-        this.buffersMeta.push(new BufferMetadata(this.byteLength, alignedLength));
-        this.byteLength += alignedLength;
+        this.buffersMeta.push(new BufferMetadata(this.byteLength, length));
+        this.byteLength += length;
         return this;
     }
-    protected getTruncatedBitmap(sliceOffset: number, length: number, bitmap: Uint8Array) {
-        const alignedLength = align(length, 64);
-        if (sliceOffset > 0 || length < alignedLength) {
+    protected getTruncatedBitmap(offset: number, length: number, bitmap: Uint8Array) {
+        const alignedLength = align(length, length <= 64 ? 8 : 64);
+        if (offset > 0 || bitmap.length < alignedLength) {
             // With a sliced array / non-zero offset, we have to copy the bitmap
-            const bytes = new Uint8Array(alignedLength).fill(255);
+            const bytes = new Uint8Array(alignedLength);
             bytes.set(
-                (sliceOffset % 8 === 0)
-                // If the sliceOffset is aligned to 1 byte, it's safe to slice the nullBitmap directly
-                ? bitmap.subarray(sliceOffset >> 3)
-                // iterate each bit starting from the sliceOffset, and repack into an aligned nullBitmap
-                : packBools(iterateBits(bitmap, sliceOffset, length, null, getBool))
+                (offset % 8 === 0)
+                // If the slice offset is aligned to 1 byte, it's safe to slice the nullBitmap directly
+                ? bitmap.subarray(offset >> 3)
+                // iterate each bit starting from the slice offset, and repack into an aligned nullBitmap
+                : packBools(iterateBits(bitmap, offset, length, null, getBool))
             );
             return bytes;
         }
         return bitmap;
     }
-    protected getZeroBasedValueOffsets(sliceOffset: number, length: number, valueOffsets: Int32Array) {
+    protected getZeroBasedValueOffsets(offset: number, length: number, valueOffsets: Int32Array) {
         // If we have a non-zero offset, then the value offsets do not start at
         // zero. We must a) create a new offsets array with shifted offsets and
         // b) slice the values array accordingly
-        if (sliceOffset > 0 || valueOffsets[0] !== 0) {
+        if (offset > 0 || valueOffsets[0] !== 0) {
             const startOffset = valueOffsets[0];
             const destOffsets = new Int32Array(length + 1);
             for (let index = -1; ++index < length;) {
@@ -524,12 +527,15 @@ export class TypeSerializer extends TypeVisitor {
     }
 }
 
-function concatBuffersWithMetadata(byteLength: number, buffers: Uint8Array[], buffersMeta: BufferMetadata[]) {
-    const data = new Uint8Array(byteLength);
-    for (let bufferIndex = -1, buffersLen = buffers.length; ++bufferIndex < buffersLen;) {
-        const { offset } = buffersMeta[bufferIndex];
-        const { buffer, byteOffset, byteLength } = buffers[bufferIndex];
-        data.set(new Uint8Array(buffer, byteOffset, byteLength), offset);
+function concatBuffersWithMetadata(totalByteLength: number, buffers: Uint8Array[], buffersMeta: BufferMetadata[]) {
+    const data = new Uint8Array(totalByteLength);
+    for (let i = -1, n = buffers.length; ++i < n;) {
+        const { offset, length } = buffersMeta[i];
+        const { buffer, byteOffset, byteLength } = buffers[i];
+        const realBufferLength = Math.min(length, byteLength);
+        if (realBufferLength > 0) {
+            data.set(new Uint8Array(buffer, byteOffset, realBufferLength), offset);
+        }
     }
     return data;
 }

From b52af25b6b60076e21836381d798393cccc57514 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sat, 12 May 2018 21:06:50 -0700
Subject: [PATCH 30/51] cleanup

---
 js/src/ipc/reader/binary.ts           |  9 +++++++--
 js/src/ipc/reader/node.ts             | 12 ++++++------
 js/test/integration/validate-tests.ts |  6 +++---
 3 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/js/src/ipc/reader/binary.ts b/js/src/ipc/reader/binary.ts
index 1f970df912f..0447440a75a 100644
--- a/js/src/ipc/reader/binary.ts
+++ b/js/src/ipc/reader/binary.ts
@@ -171,13 +171,18 @@ function readFileSchema(bb: ByteBuffer) {
 
 function readFileMessages(footer: Footer) {
     return function* (bb: ByteBuffer) {
+        let message: RecordBatchMetadata | DictionaryBatch;
         for (let i = -1, batches = footer.dictionaryBatches, n = batches.length; ++i < n;) {
             bb.setPosition(batches[i].offset);
-            yield readMessage(bb, bb.readInt32(bb.position())) as DictionaryBatch;
+            if (message = readMessage(bb, bb.readInt32(bb.position())) as DictionaryBatch) {
+                yield message;
+            }
         }
         for (let i = -1, batches = footer.recordBatches, n = batches.length; ++i < n;) {
             bb.setPosition(batches[i].offset);
-            yield readMessage(bb, bb.readInt32(bb.position())) as RecordBatchMetadata;
+            if (message = readMessage(bb, bb.readInt32(bb.position())) as RecordBatchMetadata) {
+                yield message;
+            }
         }
     };
 }
diff --git a/js/src/ipc/reader/node.ts b/js/src/ipc/reader/node.ts
index ec7a1694b28..56a484dd53e 100644
--- a/js/src/ipc/reader/node.ts
+++ b/js/src/ipc/reader/node.ts
@@ -29,12 +29,12 @@ export async function* fromNodeStream(stream: NodeJS.ReadableStream) {
 
     for await (let chunk of (stream as any as AsyncIterable<Uint8Array | Buffer | string>)) {
 
-        const grown = new Uint8Array(bytes.length + chunk.length);
+        const grown = new Uint8Array(bytes.byteLength + chunk.length);
 
         if (typeof chunk !== 'string') {
-            grown.set(bytes, 0) || grown.set(chunk, bytes.length);
+            grown.set(bytes, 0) || grown.set(chunk, bytes.byteLength);
         } else {
-            for (let i = -1, j = bytes.length, n = chunk.length; ++i < n;) {
+            for (let i = -1, j = bytes.byteLength, n = chunk.length; ++i < n;) {
                 grown[i + j] = chunk.charCodeAt(i);
             }
         }
@@ -45,7 +45,7 @@ export async function* fromNodeStream(stream: NodeJS.ReadableStream) {
             messageLength = new DataView(bytes.buffer).getInt32(0, true);
         }
 
-        while (messageLength < bytes.length) {
+        while (messageLength < bytes.byteLength) {
             if (!message) {
                 (bb = new ByteBuffer(bytes)).setPosition(4);
                 if (message = _Message.getRootAsMessage(bb)) {
@@ -55,9 +55,9 @@ export async function* fromNodeStream(stream: NodeJS.ReadableStream) {
                 throw new Error(`Invalid message at position ${bytesRead}`);
             }
             bytesRead += messageLength + PADDING;
-            yield bytes.subarray(0, messageLength);
+            yield bytes.subarray(0, messageLength + PADDING);
             bytes = bytes.subarray(messageLength + PADDING);
-            messageLength = bytes.length <= 0 ? 0 :
+            messageLength = bytes.byteLength <= 0 ? 0 :
                 new DataView(bytes.buffer).getInt32(bytes.byteOffset, true);
             message = null;
         }
diff --git a/js/test/integration/validate-tests.ts b/js/test/integration/validate-tests.ts
index 0ab8694d398..c301d6553b6 100644
--- a/js/test/integration/validate-tests.ts
+++ b/js/test/integration/validate-tests.ts
@@ -120,8 +120,8 @@ describe(`Integration`, () => {
             testReaderIntegration(json, arrowBuffer);
             testTableFromBuffersIntegration(json, arrowBuffer);
             testTableToBuffersIntegration('json', 'file')(json, arrowBuffer);
-            testTableToBuffersIntegration('json', 'stream')(json, arrowBuffer);
             testTableToBuffersIntegration('binary', 'file')(json, arrowBuffer);
+            testTableToBuffersIntegration('json', 'stream')(json, arrowBuffer);
             testTableToBuffersIntegration('binary', 'stream')(json, arrowBuffer);
         });
     }
@@ -164,10 +164,10 @@ function testTableFromBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array)
 }
 
 function testTableToBuffersIntegration(srcFormat: 'json' | 'binary', arrowFormat: 'stream' | 'file') {
+    const refFormat = srcFormat === `json` ? `binary` : `json`;
     return function testTableToBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array) {
-        test(`serializing ${srcFormat} to a ${arrowFormat} reports the same values as the alternate format`, () => {
+        test(`serialized ${srcFormat} ${arrowFormat} reports the same values as the ${refFormat} ${arrowFormat}`, () => {
             expect.hasAssertions();
-            const refFormat = srcFormat === 'json' ? `binary` : `json`;
             const refTable = Table.from(refFormat === `json` ? jsonData : arrowBuffer);
             const srcTable = Table.from(srcFormat === `json` ? jsonData : arrowBuffer);
             const dstTable = Table.from(srcTable.serialize(`binary`, arrowFormat === `stream`));

From d4b8637b779fcd4322d28249e7fff9715617d65f Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sat, 12 May 2018 21:39:41 -0700
Subject: [PATCH 31/51] add license headers

---
 js/index.js  | 17 +++++++++++++++++
 js/index.mjs | 17 +++++++++++++++++
 js/index.ts  | 17 +++++++++++++++++
 3 files changed, 51 insertions(+)

diff --git a/js/index.js b/js/index.js
index 1582b35ec83..e42cb32d1f9 100644
--- a/js/index.js
+++ b/js/index.js
@@ -1 +1,18 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 module.exports = require('./targets/apache-arrow');
\ No newline at end of file
diff --git a/js/index.mjs b/js/index.mjs
index 7ab8a28ee9b..30435371242 100644
--- a/js/index.mjs
+++ b/js/index.mjs
@@ -1 +1,18 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 export * from './targets/apache-arrow';
\ No newline at end of file
diff --git a/js/index.ts b/js/index.ts
index 45c803f8a5b..51b8676abbd 100644
--- a/js/index.ts
+++ b/js/index.ts
@@ -1 +1,18 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 export * from './src/Arrow';
\ No newline at end of file

From 0be6de3f1443df974d91be9c3fa927600b8b19c7 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sat, 12 May 2018 22:54:17 -0700
Subject: [PATCH 32/51] use node v10.1.0 in travis

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 68ab478652e..93d4605d492 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -136,7 +136,7 @@ matrix:
     - if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi
     - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
-    - nvm install 9.8
+    - nvm install v10.1.0
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:

From c0b88c2501243f82ce2881e4659dda0f4ec72450 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sun, 13 May 2018 15:23:19 -0700
Subject: [PATCH 33/51] always write flatbuffer vectors

---
 js/src/ipc/writer/binary.ts | 67 ++++++++++++++++++-------------------
 1 file changed, 32 insertions(+), 35 deletions(-)

diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
index 699819d63a2..118b1de1fca 100644
--- a/js/src/ipc/writer/binary.ts
+++ b/js/src/ipc/writer/binary.ts
@@ -542,26 +542,24 @@ function concatBuffersWithMetadata(totalByteLength: number, buffers: Uint8Array[
 
 function writeFooter(b: Builder, node: Footer) {
     let schemaOffset = writeSchema(b, node.schema);
-    let recordBatchesOffset: number | undefined = undefined;
-    let dictionaryBatchesOffset: number | undefined = undefined;
-    if (node.recordBatches && node.recordBatches.length) {
-        recordBatchesOffset =
-            _Footer.startRecordBatchesVector(b, node.recordBatches.length) ||
-            mapReverse(node.recordBatches, (rb) => writeBlock(b, rb)) &&
-            b.endVector();
-    }
-    if (node.dictionaryBatches && node.dictionaryBatches.length) {
-        dictionaryBatchesOffset =
-            _Footer.startDictionariesVector(b, node.dictionaryBatches.length) ||
-            mapReverse(node.dictionaryBatches, (db) => writeBlock(b, db)) &&
-            b.endVector();
-    }
+    let recordBatches = (node.recordBatches || []);
+    let dictionaryBatches = (node.dictionaryBatches || []);
+    let recordBatchesOffset =
+        _Footer.startRecordBatchesVector(b, recordBatches.length) ||
+            mapReverse(recordBatches, (rb) => writeBlock(b, rb)) &&
+        b.endVector();
+
+    let dictionaryBatchesOffset =
+        _Footer.startDictionariesVector(b, dictionaryBatches.length) ||
+            mapReverse(dictionaryBatches, (db) => writeBlock(b, db)) &&
+        b.endVector();
+
     return (
         _Footer.startFooter(b) ||
         _Footer.addSchema(b, schemaOffset) ||
         _Footer.addVersion(b, node.schema.version) ||
-        (recordBatchesOffset !== undefined && _Footer.addRecordBatches(b, recordBatchesOffset)) ||
-        (dictionaryBatchesOffset !== undefined && _Footer.addDictionaries(b, dictionaryBatchesOffset)) ||
+        _Footer.addRecordBatches(b, recordBatchesOffset) ||
+        _Footer.addDictionaries(b, dictionaryBatchesOffset) ||
         _Footer.endFooter(b)
     );
 }
@@ -607,25 +605,23 @@ function writeSchema(b: Builder, node: Schema) {
 }
 
 function writeRecordBatch(b: Builder, node: RecordBatchMetadata) {
-    let nodesOffset: number | undefined = undefined;
-    let buffersOffset: number | undefined = undefined;
-    if (node.nodes && node.nodes.length) {
-        nodesOffset =
-            _RecordBatch.startNodesVector(b, node.nodes.length) ||
-            mapReverse(node.nodes, (n) => writeFieldNode(b, n)) &&
-            b.endVector();
-    }
-    if (node.buffers && node.buffers.length) {
-        buffersOffset =
-            _RecordBatch.startBuffersVector(b, node.buffers.length) ||
-            mapReverse(node.buffers, (b_) => writeBuffer(b, b_)) &&
-            b.endVector();
-    }
+    let nodes = (node.nodes || []);
+    let buffers = (node.buffers || []);
+    let nodesOffset =
+        _RecordBatch.startNodesVector(b, nodes.length) ||
+        mapReverse(nodes, (n) => writeFieldNode(b, n)) &&
+        b.endVector();
+
+    let buffersOffset =
+        _RecordBatch.startBuffersVector(b, buffers.length) ||
+        mapReverse(buffers, (b_) => writeBuffer(b, b_)) &&
+        b.endVector();
+
     return (
         _RecordBatch.startRecordBatch(b) ||
         _RecordBatch.addLength(b, new Long(node.length, 0)) ||
-        (nodesOffset !== undefined && _RecordBatch.addNodes(b, nodesOffset)) ||
-        (buffersOffset !== undefined && _RecordBatch.addBuffers(b, buffersOffset)) ||
+        _RecordBatch.addNodes(b, nodesOffset) ||
+        _RecordBatch.addBuffers(b, buffersOffset) ||
         _RecordBatch.endRecordBatch(b)
     );
 }
@@ -654,9 +650,9 @@ function writeField(b: Builder, node: Field) {
     let type = node.type;
     let typeId = node.typeId;
     let name: number | undefined = undefined;
-    let children: number | undefined = undefined;
     let metadata: number | undefined = undefined;
     let dictionary: number | undefined = undefined;
+
     if (!DataType.isDictionary(type)) {
         typeOffset = new TypeSerializer(b).visit(type);
     } else {
@@ -664,7 +660,8 @@ function writeField(b: Builder, node: Field) {
         dictionary = new TypeSerializer(b).visit(type);
         typeOffset = new TypeSerializer(b).visit(type.dictionary);
     }
-    children = _Field.createChildrenVector(b, (type.children || []).map((f) => writeField(b, f)));
+
+    let children = _Field.createChildrenVector(b, (type.children || []).map((f) => writeField(b, f)));
     if (node.metadata && node.metadata.size > 0) {
         metadata = _Field.createCustomMetadataVector(
             b,
@@ -687,9 +684,9 @@ function writeField(b: Builder, node: Field) {
         _Field.startField(b) ||
         _Field.addType(b, typeOffset) ||
         _Field.addTypeType(b, typeId) ||
+        _Field.addChildren(b, children) ||
         _Field.addNullable(b, !!node.nullable) ||
         (name !== undefined && _Field.addName(b, name)) ||
-        (children !== undefined && _Field.addChildren(b, children)) ||
         (dictionary !== undefined && _Field.addDictionary(b, dictionary)) ||
         (metadata !== undefined && _Field.addCustomMetadata(b, metadata)) ||
         _Field.endField(b)

From ccaf489e7d4092eb2522bff6894156317c407de8 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sun, 13 May 2018 16:28:47 -0700
Subject: [PATCH 34/51] remove stream-to-iterator

---
 js/src/bin/arrow2csv.ts | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/js/src/bin/arrow2csv.ts b/js/src/bin/arrow2csv.ts
index b39fda48d80..43e2f51508a 100644
--- a/js/src/bin/arrow2csv.ts
+++ b/js/src/bin/arrow2csv.ts
@@ -30,15 +30,18 @@ const files = [...(argv.file || []), ...(argv._unknown || [])].filter(Boolean);
 
 (async () => {
     if (!files.length) {
-        let buffer = Buffer.from([]);
-        for await (let chunk of require('stream-to-iterator')(process.stdin as any)) {
-            chunk = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, 'binary');
-            buffer = Buffer.concat([buffer, chunk], buffer.length + chunk.length);
+        let buffers = [], totalLength = 0;
+        for await (let chunk of (process.stdin as any)) {
+            if (!Buffer.isBuffer(chunk)) {
+                chunk = Buffer.from(chunk, 'binary');
+            }
+            buffers.push(chunk);
+            totalLength += chunk.byteLength;
         }
-        if (buffer.length === 0) {
+        if (buffers.length === 0) {
             return print_usage();
         }
-        files.push(buffer);
+        files.push(Buffer.concat(buffers, totalLength));
     }
     for (let input of files) {
         printTable(typeof input === 'string' ? await readFile(input) : input);

From 081fefc5e5e58757fcb773837c285c2c48689fd5 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sun, 13 May 2018 16:28:58 -0700
Subject: [PATCH 35/51] remove bin from ts package.json

---
 js/gulp/package-task.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js
index 98812b379b5..8c0f8fb0e47 100644
--- a/js/gulp/package-task.js
+++ b/js/gulp/package-task.js
@@ -44,6 +44,7 @@ module.exports.packageTask = packageTask;
 
 const createMainPackageJson = (target, format) => (orig) => ({
     ...createTypeScriptPackageJson(target, format)(orig),
+    bin: orig.bin,
     name: npmPkgName,
     main: mainExport,
     types: `${mainExport}.d.ts`,
@@ -55,6 +56,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
 const createTypeScriptPackageJson = (target, format) => (orig) => ({
     ...createScopedPackageJSON(target, format)(orig),
     main: `${mainExport}.ts`, types: `${mainExport}.ts`,
+    bin: undefined,
     dependencies: {
         '@types/flatbuffers': '*',
         '@types/node': '*',

From a79334df15c80782aaf3e309d4cf02a550ace629 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Sun, 13 May 2018 16:31:14 -0700
Subject: [PATCH 36/51] fix typo again after rebase

---
 js/src/vector.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/js/src/vector.ts b/js/src/vector.ts
index 57101b59b05..40d8faa2a94 100644
--- a/js/src/vector.ts
+++ b/js/src/vector.ts
@@ -414,7 +414,7 @@ export class DictionaryVector<T extends DataType = DataType> extends Vector<Dict
             view = (view as any).view;
         }
         if (data instanceof DictionaryData && view instanceof DictionaryView) {
-            this.indices = view.indicies;
+            this.indices = view.indices;
             this.dictionary = data.dictionary;
         } else if (data instanceof ChunkedData && view instanceof ChunkedView) {
             const chunks = view.chunkVectors as DictionaryVector<T>[];

From a6a7ab926c6dfd8c127b217a09739acd4f9760f3 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Mon, 14 May 2018 14:46:28 -0700
Subject: [PATCH 37/51] put test tables into hoisted functions so it's easier
 to set breakpoints

---
 js/test/unit/table-tests.ts | 774 ++++++++++++++++++------------------
 1 file changed, 395 insertions(+), 379 deletions(-)

diff --git a/js/test/unit/table-tests.ts b/js/test/unit/table-tests.ts
index d4dc75fb7a1..32b15fa1154 100644
--- a/js/test/unit/table-tests.ts
+++ b/js/test/unit/table-tests.ts
@@ -23,367 +23,47 @@ const { col, lit, custom } = predicate;
 
 const F32 = 0, I32 = 1, DICT = 2;
 const test_data = [
-    {name: `single record batch`,
-     table: () => Table.from({
-          'schema': {
-            'fields': [
-              {
-                'name': 'f32',
-                'type': {
-                  'name': 'floatingpoint',
-                  'precision': 'SINGLE'
-                },
-                'nullable': false,
-                'children': [],
-              },
-              {
-                'name': 'i32',
-                'type': {
-                  'name': 'int',
-                  'isSigned': true,
-                  'bitWidth': 32
-                },
-                'nullable': false,
-                'children': [],
-              },
-              {
-                'name': 'dictionary',
-                'type': {
-                  'name': 'utf8'
-                },
-                'nullable': false,
-                'children': [],
-                'dictionary': {
-                  'id': 0,
-                  'indexType': {
-                    'name': 'int',
-                    'isSigned': true,
-                    'bitWidth': 8
-                  },
-                  'isOrdered': false
-                }
-              }
-            ]
-          },
-          'dictionaries': [{
-            'id': 0,
-            'data': {
-              'count': 3,
-              'columns': [
-                {
-                  'name': 'DICT0',
-                  'count': 3,
-                  'VALIDITY': [],
-                  'OFFSET': [
-                    0,
-                    1,
-                    2,
-                    3
-                  ],
-                  'DATA': [
-                    'a',
-                    'b',
-                    'c',
-                  ]
-                }
-              ]
-            }
-          }],
-          'batches': [{
-            'count': 7,
-            'columns': [
-              {
-                'name': 'f32',
-                'count': 7,
-                'VALIDITY': [],
-                'DATA': [-0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3]
-              },
-              {
-                'name': 'i32',
-                'count': 7,
-                'VALIDITY': [],
-                'DATA': [-1, 1, -1, 1, -1, 1, -1]
-              },
-              {
-                'name': 'dictionary',
-                'count': 7,
-                'VALIDITY': [],
-                'DATA': [0, 1, 2, 0, 1, 2, 0]
-              }
-            ]
-          }]
-        }),
+    {
+        name: `single record batch`,
+        table: getSingleRecordBatchTable,
         // Use Math.fround to coerce to float32
-     values: () => [
-         [Math.fround(-0.3), -1, 'a'],
-         [Math.fround(-0.2),  1, 'b'],
-         [Math.fround(-0.1), -1, 'c'],
-         [Math.fround( 0  ),  1, 'a'],
-         [Math.fround( 0.1), -1, 'b'],
-         [Math.fround( 0.2),  1, 'c'],
-         [Math.fround( 0.3), -1, 'a']
-     ]},
-     {name: `multiple record batches`,
-      table: () => Table.from({
-          'schema': {
-            'fields': [
-              {
-                'name': 'f32',
-                'type': {
-                  'name': 'floatingpoint',
-                  'precision': 'SINGLE'
-                },
-                'nullable': false,
-                'children': [],
-              },
-              {
-                'name': 'i32',
-                'type': {
-                  'name': 'int',
-                  'isSigned': true,
-                  'bitWidth': 32
-                },
-                'nullable': false,
-                'children': [],
-              },
-              {
-                'name': 'dictionary',
-                'type': {
-                  'name': 'utf8'
-                },
-                'nullable': false,
-                'children': [],
-                'dictionary': {
-                  'id': 0,
-                  'indexType': {
-                    'name': 'int',
-                    'isSigned': true,
-                    'bitWidth': 8
-                  },
-                  'isOrdered': false
-                }
-              }
-            ]
-          },
-          'dictionaries': [{
-            'id': 0,
-            'data': {
-              'count': 3,
-              'columns': [
-                {
-                  'name': 'DICT0',
-                  'count': 3,
-                  'VALIDITY': [],
-                  'OFFSET': [
-                    0,
-                    1,
-                    2,
-                    3
-                  ],
-                  'DATA': [
-                    'a',
-                    'b',
-                    'c',
-                  ]
-                }
-              ]
-            }
-          }],
-          'batches': [{
-            'count': 3,
-            'columns': [
-              {
-                'name': 'f32',
-                'count': 3,
-                'VALIDITY': [],
-                'DATA': [-0.3, -0.2, -0.1]
-              },
-              {
-                'name': 'i32',
-                'count': 3,
-                'VALIDITY': [],
-                'DATA': [-1, 1, -1]
-              },
-              {
-                'name': 'dictionary',
-                'count': 3,
-                'VALIDITY': [],
-                'DATA': [0, 1, 2]
-              }
-            ]
-          }, {
-            'count': 3,
-            'columns': [
-              {
-                'name': 'f32',
-                'count': 3,
-                'VALIDITY': [],
-                'DATA': [0, 0.1, 0.2]
-              },
-              {
-                'name': 'i32',
-                'count': 3,
-                'VALIDITY': [],
-                'DATA': [1, -1, 1]
-              },
-              {
-                'name': 'dictionary',
-                'count': 3,
-                'VALIDITY': [],
-                'DATA': [0, 1, 2]
-              }
-            ]
-          }, {
-            'count': 3,
-            'columns': [
-              {
-                'name': 'f32',
-                'count': 3,
-                'VALIDITY': [],
-                'DATA': [0.3, 0.2, 0.1]
-              },
-              {
-                'name': 'i32',
-                'count': 3,
-                'VALIDITY': [],
-                'DATA': [-1, 1, -1]
-              },
-              {
-                'name': 'dictionary',
-                'count': 3,
-                'VALIDITY': [],
-                'DATA': [0, 1, 2]
-              }
-            ]
-          }]
-      }),
-      values: () => [
+        values: () => [
             [Math.fround(-0.3), -1, 'a'],
-            [Math.fround(-0.2),  1, 'b'],
+            [Math.fround(-0.2), 1, 'b'],
             [Math.fround(-0.1), -1, 'c'],
-            [Math.fround( 0  ),  1, 'a'],
-            [Math.fround( 0.1), -1, 'b'],
-            [Math.fround( 0.2),  1, 'c'],
-            [Math.fround( 0.3), -1, 'a'],
-            [Math.fround( 0.2),  1, 'b'],
-            [Math.fround( 0.1), -1, 'c'],
-      ]},
-    {name: `struct`,
-     table: () => Table.fromStruct(Table.from({
-          'schema': {
-            'fields': [
-              {
-                'name': 'struct',
-                'type': {
-                  'name': 'struct'
-                },
-                'nullable': false,
-                'children': [
-                  {
-                    'name': 'f32',
-                    'type': {
-                      'name': 'floatingpoint',
-                      'precision': 'SINGLE'
-                    },
-                    'nullable': false,
-                    'children': [],
-                  },
-                  {
-                    'name': 'i32',
-                    'type': {
-                      'name': 'int',
-                      'isSigned': true,
-                      'bitWidth': 32
-                    },
-                    'nullable': false,
-                    'children': [],
-                  },
-                  {
-                    'name': 'dictionary',
-                    'type': {
-                      'name': 'utf8'
-                    },
-                    'nullable': false,
-                    'children': [],
-                    'dictionary': {
-                      'id': 0,
-                      'indexType': {
-                        'name': 'int',
-                        'isSigned': true,
-                        'bitWidth': 8
-                      },
-                      'isOrdered': false
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          'dictionaries': [{
-            'id': 0,
-            'data': {
-              'count': 3,
-              'columns': [
-                {
-                  'name': 'DICT0',
-                  'count': 3,
-                  'VALIDITY': [],
-                  'OFFSET': [
-                    0,
-                    1,
-                    2,
-                    3
-                  ],
-                  'DATA': [
-                    'a',
-                    'b',
-                    'c',
-                  ]
-                }
-              ]
-            }
-          }],
-          'batches': [{
-            'count': 7,
-            'columns': [
-              {
-                'name': 'struct',
-                'count': 7,
-                'VALIDITY': [],
-                'children': [
-                  {
-                    'name': 'f32',
-                    'count': 7,
-                    'VALIDITY': [],
-                    'DATA': [-0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3]
-                  },
-                  {
-                    'name': 'i32',
-                    'count': 7,
-                    'VALIDITY': [],
-                    'DATA': [-1, 1, -1, 1, -1, 1, -1]
-                  },
-                  {
-                    'name': 'dictionary',
-                    'count': 7,
-                    'VALIDITY': [],
-                    'DATA': [0, 1, 2, 0, 1, 2, 0]
-                  }
-                ]
-              }
-            ]
-          }]
-        }).getColumn('struct') as vector.StructVector),
+            [Math.fround(0), 1, 'a'],
+            [Math.fround(0.1), -1, 'b'],
+            [Math.fround(0.2), 1, 'c'],
+            [Math.fround(0.3), -1, 'a']
+        ]
+    }, {
+        name: `multiple record batches`,
+        table: getMultipleRecordBatchesTable,
+        values: () => [
+            [Math.fround(-0.3), -1, 'a'],
+            [Math.fround(-0.2), 1, 'b'],
+            [Math.fround(-0.1), -1, 'c'],
+            [Math.fround(0), 1, 'a'],
+            [Math.fround(0.1), -1, 'b'],
+            [Math.fround(0.2), 1, 'c'],
+            [Math.fround(0.3), -1, 'a'],
+            [Math.fround(0.2), 1, 'b'],
+            [Math.fround(0.1), -1, 'c'],
+        ]
+    }, {
+        name: `struct`,
+        table: () => Table.fromStruct(getStructTable().getColumn('struct') as vector.StructVector),
         // Use Math.fround to coerce to float32
-     values: () => [
-         [Math.fround(-0.3), -1, 'a'],
-         [Math.fround(-0.2),  1, 'b'],
-         [Math.fround(-0.1), -1, 'c'],
-         [Math.fround( 0  ),  1, 'a'],
-         [Math.fround( 0.1), -1, 'b'],
-         [Math.fround( 0.2),  1, 'c'],
-         [Math.fround( 0.3), -1, 'a']
-     ]},
+        values: () => [
+            [Math.fround(-0.3), -1, 'a'],
+            [Math.fround(-0.2), 1, 'b'],
+            [Math.fround(-0.1), -1, 'c'],
+            [Math.fround(0), 1, 'a'],
+            [Math.fround(0.1), -1, 'b'],
+            [Math.fround(0.2), 1, 'c'],
+            [Math.fround(0.3), -1, 'a']
+        ]
+    },
 ];
 
 describe(`Table`, () => {
@@ -425,7 +105,7 @@ describe(`Table`, () => {
                 });
                 test(`calls bind function with every batch`, () => {
                     let bind = jest.fn();
-                    table.scan(() => {}, bind);
+                    table.scan(() => { }, bind);
                     for (let batch of table.batches) {
                         expect(bind).toHaveBeenCalledWith(batch);
                     }
@@ -442,63 +122,63 @@ describe(`Table`, () => {
             let get_i32: (idx: number) => number, get_f32: (idx: number) => number;
             const filter_tests = [
                 {
-                    name:     `filter on f32 >= 0`,
+                    name: `filter on f32 >= 0`,
                     filtered: table.filter(col('f32').ge(0)),
                     expected: values.filter((row) => row[F32] >= 0)
                 }, {
-                    name:     `filter on 0 <= f32`,
+                    name: `filter on 0 <= f32`,
                     filtered: table.filter(lit(0).le(col('f32'))),
                     expected: values.filter((row) => 0 <= row[F32])
                 }, {
-                    name:     `filter on i32 <= 0`,
+                    name: `filter on i32 <= 0`,
                     filtered: table.filter(col('i32').le(0)),
                     expected: values.filter((row) => row[I32] <= 0)
                 }, {
-                    name:     `filter on 0 >= i32`,
+                    name: `filter on 0 >= i32`,
                     filtered: table.filter(lit(0).ge(col('i32'))),
                     expected: values.filter((row) => 0 >= row[I32])
                 }, {
-                    name:     `filter on f32 < 0`,
+                    name: `filter on f32 < 0`,
                     filtered: table.filter(col('f32').lt(0)),
                     expected: values.filter((row) => row[F32] < 0)
                 }, {
-                    name:     `filter on i32 > 1 (empty)`,
+                    name: `filter on i32 > 1 (empty)`,
                     filtered: table.filter(col('i32').gt(0)),
                     expected: values.filter((row) => row[I32] > 0)
                 }, {
-                    name:     `filter on f32 <= -.25 || f3 >= .25`,
+                    name: `filter on f32 <= -.25 || f3 >= .25`,
                     filtered: table.filter(col('f32').le(-.25).or(col('f32').ge(.25))),
                     expected: values.filter((row) => row[F32] <= -.25 || row[F32] >= .25)
                 }, {
-                    name:     `filter on !(f32 <= -.25 || f3 >= .25) (not)`,
+                    name: `filter on !(f32 <= -.25 || f3 >= .25) (not)`,
                     filtered: table.filter(col('f32').le(-.25).or(col('f32').ge(.25)).not()),
                     expected: values.filter((row) => !(row[F32] <= -.25 || row[F32] >= .25))
                 }, {
-                    name:     `filter method combines predicates (f32 >= 0 && i32 <= 0)`,
+                    name: `filter method combines predicates (f32 >= 0 && i32 <= 0)`,
                     filtered: table.filter(col('i32').le(0)).filter(col('f32').ge(0)),
                     expected: values.filter((row) => row[I32] <= 0 && row[F32] >= 0)
                 }, {
-                    name:     `filter on dictionary == 'a'`,
+                    name: `filter on dictionary == 'a'`,
                     filtered: table.filter(col('dictionary').eq('a')),
                     expected: values.filter((row) => row[DICT] === 'a')
                 }, {
-                    name:     `filter on 'a' == dictionary (commutativity)`,
+                    name: `filter on 'a' == dictionary (commutativity)`,
                     filtered: table.filter(lit('a').eq(col('dictionary'))),
                     expected: values.filter((row) => row[DICT] === 'a')
                 }, {
-                    name:     `filter on dictionary != 'b'`,
+                    name: `filter on dictionary != 'b'`,
                     filtered: table.filter(col('dictionary').ne('b')),
                     expected: values.filter((row) => row[DICT] !== 'b')
                 }, {
-                    name:     `filter on f32 >= i32`,
+                    name: `filter on f32 >= i32`,
                     filtered: table.filter(col('f32').ge(col('i32'))),
                     expected: values.filter((row) => row[F32] >= row[I32])
                 }, {
-                    name:     `filter on f32 <= i32`,
+                    name: `filter on f32 <= i32`,
                     filtered: table.filter(col('f32').le(col('i32'))),
                     expected: values.filter((row) => row[F32] <= row[I32])
                 }, {
-                    name:     `filter on f32*i32 > 0 (custom predicate)`,
+                    name: `filter on f32*i32 > 0 (custom predicate)`,
                     filtered: table.filter(custom(
                         (idx: number) => (get_f32(idx) * get_i32(idx) > 0),
                         (batch: RecordBatch) => {
@@ -528,7 +208,7 @@ describe(`Table`, () => {
                             // this test may fail in the future if we change
                             // that - and that's ok!
                             let bind = jest.fn();
-                            filtered.scan(() => {}, bind);
+                            filtered.scan(() => { }, bind);
                             for (let batch of table.batches) {
                                 expect(bind).toHaveBeenCalledWith(batch);
                             }
@@ -539,7 +219,7 @@ describe(`Table`, () => {
             test(`countBy on dictionary returns the correct counts`, () => {
                 // Make sure countBy works both with and without the Col wrapper
                 // class
-                let expected: {[key: string]: number} = {'a': 0, 'b': 0, 'c': 0};
+                let expected: { [key: string]: number } = { 'a': 0, 'b': 0, 'c': 0 };
                 for (let row of values) {
                     expected[row[DICT]] += 1;
                 }
@@ -548,7 +228,7 @@ describe(`Table`, () => {
                 expect(table.countBy('dictionary').toJSON()).toEqual(expected);
             });
             test(`countBy on dictionary with filter returns the correct counts`, () => {
-                let expected: {[key: string]: number} = {'a': 0, 'b': 0, 'c': 0};
+                let expected: { [key: string]: number } = { 'a': 0, 'b': 0, 'c': 0 };
                 for (let row of values) {
                     if (row[I32] === 1) { expected[row[DICT]] += 1; }
                 }
@@ -578,11 +258,11 @@ describe(`Table`, () => {
             });
             test(`table.toString()`, () => {
                 let selected = table.select('i32', 'dictionary');
-                let headers  = [`"row_id"`, `"i32: Int32"`, `"dictionary: Dictionary<Int8, Utf8>"`];
+                let headers = [`"row_id"`, `"i32: Int32"`, `"dictionary: Dictionary<Int8, Utf8>"`];
                 let expected = [headers.join(' | '), ...values.map((row, idx) => {
                     return [`${idx}`, `${row[I32]}`, `"${row[DICT]}"`].map((str, col) => {
-                                return leftPad(str, ' ', headers[col].length);
-                            }).join(' | ');
+                        return leftPad(str, ' ', headers[col].length);
+                    }).join(' | ');
                 })].join('\n') + '\n';
                 expect(selected.toString()).toEqual(expected);
             });
@@ -615,3 +295,339 @@ describe(`Table`, () => {
 function leftPad(str: string, fill: string, n: number) {
     return (new Array(n + 1).join(fill) + str).slice(-1 * n);
 }
+
+function getSingleRecordBatchTable() {
+    return Table.from({
+        'schema': {
+            'fields': [
+                {
+                    'name': 'f32',
+                    'type': {
+                        'name': 'floatingpoint',
+                        'precision': 'SINGLE'
+                    },
+                    'nullable': false,
+                    'children': [],
+                },
+                {
+                    'name': 'i32',
+                    'type': {
+                        'name': 'int',
+                        'isSigned': true,
+                        'bitWidth': 32
+                    },
+                    'nullable': false,
+                    'children': [],
+                },
+                {
+                    'name': 'dictionary',
+                    'type': {
+                        'name': 'utf8'
+                    },
+                    'nullable': false,
+                    'children': [],
+                    'dictionary': {
+                        'id': 0,
+                        'indexType': {
+                            'name': 'int',
+                            'isSigned': true,
+                            'bitWidth': 8
+                        },
+                        'isOrdered': false
+                    }
+                }
+            ]
+        },
+        'dictionaries': [{
+            'id': 0,
+            'data': {
+                'count': 3,
+                'columns': [
+                    {
+                        'name': 'DICT0',
+                        'count': 3,
+                        'VALIDITY': [],
+                        'OFFSET': [
+                            0,
+                            1,
+                            2,
+                            3
+                        ],
+                        'DATA': [
+                            'a',
+                            'b',
+                            'c',
+                        ]
+                    }
+                ]
+            }
+        }],
+        'batches': [{
+            'count': 7,
+            'columns': [
+                {
+                    'name': 'f32',
+                    'count': 7,
+                    'VALIDITY': [],
+                    'DATA': [-0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3]
+                },
+                {
+                    'name': 'i32',
+                    'count': 7,
+                    'VALIDITY': [],
+                    'DATA': [-1, 1, -1, 1, -1, 1, -1]
+                },
+                {
+                    'name': 'dictionary',
+                    'count': 7,
+                    'VALIDITY': [],
+                    'DATA': [0, 1, 2, 0, 1, 2, 0]
+                }
+            ]
+        }]
+    });
+}
+
+function getMultipleRecordBatchesTable() {
+    return Table.from({
+        'schema': {
+            'fields': [
+                {
+                    'name': 'f32',
+                    'type': {
+                        'name': 'floatingpoint',
+                        'precision': 'SINGLE'
+                    },
+                    'nullable': false,
+                    'children': [],
+                },
+                {
+                    'name': 'i32',
+                    'type': {
+                        'name': 'int',
+                        'isSigned': true,
+                        'bitWidth': 32
+                    },
+                    'nullable': false,
+                    'children': [],
+                },
+                {
+                    'name': 'dictionary',
+                    'type': {
+                        'name': 'utf8'
+                    },
+                    'nullable': false,
+                    'children': [],
+                    'dictionary': {
+                        'id': 0,
+                        'indexType': {
+                            'name': 'int',
+                            'isSigned': true,
+                            'bitWidth': 8
+                        },
+                        'isOrdered': false
+                    }
+                }
+            ]
+        },
+        'dictionaries': [{
+            'id': 0,
+            'data': {
+                'count': 3,
+                'columns': [
+                    {
+                        'name': 'DICT0',
+                        'count': 3,
+                        'VALIDITY': [],
+                        'OFFSET': [
+                            0,
+                            1,
+                            2,
+                            3
+                        ],
+                        'DATA': [
+                            'a',
+                            'b',
+                            'c',
+                        ]
+                    }
+                ]
+            }
+        }],
+        'batches': [{
+            'count': 3,
+            'columns': [
+                {
+                    'name': 'f32',
+                    'count': 3,
+                    'VALIDITY': [],
+                    'DATA': [-0.3, -0.2, -0.1]
+                },
+                {
+                    'name': 'i32',
+                    'count': 3,
+                    'VALIDITY': [],
+                    'DATA': [-1, 1, -1]
+                },
+                {
+                    'name': 'dictionary',
+                    'count': 3,
+                    'VALIDITY': [],
+                    'DATA': [0, 1, 2]
+                }
+            ]
+        }, {
+            'count': 3,
+            'columns': [
+                {
+                    'name': 'f32',
+                    'count': 3,
+                    'VALIDITY': [],
+                    'DATA': [0, 0.1, 0.2]
+                },
+                {
+                    'name': 'i32',
+                    'count': 3,
+                    'VALIDITY': [],
+                    'DATA': [1, -1, 1]
+                },
+                {
+                    'name': 'dictionary',
+                    'count': 3,
+                    'VALIDITY': [],
+                    'DATA': [0, 1, 2]
+                }
+            ]
+        }, {
+            'count': 3,
+            'columns': [
+                {
+                    'name': 'f32',
+                    'count': 3,
+                    'VALIDITY': [],
+                    'DATA': [0.3, 0.2, 0.1]
+                },
+                {
+                    'name': 'i32',
+                    'count': 3,
+                    'VALIDITY': [],
+                    'DATA': [-1, 1, -1]
+                },
+                {
+                    'name': 'dictionary',
+                    'count': 3,
+                    'VALIDITY': [],
+                    'DATA': [0, 1, 2]
+                }
+            ]
+        }]
+    });
+}
+
+function getStructTable() {
+    return Table.from({
+        'schema': {
+            'fields': [
+                {
+                    'name': 'struct',
+                    'type': {
+                        'name': 'struct'
+                    },
+                    'nullable': false,
+                    'children': [
+                        {
+                            'name': 'f32',
+                            'type': {
+                                'name': 'floatingpoint',
+                                'precision': 'SINGLE'
+                            },
+                            'nullable': false,
+                            'children': [],
+                        },
+                        {
+                            'name': 'i32',
+                            'type': {
+                                'name': 'int',
+                                'isSigned': true,
+                                'bitWidth': 32
+                            },
+                            'nullable': false,
+                            'children': [],
+                        },
+                        {
+                            'name': 'dictionary',
+                            'type': {
+                                'name': 'utf8'
+                            },
+                            'nullable': false,
+                            'children': [],
+                            'dictionary': {
+                                'id': 0,
+                                'indexType': {
+                                    'name': 'int',
+                                    'isSigned': true,
+                                    'bitWidth': 8
+                                },
+                                'isOrdered': false
+                            }
+                        }
+                    ]
+                }
+            ]
+        },
+        'dictionaries': [{
+            'id': 0,
+            'data': {
+                'count': 3,
+                'columns': [
+                    {
+                        'name': 'DICT0',
+                        'count': 3,
+                        'VALIDITY': [],
+                        'OFFSET': [
+                            0,
+                            1,
+                            2,
+                            3
+                        ],
+                        'DATA': [
+                            'a',
+                            'b',
+                            'c',
+                        ]
+                    }
+                ]
+            }
+        }],
+        'batches': [{
+            'count': 7,
+            'columns': [
+                {
+                    'name': 'struct',
+                    'count': 7,
+                    'VALIDITY': [],
+                    'children': [
+                        {
+                            'name': 'f32',
+                            'count': 7,
+                            'VALIDITY': [],
+                            'DATA': [-0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3]
+                        },
+                        {
+                            'name': 'i32',
+                            'count': 7,
+                            'VALIDITY': [],
+                            'DATA': [-1, 1, -1, 1, -1, 1, -1]
+                        },
+                        {
+                            'name': 'dictionary',
+                            'count': 7,
+                            'VALIDITY': [],
+                            'DATA': [0, 1, 2, 0, 1, 2, 0]
+                        }
+                    ]
+                }
+            ]
+        }]
+    });
+}
\ No newline at end of file

From 2df1a4ad1ce6b9d58552b00c5540286a46023d76 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Mon, 14 May 2018 14:50:59 -0700
Subject: [PATCH 38/51] update google-closure-compiler, remove gcc-specific
 workarounds in the build

---
 js/DEVELOP.md                  |  194 +--
 js/gulp/closure-task.js        |   26 -
 js/gulp/typescript-task.js     |   16 -
 js/package.json                |    5 +-
 js/src/Arrow.externs.js        |    4 +
 js/src/fb/File_generated.js    |  256 ----
 js/src/fb/Message_generated.js |  497 -------
 js/src/fb/Schema_generated.js  | 2231 --------------------------------
 8 files changed, 7 insertions(+), 3222 deletions(-)
 delete mode 100644 js/src/fb/File_generated.js
 delete mode 100644 js/src/fb/Message_generated.js
 delete mode 100644 js/src/fb/Schema_generated.js

diff --git a/js/DEVELOP.md b/js/DEVELOP.md
index c7ffd5c46d0..17eb8e1d00f 100644
--- a/js/DEVELOP.md
+++ b/js/DEVELOP.md
@@ -68,9 +68,7 @@ Uses [lerna](https://github.com/lerna/lerna) to publish each build target to npm
 
 # Updating the Arrow format flatbuffers generated code
 
-Once generated, the flatbuffers format code needs to be adjusted for our TS and JS build environments.
-
-## TypeScript
+Once generated, the flatbuffers format code needs to be adjusted for our build scripts.
 
 1. Generate the flatbuffers TypeScript source from the Arrow project root directory:
     ```sh
@@ -101,193 +99,3 @@ Once generated, the flatbuffers format code needs to be adjusted for our TS and
     ```
 1. Add `/* tslint:disable:class-name */` to the top of `Schema.ts`
 1. Execute `npm run lint` to fix all the linting errors
-
-## JavaScript (for Google Closure Compiler builds)
-
-1. Generate the flatbuffers JS source from the Arrow project root directory
-    ```sh
-    cd $ARROW_HOME
-
-    flatc --js --no-js-exports -o ./js/src/format ./format/*.fbs
-
-    cd ./js/src/format
-
-    # Delete Tensor_generated.js (skip this when we support Tensors)
-    rm Tensor_generated.js
-
-    # append an ES6 export to Schema_generated.js
-    echo "$(cat Schema_generated.js)
-    export { org };
-    " > Schema_generated.js
-
-    # import Schema's "org" namespace and
-    # append an ES6 export to File_generated.js
-    echo "import { org } from './Schema';
-    $(cat File_generated.js)
-    export { org };
-    " > File_generated.js
-
-    # import Schema's "org" namespace and
-    # append an ES6 export to Message_generated.js
-    echo "import { org } from './Schema';
-    $(cat Message_generated.js)
-    export { org };
-    " > Message_generated.js
-    ```
-1. Fixup the generated JS enums with the reverse value-to-key mappings to match TypeScript
-    `Message_generated.js`
-    ```js
-    // Replace this
-    org.apache.arrow.flatbuf.MessageHeader = {
-      NONE: 0,
-      Schema: 1,
-      DictionaryBatch: 2,
-      RecordBatch: 3,
-      Tensor: 4
-    };
-    // With this
-    org.apache.arrow.flatbuf.MessageHeader = {
-      NONE: 0, 0: 'NONE',
-      Schema: 1, 1: 'Schema',
-      DictionaryBatch: 2, 2: 'DictionaryBatch',
-      RecordBatch: 3, 3: 'RecordBatch',
-      Tensor: 4, 4: 'Tensor'
-    };
-    ```
-    `Schema_generated.js`
-    ```js
-    /**
-     * @enum
-     */
-    org.apache.arrow.flatbuf.MetadataVersion = {
-      /**
-       * 0.1.0
-       */
-      V1: 0, 0: 'V1',
-
-      /**
-       * 0.2.0
-       */
-      V2: 1, 1: 'V2',
-
-      /**
-       * 0.3.0 -> 0.7.1
-       */
-      V3: 2, 2: 'V3',
-
-      /**
-       * >= 0.8.0
-       */
-      V4: 3, 3: 'V4'
-    };
-
-    /**
-     * @enum
-     */
-    org.apache.arrow.flatbuf.UnionMode = {
-      Sparse: 0, 0: 'Sparse',
-      Dense: 1, 1: 'Dense',
-    };
-
-    /**
-     * @enum
-     */
-    org.apache.arrow.flatbuf.Precision = {
-      HALF: 0, 0: 'HALF',
-      SINGLE: 1, 1: 'SINGLE',
-      DOUBLE: 2, 2: 'DOUBLE',
-    };
-
-    /**
-     * @enum
-     */
-    org.apache.arrow.flatbuf.DateUnit = {
-      DAY: 0, 0: 'DAY',
-      MILLISECOND: 1, 1: 'MILLISECOND',
-    };
-
-    /**
-     * @enum
-     */
-    org.apache.arrow.flatbuf.TimeUnit = {
-      SECOND: 0, 0: 'SECOND',
-      MILLISECOND: 1, 1: 'MILLISECOND',
-      MICROSECOND: 2, 2: 'MICROSECOND',
-      NANOSECOND: 3, 3: 'NANOSECOND',
-    };
-
-    /**
-     * @enum
-     */
-    org.apache.arrow.flatbuf.IntervalUnit = {
-      YEAR_MONTH: 0, 0: 'YEAR_MONTH',
-      DAY_TIME: 1, 1: 'DAY_TIME',
-    };
-
-    /**
-     * ----------------------------------------------------------------------
-     * Top-level Type value, enabling extensible type-specific metadata. We can
-     * add new logical types to Type without breaking backwards compatibility
-     *
-     * @enum
-     */
-    org.apache.arrow.flatbuf.Type = {
-      NONE: 0, 0: 'NONE',
-      Null: 1, 1: 'Null',
-      Int: 2, 2: 'Int',
-      FloatingPoint: 3, 3: 'FloatingPoint',
-      Binary: 4, 4: 'Binary',
-      Utf8: 5, 5: 'Utf8',
-      Bool: 6, 6: 'Bool',
-      Decimal: 7, 7: 'Decimal',
-      Date: 8, 8: 'Date',
-      Time: 9, 9: 'Time',
-      Timestamp: 10, 10: 'Timestamp',
-      Interval: 11, 11: 'Interval',
-      List: 12, 12: 'List',
-      Struct_: 13, 13: 'Struct_',
-      Union: 14, 14: 'Union',
-      FixedSizeBinary: 15, 15: 'FixedSizeBinary',
-      FixedSizeList: 16, 16: 'FixedSizeList',
-      Map: 17, 17: 'Map'
-    };
-
-    /**
-     * ----------------------------------------------------------------------
-     * The possible types of a vector
-     *
-     * @enum
-     */
-    org.apache.arrow.flatbuf.VectorType = {
-      /**
-       * used in List type, Dense Union and variable length primitive types (String, Binary)
-       */
-      OFFSET: 0, 0: 'OFFSET',
-
-      /**
-       * actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector
-       */
-      DATA: 1, 1: 'DATA',
-
-      /**
-       * Bit vector indicating if each value is null
-       */
-      VALIDITY: 2, 2: 'VALIDITY',
-
-      /**
-       * Type vector used in Union type
-       */
-      TYPE: 3, 3: 'TYPE'
-    };
-
-    /**
-     * ----------------------------------------------------------------------
-     * Endianness of the platform producing the data
-     *
-     * @enum
-     */
-    org.apache.arrow.flatbuf.Endianness = {
-      Little: 0, 0: 'Little',
-      Big: 1, 1: 'Big',
-    };
-    ```
diff --git a/js/gulp/closure-task.js b/js/gulp/closure-task.js
index 8833c2c2228..547e760a7fa 100644
--- a/js/gulp/closure-task.js
+++ b/js/gulp/closure-task.js
@@ -28,8 +28,6 @@ const path = require('path');
 const sourcemaps = require('gulp-sourcemaps');
 const { memoizeTask } = require('./memoize-task');
 const { compileBinFiles } = require('./typescript-task');
-const ASTBuilders = require('ast-types').builders;
-const transformAST = require('gulp-transform-js-ast');
 const { Observable, ReplaySubject } = require('rxjs');
 const closureCompiler = require('google-closure-compiler').gulp();
 
@@ -50,9 +48,6 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form
         ], { base: `./` }),
         sourcemaps.init(),
         closureCompiler(createClosureArgs(entry, externs)),
-        // Strip out closure compiler's error-throwing iterator-return methods
-        // see this issue: https://github.com/google/closure-compiler/issues/2728
-        transformAST(iteratorReturnVisitor),
         // rename the sourcemaps from *.js.map files to *.min.js.map
         sourcemaps.write(`.`, { mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`) }),
         gulp.dest(out)
@@ -105,24 +100,3 @@ const createClosureArgs = (entry, externs) => ({
 
 module.exports = closureTask;
 module.exports.closureTask = closureTask;
-
-const iteratorReturnVisitor = {
-    visitObjectExpression(p) {
-        const node = p.node, value = p.value;
-        if (!node.properties || !(node.properties.length === 3)) { return value; }
-        if (!propertyIsThrowingIteratorReturn(node.properties[2])) { return value; }
-        value.properties = value.properties.slice(0, 2);
-        return value;
-    }
-};
-
-function propertyIsThrowingIteratorReturn(p) {
-    if (!p || !(p.kind === 'init')) { return false; }
-    if (!p.key || !(p.key.type === 'Identifier') || !(p.key.name === 'return')) { return false; }
-    if (!p.value || !(p.value.type === 'FunctionExpression') || !p.value.params || !(p.value.params.length === 0)) { return false; }
-    if (!p.value.body || !p.value.body.body || !(p.value.body.body.length === 1) || !(p.value.body.body[0].type === 'ThrowStatement')) { return false; }
-    if (!p.value.body.body[0].argument || !(p.value.body.body[0].argument.type === 'CallExpression')) { return false; }
-    if (!p.value.body.body[0].argument.arguments || !(p.value.body.body[0].argument.arguments.length === 1)) { return false; }
-    if (!p.value.body.body[0].argument.arguments[0] || !(p.value.body.body[0].argument.arguments[0].type === 'Literal')) { return false; }
-    return p.value.body.body[0].argument.arguments[0].value === 'Not yet implemented';
-}
\ No newline at end of file
diff --git a/js/gulp/typescript-task.js b/js/gulp/typescript-task.js
index 0fdd1c73266..beffab8a08c 100644
--- a/js/gulp/typescript-task.js
+++ b/js/gulp/typescript-task.js
@@ -33,7 +33,6 @@ const typescriptTask = ((cache) => memoizeTask(cache, function typescript(target
     const tsconfigPath = path.join(`tsconfig`, `tsconfig.${tsconfigName(target, format)}.json`);
     return compileTypescript(out, tsconfigPath)
         .merge(compileBinFiles(target, format)).takeLast(1)
-        .concat(maybeCopyRawJSArrowFormatFiles(target, format))
         .publish(new ReplaySubject()).refCount();
 }))({});
 
@@ -57,18 +56,3 @@ function compileTypescript(out, tsconfigPath) {
 module.exports = typescriptTask;
 module.exports.typescriptTask = typescriptTask;
 module.exports.compileBinFiles = compileBinFiles;
-
-function maybeCopyRawJSArrowFormatFiles(target, format) {
-    if (target !== `es5` || format !== `cls`) {
-        return Observable.empty();
-    }
-    return Observable.defer(async () => {
-        const outFormatDir = path.join(targetDir(target, format), `fb`);
-        await del(path.join(outFormatDir, '*.js'));
-        await observableFromStreams(
-            gulp.src(path.join(`src`, `fb`, `*_generated.js`)),
-            gulpRename((p) => { p.basename = p.basename.replace(`_generated`, ``); }),
-            gulp.dest(outFormatDir)
-        ).toPromise();
-    });
-}
diff --git a/js/package.json b/js/package.json
index ba47cb96ed6..da75c257bd6 100644
--- a/js/package.json
+++ b/js/package.json
@@ -67,18 +67,16 @@
     "@std/esm": "0.26.0",
     "@types/glob": "5.0.35",
     "@types/jest": "22.2.3",
-    "ast-types": "0.10.1",
     "babel-jest": "22.4.3",
     "benchmark": "2.1.4",
     "coveralls": "3.0.0",
     "del": "3.0.0",
     "glob": "7.1.2",
-    "google-closure-compiler": "20180101.0.0",
+    "google-closure-compiler": "20180506.0.0",
     "gulp": "github:gulpjs/gulp#6d71a658c61edb3090221579d8f97dbe086ba2ed",
     "gulp-json-transform": "0.4.5",
     "gulp-rename": "1.2.2",
     "gulp-sourcemaps": "2.6.3",
-    "gulp-transform-js-ast": "1.0.2",
     "gulp-typescript": "3.2.4",
     "ix": "2.3.4",
     "jest": "22.4.3",
@@ -115,6 +113,7 @@
   },
   "jest": {
     "verbose": false,
+    "testEnvironment": "node",
     "globals": {
       "ts-jest": {
         "skipBabel": true,
diff --git a/js/src/Arrow.externs.js b/js/src/Arrow.externs.js
index b22ee55d56a..067a61ce1dd 100644
--- a/js/src/Arrow.externs.js
+++ b/js/src/Arrow.externs.js
@@ -238,6 +238,8 @@ Type.Int = function() {};
 /** @type {?} */
 Type.Float = function() {};
 /** @type {?} */
+Type.FloatingPoint = function() {};
+/** @type {?} */
 Type.Binary = function() {};
 /** @type {?} */
 Type.Utf8 = function() {};
@@ -258,6 +260,8 @@ Type.List = function() {};
 /** @type {?} */
 Type.Struct = function() {};
 /** @type {?} */
+Type.Struct_ = function() {};
+/** @type {?} */
 Type.Union = function() {};
 /** @type {?} */
 Type.FixedSizeBinary = function() {};
diff --git a/js/src/fb/File_generated.js b/js/src/fb/File_generated.js
deleted file mode 100644
index 12aae293ea4..00000000000
--- a/js/src/fb/File_generated.js
+++ /dev/null
@@ -1,256 +0,0 @@
-import { org } from './Schema';
-// automatically generated by the FlatBuffers compiler, do not modify
-
-/**
- * @const
- * @namespace
- */
-org.apache = org.apache || {};
-
-/**
- * @const
- * @namespace
- */
-org.apache.arrow = org.apache.arrow || {};
-
-/**
- * @const
- * @namespace
- */
-org.apache.arrow.flatbuf = org.apache.arrow.flatbuf || {};
-
-/**
- * ----------------------------------------------------------------------
- * Arrow File metadata
- *
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.Footer = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Footer}
- */
-org.apache.arrow.flatbuf.Footer.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Footer=} obj
- * @returns {org.apache.arrow.flatbuf.Footer}
- */
-org.apache.arrow.flatbuf.Footer.getRootAsFooter = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Footer).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @returns {org.apache.arrow.flatbuf.MetadataVersion}
- */
-org.apache.arrow.flatbuf.Footer.prototype.version = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? /** @type {org.apache.arrow.flatbuf.MetadataVersion} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.MetadataVersion.V1;
-};
-
-/**
- * @param {org.apache.arrow.flatbuf.Schema=} obj
- * @returns {org.apache.arrow.flatbuf.Schema|null}
- */
-org.apache.arrow.flatbuf.Footer.prototype.schema = function(obj) {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? (obj || new org.apache.arrow.flatbuf.Schema).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
-};
-
-/**
- * @param {number} index
- * @param {org.apache.arrow.flatbuf.Block=} obj
- * @returns {org.apache.arrow.flatbuf.Block}
- */
-org.apache.arrow.flatbuf.Footer.prototype.dictionaries = function(index, obj) {
-  var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? (obj || new org.apache.arrow.flatbuf.Block).__init(this.bb.__vector(this.bb_pos + offset) + index * 24, this.bb) : null;
-};
-
-/**
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Footer.prototype.dictionariesLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @param {number} index
- * @param {org.apache.arrow.flatbuf.Block=} obj
- * @returns {org.apache.arrow.flatbuf.Block}
- */
-org.apache.arrow.flatbuf.Footer.prototype.recordBatches = function(index, obj) {
-  var offset = this.bb.__offset(this.bb_pos, 10);
-  return offset ? (obj || new org.apache.arrow.flatbuf.Block).__init(this.bb.__vector(this.bb_pos + offset) + index * 24, this.bb) : null;
-};
-
-/**
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Footer.prototype.recordBatchesLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 10);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Footer.startFooter = function(builder) {
-  builder.startObject(4);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {org.apache.arrow.flatbuf.MetadataVersion} version
- */
-org.apache.arrow.flatbuf.Footer.addVersion = function(builder, version) {
-  builder.addFieldInt16(0, version, org.apache.arrow.flatbuf.MetadataVersion.V1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} schemaOffset
- */
-org.apache.arrow.flatbuf.Footer.addSchema = function(builder, schemaOffset) {
-  builder.addFieldOffset(1, schemaOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} dictionariesOffset
- */
-org.apache.arrow.flatbuf.Footer.addDictionaries = function(builder, dictionariesOffset) {
-  builder.addFieldOffset(2, dictionariesOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-org.apache.arrow.flatbuf.Footer.startDictionariesVector = function(builder, numElems) {
-  builder.startVector(24, numElems, 8);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} recordBatchesOffset
- */
-org.apache.arrow.flatbuf.Footer.addRecordBatches = function(builder, recordBatchesOffset) {
-  builder.addFieldOffset(3, recordBatchesOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-org.apache.arrow.flatbuf.Footer.startRecordBatchesVector = function(builder, numElems) {
-  builder.startVector(24, numElems, 8);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Footer.endFooter = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} offset
- */
-org.apache.arrow.flatbuf.Footer.finishFooterBuffer = function(builder, offset) {
-  builder.finish(offset);
-};
-
-/**
- * @constructor
- */
-org.apache.arrow.flatbuf.Block = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Block}
- */
-org.apache.arrow.flatbuf.Block.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * Index to the start of the RecordBlock (note this is past the Message header)
- *
- * @returns {flatbuffers.Long}
- */
-org.apache.arrow.flatbuf.Block.prototype.offset = function() {
-  return this.bb.readInt64(this.bb_pos);
-};
-
-/**
- * Length of the metadata
- *
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Block.prototype.metaDataLength = function() {
-  return this.bb.readInt32(this.bb_pos + 8);
-};
-
-/**
- * Length of the data (this is aligned so there can be a gap between this and
- * the metatdata).
- *
- * @returns {flatbuffers.Long}
- */
-org.apache.arrow.flatbuf.Block.prototype.bodyLength = function() {
-  return this.bb.readInt64(this.bb_pos + 16);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Long} offset
- * @param {number} metaDataLength
- * @param {flatbuffers.Long} bodyLength
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Block.createBlock = function(builder, offset, metaDataLength, bodyLength) {
-  builder.prep(8, 24);
-  builder.writeInt64(bodyLength);
-  builder.pad(4);
-  builder.writeInt32(metaDataLength);
-  builder.writeInt64(offset);
-  return builder.offset();
-};
-export { org };
-
diff --git a/js/src/fb/Message_generated.js b/js/src/fb/Message_generated.js
deleted file mode 100644
index ef46c98057c..00000000000
--- a/js/src/fb/Message_generated.js
+++ /dev/null
@@ -1,497 +0,0 @@
-import { org } from './Schema';
-// automatically generated by the FlatBuffers compiler, do not modify
-
-/**
- * @const
- * @namespace
- */
-org.apache = org.apache || {};
-
-/**
- * @const
- * @namespace
- */
-org.apache.arrow = org.apache.arrow || {};
-
-/**
- * @const
- * @namespace
- */
-org.apache.arrow.flatbuf = org.apache.arrow.flatbuf || {};
-
-/**
- * ----------------------------------------------------------------------
- * The root Message type
- * This union enables us to easily send different message types without
- * redundant storage, and in the future we can easily add new message types.
- *
- * Arrow implementations do not need to implement all of the message types,
- * which may include experimental metadata types. For maximum compatibility,
- * it is best to send data using RecordBatch
- *
- * @enum
- */
-org.apache.arrow.flatbuf.MessageHeader = {
-  NONE: 0, 0: 'NONE',
-  Schema: 1, 1: 'Schema',
-  DictionaryBatch: 2, 2: 'DictionaryBatch',
-  RecordBatch: 3, 3: 'RecordBatch',
-  Tensor: 4, 4: 'Tensor',
-};
-
-/**
- * ----------------------------------------------------------------------
- * Data structures for describing a table row batch (a collection of
- * equal-length Arrow arrays)
- * Metadata about a field at some level of a nested type tree (but not
- * its children).
- *
- * For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
- * would have {length: 5, null_count: 2} for its List node, and {length: 6,
- * null_count: 0} for its Int16 node, as separate FieldNode structs
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.FieldNode = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.FieldNode}
- */
-org.apache.arrow.flatbuf.FieldNode.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * The number of value slots in the Arrow array at this level of a nested
- * tree
- *
- * @returns {flatbuffers.Long}
- */
-org.apache.arrow.flatbuf.FieldNode.prototype.length = function() {
-  return this.bb.readInt64(this.bb_pos);
-};
-
-/**
- * The number of observed nulls. Fields with null_count == 0 may choose not
- * to write their physical validity bitmap out as a materialized buffer,
- * instead setting the length of the bitmap buffer to 0.
- *
- * @returns {flatbuffers.Long}
- */
-org.apache.arrow.flatbuf.FieldNode.prototype.nullCount = function() {
-  return this.bb.readInt64(this.bb_pos + 8);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Long} length
- * @param {flatbuffers.Long} null_count
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.FieldNode.createFieldNode = function(builder, length, null_count) {
-  builder.prep(8, 16);
-  builder.writeInt64(null_count);
-  builder.writeInt64(length);
-  return builder.offset();
-};
-
-/**
- * A data header describing the shared memory layout of a "record" or "row"
- * batch. Some systems call this a "row batch" internally and others a "record
- * batch".
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.RecordBatch = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.RecordBatch}
- */
-org.apache.arrow.flatbuf.RecordBatch.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.RecordBatch=} obj
- * @returns {org.apache.arrow.flatbuf.RecordBatch}
- */
-org.apache.arrow.flatbuf.RecordBatch.getRootAsRecordBatch = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.RecordBatch).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * number of records / rows. The arrays in the batch should all have this
- * length
- *
- * @returns {flatbuffers.Long}
- */
-org.apache.arrow.flatbuf.RecordBatch.prototype.length = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
-};
-
-/**
- * Nodes correspond to the pre-ordered flattened logical schema
- *
- * @param {number} index
- * @param {org.apache.arrow.flatbuf.FieldNode=} obj
- * @returns {org.apache.arrow.flatbuf.FieldNode}
- */
-org.apache.arrow.flatbuf.RecordBatch.prototype.nodes = function(index, obj) {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? (obj || new org.apache.arrow.flatbuf.FieldNode).__init(this.bb.__vector(this.bb_pos + offset) + index * 16, this.bb) : null;
-};
-
-/**
- * @returns {number}
- */
-org.apache.arrow.flatbuf.RecordBatch.prototype.nodesLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * Buffers correspond to the pre-ordered flattened buffer tree
- *
- * The number of buffers appended to this list depends on the schema. For
- * example, most primitive arrays will have 2 buffers, 1 for the validity
- * bitmap and 1 for the values. For struct arrays, there will only be a
- * single buffer for the validity (nulls) bitmap
- *
- * @param {number} index
- * @param {org.apache.arrow.flatbuf.Buffer=} obj
- * @returns {org.apache.arrow.flatbuf.Buffer}
- */
-org.apache.arrow.flatbuf.RecordBatch.prototype.buffers = function(index, obj) {
-  var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? (obj || new org.apache.arrow.flatbuf.Buffer).__init(this.bb.__vector(this.bb_pos + offset) + index * 16, this.bb) : null;
-};
-
-/**
- * @returns {number}
- */
-org.apache.arrow.flatbuf.RecordBatch.prototype.buffersLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.RecordBatch.startRecordBatch = function(builder) {
-  builder.startObject(3);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Long} length
- */
-org.apache.arrow.flatbuf.RecordBatch.addLength = function(builder, length) {
-  builder.addFieldInt64(0, length, builder.createLong(0, 0));
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} nodesOffset
- */
-org.apache.arrow.flatbuf.RecordBatch.addNodes = function(builder, nodesOffset) {
-  builder.addFieldOffset(1, nodesOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-org.apache.arrow.flatbuf.RecordBatch.startNodesVector = function(builder, numElems) {
-  builder.startVector(16, numElems, 8);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} buffersOffset
- */
-org.apache.arrow.flatbuf.RecordBatch.addBuffers = function(builder, buffersOffset) {
-  builder.addFieldOffset(2, buffersOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-org.apache.arrow.flatbuf.RecordBatch.startBuffersVector = function(builder, numElems) {
-  builder.startVector(16, numElems, 8);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.RecordBatch.endRecordBatch = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * For sending dictionary encoding information. Any Field can be
- * dictionary-encoded, but in this case none of its children may be
- * dictionary-encoded.
- * There is one vector / column per dictionary, but that vector / column
- * may be spread across multiple dictionary batches by using the isDelta
- * flag
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.DictionaryBatch = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.DictionaryBatch}
- */
-org.apache.arrow.flatbuf.DictionaryBatch.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.DictionaryBatch=} obj
- * @returns {org.apache.arrow.flatbuf.DictionaryBatch}
- */
-org.apache.arrow.flatbuf.DictionaryBatch.getRootAsDictionaryBatch = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.DictionaryBatch).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @returns {flatbuffers.Long}
- */
-org.apache.arrow.flatbuf.DictionaryBatch.prototype.id = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
-};
-
-/**
- * @param {org.apache.arrow.flatbuf.RecordBatch=} obj
- * @returns {org.apache.arrow.flatbuf.RecordBatch|null}
- */
-org.apache.arrow.flatbuf.DictionaryBatch.prototype.data = function(obj) {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? (obj || new org.apache.arrow.flatbuf.RecordBatch).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
-};
-
-/**
- * If isDelta is true the values in the dictionary are to be appended to a
- * dictionary with the indicated id
- *
- * @returns {boolean}
- */
-org.apache.arrow.flatbuf.DictionaryBatch.prototype.isDelta = function() {
-  var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.DictionaryBatch.startDictionaryBatch = function(builder) {
-  builder.startObject(3);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Long} id
- */
-org.apache.arrow.flatbuf.DictionaryBatch.addId = function(builder, id) {
-  builder.addFieldInt64(0, id, builder.createLong(0, 0));
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} dataOffset
- */
-org.apache.arrow.flatbuf.DictionaryBatch.addData = function(builder, dataOffset) {
-  builder.addFieldOffset(1, dataOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {boolean} isDelta
- */
-org.apache.arrow.flatbuf.DictionaryBatch.addIsDelta = function(builder, isDelta) {
-  builder.addFieldInt8(2, +isDelta, +false);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.DictionaryBatch.endDictionaryBatch = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-org.apache.arrow.flatbuf.Message = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Message}
- */
-org.apache.arrow.flatbuf.Message.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Message=} obj
- * @returns {org.apache.arrow.flatbuf.Message}
- */
-org.apache.arrow.flatbuf.Message.getRootAsMessage = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Message).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @returns {org.apache.arrow.flatbuf.MetadataVersion}
- */
-org.apache.arrow.flatbuf.Message.prototype.version = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? /** @type {org.apache.arrow.flatbuf.MetadataVersion} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.MetadataVersion.V1;
-};
-
-/**
- * @returns {org.apache.arrow.flatbuf.MessageHeader}
- */
-org.apache.arrow.flatbuf.Message.prototype.headerType = function() {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? /** @type {org.apache.arrow.flatbuf.MessageHeader} */ (this.bb.readUint8(this.bb_pos + offset)) : org.apache.arrow.flatbuf.MessageHeader.NONE;
-};
-
-/**
- * @param {flatbuffers.Table} obj
- * @returns {?flatbuffers.Table}
- */
-org.apache.arrow.flatbuf.Message.prototype.header = function(obj) {
-  var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
-};
-
-/**
- * @returns {flatbuffers.Long}
- */
-org.apache.arrow.flatbuf.Message.prototype.bodyLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 10);
-  return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Message.startMessage = function(builder) {
-  builder.startObject(4);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {org.apache.arrow.flatbuf.MetadataVersion} version
- */
-org.apache.arrow.flatbuf.Message.addVersion = function(builder, version) {
-  builder.addFieldInt16(0, version, org.apache.arrow.flatbuf.MetadataVersion.V1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {org.apache.arrow.flatbuf.MessageHeader} headerType
- */
-org.apache.arrow.flatbuf.Message.addHeaderType = function(builder, headerType) {
-  builder.addFieldInt8(1, headerType, org.apache.arrow.flatbuf.MessageHeader.NONE);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} headerOffset
- */
-org.apache.arrow.flatbuf.Message.addHeader = function(builder, headerOffset) {
-  builder.addFieldOffset(2, headerOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Long} bodyLength
- */
-org.apache.arrow.flatbuf.Message.addBodyLength = function(builder, bodyLength) {
-  builder.addFieldInt64(3, bodyLength, builder.createLong(0, 0));
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Message.endMessage = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} offset
- */
-org.apache.arrow.flatbuf.Message.finishMessageBuffer = function(builder, offset) {
-  builder.finish(offset);
-};
-export { org };
-
diff --git a/js/src/fb/Schema_generated.js b/js/src/fb/Schema_generated.js
deleted file mode 100644
index ebed8a90645..00000000000
--- a/js/src/fb/Schema_generated.js
+++ /dev/null
@@ -1,2231 +0,0 @@
-// automatically generated by the FlatBuffers compiler, do not modify
-
-/**
- * @const
- * @namespace
- */
-var org = org || {};
-
-/**
- * @const
- * @namespace
- */
-org.apache = org.apache || {};
-
-/**
- * @const
- * @namespace
- */
-org.apache.arrow = org.apache.arrow || {};
-
-/**
- * @const
- * @namespace
- */
-org.apache.arrow.flatbuf = org.apache.arrow.flatbuf || {};
-
-/**
- * @enum
- */
-org.apache.arrow.flatbuf.MetadataVersion = {
-  /**
-   * 0.1.0
-   */
-  'V1': 0, 0: 'V1',
-
-  /**
-   * 0.2.0
-   */
-  'V2': 1, 1: 'V2',
-
-  /**
-   * 0.3.0 -> 0.7.1
-   */
-  'V3': 2, 2: 'V3',
-
-  /**
-   * >= 0.8.0
-   */
-  'V4': 3, 3: 'V4'
-};
-
-/**
- * @enum
- */
-org.apache.arrow.flatbuf.UnionMode = {
-  'Sparse': 0, 0: 'Sparse',
-  'Dense': 1, 1: 'Dense',
-};
-
-/**
- * @enum
- */
-org.apache.arrow.flatbuf.Precision = {
-  'HALF': 0, 0: 'HALF',
-  'SINGLE': 1, 1: 'SINGLE',
-  'DOUBLE': 2, 2: 'DOUBLE',
-};
-
-/**
- * @enum
- */
-org.apache.arrow.flatbuf.DateUnit = {
-  'DAY': 0, 0: 'DAY',
-  'MILLISECOND': 1, 1: 'MILLISECOND',
-};
-
-/**
- * @enum
- */
-org.apache.arrow.flatbuf.TimeUnit = {
-  'SECOND': 0, 0: 'SECOND',
-  'MILLISECOND': 1, 1: 'MILLISECOND',
-  'MICROSECOND': 2, 2: 'MICROSECOND',
-  'NANOSECOND': 3, 3: 'NANOSECOND',
-};
-
-/**
- * @enum
- */
-org.apache.arrow.flatbuf.IntervalUnit = {
-  'YEAR_MONTH': 0, 0: 'YEAR_MONTH',
-  'DAY_TIME': 1, 1: 'DAY_TIME',
-};
-
-/**
- * ----------------------------------------------------------------------
- * Top-level Type value, enabling extensible type-specific metadata. We can
- * add new logical types to Type without breaking backwards compatibility
- *
- * @enum
- */
-org.apache.arrow.flatbuf.Type = {
-  'NONE': 0, 0: 'NONE',
-  'Null': 1, 1: 'Null',
-  'Int': 2, 2: 'Int',
-  'FloatingPoint': 3, 3: 'FloatingPoint',
-  'Binary': 4, 4: 'Binary',
-  'Utf8': 5, 5: 'Utf8',
-  'Bool': 6, 6: 'Bool',
-  'Decimal': 7, 7: 'Decimal',
-  'Date': 8, 8: 'Date',
-  'Time': 9, 9: 'Time',
-  'Timestamp': 10, 10: 'Timestamp',
-  'Interval': 11, 11: 'Interval',
-  'List': 12, 12: 'List',
-  'Struct_': 13, 13: 'Struct_',
-  'Union': 14, 14: 'Union',
-  'FixedSizeBinary': 15, 15: 'FixedSizeBinary',
-  'FixedSizeList': 16, 16: 'FixedSizeList',
-  'Map': 17, 17: 'Map'
-};
-
-/**
- * ----------------------------------------------------------------------
- * The possible types of a vector
- *
- * @enum
- */
-org.apache.arrow.flatbuf.VectorType = {
-  /**
-   * used in List type, Dense Union and variable length primitive types (String, Binary)
-   */
-  'OFFSET': 0, 0: 'OFFSET',
-
-  /**
-   * actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector
-   */
-  'DATA': 1, 1: 'DATA',
-
-  /**
-   * Bit vector indicating if each value is null
-   */
-  'VALIDITY': 2, 2: 'VALIDITY',
-
-  /**
-   * Type vector used in Union type
-   */
-  'TYPE': 3, 3: 'TYPE'
-};
-
-/**
- * ----------------------------------------------------------------------
- * Endianness of the platform producing the data
- *
- * @enum
- */
-org.apache.arrow.flatbuf.Endianness = {
-  'Little': 0, 0: 'Little',
-  'Big': 1, 1: 'Big',
-};
-
-/**
- * These are stored in the flatbuffer in the Type union below
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.Null = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Null}
- */
-org.apache.arrow.flatbuf.Null.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Null=} obj
- * @returns {org.apache.arrow.flatbuf.Null}
- */
-org.apache.arrow.flatbuf.Null.getRootAsNull = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Null).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Null.startNull = function(builder) {
-  builder.startObject(0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Null.endNull = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * A Struct_ in the flatbuffer metadata is the same as an Arrow Struct
- * (according to the physical memory layout). We used Struct_ here as
- * Struct is a reserved word in Flatbuffers
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.Struct_ = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Struct_}
- */
-org.apache.arrow.flatbuf.Struct_.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Struct_=} obj
- * @returns {org.apache.arrow.flatbuf.Struct_}
- */
-org.apache.arrow.flatbuf.Struct_.getRootAsStruct_ = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Struct_).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Struct_.startStruct_ = function(builder) {
-  builder.startObject(0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Struct_.endStruct_ = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-org.apache.arrow.flatbuf.List = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.List}
- */
-org.apache.arrow.flatbuf.List.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.List=} obj
- * @returns {org.apache.arrow.flatbuf.List}
- */
-org.apache.arrow.flatbuf.List.getRootAsList = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.List).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.List.startList = function(builder) {
-  builder.startObject(0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.List.endList = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-org.apache.arrow.flatbuf.FixedSizeList = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.FixedSizeList}
- */
-org.apache.arrow.flatbuf.FixedSizeList.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.FixedSizeList=} obj
- * @returns {org.apache.arrow.flatbuf.FixedSizeList}
- */
-org.apache.arrow.flatbuf.FixedSizeList.getRootAsFixedSizeList = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.FixedSizeList).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * Number of list items per value
- *
- * @returns {number}
- */
-org.apache.arrow.flatbuf.FixedSizeList.prototype.listSize = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.FixedSizeList.startFixedSizeList = function(builder) {
-  builder.startObject(1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} listSize
- */
-org.apache.arrow.flatbuf.FixedSizeList.addListSize = function(builder, listSize) {
-  builder.addFieldInt32(0, listSize, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.FixedSizeList.endFixedSizeList = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * A Map is a logical nested type that is represented as
- *
- * List<entry: Struct<key: K, value: V>>
- *
- * In this layout, the keys and values are each respectively contiguous. We do
- * not constrain the key and value types, so the application is responsible
- * for ensuring that the keys are hashable and unique. Whether the keys are sorted
- * may be set in the metadata for this field
- *
- * In a Field with Map type, the Field has a child Struct field, which then
- * has two children: key type and the second the value type. The names of the
- * child fields may be respectively "entry", "key", and "value", but this is
- * not enforced
- *
- * Map
- *   - child[0] entry: Struct
- *     - child[0] key: K
- *     - child[1] value: V
- *
- * Neither the "entry" field nor the "key" field may be nullable.
- *
- * The metadata is structured so that Arrow systems without special handling
- * for Map can make Map an alias for List. The "layout" attribute for the Map
- * field must have the same contents as a List.
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.Map = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Map}
- */
-org.apache.arrow.flatbuf.Map.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Map=} obj
- * @returns {org.apache.arrow.flatbuf.Map}
- */
-org.apache.arrow.flatbuf.Map.getRootAsMap = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Map).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * Set to true if the keys within each value are sorted
- *
- * @returns {boolean}
- */
-org.apache.arrow.flatbuf.Map.prototype.keysSorted = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Map.startMap = function(builder) {
-  builder.startObject(1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {boolean} keysSorted
- */
-org.apache.arrow.flatbuf.Map.addKeysSorted = function(builder, keysSorted) {
-  builder.addFieldInt8(0, +keysSorted, +false);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Map.endMap = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * A union is a complex type with children in Field
- * By default ids in the type vector refer to the offsets in the children
- * optionally typeIds provides an indirection between the child offset and the type id
- * for each child typeIds[offset] is the id used in the type vector
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.Union = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Union}
- */
-org.apache.arrow.flatbuf.Union.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Union=} obj
- * @returns {org.apache.arrow.flatbuf.Union}
- */
-org.apache.arrow.flatbuf.Union.getRootAsUnion = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Union).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @returns {org.apache.arrow.flatbuf.UnionMode}
- */
-org.apache.arrow.flatbuf.Union.prototype.mode = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? /** @type {org.apache.arrow.flatbuf.UnionMode} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.UnionMode.Sparse;
-};
-
-/**
- * @param {number} index
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Union.prototype.typeIds = function(index) {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? this.bb.readInt32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
-};
-
-/**
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Union.prototype.typeIdsLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @returns {Int32Array}
- */
-org.apache.arrow.flatbuf.Union.prototype.typeIdsArray = function() {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? new Int32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Union.startUnion = function(builder) {
-  builder.startObject(2);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {org.apache.arrow.flatbuf.UnionMode} mode
- */
-org.apache.arrow.flatbuf.Union.addMode = function(builder, mode) {
-  builder.addFieldInt16(0, mode, org.apache.arrow.flatbuf.UnionMode.Sparse);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} typeIdsOffset
- */
-org.apache.arrow.flatbuf.Union.addTypeIds = function(builder, typeIdsOffset) {
-  builder.addFieldOffset(1, typeIdsOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {Array.<number>} data
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Union.createTypeIdsVector = function(builder, data) {
-  builder.startVector(4, data.length, 4);
-  for (var i = data.length - 1; i >= 0; i--) {
-    builder.addInt32(data[i]);
-  }
-  return builder.endVector();
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-org.apache.arrow.flatbuf.Union.startTypeIdsVector = function(builder, numElems) {
-  builder.startVector(4, numElems, 4);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Union.endUnion = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-org.apache.arrow.flatbuf.Int = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Int}
- */
-org.apache.arrow.flatbuf.Int.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Int=} obj
- * @returns {org.apache.arrow.flatbuf.Int}
- */
-org.apache.arrow.flatbuf.Int.getRootAsInt = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Int).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Int.prototype.bitWidth = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
-};
-
-/**
- * @returns {boolean}
- */
-org.apache.arrow.flatbuf.Int.prototype.isSigned = function() {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Int.startInt = function(builder) {
-  builder.startObject(2);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} bitWidth
- */
-org.apache.arrow.flatbuf.Int.addBitWidth = function(builder, bitWidth) {
-  builder.addFieldInt32(0, bitWidth, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {boolean} isSigned
- */
-org.apache.arrow.flatbuf.Int.addIsSigned = function(builder, isSigned) {
-  builder.addFieldInt8(1, +isSigned, +false);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Int.endInt = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-org.apache.arrow.flatbuf.FloatingPoint = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.FloatingPoint}
- */
-org.apache.arrow.flatbuf.FloatingPoint.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.FloatingPoint=} obj
- * @returns {org.apache.arrow.flatbuf.FloatingPoint}
- */
-org.apache.arrow.flatbuf.FloatingPoint.getRootAsFloatingPoint = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.FloatingPoint).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @returns {org.apache.arrow.flatbuf.Precision}
- */
-org.apache.arrow.flatbuf.FloatingPoint.prototype.precision = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? /** @type {org.apache.arrow.flatbuf.Precision} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.Precision.HALF;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.FloatingPoint.startFloatingPoint = function(builder) {
-  builder.startObject(1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {org.apache.arrow.flatbuf.Precision} precision
- */
-org.apache.arrow.flatbuf.FloatingPoint.addPrecision = function(builder, precision) {
-  builder.addFieldInt16(0, precision, org.apache.arrow.flatbuf.Precision.HALF);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.FloatingPoint.endFloatingPoint = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * Unicode with UTF-8 encoding
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.Utf8 = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Utf8}
- */
-org.apache.arrow.flatbuf.Utf8.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Utf8=} obj
- * @returns {org.apache.arrow.flatbuf.Utf8}
- */
-org.apache.arrow.flatbuf.Utf8.getRootAsUtf8 = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Utf8).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Utf8.startUtf8 = function(builder) {
-  builder.startObject(0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Utf8.endUtf8 = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-org.apache.arrow.flatbuf.Binary = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Binary}
- */
-org.apache.arrow.flatbuf.Binary.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Binary=} obj
- * @returns {org.apache.arrow.flatbuf.Binary}
- */
-org.apache.arrow.flatbuf.Binary.getRootAsBinary = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Binary).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Binary.startBinary = function(builder) {
-  builder.startObject(0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Binary.endBinary = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-org.apache.arrow.flatbuf.FixedSizeBinary = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.FixedSizeBinary}
- */
-org.apache.arrow.flatbuf.FixedSizeBinary.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.FixedSizeBinary=} obj
- * @returns {org.apache.arrow.flatbuf.FixedSizeBinary}
- */
-org.apache.arrow.flatbuf.FixedSizeBinary.getRootAsFixedSizeBinary = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.FixedSizeBinary).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * Number of bytes per value
- *
- * @returns {number}
- */
-org.apache.arrow.flatbuf.FixedSizeBinary.prototype.byteWidth = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.FixedSizeBinary.startFixedSizeBinary = function(builder) {
-  builder.startObject(1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} byteWidth
- */
-org.apache.arrow.flatbuf.FixedSizeBinary.addByteWidth = function(builder, byteWidth) {
-  builder.addFieldInt32(0, byteWidth, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.FixedSizeBinary.endFixedSizeBinary = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-org.apache.arrow.flatbuf.Bool = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Bool}
- */
-org.apache.arrow.flatbuf.Bool.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Bool=} obj
- * @returns {org.apache.arrow.flatbuf.Bool}
- */
-org.apache.arrow.flatbuf.Bool.getRootAsBool = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Bool).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Bool.startBool = function(builder) {
-  builder.startObject(0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Bool.endBool = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-org.apache.arrow.flatbuf.Decimal = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Decimal}
- */
-org.apache.arrow.flatbuf.Decimal.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Decimal=} obj
- * @returns {org.apache.arrow.flatbuf.Decimal}
- */
-org.apache.arrow.flatbuf.Decimal.getRootAsDecimal = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Decimal).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * Total number of decimal digits
- *
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Decimal.prototype.precision = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
-};
-
-/**
- * Number of digits after the decimal point "."
- *
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Decimal.prototype.scale = function() {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Decimal.startDecimal = function(builder) {
-  builder.startObject(2);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} precision
- */
-org.apache.arrow.flatbuf.Decimal.addPrecision = function(builder, precision) {
-  builder.addFieldInt32(0, precision, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} scale
- */
-org.apache.arrow.flatbuf.Decimal.addScale = function(builder, scale) {
-  builder.addFieldInt32(1, scale, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Decimal.endDecimal = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * Date is either a 32-bit or 64-bit type representing elapsed time since UNIX
- * epoch (1970-01-01), stored in either of two units:
- *
- * * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no
- *   leap seconds), where the values are evenly divisible by 86400000
- * * Days (32 bits) since the UNIX epoch
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.Date = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Date}
- */
-org.apache.arrow.flatbuf.Date.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Date=} obj
- * @returns {org.apache.arrow.flatbuf.Date}
- */
-org.apache.arrow.flatbuf.Date.getRootAsDate = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Date).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @returns {org.apache.arrow.flatbuf.DateUnit}
- */
-org.apache.arrow.flatbuf.Date.prototype.unit = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? /** @type {org.apache.arrow.flatbuf.DateUnit} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.DateUnit.MILLISECOND;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Date.startDate = function(builder) {
-  builder.startObject(1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {org.apache.arrow.flatbuf.DateUnit} unit
- */
-org.apache.arrow.flatbuf.Date.addUnit = function(builder, unit) {
-  builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.DateUnit.MILLISECOND);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Date.endDate = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * Time type. The physical storage type depends on the unit
- * - SECOND and MILLISECOND: 32 bits
- * - MICROSECOND and NANOSECOND: 64 bits
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.Time = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Time}
- */
-org.apache.arrow.flatbuf.Time.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Time=} obj
- * @returns {org.apache.arrow.flatbuf.Time}
- */
-org.apache.arrow.flatbuf.Time.getRootAsTime = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Time).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @returns {org.apache.arrow.flatbuf.TimeUnit}
- */
-org.apache.arrow.flatbuf.Time.prototype.unit = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? /** @type {org.apache.arrow.flatbuf.TimeUnit} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.TimeUnit.MILLISECOND;
-};
-
-/**
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Time.prototype.bitWidth = function() {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? this.bb.readInt32(this.bb_pos + offset) : 32;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Time.startTime = function(builder) {
-  builder.startObject(2);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {org.apache.arrow.flatbuf.TimeUnit} unit
- */
-org.apache.arrow.flatbuf.Time.addUnit = function(builder, unit) {
-  builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.TimeUnit.MILLISECOND);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} bitWidth
- */
-org.apache.arrow.flatbuf.Time.addBitWidth = function(builder, bitWidth) {
-  builder.addFieldInt32(1, bitWidth, 32);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Time.endTime = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, excluding
- * leap seconds, as a 64-bit integer. Note that UNIX time does not include
- * leap seconds.
- *
- * The Timestamp metadata supports both "time zone naive" and "time zone
- * aware" timestamps. Read about the timezone attribute for more detail
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.Timestamp = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Timestamp}
- */
-org.apache.arrow.flatbuf.Timestamp.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Timestamp=} obj
- * @returns {org.apache.arrow.flatbuf.Timestamp}
- */
-org.apache.arrow.flatbuf.Timestamp.getRootAsTimestamp = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Timestamp).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @returns {org.apache.arrow.flatbuf.TimeUnit}
- */
-org.apache.arrow.flatbuf.Timestamp.prototype.unit = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? /** @type {org.apache.arrow.flatbuf.TimeUnit} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.TimeUnit.SECOND;
-};
-
-/**
- * The time zone is a string indicating the name of a time zone, one of:
- *
- * * As used in the Olson time zone database (the "tz database" or
- *   "tzdata"), such as "America/New_York"
- * * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30
- *
- * Whether a timezone string is present indicates different semantics about
- * the data:
- *
- * * If the time zone is null or equal to an empty string, the data is "time
- *   zone naive" and shall be displayed *as is* to the user, not localized
- *   to the locale of the user. This data can be though of as UTC but
- *   without having "UTC" as the time zone, it is not considered to be
- *   localized to any time zone
- *
- * * If the time zone is set to a valid value, values can be displayed as
- *   "localized" to that time zone, even though the underlying 64-bit
- *   integers are identical to the same data stored in UTC. Converting
- *   between time zones is a metadata-only operation and does not change the
- *   underlying values
- *
- * @param {flatbuffers.Encoding=} optionalEncoding
- * @returns {string|Uint8Array|null}
- */
-org.apache.arrow.flatbuf.Timestamp.prototype.timezone = function(optionalEncoding) {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Timestamp.startTimestamp = function(builder) {
-  builder.startObject(2);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {org.apache.arrow.flatbuf.TimeUnit} unit
- */
-org.apache.arrow.flatbuf.Timestamp.addUnit = function(builder, unit) {
-  builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.TimeUnit.SECOND);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} timezoneOffset
- */
-org.apache.arrow.flatbuf.Timestamp.addTimezone = function(builder, timezoneOffset) {
-  builder.addFieldOffset(1, timezoneOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Timestamp.endTimestamp = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @constructor
- */
-org.apache.arrow.flatbuf.Interval = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Interval}
- */
-org.apache.arrow.flatbuf.Interval.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Interval=} obj
- * @returns {org.apache.arrow.flatbuf.Interval}
- */
-org.apache.arrow.flatbuf.Interval.getRootAsInterval = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Interval).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @returns {org.apache.arrow.flatbuf.IntervalUnit}
- */
-org.apache.arrow.flatbuf.Interval.prototype.unit = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? /** @type {org.apache.arrow.flatbuf.IntervalUnit} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Interval.startInterval = function(builder) {
-  builder.startObject(1);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {org.apache.arrow.flatbuf.IntervalUnit} unit
- */
-org.apache.arrow.flatbuf.Interval.addUnit = function(builder, unit) {
-  builder.addFieldInt16(0, unit, org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Interval.endInterval = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * ----------------------------------------------------------------------
- * represents the physical layout of a buffer
- * buffers have fixed width slots of a given type
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.VectorLayout = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.VectorLayout}
- */
-org.apache.arrow.flatbuf.VectorLayout.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.VectorLayout=} obj
- * @returns {org.apache.arrow.flatbuf.VectorLayout}
- */
-org.apache.arrow.flatbuf.VectorLayout.getRootAsVectorLayout = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.VectorLayout).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * the width of a slot in the buffer (typically 1, 8, 16, 32 or 64)
- *
- * @returns {number}
- */
-org.apache.arrow.flatbuf.VectorLayout.prototype.bitWidth = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readInt16(this.bb_pos + offset) : 0;
-};
-
-/**
- * the purpose of the vector
- *
- * @returns {org.apache.arrow.flatbuf.VectorType}
- */
-org.apache.arrow.flatbuf.VectorLayout.prototype.type = function() {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? /** @type {org.apache.arrow.flatbuf.VectorType} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.VectorType.OFFSET;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.VectorLayout.startVectorLayout = function(builder) {
-  builder.startObject(2);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} bitWidth
- */
-org.apache.arrow.flatbuf.VectorLayout.addBitWidth = function(builder, bitWidth) {
-  builder.addFieldInt16(0, bitWidth, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {org.apache.arrow.flatbuf.VectorType} type
- */
-org.apache.arrow.flatbuf.VectorLayout.addType = function(builder, type) {
-  builder.addFieldInt16(1, type, org.apache.arrow.flatbuf.VectorType.OFFSET);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.VectorLayout.endVectorLayout = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * ----------------------------------------------------------------------
- * user defined key value pairs to add custom metadata to arrow
- * key namespacing is the responsibility of the user
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.KeyValue = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.KeyValue}
- */
-org.apache.arrow.flatbuf.KeyValue.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.KeyValue=} obj
- * @returns {org.apache.arrow.flatbuf.KeyValue}
- */
-org.apache.arrow.flatbuf.KeyValue.getRootAsKeyValue = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.KeyValue).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {flatbuffers.Encoding=} optionalEncoding
- * @returns {string|Uint8Array|null}
- */
-org.apache.arrow.flatbuf.KeyValue.prototype.key = function(optionalEncoding) {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
-};
-
-/**
- * @param {flatbuffers.Encoding=} optionalEncoding
- * @returns {string|Uint8Array|null}
- */
-org.apache.arrow.flatbuf.KeyValue.prototype.value = function(optionalEncoding) {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.KeyValue.startKeyValue = function(builder) {
-  builder.startObject(2);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} keyOffset
- */
-org.apache.arrow.flatbuf.KeyValue.addKey = function(builder, keyOffset) {
-  builder.addFieldOffset(0, keyOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} valueOffset
- */
-org.apache.arrow.flatbuf.KeyValue.addValue = function(builder, valueOffset) {
-  builder.addFieldOffset(1, valueOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.KeyValue.endKeyValue = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * ----------------------------------------------------------------------
- * Dictionary encoding metadata
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.DictionaryEncoding = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.DictionaryEncoding}
- */
-org.apache.arrow.flatbuf.DictionaryEncoding.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.DictionaryEncoding=} obj
- * @returns {org.apache.arrow.flatbuf.DictionaryEncoding}
- */
-org.apache.arrow.flatbuf.DictionaryEncoding.getRootAsDictionaryEncoding = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.DictionaryEncoding).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * The known dictionary id in the application where this data is used. In
- * the file or streaming formats, the dictionary ids are found in the
- * DictionaryBatch messages
- *
- * @returns {flatbuffers.Long}
- */
-org.apache.arrow.flatbuf.DictionaryEncoding.prototype.id = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
-};
-
-/**
- * The dictionary indices are constrained to be positive integers. If this
- * field is null, the indices must be signed int32
- *
- * @param {org.apache.arrow.flatbuf.Int=} obj
- * @returns {org.apache.arrow.flatbuf.Int|null}
- */
-org.apache.arrow.flatbuf.DictionaryEncoding.prototype.indexType = function(obj) {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? (obj || new org.apache.arrow.flatbuf.Int).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
-};
-
-/**
- * By default, dictionaries are not ordered, or the order does not have
- * semantic meaning. In some statistical, applications, dictionary-encoding
- * is used to represent ordered categorical data, and we provide a way to
- * preserve that metadata here
- *
- * @returns {boolean}
- */
-org.apache.arrow.flatbuf.DictionaryEncoding.prototype.isOrdered = function() {
-  var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.DictionaryEncoding.startDictionaryEncoding = function(builder) {
-  builder.startObject(3);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Long} id
- */
-org.apache.arrow.flatbuf.DictionaryEncoding.addId = function(builder, id) {
-  builder.addFieldInt64(0, id, builder.createLong(0, 0));
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} indexTypeOffset
- */
-org.apache.arrow.flatbuf.DictionaryEncoding.addIndexType = function(builder, indexTypeOffset) {
-  builder.addFieldOffset(1, indexTypeOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {boolean} isOrdered
- */
-org.apache.arrow.flatbuf.DictionaryEncoding.addIsOrdered = function(builder, isOrdered) {
-  builder.addFieldInt8(2, +isOrdered, +false);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.DictionaryEncoding.endDictionaryEncoding = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * ----------------------------------------------------------------------
- * A field represents a named column in a record / row batch or child of a
- * nested type.
- *
- * - children is only for nested Arrow arrays
- * - For primitive types, children will have length 0
- * - nullable should default to true in general
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.Field = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Field}
- */
-org.apache.arrow.flatbuf.Field.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Field=} obj
- * @returns {org.apache.arrow.flatbuf.Field}
- */
-org.apache.arrow.flatbuf.Field.getRootAsField = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Field).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * @param {flatbuffers.Encoding=} optionalEncoding
- * @returns {string|Uint8Array|null}
- */
-org.apache.arrow.flatbuf.Field.prototype.name = function(optionalEncoding) {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
-};
-
-/**
- * @returns {boolean}
- */
-org.apache.arrow.flatbuf.Field.prototype.nullable = function() {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
-};
-
-/**
- * @returns {org.apache.arrow.flatbuf.Type}
- */
-org.apache.arrow.flatbuf.Field.prototype.typeType = function() {
-  var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? /** @type {org.apache.arrow.flatbuf.Type} */ (this.bb.readUint8(this.bb_pos + offset)) : org.apache.arrow.flatbuf.Type.NONE;
-};
-
-/**
- * @param {flatbuffers.Table} obj
- * @returns {?flatbuffers.Table}
- */
-org.apache.arrow.flatbuf.Field.prototype.type = function(obj) {
-  var offset = this.bb.__offset(this.bb_pos, 10);
-  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
-};
-
-/**
- * @param {org.apache.arrow.flatbuf.DictionaryEncoding=} obj
- * @returns {org.apache.arrow.flatbuf.DictionaryEncoding|null}
- */
-org.apache.arrow.flatbuf.Field.prototype.dictionary = function(obj) {
-  var offset = this.bb.__offset(this.bb_pos, 12);
-  return offset ? (obj || new org.apache.arrow.flatbuf.DictionaryEncoding).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null;
-};
-
-/**
- * @param {number} index
- * @param {org.apache.arrow.flatbuf.Field=} obj
- * @returns {org.apache.arrow.flatbuf.Field}
- */
-org.apache.arrow.flatbuf.Field.prototype.children = function(index, obj) {
-  var offset = this.bb.__offset(this.bb_pos, 14);
-  return offset ? (obj || new org.apache.arrow.flatbuf.Field).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
-};
-
-/**
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Field.prototype.childrenLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 14);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * layout of buffers produced for this type (as derived from the Type)
- * does not include children
- * each recordbatch will return instances of those Buffers.
- *
- * @param {number} index
- * @param {org.apache.arrow.flatbuf.VectorLayout=} obj
- * @returns {org.apache.arrow.flatbuf.VectorLayout}
- */
-org.apache.arrow.flatbuf.Field.prototype.layout = function(index, obj) {
-  var offset = this.bb.__offset(this.bb_pos, 16);
-  return offset ? (obj || new org.apache.arrow.flatbuf.VectorLayout).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
-};
-
-/**
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Field.prototype.layoutLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 16);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @param {number} index
- * @param {org.apache.arrow.flatbuf.KeyValue=} obj
- * @returns {org.apache.arrow.flatbuf.KeyValue}
- */
-org.apache.arrow.flatbuf.Field.prototype.customMetadata = function(index, obj) {
-  var offset = this.bb.__offset(this.bb_pos, 18);
-  return offset ? (obj || new org.apache.arrow.flatbuf.KeyValue).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
-};
-
-/**
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Field.prototype.customMetadataLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 18);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Field.startField = function(builder) {
-  builder.startObject(8);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} nameOffset
- */
-org.apache.arrow.flatbuf.Field.addName = function(builder, nameOffset) {
-  builder.addFieldOffset(0, nameOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {boolean} nullable
- */
-org.apache.arrow.flatbuf.Field.addNullable = function(builder, nullable) {
-  builder.addFieldInt8(1, +nullable, +false);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {org.apache.arrow.flatbuf.Type} typeType
- */
-org.apache.arrow.flatbuf.Field.addTypeType = function(builder, typeType) {
-  builder.addFieldInt8(2, typeType, org.apache.arrow.flatbuf.Type.NONE);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} typeOffset
- */
-org.apache.arrow.flatbuf.Field.addType = function(builder, typeOffset) {
-  builder.addFieldOffset(3, typeOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} dictionaryOffset
- */
-org.apache.arrow.flatbuf.Field.addDictionary = function(builder, dictionaryOffset) {
-  builder.addFieldOffset(4, dictionaryOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} childrenOffset
- */
-org.apache.arrow.flatbuf.Field.addChildren = function(builder, childrenOffset) {
-  builder.addFieldOffset(5, childrenOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {Array.<flatbuffers.Offset>} data
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Field.createChildrenVector = function(builder, data) {
-  builder.startVector(4, data.length, 4);
-  for (var i = data.length - 1; i >= 0; i--) {
-    builder.addOffset(data[i]);
-  }
-  return builder.endVector();
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-org.apache.arrow.flatbuf.Field.startChildrenVector = function(builder, numElems) {
-  builder.startVector(4, numElems, 4);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} layoutOffset
- */
-org.apache.arrow.flatbuf.Field.addLayout = function(builder, layoutOffset) {
-  builder.addFieldOffset(6, layoutOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {Array.<flatbuffers.Offset>} data
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Field.createLayoutVector = function(builder, data) {
-  builder.startVector(4, data.length, 4);
-  for (var i = data.length - 1; i >= 0; i--) {
-    builder.addOffset(data[i]);
-  }
-  return builder.endVector();
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-org.apache.arrow.flatbuf.Field.startLayoutVector = function(builder, numElems) {
-  builder.startVector(4, numElems, 4);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} customMetadataOffset
- */
-org.apache.arrow.flatbuf.Field.addCustomMetadata = function(builder, customMetadataOffset) {
-  builder.addFieldOffset(7, customMetadataOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {Array.<flatbuffers.Offset>} data
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Field.createCustomMetadataVector = function(builder, data) {
-  builder.startVector(4, data.length, 4);
-  for (var i = data.length - 1; i >= 0; i--) {
-    builder.addOffset(data[i]);
-  }
-  return builder.endVector();
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-org.apache.arrow.flatbuf.Field.startCustomMetadataVector = function(builder, numElems) {
-  builder.startVector(4, numElems, 4);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Field.endField = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * ----------------------------------------------------------------------
- * A Buffer represents a single contiguous memory segment
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.Buffer = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Buffer}
- */
-org.apache.arrow.flatbuf.Buffer.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * The relative offset into the shared memory page where the bytes for this
- * buffer starts
- *
- * @returns {flatbuffers.Long}
- */
-org.apache.arrow.flatbuf.Buffer.prototype.offset = function() {
-  return this.bb.readInt64(this.bb_pos);
-};
-
-/**
- * The absolute length (in bytes) of the memory buffer. The memory is found
- * from offset (inclusive) to offset + length (non-inclusive).
- *
- * @returns {flatbuffers.Long}
- */
-org.apache.arrow.flatbuf.Buffer.prototype.length = function() {
-  return this.bb.readInt64(this.bb_pos + 8);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Long} offset
- * @param {flatbuffers.Long} length
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Buffer.createBuffer = function(builder, offset, length) {
-  builder.prep(8, 16);
-  builder.writeInt64(length);
-  builder.writeInt64(offset);
-  return builder.offset();
-};
-
-/**
- * ----------------------------------------------------------------------
- * A Schema describes the columns in a row batch
- *
- * @constructor
- */
-org.apache.arrow.flatbuf.Schema = function() {
-  /**
-   * @type {flatbuffers.ByteBuffer}
-   */
-  this.bb = null;
-
-  /**
-   * @type {number}
-   */
-  this.bb_pos = 0;
-};
-
-/**
- * @param {number} i
- * @param {flatbuffers.ByteBuffer} bb
- * @returns {org.apache.arrow.flatbuf.Schema}
- */
-org.apache.arrow.flatbuf.Schema.prototype.__init = function(i, bb) {
-  this.bb_pos = i;
-  this.bb = bb;
-  return this;
-};
-
-/**
- * @param {flatbuffers.ByteBuffer} bb
- * @param {org.apache.arrow.flatbuf.Schema=} obj
- * @returns {org.apache.arrow.flatbuf.Schema}
- */
-org.apache.arrow.flatbuf.Schema.getRootAsSchema = function(bb, obj) {
-  return (obj || new org.apache.arrow.flatbuf.Schema).__init(bb.readInt32(bb.position()) + bb.position(), bb);
-};
-
-/**
- * endianness of the buffer
- * it is Little Endian by default
- * if endianness doesn't match the underlying system then the vectors need to be converted
- *
- * @returns {org.apache.arrow.flatbuf.Endianness}
- */
-org.apache.arrow.flatbuf.Schema.prototype.endianness = function() {
-  var offset = this.bb.__offset(this.bb_pos, 4);
-  return offset ? /** @type {org.apache.arrow.flatbuf.Endianness} */ (this.bb.readInt16(this.bb_pos + offset)) : org.apache.arrow.flatbuf.Endianness.Little;
-};
-
-/**
- * @param {number} index
- * @param {org.apache.arrow.flatbuf.Field=} obj
- * @returns {org.apache.arrow.flatbuf.Field}
- */
-org.apache.arrow.flatbuf.Schema.prototype.fields = function(index, obj) {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? (obj || new org.apache.arrow.flatbuf.Field).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
-};
-
-/**
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Schema.prototype.fieldsLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 6);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @param {number} index
- * @param {org.apache.arrow.flatbuf.KeyValue=} obj
- * @returns {org.apache.arrow.flatbuf.KeyValue}
- */
-org.apache.arrow.flatbuf.Schema.prototype.customMetadata = function(index, obj) {
-  var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? (obj || new org.apache.arrow.flatbuf.KeyValue).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
-};
-
-/**
- * @returns {number}
- */
-org.apache.arrow.flatbuf.Schema.prototype.customMetadataLength = function() {
-  var offset = this.bb.__offset(this.bb_pos, 8);
-  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- */
-org.apache.arrow.flatbuf.Schema.startSchema = function(builder) {
-  builder.startObject(3);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {org.apache.arrow.flatbuf.Endianness} endianness
- */
-org.apache.arrow.flatbuf.Schema.addEndianness = function(builder, endianness) {
-  builder.addFieldInt16(0, endianness, org.apache.arrow.flatbuf.Endianness.Little);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} fieldsOffset
- */
-org.apache.arrow.flatbuf.Schema.addFields = function(builder, fieldsOffset) {
-  builder.addFieldOffset(1, fieldsOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {Array.<flatbuffers.Offset>} data
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Schema.createFieldsVector = function(builder, data) {
-  builder.startVector(4, data.length, 4);
-  for (var i = data.length - 1; i >= 0; i--) {
-    builder.addOffset(data[i]);
-  }
-  return builder.endVector();
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-org.apache.arrow.flatbuf.Schema.startFieldsVector = function(builder, numElems) {
-  builder.startVector(4, numElems, 4);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} customMetadataOffset
- */
-org.apache.arrow.flatbuf.Schema.addCustomMetadata = function(builder, customMetadataOffset) {
-  builder.addFieldOffset(2, customMetadataOffset, 0);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {Array.<flatbuffers.Offset>} data
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Schema.createCustomMetadataVector = function(builder, data) {
-  builder.startVector(4, data.length, 4);
-  for (var i = data.length - 1; i >= 0; i--) {
-    builder.addOffset(data[i]);
-  }
-  return builder.endVector();
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {number} numElems
- */
-org.apache.arrow.flatbuf.Schema.startCustomMetadataVector = function(builder, numElems) {
-  builder.startVector(4, numElems, 4);
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @returns {flatbuffers.Offset}
- */
-org.apache.arrow.flatbuf.Schema.endSchema = function(builder) {
-  var offset = builder.endObject();
-  return offset;
-};
-
-/**
- * @param {flatbuffers.Builder} builder
- * @param {flatbuffers.Offset} offset
- */
-org.apache.arrow.flatbuf.Schema.finishSchemaBuffer = function(builder, offset) {
-  builder.finish(offset);
-};
-export { org };
-

From ed855723409073f7ceccd32115199092c6c39982 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Mon, 14 May 2018 16:18:07 -0700
Subject: [PATCH 39/51] fix instanceof ArrayBuffer in jest/node 10

---
 js/package.json  | 2 +-
 js/test/Arrow.ts | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/js/package.json b/js/package.json
index da75c257bd6..15869394f1e 100644
--- a/js/package.json
+++ b/js/package.json
@@ -60,7 +60,7 @@
     "command-line-usage": "4.1.0",
     "flatbuffers": "trxcllnt/flatbuffers-esm",
     "json-bignum": "0.0.3",
-    "text-encoding-utf-8": "trxcllnt/text-encoding-utf-8#6a959afc86b798a435bd7b97682a7eb1f735ee2e",
+    "text-encoding-utf-8": "1.0.2",
     "tslib": "1.9.0"
   },
   "devDependencies": {
diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts
index 9e027377c09..36f98d45571 100644
--- a/js/test/Arrow.ts
+++ b/js/test/Arrow.ts
@@ -18,6 +18,15 @@
 /* tslint:disable */
 // Dynamically load an Arrow target build based on command line arguments
 
+// Fix for Jest in node v10.x
+Object.defineProperty(ArrayBuffer, Symbol.hasInstance, {
+    writable: true,
+    configurable: true,
+    value(inst: any) {
+        return inst && inst.constructor && inst.constructor.name === 'ArrayBuffer';
+    }
+});
+
 const path = require('path');
 const target = process.env.TEST_TARGET!;
 const format = process.env.TEST_MODULE!;

From a06180b34f949c2f9e5dca531e66222adb16d3ac Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Mon, 14 May 2018 18:22:14 -0700
Subject: [PATCH 40/51] don't run JS integration tests in src-only mode when
 --debug=true

---
 integration/integration_test.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/integration/integration_test.py b/integration/integration_test.py
index 1f604be9f95..8c8ed040399 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -1115,7 +1115,6 @@ def _run(self, exe_cmd, arrow_path=None, json_path=None, command='VALIDATE'):
         cmd.extend(['--mode', command])
 
         if self.debug:
-            cmd.extend(['-t', 'src'])
             print(' '.join(cmd))
 
         run_cmd(cmd)

From efc7225ebdeb252386266e5a3427cdb45fab7c8c Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Tue, 15 May 2018 10:31:32 -0700
Subject: [PATCH 41/51] fix perf tests

---
 js/perf/index.js        | 6 +++---
 js/perf/table_config.js | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/js/perf/index.js b/js/perf/index.js
index 42cb6abe29c..2c075919253 100644
--- a/js/perf/index.js
+++ b/js/perf/index.js
@@ -138,7 +138,7 @@ function createGetByIndexTest(vector, name) {
 function createDataFrameDirectCountTest(table, column, test, value) {
     let sum, colidx = table.schema.fields.findIndex((c)=>c.name === column);
 
-    if (test == 'gteq') {
+    if (test == 'gt') {
         op = function () {
             sum = 0;
             let batches = table.batches;
@@ -195,8 +195,8 @@ function createDataFrameFilterCountTest(table, column, test, value) {
     let colidx = table.schema.fields.findIndex((c)=> c.name === column);
     let df;
 
-    if (test == 'gteq') {
-        df = table.filter(col(column).gteq(value));
+    if (test == 'gt') {
+        df = table.filter(col(column).gt(value));
     } else if (test == 'eq') {
         df = table.filter(col(column).eq(value));
     } else {
diff --git a/js/perf/table_config.js b/js/perf/table_config.js
index e3c332c870f..190908bc328 100644
--- a/js/perf/table_config.js
+++ b/js/perf/table_config.js
@@ -27,9 +27,9 @@ countBys = {
 }
 counts = {
     "tracks": [
-        {col: 'lat',    test: 'gteq', value: 0        },
-        {col: 'lng',    test: 'gteq', value: 0        },
-        {col: 'origin', test:   'eq', value: 'Seattle'},
+        {col: 'lat',    test: 'gt', value: 0        },
+        {col: 'lng',    test: 'gt', value: 0        },
+        {col: 'origin', test: 'eq', value: 'Seattle'},
     ]
 }
 

From 7924e677301545aad55ad20165b08ad610b103db Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Tue, 15 May 2018 12:00:37 -0700
Subject: [PATCH 42/51] rename readNodeStream -> readStream, fromNodeStream ->
 fromReadableStream, add support for reading File format

---
 js/bin/file-to-stream.js    | 16 ++--------------
 js/bin/stream-to-file.js    |  4 ++--
 js/src/Arrow.ts             | 12 ++++++------
 js/src/ipc/magic.ts         | 16 ++++++++++++++++
 js/src/ipc/reader/arrow.ts  |  6 +++---
 js/src/ipc/reader/binary.ts | 15 ++++++---------
 js/src/ipc/reader/node.ts   | 13 +++++++++++--
 7 files changed, 46 insertions(+), 36 deletions(-)

diff --git a/js/bin/file-to-stream.js b/js/bin/file-to-stream.js
index 3943785987c..fa4e5d17bbd 100755
--- a/js/bin/file-to-stream.js
+++ b/js/bin/file-to-stream.js
@@ -23,7 +23,7 @@ const path = require('path');
 const encoding = 'binary';
 const ext = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : '';
 const { util: { PipeIterator } } = require(`../index${ext}`);
-const { Table, serializeStream } = require(`../index${ext}`);
+const { Table, serializeStream, fromReadableStream } = require(`../index${ext}`);
 
 (async () => {
     // Todo (ptaylor): implement `serializeStreamAsync` that accepts an
@@ -32,18 +32,6 @@ const { Table, serializeStream } = require(`../index${ext}`);
         ? process.stdin : fs.createReadStream(path.resolve(process.argv[2]));
     const out = process.argv.length < 4
         ? process.stdout : fs.createWriteStream(path.resolve(process.argv[3]));
-    new PipeIterator(serializeStream(await Table.fromAsync(readArrowFile(in_))), encoding).pipe(out);
+    new PipeIterator(serializeStream(await Table.fromAsync(fromReadableStream(in_))), encoding).pipe(out);
 
 })().catch((e) => { console.error(e); process.exit(1); });
-
-async function readArrowFile(stream) {
-    let buffers = [], totalLength = 0;
-    for await (let chunk of stream) {
-        if (!Buffer.isBuffer(chunk)) {
-            chunk = Buffer.from(chunk, encoding);
-        }
-        buffers.push(chunk);
-        totalLength += chunk.byteLength;
-    }
-    return Buffer.concat(buffers, totalLength);
-}
diff --git a/js/bin/stream-to-file.js b/js/bin/stream-to-file.js
index cf32e274f33..f33646ac61a 100755
--- a/js/bin/stream-to-file.js
+++ b/js/bin/stream-to-file.js
@@ -23,7 +23,7 @@ const path = require('path');
 const encoding = 'binary';
 const ext = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : '';
 const { util: { PipeIterator } } = require(`../index${ext}`);
-const { Table, serializeFile, fromNodeStream } = require(`../index${ext}`);
+const { Table, serializeFile, fromReadableStream } = require(`../index${ext}`);
 
 (async () => {
     // Todo (ptaylor): implement `serializeFileAsync` that accepts an
@@ -32,6 +32,6 @@ const { Table, serializeFile, fromNodeStream } = require(`../index${ext}`);
         ? process.stdin : fs.createReadStream(path.resolve(process.argv[2]));
     const out = process.argv.length < 4
         ? process.stdout : fs.createWriteStream(path.resolve(process.argv[3]));
-    new PipeIterator(serializeFile(await Table.fromAsync(fromNodeStream(in_))), encoding).pipe(out);
+    new PipeIterator(serializeFile(await Table.fromAsync(fromReadableStream(in_))), encoding).pipe(out);
 
 })().catch((e) => { console.error(e); process.exit(1); });
diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts
index ad4a3c41399..58ec6aa47ae 100644
--- a/js/src/Arrow.ts
+++ b/js/src/Arrow.ts
@@ -28,8 +28,8 @@ import { Vector } from './vector';
 import { RecordBatch } from './recordbatch';
 import { Schema, Field, Type } from './type';
 import { Table, DataFrame, NextFunc, BindFunc, CountByResult } from './table';
-import { fromNodeStream } from './ipc/reader/node';
-import { read, readAsync, readNodeStream } from './ipc/reader/arrow';
+import { fromReadableStream } from './ipc/reader/node';
+import { read, readAsync, readStream } from './ipc/reader/arrow';
 import { serializeFile, serializeStream } from './ipc/writer/binary';
 
 export import View = vector_.View;
@@ -39,8 +39,8 @@ export import IntBitWidth = type_.IntBitWidth;
 export import TimeBitWidth = type_.TimeBitWidth;
 export import TypedArrayConstructor = type_.TypedArrayConstructor;
 
-export { fromNodeStream };
-export { read, readAsync, readNodeStream };
+export { fromReadableStream };
+export { read, readAsync, readStream };
 export { serializeFile, serializeStream };
 export { Table, DataFrame, NextFunc, BindFunc, CountByResult };
 export { Field, Schema, RecordBatch, Vector, Type };
@@ -209,8 +209,8 @@ try {
 
         Arrow['read'] = read;
         Arrow['readAsync'] = readAsync;
-        Arrow['readNodeStream'] = readNodeStream;
-        Arrow['fromNodeStream'] = fromNodeStream;
+        Arrow['readStream'] = readStream;
+        Arrow['fromReadableStream'] = fromReadableStream;
 
         Arrow['serializeFile'] = serializeFile;
         Arrow['serializeStream'] = serializeStream;
diff --git a/js/src/ipc/magic.ts b/js/src/ipc/magic.ts
index c6eb36a834e..0688d1a2d1e 100644
--- a/js/src/ipc/magic.ts
+++ b/js/src/ipc/magic.ts
@@ -15,6 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import { flatbuffers } from 'flatbuffers';
+import ByteBuffer = flatbuffers.ByteBuffer;
+
 export const PADDING = 4;
 export const MAGIC_STR = 'ARROW1';
 export const MAGIC = new Uint8Array(MAGIC_STR.length);
@@ -32,6 +35,19 @@ export function checkForMagicArrowString(buffer: Uint8Array, index = 0) {
     return true;
 }
 
+export function isValidArrowFile(bb: ByteBuffer) {
+    let fileLength = bb.capacity(), footerLength: number, lengthOffset: number;
+    if ((fileLength < magicX2AndPadding /*                     Arrow buffer too small */) ||
+        (!checkForMagicArrowString(bb.bytes(), 0) /*                        Missing magic start    */) ||
+        (!checkForMagicArrowString(bb.bytes(), fileLength - magicLength) /* Missing magic end      */) ||
+        (/*                                                    Invalid footer length  */
+        (footerLength = bb.readInt32(lengthOffset = fileLength - magicAndPadding)) < 1 &&
+        (footerLength + lengthOffset > fileLength))) {
+        return false;
+    }
+    return true;
+}
+
 export const magicLength = MAGIC.length;
 export const magicAndPadding = magicLength + PADDING;
 export const magicX2AndPadding = magicLength * 2 + PADDING;
diff --git a/js/src/ipc/reader/arrow.ts b/js/src/ipc/reader/arrow.ts
index ad222a1054e..1847c9c2eb6 100644
--- a/js/src/ipc/reader/arrow.ts
+++ b/js/src/ipc/reader/arrow.ts
@@ -16,7 +16,7 @@
 // under the License.
 
 import { readJSON } from './json';
-import { fromNodeStream } from './node';
+import { fromReadableStream } from './node';
 import { RecordBatch } from '../../recordbatch';
 import { readBuffers, readBuffersAsync } from './binary';
 import { readRecordBatches, readRecordBatchesAsync, TypeDataLoader } from './vector';
@@ -48,8 +48,8 @@ export async function* readAsync(sources: AsyncIterable<Uint8Array | Buffer | st
     }
 }
 
-export async function* readNodeStream(stream: NodeJS.ReadableStream) {
-    for await (const recordBatch of readAsync(fromNodeStream(stream))) {
+export async function* readStream(stream: NodeJS.ReadableStream) {
+    for await (const recordBatch of readAsync(fromReadableStream(stream))) {
         yield recordBatch as RecordBatch;
     }
 }
diff --git a/js/src/ipc/reader/binary.ts b/js/src/ipc/reader/binary.ts
index 0447440a75a..15e21af500b 100644
--- a/js/src/ipc/reader/binary.ts
+++ b/js/src/ipc/reader/binary.ts
@@ -18,7 +18,7 @@
 import { Vector } from '../../vector';
 import { flatbuffers } from 'flatbuffers';
 import { TypeDataLoader } from './vector';
-import { checkForMagicArrowString, PADDING, magicLength, magicAndPadding, magicX2AndPadding } from '../magic';
+import { checkForMagicArrowString, PADDING, magicAndPadding, isValidArrowFile } from '../magic';
 import { Message, Footer, FileBlock, RecordBatchMetadata, DictionaryBatch, BufferMetadata, FieldMetadata, } from '../metadata';
 import {
     Schema, Field,
@@ -156,16 +156,13 @@ function* readStreamMessages(bb: ByteBuffer) {
 }
 
 function readFileSchema(bb: ByteBuffer) {
-    let fileLength = bb.capacity(), footerLength: number, footerOffset: number;
-    if ((fileLength < magicX2AndPadding /*                     Arrow buffer too small */) ||
-        (!checkForMagicArrowString(bb.bytes(), 0) /*                        Missing magic start    */) ||
-        (!checkForMagicArrowString(bb.bytes(), fileLength - magicLength) /* Missing magic end      */) ||
-        (/*                                                    Invalid footer length  */
-        (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 &&
-        (footerLength + footerOffset > fileLength))) {
+    if (!isValidArrowFile(bb)) {
         return null;
     }
-    bb.setPosition(footerOffset - footerLength);
+    let fileLength = bb.capacity();
+    let lengthOffset = fileLength - magicAndPadding;
+    let footerLength = bb.readInt32(lengthOffset);
+    bb.setPosition(lengthOffset - footerLength);
     return footerFromByteBuffer(bb);
 }
 
diff --git a/js/src/ipc/reader/node.ts b/js/src/ipc/reader/node.ts
index 56a484dd53e..7fbd7bfbbfc 100644
--- a/js/src/ipc/reader/node.ts
+++ b/js/src/ipc/reader/node.ts
@@ -15,13 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { PADDING } from '../magic';
 import { flatbuffers } from 'flatbuffers';
 import * as Message_ from '../../fb/Message';
 import ByteBuffer = flatbuffers.ByteBuffer;
 import _Message = Message_.org.apache.arrow.flatbuf.Message;
+import { PADDING, isValidArrowFile, checkForMagicArrowString } from '../magic';
 
-export async function* fromNodeStream(stream: NodeJS.ReadableStream) {
+export async function* fromReadableStream(stream: NodeJS.ReadableStream) {
 
     let bb: ByteBuffer;
     let bytesRead = 0, bytes = new Uint8Array(0);
@@ -41,6 +41,15 @@ export async function* fromNodeStream(stream: NodeJS.ReadableStream) {
 
         bytes = grown;
 
+        // If we're reading in an Arrow File, just concatenate the bytes until
+        // the file is fully read in
+        if (checkForMagicArrowString(bytes)) {
+            if (!isValidArrowFile(new ByteBuffer(bytes))) {
+                continue;
+            }
+            return yield bytes;
+        }
+
         if (messageLength <= 0) {
             messageLength = new DataView(bytes.buffer).getInt32(0, true);
         }

From df43bc55ab51711a8a8c9eca70e5f37d166fd753 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Tue, 15 May 2018 12:01:40 -0700
Subject: [PATCH 43/51] make arrow2csv support streaming files from stdin, add
 rowsToString() method to RecordBatch

---
 js/src/bin/arrow2csv.ts | 35 +++++++++++++++++------------------
 js/src/recordbatch.ts   | 30 ++++++++++++++++++++++++++++++
 js/src/table.ts         | 35 +++++++++++------------------------
 js/src/util/pretty.ts   |  8 ++++++++
 js/src/vector/nested.ts |  7 ++-----
 5 files changed, 68 insertions(+), 47 deletions(-)
 create mode 100644 js/src/util/pretty.ts

diff --git a/js/src/bin/arrow2csv.ts b/js/src/bin/arrow2csv.ts
index 43e2f51508a..510f00740fe 100644
--- a/js/src/bin/arrow2csv.ts
+++ b/js/src/bin/arrow2csv.ts
@@ -21,7 +21,7 @@
 
 import * as fs from 'fs';
 import { promisify } from 'util';
-import * as Arrow from '../Arrow';
+import { Table, readStream } from '../Arrow';
 
 const readFile = promisify(fs.readFile);
 const { parse } = require('json-bignum');
@@ -29,31 +29,30 @@ const argv = require(`command-line-args`)(cliOpts(), { partial: true });
 const files = [...(argv.file || []), ...(argv._unknown || [])].filter(Boolean);
 
 (async () => {
-    if (!files.length) {
-        let buffers = [], totalLength = 0;
-        for await (let chunk of (process.stdin as any)) {
-            if (!Buffer.isBuffer(chunk)) {
-                chunk = Buffer.from(chunk, 'binary');
-            }
-            buffers.push(chunk);
-            totalLength += chunk.byteLength;
+    let hasRecords = false;
+    if (files.length > 0) {
+        hasRecords = true;
+        for (let input of files) {
+            printTable(await readFile(input));
         }
-        if (buffers.length === 0) {
-            return print_usage();
+    } else {
+        let rowOffset = 0;
+        let maxColumnWidths: number[] = [];
+        for await (const recordBatch of readStream(process.stdin)) {
+            hasRecords = true;
+            recordBatch.rowsToString(' | ', rowOffset, maxColumnWidths).pipe(process.stdout);
+            rowOffset += recordBatch.length;
         }
-        files.push(Buffer.concat(buffers, totalLength));
-    }
-    for (let input of files) {
-        printTable(typeof input === 'string' ? await readFile(input) : input);
     }
+    return hasRecords ? null : print_usage();
 })().catch((e) => { console.error(e); process.exit(1); });
 
 function printTable(input: any) {
-    let table: Arrow.Table;
+    let table: Table;
     try {
-        table = Arrow.Table.from(input);
+        table = Table.from(input);
     } catch (e) {
-        table = Arrow.Table.from(parse(input + ''));
+        table = Table.from(parse(input + ''));
     }
     if (argv.schema && argv.schema.length) {
         table = table.select(...argv.schema);
diff --git a/js/src/recordbatch.ts b/js/src/recordbatch.ts
index 07d94a9d496..0515278f33e 100644
--- a/js/src/recordbatch.ts
+++ b/js/src/recordbatch.ts
@@ -19,6 +19,8 @@ import { Schema, Struct, DataType } from './type';
 import { flatbuffers } from 'flatbuffers';
 import { View, Vector, StructVector } from './vector';
 import { Data, NestedData } from './data';
+import { PipeIterator } from './util/node';
+import { valueToString, leftPad } from './util/pretty';
 
 import Long = flatbuffers.Long;
 
@@ -67,4 +69,32 @@ export class RecordBatch extends StructVector {
             this.childData.filter((_, i) => namesToKeep[fields[i].name])
         );
     }
+    public rowsToString(separator = ' | ', rowOffset = 0, maxColumnWidths: number[] = []) {
+        return new PipeIterator(recordBatchRowsToString(this, separator, rowOffset, maxColumnWidths), 'utf8');
+    }
+}
+
+function* recordBatchRowsToString(recordBatch: RecordBatch, separator = ' | ', rowOffset = 0, maxColumnWidths: number[] = []) {
+    const fields = recordBatch.schema.fields;
+    const header = ['row_id', ...fields.map((f) => `${f}`)].map(valueToString);
+    header.forEach((x, i) => {
+        maxColumnWidths[i] = Math.max(maxColumnWidths[i] || 0, x.length);
+    });
+    // Pass one to convert to strings and count max column widths
+    for (let i = -1, n = recordBatch.length - 1; ++i < n;) {
+        let val, row = [rowOffset + i, ...recordBatch.get(i) as Struct['TValue']];
+        for (let j = -1, k = row.length; ++j < k; ) {
+            val = valueToString(row[j]);
+            maxColumnWidths[j] = Math.max(maxColumnWidths[j] || 0, val.length);
+        }
+    }
+    for (let i = -1; ++i < recordBatch.length;) {
+        if ((rowOffset + i) % 1000 === 0) {
+            yield header.map((x, j) => leftPad(x, ' ', maxColumnWidths[j])).join(separator);
+        }
+        yield [rowOffset + i, ...recordBatch.get(i) as Struct['TValue']]
+            .map((x) => valueToString(x))
+            .map((x, j) => leftPad(x, ' ', maxColumnWidths[j]))
+            .join(separator);
+    }
 }
diff --git a/js/src/table.ts b/js/src/table.ts
index a365dbfe258..a988a48af5b 100644
--- a/js/src/table.ts
+++ b/js/src/table.ts
@@ -297,30 +297,17 @@ export class CountByResult extends Table implements DataFrame {
 }
 
 function* tableRowsToString(table: Table, separator = ' | ') {
-    const fields = table.schema.fields;
-    const header = ['row_id', ...fields.map((f) => `${f}`)].map(stringify);
-    const maxColumnWidths = header.map(x => x.length);
-    // Pass one to convert to strings and count max column widths
-    for (let i = -1, n = table.length - 1; ++i < n;) {
-        let val, row = [i, ...table.get(i)];
-        for (let j = -1, k = row.length; ++j < k; ) {
-            val = stringify(row[j]);
-            maxColumnWidths[j] = Math.max(maxColumnWidths[j], val.length);
-        }
+    let rowOffset = 0;
+    let maxColumnWidths: number[] = [];
+    let iterators: IterableIterator<string>[] = [];
+    // Gather all the `rowsToString` iterators into a list before iterating,
+    // so that `maxColumnWidths` is filled with the maxWidth for each column
+    // across all RecordBatches.
+    for (const batch of table.batches) {
+        iterators.push(batch.rowsToString(separator, rowOffset, maxColumnWidths));
+        rowOffset += batch.length;
     }
-    yield header.map((x, j) => leftPad(x, ' ', maxColumnWidths[j])).join(separator);
-    for (let i = -1; ++i < table.length;) {
-        yield [i, ...table.get(i)]
-            .map((x) => stringify(x))
-            .map((x, j) => leftPad(x, ' ', maxColumnWidths[j]))
-            .join(separator);
+    for (const iterator of iterators) {
+        yield* iterator;
     }
 }
-
-function leftPad(str: string, fill: string, n: number) {
-    return (new Array(n + 1).join(fill) + str).slice(-1 * n);
-}
-
-function stringify(x: any) {
-    return typeof x === 'string' ? `"${x}"` : ArrayBuffer.isView(x) ? `[${x}]` : JSON.stringify(x);
-}
diff --git a/js/src/util/pretty.ts b/js/src/util/pretty.ts
new file mode 100644
index 00000000000..c2d7a414efd
--- /dev/null
+++ b/js/src/util/pretty.ts
@@ -0,0 +1,8 @@
+
+export function leftPad(str: string, fill: string, n: number) {
+    return (new Array(n + 1).join(fill) + str).slice(-1 * n);
+}
+
+export function valueToString(x: any) {
+    return typeof x === 'string' ? `"${x}"` : ArrayBuffer.isView(x) ? `[${x}]` : JSON.stringify(x);
+}
diff --git a/js/src/vector/nested.ts b/js/src/vector/nested.ts
index a45a912c94e..1102fe83b71 100644
--- a/js/src/vector/nested.ts
+++ b/js/src/vector/nested.ts
@@ -18,6 +18,7 @@
 import { Data } from '../data';
 import { View, Vector } from '../vector';
 import { IterableArrayLike } from '../type';
+import { valueToString } from '../util/pretty';
 import { DataType, NestedType, DenseUnion, SparseUnion, Struct, Map_ } from '../type';
 
 export abstract class NestedView<T extends NestedType> implements View<T> {
@@ -45,7 +46,7 @@ export abstract class NestedView<T extends NestedType> implements View<T> {
     }
     public toJSON(): any { return this.toArray(); }
     public toString() {
-        return [...this].map((x) => stringify(x)).join(', ');
+        return [...this].map((x) => valueToString(x)).join(', ');
     }
     public get(index: number): T['TValue'] {
         return this.getNested(this, index);
@@ -214,7 +215,3 @@ export class MapRowView extends RowView {
         return child ? child.set(self.rowIndex, value) : null;
     }
 }
-
-function stringify(x: any) {
-    return typeof x === 'string' ? `"${x}"` : Array.isArray(x) ? JSON.stringify(x) : ArrayBuffer.isView(x) ? `[${x}]` : `${x}`;
-}

From 14e6b387c01b670f646825006bebce51267697a9 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Tue, 15 May 2018 12:01:55 -0700
Subject: [PATCH 44/51] cleanup: remove dead code

---
 js/src/util/layout.ts | 193 ------------------------------------------
 1 file changed, 193 deletions(-)
 delete mode 100644 js/src/util/layout.ts

diff --git a/js/src/util/layout.ts b/js/src/util/layout.ts
deleted file mode 100644
index 29698fb3d2b..00000000000
--- a/js/src/util/layout.ts
+++ /dev/null
@@ -1,193 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { align } from './bit';
-import { TextEncoder } from 'text-encoding-utf-8';
-import { TypedArrayConstructor, TypedArray } from '../type';
-
-export type NullableLayout = { nullCount: number, validity: Uint8Array };
-export type BufferLayout<TArray = ArrayLike<number>> = { data: TArray };
-export type DictionaryLayout<TArray = ArrayLike<number>> = { data: TArray, keys: number[] };
-export type VariableWidthLayout<TArray = ArrayLike<number>> = { data: TArray, offsets: number[] };
-export type VariableWidthDictionaryLayout<TArray = ArrayLike<number>> = { data: TArray, keys: number[], offsets: number[] };
-
-export type values<T, TNull> = ArrayLike<T | TNull | null | undefined>;
-export type BufferValueWriter<T> = (src: ArrayLike<T>, dst: number[], index: number) => boolean | void;
-export type BufferWriter<T, TNull> = (values: values<T, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout;
-export type BufferLayoutWriter<T, TNull> = (write: BufferValueWriter<T>, values: values<T, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout;
-
-const writeNumeric64Value = writeFixedWidthValue.bind(null, 64);
-const writeNumeric128Value = writeFixedWidthValue.bind(null, 128);
-const utf8Encoder = new TextEncoder() as { encode: (s: string) => Uint8Array };
-
-const stride1Encode = writeValidityLayout.bind(null, writeFixedWidthLayoutWithStride.bind(null, 1));
-const stride1FixedWidth = writeFixedWidthLayout.bind(null, writeValidityLayout.bind(null, stride1Encode));
-const stride2FixedWidth = writeFixedWidthLayout.bind(null, writeValidityLayout.bind(null, writeFixedWidthLayoutWithStride.bind(null, 2)));
-const stride4FixedWidth = writeFixedWidthLayout.bind(null, writeValidityLayout.bind(null, writeFixedWidthLayoutWithStride.bind(null, 4)));
-
-export const writeBools = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeBooleanValue), Uint8Array)                                                as <TNull>(values: values<boolean | number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Uint8Array>;
-export const writeInt8s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Int8Array)                                                 as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Int8Array>;
-export const writeInt16s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Int16Array)                                               as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Int16Array>;
-export const writeInt32s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Int32Array)                                               as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Int32Array>;
-export const writeInt64s = writeTypedLayout.bind(null, stride2FixedWidth.bind(null, writeNumeric64Value), Int32Array)                                             as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Int32Array>;
-export const writeUint8s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Uint8Array)                                               as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Uint8Array>;
-export const writeUint16s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Uint16Array)                                             as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Uint16Array>;
-export const writeUint32s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Uint32Array)                                             as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Uint32Array>;
-export const writeUint64s = writeTypedLayout.bind(null, stride2FixedWidth.bind(null, writeNumeric64Value), Uint32Array)                                           as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Uint32Array>;
-export const writeDecimals = writeTypedLayout.bind(null, stride4FixedWidth.bind(null, writeNumeric128Value), Uint32Array)                                         as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Uint32Array>;
-export const writeFloat32s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Float32Array)                                           as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Float32Array>;
-export const writeFloat64s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Float64Array)                                           as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => BufferLayout<Float64Array>;
-export const writeVariableWidth = writeVariableWidthLayout.bind(null, stride1Encode)                                                                              as <T, TNull>(writeValue: BufferValueWriter<T>, values: values<T, TNull>, nulls?: ArrayLike<TNull>) => VariableWidthLayout<Uint8Array>;
-export const writeBinary = writeTypedLayout.bind(null, writeVariableWidth.bind(null, writeBinaryValue))                                                           as <TNull>(values: values<Iterable<number>, TNull>, nulls?: ArrayLike<TNull>) => VariableWidthLayout<Uint8Array>;
-export const writeUtf8s = writeTypedLayout.bind(null, writeVariableWidth.bind(null, writeUtf8Value), Uint8Array)                                                  as <TNull>(values: values<string, TNull>, nulls?: ArrayLike<TNull>) => VariableWidthLayout<Uint8Array>;
-export const writeDictionaryEncoded = writeDictionaryLayout.bind(null, stride1Encode)                                                                             as <T, TNull>(writeValue: BufferValueWriter<T>, values: values<T, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint8Array>;
-export const writeDictionaryEncodedBools = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeBooleanValue), Uint8Array)        as <TNull>(values: values<boolean | number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint8Array>;
-export const writeDictionaryEncodedInt8s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Int8Array)         as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Int8Array>;
-export const writeDictionaryEncodedInt16s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Int16Array)       as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Int16Array>;
-export const writeDictionaryEncodedInt32s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Int32Array)       as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Int32Array>;
-export const writeDictionaryEncodedInt64s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride2FixedWidth, writeNumeric64Value), Int32Array)     as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Int32Array>;
-export const writeDictionaryEncodedUint8s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Uint8Array)       as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint8Array>;
-export const writeDictionaryEncodedUint16s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Uint16Array)     as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint16Array>;
-export const writeDictionaryEncodedUint32s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Uint32Array)     as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint32Array>;
-export const writeDictionaryEncodedUint64s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride2FixedWidth, writeNumeric64Value), Uint32Array)   as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint32Array>;
-export const writeDictionaryEncodedDecimals = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride4FixedWidth, writeNumeric128Value), Uint32Array) as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Uint32Array>;
-export const writeDictionaryEncodedFloat32s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Float32Array)   as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Float32Array>;
-export const writeDictionaryEncodedFloat64s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Float64Array)   as <TNull>(values: values<number, TNull>, nulls?: ArrayLike<TNull>) => DictionaryLayout<Float64Array>;
-export const writeDictionaryEncodedVariableWidth = writeDictionaryLayout.bind(null, writeVariableWidth)                                                           as <T, TNull>(writeValue: BufferValueWriter<T>, values: values<T, TNull>, nulls?: ArrayLike<TNull>) => VariableWidthDictionaryLayout<Uint8Array>;
-export const writeDictionaryEncodedBinary = writeTypedLayout.bind(null, writeDictionaryEncodedVariableWidth.bind(null, writeBinaryValue))                         as <TNull>(values: values<Iterable<number>, TNull>, nulls?: ArrayLike<TNull>) => VariableWidthDictionaryLayout<Uint8Array>;
-export const writeDictionaryEncodedUtf8s = writeTypedLayout.bind(null, writeDictionaryEncodedVariableWidth.bind(null, writeUtf8Value), Uint8Array)                as <TNull>(values: values<string, TNull>, nulls?: ArrayLike<TNull>) => VariableWidthDictionaryLayout<Uint8Array>;
-
-function writeFixedWidthLayoutWithStride<T, TNull>(
-    stride: number,
-    writeValue: BufferValueWriter<T>,
-    values: values<T, TNull>
-) {
-    let index = -stride;
-    const data = [] as number[];
-    const length = values.length;
-    while ((index += stride) < length) {
-        writeValue(values as ArrayLike<T>, data, index);
-    }
-    return { data: data as ArrayLike<number> };
-}
-
-function writeFixedWidthLayout<T, TNull>(
-    writeLayout: BufferLayoutWriter<T, TNull>,
-    writeValue: BufferValueWriter<T>,
-    values: values<T, TNull>,
-    nulls?: ArrayLike<TNull>
-) {
-    return writeLayout(writeValue, values, nulls);
-}
-
-function writeValidityLayout<T, TNull>(
-    writeLayout: BufferLayoutWriter<T, TNull>,
-    writeValue: BufferValueWriter<T>,
-    values: values<T, TNull>,
-    nulls?: ArrayLike<TNull>
-) {
-    let nullCount = 0;
-    let nullsLength = nulls && nulls.length || 0;
-    let validity = new Uint8Array(align(values.length >>> 3, 8)).fill(255);
-    return {
-        ...writeLayout(writeValueOrValidity, values),
-        nullCount, validity: (nullCount > 0 && validity) || new Uint8Array(0)
-    } as BufferLayout & NullableLayout;
-    function writeValueOrValidity(src: ArrayLike<T>, dst: number[], index: number) {
-        writeValue(src, dst, index);
-        let i = -1, x = src[index] as T | TNull;
-        let isNull = x === null || x === undefined;
-        while (!isNull && ++i < nullsLength) {
-            isNull = x === nulls![i];
-        }
-        if (isNull) {
-            nullCount++;
-            validity[index >> 3] &= ~(1 << (index % 8));
-        }
-    }
-}
-
-function writeVariableWidthLayout<T, TNull>(
-    writeLayout: BufferLayoutWriter<T, TNull>,
-    writeValue: BufferValueWriter<T>,
-    values: values<T, TNull>,
-    nulls?: ArrayLike<TNull>
-) {
-    let offsets = [0], offsetsIndex = 0;
-    return { ...writeLayout(writeValueAndOffset, values, nulls), offsets } as VariableWidthLayout;
-    function writeValueAndOffset(src: ArrayLike<T>, dst: number[], index: number) {
-        if (!writeValue(src, dst, index)) {
-            offsets[++offsetsIndex] = dst.length;
-        }
-    }
-}
-
-function writeDictionaryLayout<T, TNull>(
-    writeLayout: BufferLayoutWriter<T, TNull>,
-    writeValue: BufferValueWriter<T>,
-    values: values<T, TNull>,
-    nulls?: ArrayLike<TNull>
-) {
-    let keys = [] as number[], keysIndex = 0, keysMap = Object.create(null);
-    return { ...writeLayout(writeKeysOrValues, values, nulls), keys };
-    function writeKeysOrValues(src: ArrayLike<T>, dst: number[], index: number) {
-        const x: any = src[index];
-        if (x in keysMap) {
-            return (keys[index] = keysMap[x]) || true;
-        } else if (!writeValue(src, dst, index)) {
-            keys[index] = keysMap[x] = keysIndex++;
-        }
-    }
-}
-
-function writeTypedLayout<T, TNull, TArray extends TypedArray>(
-    writeBuffers: BufferWriter<T, TNull>,
-    ArrayBufferView: TypedArrayConstructor<TArray>,
-    values: values<T, TNull>,
-    nulls?: ArrayLike<TNull>
-) {
-    const result = writeBuffers(values, nulls);
-    result.data = new ArrayBufferView(result.data);
-    return result as BufferLayout<TArray>;
-}
-
-function writeBooleanValue(src: ArrayLike<boolean>, dst: number[], index: number) {
-    if (src[index]) {
-        let i = index >>> 3;
-        let b = dst[i] || 0;
-        dst[i] = b | 1 << (index % 8);
-    }
-}
-
-function writeNumericValue(src: ArrayLike<number>, dst: number[], index: number) {
-    dst[index] = +src[index];
-}
-
-function writeFixedWidthValue(bitWidth: number, src: ArrayLike<number>, dst: number[], index: number) {
-    const bytesLen = bitWidth / 32;
-    for (let i = -1; ++i < bytesLen;) {
-        dst[index + i] = src[index + i];
-    }
-}
-
-function writeUtf8Value(src: ArrayLike<string>, dst: number[], index: number) {
-    dst.push(...utf8Encoder.encode(src[index]));
-}
-
-function writeBinaryValue(src: ArrayLike<Iterable<number>>, dst: number[], index: number) {
-    dst.push(...src[index]);
-}

From f497f7a216d902b90a4f07cbb28493a43e00e8dc Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Tue, 15 May 2018 12:15:06 -0700
Subject: [PATCH 45/51] measure maxColumnWidths across all recordBatches when
 printing a table

---
 js/src/table.ts | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/js/src/table.ts b/js/src/table.ts
index a988a48af5b..de06dd71479 100644
--- a/js/src/table.ts
+++ b/js/src/table.ts
@@ -298,16 +298,23 @@ export class CountByResult extends Table implements DataFrame {
 
 function* tableRowsToString(table: Table, separator = ' | ') {
     let rowOffset = 0;
+    let firstValues = [];
     let maxColumnWidths: number[] = [];
     let iterators: IterableIterator<string>[] = [];
     // Gather all the `rowsToString` iterators into a list before iterating,
     // so that `maxColumnWidths` is filled with the maxWidth for each column
     // across all RecordBatches.
     for (const batch of table.batches) {
-        iterators.push(batch.rowsToString(separator, rowOffset, maxColumnWidths));
-        rowOffset += batch.length;
+        const iterator = batch.rowsToString(separator, rowOffset, maxColumnWidths);
+        const { done, value } = iterator.next();
+        if (!done) {
+            firstValues.push(value);
+            iterators.push(iterator);
+            rowOffset += batch.length;
+        }
     }
     for (const iterator of iterators) {
+        yield firstValues.shift();
         yield* iterator;
     }
 }

From b765b12a98eda30a705c074bcc2152a9b811b0b1 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Wed, 16 May 2018 13:14:48 -0700
Subject: [PATCH 46/51] speed up integration_test.py by only testing the JS
 source, not every compilation target

---
 ci/travis_env_common.sh         | 2 ++
 integration/integration_test.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index a4ae5bd271a..0b39aaa1184 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -17,6 +17,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# hide nodejs experimental-feature warnings
+export NODE_NO_WARNINGS=1
 export MINICONDA=$HOME/miniconda
 export PATH="$MINICONDA/bin:$PATH"
 export CONDA_PKGS_DIRS=$HOME/.conda_packages
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 8c8ed040399..10b9eafdd26 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -1112,7 +1112,7 @@ def _run(self, exe_cmd, arrow_path=None, json_path=None, command='VALIDATE'):
         if json_path is not None:
             cmd.extend(['-j', json_path])
 
-        cmd.extend(['--mode', command])
+        cmd.extend(['--mode', command, '-t', 'src'])
 
         if self.debug:
             print(' '.join(cmd))

From e34afaa8440f990737ced8a1ee908339011ca635 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Wed, 16 May 2018 19:10:05 -0700
Subject: [PATCH 47/51] export the RecordBatchSerializer

---
 js/src/ipc/writer/binary.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
index 118b1de1fca..a267b02fc7f 100644
--- a/js/src/ipc/writer/binary.ts
+++ b/js/src/ipc/writer/binary.ts
@@ -139,7 +139,7 @@ export function serializeFooter(footer: Footer) {
     return { metadataLength, buffer: footerBytes };
 }
 
-class RecordBatchSerializer extends VectorVisitor {
+export class RecordBatchSerializer extends VectorVisitor {
     public byteLength = 0;
     public buffers: TypedArray[] = [];
     public fieldNodes: FieldMetadata[] = [];

From 1a9864cfb8e264d02baa648365520693d876e7ee Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Wed, 16 May 2018 19:12:25 -0700
Subject: [PATCH 48/51] read message bodyLength from flatbuffer object

---
 js/src/ipc/metadata.ts      |  7 +++++--
 js/src/ipc/reader/binary.ts | 12 ++++++------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/js/src/ipc/metadata.ts b/js/src/ipc/metadata.ts
index 9d2ea26b45e..25b94b1dd55 100644
--- a/js/src/ipc/metadata.ts
+++ b/js/src/ipc/metadata.ts
@@ -51,8 +51,11 @@ export class RecordBatchMetadata extends Message {
     public length: number;
     public nodes: FieldMetadata[];
     public buffers: BufferMetadata[];
-    constructor(version: MetadataVersion, length: Long | number, nodes: FieldMetadata[], buffers: BufferMetadata[]) {
-        super(version, buffers.reduce((s, b) => align(s + b.length + (b.offset - s), 8), 0), MessageHeader.RecordBatch);
+    constructor(version: MetadataVersion, length: Long | number, nodes: FieldMetadata[], buffers: BufferMetadata[], bodyLength?: Long | number) {
+        if (bodyLength === void(0)) {
+            bodyLength = buffers.reduce((s, b) => align(s + b.length + (b.offset - s), 8), 0);
+        }
+        super(version, bodyLength, MessageHeader.RecordBatch);
         this.nodes = nodes;
         this.buffers = buffers;
         this.length = typeof length === 'number' ? length : length.low;
diff --git a/js/src/ipc/reader/binary.ts b/js/src/ipc/reader/binary.ts
index 15e21af500b..65dc91a7693 100644
--- a/js/src/ipc/reader/binary.ts
+++ b/js/src/ipc/reader/binary.ts
@@ -250,8 +250,8 @@ function messageFromByteBuffer(bb: ByteBuffer) {
     const m = _Message.getRootAsMessage(bb)!, type = m.headerType(), version = m.version();
     switch (type) {
         case MessageHeader.Schema: return schemaFromMessage(version, m.header(new _Schema())!, new Map());
-        case MessageHeader.RecordBatch: return recordBatchFromMessage(version, m.header(new _RecordBatch())!);
-        case MessageHeader.DictionaryBatch: return dictionaryBatchFromMessage(version, m.header(new _DictionaryBatch())!);
+        case MessageHeader.RecordBatch: return recordBatchFromMessage(version, m, m.header(new _RecordBatch())!);
+        case MessageHeader.DictionaryBatch: return dictionaryBatchFromMessage(version, m, m.header(new _DictionaryBatch())!);
     }
     return null;
     // throw new Error(`Unrecognized Message type '${type}'`);
@@ -261,12 +261,12 @@ function schemaFromMessage(version: MetadataVersion, s: _Schema, dictionaryField
     return new Schema(fieldsFromSchema(s, dictionaryFields), customMetadata(s), version, dictionaryFields);
 }
 
-function recordBatchFromMessage(version: MetadataVersion, b: _RecordBatch) {
-    return new RecordBatchMetadata(version, b.length(), fieldNodesFromRecordBatch(b), buffersFromRecordBatch(b, version));
+function recordBatchFromMessage(version: MetadataVersion, m: _Message, b: _RecordBatch) {
+    return new RecordBatchMetadata(version, b.length(), fieldNodesFromRecordBatch(b), buffersFromRecordBatch(b, version), m.bodyLength());
 }
 
-function dictionaryBatchFromMessage(version: MetadataVersion, d: _DictionaryBatch) {
-    return new DictionaryBatch(version, recordBatchFromMessage(version, d.data()!), d.id(), d.isDelta());
+function dictionaryBatchFromMessage(version: MetadataVersion, m: _Message, d: _DictionaryBatch) {
+    return new DictionaryBatch(version, recordBatchFromMessage(version, m, d.data()!), d.id(), d.isDelta());
 }
 
 function dictionaryBatchesFromFooter(f: _Footer) {

From 4594fe3d7c8c0b231ca1cda0ce7f6f5e2e6977cd Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 17 May 2018 14:43:58 -0700
Subject: [PATCH 49/51] align to 8-byte boundaries only

---
 js/src/ipc/writer/binary.ts | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
index a267b02fc7f..d8b1d7eb237 100644
--- a/js/src/ipc/writer/binary.ts
+++ b/js/src/ipc/writer/binary.ts
@@ -163,12 +163,10 @@ export class RecordBatchSerializer extends VectorVisitor {
                 throw new RangeError('Cannot write arrays larger than 2^31 - 1 in length');
             }
             this.fieldNodes.push(new FieldMetadata(length, nullCount));
-
-            const nullBitmapAlignment = length <= 64 ? 8 : 64;
-            const nullBitmap = nullCount <= 0
+            this.addBuffer(nullCount <= 0
                 ? new Uint8Array(0) // placeholder validity buffer
-                : this.getTruncatedBitmap(data.offset, length, data.nullBitmap!);
-            this.addBuffer(nullBitmap, nullBitmapAlignment);
+                : this.getTruncatedBitmap(data.offset, length, data.nullBitmap!)
+            );
         }
         return super.visit(vector);
     }
@@ -244,7 +242,6 @@ export class RecordBatchSerializer extends VectorVisitor {
         // Bool vector is a special case of FlatVector, as its data buffer needs to stay packed
         let bitmap: Uint8Array;
         let values, { data, length } = vector;
-        let alignment = length <= 64 ? 8 : 64;
         if (vector.nullCount >= length) {
             // If all values are null, just insert a placeholder empty data buffer (fastest path)
             bitmap = new Uint8Array(0);
@@ -256,8 +253,7 @@ export class RecordBatchSerializer extends VectorVisitor {
             // otherwise just slice the bitmap (fast path)
             bitmap = this.getTruncatedBitmap(data.offset, length, values);
         }
-        this.addBuffer(bitmap, alignment);
-        return this;
+        return this.addBuffer(bitmap);
     }
     protected visitFlatVector<T extends FlatType>(vector: Vector<T>) {
         const { view, data } = vector;
@@ -298,16 +294,16 @@ export class RecordBatchSerializer extends VectorVisitor {
         }
         return this;
     }
-    protected addBuffer(values: TypedArray, alignment: number = 64) {
-        const length = align(values.byteLength, alignment);
+    protected addBuffer(values: TypedArray) {
+        const byteLength = align(values.byteLength, 8);
         this.buffers.push(values);
-        this.buffersMeta.push(new BufferMetadata(this.byteLength, length));
-        this.byteLength += length;
+        this.buffersMeta.push(new BufferMetadata(this.byteLength, byteLength));
+        this.byteLength += byteLength;
         return this;
     }
     protected getTruncatedBitmap(offset: number, length: number, bitmap: Uint8Array) {
-        const alignedLength = align(length, length <= 64 ? 8 : 64);
-        if (offset > 0 || bitmap.length < alignedLength) {
+        const alignedLength = align(bitmap.byteLength, 8);
+        if (offset > 0 || bitmap.byteLength < alignedLength) {
             // With a sliced array / non-zero offset, we have to copy the bitmap
             const bytes = new Uint8Array(alignedLength);
             bytes.set(

From 7a346dca8b507972f58019788e93a3656004b341 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 17 May 2018 14:58:52 -0700
Subject: [PATCH 50/51] add a handy script for printing the alignment of
 buffers in a table

---
 js/bin/print-buffer-alignment.js | 50 ++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100755 js/bin/print-buffer-alignment.js

diff --git a/js/bin/print-buffer-alignment.js b/js/bin/print-buffer-alignment.js
new file mode 100755
index 00000000000..a4cd9bb2351
--- /dev/null
+++ b/js/bin/print-buffer-alignment.js
@@ -0,0 +1,50 @@
+#! /usr/bin/env node
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const fs = require('fs');
+const path = require('path');
+
+const ext = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : '';
+const base = process.env.ARROW_JS_DEBUG === 'src' ? '../src' : '../targets/apache-arrow';
+const { Message } = require(`${base}/ipc/metadata${ext}`);
+const { readBuffersAsync } = require(`${base}/ipc/reader/binary${ext}`);
+const { Table, VectorVisitor, fromReadableStream } = require(`../index${ext}`);
+
+(async () => {
+    const in_ = process.argv.length < 3
+        ? process.stdin : fs.createReadStream(path.resolve(process.argv[2]));
+    
+    let recordBatchIndex = 0;
+    let dictionaryBatchIndex = 0;
+
+    for await (let { message, loader } of readBuffersAsync(fromReadableStream(in_))) {
+
+        if (Message.isRecordBatch(message)) {
+            console.log(`record batch ${++recordBatchIndex}, offset ${loader.messageOffset}`);
+        } else if (Message.isDictionaryBatch(message)) {
+            message = message.data;
+            console.log(`dictionary batch ${++dictionaryBatchIndex}, offset ${loader.messageOffset}`);
+        } else { continue; }
+        
+        message.buffers.forEach(({offset, length}, i) => {
+            console.log(`\tbuffer ${i+1}: { offset: ${offset},  length: ${length} }`);
+        });
+    }
+
+})().catch((e) => { console.error(e); process.exit(1); });

From 917c2fc8284c3e124a123bed3e17d2c854cc047b Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Thu, 17 May 2018 14:59:11 -0700
Subject: [PATCH 51/51] test the ES5/UMD bundle in the integration tests

---
 integration/integration_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/integration_test.py b/integration/integration_test.py
index 10b9eafdd26..173fe549092 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -1112,7 +1112,7 @@ def _run(self, exe_cmd, arrow_path=None, json_path=None, command='VALIDATE'):
         if json_path is not None:
             cmd.extend(['-j', json_path])
 
-        cmd.extend(['--mode', command, '-t', 'src'])
+        cmd.extend(['--mode', command, '-t', 'es5', '-m', 'umd'])
 
         if self.debug:
             print(' '.join(cmd))