From d591b3c789b153e426b2dc5e62ab973c5bcc321b Mon Sep 17 00:00:00 2001 From: Brian Hulette Date: Tue, 9 May 2017 16:12:28 -0400 Subject: [PATCH 01/10] add tslint config --- js/tslint.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 js/tslint.json diff --git a/js/tslint.json b/js/tslint.json new file mode 100644 index 00000000000..40a1bb03f34 --- /dev/null +++ b/js/tslint.json @@ -0,0 +1,11 @@ +{ + "defaultSeverity": "error", + "extends": [ + "tslint:recommended" + ], + "jsRules": {}, + "rules": { + "no-bitwise": false + }, + "rulesDirectory": [] +} From 81c9868afe9283fd6904b7bc3069d6b57157a9eb Mon Sep 17 00:00:00 2001 From: Brian Hulette Date: Tue, 9 May 2017 17:39:24 -0400 Subject: [PATCH 02/10] Replace vars with let/const, one def per line --- js/src/arrow.ts | 186 +++++++++++++++++++++++++-------------------- js/src/bitarray.ts | 12 +-- js/src/types.ts | 65 ++++++++-------- 3 files changed, 142 insertions(+), 121 deletions(-) diff --git a/js/src/arrow.ts b/js/src/arrow.ts index 74def4dc94d..0e0fdfa1397 100644 --- a/js/src/arrow.ts +++ b/js/src/arrow.ts @@ -20,14 +20,14 @@ import { org } from './Arrow_generated'; import { vectorFromField, Vector } from './types'; import ByteBuffer = flatbuffers.ByteBuffer; -var Footer = org.apache.arrow.flatbuf.Footer; -var Message = org.apache.arrow.flatbuf.Message; -var MessageHeader = org.apache.arrow.flatbuf.MessageHeader; -var RecordBatch = org.apache.arrow.flatbuf.RecordBatch; -var DictionaryBatch = org.apache.arrow.flatbuf.DictionaryBatch; -var Schema = org.apache.arrow.flatbuf.Schema; -var Type = org.apache.arrow.flatbuf.Type; -var VectorType = org.apache.arrow.flatbuf.VectorType; +const Footer = org.apache.arrow.flatbuf.Footer; +const Message = org.apache.arrow.flatbuf.Message; +const MessageHeader = org.apache.arrow.flatbuf.MessageHeader; +const RecordBatch = org.apache.arrow.flatbuf.RecordBatch; +const DictionaryBatch = org.apache.arrow.flatbuf.DictionaryBatch; +const Schema = org.apache.arrow.flatbuf.Schema; +const Type = org.apache.arrow.flatbuf.Type; +const VectorType = org.apache.arrow.flatbuf.VectorType; export class ArrowReader { @@ -43,7 +43,7 @@ export class ArrowReader { this.bb = bb; this.schema = schema; this.vectors = vectors; - for (var i = 0; i < vectors.length; i += 1|0) { + for (let i = 0; i < vectors.length; i += 1|0) { this.vectorMap[vectors[i].name] = vectors[i] } this.batches = batches; @@ -52,7 +52,7 @@ export class ArrowReader { loadNextBatch() { if (this.batchIndex < this.batches.length) { - var batch = this.batches[this.batchIndex]; + const batch = this.batches[this.batchIndex]; this.batchIndex += 1; loadVectors(this.bb, this.vectors, batch); return batch.length; @@ -99,17 +99,17 @@ export function getReader(buf) : ArrowReader { } export function getStreamReader(buf) : ArrowReader { - var bb = new ByteBuffer(buf); - - var schema = _loadSchema(bb), - field, - vectors: Vector[] = [], - i,j, - iLen,jLen, - batch, - recordBatches = [], - dictionaryBatches = [], - dictionaries = {}; + const bb = new ByteBuffer(buf); + + const schema = _loadSchema(bb); + let field; + const vectors: Vector[] = []; + let i; + let iLen; + let batch; + const recordBatches = []; + const dictionaryBatches = []; + const dictionaries = {}; for (i = 0, iLen = schema.fieldsLength(); i < iLen; i += 1|0) { field = schema.fields(i); @@ -141,18 +141,20 @@ export function getStreamReader(buf) : ArrowReader { } export function getFileReader (buf) : ArrowReader { - var bb = new ByteBuffer(buf); + const bb = new ByteBuffer(buf); - var footer = _loadFooter(bb); + const footer = _loadFooter(bb); - var schema = footer.schema(); - var i, len, field, - vectors: Vector[] = [], - block, - batch, - recordBatchBlocks = [], - dictionaryBatchBlocks = [], - dictionaries = {}; + const schema = footer.schema(); + let i; + let len; + let field; + const vectors: Vector[] = []; + let block; + let batch; + const recordBatchBlocks = []; + const dictionaryBatchBlocks = []; + const dictionaries = {}; for (i = 0, len = schema.fieldsLength(); i < len; i += 1|0) { field = schema.fields(i); @@ -178,13 +180,13 @@ export function getFileReader (buf) : ArrowReader { }) } - var dictionaryBatches = dictionaryBatchBlocks.map(function (block) { + const dictionaryBatches = dictionaryBatchBlocks.map(function (block) { bb.setPosition(block.offset); // TODO: Make sure this is a dictionary batch return _loadBatch(bb); }); - var recordBatches = recordBatchBlocks.map(function (block) { + const recordBatches = recordBatchBlocks.map(function (block) { bb.setPosition(block.offset); // TODO: Make sure this is a record batch return _loadBatch(bb); @@ -200,7 +202,7 @@ export function getFileReader (buf) : ArrowReader { } function _loadFooter(bb) { - var fileLength: number = bb.bytes_.length; + const fileLength: number = bb.bytes_.length; if (fileLength < MAGIC.length*2 + 4) { console.error("file too small " + fileLength); @@ -217,23 +219,23 @@ function _loadFooter(bb) { return; } - var footerLengthOffset: number = fileLength - MAGIC.length - 4; + const footerLengthOffset: number = fileLength - MAGIC.length - 4; bb.setPosition(footerLengthOffset); - var footerLength: number = Int32FromByteBuffer(bb, footerLengthOffset) + const footerLength: number = Int32FromByteBuffer(bb, footerLengthOffset) if (footerLength <= 0 || footerLength + MAGIC.length*2 + 4 > fileLength) { console.log("Invalid footer length: " + footerLength) } - var footerOffset: number = footerLengthOffset - footerLength; + const footerOffset: number = footerLengthOffset - footerLength; bb.setPosition(footerOffset); - var footer = Footer.getRootAsFooter(bb); + const footer = Footer.getRootAsFooter(bb); return footer; } function _loadSchema(bb) { - var message =_loadMessage(bb); + const message =_loadMessage(bb); if (message.headerType() != MessageHeader.Schema) { console.error("Expected header type " + MessageHeader.Schema + " but got " + message.headerType()); return; @@ -242,14 +244,14 @@ function _loadSchema(bb) { } function _loadBatch(bb) { - var message = _loadMessage(bb); + const message = _loadMessage(bb); if (message == null) { return; } else if (message.headerType() == MessageHeader.RecordBatch) { - var batch = { header: message.header(new RecordBatch()), length: message.bodyLength().low } + const batch = { header: message.header(new RecordBatch()), length: message.bodyLength().low } return _loadRecordBatch(bb, batch); } else if (message.headerType() == MessageHeader.DictionaryBatch) { - var batch = { header: message.header(new DictionaryBatch()), length: message.bodyLength().low } + const batch = { header: message.header(new DictionaryBatch()), length: message.bodyLength().low } return _loadDictionaryBatch(bb, batch); } else { console.error("Expected header type " + MessageHeader.RecordBatch + " or " + MessageHeader.DictionaryBatch + @@ -259,9 +261,13 @@ function _loadBatch(bb) { } function _loadRecordBatch(bb, batch) { - var data = batch.header; - var i, nodes_ = [], nodesLength = data.nodesLength(); - var buffer, buffers_ = [], buffersLength = data.buffersLength(); + const data = batch.header; + let i; + const nodes_ = []; + const nodesLength = data.nodesLength(); + let buffer; + const buffers_ = [] + const buffersLength = data.buffersLength(); for (i = 0; i < nodesLength; i += 1) { nodes_.push(data.nodes(i)); @@ -277,9 +283,14 @@ function _loadRecordBatch(bb, batch) { } function _loadDictionaryBatch(bb, batch) { - var id_ = batch.header.id().toFloat64().toString(), data = batch.header.data(); - var i, nodes_ = [], nodesLength = data.nodesLength(); - var buffer, buffers_ = [], buffersLength = data.buffersLength(); + const id_ = batch.header.id().toFloat64().toString(); + const data = batch.header.data(); + let i; + const nodes_ = []; + const nodesLength = data.nodesLength(); + let buffer; + const buffers_ = []; + const buffersLength = data.buffersLength(); for (i = 0; i < nodesLength; i += 1) { nodes_.push(data.nodes(i)); @@ -291,16 +302,20 @@ function _loadDictionaryBatch(bb, batch) { // position the buffer after the body to read the next message bb.setPosition(bb.position() + batch.length); - return { id: id_, nodes: nodes_, buffers: buffers_, length: data.length().low, type: MessageHeader.DictionaryBatch }; + return {id: id_, + nodes: nodes_, + buffers: buffers_, + length: data.length().low, + type: MessageHeader.DictionaryBatch }; } function _loadMessage(bb) { - var messageLength: number = Int32FromByteBuffer(bb, bb.position()); + const messageLength: number = Int32FromByteBuffer(bb, bb.position()); if (messageLength == 0) { return; } bb.setPosition(bb.position() + 4); - var message = Message.getRootAsMessage(bb); + const message = Message.getRootAsMessage(bb); // position the buffer at the end of the message so it's ready to read further bb.setPosition(bb.position() + messageLength); @@ -308,36 +323,36 @@ function _loadMessage(bb) { } function _createDictionaryVectors(field, dictionaries) { - var encoding = field.dictionary(); + const encoding = field.dictionary(); if (encoding != null) { - var id = encoding.id().toFloat64().toString(); + const id = encoding.id().toFloat64().toString(); if (dictionaries[id] == null) { // create a field for the dictionary - var dictionaryField = _createDictionaryField(id, field); + const dictionaryField = _createDictionaryField(id, field); dictionaries[id] = vectorFromField(dictionaryField, null); } } // recursively examine child fields - for (var i = 0, len = field.childrenLength(); i < len; i += 1|0) { + for (let i = 0, len = field.childrenLength(); i < len; i += 1|0) { _createDictionaryVectors(field.children(i), dictionaries); } } function _createDictionaryField(id, field) { - var builder = new flatbuffers.Builder(); - var nameOffset = builder.createString("dict-" + id); + const builder = new flatbuffers.Builder(); + const nameOffset = builder.createString("dict-" + id); - var typeType = field.typeType(); - var typeOffset; + const typeType = field.typeType(); + let typeOffset; if (typeType === Type.Int) { - var type = field.type(new org.apache.arrow.flatbuf.Int()); + const type = field.type(new org.apache.arrow.flatbuf.Int()); org.apache.arrow.flatbuf.Int.startInt(builder); org.apache.arrow.flatbuf.Int.addBitWidth(builder, type.bitWidth()); org.apache.arrow.flatbuf.Int.addIsSigned(builder, type.isSigned()); typeOffset = org.apache.arrow.flatbuf.Int.endInt(builder); } else if (typeType === Type.FloatingPoint) { - var type = field.type(new org.apache.arrow.flatbuf.FloatingPoint()); + const type = field.type(new org.apache.arrow.flatbuf.FloatingPoint()); org.apache.arrow.flatbuf.FloatingPoint.startFloatingPoint(builder); org.apache.arrow.flatbuf.FloatingPoint.addPrecision(builder, type.precision()); typeOffset = org.apache.arrow.flatbuf.FloatingPoint.endFloatingPoint(builder); @@ -345,7 +360,7 @@ function _createDictionaryField(id, field) { org.apache.arrow.flatbuf.Utf8.startUtf8(builder); typeOffset = org.apache.arrow.flatbuf.Utf8.endUtf8(builder); } else if (typeType === Type.Date) { - var type = field.type(new org.apache.arrow.flatbuf.Date()); + const type = field.type(new org.apache.arrow.flatbuf.Date()); org.apache.arrow.flatbuf.Date.startDate(builder); org.apache.arrow.flatbuf.Date.addUnit(builder, type.unit()); typeOffset = org.apache.arrow.flatbuf.Date.endDate(builder); @@ -355,17 +370,18 @@ function _createDictionaryField(id, field) { if (field.childrenLength() > 0) { throw "Dictionary encoded fields can't have children" } - var childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, []); + const childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, []); - var layout, layoutOffsets = []; - for (var i = 0, len = field.layoutLength(); i < len; i += 1|0) { + let layout; + const layoutOffsets = []; + for (let i = 0, len = field.layoutLength(); i < len; i += 1|0) { layout = field.layout(i); org.apache.arrow.flatbuf.VectorLayout.startVectorLayout(builder); org.apache.arrow.flatbuf.VectorLayout.addBitWidth(builder, layout.bitWidth()); org.apache.arrow.flatbuf.VectorLayout.addType(builder, layout.type()); layoutOffsets.push(org.apache.arrow.flatbuf.VectorLayout.endVectorLayout(builder)); } - var layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, layoutOffsets); + const layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, layoutOffsets); org.apache.arrow.flatbuf.Field.startField(builder); org.apache.arrow.flatbuf.Field.addName(builder, nameOffset); @@ -374,7 +390,7 @@ function _createDictionaryField(id, field) { org.apache.arrow.flatbuf.Field.addType(builder, typeOffset); org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset); org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset); - var offset = org.apache.arrow.flatbuf.Field.endField(builder); + const offset = org.apache.arrow.flatbuf.Field.endField(builder); builder.finish(offset); return org.apache.arrow.flatbuf.Field.getRootAsField(builder.bb); @@ -387,14 +403,14 @@ function Int32FromByteBuffer(bb, offset) { ((bb.bytes_[offset] & 255)); } -var MAGIC_STR = "ARROW1"; -var MAGIC = new Uint8Array(MAGIC_STR.length); -for (var i = 0; i < MAGIC_STR.length; i += 1|0) { +const MAGIC_STR = "ARROW1"; +const MAGIC = new Uint8Array(MAGIC_STR.length); +for (let i = 0; i < MAGIC_STR.length; i += 1|0) { MAGIC[i] = MAGIC_STR.charCodeAt(i); } function _checkMagic(buf, index) { - for (var i = 0; i < MAGIC.length; i += 1|0) { + for (let i = 0; i < MAGIC.length; i += 1|0) { if (MAGIC[i] != buf[index + i]) { return false; } @@ -402,7 +418,7 @@ function _checkMagic(buf, index) { return true; } -var TYPEMAP = {} +const TYPEMAP = {} TYPEMAP[Type.NONE] = "NONE"; TYPEMAP[Type.Null] = "Null"; TYPEMAP[Type.Int] = "Int"; @@ -420,20 +436,20 @@ TYPEMAP[Type.FixedSizeList] = "FixedSizeList"; TYPEMAP[Type.Struct_] = "Struct"; TYPEMAP[Type.Union] = "Union"; -var VECTORTYPEMAP = {}; +const VECTORTYPEMAP = {}; VECTORTYPEMAP[VectorType.OFFSET] = 'OFFSET'; VECTORTYPEMAP[VectorType.DATA] = 'DATA'; VECTORTYPEMAP[VectorType.VALIDITY] = 'VALIDITY'; VECTORTYPEMAP[VectorType.TYPE] = 'TYPE'; function parseField(field) { - var children = []; - for (var i = 0; i < field.childrenLength(); i += 1|0) { + const children = []; + for (let i = 0; i < field.childrenLength(); i += 1|0) { children.push(parseField(field.children(i))); } - var layouts = []; - for (var i = 0; i < field.layoutLength(); i += 1|0) { + const layouts = []; + for (let i = 0; i < field.layoutLength(); i += 1|0) { layouts.push(VECTORTYPEMAP[field.layout(i).type()]); } @@ -447,17 +463,16 @@ function parseField(field) { } function parseSchema(schema) { - var result = []; - var this_result, type; - for (var i = 0, len = schema.fieldsLength(); i < len; i += 1|0) { + const result = []; + for (let i = 0, len = schema.fieldsLength(); i < len; i += 1|0) { result.push(parseField(schema.fields(i))); } return result; } function loadVectors(bb, vectors: Vector[], recordBatch) { - var indices = { bufferIndex: 0, nodeIndex: 0 }, i; - for (i = 0; i < vectors.length; i += 1) { + const indices = { bufferIndex: 0, nodeIndex: 0 }; + for (let i = 0; i < vectors.length; i += 1) { loadVector(bb, vectors[i], recordBatch, indices); } } @@ -467,7 +482,10 @@ function loadVectors(bb, vectors: Vector[], recordBatch) { * recordBatch: { nodes: org.apache.arrow.flatbuf.FieldNode[], buffers: { offset: number, length: number }[] } */ function loadVector(bb, vector: Vector, recordBatch, indices) { - var node = recordBatch.nodes[indices.nodeIndex], ownBuffersLength, ownBuffers = [], i; + const node = recordBatch.nodes[indices.nodeIndex] + let ownBuffersLength; + const ownBuffers = []; + let i; indices.nodeIndex += 1; // dictionary vectors are always ints, so will have a data vector plus optional null vector @@ -486,7 +504,7 @@ function loadVector(bb, vector: Vector, recordBatch, indices) { vector.loadData(bb, node, ownBuffers); - var children = vector.getChildVectors(); + const children = vector.getChildVectors(); for (i = 0; i < children.length; i++) { loadVector(bb, children[i], recordBatch, indices); } diff --git a/js/src/bitarray.ts b/js/src/bitarray.ts index fc3c0918eaf..82dec4eb148 100644 --- a/js/src/bitarray.ts +++ b/js/src/bitarray.ts @@ -23,20 +23,20 @@ export class BitArray { } get(i) { - var index = (i >> 3) | 0; // | 0 converts to an int. Math.floor works too. - var bit = i % 8; // i % 8 is just as fast as i & 7 + const index = (i >> 3) | 0; // | 0 converts to an int. Math.floor works too. + const bit = i % 8; // i % 8 is just as fast as i & 7 return (this.view[index] & (1 << bit)) !== 0; } set(i) { - var index = (i >> 3) | 0; - var bit = i % 8; + const index = (i >> 3) | 0; + const bit = i % 8; this.view[index] |= 1 << bit; } unset(i) { - var index = (i >> 3) | 0; - var bit = i % 8; + const index = (i >> 3) | 0; + const bit = i % 8; this.view[index] &= ~(1 << bit); } } diff --git a/js/src/types.ts b/js/src/types.ts index d656c6aa98c..9e38e742911 100644 --- a/js/src/types.ts +++ b/js/src/types.ts @@ -19,7 +19,7 @@ import { BitArray } from './bitarray'; import { TextDecoder } from 'text-encoding'; import { org } from './Arrow_generated'; -var Type = org.apache.arrow.flatbuf.Type; +const Type = org.apache.arrow.flatbuf.Type; interface ArrayView { slice(start: number, end: number) : ArrayView @@ -64,8 +64,8 @@ export abstract class Vector { * buffer: org.apache.arrow.flatbuf.Buffer */ static loadValidityBuffer(bb, buffer) : BitArray { - var arrayBuffer = bb.bytes_.buffer; - var offset = bb.bytes_.byteOffset + buffer.offset; + const arrayBuffer = bb.bytes_.buffer; + const offset = bb.bytes_.byteOffset + buffer.offset; return new BitArray(arrayBuffer, offset, buffer.length * 8); } @@ -74,9 +74,9 @@ export abstract class Vector { * buffer: org.apache.arrow.flatbuf.Buffer */ static loadOffsetBuffer(bb, buffer) : Int32Array { - var arrayBuffer = bb.bytes_.buffer; - var offset = bb.bytes_.byteOffset + buffer.offset; - var length = buffer.length / Int32Array.BYTES_PER_ELEMENT; + const arrayBuffer = bb.bytes_.buffer; + const offset = bb.bytes_.byteOffset + buffer.offset; + const length = buffer.length / Int32Array.BYTES_PER_ELEMENT; return new Int32Array(arrayBuffer, offset, length); } @@ -107,9 +107,9 @@ class SimpleVector extends Vector { * buffer: org.apache.arrow.flatbuf.Buffer */ protected loadDataBuffer(bb, buffer) { - var arrayBuffer = bb.bytes_.buffer; - var offset = bb.bytes_.byteOffset + buffer.offset; - var length = buffer.length / this.TypedArray.BYTES_PER_ELEMENT; + const arrayBuffer = bb.bytes_.buffer; + const offset = bb.bytes_.byteOffset + buffer.offset; + const length = buffer.length / this.TypedArray.BYTES_PER_ELEMENT; this.dataView = new this.TypedArray(arrayBuffer, offset, length); } @@ -263,8 +263,8 @@ class Utf8Vector extends SimpleVector { } slice(start: number, end: number) { - var result: string[] = []; - for (var i: number = start; i < end; i += 1|0) { + const result: string[] = []; + for (let i: number = start; i < end; i += 1|0) { result.push(this.get(i)); } return result; @@ -316,11 +316,11 @@ class ListVector extends Uint32Vector { } get(i) { - var offset = super.get(i) + const offset = super.get(i) if (offset === null) { return null; } - var next_offset = super.get(i + 1) + const next_offset = super.get(i + 1) return this.dataVector.slice(offset, next_offset) } @@ -329,8 +329,8 @@ class ListVector extends Uint32Vector { } slice(start: number, end: number) { - var result = []; - for (var i = start; i < end; i += 1|0) { + const result = []; + for (let i = start; i < end; i += 1|0) { result.push(this.get(i)); } return result; @@ -382,8 +382,8 @@ class FixedSizeListVector extends Vector { } slice(start : number, end : number) { - var result = []; - for (var i = start; i < end; i += 1|0) { + const result = []; + for (let i = start; i < end; i += 1|0) { result.push(this.get(i)); } return result; @@ -440,8 +440,8 @@ class StructVector extends Vector { } slice(start : number, end : number) { - var result = []; - for (var i = start; i < end; i += 1|0) { + const result = []; + for (let i = start; i < end; i += 1|0) { result.push(this.get(i)); } return result; @@ -464,7 +464,7 @@ class DictionaryVector extends Vector { } get(i) { - var encoded = this.indices.get(i); + const encoded = this.indices.get(i); if (encoded == null) { return null; } else { @@ -505,32 +505,33 @@ class DictionaryVector extends Vector { } export function vectorFromField(field, dictionaries) : Vector { - var dictionary = field.dictionary(), nullable = field.nullable(); + const dictionary = field.dictionary(); + const nullable = field.nullable(); if (dictionary == null) { - var typeType = field.typeType(); + const typeType = field.typeType(); if (typeType === Type.List) { - var dataVector = vectorFromField(field.children(0), dictionaries); + const dataVector = vectorFromField(field.children(0), dictionaries); return nullable ? new NullableListVector(field, dataVector) : new ListVector(field, dataVector); } else if (typeType === Type.FixedSizeList) { - var dataVector = vectorFromField(field.children(0), dictionaries); - var size = field.type(new org.apache.arrow.flatbuf.FixedSizeList()).listSize(); + const dataVector = vectorFromField(field.children(0), dictionaries); + const size = field.type(new org.apache.arrow.flatbuf.FixedSizeList()).listSize(); if (nullable) { return new NullableFixedSizeListVector(field, size, dataVector); } else { return new FixedSizeListVector(field, size, dataVector); } } else if (typeType === Type.Struct_) { - var vectors : Vector[] = []; - for (var i : number = 0; i < field.childrenLength(); i += 1|0) { + const vectors : Vector[] = []; + for (let i : number = 0; i < field.childrenLength(); i += 1|0) { vectors.push(vectorFromField(field.children(i), dictionaries)); } return new StructVector(field, vectors); } else { if (typeType === Type.Int) { - var type = field.type(new org.apache.arrow.flatbuf.Int()); + const type = field.type(new org.apache.arrow.flatbuf.Int()); return _createIntVector(field, type.bitWidth(), type.isSigned(), nullable) } else if (typeType === Type.FloatingPoint) { - var precision = field.type(new org.apache.arrow.flatbuf.FloatingPoint()).precision(); + const precision = field.type(new org.apache.arrow.flatbuf.FloatingPoint()).precision(); if (precision == org.apache.arrow.flatbuf.Precision.SINGLE) { return nullable ? new NullableFloat32Vector(field) : new Float32Vector(field); } else if (precision == org.apache.arrow.flatbuf.Precision.DOUBLE) { @@ -548,12 +549,14 @@ export function vectorFromField(field, dictionaries) : Vector { } } else { // determine arrow type - default is signed 32 bit int - var type = dictionary.indexType(), bitWidth = 32, signed = true; + const type = dictionary.indexType(); + let bitWidth = 32; + let signed = true; if (type != null) { bitWidth = type.bitWidth(); signed = type.isSigned(); } - var indices = _createIntVector(field, bitWidth, signed, nullable); + const indices = _createIntVector(field, bitWidth, signed, nullable); return new DictionaryVector(field, indices, dictionaries[dictionary.id().toFloat64().toString()]); } } From 2b4ff287870c7d3eb69b1b92653555834aac6a97 Mon Sep 17 00:00:00 2001 From: Brian Hulette Date: Sun, 28 May 2017 22:14:23 -0400 Subject: [PATCH 03/10] added public, private, protected to all members --- js/src/arrow.ts | 14 ++--- js/src/bitarray.ts | 6 +-- js/src/types.ts | 127 ++++++++++++++++++++++----------------------- 3 files changed, 72 insertions(+), 75 deletions(-) diff --git a/js/src/arrow.ts b/js/src/arrow.ts index 0e0fdfa1397..4a15d1a63a1 100644 --- a/js/src/arrow.ts +++ b/js/src/arrow.ts @@ -50,7 +50,7 @@ export class ArrowReader { this.dictionaries = dictionaries; } - loadNextBatch() { + public loadNextBatch() { if (this.batchIndex < this.batches.length) { const batch = this.batches[this.batchIndex]; this.batchIndex += 1; @@ -61,29 +61,29 @@ export class ArrowReader { } } - getSchema() { + public getSchema() { return this.schema; } - getVectors() { + public getVectors() { return this.vectors; } - getVector(name) { + public getVector(name) { return this.vectorMap[name]; } - getBatchCount() { + public getBatchCount() { return this.batches.length; } // the index of the next batch to be loaded - getBatchIndex() { + public getBatchIndex() { return this.batchIndex; } // set the index of the next batch to be loaded - setBatchIndex(i: number) { + public setBatchIndex(i: number) { this.batchIndex = i; } } diff --git a/js/src/bitarray.ts b/js/src/bitarray.ts index 82dec4eb148..6b0a91aed60 100644 --- a/js/src/bitarray.ts +++ b/js/src/bitarray.ts @@ -22,19 +22,19 @@ export class BitArray { this.view = new Uint8Array(buffer, offset || 0, Math.ceil(length / 8)); } - get(i) { + public get(i) { const index = (i >> 3) | 0; // | 0 converts to an int. Math.floor works too. const bit = i % 8; // i % 8 is just as fast as i & 7 return (this.view[index] & (1 << bit)) !== 0; } - set(i) { + public set(i) { const index = (i >> 3) | 0; const bit = i % 8; this.view[index] |= 1 << bit; } - unset(i) { + public unset(i) { const index = (i >> 3) | 0; const bit = i % 8; this.view[index] &= ~(1 << bit); diff --git a/js/src/types.ts b/js/src/types.ts index 9e38e742911..2c54f2570c8 100644 --- a/js/src/types.ts +++ b/js/src/types.ts @@ -27,10 +27,10 @@ interface ArrayView { } export abstract class Vector { - field: any; - name: string; - length: number; - null_count: number; + public field: any; + public name: string; + public length: number; + public null_count: number; constructor(field) { this.field = field; @@ -38,11 +38,13 @@ export abstract class Vector { } /* Access datum at index i */ - abstract get(i); + public abstract get(i); /* Return array representing data in the range [start, end) */ - abstract slice(start: number, end: number); + public abstract slice(start: number, end: number); /* Return array of child vectors, for container types */ - abstract getChildVectors(); + public getChildVectors() { + return []; + } /** * Use recordBatch fieldNodes and Buffers to construct this Vector @@ -63,7 +65,7 @@ export abstract class Vector { * bb: flatbuffers.ByteBuffer * buffer: org.apache.arrow.flatbuf.Buffer */ - static loadValidityBuffer(bb, buffer) : BitArray { + public static loadValidityBuffer(bb, buffer) : BitArray { const arrayBuffer = bb.bytes_.buffer; const offset = bb.bytes_.byteOffset + buffer.offset; return new BitArray(arrayBuffer, offset, buffer.length * 8); @@ -73,7 +75,7 @@ export abstract class Vector { * Helper function for loading an OFFSET buffer * buffer: org.apache.arrow.flatbuf.Buffer */ - static loadOffsetBuffer(bb, buffer) : Int32Array { + public static loadOffsetBuffer(bb, buffer) : Int32Array { const arrayBuffer = bb.bytes_.buffer; const offset = bb.bytes_.byteOffset + buffer.offset; const length = buffer.length / Int32Array.BYTES_PER_ELEMENT; @@ -91,15 +93,11 @@ class SimpleVector extends Vector { this.TypedArray = TypedArray; } - getChildVectors() { - return []; - } - - get(i) { + public get(i) { return this.dataView[i]; } - loadBuffers(bb, node, buffers) { + protected loadBuffers(bb, node, buffers) { this.loadDataBuffer(bb, buffers[0]); } @@ -113,15 +111,15 @@ class SimpleVector extends Vector { this.dataView = new this.TypedArray(arrayBuffer, offset, length); } - getDataView() { + public getDataView() { return this.dataView; } - toString() { + public toString() { return this.dataView.toString(); } - slice(start, end) { + public slice(start, end) { return this.dataView.slice(start, end); } } @@ -130,7 +128,7 @@ class NullableSimpleVector extends SimpleVector { protected validityView: BitArray; - get(i: number) { + public get(i: number) { if (this.validityView.get(i)) { return this.dataView[i]; } else { @@ -138,12 +136,12 @@ class NullableSimpleVector extends SimpleVector { } } - loadBuffers(bb, node, buffers) { + protected loadBuffers(bb, node, buffers) { this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); this.loadDataBuffer(bb, buffers[1]); } - getValidityVector() { + public getValidityVector() { return this.validityView; } } @@ -171,7 +169,7 @@ class Uint64Vector extends SimpleVector { super(field, Uint32Array); } - get(i: number) { + public get(i: number) { return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] }; } } @@ -181,7 +179,7 @@ class NullableUint64Vector extends NullableSimpleVector { super(field, Uint32Array); } - get(i: number) { + public get(i: number) { if (this.validityView.get(i)) { return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] }; } else { @@ -195,7 +193,7 @@ class Int64Vector extends NullableSimpleVector { super(field, Uint32Array); } - get(i: number) { + public get(i: number) { return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] }; } } @@ -205,7 +203,7 @@ class NullableInt64Vector extends NullableSimpleVector { super(field, Uint32Array); } - get(i: number) { + public get(i: number) { if (this.validityView.get(i)) { return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] }; } else { @@ -219,7 +217,7 @@ class DateVector extends SimpleVector { super(field, Uint32Array); } - get (i) { + public get (i) { return new Date(super.get(2*i+1)*Math.pow(2,32) + super.get(2*i)); } } @@ -227,12 +225,12 @@ class DateVector extends SimpleVector { class NullableDateVector extends DateVector { private validityView: BitArray; - loadBuffers(bb, node, buffers) { + protected loadBuffers(bb, node, buffers) { this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); this.loadDataBuffer(bb, buffers[1]); } - get (i) { + public get (i) { if (this.validityView.get(i)) { return super.get(i); } else { @@ -240,29 +238,29 @@ class NullableDateVector extends DateVector { } } - getValidityVector() { + public getValidityVector() { return this.validityView; } } class Utf8Vector extends SimpleVector { protected offsetView: Int32Array; - static decoder: TextDecoder = new TextDecoder('utf8'); + private static decoder: TextDecoder = new TextDecoder('utf8'); constructor(field) { super(field, Uint8Array); } - loadBuffers(bb, node, buffers) { + protected loadBuffers(bb, node, buffers) { this.offsetView = Vector.loadOffsetBuffer(bb, buffers[0]); this.loadDataBuffer(bb, buffers[1]); } - get(i) { + public get(i) { return Utf8Vector.decoder.decode(this.dataView.slice(this.offsetView[i], this.offsetView[i + 1])); } - slice(start: number, end: number) { + public slice(start: number, end: number) { const result: string[] = []; for (let i: number = start; i < end; i += 1|0) { result.push(this.get(i)); @@ -270,7 +268,7 @@ class Utf8Vector extends SimpleVector { return result; } - getOffsetView() { + public getOffsetView() { return this.offsetView; } } @@ -278,13 +276,13 @@ class Utf8Vector extends SimpleVector { class NullableUtf8Vector extends Utf8Vector { private validityView: BitArray; - loadBuffers(bb, node, buffers) { + protected loadBuffers(bb, node, buffers) { this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); this.offsetView = Vector.loadOffsetBuffer(bb, buffers[1]); this.loadDataBuffer(bb, buffers[2]); } - get(i) { + public get(i) { if (this.validityView.get(i)) { return super.get(i); } else { @@ -292,7 +290,7 @@ class NullableUtf8Vector extends Utf8Vector { } } - getValidityVector() { + public getValidityVector() { return this.validityView; } } @@ -306,16 +304,16 @@ class ListVector extends Uint32Vector { this.dataVector = dataVector; } - getChildVectors() { + public getChildVectors() { return [this.dataVector]; } - loadBuffers(bb, node, buffers) { + protected loadBuffers(bb, node, buffers) { super.loadBuffers(bb, node, buffers); this.length -= 1; } - get(i) { + public get(i) { const offset = super.get(i) if (offset === null) { return null; @@ -324,11 +322,11 @@ class ListVector extends Uint32Vector { return this.dataVector.slice(offset, next_offset) } - toString() { + public toString() { return "length: " + (this.length); } - slice(start: number, end: number) { + public slice(start: number, end: number) { const result = []; for (let i = start; i < end; i += 1|0) { result.push(this.get(i)); @@ -340,13 +338,13 @@ class ListVector extends Uint32Vector { class NullableListVector extends ListVector { private validityView: BitArray; - loadBuffers(bb, node, buffers) { + protected loadBuffers(bb, node, buffers) { this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); this.loadDataBuffer(bb, buffers[1]); this.length -= 1; } - get(i) { + public get(i) { if (this.validityView.get(i)) { return super.get(i); } else { @@ -354,13 +352,13 @@ class NullableListVector extends ListVector { } } - getValidityVector() { + public getValidityVector() { return this.validityView; } } class FixedSizeListVector extends Vector { - private size: number + public size: number private dataVector: Vector; constructor(field, size: number, dataVector: Vector) { @@ -369,19 +367,19 @@ class FixedSizeListVector extends Vector { this.dataVector = dataVector; } - getChildVectors() { + public getChildVectors() { return [this.dataVector]; } - loadBuffers(bb, node, buffers) { + protected loadBuffers(bb, node, buffers) { // no buffers to load } - get(i: number) { + public get(i: number) { return this.dataVector.slice(i * this.size, (i + 1) * this.size); } - slice(start : number, end : number) { + public slice(start : number, end : number) { const result = []; for (let i = start; i < end; i += 1|0) { result.push(this.get(i)); @@ -389,7 +387,7 @@ class FixedSizeListVector extends Vector { return result; } - getListSize() { + public getListSize() { return this.size; } } @@ -397,11 +395,11 @@ class FixedSizeListVector extends Vector { class NullableFixedSizeListVector extends FixedSizeListVector { private validityView: BitArray; - loadBuffers(bb, node, buffers) { + protected loadBuffers(bb, node, buffers) { this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); } - get(i: number) { + public get(i: number) { if (this.validityView.get(i)) { return super.get(i); } else { @@ -409,7 +407,7 @@ class NullableFixedSizeListVector extends FixedSizeListVector { } } - getValidityVector() { + public getValidityVector() { return this.validityView; } } @@ -423,15 +421,15 @@ class StructVector extends Vector { this.vectors = vectors; } - getChildVectors() { + public getChildVectors() { return this.vectors; } - loadBuffers(bb, node, buffers) { + public loadBuffers(bb, node, buffers) { this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); } - get(i : number) { + public get(i : number) { if (this.validityView.get(i)) { return this.vectors.map((v: Vector) => v.get(i)); } else { @@ -439,7 +437,7 @@ class StructVector extends Vector { } } - slice(start : number, end : number) { + public slice(start : number, end : number) { const result = []; for (let i = start; i < end; i += 1|0) { result.push(this.get(i)); @@ -447,13 +445,12 @@ class StructVector extends Vector { return result; } - getValidityVector() { + public getValidityVector() { return this.validityView; } } class DictionaryVector extends Vector { - private indices: Vector; private dictionary: Vector; @@ -463,7 +460,7 @@ class DictionaryVector extends Vector { this.dictionary = dictionary; } - get(i) { + public get(i) { const encoded = this.indices.get(i); if (encoded == null) { return null; @@ -477,15 +474,15 @@ class DictionaryVector extends Vector { return this.indices.get(i); } - slice(start, end) { + public slice(start, end) { return this.indices.slice(start, end); // TODO decode } - getChildVectors() { + public getChildVectors() { return this.indices.getChildVectors(); } - loadBuffers(bb, node, buffers) { + protected loadBuffers(bb, node, buffers) { this.indices.loadData(bb, node, buffers); } @@ -499,7 +496,7 @@ class DictionaryVector extends Vector { return this.dictionary; } - toString() { + public toString() { return this.indices.toString(); } } From c5d85f7786e6470e9500c767438903bdb25a1bcc Mon Sep 17 00:00:00 2001 From: Brian Hulette Date: Sun, 28 May 2017 22:46:42 -0400 Subject: [PATCH 04/10] whitespace, semicolons, Errors --- js/src/arrow.ts | 87 +++++++++++++++++++++++------------------------- js/src/types.ts | 88 ++++++++++++++++++++++++------------------------- 2 files changed, 85 insertions(+), 90 deletions(-) diff --git a/js/src/arrow.ts b/js/src/arrow.ts index 4a15d1a63a1..026ccc840a7 100644 --- a/js/src/arrow.ts +++ b/js/src/arrow.ts @@ -43,8 +43,8 @@ export class ArrowReader { this.bb = bb; this.schema = schema; this.vectors = vectors; - for (let i = 0; i < vectors.length; i += 1|0) { - this.vectorMap[vectors[i].name] = vectors[i] + for (let i = 0; i < vectors.length; i++) { + this.vectorMap[vectors[i].name] = vectors[i]; } this.batches = batches; this.dictionaries = dictionaries; @@ -90,7 +90,7 @@ export class ArrowReader { export function getSchema(buf) { return getReader(buf).getSchema(); } -export function getReader(buf) : ArrowReader { +export function getReader(buf): ArrowReader { if (_checkMagic(buf, 0)) { return getFileReader(buf); } else { @@ -98,7 +98,7 @@ export function getReader(buf) : ArrowReader { } } -export function getStreamReader(buf) : ArrowReader { +export function getStreamReader(buf): ArrowReader { const bb = new ByteBuffer(buf); const schema = _loadSchema(bb); @@ -111,7 +111,7 @@ export function getStreamReader(buf) : ArrowReader { const dictionaryBatches = []; const dictionaries = {}; - for (i = 0, iLen = schema.fieldsLength(); i < iLen; i += 1|0) { + for (i = 0, iLen = schema.fieldsLength(); i < iLen; i++) { field = schema.fields(i); _createDictionaryVectors(field, dictionaries); vectors.push(vectorFromField(field, dictionaries)); @@ -124,15 +124,15 @@ export function getStreamReader(buf) : ArrowReader { } else if (batch.type == MessageHeader.DictionaryBatch) { dictionaryBatches.push(batch); } else if (batch.type == MessageHeader.RecordBatch) { - recordBatches.push(batch) + recordBatches.push(batch); } else { - console.error("Expected batch type" + MessageHeader.RecordBatch + " or " + + throw new Error("Expected batch type" + MessageHeader.RecordBatch + " or " + MessageHeader.DictionaryBatch + " but got " + batch.type); } } // load dictionary vectors - for (i = 0; i < dictionaryBatches.length; i += 1|0) { + for (i = 0; i < dictionaryBatches.length; i++) { batch = dictionaryBatches[i]; loadVectors(bb, [dictionaries[batch.id]], batch); } @@ -140,7 +140,7 @@ export function getStreamReader(buf) : ArrowReader { return new ArrowReader(bb, parseSchema(schema), vectors, recordBatches, dictionaries); } -export function getFileReader (buf) : ArrowReader { +export function getFileReader(buf): ArrowReader { const bb = new ByteBuffer(buf); const footer = _loadFooter(bb); @@ -156,44 +156,44 @@ export function getFileReader (buf) : ArrowReader { const dictionaryBatchBlocks = []; const dictionaries = {}; - for (i = 0, len = schema.fieldsLength(); i < len; i += 1|0) { + for (i = 0, len = schema.fieldsLength(); i < len; i++) { field = schema.fields(i); _createDictionaryVectors(field, dictionaries); vectors.push(vectorFromField(field, dictionaries)); } - for (i = 0; i < footer.dictionariesLength(); i += 1|0) { + for (i = 0; i < footer.dictionariesLength(); i++) { block = footer.dictionaries(i); dictionaryBatchBlocks.push({ offset: block.offset().low, metaDataLength: block.metaDataLength(), bodyLength: block.bodyLength().low, - }) + }); } - for (i = 0; i < footer.recordBatchesLength(); i += 1|0) { + for (i = 0; i < footer.recordBatchesLength(); i++) { block = footer.recordBatches(i); recordBatchBlocks.push({ offset: block.offset().low, metaDataLength: block.metaDataLength(), bodyLength: block.bodyLength().low, - }) + }); } - const dictionaryBatches = dictionaryBatchBlocks.map(function (block) { + const dictionaryBatches = dictionaryBatchBlocks.map(function(block) { bb.setPosition(block.offset); // TODO: Make sure this is a dictionary batch return _loadBatch(bb); }); - const recordBatches = recordBatchBlocks.map(function (block) { + const recordBatches = recordBatchBlocks.map(function(block) { bb.setPosition(block.offset); // TODO: Make sure this is a record batch return _loadBatch(bb); }); // load dictionary vectors - for (i = 0; i < dictionaryBatches.length; i += 1|0) { + for (i = 0; i < dictionaryBatches.length; i++) { batch = dictionaryBatches[i]; loadVectors(bb, [dictionaries[batch.id]], batch); } @@ -204,27 +204,24 @@ export function getFileReader (buf) : ArrowReader { function _loadFooter(bb) { const fileLength: number = bb.bytes_.length; - if (fileLength < MAGIC.length*2 + 4) { - console.error("file too small " + fileLength); - return; + if (fileLength < MAGIC.length * 2 + 4) { + throw new Error("file too small " + fileLength); } if (!_checkMagic(bb.bytes_, 0)) { - console.error("missing magic bytes at beginning of file") - return; + throw new Error("missing magic bytes at beginning of file"); } if (!_checkMagic(bb.bytes_, fileLength - MAGIC.length)) { - console.error("missing magic bytes at end of file") - return; + throw new Error("missing magic bytes at end of file"); } const footerLengthOffset: number = fileLength - MAGIC.length - 4; bb.setPosition(footerLengthOffset); - const footerLength: number = Int32FromByteBuffer(bb, footerLengthOffset) + const footerLength: number = Int32FromByteBuffer(bb, footerLengthOffset); - if (footerLength <= 0 || footerLength + MAGIC.length*2 + 4 > fileLength) { - console.log("Invalid footer length: " + footerLength) + if (footerLength <= 0 || footerLength + MAGIC.length * 2 + 4 > fileLength) { + throw new Error("Invalid footer length: " + footerLength); } const footerOffset: number = footerLengthOffset - footerLength; @@ -235,10 +232,9 @@ function _loadFooter(bb) { } function _loadSchema(bb) { - const message =_loadMessage(bb); + const message = _loadMessage(bb); if (message.headerType() != MessageHeader.Schema) { - console.error("Expected header type " + MessageHeader.Schema + " but got " + message.headerType()); - return; + throw new Error("Expected header type " + MessageHeader.Schema + " but got " + message.headerType()); } return message.header(new Schema()); } @@ -248,15 +244,14 @@ function _loadBatch(bb) { if (message == null) { return; } else if (message.headerType() == MessageHeader.RecordBatch) { - const batch = { header: message.header(new RecordBatch()), length: message.bodyLength().low } + const batch = { header: message.header(new RecordBatch()), length: message.bodyLength().low }; return _loadRecordBatch(bb, batch); } else if (message.headerType() == MessageHeader.DictionaryBatch) { - const batch = { header: message.header(new DictionaryBatch()), length: message.bodyLength().low } + const batch = { header: message.header(new DictionaryBatch()), length: message.bodyLength().low }; return _loadDictionaryBatch(bb, batch); } else { - console.error("Expected header type " + MessageHeader.RecordBatch + " or " + MessageHeader.DictionaryBatch + + throw new Error("Expected header type " + MessageHeader.RecordBatch + " or " + MessageHeader.DictionaryBatch + " but got " + message.headerType()); - return; } } @@ -266,7 +261,7 @@ function _loadRecordBatch(bb, batch) { const nodes_ = []; const nodesLength = data.nodesLength(); let buffer; - const buffers_ = [] + const buffers_ = []; const buffersLength = data.buffersLength(); for (i = 0; i < nodesLength; i += 1) { @@ -334,7 +329,7 @@ function _createDictionaryVectors(field, dictionaries) { } // recursively examine child fields - for (let i = 0, len = field.childrenLength(); i < len; i += 1|0) { + for (let i = 0, len = field.childrenLength(); i < len; i++) { _createDictionaryVectors(field.children(i), dictionaries); } } @@ -365,16 +360,16 @@ function _createDictionaryField(id, field) { org.apache.arrow.flatbuf.Date.addUnit(builder, type.unit()); typeOffset = org.apache.arrow.flatbuf.Date.endDate(builder); } else { - throw "Unimplemented dictionary type " + typeType; + throw new Error("Unimplemented dictionary type " + typeType); } if (field.childrenLength() > 0) { - throw "Dictionary encoded fields can't have children" + throw new Error("Dictionary encoded fields can't have children"); } const childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, []); let layout; const layoutOffsets = []; - for (let i = 0, len = field.layoutLength(); i < len; i += 1|0) { + for (let i = 0, len = field.layoutLength(); i < len; i++) { layout = field.layout(i); org.apache.arrow.flatbuf.VectorLayout.startVectorLayout(builder); org.apache.arrow.flatbuf.VectorLayout.addBitWidth(builder, layout.bitWidth()); @@ -405,12 +400,12 @@ function Int32FromByteBuffer(bb, offset) { const MAGIC_STR = "ARROW1"; const MAGIC = new Uint8Array(MAGIC_STR.length); -for (let i = 0; i < MAGIC_STR.length; i += 1|0) { +for (let i = 0; i < MAGIC_STR.length; i++) { MAGIC[i] = MAGIC_STR.charCodeAt(i); } function _checkMagic(buf, index) { - for (let i = 0; i < MAGIC.length; i += 1|0) { + for (let i = 0; i < MAGIC.length; i++) { if (MAGIC[i] != buf[index + i]) { return false; } @@ -418,7 +413,7 @@ function _checkMagic(buf, index) { return true; } -const TYPEMAP = {} +const TYPEMAP = {}; TYPEMAP[Type.NONE] = "NONE"; TYPEMAP[Type.Null] = "Null"; TYPEMAP[Type.Int] = "Int"; @@ -444,12 +439,12 @@ VECTORTYPEMAP[VectorType.TYPE] = 'TYPE'; function parseField(field) { const children = []; - for (let i = 0; i < field.childrenLength(); i += 1|0) { + for (let i = 0; i < field.childrenLength(); i++) { children.push(parseField(field.children(i))); } const layouts = []; - for (let i = 0; i < field.layoutLength(); i += 1|0) { + for (let i = 0; i < field.layoutLength(); i++) { layouts.push(VECTORTYPEMAP[field.layout(i).type()]); } @@ -464,7 +459,7 @@ function parseField(field) { function parseSchema(schema) { const result = []; - for (let i = 0, len = schema.fieldsLength(); i < len; i += 1|0) { + for (let i = 0, len = schema.fieldsLength(); i < len; i++) { result.push(parseField(schema.fields(i))); } return result; @@ -482,7 +477,7 @@ function loadVectors(bb, vectors: Vector[], recordBatch) { * recordBatch: { nodes: org.apache.arrow.flatbuf.FieldNode[], buffers: { offset: number, length: number }[] } */ function loadVector(bb, vector: Vector, recordBatch, indices) { - const node = recordBatch.nodes[indices.nodeIndex] + const node = recordBatch.nodes[indices.nodeIndex]; let ownBuffersLength; const ownBuffers = []; let i; diff --git a/js/src/types.ts b/js/src/types.ts index 2c54f2570c8..94d25eaf32b 100644 --- a/js/src/types.ts +++ b/js/src/types.ts @@ -22,8 +22,8 @@ import { org } from './Arrow_generated'; const Type = org.apache.arrow.flatbuf.Type; interface ArrayView { - slice(start: number, end: number) : ArrayView - toString() : string + slice(start: number, end: number): ArrayView; + toString(): string; } export abstract class Vector { @@ -65,7 +65,7 @@ export abstract class Vector { * bb: flatbuffers.ByteBuffer * buffer: org.apache.arrow.flatbuf.Buffer */ - public static loadValidityBuffer(bb, buffer) : BitArray { + public static loadValidityBuffer(bb, buffer): BitArray { const arrayBuffer = bb.bytes_.buffer; const offset = bb.bytes_.byteOffset + buffer.offset; return new BitArray(arrayBuffer, offset, buffer.length * 8); @@ -75,7 +75,7 @@ export abstract class Vector { * Helper function for loading an OFFSET buffer * buffer: org.apache.arrow.flatbuf.Buffer */ - public static loadOffsetBuffer(bb, buffer) : Int32Array { + public static loadOffsetBuffer(bb, buffer): Int32Array { const arrayBuffer = bb.bytes_.buffer; const offset = bb.bytes_.byteOffset + buffer.offset; const length = buffer.length / Int32Array.BYTES_PER_ELEMENT; @@ -88,7 +88,7 @@ class SimpleVector extends Vector { protected dataView: T; private TypedArray: { new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number }; - constructor (field, TypedArray: { new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number }) { + constructor(field, TypedArray: { new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number }) { super(field); this.TypedArray = TypedArray; } @@ -146,23 +146,23 @@ class NullableSimpleVector extends SimpleVector { } } -class Uint8Vector extends SimpleVector { constructor(field) { super(field, Uint8Array); }; } -class Uint16Vector extends SimpleVector { constructor(field) { super(field, Uint16Array); }; } -class Uint32Vector extends SimpleVector { constructor(field) { super(field, Uint32Array); }; } -class Int8Vector extends SimpleVector { constructor(field) { super(field, Uint8Array); }; } -class Int16Vector extends SimpleVector { constructor(field) { super(field, Uint16Array); }; } -class Int32Vector extends SimpleVector { constructor(field) { super(field, Uint32Array); }; } -class Float32Vector extends SimpleVector { constructor(field) { super(field, Float32Array); }; } -class Float64Vector extends SimpleVector { constructor(field) { super(field, Float64Array); }; } - -class NullableUint8Vector extends NullableSimpleVector { constructor(field) { super(field, Uint8Array); }; } -class NullableUint16Vector extends NullableSimpleVector { constructor(field) { super(field, Uint16Array); }; } -class NullableUint32Vector extends NullableSimpleVector { constructor(field) { super(field, Uint32Array); }; } -class NullableInt8Vector extends NullableSimpleVector { constructor(field) { super(field, Uint8Array); }; } -class NullableInt16Vector extends NullableSimpleVector { constructor(field) { super(field, Uint16Array); }; } -class NullableInt32Vector extends NullableSimpleVector { constructor(field) { super(field, Uint32Array); }; } -class NullableFloat32Vector extends NullableSimpleVector { constructor(field) { super(field, Float32Array); }; } -class NullableFloat64Vector extends NullableSimpleVector { constructor(field) { super(field, Float64Array); }; } +class Uint8Vector extends SimpleVector { constructor(field) { super(field, Uint8Array); } } +class Uint16Vector extends SimpleVector { constructor(field) { super(field, Uint16Array); } } +class Uint32Vector extends SimpleVector { constructor(field) { super(field, Uint32Array); } } +class Int8Vector extends SimpleVector { constructor(field) { super(field, Uint8Array); } } +class Int16Vector extends SimpleVector { constructor(field) { super(field, Uint16Array); } } +class Int32Vector extends SimpleVector { constructor(field) { super(field, Uint32Array); } } +class Float32Vector extends SimpleVector { constructor(field) { super(field, Float32Array); } } +class Float64Vector extends SimpleVector { constructor(field) { super(field, Float64Array); } } + +class NullableUint8Vector extends NullableSimpleVector { constructor(field) { super(field, Uint8Array); } } +class NullableUint16Vector extends NullableSimpleVector { constructor(field) { super(field, Uint16Array); } } +class NullableUint32Vector extends NullableSimpleVector { constructor(field) { super(field, Uint32Array); } } +class NullableInt8Vector extends NullableSimpleVector { constructor(field) { super(field, Uint8Array); } } +class NullableInt16Vector extends NullableSimpleVector { constructor(field) { super(field, Uint16Array); } } +class NullableInt32Vector extends NullableSimpleVector { constructor(field) { super(field, Uint32Array); } } +class NullableFloat32Vector extends NullableSimpleVector { constructor(field) { super(field, Float32Array); } } +class NullableFloat64Vector extends NullableSimpleVector { constructor(field) { super(field, Float64Array); } } class Uint64Vector extends SimpleVector { constructor(field) { @@ -217,8 +217,8 @@ class DateVector extends SimpleVector { super(field, Uint32Array); } - public get (i) { - return new Date(super.get(2*i+1)*Math.pow(2,32) + super.get(2*i)); + public get(i) { + return new Date(super.get(2 * i + 1) * Math.pow(2, 32) + super.get(2 * i)); } } @@ -230,7 +230,7 @@ class NullableDateVector extends DateVector { this.loadDataBuffer(bb, buffers[1]); } - public get (i) { + public get(i) { if (this.validityView.get(i)) { return super.get(i); } else { @@ -262,7 +262,7 @@ class Utf8Vector extends SimpleVector { public slice(start: number, end: number) { const result: string[] = []; - for (let i: number = start; i < end; i += 1|0) { + for (let i: number = start; i < end; i++) { result.push(this.get(i)); } return result; @@ -314,12 +314,12 @@ class ListVector extends Uint32Vector { } public get(i) { - const offset = super.get(i) + const offset = super.get(i); if (offset === null) { return null; } - const next_offset = super.get(i + 1) - return this.dataVector.slice(offset, next_offset) + const next_offset = super.get(i + 1); + return this.dataVector.slice(offset, next_offset); } public toString() { @@ -328,7 +328,7 @@ class ListVector extends Uint32Vector { public slice(start: number, end: number) { const result = []; - for (let i = start; i < end; i += 1|0) { + for (let i = start; i < end; i++) { result.push(this.get(i)); } return result; @@ -358,7 +358,7 @@ class NullableListVector extends ListVector { } class FixedSizeListVector extends Vector { - public size: number + public size: number; private dataVector: Vector; constructor(field, size: number, dataVector: Vector) { @@ -379,9 +379,9 @@ class FixedSizeListVector extends Vector { return this.dataVector.slice(i * this.size, (i + 1) * this.size); } - public slice(start : number, end : number) { + public slice(start: number, end: number) { const result = []; - for (let i = start; i < end; i += 1|0) { + for (let i = start; i < end; i++) { result.push(this.get(i)); } return result; @@ -429,7 +429,7 @@ class StructVector extends Vector { this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); } - public get(i : number) { + public get(i: number) { if (this.validityView.get(i)) { return this.vectors.map((v: Vector) => v.get(i)); } else { @@ -437,9 +437,9 @@ class StructVector extends Vector { } } - public slice(start : number, end : number) { + public slice(start: number, end: number) { const result = []; - for (let i = start; i < end; i += 1|0) { + for (let i = start; i < end; i++) { result.push(this.get(i)); } return result; @@ -454,7 +454,7 @@ class DictionaryVector extends Vector { private indices: Vector; private dictionary: Vector; - constructor (field, indices: Vector, dictionary: Vector) { + constructor(field, indices: Vector, dictionary: Vector) { super(field); this.indices = indices; this.dictionary = dictionary; @@ -501,7 +501,7 @@ class DictionaryVector extends Vector { } } -export function vectorFromField(field, dictionaries) : Vector { +export function vectorFromField(field, dictionaries): Vector { const dictionary = field.dictionary(); const nullable = field.nullable(); if (dictionary == null) { @@ -518,15 +518,15 @@ export function vectorFromField(field, dictionaries) : Vector { return new FixedSizeListVector(field, size, dataVector); } } else if (typeType === Type.Struct_) { - const vectors : Vector[] = []; - for (let i : number = 0; i < field.childrenLength(); i += 1|0) { + const vectors: Vector[] = []; + for (let i: number = 0; i < field.childrenLength(); i++) { vectors.push(vectorFromField(field.children(i), dictionaries)); } return new StructVector(field, vectors); } else { if (typeType === Type.Int) { const type = field.type(new org.apache.arrow.flatbuf.Int()); - return _createIntVector(field, type.bitWidth(), type.isSigned(), nullable) + return _createIntVector(field, type.bitWidth(), type.isSigned(), nullable); } else if (typeType === Type.FloatingPoint) { const precision = field.type(new org.apache.arrow.flatbuf.FloatingPoint()).precision(); if (precision == org.apache.arrow.flatbuf.Precision.SINGLE) { @@ -534,14 +534,14 @@ export function vectorFromField(field, dictionaries) : Vector { } else if (precision == org.apache.arrow.flatbuf.Precision.DOUBLE) { return nullable ? new NullableFloat64Vector(field) : new Float64Vector(field); } else { - throw "Unimplemented FloatingPoint precision " + precision; + throw new Error("Unimplemented FloatingPoint precision " + precision); } } else if (typeType === Type.Utf8) { return nullable ? new NullableUtf8Vector(field) : new Utf8Vector(field); } else if (typeType === Type.Date) { return nullable ? new NullableDateVector(field) : new DateVector(field); } else { - throw "Unimplemented type " + typeType; + throw new Error("Unimplemented type " + typeType); } } } else { @@ -584,6 +584,6 @@ function _createIntVector(field, bitWidth, signed, nullable) { return nullable ? new NullableUint8Vector(field) : new Uint8Vector(field); } } else { - throw "Unimplemented Int bit width " + bitWidth; + throw new Error("Unimplemented Int bit width " + bitWidth); } } From 08d60e67c3373dd713d733f982ccce4c014b4931 Mon Sep 17 00:00:00 2001 From: Brian Hulette Date: Mon, 29 May 2017 11:49:16 -0400 Subject: [PATCH 05/10] quotes, equality checks --- js/src/arrow.ts | 42 +++++++++++++++++++++--------------------- js/src/types.ts | 20 ++++++++++---------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/js/src/arrow.ts b/js/src/arrow.ts index 026ccc840a7..d5963e315da 100644 --- a/js/src/arrow.ts +++ b/js/src/arrow.ts @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -import { flatbuffers } from 'flatbuffers'; -import { org } from './Arrow_generated'; -import { vectorFromField, Vector } from './types'; +import { flatbuffers } from "flatbuffers"; +import { org } from "./Arrow_generated"; +import { Vector, vectorFromField } from "./types"; import ByteBuffer = flatbuffers.ByteBuffer; const Footer = org.apache.arrow.flatbuf.Footer; @@ -121,9 +121,9 @@ export function getStreamReader(buf): ArrowReader { batch = _loadBatch(bb); if (batch == null) { break; - } else if (batch.type == MessageHeader.DictionaryBatch) { + } else if (batch.type === MessageHeader.DictionaryBatch) { dictionaryBatches.push(batch); - } else if (batch.type == MessageHeader.RecordBatch) { + } else if (batch.type === MessageHeader.RecordBatch) { recordBatches.push(batch); } else { throw new Error("Expected batch type" + MessageHeader.RecordBatch + " or " + @@ -180,14 +180,14 @@ export function getFileReader(buf): ArrowReader { }); } - const dictionaryBatches = dictionaryBatchBlocks.map(function(block) { - bb.setPosition(block.offset); + const dictionaryBatches = dictionaryBatchBlocks.map((batchBlock) => { + bb.setPosition(batchBlock.offset); // TODO: Make sure this is a dictionary batch return _loadBatch(bb); }); - const recordBatches = recordBatchBlocks.map(function(block) { - bb.setPosition(block.offset); + const recordBatches = recordBatchBlocks.map((batchBlock) => { + bb.setPosition(batchBlock.offset); // TODO: Make sure this is a record batch return _loadBatch(bb); }); @@ -233,7 +233,7 @@ function _loadFooter(bb) { function _loadSchema(bb) { const message = _loadMessage(bb); - if (message.headerType() != MessageHeader.Schema) { + if (message.headerType() !== MessageHeader.Schema) { throw new Error("Expected header type " + MessageHeader.Schema + " but got " + message.headerType()); } return message.header(new Schema()); @@ -243,10 +243,10 @@ function _loadBatch(bb) { const message = _loadMessage(bb); if (message == null) { return; - } else if (message.headerType() == MessageHeader.RecordBatch) { + } else if (message.headerType() === MessageHeader.RecordBatch) { const batch = { header: message.header(new RecordBatch()), length: message.bodyLength().low }; return _loadRecordBatch(bb, batch); - } else if (message.headerType() == MessageHeader.DictionaryBatch) { + } else if (message.headerType() === MessageHeader.DictionaryBatch) { const batch = { header: message.header(new DictionaryBatch()), length: message.bodyLength().low }; return _loadDictionaryBatch(bb, batch); } else { @@ -306,7 +306,7 @@ function _loadDictionaryBatch(bb, batch) { function _loadMessage(bb) { const messageLength: number = Int32FromByteBuffer(bb, bb.position()); - if (messageLength == 0) { + if (messageLength === 0) { return; } bb.setPosition(bb.position() + 4); @@ -406,7 +406,7 @@ for (let i = 0; i < MAGIC_STR.length; i++) { function _checkMagic(buf, index) { for (let i = 0; i < MAGIC.length; i++) { - if (MAGIC[i] != buf[index + i]) { + if (MAGIC[i] !== buf[index + i]) { return false; } } @@ -432,10 +432,10 @@ TYPEMAP[Type.Struct_] = "Struct"; TYPEMAP[Type.Union] = "Union"; const VECTORTYPEMAP = {}; -VECTORTYPEMAP[VectorType.OFFSET] = 'OFFSET'; -VECTORTYPEMAP[VectorType.DATA] = 'DATA'; -VECTORTYPEMAP[VectorType.VALIDITY] = 'VALIDITY'; -VECTORTYPEMAP[VectorType.TYPE] = 'TYPE'; +VECTORTYPEMAP[VectorType.OFFSET] = "OFFSET"; +VECTORTYPEMAP[VectorType.DATA] = "DATA"; +VECTORTYPEMAP[VectorType.VALIDITY] = "VALIDITY"; +VECTORTYPEMAP[VectorType.TYPE] = "TYPE"; function parseField(field) { const children = []; @@ -453,7 +453,7 @@ function parseField(field) { nullable: field.nullable(), type: TYPEMAP[field.typeType()], children: children, - layout: layouts + layout: layouts, }; } @@ -467,8 +467,8 @@ function parseSchema(schema) { function loadVectors(bb, vectors: Vector[], recordBatch) { const indices = { bufferIndex: 0, nodeIndex: 0 }; - for (let i = 0; i < vectors.length; i += 1) { - loadVector(bb, vectors[i], recordBatch, indices); + for (const vector of vectors) { + loadVector(bb, vector, recordBatch, indices); } } diff --git a/js/src/types.ts b/js/src/types.ts index 94d25eaf32b..f691d7e2634 100644 --- a/js/src/types.ts +++ b/js/src/types.ts @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -import { BitArray } from './bitarray'; -import { TextDecoder } from 'text-encoding'; -import { org } from './Arrow_generated'; +import { BitArray } from "./bitarray"; +import { TextDecoder } from "text-encoding"; +import { org } from "./Arrow_generated"; const Type = org.apache.arrow.flatbuf.Type; @@ -245,7 +245,7 @@ class NullableDateVector extends DateVector { class Utf8Vector extends SimpleVector { protected offsetView: Int32Array; - private static decoder: TextDecoder = new TextDecoder('utf8'); + private static decoder: TextDecoder = new TextDecoder("utf8"); constructor(field) { super(field, Uint8Array); @@ -529,9 +529,9 @@ export function vectorFromField(field, dictionaries): Vector { return _createIntVector(field, type.bitWidth(), type.isSigned(), nullable); } else if (typeType === Type.FloatingPoint) { const precision = field.type(new org.apache.arrow.flatbuf.FloatingPoint()).precision(); - if (precision == org.apache.arrow.flatbuf.Precision.SINGLE) { + if (precision === org.apache.arrow.flatbuf.Precision.SINGLE) { return nullable ? new NullableFloat32Vector(field) : new Float32Vector(field); - } else if (precision == org.apache.arrow.flatbuf.Precision.DOUBLE) { + } else if (precision === org.apache.arrow.flatbuf.Precision.DOUBLE) { return nullable ? new NullableFloat64Vector(field) : new Float64Vector(field); } else { throw new Error("Unimplemented FloatingPoint precision " + precision); @@ -559,25 +559,25 @@ export function vectorFromField(field, dictionaries): Vector { } function _createIntVector(field, bitWidth, signed, nullable) { - if (bitWidth == 64) { + if (bitWidth === 64) { if (signed) { return nullable ? new NullableInt64Vector(field) : new Int64Vector(field); } else { return nullable ? new NullableUint64Vector(field) : new Uint64Vector(field); } - } else if (bitWidth == 32) { + } else if (bitWidth === 32) { if (signed) { return nullable ? new NullableInt32Vector(field) : new Int32Vector(field); } else { return nullable ? new NullableUint32Vector(field) : new Uint32Vector(field); } - } else if (bitWidth == 16) { + } else if (bitWidth === 16) { if (signed) { return nullable ? new NullableInt16Vector(field) : new Int16Vector(field); } else { return nullable ? new NullableUint16Vector(field) : new Uint16Vector(field); } - } else if (bitWidth == 8) { + } else if (bitWidth === 8) { if (signed) { return nullable ? new NullableInt8Vector(field) : new Int8Vector(field); } else { From 0a1b872f56838365af198321d205626b05fc1fa3 Mon Sep 17 00:00:00 2001 From: Brian Hulette Date: Mon, 29 May 2017 12:46:28 -0400 Subject: [PATCH 06/10] fix public, private, protected ordering issues --- js/src/types.ts | 172 ++++++++++++++++++++++++------------------------ 1 file changed, 87 insertions(+), 85 deletions(-) diff --git a/js/src/types.ts b/js/src/types.ts index f691d7e2634..2952ca011ec 100644 --- a/js/src/types.ts +++ b/js/src/types.ts @@ -15,9 +15,10 @@ // specific language governing permissions and limitations // under the License. +import { org } from "./Arrow_generated"; import { BitArray } from "./bitarray"; + import { TextDecoder } from "text-encoding"; -import { org } from "./Arrow_generated"; const Type = org.apache.arrow.flatbuf.Type; @@ -27,6 +28,28 @@ interface ArrayView { } export abstract class Vector { + /** + * Helper function for loading a VALIDITY buffer (for Nullable types) + * bb: flatbuffers.ByteBuffer + * buffer: org.apache.arrow.flatbuf.Buffer + */ + public static loadValidityBuffer(bb, buffer): BitArray { + const arrayBuffer = bb.bytes_.buffer; + const offset = bb.bytes_.byteOffset + buffer.offset; + return new BitArray(arrayBuffer, offset, buffer.length * 8); + } + + /** + * Helper function for loading an OFFSET buffer + * buffer: org.apache.arrow.flatbuf.Buffer + */ + public static loadOffsetBuffer(bb, buffer): Int32Array { + const arrayBuffer = bb.bytes_.buffer; + const offset = bb.bytes_.byteOffset + buffer.offset; + const length = buffer.length / Int32Array.BYTES_PER_ELEMENT; + return new Int32Array(arrayBuffer, offset, length); + } + public field: any; public name: string; public length: number; @@ -59,36 +82,13 @@ export abstract class Vector { } protected abstract loadBuffers(bb, node, buffers); - - /** - * Helper function for loading a VALIDITY buffer (for Nullable types) - * bb: flatbuffers.ByteBuffer - * buffer: org.apache.arrow.flatbuf.Buffer - */ - public static loadValidityBuffer(bb, buffer): BitArray { - const arrayBuffer = bb.bytes_.buffer; - const offset = bb.bytes_.byteOffset + buffer.offset; - return new BitArray(arrayBuffer, offset, buffer.length * 8); - } - - /** - * Helper function for loading an OFFSET buffer - * buffer: org.apache.arrow.flatbuf.Buffer - */ - public static loadOffsetBuffer(bb, buffer): Int32Array { - const arrayBuffer = bb.bytes_.buffer; - const offset = bb.bytes_.byteOffset + buffer.offset; - const length = buffer.length / Int32Array.BYTES_PER_ELEMENT; - return new Int32Array(arrayBuffer, offset, length); - } - } class SimpleVector extends Vector { protected dataView: T; - private TypedArray: { new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number }; + private TypedArray: { BYTES_PER_ELEMENT: number, new(buffer: any, offset: number, length: number): T }; - constructor(field, TypedArray: { new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number }) { + constructor(field, TypedArray: { BYTES_PER_ELEMENT: number, new(buffer: any, offset: number, length: number): T }) { super(field); this.TypedArray = TypedArray; } @@ -97,6 +97,18 @@ class SimpleVector extends Vector { return this.dataView[i]; } + public getDataView() { + return this.dataView; + } + + public toString() { + return this.dataView.toString(); + } + + public slice(start, end) { + return this.dataView.slice(start, end); + } + protected loadBuffers(bb, node, buffers) { this.loadDataBuffer(bb, buffers[0]); } @@ -111,17 +123,6 @@ class SimpleVector extends Vector { this.dataView = new this.TypedArray(arrayBuffer, offset, length); } - public getDataView() { - return this.dataView; - } - - public toString() { - return this.dataView.toString(); - } - - public slice(start, end) { - return this.dataView.slice(start, end); - } } class NullableSimpleVector extends SimpleVector { @@ -136,14 +137,14 @@ class NullableSimpleVector extends SimpleVector { } } + public getValidityVector() { + return this.validityView; + } + protected loadBuffers(bb, node, buffers) { this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); this.loadDataBuffer(bb, buffers[1]); } - - public getValidityVector() { - return this.validityView; - } } class Uint8Vector extends SimpleVector { constructor(field) { super(field, Uint8Array); } } @@ -225,11 +226,6 @@ class DateVector extends SimpleVector { class NullableDateVector extends DateVector { private validityView: BitArray; - protected loadBuffers(bb, node, buffers) { - this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); - this.loadDataBuffer(bb, buffers[1]); - } - public get(i) { if (this.validityView.get(i)) { return super.get(i); @@ -241,21 +237,22 @@ class NullableDateVector extends DateVector { public getValidityVector() { return this.validityView; } + + protected loadBuffers(bb, node, buffers) { + this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); + this.loadDataBuffer(bb, buffers[1]); + } } class Utf8Vector extends SimpleVector { - protected offsetView: Int32Array; private static decoder: TextDecoder = new TextDecoder("utf8"); + protected offsetView: Int32Array; + constructor(field) { super(field, Uint8Array); } - protected loadBuffers(bb, node, buffers) { - this.offsetView = Vector.loadOffsetBuffer(bb, buffers[0]); - this.loadDataBuffer(bb, buffers[1]); - } - public get(i) { return Utf8Vector.decoder.decode(this.dataView.slice(this.offsetView[i], this.offsetView[i + 1])); } @@ -271,17 +268,16 @@ class Utf8Vector extends SimpleVector { public getOffsetView() { return this.offsetView; } + + protected loadBuffers(bb, node, buffers) { + this.offsetView = Vector.loadOffsetBuffer(bb, buffers[0]); + this.loadDataBuffer(bb, buffers[1]); + } } class NullableUtf8Vector extends Utf8Vector { private validityView: BitArray; - protected loadBuffers(bb, node, buffers) { - this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); - this.offsetView = Vector.loadOffsetBuffer(bb, buffers[1]); - this.loadDataBuffer(bb, buffers[2]); - } - public get(i) { if (this.validityView.get(i)) { return super.get(i); @@ -293,6 +289,12 @@ class NullableUtf8Vector extends Utf8Vector { public getValidityVector() { return this.validityView; } + + protected loadBuffers(bb, node, buffers) { + this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); + this.offsetView = Vector.loadOffsetBuffer(bb, buffers[1]); + this.loadDataBuffer(bb, buffers[2]); + } } // Nested Types @@ -308,11 +310,6 @@ class ListVector extends Uint32Vector { return [this.dataVector]; } - protected loadBuffers(bb, node, buffers) { - super.loadBuffers(bb, node, buffers); - this.length -= 1; - } - public get(i) { const offset = super.get(i); if (offset === null) { @@ -333,16 +330,15 @@ class ListVector extends Uint32Vector { } return result; } -} - -class NullableListVector extends ListVector { - private validityView: BitArray; protected loadBuffers(bb, node, buffers) { - this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); - this.loadDataBuffer(bb, buffers[1]); + super.loadBuffers(bb, node, buffers); this.length -= 1; } +} + +class NullableListVector extends ListVector { + private validityView: BitArray; public get(i) { if (this.validityView.get(i)) { @@ -355,6 +351,12 @@ class NullableListVector extends ListVector { public getValidityVector() { return this.validityView; } + + protected loadBuffers(bb, node, buffers) { + this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); + this.loadDataBuffer(bb, buffers[1]); + this.length -= 1; + } } class FixedSizeListVector extends Vector { @@ -371,10 +373,6 @@ class FixedSizeListVector extends Vector { return [this.dataVector]; } - protected loadBuffers(bb, node, buffers) { - // no buffers to load - } - public get(i: number) { return this.dataVector.slice(i * this.size, (i + 1) * this.size); } @@ -390,15 +388,15 @@ class FixedSizeListVector extends Vector { public getListSize() { return this.size; } + + protected loadBuffers(bb, node, buffers) { + // no buffers to load + } } class NullableFixedSizeListVector extends FixedSizeListVector { private validityView: BitArray; - protected loadBuffers(bb, node, buffers) { - this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); - } - public get(i: number) { if (this.validityView.get(i)) { return super.get(i); @@ -410,6 +408,10 @@ class NullableFixedSizeListVector extends FixedSizeListVector { public getValidityVector() { return this.validityView; } + + protected loadBuffers(bb, node, buffers) { + this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); + } } class StructVector extends Vector { @@ -425,10 +427,6 @@ class StructVector extends Vector { return this.vectors; } - public loadBuffers(bb, node, buffers) { - this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); - } - public get(i: number) { if (this.validityView.get(i)) { return this.vectors.map((v: Vector) => v.get(i)); @@ -448,6 +446,10 @@ class StructVector extends Vector { public getValidityVector() { return this.validityView; } + + protected loadBuffers(bb, node, buffers) { + this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); + } } class DictionaryVector extends Vector { @@ -482,10 +484,6 @@ class DictionaryVector extends Vector { return this.indices.getChildVectors(); } - protected loadBuffers(bb, node, buffers) { - this.indices.loadData(bb, node, buffers); - } - /** Get the index (encoded) vector */ public getIndexVector() { return this.indices; @@ -499,6 +497,10 @@ class DictionaryVector extends Vector { public toString() { return this.indices.toString(); } + + protected loadBuffers(bb, node, buffers) { + this.indices.loadData(bb, node, buffers); + } } export function vectorFromField(field, dictionaries): Vector { From 67d82cac720c8ba8b908c331182792f51a28586a Mon Sep 17 00:00:00 2001 From: Brian Hulette Date: Mon, 29 May 2017 13:01:17 -0400 Subject: [PATCH 07/10] variable names, object shorthand --- js/src/arrow.ts | 35 +++++++++++++++++++++-------------- js/src/types.ts | 8 ++++---- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/js/src/arrow.ts b/js/src/arrow.ts index d5963e315da..3d3c8730fda 100644 --- a/js/src/arrow.ts +++ b/js/src/arrow.ts @@ -258,48 +258,55 @@ function _loadBatch(bb) { function _loadRecordBatch(bb, batch) { const data = batch.header; let i; - const nodes_ = []; const nodesLength = data.nodesLength(); + const nodes = new Array(nodesLength); let buffer; - const buffers_ = []; const buffersLength = data.buffersLength(); + const buffers = new Array(buffersLength); for (i = 0; i < nodesLength; i += 1) { - nodes_.push(data.nodes(i)); + nodes[i] = data.nodes(i); } + for (i = 0; i < buffersLength; i += 1) { buffer = data.buffers(i); - buffers_.push({ offset: bb.position() + buffer.offset().low, length: buffer.length().low }); + buffers[i] = { + offset: bb.position() + buffer.offset().low, + length: buffer.length().low, + }; } // position the buffer after the body to read the next message bb.setPosition(bb.position() + batch.length); - return { nodes: nodes_, buffers: buffers_, length: data.length().low, type: MessageHeader.RecordBatch }; + return { nodes, buffers, length: data.length().low, type: MessageHeader.RecordBatch }; } function _loadDictionaryBatch(bb, batch) { - const id_ = batch.header.id().toFloat64().toString(); + const id = batch.header.id().toFloat64().toString(); const data = batch.header.data(); let i; - const nodes_ = []; const nodesLength = data.nodesLength(); + const nodes = new Array(nodesLength); let buffer; - const buffers_ = []; const buffersLength = data.buffersLength(); + const buffers = new Array(buffersLength); for (i = 0; i < nodesLength; i += 1) { - nodes_.push(data.nodes(i)); + nodes[i] = data.nodes(i); } for (i = 0; i < buffersLength; i += 1) { buffer = data.buffers(i); - buffers_.push({ offset: bb.position() + buffer.offset().low, length: buffer.length().low }); + buffers[i] = { + offset: bb.position() + buffer.offset().low, + length: buffer.length().low, + }; } // position the buffer after the body to read the next message bb.setPosition(bb.position() + batch.length); - return {id: id_, - nodes: nodes_, - buffers: buffers_, + return {id, + nodes, + buffers, length: data.length().low, type: MessageHeader.DictionaryBatch }; } @@ -452,7 +459,7 @@ function parseField(field) { name: field.name(), nullable: field.nullable(), type: TYPEMAP[field.typeType()], - children: children, + children, layout: layouts, }; } diff --git a/js/src/types.ts b/js/src/types.ts index 2952ca011ec..29a1c09c0d5 100644 --- a/js/src/types.ts +++ b/js/src/types.ts @@ -53,7 +53,7 @@ export abstract class Vector { public field: any; public name: string; public length: number; - public null_count: number; + public nullCount: number; constructor(field) { this.field = field; @@ -77,7 +77,7 @@ export abstract class Vector { */ public loadData(bb, node, buffers) { this.length = node.length().low; - this.null_count = node.nullCount().low; + this.nullCount = node.nullCount().low; this.loadBuffers(bb, node, buffers); } @@ -315,8 +315,8 @@ class ListVector extends Uint32Vector { if (offset === null) { return null; } - const next_offset = super.get(i + 1); - return this.dataVector.slice(offset, next_offset); + const nextOffset = super.get(i + 1); + return this.dataVector.slice(offset, nextOffset); } public toString() { From 6f1583e552d2c178364908ccba5819450e5b2d63 Mon Sep 17 00:00:00 2001 From: Brian Hulette Date: Mon, 29 May 2017 13:03:43 -0400 Subject: [PATCH 08/10] sort object literals --- js/src/arrow.ts | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/js/src/arrow.ts b/js/src/arrow.ts index 3d3c8730fda..ac87a8c0ed7 100644 --- a/js/src/arrow.ts +++ b/js/src/arrow.ts @@ -165,18 +165,18 @@ export function getFileReader(buf): ArrowReader { for (i = 0; i < footer.dictionariesLength(); i++) { block = footer.dictionaries(i); dictionaryBatchBlocks.push({ - offset: block.offset().low, - metaDataLength: block.metaDataLength(), bodyLength: block.bodyLength().low, + metaDataLength: block.metaDataLength(), + offset: block.offset().low, }); } for (i = 0; i < footer.recordBatchesLength(); i++) { block = footer.recordBatches(i); recordBatchBlocks.push({ - offset: block.offset().low, - metaDataLength: block.metaDataLength(), bodyLength: block.bodyLength().low, + metaDataLength: block.metaDataLength(), + offset: block.offset().low, }); } @@ -271,8 +271,8 @@ function _loadRecordBatch(bb, batch) { for (i = 0; i < buffersLength; i += 1) { buffer = data.buffers(i); buffers[i] = { - offset: bb.position() + buffer.offset().low, length: buffer.length().low, + offset: bb.position() + buffer.offset().low, }; } // position the buffer after the body to read the next message @@ -297,18 +297,20 @@ function _loadDictionaryBatch(bb, batch) { for (i = 0; i < buffersLength; i += 1) { buffer = data.buffers(i); buffers[i] = { - offset: bb.position() + buffer.offset().low, length: buffer.length().low, + offset: bb.position() + buffer.offset().low, }; } // position the buffer after the body to read the next message bb.setPosition(bb.position() + batch.length); - return {id, - nodes, - buffers, - length: data.length().low, - type: MessageHeader.DictionaryBatch }; + return { + buffers, + id, + length: data.length().low, + nodes, + type: MessageHeader.DictionaryBatch, + }; } function _loadMessage(bb) { @@ -456,11 +458,11 @@ function parseField(field) { } return { + children, + layout: layouts, name: field.name(), nullable: field.nullable(), type: TYPEMAP[field.typeType()], - children, - layout: layouts, }; } From 68b7c8fc1164d1198f877603d6dfb726dcfd490c Mon Sep 17 00:00:00 2001 From: Brian Hulette Date: Mon, 29 May 2017 13:16:25 -0400 Subject: [PATCH 09/10] misc tslint fixes --- js/src/types.ts | 26 ++++++++++++++++---------- js/tslint.json | 3 ++- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/js/src/types.ts b/js/src/types.ts index 29a1c09c0d5..c541098d534 100644 --- a/js/src/types.ts +++ b/js/src/types.ts @@ -22,11 +22,16 @@ import { TextDecoder } from "text-encoding"; const Type = org.apache.arrow.flatbuf.Type; -interface ArrayView { - slice(start: number, end: number): ArrayView; +interface IArrayView { + slice(start: number, end: number): IArrayView; toString(): string; } +interface IViewConstructor { + BYTES_PER_ELEMENT: number; + new(buffer: any, offset: number, length: number): T; +} + export abstract class Vector { /** * Helper function for loading a VALIDITY buffer (for Nullable types) @@ -84,11 +89,11 @@ export abstract class Vector { protected abstract loadBuffers(bb, node, buffers); } -class SimpleVector extends Vector { +class SimpleVector extends Vector { protected dataView: T; - private TypedArray: { BYTES_PER_ELEMENT: number, new(buffer: any, offset: number, length: number): T }; + private TypedArray: IViewConstructor; - constructor(field, TypedArray: { BYTES_PER_ELEMENT: number, new(buffer: any, offset: number, length: number): T }) { + constructor(field, TypedArray: IViewConstructor) { super(field); this.TypedArray = TypedArray; } @@ -113,9 +118,9 @@ class SimpleVector extends Vector { this.loadDataBuffer(bb, buffers[0]); } - /** - * buffer: org.apache.arrow.flatbuf.Buffer - */ + /* + * buffer: org.apache.arrow.flatbuf.Buffer + */ protected loadDataBuffer(bb, buffer) { const arrayBuffer = bb.bytes_.buffer; const offset = bb.bytes_.byteOffset + buffer.offset; @@ -125,8 +130,7 @@ class SimpleVector extends Vector { } -class NullableSimpleVector extends SimpleVector { - +class NullableSimpleVector extends SimpleVector { protected validityView: BitArray; public get(i: number) { @@ -147,6 +151,7 @@ class NullableSimpleVector extends SimpleVector { } } +/* tslint:disable max-line-length */ class Uint8Vector extends SimpleVector { constructor(field) { super(field, Uint8Array); } } class Uint16Vector extends SimpleVector { constructor(field) { super(field, Uint16Array); } } class Uint32Vector extends SimpleVector { constructor(field) { super(field, Uint32Array); } } @@ -164,6 +169,7 @@ class NullableInt16Vector extends NullableSimpleVector { constru class NullableInt32Vector extends NullableSimpleVector { constructor(field) { super(field, Uint32Array); } } class NullableFloat32Vector extends NullableSimpleVector { constructor(field) { super(field, Float32Array); } } class NullableFloat64Vector extends NullableSimpleVector { constructor(field) { super(field, Float64Array); } } +/* tslint:enable max-line-length */ class Uint64Vector extends SimpleVector { constructor(field) { diff --git a/js/tslint.json b/js/tslint.json index 40a1bb03f34..5f2f6cdcf36 100644 --- a/js/tslint.json +++ b/js/tslint.json @@ -5,7 +5,8 @@ ], "jsRules": {}, "rules": { - "no-bitwise": false + "no-bitwise": false, + "max-classes-per-file": false }, "rulesDirectory": [] } From 727d80f2dc94fe855786e2261bab112e757bd74d Mon Sep 17 00:00:00 2001 From: Brian Hulette Date: Mon, 29 May 2017 13:35:50 -0400 Subject: [PATCH 10/10] added npm lint script --- js/README.md | 1 + js/package.json | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/js/README.md b/js/README.md index 167bafcd724..de9070c59aa 100644 --- a/js/README.md +++ b/js/README.md @@ -18,6 +18,7 @@ From this directory, run: ``` bash $ npm install # pull dependencies +$ npm run lint -- # run tslint $ npm run build # build typescript (run tsc and webpack) $ npm run test # run the unit tests (node.js only) ``` diff --git a/js/package.json b/js/package.json index 1739e389265..e46b5bdc23c 100644 --- a/js/package.json +++ b/js/package.json @@ -6,7 +6,8 @@ "scripts": { "build": "./flatbuffers.sh && tsc && tsc -m es6 --outDir lib-esm && webpack", "clean": "rm -rf lib lib-esm _bundles", - "test": "./node_modules/mocha/bin/mocha ./spec/arrow.js" + "test": "./node_modules/mocha/bin/mocha ./spec/arrow.js", + "lint": "./node_modules/tslint/bin/tslint" }, "author": "", "repository": "https://github.com/apache/arrow/", @@ -15,6 +16,7 @@ "awesome-typescript-loader": "^3.1.3", "chai": "^3.5.0", "mocha": "^3.3.0", + "tslint": "^5.3.2", "typescript": "^2.3.2", "uglifyjs-webpack-plugin": "^0.4.3", "webpack": "^2.3.3"