util/rowcodec: add new row codec lib

util/rowcodec/common.go

@@ -0,0 +1,198 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rowcodec
+
+import (
+	"encoding/binary"
+	"reflect"
+	"unsafe"
+
+	"github.com/pingcap/errors"
+)
+
+// CodecVer is the constant number that represent the new row format.
+const CodecVer = 128
+
+var errInvalidCodecVer = errors.New("invalid codec version")
+
+// row is the struct type used to access a row.
+// There are two types of row, small and large.
+// A small row takes one byte colID and two bytes offset, optimized for most cases.
+// If the max colID is larger than 255 or total value size is larger than 65535, the row type would be large.
+// A large row takes four bytes colID and four bytes offset.
+type row struct {
+	isLarge        bool
+	numNotNullCols uint16
+	numNullCols    uint16
+	data           []byte
+
+	// for small rows
+	colIDs  []byte
+	offsets []uint16
+
+	// for large row
+	colIDs32  []uint32
+	offsets32 []uint32
+}
+
+func (r *row) getData(i int) []byte {
+	var start, end uint32
+	if r.isLarge {
+		if i > 0 {
+			start = r.offsets32[i-1]
+		}
+		end = r.offsets32[i]
+	} else {
+		if i > 0 {
+			start = uint32(r.offsets[i-1])
+		}
+		end = uint32(r.offsets[i])
+	}
+	return r.data[start:end]
+}
+
+func (r *row) setRowData(rowData []byte) error {
+	if rowData[0] != CodecVer {
+		return errInvalidCodecVer
+	}
+	r.isLarge = rowData[1]&1 > 0
+	r.numNotNullCols = binary.LittleEndian.Uint16(rowData[2:])
+	r.numNullCols = binary.LittleEndian.Uint16(rowData[4:])
+	cursor := 6
+	if r.isLarge {
+		colIDsLen := int(r.numNotNullCols+r.numNullCols) * 4
+		r.colIDs32 = bytesToU32Slice(rowData[cursor : cursor+colIDsLen])
+		cursor += colIDsLen
+		offsetsLen := int(r.numNotNullCols) * 4
+		r.offsets32 = bytesToU32Slice(rowData[cursor : cursor+offsetsLen])
+		cursor += offsetsLen
+	} else {
+		colIDsLen := int(r.numNotNullCols + r.numNullCols)
+		r.colIDs = rowData[cursor : cursor+colIDsLen]
+		cursor += colIDsLen
+		offsetsLen := int(r.numNotNullCols) * 2
+		r.offsets = bytes2U16Slice(rowData[cursor : cursor+offsetsLen])
+		cursor += offsetsLen
+	}
+	r.data = rowData[cursor:]
+	return nil
+}
+
+func bytesToU32Slice(b []byte) []uint32 {
+	if len(b) == 0 {
+		return nil
+	}
+	var u32s []uint32
+	hdr := (*reflect.SliceHeader)(unsafe.Pointer(&u32s))
+	hdr.Len = len(b) / 4
+	hdr.Cap = hdr.Len
+	hdr.Data = uintptr(unsafe.Pointer(&b[0]))
+	return u32s
+}
+
+func bytes2U16Slice(b []byte) []uint16 {
+	if len(b) == 0 {
+		return nil
+	}
+	var u16s []uint16
+	hdr := (*reflect.SliceHeader)(unsafe.Pointer(&u16s))
+	hdr.Len = len(b) / 2
+	hdr.Cap = hdr.Len
+	hdr.Data = uintptr(unsafe.Pointer(&b[0]))
+	return u16s
+}
+
+func u16SliceToBytes(u16s []uint16) []byte {
+	if len(u16s) == 0 {
+		return nil
+	}
+	var b []byte
+	hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	hdr.Len = len(u16s) * 2
+	hdr.Cap = hdr.Len
+	hdr.Data = uintptr(unsafe.Pointer(&u16s[0]))
+	return b
+}
+
+func u32SliceToBytes(u32s []uint32) []byte {
+	if len(u32s) == 0 {
+		return nil
+	}
+	var b []byte
+	hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	hdr.Len = len(u32s) * 4
+	hdr.Cap = hdr.Len
+	hdr.Data = uintptr(unsafe.Pointer(&u32s[0]))
+	return b
+}
+
+func encodeInt(buf []byte, iVal int64) []byte {
+	var tmp [8]byte
+	if int64(int8(iVal)) == iVal {
+		buf = append(buf, byte(iVal))
+	} else if int64(int16(iVal)) == iVal {
+		binary.LittleEndian.PutUint16(tmp[:], uint16(iVal))
+		buf = append(buf, tmp[:2]...)
+	} else if int64(int32(iVal)) == iVal {
+		binary.LittleEndian.PutUint32(tmp[:], uint32(iVal))
+		buf = append(buf, tmp[:4]...)
+	} else {
+		binary.LittleEndian.PutUint64(tmp[:], uint64(iVal))
+		buf = append(buf, tmp[:8]...)
+	}
+	return buf
+}
+
+func decodeInt(val []byte) int64 {
+	switch len(val) {
+	case 1:
+		return int64(int8(val[0]))
+	case 2:
+		return int64(int16(binary.LittleEndian.Uint16(val)))
+	case 4:
+		return int64(int32(binary.LittleEndian.Uint32(val)))
+	default:
+		return int64(binary.LittleEndian.Uint64(val))
+	}
+}
+
+func encodeUint(buf []byte, uVal uint64) []byte {
+	var tmp [8]byte
+	if uint64(uint8(uVal)) == uVal {
+		buf = append(buf, byte(uVal))
+	} else if uint64(uint16(uVal)) == uVal {
+		binary.LittleEndian.PutUint16(tmp[:], uint16(uVal))
+		buf = append(buf, tmp[:2]...)
+	} else if uint64(uint32(uVal)) == uVal {
+		binary.LittleEndian.PutUint32(tmp[:], uint32(uVal))
+		buf = append(buf, tmp[:4]...)
+	} else {
+		binary.LittleEndian.PutUint64(tmp[:], uint64(uVal))
+		buf = append(buf, tmp[:8]...)
+	}
+	return buf
+}
+
+func decodeUint(val []byte) uint64 {
+	switch len(val) {
+	case 1:
+		return uint64(val[0])
+	case 2:
+		return uint64(binary.LittleEndian.Uint16(val))
+	case 4:
+		return uint64(binary.LittleEndian.Uint32(val))
+	default:
+		return binary.LittleEndian.Uint64(val)
+	}
+}