Skip to content

Faster JSON marshaling (and easyjson support) #112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jacksontj wants to merge 3 commits into from
Closed

Faster JSON marshaling (and easyjson support) #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fa42251
Add benchmark to value marshaling
jacksontj Dec 21, 2017
a347fbf
Add time marshal tests
jacksontj Jan 3, 2018
960a398
Add easyjson support
jacksontj Dec 21, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions model/labelset.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"fmt"
"sort"
"strings"

"github.com/mailru/easyjson/jwriter"
Copy link
Contributor

@brian-brazil brian-brazil Dec 27, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this is an appropriate dependency to have in this package. Data models should just be data models.

Copy link
Contributor Author

@jacksontj jacksontj Dec 27, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@brian-brazil this package already has a dep on encoding/json meaning marshaling is already a part of the model. Similarly custom marshal and unmarshal methods already exist in this package (and even in that file).

In golang you can't define methods on a struct outside of the package where the struct is defined, meaning that if you want a struct to adhere to an interface it must define the methods in the same package. We could move the marshal methods into a different file in the same package, but the existing ones are in the same files-- but if separate files in the package is preferred I'd be more than happy to move them around.

)

// A LabelSet is a collection of LabelName and LabelValue pairs. The LabelSet
Expand Down Expand Up @@ -150,6 +152,29 @@ func (ls LabelSet) FastFingerprint() Fingerprint {
return labelSetToFastFingerprint(ls)
}

func (l LabelSet) MarshalEasyJSON(w *jwriter.Writer) {
w.RawByte('{')
first := true
for k, v := range l {
if !first {
w.RawByte(',')
} else {
first = false
}
w.RawString(`"` + string(k) + `"`)
w.RawByte(':')
w.RawString(`"` + string(v) + `"`)
}
w.RawByte('}')
}

// MarshalJSON implements the json.Marshaler interface.
func (l LabelSet) MarshalJSON() ([]byte, error) {
w := jwriter.Writer{}
l.MarshalEasyJSON(&w)
return w.Buffer.BuildBytes(), w.Error
}

// UnmarshalJSON implements the json.Unmarshaler interface.
func (l *LabelSet) UnmarshalJSON(b []byte) error {
var m map[LabelName]LabelValue
Expand Down
25 changes: 25 additions & 0 deletions model/metric.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"regexp"
"sort"
"strings"

"github.com/mailru/easyjson/jwriter"
)

var (
Expand Down Expand Up @@ -87,6 +89,29 @@ func (m Metric) FastFingerprint() Fingerprint {
return LabelSet(m).FastFingerprint()
}

func (m Metric) MarshalEasyJSON(w *jwriter.Writer) {
w.RawByte('{')
first := true
for k, v := range m {
if !first {
w.RawByte(',')
} else {
first = false
Copy link
Member

@juliusv juliusv Dec 27, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: be consistent how you handle the if-else here and in the LabelSet marshaling method (I guess I prefer the if-else variant, since it only assigns the first variable once - not that I think it'd make a real difference).

Copy link
Contributor Author

@jacksontj jacksontj Dec 27, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch, I switched to the else, since it seems better :) (tests don't show much of a difference, but it looks better at least)

}
w.RawString(`"` + string(k) + `"`)
w.RawByte(':')
w.RawString(`"` + string(v) + `"`)
}
w.RawByte('}')
}

// MarshalJSON implements the json.Marshaler interface.
func (m Metric) MarshalJSON() ([]byte, error) {
w := jwriter.Writer{}
m.MarshalEasyJSON(&w)
return w.Buffer.BuildBytes(), w.Error
}

// IsValidMetricName returns true iff name matches the pattern of MetricNameRE.
// This function, however, does not use MetricNameRE for the check but a much
// faster hardcoded implementation.
Expand Down
50 changes: 48 additions & 2 deletions model/time.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (
"strconv"
"strings"
"time"

"github.com/mailru/easyjson/jwriter"
)

const (
Expand All @@ -39,6 +41,9 @@ const (
Latest = Time(math.MaxInt64)
)

// How many digits the second variable is (needed for formatting to string)
var secondDigitLen = len(strconv.FormatInt(second, 10)) - 1

// Time is the number of milliseconds since the epoch
// (1970-01-01 00:00 UTC) excluding leap seconds.
type Time int64
Expand Down Expand Up @@ -112,12 +117,53 @@ var dotPrecision = int(math.Log10(float64(second)))

// String returns a string representation of the Time.
func (t Time) String() string {
return strconv.FormatFloat(float64(t)/float64(second), 'f', -1, 64)
timeStr := strconv.FormatInt(int64(t), 10)
lenDelta := secondDigitLen - len(timeStr)

retString := ""

// Put out anything before a decimal
if len(timeStr) > secondDigitLen {
retString = timeStr[:len(timeStr)-secondDigitLen]
}

// put the decimal there
retString += "."

// pad (if needed)
if lenDelta > 0 {
retString += strings.Repeat("0", lenDelta) + timeStr
} else {
retString += timeStr[len(timeStr)-secondDigitLen:]
}
return retString
}

func (t Time) MarshalEasyJSON(w *jwriter.Writer) {
timeStr := strconv.FormatInt(int64(t), 10)
lenDelta := secondDigitLen - len(timeStr)

// Put out anything before a decimal
if len(timeStr) > secondDigitLen {
w.RawString(timeStr[:len(timeStr)-secondDigitLen])
}

// put the decimal there
w.RawByte('.')

// pad (if needed)
if lenDelta > 0 {
w.RawString(strings.Repeat("0", lenDelta) + timeStr)
} else {
w.RawString(timeStr[len(timeStr)-secondDigitLen:])
}
}

// MarshalJSON implements the json.Marshaler interface.
func (t Time) MarshalJSON() ([]byte, error) {
return []byte(t.String()), nil
w := jwriter.Writer{}
t.MarshalEasyJSON(&w)
return w.Buffer.BuildBytes(), w.Error
}

// UnmarshalJSON implements the json.Unmarshaler interface.
Expand Down
74 changes: 74 additions & 0 deletions model/time_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,77 @@ func TestParseDuration(t *testing.T) {
}
}
}

func TestTimeString(t *testing.T) {
var cases = []struct {
in Time
out string
}{
{
in: Time(1),
out: ".001",
},
{
in: Time(10),
out: ".010",
},
{
in: Time(100),
out: ".100",
},
{
in: Time(10000),
out: "10.000",
},
{
in: Time(1000000),
out: "1000.000",
},
}

for _, c := range cases {
actual := c.in.String()
if actual != c.out {
t.Fatalf("Mismatched time output expected=%s actual=%s", c.out, actual)
}
}
}

func TestTimeJson(t *testing.T) {
var cases = []struct {
in Time
out string
}{
{
in: Time(1),
out: ".001",
},
{
in: Time(10),
out: ".010",
},
{
in: Time(100),
out: ".100",
},
{
in: Time(10000),
out: "10.000",
},
{
in: Time(1000000),
out: "1000.000",
},
}

for _, c := range cases {
b, err := c.in.MarshalJSON()
if err != nil {
t.Fatalf("Error marshaling time: %v", err)
}
actual := string(b)
if actual != c.out {
t.Fatalf("Mismatched time output expected=%s actual=%s", c.out, actual)
}
}
}
54 changes: 39 additions & 15 deletions model/value.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (
"sort"
"strconv"
"strings"

"github.com/mailru/easyjson/jwriter"
)

var (
Expand All @@ -42,9 +44,18 @@ var (
// time.
type SampleValue float64

func (v SampleValue) MarshalEasyJSON(w *jwriter.Writer) {
w.RawByte('"')
w.Buffer.EnsureSpace(20)
w.Buffer.Buf = strconv.AppendFloat(w.Buffer.Buf, float64(v), 'f', -1, 64)
w.RawByte('"')
}

// MarshalJSON implements json.Marshaler.
func (v SampleValue) MarshalJSON() ([]byte, error) {
return json.Marshal(v.String())
w := jwriter.Writer{}
v.MarshalEasyJSON(&w)
return w.Buffer.BuildBytes(), w.Error
}

// UnmarshalJSON implements json.Unmarshaler.
Expand Down Expand Up @@ -80,17 +91,19 @@ type SamplePair struct {
Value SampleValue
}

func (s SamplePair) MarshalEasyJSON(w *jwriter.Writer) {
w.RawByte('[')
s.Timestamp.MarshalEasyJSON(w)
w.RawByte(',')
s.Value.MarshalEasyJSON(w)
w.RawByte(']')
}

// MarshalJSON implements json.Marshaler.
func (s SamplePair) MarshalJSON() ([]byte, error) {
t, err := json.Marshal(s.Timestamp)
if err != nil {
return nil, err
}
v, err := json.Marshal(s.Value)
if err != nil {
return nil, err
}
return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil
w := jwriter.Writer{}
s.MarshalEasyJSON(&w)
return w.Buffer.BuildBytes(), w.Error
}

// UnmarshalJSON implements json.Unmarshaler.
Expand Down Expand Up @@ -220,6 +233,7 @@ func (s Samples) Equal(o Samples) bool {
}

// SampleStream is a stream of Values belonging to an attached COWMetric.
//easyjson:json
type SampleStream struct {
Metric Metric `json:"metric"`
Values []SamplePair `json:"values"`
Expand Down Expand Up @@ -303,17 +317,21 @@ type Scalar struct {
Timestamp Time `json:"timestamp"`
}

func (s Scalar) String() string {
return fmt.Sprintf("scalar: %v @[%v]", s.Value, s.Timestamp)
func (s Scalar) MarshalEasyJSON(w *jwriter.Writer) {
w.RawByte('[')
s.Timestamp.MarshalEasyJSON(w)
w.RawByte(',')
s.Value.MarshalEasyJSON(w)
w.RawByte(']')
}

// MarshalJSON implements json.Marshaler.
func (s Scalar) MarshalJSON() ([]byte, error) {
v := strconv.FormatFloat(float64(s.Value), 'f', -1, 64)
return json.Marshal([...]interface{}{s.Timestamp, string(v)})
w := jwriter.Writer{}
s.MarshalEasyJSON(&w)
return w.Buffer.BuildBytes(), w.Error
}

// UnmarshalJSON implements json.Unmarshaler.
func (s *Scalar) UnmarshalJSON(b []byte) error {
var f string
v := [...]interface{}{&s.Timestamp, &f}
Expand All @@ -330,6 +348,10 @@ func (s *Scalar) UnmarshalJSON(b []byte) error {
return nil
}

func (s Scalar) String() string {
return fmt.Sprintf("scalar: %v @[%v]", s.Value, s.Timestamp)
}

// String is a string value evaluated at the set timestamp.
type String struct {
Value string `json:"value"`
Expand All @@ -353,6 +375,7 @@ func (s *String) UnmarshalJSON(b []byte) error {

// Vector is basically only an alias for Samples, but the
// contract is that in a Vector, all Samples have the same timestamp.
//easyjson:json
type Vector []*Sample

func (vec Vector) String() string {
Expand Down Expand Up @@ -395,6 +418,7 @@ func (vec Vector) Equal(o Vector) bool {
}

// Matrix is a list of time series.
//easyjson:json
type Matrix []*SampleStream

func (m Matrix) Len() int { return len(m) }
Expand Down
Loading