Get rid of the centroid abstraction 💬 by Caio 8 years ago (log)
This was only being used to pack {float64,uint32}, all the
other functionality was skipped or became unused over
time for performance reasons. Away it goes.
Blob serialization.go
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 |
package tdigest
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
)
const smallEncoding int32 = 2
var endianess = binary.BigEndian
// AsBytes serializes the digest into a byte array so it can be
// saved to disk or sent over the wire.
func (t TDigest) AsBytes() ([]byte, error) {
buffer := new(bytes.Buffer)
err := binary.Write(buffer, endianess, smallEncoding)
if err != nil {
return nil, err
}
err = binary.Write(buffer, endianess, t.compression)
if err != nil {
return nil, err
}
err = binary.Write(buffer, endianess, int32(t.summary.Len()))
if err != nil {
return nil, err
}
var x float64
t.summary.ForEach(func(mean float64, count uint32) bool {
delta := mean - x
x = mean
err = binary.Write(buffer, endianess, float32(delta))
return err == nil
})
if err != nil {
return nil, err
}
t.summary.ForEach(func(mean float64, count uint32) bool {
err = encodeUint(buffer, count)
return err == nil
})
if err != nil {
return nil, err
}
return buffer.Bytes(), nil
}
// FromBytes reads a byte buffer with a serialized digest (from AsBytes)
// and deserializes it.
func FromBytes(buf *bytes.Reader) (*TDigest, error) {
var encoding int32
err := binary.Read(buf, endianess, &encoding)
if err != nil {
return nil, err
}
if encoding != smallEncoding {
return nil, fmt.Errorf("Unsupported encoding version: %d", encoding)
}
var compression float64
err = binary.Read(buf, endianess, &compression)
if err != nil {
return nil, err
}
t, err := New(Compression(uint32(compression)))
if err != nil {
return nil, err
}
var numCentroids int32
err = binary.Read(buf, endianess, &numCentroids)
if err != nil {
return nil, err
}
if numCentroids < 0 || numCentroids > 1<<22 {
return nil, errors.New("bad number of centroids in serialization")
}
means := make([]float64, numCentroids)
var delta float32
var x float64
for i := 0; i < int(numCentroids); i++ {
err = binary.Read(buf, endianess, &delta)
if err != nil {
return nil, err
}
x += float64(delta)
means[i] = x
}
for i := 0; i < int(numCentroids); i++ {
decUint, err := decodeUint(buf)
if err != nil {
return nil, err
}
err = t.AddWeighted(means[i], decUint)
if err != nil {
return nil, err
}
}
return t, nil
}
func encodeUint(buf *bytes.Buffer, n uint32) error {
var b [binary.MaxVarintLen32]byte
l := binary.PutUvarint(b[:], uint64(n))
_, err := buf.Write(b[:l])
return err
}
func decodeUint(buf *bytes.Reader) (uint32, error) {
v, err := binary.ReadUvarint(buf)
if v > 0xffffffff {
return 0, errors.New("Something wrong, this number looks too big")
}
return uint32(v), err
}
|