Use a fenwick tree to optimize prefix sums
This patch makes summary use the excellent fenwick package from
https://github.com/yourbasic/fenwick to dramatically speed
up Add() calls while adding a bounded cost to memory.
benchmark old ns/op new ns/op delta
BenchmarkAdd1-4 187 206 +10.16%
BenchmarkAdd10-4 332 274 -17.47%
BenchmarkAdd100-4 1092 325 -70.24%
The tree creation could be faster, but profiling has shown that
it doesn't seem to be that much of a problem (that's because
`summary.setAt()` happens a *lot* more than `summary.Add()`)
- Id
- c475ad375995a32e8f72759c29ff1b80b6d9e82b
- Author
- Caio
- Commit time
- 2017-10-29T13:46:38+01:00
Modified Gopkg.lock
packages = ["."]
revision = "5344a9259b21627d94279721ab1f27eb029194e7"
+[[projects]]
+ name = "github.com/yourbasic/fenwick"
+ packages = ["."]
+ revision = "7cb325001daae879940edf7b19da8557a51ca5f7"
+ version = "1.2.0"
+
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
- inputs-digest = "19f9623e18e30932906e83f04c0f62bf28811eca7762f2fb26c0d3bd55d45d5c"
+ inputs-digest = "eac5a7bc22a9b8d592de2752be2ab1f8707df0caf1a1e77087c88671cbb8c94d"
solver-name = "gps-cdcl"
solver-version = 1
Modified Gopkg.toml
[[constraint]]
revision = "5344a9259b21627d94279721ab1f27eb029194e7"
name = "github.com/leesper/go_rng"
+
+[[constraint]]
+ version = "1.2.0"
+ name = "github.com/yourbasic/fenwick"
Modified summary.go
package tdigest
import (
"fmt"
"math"
"sort"
+
+ "github.com/yourbasic/fenwick"
)
type summary struct {
means []float64
counts []uint32
+ bitree *fenwick.List
}
func newSummary(initialCapacity uint) *summary {
- return &summary{
+ s := &summary{
means: make([]float64, 0, initialCapacity),
counts: make([]uint32, 0, initialCapacity),
}
+ s.rebuildFenwickTree()
+ return s
}
func (s summary) Len() int {
s.means[idx] = key
s.counts[idx] = value
+ // Reinitialize the prefixSum cache
+ // we can likely be smarter when doing this
+ s.rebuildFenwickTree()
+
return nil
+}
+
+func (s *summary) rebuildFenwickTree() {
+ x := make([]int64, s.Len())
+ for i := 0; i < s.Len(); i++ {
+ x[i] = int64(s.counts[i])
+ }
+ s.bitree = fenwick.New(x...)
}
func (s summary) Floor(x float64) int {
}
func (s summary) HeadSum(index int) (sum float64) {
- for i := 0; i < index; i++ {
- sum += float64(s.counts[i])
- }
- return sum
+ return float64(s.bitree.Sum(index))
}
func (s summary) FindIndex(x float64) int {
s.counts[index] = count
s.adjustRight(index)
s.adjustLeft(index)
+ s.bitree.Set(index, int64(count))
}
func (s *summary) adjustRight(index int) {