diff --git a/consistent.go b/consistent.go index c9be47a..aaf899d 100644 --- a/consistent.go +++ b/consistent.go @@ -11,15 +11,14 @@ import ( "errors" "fmt" "math" - "sort" "sync" "sync/atomic" + "github.com/google/btree" + blake2b "github.com/minio/blake2b-simd" ) -const replicationFactor = 10 - var ErrNoHosts = errors.New("no hosts added") type Host struct { @@ -28,170 +27,201 @@ type Host struct { } type Consistent struct { - hosts map[uint64]string - sortedSet []uint64 - loadMap map[string]*Host - totalLoad int64 + servers map[uint64]string + clients *btree.BTree + loadMap map[string]*Host + totalLoad int64 + replicationFactor int sync.RWMutex } +type item struct { + value uint64 +} + +func (i item) Less(than btree.Item) bool { + return i.value < than.(item).value +} + func New() *Consistent { return &Consistent{ - hosts: map[uint64]string{}, - sortedSet: []uint64{}, - loadMap: map[string]*Host{}, + servers: map[uint64]string{}, + clients: btree.New(2), + loadMap: map[string]*Host{}, + replicationFactor: 1000, } } -func (c *Consistent) Add(host string) { +func NewWithReplicationFactor(replicationFactor int) *Consistent { + return &Consistent{ + servers: map[uint64]string{}, + clients: btree.New(2), + loadMap: map[string]*Host{}, + replicationFactor: replicationFactor, + } +} +func (c *Consistent) Add(server string) { c.Lock() defer c.Unlock() - if _, ok := c.loadMap[host]; ok { + if _, ok := c.loadMap[server]; ok { return } - c.loadMap[host] = &Host{Name: host, Load: 0} - for i := 0; i < replicationFactor; i++ { - h := c.hash(fmt.Sprintf("%s%d", host, i)) - c.hosts[h] = host - c.sortedSet = append(c.sortedSet, h) - + c.loadMap[server] = &Host{Name: server, Load: 0} + for i := 0; i < c.replicationFactor; i++ { + h := c.hash(fmt.Sprintf("%s%d", server, i)) + c.servers[h] = server + c.clients.ReplaceOrInsert(item{h}) } - // sort hashes ascendingly - sort.Slice(c.sortedSet, func(i int, j int) bool { - if c.sortedSet[i] < c.sortedSet[j] { - return true - } - return false - }) } -// Returns the host that owns `key`. -// +// Get returns the server that owns the given client. // As described in https://en.wikipedia.org/wiki/Consistent_hashing -// -// It returns ErrNoHosts if the ring has no hosts in it. -func (c *Consistent) Get(key string) (string, error) { +// It returns ErrNoHosts if the ring has no servers in it. +func (c *Consistent) Get(client string) (string, error) { c.RLock() defer c.RUnlock() - if len(c.hosts) == 0 { + if c.clients.Len() == 0 { return "", ErrNoHosts } - h := c.hash(key) - idx := c.search(h) - return c.hosts[c.sortedSet[idx]], nil + h := c.hash(client) + var foundItem btree.Item + c.clients.AscendGreaterOrEqual(item{h}, func(i btree.Item) bool { + foundItem = i + return false // stop the iteration + }) + + if foundItem == nil { + // If no host found, wrap around to the first one. + foundItem = c.clients.Min() + } + + host := c.servers[foundItem.(item).value] + + return host, nil } -// It uses Consistent Hashing With Bounded loads -// +// GetLeast returns the least loaded host that can serve the key. +// It uses Consistent Hashing With Bounded loads. // https://research.googleblog.com/2017/04/consistent-hashing-with-bounded-loads.html -// -// to pick the least loaded host that can serve the key -// // It returns ErrNoHosts if the ring has no hosts in it. -// -func (c *Consistent) GetLeast(key string) (string, error) { +func (c *Consistent) GetLeast(client string) (string, error) { c.RLock() defer c.RUnlock() - if len(c.hosts) == 0 { + if c.clients.Len() == 0 { return "", ErrNoHosts } - h := c.hash(key) + h := c.hash(client) idx := c.search(h) i := idx for { - host := c.hosts[c.sortedSet[i]] - if c.loadOK(host) { - return host, nil - } - i++ - if i >= len(c.hosts) { - i = 0 + x := item{uint64(i)} + key := c.clients.Get(x) + if key != nil { + host := c.servers[key.(*item).value] + if c.loadOK(host) { + return host, nil + } + i++ + if i >= c.clients.Len() { + i = 0 + } + } else { + return client, nil } } } func (c *Consistent) search(key uint64) int { - idx := sort.Search(len(c.sortedSet), func(i int) bool { - return c.sortedSet[i] >= key + idx := 0 + found := false + + c.clients.Ascend(func(i btree.Item) bool { + if i.(item).value >= key { + found = true + return false // stop the iteration + } + idx++ + return true }) - if idx >= len(c.sortedSet) { + if !found { idx = 0 } + return idx } -// Sets the load of `host` to the given `load` -func (c *Consistent) UpdateLoad(host string, load int64) { +// Sets the load of `server` to the given `load` +func (c *Consistent) UpdateLoad(server string, load int64) { c.Lock() defer c.Unlock() - if _, ok := c.loadMap[host]; !ok { + if _, ok := c.loadMap[server]; !ok { return } - c.totalLoad -= c.loadMap[host].Load - c.loadMap[host].Load = load + c.totalLoad -= c.loadMap[server].Load + c.loadMap[server].Load = load c.totalLoad += load } // Increments the load of host by 1 // // should only be used with if you obtained a host with GetLeast -func (c *Consistent) Inc(host string) { +func (c *Consistent) Inc(server string) { c.Lock() defer c.Unlock() - if _, ok := c.loadMap[host]; !ok { + if _, ok := c.loadMap[server]; !ok { return } - atomic.AddInt64(&c.loadMap[host].Load, 1) + atomic.AddInt64(&c.loadMap[server].Load, 1) atomic.AddInt64(&c.totalLoad, 1) } // Decrements the load of host by 1 // // should only be used with if you obtained a host with GetLeast -func (c *Consistent) Done(host string) { +func (c *Consistent) Done(server string) { c.Lock() defer c.Unlock() - if _, ok := c.loadMap[host]; !ok { + if _, ok := c.loadMap[server]; !ok { return } - atomic.AddInt64(&c.loadMap[host].Load, -1) + atomic.AddInt64(&c.loadMap[server].Load, -1) atomic.AddInt64(&c.totalLoad, -1) } // Deletes host from the ring -func (c *Consistent) Remove(host string) bool { +func (c *Consistent) Remove(server string) bool { c.Lock() defer c.Unlock() - for i := 0; i < replicationFactor; i++ { - h := c.hash(fmt.Sprintf("%s%d", host, i)) - delete(c.hosts, h) + for i := 0; i < c.replicationFactor; i++ { + h := c.hash(fmt.Sprintf("%s%d", server, i)) + delete(c.servers, h) c.delSlice(h) } - delete(c.loadMap, host) + delete(c.loadMap, server) return true } -// Return the list of hosts in the ring -func (c *Consistent) Hosts() (hosts []string) { +// Return the list of servers in the ring +func (c *Consistent) Servers() (servers []string) { c.RLock() defer c.RUnlock() for k, _ := range c.loadMap { - hosts = append(hosts, k) + servers = append(servers, k) } - return hosts + return servers } // Returns the loads of all the hosts @@ -223,7 +253,7 @@ func (c *Consistent) MaxLoad() int64 { return int64(avgLoadPerNode) } -func (c *Consistent) loadOK(host string) bool { +func (c *Consistent) loadOK(server string) bool { // a safety check if someone performed c.Done more than needed if c.totalLoad < 0 { c.totalLoad = 0 @@ -236,12 +266,12 @@ func (c *Consistent) loadOK(host string) bool { } avgLoadPerNode = math.Ceil(avgLoadPerNode * 1.25) - bhost, ok := c.loadMap[host] + bserver, ok := c.loadMap[server] if !ok { - panic(fmt.Sprintf("given host(%s) not in loadsMap", bhost.Name)) + panic(fmt.Sprintf("given host(%s) not in loadsMap", bserver.Name)) } - if float64(bhost.Load)+1 <= avgLoadPerNode { + if float64(bserver.Load)+1 <= avgLoadPerNode { return true } @@ -249,23 +279,7 @@ func (c *Consistent) loadOK(host string) bool { } func (c *Consistent) delSlice(val uint64) { - idx := -1 - l := 0 - r := len(c.sortedSet) - 1 - for l <= r { - m := (l + r) / 2 - if c.sortedSet[m] == val { - idx = m - break - } else if c.sortedSet[m] < val { - l = m + 1 - } else if c.sortedSet[m] > val { - r = m - 1 - } - } - if idx != -1 { - c.sortedSet = append(c.sortedSet[:idx], c.sortedSet[idx+1:]...) - } + c.clients.Delete(item{val}) } func (c *Consistent) hash(key string) uint64 { diff --git a/consistent_test.go b/consistent_test.go index b639c46..dbc6d33 100644 --- a/consistent_test.go +++ b/consistent_test.go @@ -7,34 +7,34 @@ import ( func TestAdd(t *testing.T) { c := New() - - c.Add("127.0.0.1:8000") - if len(c.sortedSet) != replicationFactor { + c.Add("server-001") + if c.clients.Len() != c.replicationFactor { t.Fatal("vnodes number is incorrect") } } func TestGet(t *testing.T) { c := New() - - c.Add("127.0.0.1:8000") - host, err := c.Get("127.0.0.1:8000") + client := "client-001" + server := "server-001" + c.Add(server) + host, err := c.Get(client) if err != nil { t.Fatal(err) } - - if host != "127.0.0.1:8000" { - t.Fatal("returned host is not what expected") + // there is only one server, so it should process the client + if host != server { + t.Fatalf("returned host is not what expected: %s got %s", host, client) } } func TestRemove(t *testing.T) { c := New() + server := "server-001" + c.Add(server) + c.Remove(server) - c.Add("127.0.0.1:8000") - c.Remove("127.0.0.1:8000") - - if len(c.sortedSet) != 0 && len(c.hosts) != 0 { + if c.clients.Len() != 0 && len(c.servers) != 0 { t.Fatal(("remove is not working")) } @@ -43,8 +43,8 @@ func TestRemove(t *testing.T) { func TestGetLeast(t *testing.T) { c := New() - c.Add("127.0.0.1:8000") - c.Add("92.0.0.1:8000") + c.Add("shard-1") + c.Add("shard-2") for i := 0; i < 100; i++ { host, err := c.GetLeast("92.0.0.1:80001") @@ -70,7 +70,7 @@ func TestIncDone(t *testing.T) { c.Add("127.0.0.1:8000") c.Add("92.0.0.1:8000") - host, err := c.GetLeast("92.0.0.1:80001") + host, err := c.GetLeast("92.0.0.1:8000") if err != nil { t.Fatal(err) } @@ -97,9 +97,9 @@ func TestHosts(t *testing.T) { for _, h := range hosts { c.Add(h) } - fmt.Println("hosts in the ring", c.Hosts()) + fmt.Println("hosts in the ring", c.Servers()) - addedHosts := c.Hosts() + addedHosts := c.Servers() for _, h := range hosts { found := false for _, ah := range addedHosts { @@ -113,30 +113,6 @@ func TestHosts(t *testing.T) { } } c.Remove("127.0.0.1:8000") - fmt.Println("hosts in the ring", c.Hosts()) + fmt.Println("hosts in the ring", c.Servers()) } - -func TestDelSlice(t *testing.T) { - items := []uint64{0, 1, 2, 3, 5, 20, 22, 23, 25, 27, 28, 30, 35, 37, 1008, 1009} - deletes := []uint64{25, 37, 1009, 3, 100000} - - c := &Consistent{} - c.sortedSet = append(c.sortedSet, items...) - - fmt.Printf("before deletion%+v\n", c.sortedSet) - - for _, val := range deletes { - c.delSlice(val) - } - - for _, val := range deletes { - for _, item := range c.sortedSet { - if item == val { - t.Fatalf("%d wasn't deleted\n", val) - } - } - } - - fmt.Printf("after deletions: %+v\n", c.sortedSet) -} \ No newline at end of file diff --git a/example_test.go b/example_test.go index 164854f..0497615 100644 --- a/example_test.go +++ b/example_test.go @@ -27,7 +27,7 @@ func Example_consistent(t *testing.T) { log.Println(host) } -func Example_bounded() { +func Example_bounded(t *testing.T) { c := consistent.New() // adds the hosts to the ring @@ -43,6 +43,7 @@ func Example_bounded() { host, err := c.GetLeast("/app.html") if err != nil { log.Fatal(err) + t.Fail() } // increases the load of `host`, we have to call it before sending the request c.Inc(host) @@ -50,4 +51,5 @@ func Example_bounded() { log.Println("send request to", host) // call it when the work is done, to update the load of `host`. c.Done(host) + } diff --git a/examples/load_distribution.go b/examples/load_distribution.go new file mode 100644 index 0000000..1967c7d --- /dev/null +++ b/examples/load_distribution.go @@ -0,0 +1,69 @@ +package main + +import ( + "fmt" + "log" + + "github.com/lafikl/consistent" +) + +func main() { + prefix := "cluster" + consistentHashing := consistent.New() + clusters := []string{} + clusterCount := 120 + + for i := 1; i <= clusterCount; i++ { + cluster := fmt.Sprintf("%s-%d", prefix, i) + clusters = append(clusters, cluster) + } + // adds the hosts to the ring + consistentHashing.Add("shard-1") + consistentHashing.Add("shard-2") + consistentHashing.Add("shard-3") + log.Printf("------------- %d shards and %d clusters -------------", len(consistentHashing.Servers()), len(clusters)) + loadDistribution := distribute(clusters, consistentHashing) + printLoadDistribution(loadDistribution) + + consistentHashing.Remove("shard-2") + log.Printf("------------- %d shards and %d clusters -------------", len(consistentHashing.Servers()), len(clusters)) + loadDistribution = distribute(clusters, consistentHashing) + printLoadDistribution(loadDistribution) + + consistentHashing.Add("shard-2") + consistentHashing.Add("shard-4") + + loadDistribution = distribute(clusters, consistentHashing) + printLoadDistribution(loadDistribution) + + consistentHashing.Remove("shard-3") + consistentHashing.Remove("shard-4") + consistentHashing.Remove("shard-5") + loadDistribution = distribute(clusters, consistentHashing) + printLoadDistribution(loadDistribution) + + for i := clusterCount; i <= 2*clusterCount; i++ { + cluster := fmt.Sprintf("%s-%d", prefix, i) + clusters = append(clusters, cluster) + } + log.Printf("------------- %d shards and %d clusters -------------", len(consistentHashing.Servers()), len(clusters)) + loadDistribution = distribute(clusters, consistentHashing) + printLoadDistribution(loadDistribution) + +} + +func distribute(clusters []string, c *consistent.Consistent) map[string]int { + loadDistribution := map[string]int{} + for _, cluster := range clusters { + shard, _ := c.Get(cluster) + log.Printf("cluster: %s managed by shard: %s", cluster, shard) + loadDistribution[shard]++ + } + return loadDistribution +} + +func printLoadDistribution(distribution map[string]int) { + for key, value := range distribution { + log.Printf("shard: %s processes: %d clusters", key, value) + } +} diff --git a/go.mod b/go.mod index 48557b4..c9c2593 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module github.com/lafikl/consistent go 1.16 -require github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 +require ( + github.com/google/btree v1.1.2 + github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 +) diff --git a/go.sum b/go.sum index a228a6f..e57124d 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,6 @@ +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=