The cache can now do reference counting so that the LRU algorithm is aware of

long-lived objects and won't clean them up. Oftentimes, the value returned
from a cache hit is short-lived. As a silly example:

	func GetUser(http.responseWrite) {
		user := cache.Get("user:1")
		response.Write(serialize(user))
	}

It's fine if the cache's GC cleans up "user:1" while the user variable has a reference to the
object..the cache's reference is removed and the real GC will clean it up
at some point after the user variable falls out of scope.

However, what if user is long-lived? Possibly stored as a reference to another
cached object? Normally (without this commit) the next time you call
cache.Get("user:1"), you'll get a miss and will need to refetch the object; even
though the original user object is still somewhere in memory - you just lost
your reference to it from the cache.

By enabling the Track() configuration flag, and calling TrackingGet() (instead
of Get), the cache will track that the object is in-use and won't GC it (even
if there's great memory pressure (what's the point? something else is holding on
to it anyways). Calling item.Release() will decrement the number of references.
When the count is 0, the item can be pruned from the cache.

The returned value is a TrackedItem which exposes:

- Value() interface{} (to get the actual cached value)
- Release() to release the item back in the cache

bucket.go

@@ -1,53 +1,53 @@
 package ccache
 
 import (
-  "sync"
+	"sync"
-  "time"
+	"time"
 )
 
 type Bucket struct {
-  sync.RWMutex
+	sync.RWMutex
-  lookup map[string]*Item
+	lookup map[string]*Item
 }
 
 func (b *Bucket) get(key string) *Item {
-  b.RLock()
+	b.RLock()
-  defer b.RUnlock()
+	defer b.RUnlock()
-  return b.lookup[key]
+	return b.lookup[key]
 }
 
 func (b *Bucket) set(key string, value interface{}, duration time.Duration) (*Item, bool) {
-  expires := time.Now().Add(duration)
+	expires := time.Now().Add(duration)
-  b.Lock()
+	b.Lock()
-  defer b.Unlock()
+	defer b.Unlock()
-  if existing, exists := b.lookup[key]; exists {
+	if existing, exists := b.lookup[key]; exists {
-    existing.Lock()
+		existing.Lock()
-    existing.value = value
+		existing.value = value
-    existing.expires = expires
+		existing.expires = expires
-    existing.Unlock()
+		existing.Unlock()
-    return existing, false
+		return existing, false
-  }
+	}
-  item := newItem(key, value, expires)
+	item := newItem(key, value, expires)
-  b.lookup[key] = item
+	b.lookup[key] = item
-  return item, true
+	return item, true
 }
 
 func (b *Bucket) delete(key string) {
-  b.Lock()
+	b.Lock()
-  defer b.Unlock()
+	defer b.Unlock()
-  delete(b.lookup, key)
+	delete(b.lookup, key)
 }
 
-func (b *Bucket) getAndDelete(key string) *Item{
+func (b *Bucket) getAndDelete(key string) *Item {
-  b.Lock()
+	b.Lock()
-  defer b.Unlock()
+	defer b.Unlock()
-  item := b.lookup[key]
+	item := b.lookup[key]
-  delete(b.lookup, key)
+	delete(b.lookup, key)
-  return item
+	return item
 }
 
 func (b *Bucket) clear() {
-  b.Lock()
+	b.Lock()
-  defer b.Unlock()
+	defer b.Unlock()
-  b.lookup = make(map[string]*Item)
+	b.lookup = make(map[string]*Item)
 }

bucket_test.go

@@ -1,64 +1,64 @@
 package ccache
 
 import (
-  "time"
+	"github.com/karlseguin/gspec"
-  "testing"
+	"testing"
-  "github.com/viki-org/gspec"
+	"time"
 )
 
 func TestGetMissFromBucket(t *testing.T) {
-  bucket := testBucket()
+	bucket := testBucket()
-  gspec.New(t).Expect(bucket.get("invalid")).ToBeNil()
+	gspec.New(t).Expect(bucket.get("invalid")).ToBeNil()
 }
 
 func TestGetHitFromBucket(t *testing.T) {
-  bucket := testBucket()
+	bucket := testBucket()
-  item := bucket.get("power")
+	item := bucket.get("power")
-  assertValue(t, item, "9000")
+	assertValue(t, item, "9000")
 }
 
 func TestDeleteItemFromBucket(t *testing.T) {
-  bucket := testBucket()
+	bucket := testBucket()
-  bucket.delete("power")
+	bucket.delete("power")
-  gspec.New(t).Expect(bucket.get("power")).ToBeNil()
+	gspec.New(t).Expect(bucket.get("power")).ToBeNil()
 }
 
 func TestSetsANewBucketItem(t *testing.T) {
-  spec := gspec.New(t)
+	spec := gspec.New(t)
-  bucket := testBucket()
+	bucket := testBucket()
-  item, new := bucket.set("spice", TestValue("flow"), time.Minute)
+	item, new := bucket.set("spice", TestValue("flow"), time.Minute)
-  assertValue(t, item, "flow")
+	assertValue(t, item, "flow")
-  item = bucket.get("spice")
+	item = bucket.get("spice")
-  assertValue(t, item, "flow")
+	assertValue(t, item, "flow")
-  spec.Expect(new).ToEqual(true)
+	spec.Expect(new).ToEqual(true)
 }
 
 func TestSetsAnExistingItem(t *testing.T) {
-  spec := gspec.New(t)
+	spec := gspec.New(t)
-  bucket := testBucket()
+	bucket := testBucket()
-  item, new := bucket.set("power", TestValue("9002"), time.Minute)
+	item, new := bucket.set("power", TestValue("9002"), time.Minute)
-  assertValue(t, item, "9002")
+	assertValue(t, item, "9002")
-  item = bucket.get("power")
+	item = bucket.get("power")
-  assertValue(t, item, "9002")
+	assertValue(t, item, "9002")
-  spec.Expect(new).ToEqual(false)
+	spec.Expect(new).ToEqual(false)
 }
 
 func testBucket() *Bucket {
-  b := &Bucket{lookup: make(map[string]*Item),}
+	b := &Bucket{lookup: make(map[string]*Item)}
-  b.lookup["power"] = &Item{
+	b.lookup["power"] = &Item{
-    key: "power",
+		key:   "power",
-    value: TestValue("9000"),
+		value: TestValue("9000"),
-  }
+	}
-  return b
+	return b
 }
 
 func assertValue(t *testing.T, item *Item, expected string) {
-  value := item.value.(TestValue)
+	value := item.value.(TestValue)
-  gspec.New(t).Expect(value).ToEqual(TestValue(expected))
+	gspec.New(t).Expect(value).ToEqual(TestValue(expected))
 }
 
 type TestValue string
 
 func (v TestValue) Expires() time.Time {
-  return time.Now()
+	return time.Now()
 }

cache.go

@@ -1,139 +1,172 @@
+// An LRU cached aimed at high concurrency
 package ccache
 
 import (
-  "time"
+	"container/list"
-  "runtime"
+	"hash/fnv"
-  "hash/fnv"
+	"runtime"
-  "container/list"
+	"sync/atomic"
+	"time"
 )
 
 type Cache struct {
-  *Configuration
+	*Configuration
-  list *list.List
+	list        *list.List
-  buckets []*Bucket
+	buckets     []*Bucket
-  bucketCount uint32
+	bucketCount uint32
-  deletables chan *Item
+	deletables  chan *Item
-  promotables chan *Item
+	promotables chan *Item
 }
 
 func New(config *Configuration) *Cache {
-  c := &Cache{
+	c := &Cache{
-    list: list.New(),
+		list:          list.New(),
-    Configuration: config,
+		Configuration: config,
-    bucketCount: uint32(config.buckets),
+		bucketCount:   uint32(config.buckets),
-    buckets: make([]*Bucket, config.buckets),
+		buckets:       make([]*Bucket, config.buckets),
-    deletables: make(chan *Item, config.deleteBuffer),
+		deletables:    make(chan *Item, config.deleteBuffer),
-    promotables: make(chan *Item, config.promoteBuffer),
+		promotables:   make(chan *Item, config.promoteBuffer),
-  }
+	}
-  for i := 0; i < config.buckets; i++ {
+	for i := 0; i < config.buckets; i++ {
-    c.buckets[i] = &Bucket{
+		c.buckets[i] = &Bucket{
-      lookup: make(map[string]*Item),
+			lookup: make(map[string]*Item),
-    }
+		}
-  }
+	}
-  go c.worker()
+	go c.worker()
-  return c
+	return c
 }
 
 func (c *Cache) Get(key string) interface{} {
-  bucket := c.bucket(key)
+	if item := c.get(key); item != nil {
-  item := bucket.get(key)
+		return item.value
-  if item == nil { return nil }
+	}
-  if item.expires.Before(time.Now()) {
+	return nil
-    c.deleteItem(bucket, item)
+}
-    return nil
+
-  }
+func (c *Cache) TrackingGet(key string) TrackedItem {
-  c.conditionalPromote(item)
+	item := c.get(key)
-  return item.value
+	if item == nil {
+		return NilTracked
+	}
+	item.track()
+	return item
+}
+
+func (c *Cache) get(key string) *Item {
+	bucket := c.bucket(key)
+	item := bucket.get(key)
+	if item == nil {
+		return nil
+	}
+	if item.expires.Before(time.Now()) {
+		c.deleteItem(bucket, item)
+		return nil
+	}
+	c.conditionalPromote(item)
+	return item
 }
 
 func (c *Cache) Set(key string, value interface{}, duration time.Duration) {
-  item, new := c.bucket(key).set(key, value, duration)
+	item, new := c.bucket(key).set(key, value, duration)
-  if new {
+	if new {
-    c.promote(item)
+		c.promote(item)
-  } else {
+	} else {
-    c.conditionalPromote(item)
+		c.conditionalPromote(item)
-  }
+	}
 }
 
 func (c *Cache) Fetch(key string, duration time.Duration, fetch func() (interface{}, error)) (interface{}, error) {
-  item := c.Get(key)
+	item := c.Get(key)
-  if item != nil { return item, nil }
+	if item != nil {
-  value, err := fetch()
+		return item, nil
-  if err == nil {
+	}
-    c.Set(key, value, duration)
+	value, err := fetch()
-  }
+	if err == nil {
-  return value, err
+		c.Set(key, value, duration)
+	}
+	return value, err
 }
 
 func (c *Cache) Delete(key string) {
-  item := c.bucket(key).getAndDelete(key)
+	item := c.bucket(key).getAndDelete(key)
-  if item != nil {
+	if item != nil {
-    c.deletables <- item
+		c.deletables <- item
-  }
+	}
 }
 
 //this isn't thread safe. It's meant to be called from non-concurrent tests
 func (c *Cache) Clear() {
-  for _, bucket := range c.buckets {
+	for _, bucket := range c.buckets {
-    bucket.clear()
+		bucket.clear()
-  }
+	}
-  c.list = list.New()
+	c.list = list.New()
 }
 
 func (c *Cache) deleteItem(bucket *Bucket, item *Item) {
-  bucket.delete(item.key) //stop othe GETs from getting it
+	bucket.delete(item.key) //stop othe GETs from getting it
-  c.deletables <- item
+	c.deletables <- item
 }
 
 func (c *Cache) bucket(key string) *Bucket {
-  h := fnv.New32a()
+	h := fnv.New32a()
-  h.Write([]byte(key))
+	h.Write([]byte(key))
-  index := h.Sum32() % c.bucketCount
+	index := h.Sum32() % c.bucketCount
-  return c.buckets[index]
+	return c.buckets[index]
 }
 
 func (c *Cache) conditionalPromote(item *Item) {
-  if item.shouldPromote(c.getsPerPromote) == false { return }
+	if item.shouldPromote(c.getsPerPromote) == false {
-  c.promote(item)
+		return
+	}
+	c.promote(item)
 }
 
 func (c *Cache) promote(item *Item) {
-  c.promotables <- item
+	c.promotables <- item
 }
 
 func (c *Cache) worker() {
-  ms := new(runtime.MemStats)
+	ms := new(runtime.MemStats)
-  for {
+	for {
-    select {
+		select {
-    case item := <- c.promotables:
+		case item := <-c.promotables:
-      wasNew := c.doPromote(item)
+			if wasNew := c.doPromote(item); wasNew == false {
-      if wasNew == false { continue }
+				continue
-      runtime.ReadMemStats(ms)
+			}
-      if ms.HeapAlloc > c.size { c.gc() }
+			runtime.ReadMemStats(ms)
-    case item := <- c.deletables:
+			if ms.HeapAlloc > c.size {
-      c.list.Remove(item.element)
+				c.gc()
-    }
+			}
-  }
+		case item := <-c.deletables:
+			c.list.Remove(item.element)
+		}
+	}
 }
 
 func (c *Cache) doPromote(item *Item) bool {
-  item.Lock()
+	item.Lock()
-  defer item.Unlock()
+	defer item.Unlock()
-  item.promotions = 0
+	item.promotions = 0
-  if item.element != nil { //not a new item
+	if item.element != nil { //not a new item
-    c.list.MoveToFront(item.element)
+		c.list.MoveToFront(item.element)
-    return false
+		return false
-  }
+	}
-  item.element = c.list.PushFront(item)
+	item.element = c.list.PushFront(item)
-  return true
+	return true
 }
 
 func (c *Cache) gc() {
-  for i := 0; i < c.itemsToPrune; i++ {
+	element := c.list.Back()
-    element := c.list.Back()
+	for i := 0; i < c.itemsToPrune; i++ {
-    if element == nil { return }
+		if element == nil {
-    item := element.Value.(*Item)
+			return
-    c.bucket(item.key).delete(item.key)
+		}
-    c.list.Remove(element)
+		prev := element.Prev()
-  }
+		item := element.Value.(*Item)
+		if c.tracking == false || atomic.LoadInt32(&item.refCount) == 0 {
+			c.bucket(item.key).delete(item.key)
+			c.list.Remove(element)
+		}
+		element = prev
+	}
 }

cache_test.go

@@ -0,0 +1,49 @@
+package ccache
+
+import (
+	"github.com/karlseguin/gspec"
+	"testing"
+	"strconv"
+	"time"
+)
+
+func TestGCsTheOldestItems(t *testing.T) {
+	spec := gspec.New(t)
+	cache := New(Configure().ItemsToPrune(10))
+	for i := 0; i < 500; i++ {
+		cache.Set(strconv.Itoa(i), i, time.Minute)
+	}
+	cache.gc()
+	spec.Expect(cache.Get("9")).ToBeNil()
+	spec.Expect(cache.Get("10").(int)).ToEqual(10)
+}
+
+func TestPromotedItemsDontGetPruned(t *testing.T) {
+	spec := gspec.New(t)
+	cache := New(Configure().ItemsToPrune(10).GetsPerPromote(1))
+	for i := 0; i < 500; i++ {
+		cache.Set(strconv.Itoa(i), i, time.Minute)
+	}
+	cache.Get("9")
+	time.Sleep(time.Millisecond * 10)
+	cache.gc()
+	spec.Expect(cache.Get("9").(int)).ToEqual(9)
+	spec.Expect(cache.Get("10")).ToBeNil()
+	spec.Expect(cache.Get("11").(int)).ToEqual(11)
+}
+
+func TestTrackerDoesNotCleanupHeldInstance(t *testing.T) {
+	spec := gspec.New(t)
+	cache := New(Configure().ItemsToPrune(10).Track())
+	for i := 0; i < 10; i++ {
+		cache.Set(strconv.Itoa(i), i, time.Minute)
+	}
+	item := cache.TrackingGet("0")
+	time.Sleep(time.Millisecond * 10)
+	cache.gc()
+	spec.Expect(cache.Get("0").(int)).ToEqual(0)
+	spec.Expect(cache.Get("1")).ToBeNil()
+	item.Release()
+	cache.gc()
+	spec.Expect(cache.Get("0")).ToBeNil()
+}

configuration.go

@@ -1,51 +1,58 @@
 package ccache
 
 type Configuration struct {
-  size uint64
+	size           uint64
-  buckets int
+	buckets        int
-  itemsToPrune int
+	itemsToPrune   int
-  deleteBuffer int
+	deleteBuffer   int
-  promoteBuffer int
+	promoteBuffer  int
-  getsPerPromote int32
+	getsPerPromote int32
+	tracking       bool
 }
 
 func Configure() *Configuration {
-  return &Configuration {
+	return &Configuration{
-    buckets: 64,
+		buckets:        64,
-    itemsToPrune: 500,
+		itemsToPrune:   500,
-    deleteBuffer: 1024,
+		deleteBuffer:   1024,
-    getsPerPromote: 10,
+		getsPerPromote: 10,
-    promoteBuffer: 1024,
+		promoteBuffer:  1024,
-    size: 500 * 1024 * 1024,
+		size:           500 * 1024 * 1024,
-  }
+		tracking:       false,
+	}
 }
 
 func (c *Configuration) Size(bytes uint64) *Configuration {
-  c.size = bytes
+	c.size = bytes
-  return c
+	return c
 }
 
 func (c *Configuration) Buckets(count int) *Configuration {
-  c.buckets = count
+	c.buckets = count
-  return c
+	return c
 }
 
 func (c *Configuration) ItemsToPrune(count int) *Configuration {
-  c.itemsToPrune = count
+	c.itemsToPrune = count
-  return c
+	return c
 }
 
 func (c *Configuration) PromoteBuffer(size int) *Configuration {
-  c.promoteBuffer = size
+	c.promoteBuffer = size
-  return c
+	return c
 }
 
 func (c *Configuration) DeleteBuffer(size int) *Configuration {
-  c.deleteBuffer = size
+	c.deleteBuffer = size
-  return c
+	return c
 }
 
 func (c *Configuration) GetsPerPromote(count int) *Configuration {
-  c.getsPerPromote = int32(count)
+	c.getsPerPromote = int32(count)
-  return c
+	return c
+}
+
+func (c *Configuration) Track() *Configuration {
+	c.tracking = true
+	return c
 }

item.go

@@ -1,30 +1,55 @@
 package ccache
 
 import (
-  "sync"
+	"container/list"
-  "time"
+	"sync"
-  "sync/atomic"
+	"sync/atomic"
-  "container/list"
+	"time"
 )
 
+type TrackedItem interface {
+	Value() interface{}
+	Release()
+}
+
+type nilItem struct{}
+
+func (n *nilItem) Value() interface{} { return nil }
+func (n *nilItem) Release()           {}
+
+var NilTracked = new(nilItem)
+
 type Item struct {
-  key string
+	key string
-  sync.RWMutex
+	sync.RWMutex
-  promotions int32
+	promotions int32
-  expires time.Time
+	refCount   int32
-  value interface{}
+	expires    time.Time
-  element *list.Element
+	value      interface{}
+	element    *list.Element
 }
 
 func newItem(key string, value interface{}, expires time.Time) *Item {
-  return &Item{
+	return &Item{
-    key: key,
+		key:        key,
-    value: value,
+		value:      value,
-    promotions: -1,
+		promotions: -1,
-    expires: expires,
+		expires:    expires,
-  }
+	}
 }
 
 func (i *Item) shouldPromote(getsPerPromote int32) bool {
-  return atomic.AddInt32(&i.promotions, 1) == getsPerPromote
+	return atomic.AddInt32(&i.promotions, 1) == getsPerPromote
+}
+
+func (i *Item) Value() interface{} {
+	return i.value
+}
+
+func (i *Item) track() {
+	atomic.AddInt32(&i.refCount, 1)
+}
+
+func (i *Item) Release() {
+	atomic.AddInt32(&i.refCount, -1)
 }

item_test.go

@@ -1,13 +1,13 @@
 package ccache
 
 import (
-  "testing"
+	"github.com/karlseguin/gspec"
-  "github.com/viki-org/gspec"
+	"testing"
 )
 
 func TestItemPromotability(t *testing.T) {
-  spec := gspec.New(t)
+	spec := gspec.New(t)
-  item := &Item{promotions: 4}
+	item := &Item{promotions: 4}
-  spec.Expect(item.shouldPromote(5)).ToEqual(true)
+	spec.Expect(item.shouldPromote(5)).ToEqual(true)
-  spec.Expect(item.shouldPromote(5)).ToEqual(false)
+	spec.Expect(item.shouldPromote(5)).ToEqual(false)
 }