diff --git a/components/engine/daemon/graphdriver/counter.go b/components/engine/daemon/graphdriver/counter.go
index 53123f52ce..2772bd247d 100644
--- a/components/engine/daemon/graphdriver/counter.go
+++ b/components/engine/daemon/graphdriver/counter.go
@@ -54,6 +54,9 @@ func (c *RefCounter) incdec(path string, infoOp func(minfo *minfo)) int {
 	}
 	infoOp(m)
 	count := m.count
+	if count <= 0 {
+		delete(c.counts, path)
+	}
 	c.mu.Unlock()
 	return count
 }