From 080b2a353f928f1478277ef785a1996f67f0c5e5 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 14 Dec 2016 21:21:18 +0000 Subject: [PATCH 1/2] Remove btrfs quota groups after containers destroyed This fix tries to address the issue raised in 29325 where btrfs quota groups are not clean up even after containers have been destroyed. The reason for the issue is that btrfs quota groups have to be explicitly destroyed. This fix fixes this issue. This fix is tested manually in Ubuntu 16.04, with steps specified in 29325. This fix fixes 29325. Signed-off-by: Yong Tang Upstream-commit: e907c6418a2ae7754f69fede5897a7176250e8bd Component: engine --- .../engine/daemon/graphdriver/btrfs/btrfs.go | 99 +++++++++++++++---- 1 file changed, 79 insertions(+), 20 deletions(-) diff --git a/components/engine/daemon/graphdriver/btrfs/btrfs.go b/components/engine/daemon/graphdriver/btrfs/btrfs.go index 2ff41a0b8b..63c68dc6a2 100644 --- a/components/engine/daemon/graphdriver/btrfs/btrfs.go +++ b/components/engine/daemon/graphdriver/btrfs/btrfs.go @@ -16,13 +16,16 @@ import "C" import ( "fmt" + "math" "os" "path" "path/filepath" "strings" + "sync" "syscall" "unsafe" + "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/mount" @@ -119,6 +122,7 @@ type Driver struct { gidMaps []idtools.IDMap options btrfsOptions quotaEnabled bool + once sync.Once } // String prints the name of the driver (btrfs). @@ -237,7 +241,7 @@ func isSubvolume(p string) (bool, error) { return bufStat.Ino == C.BTRFS_FIRST_FREE_OBJECTID, nil } -func subvolDelete(dirpath, name string) error { +func subvolDelete(dirpath, name string, quotaEnabled bool) error { dir, err := openDir(dirpath) if err != nil { return err @@ -265,7 +269,7 @@ func subvolDelete(dirpath, name string) error { return fmt.Errorf("Failed to test if %s is a btrfs subvolume: %v", p, err) } if sv { - if err := subvolDelete(path.Dir(p), f.Name()); err != nil { + if err := subvolDelete(path.Dir(p), f.Name(), quotaEnabled); err != nil { return fmt.Errorf("Failed to destroy btrfs child subvolume (%s) of parent (%s): %v", p, dirpath, err) } } @@ -276,6 +280,21 @@ func subvolDelete(dirpath, name string) error { return fmt.Errorf("Recursively walking subvolumes for %s failed: %v", dirpath, err) } + if quotaEnabled { + if qgroupid, err := subvolLookupQgroup(fullPath); err == nil { + var args C.struct_btrfs_ioctl_qgroup_create_args + args.qgroupid = C.__u64(qgroupid) + + _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_CREATE, + uintptr(unsafe.Pointer(&args))) + if errno != 0 { + logrus.Errorf("Failed to delete btrfs qgroup %v for %s: %v", qgroupid, fullPath, errno.Error()) + } + } else { + logrus.Errorf("Failed to lookup btrfs qgroup for %s: %v", fullPath, err.Error()) + } + } + // all subvolumes have been removed // now remove the one originally passed in for i, c := range []byte(name) { @@ -289,15 +308,25 @@ func subvolDelete(dirpath, name string) error { return nil } +func (d *Driver) updateQuotaStatus() { + d.once.Do(func() { + if !d.quotaEnabled { + // In case quotaEnabled is not set, check qgroup and update quotaEnabled as needed + if err := subvolQgroupStatus(d.home); err != nil { + // quota is still not enabled + return + } + d.quotaEnabled = true + } + }) +} + func (d *Driver) subvolEnableQuota() error { + d.updateQuotaStatus() + if d.quotaEnabled { return nil } - // In case quotaEnabled is not set, check qgroup and update quotaEnabled as needed - if _, err := subvolLookupQgroup(d.home); err == nil { - d.quotaEnabled = true - return nil - } dir, err := openDir(d.home) if err != nil { @@ -319,13 +348,10 @@ func (d *Driver) subvolEnableQuota() error { } func (d *Driver) subvolDisableQuota() error { + d.updateQuotaStatus() + if !d.quotaEnabled { - // In case quotaEnabled is not set, check qgroup and update quotaEnabled as needed - if _, err := subvolLookupQgroup(d.home); err != nil { - // quota is still not enabled - return nil - } - d.quotaEnabled = true + return nil } dir, err := openDir(d.home) @@ -348,13 +374,10 @@ func (d *Driver) subvolDisableQuota() error { } func (d *Driver) subvolRescanQuota() error { + d.updateQuotaStatus() + if !d.quotaEnabled { - // In case quotaEnabled is not set, check qgroup and update quotaEnabled as needed - if _, err := subvolLookupQgroup(d.home); err != nil { - // quota is still not enabled - return nil - } - d.quotaEnabled = true + return nil } dir, err := openDir(d.home) @@ -392,6 +415,38 @@ func subvolLimitQgroup(path string, size uint64) error { return nil } +// subvolQgroupStatus performs a BTRFS_IOC_TREE_SEARCH on the root path +// with search key of BTRFS_QGROUP_STATUS_KEY. +// In case qgroup is enabled, the retuned key type will match BTRFS_QGROUP_STATUS_KEY. +// For more details please see https://github.com/kdave/btrfs-progs/blob/v4.9/qgroup.c#L1035 +func subvolQgroupStatus(path string) error { + dir, err := openDir(path) + if err != nil { + return err + } + defer closeDir(dir) + + var args C.struct_btrfs_ioctl_search_args + args.key.tree_id = C.BTRFS_QUOTA_TREE_OBJECTID + args.key.min_type = C.BTRFS_QGROUP_STATUS_KEY + args.key.max_type = C.BTRFS_QGROUP_STATUS_KEY + args.key.max_objectid = C.__u64(math.MaxUint64) + args.key.max_offset = C.__u64(math.MaxUint64) + args.key.max_transid = C.__u64(math.MaxUint64) + args.key.nr_items = 4096 + + _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_TREE_SEARCH, + uintptr(unsafe.Pointer(&args))) + if errno != 0 { + return fmt.Errorf("Failed to search qgroup for %s: %v", path, errno.Error()) + } + sh := (*C.struct_btrfs_ioctl_search_header)(unsafe.Pointer(&args.buf)) + if sh._type != C.BTRFS_QGROUP_STATUS_KEY { + return fmt.Errorf("Invalid qgroup search header type for %s: %v", path, sh._type) + } + return nil +} + func subvolLookupQgroup(path string) (uint64, error) { dir, err := openDir(path) if err != nil { @@ -533,7 +588,11 @@ func (d *Driver) Remove(id string) error { if _, err := os.Stat(dir); err != nil { return err } - if err := subvolDelete(d.subvolumesDir(), id); err != nil { + + // Call updateQuotaStatus() to invoke status update + d.updateQuotaStatus() + + if err := subvolDelete(d.subvolumesDir(), id, d.quotaEnabled); err != nil { return err } if err := system.EnsureRemoveAll(dir); err != nil { From a81b0c835e5ac56a17524efc3cd14275bcaadba3 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 24 May 2017 18:00:02 -0700 Subject: [PATCH 2/2] Persist the quota size for btrfs so that daemon restart keeps quota This commit is an extension of fix for 29325 based on the review comment. In this commit, the quota size for btrfs is kept in `/var/lib/docker/btrfs/quotas` so that a daemon restart keeps quota. Signed-off-by: Yong Tang Upstream-commit: 16328cc207a493ecff0cabc11ebf51e958131418 Component: engine --- .../engine/daemon/graphdriver/btrfs/btrfs.go | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/components/engine/daemon/graphdriver/btrfs/btrfs.go b/components/engine/daemon/graphdriver/btrfs/btrfs.go index 63c68dc6a2..3eb4ce8ed1 100644 --- a/components/engine/daemon/graphdriver/btrfs/btrfs.go +++ b/components/engine/daemon/graphdriver/btrfs/btrfs.go @@ -16,10 +16,12 @@ import "C" import ( "fmt" + "io/ioutil" "math" "os" "path" "path/filepath" + "strconv" "strings" "sync" "syscall" @@ -477,6 +479,14 @@ func (d *Driver) subvolumesDirID(id string) string { return path.Join(d.subvolumesDir(), id) } +func (d *Driver) quotasDir() string { + return path.Join(d.home, "quotas") +} + +func (d *Driver) quotasDirID(id string) string { + return path.Join(d.quotasDir(), id) +} + // CreateReadWrite creates a layer that is writable for use as a container // file system. func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { @@ -485,6 +495,7 @@ func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts // Create the filesystem with given id. func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { + quotas := path.Join(d.home, "quotas") subvolumes := path.Join(d.home, "subvolumes") rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) if err != nil { @@ -521,9 +532,16 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { if err := d.parseStorageOpt(storageOpt, driver); err != nil { return err } + if err := d.setStorageSize(path.Join(subvolumes, id), driver); err != nil { return err } + if err := idtools.MkdirAllAs(quotas, 0700, rootUID, rootGID); err != nil { + return err + } + if err := ioutil.WriteFile(path.Join(quotas, id), []byte(fmt.Sprint(driver.options.size)), 0644); err != nil { + return err + } } // if we have a remapped root (user namespaces enabled), change the created snapshot @@ -588,6 +606,14 @@ func (d *Driver) Remove(id string) error { if _, err := os.Stat(dir); err != nil { return err } + quotasDir := d.quotasDirID(id) + if _, err := os.Stat(quotasDir); err == nil { + if err := os.Remove(quotasDir); err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err + } // Call updateQuotaStatus() to invoke status update d.updateQuotaStatus() @@ -616,6 +642,17 @@ func (d *Driver) Get(id, mountLabel string) (string, error) { return "", fmt.Errorf("%s: not a directory", dir) } + if quota, err := ioutil.ReadFile(d.quotasDirID(id)); err == nil { + if size, err := strconv.ParseUint(string(quota), 10, 64); err == nil && size >= d.options.minSpace { + if err := d.subvolEnableQuota(); err != nil { + return "", err + } + if err := subvolLimitQgroup(dir, size); err != nil { + return "", err + } + } + } + return dir, nil }