forked from coop-cloud/abra
feat: catalogue generate now rate limits
Closes coop-cloud/organising#231.
This commit is contained in:
parent
08aca28d9d
commit
e4e606efb0
|
@ -11,6 +11,7 @@ import (
|
||||||
"coopcloud.tech/abra/pkg/catalogue"
|
"coopcloud.tech/abra/pkg/catalogue"
|
||||||
"coopcloud.tech/abra/pkg/config"
|
"coopcloud.tech/abra/pkg/config"
|
||||||
"coopcloud.tech/abra/pkg/git"
|
"coopcloud.tech/abra/pkg/git"
|
||||||
|
"coopcloud.tech/abra/pkg/limit"
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
"github.com/urfave/cli/v2"
|
"github.com/urfave/cli/v2"
|
||||||
)
|
)
|
||||||
|
@ -74,10 +75,14 @@ var catalogueGenerateCommand = &cli.Command{
|
||||||
|
|
||||||
logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(repos))
|
logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(repos))
|
||||||
|
|
||||||
|
cloneLimiter := limit.New(10)
|
||||||
retrieveBar := formatter.CreateProgressbar(len(repos), "retrieving recipes...")
|
retrieveBar := formatter.CreateProgressbar(len(repos), "retrieving recipes...")
|
||||||
ch := make(chan string, len(repos))
|
ch := make(chan string, len(repos))
|
||||||
for _, repoMeta := range repos {
|
for _, repoMeta := range repos {
|
||||||
go func(rm catalogue.RepoMeta) {
|
go func(rm catalogue.RepoMeta) {
|
||||||
|
cloneLimiter.Begin()
|
||||||
|
defer cloneLimiter.End()
|
||||||
|
|
||||||
if recipeName != "" && recipeName != rm.Name {
|
if recipeName != "" && recipeName != rm.Name {
|
||||||
ch <- rm.Name
|
ch <- rm.Name
|
||||||
retrieveBar.Add(1)
|
retrieveBar.Add(1)
|
||||||
|
|
|
@ -0,0 +1,20 @@
|
||||||
|
package limit // https://github.com/tidwall/limiter
|
||||||
|
|
||||||
|
// Limiter is for limiting the number of concurrent operations. This
|
||||||
|
type Limiter struct{ sem chan struct{} }
|
||||||
|
|
||||||
|
// New returns a new Limiter. The limit param is the maximum number of
|
||||||
|
// concurrent operations.
|
||||||
|
func New(limit int) *Limiter {
|
||||||
|
return &Limiter{make(chan struct{}, limit)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Begin an operation.
|
||||||
|
func (l *Limiter) Begin() {
|
||||||
|
l.sem <- struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// End the operation.
|
||||||
|
func (l *Limiter) End() {
|
||||||
|
<-l.sem
|
||||||
|
}
|
Loading…
Reference in New Issue