feat: catalogue generate now rate limits

Closes coop-cloud/organising#231.
This commit is contained in:
decentral1se 2021-11-03 06:53:19 +01:00
parent 08aca28d9d
commit e4e606efb0
No known key found for this signature in database
GPG Key ID: 5E2EF5A63E3718CC
2 changed files with 25 additions and 0 deletions

View File

@ -11,6 +11,7 @@ import (
"coopcloud.tech/abra/pkg/catalogue"
"coopcloud.tech/abra/pkg/config"
"coopcloud.tech/abra/pkg/git"
"coopcloud.tech/abra/pkg/limit"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"
)
@ -74,10 +75,14 @@ var catalogueGenerateCommand = &cli.Command{
logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(repos))
cloneLimiter := limit.New(10)
retrieveBar := formatter.CreateProgressbar(len(repos), "retrieving recipes...")
ch := make(chan string, len(repos))
for _, repoMeta := range repos {
go func(rm catalogue.RepoMeta) {
cloneLimiter.Begin()
defer cloneLimiter.End()
if recipeName != "" && recipeName != rm.Name {
ch <- rm.Name
retrieveBar.Add(1)

20
pkg/limit/limit.go Normal file
View File

@ -0,0 +1,20 @@
package limit // https://github.com/tidwall/limiter
// Limiter is for limiting the number of concurrent operations. This
type Limiter struct{ sem chan struct{} }
// New returns a new Limiter. The limit param is the maximum number of
// concurrent operations.
func New(limit int) *Limiter {
return &Limiter{make(chan struct{}, limit)}
}
// Begin an operation.
func (l *Limiter) Begin() {
l.sem <- struct{}{}
}
// End the operation.
func (l *Limiter) End() {
<-l.sem
}