forked from toolshed/abra
		
	feat: catalogue generate now rate limits
Closes coop-cloud/organising#231.
This commit is contained in:
		@ -11,6 +11,7 @@ import (
 | 
			
		||||
	"coopcloud.tech/abra/pkg/catalogue"
 | 
			
		||||
	"coopcloud.tech/abra/pkg/config"
 | 
			
		||||
	"coopcloud.tech/abra/pkg/git"
 | 
			
		||||
	"coopcloud.tech/abra/pkg/limit"
 | 
			
		||||
	"github.com/sirupsen/logrus"
 | 
			
		||||
	"github.com/urfave/cli/v2"
 | 
			
		||||
)
 | 
			
		||||
@ -74,10 +75,14 @@ var catalogueGenerateCommand = &cli.Command{
 | 
			
		||||
 | 
			
		||||
		logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(repos))
 | 
			
		||||
 | 
			
		||||
		cloneLimiter := limit.New(10)
 | 
			
		||||
		retrieveBar := formatter.CreateProgressbar(len(repos), "retrieving recipes...")
 | 
			
		||||
		ch := make(chan string, len(repos))
 | 
			
		||||
		for _, repoMeta := range repos {
 | 
			
		||||
			go func(rm catalogue.RepoMeta) {
 | 
			
		||||
				cloneLimiter.Begin()
 | 
			
		||||
				defer cloneLimiter.End()
 | 
			
		||||
 | 
			
		||||
				if recipeName != "" && recipeName != rm.Name {
 | 
			
		||||
					ch <- rm.Name
 | 
			
		||||
					retrieveBar.Add(1)
 | 
			
		||||
 | 
			
		||||
							
								
								
									
										20
									
								
								pkg/limit/limit.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										20
									
								
								pkg/limit/limit.go
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,20 @@
 | 
			
		||||
package limit // https://github.com/tidwall/limiter
 | 
			
		||||
 | 
			
		||||
// Limiter is for limiting the number of concurrent operations. This
 | 
			
		||||
type Limiter struct{ sem chan struct{} }
 | 
			
		||||
 | 
			
		||||
// New returns a new Limiter. The limit param is the maximum number of
 | 
			
		||||
// concurrent operations.
 | 
			
		||||
func New(limit int) *Limiter {
 | 
			
		||||
	return &Limiter{make(chan struct{}, limit)}
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
// Begin an operation.
 | 
			
		||||
func (l *Limiter) Begin() {
 | 
			
		||||
	l.sem <- struct{}{}
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
// End the operation.
 | 
			
		||||
func (l *Limiter) End() {
 | 
			
		||||
	<-l.sem
 | 
			
		||||
}
 | 
			
		||||
		Reference in New Issue
	
	Block a user