forked from toolshed/abra
		
	feat: catalogue generate now rate limits
Closes coop-cloud/organising#231.
This commit is contained in:
		| @ -11,6 +11,7 @@ import ( | ||||
| 	"coopcloud.tech/abra/pkg/catalogue" | ||||
| 	"coopcloud.tech/abra/pkg/config" | ||||
| 	"coopcloud.tech/abra/pkg/git" | ||||
| 	"coopcloud.tech/abra/pkg/limit" | ||||
| 	"github.com/sirupsen/logrus" | ||||
| 	"github.com/urfave/cli/v2" | ||||
| ) | ||||
| @ -74,10 +75,14 @@ var catalogueGenerateCommand = &cli.Command{ | ||||
|  | ||||
| 		logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(repos)) | ||||
|  | ||||
| 		cloneLimiter := limit.New(10) | ||||
| 		retrieveBar := formatter.CreateProgressbar(len(repos), "retrieving recipes...") | ||||
| 		ch := make(chan string, len(repos)) | ||||
| 		for _, repoMeta := range repos { | ||||
| 			go func(rm catalogue.RepoMeta) { | ||||
| 				cloneLimiter.Begin() | ||||
| 				defer cloneLimiter.End() | ||||
|  | ||||
| 				if recipeName != "" && recipeName != rm.Name { | ||||
| 					ch <- rm.Name | ||||
| 					retrieveBar.Add(1) | ||||
|  | ||||
							
								
								
									
										20
									
								
								pkg/limit/limit.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										20
									
								
								pkg/limit/limit.go
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,20 @@ | ||||
| package limit // https://github.com/tidwall/limiter | ||||
|  | ||||
| // Limiter is for limiting the number of concurrent operations. This | ||||
| type Limiter struct{ sem chan struct{} } | ||||
|  | ||||
| // New returns a new Limiter. The limit param is the maximum number of | ||||
| // concurrent operations. | ||||
| func New(limit int) *Limiter { | ||||
| 	return &Limiter{make(chan struct{}, limit)} | ||||
| } | ||||
|  | ||||
| // Begin an operation. | ||||
| func (l *Limiter) Begin() { | ||||
| 	l.sem <- struct{}{} | ||||
| } | ||||
|  | ||||
| // End the operation. | ||||
| func (l *Limiter) End() { | ||||
| 	<-l.sem | ||||
| } | ||||
		Reference in New Issue
	
	Block a user