Add rate limiting
This commit is contained in:
19
main.go
19
main.go
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
@@ -10,15 +11,21 @@ import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
// HTTP клиент с таймаутом
|
||||
var httpClient *http.Client
|
||||
|
||||
// Глобальный rate limiter
|
||||
var limiter *rate.Limiter
|
||||
|
||||
func main() {
|
||||
timeout := flag.Duration("t", 60*time.Second, "request timeout")
|
||||
retries := flag.Int("r", 5, "number of download attempts")
|
||||
jobs := flag.Int("j", 5, "concurrent downloads")
|
||||
rateLimit := flag.Float64("rate", 0, "maximum downloads per second (0 = no limit)")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
@@ -36,6 +43,11 @@ func main() {
|
||||
Timeout: *timeout,
|
||||
}
|
||||
|
||||
// Инициализируем rate limiter
|
||||
if *rateLimit > 0 {
|
||||
limiter = rate.NewLimiter(rate.Limit(*rateLimit), 1)
|
||||
}
|
||||
|
||||
// Создаем директорию для загрузок
|
||||
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||
fmt.Printf("Error creating directory: %v\n", err)
|
||||
@@ -128,6 +140,13 @@ func downloadImage(url, filename string, maxRetries int) error {
|
||||
|
||||
// Одна попытка скачивания
|
||||
func attemptDownload(url, filename string) error {
|
||||
// Применяем rate limiting если он включен
|
||||
if limiter != nil {
|
||||
if err := limiter.Wait(context.Background()); err != nil {
|
||||
return fmt.Errorf("rate limiter error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := httpClient.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
|
Reference in New Issue
Block a user