Add rate limiting

This commit is contained in:
2025-06-13 17:51:31 +03:00
parent 14f115fbe0
commit 1883d62cc6
3 changed files with 23 additions and 0 deletions

2
go.mod
View File

@@ -1,3 +1,5 @@
module imgdownloader module imgdownloader
go 1.24.3 go 1.24.3
require golang.org/x/time v0.12.0

2
go.sum Normal file
View File

@@ -0,0 +1,2 @@
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=

19
main.go
View File

@@ -1,6 +1,7 @@
package main package main
import ( import (
"context"
"encoding/json" "encoding/json"
"flag" "flag"
"fmt" "fmt"
@@ -10,15 +11,21 @@ import (
"path/filepath" "path/filepath"
"sync" "sync"
"time" "time"
"golang.org/x/time/rate"
) )
// HTTP клиент с таймаутом // HTTP клиент с таймаутом
var httpClient *http.Client var httpClient *http.Client
// Глобальный rate limiter
var limiter *rate.Limiter
func main() { func main() {
timeout := flag.Duration("t", 60*time.Second, "request timeout") timeout := flag.Duration("t", 60*time.Second, "request timeout")
retries := flag.Int("r", 5, "number of download attempts") retries := flag.Int("r", 5, "number of download attempts")
jobs := flag.Int("j", 5, "concurrent downloads") jobs := flag.Int("j", 5, "concurrent downloads")
rateLimit := flag.Float64("rate", 0, "maximum downloads per second (0 = no limit)")
flag.Parse() flag.Parse()
@@ -36,6 +43,11 @@ func main() {
Timeout: *timeout, Timeout: *timeout,
} }
// Инициализируем rate limiter
if *rateLimit > 0 {
limiter = rate.NewLimiter(rate.Limit(*rateLimit), 1)
}
// Создаем директорию для загрузок // Создаем директорию для загрузок
if err := os.MkdirAll(outputDir, 0755); err != nil { if err := os.MkdirAll(outputDir, 0755); err != nil {
fmt.Printf("Error creating directory: %v\n", err) fmt.Printf("Error creating directory: %v\n", err)
@@ -128,6 +140,13 @@ func downloadImage(url, filename string, maxRetries int) error {
// Одна попытка скачивания // Одна попытка скачивания
func attemptDownload(url, filename string) error { func attemptDownload(url, filename string) error {
// Применяем rate limiting если он включен
if limiter != nil {
if err := limiter.Wait(context.Background()); err != nil {
return fmt.Errorf("rate limiter error: %v", err)
}
}
resp, err := httpClient.Get(url) resp, err := httpClient.Get(url)
if err != nil { if err != nil {
return err return err