Add rate limiting
This commit is contained in:
2
go.mod
2
go.mod
@@ -1,3 +1,5 @@
|
|||||||
module imgdownloader
|
module imgdownloader
|
||||||
|
|
||||||
go 1.24.3
|
go 1.24.3
|
||||||
|
|
||||||
|
require golang.org/x/time v0.12.0
|
||||||
|
2
go.sum
Normal file
2
go.sum
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
|
||||||
|
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
|
19
main.go
19
main.go
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -10,15 +11,21 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/time/rate"
|
||||||
)
|
)
|
||||||
|
|
||||||
// HTTP клиент с таймаутом
|
// HTTP клиент с таймаутом
|
||||||
var httpClient *http.Client
|
var httpClient *http.Client
|
||||||
|
|
||||||
|
// Глобальный rate limiter
|
||||||
|
var limiter *rate.Limiter
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
timeout := flag.Duration("t", 60*time.Second, "request timeout")
|
timeout := flag.Duration("t", 60*time.Second, "request timeout")
|
||||||
retries := flag.Int("r", 5, "number of download attempts")
|
retries := flag.Int("r", 5, "number of download attempts")
|
||||||
jobs := flag.Int("j", 5, "concurrent downloads")
|
jobs := flag.Int("j", 5, "concurrent downloads")
|
||||||
|
rateLimit := flag.Float64("rate", 0, "maximum downloads per second (0 = no limit)")
|
||||||
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
@@ -36,6 +43,11 @@ func main() {
|
|||||||
Timeout: *timeout,
|
Timeout: *timeout,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Инициализируем rate limiter
|
||||||
|
if *rateLimit > 0 {
|
||||||
|
limiter = rate.NewLimiter(rate.Limit(*rateLimit), 1)
|
||||||
|
}
|
||||||
|
|
||||||
// Создаем директорию для загрузок
|
// Создаем директорию для загрузок
|
||||||
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||||
fmt.Printf("Error creating directory: %v\n", err)
|
fmt.Printf("Error creating directory: %v\n", err)
|
||||||
@@ -128,6 +140,13 @@ func downloadImage(url, filename string, maxRetries int) error {
|
|||||||
|
|
||||||
// Одна попытка скачивания
|
// Одна попытка скачивания
|
||||||
func attemptDownload(url, filename string) error {
|
func attemptDownload(url, filename string) error {
|
||||||
|
// Применяем rate limiting если он включен
|
||||||
|
if limiter != nil {
|
||||||
|
if err := limiter.Wait(context.Background()); err != nil {
|
||||||
|
return fmt.Errorf("rate limiter error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
resp, err := httpClient.Get(url)
|
resp, err := httpClient.Get(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
Reference in New Issue
Block a user