150 lines
3.4 KiB
Go
150 lines
3.4 KiB
Go
package main
|
||
|
||
import (
|
||
"encoding/json"
|
||
"flag"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"os"
|
||
"path/filepath"
|
||
"sync"
|
||
"time"
|
||
)
|
||
|
||
// HTTP клиент с таймаутом
|
||
var httpClient *http.Client
|
||
|
||
func main() {
|
||
timeout := flag.Duration("t", 60*time.Second, "request timeout")
|
||
retries := flag.Int("r", 5, "number of download attempts")
|
||
jobs := flag.Int("j", 5, "concurrent downloads")
|
||
|
||
flag.Parse()
|
||
|
||
args := flag.Args()
|
||
|
||
if len(args) != 2 {
|
||
fmt.Println("Usage: program <json-file> <output-dir>")
|
||
os.Exit(1)
|
||
}
|
||
|
||
jsonFile := args[0]
|
||
outputDir := args[1]
|
||
|
||
httpClient = &http.Client{
|
||
Timeout: *timeout,
|
||
}
|
||
|
||
// Создаем директорию для загрузок
|
||
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||
fmt.Printf("Error creating directory: %v\n", err)
|
||
os.Exit(1)
|
||
}
|
||
|
||
// Читаем JSON файл
|
||
data, err := os.ReadFile(jsonFile)
|
||
if err != nil {
|
||
fmt.Printf("Error reading JSON file: %v\n", err)
|
||
os.Exit(1)
|
||
}
|
||
|
||
// Парсим JSON в массив строк
|
||
var urls []string
|
||
if err := json.Unmarshal(data, &urls); err != nil {
|
||
fmt.Printf("Error parsing JSON: %v\n", err)
|
||
os.Exit(1)
|
||
}
|
||
|
||
// Семафор для ограничения параллелизма
|
||
sem := make(chan struct{}, *jobs)
|
||
var wg sync.WaitGroup
|
||
|
||
start := time.Now()
|
||
for i, url := range urls {
|
||
wg.Add(1)
|
||
sem <- struct{}{} // Занимаем слот
|
||
|
||
go func(idx int, url string) {
|
||
defer wg.Done()
|
||
defer func() { <-sem }() // Освобождаем слот
|
||
|
||
filename := filepath.Join(outputDir, fmt.Sprintf("image_%06d%s", idx, fileExtension(url)))
|
||
if err := downloadImage(url, filename, *retries); err != nil {
|
||
fmt.Printf("Error downloading %s: %v\n", url, err)
|
||
} else {
|
||
fmt.Printf("Downloaded %s -> %s\n", url, filename)
|
||
}
|
||
}(i, url)
|
||
}
|
||
|
||
wg.Wait()
|
||
fmt.Printf("\nDownloaded %d images in %v\n", len(urls), time.Since(start))
|
||
}
|
||
|
||
// Определяем расширение файла по Content-Type
|
||
func fileExtension(url string) string {
|
||
contentTypes := map[string]string{
|
||
"image/jpeg": ".jpg",
|
||
"image/png": ".png",
|
||
"image/gif": ".gif",
|
||
"image/webp": ".webp",
|
||
"image/svg+xml": ".svg",
|
||
}
|
||
|
||
resp, err := httpClient.Head(url)
|
||
if err == nil {
|
||
ct := resp.Header.Get("Content-Type")
|
||
if ext, ok := contentTypes[ct]; ok {
|
||
return ext
|
||
}
|
||
}
|
||
return ".bin" // расширение по умолчанию
|
||
}
|
||
|
||
// Скачиваем и сохраняем изображение
|
||
func downloadImage(url, filename string, maxRetries int) error {
|
||
const retryDelay = 1 * time.Second
|
||
|
||
var lastErr error
|
||
for attempt := 1; attempt <= maxRetries; attempt++ {
|
||
// Попытка скачать изображение
|
||
err := attemptDownload(url, filename)
|
||
if err == nil {
|
||
// Успешно скачали
|
||
return nil
|
||
}
|
||
|
||
lastErr = err
|
||
if attempt < maxRetries {
|
||
fmt.Printf("Attempt %d failed for %s: %v. Retrying in %v...\n",
|
||
attempt, url, err, retryDelay)
|
||
time.Sleep(retryDelay)
|
||
}
|
||
}
|
||
|
||
return fmt.Errorf("all %d download attempts failed: %v", maxRetries, lastErr)
|
||
}
|
||
|
||
// Одна попытка скачивания
|
||
func attemptDownload(url, filename string) error {
|
||
resp, err := httpClient.Get(url)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
if resp.StatusCode != http.StatusOK {
|
||
return fmt.Errorf("HTTP error: %s", resp.Status)
|
||
}
|
||
|
||
file, err := os.Create(filename)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer file.Close()
|
||
|
||
_, err = io.Copy(file, resp.Body)
|
||
return err
|
||
}
|