Compare commits
11 Commits
9539c193c6
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
61a6fcee03
|
|||
|
ad31687111
|
|||
|
43c2c22de7
|
|||
|
fbc43946f8
|
|||
|
daf99c5b66
|
|||
|
4f8118562e
|
|||
|
1e32de279e
|
|||
|
b5c8ec45aa
|
|||
|
6e51b420d4
|
|||
|
f9732d9c33
|
|||
|
cfeced8bd9
|
@@ -1,39 +0,0 @@
|
||||
name: docker-release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.25.5'
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Yandex Cloud Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: cr.yandex
|
||||
username: oauth
|
||||
password: ${{ secrets.YANDEX_CLOUD_OAUTH_TOKEN }}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
push: true
|
||||
tags: |
|
||||
cr.yandex/${{ secrets.YANDEX_CLOUD_REGISTRY_ID }}/trackers:${{ github.ref_name }}
|
||||
cr.yandex/${{ secrets.YANDEX_CLOUD_REGISTRY_ID }}/trackers:latest
|
||||
platforms: linux/amd64
|
||||
@@ -6,16 +6,20 @@ on:
|
||||
- 'v*'
|
||||
|
||||
jobs:
|
||||
|
||||
goreleaser:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.25.5'
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Run GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v6
|
||||
@@ -25,3 +29,30 @@ jobs:
|
||||
args: release --clean
|
||||
env:
|
||||
GITEA_TOKEN: '${{ secrets.RELEASE_TOKEN }}'
|
||||
|
||||
docker-image:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Yandex Cloud Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: cr.yandex
|
||||
username: oauth
|
||||
password: ${{ secrets.YANDEX_CLOUD_OAUTH_TOKEN }}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
file: ./Dockerfile
|
||||
context: .
|
||||
push: true
|
||||
tags: |
|
||||
cr.yandex/${{ secrets.YANDEX_CLOUD_REGISTRY_ID }}/trackers:${{ github.ref_name }}
|
||||
cr.yandex/${{ secrets.YANDEX_CLOUD_REGISTRY_ID }}/trackers:latest
|
||||
platforms: linux/amd64
|
||||
107
AGENTS.md
107
AGENTS.md
@@ -1,9 +1,104 @@
|
||||
## Role
|
||||
# AGENTS.md
|
||||
|
||||
Write code like a Go Senior Developer.
|
||||
Use the best practices, write modern idiomatic code, and handle all errors.
|
||||
Keep an eye on security.
|
||||
This file provides guidance to LLM agents when working with code in this repository.
|
||||
|
||||
## Context
|
||||
## Project Overview
|
||||
|
||||
- go 1.25.5
|
||||
Trackers is a Go application that aggregates torrent tracker links from multiple sources (HTTP, HTTPS, and local files). It:
|
||||
- Polls configured sources at regular intervals
|
||||
- Deduplicates and validates tracker links
|
||||
- Caches results to disk for resilience
|
||||
- Serves aggregated tracker lists via HTTP API
|
||||
- Runs a single HTTP server with minimal dependencies
|
||||
|
||||
This is an educational project written in Go 1.25.5.
|
||||
|
||||
## Development Commands
|
||||
|
||||
### Build
|
||||
```bash
|
||||
go build -o trackers ./main.go
|
||||
```
|
||||
|
||||
### Run
|
||||
```bash
|
||||
go run main.go -config config.toml
|
||||
```
|
||||
|
||||
### Test
|
||||
The codebase has no tests currently. Tests can be added with:
|
||||
```bash
|
||||
go test ./...
|
||||
```
|
||||
|
||||
To run a single test:
|
||||
```bash
|
||||
go test -run TestName
|
||||
```
|
||||
|
||||
### Lint
|
||||
```bash
|
||||
go fmt ./...
|
||||
go vet ./...
|
||||
```
|
||||
|
||||
### Release Build
|
||||
Uses goreleaser for cross-platform builds (Linux amd64/arm64):
|
||||
```bash
|
||||
goreleaser build --snapshot --clean
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
**Single File Design**: All code is in `main.go` with clear functional separation:
|
||||
|
||||
1. **Config System** (`loadConfig`): Parses TOML configuration with defaults
|
||||
- `port`: HTTP server port (default: 8080)
|
||||
- `cache_dir`: Directory for caching tracker lists (default: cache/)
|
||||
- `poll_interval`: How often to refresh sources (default: 60m)
|
||||
- `sources`: Array of URLs/file paths to fetch tracker lists from
|
||||
|
||||
2. **Aggregator** (type `Aggregator`): Thread-safe in-memory deduplication
|
||||
- Maintains per-source tracker sets using `sync.RWMutex`
|
||||
- `Update()`: Stores new tracker list for a source
|
||||
- `List()`: Returns combined sorted list across all sources
|
||||
|
||||
3. **Polling System** (`pollSource`/`runOnce`): Background goroutine per source
|
||||
- Fetches source on startup and at configured intervals
|
||||
- Updates aggregator and writes to cache on success
|
||||
- Graceful shutdown on SIGINT/SIGTERM
|
||||
|
||||
4. **HTTP Handler** (`/list` endpoint): Returns deduplicated tracker list as plain text
|
||||
- Links separated by double newlines
|
||||
- Read timeouts enforce reasonable request handling
|
||||
|
||||
5. **Source Fetching** (`fetchSource`): Pluggable source handlers
|
||||
- **HTTP/HTTPS**: Makes requests with context support and timeout (15s)
|
||||
- **File**: Reads local files via `file://` URLs
|
||||
- Response is parsed line-by-line
|
||||
|
||||
6. **Link Validation** (`normalizeLinks`/`isValidTrackerLink`):
|
||||
- Strips whitespace and empty lines
|
||||
- Validates URL format and supported schemes: `http`, `https`, `udp`, `ws`, `wss`
|
||||
- Deduplicates via map-based set
|
||||
|
||||
7. **Caching** (`writeCache`/`loadCachedLinks`): SHA1-hashed filenames in `cache_dir/`
|
||||
- Enables graceful degradation if source becomes unavailable
|
||||
- Filenames are hex-encoded SHA1(source_url) + ".txt"
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
- **No external dependencies except go-toml**: Keeps binary small and build simple
|
||||
- **Simple HTTP server**: Uses stdlib `net/http` instead of frameworks
|
||||
- **Per-source goroutines**: Allows independent polling without blocking
|
||||
- **RWMutex for reads**: Readers don't block each other when listing trackers
|
||||
- **Context propagation**: Respects shutdown signals in all async operations
|
||||
- **Line-based parsing**: Flexible input format (handles various tracker list formats)
|
||||
|
||||
## Testing Notes
|
||||
|
||||
The project follows Go conventions but has no test files. Consider adding tests for:
|
||||
- Link validation edge cases
|
||||
- Config parsing with invalid inputs
|
||||
- Concurrent aggregator updates
|
||||
- Cache file I/O
|
||||
51
README.md
51
README.md
@@ -1,32 +1,51 @@
|
||||
# Trackers
|
||||
|
||||
Опрашивает ссылки со списками torrent-трекеров,
|
||||
соединяет их в один список. Поддерживает кеширование,
|
||||
http, file источники.
|
||||
A torrent tracker aggregator that polls multiple sources and serves a unified,
|
||||
deduplicated list of tracker URLs via HTTP API.
|
||||
|
||||
Учебный проект.
|
||||
**Educational project written in Go 1.25.5**
|
||||
|
||||
## Запуск
|
||||
## Features
|
||||
|
||||
- Polls tracker sources at configurable intervals
|
||||
- Supports HTTP/HTTPS and local file sources
|
||||
- Deduplicates and validates tracker URLs
|
||||
- Persistent disk caching for resilience
|
||||
- HTTP proxy support via environment variables
|
||||
- Request logging for monitoring
|
||||
- Single binary with minimal dependencies
|
||||
|
||||
## Usage
|
||||
|
||||
```shell
|
||||
trackers --config config.toml
|
||||
trackers -config config.toml
|
||||
```
|
||||
|
||||
Получение списка трекеров:
|
||||
Get the aggregated tracker list:
|
||||
|
||||
```shell
|
||||
curl http://127.0.0.1:8080/list
|
||||
```
|
||||
|
||||
## Конфигурация
|
||||
## Configuration
|
||||
|
||||
Источники для трекеров описываются в toml-конфиге:
|
||||
See [config.dist.toml](config.dist.toml) for a complete configuration example with comments.
|
||||
|
||||
```toml
|
||||
port = 8080
|
||||
Copy the example file and customize:
|
||||
|
||||
sources = [
|
||||
"https://example.com",
|
||||
"file:///home/user/local-file.txt",
|
||||
]
|
||||
```
|
||||
```shell
|
||||
cp config.dist.toml config.toml
|
||||
# Edit config.toml with your sources
|
||||
```
|
||||
|
||||
## Proxy Support
|
||||
|
||||
HTTP requests respect standard proxy environment variables:
|
||||
|
||||
```shell
|
||||
export HTTP_PROXY=http://proxy.example.com:8080
|
||||
export HTTPS_PROXY=http://proxy.example.com:8080
|
||||
trackers -config config.toml
|
||||
```
|
||||
|
||||
See [config.dist.toml](config.dist.toml) for details.
|
||||
44
config.dist.toml
Normal file
44
config.dist.toml
Normal file
@@ -0,0 +1,44 @@
|
||||
# Trackers Configuration Example
|
||||
# Copy this file to config.toml and adjust values for your setup
|
||||
|
||||
# HTTP server port
|
||||
# Default: 8080
|
||||
port = 8080
|
||||
|
||||
# Directory for caching tracker lists
|
||||
# Used to persist tracker data between restarts and handle source failures
|
||||
# Default: "cache"
|
||||
cache_dir = "cache"
|
||||
|
||||
# Interval between polling each source
|
||||
# Valid units: s (seconds), m (minutes), h (hours)
|
||||
# Examples: "30s", "5m", "1h", "90m"
|
||||
# Default: "60m"
|
||||
poll_interval = "60m"
|
||||
|
||||
# List of tracker sources to aggregate
|
||||
# Supported schemes:
|
||||
# - http:// / https:// - Remote HTTP endpoints
|
||||
# - file:// - Local files (e.g., file:///path/to/trackers.txt)
|
||||
#
|
||||
# Each source should return tracker URLs, one per line
|
||||
# Blank lines and duplicates are automatically filtered
|
||||
sources = [
|
||||
"https://example.com/trackers/all.txt",
|
||||
"https://another-source.org/trackers.txt",
|
||||
# "file:///etc/trackers/local.txt",
|
||||
]
|
||||
|
||||
# Proxy Configuration
|
||||
# ==================
|
||||
# HTTP requests automatically respect standard proxy environment variables:
|
||||
#
|
||||
# HTTP_PROXY - Proxy for HTTP requests (e.g., http://proxy.example.com:8080)
|
||||
# HTTPS_PROXY - Proxy for HTTPS requests (e.g., http://proxy.example.com:8080)
|
||||
# NO_PROXY - Comma-separated list of hosts to bypass proxy (e.g., localhost,127.0.0.1)
|
||||
#
|
||||
# Example usage:
|
||||
# export HTTP_PROXY=http://proxy.example.com:8080
|
||||
# export HTTPS_PROXY=http://proxy.example.com:8080
|
||||
# export NO_PROXY=localhost,127.0.0.1,.local
|
||||
# ./trackers -config config.toml
|
||||
16
main.go
16
main.go
@@ -122,7 +122,13 @@ func main() {
|
||||
|
||||
agg := NewAggregator()
|
||||
|
||||
client := &http.Client{Timeout: 15 * time.Second}
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: 15 * time.Second,
|
||||
Transport: transport,
|
||||
}
|
||||
|
||||
for _, source := range cfg.Sources {
|
||||
cached, err := loadCachedLinks(cfg.CacheDir, source)
|
||||
@@ -143,6 +149,7 @@ func main() {
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/list", func(w http.ResponseWriter, r *http.Request) {
|
||||
log.Printf("request /list from %s [%s %s]", r.RemoteAddr, r.Method, r.UserAgent())
|
||||
links := agg.List()
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
for i, link := range links {
|
||||
@@ -151,6 +158,7 @@ func main() {
|
||||
}
|
||||
_, _ = w.Write([]byte(link))
|
||||
}
|
||||
log.Printf("response /list to %s: %d links", r.RemoteAddr, len(links))
|
||||
})
|
||||
|
||||
server := &http.Server{
|
||||
@@ -197,10 +205,11 @@ func pollSource(ctx context.Context, source string, interval time.Duration, cach
|
||||
func runOnce(ctx context.Context, source string, cacheDir string, agg *Aggregator, client *http.Client) {
|
||||
links, err := fetchSource(ctx, source, client)
|
||||
if err != nil {
|
||||
log.Printf("poll %s: %v", source, err)
|
||||
log.Printf("poll %s: failed - %v", source, err)
|
||||
return
|
||||
}
|
||||
if len(links) == 0 {
|
||||
log.Printf("poll %s: success - 0 links", source)
|
||||
agg.Update(source, nil)
|
||||
if err := writeCache(cacheDir, source, nil); err != nil {
|
||||
log.Printf("write cache %s: %v", source, err)
|
||||
@@ -208,6 +217,7 @@ func runOnce(ctx context.Context, source string, cacheDir string, agg *Aggregato
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("poll %s: success - %d links", source, len(links))
|
||||
agg.Update(source, links)
|
||||
if err := writeCache(cacheDir, source, links); err != nil {
|
||||
log.Printf("write cache %s: %v", source, err)
|
||||
@@ -234,6 +244,7 @@ func fetchSource(ctx context.Context, source string, client *http.Client) ([]str
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("unexpected status: %s", resp.Status)
|
||||
}
|
||||
log.Printf("fetch %s: HTTP %d", source, resp.StatusCode)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read body: %w", err)
|
||||
@@ -248,6 +259,7 @@ func fetchSource(ctx context.Context, source string, client *http.Client) ([]str
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read file: %w", err)
|
||||
}
|
||||
log.Printf("fetch %s: file read %d bytes", source, len(data))
|
||||
return normalizeLinks(string(data)), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported source scheme: %s", u.Scheme)
|
||||
|
||||
Reference in New Issue
Block a user