diff --git a/go.mod b/go.mod index 79044fc..a2a663e 100644 --- a/go.mod +++ b/go.mod @@ -34,8 +34,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 // indirect github.com/aws/smithy-go v1.22.5 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -51,8 +53,13 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.23.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.65.0 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.8.0 // indirect diff --git a/go.sum b/go.sum index e3afabf..2cc955e 100644 --- a/go.sum +++ b/go.sum @@ -38,10 +38,14 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 h1:bRP/a9llXSSgDPk7Rqn5GD/DQCGo github.com/aws/aws-sdk-go-v2/service/sts v1.36.0/go.mod h1:tgBsFzxwl65BWkuJ/x2EUs59bD4SfYKgikvFDJi1S58= github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= @@ -108,12 +112,22 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pressly/goose/v3 v3.15.1 h1:dKaJ1SdLvS/+HtS8PzFT0KBEtICC1jewLXM+b3emlv8= github.com/pressly/goose/v3 v3.15.1/go.mod h1:0E3Yg/+EwYzO6Rz2P98MlClFgIcoujbVRs575yi3iIM= +github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= +github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= +github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/internal/controller/worker/worker.go b/internal/controller/worker/worker.go index 0eecbc5..46a13e6 100644 --- a/internal/controller/worker/worker.go +++ b/internal/controller/worker/worker.go @@ -3,8 +3,10 @@ package worker import ( "context" "log" + "strconv" "time" + "git.vakhrushev.me/av/transcriber/internal/metrics" "git.vakhrushev.me/av/transcriber/internal/service" ) @@ -39,6 +41,7 @@ func (w *ConversionWorker) Start(ctx context.Context) { return default: err := w.transcribeService.FindAndRunConversionJob() + metrics.WorkerJobCounter.WithLabelValues(w.Name(), strconv.FormatBool(err != nil)).Inc() if err != nil { log.Printf("%s error: %v", w.Name(), err) } @@ -80,6 +83,7 @@ func (w *TranscribeWorker) Start(ctx context.Context) { return default: err := w.transcribeService.FindAndRunTranscribeJob() + metrics.WorkerJobCounter.WithLabelValues(w.Name(), strconv.FormatBool(err != nil)).Inc() if err != nil { log.Printf("%s error: %v", w.Name(), err) } @@ -121,6 +125,7 @@ func (w *CheckWorker) Start(ctx context.Context) { return default: err := w.transcribeService.FindAndRunTranscribeCheckJob() + metrics.WorkerJobCounter.WithLabelValues(w.Name(), strconv.FormatBool(err != nil)).Inc() if err != nil { log.Printf("%s error: %v", w.Name(), err) } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..f126d8e --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,46 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + WorkerJobCounter = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "transcriber_worker_job_count", + Help: "Count of jobs handled by each worker", + }, + []string{"name", "error"}, + ) + + // Размер принятых на обработку файлов (в байтах) + InputFileSizeHistogram = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "transcriber_input_file_size_bytes", + Help: "Size of input files received for processing", + Buckets: []float64{1024, 10240, 102400, 1048576, 10485760, 104857600, 1073741824}, // 1KB, 10KB, 100KB, 1MB, 10MB, 100MB, 1GB + }, + []string{"file_extension"}, + ) + + // Время конвертации файлов (в секундах) + ConversionDurationHistogram = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "transcriber_conversion_duration_seconds", + Help: "Time taken to convert audio files", + Buckets: []float64{0.1, 0.5, 1, 5, 10, 30, 60, 120, 300}, // 0.1s, 0.5s, 1s, 5s, 10s, 30s, 1m, 2m, 5m + }, + []string{"source_format", "target_format", "error"}, + ) + + // Размер файла после конвертации (в байтах) + OutputFileSizeHistogram = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "transcriber_output_file_size_bytes", + Help: "Size of files after conversion", + Buckets: []float64{1024, 10240, 102400, 1048576, 10485760, 104857600, 1073741824}, // 1KB, 10KB, 100KB, 1MB, 10MB, 100MB, 1GB + }, + []string{"format"}, + ) +) diff --git a/internal/service/transcribe.go b/internal/service/transcribe.go index 14b3a7d..905085c 100644 --- a/internal/service/transcribe.go +++ b/internal/service/transcribe.go @@ -6,14 +6,21 @@ import ( "log" "os" "path/filepath" + "strconv" + "strings" "time" "git.vakhrushev.me/av/transcriber/internal/contract" "git.vakhrushev.me/av/transcriber/internal/entity" + "git.vakhrushev.me/av/transcriber/internal/metrics" "github.com/google/uuid" ) -const baseStorageDir = "data/files" +const ( + baseStorageDir = "data/files" + + defaultAudioExt = "audio" +) type TranscribeService struct { jobRepo contract.TranscriptJobRepository @@ -43,7 +50,7 @@ func (s *TranscribeService) CreateTranscribeJob(file io.Reader, fileName string) // Определяем расширение файла ext := filepath.Ext(fileName) if ext == "" { - ext = ".audio" // fallback если расширение не определено + ext = fmt.Sprintf(".%s", defaultAudioExt) // fallback если расширение не определено } // Создаем путь для сохранения файла @@ -63,6 +70,12 @@ func (s *TranscribeService) CreateTranscribeJob(file io.Reader, fileName string) return nil, err } + if err := dst.Close(); err != nil { + return nil, err + } + + metrics.InputFileSizeHistogram.WithLabelValues(ext).Observe(float64(size)) + // Создаем запись в таблице files fileRecord := &entity.File{ Id: fileId, @@ -121,7 +134,22 @@ func (s *TranscribeService) FindAndRunConversionJob() error { destFileName := fmt.Sprintf("%s%s", destFileId, ".ogg") destFilePath := filepath.Join(baseStorageDir, destFileName) + // Получаем расширение исходного файла для метрики + srcExt := strings.TrimPrefix(filepath.Ext(srcFile.FileName), ".") + if srcExt == "" { + srcExt = defaultAudioExt + } + + // Измеряем время конвертации + startTime := time.Now() err = s.converter.Convert(srcFilePath, destFilePath) + conversionDuration := time.Since(startTime) + + // Записываем метрику времени конвертации + metrics.ConversionDurationHistogram. + WithLabelValues(srcExt, "ogg", strconv.FormatBool(err != nil)). + Observe(conversionDuration.Seconds()) + if err != nil { return err } @@ -131,6 +159,9 @@ func (s *TranscribeService) FindAndRunConversionJob() error { return err } + // Записываем метрику размера выходного файла + metrics.OutputFileSizeHistogram.WithLabelValues("ogg").Observe(float64(stat.Size())) + // Создаем запись в таблице files destFileRecord := &entity.File{ Id: destFileId, @@ -218,7 +249,7 @@ func (s *TranscribeService) FindAndRunTranscribeCheckJob() error { if _, ok := err.(*contract.JobNotFoundError); ok { return nil } - return err + return fmt.Errorf("failed find and acquire job: %s, %w", entity.StateTranscribe, err) } if job.RecognitionOpID == nil { diff --git a/main.go b/main.go index e3278c5..6cbc5a7 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ import ( "github.com/joho/godotenv" _ "github.com/mattn/go-sqlite3" "github.com/pressly/goose/v3" + "github.com/prometheus/client_golang/prometheus/promhttp" ) func main() { @@ -132,6 +133,9 @@ func main() { }) }) + // Добавляем эндпоинт для метрик Prometheus + router.GET("/metrics", gin.WrapH(promhttp.Handler())) + // Создаем HTTP сервер srv := &http.Server{ Addr: ":8080",