av/transcriber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add prometheus metrics

Browse Source
This commit is contained in:
Anton Vakhrushev
2025-08-13 11:17:35 +03:00
parent 22cbaf0bca
commit eb0dea6113
6 changed files with 110 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
go.mod
View File

@@ -34,8 +34,10 @@ require (
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 // indirect
github.com/aws/smithy-go v1.22.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bytedance/sonic v1.11.6 // indirect
github.com/bytedance/sonic/loader v0.1.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect
github.com/cloudwego/iasm v0.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
@@ -51,8 +53,13 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.23.0 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.65.0 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
golang.org/x/arch v0.8.0 // indirect

14
go.sum
View File

@@ -38,10 +38,14 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 h1:bRP/a9llXSSgDPk7Rqn5GD/DQCGo
github.com/aws/aws-sdk-go-v2/service/sts v1.36.0/go.mod h1:tgBsFzxwl65BWkuJ/x2EUs59bD4SfYKgikvFDJi1S58=
github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw=
github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
@@ -108,12 +112,22 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pressly/goose/v3 v3.15.1 h1:dKaJ1SdLvS/+HtS8PzFT0KBEtICC1jewLXM+b3emlv8=
github.com/pressly/goose/v3 v3.15.1/go.mod h1:0E3Yg/+EwYzO6Rz2P98MlClFgIcoujbVRs575yi3iIM=
github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc=
github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE=
github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

5
internal/controller/worker/worker.go
View File

@@ -3,8 +3,10 @@ package worker
import (
"context"
"log"
"strconv"
"time"
"git.vakhrushev.me/av/transcriber/internal/metrics"
"git.vakhrushev.me/av/transcriber/internal/service"
)
@@ -39,6 +41,7 @@ func (w *ConversionWorker) Start(ctx context.Context) {
return
default:
err := w.transcribeService.FindAndRunConversionJob()
metrics.WorkerJobCounter.WithLabelValues(w.Name(), strconv.FormatBool(err != nil)).Inc()
if err != nil {
log.Printf("%s error: %v", w.Name(), err)
}
@@ -80,6 +83,7 @@ func (w *TranscribeWorker) Start(ctx context.Context) {
return
default:
err := w.transcribeService.FindAndRunTranscribeJob()
metrics.WorkerJobCounter.WithLabelValues(w.Name(), strconv.FormatBool(err != nil)).Inc()
if err != nil {
log.Printf("%s error: %v", w.Name(), err)
}
@@ -121,6 +125,7 @@ func (w *CheckWorker) Start(ctx context.Context) {
return
default:
err := w.transcribeService.FindAndRunTranscribeCheckJob()
metrics.WorkerJobCounter.WithLabelValues(w.Name(), strconv.FormatBool(err != nil)).Inc()
if err != nil {
log.Printf("%s error: %v", w.Name(), err)
}

46
internal/metrics/metrics.go Normal file
View File

@@ -0,0 +1,46 @@
package metrics
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
WorkerJobCounter = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "transcriber_worker_job_count",
Help: "Count of jobs handled by each worker",
},
[]string{"name", "error"},
)
// Размер принятых на обработку файлов (в байтах)
InputFileSizeHistogram = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "transcriber_input_file_size_bytes",
Help: "Size of input files received for processing",
Buckets: []float64{1024, 10240, 102400, 1048576, 10485760, 104857600, 1073741824}, // 1KB, 10KB, 100KB, 1MB, 10MB, 100MB, 1GB
},
[]string{"file_extension"},
)
// Время конвертации файлов (в секундах)
ConversionDurationHistogram = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "transcriber_conversion_duration_seconds",
Help: "Time taken to convert audio files",
Buckets: []float64{0.1, 0.5, 1, 5, 10, 30, 60, 120, 300}, // 0.1s, 0.5s, 1s, 5s, 10s, 30s, 1m, 2m, 5m
},
[]string{"source_format", "target_format", "error"},
)
// Размер файла после конвертации (в байтах)
OutputFileSizeHistogram = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "transcriber_output_file_size_bytes",
Help: "Size of files after conversion",
Buckets: []float64{1024, 10240, 102400, 1048576, 10485760, 104857600, 1073741824}, // 1KB, 10KB, 100KB, 1MB, 10MB, 100MB, 1GB
},
[]string{"format"},
)
)

37
internal/service/transcribe.go
View File

@@ -6,14 +6,21 @@ import (
"log"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"git.vakhrushev.me/av/transcriber/internal/contract"
"git.vakhrushev.me/av/transcriber/internal/entity"
"git.vakhrushev.me/av/transcriber/internal/metrics"
"github.com/google/uuid"
)
const baseStorageDir = "data/files"
const (
baseStorageDir = "data/files"
defaultAudioExt = "audio"
)
type TranscribeService struct {
jobRepo contract.TranscriptJobRepository
@@ -43,7 +50,7 @@ func (s *TranscribeService) CreateTranscribeJob(file io.Reader, fileName string)
// Определяем расширение файла
ext := filepath.Ext(fileName)
if ext == "" {
ext = ".audio" // fallback если расширение не определено
ext = fmt.Sprintf(".%s", defaultAudioExt) // fallback если расширение не определено
}
// Создаем путь для сохранения файла
@@ -63,6 +70,12 @@ func (s *TranscribeService) CreateTranscribeJob(file io.Reader, fileName string)
return nil, err
}
if err := dst.Close(); err != nil {
return nil, err
}
metrics.InputFileSizeHistogram.WithLabelValues(ext).Observe(float64(size))
// Создаем запись в таблице files
fileRecord := &entity.File{
Id: fileId,
@@ -121,7 +134,22 @@ func (s *TranscribeService) FindAndRunConversionJob() error {
destFileName := fmt.Sprintf("%s%s", destFileId, ".ogg")
destFilePath := filepath.Join(baseStorageDir, destFileName)
// Получаем расширение исходного файла для метрики
srcExt := strings.TrimPrefix(filepath.Ext(srcFile.FileName), ".")
if srcExt == "" {
srcExt = defaultAudioExt
}
// Измеряем время конвертации
startTime := time.Now()
err = s.converter.Convert(srcFilePath, destFilePath)
conversionDuration := time.Since(startTime)
// Записываем метрику времени конвертации
metrics.ConversionDurationHistogram.
WithLabelValues(srcExt, "ogg", strconv.FormatBool(err != nil)).
Observe(conversionDuration.Seconds())
if err != nil {
return err
}
@@ -131,6 +159,9 @@ func (s *TranscribeService) FindAndRunConversionJob() error {
return err
}
// Записываем метрику размера выходного файла
metrics.OutputFileSizeHistogram.WithLabelValues("ogg").Observe(float64(stat.Size()))
// Создаем запись в таблице files
destFileRecord := &entity.File{
Id: destFileId,
@@ -218,7 +249,7 @@ func (s *TranscribeService) FindAndRunTranscribeCheckJob() error {
if _, ok := err.(*contract.JobNotFoundError); ok {
return nil
}
return err
return fmt.Errorf("failed find and acquire job: %s, %w", entity.StateTranscribe, err)
}
if job.RecognitionOpID == nil {

4
main.go
View File

@@ -24,6 +24,7 @@ import (
"github.com/joho/godotenv"
_ "github.com/mattn/go-sqlite3"
"github.com/pressly/goose/v3"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
func main() {
@@ -132,6 +133,9 @@ func main() {
})
})
// Добавляем эндпоинт для метрик Prometheus
router.GET("/metrics", gin.WrapH(promhttp.Handler()))
// Создаем HTTP сервер
srv := &http.Server{
Addr: ":8080",