From c1da998c02d170e268016fbc9afc82c0578892f5 Mon Sep 17 00:00:00 2001 From: Anton Vakhrushev Date: Mon, 11 Aug 2025 11:31:08 +0300 Subject: [PATCH] Upload files into yandex object storage (s3) --- .env.example | 15 +++++ go.mod | 20 +++++++ go.sum | 40 +++++++++++++ internal/controller/http/transcribe.go | 63 +++++++++++++++++++- internal/entity/job.go | 11 ++-- internal/service/s3/s3.go | 80 ++++++++++++++++++++++++++ main.go | 7 +++ 7 files changed, 231 insertions(+), 5 deletions(-) create mode 100644 .env.example create mode 100644 internal/service/s3/s3.go diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..c7acc17 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# AWS S3 Configuration +# Регион AWS (например: us-east-1, eu-west-1) +AWS_REGION=us-east-1 + +# AWS Access Key ID (получить в AWS Console) +AWS_ACCESS_KEY_ID=your_access_key_id + +# AWS Secret Access Key (получить в AWS Console) +AWS_SECRET_ACCESS_KEY=your_secret_access_key + +# Имя S3 bucket для загрузки файлов +S3_BUCKET_NAME=your_bucket_name + +# Кастомный endpoint для S3 (оставить пустым для AWS S3, заполнить для MinIO или других S3-совместимых сервисов) +S3_ENDPOINT= diff --git a/go.mod b/go.mod index 2453545..9adc8f1 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,35 @@ module git.vakhrushev.me/av/transcriber go 1.24.5 require ( + github.com/aws/aws-sdk-go-v2 v1.37.2 + github.com/aws/aws-sdk-go-v2/config v1.30.3 + github.com/aws/aws-sdk-go-v2/credentials v1.18.3 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.3 + github.com/aws/aws-sdk-go-v2/service/s3 v1.86.0 github.com/doug-martin/goqu/v9 v9.19.0 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.4.0 + github.com/joho/godotenv v1.5.1 github.com/mattn/go-sqlite3 v1.14.17 github.com/pressly/goose/v3 v3.15.1 github.com/stretchr/testify v1.10.0 ) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.27.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 // indirect + github.com/aws/smithy-go v1.22.5 // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cloudwego/base64x v0.1.4 // indirect diff --git a/go.sum b/go.sum index eebbaac..949ec7f 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,43 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/aws/aws-sdk-go-v2 v1.37.2 h1:xkW1iMYawzcmYFYEV0UCMxc8gSsjCGEhBXQkdQywVbo= +github.com/aws/aws-sdk-go-v2 v1.37.2/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg= +github.com/aws/aws-sdk-go-v2/config v1.30.3 h1:utupeVnE3bmB221W08P0Moz1lDI3OwYa2fBtUhl7TCc= +github.com/aws/aws-sdk-go-v2/config v1.30.3/go.mod h1:NDGwOEBdpyZwLPlQkpKIO7frf18BW8PaCmAM9iUxQmI= +github.com/aws/aws-sdk-go-v2/credentials v1.18.3 h1:ptfyXmv+ooxzFwyuBth0yqABcjVIkjDL0iTYZBSbum8= +github.com/aws/aws-sdk-go-v2/credentials v1.18.3/go.mod h1:Q43Nci++Wohb0qUh4m54sNln0dbxJw8PvQWkrwOkGOI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.2 h1:nRniHAvjFJGUCl04F3WaAj7qp/rcz5Gi1OVoj5ErBkc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.2/go.mod h1:eJDFKAMHHUvv4a0Zfa7bQb//wFNUXGrbFpYRCHe2kD0= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.3 h1:Nb2pUE30lySKPGdkiIJ1SZgHsjiebOiRNI7R9NA1WtM= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.3/go.mod h1:BO5EKulvhBF1NXwui8lfnuDPBQQU5807yvWASZ/5n6k= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2 h1:sPiRHLVUIIQcoVZTNwqQcdtjkqkPopyYmIX0M5ElRf4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2/go.mod h1:ik86P3sgV+Bk7c1tBFCwI3VxMoSEwl4YkRB9xn1s340= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2 h1:ZdzDAg075H6stMZtbD2o+PyB933M/f20e9WmCBC17wA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2/go.mod h1:eE1IIzXG9sdZCB0pNNpMpsYTLl4YdOQD3njiVN1e/E4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.2 h1:sBpc8Ph6CpfZsEdkz/8bfg8WhKlWMCms5iWj6W/AW2U= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.2/go.mod h1:Z2lDojZB+92Wo6EKiZZmJid9pPrDJW2NNIXSlaEfVlU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.2 h1:blV3dY6WbxIVOFggfYIo2E1Q2lZoy5imS7nKgu5m6Tc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.2/go.mod h1:cBWNeLBjHJRSmXAxdS7mwiMUEgx6zup4wQ9J+/PcsRQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.2 h1:oxmDEO14NBZJbK/M8y3brhMFEIGN4j8a6Aq8eY0sqlo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.2/go.mod h1:4hH+8QCrk1uRWDPsVfsNDUup3taAjO8Dnx63au7smAU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.2 h1:0hBNFAPwecERLzkhhBY+lQKUMpXSKVv4Sxovikrioms= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.2/go.mod h1:Vcnh4KyR4imrrjGN7A2kP2v9y6EPudqoPKXtnmBliPU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.86.0 h1:utPhv4ECQzJIUbtx7vMN4A8uZxlQ5tSt1H1toPI41h8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.86.0/go.mod h1:1/eZYtTWazDgVl96LmGdGktHFi7prAcGCrJ9JGvBITU= +github.com/aws/aws-sdk-go-v2/service/sso v1.27.0 h1:j7/jTOjWeJDolPwZ/J4yZ7dUsxsWZEsxNwH5O7F8eEA= +github.com/aws/aws-sdk-go-v2/service/sso v1.27.0/go.mod h1:M0xdEPQtgpNT7kdAX4/vOAPkFj60hSQRb7TvW9B0iug= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0 h1:ywQF2N4VjqX+Psw+jLjMmUL2g1RDHlvri3NxHA08MGI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0/go.mod h1:Z+qv5Q6b7sWiclvbJyPSOT1BRVU9wfSUPaqQzZ1Xg3E= +github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 h1:bRP/a9llXSSgDPk7Rqn5GD/DQCGo6uk95plBFKoXt2M= +github.com/aws/aws-sdk-go-v2/service/sts v1.36.0/go.mod h1:tgBsFzxwl65BWkuJ/x2EUs59bD4SfYKgikvFDJi1S58= +github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= +github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= @@ -39,6 +77,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= diff --git a/internal/controller/http/transcribe.go b/internal/controller/http/transcribe.go index 65ef2d3..8b470b8 100644 --- a/internal/controller/http/transcribe.go +++ b/internal/controller/http/transcribe.go @@ -11,6 +11,7 @@ import ( "git.vakhrushev.me/av/transcriber/internal/entity" "git.vakhrushev.me/av/transcriber/internal/repo" "git.vakhrushev.me/av/transcriber/internal/repo/ffmpeg" + "git.vakhrushev.me/av/transcriber/internal/service/s3" "github.com/gin-gonic/gin" "github.com/google/uuid" ) @@ -147,7 +148,6 @@ func (h *TranscribeHandler) RunConversionJob(c *gin.Context) { srcFilePath := filepath.Join("data", "files", srcFile.FileName) destFileId := uuid.New().String() - destFileName := fmt.Sprintf("%s%s", destFileId, ".ogg") destFilePath := filepath.Join("data", "files", destFileName) @@ -190,3 +190,64 @@ func (h *TranscribeHandler) RunConversionJob(c *gin.Context) { c.Status(http.StatusOK) } + +func (h *TranscribeHandler) RunUploadJob(c *gin.Context) { + acquisitionId := uuid.NewString() + rottingTime := time.Now().Add(-1 * time.Hour) + + job, err := h.jobRepo.FindAndAcquire(entity.StateConverted, acquisitionId, rottingTime) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + fileRecord, err := h.fileRepo.GetByID(*job.FileID) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + filePath := filepath.Join("data", "files", fileRecord.FileName) + + destFileId := uuid.New().String() + destFileRecord := &entity.File{ + Id: destFileId, + Storage: entity.StorageS3, + FileName: fileRecord.FileName, + Size: fileRecord.Size, + CreatedAt: time.Now(), + } + + // Создаем S3 сервис + s3Service, err := s3.NewS3Service() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to initialize S3 service: " + err.Error()}) + return + } + + // Загружаем файл на S3 + err = s3Service.UploadFile(filePath, destFileRecord.FileName) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to upload file to S3: " + err.Error()}) + return + } + + job.FileID = &destFileId + job.MoveToState(entity.StateTranscribeReady) + + // Сохраняем информацию о загрузке файла на S3 + err = h.fileRepo.Create(destFileRecord) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update file record: " + err.Error()}) + return + } + + // Обновляем состояние задачи + err = h.jobRepo.Save(job) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update job state: " + err.Error()}) + return + } + + c.Status(http.StatusOK) +} diff --git a/internal/entity/job.go b/internal/entity/job.go index fab7ecb..c2df303 100644 --- a/internal/entity/job.go +++ b/internal/entity/job.go @@ -16,10 +16,13 @@ type TranscribeJob struct { } const ( - StateCreated = "created" - StateConverted = "converted" - StateUploaded = "uploaded" - StatusFailed = "failed" + StateCreated = "created" + StateConverted = "converted" + StateUploaded = "uploaded" + StateTranscribeReady = "transcribe_ready" + StateTranscribeWait = "transcribe_wait" + StateDone = "done" + StatusFailed = "failed" ) func (j *TranscribeJob) MoveToState(state string) { diff --git a/internal/service/s3/s3.go b/internal/service/s3/s3.go new file mode 100644 index 0000000..c0eac8c --- /dev/null +++ b/internal/service/s3/s3.go @@ -0,0 +1,80 @@ +package s3 + +import ( + "context" + "fmt" + "os" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type S3Service struct { + client *s3.Client + uploader *manager.Uploader + bucketName string +} + +func NewS3Service() (*S3Service, error) { + region := os.Getenv("AWS_REGION") + accessKey := os.Getenv("AWS_ACCESS_KEY_ID") + secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY") + bucketName := os.Getenv("S3_BUCKET_NAME") + endpoint := os.Getenv("S3_ENDPOINT") + + if region == "" || accessKey == "" || secretKey == "" || bucketName == "" { + return nil, fmt.Errorf("missing required S3 environment variables") + } + + // Создаем конфигурацию + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(region), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")), + ) + if err != nil { + return nil, fmt.Errorf("failed to load AWS config: %w", err) + } + + // Создаем клиент S3 + var client *s3.Client + if endpoint != "" { + // Кастомный endpoint (например, для MinIO) + client = s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(endpoint) + o.UsePathStyle = true + }) + } else { + // Стандартный AWS S3 + client = s3.NewFromConfig(cfg) + } + + uploader := manager.NewUploader(client) + + return &S3Service{ + client: client, + uploader: uploader, + bucketName: bucketName, + }, nil +} + +func (s *S3Service) UploadFile(filePath, fileName string) error { + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", filePath, err) + } + defer file.Close() + + _, err = s.uploader.Upload(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(s.bucketName), + Key: aws.String(fileName), + Body: file, + }) + if err != nil { + return fmt.Errorf("failed to upload file to S3: %w", err) + } + + return nil +} diff --git a/main.go b/main.go index dd545d5..1ed719e 100644 --- a/main.go +++ b/main.go @@ -11,11 +11,17 @@ import ( "github.com/doug-martin/goqu/v9" _ "github.com/doug-martin/goqu/v9/dialect/sqlite3" "github.com/gin-gonic/gin" + "github.com/joho/godotenv" _ "github.com/mattn/go-sqlite3" "github.com/pressly/goose/v3" ) func main() { + // Загружаем переменные окружения из .env файла + if err := godotenv.Load(); err != nil { + log.Println("Warning: .env file not found, using system environment variables") + } + // Создаем директории если они не существуют if err := os.MkdirAll("data/files", 0755); err != nil { log.Fatal("Failed to create data/files directory:", err) @@ -54,6 +60,7 @@ func main() { api.GET("/transcribe/:id", transcribeHandler.GetTranscribeJobStatus) api.POST("/transcribe/convert", transcribeHandler.RunConversionJob) + api.POST("/transcribe/upload", transcribeHandler.RunUploadJob) } // Добавляем middleware для обработки больших файлов