av/transcriber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload files into yandex object storage (s3)

Browse Source
This commit is contained in:
Anton Vakhrushev
2025-08-11 11:31:08 +03:00
parent c0d55c2088
commit c1da998c02
7 changed files with 231 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
.env.example Normal file
View File

@@ -0,0 +1,15 @@
# AWS S3 Configuration
# Регион AWS (например: us-east-1, eu-west-1)
AWS_REGION=us-east-1
# AWS Access Key ID (получить в AWS Console)
AWS_ACCESS_KEY_ID=your_access_key_id
# AWS Secret Access Key (получить в AWS Console)
AWS_SECRET_ACCESS_KEY=your_secret_access_key
# Имя S3 bucket для загрузки файлов
S3_BUCKET_NAME=your_bucket_name
# Кастомный endpoint для S3 (оставить пустым для AWS S3, заполнить для MinIO или других S3-совместимых сервисов)
S3_ENDPOINT=

20
go.mod
View File

@@ -3,15 +3,35 @@ module git.vakhrushev.me/av/transcriber
go 1.24.5
require (
github.com/aws/aws-sdk-go-v2 v1.37.2
github.com/aws/aws-sdk-go-v2/config v1.30.3
github.com/aws/aws-sdk-go-v2/credentials v1.18.3
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.3
github.com/aws/aws-sdk-go-v2/service/s3 v1.86.0
github.com/doug-martin/goqu/v9 v9.19.0
github.com/gin-gonic/gin v1.10.1
github.com/google/uuid v1.4.0
github.com/joho/godotenv v1.5.1
github.com/mattn/go-sqlite3 v1.14.17
github.com/pressly/goose/v3 v3.15.1
github.com/stretchr/testify v1.10.0
)
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.27.0 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 // indirect
github.com/aws/smithy-go v1.22.5 // indirect
github.com/bytedance/sonic v1.11.6 // indirect
github.com/bytedance/sonic/loader v0.1.1 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect

40
go.sum
View File

@@ -1,5 +1,43 @@
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/aws/aws-sdk-go-v2 v1.37.2 h1:xkW1iMYawzcmYFYEV0UCMxc8gSsjCGEhBXQkdQywVbo=
github.com/aws/aws-sdk-go-v2 v1.37.2/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg=
github.com/aws/aws-sdk-go-v2/config v1.30.3 h1:utupeVnE3bmB221W08P0Moz1lDI3OwYa2fBtUhl7TCc=
github.com/aws/aws-sdk-go-v2/config v1.30.3/go.mod h1:NDGwOEBdpyZwLPlQkpKIO7frf18BW8PaCmAM9iUxQmI=
github.com/aws/aws-sdk-go-v2/credentials v1.18.3 h1:ptfyXmv+ooxzFwyuBth0yqABcjVIkjDL0iTYZBSbum8=
github.com/aws/aws-sdk-go-v2/credentials v1.18.3/go.mod h1:Q43Nci++Wohb0qUh4m54sNln0dbxJw8PvQWkrwOkGOI=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.2 h1:nRniHAvjFJGUCl04F3WaAj7qp/rcz5Gi1OVoj5ErBkc=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.2/go.mod h1:eJDFKAMHHUvv4a0Zfa7bQb//wFNUXGrbFpYRCHe2kD0=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.3 h1:Nb2pUE30lySKPGdkiIJ1SZgHsjiebOiRNI7R9NA1WtM=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.3/go.mod h1:BO5EKulvhBF1NXwui8lfnuDPBQQU5807yvWASZ/5n6k=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2 h1:sPiRHLVUIIQcoVZTNwqQcdtjkqkPopyYmIX0M5ElRf4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2/go.mod h1:ik86P3sgV+Bk7c1tBFCwI3VxMoSEwl4YkRB9xn1s340=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2 h1:ZdzDAg075H6stMZtbD2o+PyB933M/f20e9WmCBC17wA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2/go.mod h1:eE1IIzXG9sdZCB0pNNpMpsYTLl4YdOQD3njiVN1e/E4=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.2 h1:sBpc8Ph6CpfZsEdkz/8bfg8WhKlWMCms5iWj6W/AW2U=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.2/go.mod h1:Z2lDojZB+92Wo6EKiZZmJid9pPrDJW2NNIXSlaEfVlU=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.2 h1:blV3dY6WbxIVOFggfYIo2E1Q2lZoy5imS7nKgu5m6Tc=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.2/go.mod h1:cBWNeLBjHJRSmXAxdS7mwiMUEgx6zup4wQ9J+/PcsRQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.2 h1:oxmDEO14NBZJbK/M8y3brhMFEIGN4j8a6Aq8eY0sqlo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.2/go.mod h1:4hH+8QCrk1uRWDPsVfsNDUup3taAjO8Dnx63au7smAU=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.2 h1:0hBNFAPwecERLzkhhBY+lQKUMpXSKVv4Sxovikrioms=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.2/go.mod h1:Vcnh4KyR4imrrjGN7A2kP2v9y6EPudqoPKXtnmBliPU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.86.0 h1:utPhv4ECQzJIUbtx7vMN4A8uZxlQ5tSt1H1toPI41h8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.86.0/go.mod h1:1/eZYtTWazDgVl96LmGdGktHFi7prAcGCrJ9JGvBITU=
github.com/aws/aws-sdk-go-v2/service/sso v1.27.0 h1:j7/jTOjWeJDolPwZ/J4yZ7dUsxsWZEsxNwH5O7F8eEA=
github.com/aws/aws-sdk-go-v2/service/sso v1.27.0/go.mod h1:M0xdEPQtgpNT7kdAX4/vOAPkFj60hSQRb7TvW9B0iug=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0 h1:ywQF2N4VjqX+Psw+jLjMmUL2g1RDHlvri3NxHA08MGI=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0/go.mod h1:Z+qv5Q6b7sWiclvbJyPSOT1BRVU9wfSUPaqQzZ1Xg3E=
github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 h1:bRP/a9llXSSgDPk7Rqn5GD/DQCGo6uk95plBFKoXt2M=
github.com/aws/aws-sdk-go-v2/service/sts v1.36.0/go.mod h1:tgBsFzxwl65BWkuJ/x2EUs59bD4SfYKgikvFDJi1S58=
github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw=
github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
@@ -39,6 +77,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=

63
internal/controller/http/transcribe.go
View File

@@ -11,6 +11,7 @@ import (
"git.vakhrushev.me/av/transcriber/internal/entity"
"git.vakhrushev.me/av/transcriber/internal/repo"
"git.vakhrushev.me/av/transcriber/internal/repo/ffmpeg"
"git.vakhrushev.me/av/transcriber/internal/service/s3"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
)
@@ -147,7 +148,6 @@ func (h *TranscribeHandler) RunConversionJob(c *gin.Context) {
srcFilePath := filepath.Join("data", "files", srcFile.FileName)
destFileId := uuid.New().String()
destFileName := fmt.Sprintf("%s%s", destFileId, ".ogg")
destFilePath := filepath.Join("data", "files", destFileName)
@@ -190,3 +190,64 @@ func (h *TranscribeHandler) RunConversionJob(c *gin.Context) {
c.Status(http.StatusOK)
}
func (h *TranscribeHandler) RunUploadJob(c *gin.Context) {
acquisitionId := uuid.NewString()
rottingTime := time.Now().Add(-1 * time.Hour)
job, err := h.jobRepo.FindAndAcquire(entity.StateConverted, acquisitionId, rottingTime)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
fileRecord, err := h.fileRepo.GetByID(*job.FileID)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
filePath := filepath.Join("data", "files", fileRecord.FileName)
destFileId := uuid.New().String()
destFileRecord := &entity.File{
Id: destFileId,
Storage: entity.StorageS3,
FileName: fileRecord.FileName,
Size: fileRecord.Size,
CreatedAt: time.Now(),
}
// Создаем S3 сервис
s3Service, err := s3.NewS3Service()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to initialize S3 service: " + err.Error()})
return
}
// Загружаем файл на S3
err = s3Service.UploadFile(filePath, destFileRecord.FileName)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to upload file to S3: " + err.Error()})
return
}
job.FileID = &destFileId
job.MoveToState(entity.StateTranscribeReady)
// Сохраняем информацию о загрузке файла на S3
err = h.fileRepo.Create(destFileRecord)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update file record: " + err.Error()})
return
}
// Обновляем состояние задачи
err = h.jobRepo.Save(job)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update job state: " + err.Error()})
return
}
c.Status(http.StatusOK)
}

3
internal/entity/job.go
View File

@@ -19,6 +19,9 @@ const (
StateCreated = "created"
StateConverted = "converted"
StateUploaded = "uploaded"
StateTranscribeReady = "transcribe_ready"
StateTranscribeWait = "transcribe_wait"
StateDone = "done"
StatusFailed = "failed"
)

80
internal/service/s3/s3.go Normal file
View File

@@ -0,0 +1,80 @@
package s3
import (
"context"
"fmt"
"os"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
type S3Service struct {
client *s3.Client
uploader *manager.Uploader
bucketName string
}
func NewS3Service() (*S3Service, error) {
region := os.Getenv("AWS_REGION")
accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
bucketName := os.Getenv("S3_BUCKET_NAME")
endpoint := os.Getenv("S3_ENDPOINT")
if region == "" || accessKey == "" || secretKey == "" || bucketName == "" {
return nil, fmt.Errorf("missing required S3 environment variables")
}
// Создаем конфигурацию
cfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithRegion(region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
)
if err != nil {
return nil, fmt.Errorf("failed to load AWS config: %w", err)
}
// Создаем клиент S3
var client *s3.Client
if endpoint != "" {
// Кастомный endpoint (например, для MinIO)
client = s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(endpoint)
o.UsePathStyle = true
})
} else {
// Стандартный AWS S3
client = s3.NewFromConfig(cfg)
}
uploader := manager.NewUploader(client)
return &S3Service{
client: client,
uploader: uploader,
bucketName: bucketName,
}, nil
}
func (s *S3Service) UploadFile(filePath, fileName string) error {
file, err := os.Open(filePath)
if err != nil {
return fmt.Errorf("failed to open file %s: %w", filePath, err)
}
defer file.Close()
_, err = s.uploader.Upload(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(fileName),
Body: file,
})
if err != nil {
return fmt.Errorf("failed to upload file to S3: %w", err)
}
return nil
}

7
main.go
View File

@@ -11,11 +11,17 @@ import (
"github.com/doug-martin/goqu/v9"
_ "github.com/doug-martin/goqu/v9/dialect/sqlite3"
"github.com/gin-gonic/gin"
"github.com/joho/godotenv"
_ "github.com/mattn/go-sqlite3"
"github.com/pressly/goose/v3"
)
func main() {
// Загружаем переменные окружения из .env файла
if err := godotenv.Load(); err != nil {
log.Println("Warning: .env file not found, using system environment variables")
}
// Создаем директории если они не существуют
if err := os.MkdirAll("data/files", 0755); err != nil {
log.Fatal("Failed to create data/files directory:", err)
@@ -54,6 +60,7 @@ func main() {
api.GET("/transcribe/:id", transcribeHandler.GetTranscribeJobStatus)
api.POST("/transcribe/convert", transcribeHandler.RunConversionJob)
api.POST("/transcribe/upload", transcribeHandler.RunUploadJob)
}
// Добавляем middleware для обработки больших файлов