From 0d295b81e9cbf66cdcf5c776db0e4c3f1c739577 Mon Sep 17 00:00:00 2001 From: MenD32 Date: Sat, 14 Sep 2024 14:09:14 +0300 Subject: [PATCH 1/2] added s3 put functions with s3 tag and metadata support Signed-off-by: MenD32 --- s3/s3.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/s3/s3.go b/s3/s3.go index 5e6a7106..d5d06b02 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -28,10 +28,17 @@ type S3Client interface { // PutFile puts a single file to a bucket at the specified key PutFile(bucket, key, path string) error + // PutFileWithMetadata puts a single file to a bucket at the specified key with specified metadata and tags + PutFileWithMetadata(bucket, key, path string, metadata, tags map[string]string) error + // PutDirectory puts a complete directory into a bucket key prefix, with each file in the directory // a separate key in the bucket. PutDirectory(bucket, key, path string) error + // PutDirectoryWithMetadata puts a complete directory into a bucket key prefix, with each file in the directory + // a separate key in the bucket with specified metadata and tags. + PutDirectoryWithMetadata(bucket, key, path string, metadata, tags map[string]string) error + // GetFile downloads a file to a local file path GetFile(bucket, key, path string) error @@ -225,6 +232,24 @@ func (s *s3client) PutFile(bucket, key, path string) error { return nil } +// PutFile puts a single file to a bucket at the specified key +func (s *s3client) PutFileWithMetadata(bucket, key, path string, metadata, tags map[string]string) error { + log.WithFields(log.Fields{"endpoint": s.Endpoint, "bucket": bucket, "key": key, "path": path}).Info("Saving file to s3") + // NOTE: minio will detect proper mime-type based on file extension + + encOpts, err := s.EncryptOpts.buildServerSideEnc(bucket, key) + + if err != nil { + return errors.WithStack(err) + } + + _, err = s.minioClient.FPutObject(s.ctx, bucket, key, path, minio.PutObjectOptions{ServerSideEncryption: encOpts, UserMetadata: metadata, UserTags: tags}) + if err != nil { + return errors.WithStack(err) + } + return nil +} + func (s *s3client) BucketExists(bucketName string) (bool, error) { log.WithField("bucket", bucketName).Info("Checking if bucket exists") result, err := s.minioClient.BucketExists(s.ctx, bucketName) @@ -288,6 +313,16 @@ func (s *s3client) PutDirectory(bucket, key, path string) error { return nil } +func (s *s3client) PutDirectoryWithMetadata(bucket, key, path string, metadata, tags map[string]string) error { + for putTask := range generatePutTasks(key, path) { + err := s.PutFileWithMetadata(bucket, putTask.key, putTask.path, metadata, tags) + if err != nil { + return err + } + } + return nil +} + // GetFile downloads a file to a local file path func (s *s3client) GetFile(bucket, key, path string) error { log.WithFields(log.Fields{"endpoint": s.Endpoint, "bucket": bucket, "key": key, "path": path}).Info("Getting file from s3") From 8482974249182ff71d85d5ab1e975c115aee142c Mon Sep 17 00:00:00 2001 From: MenD32 Date: Sat, 14 Sep 2024 20:44:25 +0300 Subject: [PATCH 2/2] better implementation Signed-off-by: MenD32 --- s3/s3.go | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/s3/s3.go b/s3/s3.go index d5d06b02..d45774fd 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -214,8 +214,9 @@ func NewS3Client(ctx context.Context, opts S3ClientOpts) (S3Client, error) { return &s3cli, nil } -// PutFile puts a single file to a bucket at the specified key -func (s *s3client) PutFile(bucket, key, path string) error { + +// putFile puts a single file to a bucket at the specified key +func (s *s3client) putFile(bucket, key, path string, opts minio.PutObjectOptions) error { log.WithFields(log.Fields{"endpoint": s.Endpoint, "bucket": bucket, "key": key, "path": path}).Info("Saving file to s3") // NOTE: minio will detect proper mime-type based on file extension @@ -225,7 +226,7 @@ func (s *s3client) PutFile(bucket, key, path string) error { return errors.WithStack(err) } - _, err = s.minioClient.FPutObject(s.ctx, bucket, key, path, minio.PutObjectOptions{ServerSideEncryption: encOpts}) + _, err = s.minioClient.FPutObject(s.ctx, bucket, key, path, opts) if err != nil { return errors.WithStack(err) } @@ -233,21 +234,13 @@ func (s *s3client) PutFile(bucket, key, path string) error { } // PutFile puts a single file to a bucket at the specified key -func (s *s3client) PutFileWithMetadata(bucket, key, path string, metadata, tags map[string]string) error { - log.WithFields(log.Fields{"endpoint": s.Endpoint, "bucket": bucket, "key": key, "path": path}).Info("Saving file to s3") - // NOTE: minio will detect proper mime-type based on file extension - - encOpts, err := s.EncryptOpts.buildServerSideEnc(bucket, key) - - if err != nil { - return errors.WithStack(err) - } +func (s *s3client) PutFile(bucket, key, path string) error { + return s.putFile(bucket, key, path, minio.PutObjectOptions{ServerSideEncryption: encOpts}) +} - _, err = s.minioClient.FPutObject(s.ctx, bucket, key, path, minio.PutObjectOptions{ServerSideEncryption: encOpts, UserMetadata: metadata, UserTags: tags}) - if err != nil { - return errors.WithStack(err) - } - return nil +// PutFile puts a single file to a bucket at the specified key +func (s *s3client) PutFileWithMetadata(bucket, key, path string, metadata, tags map[string]string) error { + return s.putFile(bucket, key, path, minio.PutObjectOptions{ServerSideEncryption: encOpts, UserMetadata: metadata, UserTags: tags}) } func (s *s3client) BucketExists(bucketName string) (bool, error) {