Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: add s3 put functions with tag and metadata support #727

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

feat: add s3 put functions with tag and metadata support #727

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0d295b8
added s3 put functions with s3 tag and metadata support
Sep 14, 2024
8482974
better implementation
Sep 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 31 additions & 3 deletions s3/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,17 @@ type S3Client interface {
// PutFile puts a single file to a bucket at the specified key
PutFile(bucket, key, path string) error

// PutFileWithMetadata puts a single file to a bucket at the specified key with specified metadata and tags
PutFileWithMetadata(bucket, key, path string, metadata, tags map[string]string) error

// PutDirectory puts a complete directory into a bucket key prefix, with each file in the directory
// a separate key in the bucket.
PutDirectory(bucket, key, path string) error

// PutDirectoryWithMetadata puts a complete directory into a bucket key prefix, with each file in the directory
// a separate key in the bucket with specified metadata and tags.
PutDirectoryWithMetadata(bucket, key, path string, metadata, tags map[string]string) error

// GetFile downloads a file to a local file path
GetFile(bucket, key, path string) error

Expand Down Expand Up @@ -207,8 +214,9 @@ func NewS3Client(ctx context.Context, opts S3ClientOpts) (S3Client, error) {
return &s3cli, nil
}

// PutFile puts a single file to a bucket at the specified key
func (s *s3client) PutFile(bucket, key, path string) error {

// putFile puts a single file to a bucket at the specified key
func (s *s3client) putFile(bucket, key, path string, opts minio.PutObjectOptions) error {
log.WithFields(log.Fields{"endpoint": s.Endpoint, "bucket": bucket, "key": key, "path": path}).Info("Saving file to s3")
// NOTE: minio will detect proper mime-type based on file extension

Expand All @@ -218,13 +226,23 @@ func (s *s3client) PutFile(bucket, key, path string) error {
return errors.WithStack(err)
}

_, err = s.minioClient.FPutObject(s.ctx, bucket, key, path, minio.PutObjectOptions{ServerSideEncryption: encOpts})
_, err = s.minioClient.FPutObject(s.ctx, bucket, key, path, opts)
if err != nil {
return errors.WithStack(err)
}
return nil
}

// PutFile puts a single file to a bucket at the specified key
func (s *s3client) PutFile(bucket, key, path string) error {
return s.putFile(bucket, key, path, minio.PutObjectOptions{ServerSideEncryption: encOpts})
}

// PutFile puts a single file to a bucket at the specified key
func (s *s3client) PutFileWithMetadata(bucket, key, path string, metadata, tags map[string]string) error {
return s.putFile(bucket, key, path, minio.PutObjectOptions{ServerSideEncryption: encOpts, UserMetadata: metadata, UserTags: tags})
}

func (s *s3client) BucketExists(bucketName string) (bool, error) {
log.WithField("bucket", bucketName).Info("Checking if bucket exists")
result, err := s.minioClient.BucketExists(s.ctx, bucketName)
Expand Down Expand Up @@ -288,6 +306,16 @@ func (s *s3client) PutDirectory(bucket, key, path string) error {
return nil
}

func (s *s3client) PutDirectoryWithMetadata(bucket, key, path string, metadata, tags map[string]string) error {
for putTask := range generatePutTasks(key, path) {
err := s.PutFileWithMetadata(bucket, putTask.key, putTask.path, metadata, tags)
if err != nil {
return err
}
}
return nil
}

// GetFile downloads a file to a local file path
func (s *s3client) GetFile(bucket, key, path string) error {
log.WithFields(log.Fields{"endpoint": s.Endpoint, "bucket": bucket, "key": key, "path": path}).Info("Getting file from s3")
Expand Down