I'm trying to implements download/upload a file from/to a bucket in cloud storage via the s3 go sdk aws-sdk-go-v2 using the Interoperability feature
The download is working as expected, but the upload isnt working, with this error message: SDK 2022/09/14 11:24:43 DEBUG request failed with unretryable error https response error StatusCode: 403, RequestID: , HostID: , api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.
As I use same access_key and secret_key for both download and upload, it does not seems to be a credentials problem. Plus, the service account behind the hmac keys has the Storage object Admin Role.
Here the code:
main.go
package main
import (
"context"
"fmt"
"os"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
var BUCKET_NAME = ""
func main() {
//prepare gcp resolver
gcpResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: "https://storage.googleapis.com",
SigningRegion: "auto",
Source: aws.EndpointSourceCustom,
HostnameImmutable: true,
}, nil
})
//file with fornat : $accessKey:$secretKey
file, _ := os.ReadFile("/home/bapt/creds/amz-keys-gcp-2")
keys := strings.Split(string(file), ":")
//init the config options
optConfig := []func(*config.LoadOptions) error{
config.WithRegion("auto"),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(keys[0], strings.TrimRight(keys[1], "\n"), "")),
config.WithClientLogMode(aws.LogRetries | aws.LogRequestWithBody | aws.LogResponseWithBody | aws.LogRequestEventMessage | aws.LogResponseEventMessage | aws.LogSigning),
config.WithEndpointResolverWithOptions(gcpResolver),
}
//init config
cfg, _ := config.LoadDefaultConfig(context.TODO(), optConfig...)
//init service
svc := s3.NewFromConfig(cfg)
tempFile, _ := os.CreateTemp("/tmp", "test-gcp-*")
defer tempFile.Close()
downloader := manager.NewDownloader(svc)
downloader.Download(context.TODO(),tempFile, &s3.GetObjectInput{
Bucket: aws.String(BUCKET_NAME),
Key: aws.String("file-test.txt"),
})
//init uploader ( no multipart)
uploader := manager.NewUploader(svc, func(u *manager.Uploader) {
u.Concurrency = 1
u.MaxUploadParts = 1
})
//upload
_, err := uploader.Upload(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(BUCKET_NAME),
Key: aws.String("file-test.txt"),
Body: strings.NewReader("HELLO"),
})
fmt.Println(err)
}
go.mod
module gcps3/v2
go 1.18
require (
github.com/aws/aws-sdk-go-v2 v1.16.14
github.com/aws/aws-sdk-go-v2/config v1.17.5
github.com/aws/aws-sdk-go-v2/credentials v1.12.18
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.31
github.com/aws/aws-sdk-go-v2/service/s3 v1.27.9
)
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.7 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.15 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.21 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.17 // indirect
github.com/aws/smithy-go v1.13.2 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
)
And here the debug trace of the PUT ( my bucket name is replaced by BUCKET, and access_key with GOOG1ID:
SDK 2022/09/14 14:52:37 DEBUG Request Signature:
---[ CANONICAL STRING ]-----------------------------
PUT
/BUCKET/file-test.txt
x-id=PutObject
accept-encoding:identity
amz-sdk-invocation-id:d6776820-e336-4bdf-afa3-0ca3b6d5b0a0
amz-sdk-request:attempt=1; max=3
content-length:5
content-type:application/octet-stream
host:storage.googleapis.com
x-amz-content-sha256:UNSIGNED-PAYLOAD
x-amz-date:20220914T125237Z
accept-encoding;amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD
---[ STRING TO SIGN ]--------------------------------
AWS4-HMAC-SHA256
20220914T125237Z
20220914/auto/s3/aws4_request
bc09.....daf520
-----------------------------------------------------
SDK 2022/09/14 14:52:37 DEBUG Request
PUT /BUCKET/file-test.txt?x-id=PutObject HTTP/1.1
Host: storage.googleapis.com
User-Agent: aws-sdk-go-v2/1.16.14 os/linux lang/go/1.18.1 md/GOOS/linux md/GOARCH/amd64 api/s3/1.27.9 ft/s3-transfer
Content-Length: 5
Accept-Encoding: identity
Amz-Sdk-Invocation-Id: d6776820-e336-4bdf-afa3-0ca3b6d5b0a0
Amz-Sdk-Request: attempt=1; max=3
Authorization: AWS4-HMAC-SHA256 Credential=GOOG1ID/20220914/auto/s3/aws4_request, SignedHeaders=accept-encoding;amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date, Signature=c994....37d0
Content-Type: application/octet-stream
X-Amz-Content-Sha256: UNSIGNED-PAYLOAD
X-Amz-Date: 20220914T125237Z
HELLO
SDK 2022/09/14 14:52:37 DEBUG Response
HTTP/2.0 403 Forbidden
Content-Length: 883
Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"
Content-Type: application/xml; charset=UTF-8
Date: Wed, 14 Sep 2022 12:52:38 GMT
Server: UploadServer
X-Guploader-Uploadid: ADPycdt0aCu6BTmzWQl2Ehc4q2sP8rtexDb4Keyn6cQL_GigREvc8T1CzX0HH-ZXgw_6XWLJPPYXufwRCr0Sl7uSsiIi0Q
<?xml version='1.0' encoding='UTF-8'?><Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.</Message><StringToSign>AWS4-HMAC-SHA256
20220914T125237Z
20220914/auto/s3/aws4_request
0a10....d63e</StringToSign><CanonicalRequest>PUT
/BUCKET/file-test.txt
x-id=PutObject
accept-encoding:identity,gzip(gfe)
amz-sdk-invocation-id:d6776820-e336-4bdf-afa3-0ca3b6d5b0a0
amz-sdk-request:attempt=1; max=3
content-length:5
content-type:application/octet-stream
host:storage.googleapis.com
x-amz-content-sha256:UNSIGNED-PAYLOAD
x-amz-date:20220914T125237Z
accept-encoding;amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD</CanonicalRequest></Error>
SDK 2022/09/14 14:52:37 DEBUG request failed with unretryable error https response error StatusCode: 403, RequestID: , HostID: , api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.
operation error S3: PutObject, https response error StatusCode: 403, RequestID: , HostID: , api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.
I tested the upload, with this python script, and it works (same creds)
boto3.set_stream_logger('', logging.DEBUG)
GCP_BUCKET = True
FILE_TO_BE_UPLOADED = '/tmp/toto'
if GCP_BUCKET:
ACCESS_KEY = "ACESS"
SECRET_KEY = "SECRET"
bucket_name = "BUCKET"
region_name="auto"
endpoint_url="https://storage.googleapis.com"
def makeS3Client():
s3 = boto3.client("s3",
region_name=region_name,
endpoint_url=endpoint_url,
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
)
return s3
def upload_file(s3,bucket_name,fname):
"""
Uploads file to S3 bucket using S3 client object
:return: None
"""
object_name = os.path.basename(fname)
file_name = os.path.abspath(fname)
#file_name = os.path.join(pathlib.Path(__file__).parent.resolve(), fname)
response = s3.upload_file(file_name, bucket_name, object_name)
#print(response) # prints None
The only different between go and python i saw in the https requests made is the signedHeaders used.
But the go code with an s3 aws bucket is working fine...
Am I missing an option ?
Thanks for your help.