Add Infra

This commit is contained in:
aditya.siregar
2025-03-03 18:59:25 +07:00
parent c02d8cda69
commit c9f1e33c24
1782 changed files with 692894 additions and 21 deletions
+45918
View File
File diff suppressed because it is too large Load Diff
+202
View File
@@ -0,0 +1,202 @@
package s3
import (
"bytes"
"crypto/md5"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"hash"
"io"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
)
const (
contentMD5Header = "Content-Md5"
contentSha256Header = "X-Amz-Content-Sha256"
amzTeHeader = "X-Amz-Te"
amzTxEncodingHeader = "X-Amz-Transfer-Encoding"
appendMD5TxEncoding = "append-md5"
)
// computeBodyHashes will add Content MD5 and Content Sha256 hashes to the
// request. If the body is not seekable or S3DisableContentMD5Validation set
// this handler will be ignored.
func computeBodyHashes(r *request.Request) {
if aws.BoolValue(r.Config.S3DisableContentMD5Validation) {
return
}
if r.IsPresigned() {
return
}
if r.Error != nil || !aws.IsReaderSeekable(r.Body) {
return
}
var md5Hash, sha256Hash hash.Hash
hashers := make([]io.Writer, 0, 2)
// Determine upfront which hashes can be set without overriding user
// provide header data.
if v := r.HTTPRequest.Header.Get(contentMD5Header); len(v) == 0 {
md5Hash = md5.New()
hashers = append(hashers, md5Hash)
}
if v := r.HTTPRequest.Header.Get(contentSha256Header); len(v) == 0 {
sha256Hash = sha256.New()
hashers = append(hashers, sha256Hash)
}
// Create the destination writer based on the hashes that are not already
// provided by the user.
var dst io.Writer
switch len(hashers) {
case 0:
return
case 1:
dst = hashers[0]
default:
dst = io.MultiWriter(hashers...)
}
if _, err := aws.CopySeekableBody(dst, r.Body); err != nil {
r.Error = awserr.New("BodyHashError", "failed to compute body hashes", err)
return
}
// For the hashes created, set the associated headers that the user did not
// already provide.
if md5Hash != nil {
sum := make([]byte, md5.Size)
encoded := make([]byte, md5Base64EncLen)
base64.StdEncoding.Encode(encoded, md5Hash.Sum(sum[0:0]))
r.HTTPRequest.Header[contentMD5Header] = []string{string(encoded)}
}
if sha256Hash != nil {
encoded := make([]byte, sha256HexEncLen)
sum := make([]byte, sha256.Size)
hex.Encode(encoded, sha256Hash.Sum(sum[0:0]))
r.HTTPRequest.Header[contentSha256Header] = []string{string(encoded)}
}
}
const (
md5Base64EncLen = (md5.Size + 2) / 3 * 4 // base64.StdEncoding.EncodedLen
sha256HexEncLen = sha256.Size * 2 // hex.EncodedLen
)
// Adds the x-amz-te: append_md5 header to the request. This requests the service
// responds with a trailing MD5 checksum.
//
// Will not ask for append MD5 if disabled, the request is presigned or,
// or the API operation does not support content MD5 validation.
func askForTxEncodingAppendMD5(r *request.Request) {
if aws.BoolValue(r.Config.S3DisableContentMD5Validation) {
return
}
if r.IsPresigned() {
return
}
r.HTTPRequest.Header.Set(amzTeHeader, appendMD5TxEncoding)
}
func useMD5ValidationReader(r *request.Request) {
if r.Error != nil {
return
}
if v := r.HTTPResponse.Header.Get(amzTxEncodingHeader); v != appendMD5TxEncoding {
return
}
var bodyReader *io.ReadCloser
var contentLen int64
switch tv := r.Data.(type) {
case *GetObjectOutput:
bodyReader = &tv.Body
contentLen = aws.Int64Value(tv.ContentLength)
// Update ContentLength hiden the trailing MD5 checksum.
tv.ContentLength = aws.Int64(contentLen - md5.Size)
tv.ContentRange = aws.String(r.HTTPResponse.Header.Get("X-Amz-Content-Range"))
default:
r.Error = awserr.New("ChecksumValidationError",
fmt.Sprintf("%s: %s header received on unsupported API, %s",
amzTxEncodingHeader, appendMD5TxEncoding, r.Operation.Name,
), nil)
return
}
if contentLen < md5.Size {
r.Error = awserr.New("ChecksumValidationError",
fmt.Sprintf("invalid Content-Length %d for %s %s",
contentLen, appendMD5TxEncoding, amzTxEncodingHeader,
), nil)
return
}
// Wrap and swap the response body reader with the validation reader.
*bodyReader = newMD5ValidationReader(*bodyReader, contentLen-md5.Size)
}
type md5ValidationReader struct {
rawReader io.ReadCloser
payload io.Reader
hash hash.Hash
payloadLen int64
read int64
}
func newMD5ValidationReader(reader io.ReadCloser, payloadLen int64) *md5ValidationReader {
h := md5.New()
return &md5ValidationReader{
rawReader: reader,
payload: io.TeeReader(&io.LimitedReader{R: reader, N: payloadLen}, h),
hash: h,
payloadLen: payloadLen,
}
}
func (v *md5ValidationReader) Read(p []byte) (n int, err error) {
n, err = v.payload.Read(p)
if err != nil && err != io.EOF {
return n, err
}
v.read += int64(n)
if err == io.EOF {
if v.read != v.payloadLen {
return n, io.ErrUnexpectedEOF
}
expectSum := make([]byte, md5.Size)
actualSum := make([]byte, md5.Size)
if _, sumReadErr := io.ReadFull(v.rawReader, expectSum); sumReadErr != nil {
return n, sumReadErr
}
actualSum = v.hash.Sum(actualSum[0:0])
if !bytes.Equal(expectSum, actualSum) {
return n, awserr.New("InvalidChecksum",
fmt.Sprintf("expected MD5 checksum %s, got %s",
hex.EncodeToString(expectSum),
hex.EncodeToString(actualSum),
),
nil)
}
}
return n, err
}
func (v *md5ValidationReader) Close() error {
return v.rawReader.Close()
}
+107
View File
@@ -0,0 +1,107 @@
package s3
import (
"io/ioutil"
"regexp"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/awsutil"
"github.com/aws/aws-sdk-go/aws/request"
)
var reBucketLocation = regexp.MustCompile(`>([^<>]+)<\/Location`)
// NormalizeBucketLocation is a utility function which will update the
// passed in value to always be a region ID. Generally this would be used
// with GetBucketLocation API operation.
//
// Replaces empty string with "us-east-1", and "EU" with "eu-west-1".
//
// See http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETlocation.html
// for more information on the values that can be returned.
func NormalizeBucketLocation(loc string) string {
switch loc {
case "":
loc = "us-east-1"
case "EU":
loc = "eu-west-1"
}
return loc
}
// NormalizeBucketLocationHandler is a request handler which will update the
// GetBucketLocation's result LocationConstraint value to always be a region ID.
//
// Replaces empty string with "us-east-1", and "EU" with "eu-west-1".
//
// See http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETlocation.html
// for more information on the values that can be returned.
//
// req, result := svc.GetBucketLocationRequest(&s3.GetBucketLocationInput{
// Bucket: aws.String(bucket),
// })
// req.Handlers.Unmarshal.PushBackNamed(NormalizeBucketLocationHandler)
// err := req.Send()
var NormalizeBucketLocationHandler = request.NamedHandler{
Name: "awssdk.s3.NormalizeBucketLocation",
Fn: func(req *request.Request) {
if req.Error != nil {
return
}
out := req.Data.(*GetBucketLocationOutput)
loc := NormalizeBucketLocation(aws.StringValue(out.LocationConstraint))
out.LocationConstraint = aws.String(loc)
},
}
// WithNormalizeBucketLocation is a request option which will update the
// GetBucketLocation's result LocationConstraint value to always be a region ID.
//
// Replaces empty string with "us-east-1", and "EU" with "eu-west-1".
//
// See http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETlocation.html
// for more information on the values that can be returned.
//
// result, err := svc.GetBucketLocationWithContext(ctx,
// &s3.GetBucketLocationInput{
// Bucket: aws.String(bucket),
// },
// s3.WithNormalizeBucketLocation,
// )
func WithNormalizeBucketLocation(r *request.Request) {
r.Handlers.Unmarshal.PushBackNamed(NormalizeBucketLocationHandler)
}
func buildGetBucketLocation(r *request.Request) {
if r.DataFilled() {
out := r.Data.(*GetBucketLocationOutput)
b, err := ioutil.ReadAll(r.HTTPResponse.Body)
if err != nil {
r.Error = awserr.New(request.ErrCodeSerialization,
"failed reading response body", err)
return
}
match := reBucketLocation.FindSubmatch(b)
if len(match) > 1 {
loc := string(match[1])
out.LocationConstraint = aws.String(loc)
}
}
}
func populateLocationConstraint(r *request.Request) {
if r.ParamsFilled() && aws.StringValue(r.Config.Region) != "us-east-1" {
in := r.Params.(*CreateBucketInput)
if in.CreateBucketConfiguration == nil {
r.Params = awsutil.CopyOf(r.Params)
in = r.Params.(*CreateBucketInput)
in.CreateBucketConfiguration = &CreateBucketConfiguration{
LocationConstraint: r.Config.Region,
}
}
}
}
+89
View File
@@ -0,0 +1,89 @@
package s3
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/s3shared/arn"
"github.com/aws/aws-sdk-go/internal/s3shared/s3err"
)
func init() {
initClient = defaultInitClientFn
initRequest = defaultInitRequestFn
}
func defaultInitClientFn(c *client.Client) {
if c.Config.UseDualStackEndpoint == endpoints.DualStackEndpointStateUnset {
if aws.BoolValue(c.Config.UseDualStack) {
c.Config.UseDualStackEndpoint = endpoints.DualStackEndpointStateEnabled
} else {
c.Config.UseDualStackEndpoint = endpoints.DualStackEndpointStateDisabled
}
}
// Support building custom endpoints based on config
c.Handlers.Build.PushFront(endpointHandler)
// Require SSL when using SSE keys
c.Handlers.Validate.PushBack(validateSSERequiresSSL)
c.Handlers.Build.PushBack(computeSSEKeyMD5)
c.Handlers.Build.PushBack(computeCopySourceSSEKeyMD5)
// S3 uses custom error unmarshaling logic
c.Handlers.UnmarshalError.Clear()
c.Handlers.UnmarshalError.PushBack(unmarshalError)
c.Handlers.UnmarshalError.PushBackNamed(s3err.RequestFailureWrapperHandler())
}
func defaultInitRequestFn(r *request.Request) {
// Add request handlers for specific platforms.
// e.g. 100-continue support for PUT requests using Go 1.6
platformRequestHandlers(r)
switch r.Operation.Name {
case opGetBucketLocation:
// GetBucketLocation has custom parsing logic
r.Handlers.Unmarshal.PushFront(buildGetBucketLocation)
case opCreateBucket:
// Auto-populate LocationConstraint with current region
r.Handlers.Validate.PushFront(populateLocationConstraint)
case opCopyObject, opUploadPartCopy, opCompleteMultipartUpload:
r.Handlers.Unmarshal.PushFront(copyMultipartStatusOKUnmarshalError)
r.Handlers.Unmarshal.PushBackNamed(s3err.RequestFailureWrapperHandler())
case opPutObject, opUploadPart:
r.Handlers.Build.PushBack(computeBodyHashes)
// Disabled until #1837 root issue is resolved.
// case opGetObject:
// r.Handlers.Build.PushBack(askForTxEncodingAppendMD5)
// r.Handlers.Unmarshal.PushBack(useMD5ValidationReader)
case opWriteGetObjectResponse:
r.Handlers.Build.PushFront(buildWriteGetObjectResponseEndpoint)
}
}
// bucketGetter is an accessor interface to grab the "Bucket" field from
// an S3 type.
type bucketGetter interface {
getBucket() string
}
// sseCustomerKeyGetter is an accessor interface to grab the "SSECustomerKey"
// field from an S3 type.
type sseCustomerKeyGetter interface {
getSSECustomerKey() string
}
// copySourceSSECustomerKeyGetter is an accessor interface to grab the
// "CopySourceSSECustomerKey" field from an S3 type.
type copySourceSSECustomerKeyGetter interface {
getCopySourceSSECustomerKey() string
}
// endpointARNGetter is an accessor interface to grab the
// the field corresponding to an endpoint ARN input.
type endpointARNGetter interface {
getEndpointARN() (arn.Resource, error)
hasEndpointARN() bool
}
+26
View File
@@ -0,0 +1,26 @@
// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
// Package s3 provides the client and types for making API
// requests to Amazon Simple Storage Service.
//
// See https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01 for more information on this service.
//
// See s3 package documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/
//
// # Using the Client
//
// To contact Amazon Simple Storage Service with the SDK use the New function to create
// a new service client. With that client you can make API requests to the service.
// These clients are safe to use concurrently.
//
// See the SDK's documentation for more information on how to use the SDK.
// https://docs.aws.amazon.com/sdk-for-go/api/
//
// See aws.Config documentation for more information on configuring SDK clients.
// https://docs.aws.amazon.com/sdk-for-go/api/aws/#Config
//
// See the Amazon Simple Storage Service client S3 for more
// information on creating client for this service.
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#New
package s3
+109
View File
@@ -0,0 +1,109 @@
// Upload Managers
//
// The s3manager package's Uploader provides concurrent upload of content to S3
// by taking advantage of S3's Multipart APIs. The Uploader also supports both
// io.Reader for streaming uploads, and will also take advantage of io.ReadSeeker
// for optimizations if the Body satisfies that type. Once the Uploader instance
// is created you can call Upload concurrently from multiple goroutines safely.
//
// // The session the S3 Uploader will use
// sess := session.Must(session.NewSession())
//
// // Create an uploader with the session and default options
// uploader := s3manager.NewUploader(sess)
//
// f, err := os.Open(filename)
// if err != nil {
// return fmt.Errorf("failed to open file %q, %v", filename, err)
// }
//
// // Upload the file to S3.
// result, err := uploader.Upload(&s3manager.UploadInput{
// Bucket: aws.String(myBucket),
// Key: aws.String(myString),
// Body: f,
// })
// if err != nil {
// return fmt.Errorf("failed to upload file, %v", err)
// }
// fmt.Printf("file uploaded to, %s\n", aws.StringValue(result.Location))
//
// See the s3manager package's Uploader type documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#Uploader
//
// # Download Manager
//
// The s3manager package's Downloader provides concurrently downloading of Objects
// from S3. The Downloader will write S3 Object content with an io.WriterAt.
// Once the Downloader instance is created you can call Download concurrently from
// multiple goroutines safely.
//
// // The session the S3 Downloader will use
// sess := session.Must(session.NewSession())
//
// // Create a downloader with the session and default options
// downloader := s3manager.NewDownloader(sess)
//
// // Create a file to write the S3 Object contents to.
// f, err := os.Create(filename)
// if err != nil {
// return fmt.Errorf("failed to create file %q, %v", filename, err)
// }
//
// // Write the contents of S3 Object to the file
// n, err := downloader.Download(f, &s3.GetObjectInput{
// Bucket: aws.String(myBucket),
// Key: aws.String(myString),
// })
// if err != nil {
// return fmt.Errorf("failed to download file, %v", err)
// }
// fmt.Printf("file downloaded, %d bytes\n", n)
//
// See the s3manager package's Downloader type documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#Downloader
//
// # Automatic URI cleaning
//
// Interacting with objects whose keys contain adjacent slashes (e.g. bucketname/foo//bar/objectname)
// requires setting DisableRestProtocolURICleaning to true in the aws.Config struct
// used by the service client.
//
// svc := s3.New(sess, &aws.Config{
// DisableRestProtocolURICleaning: aws.Bool(true),
// })
// out, err := svc.GetObject(&s3.GetObjectInput {
// Bucket: aws.String("bucketname"),
// Key: aws.String("//foo//bar//moo"),
// })
//
// # Get Bucket Region
//
// GetBucketRegion will attempt to get the region for a bucket using a region
// hint to determine which AWS partition to perform the query on. Use this utility
// to determine the region a bucket is in.
//
// sess := session.Must(session.NewSession())
//
// bucket := "my-bucket"
// region, err := s3manager.GetBucketRegion(ctx, sess, bucket, "us-west-2")
// if err != nil {
// if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
// fmt.Fprintf(os.Stderr, "unable to find bucket %s's region not found\n", bucket)
// }
// return err
// }
// fmt.Printf("Bucket %s is in %s region\n", bucket, region)
//
// See the s3manager package's GetBucketRegion function documentation for more information
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#GetBucketRegion
//
// # S3 Crypto Client
//
// The s3crypto package provides the tools to upload and download encrypted
// content from S3. The Encryption and Decryption clients can be used concurrently
// once the client is created.
//
// See the s3crypto package documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3crypto/
package s3
+298
View File
@@ -0,0 +1,298 @@
package s3
import (
"fmt"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/endpoints"
"net/url"
"strings"
"github.com/aws/aws-sdk-go/aws"
awsarn "github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/s3shared"
"github.com/aws/aws-sdk-go/internal/s3shared/arn"
)
const (
s3Namespace = "s3"
s3AccessPointNamespace = "s3-accesspoint"
s3ObjectsLambdaNamespace = "s3-object-lambda"
s3OutpostsNamespace = "s3-outposts"
)
// Used by shapes with members decorated as endpoint ARN.
func parseEndpointARN(v string) (arn.Resource, error) {
return arn.ParseResource(v, accessPointResourceParser)
}
func accessPointResourceParser(a awsarn.ARN) (arn.Resource, error) {
resParts := arn.SplitResource(a.Resource)
switch resParts[0] {
case "accesspoint":
switch a.Service {
case s3Namespace:
return arn.ParseAccessPointResource(a, resParts[1:])
case s3ObjectsLambdaNamespace:
return parseS3ObjectLambdaAccessPointResource(a, resParts)
default:
return arn.AccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: fmt.Sprintf("service is not %s or %s", s3Namespace, s3ObjectsLambdaNamespace)}
}
case "outpost":
if a.Service != "s3-outposts" {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "service is not s3-outposts"}
}
return parseOutpostAccessPointResource(a, resParts[1:])
default:
return nil, arn.InvalidARNError{ARN: a, Reason: "unknown resource type"}
}
}
// parseOutpostAccessPointResource attempts to parse the ARNs resource as an
// outpost access-point resource.
//
// Supported Outpost AccessPoint ARN format:
// - ARN format: arn:{partition}:s3-outposts:{region}:{accountId}:outpost/{outpostId}/accesspoint/{accesspointName}
// - example: arn:aws:s3-outposts:us-west-2:012345678901:outpost/op-1234567890123456/accesspoint/myaccesspoint
func parseOutpostAccessPointResource(a awsarn.ARN, resParts []string) (arn.OutpostAccessPointARN, error) {
// outpost accesspoint arn is only valid if service is s3-outposts
if a.Service != "s3-outposts" {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "service is not s3-outposts"}
}
if len(resParts) == 0 {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "outpost resource-id not set"}
}
if len(resParts) < 3 {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{
ARN: a, Reason: "access-point resource not set in Outpost ARN",
}
}
resID := strings.TrimSpace(resParts[0])
if len(resID) == 0 {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "outpost resource-id not set"}
}
var outpostAccessPointARN = arn.OutpostAccessPointARN{}
switch resParts[1] {
case "accesspoint":
accessPointARN, err := arn.ParseAccessPointResource(a, resParts[2:])
if err != nil {
return arn.OutpostAccessPointARN{}, err
}
// set access-point arn
outpostAccessPointARN.AccessPointARN = accessPointARN
default:
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "access-point resource not set in Outpost ARN"}
}
// set outpost id
outpostAccessPointARN.OutpostID = resID
return outpostAccessPointARN, nil
}
func parseS3ObjectLambdaAccessPointResource(a awsarn.ARN, resParts []string) (arn.S3ObjectLambdaAccessPointARN, error) {
if a.Service != s3ObjectsLambdaNamespace {
return arn.S3ObjectLambdaAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: fmt.Sprintf("service is not %s", s3ObjectsLambdaNamespace)}
}
accessPointARN, err := arn.ParseAccessPointResource(a, resParts[1:])
if err != nil {
return arn.S3ObjectLambdaAccessPointARN{}, err
}
if len(accessPointARN.Region) == 0 {
return arn.S3ObjectLambdaAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: fmt.Sprintf("%s region not set", s3ObjectsLambdaNamespace)}
}
return arn.S3ObjectLambdaAccessPointARN{
AccessPointARN: accessPointARN,
}, nil
}
func endpointHandler(req *request.Request) {
endpoint, ok := req.Params.(endpointARNGetter)
if !ok || !endpoint.hasEndpointARN() {
updateBucketEndpointFromParams(req)
return
}
resource, err := endpoint.getEndpointARN()
if err != nil {
req.Error = s3shared.NewInvalidARNError(nil, err)
return
}
resReq := s3shared.ResourceRequest{
Resource: resource,
Request: req,
}
if len(resReq.Request.ClientInfo.PartitionID) != 0 && resReq.IsCrossPartition() {
req.Error = s3shared.NewClientPartitionMismatchError(resource,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
return
}
if !resReq.AllowCrossRegion() && resReq.IsCrossRegion() {
req.Error = s3shared.NewClientRegionMismatchError(resource,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
return
}
switch tv := resource.(type) {
case arn.AccessPointARN:
err = updateRequestAccessPointEndpoint(req, tv)
if err != nil {
req.Error = err
}
case arn.S3ObjectLambdaAccessPointARN:
err = updateRequestS3ObjectLambdaAccessPointEndpoint(req, tv)
if err != nil {
req.Error = err
}
case arn.OutpostAccessPointARN:
// outposts does not support FIPS regions
if req.Config.UseFIPSEndpoint == endpoints.FIPSEndpointStateEnabled {
req.Error = s3shared.NewFIPSConfigurationError(resource, req.ClientInfo.PartitionID,
aws.StringValue(req.Config.Region), nil)
return
}
err = updateRequestOutpostAccessPointEndpoint(req, tv)
if err != nil {
req.Error = err
}
default:
req.Error = s3shared.NewInvalidARNError(resource, nil)
}
}
func updateBucketEndpointFromParams(r *request.Request) {
bucket, ok := bucketNameFromReqParams(r.Params)
if !ok {
// Ignore operation requests if the bucket name was not provided
// if this is an input validation error the validation handler
// will report it.
return
}
updateEndpointForS3Config(r, bucket)
}
func updateRequestAccessPointEndpoint(req *request.Request, accessPoint arn.AccessPointARN) error {
// Accelerate not supported
if aws.BoolValue(req.Config.S3UseAccelerate) {
return s3shared.NewClientConfiguredForAccelerateError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Ignore the disable host prefix for access points
req.Config.DisableEndpointHostPrefix = aws.Bool(false)
if err := accessPointEndpointBuilder(accessPoint).build(req); err != nil {
return err
}
removeBucketFromPath(req.HTTPRequest.URL)
return nil
}
func updateRequestS3ObjectLambdaAccessPointEndpoint(req *request.Request, accessPoint arn.S3ObjectLambdaAccessPointARN) error {
// DualStack not supported
if isUseDualStackEndpoint(req) {
return s3shared.NewClientConfiguredForDualStackError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Accelerate not supported
if aws.BoolValue(req.Config.S3UseAccelerate) {
return s3shared.NewClientConfiguredForAccelerateError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Ignore the disable host prefix for access points
req.Config.DisableEndpointHostPrefix = aws.Bool(false)
if err := s3ObjectLambdaAccessPointEndpointBuilder(accessPoint).build(req); err != nil {
return err
}
removeBucketFromPath(req.HTTPRequest.URL)
return nil
}
func updateRequestOutpostAccessPointEndpoint(req *request.Request, accessPoint arn.OutpostAccessPointARN) error {
// Accelerate not supported
if aws.BoolValue(req.Config.S3UseAccelerate) {
return s3shared.NewClientConfiguredForAccelerateError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Dualstack not supported
if isUseDualStackEndpoint(req) {
return s3shared.NewClientConfiguredForDualStackError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Ignore the disable host prefix for access points
req.Config.DisableEndpointHostPrefix = aws.Bool(false)
if err := outpostAccessPointEndpointBuilder(accessPoint).build(req); err != nil {
return err
}
removeBucketFromPath(req.HTTPRequest.URL)
return nil
}
func removeBucketFromPath(u *url.URL) {
u.Path = strings.Replace(u.Path, "/{Bucket}", "", -1)
if u.Path == "" {
u.Path = "/"
}
}
func buildWriteGetObjectResponseEndpoint(req *request.Request) {
// DualStack not supported
if isUseDualStackEndpoint(req) {
req.Error = awserr.New("ConfigurationError", "client configured for dualstack but not supported for operation", nil)
return
}
// Accelerate not supported
if aws.BoolValue(req.Config.S3UseAccelerate) {
req.Error = awserr.New("ConfigurationError", "client configured for accelerate but not supported for operation", nil)
return
}
signingName := s3ObjectsLambdaNamespace
signingRegion := req.ClientInfo.SigningRegion
if !hasCustomEndpoint(req) {
endpoint, err := resolveRegionalEndpoint(req, aws.StringValue(req.Config.Region), req.ClientInfo.ResolvedRegion, EndpointsID)
if err != nil {
req.Error = awserr.New(request.ErrCodeSerialization, "failed to resolve endpoint", err)
return
}
signingRegion = endpoint.SigningRegion
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
req.Error = err
return
}
updateS3HostPrefixForS3ObjectLambda(req)
}
redirectSigner(req, signingName, signingRegion)
}
func isUseDualStackEndpoint(req *request.Request) bool {
if req.Config.UseDualStackEndpoint != endpoints.DualStackEndpointStateUnset {
return req.Config.UseDualStackEndpoint == endpoints.DualStackEndpointStateEnabled
}
return aws.BoolValue(req.Config.UseDualStack)
}
+239
View File
@@ -0,0 +1,239 @@
package s3
import (
"net/url"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/s3shared"
"github.com/aws/aws-sdk-go/internal/s3shared/arn"
"github.com/aws/aws-sdk-go/private/protocol"
)
const (
accessPointPrefixLabel = "accesspoint"
accountIDPrefixLabel = "accountID"
accessPointPrefixTemplate = "{" + accessPointPrefixLabel + "}-{" + accountIDPrefixLabel + "}."
outpostPrefixLabel = "outpost"
outpostAccessPointPrefixTemplate = accessPointPrefixTemplate + "{" + outpostPrefixLabel + "}."
)
// hasCustomEndpoint returns true if endpoint is a custom endpoint
func hasCustomEndpoint(r *request.Request) bool {
return len(aws.StringValue(r.Config.Endpoint)) > 0
}
// accessPointEndpointBuilder represents the endpoint builder for access point arn
type accessPointEndpointBuilder arn.AccessPointARN
// build builds the endpoint for corresponding access point arn
//
// For building an endpoint from access point arn, format used is:
// - Access point endpoint format : {accesspointName}-{accountId}.s3-accesspoint.{region}.{dnsSuffix}
// - example : myaccesspoint-012345678901.s3-accesspoint.us-west-2.amazonaws.com
//
// Access Point Endpoint requests are signed using "s3" as signing name.
func (a accessPointEndpointBuilder) build(req *request.Request) error {
resolveService := arn.AccessPointARN(a).Service
resolveRegion := arn.AccessPointARN(a).Region
endpoint, err := resolveRegionalEndpoint(req, resolveRegion, "", resolveService)
if err != nil {
return s3shared.NewFailedToResolveEndpointError(arn.AccessPointARN(a),
req.ClientInfo.PartitionID, resolveRegion, err)
}
endpoint.URL = endpoints.AddScheme(endpoint.URL, aws.BoolValue(req.Config.DisableSSL))
if !hasCustomEndpoint(req) {
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
return err
}
// dual stack provided by endpoint resolver
updateS3HostForS3AccessPoint(req)
}
protocol.HostPrefixBuilder{
Prefix: accessPointPrefixTemplate,
LabelsFn: a.hostPrefixLabelValues,
}.Build(req)
// signer redirection
redirectSigner(req, endpoint.SigningName, endpoint.SigningRegion)
err = protocol.ValidateEndpointHost(req.Operation.Name, req.HTTPRequest.URL.Host)
if err != nil {
return s3shared.NewInvalidARNError(arn.AccessPointARN(a), err)
}
return nil
}
func (a accessPointEndpointBuilder) hostPrefixLabelValues() map[string]string {
return map[string]string{
accessPointPrefixLabel: arn.AccessPointARN(a).AccessPointName,
accountIDPrefixLabel: arn.AccessPointARN(a).AccountID,
}
}
// s3ObjectLambdaAccessPointEndpointBuilder represents the endpoint builder for an s3 object lambda access point arn
type s3ObjectLambdaAccessPointEndpointBuilder arn.S3ObjectLambdaAccessPointARN
// build builds the endpoint for corresponding access point arn
//
// For building an endpoint from access point arn, format used is:
// - Access point endpoint format : {accesspointName}-{accountId}.s3-object-lambda.{region}.{dnsSuffix}
// - example : myaccesspoint-012345678901.s3-object-lambda.us-west-2.amazonaws.com
//
// Access Point Endpoint requests are signed using "s3-object-lambda" as signing name.
func (a s3ObjectLambdaAccessPointEndpointBuilder) build(req *request.Request) error {
resolveRegion := arn.S3ObjectLambdaAccessPointARN(a).Region
endpoint, err := resolveRegionalEndpoint(req, resolveRegion, "", EndpointsID)
if err != nil {
return s3shared.NewFailedToResolveEndpointError(arn.S3ObjectLambdaAccessPointARN(a),
req.ClientInfo.PartitionID, resolveRegion, err)
}
endpoint.URL = endpoints.AddScheme(endpoint.URL, aws.BoolValue(req.Config.DisableSSL))
endpoint.SigningName = s3ObjectsLambdaNamespace
if !hasCustomEndpoint(req) {
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
return err
}
updateS3HostPrefixForS3ObjectLambda(req)
}
protocol.HostPrefixBuilder{
Prefix: accessPointPrefixTemplate,
LabelsFn: a.hostPrefixLabelValues,
}.Build(req)
// signer redirection
redirectSigner(req, endpoint.SigningName, endpoint.SigningRegion)
err = protocol.ValidateEndpointHost(req.Operation.Name, req.HTTPRequest.URL.Host)
if err != nil {
return s3shared.NewInvalidARNError(arn.S3ObjectLambdaAccessPointARN(a), err)
}
return nil
}
func (a s3ObjectLambdaAccessPointEndpointBuilder) hostPrefixLabelValues() map[string]string {
return map[string]string{
accessPointPrefixLabel: arn.S3ObjectLambdaAccessPointARN(a).AccessPointName,
accountIDPrefixLabel: arn.S3ObjectLambdaAccessPointARN(a).AccountID,
}
}
// outpostAccessPointEndpointBuilder represents the Endpoint builder for outpost access point arn.
type outpostAccessPointEndpointBuilder arn.OutpostAccessPointARN
// build builds an endpoint corresponding to the outpost access point arn.
//
// For building an endpoint from outpost access point arn, format used is:
// - Outpost access point endpoint format : {accesspointName}-{accountId}.{outpostId}.s3-outposts.{region}.{dnsSuffix}
// - example : myaccesspoint-012345678901.op-01234567890123456.s3-outposts.us-west-2.amazonaws.com
//
// Outpost AccessPoint Endpoint request are signed using "s3-outposts" as signing name.
func (o outpostAccessPointEndpointBuilder) build(req *request.Request) error {
resolveRegion := o.Region
resolveService := o.Service
endpointsID := resolveService
if resolveService == s3OutpostsNamespace {
endpointsID = "s3"
}
endpoint, err := resolveRegionalEndpoint(req, resolveRegion, "", endpointsID)
if err != nil {
return s3shared.NewFailedToResolveEndpointError(o,
req.ClientInfo.PartitionID, resolveRegion, err)
}
endpoint.URL = endpoints.AddScheme(endpoint.URL, aws.BoolValue(req.Config.DisableSSL))
if !hasCustomEndpoint(req) {
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
return err
}
updateHostPrefix(req, endpointsID, resolveService)
}
protocol.HostPrefixBuilder{
Prefix: outpostAccessPointPrefixTemplate,
LabelsFn: o.hostPrefixLabelValues,
}.Build(req)
// set the signing region, name to resolved names from ARN
redirectSigner(req, resolveService, resolveRegion)
err = protocol.ValidateEndpointHost(req.Operation.Name, req.HTTPRequest.URL.Host)
if err != nil {
return s3shared.NewInvalidARNError(o, err)
}
return nil
}
func (o outpostAccessPointEndpointBuilder) hostPrefixLabelValues() map[string]string {
return map[string]string{
accessPointPrefixLabel: o.AccessPointName,
accountIDPrefixLabel: o.AccountID,
outpostPrefixLabel: o.OutpostID,
}
}
func resolveRegionalEndpoint(r *request.Request, region, resolvedRegion, endpointsID string) (endpoints.ResolvedEndpoint, error) {
return r.Config.EndpointResolver.EndpointFor(endpointsID, region, func(opts *endpoints.Options) {
opts.DisableSSL = aws.BoolValue(r.Config.DisableSSL)
opts.UseDualStack = aws.BoolValue(r.Config.UseDualStack)
opts.UseDualStackEndpoint = r.Config.UseDualStackEndpoint
opts.UseFIPSEndpoint = r.Config.UseFIPSEndpoint
opts.S3UsEast1RegionalEndpoint = endpoints.RegionalS3UsEast1Endpoint
opts.ResolvedRegion = resolvedRegion
opts.Logger = r.Config.Logger
opts.LogDeprecated = r.Config.LogLevel.Matches(aws.LogDebugWithDeprecated)
})
}
func updateRequestEndpoint(r *request.Request, endpoint string) (err error) {
r.HTTPRequest.URL, err = url.Parse(endpoint + r.Operation.HTTPPath)
if err != nil {
return awserr.New(request.ErrCodeSerialization,
"failed to parse endpoint URL", err)
}
return nil
}
// redirectSigner sets signing name, signing region for a request
func redirectSigner(req *request.Request, signingName string, signingRegion string) {
req.ClientInfo.SigningName = signingName
req.ClientInfo.SigningRegion = signingRegion
}
func updateS3HostForS3AccessPoint(req *request.Request) {
updateHostPrefix(req, "s3", s3AccessPointNamespace)
}
func updateS3HostPrefixForS3ObjectLambda(req *request.Request) {
updateHostPrefix(req, "s3", s3ObjectsLambdaNamespace)
}
func updateHostPrefix(req *request.Request, oldEndpointPrefix, newEndpointPrefix string) {
host := req.HTTPRequest.URL.Host
if strings.HasPrefix(host, oldEndpointPrefix) {
// replace service hostlabel oldEndpointPrefix to newEndpointPrefix
req.HTTPRequest.URL.Host = newEndpointPrefix + host[len(oldEndpointPrefix):]
}
}
+69
View File
@@ -0,0 +1,69 @@
// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
package s3
const (
// ErrCodeBucketAlreadyExists for service response error code
// "BucketAlreadyExists".
//
// The requested bucket name is not available. The bucket namespace is shared
// by all users of the system. Select a different name and try again.
ErrCodeBucketAlreadyExists = "BucketAlreadyExists"
// ErrCodeBucketAlreadyOwnedByYou for service response error code
// "BucketAlreadyOwnedByYou".
//
// The bucket you tried to create already exists, and you own it. Amazon S3
// returns this error in all Amazon Web Services Regions except in the North
// Virginia Region. For legacy compatibility, if you re-create an existing bucket
// that you already own in the North Virginia Region, Amazon S3 returns 200
// OK and resets the bucket access control lists (ACLs).
ErrCodeBucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou"
// ErrCodeInvalidObjectState for service response error code
// "InvalidObjectState".
//
// Object is archived and inaccessible until restored.
//
// If the object you are retrieving is stored in the S3 Glacier Flexible Retrieval
// storage class, the S3 Glacier Deep Archive storage class, the S3 Intelligent-Tiering
// Archive Access tier, or the S3 Intelligent-Tiering Deep Archive Access tier,
// before you can retrieve the object you must first restore a copy using RestoreObject
// (https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html).
// Otherwise, this operation returns an InvalidObjectState error. For information
// about restoring archived objects, see Restoring Archived Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/restoring-objects.html)
// in the Amazon S3 User Guide.
ErrCodeInvalidObjectState = "InvalidObjectState"
// ErrCodeNoSuchBucket for service response error code
// "NoSuchBucket".
//
// The specified bucket does not exist.
ErrCodeNoSuchBucket = "NoSuchBucket"
// ErrCodeNoSuchKey for service response error code
// "NoSuchKey".
//
// The specified key does not exist.
ErrCodeNoSuchKey = "NoSuchKey"
// ErrCodeNoSuchUpload for service response error code
// "NoSuchUpload".
//
// The specified multipart upload does not exist.
ErrCodeNoSuchUpload = "NoSuchUpload"
// ErrCodeObjectAlreadyInActiveTierError for service response error code
// "ObjectAlreadyInActiveTierError".
//
// This action is not allowed against this storage tier.
ErrCodeObjectAlreadyInActiveTierError = "ObjectAlreadyInActiveTierError"
// ErrCodeObjectNotInActiveTierError for service response error code
// "ObjectNotInActiveTierError".
//
// The source object of the COPY action is not in the active tier and is only
// stored in Amazon S3 Glacier.
ErrCodeObjectNotInActiveTierError = "ObjectNotInActiveTierError"
)
+136
View File
@@ -0,0 +1,136 @@
package s3
import (
"fmt"
"net/url"
"regexp"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
)
// an operationBlacklist is a list of operation names that should a
// request handler should not be executed with.
type operationBlacklist []string
// Continue will return true of the Request's operation name is not
// in the blacklist. False otherwise.
func (b operationBlacklist) Continue(r *request.Request) bool {
for i := 0; i < len(b); i++ {
if b[i] == r.Operation.Name {
return false
}
}
return true
}
var accelerateOpBlacklist = operationBlacklist{
opListBuckets, opCreateBucket, opDeleteBucket,
}
// Automatically add the bucket name to the endpoint domain
// if possible. This style of bucket is valid for all bucket names which are
// DNS compatible and do not contain "."
func updateEndpointForS3Config(r *request.Request, bucketName string) {
forceHostStyle := aws.BoolValue(r.Config.S3ForcePathStyle)
accelerate := aws.BoolValue(r.Config.S3UseAccelerate)
if accelerate && accelerateOpBlacklist.Continue(r) {
if forceHostStyle {
if r.Config.Logger != nil {
r.Config.Logger.Log("ERROR: aws.Config.S3UseAccelerate is not compatible with aws.Config.S3ForcePathStyle, ignoring S3ForcePathStyle.")
}
}
updateEndpointForAccelerate(r, bucketName)
} else if !forceHostStyle && r.Operation.Name != opGetBucketLocation {
updateEndpointForHostStyle(r, bucketName)
}
}
func updateEndpointForHostStyle(r *request.Request, bucketName string) {
if !hostCompatibleBucketName(r.HTTPRequest.URL, bucketName) {
// bucket name must be valid to put into the host
return
}
moveBucketToHost(r.HTTPRequest.URL, bucketName)
}
var (
accelElem = []byte("s3-accelerate.dualstack.")
)
func updateEndpointForAccelerate(r *request.Request, bucketName string) {
if !hostCompatibleBucketName(r.HTTPRequest.URL, bucketName) {
r.Error = awserr.New("InvalidParameterException",
fmt.Sprintf("bucket name %s is not compatible with S3 Accelerate", bucketName),
nil)
return
}
parts := strings.Split(r.HTTPRequest.URL.Host, ".")
if len(parts) < 3 {
r.Error = awserr.New("InvalidParameterExecption",
fmt.Sprintf("unable to update endpoint host for S3 accelerate, hostname invalid, %s",
r.HTTPRequest.URL.Host), nil)
return
}
if parts[0] == "s3" || strings.HasPrefix(parts[0], "s3-") {
parts[0] = "s3-accelerate"
}
for i := 1; i+1 < len(parts); i++ {
if parts[i] == aws.StringValue(r.Config.Region) {
parts = append(parts[:i], parts[i+1:]...)
break
}
}
r.HTTPRequest.URL.Host = strings.Join(parts, ".")
moveBucketToHost(r.HTTPRequest.URL, bucketName)
}
// Attempts to retrieve the bucket name from the request input parameters.
// If no bucket is found, or the field is empty "", false will be returned.
func bucketNameFromReqParams(params interface{}) (string, bool) {
if iface, ok := params.(bucketGetter); ok {
b := iface.getBucket()
return b, len(b) > 0
}
return "", false
}
// hostCompatibleBucketName returns true if the request should
// put the bucket in the host. This is false if S3ForcePathStyle is
// explicitly set or if the bucket is not DNS compatible.
func hostCompatibleBucketName(u *url.URL, bucket string) bool {
// Bucket might be DNS compatible but dots in the hostname will fail
// certificate validation, so do not use host-style.
if u.Scheme == "https" && strings.Contains(bucket, ".") {
return false
}
// if the bucket is DNS compatible
return dnsCompatibleBucketName(bucket)
}
var reDomain = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
var reIPAddress = regexp.MustCompile(`^(\d+\.){3}\d+$`)
// dnsCompatibleBucketName returns true if the bucket name is DNS compatible.
// Buckets created outside of the classic region MUST be DNS compatible.
func dnsCompatibleBucketName(bucket string) bool {
return reDomain.MatchString(bucket) &&
!reIPAddress.MatchString(bucket) &&
!strings.Contains(bucket, "..")
}
// moveBucketToHost moves the bucket name from the URI path to URL host.
func moveBucketToHost(u *url.URL, bucket string) {
u.Host = bucket + "." + u.Host
removeBucketFromPath(u)
}
+9
View File
@@ -0,0 +1,9 @@
//go:build !go1.6
// +build !go1.6
package s3
import "github.com/aws/aws-sdk-go/aws/request"
func platformRequestHandlers(r *request.Request) {
}
+29
View File
@@ -0,0 +1,29 @@
//go:build go1.6
// +build go1.6
package s3
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
)
func platformRequestHandlers(r *request.Request) {
if r.Operation.HTTPMethod == "PUT" {
// 100-Continue should only be used on put requests.
r.Handlers.Sign.PushBack(add100Continue)
}
}
func add100Continue(r *request.Request) {
if aws.BoolValue(r.Config.S3Disable100Continue) {
return
}
if r.HTTPRequest.ContentLength < 1024*1024*2 {
// Ignore requests smaller than 2MB. This helps prevent delaying
// requests unnecessarily.
return
}
r.HTTPRequest.Header.Set("Expect", "100-continue")
}
+108
View File
@@ -0,0 +1,108 @@
// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
package s3
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/client/metadata"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/aws/aws-sdk-go/private/protocol/restxml"
)
// S3 provides the API operation methods for making requests to
// Amazon Simple Storage Service. See this package's package overview docs
// for details on the service.
//
// S3 methods are safe to use concurrently. It is not safe to
// modify mutate any of the struct's properties though.
type S3 struct {
*client.Client
}
// Used for custom client initialization logic
var initClient func(*client.Client)
// Used for custom request initialization logic
var initRequest func(*request.Request)
// Service information constants
const (
ServiceName = "s3" // Name of service.
EndpointsID = ServiceName // ID to lookup a service endpoint with.
ServiceID = "S3" // ServiceID is a unique identifier of a specific service.
)
// New creates a new instance of the S3 client with a session.
// If additional configuration is needed for the client instance use the optional
// aws.Config parameter to add your extra config.
//
// Example:
//
// mySession := session.Must(session.NewSession())
//
// // Create a S3 client from just a session.
// svc := s3.New(mySession)
//
// // Create a S3 client with additional configuration
// svc := s3.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
func New(p client.ConfigProvider, cfgs ...*aws.Config) *S3 {
c := p.ClientConfig(EndpointsID, cfgs...)
if c.SigningNameDerived || len(c.SigningName) == 0 {
c.SigningName = "s3"
}
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName, c.ResolvedRegion)
}
// newClient creates, initializes and returns a new service client instance.
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName, resolvedRegion string) *S3 {
svc := &S3{
Client: client.New(
cfg,
metadata.ClientInfo{
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
PartitionID: partitionID,
Endpoint: endpoint,
APIVersion: "2006-03-01",
ResolvedRegion: resolvedRegion,
},
handlers,
),
}
// Handlers
svc.Handlers.Sign.PushBackNamed(v4.BuildNamedHandler(v4.SignRequestHandler.Name, func(s *v4.Signer) {
s.DisableURIPathEscaping = true
}))
svc.Handlers.Build.PushBackNamed(restxml.BuildHandler)
svc.Handlers.Unmarshal.PushBackNamed(restxml.UnmarshalHandler)
svc.Handlers.UnmarshalMeta.PushBackNamed(restxml.UnmarshalMetaHandler)
svc.Handlers.UnmarshalError.PushBackNamed(restxml.UnmarshalErrorHandler)
svc.Handlers.BuildStream.PushBackNamed(restxml.BuildHandler)
svc.Handlers.UnmarshalStream.PushBackNamed(restxml.UnmarshalHandler)
// Run custom client initialization if present
if initClient != nil {
initClient(svc.Client)
}
return svc
}
// newRequest creates a new request for a S3 operation and runs any
// custom request initialization.
func (c *S3) newRequest(op *request.Operation, params, data interface{}) *request.Request {
req := c.NewRequest(op, params, data)
// Run custom request initialization if present
if initRequest != nil {
initRequest(req)
}
return req
}
+84
View File
@@ -0,0 +1,84 @@
package s3
import (
"crypto/md5"
"encoding/base64"
"net/http"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
)
var errSSERequiresSSL = awserr.New("ConfigError", "cannot send SSE keys over HTTP.", nil)
func validateSSERequiresSSL(r *request.Request) {
if r.HTTPRequest.URL.Scheme == "https" {
return
}
if iface, ok := r.Params.(sseCustomerKeyGetter); ok {
if len(iface.getSSECustomerKey()) > 0 {
r.Error = errSSERequiresSSL
return
}
}
if iface, ok := r.Params.(copySourceSSECustomerKeyGetter); ok {
if len(iface.getCopySourceSSECustomerKey()) > 0 {
r.Error = errSSERequiresSSL
return
}
}
}
const (
sseKeyHeader = "x-amz-server-side-encryption-customer-key"
sseKeyMD5Header = sseKeyHeader + "-md5"
)
func computeSSEKeyMD5(r *request.Request) {
var key string
if g, ok := r.Params.(sseCustomerKeyGetter); ok {
key = g.getSSECustomerKey()
}
computeKeyMD5(sseKeyHeader, sseKeyMD5Header, key, r.HTTPRequest)
}
const (
copySrcSSEKeyHeader = "x-amz-copy-source-server-side-encryption-customer-key"
copySrcSSEKeyMD5Header = copySrcSSEKeyHeader + "-md5"
)
func computeCopySourceSSEKeyMD5(r *request.Request) {
var key string
if g, ok := r.Params.(copySourceSSECustomerKeyGetter); ok {
key = g.getCopySourceSSECustomerKey()
}
computeKeyMD5(copySrcSSEKeyHeader, copySrcSSEKeyMD5Header, key, r.HTTPRequest)
}
func computeKeyMD5(keyHeader, keyMD5Header, key string, r *http.Request) {
if len(key) == 0 {
// Backwards compatiablity where user just set the header value instead
// of using the API parameter, or setting the header value for an
// operation without the parameters modeled.
key = r.Header.Get(keyHeader)
if len(key) == 0 {
return
}
// In backwards compatible, the header's value is not base64 encoded,
// and needs to be encoded and updated by the SDK's customizations.
b64Key := base64.StdEncoding.EncodeToString([]byte(key))
r.Header.Set(keyHeader, b64Key)
}
// Only update Key's MD5 if not already set.
if len(r.Header.Get(keyMD5Header)) == 0 {
sum := md5.Sum([]byte(key))
keyMD5 := base64.StdEncoding.EncodeToString(sum[:])
r.Header.Set(keyMD5Header, keyMD5)
}
}
+47
View File
@@ -0,0 +1,47 @@
package s3
import (
"bytes"
"io"
"io/ioutil"
"net/http"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/sdkio"
)
func copyMultipartStatusOKUnmarshalError(r *request.Request) {
b, err := ioutil.ReadAll(r.HTTPResponse.Body)
r.HTTPResponse.Body.Close()
if err != nil {
r.Error = awserr.NewRequestFailure(
awserr.New(request.ErrCodeSerialization, "unable to read response body", err),
r.HTTPResponse.StatusCode,
r.RequestID,
)
// Note, some middleware later in the stack like restxml.Unmarshal expect a valid, non-closed Body
// even in case of an error, so we replace it with an empty Reader.
r.HTTPResponse.Body = ioutil.NopCloser(bytes.NewBuffer(nil))
return
}
body := bytes.NewReader(b)
r.HTTPResponse.Body = ioutil.NopCloser(body)
defer body.Seek(0, sdkio.SeekStart)
unmarshalError(r)
if err, ok := r.Error.(awserr.Error); ok && err != nil {
if err.Code() == request.ErrCodeSerialization &&
err.OrigErr() != io.EOF {
r.Error = nil
return
}
// if empty payload
if err.OrigErr() == io.EOF {
r.HTTPResponse.StatusCode = http.StatusInternalServerError
} else {
r.HTTPResponse.StatusCode = http.StatusServiceUnavailable
}
}
}
+114
View File
@@ -0,0 +1,114 @@
package s3
import (
"bytes"
"encoding/xml"
"fmt"
"io"
"io/ioutil"
"net/http"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
)
type xmlErrorResponse struct {
XMLName xml.Name `xml:"Error"`
Code string `xml:"Code"`
Message string `xml:"Message"`
}
func unmarshalError(r *request.Request) {
defer r.HTTPResponse.Body.Close()
defer io.Copy(ioutil.Discard, r.HTTPResponse.Body)
// Bucket exists in a different region, and request needs
// to be made to the correct region.
if r.HTTPResponse.StatusCode == http.StatusMovedPermanently {
msg := fmt.Sprintf(
"incorrect region, the bucket is not in '%s' region at endpoint '%s'",
aws.StringValue(r.Config.Region),
aws.StringValue(r.Config.Endpoint),
)
if v := r.HTTPResponse.Header.Get("x-amz-bucket-region"); len(v) != 0 {
msg += fmt.Sprintf(", bucket is in '%s' region", v)
}
r.Error = awserr.NewRequestFailure(
awserr.New("BucketRegionError", msg, nil),
r.HTTPResponse.StatusCode,
r.RequestID,
)
return
}
// Attempt to parse error from body if it is known
var errResp xmlErrorResponse
var err error
if r.HTTPResponse.StatusCode >= 200 && r.HTTPResponse.StatusCode < 300 {
err = s3unmarshalXMLError(&errResp, r.HTTPResponse.Body)
} else {
err = xmlutil.UnmarshalXMLError(&errResp, r.HTTPResponse.Body)
}
if err != nil {
var errorMsg string
if err == io.EOF {
errorMsg = "empty response payload"
} else {
errorMsg = "failed to unmarshal error message"
}
r.Error = awserr.NewRequestFailure(
awserr.New(request.ErrCodeSerialization,
errorMsg, err),
r.HTTPResponse.StatusCode,
r.RequestID,
)
return
}
// Fallback to status code converted to message if still no error code
if len(errResp.Code) == 0 {
statusText := http.StatusText(r.HTTPResponse.StatusCode)
errResp.Code = strings.Replace(statusText, " ", "", -1)
errResp.Message = statusText
}
r.Error = awserr.NewRequestFailure(
awserr.New(errResp.Code, errResp.Message, err),
r.HTTPResponse.StatusCode,
r.RequestID,
)
}
// A RequestFailure provides access to the S3 Request ID and Host ID values
// returned from API operation errors. Getting the error as a string will
// return the formated error with the same information as awserr.RequestFailure,
// while also adding the HostID value from the response.
type RequestFailure interface {
awserr.RequestFailure
// Host ID is the S3 Host ID needed for debug, and contacting support
HostID() string
}
// s3unmarshalXMLError is s3 specific xml error unmarshaler
// for 200 OK errors and response payloads.
// This function differs from the xmlUtil.UnmarshalXMLError
// func. It does not ignore the EOF error and passes it up.
// Related to bug fix for `s3 200 OK response with empty payload`
func s3unmarshalXMLError(v interface{}, stream io.Reader) error {
var errBuf bytes.Buffer
body := io.TeeReader(stream, &errBuf)
err := xml.NewDecoder(body).Decode(v)
if err != nil && err != io.EOF {
return awserr.NewUnmarshalError(err,
"failed to unmarshal error message", errBuf.Bytes())
}
return err
}
+214
View File
@@ -0,0 +1,214 @@
// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
package s3
import (
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
)
// WaitUntilBucketExists uses the Amazon S3 API operation
// HeadBucket to wait for a condition to be met before returning.
// If the condition is not met within the max attempt window, an error will
// be returned.
func (c *S3) WaitUntilBucketExists(input *HeadBucketInput) error {
return c.WaitUntilBucketExistsWithContext(aws.BackgroundContext(), input)
}
// WaitUntilBucketExistsWithContext is an extended version of WaitUntilBucketExists.
// With the support for passing in a context and options to configure the
// Waiter and the underlying request options.
//
// The context must be non-nil and will be used for request cancellation. If
// the context is nil a panic will occur. In the future the SDK may create
// sub-contexts for http.Requests. See https://golang.org/pkg/context/
// for more information on using Contexts.
func (c *S3) WaitUntilBucketExistsWithContext(ctx aws.Context, input *HeadBucketInput, opts ...request.WaiterOption) error {
w := request.Waiter{
Name: "WaitUntilBucketExists",
MaxAttempts: 20,
Delay: request.ConstantWaiterDelay(5 * time.Second),
Acceptors: []request.WaiterAcceptor{
{
State: request.SuccessWaiterState,
Matcher: request.StatusWaiterMatch,
Expected: 200,
},
{
State: request.SuccessWaiterState,
Matcher: request.StatusWaiterMatch,
Expected: 301,
},
{
State: request.SuccessWaiterState,
Matcher: request.StatusWaiterMatch,
Expected: 403,
},
{
State: request.RetryWaiterState,
Matcher: request.StatusWaiterMatch,
Expected: 404,
},
},
Logger: c.Config.Logger,
NewRequest: func(opts []request.Option) (*request.Request, error) {
var inCpy *HeadBucketInput
if input != nil {
tmp := *input
inCpy = &tmp
}
req, _ := c.HeadBucketRequest(inCpy)
req.SetContext(ctx)
req.ApplyOptions(opts...)
return req, nil
},
}
w.ApplyOptions(opts...)
return w.WaitWithContext(ctx)
}
// WaitUntilBucketNotExists uses the Amazon S3 API operation
// HeadBucket to wait for a condition to be met before returning.
// If the condition is not met within the max attempt window, an error will
// be returned.
func (c *S3) WaitUntilBucketNotExists(input *HeadBucketInput) error {
return c.WaitUntilBucketNotExistsWithContext(aws.BackgroundContext(), input)
}
// WaitUntilBucketNotExistsWithContext is an extended version of WaitUntilBucketNotExists.
// With the support for passing in a context and options to configure the
// Waiter and the underlying request options.
//
// The context must be non-nil and will be used for request cancellation. If
// the context is nil a panic will occur. In the future the SDK may create
// sub-contexts for http.Requests. See https://golang.org/pkg/context/
// for more information on using Contexts.
func (c *S3) WaitUntilBucketNotExistsWithContext(ctx aws.Context, input *HeadBucketInput, opts ...request.WaiterOption) error {
w := request.Waiter{
Name: "WaitUntilBucketNotExists",
MaxAttempts: 20,
Delay: request.ConstantWaiterDelay(5 * time.Second),
Acceptors: []request.WaiterAcceptor{
{
State: request.SuccessWaiterState,
Matcher: request.StatusWaiterMatch,
Expected: 404,
},
},
Logger: c.Config.Logger,
NewRequest: func(opts []request.Option) (*request.Request, error) {
var inCpy *HeadBucketInput
if input != nil {
tmp := *input
inCpy = &tmp
}
req, _ := c.HeadBucketRequest(inCpy)
req.SetContext(ctx)
req.ApplyOptions(opts...)
return req, nil
},
}
w.ApplyOptions(opts...)
return w.WaitWithContext(ctx)
}
// WaitUntilObjectExists uses the Amazon S3 API operation
// HeadObject to wait for a condition to be met before returning.
// If the condition is not met within the max attempt window, an error will
// be returned.
func (c *S3) WaitUntilObjectExists(input *HeadObjectInput) error {
return c.WaitUntilObjectExistsWithContext(aws.BackgroundContext(), input)
}
// WaitUntilObjectExistsWithContext is an extended version of WaitUntilObjectExists.
// With the support for passing in a context and options to configure the
// Waiter and the underlying request options.
//
// The context must be non-nil and will be used for request cancellation. If
// the context is nil a panic will occur. In the future the SDK may create
// sub-contexts for http.Requests. See https://golang.org/pkg/context/
// for more information on using Contexts.
func (c *S3) WaitUntilObjectExistsWithContext(ctx aws.Context, input *HeadObjectInput, opts ...request.WaiterOption) error {
w := request.Waiter{
Name: "WaitUntilObjectExists",
MaxAttempts: 20,
Delay: request.ConstantWaiterDelay(5 * time.Second),
Acceptors: []request.WaiterAcceptor{
{
State: request.SuccessWaiterState,
Matcher: request.StatusWaiterMatch,
Expected: 200,
},
{
State: request.RetryWaiterState,
Matcher: request.StatusWaiterMatch,
Expected: 404,
},
},
Logger: c.Config.Logger,
NewRequest: func(opts []request.Option) (*request.Request, error) {
var inCpy *HeadObjectInput
if input != nil {
tmp := *input
inCpy = &tmp
}
req, _ := c.HeadObjectRequest(inCpy)
req.SetContext(ctx)
req.ApplyOptions(opts...)
return req, nil
},
}
w.ApplyOptions(opts...)
return w.WaitWithContext(ctx)
}
// WaitUntilObjectNotExists uses the Amazon S3 API operation
// HeadObject to wait for a condition to be met before returning.
// If the condition is not met within the max attempt window, an error will
// be returned.
func (c *S3) WaitUntilObjectNotExists(input *HeadObjectInput) error {
return c.WaitUntilObjectNotExistsWithContext(aws.BackgroundContext(), input)
}
// WaitUntilObjectNotExistsWithContext is an extended version of WaitUntilObjectNotExists.
// With the support for passing in a context and options to configure the
// Waiter and the underlying request options.
//
// The context must be non-nil and will be used for request cancellation. If
// the context is nil a panic will occur. In the future the SDK may create
// sub-contexts for http.Requests. See https://golang.org/pkg/context/
// for more information on using Contexts.
func (c *S3) WaitUntilObjectNotExistsWithContext(ctx aws.Context, input *HeadObjectInput, opts ...request.WaiterOption) error {
w := request.Waiter{
Name: "WaitUntilObjectNotExists",
MaxAttempts: 20,
Delay: request.ConstantWaiterDelay(5 * time.Second),
Acceptors: []request.WaiterAcceptor{
{
State: request.SuccessWaiterState,
Matcher: request.StatusWaiterMatch,
Expected: 404,
},
},
Logger: c.Config.Logger,
NewRequest: func(opts []request.Option) (*request.Request, error) {
var inCpy *HeadObjectInput
if input != nil {
tmp := *input
inCpy = &tmp
}
req, _ := c.HeadObjectRequest(inCpy)
req.SetContext(ctx)
req.ApplyOptions(opts...)
return req, nil
},
}
w.ApplyOptions(opts...)
return w.WaitWithContext(ctx)
}