From 5c596e7612de61deffbd0bcea1219fdb27d47a17 Mon Sep 17 00:00:00 2001 From: shabbyrobe Date: Thu, 17 Jan 2019 22:53:15 +1100 Subject: [PATCH] Bucket name rewriting --- cmd/gofakes3/main.go | 3 +++ cors.go | 16 --------------- gofakes3.go | 41 +++++++++++++++++++++++++++++++++------ gofakes3_internal_test.go | 30 ++++++++++++++++++++++++++++ option.go | 10 ++++++++++ 5 files changed, 78 insertions(+), 22 deletions(-) diff --git a/cmd/gofakes3/main.go b/cmd/gofakes3/main.go index 0f2bc18..2ea4b50 100644 --- a/cmd/gofakes3/main.go +++ b/cmd/gofakes3/main.go @@ -31,6 +31,7 @@ type fakeS3Flags struct { initialBucket string fixedTimeStr string noIntegrity bool + hostBucket bool boltDb string directFsPath string @@ -45,6 +46,7 @@ func (f *fakeS3Flags) attach(flagSet *flag.FlagSet) { flagSet.StringVar(&f.fixedTimeStr, "time", "", "RFC3339 format. If passed, the server's clock will always see this time; does not affect existing stored dates.") flagSet.StringVar(&f.initialBucket, "initialbucket", "", "If passed, this bucket will be created on startup if it does not already exist.") flagSet.BoolVar(&f.noIntegrity, "no-integrity", false, "Pass this flag to disable Content-MD5 validation when uploading.") + flagSet.BoolVar(&f.hostBucket, "hostbucket", false, "If passed, the bucket name will be extracted from the first segment of the hostname, rather than the first part of the URL path.") // Backend specific: flagSet.StringVar(&f.backendKind, "backend", "", "Backend to use to store data (memory, bolt, directfs, fs)") @@ -181,6 +183,7 @@ func run() error { gofakes3.WithTimeSkewLimit(timeSkewLimit), gofakes3.WithTimeSource(timeSource), gofakes3.WithLogger(gofakes3.GlobalLog()), + gofakes3.WithHostBucket(hostBucket), ) return listenAndServe(values.host, faker.Server()) diff --git a/cors.go b/cors.go index 8347819..cbbbf16 100644 --- a/cors.go +++ b/cors.go @@ -2,7 +2,6 @@ package gofakes3 import ( "net/http" - "regexp" "strings" ) @@ -22,8 +21,6 @@ var ( "x-amz-meta-to", } corsHeadersString = strings.Join(corsHeaders, ", ") - - bucketRewritePattern = regexp.MustCompile("(127.0.0.1:\\d{1,7})|(.localhost:\\d{1,7})|(localhost:\\d{1,7})") ) type withCORS struct { @@ -40,18 +37,5 @@ func (s *withCORS) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // Bucket name rewriting - // this is due to some inconsistencies in the AWS SDKs - bucket := bucketRewritePattern.ReplaceAllString(r.Host, "") - if len(bucket) > 0 { - s.log.Print(LogInfo, "rewrite bucket ->", bucket) - p := r.URL.Path - r.URL.Path = "/" + bucket - if p != "/" { - r.URL.Path += p - } - } - s.log.Print(LogInfo, "=>", r.URL) - s.r.ServeHTTP(w, r) } diff --git a/gofakes3.go b/gofakes3.go index 1c8a26b..74f5d0d 100644 --- a/gofakes3.go +++ b/gofakes3.go @@ -60,6 +60,7 @@ type GoFakeS3 struct { timeSkew time.Duration metadataSizeLimit int integrityCheck bool + hostBucket bool uploader *uploader requestID uint64 log Logger @@ -96,12 +97,24 @@ func (g *GoFakeS3) nextRequestID() uint64 { // Create the AWS S3 API func (g *GoFakeS3) Server() http.Handler { - wc := &withCORS{r: http.HandlerFunc(g.routeBase), log: g.log} + var handler http.Handler = &withCORS{r: http.HandlerFunc(g.routeBase), log: g.log} - hf := func(w http.ResponseWriter, rq *http.Request) { + if g.timeSkew != 0 { + handler = g.timeSkewMiddleware(handler) + } + + if g.hostBucket { + handler = g.hostBucketMiddleware(handler) + } + + return handler +} + +func (g *GoFakeS3) timeSkewMiddleware(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, rq *http.Request) { timeHdr := rq.Header.Get("x-amz-date") - if g.timeSkew > 0 && timeHdr != "" { + if timeHdr != "" { rqTime, _ := time.Parse("20060102T150405Z", timeHdr) at := g.timeSource.Now() skew := at.Sub(rqTime) @@ -112,10 +125,26 @@ func (g *GoFakeS3) Server() http.Handler { } } - wc.ServeHTTP(w, rq) - } + handler.ServeHTTP(w, rq) + }) +} - return http.HandlerFunc(hf) +// hostBucketMiddleware forces the server to use VirtualHost-style bucket URLs: +// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html +func (g *GoFakeS3) hostBucketMiddleware(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, rq *http.Request) { + parts := strings.SplitN(rq.Host, ".", 2) + bucket := parts[0] + + p := rq.URL.Path + rq.URL.Path = "/" + bucket + if p != "/" { + rq.URL.Path += p + } + g.log.Print(LogInfo, p, "=>", rq.URL) + + handler.ServeHTTP(w, rq) + }) } func (g *GoFakeS3) httpError(w http.ResponseWriter, r *http.Request, err error) { diff --git a/gofakes3_internal_test.go b/gofakes3_internal_test.go index 753b011..8759d12 100644 --- a/gofakes3_internal_test.go +++ b/gofakes3_internal_test.go @@ -5,6 +5,7 @@ import ( "encoding/xml" "fmt" "log" + "net/http" "net/http/httptest" "testing" ) @@ -52,6 +53,35 @@ func TestHttpErrorWriteFailure(t *testing.T) { } } +func TestHostBucketMiddleware(t *testing.T) { + for _, tc := range []struct { + in string + host string + out string + }{ + {"/", "foo", "/foo"}, + {"/", "mybucket.localhost", "/mybucket"}, + {"/object", "mybucket.localhost", "/mybucket/object"}, + } { + t.Run("", func(t *testing.T) { + var g GoFakeS3 + g.log = DiscardLog() + + inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != tc.out { + t.Fatal(r.URL.Path, "!=", tc.out) + } + }) + + handler := g.hostBucketMiddleware(inner) + rq := httptest.NewRequest("GET", tc.in, nil) + rq.Host = tc.host + rs := httptest.NewRecorder() + handler.ServeHTTP(rs, rq) + }) + } +} + type failingResponseWriter struct { *httptest.ResponseRecorder } diff --git a/option.go b/option.go index 0f52649..77c47f0 100644 --- a/option.go +++ b/option.go @@ -55,3 +55,13 @@ func WithGlobalLog() Option { func WithRequestID(id uint64) Option { return func(g *GoFakeS3) { g.requestID = id } } + +// WithHostBucket enables or disables bucket rewriting in the router. +// If active, the URL 'http://mybucket.localhost/object' will be routed +// as if the URL path was '/mybucket/object'. +// +// See https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html +// for details. +func WithHostBucket(enabled bool) Option { + return func(g *GoFakeS3) { g.hostBucket = enabled } +}