From 231f0c3efbd6b80cb40a5e1358e21aa3bf6bb77d Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 8 Feb 2015 17:56:28 -0800 Subject: [PATCH] Squash minio package into s3 package --- .gitignore | 3 +- Makefile | 18 +- common.go | 60 +------ configure.go | 55 +----- docs/index.md | 17 ++ docs_linux.go | 3 + minio-api-options.go | 149 --------------- minio-get-object.go | 59 ------ minio-list-buckets.go | 30 ---- minio-list-objects.go | 60 ------- minio-put-bucket.go | 46 ----- minio-put-object.go | 104 ----------- mkdocs.yml | 1 + pkg/minio/.gitignore | 1 - pkg/minio/auth.go | 69 ------- pkg/minio/client.go | 380 --------------------------------------- pkg/minio/client_test.go | 51 ------ pkg/minio/doc.go | 1 - pkg/s3/auth.go | 67 ++++++- s3-api-options.go | 2 +- 20 files changed, 100 insertions(+), 1076 deletions(-) create mode 100644 docs/index.md create mode 100644 docs_linux.go delete mode 100644 minio-api-options.go delete mode 100644 minio-get-object.go delete mode 100644 minio-list-buckets.go delete mode 100644 minio-list-objects.go delete mode 100644 minio-put-bucket.go delete mode 100644 minio-put-object.go create mode 100644 mkdocs.yml delete mode 100644 pkg/minio/.gitignore delete mode 100644 pkg/minio/auth.go delete mode 100644 pkg/minio/client.go delete mode 100644 pkg/minio/client_test.go delete mode 100644 pkg/minio/doc.go diff --git a/.gitignore b/.gitignore index ae68139b..22014e67 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ cover.out -mc \ No newline at end of file +mc +site/ \ No newline at end of file diff --git a/Makefile b/Makefile index ffc52f9d..43b4e58f 100644 --- a/Makefile +++ b/Makefile @@ -7,12 +7,14 @@ getdeps: checkdeps @go get github.com/tools/godep && echo "Installed godep" @go get golang.org/x/tools/cmd/cover && echo "Installed cover" -install: pkgs uri +build-all: getdeps + @echo "Building Libraries" + @godep go generate ./... + @godep go build ./... + +test-all: build-all + @echo "Running Test Suites:" + @godep go test -race ./... + +install: test-all @godep go install github.com/minio-io/mc && echo "Installed mc" - -pkgs: - @godep go test -race -coverprofile=cover.out github.com/minio-io/mc/pkg/s3 - @godep go test -race -coverprofile=cover.out github.com/minio-io/mc/pkg/minio - -uri: - @godep go test -race -coverprofile=cover.out github.com/minio-io/mc/pkg/uri diff --git a/common.go b/common.go index c41d0501..9289c9f4 100644 --- a/common.go +++ b/common.go @@ -22,7 +22,6 @@ import ( "path" "github.com/codegangsta/cli" - "github.com/minio-io/mc/pkg/minio" "github.com/minio-io/mc/pkg/s3" ) @@ -37,11 +36,6 @@ var Options = []cli.Command{ Usage: "", Subcommands: subS3APIOptions, }, - { - Name: "minio", - Usage: "", - Subcommands: subMinioApiOptions, - }, } var subS3Options = []cli.Command{ @@ -59,63 +53,15 @@ var subS3APIOptions = []cli.Command{ Configure, } -var subMinioApiOptions = []cli.Command{ - MinioGetObject, - MinioPutObject, - MinioPutBucket, - MinioListObjects, - MinioListBuckets, - MinioConfigure, -} - -func getAWSAuthFilePath() string { +func getAuthFilePath() string { home := os.Getenv("HOME") - return path.Join(home, S3_AUTH) -} - -func getMinioAuthFilePath() string { - home := os.Getenv("HOME") - return path.Join(home, MINIO_AUTH) -} - -func getMinioEnvironment() (auth *minio.Auth, err error) { - var accessKey, secretKey string - minioAuth, err := os.OpenFile(getMinioAuthFilePath(), os.O_RDWR, 0666) - defer minioAuth.Close() - if err != nil { - accessKey = os.Getenv("MINIO_ACCESS_KEY_ID") - secretKey = os.Getenv("MINIO_SECRET_ACCESS_KEY") - if accessKey == "" && secretKey == "" { - return nil, missingAccessSecretErr - } - if accessKey == "" { - return nil, missingAccessErr - } - - if secretKey == "" { - return nil, missingSecretErr - } - auth = &minio.Auth{ - AccessKey: accessKey, - SecretKey: secretKey, - Hostname: "127.0.0.1:8080", - } - } else { - var n int - minioAuthbytes := make([]byte, 1024) - n, err = minioAuth.Read(minioAuthbytes) - err = json.Unmarshal(minioAuthbytes[:n], &auth) - if err != nil { - return nil, err - } - } - return auth, nil + return path.Join(home, AUTH) } func getAWSEnvironment() (auth *s3.Auth, err error) { var s3Auth *os.File var accessKey, secretKey string - s3Auth, err = os.OpenFile(getAWSAuthFilePath(), os.O_RDWR, 0666) + s3Auth, err = os.OpenFile(getAuthFilePath(), os.O_RDWR, 0666) defer s3Auth.Close() if err != nil { accessKey = os.Getenv("AWS_ACCESS_KEY_ID") diff --git a/configure.go b/configure.go index 5a085e4e..39966f80 100644 --- a/configure.go +++ b/configure.go @@ -7,7 +7,6 @@ import ( "path" "github.com/codegangsta/cli" - "github.com/minio-io/mc/pkg/minio" "github.com/minio-io/mc/pkg/s3" ) @@ -23,58 +22,6 @@ func parseConfigureInput(c *cli.Context) (accessKey, secretKey string, err error return accessKey, secretKey, nil } -func doMinioConfigure(c *cli.Context) { - hostname := c.String("hostname") - if hostname == "" { - log.Fatal("Invalid hostname") - } - - /* - certFile := c.String("cert") - if certFile == "" { - log.Fatal("invalid certificate") - } - - keyFile := c.String("key") - if keyFile == "" { - log.Fatal("invalid key") - } - - var accessKey, secretKey string - var err error - accessKey, secretKey, err = parseConfigureInput(c) - if err != nil { - log.Fatal(err) - } - */ - auth := &minio.Auth{ - //AccessKey: accessKey, - //SecretKey: secretKey, - Hostname: hostname, - // CertPEM: certFile, - // KeyPEM: keyFile, - } - - jAuth, err := json.Marshal(auth) - if err != nil { - log.Fatal(err) - } - - var minioFile *os.File - home := os.Getenv("HOME") - minioFile, err = os.OpenFile(path.Join(home, MINIO_AUTH), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) - defer minioFile.Close() - if err != nil { - log.Fatal(err) - } - - _, err = minioFile.Write(jAuth) - if err != nil { - log.Fatal(err) - } - -} - func doS3Configure(c *cli.Context) { var err error var jAuth []byte @@ -92,7 +39,7 @@ func doS3Configure(c *cli.Context) { var s3File *os.File home := os.Getenv("HOME") - s3File, err = os.OpenFile(path.Join(home, S3_AUTH), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + s3File, err = os.OpenFile(path.Join(home, AUTH), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) defer s3File.Close() if err != nil { log.Fatal(err) diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..6482cd60 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,17 @@ +# Welcome to MkDocs + +For full documentation visit [mkdocs.org](http://mkdocs.org). + +## Commands + +* `mkdocs new [dir-name]` - Create a new project. +* `mkdocs serve` - Start the live-reloading docs server. +* `mkdocs build` - Build the documentation site. +* `mkdocs help` - Print this help message. + +## Project layout + + mkdocs.yml # The configuration file. + docs/ + index.md # The documentation homepage. + ... # Other markdown pages, images and other files. diff --git a/docs_linux.go b/docs_linux.go new file mode 100644 index 00000000..c93d25d1 --- /dev/null +++ b/docs_linux.go @@ -0,0 +1,3 @@ +package main + +//go:generate mkdocs build --clean diff --git a/minio-api-options.go b/minio-api-options.go deleted file mode 100644 index 822d6404..00000000 --- a/minio-api-options.go +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Mini Object Storage, (C) 2014,2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "github.com/codegangsta/cli" -) - -var MinioGetObject = cli.Command{ - Name: "get-object", - Usage: "", - Description: "Retrieves objects from Amazon S3.", - Action: minioGetObject, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "bucket", - Value: "", - Usage: "bucket name", - }, - cli.StringFlag{ - Name: "key", - Value: "", - Usage: "path to Object", - }, - }, -} - -var MinioPutBucket = cli.Command{ - Name: "put-object", - Usage: "", - Description: "Adds an object to a bucket.", - Action: minioPutObject, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "bucket", - Value: "", - Usage: "bucket name", - }, - cli.StringFlag{ - Name: "key", - Value: "", - Usage: "Object name", - }, - cli.StringFlag{ - Name: "body", - Value: "", - Usage: "Object blob", - }, - }, -} - -var MinioPutObject = cli.Command{ - Name: "put-bucket", - Usage: "", - Description: "Creates a new bucket.", - Action: minioPutBucket, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "bucket", - Value: "", - Usage: "bucket name", - }, - }, -} - -var MinioListObjects = cli.Command{ - Name: "list-objects", - Usage: "", - Description: `Returns some or all (up to 1000) of the objects in a bucket. - You can use the request parameters as selection criteria to - return a subset of the objects in a bucket.`, - Action: minioListObjects, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "bucket", - Value: "", - Usage: "Bucket name", - }, - }, -} - -var MinioListBuckets = cli.Command{ - Name: "list-buckets", - Usage: "", - Description: `Returns a list of all buckets owned by the authenticated - sender of the request.`, - Action: minioListBuckets, -} - -var MinioConfigure = cli.Command{ - Name: "configure", - Usage: "", - Description: `Configure minio client configuration data. If your config - file does not exist (the default location is ~/.auth), it will be - automatically created for you. Note that the configure command only writes - values to the config file. It does not use any configuration values from - the environment variables.`, - Action: doMinioConfigure, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "hostname", - Value: "127.0.0.1:8080", - Usage: "Minio object server", - }, - cli.StringFlag{ - Name: "accesskey", - Value: "", - Usage: "Minio access key", - }, - cli.StringFlag{ - Name: "secretKey", - Value: "", - Usage: "Minio secret key", - }, - cli.StringFlag{ - Name: "cacert", - Value: "", - Usage: "CA authority cert", - }, - cli.StringFlag{ - Name: "cert", - Value: "", - Usage: "Minio server certificate", - }, - cli.StringFlag{ - Name: "key", - Value: "", - Usage: "Minio server private key", - }, - }, -} - -const ( - MINIO_AUTH = ".minioauth" -) diff --git a/minio-get-object.go b/minio-get-object.go deleted file mode 100644 index 56da0a2d..00000000 --- a/minio-get-object.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Mini Object Storage, (C) 2014,2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "io" - "log" - "os" - - "github.com/codegangsta/cli" - "github.com/minio-io/mc/pkg/minio" -) - -func minioGetObject(c *cli.Context) { - var bucket, key string - var err error - var objectReader io.ReadCloser - var objectSize int64 - var auth *minio.Auth - - auth, err = getMinioEnvironment() - if err != nil { - log.Fatal(err) - } - - bucket = c.String("bucket") - key = c.String("key") - if bucket == "" { - log.Fatal(bucketNameErr) - } - if key == "" { - log.Fatal(objectNameErr) - } - - minio, _ := minio.NewMinioClient(auth) - objectReader, objectSize, err = minio.Get(bucket, key) - if err != nil { - log.Fatal(err) - } - - _, err = io.CopyN(os.Stdout, objectReader, objectSize) - if err != nil { - log.Fatal(err) - } -} diff --git a/minio-list-buckets.go b/minio-list-buckets.go deleted file mode 100644 index f13e7b3d..00000000 --- a/minio-list-buckets.go +++ /dev/null @@ -1,30 +0,0 @@ -package main - -import ( - "log" - - "github.com/codegangsta/cli" - "github.com/minio-io/mc/pkg/minio" -) - -func minioDumpBuckets(v []*minio.Bucket) { - for _, b := range v { - log.Printf("Bucket :%#v", b) - } -} - -func minioListBuckets(c *cli.Context) { - auth, err := getMinioEnvironment() - if err != nil { - log.Fatal(err) - } - - var buckets []*minio.Bucket - mc, _ := minio.NewMinioClient(auth) - buckets, err = mc.Buckets() - if err != nil { - log.Fatal(err) - } - - minioDumpBuckets(buckets) -} diff --git a/minio-list-objects.go b/minio-list-objects.go deleted file mode 100644 index 4c2b78ca..00000000 --- a/minio-list-objects.go +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Mini Object Storage, (C) 2014,2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "log" - - "github.com/codegangsta/cli" - "github.com/minio-io/mc/pkg/minio" -) - -func minioParseListObjectsInput(c *cli.Context) (bucket string, err error) { - bucket = c.String("bucket") - if bucket == "" { - return "", bucketNameErr - } - return bucket, nil -} - -func minioListObjects(c *cli.Context) { - var err error - var bucket string - var auth *minio.Auth - - auth, err = getMinioEnvironment() - if err != nil { - log.Fatal(err) - } - - bucket, err = minioParseListObjectsInput(c) - if err != nil { - log.Fatal(err) - } - - var items []*minio.Item - mc, _ := minio.NewMinioClient(auth) - // Gets 1000 maxkeys supported with GET Bucket API - items, err = mc.GetBucket(bucket, "", minio.MAX_OBJECT_LIST) - if err != nil { - log.Fatal(err) - } - - for _, item := range items { - log.Println(item) - } -} diff --git a/minio-put-bucket.go b/minio-put-bucket.go deleted file mode 100644 index 7825ce87..00000000 --- a/minio-put-bucket.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Mini Object Storage, (C) 2014,2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "log" - - "github.com/codegangsta/cli" - "github.com/minio-io/mc/pkg/minio" -) - -func minioPutBucket(c *cli.Context) { - var bucket string - var auth *minio.Auth - var err error - - auth, err = getMinioEnvironment() - if err != nil { - log.Fatal(err) - } - - bucket = c.String("bucket") - if bucket == "" { - log.Fatal(bucketNameErr) - } - - mc, _ := minio.NewMinioClient(auth) - err = mc.PutBucket(bucket) - if err != nil { - log.Fatal(err) - } -} diff --git a/minio-put-object.go b/minio-put-object.go deleted file mode 100644 index 5b767449..00000000 --- a/minio-put-object.go +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Mini Object Storage, (C) 2014,2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "bytes" - "io" - "log" - "os" - - "github.com/codegangsta/cli" - "github.com/minio-io/mc/pkg/minio" -) - -func minioPutMetadata(reader io.Reader) (bodyBuf io.Reader, size int64, err error) { - var length int - var bodyBuffer bytes.Buffer - - for err == nil { - byteBuffer := make([]byte, 1024*1024) - length, err = reader.Read(byteBuffer) - // It is necessary for us to verify this and break - if length == 0 { - break - } - byteBuffer = byteBuffer[0:length] - _, err = bodyBuffer.Write(byteBuffer) - if err != nil { - break - } - } - - if err != io.EOF { - return nil, 0, err - } - return &bodyBuffer, int64(bodyBuffer.Len()), nil -} - -func minioParsePutObjectInput(c *cli.Context) (bucket, key, body string, err error) { - bucket = c.String("bucket") - key = c.String("key") - body = c.String("body") - - if bucket == "" { - return "", "", "", bucketNameErr - } - - if key == "" { - return "", "", "", objectNameErr - } - - if body == "" { - return "", "", "", objectBlobErr - } - - return bucket, key, body, nil -} - -func minioPutObject(c *cli.Context) { - var err error - var bucket, key, body string - var auth *minio.Auth - auth, err = getMinioEnvironment() - if err != nil { - log.Fatal(err) - } - - bucket, key, body, err = minioParsePutObjectInput(c) - if err != nil { - log.Fatal(err) - } - mc, _ := minio.NewMinioClient(auth) - var bodyFile *os.File - bodyFile, err = os.Open(body) - if err != nil { - log.Fatal(err) - } - - var bodyBuffer io.Reader - var size int64 - bodyBuffer, size, err = minioPutMetadata(bodyFile) - if err != nil { - log.Fatal(err) - } - - err = mc.Put(bucket, key, nil, size, bodyBuffer) - if err != nil { - log.Fatal(err) - } -} diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..c97182f5 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1 @@ +site_name: My Docs diff --git a/pkg/minio/.gitignore b/pkg/minio/.gitignore deleted file mode 100644 index d852bed5..00000000 --- a/pkg/minio/.gitignore +++ /dev/null @@ -1 +0,0 @@ -rootkey.csv \ No newline at end of file diff --git a/pkg/minio/auth.go b/pkg/minio/auth.go deleted file mode 100644 index 0b7ebe97..00000000 --- a/pkg/minio/auth.go +++ /dev/null @@ -1,69 +0,0 @@ -package minio - -import ( - "crypto/tls" - "io/ioutil" - "net" - "net/http" - "time" -) - -type Auth struct { - AccessKey string - SecretKey string - Hostname string - CertPEM string - KeyPEM string -} - -type TlsConfig struct { - CertPEMBlock []byte - KeyPEMBlock []byte -} - -func (a *Auth) loadKeys(cert string, key string) (*TlsConfig, error) { - certBlock, err := ioutil.ReadFile(cert) - if err != nil { - return nil, err - } - keyBlock, err := ioutil.ReadFile(key) - if err != nil { - return nil, err - } - t := &TlsConfig{} - t.CertPEMBlock = certBlock - t.KeyPEMBlock = keyBlock - return t, nil -} - -func (a *Auth) getTlsTransport() (*http.Transport, error) { - if a.CertPEM == "" || a.KeyPEM == "" { - return &http.Transport{ - Dial: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).Dial, - TLSHandshakeTimeout: 10 * time.Second, - }, nil - } - - tlsconfig, err := a.loadKeys(a.CertPEM, a.KeyPEM) - if err != nil { - return nil, err - } - var cert tls.Certificate - cert, err = tls.X509KeyPair(tlsconfig.CertPEMBlock, tlsconfig.KeyPEMBlock) - if err != nil { - return nil, err - } - - // Setup HTTPS client - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{cert}, - InsecureSkipVerify: true, - } - - tlsConfig.BuildNameToCertificate() - transport := &http.Transport{TLSClientConfig: tlsConfig} - return transport, nil -} diff --git a/pkg/minio/client.go b/pkg/minio/client.go deleted file mode 100644 index bf1499b7..00000000 --- a/pkg/minio/client.go +++ /dev/null @@ -1,380 +0,0 @@ -/* - * Mini Object Storage, (C) 2014,2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package minio implements a generic Minio client -package minio - -import ( - "bytes" - "encoding/base64" - "encoding/hex" - "encoding/xml" - "errors" - "fmt" - "hash" - "io" - "io/ioutil" - "log" - "net/http" - "net/url" - "os" - "strconv" - "strings" - "time" -) - -const ( - MAX_OBJECT_LIST = 1000 -) - -// Client is an Minio MINIO client. -type Client struct { - Hostname string - Transport http.RoundTripper // or nil for the default -} - -type Bucket struct { - Name string - CreationDate string // 2006-02-03T16:45:09.000Z -} - -func (c *Client) transport() http.RoundTripper { - if c.Transport != nil { - return c.Transport - } - return http.DefaultTransport -} - -func (c *Client) hostname() string { - if c.Hostname != "" { - return c.Hostname - } - return "localhost" -} - -// bucketURL returns the URL prefix of the bucket, with trailing slash -func (c *Client) bucketURL(bucket string) string { - return fmt.Sprintf("http://%s/%s/", c.hostname(), bucket) -} - -func (c *Client) keyURL(bucket, key string) string { - return c.bucketURL(bucket) + key -} - -func newReq(url_ string) *http.Request { - req, err := http.NewRequest("GET", url_, nil) - if err != nil { - panic(fmt.Sprintf("minio client; invalid URL: %v", err)) - } - req.Header.Set("User-Agent", "Minio") - return req -} - -func (c *Client) Buckets() ([]*Bucket, error) { - req := newReq("http://" + c.hostname() + "/") - res, err := c.transport().RoundTrip(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("minio: Unexpected status code %d fetching bucket list", res.StatusCode) - } - return parseListAllMyBuckets(res.Body) -} - -func parseListAllMyBuckets(r io.Reader) ([]*Bucket, error) { - type allMyBuckets struct { - Buckets struct { - Bucket []*Bucket - } - } - var res allMyBuckets - if err := xml.NewDecoder(r).Decode(&res); err != nil { - return nil, err - } - return res.Buckets.Bucket, nil -} - -// Returns 0, os.ErrNotExist if not on MINIO, otherwise reterr is real. -func (c *Client) Stat(key, bucket string) (size int64, reterr error) { - req := newReq(c.keyURL(bucket, key)) - req.Method = "HEAD" - res, err := c.transport().RoundTrip(req) - if err != nil { - return 0, err - } - if res.Body != nil { - defer res.Body.Close() - } - switch res.StatusCode { - case http.StatusNotFound: - return 0, os.ErrNotExist - case http.StatusOK: - return strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64) - } - return 0, fmt.Errorf("minio: Unexpected status code %d statting object %v", res.StatusCode, key) -} - -func (c *Client) PutBucket(bucket string) error { - req := newReq(c.bucketURL(bucket)) - req.Method = "PUT" - res, err := c.transport().RoundTrip(req) - if res != nil && res.Body != nil { - defer res.Body.Close() - } - - if err != nil { - return err - } - if res.StatusCode != http.StatusOK { - // res.Write(os.Stderr) - return fmt.Errorf("Got response code %d from minio", res.StatusCode) - } - return nil - -} - -func (c *Client) Put(bucket, key string, md5 hash.Hash, size int64, body io.Reader) error { - req := newReq(c.keyURL(bucket, key)) - req.Method = "PUT" - req.ContentLength = size - if md5 != nil { - b64 := new(bytes.Buffer) - encoder := base64.NewEncoder(base64.StdEncoding, b64) - encoder.Write(md5.Sum(nil)) - encoder.Close() - req.Header.Set("Content-MD5", b64.String()) - } - req.Body = ioutil.NopCloser(body) - - res, err := c.transport().RoundTrip(req) - if res != nil && res.Body != nil { - defer res.Body.Close() - } - - if err != nil { - return err - } - if res.StatusCode != http.StatusOK { - // res.Write(os.Stderr) - return fmt.Errorf("Got response code %d from minio", res.StatusCode) - } - return nil -} - -type Item struct { - Key string - LastModified string - Size int64 -} - -type listBucketResults struct { - Contents []*Item - IsTruncated bool - MaxKeys int - Name string // bucket name - Marker string -} - -// GetBucket (List Objects) returns 0 to maxKeys (inclusive) items from the -// provided bucket. Keys before startAt will be skipped. (This is the MINIO -// 'marker' value). If the length of the returned items is equal to -// maxKeys, there is no indication whether or not the returned list is truncated. -func (c *Client) GetBucket(bucket string, startAt string, maxKeys int) (items []*Item, err error) { - if maxKeys < 0 { - return nil, errors.New("invalid negative maxKeys") - } - marker := startAt - for len(items) < maxKeys { - fetchN := maxKeys - len(items) - if fetchN > MAX_OBJECT_LIST { - fetchN = MAX_OBJECT_LIST - } - var bres listBucketResults - - url_ := fmt.Sprintf("%s?marker=%s&max-keys=%d", - c.bucketURL(bucket), url.QueryEscape(marker), fetchN) - - // Try the enumerate three times, since s3 likes to close - // https connections a lot, and Go sucks at dealing with it: - // https://code.google.com/p/go/issues/detail?id=3514 - const maxTries = 5 - for try := 1; try <= maxTries; try++ { - time.Sleep(time.Duration(try-1) * 100 * time.Millisecond) - req := newReq(url_) - res, err := c.transport().RoundTrip(req) - if err != nil { - if try < maxTries { - continue - } - return nil, err - } - if res.StatusCode != http.StatusOK { - if res.StatusCode < 500 { - body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20)) - aerr := &Error{ - Op: "ListBucket", - Code: res.StatusCode, - Body: body, - Header: res.Header, - } - aerr.parseXML() - res.Body.Close() - return nil, aerr - } - } else { - bres = listBucketResults{} - var logbuf bytes.Buffer - err = xml.NewDecoder(io.TeeReader(res.Body, &logbuf)).Decode(&bres) - if err != nil { - log.Printf("Error parsing minio XML response: %v for %q", err, logbuf.Bytes()) - } else if bres.MaxKeys != fetchN || bres.Name != bucket || bres.Marker != marker { - err = fmt.Errorf("Unexpected parse from server: %#v from: %s", bres, logbuf.Bytes()) - log.Print(err) - } - } - res.Body.Close() - if err != nil { - if try < maxTries-1 { - continue - } - log.Print(err) - return nil, err - } - break - } - for _, it := range bres.Contents { - if it.Key == marker && it.Key != startAt { - // Skip first dup on pages 2 and higher. - continue - } - if it.Key < startAt { - return nil, fmt.Errorf("Unexpected response from Minio: item key %q but wanted greater than %q", it.Key, startAt) - } - items = append(items, it) - marker = it.Key - } - if !bres.IsTruncated { - // log.Printf("Not truncated. so breaking. items = %d; len Contents = %d, url = %s", len(items), len(bres.Contents), url_) - break - } - } - return items, nil -} - -func (c *Client) Get(bucket, key string) (body io.ReadCloser, size int64, err error) { - req := newReq(c.keyURL(bucket, key)) - res, err := c.transport().RoundTrip(req) - if err != nil { - return - } - switch res.StatusCode { - case http.StatusOK: - return res.Body, res.ContentLength, nil - case http.StatusNotFound: - res.Body.Close() - return nil, 0, os.ErrNotExist - default: - res.Body.Close() - return nil, 0, fmt.Errorf("Minio HTTP error on GET: %d", res.StatusCode) - } -} - -// GetPartial fetches part of the minio key object in bucket. -// If length is negative, the rest of the object is returned. -// The caller must close rc. -func (c *Client) GetPartial(bucket, key string, offset, length int64) (rc io.ReadCloser, err error) { - if offset < 0 { - return nil, errors.New("invalid negative length") - } - - req := newReq(c.keyURL(bucket, key)) - if length >= 0 { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) - } else { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) - } - - res, err := c.transport().RoundTrip(req) - if err != nil { - return - } - switch res.StatusCode { - case http.StatusOK, http.StatusPartialContent: - return res.Body, nil - case http.StatusNotFound: - res.Body.Close() - return nil, os.ErrNotExist - default: - res.Body.Close() - return nil, fmt.Errorf("Minio HTTP error on GET: %d", res.StatusCode) - } -} - -func NewMinioClient(auth *Auth) (client *Client, err error) { - transport, err := auth.getTlsTransport() - if err != nil { - return nil, err - } - client = &Client{auth.Hostname, transport} - return client, nil -} - -// Error is the type returned by some API operations. -type Error struct { - Op string - Code int // HTTP status code - Body []byte // response body - Header http.Header // response headers - - // UsedEndpoint and MinioCode are the XML response's Endpoint and - // Code fields, respectively. - UseEndpoint string // if a temporary redirect (wrong hostname) - MinioCode string -} - -func (e *Error) Error() string { - if bytes.Contains(e.Body, []byte("")) { - return fmt.Sprintf("minio.%s: status %d: %s", e.Op, e.Code, e.Body) - } - return fmt.Sprintf("minio.%s: status %d", e.Op, e.Code) -} - -func (e *Error) parseXML() { - var xe xmlError - _ = xml.NewDecoder(bytes.NewReader(e.Body)).Decode(&xe) - e.MinioCode = xe.Code - if xe.Code == "TemporaryRedirect" { - e.UseEndpoint = xe.Endpoint - } - if xe.Code == "SignatureDoesNotMatch" { - want, _ := hex.DecodeString(strings.Replace(xe.StringToSignBytes, " ", "", -1)) - log.Printf("Minio SignatureDoesNotMatch. StringToSign should be %d bytes: %q (%x)", len(want), want, want) - } - -} - -// xmlError is the Error response from Minio. -type xmlError struct { - XMLName xml.Name `xml:"Error"` - Code string - Message string - RequestId string - Bucket string - Endpoint string - StringToSignBytes string -} diff --git a/pkg/minio/client_test.go b/pkg/minio/client_test.go deleted file mode 100644 index dc0712df..00000000 --- a/pkg/minio/client_test.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Mini Object Storage, (C) 2014,2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "reflect" - "strings" - "testing" -) - -var tc *Client - -func TestParseBuckets(t *testing.T) { - res := "\nownerIDFieldbobDisplayNamebucketOne2006-06-21T07:04:31.000ZbucketTwo2006-06-21T07:04:32.000Z" - buckets, err := parseListAllMyBuckets(strings.NewReader(res)) - if err != nil { - t.Fatal(err) - } - if g, w := len(buckets), 2; g != w { - t.Errorf("num parsed buckets = %d; want %d", g, w) - } - want := []*Bucket{ - {Name: "bucketOne", CreationDate: "2006-06-21T07:04:31.000Z"}, - {Name: "bucketTwo", CreationDate: "2006-06-21T07:04:32.000Z"}, - } - dump := func(v []*Bucket) { - for i, b := range v { - t.Logf("Bucket #%d: %#v", i, b) - } - } - if !reflect.DeepEqual(buckets, want) { - t.Error("mismatch; GOT:") - dump(buckets) - t.Error("WANT:") - dump(want) - } -} diff --git a/pkg/minio/doc.go b/pkg/minio/doc.go deleted file mode 100644 index c5576b1f..00000000 --- a/pkg/minio/doc.go +++ /dev/null @@ -1 +0,0 @@ -package minio diff --git a/pkg/s3/auth.go b/pkg/s3/auth.go index f543159b..bdd3db1c 100644 --- a/pkg/s3/auth.go +++ b/pkg/s3/auth.go @@ -20,9 +20,12 @@ import ( "bytes" "crypto/hmac" "crypto/sha1" + "crypto/tls" "encoding/base64" "fmt" "io" + "io/ioutil" + "net" //"log" "net/http" "net/url" @@ -37,13 +40,20 @@ type Auth struct { AccessKey string SecretKey string - // Hostname is the S3 hostname to use. - // If empty, the standard US region of "s3.amazonaws.com" is - // used. + // If empty, the standard US region of "s3.amazonaws.com" is used. Hostname string + + // Used for SSL transport layer + CertPEM string + KeyPEM string } -const standardUSRegionAWS = "https://s3.amazonaws.com" +type TlsConfig struct { + CertPEMBlock []byte + KeyPEMBlock []byte +} + +const standardUSRegionAWS = "s3.amazonaws.com" func (a *Auth) hostname() string { // Prefix with https for Amazon hostnames @@ -54,7 +64,54 @@ func (a *Auth) hostname() string { return "http://" + a.Hostname } } - return standardUSRegionAWS + return "https://" + standardUSRegionAWS +} + +func (a *Auth) loadKeys(cert string, key string) (*TlsConfig, error) { + certBlock, err := ioutil.ReadFile(cert) + if err != nil { + return nil, err + } + keyBlock, err := ioutil.ReadFile(key) + if err != nil { + return nil, err + } + t := &TlsConfig{} + t.CertPEMBlock = certBlock + t.KeyPEMBlock = keyBlock + return t, nil +} + +func (a *Auth) getTlsTransport() (*http.Transport, error) { + if a.CertPEM == "" || a.KeyPEM == "" { + return &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + TLSHandshakeTimeout: 10 * time.Second, + }, nil + } + + tlsconfig, err := a.loadKeys(a.CertPEM, a.KeyPEM) + if err != nil { + return nil, err + } + var cert tls.Certificate + cert, err = tls.X509KeyPair(tlsconfig.CertPEMBlock, tlsconfig.KeyPEMBlock) + if err != nil { + return nil, err + } + + // Setup HTTPS client + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + InsecureSkipVerify: true, + } + + tlsConfig.BuildNameToCertificate() + transport := &http.Transport{TLSClientConfig: tlsConfig} + return transport, nil } func (a *Auth) SignRequest(req *http.Request) { diff --git a/s3-api-options.go b/s3-api-options.go index bd983a9d..75fa6394 100644 --- a/s3-api-options.go +++ b/s3-api-options.go @@ -111,5 +111,5 @@ var Configure = cli.Command{ } const ( - S3_AUTH = ".auth" + AUTH = ".auth" )