Add `share` command leveraging new PresignedURL API from minio-go

Godeps/Godeps.json

@@ -29,8 +29,8 @@
 		},
 		{
 			"ImportPath": "github.com/minio/minio-go",
-			"Comment": "v0.2.0-19-g44dd766",
+			"Comment": "v0.2.0-21-gd9184e5",
-			"Rev": "44dd766b3563b2f95cc7cf03c4789b28254f3d8a"
+			"Rev": "d9184e5834efb7cd4a24df4d6ebab9b51dad4d9c"
 		},
 		{
 			"ImportPath": "github.com/minio/minio/pkg/probe",

Godeps/_workspace/src/github.com/minio/minio-go/api-v1.go

@@ -524,6 +524,42 @@ func (a apiV1) putObject(bucket, object, contentType string, md5SumBytes []byte,
 	return metadata, nil
 }
 
+func (a apiV1) presignedGetObjectRequest(bucket, object string, expires, offset, length int64) (*request, error) {
+	encodedObject, err := urlEncodeName(object)
+	if err != nil {
+		return nil, err
+	}
+	op := &operation{
+		HTTPServer: a.config.Endpoint,
+		HTTPMethod: "GET",
+		HTTPPath:   separator + bucket + separator + encodedObject,
+	}
+	r, err := newPresignedRequest(op, a.config, strconv.FormatInt(expires, 10))
+	if err != nil {
+		return nil, err
+	}
+	switch {
+	case length > 0 && offset > 0:
+		r.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1))
+	case offset > 0 && length == 0:
+		r.Set("Range", fmt.Sprintf("bytes=%d-", offset))
+	case length > 0 && offset == 0:
+		r.Set("Range", fmt.Sprintf("bytes=-%d", length))
+	}
+	return r, nil
+}
+
+func (a apiV1) presignedGetObject(bucket, object string, expires, offset, length int64) (string, error) {
+	if err := invalidArgumentError(object); err != nil {
+		return "", err
+	}
+	req, err := a.presignedGetObjectRequest(bucket, object, expires, offset, length)
+	if err != nil {
+		return "", err
+	}
+	return req.PreSignV4(), nil
+}
+
 // getObjectRequest wrapper creates a new getObject request
 func (a apiV1) getObjectRequest(bucket, object string, offset, length int64) (*request, error) {
 	encodedObject, err := urlEncodeName(object)

Godeps/_workspace/src/github.com/minio/minio-go/api-v2.go

@@ -36,6 +36,9 @@ type API interface {
 
 	// Object Read/Write/Stat operations
 	ObjectAPI
+
+	// Presigned API
+	PresignedAPI
 }
 
 // BucketAPI - bucket specific Read/Write/Stat interface
@@ -65,6 +68,12 @@ type ObjectAPI interface {
 	DropIncompleteUploads(bucket, object string) <-chan error
 }
 
+// PresignedAPI - object specific for now
+type PresignedAPI interface {
+	PresignedGetObject(bucket, object string, expires time.Duration) (string, error)
+	PresignedGetPartialObject(bucket, object string, expires time.Duration, offset, length int64) (string, error)
+}
+
 // BucketStatCh - bucket metadata over read channel
 type BucketStatCh struct {
 	Stat BucketStat
@@ -214,6 +223,25 @@ func New(config Config) (API, error) {
 
 /// Object operations
 
+/// Expires maximum is 7days - ie. 604800 and minimum is 1
+
+// PresignedGetObject get a presigned URL to retrieve an object for third party apps
+func (a apiV2) PresignedGetObject(bucket, object string, expires time.Duration) (string, error) {
+	expireSeconds := int64(expires / time.Second)
+	if expireSeconds < 1 || expireSeconds > 604800 {
+		return "", invalidArgumentError("")
+	}
+	return a.presignedGetObject(bucket, object, expireSeconds, 0, 0)
+}
+
+func (a apiV2) PresignedGetPartialObject(bucket, object string, expires time.Duration, offset, length int64) (string, error) {
+	expireSeconds := int64(expires / time.Second)
+	if expireSeconds < 1 || expireSeconds > 604800 {
+		return "", invalidArgumentError("")
+	}
+	return a.presignedGetObject(bucket, object, expireSeconds, offset, length)
+}
+
 // GetObject retrieve object
 
 // Downloads full object with no ranges, if you need ranges use GetPartialObject

Godeps/_workspace/src/github.com/minio/minio-go/examples/s3/presignedgetobject.go

@@ -0,0 +1,43 @@
+// +build ignore
+
+/*
+ * Minio Go Library for Amazon S3 compatible cloud storage (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"log"
+	"time"
+
+	"github.com/minio/minio-go"
+)
+
+func main() {
+	config := minio.Config{
+		AccessKeyID:     "YOUR-ACCESS-KEY-HERE",
+		SecretAccessKey: "YOUR-PASSWORD-HERE",
+		Endpoint:        "https://s3.amazonaws.com",
+	}
+	s3Client, err := minio.New(config)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	string, err := s3Client.PresignedGetObject("mybucket", "myobject", time.Duration(1000)*time.Second)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	log.Println(string)
+}

Godeps/_workspace/src/github.com/minio/minio-go/request.go

@@ -40,6 +40,7 @@ type request struct {
 	req     *http.Request
 	config  *Config
 	body    io.ReadSeeker
+	expires string
 }
 
 const (
@@ -157,9 +158,9 @@ func httpNewRequest(method, urlStr string, body io.Reader) (*http.Request, error
 		if err != nil {
 			return nil, err
 		}
-		uEncoded.Opaque = separator + bucketName + separator + encodedObjectName
+		uEncoded.Opaque = "//" + uEncoded.Host + separator + bucketName + separator + encodedObjectName
 	} else {
-		uEncoded.Opaque = separator + bucketName
+		uEncoded.Opaque = "//" + uEncoded.Host + separator + bucketName
 	}
 	rc, ok := body.(io.ReadCloser)
 	if !ok && body != nil {
@@ -188,6 +189,39 @@ func httpNewRequest(method, urlStr string, body io.Reader) (*http.Request, error
 	return req, nil
 }
 
+func newPresignedRequest(op *operation, config *Config, expires string) (*request, error) {
+	// if no method default to POST
+	method := op.HTTPMethod
+	if method == "" {
+		method = "POST"
+	}
+
+	u := op.getRequestURL(*config)
+
+	// get a new HTTP request, for the requested method
+	req, err := httpNewRequest(method, u, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// set UserAgent
+	req.Header.Set("User-Agent", config.userAgent)
+
+	// set Accept header for response encoding style, if available
+	if config.AcceptType != "" {
+		req.Header.Set("Accept", config.AcceptType)
+	}
+
+	// save for subsequent use
+	r := new(request)
+	r.config = config
+	r.expires = expires
+	r.req = req
+	r.body = nil
+
+	return r, nil
+}
+
 // newRequest - instantiate a new request
 func newRequest(op *operation, config *Config, body io.ReadSeeker) (*request, error) {
 	// if no method default to POST
@@ -249,6 +283,14 @@ func (r *request) Do() (resp *http.Response, err error) {
 	return transport.RoundTrip(r.req)
 }
 
+func (r *request) SetQuery(key, value string) {
+	r.req.URL.Query().Set(key, value)
+}
+
+func (r *request) AddQuery(key, value string) {
+	r.req.URL.Query().Add(key, value)
+}
+
 // Set - set additional headers if any
 func (r *request) Set(key, value string) {
 	r.req.Header.Set(key, value)
@@ -263,6 +305,8 @@ func (r *request) Get(key string) string {
 func (r *request) getHashedPayload() string {
 	hash := func() string {
 		switch {
+		case r.expires != "":
+			return "UNSIGNED-PAYLOAD"
 		case r.body == nil:
 			return hex.EncodeToString(sum256([]byte{}))
 		default:
@@ -271,7 +315,10 @@ func (r *request) getHashedPayload() string {
 		}
 	}
 	hashedPayload := hash()
-	r.req.Header.Add("x-amz-content-sha256", hashedPayload)
+	r.req.Header.Set("X-Amz-Content-Sha256", hashedPayload)
+	if hashedPayload == "UNSIGNED-PAYLOAD" {
+		r.req.Header.Set("X-Amz-Content-Sha256", "")
+	}
 	return hashedPayload
 }
 
@@ -336,9 +383,13 @@ func (r *request) getSignedHeaders() string {
 //
 func (r *request) getCanonicalRequest(hashedPayload string) string {
 	r.req.URL.RawQuery = strings.Replace(r.req.URL.Query().Encode(), "+", "%20", -1)
+
+	// get path URI from Opaque
+	uri := strings.Join(strings.Split(r.req.URL.Opaque, "/")[3:], "/")
+
 	canonicalRequest := strings.Join([]string{
 		r.req.Method,
-		r.req.URL.Opaque,
+		"/" + uri,
 		r.req.URL.RawQuery,
 		r.getCanonicalHeaders(),
 		r.getSignedHeaders(),
@@ -381,22 +432,45 @@ func (r *request) getSignature(signingKey []byte, stringToSign string) string {
 	return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
 }
 
+// Presign the request, in accordance with - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+func (r *request) PreSignV4() string {
+	r.SignV4()
+	return r.req.URL.String()
+}
+
 // SignV4 the request before Do(), in accordance with - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
 func (r *request) SignV4() {
+	query := r.req.URL.Query()
+	if r.expires != "" {
+		query.Set("X-Amz-Algorithm", authHeader)
+	}
 	t := time.Now().UTC()
 	// Add date if not present
-	if date := r.Get("Date"); date == "" {
+	if r.expires != "" {
-		r.Set("x-amz-date", t.Format(iso8601Format))
+		query.Set("X-Amz-Date", t.Format(iso8601Format))
+		query.Set("X-Amz-Expires", r.expires)
+	} else {
+		r.Set("X-Amz-Date", t.Format(iso8601Format))
 	}
 
 	hashedPayload := r.getHashedPayload()
 	signedHeaders := r.getSignedHeaders()
-	canonicalRequest := r.getCanonicalRequest(hashedPayload)
+	if r.expires != "" {
+		query.Set("X-Amz-SignedHeaders", signedHeaders)
+	}
 	scope := r.getScope(t)
+	if r.expires != "" {
+		query.Set("X-Amz-Credential", r.config.AccessKeyID+"/"+scope)
+		r.req.URL.RawQuery = query.Encode()
+	}
+	canonicalRequest := r.getCanonicalRequest(hashedPayload)
 	stringToSign := r.getStringToSign(canonicalRequest, t)
 	signingKey := r.getSigningKey(t)
 	signature := r.getSignature(signingKey, stringToSign)
 
+	if r.expires != "" {
+		r.req.URL.RawQuery += "&X-Amz-Signature=" + signature
+	} else {
 		// final Authorization header
 		parts := []string{
 			authHeader + " Credential=" + r.config.AccessKeyID + "/" + scope,
@@ -405,4 +479,5 @@ func (r *request) SignV4() {
 		}
 		auth := strings.Join(parts, ", ")
 		r.Set("Authorization", auth)
+	}
 }

main.go

@@ -111,6 +111,7 @@ func registerApp() *cli.App {
 	registerCmd(cpCmd)      // copy objects and files from multiple sources to single destination
 	registerCmd(mirrorCmd)  // mirror objects and files from single source to multiple destinations
 	registerCmd(sessionCmd) // session handling for resuming copy and mirror operations
+	registerCmd(shareCmd)   // share any given url for third party access
 	registerCmd(diffCmd)    // compare two objects
 	registerCmd(accessCmd)  // set permissions [public, private, readonly, authenticated] for buckets and folders.
 	registerCmd(configCmd)  // generate configuration "/home/harsha/.mc/config.json" file.

pkg/client/client.go

@@ -35,6 +35,7 @@ type Client interface {
 	SetBucketACL(acl string) *probe.Error
 
 	// Object operations
+	PresignedGetObject(expires time.Duration, offset, length int64) (string, *probe.Error)
 	GetObject(offset, length int64) (body io.ReadCloser, size int64, err *probe.Error)
 	PutObject(size int64, data io.Reader) *probe.Error
 

pkg/client/errors.go

@@ -95,6 +95,15 @@ type GenericObjectError struct {
 	Object string
 }
 
+// ObjectAlreadyExists - typed return for MethodNotAllowed
+type ObjectAlreadyExists struct {
+	Object string
+}
+
+func (e ObjectAlreadyExists) Error() string {
+	return "Object #" + e.Object + " already exists."
+}
+
 // ObjectNotFound - object requested does not exist
 type ObjectNotFound GenericObjectError
 

pkg/client/fs/fs.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"io/ioutil"
 
@@ -121,6 +122,10 @@ func (f *fsClient) get() (io.ReadCloser, int64, *probe.Error) {
 	}
 }
 
+func (f *fsClient) PresignedGetObject(expires time.Duration, offset, length int64) (string, *probe.Error) {
+	return "", probe.New(client.APINotImplemented{API: "PresignedGetObject"})
+}
+
 // GetObject download an full or part object from bucket
 // getobject returns a reader, length and nil for no errors
 // with errors getobject will return nil reader, length and typed errors

pkg/client/s3/s3.go

@@ -101,13 +101,14 @@ func (c *s3Client) GetObject(offset, length int64) (io.ReadCloser, int64, *probe
 	return reader, metadata.Size, nil
 }
 
-// ObjectAlreadyExists - typed return for MethodNotAllowed
+// PresignedGetObject - get a presigned usable get object url to share
-type ObjectAlreadyExists struct {
+func (c *s3Client) PresignedGetObject(expires time.Duration, offset, length int64) (string, *probe.Error) {
-	Object string
+	bucket, object := c.url2BucketAndObject()
-}
+	presignedURL, err := c.api.PresignedGetPartialObject(bucket, object, expires, offset, length)
-
+	if err != nil {
-func (e ObjectAlreadyExists) Error() string {
+		return "", probe.New(err)
-	return "Object #" + e.Object + " already exists."
+	}
+	return presignedURL, nil
 }
 
 // PutObject - put object
@@ -121,7 +122,7 @@ func (c *s3Client) PutObject(size int64, data io.Reader) *probe.Error {
 		errResponse := minio.ToErrorResponse(err)
 		if errResponse != nil {
 			if errResponse.Code == "MethodNotAllowed" {
-				return probe.New(ObjectAlreadyExists{Object: object})
+				return probe.New(client.ObjectAlreadyExists{Object: object})
 			}
 		}
 		return probe.New(err)

share-main.go

@@ -0,0 +1,137 @@
+/*
+ * Minio Client (C) 2014, 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/minio/cli"
+	"github.com/minio/mc/pkg/client"
+	"github.com/minio/mc/pkg/console"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// Help message.
+var shareCmd = cli.Command{
+	Name:   "share",
+	Usage:  "Share presigned URLs from cloud storage",
+	Action: runShareCmd,
+	CustomHelpTemplate: `NAME:
+   mc {{.Name}} - {{.Usage}}
+
+USAGE:
+   mc {{.Name}} TARGET [TARGET...] {{if .Description}}
+
+DESCRIPTION:
+   {{.Description}}{{end}}{{if .Flags}}
+
+FLAGS:
+   {{range .Flags}}{{.}}
+   {{end}}{{ end }}
+
+EXAMPLES:
+   1. Generate presigned url for an object with expiration of 10minutes
+      $ mc {{.Name}} https://s3.amazonaws.com/backup/2006-Mar-1/backup.tar.gz expire 10m
+
+   2. Generate presigned url for all objects at given path
+      $ mc {{.Name}} https://s3.amazonaws.com/backup... expire 20m
+
+`,
+}
+
+// runShareCmd - is a handler for mc share command
+func runShareCmd(ctx *cli.Context) {
+	if !ctx.Args().Present() || ctx.Args().First() == "help" {
+		cli.ShowCommandHelpAndExit(ctx, "share", 1) // last argument is exit code
+	}
+	args := ctx.Args()
+	config := mustGetMcConfig()
+	for _, arg := range args {
+		targetURL, err := getExpandedURL(arg, config.Aliases)
+		Fatal(err)
+		// if recursive strip off the "..."
+		newTargetURL := stripRecursiveURL(targetURL)
+		Fatal(doShareCmd(newTargetURL, isURLRecursive(targetURL)))
+	}
+}
+
+// doShareCmd share files from target
+func doShareCmd(targetURL string, recursive bool) *probe.Error {
+	clnt, err := target2Client(targetURL)
+	if err != nil {
+		return err.Trace()
+	}
+	err = doShare(clnt, recursive)
+	if err != nil {
+		return err.Trace()
+	}
+	return nil
+}
+
+func path2Bucket(u *client.URL) (bucketName string) {
+	pathSplits := strings.SplitN(u.Path, "?", 2)
+	splits := strings.SplitN(pathSplits[0], string(u.Separator), 3)
+	switch len(splits) {
+	case 0, 1:
+		bucketName = ""
+	case 2:
+		bucketName = splits[1]
+	case 3:
+		bucketName = splits[1]
+	}
+	return bucketName
+}
+
+func doShare(clnt client.Client, recursive bool) *probe.Error {
+	var err *probe.Error
+	for contentCh := range clnt.List(recursive) {
+		if contentCh.Err != nil {
+			switch contentCh.Err.ToError().(type) {
+			// handle this specifically for filesystem
+			case client.ISBrokenSymlink:
+				Error(contentCh.Err)
+				continue
+			}
+			if os.IsNotExist(contentCh.Err.ToError()) || os.IsPermission(contentCh.Err.ToError()) {
+				Error(contentCh.Err)
+				continue
+			}
+			err = contentCh.Err
+			break
+		}
+		if err != nil {
+			return err.Trace()
+		}
+		targetParser := clnt.URL()
+		targetParser.Path = path2Bucket(targetParser) + string(targetParser.Separator) + contentCh.Content.Name
+		newClnt, err := url2Client(targetParser.String())
+		if err != nil {
+			return err.Trace()
+		}
+		// TODO enable expiry
+		expire := time.Duration(1000) * time.Second
+		presignedURL, err := newClnt.PresignedGetObject(time.Duration(1000)*time.Second, 0, 0)
+		if err != nil {
+			return err.Trace()
+		}
+		console.PrintC(fmt.Sprintf("Succesfully generated shared URL with expiry %s, please copy: %s\n", expire, presignedURL))
+	}
+	return nil
+}