Merge pull request #600 from harshavardhana/pr_out_combine_get_and_getpartial_to_be_getobject_with_offset_and_length_parameters

cmd-access.go

@@ -91,12 +91,12 @@ func doUpdateAccessCmd(targetURL, targetACL string, targetConfig *hostConfig, de
 }
 
 func doUpdateAccess(clnt client.Client, targetURL, targetACL string) (string, error) {
-	err := clnt.PutBucketACL(targetACL)
+	err := clnt.SetBucketACL(targetACL)
 	for i := 0; i < globalMaxRetryFlag && err != nil && isValidRetry(err); i++ {
 		fmt.Println(console.Retry("Retrying ... %d", i))
 		// Progressively longer delays
 		time.Sleep(time.Duration(i*i) * time.Second)
-		err = clnt.PutBucketACL(targetACL)
+		err = clnt.SetBucketACL(targetACL)
 	}
 	if err != nil {
 		err := iodine.New(err, nil)

cmd-cat.go

@@ -73,7 +73,7 @@ func doCatCmd(sourceURLConfigMap map[string]*hostConfig, debug bool) (string, er
 		if err != nil {
 			return "Unable to create client: " + url, iodine.New(err, nil)
 		}
-		reader, size, sourceMd5, err := sourceClnt.Get()
+		reader, size, sourceMd5, err := sourceClnt.GetObject(0, 0)
 		if err != nil {
 			return "Unable to retrieve file: " + url, iodine.New(err, nil)
 		}

cmd-mb.go

@@ -86,14 +86,14 @@ func doMakeBucketCmd(targetURL string, targetConfig *hostConfig, debug bool) (st
 	return doMakeBucket(clnt, targetURL)
 }
 
-// doMakeBucket - wrapper around PutBucket() API
+// doMakeBucket - wrapper around CreateBucket() API
 func doMakeBucket(clnt client.Client, targetURL string) (string, error) {
-	err := clnt.PutBucket()
+	err := clnt.CreateBucket()
 	for i := 0; i < globalMaxRetryFlag && err != nil && isValidRetry(err); i++ {
 		fmt.Println(console.Retry("Retrying ... %d", i))
 		// Progressively longer delays
 		time.Sleep(time.Duration(i*i) * time.Second)
-		err = clnt.PutBucket()
+		err = clnt.CreateBucket()
 	}
 	if err != nil {
 		err := iodine.New(err, nil)

common-methods.go

@@ -26,14 +26,6 @@ import (
 	"github.com/minio-io/minio/pkg/iodine"
 )
 
-/*
-type sourceReader struct {
-	reader io.ReadCloser
-	length int64
-	md5hex string
-}
-*/
-
 // getSourceReader -
 func getSourceReader(sourceURL string, sourceConfig *hostConfig) (reader io.ReadCloser, length int64, md5hex string, err error) {
 	sourceClnt, err := getNewClient(sourceURL, sourceConfig, globalDebugFlag)
@@ -43,7 +35,7 @@ func getSourceReader(sourceURL string, sourceConfig *hostConfig) (reader io.Read
 	if _, err := sourceClnt.Stat(); err != nil {
 		return nil, 0, "", iodine.New(err, map[string]string{"failedURL": sourceURL})
 	}
-	return sourceClnt.Get()
+	return sourceClnt.GetObject(0, 0)
 }
 
 // getTargetWriter -
@@ -52,7 +44,7 @@ func getTargetWriter(targetURL string, targetConfig *hostConfig, md5hex string,
 	if err != nil {
 		return nil, iodine.New(err, nil)
 	}
-	return targetClnt.Put(md5hex, length)
+	return targetClnt.CreateObject(md5hex, length)
 }
 
 // getNewClient gives a new client interface

pkg/client/client.go

@@ -24,46 +24,18 @@ import (
 
 // Client - client interface
 type Client interface {
-	//	MultipartUpload - TODO
-
 	// Common operations
 	Stat() (content *Content, err error)
 	List() <-chan ContentOnChannel
 	ListRecursive() <-chan ContentOnChannel
 
 	// Bucket operations
-	PutBucket() error
-	PutBucketACL(acl string) error
+	CreateBucket() error
+	SetBucketACL(acl string) error
 
 	// Object operations
-	Get() (body io.ReadCloser, size int64, md5 string, err error)
-	GetPartial(offset, length int64) (body io.ReadCloser, size int64, md5 string, err error)
-	Put(md5 string, size int64) (io.WriteCloser, error)
-}
-
-// MultipartUpload - multi part upload interface
-//type MultipartUpload interface {
-//	InitiateMultiPartUpload() (objectID string, err error)
-//	UploadPart(uploadID string, body io.ReadSeeker, contentLength, partNumber int64) (md5hex string, err error)
-//	CompleteMultiPartUpload(uploadID string) (location, md5hex string, err error)
-//	AbortMultiPartUpload(uploadID string) error
-//	ListParts(uploadID string) (contents *PartContents, err error)
-//}
-
-// Part - part xml response
-type Part struct {
-	ETag         string
-	LastModified time.Time
-	PartNumber   int64
-	Size         int64
-}
-
-// PartContents - part xml contents response
-type PartContents struct {
-	Key         string
-	UploadID    string
-	IsTruncated bool
-	Parts       []*Part
+	GetObject(offset, length int64) (body io.ReadCloser, size int64, md5 string, err error)
+	CreateObject(md5 string, size int64) (io.WriteCloser, error)
 }
 
 // ContentOnChannel - List contents on channel

pkg/client/fs/fs.go

@@ -57,15 +57,7 @@ func (f *fsClient) fsStat() (os.FileInfo, error) {
 	return st, nil
 }
 
-// Get - download an object from bucket
-func (f *fsClient) Get() (io.ReadCloser, int64, string, error) {
-	content, err := f.getFSMetadata()
-	if err != nil {
-		return nil, 0, "", iodine.New(err, nil)
-	}
-	if content.FileType.IsDir() {
-		return nil, 0, "", iodine.New(ISFolder{path: f.path}, nil)
-	}
+func (f *fsClient) get(content *client.Content) (io.ReadCloser, int64, string, error) {
 	body, err := os.Open(f.path)
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
@@ -85,8 +77,8 @@ func (f *fsClient) Get() (io.ReadCloser, int64, string, error) {
 	return body, content.Size, md5Str, nil
 }
 
-// GetPartial - download a partial object from bucket
-func (f *fsClient) GetPartial(offset, length int64) (io.ReadCloser, int64, string, error) {
+// GetObject download an full or part object from bucket
+func (f *fsClient) GetObject(offset, length int64) (io.ReadCloser, int64, string, error) {
 	if offset < 0 {
 		return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil)
 	}
@@ -100,6 +92,9 @@ func (f *fsClient) GetPartial(offset, length int64) (io.ReadCloser, int64, strin
 	if offset > content.Size || (offset+length-1) > content.Size {
 		return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil)
 	}
+	if offset == 0 && length == 0 {
+		return f.get(content)
+	}
 	body, err := os.Open(f.path)
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
@@ -258,8 +253,8 @@ func aclToPerm(acl string) os.FileMode {
 	}
 }
 
-// PutBucket - create a new bucket
-func (f *fsClient) PutBucket() error {
+// CreateBucket - create a new bucket
+func (f *fsClient) CreateBucket() error {
 	err := os.MkdirAll(f.path, 0775)
 	if err != nil {
 		return iodine.New(err, nil)
@@ -267,8 +262,8 @@ func (f *fsClient) PutBucket() error {
 	return nil
 }
 
-// PutBucket - create a new bucket
-func (f *fsClient) PutBucketACL(acl string) error {
+// SetBucketACL - create a new bucket
+func (f *fsClient) SetBucketACL(acl string) error {
 	if !isValidBucketACL(acl) {
 		return iodine.New(errors.New("invalid acl"), nil)
 	}

pkg/client/fs/fs_put.go

@@ -29,8 +29,8 @@ import (
 	"github.com/minio-io/minio/pkg/iodine"
 )
 
-// Put - upload new object to bucket
-func (f *fsClient) Put(md5HexString string, size int64) (io.WriteCloser, error) {
+// CreateObject - upload new object to bucket
+func (f *fsClient) CreateObject(md5HexString string, size int64) (io.WriteCloser, error) {
 	r, w := io.Pipe()
 	blockingWriter := client.NewBlockingWriteCloser(w)
 	go func() {

pkg/client/fs/fs_test.go

@@ -50,7 +50,7 @@ func (s *MySuite) TestList(c *C) {
 	etag := base64.StdEncoding.EncodeToString(binarySum[:])
 	dataLen := int64(len(data))
 
-	writer, err := fsc.Put(etag, dataLen)
+	writer, err := fsc.CreateObject(etag, dataLen)
 	c.Assert(err, IsNil)
 
 	size, err := io.CopyN(writer, bytes.NewBufferString(data), dataLen)
@@ -60,7 +60,7 @@ func (s *MySuite) TestList(c *C) {
 	objectPath = filepath.Join(root, "object2")
 	fsc = New(objectPath)
 
-	writer, err = fsc.Put(etag, dataLen)
+	writer, err = fsc.CreateObject(etag, dataLen)
 	c.Assert(err, IsNil)
 
 	size, err = io.CopyN(writer, bytes.NewBufferString(data), dataLen)
@@ -83,7 +83,7 @@ func (s *MySuite) TestPutBucket(c *C) {
 
 	bucketPath := filepath.Join(root, "bucket")
 	fsc := New(bucketPath)
-	err = fsc.PutBucket()
+	err = fsc.CreateBucket()
 	c.Assert(err, IsNil)
 }
 
@@ -94,7 +94,7 @@ func (s *MySuite) TestStatBucket(c *C) {
 
 	bucketPath := filepath.Join(root, "bucket")
 	fsc := New(bucketPath)
-	err = fsc.PutBucket()
+	err = fsc.CreateBucket()
 	c.Assert(err, IsNil)
 	_, err = fsc.Stat()
 	c.Assert(err, IsNil)
@@ -107,10 +107,10 @@ func (s *MySuite) TestPutBucketACL(c *C) {
 
 	bucketPath := filepath.Join(root, "bucket")
 	fsc := New(bucketPath)
-	err = fsc.PutBucket()
+	err = fsc.CreateBucket()
 	c.Assert(err, IsNil)
 
-	err = fsc.PutBucketACL("private")
+	err = fsc.SetBucketACL("private")
 	c.Assert(err, IsNil)
 }
 
@@ -127,7 +127,7 @@ func (s *MySuite) TestPutObject(c *C) {
 	etag := base64.StdEncoding.EncodeToString(binarySum[:])
 	dataLen := int64(len(data))
 
-	writer, err := fsc.Put(etag, dataLen)
+	writer, err := fsc.CreateObject(etag, dataLen)
 	c.Assert(err, IsNil)
 
 	size, err := io.CopyN(writer, bytes.NewBufferString(data), dataLen)
@@ -148,13 +148,13 @@ func (s *MySuite) TestGetObject(c *C) {
 	etag := hex.EncodeToString(binarySum[:])
 	dataLen := int64(len(data))
 
-	writer, err := fsc.Put(etag, dataLen)
+	writer, err := fsc.CreateObject(etag, dataLen)
 	c.Assert(err, IsNil)
 
 	_, err = io.CopyN(writer, bytes.NewBufferString(data), dataLen)
 	c.Assert(err, IsNil)
 
-	reader, size, md5Sum, err := fsc.Get()
+	reader, size, md5Sum, err := fsc.GetObject(0, 0)
 	c.Assert(err, IsNil)
 	var results bytes.Buffer
 	c.Assert(etag, Equals, md5Sum)
@@ -177,7 +177,7 @@ func (s *MySuite) TestStat(c *C) {
 	etag := base64.StdEncoding.EncodeToString(binarySum[:])
 	dataLen := int64(len(data))
 
-	writer, err := fsc.Put(etag, dataLen)
+	writer, err := fsc.CreateObject(etag, dataLen)
 	c.Assert(err, IsNil)
 
 	_, err = io.CopyN(writer, bytes.NewBufferString(data), dataLen)

pkg/client/s3/bucket.go

@@ -42,8 +42,8 @@ func isValidBucketACL(acl string) bool {
 
 /// Bucket API operations
 
-// PutBucket - create a new bucket
-func (c *s3Client) PutBucket() error {
+// CreateBucket - create a new bucket
+func (c *s3Client) CreateBucket() error {
 	_, object := c.url2BucketAndObject()
 	if object != "" {
 		return iodine.New(InvalidQueryURL{URL: ""}, nil)
@@ -76,7 +76,8 @@ func (c *s3Client) PutBucket() error {
 	return nil
 }
 
-func (c *s3Client) PutBucketACL(acl string) error {
+// SetBucketACL add canned acl's on a bucket
+func (c *s3Client) SetBucketACL(acl string) error {
 	if !isValidBucketACL(acl) {
 		return iodine.New(InvalidACL{ACL: acl}, nil)
 	}

pkg/client/s3/object_get.go

@@ -56,37 +56,18 @@ func (c *s3Client) get() (*http.Request, error) {
 	return req, nil
 }
 
-// Get - download a requested object from a given bucket
-func (c *s3Client) Get() (body io.ReadCloser, size int64, md5 string, err error) {
-	req, err := c.get()
-	if err != nil {
-		return nil, 0, "", iodine.New(err, nil)
-	}
-	if c.AccessKeyID != "" && c.SecretAccessKey != "" {
-		c.signRequest(req, c.Host)
-	}
-	res, err := c.Transport.RoundTrip(req)
-	if err != nil {
-		return nil, 0, "", iodine.New(err, nil)
-	}
-
-	if res.StatusCode != http.StatusOK {
-		return nil, 0, "", iodine.New(ResponseToError(res), nil)
-	}
-	md5sum := strings.Trim(res.Header.Get("ETag"), "\"") // trim off the erroneous double quotes
-	return res.Body, res.ContentLength, md5sum, nil
-}
-
-// GetPartial fetches part of the s3 object in bucket.
+// GetObject fetches full object or part of the s3 object in bucket.
 // If length is negative, the rest of the object is returned.
-func (c *s3Client) GetPartial(offset, length int64) (body io.ReadCloser, size int64, md5 string, err error) {
+func (c *s3Client) GetObject(offset, length int64) (body io.ReadCloser, size int64, md5 string, err error) {
 	req, err := c.get()
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
 	}
-	req, err = c.setRange(req, offset, length)
-	if err != nil {
-		return nil, 0, "", iodine.New(err, nil)
+	if offset == 0 && length == 0 {
+		req, err = c.setRange(req, offset, length)
+		if err != nil {
+			return nil, 0, "", iodine.New(err, nil)
+		}
 	}
 	if c.AccessKeyID != "" && c.SecretAccessKey != "" {
 		c.signRequest(req, c.Host)
@@ -95,9 +76,11 @@ func (c *s3Client) GetPartial(offset, length int64) (body io.ReadCloser, size in
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
 	}
+
 	switch res.StatusCode {
 	case http.StatusOK, http.StatusPartialContent:
-		return res.Body, res.ContentLength, res.Header.Get("ETag"), nil
+		md5sum := strings.Trim(res.Header.Get("ETag"), "\"") // trim off the erroneous double quotes
+		return res.Body, res.ContentLength, md5sum, nil
 	default:
 		return nil, 0, "", iodine.New(ResponseToError(res), nil)
 	}

pkg/client/s3/object_put.go

@@ -46,8 +46,8 @@ func (c *s3Client) put(size int64) (*http.Request, error) {
 	return req, nil
 }
 
-// Put - upload new object to bucket
-func (c *s3Client) Put(md5HexString string, size int64) (io.WriteCloser, error) {
+// CreateObject - upload new object to bucket
+func (c *s3Client) CreateObject(md5HexString string, size int64) (io.WriteCloser, error) {
 	// if size is negative
 	if size < 0 {
 		return nil, iodine.New(client.InvalidArgument{Err: errors.New("invalid argument")}, nil)