Move from file to fs and finish ListObjects() - add test stubs

cmd-common.go

@@ -17,12 +17,12 @@
 package main
 
 import (
-	"net/http"
 	"path"
 	"time"
 
 	"github.com/cheggaaa/pb"
 	"github.com/minio-io/mc/pkg/client"
+	"github.com/minio-io/mc/pkg/client/fs"
 	"github.com/minio-io/mc/pkg/client/s3"
 	"github.com/minio-io/mc/pkg/console"
 	"github.com/minio-io/minio/pkg/iodine"
@@ -60,27 +60,23 @@ func mustGetMcBashCompletionFilename() string {
 // NewClient - get new client
 // TODO refactor this to be more testable
 func getNewClient(debug bool, urlStr string) (clnt client.Client, err error) {
-	hostCfg, err := getHostConfig(urlStr)
-	if err != nil {
-		return nil, iodine.New(err, nil)
-	}
-
-	var auth s3.Auth
-	auth.AccessKeyID = hostCfg.Auth.AccessKeyID
-	auth.SecretAccessKey = hostCfg.Auth.SecretAccessKey
 	uType, err := getURLType(urlStr)
 	if err != nil {
 		return nil, iodine.New(err, nil)
 	}
-
 	switch uType {
 	case urlObjectStorage: // Minio and S3 compatible object storage
-		traceTransport := s3.GetNewTraceTransport(s3.NewTrace(false, true, nil), http.DefaultTransport)
-		if debug {
-			clnt = s3.GetNewClient(&auth, urlStr, traceTransport)
-		} else {
-			clnt = s3.GetNewClient(&auth, urlStr, http.DefaultTransport)
+		hostCfg, err := getHostConfig(urlStr)
+		if err != nil {
+			return nil, iodine.New(err, nil)
 		}
+		var auth s3.Auth
+		auth.AccessKeyID = hostCfg.Auth.AccessKeyID
+		auth.SecretAccessKey = hostCfg.Auth.SecretAccessKey
+		clnt = s3.GetNewClient(urlStr, &auth, debug)
+		return clnt, nil
+	case urlFile:
+		clnt = fs.GetNewClient(urlStr)
 		return clnt, nil
 	default:
 		return nil, iodine.New(errUnsupportedScheme, nil)

pkg/client/fs/fs.go

@@ -2,7 +2,7 @@
  * Minimalist Object Storage, (C) 2015 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
+ * you may not use this fs except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package file
+package fs
 
 import (
 	"io"
@@ -22,6 +22,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -29,20 +30,17 @@ import (
 	"github.com/minio-io/minio/pkg/iodine"
 )
 
-type fileClient struct {
-	// Supports URL in following formats
-	//  - http://<ipaddress>/<bucketname>/<object>
-	//  - http://<bucketname>.<domain>/<object>
+type fsClient struct {
 	*url.URL
 }
 
-// GetNewClient -
+// GetNewClient - instantiate a new fs client
 func GetNewClient(path string) client.Client {
 	u, err := url.Parse(path)
 	if err != nil {
 		return nil
 	}
-	return &fileClient{u}
+	return &fsClient{u}
 }
 
 /// Object operations
@@ -67,7 +65,7 @@ func isValidObject(bucket, object string) (string, os.FileInfo, error) {
 }
 
 // Put - upload new object to bucket
-func (f *fileClient) Put(bucket, object, md5HexString string, size int64, contents io.Reader) error {
+func (f *fsClient) Put(bucket, object, md5HexString string, size int64, contents io.Reader) error {
 	// handle md5HexString match internally
 	if bucket == "" || object == "" {
 		return iodine.New(client.InvalidArgument{}, nil)
@@ -76,14 +74,14 @@ func (f *fileClient) Put(bucket, object, md5HexString string, size int64, conten
 	if size < 0 {
 		return iodine.New(client.InvalidArgument{}, nil)
 	}
-	file, err := os.OpenFile(objectPath, os.O_CREATE|os.O_EXCL, 0600)
+	fs, err := os.Create(objectPath)
 	if os.IsExist(err) {
 		return iodine.New(client.ObjectExists{Bucket: bucket, Object: object}, nil)
 	}
 	if err != nil {
 		return iodine.New(err, nil)
 	}
-	_, err = io.CopyN(file, contents, size)
+	_, err = io.CopyN(fs, contents, size)
 	if err != nil {
 		return iodine.New(err, nil)
 	}
@@ -91,12 +89,12 @@ func (f *fileClient) Put(bucket, object, md5HexString string, size int64, conten
 }
 
 // Get - download an object from bucket
-func (f *fileClient) Get(bucket, object string) (body io.ReadCloser, size int64, md5 string, err error) {
+func (f *fsClient) Get(bucket, object string) (body io.ReadCloser, size int64, md5 string, err error) {
 	objectPath, st, err := isValidObject(bucket, object)
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
 	}
-	body, err = os.OpenFile(objectPath, os.O_RDONLY, 0600)
+	body, err = os.Open(objectPath)
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
 	}
@@ -106,7 +104,7 @@ func (f *fileClient) Get(bucket, object string) (body io.ReadCloser, size int64,
 }
 
 // GetPartial - download a partial object from bucket
-func (f *fileClient) GetPartial(bucket, key string, offset, length int64) (body io.ReadCloser, size int64, md5 string, err error) {
+func (f *fsClient) GetPartial(bucket, key string, offset, length int64) (body io.ReadCloser, size int64, md5 string, err error) {
 	if offset < 0 {
 		return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil)
 	}
@@ -114,7 +112,7 @@ func (f *fileClient) GetPartial(bucket, key string, offset, length int64) (body
 }
 
 // StatObject -
-func (f *fileClient) StatObject(bucket, object string) (size int64, date time.Time, reterr error) {
+func (f *fsClient) StatObject(bucket, object string) (size int64, date time.Time, reterr error) {
 	_, st, err := isValidObject(bucket, object)
 	if size < 0 {
 		return 0, date, iodine.New(client.InvalidArgument{}, nil)
@@ -128,7 +126,7 @@ func (f *fileClient) StatObject(bucket, object string) (size int64, date time.Ti
 /// Bucket operations
 
 // ListBuckets - get list of buckets
-func (f *fileClient) ListBuckets() ([]*client.Bucket, error) {
+func (f *fsClient) ListBuckets() ([]*client.Bucket, error) {
 	buckets, err := ioutil.ReadDir(f.Path)
 	if err != nil {
 		return nil, iodine.New(err, nil)
@@ -145,12 +143,42 @@ func (f *fileClient) ListBuckets() ([]*client.Bucket, error) {
 }
 
 // ListObjects - get a list of objects
-func (f *fileClient) ListObjects(bucket, keyPrefix string) (items []*client.Item, err error) {
-	return nil, iodine.New(client.APINotImplemented{API: "ListObjects"}, nil)
+func (f *fsClient) ListObjects(bucket, prefix string) (items []*client.Item, err error) {
+	visitFS := func(fp string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err // fatal
+		}
+		if fi.IsDir() {
+			return nil // not a fs skip
+		}
+		if prefix != "" {
+			if strings.Contains(fp, prefix) {
+				item := &client.Item{
+					Key:          fp,
+					LastModified: fi.ModTime(),
+					Size:         fi.Size(),
+				}
+				items = append(items, item)
+			}
+		} else {
+			item := &client.Item{
+				Key:          fp,
+				LastModified: fi.ModTime(),
+				Size:         fi.Size(),
+			}
+			items = append(items, item)
+		}
+		return nil
+	}
+	err = filepath.Walk(bucket, visitFS)
+	if err != nil {
+		return nil, iodine.New(err, nil)
+	}
+	return items, nil
 }
 
 // PutBucket - create a new bucket
-func (f *fileClient) PutBucket(bucket string) error {
+func (f *fsClient) PutBucket(bucket string) error {
 	if bucket == "" || strings.TrimSpace(bucket) == "" {
 		return iodine.New(client.InvalidArgument{}, nil)
 	}
@@ -168,7 +196,7 @@ func (f *fileClient) PutBucket(bucket string) error {
 }
 
 // StatBucket -
-func (f *fileClient) StatBucket(bucket string) error {
+func (f *fsClient) StatBucket(bucket string) error {
 	if bucket == "" {
 		return iodine.New(client.InvalidArgument{}, nil)
 	}

pkg/client/fs/fs_test.go

@@ -0,0 +1,76 @@
+/*
+ * Minimalist Object Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this fs except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	. "github.com/minio-io/check"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+// TODO - implement these steps
+
+func (s *MySuite) TestListBuckets(c *C) {
+	root, err := ioutil.TempDir(os.TempDir(), "fs-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(root)
+}
+
+func (s *MySuite) TestListObjects(c *C) {
+	root, err := ioutil.TempDir(os.TempDir(), "fs-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(root)
+}
+
+func (s *MySuite) TestPutBucket(c *C) {
+	root, err := ioutil.TempDir(os.TempDir(), "fs-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(root)
+
+}
+
+func (s *MySuite) TestPutObject(c *C) {
+	root, err := ioutil.TempDir(os.TempDir(), "fs-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(root)
+}
+
+func (s *MySuite) TestGetObject(c *C) {
+	root, err := ioutil.TempDir(os.TempDir(), "fs-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(root)
+}
+
+func (s *MySuite) TestStatObject(c *C) {
+	root, err := ioutil.TempDir(os.TempDir(), "fs-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(root)
+}
+
+func (s *MySuite) TestStatBucket(c *C) {
+	root, err := ioutil.TempDir(os.TempDir(), "fs-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(root)
+}

pkg/client/s3/client.go

@@ -94,15 +94,24 @@ type s3Client struct {
 }
 
 // GetNewClient returns an initialized s3Client structure.
-func GetNewClient(auth *Auth, urlStr string, transport http.RoundTripper) client.Client {
-	u, err := url.Parse(urlStr)
+// if debug use a internal trace transport
+func GetNewClient(hostURL string, auth *Auth, debug bool) client.Client {
+	u, err := url.Parse(hostURL)
 	if err != nil {
 		return nil
 	}
+	var traceTransport RoundTripTrace
+	var transport http.RoundTripper
+	if debug {
+		traceTransport = GetNewTraceTransport(NewTrace(false, true, nil), http.DefaultTransport)
+		transport = GetNewTraceTransport(s3Verify{}, traceTransport)
+	} else {
+		transport = http.DefaultTransport
+	}
 	s3c := &s3Client{
 		&Meta{
 			Auth:      auth,
-			Transport: GetNewTraceTransport(s3Verify{}, transport),
+			Transport: transport,
 		}, u,
 	}
 	return s3c

pkg/client/s3/object.go

@@ -61,13 +61,16 @@ func (c *s3Client) Put(bucket, key, md5HexString string, size int64, contents io
 	req := newReq(c.keyURL(bucket, key))
 	req.Method = "PUT"
 	req.ContentLength = size
-
 	req.Body = ioutil.NopCloser(contents)
+
+	// set Content-MD5 only if md5 is provided
+	if strings.TrimSpace(md5HexString) != "" {
 		md5, err := hex.DecodeString(md5HexString)
 		if err != nil {
 			return iodine.New(err, nil)
 		}
 		req.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5))
+	}
 	c.signRequest(req, c.Host)
 
 	res, err := c.Transport.RoundTrip(req)

url-parser.go

@@ -79,7 +79,6 @@ func fixFileURL(urlStr string) (fixedURL string, err error) {
 	if urlStr == "" {
 		return "", iodine.New(errEmptyURL, nil)
 	}
-
 	utype, e := getURLType(urlStr)
 	if e != nil || utype != urlFile {
 		return "", iodine.New(e, nil)
@@ -95,14 +94,10 @@ func fixFileURL(urlStr string) (fixedURL string, err error) {
 		// Not really a file URL. Host is not empty.
 		return "", iodine.New(errInvalidURL, nil)
 	}
-
-	// fill missing scheme
-	if u.Scheme == "" {
-		// Set it to file
-		u.Scheme = "file"
-	}
-
-	fixedURL = u.String()
+	// do not use u.Scheme since that would construct a path in the form
+	// file:// which is an invalid file but url Parse doesn't report error
+	// so we construct manually instead
+	fixedURL = "file:///" + u.Path
 	return fixedURL, nil
 
 }
@@ -146,11 +141,11 @@ func parseURL(arg string, aliases map[string]string) (urlStr string, err error)
 	if !isValidURL(urlStr) {
 		return "", iodine.New(errUnsupportedScheme, nil)
 	}
-
 	// If it is a file URL, rewrite to file:///path/to form
 	if isValidFileURL(urlStr) {
 		return fixFileURL(urlStr)
 	}
+
 	return urlStr, nil
 }