From afca8b620ab5859b1427b6decdca108af84c56e8 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 11 Apr 2015 03:42:36 -0700 Subject: [PATCH] Move from file to fs and finish ListObjects() - add test stubs --- cmd-common.go | 26 ++++----- pkg/client/{file => fs}/.gitignore | 0 pkg/client/{file/file.go => fs/fs.go} | 68 +++++++++++++++++------- pkg/client/fs/fs_test.go | 76 +++++++++++++++++++++++++++ pkg/client/s3/client.go | 15 ++++-- pkg/client/s3/object.go | 13 +++-- url-parser.go | 15 ++---- 7 files changed, 160 insertions(+), 53 deletions(-) rename pkg/client/{file => fs}/.gitignore (100%) rename pkg/client/{file/file.go => fs/fs.go} (72%) create mode 100644 pkg/client/fs/fs_test.go diff --git a/cmd-common.go b/cmd-common.go index 6b2bbd97..8ccd9fc2 100644 --- a/cmd-common.go +++ b/cmd-common.go @@ -17,12 +17,12 @@ package main import ( - "net/http" "path" "time" "github.com/cheggaaa/pb" "github.com/minio-io/mc/pkg/client" + "github.com/minio-io/mc/pkg/client/fs" "github.com/minio-io/mc/pkg/client/s3" "github.com/minio-io/mc/pkg/console" "github.com/minio-io/minio/pkg/iodine" @@ -60,27 +60,23 @@ func mustGetMcBashCompletionFilename() string { // NewClient - get new client // TODO refactor this to be more testable func getNewClient(debug bool, urlStr string) (clnt client.Client, err error) { - hostCfg, err := getHostConfig(urlStr) - if err != nil { - return nil, iodine.New(err, nil) - } - - var auth s3.Auth - auth.AccessKeyID = hostCfg.Auth.AccessKeyID - auth.SecretAccessKey = hostCfg.Auth.SecretAccessKey uType, err := getURLType(urlStr) if err != nil { return nil, iodine.New(err, nil) } - switch uType { case urlObjectStorage: // Minio and S3 compatible object storage - traceTransport := s3.GetNewTraceTransport(s3.NewTrace(false, true, nil), http.DefaultTransport) - if debug { - clnt = s3.GetNewClient(&auth, urlStr, traceTransport) - } else { - clnt = s3.GetNewClient(&auth, urlStr, http.DefaultTransport) + hostCfg, err := getHostConfig(urlStr) + if err != nil { + return nil, iodine.New(err, nil) } + var auth s3.Auth + auth.AccessKeyID = hostCfg.Auth.AccessKeyID + auth.SecretAccessKey = hostCfg.Auth.SecretAccessKey + clnt = s3.GetNewClient(urlStr, &auth, debug) + return clnt, nil + case urlFile: + clnt = fs.GetNewClient(urlStr) return clnt, nil default: return nil, iodine.New(errUnsupportedScheme, nil) diff --git a/pkg/client/file/.gitignore b/pkg/client/fs/.gitignore similarity index 100% rename from pkg/client/file/.gitignore rename to pkg/client/fs/.gitignore diff --git a/pkg/client/file/file.go b/pkg/client/fs/fs.go similarity index 72% rename from pkg/client/file/file.go rename to pkg/client/fs/fs.go index b98f4303..5d67a811 100644 --- a/pkg/client/file/file.go +++ b/pkg/client/fs/fs.go @@ -2,7 +2,7 @@ * Minimalist Object Storage, (C) 2015 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. + * you may not use this fs except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 @@ -14,7 +14,7 @@ * limitations under the License. */ -package file +package fs import ( "io" @@ -22,6 +22,7 @@ import ( "net/url" "os" "path" + "path/filepath" "strings" "time" @@ -29,20 +30,17 @@ import ( "github.com/minio-io/minio/pkg/iodine" ) -type fileClient struct { - // Supports URL in following formats - // - http://// - // - http://./ +type fsClient struct { *url.URL } -// GetNewClient - +// GetNewClient - instantiate a new fs client func GetNewClient(path string) client.Client { u, err := url.Parse(path) if err != nil { return nil } - return &fileClient{u} + return &fsClient{u} } /// Object operations @@ -67,7 +65,7 @@ func isValidObject(bucket, object string) (string, os.FileInfo, error) { } // Put - upload new object to bucket -func (f *fileClient) Put(bucket, object, md5HexString string, size int64, contents io.Reader) error { +func (f *fsClient) Put(bucket, object, md5HexString string, size int64, contents io.Reader) error { // handle md5HexString match internally if bucket == "" || object == "" { return iodine.New(client.InvalidArgument{}, nil) @@ -76,14 +74,14 @@ func (f *fileClient) Put(bucket, object, md5HexString string, size int64, conten if size < 0 { return iodine.New(client.InvalidArgument{}, nil) } - file, err := os.OpenFile(objectPath, os.O_CREATE|os.O_EXCL, 0600) + fs, err := os.Create(objectPath) if os.IsExist(err) { return iodine.New(client.ObjectExists{Bucket: bucket, Object: object}, nil) } if err != nil { return iodine.New(err, nil) } - _, err = io.CopyN(file, contents, size) + _, err = io.CopyN(fs, contents, size) if err != nil { return iodine.New(err, nil) } @@ -91,12 +89,12 @@ func (f *fileClient) Put(bucket, object, md5HexString string, size int64, conten } // Get - download an object from bucket -func (f *fileClient) Get(bucket, object string) (body io.ReadCloser, size int64, md5 string, err error) { +func (f *fsClient) Get(bucket, object string) (body io.ReadCloser, size int64, md5 string, err error) { objectPath, st, err := isValidObject(bucket, object) if err != nil { return nil, 0, "", iodine.New(err, nil) } - body, err = os.OpenFile(objectPath, os.O_RDONLY, 0600) + body, err = os.Open(objectPath) if err != nil { return nil, 0, "", iodine.New(err, nil) } @@ -106,7 +104,7 @@ func (f *fileClient) Get(bucket, object string) (body io.ReadCloser, size int64, } // GetPartial - download a partial object from bucket -func (f *fileClient) GetPartial(bucket, key string, offset, length int64) (body io.ReadCloser, size int64, md5 string, err error) { +func (f *fsClient) GetPartial(bucket, key string, offset, length int64) (body io.ReadCloser, size int64, md5 string, err error) { if offset < 0 { return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil) } @@ -114,7 +112,7 @@ func (f *fileClient) GetPartial(bucket, key string, offset, length int64) (body } // StatObject - -func (f *fileClient) StatObject(bucket, object string) (size int64, date time.Time, reterr error) { +func (f *fsClient) StatObject(bucket, object string) (size int64, date time.Time, reterr error) { _, st, err := isValidObject(bucket, object) if size < 0 { return 0, date, iodine.New(client.InvalidArgument{}, nil) @@ -128,7 +126,7 @@ func (f *fileClient) StatObject(bucket, object string) (size int64, date time.Ti /// Bucket operations // ListBuckets - get list of buckets -func (f *fileClient) ListBuckets() ([]*client.Bucket, error) { +func (f *fsClient) ListBuckets() ([]*client.Bucket, error) { buckets, err := ioutil.ReadDir(f.Path) if err != nil { return nil, iodine.New(err, nil) @@ -145,12 +143,42 @@ func (f *fileClient) ListBuckets() ([]*client.Bucket, error) { } // ListObjects - get a list of objects -func (f *fileClient) ListObjects(bucket, keyPrefix string) (items []*client.Item, err error) { - return nil, iodine.New(client.APINotImplemented{API: "ListObjects"}, nil) +func (f *fsClient) ListObjects(bucket, prefix string) (items []*client.Item, err error) { + visitFS := func(fp string, fi os.FileInfo, err error) error { + if err != nil { + return err // fatal + } + if fi.IsDir() { + return nil // not a fs skip + } + if prefix != "" { + if strings.Contains(fp, prefix) { + item := &client.Item{ + Key: fp, + LastModified: fi.ModTime(), + Size: fi.Size(), + } + items = append(items, item) + } + } else { + item := &client.Item{ + Key: fp, + LastModified: fi.ModTime(), + Size: fi.Size(), + } + items = append(items, item) + } + return nil + } + err = filepath.Walk(bucket, visitFS) + if err != nil { + return nil, iodine.New(err, nil) + } + return items, nil } // PutBucket - create a new bucket -func (f *fileClient) PutBucket(bucket string) error { +func (f *fsClient) PutBucket(bucket string) error { if bucket == "" || strings.TrimSpace(bucket) == "" { return iodine.New(client.InvalidArgument{}, nil) } @@ -168,7 +196,7 @@ func (f *fileClient) PutBucket(bucket string) error { } // StatBucket - -func (f *fileClient) StatBucket(bucket string) error { +func (f *fsClient) StatBucket(bucket string) error { if bucket == "" { return iodine.New(client.InvalidArgument{}, nil) } diff --git a/pkg/client/fs/fs_test.go b/pkg/client/fs/fs_test.go new file mode 100644 index 00000000..2a83574f --- /dev/null +++ b/pkg/client/fs/fs_test.go @@ -0,0 +1,76 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this fs except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs + +import ( + "io/ioutil" + "os" + "testing" + + . "github.com/minio-io/check" +) + +func Test(t *testing.T) { TestingT(t) } + +type MySuite struct{} + +var _ = Suite(&MySuite{}) + +// TODO - implement these steps + +func (s *MySuite) TestListBuckets(c *C) { + root, err := ioutil.TempDir(os.TempDir(), "fs-") + c.Assert(err, IsNil) + defer os.RemoveAll(root) +} + +func (s *MySuite) TestListObjects(c *C) { + root, err := ioutil.TempDir(os.TempDir(), "fs-") + c.Assert(err, IsNil) + defer os.RemoveAll(root) +} + +func (s *MySuite) TestPutBucket(c *C) { + root, err := ioutil.TempDir(os.TempDir(), "fs-") + c.Assert(err, IsNil) + defer os.RemoveAll(root) + +} + +func (s *MySuite) TestPutObject(c *C) { + root, err := ioutil.TempDir(os.TempDir(), "fs-") + c.Assert(err, IsNil) + defer os.RemoveAll(root) +} + +func (s *MySuite) TestGetObject(c *C) { + root, err := ioutil.TempDir(os.TempDir(), "fs-") + c.Assert(err, IsNil) + defer os.RemoveAll(root) +} + +func (s *MySuite) TestStatObject(c *C) { + root, err := ioutil.TempDir(os.TempDir(), "fs-") + c.Assert(err, IsNil) + defer os.RemoveAll(root) +} + +func (s *MySuite) TestStatBucket(c *C) { + root, err := ioutil.TempDir(os.TempDir(), "fs-") + c.Assert(err, IsNil) + defer os.RemoveAll(root) +} diff --git a/pkg/client/s3/client.go b/pkg/client/s3/client.go index fd09c951..5a00e0a5 100644 --- a/pkg/client/s3/client.go +++ b/pkg/client/s3/client.go @@ -94,15 +94,24 @@ type s3Client struct { } // GetNewClient returns an initialized s3Client structure. -func GetNewClient(auth *Auth, urlStr string, transport http.RoundTripper) client.Client { - u, err := url.Parse(urlStr) +// if debug use a internal trace transport +func GetNewClient(hostURL string, auth *Auth, debug bool) client.Client { + u, err := url.Parse(hostURL) if err != nil { return nil } + var traceTransport RoundTripTrace + var transport http.RoundTripper + if debug { + traceTransport = GetNewTraceTransport(NewTrace(false, true, nil), http.DefaultTransport) + transport = GetNewTraceTransport(s3Verify{}, traceTransport) + } else { + transport = http.DefaultTransport + } s3c := &s3Client{ &Meta{ Auth: auth, - Transport: GetNewTraceTransport(s3Verify{}, transport), + Transport: transport, }, u, } return s3c diff --git a/pkg/client/s3/object.go b/pkg/client/s3/object.go index ec66e6e2..f36bb843 100644 --- a/pkg/client/s3/object.go +++ b/pkg/client/s3/object.go @@ -61,13 +61,16 @@ func (c *s3Client) Put(bucket, key, md5HexString string, size int64, contents io req := newReq(c.keyURL(bucket, key)) req.Method = "PUT" req.ContentLength = size - req.Body = ioutil.NopCloser(contents) - md5, err := hex.DecodeString(md5HexString) - if err != nil { - return iodine.New(err, nil) + + // set Content-MD5 only if md5 is provided + if strings.TrimSpace(md5HexString) != "" { + md5, err := hex.DecodeString(md5HexString) + if err != nil { + return iodine.New(err, nil) + } + req.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5)) } - req.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5)) c.signRequest(req, c.Host) res, err := c.Transport.RoundTrip(req) diff --git a/url-parser.go b/url-parser.go index cc2a9aa6..57981671 100644 --- a/url-parser.go +++ b/url-parser.go @@ -79,7 +79,6 @@ func fixFileURL(urlStr string) (fixedURL string, err error) { if urlStr == "" { return "", iodine.New(errEmptyURL, nil) } - utype, e := getURLType(urlStr) if e != nil || utype != urlFile { return "", iodine.New(e, nil) @@ -95,14 +94,10 @@ func fixFileURL(urlStr string) (fixedURL string, err error) { // Not really a file URL. Host is not empty. return "", iodine.New(errInvalidURL, nil) } - - // fill missing scheme - if u.Scheme == "" { - // Set it to file - u.Scheme = "file" - } - - fixedURL = u.String() + // do not use u.Scheme since that would construct a path in the form + // file:// which is an invalid file but url Parse doesn't report error + // so we construct manually instead + fixedURL = "file:///" + u.Path return fixedURL, nil } @@ -146,11 +141,11 @@ func parseURL(arg string, aliases map[string]string) (urlStr string, err error) if !isValidURL(urlStr) { return "", iodine.New(errUnsupportedScheme, nil) } - // If it is a file URL, rewrite to file:///path/to form if isValidFileURL(urlStr) { return fixFileURL(urlStr) } + return urlStr, nil }