Add filesystem Windows support, also cannot reproduce #281 anymore - fixes #281

auth.go

@@ -16,14 +16,10 @@
 
 package main
 
-import (
+import "regexp"
-	"log"
-	"regexp"
-)
 
 // isValidSecretKey - validate secret key
 func isValidSecretKey(secretAccessKey string) bool {
-	log.Println(secretAccessKey)
 	regex := regexp.MustCompile("^.{40}$")
 	return regex.MatchString(secretAccessKey)
 }

cmd-common.go

@@ -115,6 +115,29 @@ func (manager mcClientManager) getTargetWriter(targetURL string, md5Hex string,
 	return targetClnt.Put(md5Hex, length)
 }
 
+func getFilesystemAbsURL(u *url.URL) (string, error) {
+	var absURLStr string
+	var err error
+	switch true {
+	case u.Scheme == "file" && u.IsAbs():
+		absURLStr, err = filepath.Abs(filepath.Clean(u.Path))
+		if err != nil {
+			return "", iodine.New(err, nil)
+		}
+	case filepath.IsAbs(u.String()):
+		absURLStr, err = filepath.Abs(filepath.Clean(u.String()))
+		if err != nil {
+			return "", iodine.New(err, nil)
+		}
+	default:
+		absURLStr, err = filepath.Abs(filepath.Clean(u.String()))
+		if err != nil {
+			return "", iodine.New(err, nil)
+		}
+	}
+	return absURLStr, nil
+}
+
 // getNewClient gives a new client interface
 func (manager mcClientManager) getNewClient(urlStr string, debug bool) (clnt client.Client, err error) {
 	u, err := url.Parse(urlStr)
@@ -140,23 +163,17 @@ func (manager mcClientManager) getNewClient(urlStr string, debug bool) (clnt cli
 		clnt = s3.GetNewClient(urlStr, auth, mcUserAgent, debug)
 		return clnt, nil
 	case client.URLFilesystem:
-		var absURLStr string
+		absURLStr, err := getFilesystemAbsURL(u)
-		var err error
+		if err != nil {
-		if u.IsAbs() {
+			return nil, iodine.New(err, nil)
-			absURLStr, err = filepath.Abs(filepath.Clean(u.Path))
-			if err != nil {
-				return nil, iodine.New(err, nil)
-			}
-		} else {
-			absURLStr, err = filepath.Abs(filepath.Clean(urlStr))
-			if err != nil {
-				return nil, iodine.New(err, nil)
-			}
 		}
 		clnt = fs.GetNewClient(absURLStr)
 		return clnt, nil
 	default:
-		return nil, iodine.New(errUnsupportedScheme{scheme: client.GetURLType(urlStr)}, nil)
+		return nil, iodine.New(errUnsupportedScheme{
+			scheme: client.GetURLType(urlStr),
+			url:    urlStr,
+		}, nil)
 	}
 }
 

cmd-cp.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/cheggaaa/pb"
 	"github.com/minio-io/cli"
+	"github.com/minio-io/mc/pkg/client"
 	"github.com/minio-io/mc/pkg/console"
 	"github.com/minio-io/minio/pkg/iodine"
 	"github.com/minio-io/minio/pkg/utils/log"
@@ -44,10 +45,11 @@ func runCopyCmd(ctx *cli.Context) {
 		switch e := iodine.ToError(err).(type) {
 		case errUnsupportedScheme:
 			log.Debug.Println(iodine.New(err, nil))
-			console.Fatalf("mc: parsing URL failed with following reason: [%s]\n", e)
+			console.Fatalf("mc: reading URL [%s] failed with following reason: [%s], possible suggestions [%s]\n",
+				e.url, e, client.GuessPossibleURL(e.url))
 		default:
 			log.Debug.Println(iodine.New(err, nil))
-			console.Fatalf("mc: parsing URL failed with following reason: [%s]\n", e)
+			console.Fatalf("mc: reading URLs failed with following reason: [%s]\n", e)
 		}
 	}
 

errors.go

@@ -30,6 +30,7 @@ func (e errInvalidArgument) Error() string {
 
 type errUnsupportedScheme struct {
 	scheme client.URLType
+	url    string
 }
 
 func (e errUnsupportedScheme) Error() string {

pkg/client/fs/fs_multi_unix.go

@@ -1,3 +1,5 @@
+// +build darwin drangofly freebsd linux nacl netbsd openbsd solaris
+
 /*
  * Mini Copy, (C) 2015 Minio, Inc.
  *

pkg/client/fs/fs_multi_windows.go

@@ -0,0 +1,49 @@
+/*
+ * Mini Copy, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this fs except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"github.com/minio-io/mc/pkg/client"
+	"github.com/minio-io/minio/pkg/iodine"
+)
+
+// Multipart object upload handlers
+
+// InitiateMultiPartUpload -
+func (c *fsClient) InitiateMultiPartUpload() (objectID string, err error) {
+	return "", iodine.New(client.APINotImplemented{API: "InitiateMultiPartUpload"}, nil)
+}
+
+// UploadPart -
+func (c *fsClient) UploadPart(uploadID string, partNumber int) (md5hex string, err error) {
+	return "", iodine.New(client.APINotImplemented{API: "UploadPart"}, nil)
+}
+
+// CompleteMultiPartUpload -
+func (c *fsClient) CompleteMultiPartUpload(uploadID string) (location, md5hex string, err error) {
+	return "", "", iodine.New(client.APINotImplemented{API: "CompleteMultiPartUpload"}, nil)
+}
+
+// AbortMultiPartUpload -
+func (c *fsClient) AbortMultiPartUpload(uploadID string) error {
+	return iodine.New(client.APINotImplemented{API: "AbortMultiPartUpload"}, nil)
+}
+
+// ListParts -
+func (c *fsClient) ListParts(uploadID string) (items *client.PartItems, err error) {
+	return nil, iodine.New(client.APINotImplemented{API: "ListParts"}, nil)
+}

pkg/client/fs/fs_put_unix.go

@@ -1,3 +1,21 @@
+// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris
+
+/*
+ * Mini Copy, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fs
 
 import (

pkg/client/fs/fs_put_windows.go

@@ -0,0 +1,108 @@
+/*
+ * Mini Copy, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/minio-io/mc/pkg/client"
+	"github.com/minio-io/minio/pkg/iodine"
+)
+
+// Put - upload new object to bucket
+func (f *fsClient) Put(md5HexString string, size int64) (io.WriteCloser, error) {
+	r, w := io.Pipe()
+	blockingWriter := NewBlockingWriteCloser(w)
+	go func() {
+		// handle md5HexString match internally
+		if size < 0 {
+			err := iodine.New(client.InvalidArgument{Err: errors.New("invalid argument")}, nil)
+			r.CloseWithError(err)
+			blockingWriter.Release(err)
+			return
+		}
+		objectDir, _ := filepath.Split(f.String())
+		objectPath := f.String()
+		if err := os.MkdirAll(objectDir, 0700); err != nil {
+			err := iodine.New(err, nil)
+			r.CloseWithError(err)
+			blockingWriter.Release(err)
+			return
+		}
+		fs, err := os.Create(objectPath)
+		if err != nil {
+			err := iodine.New(err, nil)
+			r.CloseWithError(err)
+			blockingWriter.Release(err)
+			return
+		}
+		_, err = io.CopyN(fs, r, size)
+		if err != nil {
+			err := iodine.New(err, nil)
+			r.CloseWithError(err)
+			blockingWriter.Release(err)
+			return
+		}
+		blockingWriter.Release(nil)
+		r.Close()
+	}()
+	return blockingWriter, nil
+}
+
+// BlockingWriteCloser is a WriteCloser that blocks until released
+type BlockingWriteCloser struct {
+	w       io.WriteCloser
+	release *sync.WaitGroup
+	err     error
+}
+
+// Write to the underlying writer
+func (b *BlockingWriteCloser) Write(p []byte) (int, error) {
+	n, err := b.w.Write(p)
+	err = iodine.New(err, nil)
+	return n, err
+}
+
+// Close blocks until another goroutine calls Release(error). Returns error code if either
+// writer fails or Release is called with an error.
+func (b *BlockingWriteCloser) Close() error {
+	err := b.w.Close()
+	if err != nil {
+		b.err = err
+	}
+	b.release.Wait()
+	return b.err
+}
+
+// Release the Close, causing it to unblock. Only call this once. Calling it multiple times results in a panic.
+func (b *BlockingWriteCloser) Release(err error) {
+	b.release.Done()
+	if err != nil {
+		b.err = err
+	}
+}
+
+// NewBlockingWriteCloser Creates a new write closer that must be released by the read consumer.
+func NewBlockingWriteCloser(w io.WriteCloser) *BlockingWriteCloser {
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+	return &BlockingWriteCloser{w: w, release: wg}
+}

pkg/client/fs/fs_unix.go

@@ -1,3 +1,5 @@
+// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris
+
 /*
  * Mini Copy, (C) 2015 Minio, Inc.
  *

pkg/client/fs/fs_windows.go

@@ -0,0 +1,193 @@
+/*
+ * Mini Copy, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this fs except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"io"
+	"os"
+	"sort"
+	"strings"
+
+	"io/ioutil"
+	"net/url"
+	"path/filepath"
+
+	"github.com/minio-io/mc/pkg/client"
+	"github.com/minio-io/minio/pkg/iodine"
+)
+
+type fsClient struct {
+	*url.URL
+}
+
+// GetNewClient - instantiate a new fs client
+func GetNewClient(path string) client.Client {
+	u, err := url.Parse(path)
+	if err != nil {
+		return nil
+	}
+	return &fsClient{u}
+}
+
+/// Object operations
+
+// url2Object converts URL to bucketName and objectName
+func (f *fsClient) url2Object() (bucketName, objectName string) {
+	return filepath.Split(f.Path)
+}
+
+// getObjectMetadata - wrapper function to get file stat
+func (f *fsClient) getObjectMetadata() (os.FileInfo, error) {
+	bucket, object := f.url2Object()
+	st, err := os.Stat(f.Path)
+	if os.IsNotExist(err) {
+		return nil, iodine.New(client.ObjectNotFound{Bucket: bucket, Object: object}, nil)
+	}
+	if st.IsDir() {
+		return nil, iodine.New(client.InvalidObjectName{Bucket: bucket, Object: object}, nil)
+	}
+	if err != nil {
+		return nil, iodine.New(err, nil)
+	}
+	return st, nil
+}
+
+// Get - download an object from bucket
+func (f *fsClient) Get() (body io.ReadCloser, size int64, md5 string, err error) {
+	st, err := f.getObjectMetadata()
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	body, err = os.Open(f.Path)
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	// TODO: support md5sum - there is no easier way to do it right now without temporary buffer
+	// so avoiding it to ensure no out of memory situations
+	return body, st.Size(), "", nil
+}
+
+// GetPartial - download a partial object from bucket
+func (f *fsClient) GetPartial(offset, length int64) (body io.ReadCloser, size int64, md5 string, err error) {
+	if offset < 0 {
+		return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil)
+	}
+	st, err := f.getObjectMetadata()
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	body, err = os.Open(f.Path)
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	if offset > st.Size() || (offset+length-1) > st.Size() {
+		return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil)
+	}
+	_, err = io.CopyN(ioutil.Discard, body, offset)
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	return body, length, "", nil
+}
+
+// GetObjectMetadata -
+func (f *fsClient) GetObjectMetadata() (item *client.Item, reterr error) {
+	st, err := f.getObjectMetadata()
+	if err != nil {
+		return nil, iodine.New(err, nil)
+	}
+	item = new(client.Item)
+	item.Name = st.Name()
+	item.Size = st.Size()
+	item.Time = st.ModTime()
+	return item, nil
+}
+
+/// Bucket operations
+
+// listBuckets - get list of buckets
+func (f *fsClient) listBuckets() ([]*client.Item, error) {
+	buckets, err := ioutil.ReadDir(f.Path)
+	if err != nil {
+		return nil, iodine.New(err, nil)
+	}
+	var results []*client.Item
+	for _, bucket := range buckets {
+		result := new(client.Item)
+		result.Name = bucket.Name()
+		result.Time = bucket.ModTime()
+		results = append(results, result)
+	}
+	return results, nil
+}
+
+// List - get a list of items
+func (f *fsClient) List() (items []*client.Item, err error) {
+	item, err := f.GetObjectMetadata()
+	switch err {
+	case nil:
+		items = append(items, item)
+		return items, nil
+	default:
+		visitFS := func(fp string, fi os.FileInfo, err error) error {
+			if err != nil {
+				if os.IsPermission(err) { // skip inaccessible files
+					return nil
+				}
+				return err // fatal
+			}
+			if fi.IsDir() {
+				return nil // not a fs skip
+			}
+			// trim f.Path
+			item := &client.Item{
+				Name: strings.TrimPrefix(fp, f.Path+string(filepath.Separator)),
+				Time: fi.ModTime(),
+				Size: fi.Size(),
+			}
+			items = append(items, item)
+			return nil
+		}
+		err = filepath.Walk(f.Path, visitFS)
+		if err != nil {
+			return nil, iodine.New(err, nil)
+		}
+		sort.Sort(client.BySize(items))
+		return items, nil
+	}
+}
+
+// PutBucket - create a new bucket
+func (f *fsClient) PutBucket() error {
+	err := os.MkdirAll(f.Path, 0700)
+	if err != nil {
+		return iodine.New(err, nil)
+	}
+	return nil
+}
+
+// Stat -
+func (f *fsClient) Stat() error {
+	st, err := os.Stat(f.Path)
+	if os.IsNotExist(err) {
+		return iodine.New(client.BucketNotFound{Bucket: ""}, nil)
+	}
+	if !st.IsDir() {
+		return iodine.New(client.InvalidBucketName{Bucket: ""}, nil)
+	}
+	return nil
+}

pkg/client/s3/bucket_list.go

@@ -75,8 +75,7 @@ func (c *s3Client) decodeBucketResults(urlReq string) (listBucketResults, error)
 	var logbuf bytes.Buffer
 	err = xml.NewDecoder(io.TeeReader(res.Body, &logbuf)).Decode(&bres)
 	if err != nil {
-		fmt.Printf("Error parsing s3 XML response: %v for %q\n", err, logbuf.Bytes())
+		return listBucketResults{}, iodine.New(err, map[string]string{"XMLError": logbuf.String()})
-		return listBucketResults{}, iodine.New(err, nil)
 	}
 	return bres, nil
 }

pkg/client/url.go

@@ -3,7 +3,7 @@ package client
 import (
 	"fmt"
 	"net/url"
-	"regexp"
+	"path/filepath"
 	"runtime"
 	"strings"
 )
@@ -45,18 +45,19 @@ func GetURLType(urlStr string) URLType {
 	}
 
 	// while Scheme file, host should be empty
-	if u.Scheme == "file" && u.Host == "" && strings.Contains(urlStr, ":///") {
+	// if windows skip this check, not going to support file:/// style on windows
-		return URLFilesystem
+	// we should just check for VolumeName on windows
-	}
+	if runtime.GOOS != "windows" {
-
+		if u.Scheme == "file" && u.Host == "" && strings.Contains(urlStr, ":///") {
-	// MS Windows OS: Match drive letters
+			return URLFilesystem
-	if runtime.GOOS == "windows" {
+		}
-		if regexp.MustCompile(`^[a-zA-Z]?$`).MatchString(u.Scheme) {
+	} else {
+		if filepath.VolumeName(urlStr) != "" {
 			return URLFilesystem
 		}
 	}
 
-	// local path, without the file:///
+	// local path, without the file:/// or C:\
 	if u.Scheme == "" {
 		return URLFilesystem
 	}

url.go

@@ -36,7 +36,10 @@ func getURL(arg string, aliases map[string]string) (urlStr string, err error) {
 		return "", iodine.New(err, nil)
 	}
 	if client.GetURLType(urlStr) == client.URLUnknown {
-		return "", iodine.New(errUnsupportedScheme{scheme: client.URLUnknown}, map[string]string{"URL": urlStr})
+		return "", iodine.New(errUnsupportedScheme{
+			scheme: client.URLUnknown,
+			url:    urlStr,
+		}, nil)
 	}
 	return urlStr, nil
 }