first step to rewrite copy cmd

cmd-common.go

@@ -46,37 +46,6 @@ func startBar(size int64) *pb.ProgressBar {
 	return bar
 }
 
-// NewClient - get new client
-func getNewClient(debug bool, url string) (cl client.Client, err error) {
-	config, err := getMcConfig()
-	if err != nil {
-		return nil, err
-	}
-
-	hostCfg, err := getHostConfig(config.DefaultHost)
-	if err != nil {
-		return nil, err
-	}
-
-	var auth s3.Auth
-	auth.AccessKeyID = hostCfg.Auth.AccessKeyID
-	auth.SecretAccessKey = hostCfg.Auth.SecretAccessKey
-
-	if debug {
-		trace := s3.Trace{
-			BodyTraceFlag:        false,
-			RequestTransportFlag: true,
-			Writer:               nil,
-		}
-		traceTransport := s3.GetNewTraceTransport(trace, http.DefaultTransport)
-		cl = s3.GetNewClient(&auth, url, traceTransport)
-	} else {
-		cl = s3.GetNewClient(&auth, url, http.DefaultTransport)
-	}
-
-	return cl, nil
-}
-
 func parseDestinationArgs(urlParsed *url.URL, destination, source object) (object, error) {
 	switch true {
 	case urlParsed.Scheme == "http" || urlParsed.Scheme == "https":
@@ -226,3 +195,34 @@ func parseArgs(c *cli.Context) (args *cmdArgs, err error) {
 func getMcBashCompletionFilename() string {
 	return path.Join(getMcConfigDir(), "mc.bash_completion")
 }
+
+// NewClient - get new client
+func getNewClient(debug bool, urlStr string) (clnt client.Client, err error) {
+	config, err := getMcConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	hostCfg, err := getHostConfig(config.DefaultHost)
+	if err != nil {
+		return nil, err
+	}
+
+	var auth s3.Auth
+	auth.AccessKeyID = hostCfg.Auth.AccessKeyID
+	auth.SecretAccessKey = hostCfg.Auth.SecretAccessKey
+
+	if debug {
+		trace := s3.Trace{
+			BodyTraceFlag:        false,
+			RequestTransportFlag: true,
+			Writer:               nil,
+		}
+		traceTransport := s3.GetNewTraceTransport(trace, http.DefaultTransport)
+		clnt = s3.GetNewClient(&auth, urlStr, traceTransport)
+	} else {
+		clnt = s3.GetNewClient(&auth, urlStr, http.DefaultTransport)
+	}
+
+	return clnt, nil
+}

cmd-config.go

@@ -132,8 +132,8 @@ func loadMcConfig() (config *mcConfig, err error) {
 }
 
 // saveConfig writes configuration data in json format to config file.
-func saveConfig(c *cli.Context) error {
+func saveConfig(ctx *cli.Context) error {
-	configData, err := parseConfigInput(c)
+	configData, err := parseConfigInput(ctx)
 	if err != nil {
 		return err
 	}
@@ -284,12 +284,12 @@ func getHostConfig(hostURL string) (*hostConfig, error) {
 }
 
 // doConfigCmd is the handler for "mc config" sub-command.
-func doConfigCmd(c *cli.Context) {
+func doConfigCmd(ctx *cli.Context) {
 	switch true {
-	case c.Bool("completion") == true:
+	case ctx.Bool("completion") == true:
 		getBashCompletion()
 	default:
-		err := saveConfig(c)
+		err := saveConfig(ctx)
 		if os.IsExist(err) {
 			log.Fatalf("mc: Please rename your current configuration file [%s]\n", getMcConfigFilename())
 		}
@@ -298,6 +298,5 @@ func doConfigCmd(c *cli.Context) {
 			log.Fatalf("mc: Unable to generate config file [%s]. \nError: %v\n", getMcConfigFilename(), err)
 		}
 		info("Configuration written to " + getMcConfigFilename() + "\n")
-
 	}
 }

cmd-cp-recursive.go

@@ -1,163 +0,0 @@
-/*
- * Minimalist Object Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-
-	"github.com/cheggaaa/pb"
-	"github.com/minio-io/cli"
-	"github.com/minio-io/mc/pkg/client"
-	"github.com/minio-io/mc/pkg/client/s3"
-)
-
-type walk struct {
-	s3   client.Client
-	args *cmdArgs
-}
-
-func (w *walk) putWalk(p string, i os.FileInfo, err error) error {
-	if i.IsDir() {
-		return nil
-	}
-	if !i.Mode().IsRegular() {
-		return nil
-	}
-	parts := strings.SplitN(p, "/", 2)
-	bucketname := w.args.destination.bucket
-	key := parts[1]
-
-	bodyFile, err := os.Open(p)
-	defer bodyFile.Close()
-
-	var size int64
-	size, _, err = w.s3.Stat(bucketname, key)
-	if os.IsExist(err) || size != 0 {
-		msg := fmt.Sprintf("%s is already uploaded -- to bucket:%s/%s/%s",
-			key, w.args.destination.host, bucketname, key)
-		info(msg)
-		return nil
-	}
-	var bar *pb.ProgressBar
-	if !w.args.quiet {
-		// get progress bar
-		bar = startBar(i.Size())
-	}
-	newreader := io.Reader(bodyFile)
-	if !w.args.quiet {
-		bar.Start()
-		newreader = io.TeeReader(bodyFile, bar)
-	}
-	err = w.s3.Put(bucketname, key, i.Size(), newreader)
-	if err != nil {
-		return err
-	}
-	if !w.args.quiet {
-		bar.Finish()
-		info("Success!")
-	}
-	return nil
-}
-
-// isBucketExist checks if a bucket exists
-func isBucketExist(bucketName string, v []*client.Bucket) bool {
-	for _, b := range v {
-		if bucketName == b.Name {
-			return true
-		}
-	}
-	return false
-}
-
-func sourceValidate(input string) error {
-	if !s3.IsValidBucketName(input) {
-		return fmt.Errorf("Invalid input bucket name [%s]", input)
-	}
-	st, err := os.Stat(input)
-	if os.IsNotExist(err) {
-		return err
-	}
-	if !st.IsDir() {
-		return errors.New("Should be a directory")
-	}
-	return nil
-}
-
-// doRecursiveCP recursively copies objects from source to destination
-func doRecursiveCP(c *cli.Context, args *cmdArgs) error {
-	var buckets []*client.Bucket
-	switch true {
-	case args.source.bucket == "":
-		input := path.Clean(args.source.key)
-		if err := sourceValidate(input); err != nil {
-			return err
-		}
-		s3c, err := getNewClient(globalDebugFlag, args.destination.url.String())
-		if err != nil {
-			return err
-		}
-		p := &walk{s3c, args}
-		buckets, err = s3c.ListBuckets()
-		if !isBucketExist(args.destination.bucket, buckets) {
-			// Create bucketname, before uploading files
-			err = s3c.PutBucket(args.destination.bucket)
-			if err != nil {
-				return err
-			}
-		}
-		err = filepath.Walk(input, p.putWalk)
-		if err != nil {
-			return err
-		}
-	case args.destination.bucket == "":
-		s3c, err := getNewClient(globalDebugFlag, args.source.url.String())
-		if err != nil {
-			return err
-		}
-		items, _, err := s3c.ListObjects(args.source.bucket, "", "", "", globalMaxKeys)
-		if err != nil {
-			return err
-		}
-		root := args.destination.key
-		writeObjects := func(v []*client.Item) error {
-			if len(v) > 0 {
-				// Items are already sorted
-				for _, b := range v {
-					args.source.key = b.Key
-					os.MkdirAll(path.Join(root, path.Dir(b.Key)), 0755)
-					args.destination.key = path.Join(root, b.Key)
-					err := secondMode(c, args)
-					if err != nil {
-						return err
-					}
-				}
-			}
-			return nil
-		}
-		err = writeObjects(items)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}

cmd-cp.go

@@ -17,286 +17,80 @@
 package main
 
 import (
-	"errors"
-	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 
 	"github.com/cheggaaa/pb"
 	"github.com/minio-io/cli"
 )
 
-// Different modes of cp operation
-const (
-	first  = iota // <Object> <S3Object> or <Object> <S3Bucket>
-	second        // <S3Object> <Object> or <S3Object> .
-	third         // <S3Object> <S3Object> or <S3Object> <S3Bucket>
-	fourth        // <Dir> <S3Bucket> or <S3Bucket> <Dir> or <Dir> <S3Uri>
-	invalid
-)
-
-// Get current mode of operation from available arguments and options
-func getMode(recursive bool, args *cmdArgs) int {
-	switch recursive {
-	case false:
-		switch true {
-		// <Object> <S3Object> or <Object> <S3Bucket>
-		case args.source.bucket == "" && args.destination.bucket != "":
-			return first
-		// <S3Object> <Object> or <S3Object> .
-		case args.source.bucket != "" && args.source.key != "" && args.destination.bucket == "":
-			return second
-		// <S3Object> <S3Object> or <S3Object> <S3Bucket>
-		case args.source.bucket != "" && args.destination.bucket != "" && args.source.key != "":
-			return third
-		}
-	case true:
-		switch true {
-		// <Dir> <S3Bucket> or <S3Bucket> <Dir> or <Dir> <S3Uri>
-		case args.source.bucket != "" || args.source.key != "":
-			return fourth
-		}
-	}
-	return invalid
-}
-
-// First mode <Object> <S3Object> or <Object> <S3Bucket>
-func firstMode(c *cli.Context, args *cmdArgs) error {
-	if args.source.key == "" {
-		return errors.New("invalid args")
-	}
-	st, err := os.Stat(args.source.key)
-	if os.IsNotExist(err) {
-		return err
-	}
-	if st.IsDir() {
-		msg := fmt.Sprintf("omitting directory '%s'", st.Name())
-		return errors.New(msg)
-	}
-	size := st.Size()
-	source, err := os.Open(args.source.key)
-	defer source.Close()
-	if err != nil {
-		return err
-	}
-	var bar *pb.ProgressBar
-	if !args.quiet {
-		// get progress bar
-		bar = startBar(size)
-	}
-	// http://<bucket>.<hostname> is specified without key
-	if args.destination.key == "" {
-		args.destination.key = args.source.key
-	}
-	s3c, err := getNewClient(globalDebugFlag, args.destination.url.String())
-	if err != nil {
-		return err
-	}
-	newreader := io.Reader(source)
-	if !args.quiet {
-		bar.Start()
-		newreader = io.TeeReader(source, bar)
-	}
-	err = s3c.Put(args.destination.bucket, args.destination.key, size, newreader)
-	if err != nil {
-		return err
-	}
-	if !args.quiet {
-		bar.Finish()
-		info("Success!")
-	}
-	return nil
-}
-
-// Second mode <S3Object> <Object> or <S3Object> .
-func secondMode(c *cli.Context, args *cmdArgs) error {
-	var objectReader io.ReadCloser
-	var objectSize, downloadedSize int64
-	var destination *os.File
-	var err error
-	var st os.FileInfo
-
-	s3c, err := getNewClient(globalDebugFlag, args.source.url.String())
-	if err != nil {
-		return err
-	}
-	// Send HEAD request to validate if file exists.
-	objectSize, _, err = s3c.Stat(args.source.bucket, args.source.key)
-	if err != nil {
-		return err
-	}
-
-	var bar *pb.ProgressBar
-	if !args.quiet {
-		// get progress bar
-		bar = startBar(objectSize)
-	}
-
-	// Check if the object already exists
-	st, err = os.Stat(args.destination.key)
-	switch os.IsNotExist(err) {
-	case true:
-		// Create if it doesn't exist
-		destination, err = os.Create(args.destination.key)
-		defer destination.Close()
-		if err != nil {
-			return err
-		}
-		objectReader, _, err = s3c.Get(args.source.bucket, args.source.key)
-		if err != nil {
-			return err
-		}
-	case false:
-		downloadedSize = st.Size()
-		// Verify if file is already downloaded
-		if downloadedSize == objectSize {
-			msg := fmt.Sprintf("%s object has been already downloaded", args.destination.key)
-			return errors.New(msg)
-		}
-
-		destination, err = os.OpenFile(args.destination.key, os.O_RDWR, 0600)
-		defer destination.Close()
-		if err != nil {
-			return err
-		}
-
-		_, err := destination.Seek(downloadedSize, os.SEEK_SET)
-		if err != nil {
-			return err
-		}
-
-		remainingSize := objectSize - downloadedSize
-		objectReader, objectSize, err = s3c.GetPartial(args.source.bucket,
-			args.source.key, downloadedSize, remainingSize)
-		if err != nil {
-			return err
-		}
-
-		if !args.quiet {
-			bar.Set(int(downloadedSize))
-		}
-	}
-
-	writer := io.Writer(destination)
-	if !args.quiet {
-		// Start the bar now
-		bar.Start()
-		// create multi writer to feed data
-		writer = io.MultiWriter(destination, bar)
-	}
-
-	_, err = io.CopyN(writer, objectReader, objectSize)
-	if err != nil {
-		return err
-	}
-
-	bar.Finish()
-	info("Success!")
-	return nil
-}
-
-// <S3Object> <S3Object> or <S3Object> <S3Bucket>
-func thirdMode(c *cli.Context, args *cmdArgs) error {
-	var objectReader io.ReadCloser
-	var objectSize int64
-	var err error
-
-	s3cSource, err := getNewClient(globalDebugFlag, args.source.url.String())
-	if err != nil {
-		return err
-	}
-	// Send HEAD request to validate if file exists.
-	objectSize, _, err = s3cSource.Stat(args.source.bucket, args.source.key)
-	if err != nil {
-		return err
-	}
-
-	if args.destination.key == "" {
-		args.destination.key = args.source.key
-	}
-
-	// Check if the object already exists
-	s3cDest, err := getNewClient(globalDebugFlag, args.destination.url.String())
-	if err != nil {
-		return err
-	}
-	_, _, err = s3cDest.Stat(args.destination.bucket, args.destination.key)
-	switch os.IsNotExist(err) {
-	case true:
-		objectReader, _, err = s3cSource.Get(args.source.bucket, args.source.key)
-		if err != nil {
-			return err
-		}
-		err = s3cDest.Put(args.destination.bucket, args.destination.key, objectSize, objectReader)
-		if err != nil {
-			return err
-		}
-	case false:
-		return errors.New("Ranges not supported")
-	}
-
-	msg := fmt.Sprintf("%s/%s/%s uploaded -- to bucket:(%s/%s/%s)", args.source.host, args.source.bucket, args.source.key,
-		args.destination.host, args.destination.bucket, args.destination.key)
-	info(msg)
-	return nil
-}
-
-func fourthMode(c *cli.Context, args *cmdArgs) error {
-	if args.source.bucket == "" {
-		_, err := os.Stat(args.source.key)
-		if os.IsNotExist(err) {
-			return err
-		}
-		if args.destination.bucket == "" {
-			args.destination.bucket = args.source.key
-		}
-	} else {
-		if args.destination.key == "" {
-			args.destination.key = args.source.bucket
-		}
-		_, err := os.Stat(args.destination.key)
-		if os.IsNotExist(err) {
-			os.MkdirAll(args.destination.key, 0755)
-		}
-	}
-	return doRecursiveCP(c, args)
-}
-
 // doCopyCmd copies objects into and from a bucket or between buckets
-func doCopyCmd(c *cli.Context) {
+func doCopyCmd(ctx *cli.Context) {
-	var args *cmdArgs
+	if len(ctx.Args()) != 2 {
-	var err error
+		cli.ShowCommandHelp(ctx, "cp")
-
+		os.Exit(1)
-	args, err = parseArgs(c)
-	if err != nil {
-		fatal(err.Error())
 	}
 
-	if len(c.Args()) != 2 {
+	// var recursiveMode = ctx.Bool("recursive")
-		fatal("Invalid number of args")
+	sourceURL, err := parseURL(ctx.Args().First())
-	}
-	switch getMode(c.Bool("recursive"), args) {
-	case first:
-		err := firstMode(c, args)
 	if err != nil {
 		fatal(err.Error())
+		return
 	}
-	case second:
+
-		err := secondMode(c, args)
+	targetURL, err := parseURL(ctx.Args()[1])
 	if err != nil {
 		fatal(err.Error())
+		return
 	}
-	case third:
+
-		err := thirdMode(c, args)
+	sourceBucket, sourceObject, err := url2Object(sourceURL)
 	if err != nil {
 		fatal(err.Error())
+		return
 	}
-	case fourth:
+
-		err := fourthMode(c, args)
+	targetBucket, targetObject, err := url2Object(targetURL)
 	if err != nil {
 		fatal(err.Error())
+		return
 	}
-	default:
+
-		fatal("invalid args")
+	sourceClnt, err := getNewClient(globalDebugFlag, sourceURL)
+	if err != nil {
+		fatal(err.Error())
+		return
 	}
+
+	targetClnt, err := getNewClient(globalDebugFlag, targetURL)
+	if err != nil {
+		fatal(err.Error())
+		return
+	}
+
+	sourceReader, sourceSize, err := sourceClnt.Get(sourceBucket, sourceObject)
+	if err != nil {
+		fatal(err.Error())
+		return
+	}
+
+	var bar *pb.ProgressBar
+	if !globalQuietFlag {
+		bar = startBar(sourceSize)
+		bar.Start()
+		sourceReader = ioutil.NopCloser(io.TeeReader(sourceReader, bar))
+	}
+
+	if err = targetClnt.Put(targetBucket, targetObject, sourceSize, sourceReader); err != nil {
+		fatal(err.Error())
+		return
+	}
+
+	if !globalQuietFlag {
+		bar.Finish()
+		info("Success!")
+	}
+
+	return
 }

cmd-ls.go

@@ -55,19 +55,19 @@ func printObject(date time.Time, v int64, key string) {
 }
 
 // listObjectPrefix prints matching key prefix
-func listObjectPrefix(s3c client.Client, bucketName, objectName string, maxkeys int) {
+func listObjectPrefix(clnt client.Client, bucketName, objectName string, maxkeys int) {
 	var date time.Time
 	var size int64
 	var err error
 
-	size, date, err = s3c.Stat(bucketName, objectName)
+	size, date, err = clnt.Stat(bucketName, objectName)
 	var items []*client.Item
 	switch err {
 	case nil: // List a single object. Exact key
 		printObject(date, size, objectName)
 	case os.ErrNotExist:
 		// List all objects matching the key prefix
-		items, _, err = s3c.ListObjects(bucketName, "", objectName, "", maxkeys)
+		items, _, err = clnt.ListObjects(bucketName, "", objectName, "", maxkeys)
 		if err != nil {
 			fatal(err.Error())
 		}
@@ -82,11 +82,10 @@ func listObjectPrefix(s3c client.Client, bucketName, objectName string, maxkeys
 }
 
 // doListCmd lists objects inside a bucket
-func doListCmd(c *cli.Context) {
+func doListCmd(ctx *cli.Context) {
 	var items []*client.Item
-	// quiet := globalQuietFlag
 
-	urlStr, err := parseURL(c.Args().First())
+	urlStr, err := parseURL(ctx.Args().First())
 	if err != nil {
 		fatal(err.Error())
 	}
@@ -96,26 +95,26 @@ func doListCmd(c *cli.Context) {
 		fatal(err.Error())
 	}
 
-	s3c, err := getNewClient(globalDebugFlag, urlStr)
+	client, err := getNewClient(globalDebugFlag, urlStr)
 	if err != nil {
 		fatal(err.Error())
 	}
 
 	switch true {
 	case bucketName == "": // List all buckets
-		buckets, err := s3c.ListBuckets()
+		buckets, err := client.ListBuckets()
 		if err != nil {
 			fatal(err.Error())
 		}
 		printBuckets(buckets)
 	case objectName == "": // List objects in a bucket
-		items, _, err = s3c.ListObjects(bucketName, "", "", "", globalMaxKeys)
+		items, _, err = client.ListObjects(bucketName, "", "", "", globalMaxKeys)
 		if err != nil {
 			fatal(err.Error())
 		}
 		printObjects(items)
 	case objectName != "": // List objects matching the key prefix
-		listObjectPrefix(s3c, bucketName, objectName, globalMaxKeys)
+		listObjectPrefix(client, bucketName, objectName, globalMaxKeys)
 	default:
 		fatal(err.Error())
 	}

cmd-mb.go

@@ -22,8 +22,8 @@ import (
 )
 
 // doMakeBucketCmd creates a new bucket
-func doMakeBucketCmd(c *cli.Context) {
+func doMakeBucketCmd(ctx *cli.Context) {
-	urlStr, err := parseURL(c.Args().First())
+	urlStr, err := parseURL(ctx.Args().First())
 	if err != nil {
 		fatal(err.Error())
 	}
@@ -33,7 +33,7 @@ func doMakeBucketCmd(c *cli.Context) {
 		fatal(err.Error())
 	}
 
-	s3c, err := getNewClient(globalDebugFlag, urlStr)
+	clnt, err := getNewClient(globalDebugFlag, urlStr)
 	if err != nil {
 		fatal(err.Error())
 	}
@@ -42,7 +42,7 @@ func doMakeBucketCmd(c *cli.Context) {
 		fatal(errInvalidbucket.Error())
 	}
 
-	err = s3c.PutBucket(bucket)
+	err = clnt.PutBucket(bucket)
 	if err != nil {
 		fatal(err.Error())
 	}