diff --git a/cmd-common.go b/cmd-common.go index d83e35ea..4ad4e8ad 100644 --- a/cmd-common.go +++ b/cmd-common.go @@ -46,37 +46,6 @@ func startBar(size int64) *pb.ProgressBar { return bar } -// NewClient - get new client -func getNewClient(debug bool, url string) (cl client.Client, err error) { - config, err := getMcConfig() - if err != nil { - return nil, err - } - - hostCfg, err := getHostConfig(config.DefaultHost) - if err != nil { - return nil, err - } - - var auth s3.Auth - auth.AccessKeyID = hostCfg.Auth.AccessKeyID - auth.SecretAccessKey = hostCfg.Auth.SecretAccessKey - - if debug { - trace := s3.Trace{ - BodyTraceFlag: false, - RequestTransportFlag: true, - Writer: nil, - } - traceTransport := s3.GetNewTraceTransport(trace, http.DefaultTransport) - cl = s3.GetNewClient(&auth, url, traceTransport) - } else { - cl = s3.GetNewClient(&auth, url, http.DefaultTransport) - } - - return cl, nil -} - func parseDestinationArgs(urlParsed *url.URL, destination, source object) (object, error) { switch true { case urlParsed.Scheme == "http" || urlParsed.Scheme == "https": @@ -226,3 +195,34 @@ func parseArgs(c *cli.Context) (args *cmdArgs, err error) { func getMcBashCompletionFilename() string { return path.Join(getMcConfigDir(), "mc.bash_completion") } + +// NewClient - get new client +func getNewClient(debug bool, urlStr string) (clnt client.Client, err error) { + config, err := getMcConfig() + if err != nil { + return nil, err + } + + hostCfg, err := getHostConfig(config.DefaultHost) + if err != nil { + return nil, err + } + + var auth s3.Auth + auth.AccessKeyID = hostCfg.Auth.AccessKeyID + auth.SecretAccessKey = hostCfg.Auth.SecretAccessKey + + if debug { + trace := s3.Trace{ + BodyTraceFlag: false, + RequestTransportFlag: true, + Writer: nil, + } + traceTransport := s3.GetNewTraceTransport(trace, http.DefaultTransport) + clnt = s3.GetNewClient(&auth, urlStr, traceTransport) + } else { + clnt = s3.GetNewClient(&auth, urlStr, http.DefaultTransport) + } + + return clnt, nil +} diff --git a/cmd-config.go b/cmd-config.go index 89da74fa..ded4c585 100644 --- a/cmd-config.go +++ b/cmd-config.go @@ -132,8 +132,8 @@ func loadMcConfig() (config *mcConfig, err error) { } // saveConfig writes configuration data in json format to config file. -func saveConfig(c *cli.Context) error { - configData, err := parseConfigInput(c) +func saveConfig(ctx *cli.Context) error { + configData, err := parseConfigInput(ctx) if err != nil { return err } @@ -284,12 +284,12 @@ func getHostConfig(hostURL string) (*hostConfig, error) { } // doConfigCmd is the handler for "mc config" sub-command. -func doConfigCmd(c *cli.Context) { +func doConfigCmd(ctx *cli.Context) { switch true { - case c.Bool("completion") == true: + case ctx.Bool("completion") == true: getBashCompletion() default: - err := saveConfig(c) + err := saveConfig(ctx) if os.IsExist(err) { log.Fatalf("mc: Please rename your current configuration file [%s]\n", getMcConfigFilename()) } @@ -298,6 +298,5 @@ func doConfigCmd(c *cli.Context) { log.Fatalf("mc: Unable to generate config file [%s]. \nError: %v\n", getMcConfigFilename(), err) } info("Configuration written to " + getMcConfigFilename() + "\n") - } } diff --git a/cmd-cp-recursive.go b/cmd-cp-recursive.go deleted file mode 100644 index 26b11871..00000000 --- a/cmd-cp-recursive.go +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "errors" - "fmt" - "io" - "os" - "path" - "path/filepath" - "strings" - - "github.com/cheggaaa/pb" - "github.com/minio-io/cli" - "github.com/minio-io/mc/pkg/client" - "github.com/minio-io/mc/pkg/client/s3" -) - -type walk struct { - s3 client.Client - args *cmdArgs -} - -func (w *walk) putWalk(p string, i os.FileInfo, err error) error { - if i.IsDir() { - return nil - } - if !i.Mode().IsRegular() { - return nil - } - parts := strings.SplitN(p, "/", 2) - bucketname := w.args.destination.bucket - key := parts[1] - - bodyFile, err := os.Open(p) - defer bodyFile.Close() - - var size int64 - size, _, err = w.s3.Stat(bucketname, key) - if os.IsExist(err) || size != 0 { - msg := fmt.Sprintf("%s is already uploaded -- to bucket:%s/%s/%s", - key, w.args.destination.host, bucketname, key) - info(msg) - return nil - } - var bar *pb.ProgressBar - if !w.args.quiet { - // get progress bar - bar = startBar(i.Size()) - } - newreader := io.Reader(bodyFile) - if !w.args.quiet { - bar.Start() - newreader = io.TeeReader(bodyFile, bar) - } - err = w.s3.Put(bucketname, key, i.Size(), newreader) - if err != nil { - return err - } - if !w.args.quiet { - bar.Finish() - info("Success!") - } - return nil -} - -// isBucketExist checks if a bucket exists -func isBucketExist(bucketName string, v []*client.Bucket) bool { - for _, b := range v { - if bucketName == b.Name { - return true - } - } - return false -} - -func sourceValidate(input string) error { - if !s3.IsValidBucketName(input) { - return fmt.Errorf("Invalid input bucket name [%s]", input) - } - st, err := os.Stat(input) - if os.IsNotExist(err) { - return err - } - if !st.IsDir() { - return errors.New("Should be a directory") - } - return nil -} - -// doRecursiveCP recursively copies objects from source to destination -func doRecursiveCP(c *cli.Context, args *cmdArgs) error { - var buckets []*client.Bucket - switch true { - case args.source.bucket == "": - input := path.Clean(args.source.key) - if err := sourceValidate(input); err != nil { - return err - } - s3c, err := getNewClient(globalDebugFlag, args.destination.url.String()) - if err != nil { - return err - } - p := &walk{s3c, args} - buckets, err = s3c.ListBuckets() - if !isBucketExist(args.destination.bucket, buckets) { - // Create bucketname, before uploading files - err = s3c.PutBucket(args.destination.bucket) - if err != nil { - return err - } - } - err = filepath.Walk(input, p.putWalk) - if err != nil { - return err - } - case args.destination.bucket == "": - s3c, err := getNewClient(globalDebugFlag, args.source.url.String()) - if err != nil { - return err - } - items, _, err := s3c.ListObjects(args.source.bucket, "", "", "", globalMaxKeys) - if err != nil { - return err - } - root := args.destination.key - writeObjects := func(v []*client.Item) error { - if len(v) > 0 { - // Items are already sorted - for _, b := range v { - args.source.key = b.Key - os.MkdirAll(path.Join(root, path.Dir(b.Key)), 0755) - args.destination.key = path.Join(root, b.Key) - err := secondMode(c, args) - if err != nil { - return err - } - } - } - return nil - } - err = writeObjects(items) - if err != nil { - return err - } - } - return nil -} diff --git a/cmd-cp.go b/cmd-cp.go index 2da1157b..e0ae59b6 100644 --- a/cmd-cp.go +++ b/cmd-cp.go @@ -17,286 +17,80 @@ package main import ( - "errors" - "fmt" "io" + "io/ioutil" "os" "github.com/cheggaaa/pb" "github.com/minio-io/cli" ) -// Different modes of cp operation -const ( - first = iota // or - second // or . - third // or - fourth // or or - invalid -) +// doCopyCmd copies objects into and from a bucket or between buckets +func doCopyCmd(ctx *cli.Context) { + if len(ctx.Args()) != 2 { + cli.ShowCommandHelp(ctx, "cp") + os.Exit(1) + } -// Get current mode of operation from available arguments and options -func getMode(recursive bool, args *cmdArgs) int { - switch recursive { - case false: - switch true { - // or - case args.source.bucket == "" && args.destination.bucket != "": - return first - // or . - case args.source.bucket != "" && args.source.key != "" && args.destination.bucket == "": - return second - // or - case args.source.bucket != "" && args.destination.bucket != "" && args.source.key != "": - return third - } - case true: - switch true { - // or or - case args.source.bucket != "" || args.source.key != "": - return fourth - } - } - return invalid -} - -// First mode or -func firstMode(c *cli.Context, args *cmdArgs) error { - if args.source.key == "" { - return errors.New("invalid args") - } - st, err := os.Stat(args.source.key) - if os.IsNotExist(err) { - return err - } - if st.IsDir() { - msg := fmt.Sprintf("omitting directory '%s'", st.Name()) - return errors.New(msg) - } - size := st.Size() - source, err := os.Open(args.source.key) - defer source.Close() + // var recursiveMode = ctx.Bool("recursive") + sourceURL, err := parseURL(ctx.Args().First()) if err != nil { - return err + fatal(err.Error()) + return } + + targetURL, err := parseURL(ctx.Args()[1]) + if err != nil { + fatal(err.Error()) + return + } + + sourceBucket, sourceObject, err := url2Object(sourceURL) + if err != nil { + fatal(err.Error()) + return + } + + targetBucket, targetObject, err := url2Object(targetURL) + if err != nil { + fatal(err.Error()) + return + } + + sourceClnt, err := getNewClient(globalDebugFlag, sourceURL) + if err != nil { + fatal(err.Error()) + return + } + + targetClnt, err := getNewClient(globalDebugFlag, targetURL) + if err != nil { + fatal(err.Error()) + return + } + + sourceReader, sourceSize, err := sourceClnt.Get(sourceBucket, sourceObject) + if err != nil { + fatal(err.Error()) + return + } + var bar *pb.ProgressBar - if !args.quiet { - // get progress bar - bar = startBar(size) - } - // http://. is specified without key - if args.destination.key == "" { - args.destination.key = args.source.key - } - s3c, err := getNewClient(globalDebugFlag, args.destination.url.String()) - if err != nil { - return err - } - newreader := io.Reader(source) - if !args.quiet { + if !globalQuietFlag { + bar = startBar(sourceSize) bar.Start() - newreader = io.TeeReader(source, bar) + sourceReader = ioutil.NopCloser(io.TeeReader(sourceReader, bar)) } - err = s3c.Put(args.destination.bucket, args.destination.key, size, newreader) - if err != nil { - return err + + if err = targetClnt.Put(targetBucket, targetObject, sourceSize, sourceReader); err != nil { + fatal(err.Error()) + return } - if !args.quiet { + + if !globalQuietFlag { bar.Finish() info("Success!") } - return nil -} - -// Second mode or . -func secondMode(c *cli.Context, args *cmdArgs) error { - var objectReader io.ReadCloser - var objectSize, downloadedSize int64 - var destination *os.File - var err error - var st os.FileInfo - - s3c, err := getNewClient(globalDebugFlag, args.source.url.String()) - if err != nil { - return err - } - // Send HEAD request to validate if file exists. - objectSize, _, err = s3c.Stat(args.source.bucket, args.source.key) - if err != nil { - return err - } - - var bar *pb.ProgressBar - if !args.quiet { - // get progress bar - bar = startBar(objectSize) - } - - // Check if the object already exists - st, err = os.Stat(args.destination.key) - switch os.IsNotExist(err) { - case true: - // Create if it doesn't exist - destination, err = os.Create(args.destination.key) - defer destination.Close() - if err != nil { - return err - } - objectReader, _, err = s3c.Get(args.source.bucket, args.source.key) - if err != nil { - return err - } - case false: - downloadedSize = st.Size() - // Verify if file is already downloaded - if downloadedSize == objectSize { - msg := fmt.Sprintf("%s object has been already downloaded", args.destination.key) - return errors.New(msg) - } - - destination, err = os.OpenFile(args.destination.key, os.O_RDWR, 0600) - defer destination.Close() - if err != nil { - return err - } - - _, err := destination.Seek(downloadedSize, os.SEEK_SET) - if err != nil { - return err - } - - remainingSize := objectSize - downloadedSize - objectReader, objectSize, err = s3c.GetPartial(args.source.bucket, - args.source.key, downloadedSize, remainingSize) - if err != nil { - return err - } - - if !args.quiet { - bar.Set(int(downloadedSize)) - } - } - - writer := io.Writer(destination) - if !args.quiet { - // Start the bar now - bar.Start() - // create multi writer to feed data - writer = io.MultiWriter(destination, bar) - } - - _, err = io.CopyN(writer, objectReader, objectSize) - if err != nil { - return err - } - - bar.Finish() - info("Success!") - return nil -} - -// or -func thirdMode(c *cli.Context, args *cmdArgs) error { - var objectReader io.ReadCloser - var objectSize int64 - var err error - - s3cSource, err := getNewClient(globalDebugFlag, args.source.url.String()) - if err != nil { - return err - } - // Send HEAD request to validate if file exists. - objectSize, _, err = s3cSource.Stat(args.source.bucket, args.source.key) - if err != nil { - return err - } - - if args.destination.key == "" { - args.destination.key = args.source.key - } - - // Check if the object already exists - s3cDest, err := getNewClient(globalDebugFlag, args.destination.url.String()) - if err != nil { - return err - } - _, _, err = s3cDest.Stat(args.destination.bucket, args.destination.key) - switch os.IsNotExist(err) { - case true: - objectReader, _, err = s3cSource.Get(args.source.bucket, args.source.key) - if err != nil { - return err - } - err = s3cDest.Put(args.destination.bucket, args.destination.key, objectSize, objectReader) - if err != nil { - return err - } - case false: - return errors.New("Ranges not supported") - } - - msg := fmt.Sprintf("%s/%s/%s uploaded -- to bucket:(%s/%s/%s)", args.source.host, args.source.bucket, args.source.key, - args.destination.host, args.destination.bucket, args.destination.key) - info(msg) - return nil -} - -func fourthMode(c *cli.Context, args *cmdArgs) error { - if args.source.bucket == "" { - _, err := os.Stat(args.source.key) - if os.IsNotExist(err) { - return err - } - if args.destination.bucket == "" { - args.destination.bucket = args.source.key - } - } else { - if args.destination.key == "" { - args.destination.key = args.source.bucket - } - _, err := os.Stat(args.destination.key) - if os.IsNotExist(err) { - os.MkdirAll(args.destination.key, 0755) - } - } - return doRecursiveCP(c, args) -} - -// doCopyCmd copies objects into and from a bucket or between buckets -func doCopyCmd(c *cli.Context) { - var args *cmdArgs - var err error - - args, err = parseArgs(c) - if err != nil { - fatal(err.Error()) - } - - if len(c.Args()) != 2 { - fatal("Invalid number of args") - } - switch getMode(c.Bool("recursive"), args) { - case first: - err := firstMode(c, args) - if err != nil { - fatal(err.Error()) - } - case second: - err := secondMode(c, args) - if err != nil { - fatal(err.Error()) - } - case third: - err := thirdMode(c, args) - if err != nil { - fatal(err.Error()) - } - case fourth: - err := fourthMode(c, args) - if err != nil { - fatal(err.Error()) - } - default: - fatal("invalid args") - } + + return } diff --git a/cmd-ls.go b/cmd-ls.go index 7652c85a..95caee60 100644 --- a/cmd-ls.go +++ b/cmd-ls.go @@ -55,19 +55,19 @@ func printObject(date time.Time, v int64, key string) { } // listObjectPrefix prints matching key prefix -func listObjectPrefix(s3c client.Client, bucketName, objectName string, maxkeys int) { +func listObjectPrefix(clnt client.Client, bucketName, objectName string, maxkeys int) { var date time.Time var size int64 var err error - size, date, err = s3c.Stat(bucketName, objectName) + size, date, err = clnt.Stat(bucketName, objectName) var items []*client.Item switch err { case nil: // List a single object. Exact key printObject(date, size, objectName) case os.ErrNotExist: // List all objects matching the key prefix - items, _, err = s3c.ListObjects(bucketName, "", objectName, "", maxkeys) + items, _, err = clnt.ListObjects(bucketName, "", objectName, "", maxkeys) if err != nil { fatal(err.Error()) } @@ -82,11 +82,10 @@ func listObjectPrefix(s3c client.Client, bucketName, objectName string, maxkeys } // doListCmd lists objects inside a bucket -func doListCmd(c *cli.Context) { +func doListCmd(ctx *cli.Context) { var items []*client.Item - // quiet := globalQuietFlag - urlStr, err := parseURL(c.Args().First()) + urlStr, err := parseURL(ctx.Args().First()) if err != nil { fatal(err.Error()) } @@ -96,26 +95,26 @@ func doListCmd(c *cli.Context) { fatal(err.Error()) } - s3c, err := getNewClient(globalDebugFlag, urlStr) + client, err := getNewClient(globalDebugFlag, urlStr) if err != nil { fatal(err.Error()) } switch true { case bucketName == "": // List all buckets - buckets, err := s3c.ListBuckets() + buckets, err := client.ListBuckets() if err != nil { fatal(err.Error()) } printBuckets(buckets) case objectName == "": // List objects in a bucket - items, _, err = s3c.ListObjects(bucketName, "", "", "", globalMaxKeys) + items, _, err = client.ListObjects(bucketName, "", "", "", globalMaxKeys) if err != nil { fatal(err.Error()) } printObjects(items) case objectName != "": // List objects matching the key prefix - listObjectPrefix(s3c, bucketName, objectName, globalMaxKeys) + listObjectPrefix(client, bucketName, objectName, globalMaxKeys) default: fatal(err.Error()) } diff --git a/cmd-mb.go b/cmd-mb.go index bc09e04e..11f58e1d 100644 --- a/cmd-mb.go +++ b/cmd-mb.go @@ -22,8 +22,8 @@ import ( ) // doMakeBucketCmd creates a new bucket -func doMakeBucketCmd(c *cli.Context) { - urlStr, err := parseURL(c.Args().First()) +func doMakeBucketCmd(ctx *cli.Context) { + urlStr, err := parseURL(ctx.Args().First()) if err != nil { fatal(err.Error()) } @@ -33,7 +33,7 @@ func doMakeBucketCmd(c *cli.Context) { fatal(err.Error()) } - s3c, err := getNewClient(globalDebugFlag, urlStr) + clnt, err := getNewClient(globalDebugFlag, urlStr) if err != nil { fatal(err.Error()) } @@ -42,7 +42,7 @@ func doMakeBucketCmd(c *cli.Context) { fatal(errInvalidbucket.Error()) } - err = s3c.PutBucket(bucket) + err = clnt.PutBucket(bucket) if err != nil { fatal(err.Error()) }