mirror of
https://github.com/minio/mc.git
synced 2025-11-12 01:02:26 +03:00
428 lines
12 KiB
Go
428 lines
12 KiB
Go
/*
|
||
* Minio Client (C) 2014, 2015 Minio, Inc.
|
||
*
|
||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||
* you may not use this file except in compliance with the License.
|
||
* You may obtain a copy of the License at
|
||
*
|
||
* http://www.apache.org/licenses/LICENSE-2.0
|
||
*
|
||
* Unless required by applicable law or agreed to in writing, software
|
||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
* See the License for the specific language governing permissions and
|
||
* limitations under the License.
|
||
*/
|
||
|
||
package cmd
|
||
|
||
import (
|
||
"bufio"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"syscall"
|
||
|
||
"github.com/cheggaaa/pb"
|
||
"github.com/fatih/color"
|
||
"github.com/minio/cli"
|
||
"github.com/minio/mc/pkg/console"
|
||
"github.com/minio/minio/pkg/probe"
|
||
)
|
||
|
||
// cp command flags.
|
||
var (
|
||
cpFlags = []cli.Flag{
|
||
cli.BoolFlag{
|
||
Name: "help, h",
|
||
Usage: "Help of cp.",
|
||
},
|
||
cli.BoolFlag{
|
||
Name: "recursive, r",
|
||
Usage: "Copy recursively.",
|
||
},
|
||
}
|
||
)
|
||
|
||
// Copy command.
|
||
var cpCmd = cli.Command{
|
||
Name: "cp",
|
||
Usage: "Copy one or more objects to a target.",
|
||
Action: mainCopy,
|
||
Flags: append(cpFlags, globalFlags...),
|
||
CustomHelpTemplate: `NAME:
|
||
mc {{.Name}} - {{.Usage}}
|
||
|
||
USAGE:
|
||
mc {{.Name}} [FLAGS] SOURCE [SOURCE...] TARGET
|
||
|
||
FLAGS:
|
||
{{range .Flags}}{{.}}
|
||
{{end}}
|
||
EXAMPLES:
|
||
1. Copy a list of objects from local file system to Amazon S3 cloud storage.
|
||
$ mc {{.Name}} Music/*.ogg s3/jukebox/
|
||
|
||
2. Copy a folder recursively from Minio cloud storage to Amazon S3 cloud storage.
|
||
$ mc {{.Name}} --recursive play/mybucket/burningman2011/ s3/mybucket/
|
||
|
||
3. Copy multiple local folders recursively to Minio cloud storage.
|
||
$ mc {{.Name}} --recursive backup/2014/ backup/2015/ play/archive/
|
||
|
||
4. Copy a bucket recursively from aliased Amazon S3 cloud storage to local filesystem on Windows.
|
||
$ mc {{.Name}} --recursive s3\documents\2014\ C:\Backups\2014
|
||
|
||
5. Copy an object with name containing unicode characters to Amazon S3 cloud storage.
|
||
$ mc {{.Name}} 本語 s3/andoria/
|
||
|
||
6. Copy a local folder with space separated characters to Amazon S3 cloud storage.
|
||
$ mc {{.Name}} --recursive 'workdir/documents/May 2014/' s3/miniocloud
|
||
`,
|
||
}
|
||
|
||
// copyMessage container for file copy messages
|
||
type copyMessage struct {
|
||
Status string `json:"status"`
|
||
Source string `json:"source"`
|
||
Target string `json:"target"`
|
||
}
|
||
|
||
// String colorized copy message
|
||
func (c copyMessage) String() string {
|
||
return console.Colorize("Copy", fmt.Sprintf("‘%s’ -> ‘%s’", c.Source, c.Target))
|
||
}
|
||
|
||
// JSON jsonified copy message
|
||
func (c copyMessage) JSON() string {
|
||
c.Status = "success"
|
||
copyMessageBytes, e := json.Marshal(c)
|
||
fatalIf(probe.NewError(e), "Failed to marshal copy message.")
|
||
|
||
return string(copyMessageBytes)
|
||
}
|
||
|
||
// copyStatMessage container for copy accounting message
|
||
type copyStatMessage struct {
|
||
Total int64
|
||
Transferred int64
|
||
Speed float64
|
||
}
|
||
|
||
const (
|
||
// 5GiB.
|
||
fiveGB = 5 * 1024 * 1024 * 1024
|
||
)
|
||
|
||
// copyStatMessage copy accounting message
|
||
func (c copyStatMessage) String() string {
|
||
speedBox := pb.Format(int64(c.Speed)).To(pb.U_BYTES).String()
|
||
if speedBox == "" {
|
||
speedBox = "0 MB"
|
||
} else {
|
||
speedBox = speedBox + "/s"
|
||
}
|
||
message := fmt.Sprintf("Total: %s, Transferred: %s, Speed: %s", pb.Format(c.Total).To(pb.U_BYTES),
|
||
pb.Format(c.Transferred).To(pb.U_BYTES), speedBox)
|
||
return message
|
||
}
|
||
|
||
// doCopy - Copy a singe file from source to destination
|
||
func doCopy(cpURLs URLs, progressReader *progressBar, accountingReader *accounter) URLs {
|
||
if cpURLs.Error != nil {
|
||
cpURLs.Error = cpURLs.Error.Trace()
|
||
return cpURLs
|
||
}
|
||
|
||
if !globalQuiet && !globalJSON {
|
||
progressReader = progressReader.SetCaption(cpURLs.SourceContent.URL.String() + ": ")
|
||
}
|
||
|
||
sourceAlias := cpURLs.SourceAlias
|
||
sourceURL := cpURLs.SourceContent.URL
|
||
targetAlias := cpURLs.TargetAlias
|
||
targetURL := cpURLs.TargetContent.URL
|
||
length := cpURLs.SourceContent.Size
|
||
|
||
var progress io.Reader
|
||
if globalQuiet || globalJSON {
|
||
sourcePath := filepath.ToSlash(filepath.Join(sourceAlias, sourceURL.Path))
|
||
targetPath := filepath.ToSlash(filepath.Join(targetAlias, targetURL.Path))
|
||
printMsg(copyMessage{
|
||
Source: sourcePath,
|
||
Target: targetPath,
|
||
})
|
||
// Proxy reader to accounting reader only during quiet mode.
|
||
if globalQuiet || globalJSON {
|
||
progress = accountingReader
|
||
}
|
||
} else {
|
||
// Set up progress reader.
|
||
progress = progressReader.ProgressBar
|
||
}
|
||
// If source size is <= 5GB and operation is across same server type try to use Copy.
|
||
if length <= fiveGB && (sourceURL.Type == targetURL.Type) {
|
||
// FS -> FS Copy includes alias in path.
|
||
if sourceURL.Type == fileSystem {
|
||
sourcePath := filepath.ToSlash(filepath.Join(sourceAlias, sourceURL.Path))
|
||
err := copySourceStreamFromAlias(targetAlias, targetURL.String(), sourcePath, length, progress)
|
||
if err != nil {
|
||
cpURLs.Error = err.Trace(sourceURL.String())
|
||
return cpURLs
|
||
}
|
||
} else if sourceURL.Type == objectStorage {
|
||
// If source/target are object storage their aliases must be the same.
|
||
if sourceAlias == targetAlias {
|
||
// Do not include alias inside path for ObjStore -> ObjStore.
|
||
err := copySourceStreamFromAlias(targetAlias, targetURL.String(), sourceURL.Path, length, progress)
|
||
if err != nil {
|
||
cpURLs.Error = err.Trace(sourceURL.String())
|
||
return cpURLs
|
||
}
|
||
} else {
|
||
reader, err := getSourceStreamFromAlias(sourceAlias, sourceURL.String())
|
||
if err != nil {
|
||
cpURLs.Error = err.Trace(sourceURL.String())
|
||
return cpURLs
|
||
}
|
||
_, err = putTargetStreamFromAlias(targetAlias, targetURL.String(), reader, length, progress)
|
||
if err != nil {
|
||
cpURLs.Error = err.Trace(targetURL.String())
|
||
return cpURLs
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
// Standard GET/PUT for size > 5GB.
|
||
reader, err := getSourceStreamFromAlias(sourceAlias, sourceURL.String())
|
||
if err != nil {
|
||
cpURLs.Error = err.Trace(sourceURL.String())
|
||
return cpURLs
|
||
}
|
||
_, err = putTargetStreamFromAlias(targetAlias, targetURL.String(), reader, length, progress)
|
||
if err != nil {
|
||
cpURLs.Error = err.Trace(targetURL.String())
|
||
return cpURLs
|
||
}
|
||
}
|
||
cpURLs.Error = nil // just for safety
|
||
return cpURLs
|
||
}
|
||
|
||
// doCopyFake - Perform a fake copy to update the progress bar appropriately.
|
||
func doCopyFake(cpURLs URLs, progressReader *progressBar) URLs {
|
||
if !globalQuiet && !globalJSON {
|
||
progressReader.ProgressBar.Add64(cpURLs.SourceContent.Size)
|
||
}
|
||
return cpURLs
|
||
}
|
||
|
||
// doPrepareCopyURLs scans the source URL and prepares a list of objects for copying.
|
||
func doPrepareCopyURLs(session *sessionV7, trapCh <-chan bool) {
|
||
// Separate source and target. 'cp' can take only one target,
|
||
// but any number of sources.
|
||
sourceURLs := session.Header.CommandArgs[:len(session.Header.CommandArgs)-1]
|
||
targetURL := session.Header.CommandArgs[len(session.Header.CommandArgs)-1] // Last one is target
|
||
|
||
var totalBytes int64
|
||
var totalObjects int
|
||
|
||
// Access recursive flag inside the session header.
|
||
isRecursive := session.Header.CommandBoolFlags["recursive"]
|
||
|
||
// Create a session data file to store the processed URLs.
|
||
dataFP := session.NewDataWriter()
|
||
|
||
var scanBar scanBarFunc
|
||
if !globalQuiet && !globalJSON { // set up progress bar
|
||
scanBar = scanBarFactory()
|
||
}
|
||
|
||
URLsCh := prepareCopyURLs(sourceURLs, targetURL, isRecursive)
|
||
done := false
|
||
for !done {
|
||
select {
|
||
case cpURLs, ok := <-URLsCh:
|
||
if !ok { // Done with URL preparation
|
||
done = true
|
||
break
|
||
}
|
||
if cpURLs.Error != nil {
|
||
// Print in new line and adjust to top so that we don't print over the ongoing scan bar
|
||
if !globalQuiet && !globalJSON {
|
||
console.Eraseline()
|
||
}
|
||
if strings.Contains(cpURLs.Error.ToGoError().Error(), " is a folder.") {
|
||
errorIf(cpURLs.Error.Trace(), "Folder cannot be copied. Please use ‘...’ suffix.")
|
||
} else {
|
||
errorIf(cpURLs.Error.Trace(), "Unable to prepare URL for copying.")
|
||
}
|
||
break
|
||
}
|
||
|
||
jsonData, e := json.Marshal(cpURLs)
|
||
if e != nil {
|
||
session.Delete()
|
||
fatalIf(probe.NewError(e), "Unable to prepare URL for copying. Error in JSON marshaling.")
|
||
}
|
||
fmt.Fprintln(dataFP, string(jsonData))
|
||
if !globalQuiet && !globalJSON {
|
||
scanBar(cpURLs.SourceContent.URL.String())
|
||
}
|
||
|
||
totalBytes += cpURLs.SourceContent.Size
|
||
totalObjects++
|
||
case <-trapCh:
|
||
// Print in new line and adjust to top so that we don't print over the ongoing scan bar
|
||
if !globalQuiet && !globalJSON {
|
||
console.Eraseline()
|
||
}
|
||
session.Delete() // If we are interrupted during the URL scanning, we drop the session.
|
||
os.Exit(0)
|
||
}
|
||
}
|
||
session.Header.TotalBytes = totalBytes
|
||
session.Header.TotalObjects = totalObjects
|
||
session.Save()
|
||
}
|
||
|
||
func doCopySession(session *sessionV7) {
|
||
trapCh := signalTrap(os.Interrupt, syscall.SIGTERM)
|
||
|
||
if !session.HasData() {
|
||
doPrepareCopyURLs(session, trapCh)
|
||
}
|
||
|
||
// Enable accounting reader by default.
|
||
accntReader := newAccounter(session.Header.TotalBytes)
|
||
|
||
// Prepare URL scanner from session data file.
|
||
urlScanner := bufio.NewScanner(session.NewDataReader())
|
||
// isCopied returns true if an object has been already copied
|
||
// or not. This is useful when we resume from a session.
|
||
isCopied := isLastFactory(session.Header.LastCopied)
|
||
|
||
// Enable progress bar reader only during default mode.
|
||
var progressReader *progressBar
|
||
if !globalQuiet && !globalJSON { // set up progress bar
|
||
progressReader = newProgressBar(session.Header.TotalBytes)
|
||
}
|
||
|
||
// Wait on status of doCopy() operation.
|
||
var statusCh = make(chan URLs)
|
||
|
||
// Add a wait group.
|
||
var wg = new(sync.WaitGroup)
|
||
wg.Add(1)
|
||
|
||
// Go routine to monitor signal traps if any.
|
||
go func() {
|
||
defer wg.Done()
|
||
for {
|
||
select {
|
||
case <-trapCh:
|
||
// Receive interrupt notification.
|
||
if !globalQuiet && !globalJSON {
|
||
console.Eraseline()
|
||
}
|
||
session.CloseAndDie()
|
||
case cpURLs, ok := <-statusCh:
|
||
// Status channel is closed, we should return.
|
||
if !ok {
|
||
return
|
||
}
|
||
if cpURLs.Error == nil {
|
||
session.Header.LastCopied = cpURLs.SourceContent.URL.String()
|
||
session.Save()
|
||
} else {
|
||
// Print in new line and adjust to top so that we
|
||
// don't print over the ongoing progress bar.
|
||
if !globalQuiet && !globalJSON {
|
||
console.Eraseline()
|
||
}
|
||
errorIf(cpURLs.Error.Trace(cpURLs.SourceContent.URL.String()),
|
||
fmt.Sprintf("Failed to copy ‘%s’.", cpURLs.SourceContent.URL.String()))
|
||
// For all non critical errors we can continue for the
|
||
// remaining files.
|
||
switch cpURLs.Error.ToGoError().(type) {
|
||
// Handle this specifically for filesystem related errors.
|
||
case BrokenSymlink, TooManyLevelsSymlink, PathNotFound, PathInsufficientPermission:
|
||
continue
|
||
// Handle these specifically for object storage related errors.
|
||
case BucketNameEmpty, ObjectMissing, ObjectAlreadyExists, BucketDoesNotExist, BucketInvalid, ObjectOnGlacier:
|
||
continue
|
||
}
|
||
// For critical errors we should exit. Session
|
||
// can be resumed after the user figures out
|
||
// the problem.
|
||
session.CloseAndDie()
|
||
}
|
||
}
|
||
}
|
||
}()
|
||
|
||
// Loop through all urls.
|
||
for urlScanner.Scan() {
|
||
var cpURLs URLs
|
||
// Unmarshal copyURLs from each line.
|
||
json.Unmarshal([]byte(urlScanner.Text()), &cpURLs)
|
||
|
||
// Verify if previously copied, notify progress bar.
|
||
if isCopied(cpURLs.SourceContent.URL.String()) {
|
||
statusCh <- doCopyFake(cpURLs, progressReader)
|
||
} else {
|
||
statusCh <- doCopy(cpURLs, progressReader, accntReader)
|
||
}
|
||
}
|
||
|
||
// Close the goroutine.
|
||
close(statusCh)
|
||
|
||
// Wait for the goroutines to finish.
|
||
wg.Wait()
|
||
|
||
if !globalQuiet && !globalJSON {
|
||
if progressReader.ProgressBar.Get() > 0 {
|
||
progressReader.ProgressBar.Finish()
|
||
}
|
||
} else {
|
||
accntStat := accntReader.Stat()
|
||
cpStatMessage := copyStatMessage{
|
||
Total: accntStat.Total,
|
||
Transferred: accntStat.Transferred,
|
||
Speed: accntStat.Speed,
|
||
}
|
||
console.Println(console.Colorize("Copy", cpStatMessage.String()))
|
||
}
|
||
}
|
||
|
||
// mainCopy is the entry point for cp command.
|
||
func mainCopy(ctx *cli.Context) {
|
||
// Set global flags from context.
|
||
setGlobalsFromContext(ctx)
|
||
|
||
// check 'copy' cli arguments.
|
||
checkCopySyntax(ctx)
|
||
|
||
// Additional command speific theme customization.
|
||
console.SetColor("Copy", color.New(color.FgGreen, color.Bold))
|
||
|
||
session := newSessionV7()
|
||
session.Header.CommandType = "cp"
|
||
session.Header.CommandBoolFlags["recursive"] = ctx.Bool("recursive")
|
||
|
||
var e error
|
||
if session.Header.RootPath, e = os.Getwd(); e != nil {
|
||
session.Delete()
|
||
fatalIf(probe.NewError(e), "Unable to get current working folder.")
|
||
}
|
||
|
||
// extract URLs.
|
||
session.Header.CommandArgs = ctx.Args()
|
||
doCopySession(session)
|
||
session.Delete()
|
||
}
|