mc/cmd-cp-recursive.go

/*
 * Mini Copy, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"io"
	"runtime"
	"strings"

	"fmt"

	"github.com/cheggaaa/pb"
	"github.com/minio-io/mc/pkg/client"
	"github.com/minio-io/mc/pkg/console"
	"github.com/minio-io/minio/pkg/iodine"
)

const (
	pathSeparator        = "/"
	pathSeparatorWindows = "\\"
)

func getSourceURL(sourceURL, objectName string) string {
	if client.GetURLType(sourceURL) == client.URLFilesystem {
		if runtime.GOOS == "windows" {
			return strings.TrimSuffix(sourceURL, pathSeparator) + pathSeparatorWindows + objectName
		}
	}
	return strings.TrimSuffix(sourceURL, pathSeparator) + pathSeparator + objectName
}

// getSourceObjectURLMap - get list of all source object and its URLs in a map
func getSourceObjectURLMap(manager clientManager, sourceURL string) (sourceObjectURLMap map[string]string, err error) {
	sourceClnt, err := manager.getNewClient(sourceURL, globalDebugFlag)
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	objects, err := sourceClnt.List()
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	sourceObjectURLMap = make(map[string]string)
	for _, object := range objects {
		sourceObjectURLMap[object.Name] = getSourceURL(sourceURL, object.Name)
	}
	return sourceObjectURLMap, nil
}

// getRecursiveTargetWriter - recursively get target writers to initiate copying data
func getRecursiveTargetWriter(manager clientManager, targetURL, md5Hex string, length int64) (io.WriteCloser, error) {
	targetClnt, err := manager.getNewClient(targetURL, globalDebugFlag)
	if err != nil {
		return nil, iodine.New(err, map[string]string{"failedURL": targetURL})
	}

	// For object storage URL's do a StatBucket() and PutBucket(), not necessary for fs client
	if client.GetURLType(targetURL) != client.URLFilesystem {
		// check if bucket is valid, if not found create it
		if err := targetClnt.Stat(); err != nil {
			switch iodine.ToError(err).(type) {
			case client.BucketNotFound:
				err := targetClnt.PutBucket()
				if err != nil {
					return nil, iodine.New(err, map[string]string{"failedURL": targetURL})
				}
			default:
				return nil, iodine.New(err, map[string]string{"failedURL": targetURL})
			}
		}
	}
	return targetClnt.Put(md5Hex, length)
}

// getRecursiveTargetWriters - convenient wrapper around getRecursiveTarget
func getRecursiveTargetWriters(manager clientManager, targetURLs []string, md5Hex string, length int64) ([]io.WriteCloser, error) {
	var targetWriters []io.WriteCloser
	for _, targetURL := range targetURLs {
		writer, err := getRecursiveTargetWriter(manager, targetURL, md5Hex, length)
		if err != nil {
			// on error close all writers
			for _, targetWriter := range targetWriters {
				targetWriter.Close()
			}
			// return error so that Multiwriter doesn't get control
			return nil, iodine.New(err, nil)
		}
		targetWriters = append(targetWriters, writer)
	}
	return targetWriters, nil
}

// doCopyCmdRecursive - copy bucket to bucket
func doCopyCmdRecursive(manager clientManager, sourceURL string, targetURLs []string) (string, error) {
	sourceObjectURLMap, err := getSourceObjectURLMap(manager, sourceURL)
	if err != nil {
		err = iodine.New(err, nil)
		msg := fmt.Sprintf("mc: Getting list of objects from source URL: [%s] failed with following reason: [%s]\n",
			sourceURL, iodine.ToError(err))
		return msg, err
	}
	// do not exit, continue even for failures
	for sourceObjectName, sourceObjectURL := range sourceObjectURLMap {
		sourceClnt, err := manager.getNewClient(sourceObjectURL, globalDebugFlag)
		if err != nil {
			err := iodine.New(err, nil)
			msg := fmt.Sprintf("Instantiating a new client for URL [%s] failed", sourceObjectURL)
			return msg, err
		}
		reader, length, md5hex, err := sourceClnt.Get()
		if err != nil {
			err = iodine.New(err, nil)
			msg := fmt.Sprintf("Reading from source URL: [%s] failed", sourceURL)
			return msg, err
		}
		// Construct full target URL path based on source object name
		var newTargetURLs []string
		for _, targetURL := range targetURLs {
			targetURL := strings.TrimSuffix(targetURL, pathSeparator) + pathSeparator + sourceObjectName
			newTargetURLs = append(newTargetURLs, targetURL)
		}
		writeClosers, err := getRecursiveTargetWriters(manager, newTargetURLs, md5hex, length)
		if err != nil {
			return "Writing to target URLs failed", iodine.New(err, nil)
		}

		var writers []io.Writer
		for _, writer := range writeClosers {
			writers = append(writers, writer)
		}

		// set up progress bar
		var bar *pb.ProgressBar
		if !globalQuietFlag {
			bar = startBar(length)
			bar.Start()
			writers = append(writers, bar)
		}

		// write progress bar
		multiWriter := io.MultiWriter(writers...)

		// copy data to writers
		_, err = io.CopyN(multiWriter, reader, length)
		if err != nil {
			err := iodine.New(err, nil)
			return "Copying data from source to target(s) failed", err
		}

		// close writers
		for _, writer := range writeClosers {
			err := writer.Close()
			if err != nil {
				err := iodine.New(err, nil)
				return "Connections still active, one or more writes have failed", err
			}
		}
		if !globalDebugFlag {
			bar.Finish()
			console.Infoln("Success!")
		}
	}
	return "", nil
}