mc/cmd-common.go

/*
 * Mini Copy, (C) 2014, 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"net"
	"net/url"
	"path/filepath"
	"time"

	"io"

	"github.com/cheggaaa/pb"
	"github.com/minio-io/mc/pkg/client"
	"github.com/minio-io/mc/pkg/client/fs"
	"github.com/minio-io/mc/pkg/client/s3"
	"github.com/minio-io/mc/pkg/console"
	"github.com/minio-io/minio/pkg/iodine"
)

// isValidRetry - check if we should retry for the given error sequence
func isValidRetry(err error) bool {
	if err == nil {
		return false
	}
	// DNSError, Network Operation error
	switch e := iodine.ToError(err).(type) {
	case *net.DNSError:
		return true
	case *net.OpError:
		switch e.Op {
		case "read", "write", "dial":
			return true
		}
	}
	return false
}

// StartBar -- instantiate a progressbar
func startBar(size int64) *pb.ProgressBar {
	bar := pb.New(int(size))
	bar.SetUnits(pb.U_BYTES)
	bar.SetRefreshRate(time.Millisecond * 10)
	bar.NotPrint = true
	bar.ShowSpeed = true
	bar.Callback = func(s string) {
		// Colorize
		console.Info("\r" + s)
	}
	// Feels like wget
	bar.Format("[=> ]")
	return bar
}

// clientManager interface for mock tests
type clientManager interface {
	getSourceReader(sourceURL string) (reader io.ReadCloser, length int64, md5hex string, err error)
	getTargetWriter(targetURL string, md5Hex string, length int64) (io.WriteCloser, error)
	getNewClient(urlStr string, debug bool) (clnt client.Client, err error)
}

type mcClientManager struct{}

// getSourceReader -
func (manager mcClientManager) getSourceReader(sourceURL string) (reader io.ReadCloser, length int64, md5hex string, err error) {
	sourceClnt, err := manager.getNewClient(sourceURL, globalDebugFlag)
	if err != nil {
		return nil, 0, "", iodine.New(err, map[string]string{"failedURL": sourceURL})
	}
	// check if the bucket is valid
	// For object storage URL's do a StatBucket(), not necessary for fs client
	if client.GetURLType(sourceURL) != client.URLFilesystem {
		if err := sourceClnt.Stat(); err != nil {
			return nil, 0, "", iodine.New(err, map[string]string{"failedURL": sourceURL})
		}
	}
	return sourceClnt.Get()
}

// getTargetWriter -
func (manager mcClientManager) getTargetWriter(targetURL string, md5Hex string, length int64) (io.WriteCloser, error) {
	targetClnt, err := manager.getNewClient(targetURL, globalDebugFlag)
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	// check if bucket is valid, if not create it on target
	// For object storage URL's do a StatBucket() and PutBucket(), not necessary for fs client
	if client.GetURLType(targetURL) != client.URLFilesystem {
		if err := targetClnt.Stat(); err != nil {
			switch iodine.ToError(err).(type) {
			case client.BucketNotFound:
				err := targetClnt.PutBucket()
				if err != nil {
					return nil, iodine.New(err, map[string]string{"failedURL": targetURL})
				}
			default:
				return nil, iodine.New(err, map[string]string{"failedURL": targetURL})
			}
		}
	}
	return targetClnt.Put(md5Hex, length)
}

func getFilesystemAbsURL(u *url.URL) (string, error) {
	var absURLStr string
	var err error
	switch true {
	case u.Scheme == "file" && u.IsAbs():
		absURLStr, err = filepath.Abs(filepath.Clean(u.Path))
		if err != nil {
			return "", iodine.New(err, nil)
		}
	case filepath.IsAbs(u.String()):
		absURLStr, err = filepath.Abs(filepath.Clean(u.String()))
		if err != nil {
			return "", iodine.New(err, nil)
		}
	default:
		absURLStr, err = filepath.Abs(filepath.Clean(u.String()))
		if err != nil {
			return "", iodine.New(err, nil)
		}
	}
	return absURLStr, nil
}

// getNewClient gives a new client interface
func (manager mcClientManager) getNewClient(urlStr string, debug bool) (clnt client.Client, err error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, iodine.New(errInvalidURL{url: urlStr}, nil)
	}
	switch client.GetURLType(urlStr) {
	case client.URLObject: // Minio and S3 compatible object storage
		hostCfg, err := getHostConfig(u.String())
		if err != nil {
			return nil, iodine.New(err, nil)
		}
		if hostCfg == nil {
			return nil, iodine.New(errInvalidAuth{}, nil)
		}
		auth := new(s3.Auth)
		if _, ok := hostCfg["Auth.AccessKeyID"]; ok {
			auth.AccessKeyID = hostCfg["Auth.AccessKeyID"]
		}
		if _, ok := hostCfg["Auth.SecretAccessKey"]; ok {
			auth.SecretAccessKey = hostCfg["Auth.SecretAccessKey"]
		}
		clnt = s3.GetNewClient(urlStr, auth, mcUserAgent, debug)
		return clnt, nil
	case client.URLFilesystem:
		absURLStr, err := getFilesystemAbsURL(u)
		if err != nil {
			return nil, iodine.New(err, nil)
		}
		clnt = fs.GetNewClient(absURLStr)
		return clnt, nil
	default:
		return nil, iodine.New(errUnsupportedScheme{
			scheme: client.GetURLType(urlStr),
			url:    urlStr,
		}, nil)
	}
}

// getTargetWriters -
func getTargetWriters(manager clientManager, targetURLs []string, md5Hex string, length int64) ([]io.WriteCloser, error) {
	var targetWriters []io.WriteCloser
	for _, targetURL := range targetURLs {
		writer, err := manager.getTargetWriter(targetURL, md5Hex, length)
		if err != nil {
			// close all writers
			for _, targetWriter := range targetWriters {
				targetWriter.Close()
			}
			return nil, iodine.New(err, nil)
		}
		targetWriters = append(targetWriters, writer)
	}
	return targetWriters, nil
}