/* * Minio Client (C) 2015 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "io" "os" "runtime" "strings" "sync" "github.com/minio/mc/pkg/client" "github.com/minio/mc/pkg/client/fs" "github.com/minio/mc/pkg/client/s3v2" "github.com/minio/mc/pkg/client/s3v4" "github.com/minio/minio-xl/pkg/probe" ) // Check if the target URL represents folder. It may or may not exist yet. func isTargetURLDir(targetURL string) bool { targetURLParse := client.NewURL(targetURL) _, targetContent, perr := url2Stat(targetURL) if perr != nil { if targetURLParse.Path == string(targetURLParse.Separator) && targetURLParse.Scheme != "" { return false } if strings.HasSuffix(targetURLParse.Path, string(targetURLParse.Separator)) { return true } return false } if !targetContent.Type.IsDir() { // Target is a dir. return false } return true } // getSource gets a reader from URL func getSource(sourceURL string) (reader io.ReadCloser, length int64, err *probe.Error) { sourceClnt, err := url2Client(sourceURL) if err != nil { return nil, 0, err.Trace() } return sourceClnt.GetObject(0, 0) } // putTarget writes to URL from reader. If length=0, read until EOF. func putTarget(targetURL string, length int64, reader io.Reader) *probe.Error { targetClnt, err := url2Client(targetURL) if err != nil { return err.Trace(targetURL) } err = targetClnt.PutObject(length, reader) if err != nil { return err.Trace(targetURL) } return nil } // putTargets writes to URL from reader. If length=0, read until EOF. func putTargets(targetURLs []string, length int64, reader io.Reader) *probe.Error { var tgtReaders []*io.PipeReader var tgtWriters []*io.PipeWriter var tgtClients []client.Client errCh := make(chan *probe.Error) defer close(errCh) for _, targetURL := range targetURLs { tgtClient, err := url2Client(targetURL) if err != nil { return err.Trace(targetURL) } tgtClients = append(tgtClients, tgtClient) tgtReader, tgtWriter := io.Pipe() tgtReaders = append(tgtReaders, tgtReader) tgtWriters = append(tgtWriters, tgtWriter) } go func() { var writers []io.Writer for _, tgtWriter := range tgtWriters { writers = append(writers, io.Writer(tgtWriter)) } multiTgtWriter := io.MultiWriter(writers...) var e error switch length { case 0: _, e = io.Copy(multiTgtWriter, reader) default: _, e = io.CopyN(multiTgtWriter, reader, length) } for _, tgtWriter := range tgtWriters { if e != nil { tgtWriter.CloseWithError(e) } tgtWriter.Close() } }() var wg sync.WaitGroup errorCh := make(chan *probe.Error, len(tgtClients)) func() { // Parallel putObject defer close(errorCh) // Each routine gets to return one err status. for i := range tgtClients { wg.Add(1) // make local copy for go routine tgtClient := tgtClients[i] tgtReader := tgtReaders[i] go func(targetClient client.Client, reader io.ReadCloser, errorCh chan<- *probe.Error) { defer wg.Done() defer reader.Close() err := targetClient.PutObject(length, reader) if err != nil { errorCh <- err.Trace() return } }(tgtClient, tgtReader, errorCh) } wg.Wait() }() // Return on first error encounter. err := <-errorCh if err != nil { return err.Trace() } return nil // success. } // getNewClient gives a new client interface func getNewClient(urlStr string, auth hostConfig) (client.Client, *probe.Error) { url := client.NewURL(urlStr) switch url.Type { case client.Object: // Minio and S3 compatible cloud storage s3Config := new(client.Config) s3Config.AccessKeyID = func() string { if auth.AccessKeyID == globalAccessKeyID { return "" } return auth.AccessKeyID }() s3Config.SecretAccessKey = func() string { if auth.SecretAccessKey == globalSecretAccessKey { return "" } return auth.SecretAccessKey }() s3Config.AppName = "Minio" s3Config.AppVersion = globalMCVersion s3Config.AppComments = []string{os.Args[0], runtime.GOOS, runtime.GOARCH} s3Config.HostURL = urlStr s3Config.Debug = globalDebugFlag var s3Client client.Client var err *probe.Error if auth.API == "S3v2" { s3Client, err = s3v2.New(s3Config) } else { s3Client, err = s3v4.New(s3Config) } if err != nil { return nil, err.Trace() } return s3Client, nil case client.Filesystem: fsClient, err := fs.New(urlStr) if err != nil { return nil, err.Trace() } return fsClient, nil } return nil, errInitClient(urlStr).Trace() } func url2Client(url string) (client.Client, *probe.Error) { urlconfig, err := getHostConfig(url) if err != nil { return nil, err.Trace(url) } client, err := getNewClient(url, urlconfig) if err != nil { return nil, err.Trace(url) } return client, nil } // bucketExists returns error func bucketExists(URL string) *probe.Error { _, _, err := url2Stat(URL) if err != nil { return err.Trace() } return nil }