mc/sync-url.go

/*
 * Minio Client (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"path/filepath"
	"strings"

	"github.com/minio/mc/pkg/client"
	"github.com/minio/minio/pkg/iodine"
)

//
//   NOTE: All the parse rules should reduced to A: Copy(Source, Target).
//
//   * SINGLE SOURCE - VALID
//   =======================
//   A: sync(f, f) -> copy(f, f)
//   B: sync(f, d) -> copy(f, d/f) -> A
//   C: sync(f, []d) -> []copy(f, d/f) -> []A
//      sync(d1..., d2) -> []copy(d1/f, d2/d1/f) -> []A
//      sync(d1..., []d2) -> [][]copy(d1/f, d2/d1/f) -> [][]A
//
//   * SINGLE SOURCE - INVALID
//   =========================
//   sync(d, *)
//   sync(d..., f)
//   sync(*, d...)
//
//   * MULTI-TARGET RECURSIVE - INVALID
//   ==================================
//   sync(*, f1)
//   sync(*, []f1)

type syncURLs struct {
	SourceContent  *client.Content
	TargetContents []*client.Content
	Error          error
}

func (s syncURLs) IsEmpty() bool {
	empty := false
	if s.SourceContent == nil {
		empty = true
		if s.TargetContents == nil {
			empty = true
			return empty
		}
		if len(s.TargetContents) > 0 && s.TargetContents[0] == nil {
			empty = true
			return empty
		}
	}
	return empty
}

type syncURLsType cpURLsType

// guessSyncURLType guesses the type of URL. This approach all allows prepareURL
// functions to accurately report failure causes.
func guessSyncURLType(sourceURL string, targetURLs []string) syncURLsType {
	if targetURLs == nil { // Target is empty
		return syncURLsType(cpURLsTypeInvalid)
	}
	if sourceURL == "" { // Source list is empty
		return syncURLsType(cpURLsTypeInvalid)
	}
	if isURLRecursive(sourceURL) { // Type C
		return syncURLsType(cpURLsTypeC)
	} // else Type A or Type B
	for _, targetURL := range targetURLs {
		if isTargetURLDir(targetURL) { // Type B
			return syncURLsType(cpURLsTypeB)
		}
	} // else Type A
	return syncURLsType(cpURLsTypeA)
}

// prepareSyncURLsTypeA - A: sync(f, f) -> copy(f, f)
func prepareSyncURLsTypeA(sourceURL string, targetURLs []string) <-chan syncURLs {
	syncURLsCh := make(chan syncURLs, 10000)
	go func() {
		defer close(syncURLsCh)
		var sURLs syncURLs
		for _, targetURL := range targetURLs {
			var cURLs cpURLs
			for cURLs = range prepareCopyURLsTypeA(sourceURL, targetURL) {
				if cURLs.Error != nil {
					syncURLsCh <- syncURLs{Error: iodine.New(cURLs.Error, nil)}
					continue
				}
			}
			sURLs.SourceContent = cURLs.SourceContent
			sURLs.TargetContents = append(sURLs.TargetContents, cURLs.TargetContent)
		}
		if !sURLs.IsEmpty() {
			syncURLsCh <- sURLs
		}
	}()
	return syncURLsCh
}

// prepareSyncURLsTypeB - B: sync(f, d) -> copy(f, d/f) -> A
func prepareSyncURLsTypeB(sourceURL string, targetURLs []string) <-chan syncURLs {
	syncURLsCh := make(chan syncURLs, 10000)
	go func() {
		defer close(syncURLsCh)
		var sURLs syncURLs
		for _, targetURL := range targetURLs {
			var cURLs cpURLs
			for cURLs = range prepareCopyURLsTypeB(sourceURL, targetURL) {
				if cURLs.Error != nil {
					syncURLsCh <- syncURLs{Error: iodine.New(cURLs.Error, nil)}
					continue
				}
			}
			sURLs.SourceContent = cURLs.SourceContent
			sURLs.TargetContents = append(sURLs.TargetContents, cURLs.TargetContent)
		}
		if !sURLs.IsEmpty() {
			syncURLsCh <- sURLs
		}
	}()
	return syncURLsCh
}

// prepareSyncURLsTypeC - C: sync(f, []d) -> []copy(f, d/f) -> []A
func prepareSyncURLsTypeC(sourceURL string, targetURLs []string) <-chan syncURLs {
	syncURLsCh := make(chan syncURLs, 10000)
	go func() {
		defer close(syncURLsCh)
		if !isURLRecursive(sourceURL) {
			// Source is not of recursive type.
			syncURLsCh <- syncURLs{Error: iodine.New(errSourceNotRecursive{URL: sourceURL}, nil)}
			return
		}
		// add `/` after trimming off `...` to emulate directories
		sourceURL = stripRecursiveURL(sourceURL)
		sourceClient, sourceContent, err := url2Stat(sourceURL)
		// Source exist?
		if err != nil {
			// Source does not exist or insufficient privileges.
			syncURLsCh <- syncURLs{Error: iodine.New(err, nil)}
			return
		}

		if !sourceContent.Type.IsDir() {
			// Source is not a dir.
			syncURLsCh <- syncURLs{Error: iodine.New(errSourceIsNotDir{URL: sourceURL}, nil)}
			return
		}

		for _, targetURL := range targetURLs {
			_, targetContent, err := url2Stat(targetURL)
			// Target exist?
			if err != nil {
				// Target does not exist.
				syncURLsCh <- syncURLs{Error: iodine.New(errTargetNotFound{URL: targetURL}, nil)}
				return
			}

			if !targetContent.Type.IsDir() {
				// Target exists, but is not a directory.
				syncURLsCh <- syncURLs{Error: iodine.New(errTargetIsNotDir{URL: targetURL}, nil)}
				return
			}
		}
		for sourceContent := range sourceClient.List(true) {
			if sourceContent.Err != nil {
				// Listing failed.
				syncURLsCh <- syncURLs{Error: iodine.New(sourceContent.Err, nil)}
				continue
			}
			if !sourceContent.Content.Type.IsRegular() {
				// Source is not a regular file. Skip it for copy.
				continue
			}
			// All OK.. We can proceed. Type B: source is a file, target is a directory and exists.
			sourceURLParse, err := client.Parse(sourceURL)
			if err != nil {
				syncURLsCh <- syncURLs{Error: iodine.New(errInvalidSource{URL: sourceURL}, nil)}
				continue
			}
			var newTargetURLs []string
			var sourceContentParse *client.URL
			for _, targetURL := range targetURLs {
				targetURLParse, err := client.Parse(targetURL)
				if err != nil {
					syncURLsCh <- syncURLs{Error: iodine.New(errInvalidTarget{URL: targetURL}, nil)}
					continue
				}
				sourceURLDelimited := sourceURLParse.String()[:strings.LastIndex(sourceURLParse.String(),
					string(sourceURLParse.Separator))+1]
				sourceContentName := sourceContent.Content.Name
				sourceContentURL := sourceURLDelimited + sourceContentName
				sourceContentParse, err = client.Parse(sourceContentURL)
				if err != nil {
					syncURLsCh <- syncURLs{Error: iodine.New(errInvalidSource{URL: sourceContentName}, nil)}
					continue
				}
				// Construct target path from recursive path of source without its prefix dir.
				newTargetURLParse := *targetURLParse
				newTargetURLParse.Path = filepath.Join(newTargetURLParse.Path, sourceContentName)
				newTargetURLs = append(newTargetURLs, newTargetURLParse.String())
			}
			for sURLs := range prepareSyncURLsTypeA(sourceContentParse.String(), newTargetURLs) {
				syncURLsCh <- sURLs
			}
		}
	}()
	return syncURLsCh
}

// prepareCopyURLs - prepares target and source URLs for syncing.
func prepareSyncURLs(sourceURL string, targetURLs []string) <-chan syncURLs {
	syncURLsCh := make(chan syncURLs, 10000)
	go func() {
		defer close(syncURLsCh)
		switch guessSyncURLType(sourceURL, targetURLs) {
		case syncURLsType(cpURLsTypeA):
			for sURLs := range prepareSyncURLsTypeA(sourceURL, targetURLs) {
				syncURLsCh <- sURLs
			}
		case syncURLsType(cpURLsTypeB):
			for sURLs := range prepareSyncURLsTypeB(sourceURL, targetURLs) {
				syncURLsCh <- sURLs
			}
		case syncURLsType(cpURLsTypeC):
			for sURLs := range prepareSyncURLsTypeC(sourceURL, targetURLs) {
				syncURLsCh <- sURLs
			}
		default:
			syncURLsCh <- syncURLs{Error: iodine.New(errInvalidArgument{}, nil)}
		}
	}()
	return syncURLsCh
}