cp: Copy recursive paths having the passed prefix (#3563)

cmd/client-s3.go

@@ -1315,21 +1315,8 @@ func (c *S3Client) SetAccess(ctx context.Context, bucketPolicy string, isJSON bo
 	return nil
 }
 
-func (c *S3Client) listObjectWrapperSingleKey(ctx context.Context, bucket, object string, isRecursive bool, timeRef time.Time, withVersions, withDeleteMarkers bool, metadata bool) <-chan minio.ObjectInfo {
-	if !timeRef.IsZero() || withVersions {
-		return c.listVersions(ctx, bucket, object, isRecursive, timeRef, withVersions, withDeleteMarkers)
-	}
-
-	if isGoogle(c.targetURL.Host) {
-		// Google Cloud S3 layer doesn't implement ListObjectsV2 implementation
-		// https://github.com/minio/mc/issues/3073
-		return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, UseV1: true, MaxKeys: 1})
-	}
-	return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, WithMetadata: metadata, MaxKeys: 1})
-}
-
 // listObjectWrapper - select ObjectList mode depending on arguments
-func (c *S3Client) listObjectWrapper(ctx context.Context, bucket, object string, isRecursive bool, timeRef time.Time, withVersions, withDeleteMarkers bool, metadata bool) <-chan minio.ObjectInfo {
+func (c *S3Client) listObjectWrapper(ctx context.Context, bucket, object string, isRecursive bool, timeRef time.Time, withVersions, withDeleteMarkers bool, metadata bool, maxKeys int) <-chan minio.ObjectInfo {
 	if !timeRef.IsZero() || withVersions {
 		return c.listVersions(ctx, bucket, object, isRecursive, timeRef, withVersions, withDeleteMarkers)
 	}
@@ -1337,9 +1324,9 @@ func (c *S3Client) listObjectWrapper(ctx context.Context, bucket, object string,
 	if isGoogle(c.targetURL.Host) {
 		// Google Cloud S3 layer doesn't implement ListObjectsV2 implementation
 		// https://github.com/minio/mc/issues/3073
-		return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, UseV1: true})
+		return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, UseV1: true, MaxKeys: maxKeys})
 	}
-	return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, WithMetadata: metadata})
+	return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, WithMetadata: metadata, MaxKeys: maxKeys})
 }
 
 func (c *S3Client) statIncompleteUpload(ctx context.Context, bucket, object string) (*ClientContent, *probe.Error) {
@@ -1431,7 +1418,7 @@ func (c *S3Client) Stat(ctx context.Context, opts StatOptions) (*ClientContent,
 	// Prefix to pass to minio-go listing in order to fetch if a prefix exists
 	prefix := strings.TrimRight(object, string(c.targetURL.Separator))
 
-	for objectStat := range c.listObjectWrapperSingleKey(ctx, bucket, prefix, nonRecursive, opts.timeRef, false, false, false) {
+	for objectStat := range c.listObjectWrapper(ctx, bucket, prefix, nonRecursive, opts.timeRef, false, false, false, 1) {
 		if objectStat.Err != nil {
 			return nil, probe.NewError(objectStat.Err)
 		}
@@ -1989,7 +1976,7 @@ func (c *S3Client) listInRoutine(ctx context.Context, contentCh chan *ClientCont
 		contentCh <- content
 	default:
 		isRecursive := false
-		for object := range c.listObjectWrapper(ctx, b, o, isRecursive, time.Time{}, false, false, opts.WithMetadata) {
+		for object := range c.listObjectWrapper(ctx, b, o, isRecursive, time.Time{}, false, false, opts.WithMetadata, -1) {
 			if object.Err != nil {
 				contentCh <- &ClientContent{
 					Err: probe.NewError(object.Err),
@@ -2038,7 +2025,7 @@ func (c *S3Client) listRecursiveInRoutine(ctx context.Context, contentCh chan *C
 			}
 
 			isRecursive := true
-			for object := range c.listObjectWrapper(ctx, bucket.Name, o, isRecursive, time.Time{}, false, false, opts.WithMetadata) {
+			for object := range c.listObjectWrapper(ctx, bucket.Name, o, isRecursive, time.Time{}, false, false, opts.WithMetadata, -1) {
 				if object.Err != nil {
 					contentCh <- &ClientContent{
 						Err: probe.NewError(object.Err),
@@ -2054,7 +2041,7 @@ func (c *S3Client) listRecursiveInRoutine(ctx context.Context, contentCh chan *C
 		}
 	default:
 		isRecursive := true
-		for object := range c.listObjectWrapper(ctx, b, o, isRecursive, time.Time{}, false, false, opts.WithMetadata) {
+		for object := range c.listObjectWrapper(ctx, b, o, isRecursive, time.Time{}, false, false, opts.WithMetadata, -1) {
 			if object.Err != nil {
 				contentCh <- &ClientContent{
 					Err: probe.NewError(object.Err),

cmd/client-url.go

@@ -200,6 +200,19 @@ func url2Stat(ctx context.Context, urlStr, versionID string, fileAttr bool, encK
 	return client, content, nil
 }
 
+// firstURL2Stat returns the stat info of the first object having the specified prefix
+func firstURL2Stat(ctx context.Context, prefix string, timeRef time.Time) (client Client, content *ClientContent, err *probe.Error) {
+	client, err = newClient(prefix)
+	if err != nil {
+		return nil, nil, err.Trace(prefix)
+	}
+	content = <-client.List(ctx, ListOptions{Recursive: true, TimeRef: timeRef, Count: 1})
+	if content.Err != nil {
+		return nil, nil, err.Trace(prefix)
+	}
+	return client, content, nil
+}
+
 // url2Alias separates alias and path from the URL. Aliased URL is of
 // the form alias/path/to/blah.
 func url2Alias(aliasedURL string) (alias, path string) {

cmd/client.go

@@ -69,6 +69,7 @@ type ListOptions struct {
 	WithDeleteMarkers bool
 	TimeRef           time.Time
 	ShowDir           DirOpt
+	Count             int
 }
 
 // CopyOptions holds options for copying operation

cmd/cp-url-syntax.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/minio/cli"
+	"github.com/minio/mc/pkg/probe"
 	"github.com/minio/minio/pkg/console"
 )
 
@@ -52,7 +53,12 @@ func checkCopySyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB map[stri
 
 	// Verify if source(s) exists.
 	for _, srcURL := range srcURLs {
-		_, _, err := url2Stat(ctx, srcURL, versionID, false, encKeyDB, timeRef)
+		var err *probe.Error
+		if !isRecursive {
+			_, _, err = url2Stat(ctx, srcURL, versionID, false, encKeyDB, timeRef)
+		} else {
+			_, _, err = firstURL2Stat(ctx, srcURL, timeRef)
+		}
 		if err != nil {
 			msg := "Unable to validate source `" + srcURL + "`"
 			if versionID != "" {

cmd/cp-url.go

@@ -54,8 +54,14 @@ const (
 // functions to accurately report failure causes.
 func guessCopyURLType(ctx context.Context, sourceURLs []string, targetURL string, isRecursive bool, keys map[string][]prefixSSEPair, timeRef time.Time, versionID string) (copyURLsType, string, *probe.Error) {
 	if len(sourceURLs) == 1 { // 1 Source, 1 Target
+		var err *probe.Error
+		var sourceContent *ClientContent
 		sourceURL := sourceURLs[0]
-		_, sourceContent, err := url2Stat(ctx, sourceURL, versionID, false, keys, timeRef)
+		if !isRecursive {
+			_, sourceContent, err = url2Stat(ctx, sourceURL, versionID, false, keys, timeRef)
+		} else {
+			_, sourceContent, err = firstURL2Stat(ctx, sourceURL, timeRef)
+		}
 		if err != nil {
 			return copyURLsTypeInvalid, "", err
 		}