minio/mc
1
0
Fork 0
mirror of https://github.com/minio/mc.git synced 2025-11-10 13:42:32 +03:00
Code Activity

cp: Copy recursive paths having the passed prefix (#3563)

Browse Source
This commit is contained in:
Anis Elleuch
2021-01-24 09:06:40 +01:00
committed by GitHub
parent 7e19044d98
commit 130f4393bb
5 changed files with 35 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
cmd/client-s3.go
View File

@@ -1315,21 +1315,8 @@ func (c *S3Client) SetAccess(ctx context.Context, bucketPolicy string, isJSON bo
return nil
}
func (c *S3Client) listObjectWrapperSingleKey(ctx context.Context, bucket, object string, isRecursive bool, timeRef time.Time, withVersions, withDeleteMarkers bool, metadata bool) <-chan minio.ObjectInfo {
if !timeRef.IsZero() || withVersions {
return c.listVersions(ctx, bucket, object, isRecursive, timeRef, withVersions, withDeleteMarkers)
}
if isGoogle(c.targetURL.Host) {
// Google Cloud S3 layer doesn't implement ListObjectsV2 implementation
// https://github.com/minio/mc/issues/3073
return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, UseV1: true, MaxKeys: 1})
}
return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, WithMetadata: metadata, MaxKeys: 1})
}
// listObjectWrapper - select ObjectList mode depending on arguments
func (c *S3Client) listObjectWrapper(ctx context.Context, bucket, object string, isRecursive bool, timeRef time.Time, withVersions, withDeleteMarkers bool, metadata bool) <-chan minio.ObjectInfo {
func (c *S3Client) listObjectWrapper(ctx context.Context, bucket, object string, isRecursive bool, timeRef time.Time, withVersions, withDeleteMarkers bool, metadata bool, maxKeys int) <-chan minio.ObjectInfo {
if !timeRef.IsZero() || withVersions {
return c.listVersions(ctx, bucket, object, isRecursive, timeRef, withVersions, withDeleteMarkers)
}
@@ -1337,9 +1324,9 @@ func (c *S3Client) listObjectWrapper(ctx context.Context, bucket, object string,
if isGoogle(c.targetURL.Host) {
// Google Cloud S3 layer doesn't implement ListObjectsV2 implementation
// https://github.com/minio/mc/issues/3073
return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, UseV1: true})
return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, UseV1: true, MaxKeys: maxKeys})
}
return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, WithMetadata: metadata})
return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, WithMetadata: metadata, MaxKeys: maxKeys})
}
func (c *S3Client) statIncompleteUpload(ctx context.Context, bucket, object string) (*ClientContent, *probe.Error) {
@@ -1431,7 +1418,7 @@ func (c *S3Client) Stat(ctx context.Context, opts StatOptions) (*ClientContent,
// Prefix to pass to minio-go listing in order to fetch if a prefix exists
prefix := strings.TrimRight(object, string(c.targetURL.Separator))
for objectStat := range c.listObjectWrapperSingleKey(ctx, bucket, prefix, nonRecursive, opts.timeRef, false, false, false) {
for objectStat := range c.listObjectWrapper(ctx, bucket, prefix, nonRecursive, opts.timeRef, false, false, false, 1) {
if objectStat.Err != nil {
return nil, probe.NewError(objectStat.Err)
}
@@ -1989,7 +1976,7 @@ func (c *S3Client) listInRoutine(ctx context.Context, contentCh chan *ClientCont
contentCh <- content
default:
isRecursive := false
for object := range c.listObjectWrapper(ctx, b, o, isRecursive, time.Time{}, false, false, opts.WithMetadata) {
for object := range c.listObjectWrapper(ctx, b, o, isRecursive, time.Time{}, false, false, opts.WithMetadata, -1) {
if object.Err != nil {
contentCh <- &ClientContent{
Err: probe.NewError(object.Err),
@@ -2038,7 +2025,7 @@ func (c *S3Client) listRecursiveInRoutine(ctx context.Context, contentCh chan *C
}
isRecursive := true
for object := range c.listObjectWrapper(ctx, bucket.Name, o, isRecursive, time.Time{}, false, false, opts.WithMetadata) {
for object := range c.listObjectWrapper(ctx, bucket.Name, o, isRecursive, time.Time{}, false, false, opts.WithMetadata, -1) {
if object.Err != nil {
contentCh <- &ClientContent{
Err: probe.NewError(object.Err),
@@ -2054,7 +2041,7 @@ func (c *S3Client) listRecursiveInRoutine(ctx context.Context, contentCh chan *C
}
default:
isRecursive := true
for object := range c.listObjectWrapper(ctx, b, o, isRecursive, time.Time{}, false, false, opts.WithMetadata) {
for object := range c.listObjectWrapper(ctx, b, o, isRecursive, time.Time{}, false, false, opts.WithMetadata, -1) {
if object.Err != nil {
contentCh <- &ClientContent{
Err: probe.NewError(object.Err),

13
cmd/client-url.go
View File

@@ -200,6 +200,19 @@ func url2Stat(ctx context.Context, urlStr, versionID string, fileAttr bool, encK
return client, content, nil
}
// firstURL2Stat returns the stat info of the first object having the specified prefix
func firstURL2Stat(ctx context.Context, prefix string, timeRef time.Time) (client Client, content *ClientContent, err *probe.Error) {
client, err = newClient(prefix)
if err != nil {
return nil, nil, err.Trace(prefix)
}
content = <-client.List(ctx, ListOptions{Recursive: true, TimeRef: timeRef, Count: 1})
if content.Err != nil {
return nil, nil, err.Trace(prefix)
}
return client, content, nil
}
// url2Alias separates alias and path from the URL. Aliased URL is of
// the form alias/path/to/blah.
func url2Alias(aliasedURL string) (alias, path string) {

1
cmd/client.go
View File

@@ -69,6 +69,7 @@ type ListOptions struct {
WithDeleteMarkers bool
TimeRef time.Time
ShowDir DirOpt
Count int
}
// CopyOptions holds options for copying operation

8
cmd/cp-url-syntax.go
View File

@@ -23,6 +23,7 @@ import (
"time"
"github.com/minio/cli"
"github.com/minio/mc/pkg/probe"
"github.com/minio/minio/pkg/console"
)
@@ -52,7 +53,12 @@ func checkCopySyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB map[stri
// Verify if source(s) exists.
for _, srcURL := range srcURLs {
_, _, err := url2Stat(ctx, srcURL, versionID, false, encKeyDB, timeRef)
var err *probe.Error
if !isRecursive {
_, _, err = url2Stat(ctx, srcURL, versionID, false, encKeyDB, timeRef)
} else {
_, _, err = firstURL2Stat(ctx, srcURL, timeRef)
}
if err != nil {
msg := "Unable to validate source `" + srcURL + "`"
if versionID != "" {

8
cmd/cp-url.go
View File

@@ -54,8 +54,14 @@ const (
// functions to accurately report failure causes.
func guessCopyURLType(ctx context.Context, sourceURLs []string, targetURL string, isRecursive bool, keys map[string][]prefixSSEPair, timeRef time.Time, versionID string) (copyURLsType, string, *probe.Error) {
if len(sourceURLs) == 1 { // 1 Source, 1 Target
var err *probe.Error
var sourceContent *ClientContent
sourceURL := sourceURLs[0]
_, sourceContent, err := url2Stat(ctx, sourceURL, versionID, false, keys, timeRef)
if !isRecursive {
_, sourceContent, err = url2Stat(ctx, sourceURL, versionID, false, keys, timeRef)
} else {
_, sourceContent, err = firstURL2Stat(ctx, sourceURL, timeRef)
}
if err != nil {
return copyURLsTypeInvalid, "", err
}