mirror of
https://github.com/minio/mc.git
synced 2025-11-12 01:02:26 +03:00
Add remote zip support to mc ls/cp/cat (#3977)
--zip flag is hidden for now
This commit is contained in:
@@ -43,6 +43,10 @@ var catFlags = []cli.Flag{
|
||||
Name: "version-id, vid",
|
||||
Usage: "display a specific version of an object",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "zip",
|
||||
Usage: "Extract from remote zip file (MinIO server source only)",
|
||||
},
|
||||
}
|
||||
|
||||
// Display contents of a file.
|
||||
@@ -165,7 +169,7 @@ func parseCatSyntax(ctx *cli.Context) (args []string, versionID string, timeRef
|
||||
}
|
||||
|
||||
// catURL displays contents of a URL to stdout.
|
||||
func catURL(ctx context.Context, sourceURL, sourceVersion string, timeRef time.Time, encKeyDB map[string][]prefixSSEPair) *probe.Error {
|
||||
func catURL(ctx context.Context, sourceURL, sourceVersion string, timeRef time.Time, encKeyDB map[string][]prefixSSEPair, isZip bool) *probe.Error {
|
||||
var reader io.ReadCloser
|
||||
size := int64(-1)
|
||||
switch sourceURL {
|
||||
@@ -180,7 +184,7 @@ func catURL(ctx context.Context, sourceURL, sourceVersion string, timeRef time.T
|
||||
// are ignored since some of them have zero size though they
|
||||
// have contents like files under /proc.
|
||||
// 2. extract the version ID if rewind flag is passed
|
||||
if client, content, err := url2Stat(ctx, sourceURL, sourceVersion, false, encKeyDB, timeRef); err == nil {
|
||||
if client, content, err := url2Stat(ctx, sourceURL, sourceVersion, false, encKeyDB, timeRef, isZip); err == nil {
|
||||
if sourceVersion == "" {
|
||||
versionID = content.VersionID
|
||||
}
|
||||
@@ -190,7 +194,7 @@ func catURL(ctx context.Context, sourceURL, sourceVersion string, timeRef time.T
|
||||
} else {
|
||||
return err.Trace(sourceURL)
|
||||
}
|
||||
if reader, err = getSourceStreamFromURL(ctx, sourceURL, versionID, encKeyDB); err != nil {
|
||||
if reader, err = getSourceStreamFromURL(ctx, sourceURL, versionID, encKeyDB, isZip); err != nil {
|
||||
return err.Trace(sourceURL)
|
||||
}
|
||||
defer reader.Close()
|
||||
@@ -260,6 +264,7 @@ func mainCat(cliCtx *cli.Context) error {
|
||||
if len(args) == 0 {
|
||||
stdinMode = true
|
||||
}
|
||||
isZip := cliCtx.Bool("zip")
|
||||
|
||||
// handle std input data.
|
||||
if stdinMode {
|
||||
@@ -280,7 +285,7 @@ func mainCat(cliCtx *cli.Context) error {
|
||||
|
||||
// Convert arguments to URLs: expand alias, fix format.
|
||||
for _, url := range args {
|
||||
fatalIf(catURL(ctx, url, versionID, rewind, encKeyDB).Trace(url), "Unable to read from `"+url+"`.")
|
||||
fatalIf(catURL(ctx, url, versionID, rewind, encKeyDB, isZip).Trace(url), "Unable to read from `"+url+"`.")
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -39,7 +39,7 @@ import (
|
||||
"github.com/minio/mc/pkg/disk"
|
||||
"github.com/minio/mc/pkg/hookreader"
|
||||
"github.com/minio/mc/pkg/probe"
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/encrypt"
|
||||
"github.com/minio/minio-go/v7/pkg/lifecycle"
|
||||
"github.com/minio/minio-go/v7/pkg/notification"
|
||||
@@ -542,8 +542,15 @@ func (f *fsClient) Remove(ctx context.Context, isIncomplete, isRemoveBucket, isB
|
||||
|
||||
// List - list files and folders.
|
||||
func (f *fsClient) List(ctx context.Context, opts ListOptions) <-chan *ClientContent {
|
||||
contentCh := make(chan *ClientContent)
|
||||
filteredCh := make(chan *ClientContent)
|
||||
contentCh := make(chan *ClientContent, 1)
|
||||
filteredCh := make(chan *ClientContent, 1)
|
||||
if opts.ListZip {
|
||||
contentCh <- &ClientContent{
|
||||
Err: probe.NewError(errors.New("zip listing not supported for local files")),
|
||||
}
|
||||
close(filteredCh)
|
||||
return filteredCh
|
||||
}
|
||||
|
||||
if opts.Recursive {
|
||||
if opts.ShowDir == DirNone {
|
||||
|
||||
@@ -40,7 +40,7 @@ import (
|
||||
|
||||
"github.com/minio/mc/pkg/httptracer"
|
||||
"github.com/minio/mc/pkg/probe"
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/encrypt"
|
||||
"github.com/minio/minio-go/v7/pkg/lifecycle"
|
||||
@@ -811,12 +811,15 @@ func (c *S3Client) Watch(ctx context.Context, options WatchOptions) (*WatchObjec
|
||||
// Get - get object with GET options.
|
||||
func (c *S3Client) Get(ctx context.Context, opts GetOptions) (io.ReadCloser, *probe.Error) {
|
||||
bucket, object := c.url2BucketAndObject()
|
||||
|
||||
reader, e := c.api.GetObject(ctx, bucket, object,
|
||||
minio.GetObjectOptions{
|
||||
o := minio.GetObjectOptions{
|
||||
ServerSideEncryption: opts.SSE,
|
||||
VersionID: opts.VersionID,
|
||||
})
|
||||
}
|
||||
if opts.Zip {
|
||||
o.Set("x-minio-extract", "true")
|
||||
}
|
||||
|
||||
reader, e := c.api.GetObject(ctx, bucket, object, o)
|
||||
if e != nil {
|
||||
errResponse := minio.ToErrorResponse(e)
|
||||
if errResponse.Code == "NoSuchBucket" {
|
||||
@@ -1439,7 +1442,7 @@ func (c *S3Client) SetAccess(ctx context.Context, bucketPolicy string, isJSON bo
|
||||
}
|
||||
|
||||
// listObjectWrapper - select ObjectList mode depending on arguments
|
||||
func (c *S3Client) listObjectWrapper(ctx context.Context, bucket, object string, isRecursive bool, timeRef time.Time, withVersions, withDeleteMarkers bool, metadata bool, maxKeys int) <-chan minio.ObjectInfo {
|
||||
func (c *S3Client) listObjectWrapper(ctx context.Context, bucket, object string, isRecursive bool, timeRef time.Time, withVersions, withDeleteMarkers bool, metadata bool, maxKeys int, zip bool) <-chan minio.ObjectInfo {
|
||||
if !timeRef.IsZero() || withVersions {
|
||||
return c.listVersions(ctx, bucket, object, isRecursive, timeRef, withVersions, withDeleteMarkers)
|
||||
}
|
||||
@@ -1449,7 +1452,15 @@ func (c *S3Client) listObjectWrapper(ctx context.Context, bucket, object string,
|
||||
// https://github.com/minio/mc/issues/3073
|
||||
return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, UseV1: true, MaxKeys: maxKeys})
|
||||
}
|
||||
return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, WithMetadata: metadata, MaxKeys: maxKeys})
|
||||
opts := minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, WithMetadata: metadata, MaxKeys: maxKeys}
|
||||
if zip {
|
||||
// If prefix ends with .zip, add a slash.
|
||||
if strings.HasSuffix(object, ".zip") {
|
||||
opts.Prefix = object + "/"
|
||||
}
|
||||
opts.Set("x-minio-extract", "true")
|
||||
}
|
||||
return c.api.ListObjects(ctx, bucket, opts)
|
||||
}
|
||||
|
||||
func (c *S3Client) statIncompleteUpload(ctx context.Context, bucket, object string) (*ClientContent, *probe.Error) {
|
||||
@@ -1531,7 +1542,11 @@ func (c *S3Client) Stat(ctx context.Context, opts StatOptions) (*ClientContent,
|
||||
if !strings.HasSuffix(object, string(c.targetURL.Separator)) && opts.timeRef.IsZero() {
|
||||
// Issue HEAD request first but ignore no such key error
|
||||
// so we can check if there is such prefix which exists
|
||||
ctnt, err := c.getObjectStat(ctx, bucket, object, minio.StatObjectOptions{ServerSideEncryption: opts.sse, VersionID: opts.versionID})
|
||||
o := minio.StatObjectOptions{ServerSideEncryption: opts.sse, VersionID: opts.versionID}
|
||||
if opts.isZip {
|
||||
o.Set("x-minio-extract", "true")
|
||||
}
|
||||
ctnt, err := c.getObjectStat(ctx, bucket, object, o)
|
||||
if err == nil {
|
||||
return ctnt, nil
|
||||
}
|
||||
@@ -1546,7 +1561,7 @@ func (c *S3Client) Stat(ctx context.Context, opts StatOptions) (*ClientContent,
|
||||
// Prefix to pass to minio-go listing in order to fetch if a prefix exists
|
||||
prefix := strings.TrimRight(object, string(c.targetURL.Separator))
|
||||
|
||||
for objectStat := range c.listObjectWrapper(ctx, bucket, prefix, nonRecursive, opts.timeRef, false, false, false, 1) {
|
||||
for objectStat := range c.listObjectWrapper(ctx, bucket, prefix, nonRecursive, opts.timeRef, false, false, false, 1, opts.isZip) {
|
||||
if objectStat.Err != nil {
|
||||
return nil, probe.NewError(objectStat.Err)
|
||||
}
|
||||
@@ -2074,6 +2089,12 @@ func (c *S3Client) bucketStat(ctx context.Context, bucket string) (*ClientConten
|
||||
func (c *S3Client) listInRoutine(ctx context.Context, contentCh chan *ClientContent, opts ListOptions) {
|
||||
// get bucket and object from URL.
|
||||
b, o := c.url2BucketAndObject()
|
||||
if opts.ListZip && (b == "" || o == "") {
|
||||
contentCh <- &ClientContent{
|
||||
Err: probe.NewError(errors.New("listing zip files must provide bucket and object")),
|
||||
}
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case b == "" && o == "":
|
||||
buckets, e := c.api.ListBuckets(ctx)
|
||||
@@ -2095,7 +2116,7 @@ func (c *S3Client) listInRoutine(ctx context.Context, contentCh chan *ClientCont
|
||||
contentCh <- content
|
||||
default:
|
||||
isRecursive := false
|
||||
for object := range c.listObjectWrapper(ctx, b, o, isRecursive, time.Time{}, false, false, opts.WithMetadata, -1) {
|
||||
for object := range c.listObjectWrapper(ctx, b, o, isRecursive, time.Time{}, false, false, opts.WithMetadata, -1, opts.ListZip) {
|
||||
if object.Err != nil {
|
||||
contentCh <- &ClientContent{
|
||||
Err: probe.NewError(object.Err),
|
||||
@@ -2154,7 +2175,7 @@ func (c *S3Client) listRecursiveInRoutine(ctx context.Context, contentCh chan *C
|
||||
}
|
||||
|
||||
isRecursive := true
|
||||
for object := range c.listObjectWrapper(ctx, bucket.Name, o, isRecursive, time.Time{}, false, false, opts.WithMetadata, -1) {
|
||||
for object := range c.listObjectWrapper(ctx, bucket.Name, o, isRecursive, time.Time{}, false, false, opts.WithMetadata, -1, opts.ListZip) {
|
||||
if object.Err != nil {
|
||||
contentCh <- &ClientContent{
|
||||
Err: probe.NewError(object.Err),
|
||||
@@ -2170,7 +2191,7 @@ func (c *S3Client) listRecursiveInRoutine(ctx context.Context, contentCh chan *C
|
||||
}
|
||||
default:
|
||||
isRecursive := true
|
||||
for object := range c.listObjectWrapper(ctx, b, o, isRecursive, time.Time{}, false, false, opts.WithMetadata, -1) {
|
||||
for object := range c.listObjectWrapper(ctx, b, o, isRecursive, time.Time{}, false, false, opts.WithMetadata, -1, opts.ListZip) {
|
||||
if object.Err != nil {
|
||||
contentCh <- &ClientContent{
|
||||
Err: probe.NewError(object.Err),
|
||||
|
||||
@@ -186,7 +186,7 @@ func urlJoinPath(url1, url2 string) string {
|
||||
}
|
||||
|
||||
// url2Stat returns stat info for URL.
|
||||
func url2Stat(ctx context.Context, urlStr, versionID string, fileAttr bool, encKeyDB map[string][]prefixSSEPair, timeRef time.Time) (client Client, content *ClientContent, err *probe.Error) {
|
||||
func url2Stat(ctx context.Context, urlStr, versionID string, fileAttr bool, encKeyDB map[string][]prefixSSEPair, timeRef time.Time, isZip bool) (client Client, content *ClientContent, err *probe.Error) {
|
||||
client, err = newClient(urlStr)
|
||||
if err != nil {
|
||||
return nil, nil, err.Trace(urlStr)
|
||||
@@ -194,7 +194,7 @@ func url2Stat(ctx context.Context, urlStr, versionID string, fileAttr bool, encK
|
||||
alias, _ := url2Alias(urlStr)
|
||||
sse := getSSE(urlStr, encKeyDB[alias])
|
||||
|
||||
content, err = client.Stat(ctx, StatOptions{preserve: fileAttr, sse: sse, timeRef: timeRef, versionID: versionID})
|
||||
content, err = client.Stat(ctx, StatOptions{preserve: fileAttr, sse: sse, timeRef: timeRef, versionID: versionID, isZip: isZip})
|
||||
if err != nil {
|
||||
return nil, nil, err.Trace(urlStr)
|
||||
}
|
||||
@@ -202,12 +202,12 @@ func url2Stat(ctx context.Context, urlStr, versionID string, fileAttr bool, encK
|
||||
}
|
||||
|
||||
// firstURL2Stat returns the stat info of the first object having the specified prefix
|
||||
func firstURL2Stat(ctx context.Context, prefix string, timeRef time.Time) (client Client, content *ClientContent, err *probe.Error) {
|
||||
func firstURL2Stat(ctx context.Context, prefix string, timeRef time.Time, isZip bool) (client Client, content *ClientContent, err *probe.Error) {
|
||||
client, err = newClient(prefix)
|
||||
if err != nil {
|
||||
return nil, nil, err.Trace(prefix)
|
||||
}
|
||||
content = <-client.List(ctx, ListOptions{Recursive: true, TimeRef: timeRef, Count: 1})
|
||||
content = <-client.List(ctx, ListOptions{Recursive: true, TimeRef: timeRef, Count: 1, ListZip: isZip})
|
||||
if content == nil {
|
||||
return nil, nil, probe.NewError(ObjectMissing{timeRef: timeRef}).Trace(prefix)
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/mc/pkg/probe"
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/encrypt"
|
||||
"github.com/minio/minio-go/v7/pkg/lifecycle"
|
||||
"github.com/minio/minio-go/v7/pkg/replication"
|
||||
@@ -47,6 +47,7 @@ const (
|
||||
type GetOptions struct {
|
||||
SSE encrypt.ServerSide
|
||||
VersionID string
|
||||
Zip bool
|
||||
}
|
||||
|
||||
// PutOptions holds options for PUT operation
|
||||
@@ -67,6 +68,7 @@ type StatOptions struct {
|
||||
sse encrypt.ServerSide
|
||||
timeRef time.Time
|
||||
versionID string
|
||||
isZip bool
|
||||
}
|
||||
|
||||
// ListOptions holds options for listing operation
|
||||
@@ -76,6 +78,7 @@ type ListOptions struct {
|
||||
WithMetadata bool
|
||||
WithOlderVersions bool
|
||||
WithDeleteMarkers bool
|
||||
ListZip bool
|
||||
TimeRef time.Time
|
||||
ShowDir DirOpt
|
||||
Count int
|
||||
|
||||
@@ -36,7 +36,7 @@ import (
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/mc/pkg/probe"
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/encrypt"
|
||||
"github.com/minio/pkg/env"
|
||||
)
|
||||
@@ -111,7 +111,7 @@ func getEncKeys(ctx *cli.Context) (map[string][]prefixSSEPair, *probe.Error) {
|
||||
func isAliasURLDir(ctx context.Context, aliasURL string, keys map[string][]prefixSSEPair, timeRef time.Time) bool {
|
||||
// If the target url exists, check if it is a directory
|
||||
// and return immediately.
|
||||
_, targetContent, err := url2Stat(ctx, aliasURL, "", false, keys, timeRef)
|
||||
_, targetContent, err := url2Stat(ctx, aliasURL, "", false, keys, timeRef, false)
|
||||
if err == nil {
|
||||
return targetContent.Type.IsDir()
|
||||
}
|
||||
@@ -152,24 +152,24 @@ func getSourceStreamMetadataFromURL(ctx context.Context, aliasedURL, versionID s
|
||||
return nil, nil, err.Trace(aliasedURL)
|
||||
}
|
||||
if !timeRef.IsZero() {
|
||||
_, content, err := url2Stat(ctx, aliasedURL, "", false, nil, timeRef)
|
||||
_, content, err := url2Stat(ctx, aliasedURL, "", false, nil, timeRef, false)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
versionID = content.VersionID
|
||||
}
|
||||
sseKey := getSSE(aliasedURL, encKeyDB[alias])
|
||||
return getSourceStream(ctx, alias, urlStrFull, versionID, true, sseKey, false)
|
||||
return getSourceStream(ctx, alias, urlStrFull, versionID, true, sseKey, false, false)
|
||||
}
|
||||
|
||||
// getSourceStreamFromURL gets a reader from URL.
|
||||
func getSourceStreamFromURL(ctx context.Context, urlStr, versionID string, encKeyDB map[string][]prefixSSEPair) (reader io.ReadCloser, err *probe.Error) {
|
||||
func getSourceStreamFromURL(ctx context.Context, urlStr, versionID string, encKeyDB map[string][]prefixSSEPair, isZip bool) (reader io.ReadCloser, err *probe.Error) {
|
||||
alias, urlStrFull, _, err := expandAlias(urlStr)
|
||||
if err != nil {
|
||||
return nil, err.Trace(urlStr)
|
||||
}
|
||||
sse := getSSE(urlStr, encKeyDB[alias])
|
||||
reader, _, err = getSourceStream(ctx, alias, urlStrFull, versionID, false, sse, false)
|
||||
reader, _, err = getSourceStream(ctx, alias, urlStrFull, versionID, false, sse, false, isZip)
|
||||
return reader, err
|
||||
}
|
||||
|
||||
@@ -221,12 +221,12 @@ func isReadAt(reader io.Reader) (ok bool) {
|
||||
}
|
||||
|
||||
// getSourceStream gets a reader from URL.
|
||||
func getSourceStream(ctx context.Context, alias, urlStr, versionID string, fetchStat bool, sse encrypt.ServerSide, preserve bool) (reader io.ReadCloser, metadata map[string]string, err *probe.Error) {
|
||||
func getSourceStream(ctx context.Context, alias, urlStr, versionID string, fetchStat bool, sse encrypt.ServerSide, preserve, isZip bool) (reader io.ReadCloser, metadata map[string]string, err *probe.Error) {
|
||||
sourceClnt, err := newClientFromAlias(alias, urlStr)
|
||||
if err != nil {
|
||||
return nil, nil, err.Trace(alias, urlStr)
|
||||
}
|
||||
reader, err = sourceClnt.Get(ctx, GetOptions{SSE: sse, VersionID: versionID})
|
||||
reader, err = sourceClnt.Get(ctx, GetOptions{SSE: sse, VersionID: versionID, Zip: isZip})
|
||||
if err != nil {
|
||||
return nil, nil, err.Trace(alias, urlStr)
|
||||
}
|
||||
@@ -408,7 +408,7 @@ func getAllMetadata(ctx context.Context, sourceAlias, sourceURLStr string, srcSS
|
||||
// uploadSourceToTargetURL - uploads to targetURL from source.
|
||||
// optionally optimizes copy for object sizes <= 5GiB by using
|
||||
// server side copy operation.
|
||||
func uploadSourceToTargetURL(ctx context.Context, urls URLs, progress io.Reader, encKeyDB map[string][]prefixSSEPair, preserve bool) URLs {
|
||||
func uploadSourceToTargetURL(ctx context.Context, urls URLs, progress io.Reader, encKeyDB map[string][]prefixSSEPair, preserve, isZip bool) URLs {
|
||||
sourceAlias := urls.SourceAlias
|
||||
sourceURL := urls.SourceContent.URL
|
||||
sourceVersion := urls.SourceContent.VersionID
|
||||
@@ -470,7 +470,7 @@ func uploadSourceToTargetURL(ctx context.Context, urls URLs, progress io.Reader,
|
||||
}
|
||||
|
||||
// Optimize for server side copy if the host is same.
|
||||
if sourceAlias == targetAlias {
|
||||
if sourceAlias == targetAlias && !isZip {
|
||||
// preserve new metadata and save existing ones.
|
||||
if preserve {
|
||||
currentMetadata, err := getAllMetadata(ctx, sourceAlias, sourceURL.String(), srcSSE, urls)
|
||||
@@ -538,7 +538,7 @@ func uploadSourceToTargetURL(ctx context.Context, urls URLs, progress io.Reader,
|
||||
|
||||
var reader io.ReadCloser
|
||||
// Proceed with regular stream copy.
|
||||
reader, metadata, err = getSourceStream(ctx, sourceAlias, sourceURL.String(), sourceVersion, true, srcSSE, preserve)
|
||||
reader, metadata, err = getSourceStream(ctx, sourceAlias, sourceURL.String(), sourceVersion, true, srcSSE, preserve, isZip)
|
||||
if err != nil {
|
||||
return urls.WithError(err.Trace(sourceURL.String()))
|
||||
}
|
||||
|
||||
@@ -102,6 +102,10 @@ var (
|
||||
Name: lhFlag,
|
||||
Usage: "apply legal hold to the copied object (on, off)",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "zip",
|
||||
Usage: "Extract from remote zip file (MinIO server source only)",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -239,7 +243,7 @@ type ProgressReader interface {
|
||||
}
|
||||
|
||||
// doCopy - Copy a single file from source to destination
|
||||
func doCopy(ctx context.Context, cpURLs URLs, pg ProgressReader, encKeyDB map[string][]prefixSSEPair, isMvCmd bool, preserve bool) URLs {
|
||||
func doCopy(ctx context.Context, cpURLs URLs, pg ProgressReader, encKeyDB map[string][]prefixSSEPair, isMvCmd bool, preserve, isZip bool) URLs {
|
||||
if cpURLs.Error != nil {
|
||||
cpURLs.Error = cpURLs.Error.Trace()
|
||||
return cpURLs
|
||||
@@ -265,7 +269,7 @@ func doCopy(ctx context.Context, cpURLs URLs, pg ProgressReader, encKeyDB map[st
|
||||
})
|
||||
}
|
||||
|
||||
urls := uploadSourceToTargetURL(ctx, cpURLs, pg, encKeyDB, preserve)
|
||||
urls := uploadSourceToTargetURL(ctx, cpURLs, pg, encKeyDB, preserve, isZip)
|
||||
if isMvCmd && urls.Error == nil {
|
||||
rmManager.add(ctx, sourceAlias, sourceURL.String())
|
||||
}
|
||||
@@ -308,7 +312,18 @@ func doPrepareCopyURLs(ctx context.Context, session *sessionV8, cancelCopy conte
|
||||
scanBar = scanBarFactory()
|
||||
}
|
||||
|
||||
URLsCh := prepareCopyURLs(ctx, sourceURLs, targetURL, isRecursive, encKeyDB, olderThan, newerThan, parseRewindFlag(rewind), versionID)
|
||||
opts := prepareCopyURLsOpts{
|
||||
sourceURLs: sourceURLs,
|
||||
targetURL: targetURL,
|
||||
isRecursive: isRecursive,
|
||||
encKeyDB: encKeyDB,
|
||||
olderThan: olderThan,
|
||||
newerThan: newerThan,
|
||||
timeRef: parseRewindFlag(rewind),
|
||||
versionID: versionID,
|
||||
}
|
||||
|
||||
URLsCh := prepareCopyURLs(ctx, opts)
|
||||
done := false
|
||||
for !done {
|
||||
select {
|
||||
@@ -431,8 +446,18 @@ func doCopySession(ctx context.Context, cancelCopy context.CancelFunc, cli *cli.
|
||||
|
||||
go func() {
|
||||
totalBytes := int64(0)
|
||||
for cpURLs := range prepareCopyURLs(ctx, sourceURLs, targetURL, isRecursive,
|
||||
encKeyDB, olderThan, newerThan, parseRewindFlag(rewind), versionID) {
|
||||
opts := prepareCopyURLsOpts{
|
||||
sourceURLs: sourceURLs,
|
||||
targetURL: targetURL,
|
||||
isRecursive: isRecursive,
|
||||
encKeyDB: encKeyDB,
|
||||
olderThan: olderThan,
|
||||
newerThan: newerThan,
|
||||
timeRef: parseRewindFlag(rewind),
|
||||
versionID: versionID,
|
||||
isZip: cli.Bool("zip"),
|
||||
}
|
||||
for cpURLs := range prepareCopyURLs(ctx, opts) {
|
||||
if cpURLs.Error != nil {
|
||||
// Print in new line and adjust to top so that we
|
||||
// don't print over the ongoing scan bar
|
||||
@@ -515,6 +540,7 @@ func doCopySession(ctx context.Context, cancelCopy context.CancelFunc, cli *cli.
|
||||
}
|
||||
|
||||
preserve := cli.Bool("preserve")
|
||||
isZip := cli.Bool("zip")
|
||||
if cli.String("attr") != "" {
|
||||
userMetaMap, _ := getMetaDataEntry(cli.String("attr"))
|
||||
for metadataKey, metaDataVal := range userMetaMap {
|
||||
@@ -532,7 +558,7 @@ func doCopySession(ctx context.Context, cancelCopy context.CancelFunc, cli *cli.
|
||||
}, 0)
|
||||
} else {
|
||||
parallel.queueTask(func() URLs {
|
||||
return doCopy(ctx, cpURLs, pg, encKeyDB, isMvCmd, preserve)
|
||||
return doCopy(ctx, cpURLs, pg, encKeyDB, isMvCmd, preserve, isZip)
|
||||
}, cpURLs.SourceContent.Size)
|
||||
}
|
||||
}
|
||||
@@ -641,7 +667,6 @@ func mainCopy(cliCtx *cli.Context) error {
|
||||
|
||||
// check 'copy' cli arguments.
|
||||
checkCopySyntax(ctx, cliCtx, encKeyDB, false)
|
||||
|
||||
// Additional command specific theme customization.
|
||||
console.SetColor("Copy", color.New(color.FgGreen, color.Bold))
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ func checkCopySyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB map[stri
|
||||
srcURLs := URLs[:len(URLs)-1]
|
||||
tgtURL := URLs[len(URLs)-1]
|
||||
isRecursive := cliCtx.Bool("recursive")
|
||||
isZip := cliCtx.Bool("zip")
|
||||
timeRef := parseRewindFlag(cliCtx.String("rewind"))
|
||||
versionID := cliCtx.String("version-id")
|
||||
|
||||
@@ -52,13 +53,17 @@ func checkCopySyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB map[stri
|
||||
fatalIf(errDummy().Trace(cliCtx.Args()...), "Unable to pass --version flag with multiple copy sources arguments.")
|
||||
}
|
||||
|
||||
if isZip && cliCtx.String("rewind") != "" {
|
||||
fatalIf(errDummy().Trace(cliCtx.Args()...), "--zip and --rewind cannot be used together")
|
||||
}
|
||||
|
||||
// Verify if source(s) exists.
|
||||
for _, srcURL := range srcURLs {
|
||||
var err *probe.Error
|
||||
if !isRecursive {
|
||||
_, _, err = url2Stat(ctx, srcURL, versionID, false, encKeyDB, timeRef)
|
||||
_, _, err = url2Stat(ctx, srcURL, versionID, false, encKeyDB, timeRef, isZip)
|
||||
} else {
|
||||
_, _, err = firstURL2Stat(ctx, srcURL, timeRef)
|
||||
_, _, err = firstURL2Stat(ctx, srcURL, timeRef, isZip)
|
||||
}
|
||||
if err != nil {
|
||||
msg := "Unable to validate source `" + srcURL + "`"
|
||||
@@ -92,7 +97,18 @@ func checkCopySyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB map[stri
|
||||
}
|
||||
|
||||
// Guess CopyURLsType based on source and target URLs.
|
||||
copyURLsType, _, err := guessCopyURLType(ctx, srcURLs, tgtURL, isRecursive, encKeyDB, timeRef, versionID)
|
||||
opts := prepareCopyURLsOpts{
|
||||
sourceURLs: srcURLs,
|
||||
targetURL: tgtURL,
|
||||
isRecursive: isRecursive,
|
||||
encKeyDB: encKeyDB,
|
||||
olderThan: "",
|
||||
newerThan: "",
|
||||
timeRef: timeRef,
|
||||
versionID: versionID,
|
||||
isZip: isZip,
|
||||
}
|
||||
copyURLsType, _, err := guessCopyURLType(ctx, opts)
|
||||
if err != nil {
|
||||
fatalIf(errInvalidArgument().Trace(), "Unable to guess the type of "+operation+" operation.")
|
||||
}
|
||||
@@ -111,7 +127,7 @@ func checkCopySyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB map[stri
|
||||
}
|
||||
checkCopySyntaxTypeB(ctx, srcURLs[0], versionID, tgtURL, encKeyDB, isMvCmd, timeRef)
|
||||
case copyURLsTypeC: // Folder... -> Folder.
|
||||
checkCopySyntaxTypeC(ctx, srcURLs, tgtURL, isRecursive, encKeyDB, isMvCmd, timeRef)
|
||||
checkCopySyntaxTypeC(ctx, srcURLs, tgtURL, isRecursive, isZip, encKeyDB, isMvCmd, timeRef)
|
||||
case copyURLsTypeD: // File1...FileN -> Folder.
|
||||
checkCopySyntaxTypeD(ctx, srcURLs, tgtURL, encKeyDB, isMvCmd, timeRef)
|
||||
default:
|
||||
@@ -126,7 +142,7 @@ func checkCopySyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB map[stri
|
||||
|
||||
// checkCopySyntaxTypeA verifies if the source and target are valid file arguments.
|
||||
func checkCopySyntaxTypeA(ctx context.Context, srcURL, versionID string, tgtURL string, keys map[string][]prefixSSEPair, isMvCmd bool, timeRef time.Time) {
|
||||
_, srcContent, err := url2Stat(ctx, srcURL, versionID, false, keys, timeRef)
|
||||
_, srcContent, err := url2Stat(ctx, srcURL, versionID, false, keys, timeRef, false)
|
||||
fatalIf(err.Trace(srcURL), "Unable to stat source `"+srcURL+"`.")
|
||||
|
||||
if !srcContent.Type.IsRegular() {
|
||||
@@ -136,7 +152,7 @@ func checkCopySyntaxTypeA(ctx context.Context, srcURL, versionID string, tgtURL
|
||||
|
||||
// checkCopySyntaxTypeB verifies if the source is a valid file and target is a valid folder.
|
||||
func checkCopySyntaxTypeB(ctx context.Context, srcURL, versionID string, tgtURL string, keys map[string][]prefixSSEPair, isMvCmd bool, timeRef time.Time) {
|
||||
_, srcContent, err := url2Stat(ctx, srcURL, versionID, false, keys, timeRef)
|
||||
_, srcContent, err := url2Stat(ctx, srcURL, versionID, false, keys, timeRef, false)
|
||||
fatalIf(err.Trace(srcURL), "Unable to stat source `"+srcURL+"`.")
|
||||
|
||||
if !srcContent.Type.IsRegular() {
|
||||
@@ -144,7 +160,7 @@ func checkCopySyntaxTypeB(ctx context.Context, srcURL, versionID string, tgtURL
|
||||
}
|
||||
|
||||
// Check target.
|
||||
if _, tgtContent, err := url2Stat(ctx, tgtURL, "", false, keys, timeRef); err == nil {
|
||||
if _, tgtContent, err := url2Stat(ctx, tgtURL, "", false, keys, timeRef, false); err == nil {
|
||||
if !tgtContent.Type.IsDir() {
|
||||
fatalIf(errInvalidArgument().Trace(tgtURL), "Target `"+tgtURL+"` is not a folder.")
|
||||
}
|
||||
@@ -152,21 +168,21 @@ func checkCopySyntaxTypeB(ctx context.Context, srcURL, versionID string, tgtURL
|
||||
}
|
||||
|
||||
// checkCopySyntaxTypeC verifies if the source is a valid recursive dir and target is a valid folder.
|
||||
func checkCopySyntaxTypeC(ctx context.Context, srcURLs []string, tgtURL string, isRecursive bool, keys map[string][]prefixSSEPair, isMvCmd bool, timeRef time.Time) {
|
||||
func checkCopySyntaxTypeC(ctx context.Context, srcURLs []string, tgtURL string, isRecursive, isZip bool, keys map[string][]prefixSSEPair, isMvCmd bool, timeRef time.Time) {
|
||||
// Check source.
|
||||
if len(srcURLs) != 1 {
|
||||
fatalIf(errInvalidArgument().Trace(), "Invalid number of source arguments.")
|
||||
}
|
||||
|
||||
// Check target.
|
||||
if _, tgtContent, err := url2Stat(ctx, tgtURL, "", false, keys, timeRef); err == nil {
|
||||
if _, tgtContent, err := url2Stat(ctx, tgtURL, "", false, keys, timeRef, false); err == nil {
|
||||
if !tgtContent.Type.IsDir() {
|
||||
fatalIf(errInvalidArgument().Trace(tgtURL), "Target `"+tgtURL+"` is not a folder.")
|
||||
}
|
||||
}
|
||||
|
||||
for _, srcURL := range srcURLs {
|
||||
c, srcContent, err := url2Stat(ctx, srcURL, "", false, keys, timeRef)
|
||||
c, srcContent, err := url2Stat(ctx, srcURL, "", false, keys, timeRef, isZip)
|
||||
// incomplete uploads are not necessary for copy operation, no need to verify for them.
|
||||
isIncomplete := false
|
||||
if err != nil {
|
||||
@@ -203,7 +219,7 @@ func checkCopySyntaxTypeC(ctx context.Context, srcURLs []string, tgtURL string,
|
||||
func checkCopySyntaxTypeD(ctx context.Context, srcURLs []string, tgtURL string, keys map[string][]prefixSSEPair, isMvCmd bool, timeRef time.Time) {
|
||||
// Source can be anything: file, dir, dir...
|
||||
// Check target if it is a dir
|
||||
if _, tgtContent, err := url2Stat(ctx, tgtURL, "", false, keys, timeRef); err == nil {
|
||||
if _, tgtContent, err := url2Stat(ctx, tgtURL, "", false, keys, timeRef, false); err == nil {
|
||||
if !tgtContent.Type.IsDir() {
|
||||
fatalIf(errInvalidArgument().Trace(tgtURL), "Target `"+tgtURL+"` is not a folder.")
|
||||
}
|
||||
|
||||
@@ -53,15 +53,15 @@ const (
|
||||
|
||||
// guessCopyURLType guesses the type of clientURL. This approach all allows prepareURL
|
||||
// functions to accurately report failure causes.
|
||||
func guessCopyURLType(ctx context.Context, sourceURLs []string, targetURL string, isRecursive bool, keys map[string][]prefixSSEPair, timeRef time.Time, versionID string) (copyURLsType, string, *probe.Error) {
|
||||
if len(sourceURLs) == 1 { // 1 Source, 1 Target
|
||||
func guessCopyURLType(ctx context.Context, o prepareCopyURLsOpts) (copyURLsType, string, *probe.Error) {
|
||||
if len(o.sourceURLs) == 1 { // 1 Source, 1 Target
|
||||
var err *probe.Error
|
||||
var sourceContent *ClientContent
|
||||
sourceURL := sourceURLs[0]
|
||||
if !isRecursive {
|
||||
_, sourceContent, err = url2Stat(ctx, sourceURL, versionID, false, keys, timeRef)
|
||||
sourceURL := o.sourceURLs[0]
|
||||
if !o.isRecursive {
|
||||
_, sourceContent, err = url2Stat(ctx, sourceURL, o.versionID, false, o.encKeyDB, o.timeRef, o.isZip)
|
||||
} else {
|
||||
_, sourceContent, err = firstURL2Stat(ctx, sourceURL, timeRef)
|
||||
_, sourceContent, err = firstURL2Stat(ctx, sourceURL, o.timeRef, o.isZip)
|
||||
}
|
||||
if err != nil {
|
||||
return copyURLsTypeInvalid, "", err
|
||||
@@ -69,12 +69,12 @@ func guessCopyURLType(ctx context.Context, sourceURLs []string, targetURL string
|
||||
|
||||
// If recursion is ON, it is type C.
|
||||
// If source is a folder, it is Type C.
|
||||
if sourceContent.Type.IsDir() || isRecursive {
|
||||
if sourceContent.Type.IsDir() || o.isRecursive || o.isZip {
|
||||
return copyURLsTypeC, "", nil
|
||||
}
|
||||
|
||||
// If target is a folder, it is Type B.
|
||||
if isAliasURLDir(ctx, targetURL, keys, timeRef) {
|
||||
if isAliasURLDir(ctx, o.targetURL, o.encKeyDB, o.timeRef) {
|
||||
return copyURLsTypeB, sourceContent.VersionID, nil
|
||||
}
|
||||
// else Type A.
|
||||
@@ -82,7 +82,7 @@ func guessCopyURLType(ctx context.Context, sourceURLs []string, targetURL string
|
||||
}
|
||||
|
||||
// Multiple source args and target is a folder. It is Type D.
|
||||
if isAliasURLDir(ctx, targetURL, keys, timeRef) {
|
||||
if isAliasURLDir(ctx, o.targetURL, o.encKeyDB, o.timeRef) {
|
||||
return copyURLsTypeD, "", nil
|
||||
}
|
||||
|
||||
@@ -97,7 +97,7 @@ func prepareCopyURLsTypeA(ctx context.Context, sourceURL, sourceVersion string,
|
||||
// Find alias and expanded clientURL.
|
||||
targetAlias, targetURL, _ := mustExpandAlias(targetURL)
|
||||
|
||||
_, sourceContent, err := url2Stat(ctx, sourceURL, sourceVersion, false, encKeyDB, time.Time{})
|
||||
_, sourceContent, err := url2Stat(ctx, sourceURL, sourceVersion, false, encKeyDB, time.Time{}, false)
|
||||
if err != nil {
|
||||
// Source does not exist or insufficient privileges.
|
||||
return URLs{Error: err.Trace(sourceURL)}
|
||||
@@ -130,7 +130,7 @@ func prepareCopyURLsTypeB(ctx context.Context, sourceURL, sourceVersion string,
|
||||
// Find alias and expanded clientURL.
|
||||
targetAlias, targetURL, _ := mustExpandAlias(targetURL)
|
||||
|
||||
_, sourceContent, err := url2Stat(ctx, sourceURL, sourceVersion, false, encKeyDB, time.Time{})
|
||||
_, sourceContent, err := url2Stat(ctx, sourceURL, sourceVersion, false, encKeyDB, time.Time{}, false)
|
||||
if err != nil {
|
||||
// Source does not exist or insufficient privileges.
|
||||
return URLs{Error: err.Trace(sourceURL)}
|
||||
@@ -158,7 +158,7 @@ func makeCopyContentTypeB(sourceAlias string, sourceContent *ClientContent, targ
|
||||
|
||||
// SINGLE SOURCE - Type C: copy(d1..., d2) -> []copy(d1/f, d1/d2/f) -> []A
|
||||
// prepareCopyRecursiveURLTypeC - prepares target and source clientURLs for copying.
|
||||
func prepareCopyURLsTypeC(ctx context.Context, sourceURL, targetURL string, isRecursive bool, timeRef time.Time, encKeyDB map[string][]prefixSSEPair) <-chan URLs {
|
||||
func prepareCopyURLsTypeC(ctx context.Context, sourceURL, targetURL string, isRecursive, isZip bool, timeRef time.Time, encKeyDB map[string][]prefixSSEPair) <-chan URLs {
|
||||
// Extract alias before fiddling with the clientURL.
|
||||
sourceAlias, _, _ := mustExpandAlias(sourceURL)
|
||||
// Find alias and expanded clientURL.
|
||||
@@ -173,7 +173,7 @@ func prepareCopyURLsTypeC(ctx context.Context, sourceURL, targetURL string, isRe
|
||||
return
|
||||
}
|
||||
|
||||
for sourceContent := range sourceClient.List(ctx, ListOptions{Recursive: isRecursive, TimeRef: timeRef, ShowDir: DirNone}) {
|
||||
for sourceContent := range sourceClient.List(ctx, ListOptions{Recursive: isRecursive, TimeRef: timeRef, ShowDir: DirNone, ListZip: isZip}) {
|
||||
if sourceContent.Err != nil {
|
||||
// Listing failed.
|
||||
copyURLsCh <- URLs{Error: sourceContent.Err.Trace(sourceClient.GetURL().String())}
|
||||
@@ -212,7 +212,7 @@ func prepareCopyURLsTypeD(ctx context.Context, sourceURLs []string, targetURL st
|
||||
go func(sourceURLs []string, targetURL string, copyURLsCh chan URLs) {
|
||||
defer close(copyURLsCh)
|
||||
for _, sourceURL := range sourceURLs {
|
||||
for cpURLs := range prepareCopyURLsTypeC(ctx, sourceURL, targetURL, isRecursive, timeRef, encKeyDB) {
|
||||
for cpURLs := range prepareCopyURLsTypeC(ctx, sourceURL, targetURL, isRecursive, false, timeRef, encKeyDB) {
|
||||
copyURLsCh <- cpURLs
|
||||
}
|
||||
}
|
||||
@@ -220,43 +220,54 @@ func prepareCopyURLsTypeD(ctx context.Context, sourceURLs []string, targetURL st
|
||||
return copyURLsCh
|
||||
}
|
||||
|
||||
type prepareCopyURLsOpts struct {
|
||||
sourceURLs []string
|
||||
targetURL string
|
||||
isRecursive bool
|
||||
encKeyDB map[string][]prefixSSEPair
|
||||
olderThan, newerThan string
|
||||
timeRef time.Time
|
||||
versionID string
|
||||
isZip bool
|
||||
}
|
||||
|
||||
// prepareCopyURLs - prepares target and source clientURLs for copying.
|
||||
func prepareCopyURLs(ctx context.Context, sourceURLs []string, targetURL string, isRecursive bool, encKeyDB map[string][]prefixSSEPair, olderThan, newerThan string, timeRef time.Time, versionID string) chan URLs {
|
||||
func prepareCopyURLs(ctx context.Context, o prepareCopyURLsOpts) chan URLs {
|
||||
copyURLsCh := make(chan URLs)
|
||||
go func(sourceURLs []string, targetURL string, copyURLsCh chan URLs, encKeyDB map[string][]prefixSSEPair, timeRef time.Time) {
|
||||
go func(o prepareCopyURLsOpts) {
|
||||
defer close(copyURLsCh)
|
||||
cpType, cpVersion, err := guessCopyURLType(ctx, sourceURLs, targetURL, isRecursive, encKeyDB, timeRef, versionID)
|
||||
cpType, cpVersion, err := guessCopyURLType(ctx, o)
|
||||
fatalIf(err.Trace(), "Unable to guess the type of copy operation.")
|
||||
|
||||
switch cpType {
|
||||
case copyURLsTypeA:
|
||||
copyURLsCh <- prepareCopyURLsTypeA(ctx, sourceURLs[0], cpVersion, targetURL, encKeyDB)
|
||||
copyURLsCh <- prepareCopyURLsTypeA(ctx, o.sourceURLs[0], cpVersion, o.targetURL, o.encKeyDB)
|
||||
case copyURLsTypeB:
|
||||
copyURLsCh <- prepareCopyURLsTypeB(ctx, sourceURLs[0], cpVersion, targetURL, encKeyDB)
|
||||
copyURLsCh <- prepareCopyURLsTypeB(ctx, o.sourceURLs[0], cpVersion, o.targetURL, o.encKeyDB)
|
||||
case copyURLsTypeC:
|
||||
for cURLs := range prepareCopyURLsTypeC(ctx, sourceURLs[0], targetURL, isRecursive, timeRef, encKeyDB) {
|
||||
for cURLs := range prepareCopyURLsTypeC(ctx, o.sourceURLs[0], o.targetURL, o.isRecursive, o.isZip, o.timeRef, o.encKeyDB) {
|
||||
copyURLsCh <- cURLs
|
||||
}
|
||||
case copyURLsTypeD:
|
||||
for cURLs := range prepareCopyURLsTypeD(ctx, sourceURLs, targetURL, isRecursive, timeRef, encKeyDB) {
|
||||
for cURLs := range prepareCopyURLsTypeD(ctx, o.sourceURLs, o.targetURL, o.isRecursive, o.timeRef, o.encKeyDB) {
|
||||
copyURLsCh <- cURLs
|
||||
}
|
||||
default:
|
||||
copyURLsCh <- URLs{Error: errInvalidArgument().Trace(sourceURLs...)}
|
||||
copyURLsCh <- URLs{Error: errInvalidArgument().Trace(o.sourceURLs...)}
|
||||
}
|
||||
}(sourceURLs, targetURL, copyURLsCh, encKeyDB, timeRef)
|
||||
}(o)
|
||||
|
||||
finalCopyURLsCh := make(chan URLs)
|
||||
go func() {
|
||||
defer close(finalCopyURLsCh)
|
||||
for cpURLs := range copyURLsCh {
|
||||
// Skip objects older than --older-than parameter if specified
|
||||
if olderThan != "" && isOlder(cpURLs.SourceContent.Time, olderThan) {
|
||||
if o.olderThan != "" && isOlder(cpURLs.SourceContent.Time, o.olderThan) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip objects newer than --newer-than parameter if specified
|
||||
if newerThan != "" && isNewer(cpURLs.SourceContent.Time, newerThan) {
|
||||
if o.newerThan != "" && isNewer(cpURLs.SourceContent.Time, o.newerThan) {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -130,7 +130,7 @@ func checkDiffSyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB map[stri
|
||||
// Diff only works between two directories, verify them below.
|
||||
|
||||
// Verify if firstURL is accessible.
|
||||
_, firstContent, err := url2Stat(ctx, firstURL, "", false, encKeyDB, time.Time{})
|
||||
_, firstContent, err := url2Stat(ctx, firstURL, "", false, encKeyDB, time.Time{}, false)
|
||||
if err != nil {
|
||||
fatalIf(err.Trace(firstURL), fmt.Sprintf("Unable to stat '%s'.", firstURL))
|
||||
}
|
||||
@@ -141,7 +141,7 @@ func checkDiffSyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB map[stri
|
||||
}
|
||||
|
||||
// Verify if secondURL is accessible.
|
||||
_, secondContent, err := url2Stat(ctx, secondURL, "", false, encKeyDB, time.Time{})
|
||||
_, secondContent, err := url2Stat(ctx, secondURL, "", false, encKeyDB, time.Time{}, false)
|
||||
if err != nil {
|
||||
// Destination doesn't exist is okay.
|
||||
if _, ok := err.ToGoError().(ObjectMissing); !ok {
|
||||
|
||||
@@ -174,7 +174,7 @@ func checkFindSyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB map[stri
|
||||
|
||||
// Extract input URLs and validate.
|
||||
for _, url := range args {
|
||||
_, _, err := url2Stat(ctx, url, "", false, encKeyDB, time.Time{})
|
||||
_, _, err := url2Stat(ctx, url, "", false, encKeyDB, time.Time{}, false)
|
||||
if err != nil && !isURLPrefixExists(url, false) {
|
||||
// Bucket name empty is a valid error for 'find myminio' unless we are using watch, treat it as such.
|
||||
if _, ok := err.ToGoError().(BucketNameEmpty); ok && !cliCtx.Bool("watch") {
|
||||
|
||||
@@ -57,6 +57,10 @@ var (
|
||||
Name: "storage-class, sc",
|
||||
Usage: "filter to specified storage class",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "zip",
|
||||
Usage: "list files inside zip archive (MinIO servers only)",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -154,7 +158,7 @@ func parseRewindFlag(rewind string) (timeRef time.Time) {
|
||||
}
|
||||
|
||||
// checkListSyntax - validate all the passed arguments
|
||||
func checkListSyntax(ctx context.Context, cliCtx *cli.Context) ([]string, bool, bool, bool, time.Time, bool, string) {
|
||||
func checkListSyntax(ctx context.Context, cliCtx *cli.Context) ([]string, doListOptions) {
|
||||
args := cliCtx.Args()
|
||||
if !cliCtx.Args().Present() {
|
||||
args = []string{"."}
|
||||
@@ -169,15 +173,27 @@ func checkListSyntax(ctx context.Context, cliCtx *cli.Context) ([]string, bool,
|
||||
isIncomplete := cliCtx.Bool("incomplete")
|
||||
withOlderVersions := cliCtx.Bool("versions")
|
||||
isSummary := cliCtx.Bool("summarize")
|
||||
listZip := cliCtx.Bool("zip")
|
||||
|
||||
timeRef := parseRewindFlag(cliCtx.String("rewind"))
|
||||
if timeRef.IsZero() && withOlderVersions {
|
||||
timeRef = time.Now().UTC()
|
||||
}
|
||||
|
||||
if listZip && (withOlderVersions || !timeRef.IsZero()) {
|
||||
fatalIf(errInvalidArgument().Trace(args...), "Zip file listing can only be performed on the latest version")
|
||||
}
|
||||
storageClasss := cliCtx.String("storage-class")
|
||||
|
||||
return args, isRecursive, isIncomplete, isSummary, timeRef, withOlderVersions, storageClasss
|
||||
opts := doListOptions{
|
||||
timeRef: timeRef,
|
||||
isRecursive: isRecursive,
|
||||
isIncomplete: isIncomplete,
|
||||
isSummary: isSummary,
|
||||
withOlderVersions: withOlderVersions,
|
||||
listZip: listZip,
|
||||
filter: storageClasss,
|
||||
}
|
||||
return args, opts
|
||||
}
|
||||
|
||||
// mainList - is a handler for mc ls command
|
||||
@@ -198,7 +214,7 @@ func mainList(cliCtx *cli.Context) error {
|
||||
console.SetColor("SC", color.New(color.FgBlue))
|
||||
|
||||
// check 'ls' cliCtx arguments.
|
||||
args, isRecursive, isIncomplete, isSummary, timeRef, withOlderVersions, storageClassFilter := checkListSyntax(ctx, cliCtx)
|
||||
args, opts := checkListSyntax(ctx, cliCtx)
|
||||
|
||||
var cErr error
|
||||
for _, targetURL := range args {
|
||||
@@ -206,14 +222,14 @@ func mainList(cliCtx *cli.Context) error {
|
||||
fatalIf(err.Trace(targetURL), "Unable to initialize target `"+targetURL+"`.")
|
||||
if !strings.HasSuffix(targetURL, string(clnt.GetURL().Separator)) {
|
||||
var st *ClientContent
|
||||
st, err = clnt.Stat(ctx, StatOptions{incomplete: isIncomplete})
|
||||
st, err = clnt.Stat(ctx, StatOptions{incomplete: opts.isIncomplete})
|
||||
if st != nil && err == nil && st.Type.IsDir() {
|
||||
targetURL = targetURL + string(clnt.GetURL().Separator)
|
||||
clnt, err = newClient(targetURL)
|
||||
fatalIf(err.Trace(targetURL), "Unable to initialize target `"+targetURL+"`.")
|
||||
}
|
||||
}
|
||||
if e := doList(ctx, clnt, isRecursive, isIncomplete, isSummary, timeRef, withOlderVersions, storageClassFilter); e != nil {
|
||||
if e := doList(ctx, clnt, opts); e != nil {
|
||||
cErr = e
|
||||
}
|
||||
}
|
||||
|
||||
31
cmd/ls.go
31
cmd/ls.go
@@ -25,7 +25,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/dustin/go-humanize"
|
||||
json "github.com/minio/colorjson"
|
||||
"github.com/minio/mc/pkg/probe"
|
||||
"github.com/minio/pkg/console"
|
||||
@@ -200,8 +200,18 @@ func printObjectVersions(clntURL ClientURL, ctntVersions []*ClientContent, print
|
||||
}
|
||||
}
|
||||
|
||||
type doListOptions struct {
|
||||
timeRef time.Time
|
||||
isRecursive bool
|
||||
isIncomplete bool
|
||||
isSummary bool
|
||||
withOlderVersions bool
|
||||
listZip bool
|
||||
filter string
|
||||
}
|
||||
|
||||
// doList - list all entities inside a folder.
|
||||
func doList(ctx context.Context, clnt Client, isRecursive, isIncomplete, isSummary bool, timeRef time.Time, withOlderVersions bool, filter string) error {
|
||||
func doList(ctx context.Context, clnt Client, o doListOptions) error {
|
||||
var (
|
||||
lastPath string
|
||||
perObjectVersions []*ClientContent
|
||||
@@ -211,12 +221,13 @@ func doList(ctx context.Context, clnt Client, isRecursive, isIncomplete, isSumma
|
||||
)
|
||||
|
||||
for content := range clnt.List(ctx, ListOptions{
|
||||
Recursive: isRecursive,
|
||||
Incomplete: isIncomplete,
|
||||
TimeRef: timeRef,
|
||||
WithOlderVersions: withOlderVersions || !timeRef.IsZero(),
|
||||
Recursive: o.isRecursive,
|
||||
Incomplete: o.isIncomplete,
|
||||
TimeRef: o.timeRef,
|
||||
WithOlderVersions: o.withOlderVersions || !o.timeRef.IsZero(),
|
||||
WithDeleteMarkers: true,
|
||||
ShowDir: DirNone,
|
||||
ListZip: o.listZip,
|
||||
}) {
|
||||
if content.Err != nil {
|
||||
switch content.Err.ToGoError().(type) {
|
||||
@@ -239,13 +250,13 @@ func doList(ctx context.Context, clnt Client, isRecursive, isIncomplete, isSumma
|
||||
continue
|
||||
}
|
||||
|
||||
if content.StorageClass != "" && filter != "" && filter != "*" && content.StorageClass != filter {
|
||||
if content.StorageClass != "" && o.filter != "" && o.filter != "*" && content.StorageClass != o.filter {
|
||||
continue
|
||||
}
|
||||
|
||||
if lastPath != content.URL.Path {
|
||||
// Print any object in the current list before reinitializing it
|
||||
printObjectVersions(clnt.GetURL(), perObjectVersions, withOlderVersions, isSummary)
|
||||
printObjectVersions(clnt.GetURL(), perObjectVersions, o.withOlderVersions, o.isSummary)
|
||||
lastPath = content.URL.Path
|
||||
perObjectVersions = []*ClientContent{}
|
||||
}
|
||||
@@ -255,9 +266,9 @@ func doList(ctx context.Context, clnt Client, isRecursive, isIncomplete, isSumma
|
||||
totalObjects++
|
||||
}
|
||||
|
||||
printObjectVersions(clnt.GetURL(), perObjectVersions, withOlderVersions, isSummary)
|
||||
printObjectVersions(clnt.GetURL(), perObjectVersions, o.withOlderVersions, o.isSummary)
|
||||
|
||||
if isSummary {
|
||||
if o.isSummary {
|
||||
printMsg(summaryMessage{
|
||||
TotalObjects: totalObjects,
|
||||
TotalSize: totalSize,
|
||||
|
||||
@@ -463,7 +463,7 @@ func (mj *mirrorJob) doMirror(ctx context.Context, sURLs URLs) URLs {
|
||||
sURLs.DisableMultipart = mj.opts.disableMultipart
|
||||
|
||||
now := time.Now()
|
||||
ret := uploadSourceToTargetURL(ctx, sURLs, mj.status, mj.opts.encKeyDB, mj.opts.isMetadata)
|
||||
ret := uploadSourceToTargetURL(ctx, sURLs, mj.status, mj.opts.encKeyDB, mj.opts.isMetadata, false)
|
||||
if ret.Error == nil {
|
||||
durationMs := time.Since(now) / time.Millisecond
|
||||
mirrorReplicationDurations.With(prometheus.Labels{"object_size": convertSizeToTag(sURLs.SourceContent.Size)}).Observe(float64(durationMs))
|
||||
|
||||
@@ -66,7 +66,7 @@ func checkMirrorSyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB map[st
|
||||
|
||||
/****** Generic rules *******/
|
||||
if !cliCtx.Bool("watch") && !cliCtx.Bool("active-active") && !cliCtx.Bool("multi-master") {
|
||||
_, srcContent, err := url2Stat(ctx, srcURL, "", false, encKeyDB, time.Time{})
|
||||
_, srcContent, err := url2Stat(ctx, srcURL, "", false, encKeyDB, time.Time{}, false)
|
||||
if err != nil {
|
||||
fatalIf(err.Trace(srcURL), "Unable to stat source `"+srcURL+"`.")
|
||||
}
|
||||
|
||||
@@ -272,7 +272,7 @@ func removeSingle(url, versionID string, isIncomplete, isFake, isForce, isBypass
|
||||
modTime time.Time
|
||||
)
|
||||
|
||||
_, content, pErr := url2Stat(ctx, url, versionID, false, encKeyDB, time.Time{})
|
||||
_, content, pErr := url2Stat(ctx, url, versionID, false, encKeyDB, time.Time{}, false)
|
||||
if pErr != nil {
|
||||
switch minio.ToErrorResponse(pErr.ToGoError()).StatusCode {
|
||||
case http.StatusBadRequest, http.StatusMethodNotAllowed:
|
||||
|
||||
@@ -104,7 +104,7 @@ func checkShareDownloadSyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB
|
||||
// Validate if object exists only if the `--recursive` flag was NOT specified
|
||||
if !isRecursive {
|
||||
for _, url := range cliCtx.Args() {
|
||||
_, _, err := url2Stat(ctx, url, "", false, encKeyDB, time.Time{})
|
||||
_, _, err := url2Stat(ctx, url, "", false, encKeyDB, time.Time{}, false)
|
||||
if err != nil {
|
||||
fatalIf(err.Trace(url), "Unable to stat `"+url+"`.")
|
||||
}
|
||||
|
||||
@@ -454,7 +454,7 @@ func mainSQL(cliCtx *cli.Context) error {
|
||||
URLs := cliCtx.Args()
|
||||
writeHdr := true
|
||||
for _, url := range URLs {
|
||||
if _, targetContent, err := url2Stat(ctx, url, "", false, encKeyDB, time.Time{}); err != nil {
|
||||
if _, targetContent, err := url2Stat(ctx, url, "", false, encKeyDB, time.Time{}, false); err != nil {
|
||||
errorIf(err.Trace(url), "Unable to run sql for "+url+".")
|
||||
continue
|
||||
} else if !targetContent.Type.IsDir() {
|
||||
|
||||
@@ -125,7 +125,7 @@ func parseAndCheckStatSyntax(ctx context.Context, cliCtx *cli.Context, encKeyDB
|
||||
}
|
||||
|
||||
for _, url := range URLs {
|
||||
_, _, err := url2Stat(ctx, url, versionID, false, encKeyDB, rewind)
|
||||
_, _, err := url2Stat(ctx, url, versionID, false, encKeyDB, rewind, false)
|
||||
if err != nil && !isURLPrefixExists(url, isIncomplete) {
|
||||
fatalIf(err.Trace(url), "Unable to stat `"+url+"`.")
|
||||
}
|
||||
|
||||
@@ -229,7 +229,7 @@ func statURL(ctx context.Context, targetURL, versionID string, timeRef time.Time
|
||||
continue
|
||||
}
|
||||
}
|
||||
clnt, stat, err := url2Stat(ctx, url, content.VersionID, true, encKeyDB, timeRef)
|
||||
clnt, stat, err := url2Stat(ctx, url, content.VersionID, true, encKeyDB, timeRef, false)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -131,7 +131,7 @@ func parseTreeSyntax(ctx context.Context, cliCtx *cli.Context) (args []string, d
|
||||
}
|
||||
|
||||
for _, url := range args {
|
||||
if _, _, err := url2Stat(ctx, url, "", false, nil, timeRef); err != nil && !isURLPrefixExists(url, false) {
|
||||
if _, _, err := url2Stat(ctx, url, "", false, nil, timeRef, false); err != nil && !isURLPrefixExists(url, false) {
|
||||
fatalIf(err.Trace(url), "Unable to tree `"+url+"`.")
|
||||
}
|
||||
}
|
||||
@@ -284,7 +284,16 @@ func mainTree(cliCtx *cli.Context) error {
|
||||
}
|
||||
clnt, err := newClientFromAlias(targetAlias, targetURL)
|
||||
fatalIf(err.Trace(targetURL), "Unable to initialize target `"+targetURL+"`.")
|
||||
if e := doList(ctx, clnt, true, false, false, timeRef, false, "*"); e != nil {
|
||||
opts := doListOptions{
|
||||
timeRef: timeRef,
|
||||
isRecursive: true,
|
||||
isIncomplete: false,
|
||||
isSummary: false,
|
||||
withOlderVersions: false,
|
||||
listZip: false,
|
||||
filter: "*",
|
||||
}
|
||||
if e := doList(ctx, clnt, opts); e != nil {
|
||||
cErr = e
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user