From b4c78a33ffe0894ca02491eaa69b9a24f76f00a2 Mon Sep 17 00:00:00 2001 From: Krishna Srinivas Date: Sat, 17 Oct 2015 00:46:12 -0700 Subject: [PATCH] support for "rm" of objects and buckets --- main.go | 1 + pkg/client/client.go | 1 + pkg/client/fs/fs.go | 5 + pkg/client/s3v2/s3v2.go | 12 +++ pkg/client/s3v4/s3v4.go | 12 +++ remove-main.go | 201 ++++++++++++++++++++++++++++++++++++++++ url.go | 10 ++ 7 files changed, 242 insertions(+) create mode 100644 remove-main.go diff --git a/main.go b/main.go index c5d49d12..cd714817 100644 --- a/main.go +++ b/main.go @@ -119,6 +119,7 @@ func registerApp() *cli.App { registerCmd(lsCmd) // List contents of a bucket. registerCmd(mbCmd) // Make a bucket. registerCmd(catCmd) // Display contents of a file. + registerCmd(rmCmd) // Remove a file or bucket registerCmd(pigCmd) // Write contents of stdin to a file. registerCmd(cpCmd) // Copy objects and files from multiple sources to single destination. registerCmd(mirrorCmd) // Mirror objects and files from single source to multiple destinations. diff --git a/pkg/client/client.go b/pkg/client/client.go index b25ba910..d6d62e95 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -40,6 +40,7 @@ type Client interface { ShareUpload(bool, time.Duration, string) (map[string]string, *probe.Error) GetObject(offset, length int64) (body io.ReadCloser, size int64, err *probe.Error) PutObject(size int64, data io.Reader) *probe.Error + Remove() *probe.Error // URL returns back internal url URL() *URL diff --git a/pkg/client/fs/fs.go b/pkg/client/fs/fs.go index 73a33cbf..58573d3f 100644 --- a/pkg/client/fs/fs.go +++ b/pkg/client/fs/fs.go @@ -191,6 +191,11 @@ func (f *fsClient) GetObject(offset, length int64) (io.ReadCloser, int64, *probe return body, length, nil } +func (f *fsClient) Remove() *probe.Error { + err := os.Remove(f.Path) + return probe.NewError(err) +} + // List - list files and folders func (f *fsClient) List(recursive bool) <-chan client.ContentOnChannel { contentCh := make(chan client.ContentOnChannel) diff --git a/pkg/client/s3v2/s3v2.go b/pkg/client/s3v2/s3v2.go index 1be0783f..e21e61c3 100644 --- a/pkg/client/s3v2/s3v2.go +++ b/pkg/client/s3v2/s3v2.go @@ -76,6 +76,18 @@ func (c *s3Client) GetObject(offset, length int64) (io.ReadCloser, int64, *probe return reader, metadata.Size, nil } +// Remove - remove object or bucket +func (c *s3Client) Remove() *probe.Error { + bucket, object := c.url2BucketAndObject() + var err error + if object == "" { + err = c.api.RemoveBucket(bucket) + } else { + err = c.api.RemoveObject(bucket, object) + } + return probe.NewError(err) +} + // Share - get a usable get object url to share func (c *s3Client) ShareDownload(expires time.Duration) (string, *probe.Error) { bucket, object := c.url2BucketAndObject() diff --git a/pkg/client/s3v4/s3v4.go b/pkg/client/s3v4/s3v4.go index 127ba0f7..658c971e 100644 --- a/pkg/client/s3v4/s3v4.go +++ b/pkg/client/s3v4/s3v4.go @@ -76,6 +76,18 @@ func (c *s3Client) GetObject(offset, length int64) (io.ReadCloser, int64, *probe return reader, metadata.Size, nil } +// Remove - remove object or bucket +func (c *s3Client) Remove() *probe.Error { + bucket, object := c.url2BucketAndObject() + var err error + if object == "" { + err = c.api.RemoveBucket(bucket) + } else { + err = c.api.RemoveObject(bucket, object) + } + return probe.NewError(err) +} + // Share - get a usable get object url to share func (c *s3Client) ShareDownload(expires time.Duration) (string, *probe.Error) { bucket, object := c.url2BucketAndObject() diff --git a/remove-main.go b/remove-main.go new file mode 100644 index 00000000..4432aec9 --- /dev/null +++ b/remove-main.go @@ -0,0 +1,201 @@ +/* + * Minio Client (C) 2014, 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "os" + "strings" + + "github.com/minio/cli" + "github.com/minio/mc/pkg/client" + "github.com/minio/minio-xl/pkg/probe" +) + +// remove a file or folder. +var rmCmd = cli.Command{ + Name: "rm", + Usage: "Remove file or bucket.", + Action: mainRm, + CustomHelpTemplate: `NAME: + mc {{.Name}} - {{.Usage}} + +USAGE: + mc {{.Name}} TARGET + +EXAMPLES: + 1. Remove a file on Cloud storage + $ mc {{.Name}} https://s3.amazonaws.com/jazz-songs/louis/file01.mp3 + + 2. Remove a folder recursively on Cloud storage + $ mc {{.Name}} force https://s3.amazonaws.com/jazz-songs/louis/... + + 3. Remove a bucket on Minio cloud storage + $ mc {{.Name}} https://play.minio.io:9000/mongodb-backup + + 4. Remove a bucket on Cloud storage recursively + $ mc {{.Name}} force https://s3.amazonaws.com/jazz-songs/... + + 5. Remove a file on local filesystem: + $ mc {{.Name}} march/expenses.doc + + 6. Remove a file named "force" on local filesystem: + $ mc {{.Name}} force force +`, +} + +func rmList(url string) (<-chan string, *probe.Error) { + clnt, err := url2Client(url) + if err != nil { + errorIf(err.Trace(), "Unable to get client object for "+url) + return nil, err.Trace() + } + in := clnt.List(true) + out := make(chan string) + + var depthFirst func(currentDir string) (*client.Content, bool) + + depthFirst = func(currentDir string) (*client.Content, bool) { + entry, ok := <-in + for { + if !ok || !strings.HasPrefix(entry.Content.Name, currentDir) { + return entry.Content, ok + } + if entry.Content.Type.IsRegular() { + out <- entry.Content.Name + } + if entry.Content.Type.IsDir() { + var content *client.Content + content, ok = depthFirst(entry.Content.Name) + out <- entry.Content.Name + entry = client.ContentOnChannel{Content: content} + continue + } + entry, ok = <-in + } + } + + go func() { + depthFirst("") + close(out) + }() + return out, nil +} + +func rm(url string) { + clnt, err := url2Client(url) + if err != nil { + errorIf(err.Trace(), "Unable to get client object for "+url) + return + } + err = clnt.Remove() + errorIf(err.Trace(), "Unable to remove "+url) +} + +func rmAll(url string) { + clnt, err := url2Client(url) + if err != nil { + errorIf(err.Trace(), "Unable to get client object for "+url) + return + } + urlPartial1 := url2Dir(url) + out, err := rmList(url) + if err != nil { + errorIf(err.Trace(), "Unable to List "+url) + return + } + for urlPartial2 := range out { + urlFull := urlPartial1 + urlPartial2 + newclnt, e := url2Client(urlFull) + if e != nil { + errorIf(e, "Unable to create client object : "+urlFull) + continue + } + err = newclnt.Remove() + errorIf(err, "Unable to remove : "+urlFull) + } + _, err = clnt.Stat() + if err == nil { + err = clnt.Remove() + errorIf(err, "Unable to remove : "+clnt.URL().String()) + } +} + +func checkRmSyntax(ctx *cli.Context) { + args, err := args2URLs(ctx.Args()) + fatalIf(err.Trace(), "args2URL failed") + var force bool + if len(args) == 0 { + cli.ShowCommandHelpAndExit(ctx, "rm", 1) // last argument is exit code. + } + if len(args) == 1 && args[0] == "force" { + cli.ShowCommandHelpAndExit(ctx, "rm", 1) + } + if args[0] == "force" { + force = true + args = args[1:] + } + // If input validation fails then provide context sensitive help without displaying generic help message. + // The context sensitive help is shown per argument instead of all arguments to keep the help display + // as well as the code simple. Also most of the times there will be just one arg + for _, arg := range args { + url := client.NewURL(arg) + if strings.HasSuffix(arg, string(url.Separator)) { + helpStr := "Usage : mc rm force " + arg + recursiveSeparator + fatalIf(errDummy().Trace(), helpStr) + } + if isURLRecursive(arg) && !force { + helpStr := "Usage : mc rm force " + arg + fatalIf(errDummy().Trace(), helpStr) + } + if url.Type == client.Filesystem { + // For local file system we don't support "mc rm fileprefix..." just like the behavior of "mc ls fileprefix..." + // So recursive delete has to be of the form "mc rm dir1/dir2/..." + isRecursive := isURLRecursive(arg) + path := stripRecursiveURL(arg) + if isRecursive && (strings.HasSuffix(path, string(url.Separator)) == false) { + helpStr := "Usage : mc rm force " + path + string(url.Separator) + recursiveSeparator + fatalIf(errDummy().Trace(), helpStr) + } + _, content, err := url2Stat(path) + if err != nil { + fatalIf(err.Trace(), "url2stat error on "+arg) + } + if content.Type&os.ModeDir != 0 && !isRecursive { + helpStr := "Usage : mc rm force " + arg + string(url.Separator) + recursiveSeparator + fatalIf(errDummy().Trace(), helpStr) + } + continue + } + } +} + +func mainRm(ctx *cli.Context) { + checkRmSyntax(ctx) + args, err := args2URLs(ctx.Args()) + fatalIf(err.Trace(), "args2URL failed") + if args[0] == "force" { + args = args[1:] + } + for _, arg := range args { + if isURLRecursive(arg) { + url := stripRecursiveURL(arg) + rmAll(url) + } else { + rm(arg) + } + } +} diff --git a/url.go b/url.go index 5ee0ae67..3fc9ccb5 100644 --- a/url.go +++ b/url.go @@ -85,3 +85,13 @@ func url2Stat(urlStr string) (client client.Client, content *client.Content, err } return client, content, nil } + +// just like filepath.Dir but always has a trailing url.Seperator +func url2Dir(urlStr string) string { + url := client.NewURL(urlStr) + if strings.HasSuffix(urlStr, string(url.Separator)) { + return urlStr + } + lastIndex := strings.LastIndex(urlStr, string(url.Separator)) + return urlStr[:lastIndex+1] +}