Adding mc cat

Godeps/Godeps.json

@@ -1,6 +1,9 @@
 {
 	"ImportPath": "github.com/minio-io/mc",
 	"GoVersion": "go1.4.2",
+	"Packages": [
+		"./..."
+	],
 	"Deps": [
 		{
 			"ImportPath": "github.com/cheggaaa/pb",
@@ -35,6 +38,10 @@
 			"ImportPath": "github.com/minio-io/minio/pkg/iodine",
 			"Rev": "1d4bc0c34e0c0ceb6606dab4e99288df0166c3d1"
 		},
+		{
+			"ImportPath": "github.com/minio-io/minio/pkg/utils/crypto/md5",
+			"Rev": "1d4bc0c34e0c0ceb6606dab4e99288df0166c3d1"
+		},
 		{
 			"ImportPath": "github.com/minio-io/minio/pkg/utils/log",
 			"Rev": "1d4bc0c34e0c0ceb6606dab4e99288df0166c3d1"

Godeps/_workspace/src/github.com/minio-io/minio/pkg/utils/crypto/md5/md5.go

@@ -0,0 +1,44 @@
+/*
+ * Minimalist Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package md5
+
+import (
+	"crypto/md5"
+	"io"
+)
+
+// Sum - low memory footprint io.Reader based md5sum helper
+func Sum(reader io.Reader) ([]byte, error) {
+	hash := md5.New()
+	var err error
+	var length int
+	for err == nil {
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		// While hash.Write() wouldn't mind a Nil byteBuffer
+		// It is necessary for us to verify this and break
+		if length == 0 {
+			break
+		}
+		byteBuffer = byteBuffer[0:length]
+		hash.Write(byteBuffer)
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+	return hash.Sum(nil), nil
+}

Godeps/_workspace/src/github.com/minio-io/minio/pkg/utils/crypto/md5/md5_test.go

@@ -0,0 +1,24 @@
+package md5_test
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	. "github.com/minio-io/check"
+	"github.com/minio-io/minio/pkg/utils/crypto/md5"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestMd5sum(c *C) {
+	testString := []byte("Test string")
+	expectedHash, _ := hex.DecodeString("0fd3dbec9730101bff92acc820befc34")
+	hash, err := md5.Sum(bytes.NewBuffer(testString))
+	c.Assert(err, IsNil)
+	c.Assert(bytes.Equal(expectedHash, hash), Equals, true)
+}

cmd-cat.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"sync"
+
+	"encoding/base64"
+	"github.com/minio-io/cli"
+	"github.com/minio-io/minio/pkg/iodine"
+	"github.com/minio-io/minio/pkg/utils/crypto/md5"
+	"github.com/minio-io/minio/pkg/utils/log"
+	"os"
+	"strings"
+)
+
+func runCatCmd(ctx *cli.Context) {
+	if len(ctx.Args()) != 1 {
+		cli.ShowCommandHelpAndExit(ctx, "cat", 1) // last argument is exit code
+	}
+
+	config, err := getMcConfig()
+	if err != nil {
+		log.Debug.Println(iodine.New(err, nil))
+		os.Exit(-1)
+	}
+
+	// Convert arguments to URLs: expand alias, fix format...
+	urls, err := getURLs(ctx.Args(), config.Aliases)
+	if err != nil {
+		switch iodine.ToError(err).(type) {
+		case errUnsupportedScheme:
+			log.Debug.Println(iodine.New(err, nil))
+			os.Exit(-1)
+		default:
+			log.Debug.Println(iodine.New(err, nil))
+			os.Exit(-1)
+		}
+	}
+
+	sourceURL := urls[0] // First arg is source
+	recursive := isURLRecursive(sourceURL)
+	// if recursive strip off the "..."
+	if recursive {
+		sourceURL = strings.TrimSuffix(sourceURL, recursiveSeparator)
+	}
+
+	sourceURLConfigMap := make(map[string]*hostConfig)
+	sourceConfig, err := getHostConfig(sourceURL)
+	if err != nil {
+		log.Debug.Println(iodine.New(err, nil))
+		os.Exit(-1)
+	}
+	sourceURLConfigMap[sourceURL] = sourceConfig
+
+	if err != nil {
+		log.Debug.Println(iodine.New(err, nil))
+		os.Exit(-1)
+	}
+	doCatCmd(mcClientManager{}, os.Stdout, sourceURLConfigMap, globalDebugFlag)
+}
+
+func doCatCmd(manager clientManager, writer io.Writer, sourceURLConfigMap map[string]*hostConfig, debug bool) (string, error) {
+	for url, config := range sourceURLConfigMap {
+		clnt, err := manager.getNewClient(url, config, debug)
+		if err != nil {
+			// No human readable, will return with non-zero exit
+			return "", iodine.New(err, nil)
+		}
+		reader, size, etag, err := clnt.Get()
+		if err != nil {
+			// No human readable, will return with non-zero exit
+			return "", iodine.New(err, nil)
+		}
+		wg := &sync.WaitGroup{}
+		md5Reader, md5Writer := io.Pipe()
+		var actualMd5 []byte
+		wg.Add(1)
+		go func() {
+			actualMd5, _ = md5.Sum(md5Reader)
+			// drop error, we'll catch later on if it is important
+			wg.Done()
+		}()
+		teeReader := io.TeeReader(reader, md5Writer)
+		_, err = io.CopyN(writer, teeReader, size)
+		md5Writer.Close()
+		if err != nil {
+			return "", iodine.New(err, nil)
+		}
+		wg.Wait()
+		expectedMd5, err := base64.StdEncoding.DecodeString(etag)
+		if err != nil {
+			return "", iodine.New(errors.New("Unable to read md5sum (etag)"), nil)
+		}
+		if !bytes.Equal(expectedMd5, actualMd5) {
+			return "", iodine.New(errors.New("corruption occurred"), nil)
+		}
+	}
+	return "", nil
+}

cmd-options.go

@@ -26,6 +26,32 @@ import (
 
 // List of commands
 var (
+	catCmd = cli.Command{
+		Name:   "cat",
+		Usage:  "Copy an object to standard out",
+		Action: runCatCmd,
+		CustomHelpTemplate: `NAME:
+   mc {{.Name}} - {{.Usage}}
+
+USAGE:
+   mc {{.Name}}{{if .Flags}} [ARGS...]{{end}} SOURCE {{if .Description}}
+
+DESCRIPTION:
+   {{.Description}}{{end}}{{if .Flags}}
+
+OPTIONS:
+   {{range .Flags}}{{.}}
+   {{end}}{{ end }}
+
+EXAMPLES:
+   1. Copy an object from Amazon S3 object storage to standard out.
+      $ mc {{.Name}} https://s3.amazonaws.com/jukebox/klingon_opera_aktuh_maylotah.ogg
+
+   2. Copy an object from the file system to standard out.
+      $ mc {{.Name}} klingon_opera_aktuh_maylotah.ogg
+
+`,
+	}
 	cpCmd = cli.Command{
 		Name:   "cp",
 		Usage:  "Copy objects and files",
@@ -183,6 +209,7 @@ EXAMPLES:
 )
 
 var options = []cli.Command{
+	catCmd,
 	cpCmd,
 	lsCmd,
 	mbCmd,

cmd_test.go

@@ -477,3 +477,34 @@ func (s *CmdTestSuite) TestMbCmdOnFile(c *C) {
 	manager.AssertExpectations(c)
 	cl1.AssertExpectations(c)
 }
+
+func (s *CmdTestSuite) TestCatCmd(c *C) {
+	sourceURL, err := getURL("http://example.com/bucket1/object1", nil)
+	c.Assert(err, IsNil)
+
+	manager := &MockclientManager{}
+	cl1 := &clientMocks.Client{}
+
+	data1 := "hello1"
+	binarySum1 := md5.Sum([]byte(data1))
+	etag1 := base64.StdEncoding.EncodeToString(binarySum1[:])
+	dataLen1 := int64(len(data1))
+
+	sourceURLConfigMap := make(map[string]*hostConfig)
+	sourceConfig := new(hostConfig)
+	sourceConfig.AccessKeyID = ""
+	sourceConfig.SecretAccessKey = ""
+	sourceURLConfigMap[sourceURL] = sourceConfig
+
+	var results bytes.Buffer
+	manager.On("getNewClient", sourceURL, sourceConfig, false).Return(cl1, nil).Once()
+	cl1.On("Get").Return(ioutil.NopCloser(bytes.NewBufferString(data1)), dataLen1, etag1, nil)
+	msg, err := doCatCmd(manager, &results, sourceURLConfigMap, false)
+	c.Assert(msg, Equals, "")
+	c.Assert(err, IsNil)
+
+	c.Assert(data1, Equals, results.String())
+
+	manager.AssertExpectations(c)
+	cl1.AssertExpectations(c)
+}