Filesystem now calculates md5sum and returns

pkg/client/fs/fs_put_unix.go

@@ -19,6 +19,9 @@
 package fs
 
 import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
 	"errors"
 	"io"
 	"os"
@@ -57,13 +60,30 @@ func (f *fsClient) Put(md5HexString string, size int64) (io.WriteCloser, error)
 			blockingWriter.Release(err)
 			return
 		}
-		_, err = io.CopyN(fs, r, size)
+		// calculate md5 to verify - incoming md5
+		h := md5.New()
+		mw := io.MultiWriter(fs, h)
+
+		_, err = io.CopyN(mw, r, size)
 		if err != nil {
 			err := iodine.New(err, nil)
 			r.CloseWithError(err)
 			blockingWriter.Release(err)
 			return
 		}
+		expectedMD5, err := hex.DecodeString(md5HexString)
+		if err != nil {
+			err := iodine.New(err, nil)
+			r.CloseWithError(err)
+			blockingWriter.Release(err)
+			return
+		}
+		if !bytes.Equal(expectedMD5, h.Sum(nil)) {
+			err := iodine.New(errors.New("md5sum mismatch"), nil)
+			r.CloseWithError(err)
+			blockingWriter.Release(err)
+			return
+		}
 		blockingWriter.Release(nil)
 		r.Close()
 	}()

pkg/client/fs/fs_put_windows.go

@@ -17,6 +17,9 @@
 package fs
 
 import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
 	"errors"
 	"io"
 	"os"
@@ -55,13 +58,30 @@ func (f *fsClient) Put(md5HexString string, size int64) (io.WriteCloser, error)
 			blockingWriter.Release(err)
 			return
 		}
-		_, err = io.CopyN(fs, r, size)
+		// calculate md5 to verify - incoming md5
+		h := md5.New()
+		mw := io.MultiWriter(fs, h)
+
+		_, err = io.CopyN(mw, r, size)
 		if err != nil {
 			err := iodine.New(err, nil)
 			r.CloseWithError(err)
 			blockingWriter.Release(err)
 			return
 		}
+		expectedMD5, err := hex.DecodeString(md5HexString)
+		if err != nil {
+			err := iodine.New(err, nil)
+			r.CloseWithError(err)
+			blockingWriter.Release(err)
+			return
+		}
+		if !bytes.Equal(expectedMD5, h.Sum(nil)) {
+			err := iodine.New(errors.New("md5sum mismatch"), nil)
+			r.CloseWithError(err)
+			blockingWriter.Release(err)
+			return
+		}
 		blockingWriter.Release(nil)
 		r.Close()
 	}()

pkg/client/fs/fs_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"crypto/md5"
 	"encoding/base64"
+	"encoding/hex"
 	"io"
 	"io/ioutil"
 	"os"
@@ -130,7 +131,7 @@ func (s *MySuite) TestGetObject(c *C) {
 
 	data := "hello"
 	binarySum := md5.Sum([]byte(data))
-	etag := base64.StdEncoding.EncodeToString(binarySum[:])
+	etag := hex.EncodeToString(binarySum[:])
 	dataLen := int64(len(data))
 
 	writer, err := fsc.Put(etag, dataLen)
@@ -139,10 +140,10 @@ func (s *MySuite) TestGetObject(c *C) {
 	_, err = io.CopyN(writer, bytes.NewBufferString(data), dataLen)
 	c.Assert(err, IsNil)
 
-	reader, size, _, err := fsc.Get()
+	reader, size, md5Sum, err := fsc.Get()
 	c.Assert(err, IsNil)
 	var results bytes.Buffer
-
+	c.Assert(etag, Equals, md5Sum)
 	_, err = io.CopyN(&results, reader, size)
 	c.Assert(err, IsNil)
 	c.Assert([]byte(data), DeepEquals, results.Bytes())

pkg/client/fs/fs_unix.go

@@ -19,6 +19,8 @@
 package fs
 
 import (
+	"crypto/md5"
+	"encoding/hex"
 	"errors"
 	"io"
 	"os"
@@ -58,7 +60,7 @@ func (f *fsClient) fsStat() (os.FileInfo, error) {
 }
 
 // Get - download an object from bucket
-func (f *fsClient) Get() (body io.ReadCloser, size int64, md5 string, err error) {
+func (f *fsClient) Get() (io.ReadCloser, int64, string, error) {
 	item, err := f.getFSMetadata()
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
@@ -66,17 +68,27 @@ func (f *fsClient) Get() (body io.ReadCloser, size int64, md5 string, err error)
 	if item.FileType.IsDir() {
 		return nil, 0, "", iodine.New(FileISDir{path: f.path}, nil)
 	}
-	body, err = os.Open(f.path)
+	body, err := os.Open(f.path)
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
 	}
-	// TODO: support md5sum - there is no easier way to do it right now without temporary buffer
+	h := md5.New()
-	// so avoiding it to ensure no out of memory situations
+	// calculate md5sum
-	return body, item.Size, "", nil
+	_, err = io.Copy(h, body)
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	// seek back
+	_, err = body.Seek(0, 0)
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	md5Str := hex.EncodeToString(h.Sum(nil))
+	return body, item.Size, md5Str, nil
 }
 
 // GetPartial - download a partial object from bucket
-func (f *fsClient) GetPartial(offset, length int64) (body io.ReadCloser, size int64, md5 string, err error) {
+func (f *fsClient) GetPartial(offset, length int64) (io.ReadCloser, int64, string, error) {
 	if offset < 0 {
 		return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil)
 	}
@@ -90,11 +102,27 @@ func (f *fsClient) GetPartial(offset, length int64) (body io.ReadCloser, size in
 	if offset > item.Size || (offset+length-1) > item.Size {
 		return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil)
 	}
+	body, err := os.Open(f.path)
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
 	_, err = io.CopyN(ioutil.Discard, body, offset)
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
 	}
-	return body, length, "", nil
+	h := md5.New()
+	// calculate md5sum
+	_, err = io.Copy(h, body)
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	// seek back
+	_, err = body.Seek(0, 0)
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	md5Str := hex.EncodeToString(h.Sum(nil))
+	return body, length, md5Str, nil
 }
 
 func (f *fsClient) List() <-chan client.ItemOnChannel {

pkg/client/fs/fs_windows.go

@@ -17,6 +17,8 @@
 package fs
 
 import (
+	"crypto/md5"
+	"encoding/hex"
 	"errors"
 	"io"
 	"os"
@@ -54,7 +56,7 @@ func (f *fsClient) fsStat() (os.FileInfo, error) {
 }
 
 // Get - download an object from bucket
-func (f *fsClient) Get() (body io.ReadCloser, size int64, md5 string, err error) {
+func (f *fsClient) Get() (io.ReadCloser, int64, string, error) {
 	item, err := f.getFSMetadata()
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
@@ -62,16 +64,27 @@ func (f *fsClient) Get() (body io.ReadCloser, size int64, md5 string, err error)
 	if item.FileType.IsDir() {
 		return nil, 0, "", iodine.New(FileISDir{path: f.path}, nil)
 	}
-	body, err = os.Open(f.path)
+	body, err := os.Open(f.path)
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
 	}
-	// TODO: support md5sum -
+	h := md5.New()
-	return body, st.Size(), "", nil
+	// calculate md5sum
+	_, err = io.Copy(h, body)
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	// seek back
+	_, err = body.Seek(0, 0)
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	md5Str := hex.EncodeToString(h.Sum(nil))
+	return body, item.Size, md5Str, nil
 }
 
 // GetPartial - download a partial object from bucket
-func (f *fsClient) GetPartial(offset, length int64) (body io.ReadCloser, size int64, md5 string, err error) {
+func (f *fsClient) GetPartial(offset, length int64) (io.ReadCloser, int64, string, error) {
 	if offset < 0 {
 		return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil)
 	}
@@ -82,7 +95,7 @@ func (f *fsClient) GetPartial(offset, length int64) (body io.ReadCloser, size in
 	if item.FileType.IsDir() {
 		return nil, 0, "", iodine.New(FileISDir{path: f.path}, nil)
 	}
-	body, err = os.Open(f.path)
+	body, err := os.Open(f.path)
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
 	}
@@ -93,7 +106,19 @@ func (f *fsClient) GetPartial(offset, length int64) (body io.ReadCloser, size in
 	if err != nil {
 		return nil, 0, "", iodine.New(err, nil)
 	}
-	return body, length, "", nil
+	h := md5.New()
+	// calculate md5sum
+	_, err = io.Copy(h, body)
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	// seek back
+	_, err = body.Seek(0, 0)
+	if err != nil {
+		return nil, 0, "", iodine.New(err, nil)
+	}
+	md5Str := hex.EncodeToString(h.Sum(nil))
+	return body, length, md5Str, nil
 }
 
 /// Bucket operations