diff --git a/cmd/cat-main.go b/cmd/cat-main.go index 685546af..7a344133 100644 --- a/cmd/cat-main.go +++ b/cmd/cat-main.go @@ -69,6 +69,10 @@ EXAMPLES: 4. Save an encrypted object from Amazon S3 cloud storage to a local file. $ {{.HelpName}} --encrypt-key 's3/mysql-backups=32byteslongsecretkeymustbegiven1' s3/mysql-backups/backups-201810.gz > /mnt/data/recent.gz + + 5. Display the content of encrypted object. In case the encryption key contains non-printable character like tab, pass the + base64 encoded string as key. + $ {{.HelpName}} --encrypt-key "play/my-bucket/=MzJieXRlc2xvbmdzZWNyZXRrZQltdXN0YmVnaXZlbjE=" play/my-bucket/my-object `, } diff --git a/cmd/common-methods.go b/cmd/common-methods.go index 66431c20..50b69943 100644 --- a/cmd/common-methods.go +++ b/cmd/common-methods.go @@ -18,6 +18,8 @@ package cmd import ( "context" + "encoding/base64" + "errors" "io" "os" "path/filepath" @@ -32,6 +34,36 @@ import ( "github.com/minio/minio-go/v6/pkg/encrypt" ) +// decode if the key is encoded key and returns the key +func getDecodedKey(sseKeys string) (key string, err *probe.Error) { + keyString := "" + for i, sse := range strings.Split(sseKeys, ",") { + if i > 0 { + keyString = keyString + "," + } + sseString, err := parseKey(sse) + if err != nil { + return "", err + } + keyString = keyString + sseString + } + return keyString, nil +} + +// Validate the key +func parseKey(sseKeys string) (sse string, err *probe.Error) { + encryptString := strings.SplitN(sseKeys, "=", 2) + secretValue := encryptString[1] + if len(secretValue) == 32 { + return sseKeys, nil + } + decodedString, e := base64.StdEncoding.DecodeString(secretValue) + if e != nil || len(decodedString) != 32 { + return "", probe.NewError(errors.New("Encryption key should be 32 bytes plain text key or 44 bytes base64 encoded key")) + } + return encryptString[0] + "=" + string(decodedString), nil +} + // parse and return encryption key pairs per alias. func getEncKeys(ctx *cli.Context) (map[string][]prefixSSEPair, *probe.Error) { sseServer := os.Getenv("MC_ENCRYPT") @@ -46,6 +78,13 @@ func getEncKeys(ctx *cli.Context) (map[string][]prefixSSEPair, *probe.Error) { } sseKeys = keyPrefix } + var err *probe.Error + if sseKeys != "" { + sseKeys, err = getDecodedKey(sseKeys) + if err != nil { + return nil, err.Trace(sseKeys) + } + } encKeyDB, err := parseAndValidateEncryptionKeys(sseKeys, sseServer) if err != nil { diff --git a/cmd/common-methods_test.go b/cmd/common-methods_test.go new file mode 100644 index 00000000..e30f6635 --- /dev/null +++ b/cmd/common-methods_test.go @@ -0,0 +1,70 @@ +/* + * MinIO Client (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "errors" + "reflect" + "testing" +) + +func TestGetDecodedKey(t *testing.T) { + getDecodeCases := []struct { + input string + output string + err error + status bool + }{ + //success scenario the key contains non printable (tab) character as key + {"s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=", "s3/documents/=32byteslongsecreabcdefg givenn21", nil, true}, + //success scenario the key contains non printable (tab character) as key + {"s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=,play/documents/=MzJieXRlc2xvbmdzZWNyZXRrZQltdXN0YmVnaXZlbjE=", "s3/documents/=32byteslongsecreabcdefg givenn21,play/documents/=32byteslongsecretke mustbegiven1", nil, true}, + // success scenario using a normal string + {"s3/documents/=32byteslongsecretkeymustbegiven1", "s3/documents/=32byteslongsecretkeymustbegiven1", nil, true}, + // success scenario using a normal string + {"s3/documents/=32byteslongsecretkeymustbegiven1,myminio/documents/=32byteslongsecretkeymustbegiven2", "s3/documents/=32byteslongsecretkeymustbegiven1,myminio/documents/=32byteslongsecretkeymustbegiven2", nil, true}, + // success scenario using a mix of normal string and encoded string + {"s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=,play/documents/=32byteslongsecretkeymustbegiven1", "s3/documents/=32byteslongsecreabcdefg givenn21,play/documents/=32byteslongsecretkeymustbegiven1", nil, true}, + // success scenario using a mix of normal string and encoded string + {"play/documents/=32byteslongsecretkeymustbegiven1,s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=", "play/documents/=32byteslongsecretkeymustbegiven1,s3/documents/=32byteslongsecreabcdefg givenn21", nil, true}, + // decoded key less than 32 char and conatin non printable (tab) character + {"s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE", "", errors.New("Encryption key should be 32 bytes plain text key or 44 bytes base64 encoded key"), false}, + // normal key less than 32 character + {"s3/documents/=32byteslongsecretkeymustbegiven", "", errors.New("Encryption key should be 32 bytes plain text key or 44 bytes base64 encoded key"), false}, + } + + for idx, testCase := range getDecodeCases { + decodedString, errDecode := getDecodedKey(testCase.input) + if testCase.status == true { + if errDecode != nil { + t.Fatalf("Test %d: generated error not matching, expected = `%s`, found = `%s`", idx+1, testCase.err, errDecode) + } + if !reflect.DeepEqual(decodedString, testCase.output) { + t.Fatalf("Test %d: generated key not matching, expected = `%s`, found = `%s`", idx+1, testCase.input, decodedString) + } + } + + if testCase.status == false { + if !reflect.DeepEqual(decodedString, testCase.output) { + t.Fatalf("Test %d: generated Map not matching, expected = `%s`, found = `%s`", idx+1, testCase.input, errDecode) + } + if errDecode.Cause.Error() != testCase.err.Error() { + t.Fatalf("Test %d: generated error not matching, expected = `%s`, found = `%s`", idx+1, testCase.err, errDecode) + } + } + } +} diff --git a/cmd/cp-main.go b/cmd/cp-main.go index 2d4e0469..374d2c8b 100644 --- a/cmd/cp-main.go +++ b/cmd/cp-main.go @@ -115,13 +115,17 @@ EXAMPLES: 09. Copy a folder with encrypted objects recursively from Amazon S3 to MinIO cloud storage. $ {{.HelpName}} --recursive --encrypt-key "s3/documents/=32byteslongsecretkeymustbegiven1,myminio/documents/=32byteslongsecretkeymustbegiven2" s3/documents/ myminio/documents/ - 10. Copy a list of objects from local file system to MinIO cloud storage with specified metadata. + 10. Copy a folder with encrypted objects recursively from Amazon S3 to MinIO cloud storage. In case the encryption key contains non-printable character like tab, pass the + base64 encoded string as key. + $ {{.HelpName}} --recursive --encrypt-key "s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=,myminio/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=" s3/documents/ myminio/documents/ + + 11. Copy a list of objects from local file system to MinIO cloud storage with specified metadata. $ {{.HelpName}} --attr key1=value1,key2=value2 Music/*.mp4 play/mybucket/ - 11. Copy a folder recursively from MinIO cloud storage to Amazon S3 cloud storage with specified metadata. + 12. Copy a folder recursively from MinIO cloud storage to Amazon S3 cloud storage with specified metadata. $ {{.HelpName}} --attr Cache-Control=max-age=90000,min-fresh=9000\;key1=value1\;key2=value2 --recursive play/mybucket/burningman2011/ s3/mybucket/ - 12. Copy a text file to an object storage and assign REDUCED_REDUNDANCY storage-class to the uploaded object. + 13. Copy a text file to an object storage and assign REDUCED_REDUNDANCY storage-class to the uploaded object. $ {{.HelpName}} --storage-class REDUCED_REDUNDANCY myobject.txt play/mybucket `, } @@ -489,6 +493,11 @@ func mainCopy(ctx *cli.Context) error { if key := ctx.String("encrypt-key"); key != "" { sseKeys = key } + + if sseKeys != "" { + sseKeys, err = getDecodedKey(sseKeys) + fatalIf(err, "Unable to parse encryption keys.") + } sse := ctx.String("encrypt") session := newSessionV8() diff --git a/cmd/cp-main_test.go b/cmd/cp-main_test.go index f20c6b22..24fc11d8 100644 --- a/cmd/cp-main_test.go +++ b/cmd/cp-main_test.go @@ -28,15 +28,15 @@ func TestParseMetaData(t *testing.T) { err error status bool }{ - // success scenerio using ; as delimitter + // success scenario using ; as delimiter {"key1=value1;key2=value2", map[string]string{"key1": "value1", "key2": "value2"}, nil, true}, - // success scenerio using ; as delimitter + // success scenario using ; as delimiter {"key1=m1=m2,m3=m4;key2=value2", map[string]string{"key1": "m1=m2,m3=m4", "key2": "value2"}, nil, true}, - // success scenerio using = more than once + // success scenario using = more than once {"Cache-Control=max-age=90000,min-fresh=9000;key1=value1;key2=value2", map[string]string{"Cache-Control": "max-age=90000,min-fresh=9000", "key1": "value1", "key2": "value2"}, nil, true}, - // using different delimitter, other than '=' between key value + // using different delimiter, other than '=' between key value {"key1:value1;key2:value2", nil, ErrInvalidMetadata, false}, - // using no delimitter + // using no delimiter {"key1:value1:key2:value2", nil, ErrInvalidMetadata, false}, } diff --git a/cmd/head-main.go b/cmd/head-main.go index 9570a334..ad0d102d 100644 --- a/cmd/head-main.go +++ b/cmd/head-main.go @@ -68,6 +68,10 @@ EXAMPLES: 2. Display only first line from server encrypted object on Amazon S3. $ {{.HelpName}} -n 1 --encrypt-key 's3/csv-data=32byteslongsecretkeymustbegiven1' s3/csv-data/population.csv + + 3. Display only first line from server encrypted object on Amazon S3. In case the encryption key contains non-printable character like tab, pass the + base64 encoded string as key. + $ {{.HelpName}} --encrypt-key "s3/json-data=MzJieXRlc2xvbmdzZWNyZXRrZQltdXN0YmVnaXZlbjE=" s3/json-data/population.json `, } diff --git a/cmd/mirror-main.go b/cmd/mirror-main.go index 225622f3..0460f2b8 100644 --- a/cmd/mirror-main.go +++ b/cmd/mirror-main.go @@ -146,6 +146,10 @@ EXAMPLES: 11. Mirror server encrypted objects from MinIO cloud storage to a bucket on Amazon S3 cloud storage $ {{.HelpName}} --encrypt-key "minio/photos=32byteslongsecretkeymustbegiven1,s3/archive=32byteslongsecretkeymustbegiven2" minio/photos/ s3/archive/ + + 12. Mirror server encrypted objects from MinIO cloud storage to a bucket on Amazon S3 cloud storage. In case the encryption key contains + non-printable character like tab, pass the base64 encoded string as key. + $ {{.HelpName}} --encrypt-key "s3/photos/=32byteslongsecretkeymustbegiven1,play/archive/=MzJieXRlc2xvbmdzZWNyZXRrZQltdXN0YmVnaXZlbjE=" s3/photos/ play/archive/ `, } diff --git a/cmd/stat-main.go b/cmd/stat-main.go index a63429d7..c9342aa1 100644 --- a/cmd/stat-main.go +++ b/cmd/stat-main.go @@ -65,6 +65,10 @@ EXAMPLES: 4. Stat encrypted files on Amazon S3 cloud storage. $ {{.HelpName}} --encrypt-key "s3/personal-docs/=32byteslongsecretkeymustbegiven1" s3/personal-docs/2018-account_report.docx + + 5. Stat encrypted files on Amazon S3 cloud storage. In case the encryption key contains non-printable character like tab, pass the + base64 encoded string as key. + $ {{.HelpName}} --encrypt-key "s3/personal-document/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=" s3/personal-document/2019-account_report.docx `, } diff --git a/docs/minio-client-complete-guide.md b/docs/minio-client-complete-guide.md index 603057dc..e308fc80 100644 --- a/docs/minio-client-complete-guide.md +++ b/docs/minio-client-complete-guide.md @@ -403,6 +403,13 @@ mc cat --encrypt-key "play/mybucket=32byteslongsecretkeymustbegiven1" play/mybuc Hello MinIO!! ``` +*Example: Display the contents of a server encrypted object `myencryptedobject.txt`. Pass base64 encoded string if encryption key contains non-printable character like tab* + +``` +mc cat --encrypt-key "play/mybucket=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=" play/mybucket/myencryptedobject.txt +Hello MinIO!! +``` + ### Command `sql` - Run sql queries on objects `sql` run sql queries on objects. diff --git a/functional-tests.sh b/functional-tests.sh index 37f6e693..38e76768 100755 --- a/functional-tests.sh +++ b/functional-tests.sh @@ -631,6 +631,18 @@ function test_put_object_with_sse() log_success "$start_time" "${FUNCNAME[0]}" } +function test_put_object_with_encoded_sse() +{ + show "${FUNCNAME[0]}" + start_time=$(get_time) + object_name="mc-test-object-$RANDOM" + cli_flag="${SERVER_ALIAS}/${BUCKET_NAME}=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=" + # put encrypted object; then delete with correct secret key + assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --encrypt-key "${cli_flag}" "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}" + assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm --encrypt-key "${cli_flag}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}" + log_success "$start_time" "${FUNCNAME[0]}" +} + function test_put_object_with_sse_error() { show "${FUNCNAME[0]}" @@ -845,6 +857,7 @@ function run_test() if [ "$ENABLE_HTTPS" == "1" ]; then test_put_object_with_sse + test_put_object_with_encoded_sse test_put_object_with_sse_error test_put_object_multipart_sse test_get_object_with_sse