minio/mc
1
0
Fork 0
mirror of https://github.com/minio/mc.git synced 2025-11-10 13:42:32 +03:00
Code Activity

Add support for passing non-printable charachter as key (#2851)

Browse Source
This commit is contained in:
Ashish Kumar Sinha
2019-08-30 05:27:04 +05:30
committed by Harshavardhana
parent bd7564a2d4
commit 1582b1cf35
10 changed files with 162 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/cat-main.go
View File

@@ -69,6 +69,10 @@ EXAMPLES:
4. Save an encrypted object from Amazon S3 cloud storage to a local file.
$ {{.HelpName}} --encrypt-key 's3/mysql-backups=32byteslongsecretkeymustbegiven1' s3/mysql-backups/backups-201810.gz > /mnt/data/recent.gz
5. Display the content of encrypted object. In case the encryption key contains non-printable character like tab, pass the
base64 encoded string as key.
$ {{.HelpName}} --encrypt-key "play/my-bucket/=MzJieXRlc2xvbmdzZWNyZXRrZQltdXN0YmVnaXZlbjE=" play/my-bucket/my-object
`,
}

39
cmd/common-methods.go
View File

@@ -18,6 +18,8 @@ package cmd
import (
"context"
"encoding/base64"
"errors"
"io"
"os"
"path/filepath"
@@ -32,6 +34,36 @@ import (
"github.com/minio/minio-go/v6/pkg/encrypt"
)
// decode if the key is encoded key and returns the key
func getDecodedKey(sseKeys string) (key string, err *probe.Error) {
keyString := ""
for i, sse := range strings.Split(sseKeys, ",") {
if i > 0 {
keyString = keyString + ","
}
sseString, err := parseKey(sse)
if err != nil {
return "", err
}
keyString = keyString + sseString
}
return keyString, nil
}
// Validate the key
func parseKey(sseKeys string) (sse string, err *probe.Error) {
encryptString := strings.SplitN(sseKeys, "=", 2)
secretValue := encryptString[1]
if len(secretValue) == 32 {
return sseKeys, nil
}
decodedString, e := base64.StdEncoding.DecodeString(secretValue)
if e != nil || len(decodedString) != 32 {
return "", probe.NewError(errors.New("Encryption key should be 32 bytes plain text key or 44 bytes base64 encoded key"))
}
return encryptString[0] + "=" + string(decodedString), nil
}
// parse and return encryption key pairs per alias.
func getEncKeys(ctx *cli.Context) (map[string][]prefixSSEPair, *probe.Error) {
sseServer := os.Getenv("MC_ENCRYPT")
@@ -46,6 +78,13 @@ func getEncKeys(ctx *cli.Context) (map[string][]prefixSSEPair, *probe.Error) {
}
sseKeys = keyPrefix
}
var err *probe.Error
if sseKeys != "" {
sseKeys, err = getDecodedKey(sseKeys)
if err != nil {
return nil, err.Trace(sseKeys)
}
}
encKeyDB, err := parseAndValidateEncryptionKeys(sseKeys, sseServer)
if err != nil {

70
cmd/common-methods_test.go Normal file
View File

@@ -0,0 +1,70 @@
/*
* MinIO Client (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"errors"
"reflect"
"testing"
)
func TestGetDecodedKey(t *testing.T) {
getDecodeCases := []struct {
input string
output string
err error
status bool
}{
//success scenario the key contains non printable (tab) character as key
{"s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=", "s3/documents/=32byteslongsecreabcdefg givenn21", nil, true},
//success scenario the key contains non printable (tab character) as key
{"s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=,play/documents/=MzJieXRlc2xvbmdzZWNyZXRrZQltdXN0YmVnaXZlbjE=", "s3/documents/=32byteslongsecreabcdefg givenn21,play/documents/=32byteslongsecretke mustbegiven1", nil, true},
// success scenario using a normal string
{"s3/documents/=32byteslongsecretkeymustbegiven1", "s3/documents/=32byteslongsecretkeymustbegiven1", nil, true},
// success scenario using a normal string
{"s3/documents/=32byteslongsecretkeymustbegiven1,myminio/documents/=32byteslongsecretkeymustbegiven2", "s3/documents/=32byteslongsecretkeymustbegiven1,myminio/documents/=32byteslongsecretkeymustbegiven2", nil, true},
// success scenario using a mix of normal string and encoded string
{"s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=,play/documents/=32byteslongsecretkeymustbegiven1", "s3/documents/=32byteslongsecreabcdefg givenn21,play/documents/=32byteslongsecretkeymustbegiven1", nil, true},
// success scenario using a mix of normal string and encoded string
{"play/documents/=32byteslongsecretkeymustbegiven1,s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=", "play/documents/=32byteslongsecretkeymustbegiven1,s3/documents/=32byteslongsecreabcdefg givenn21", nil, true},
// decoded key less than 32 char and conatin non printable (tab) character
{"s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE", "", errors.New("Encryption key should be 32 bytes plain text key or 44 bytes base64 encoded key"), false},
// normal key less than 32 character
{"s3/documents/=32byteslongsecretkeymustbegiven", "", errors.New("Encryption key should be 32 bytes plain text key or 44 bytes base64 encoded key"), false},
}
for idx, testCase := range getDecodeCases {
decodedString, errDecode := getDecodedKey(testCase.input)
if testCase.status == true {
if errDecode != nil {
t.Fatalf("Test %d: generated error not matching, expected = `%s`, found = `%s`", idx+1, testCase.err, errDecode)
}
if !reflect.DeepEqual(decodedString, testCase.output) {
t.Fatalf("Test %d: generated key not matching, expected = `%s`, found = `%s`", idx+1, testCase.input, decodedString)
}
}
if testCase.status == false {
if !reflect.DeepEqual(decodedString, testCase.output) {
t.Fatalf("Test %d: generated Map not matching, expected = `%s`, found = `%s`", idx+1, testCase.input, errDecode)
}
if errDecode.Cause.Error() != testCase.err.Error() {
t.Fatalf("Test %d: generated error not matching, expected = `%s`, found = `%s`", idx+1, testCase.err, errDecode)
}
}
}
}

15
cmd/cp-main.go
View File

@@ -115,13 +115,17 @@ EXAMPLES:
09. Copy a folder with encrypted objects recursively from Amazon S3 to MinIO cloud storage.
$ {{.HelpName}} --recursive --encrypt-key "s3/documents/=32byteslongsecretkeymustbegiven1,myminio/documents/=32byteslongsecretkeymustbegiven2" s3/documents/ myminio/documents/
10. Copy a list of objects from local file system to MinIO cloud storage with specified metadata.
10. Copy a folder with encrypted objects recursively from Amazon S3 to MinIO cloud storage. In case the encryption key contains non-printable character like tab, pass the
base64 encoded string as key.
$ {{.HelpName}} --recursive --encrypt-key "s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=,myminio/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=" s3/documents/ myminio/documents/
11. Copy a list of objects from local file system to MinIO cloud storage with specified metadata.
$ {{.HelpName}} --attr key1=value1,key2=value2 Music/*.mp4 play/mybucket/
11. Copy a folder recursively from MinIO cloud storage to Amazon S3 cloud storage with specified metadata.
12. Copy a folder recursively from MinIO cloud storage to Amazon S3 cloud storage with specified metadata.
$ {{.HelpName}} --attr Cache-Control=max-age=90000,min-fresh=9000\;key1=value1\;key2=value2 --recursive play/mybucket/burningman2011/ s3/mybucket/
12. Copy a text file to an object storage and assign REDUCED_REDUNDANCY storage-class to the uploaded object.
13. Copy a text file to an object storage and assign REDUCED_REDUNDANCY storage-class to the uploaded object.
$ {{.HelpName}} --storage-class REDUCED_REDUNDANCY myobject.txt play/mybucket
`,
}
@@ -489,6 +493,11 @@ func mainCopy(ctx *cli.Context) error {
if key := ctx.String("encrypt-key"); key != "" {
sseKeys = key
}
if sseKeys != "" {
sseKeys, err = getDecodedKey(sseKeys)
fatalIf(err, "Unable to parse encryption keys.")
}
sse := ctx.String("encrypt")
session := newSessionV8()

10
cmd/cp-main_test.go
View File

@@ -28,15 +28,15 @@ func TestParseMetaData(t *testing.T) {
err error
status bool
}{
// success scenerio using ; as delimitter
// success scenario using ; as delimiter
{"key1=value1;key2=value2", map[string]string{"key1": "value1", "key2": "value2"}, nil, true},
// success scenerio using ; as delimitter
// success scenario using ; as delimiter
{"key1=m1=m2,m3=m4;key2=value2", map[string]string{"key1": "m1=m2,m3=m4", "key2": "value2"}, nil, true},
// success scenerio using = more than once
// success scenario using = more than once
{"Cache-Control=max-age=90000,min-fresh=9000;key1=value1;key2=value2", map[string]string{"Cache-Control": "max-age=90000,min-fresh=9000", "key1": "value1", "key2": "value2"}, nil, true},
// using different delimitter, other than '=' between key value
// using different delimiter, other than '=' between key value
{"key1:value1;key2:value2", nil, ErrInvalidMetadata, false},
// using no delimitter
// using no delimiter
{"key1:value1:key2:value2", nil, ErrInvalidMetadata, false},
}

4
cmd/head-main.go
View File

@@ -68,6 +68,10 @@ EXAMPLES:
2. Display only first line from server encrypted object on Amazon S3.
$ {{.HelpName}} -n 1 --encrypt-key 's3/csv-data=32byteslongsecretkeymustbegiven1' s3/csv-data/population.csv
3. Display only first line from server encrypted object on Amazon S3. In case the encryption key contains non-printable character like tab, pass the
base64 encoded string as key.
$ {{.HelpName}} --encrypt-key "s3/json-data=MzJieXRlc2xvbmdzZWNyZXRrZQltdXN0YmVnaXZlbjE=" s3/json-data/population.json
`,
}

4
cmd/mirror-main.go
View File

@@ -146,6 +146,10 @@ EXAMPLES:
11. Mirror server encrypted objects from MinIO cloud storage to a bucket on Amazon S3 cloud storage
$ {{.HelpName}} --encrypt-key "minio/photos=32byteslongsecretkeymustbegiven1,s3/archive=32byteslongsecretkeymustbegiven2" minio/photos/ s3/archive/
12. Mirror server encrypted objects from MinIO cloud storage to a bucket on Amazon S3 cloud storage. In case the encryption key contains
non-printable character like tab, pass the base64 encoded string as key.
$ {{.HelpName}} --encrypt-key "s3/photos/=32byteslongsecretkeymustbegiven1,play/archive/=MzJieXRlc2xvbmdzZWNyZXRrZQltdXN0YmVnaXZlbjE=" s3/photos/ play/archive/
`,
}

4
cmd/stat-main.go
View File

@@ -65,6 +65,10 @@ EXAMPLES:
4. Stat encrypted files on Amazon S3 cloud storage.
$ {{.HelpName}} --encrypt-key "s3/personal-docs/=32byteslongsecretkeymustbegiven1" s3/personal-docs/2018-account_report.docx
5. Stat encrypted files on Amazon S3 cloud storage. In case the encryption key contains non-printable character like tab, pass the
base64 encoded string as key.
$ {{.HelpName}} --encrypt-key "s3/personal-document/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=" s3/personal-document/2019-account_report.docx
`,
}

7
docs/minio-client-complete-guide.md
View File

@@ -403,6 +403,13 @@ mc cat --encrypt-key "play/mybucket=32byteslongsecretkeymustbegiven1" play/mybuc
Hello MinIO!!
```
*Example: Display the contents of a server encrypted object `myencryptedobject.txt`. Pass base64 encoded string if encryption key contains non-printable character like tab*
```
mc cat --encrypt-key "play/mybucket=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=" play/mybucket/myencryptedobject.txt
Hello MinIO!!
```
<a name="sql"></a>
### Command `sql` - Run sql queries on objects
`sql` run sql queries on objects.

13
functional-tests.sh
View File

@@ -631,6 +631,18 @@ function test_put_object_with_sse()
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_put_object_with_encoded_sse()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_name="mc-test-object-$RANDOM"
cli_flag="${SERVER_ALIAS}/${BUCKET_NAME}=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE="
# put encrypted object; then delete with correct secret key
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --encrypt-key "${cli_flag}" "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm --encrypt-key "${cli_flag}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_put_object_with_sse_error()
{
show "${FUNCNAME[0]}"
@@ -845,6 +857,7 @@ function run_test()
if [ "$ENABLE_HTTPS" == "1" ]; then
test_put_object_with_sse
test_put_object_with_encoded_sse
test_put_object_with_sse_error
test_put_object_multipart_sse
test_get_object_with_sse