mirror of
https://github.com/minio/mc.git
synced 2025-04-18 10:04:03 +03:00
cf128de2cf
Currently, attempts to attach a policy to a user who already has the policy attached to them results in a 400. This change handles cases where policy attach/detach operations are automated using scripts/jobs. A re-run of the attach/detach step should not fail the entire job.
1161 lines
45 KiB
Bash
Executable File
1161 lines
45 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Copyright (c) 2015-2024 MinIO, Inc.
|
|
#
|
|
# This file is part of MinIO Object Storage stack
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
|
|
################################################################################
|
|
#
|
|
# This script is usable by mc functional tests, mint tests and MinIO verification
|
|
# tests.
|
|
#
|
|
# * As mc functional tests, just run this script. It uses mc executable binary
|
|
# in current working directory or in the path. The tests uses play.min.io
|
|
# as MinIO server.
|
|
#
|
|
# * For other, call this script with environment variables MINT_MODE,
|
|
# MINT_DATA_DIR, SERVER_ENDPOINT, ACCESS_KEY, SECRET_KEY and ENABLE_HTTPS. It
|
|
# uses mc executable binary in current working directory and uses given MinIO
|
|
# server to run tests. MINT_MODE is set by mint to specify what category of
|
|
# tests to run.
|
|
#
|
|
################################################################################
|
|
|
|
# Force bytewise sorting for CLI tools
|
|
LANG=C
|
|
|
|
if [ -n "$DEBUG" ]; then
|
|
set -x
|
|
fi
|
|
|
|
if [ -n "$MINT_MODE" ]; then
|
|
if [ -z "${MINT_DATA_DIR+x}" ]; then
|
|
echo "MINT_DATA_DIR not defined"
|
|
exit 1
|
|
fi
|
|
if [ -z "${SERVER_ENDPOINT+x}" ]; then
|
|
echo "SERVER_ENDPOINT not defined"
|
|
exit 1
|
|
fi
|
|
if [ -z "${ACCESS_KEY+x}" ]; then
|
|
echo "ACCESS_KEY not defined"
|
|
exit 1
|
|
fi
|
|
if [ -z "${SECRET_KEY+x}" ]; then
|
|
echo "SECRET_KEY not defined"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [ -z "${SERVER_ENDPOINT+x}" ]; then
|
|
SERVER_ENDPOINT="play.min.io"
|
|
ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
|
SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
|
ENABLE_HTTPS=1
|
|
fi
|
|
|
|
# If you want to run the complete site cleaning test, set this variable to true
|
|
COMPLETE_RB_TEST=true
|
|
|
|
WORK_DIR="$PWD"
|
|
DATA_DIR="$MINT_DATA_DIR"
|
|
if [ -z "$MINT_MODE" ]; then
|
|
WORK_DIR="$PWD/.run-$RANDOM"
|
|
DATA_DIR="$WORK_DIR/data"
|
|
fi
|
|
|
|
FILE_0_B="$DATA_DIR/datafile-0-b"
|
|
FILE_1_MB="$DATA_DIR/datafile-1-MB"
|
|
FILE_65_MB="$DATA_DIR/datafile-65-MB"
|
|
declare FILE_0_B_MD5SUM
|
|
declare FILE_1_MB_MD5SUM
|
|
declare FILE_65_MB_MD5SUM
|
|
|
|
ENDPOINT="https://$SERVER_ENDPOINT"
|
|
if [ "$ENABLE_HTTPS" != "1" ]; then
|
|
ENDPOINT="http://$SERVER_ENDPOINT"
|
|
fi
|
|
|
|
SERVER_ALIAS="myminio"
|
|
SERVER_ALIAS_TLS="myminio-ssl"
|
|
|
|
BUCKET_NAME="mc-test-bucket-$RANDOM"
|
|
WATCH_OUT_FILE="$WORK_DIR/watch.out-$RANDOM"
|
|
|
|
MC_CONFIG_DIR="/tmp/.mc-$RANDOM"
|
|
MC="$PWD/mc"
|
|
declare -a MC_CMD
|
|
|
|
function get_md5sum() {
|
|
filename="$1"
|
|
out=$(md5sum "$filename" 2>/dev/null)
|
|
rv=$?
|
|
if [ "$rv" -eq 0 ]; then
|
|
echo $(awk '{ print $1 }' <<<"$out")
|
|
fi
|
|
|
|
return "$rv"
|
|
}
|
|
|
|
function get_time() {
|
|
date +%s%N
|
|
}
|
|
|
|
function get_duration() {
|
|
start_time=$1
|
|
end_time=$(get_time)
|
|
|
|
echo $(((end_time - start_time) / 1000000))
|
|
}
|
|
|
|
function log_success() {
|
|
if [ -n "$MINT_MODE" ]; then
|
|
printf '{"name": "mc", "duration": "%d", "function": "%s", "status": "PASS"}\n' "$(get_duration "$1")" "$2"
|
|
fi
|
|
}
|
|
|
|
function show() {
|
|
if [ -z "$MINT_MODE" ]; then
|
|
func_name="$1"
|
|
echo "Running $func_name()"
|
|
fi
|
|
}
|
|
|
|
function show_on_success() {
|
|
rv="$1"
|
|
shift
|
|
|
|
if [ "$rv" -eq 0 ]; then
|
|
echo "$@"
|
|
fi
|
|
|
|
return "$rv"
|
|
}
|
|
|
|
function show_on_failure() {
|
|
rv="$1"
|
|
shift
|
|
|
|
if [ "$rv" -ne 0 ]; then
|
|
echo "$@"
|
|
fi
|
|
|
|
return "$rv"
|
|
}
|
|
|
|
function assert() {
|
|
expected_rv="$1"
|
|
shift
|
|
start_time="$1"
|
|
shift
|
|
func_name="$1"
|
|
shift
|
|
|
|
err=$("$@")
|
|
rv=$?
|
|
if [ "$rv" -ne "$expected_rv" ]; then
|
|
if [ -n "$MINT_MODE" ]; then
|
|
err=$(printf "$err" | python -c 'import sys,json; print(json.dumps(sys.stdin.read()))')
|
|
## err is already JSON string, no need to double quote
|
|
printf '{"name": "mc", "duration": "%d", "function": "%s", "status": "FAIL", "error": %s}\n' "$(get_duration "$start_time")" "$func_name" "$err"
|
|
else
|
|
echo "mc: $func_name: $err"
|
|
fi
|
|
|
|
if [ "$rv" -eq 0 ]; then
|
|
exit 1
|
|
fi
|
|
|
|
exit "$rv"
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
function assert_success() {
|
|
assert 0 "$@"
|
|
}
|
|
|
|
function assert_failure() {
|
|
assert 1 "$@"
|
|
}
|
|
|
|
function mc_cmd() {
|
|
cmd=("${MC_CMD[@]}" "$@")
|
|
err_file="$WORK_DIR/cmd.out.$RANDOM"
|
|
|
|
"${cmd[@]}" >"$err_file" 2>&1
|
|
rv=$?
|
|
if [ "$rv" -ne 0 ]; then
|
|
printf '%q ' "${cmd[@]}"
|
|
echo " >>> "
|
|
cat "$err_file"
|
|
fi
|
|
|
|
rm -f "$err_file"
|
|
return "$rv"
|
|
}
|
|
|
|
function check_md5sum() {
|
|
expected_checksum="$1"
|
|
shift
|
|
filename="$@"
|
|
|
|
checksum="$(get_md5sum "$filename")"
|
|
rv=$?
|
|
if [ "$rv" -ne 0 ]; then
|
|
echo "unable to get md5sum for $filename"
|
|
return "$rv"
|
|
fi
|
|
|
|
if [ "$checksum" != "$expected_checksum" ]; then
|
|
echo "$filename: md5sum mismatch"
|
|
return 1
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
function test_make_bucket() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
bucket_name="mc-test-bucket-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${bucket_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rb "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_make_bucket_error() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
bucket_name="MC-test%bucket%$RANDOM"
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_rb() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
bucket1="mc-test-bucket-$RANDOM-1"
|
|
bucket2="mc-test-bucket-$RANDOM-2"
|
|
object_name="mc-test-object-$RANDOM"
|
|
|
|
# Tets rb when the bucket is empty
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${bucket1}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rb "${SERVER_ALIAS}/${bucket1}"
|
|
|
|
# Test rb with --force flag when the bucket is not empty
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${bucket1}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${bucket1}/${object_name}"
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd rb "${SERVER_ALIAS}/${bucket1}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rb --force "${SERVER_ALIAS}/${bucket1}"
|
|
|
|
# Test rb with --force and --dangerous to remove a site content
|
|
if [ "${COMPLETE_RB_TEST}" == "true" ]; then
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${bucket1}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${bucket2}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${bucket1}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${bucket2}/${object_name}"
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd rb --force "${SERVER_ALIAS}/"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rb --force --dangerous "${SERVER_ALIAS}"
|
|
fi
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function setup() {
|
|
start_time=$(get_time)
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${BUCKET_NAME}"
|
|
}
|
|
|
|
function teardown() {
|
|
start_time=$(get_time)
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rb --force "${SERVER_ALIAS}/${BUCKET_NAME}"
|
|
}
|
|
|
|
# Test mc ls on a S3 prefix where a lower similar prefix exists as well e.g. dir-foo/ and dir/
|
|
function test_list_dir() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/dir-foo/object1"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/dir/object2"
|
|
diff -bB <(echo "object2") <("${MC_CMD[@]}" --json ls "${SERVER_ALIAS}/${BUCKET_NAME}/dir" | jq -r '.key') >/dev/null 2>&1
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unexpected listing dir"
|
|
|
|
# Cleanup
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/dir-foo/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/dir/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_od_object() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd od if="${FILE_1_MB}" of="${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd od of="${FILE_1_MB}" if="${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_put_object() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_put_object_error() {
|
|
show "${FUNCNAME[0]}"
|
|
start_time=$(get_time)
|
|
|
|
object_long_name=$(printf "mc-test-object-%01100d" 1)
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_long_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_put_object_multipart() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_65_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_put_object_0byte() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_0_B}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_0_B_MD5SUM" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${object_name}.downloaded" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
## Test mc cp command with storage-class flag set
|
|
function test_put_object_with_storage_class() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --storage-class REDUCED_REDUNDANCY "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
## Test mc cp command with storage-class flag set to incorrect value
|
|
function test_put_object_with_storage_class_error() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd cp --storage-class REDUCED "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
## Test mc cp command with valid metadata string
|
|
function test_put_object_with_metadata() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --attr key1=val1\;key2=val2 "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
diff -bB <(echo "val1") <("${MC_CMD[@]}" --json stat "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}" | jq -r '.metadata."X-Amz-Meta-Key1"') >/dev/null 2>&1
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to put object with metadata"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
|
|
}
|
|
|
|
function test_get_object() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${object_name}.downloaded" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_get_object_multipart() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_65_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_65_MB_MD5SUM" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${object_name}.downloaded" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_presigned_post_policy_error() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
|
|
out=$("${MC_CMD[@]}" --json share upload "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}")
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to get presigned post policy and put object url"
|
|
|
|
# Support IPv6 address, IPv6 is specifed as [host]:9000 form, we should
|
|
# replace '['']' with their escaped values as '\[' '\]'.
|
|
#
|
|
# Without escaping '['']', 'sed' command interprets them as expressions
|
|
# which fails our requirement of replacing $endpoint/$bucket URLs in the
|
|
# subsequent operations.
|
|
endpoint=$(echo "$ENDPOINT" | sed 's|[][]|\\&|g')
|
|
|
|
# Extract share field of json output, and append object name to the URL
|
|
upload=$(echo "$out" | jq -r .share | sed "s|<FILE>|$FILE_1_MB|g" | sed "s|curl|curl -sSg|g" | sed "s|${endpoint}/${BUCKET_NAME}/|${endpoint}/${BUCKET_NAME}/${object_name}|g")
|
|
|
|
# In case of virtual host style URL path, the previous replace would have failed.
|
|
# One of the following two commands will append the object name in that scenario.
|
|
upload=$(echo "$upload" | sed "s|http://${BUCKET_NAME}.${SERVER_ENDPOINT}/|http://${BUCKET_NAME}.${SERVER_ENDPOINT}/${object_name}|g")
|
|
upload=$(echo "$upload" | sed "s|https://${BUCKET_NAME}.${SERVER_ENDPOINT}/|https://${BUCKET_NAME}.${SERVER_ENDPOINT}/${object_name}|g")
|
|
|
|
ret=$($upload 2>&1 | grep -oP '(?<=Code>)[^<]+')
|
|
# Check if the command execution failed.
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unknown failure in upload of $FILE_1_MB using presigned post policy"
|
|
if [ -z "$ret" ]; then
|
|
|
|
# Check if the upload succeeded. We expect it to fail.
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" show_on_success 0 "upload of $FILE_1_MB using presigned post policy should have failed"
|
|
fi
|
|
|
|
if [ "$ret" != "MethodNotAllowed" ]; then
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" show_on_success 0 "upload of $FILE_1_MB using presigned post policy should have failed with MethodNotAllowed error, instead failed with $ret error"
|
|
fi
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_presigned_put_object() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
|
|
out=$("${MC_CMD[@]}" --json share upload "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}")
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to get presigned put object url"
|
|
upload=$(echo "$out" | jq -r .share | sed "s|<FILE>|$FILE_1_MB|g" | sed "s|curl|curl -sSg|g")
|
|
$upload >/dev/null 2>&1
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to upload $FILE_1_MB presigned put object url"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${object_name}.downloaded" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_presigned_get_object() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
out=$("${MC_CMD[@]}" --json share download "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}")
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to get presigned get object url"
|
|
download_url=$(echo "$out" | jq -r .share)
|
|
curl -sSg --output "${object_name}.downloaded" -X GET "$download_url"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to download $download_url"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${object_name}.downloaded" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_cat_object() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
"${MC_CMD[@]}" cat "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}" >"${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to download object using 'mc cat'"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${object_name}.downloaded" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_cat_stdin() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
bucket_name="mc-test-bucket-$RANDOM"
|
|
mc_cmd mb "${SERVER_ALIAS}/${bucket_name}"
|
|
echo "testcontent" | mc_cmd pipe "${SERVER_ALIAS}/${bucket_name}/${object_name}"
|
|
"${MC_CMD[@]}" cat "${SERVER_ALIAS}/${bucket_name}/${object_name}" >stdout.output
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to redirect stdin to stdout using 'mc cat'"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "42ed9fb3563d8e9c7bb522be443033f4" stdout.output
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm stdout.output
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_mirror_list_objects() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
bucket_name="mc-test-bucket-$RANDOM"
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${bucket_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mirror "$DATA_DIR" "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
diff -bB <(ls "$DATA_DIR") <("${MC_CMD[@]}" --json ls "${SERVER_ALIAS}/${bucket_name}/" | jq -r .key) >/dev/null 2>&1
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "mirror and list differs"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rb --force "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
## Tests mc mirror command with --storage-class flag set
|
|
function test_mirror_list_objects_storage_class() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
bucket_name="mc-test-bucket-$RANDOM"
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${bucket_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mirror --storage-class REDUCED_REDUNDANCY "$DATA_DIR" "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
diff -bB <(ls "$DATA_DIR") <("${MC_CMD[@]}" --json ls "${SERVER_ALIAS}/${bucket_name}/" | jq -r .key) >/dev/null 2>&1
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "mirror and list differs"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rb --force "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
## Tests find command with --older-than set to 1day, should be empty.
|
|
function test_find_empty() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
bucket_name="mc-test-bucket-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${bucket_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mirror "$DATA_DIR" "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
# find --older-than 1 day should be empty, so we compare with empty string.
|
|
diff -bB <(echo "") <("${MC_CMD[@]}" find "${SERVER_ALIAS}/${bucket_name}" --json --older-than 1d | jq -r .key | sed "s/${SERVER_ALIAS}\/${bucket_name}\///g") >/dev/null 2>&1
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "mirror and list differs"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rb --force "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
## Tests find command, should list.
|
|
function test_find() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
bucket_name="mc-test-bucket-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${bucket_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mirror "$DATA_DIR" "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
diff -bB <(ls "$DATA_DIR") <("${MC_CMD[@]}" --json find "${SERVER_ALIAS}/${bucket_name}/" | jq -r .key | sed "s/${SERVER_ALIAS}\/${bucket_name}\///g") >/dev/null 2>&1
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "mirror and list differs"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rb --force "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_watch_object() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
bucket_name="mc-test-bucket-$RANDOM"
|
|
object_name="mc-test-object-$RANDOM"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
# start a process to watch on bucket
|
|
"${MC_CMD[@]}" --json watch "${SERVER_ALIAS}/${bucket_name}" >"$WATCH_OUT_FILE" &
|
|
watch_cmd_pid=$!
|
|
sleep 1
|
|
|
|
(assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${bucket_name}/${object_name}")
|
|
rv=$?
|
|
if [ "$rv" -ne 0 ]; then
|
|
kill "$watch_cmd_pid"
|
|
exit "$rv"
|
|
fi
|
|
|
|
sleep 1
|
|
if ! jq -r .events.type "$WATCH_OUT_FILE" | grep -qi ObjectCreated; then
|
|
kill "$watch_cmd_pid"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure 1 "ObjectCreated event not found"
|
|
fi
|
|
|
|
(assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${bucket_name}/${object_name}")
|
|
rv=$?
|
|
if [ "$rv" -ne 0 ]; then
|
|
kill "$watch_cmd_pid"
|
|
exit "$rv"
|
|
fi
|
|
|
|
sleep 1
|
|
if ! jq -r .events.type "$WATCH_OUT_FILE" | grep -qi ObjectRemoved; then
|
|
kill "$watch_cmd_pid"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure 1 "ObjectRemoved event not found"
|
|
fi
|
|
|
|
kill "$watch_cmd_pid"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rb --force "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_config_host_add() {
|
|
show "${FUNCNAME[0]}"
|
|
start_time=$(get_time)
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd alias set "${SERVER_ALIAS}1" "$ENDPOINT" "$ACCESS_KEY" "$SECRET_KEY"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd alias list "${SERVER_ALIAS}1"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_config_host_add_error() {
|
|
show "${FUNCNAME[0]}"
|
|
start_time=$(get_time)
|
|
|
|
out=$("${MC_CMD[@]}" --json alias set "${SERVER_ALIAS}1" "$ENDPOINT" "$ACCESS_KEY" "invalid-secret")
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" show_on_success $? "adding host should fail"
|
|
got_code=$(echo "$out" | jq -r .error.cause.error.Code)
|
|
if [ "${got_code}" != "SignatureDoesNotMatch" ]; then
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" show_on_failure 1 "incorrect error code ${got_code} returned by server"
|
|
fi
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_put_object_with_sse() {
|
|
show "${FUNCNAME[0]}"
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
cli_flag="${SERVER_ALIAS}/${BUCKET_NAME}=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA"
|
|
# put encrypted object; then delete with correct secret key
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag}" "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_put_object_with_sse_error() {
|
|
show "${FUNCNAME[0]}"
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
cli_flag="${SERVER_ALIAS}/${BUCKET_NAME}=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MD"
|
|
# put object with invalid encryption key; should fail
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag}" "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_cat_object_with_sse() {
|
|
show "${FUNCNAME[0]}"
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
cli_flag="${SERVER_ALIAS}/${BUCKET_NAME}=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA"
|
|
# put encrypted object; then cat object correct secret key
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag}" "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cat --enc-c "${cli_flag}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_cat_object_with_sse_error() {
|
|
show "${FUNCNAME[0]}"
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
cli_flag="${SERVER_ALIAS}/${BUCKET_NAME}=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA"
|
|
# put encrypted object; then cat object with no secret key
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag}" "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd cat "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
# Test "mc cp -r" command of a directory with and without a leading slash
|
|
function test_copy_directory() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
random_dir="dir-$RANDOM-$RANDOM"
|
|
tmpdir="$(mktemp -d)"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}/object-name"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to upload an object"
|
|
|
|
# Copy a directory with a trailing slash
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp -r "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}/" "${tmpdir}/"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to copy a directory with a trailing slash"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${tmpdir}/object-name"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" rm "${tmpdir}/object-name"
|
|
|
|
# Copy a directory without a trailing slash
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp -r "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}" "${tmpdir}/"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to copy a directory without a trailing slash"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${tmpdir}/${random_dir}/object-name"
|
|
|
|
# Cleanup
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm -r --force "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}/"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" rm -r "${tmpdir}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
# Test "mc cp -a" command to see if it preserves file system attributes
|
|
function test_copy_object_preserve_filesystem_attr() {
|
|
show "${FUNCNAME[0]}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp -a "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
diff -bB <("${MC_CMD[@]}" --json stat "${FILE_1_MB}" | jq -r '.metadata."X-Amz-Meta-Mc-Attrs"') <("${MC_CMD[@]}" --json stat "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}" | jq -r '.metadata."X-Amz-Meta-Mc-Attrs"') >/dev/null 2>&1 >/dev/null 2>&1
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to put object with file system attribute"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
# Test "mc mv" command
|
|
function test_mv_object() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
random_dir="dir-$RANDOM-$RANDOM"
|
|
tmpdir="$(mktemp -d)"
|
|
|
|
# Test mv command locally
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "${FILE_1_MB}" "${tmpdir}/file.tmp"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mv "${tmpdir}/file.tmp" "${tmpdir}/file"
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd stat "${tmpdir}/file.tmp"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${tmpdir}/file"
|
|
|
|
# Test mv command from filesystem to S3
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mv "${tmpdir}/file" "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}/object-1"
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd stat "${tmpdir}/file"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd stat "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}/object-1"
|
|
|
|
# Test mv command from S3 to S3
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mv "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}/object-1" "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}/object-2"
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd stat "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}/object-1"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd stat "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}/object-2"
|
|
|
|
# Test mv command from S3 to filesystem
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mv "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}/object-2" "${tmpdir}/file"
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd stat "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}/object-2"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd stat "${tmpdir}/file"
|
|
|
|
# Cleanup
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm -r --force "${SERVER_ALIAS}/${BUCKET_NAME}/${random_dir}/"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" rm -r "${tmpdir}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_copy_object_with_sse_rewrite() {
|
|
# test server side copy and remove operation - target is unencrypted while source is encrypted
|
|
show "${FUNCNAME[0]}"
|
|
start_time=$(get_time)
|
|
prefix="prefix"
|
|
object_name="mc-test-object-$RANDOM"
|
|
|
|
cli_flag="${SERVER_ALIAS}/${BUCKET_NAME}/${prefix}=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA"
|
|
# create encrypted object on server
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag}" "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${prefix}/${object_name}"
|
|
# now do a server side copy and store it unencrypted.
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag}" "${SERVER_ALIAS}/${BUCKET_NAME}/${prefix}/${object_name}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
# cat the unencrypted destination object. should return data without any error
|
|
"${MC_CMD[@]}" cat "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}" >"${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to download object using 'mc cat'"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${object_name}.downloaded"
|
|
# mc rm on with multi-object delete, deletes encrypted object without encryption key.
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${BUCKET_NAME}/${prefix}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_copy_object_with_sse_dest() {
|
|
# test server side copy and remove operation - target is encrypted with different key
|
|
show "${FUNCNAME[0]}"
|
|
start_time=$(get_time)
|
|
prefix="prefix"
|
|
object_name="mc-test-object-$RANDOM"
|
|
|
|
cli_flag1="${SERVER_ALIAS}/${BUCKET_NAME}/${prefix}=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA"
|
|
cli_flag2="${SERVER_ALIAS}/${BUCKET_NAME}=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA"
|
|
|
|
# create encrypted object on server
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag1}" "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${prefix}/${object_name}"
|
|
# now do a server side copy and store it eith different encryption key.
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag2}" \
|
|
"${SERVER_ALIAS}/${BUCKET_NAME}/${prefix}/${object_name}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
# cat the destination object with the new key. should return data without any error
|
|
"${MC_CMD[@]}" cat --enc-c "${cli_flag2}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}" >"${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to download object using 'mc cat'"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${object_name}.downloaded"
|
|
# mc rm on src object with first encryption key should pass
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${BUCKET_NAME}/${prefix}/${object_name}"
|
|
# mc rm on encrypted destination object with second encryption key should pass
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_sse_key_rotation() {
|
|
# test server side copy and remove operation - target is encrypted with different key
|
|
show "${FUNCNAME[0]}"
|
|
start_time=$(get_time)
|
|
prefix="prefix"
|
|
object_name="mc-test-object-$RANDOM"
|
|
old_key="MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA"
|
|
new_key="MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA"
|
|
cli_flag1="${SERVER_ALIAS}/${BUCKET_NAME}/${prefix}=${old_key}"
|
|
cli_flag2="${SERVER_ALIAS_TLS}/${BUCKET_NAME}=${new_key}"
|
|
|
|
# create encrypted object on server
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag1}" "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${prefix}/${object_name}"
|
|
# now do a server side copy on same object and do a key rotation
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag2}" \
|
|
"${SERVER_ALIAS_TLS}/${BUCKET_NAME}/${prefix}/${object_name}" "${SERVER_ALIAS_TLS}/${BUCKET_NAME}/${object_name}"
|
|
# cat the object with the new key. should return data without any error
|
|
"${MC_CMD[@]}" cat --enc-c "${cli_flag2}" "${SERVER_ALIAS_TLS}/${BUCKET_NAME}/${object_name}" >"${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "unable to download object using 'mc cat'"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${object_name}.downloaded"
|
|
# mc rm on encrypted object with succeed anyways, without encrypted keys.
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS_TLS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_mirror_with_sse() {
|
|
# test if mirror operation works with encrypted objects
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
bucket_name="mc-test-bucket-$RANDOM"
|
|
cli_flag="${SERVER_ALIAS}/${bucket_name}=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mb "${SERVER_ALIAS}/${bucket_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd mirror --enc-c "${cli_flag}" "$DATA_DIR" "${SERVER_ALIAS}/${bucket_name}"
|
|
diff -bB <(ls "$DATA_DIR") <("${MC_CMD[@]}" --json ls "${SERVER_ALIAS}/${bucket_name}/" | jq -r .key) >/dev/null 2>&1
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure $? "mirror and list differs"
|
|
# Remove the test bucket with its contents
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rb --force "${SERVER_ALIAS}/${bucket_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_rm_object_with_sse() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
# test whether remove fails for encrypted object if secret key not provided.
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
cli_flag="${SERVER_ALIAS}/${BUCKET_NAME}=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag}" "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
# rm will not fail even if the encryption keys are not provided, since mc rm uses multi-object delete.
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_get_object_with_sse() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
cli_flag="${SERVER_ALIAS}/${BUCKET_NAME}=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag}" "${FILE_1_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${object_name}.downloaded" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_put_object_multipart_sse() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
object_name="mc-test-object-$RANDOM"
|
|
cli_flag="${SERVER_ALIAS}/${BUCKET_NAME}=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp --enc-c "${cli_flag}" "${FILE_65_MB}" "${SERVER_ALIAS}/${BUCKET_NAME}/${object_name}"
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function test_admin_users() {
|
|
show "${FUNCNAME[0]}"
|
|
|
|
start_time=$(get_time)
|
|
|
|
# create a user
|
|
username=foo
|
|
password=foobar12345
|
|
test_alias="aliasx"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd admin user add "$SERVER_ALIAS" "$username" "$password"
|
|
|
|
# check that user appears in the user list
|
|
"${MC_CMD[@]}" --json admin user list "${SERVER_ALIAS}" | jq -r '.accessKey' | grep --quiet "^${username}$"
|
|
rv=$?
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure ${rv} "user ${username} did NOT appear in the list of users returned by server"
|
|
|
|
# setup temporary alias to make requests as the created user.
|
|
scheme="https"
|
|
if [ "$ENABLE_HTTPS" != "1" ]; then
|
|
scheme="http"
|
|
fi
|
|
object1_name="mc-test-object-$RANDOM"
|
|
object2_name="mc-test-object-$RANDOM"
|
|
|
|
# Adding an alias for the $test_alias
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd alias set $test_alias "${scheme}://${SERVER_ENDPOINT}" ${username} ${password}
|
|
|
|
# check that alias appears in the alias list
|
|
"${MC_CMD[@]}" --json alias list | jq -r '.alias' | grep --quiet "^${test_alias}$"
|
|
rv=$?
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure ${rv} "alias ${test_alias} did NOT appear in the list of aliases returned by server"
|
|
|
|
# check that the user can write objects with readwrite policy
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd admin policy attach "$SERVER_ALIAS" readwrite --user="${username}"
|
|
|
|
# verify that re-attaching an already attached policy to a user does not result in a failure.
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd admin policy attach "$SERVER_ALIAS" readwrite --user="${username}"
|
|
|
|
# Validate that the correct policy has been added to the user
|
|
"${MC_CMD[@]}" --json admin user list "${SERVER_ALIAS}" | jq -r '.policyName' | grep --quiet "^readwrite$"
|
|
rv=$?
|
|
assert_success "$start_time" "${FUNCNAME[0]}" show_on_failure ${rv} "user ${username} did NOT have the readwrite policy attached"
|
|
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "$FILE_1_MB" "${test_alias}/${BUCKET_NAME}/${object1_name}"
|
|
|
|
# check that the user cannot write objects with readonly policy
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd admin policy detach "$SERVER_ALIAS" readwrite --user="${username}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd admin policy attach "$SERVER_ALIAS" readonly --user="${username}"
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd cp "$FILE_1_MB" "${test_alias}/${BUCKET_NAME}/${object2_name}"
|
|
|
|
# check that the user can read with readonly policy
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cat "${test_alias}/${BUCKET_NAME}/${object1_name}"
|
|
|
|
# check that user can delete with readwrite policy
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd admin policy attach "$SERVER_ALIAS" readwrite --user="${username}"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd rm "${test_alias}/${BUCKET_NAME}/${object1_name}"
|
|
|
|
# check that user cannot perform admin actions with readwrite policy
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd admin info $test_alias
|
|
|
|
# create object1_name for subsequent tests.
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cp "$FILE_1_MB" "${test_alias}/${BUCKET_NAME}/${object1_name}"
|
|
|
|
# check that user can be disabled
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd admin user disable "$SERVER_ALIAS" "$username"
|
|
|
|
# check that disabled cannot perform any action
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd cat "${test_alias}/${BUCKET_NAME}/${object1_name}"
|
|
|
|
# check that user can be enabled and can then perform an allowed action
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd admin user enable "$SERVER_ALIAS" "$username"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd cat "${test_alias}/${BUCKET_NAME}/${object1_name}"
|
|
|
|
# check that user can be removed, and then is no longer available
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd admin user remove "$SERVER_ALIAS" "$username"
|
|
assert_failure "$start_time" "${FUNCNAME[0]}" mc_cmd cat "${test_alias}/${BUCKET_NAME}/${object1_name}"
|
|
|
|
log_success "$start_time" "${FUNCNAME[0]}"
|
|
}
|
|
|
|
function run_test() {
|
|
test_make_bucket
|
|
test_make_bucket_error
|
|
test_rb
|
|
|
|
setup
|
|
test_list_dir
|
|
test_put_object
|
|
test_put_object_error
|
|
test_put_object_0byte
|
|
test_put_object_with_storage_class
|
|
test_put_object_with_storage_class_error
|
|
test_put_object_with_metadata
|
|
test_put_object_multipart
|
|
test_get_object
|
|
test_get_object_multipart
|
|
test_od_object
|
|
test_mv_object
|
|
test_presigned_post_policy_error
|
|
test_presigned_put_object
|
|
test_presigned_get_object
|
|
test_cat_object
|
|
test_cat_stdin
|
|
test_copy_directory
|
|
test_mirror_list_objects
|
|
test_mirror_list_objects_storage_class
|
|
test_copy_object_preserve_filesystem_attr
|
|
test_find
|
|
test_find_empty
|
|
if [ -z "$MINT_MODE" ]; then
|
|
test_watch_object
|
|
fi
|
|
|
|
if [ "$ENABLE_HTTPS" == "1" ]; then
|
|
test_put_object_with_sse
|
|
test_put_object_with_sse_error
|
|
test_put_object_multipart_sse
|
|
test_get_object_with_sse
|
|
test_cat_object_with_sse
|
|
test_cat_object_with_sse_error
|
|
test_copy_object_with_sse_rewrite
|
|
test_copy_object_with_sse_dest
|
|
test_sse_key_rotation
|
|
test_mirror_with_sse
|
|
test_rm_object_with_sse
|
|
fi
|
|
|
|
test_config_host_add
|
|
test_config_host_add_error
|
|
test_admin_users
|
|
|
|
teardown
|
|
}
|
|
|
|
function __init__() {
|
|
set -e
|
|
if [ -n "$DEBUG" ]; then
|
|
set -x
|
|
fi
|
|
|
|
# For Mint, setup is already done. For others, setup the environment
|
|
if [ -z "$MINT_MODE" ]; then
|
|
mkdir -p "$WORK_DIR"
|
|
mkdir -p "$DATA_DIR"
|
|
|
|
# If mc executable binary is not available in current directory, use it in the path.
|
|
if [ ! -x "$MC" ]; then
|
|
if ! MC=$(which mc 2>/dev/null); then
|
|
echo "'mc' executable binary not found in current directory and in path"
|
|
exit 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if [ ! -x "$MC" ]; then
|
|
echo "$MC executable binary not found"
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "$MC_CONFIG_DIR"
|
|
MC_CMD=("${MC}" --config-dir "$MC_CONFIG_DIR" --quiet --no-color)
|
|
|
|
if [ ! -e "$FILE_0_B" ]; then
|
|
touch "$FILE_0_B"
|
|
fi
|
|
|
|
if [ ! -e "$FILE_1_MB" ]; then
|
|
base64 -i /dev/urandom | head -c 1048576 >"$FILE_1_MB"
|
|
fi
|
|
|
|
if [ ! -e "$FILE_65_MB" ]; then
|
|
base64 -i /dev/urandom | head -c 68157440 >"$FILE_65_MB"
|
|
fi
|
|
|
|
set -E
|
|
set -o pipefail
|
|
|
|
FILE_0_B_MD5SUM="$(get_md5sum "$FILE_0_B")"
|
|
if [ $? -ne 0 ]; then
|
|
echo "unable to get md5sum of $FILE_0_B"
|
|
exit 1
|
|
fi
|
|
|
|
FILE_1_MB_MD5SUM="$(get_md5sum "$FILE_1_MB")"
|
|
if [ $? -ne 0 ]; then
|
|
echo "unable to get md5sum of $FILE_1_MB"
|
|
exit 1
|
|
fi
|
|
|
|
FILE_65_MB_MD5SUM="$(get_md5sum "$FILE_65_MB")"
|
|
if [ $? -ne 0 ]; then
|
|
echo "unable to get md5sum of $FILE_65_MB"
|
|
exit 1
|
|
fi
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd alias set "${SERVER_ALIAS}" "$ENDPOINT" "$ACCESS_KEY" "$SECRET_KEY"
|
|
assert_success "$start_time" "${FUNCNAME[0]}" mc_cmd alias set "${SERVER_ALIAS_TLS}" "$ENDPOINT" "$ACCESS_KEY" "$SECRET_KEY"
|
|
|
|
set +e
|
|
}
|
|
|
|
function validate_dependencies() {
|
|
jqVersion=$(jq --version)
|
|
if [[ $jqVersion == *"jq"* ]]; then
|
|
echo "Dependency validation complete"
|
|
else
|
|
echo "jq is missing, please install: 'sudo apt install jq'"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
function main() {
|
|
validate_dependencies
|
|
|
|
(run_test)
|
|
rv=$?
|
|
|
|
rm -fr "$MC_CONFIG_DIR" "$WATCH_OUT_FILE"
|
|
if [ -z "$MINT_MODE" ]; then
|
|
rm -fr "$WORK_DIR" "$DATA_DIR"
|
|
fi
|
|
|
|
exit "$rv"
|
|
}
|
|
|
|
__init__ "$@"
|
|
main "$@"
|