mirror of
https://github.com/minio/mc.git
synced 2025-11-12 01:02:26 +03:00
499 lines
13 KiB
Go
499 lines
13 KiB
Go
/*
|
|
* Minio Client (C) 2015 Minio, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this fs except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package fs
|
|
|
|
import (
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"time"
|
|
|
|
"io/ioutil"
|
|
|
|
"github.com/minio/mc/pkg/client"
|
|
"github.com/minio/minio-xl/pkg/probe"
|
|
)
|
|
|
|
type fsClient struct {
|
|
Path string
|
|
}
|
|
|
|
// New - instantiate a new fs client
|
|
func New(path string) (client.Client, *probe.Error) {
|
|
if strings.TrimSpace(path) == "" {
|
|
return nil, probe.NewError(client.EmptyPath{})
|
|
}
|
|
return &fsClient{
|
|
Path: normalizePath(path),
|
|
}, nil
|
|
}
|
|
|
|
// URL get url
|
|
func (f *fsClient) URL() *client.URL {
|
|
return client.NewURL(f.Path)
|
|
}
|
|
|
|
/// Object operations
|
|
|
|
// fsStat - wrapper function to get file stat
|
|
func (f *fsClient) fsStat() (os.FileInfo, *probe.Error) {
|
|
fpath := f.Path
|
|
// Golang strips trailing / if you clean(..) or
|
|
// EvalSymlinks(..). Adding '.' prevents it from doing so.
|
|
if strings.HasSuffix(fpath, string(f.URL().Separator)) {
|
|
fpath = fpath + "."
|
|
}
|
|
// Resolve symlinks
|
|
fpath, err := filepath.EvalSymlinks(fpath)
|
|
if runtime.GOOS == "windows" {
|
|
// On windows there are folder symlinks
|
|
// which are called junction files which
|
|
// carry special meaning on windows
|
|
// - which cannot be accessed with regular operations
|
|
if os.IsPermission(err) {
|
|
lfi, lerr := os.Lstat(fpath)
|
|
if lerr != nil {
|
|
return nil, probe.NewError(lerr)
|
|
}
|
|
return lfi, nil
|
|
}
|
|
}
|
|
if err != nil {
|
|
return nil, probe.NewError(err)
|
|
}
|
|
st, err := os.Stat(fpath)
|
|
if runtime.GOOS == "windows" {
|
|
// On windows there are directory symlinks
|
|
// which are called junction files which
|
|
// carry special meaning on windows
|
|
// - which cannot be accessed with regular operations
|
|
if os.IsPermission(err) {
|
|
lst, lerr := os.Lstat(fpath)
|
|
if lerr != nil {
|
|
return nil, probe.NewError(lerr)
|
|
}
|
|
return lst, nil
|
|
}
|
|
}
|
|
if os.IsNotExist(err) {
|
|
return nil, probe.NewError(client.NotFound{Path: fpath})
|
|
}
|
|
if err != nil {
|
|
return nil, probe.NewError(err)
|
|
}
|
|
return st, nil
|
|
}
|
|
|
|
// PutObject - create a new file
|
|
func (f *fsClient) PutObject(size int64, data io.Reader) *probe.Error {
|
|
objectDir, _ := filepath.Split(f.Path)
|
|
objectPath := f.Path
|
|
if objectDir != "" {
|
|
if err := os.MkdirAll(objectDir, 0700); err != nil {
|
|
return probe.NewError(err)
|
|
}
|
|
}
|
|
fs, err := os.Create(objectPath)
|
|
if err != nil {
|
|
return probe.NewError(err)
|
|
}
|
|
defer fs.Close()
|
|
|
|
// even if size is zero try to read from source
|
|
if size > 0 {
|
|
_, err = io.CopyN(fs, data, int64(size))
|
|
if err != nil {
|
|
return probe.NewError(err)
|
|
}
|
|
} else {
|
|
// size could be 0 for virtual files on certain filesystems
|
|
// for example /proc, so read till EOF for such files
|
|
_, err = io.Copy(fs, data)
|
|
if err != nil {
|
|
return probe.NewError(err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// get - download an object from bucket
|
|
func (f *fsClient) get() (io.ReadCloser, int64, *probe.Error) {
|
|
body, err := os.Open(f.Path)
|
|
if err != nil {
|
|
return nil, 0, probe.NewError(err)
|
|
}
|
|
content, perr := f.getFSMetadata()
|
|
if perr != nil {
|
|
return nil, content.Size, perr.Trace(f.Path)
|
|
}
|
|
return body, content.Size, nil
|
|
}
|
|
|
|
func (f *fsClient) ShareDownload(expires time.Duration) (string, *probe.Error) {
|
|
return "", probe.NewError(client.APINotImplemented{API: "ShareDownload", APIType: "filesystem"})
|
|
}
|
|
|
|
func (f *fsClient) ShareUpload(recursive bool, expires time.Duration, contentType string) (map[string]string, *probe.Error) {
|
|
return nil, probe.NewError(client.APINotImplemented{API: "ShareUpload", APIType: "filesystem"})
|
|
}
|
|
|
|
// GetObject download an full or part object from bucket
|
|
// getobject returns a reader, length and nil for no errors
|
|
// with errors getobject will return nil reader, length and typed errors
|
|
func (f *fsClient) GetObject(offset, length int64) (io.ReadCloser, int64, *probe.Error) {
|
|
if offset < 0 || length < 0 {
|
|
return nil, 0, probe.NewError(client.InvalidRange{Offset: offset})
|
|
}
|
|
|
|
tmppath := f.Path
|
|
// Golang strips trailing / if you clean(..) or
|
|
// EvalSymlinks(..). Adding '.' prevents it from doing so.
|
|
if strings.HasSuffix(tmppath, string(f.URL().Separator)) {
|
|
tmppath = tmppath + "."
|
|
}
|
|
|
|
// Resolve symlinks
|
|
_, err := filepath.EvalSymlinks(tmppath)
|
|
if os.IsNotExist(err) {
|
|
return nil, length, probe.NewError(err)
|
|
}
|
|
if err != nil {
|
|
return nil, length, probe.NewError(err)
|
|
}
|
|
if offset == 0 && length == 0 {
|
|
return f.get()
|
|
}
|
|
body, err := os.Open(f.Path)
|
|
if err != nil {
|
|
return nil, length, probe.NewError(err)
|
|
|
|
}
|
|
_, err = io.CopyN(ioutil.Discard, body, int64(offset))
|
|
if err != nil {
|
|
return nil, length, probe.NewError(err)
|
|
}
|
|
return body, length, nil
|
|
}
|
|
|
|
func (f *fsClient) Remove() *probe.Error {
|
|
err := os.Remove(f.Path)
|
|
return probe.NewError(err)
|
|
}
|
|
|
|
// List - list files and folders
|
|
func (f *fsClient) List(recursive bool) <-chan client.ContentOnChannel {
|
|
contentCh := make(chan client.ContentOnChannel)
|
|
switch recursive {
|
|
case true:
|
|
go f.listRecursiveInRoutine(contentCh)
|
|
default:
|
|
go f.listInRoutine(contentCh)
|
|
}
|
|
return contentCh
|
|
}
|
|
|
|
func (f *fsClient) listInRoutine(contentCh chan client.ContentOnChannel) {
|
|
defer close(contentCh)
|
|
|
|
fpath := f.Path
|
|
// Golang strips trailing / if you clean(..) or
|
|
// EvalSymlinks(..). Adding '.' prevents it from doing so.
|
|
if strings.HasSuffix(fpath, string(f.URL().Separator)) {
|
|
fpath = fpath + "."
|
|
}
|
|
|
|
fst, err := f.fsStat()
|
|
if err != nil {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: err.Trace(f.Path),
|
|
}
|
|
return
|
|
}
|
|
|
|
switch fst.Mode().IsDir() {
|
|
case true:
|
|
// do not use ioutil.ReadDir(), since it tries to sort its
|
|
// output at our scale we are expecting that to slow down
|
|
// instead we take raw output and provide it back to the
|
|
// user - this is the correct style when are moving large
|
|
// quantities of files
|
|
dir, err := os.Open(fpath)
|
|
if err != nil {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(err),
|
|
}
|
|
return
|
|
}
|
|
defer dir.Close()
|
|
|
|
files, err := dir.Readdir(-1)
|
|
if err != nil {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(err),
|
|
}
|
|
return
|
|
}
|
|
for _, file := range files {
|
|
fi := file
|
|
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
|
|
fi, err = os.Stat(filepath.Join(dir.Name(), fi.Name()))
|
|
if os.IsPermission(err) {
|
|
// On windows there are folder symlinks
|
|
// which are called junction files which
|
|
// carry special meaning on windows
|
|
// - which cannot be accessed with regular operations
|
|
if runtime.GOOS == "windows" {
|
|
lfi, lerr := os.Lstat(filepath.Join(dir.Name(), fi.Name()))
|
|
if lerr != nil {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(lerr),
|
|
}
|
|
continue
|
|
}
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: &client.Content{
|
|
Name: lfi.Name(),
|
|
Time: lfi.ModTime(),
|
|
Size: lfi.Size(),
|
|
Type: lfi.Mode(),
|
|
},
|
|
Err: probe.NewError(err),
|
|
}
|
|
continue
|
|
} else {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(err),
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
if os.IsNotExist(err) {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(client.BrokenSymlink{Path: file.Name()}),
|
|
}
|
|
continue
|
|
}
|
|
if err != nil {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(err),
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
if fi.Mode().IsRegular() || fi.Mode().IsDir() {
|
|
content := &client.Content{
|
|
Name: fi.Name(),
|
|
Time: fi.ModTime(),
|
|
Size: fi.Size(),
|
|
Type: fi.Mode(),
|
|
}
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: content,
|
|
Err: nil,
|
|
}
|
|
}
|
|
}
|
|
default:
|
|
content := &client.Content{
|
|
Name: f.Path,
|
|
Time: fst.ModTime(),
|
|
Size: fst.Size(),
|
|
Type: fst.Mode(),
|
|
}
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: content,
|
|
Err: nil,
|
|
}
|
|
}
|
|
}
|
|
|
|
func (f *fsClient) delimited(fp string) string {
|
|
var stripPrefix string
|
|
stripPrefix = f.Path[:strings.LastIndex(f.Path, string(f.URL().Separator))+1]
|
|
return strings.TrimPrefix(fp, stripPrefix)
|
|
}
|
|
|
|
func (f *fsClient) listRecursiveInRoutine(contentCh chan client.ContentOnChannel) {
|
|
defer close(contentCh)
|
|
visitFS := func(fp string, fi os.FileInfo, err error) error {
|
|
// fp also sends back itself with visitFS, ignore it we don't need it
|
|
if fp == f.Path {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
if strings.Contains(err.Error(), "operation not permitted") {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(err),
|
|
}
|
|
return nil
|
|
}
|
|
if os.IsPermission(err) {
|
|
if runtime.GOOS == "windows" {
|
|
// On windows there are folder symlinks
|
|
// which are called junction files which
|
|
// carry special meaning on windows
|
|
// - which cannot be accessed with regular operations
|
|
lfi, lerr := os.Lstat(fp)
|
|
if lerr != nil {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(lerr),
|
|
}
|
|
return nil
|
|
}
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: &client.Content{
|
|
Name: f.delimited(fp),
|
|
Time: lfi.ModTime(),
|
|
Size: lfi.Size(),
|
|
Type: lfi.Mode(),
|
|
},
|
|
Err: probe.NewError(err),
|
|
}
|
|
} else {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(err),
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
|
|
fi, err = os.Stat(fp)
|
|
if err != nil {
|
|
if os.IsPermission(err) {
|
|
if runtime.GOOS == "windows" {
|
|
// On windows there are folder symlinks
|
|
// which are called junction files which
|
|
// carry special meaning on windows
|
|
// - which cannot be accessed with regular operations
|
|
lfi, lerr := os.Lstat(fp)
|
|
if lerr != nil {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(lerr),
|
|
}
|
|
return nil
|
|
}
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: &client.Content{
|
|
Name: f.delimited(fp),
|
|
Time: lfi.ModTime(),
|
|
Size: lfi.Size(),
|
|
Type: lfi.Mode(),
|
|
},
|
|
Err: probe.NewError(err),
|
|
}
|
|
} else {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(err),
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
if os.IsNotExist(err) { // ignore broken symlinks
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(client.BrokenSymlink{Path: fp}),
|
|
}
|
|
return nil
|
|
}
|
|
if strings.Contains(err.Error(), "too many levels of symbolic links") {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(client.TooManyLevelsSymlink{Path: fp}),
|
|
}
|
|
return nil
|
|
}
|
|
|
|
return err
|
|
}
|
|
}
|
|
if fi.Mode().IsRegular() || fi.Mode().IsDir() {
|
|
content := &client.Content{
|
|
Name: f.delimited(fp),
|
|
Time: fi.ModTime(),
|
|
Size: fi.Size(),
|
|
Type: fi.Mode(),
|
|
}
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: content,
|
|
Err: nil,
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
err := filepath.Walk(f.Path, visitFS)
|
|
if err != nil {
|
|
contentCh <- client.ContentOnChannel{
|
|
Content: nil,
|
|
Err: probe.NewError(err),
|
|
}
|
|
}
|
|
}
|
|
|
|
// MakeBucket - create a new bucket
|
|
func (f *fsClient) MakeBucket() *probe.Error {
|
|
err := os.MkdirAll(f.Path, 0775)
|
|
if err != nil {
|
|
return probe.NewError(err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// GetBucketACL - get bucket acl
|
|
func (f *fsClient) GetBucketACL() (acl string, error *probe.Error) {
|
|
return "", probe.NewError(client.APINotImplemented{API: "GetBucketACL", APIType: "filesystem"})
|
|
}
|
|
|
|
// SetBucketACL - create a new bucket acl
|
|
func (f *fsClient) SetBucketACL(acl string) *probe.Error {
|
|
return probe.NewError(client.APINotImplemented{API: "SetBucketACL", APIType: "filesystem"})
|
|
}
|
|
|
|
// getFSMetadata -
|
|
func (f *fsClient) getFSMetadata() (content *client.Content, err *probe.Error) {
|
|
st, err := f.fsStat()
|
|
if err != nil {
|
|
return nil, err.Trace()
|
|
}
|
|
content = new(client.Content)
|
|
content.Name = f.Path
|
|
content.Size = st.Size()
|
|
content.Time = st.ModTime()
|
|
content.Type = st.Mode()
|
|
return content, nil
|
|
}
|
|
|
|
// Stat - get metadata from path
|
|
func (f *fsClient) Stat() (content *client.Content, err *probe.Error) {
|
|
return f.getFSMetadata()
|
|
}
|