diff --git a/diff.go b/diff.go index fd5c0e48..6e89dc65 100644 --- a/diff.go +++ b/diff.go @@ -18,7 +18,6 @@ package main import ( "encoding/json" - "os" "strings" "sync" "time" @@ -26,7 +25,6 @@ import ( "github.com/minio/mc/pkg/client" "github.com/minio/mc/pkg/console" "github.com/minio/minio/pkg/probe" - "github.com/tchap/go-patricia/patricia" ) // @@ -216,45 +214,60 @@ func dodiff(firstClnt, secondClnt client.Client, ch chan DiffMessage) { } func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) { - firstTrie := patricia.NewTrie() - secondTrie := patricia.NewTrie() - wg := new(sync.WaitGroup) - - type urlAttr struct { - Size int64 - Type os.FileMode + firstURLDelimited := firstClnt.URL().String() + secondURLDelimited := secondClnt.URL().String() + if strings.HasSuffix(firstURLDelimited, "/") == false { + firstURLDelimited = firstURLDelimited + "/" + } + if strings.HasSuffix(secondURLDelimited, "/") == false { + secondURLDelimited = secondURLDelimited + "/" + } + firstClnt, err := url2Client(firstURLDelimited) + if err != nil { + ch <- DiffMessage{Error: err.Trace()} + return + } + secondClnt, err = url2Client(secondURLDelimited) + if err != nil { + ch <- DiffMessage{Error: err.Trace()} + return } - wg.Add(1) - go func(ch chan<- DiffMessage) { - defer wg.Done() - for firstContentCh := range firstClnt.List(true) { - if firstContentCh.Err != nil { - ch <- DiffMessage{ - Error: firstContentCh.Err.Trace(firstClnt.URL().String()), - } - return - } - firstTrie.Insert(patricia.Prefix(firstContentCh.Content.Name), urlAttr{firstContentCh.Content.Size, firstContentCh.Content.Type}) - } - }(ch) - wg.Add(1) - go func(ch chan<- DiffMessage) { - defer wg.Done() - for secondContentCh := range secondClnt.List(true) { - if secondContentCh.Err != nil { - ch <- DiffMessage{ - Error: secondContentCh.Err.Trace(secondClnt.URL().String()), - } - return - } - secondTrie.Insert(patricia.Prefix(secondContentCh.Content.Name), urlAttr{secondContentCh.Content.Size, secondContentCh.Content.Type}) - } - }(ch) + wg := new(sync.WaitGroup) - doneCh := make(chan struct{}) + firstSortedList := sortedList{} + secondSortedList := sortedList{} + id := randomID(8) + firstid := id + ".1" + secondid := id + ".2" + + wg.Add(1) + go func() { + defer wg.Done() + err := firstSortedList.Create(firstClnt, firstid) + if err != nil { + ch <- DiffMessage{ + Error: err.Trace(), + } + return + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + err := secondSortedList.Create(secondClnt, secondid) + if err != nil { + ch <- DiffMessage{ + Error: err.Trace(), + } + return + } + }() + + doneCh := make(chan bool) defer close(doneCh) - go func(doneCh <-chan struct{}) { + go func(doneCh <-chan bool) { cursorCh := cursorAnimate() for { select { @@ -268,64 +281,96 @@ func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) { } }(doneCh) wg.Wait() - doneCh <- struct{}{} + doneCh <- true + if !globalQuietFlag && !globalJSONFlag { console.Eraseline() } - matchNameCh := make(chan string, 10000) - go func(matchNameCh chan<- string) { - itemFunc := func(prefix patricia.Prefix, item patricia.Item) error { - matchNameCh <- string(prefix) - return nil + fch := firstSortedList.List(true) + sch := secondSortedList.List(true) + f, fok := <-fch + s, sok := <-sch + for { + if fok == false { + break } - firstTrie.Visit(itemFunc) - defer close(matchNameCh) - }(matchNameCh) - for matchName := range matchNameCh { - firstURLDelimited := firstClnt.URL().String()[:strings.LastIndex(firstClnt.URL().String(), string(firstClnt.URL().Separator))+1] - secondURLDelimited := secondClnt.URL().String()[:strings.LastIndex(secondClnt.URL().String(), string(secondClnt.URL().Separator))+1] - firstURL := firstURLDelimited + matchName - secondURL := secondURLDelimited + matchName - if !secondTrie.Match(patricia.Prefix(matchName)) { + if f.Content.Type.IsDir() { + // skip directories + // there is no concept of directories on S3 + f, fok = <-fch + continue + } + firstURL := firstURLDelimited + f.Content.Name + secondURL := secondURLDelimited + f.Content.Name + if sok == false { + // Second list reached EOF ch <- DiffMessage{ FirstURL: firstURL, SecondURL: secondURL, Diff: "only-in-first", } - } else { - firstURLAttr := firstTrie.Get(patricia.Prefix(matchName)).(urlAttr) - secondURLAttr := secondTrie.Get(patricia.Prefix(matchName)).(urlAttr) + f, fok = <-fch + continue + } + if s.Content.Type.IsDir() { + // skip directories + s, sok = <-sch + continue + } + fC := f.Content + sC := s.Content + compare := strings.Compare(fC.Name, sC.Name) - if firstURLAttr.Type.IsRegular() { - if !secondURLAttr.Type.IsRegular() { + if compare == 0 { + if fC.Type.IsRegular() { + if !sC.Type.IsRegular() { ch <- DiffMessage{ FirstURL: firstURL, SecondURL: secondURL, Diff: "type", } - continue } - } - - if firstURLAttr.Type.IsDir() { - if !secondURLAttr.Type.IsDir() { + } else if fC.Type.IsDir() { + if !sC.Type.IsDir() { ch <- DiffMessage{ FirstURL: firstURL, SecondURL: secondURL, Diff: "type", } - continue } - } - - if firstURLAttr.Size != secondURLAttr.Size { + } else if fC.Size != sC.Size { ch <- DiffMessage{ FirstURL: firstURL, SecondURL: secondURL, Diff: "size", } } + f, fok = <-fch + s, sok = <-sch + } + if compare < 0 { + ch <- DiffMessage{ + FirstURL: firstURL, + SecondURL: secondURL, + Diff: "only-in-first", + } + f, fok = <-fch + } + if compare > 0 { + s, sok = <-sch + } + } + err = firstSortedList.Delete() + if err != nil { + ch <- DiffMessage{ + Error: err.Trace(), + } + } + err = secondSortedList.Delete() + if err != nil { + ch <- DiffMessage{ + Error: err.Trace(), } } } diff --git a/globals.go b/globals.go index 473836f3..41a4b5ae 100644 --- a/globals.go +++ b/globals.go @@ -37,6 +37,8 @@ const ( globalSessionDir = "session" globalSharedURLsDataDir = "share" + golbalSortedListDir = "sortedlist" + // default access and secret key // do not pass accesskeyid and secretaccesskey through cli // users should manually edit them, add a stub entry diff --git a/sorted-list.go b/sorted-list.go new file mode 100644 index 00000000..daec3156 --- /dev/null +++ b/sorted-list.go @@ -0,0 +1,135 @@ +/* + * Minio Client (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "encoding/gob" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/minio/mc/pkg/client" + "github.com/minio/minio/pkg/probe" +) + +type sortedList struct { + name string + file *os.File + dec *gob.Decoder + enc *gob.Encoder +} + +func getSortedListDir() (string, *probe.Error) { + configDir, err := getMcConfigDir() + if err != nil { + return "", err.Trace() + } + sortedListDir := filepath.Join(configDir, golbalSortedListDir) + return sortedListDir, nil +} + +func createSortedListDir() *probe.Error { + sortedListDir, err := getSortedListDir() + if err != nil { + return err.Trace() + } + if _, err := os.Stat(sortedListDir); err == nil { + return nil + } + if err := os.MkdirAll(sortedListDir, 0700); err != nil { + return probe.NewError(err) + } + return nil +} + +// Create create an on disk sorted file from clnt +func (sl *sortedList) Create(clnt client.Client, id string) *probe.Error { + var e error + if err := createSortedListDir(); err != nil { + return err.Trace() + } + sortedListDir, err := getSortedListDir() + if err != nil { + return err.Trace() + } + sl.name = filepath.Join(sortedListDir, id) + sl.file, e = os.OpenFile(sl.name, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666) + if e != nil { + return probe.NewError(e) + } + sl.enc = gob.NewEncoder(sl.file) + sl.dec = gob.NewDecoder(sl.file) + for content := range clnt.List(true) { + if content.Err != nil { + switch err := content.Err.ToGoError().(type) { + case client.ISBrokenSymlink: + // FIXME: send the error to caller using channel + errorIf(content.Err.Trace(), fmt.Sprintf("Skipping broken Symlink ‘%s’.", err.Path)) + continue + } + if os.IsNotExist(content.Err.ToGoError()) || os.IsPermission(content.Err.ToGoError()) { + // FIXME: abstract this at fs.go layer + if content.Content != nil { + if content.Content.Type.IsDir() && (content.Content.Type&os.ModeSymlink == os.ModeSymlink) { + continue + } + } + errorIf(content.Err.Trace(), fmt.Sprintf("Skipping ‘%s’.", content.Content.Name)) + continue + } + return content.Err.Trace() + } + sl.enc.Encode(*content.Content) + } + if _, err := sl.file.Seek(0, os.SEEK_SET); err != nil { + return probe.NewError(err) + } + return nil +} + +// List list the entries from the sorted file +func (sl *sortedList) List(recursive bool) <-chan client.ContentOnChannel { + ch := make(chan client.ContentOnChannel) + go func() { + defer close(ch) + for { + var c client.Content + err := sl.dec.Decode(&c) + if err == io.EOF { + break + } + if err != nil { + ch <- client.ContentOnChannel{Content: nil, Err: probe.NewError(err)} + break + } + ch <- client.ContentOnChannel{Content: &c, Err: nil} + } + }() + return ch +} + +// Delete close and delete the ondisk file +func (sl *sortedList) Delete() *probe.Error { + if err := sl.file.Close(); err != nil { + return probe.NewError(err) + } + if err := os.Remove(sl.name); err != nil { + return probe.NewError(err) + } + return nil +}