diff - dump the sorted list of source and destination to local disk file and then diff from the local files

diff.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"encoding/json"
-	"os"
 	"strings"
 	"sync"
 	"time"
@@ -26,7 +25,6 @@ import (
 	"github.com/minio/mc/pkg/client"
 	"github.com/minio/mc/pkg/console"
 	"github.com/minio/minio/pkg/probe"
-	"github.com/tchap/go-patricia/patricia"
 )
 
 //
@@ -216,45 +214,60 @@ func dodiff(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
 }
 
 func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
-	firstTrie := patricia.NewTrie()
-	secondTrie := patricia.NewTrie()
+	firstURLDelimited := firstClnt.URL().String()
+	secondURLDelimited := secondClnt.URL().String()
+	if strings.HasSuffix(firstURLDelimited, "/") == false {
+		firstURLDelimited = firstURLDelimited + "/"
+	}
+	if strings.HasSuffix(secondURLDelimited, "/") == false {
+		secondURLDelimited = secondURLDelimited + "/"
+	}
+	firstClnt, err := url2Client(firstURLDelimited)
+	if err != nil {
+		ch <- DiffMessage{Error: err.Trace()}
+		return
+	}
+	secondClnt, err = url2Client(secondURLDelimited)
+	if err != nil {
+		ch <- DiffMessage{Error: err.Trace()}
+		return
+	}
+
 	wg := new(sync.WaitGroup)
 
-	type urlAttr struct {
-		Size int64
-		Type os.FileMode
-	}
+	firstSortedList := sortedList{}
+	secondSortedList := sortedList{}
+	id := randomID(8)
+	firstid := id + ".1"
+	secondid := id + ".2"
 
 	wg.Add(1)
-	go func(ch chan<- DiffMessage) {
+	go func() {
 		defer wg.Done()
-		for firstContentCh := range firstClnt.List(true) {
-			if firstContentCh.Err != nil {
+		err := firstSortedList.Create(firstClnt, firstid)
+		if err != nil {
 			ch <- DiffMessage{
-					Error: firstContentCh.Err.Trace(firstClnt.URL().String()),
+				Error: err.Trace(),
 			}
 			return
 		}
-			firstTrie.Insert(patricia.Prefix(firstContentCh.Content.Name), urlAttr{firstContentCh.Content.Size, firstContentCh.Content.Type})
-		}
-	}(ch)
+	}()
+
 	wg.Add(1)
-	go func(ch chan<- DiffMessage) {
+	go func() {
 		defer wg.Done()
-		for secondContentCh := range secondClnt.List(true) {
-			if secondContentCh.Err != nil {
+		err := secondSortedList.Create(secondClnt, secondid)
+		if err != nil {
 			ch <- DiffMessage{
-					Error: secondContentCh.Err.Trace(secondClnt.URL().String()),
+				Error: err.Trace(),
 			}
 			return
 		}
-			secondTrie.Insert(patricia.Prefix(secondContentCh.Content.Name), urlAttr{secondContentCh.Content.Size, secondContentCh.Content.Type})
-		}
-	}(ch)
+	}()
 
-	doneCh := make(chan struct{})
+	doneCh := make(chan bool)
 	defer close(doneCh)
-	go func(doneCh <-chan struct{}) {
+	go func(doneCh <-chan bool) {
 		cursorCh := cursorAnimate()
 		for {
 			select {
@@ -268,64 +281,96 @@ func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
 		}
 	}(doneCh)
 	wg.Wait()
-	doneCh <- struct{}{}
+	doneCh <- true
+
 	if !globalQuietFlag && !globalJSONFlag {
 		console.Eraseline()
 	}
 
-	matchNameCh := make(chan string, 10000)
-	go func(matchNameCh chan<- string) {
-		itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
-			matchNameCh <- string(prefix)
-			return nil
+	fch := firstSortedList.List(true)
+	sch := secondSortedList.List(true)
+	f, fok := <-fch
+	s, sok := <-sch
+	for {
+		if fok == false {
+			break
 		}
-		firstTrie.Visit(itemFunc)
-		defer close(matchNameCh)
-	}(matchNameCh)
-	for matchName := range matchNameCh {
-		firstURLDelimited := firstClnt.URL().String()[:strings.LastIndex(firstClnt.URL().String(), string(firstClnt.URL().Separator))+1]
-		secondURLDelimited := secondClnt.URL().String()[:strings.LastIndex(secondClnt.URL().String(), string(secondClnt.URL().Separator))+1]
-		firstURL := firstURLDelimited + matchName
-		secondURL := secondURLDelimited + matchName
-		if !secondTrie.Match(patricia.Prefix(matchName)) {
+		if f.Content.Type.IsDir() {
+			// skip directories
+			// there is no concept of directories on S3
+			f, fok = <-fch
+			continue
+		}
+		firstURL := firstURLDelimited + f.Content.Name
+		secondURL := secondURLDelimited + f.Content.Name
+		if sok == false {
+			// Second list reached EOF
 			ch <- DiffMessage{
 				FirstURL:  firstURL,
 				SecondURL: secondURL,
 				Diff:      "only-in-first",
 			}
-		} else {
-			firstURLAttr := firstTrie.Get(patricia.Prefix(matchName)).(urlAttr)
-			secondURLAttr := secondTrie.Get(patricia.Prefix(matchName)).(urlAttr)
+			f, fok = <-fch
+			continue
+		}
+		if s.Content.Type.IsDir() {
+			// skip directories
+			s, sok = <-sch
+			continue
+		}
+		fC := f.Content
+		sC := s.Content
+		compare := strings.Compare(fC.Name, sC.Name)
 
-			if firstURLAttr.Type.IsRegular() {
-				if !secondURLAttr.Type.IsRegular() {
+		if compare == 0 {
+			if fC.Type.IsRegular() {
+				if !sC.Type.IsRegular() {
 					ch <- DiffMessage{
 						FirstURL:  firstURL,
 						SecondURL: secondURL,
 						Diff:      "type",
 					}
-					continue
 				}
-			}
-
-			if firstURLAttr.Type.IsDir() {
-				if !secondURLAttr.Type.IsDir() {
+			} else if fC.Type.IsDir() {
+				if !sC.Type.IsDir() {
 					ch <- DiffMessage{
 						FirstURL:  firstURL,
 						SecondURL: secondURL,
 						Diff:      "type",
 					}
-					continue
 				}
-			}
-
-			if firstURLAttr.Size != secondURLAttr.Size {
+			} else if fC.Size != sC.Size {
 				ch <- DiffMessage{
 					FirstURL:  firstURL,
 					SecondURL: secondURL,
 					Diff:      "size",
 				}
 			}
+			f, fok = <-fch
+			s, sok = <-sch
+		}
+		if compare < 0 {
+			ch <- DiffMessage{
+				FirstURL:  firstURL,
+				SecondURL: secondURL,
+				Diff:      "only-in-first",
+			}
+			f, fok = <-fch
+		}
+		if compare > 0 {
+			s, sok = <-sch
+		}
+	}
+	err = firstSortedList.Delete()
+	if err != nil {
+		ch <- DiffMessage{
+			Error: err.Trace(),
+		}
+	}
+	err = secondSortedList.Delete()
+	if err != nil {
+		ch <- DiffMessage{
+			Error: err.Trace(),
 		}
 	}
 }

globals.go

@@ -37,6 +37,8 @@ const (
 	globalSessionDir        = "session"
 	globalSharedURLsDataDir = "share"
 
+	golbalSortedListDir = "sortedlist"
+
 	// default access and secret key
 	// do not pass accesskeyid and secretaccesskey through cli
 	// users should manually edit them, add a stub entry

sorted-list.go

@@ -0,0 +1,135 @@
+/*
+ * Minio Client (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/gob"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/minio/mc/pkg/client"
+	"github.com/minio/minio/pkg/probe"
+)
+
+type sortedList struct {
+	name string
+	file *os.File
+	dec  *gob.Decoder
+	enc  *gob.Encoder
+}
+
+func getSortedListDir() (string, *probe.Error) {
+	configDir, err := getMcConfigDir()
+	if err != nil {
+		return "", err.Trace()
+	}
+	sortedListDir := filepath.Join(configDir, golbalSortedListDir)
+	return sortedListDir, nil
+}
+
+func createSortedListDir() *probe.Error {
+	sortedListDir, err := getSortedListDir()
+	if err != nil {
+		return err.Trace()
+	}
+	if _, err := os.Stat(sortedListDir); err == nil {
+		return nil
+	}
+	if err := os.MkdirAll(sortedListDir, 0700); err != nil {
+		return probe.NewError(err)
+	}
+	return nil
+}
+
+// Create create an on disk sorted file from clnt
+func (sl *sortedList) Create(clnt client.Client, id string) *probe.Error {
+	var e error
+	if err := createSortedListDir(); err != nil {
+		return err.Trace()
+	}
+	sortedListDir, err := getSortedListDir()
+	if err != nil {
+		return err.Trace()
+	}
+	sl.name = filepath.Join(sortedListDir, id)
+	sl.file, e = os.OpenFile(sl.name, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
+	if e != nil {
+		return probe.NewError(e)
+	}
+	sl.enc = gob.NewEncoder(sl.file)
+	sl.dec = gob.NewDecoder(sl.file)
+	for content := range clnt.List(true) {
+		if content.Err != nil {
+			switch err := content.Err.ToGoError().(type) {
+			case client.ISBrokenSymlink:
+				// FIXME: send the error to caller using channel
+				errorIf(content.Err.Trace(), fmt.Sprintf("Skipping broken Symlink ‘%s’.", err.Path))
+				continue
+			}
+			if os.IsNotExist(content.Err.ToGoError()) || os.IsPermission(content.Err.ToGoError()) {
+				// FIXME: abstract this at fs.go layer
+				if content.Content != nil {
+					if content.Content.Type.IsDir() && (content.Content.Type&os.ModeSymlink == os.ModeSymlink) {
+						continue
+					}
+				}
+				errorIf(content.Err.Trace(), fmt.Sprintf("Skipping ‘%s’.", content.Content.Name))
+				continue
+			}
+			return content.Err.Trace()
+		}
+		sl.enc.Encode(*content.Content)
+	}
+	if _, err := sl.file.Seek(0, os.SEEK_SET); err != nil {
+		return probe.NewError(err)
+	}
+	return nil
+}
+
+// List list the entries from the sorted file
+func (sl *sortedList) List(recursive bool) <-chan client.ContentOnChannel {
+	ch := make(chan client.ContentOnChannel)
+	go func() {
+		defer close(ch)
+		for {
+			var c client.Content
+			err := sl.dec.Decode(&c)
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				ch <- client.ContentOnChannel{Content: nil, Err: probe.NewError(err)}
+				break
+			}
+			ch <- client.ContentOnChannel{Content: &c, Err: nil}
+		}
+	}()
+	return ch
+}
+
+// Delete close and delete the ondisk file
+func (sl *sortedList) Delete() *probe.Error {
+	if err := sl.file.Close(); err != nil {
+		return probe.NewError(err)
+	}
+	if err := os.Remove(sl.name); err != nil {
+		return probe.NewError(err)
+	}
+	return nil
+}