/* * Minio Client (C) 2015 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "encoding/json" "path/filepath" "strings" "github.com/minio/mc/pkg/client" "github.com/minio/mc/pkg/console" "github.com/minio/minio-xl/pkg/probe" ) // // NOTE: All the parse rules should reduced to 1: Diff(First, Second). // // Valid cases // ======================= // 1: diff(f, f) -> diff(f, f) // 2: diff(f, d) -> diff(f, d/f) -> 1 // 3. diff(d1, d2) -> []diff(d1/f, d2/f) -> []1 // 4: diff(d1..., d2) -> []diff(d1/f, d2/f) -> []1 // // Invalid cases // ======================= // 1. diff(d1..., d2/f) -> INVALID // 2. diff(d1..., d2...) -> INVALID // // diffMessage json container for diff messages type diffMessage struct { FirstURL string `json:"first"` SecondURL string `json:"second"` Diff string `json:"diff"` Error *probe.Error `json:"error,omitempty"` } // String colorized diff message func (d diffMessage) String() string { msg := "" switch d.Diff { case "only-in-first": msg = console.Colorize("DiffMessage", "‘"+d.FirstURL+"’"+" and "+"‘"+d.SecondURL+"’") + console.Colorize("DiffOnlyInFirst", " - only in first.") case "type": msg = console.Colorize("DiffMessage", "‘"+d.FirstURL+"’"+" and "+"‘"+d.SecondURL+"’") + console.Colorize("DiffType", " - differ in type.") case "size": msg = console.Colorize("DiffMessage", "‘"+d.FirstURL+"’"+" and "+"‘"+d.SecondURL+"’") + console.Colorize("DiffSize", " - differ in size.") default: fatalIf(errDummy().Trace(), "Unhandled difference between ‘"+d.FirstURL+"’ and ‘"+d.SecondURL+"’.") } return msg } // JSON jsonified diff message func (d diffMessage) JSON() string { diffJSONBytes, err := json.Marshal(d) fatalIf(probe.NewError(err), "Unable to marshal diff message ‘"+d.FirstURL+"’, ‘"+d.SecondURL+"’ and ‘"+d.Diff+"’.") return string(diffJSONBytes) } // diffObjects - diff two incoming object contents, this is the most basic types. // // 1: diff(f, f) -> diff(f, f) -> VALID // 2: diff(f, d) -> diff(f, d/f) -> VALID func diffObjects(firstContent, secondContent *client.Content) *diffMessage { if firstContent.URL.String() == secondContent.URL.String() { return nil } if firstContent.Type.IsRegular() && secondContent.Type.IsRegular() { if firstContent.Size != secondContent.Size { return &diffMessage{ FirstURL: firstContent.URL.String(), SecondURL: secondContent.URL.String(), Diff: "size", } } return nil } if firstContent.Type.IsRegular() && secondContent.Type.IsDir() { newSecondURLStr := urlJoinPath(secondContent.URL.String(), firstContent.URL.String()) _, newSecondContent, err := url2Stat(newSecondURLStr) if err != nil { return &diffMessage{ FirstURL: firstContent.URL.String(), SecondURL: newSecondURLStr, Diff: "only-in-first", } } if firstContent.Size != newSecondContent.Size { return &diffMessage{ FirstURL: firstContent.URL.String(), SecondURL: newSecondContent.URL.String(), Diff: "size", } } return nil } if firstContent.Type.IsRegular() && !secondContent.Type.IsRegular() { return &diffMessage{ FirstURL: firstContent.URL.String(), SecondURL: secondContent.URL.String(), Diff: "type", } } return nil } // diffFolders - diff of contents of two folders only top level content. // // 3: diff(d1, d2) -> []diff(d1/f, d2/f) -> VALID func diffFolders(firstClnt, secondClnt client.Client, outCh chan<- diffMessage) { recursive := false // Range on the List to consume incoming content for contentCh := range firstClnt.List(recursive, false) { if contentCh.Err != nil { outCh <- diffMessage{ Error: contentCh.Err.Trace(firstClnt.GetURL().String()), } continue } // Store incoming content newFirstContent := contentCh.Content newFirstURLStr := newFirstContent.URL.String() // Construct the second URL. newSecondURL := secondClnt.GetURL() // Need to verify the same path from first URL, construct the second URL newSecondURL.Path = filepath.Join(newSecondURL.Path, filepath.Base(contentCh.Content.URL.Path)) newSecondURLStr := newSecondURL.String() // Send a stat to verify _, newSecondContent, err := url2Stat(newSecondURLStr) if err != nil { outCh <- diffMessage{ FirstURL: newFirstURLStr, SecondURL: newSecondURLStr, Diff: "only-in-first", } continue } diffMsg := diffObjects(newFirstContent, newSecondContent) if diffMsg != nil { outCh <- *diffMsg continue } } // Reached EOF } // diffFoldersRecursive diff folders for all files recursively. // // 4: diff(d1..., d2) -> []diff(d1/f, d2/f) -> VALID. func diffFoldersRecursive(firstClnt, secondClnt client.Client, outCh chan<- diffMessage) { recursive := true firstListCh := firstClnt.List(recursive, false) // Copy first list channel. for firstContentCh := range firstListCh { if firstContentCh.Err != nil { outCh <- diffMessage{Error: firstContentCh.Err.Trace()} continue } if firstContentCh.Content.Type.IsDir() { // Skip directories there is no concept of directories on S3. continue } firstContent := firstContentCh.Content secondURL := secondClnt.GetURL() secondURL.Path = filepath.Join(secondURL.Path, strings.TrimPrefix(firstContent.URL.Path, url2Dir(firstClnt.GetURL().Path))) _, secondContent, err := url2Stat(secondURL.String()) if err != nil { outCh <- diffMessage{ FirstURL: firstContent.URL.String(), SecondURL: secondURL.String(), Diff: "only-in-first", } continue } if diffMsg := diffObjects(firstContent, secondContent); diffMsg != nil { outCh <- *diffMsg continue } } }