Diff now has recursive listing and matching based on patricia trie

Godeps/Godeps.json

@@ -49,6 +49,11 @@
 			"ImportPath": "github.com/shiena/ansicolor",
 			"Rev": "a5e2b567a4dd6cc74545b8a4f27c9d63b9e7735b"
 		},
+		{
+			"ImportPath": "github.com/tchap/go-patricia/patricia",
+			"Comment": "v2.2.0",
+			"Rev": "a560f85131e8aafc2cc4bcb1e265ae3fc6ee5f50"
+		},
 		{
 			"ImportPath": "gopkg.in/check.v1",
 			"Rev": "b3d3430320d4260e5fea99841af984b3badcea63"

Godeps/_workspace/src/github.com/tchap/go-patricia/patricia/children.go

@@ -0,0 +1,244 @@
+// Copyright (c) 2014 The go-patricia AUTHORS
+//
+// Use of this source code is governed by The MIT License
+// that can be found in the LICENSE file.
+
+package patricia
+
+import "sort"
+
+type childList interface {
+	length() int
+	head() *Trie
+	add(child *Trie) childList
+	replace(b byte, child *Trie)
+	remove(child *Trie)
+	next(b byte) *Trie
+	walk(prefix *Prefix, visitor VisitorFunc) error
+}
+
+type tries []*Trie
+
+func (t tries) Len() int {
+	return len(t)
+}
+
+func (t tries) Less(i, j int) bool {
+	strings := sort.StringSlice{string(t[i].prefix), string(t[j].prefix)}
+	return strings.Less(0, 1)
+}
+
+func (t tries) Swap(i, j int) {
+	t[i], t[j] = t[j], t[i]
+}
+
+type sparseChildList struct {
+	children tries
+}
+
+func newSparseChildList(maxChildrenPerSparseNode int) childList {
+	return &sparseChildList{
+		children: make(tries, 0, maxChildrenPerSparseNode),
+	}
+}
+
+func (list *sparseChildList) length() int {
+	return len(list.children)
+}
+
+func (list *sparseChildList) head() *Trie {
+	return list.children[0]
+}
+
+func (list *sparseChildList) add(child *Trie) childList {
+	// Search for an empty spot and insert the child if possible.
+	if len(list.children) != cap(list.children) {
+		list.children = append(list.children, child)
+		return list
+	}
+
+	// Otherwise we have to transform to the dense list type.
+	return newDenseChildList(list, child)
+}
+
+func (list *sparseChildList) replace(b byte, child *Trie) {
+	// Seek the child and replace it.
+	for i, node := range list.children {
+		if node.prefix[0] == b {
+			list.children[i] = child
+			return
+		}
+	}
+}
+
+func (list *sparseChildList) remove(child *Trie) {
+	for i, node := range list.children {
+		if node.prefix[0] == child.prefix[0] {
+			list.children = append(list.children[:i], list.children[i+1:]...)
+			return
+		}
+	}
+
+	// This is not supposed to be reached.
+	panic("removing non-existent child")
+}
+
+func (list *sparseChildList) next(b byte) *Trie {
+	for _, child := range list.children {
+		if child.prefix[0] == b {
+			return child
+		}
+	}
+	return nil
+}
+
+func (list *sparseChildList) walk(prefix *Prefix, visitor VisitorFunc) error {
+
+	sort.Sort(list.children)
+
+	for _, child := range list.children {
+		*prefix = append(*prefix, child.prefix...)
+		if child.item != nil {
+			err := visitor(*prefix, child.item)
+			if err != nil {
+				if err == SkipSubtree {
+					*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
+					continue
+				}
+				*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
+				return err
+			}
+		}
+
+		err := child.children.walk(prefix, visitor)
+		*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type denseChildList struct {
+	min      int
+	max      int
+	children []*Trie
+}
+
+func newDenseChildList(list *sparseChildList, child *Trie) childList {
+	var (
+		min int = 255
+		max int = 0
+	)
+	for _, child := range list.children {
+		b := int(child.prefix[0])
+		if b < min {
+			min = b
+		}
+		if b > max {
+			max = b
+		}
+	}
+
+	b := int(child.prefix[0])
+	if b < min {
+		min = b
+	}
+	if b > max {
+		max = b
+	}
+
+	children := make([]*Trie, max-min+1)
+	for _, child := range list.children {
+		children[int(child.prefix[0])-min] = child
+	}
+	children[int(child.prefix[0])-min] = child
+
+	return &denseChildList{min, max, children}
+}
+
+func (list *denseChildList) length() int {
+	return list.max - list.min + 1
+}
+
+func (list *denseChildList) head() *Trie {
+	return list.children[0]
+}
+
+func (list *denseChildList) add(child *Trie) childList {
+	b := int(child.prefix[0])
+
+	switch {
+	case list.min <= b && b <= list.max:
+		if list.children[b-list.min] != nil {
+			panic("dense child list collision detected")
+		}
+		list.children[b-list.min] = child
+
+	case b < list.min:
+		children := make([]*Trie, list.max-b+1)
+		children[0] = child
+		copy(children[list.min-b:], list.children)
+		list.children = children
+		list.min = b
+
+	default: // b > list.max
+		children := make([]*Trie, b-list.min+1)
+		children[b-list.min] = child
+		copy(children, list.children)
+		list.children = children
+		list.max = b
+	}
+
+	return list
+}
+
+func (list *denseChildList) replace(b byte, child *Trie) {
+	list.children[int(b)-list.min] = nil
+	list.children[int(child.prefix[0])-list.min] = child
+}
+
+func (list *denseChildList) remove(child *Trie) {
+	i := int(child.prefix[0]) - list.min
+	if list.children[i] == nil {
+		// This is not supposed to be reached.
+		panic("removing non-existent child")
+	}
+	list.children[i] = nil
+}
+
+func (list *denseChildList) next(b byte) *Trie {
+	i := int(b)
+	if i < list.min || list.max < i {
+		return nil
+	}
+	return list.children[i-list.min]
+}
+
+func (list *denseChildList) walk(prefix *Prefix, visitor VisitorFunc) error {
+	for _, child := range list.children {
+		if child == nil {
+			continue
+		}
+		*prefix = append(*prefix, child.prefix...)
+		if child.item != nil {
+			if err := visitor(*prefix, child.item); err != nil {
+				if err == SkipSubtree {
+					*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
+					continue
+				}
+				*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
+				return err
+			}
+		}
+
+		err := child.children.walk(prefix, visitor)
+		*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

Godeps/_workspace/src/github.com/tchap/go-patricia/patricia/patricia.go

@@ -0,0 +1,467 @@
+// Copyright (c) 2014 The go-patricia AUTHORS
+//
+// Use of this source code is governed by The MIT License
+// that can be found in the LICENSE file.
+
+package patricia
+
+import (
+	"errors"
+)
+
+//------------------------------------------------------------------------------
+// Trie
+//------------------------------------------------------------------------------
+
+const (
+	DefaultMaxPrefixPerNode         = 10
+	DefaultMaxChildrenPerSparseNode = 8
+)
+
+type (
+	Prefix      []byte
+	Item        interface{}
+	VisitorFunc func(prefix Prefix, item Item) error
+)
+
+// Trie is a generic patricia trie that allows fast retrieval of items by prefix.
+// and other funky stuff.
+//
+// Trie is not thread-safe.
+type Trie struct {
+	prefix Prefix
+	item   Item
+
+	maxPrefixPerNode         int
+	maxChildrenPerSparseNode int
+
+	children childList
+}
+
+// Public API ------------------------------------------------------------------
+
+type Option func(*Trie)
+
+// Trie constructor.
+func NewTrie(options ...Option) *Trie {
+	trie := &Trie{}
+
+	for _, opt := range options {
+		opt(trie)
+	}
+
+	if trie.maxPrefixPerNode <= 0 {
+		trie.maxPrefixPerNode = DefaultMaxPrefixPerNode
+	}
+	if trie.maxChildrenPerSparseNode <= 0 {
+		trie.maxChildrenPerSparseNode = DefaultMaxChildrenPerSparseNode
+	}
+
+	trie.children = newSparseChildList(trie.maxChildrenPerSparseNode)
+	return trie
+}
+
+func MaxPrefixPerNode(value int) Option {
+	return func(trie *Trie) {
+		trie.maxPrefixPerNode = value
+	}
+}
+
+func MaxChildrenPerSparseNode(value int) Option {
+	return func(trie *Trie) {
+		trie.maxChildrenPerSparseNode = value
+	}
+}
+
+// Item returns the item stored in the root of this trie.
+func (trie *Trie) Item() Item {
+	return trie.item
+}
+
+// Insert inserts a new item into the trie using the given prefix. Insert does
+// not replace existing items. It returns false if an item was already in place.
+func (trie *Trie) Insert(key Prefix, item Item) (inserted bool) {
+	return trie.put(key, item, false)
+}
+
+// Set works much like Insert, but it always sets the item, possibly replacing
+// the item previously inserted.
+func (trie *Trie) Set(key Prefix, item Item) {
+	trie.put(key, item, true)
+}
+
+// Get returns the item located at key.
+//
+// This method is a bit dangerous, because Get can as well end up in an internal
+// node that is not really representing any user-defined value. So when nil is
+// a valid value being used, it is not possible to tell if the value was inserted
+// into the tree by the user or not. A possible workaround for this is not to use
+// nil interface as a valid value, even using zero value of any type is enough
+// to prevent this bad behaviour.
+func (trie *Trie) Get(key Prefix) (item Item) {
+	_, node, found, leftover := trie.findSubtree(key)
+	if !found || len(leftover) != 0 {
+		return nil
+	}
+	return node.item
+}
+
+// Match returns what Get(prefix) != nil would return. The same warning as for
+// Get applies here as well.
+func (trie *Trie) Match(prefix Prefix) (matchedExactly bool) {
+	return trie.Get(prefix) != nil
+}
+
+// MatchSubtree returns true when there is a subtree representing extensions
+// to key, that is if there are any keys in the tree which have key as prefix.
+func (trie *Trie) MatchSubtree(key Prefix) (matched bool) {
+	_, _, matched, _ = trie.findSubtree(key)
+	return
+}
+
+// Visit calls visitor on every node containing a non-nil item
+// in alphabetical order.
+//
+// If an error is returned from visitor, the function stops visiting the tree
+// and returns that error, unless it is a special error - SkipSubtree. In that
+// case Visit skips the subtree represented by the current node and continues
+// elsewhere.
+func (trie *Trie) Visit(visitor VisitorFunc) error {
+	return trie.walk(nil, visitor)
+}
+
+// VisitSubtree works much like Visit, but it only visits nodes matching prefix.
+func (trie *Trie) VisitSubtree(prefix Prefix, visitor VisitorFunc) error {
+	// Nil prefix not allowed.
+	if prefix == nil {
+		panic(ErrNilPrefix)
+	}
+
+	// Empty trie must be handled explicitly.
+	if trie.prefix == nil {
+		return nil
+	}
+
+	// Locate the relevant subtree.
+	_, root, found, leftover := trie.findSubtree(prefix)
+	if !found {
+		return nil
+	}
+	prefix = append(prefix, leftover...)
+
+	// Visit it.
+	return root.walk(prefix, visitor)
+}
+
+// VisitPrefixes visits only nodes that represent prefixes of key.
+// To say the obvious, returning SkipSubtree from visitor makes no sense here.
+func (trie *Trie) VisitPrefixes(key Prefix, visitor VisitorFunc) error {
+	// Nil key not allowed.
+	if key == nil {
+		panic(ErrNilPrefix)
+	}
+
+	// Empty trie must be handled explicitly.
+	if trie.prefix == nil {
+		return nil
+	}
+
+	// Walk the path matching key prefixes.
+	node := trie
+	prefix := key
+	offset := 0
+	for {
+		// Compute what part of prefix matches.
+		common := node.longestCommonPrefixLength(key)
+		key = key[common:]
+		offset += common
+
+		// Partial match means that there is no subtree matching prefix.
+		if common < len(node.prefix) {
+			return nil
+		}
+
+		// Call the visitor.
+		if item := node.item; item != nil {
+			if err := visitor(prefix[:offset], item); err != nil {
+				return err
+			}
+		}
+
+		if len(key) == 0 {
+			// This node represents key, we are finished.
+			return nil
+		}
+
+		// There is some key suffix left, move to the children.
+		child := node.children.next(key[0])
+		if child == nil {
+			// There is nowhere to continue, return.
+			return nil
+		}
+
+		node = child
+	}
+}
+
+// Delete deletes the item represented by the given prefix.
+//
+// True is returned if the matching node was found and deleted.
+func (trie *Trie) Delete(key Prefix) (deleted bool) {
+	// Nil prefix not allowed.
+	if key == nil {
+		panic(ErrNilPrefix)
+	}
+
+	// Empty trie must be handled explicitly.
+	if trie.prefix == nil {
+		return false
+	}
+
+	// Find the relevant node.
+	parent, node, _, leftover := trie.findSubtree(key)
+	if len(leftover) != 0 {
+		return false
+	}
+
+	// If the item is already set to nil, there is nothing to do.
+	if node.item == nil {
+		return false
+	}
+
+	// Delete the item.
+	node.item = nil
+
+	// Compact since that might be possible now.
+	if compacted := node.compact(); compacted != node {
+		if parent == nil {
+			*node = *compacted
+		} else {
+			parent.children.replace(node.prefix[0], compacted)
+			*parent = *parent.compact()
+		}
+	}
+
+	return true
+}
+
+// DeleteSubtree finds the subtree exactly matching prefix and deletes it.
+//
+// True is returned if the subtree was found and deleted.
+func (trie *Trie) DeleteSubtree(prefix Prefix) (deleted bool) {
+	// Nil prefix not allowed.
+	if prefix == nil {
+		panic(ErrNilPrefix)
+	}
+
+	// Empty trie must be handled explicitly.
+	if trie.prefix == nil {
+		return false
+	}
+
+	// Locate the relevant subtree.
+	parent, root, found, _ := trie.findSubtree(prefix)
+	if !found {
+		return false
+	}
+
+	// If we are in the root of the trie, reset the trie.
+	if parent == nil {
+		root.prefix = nil
+		root.children = newSparseChildList(trie.maxPrefixPerNode)
+		return true
+	}
+
+	// Otherwise remove the root node from its parent.
+	parent.children.remove(root)
+	return true
+}
+
+// Internal helper methods -----------------------------------------------------
+
+func (trie *Trie) put(key Prefix, item Item, replace bool) (inserted bool) {
+	// Nil prefix not allowed.
+	if key == nil {
+		panic(ErrNilPrefix)
+	}
+
+	var (
+		common int
+		node   *Trie = trie
+		child  *Trie
+	)
+
+	if node.prefix == nil {
+		if len(key) <= trie.maxPrefixPerNode {
+			node.prefix = key
+			goto InsertItem
+		}
+		node.prefix = key[:trie.maxPrefixPerNode]
+		key = key[trie.maxPrefixPerNode:]
+		goto AppendChild
+	}
+
+	for {
+		// Compute the longest common prefix length.
+		common = node.longestCommonPrefixLength(key)
+		key = key[common:]
+
+		// Only a part matches, split.
+		if common < len(node.prefix) {
+			goto SplitPrefix
+		}
+
+		// common == len(node.prefix) since never (common > len(node.prefix))
+		// common == len(former key) <-> 0 == len(key)
+		// -> former key == node.prefix
+		if len(key) == 0 {
+			goto InsertItem
+		}
+
+		// Check children for matching prefix.
+		child = node.children.next(key[0])
+		if child == nil {
+			goto AppendChild
+		}
+		node = child
+	}
+
+SplitPrefix:
+	// Split the prefix if necessary.
+	child = new(Trie)
+	*child = *node
+	*node = *NewTrie()
+	node.prefix = child.prefix[:common]
+	child.prefix = child.prefix[common:]
+	child = child.compact()
+	node.children = node.children.add(child)
+
+AppendChild:
+	// Keep appending children until whole prefix is inserted.
+	// This loop starts with empty node.prefix that needs to be filled.
+	for len(key) != 0 {
+		child := NewTrie()
+		if len(key) <= trie.maxPrefixPerNode {
+			child.prefix = key
+			node.children = node.children.add(child)
+			node = child
+			goto InsertItem
+		} else {
+			child.prefix = key[:trie.maxPrefixPerNode]
+			key = key[trie.maxPrefixPerNode:]
+			node.children = node.children.add(child)
+			node = child
+		}
+	}
+
+InsertItem:
+	// Try to insert the item if possible.
+	if replace || node.item == nil {
+		node.item = item
+		return true
+	}
+	return false
+}
+
+func (trie *Trie) compact() *Trie {
+	// Only a node with a single child can be compacted.
+	if trie.children.length() != 1 {
+		return trie
+	}
+
+	child := trie.children.head()
+
+	// If any item is set, we cannot compact since we want to retain
+	// the ability to do searching by key. This makes compaction less usable,
+	// but that simply cannot be avoided.
+	if trie.item != nil || child.item != nil {
+		return trie
+	}
+
+	// Make sure the combined prefixes fit into a single node.
+	if len(trie.prefix)+len(child.prefix) > trie.maxPrefixPerNode {
+		return trie
+	}
+
+	// Concatenate the prefixes, move the items.
+	child.prefix = append(trie.prefix, child.prefix...)
+	if trie.item != nil {
+		child.item = trie.item
+	}
+
+	return child
+}
+
+func (trie *Trie) findSubtree(prefix Prefix) (parent *Trie, root *Trie, found bool, leftover Prefix) {
+	// Find the subtree matching prefix.
+	root = trie
+	for {
+		// Compute what part of prefix matches.
+		common := root.longestCommonPrefixLength(prefix)
+		prefix = prefix[common:]
+
+		// We used up the whole prefix, subtree found.
+		if len(prefix) == 0 {
+			found = true
+			leftover = root.prefix[common:]
+			return
+		}
+
+		// Partial match means that there is no subtree matching prefix.
+		if common < len(root.prefix) {
+			leftover = root.prefix[common:]
+			return
+		}
+
+		// There is some prefix left, move to the children.
+		child := root.children.next(prefix[0])
+		if child == nil {
+			// There is nowhere to continue, there is no subtree matching prefix.
+			return
+		}
+
+		parent = root
+		root = child
+	}
+}
+
+func (trie *Trie) walk(actualRootPrefix Prefix, visitor VisitorFunc) error {
+	var prefix Prefix
+	// Allocate a bit more space for prefix at the beginning.
+	if actualRootPrefix == nil {
+		prefix = make(Prefix, 32+len(trie.prefix))
+		copy(prefix, trie.prefix)
+		prefix = prefix[:len(trie.prefix)]
+	} else {
+		prefix = make(Prefix, 32+len(actualRootPrefix))
+		copy(prefix, actualRootPrefix)
+		prefix = prefix[:len(actualRootPrefix)]
+	}
+
+	// Visit the root first. Not that this works for empty trie as well since
+	// in that case item == nil && len(children) == 0.
+	if trie.item != nil {
+		if err := visitor(prefix, trie.item); err != nil {
+			if err == SkipSubtree {
+				return nil
+			}
+			return err
+		}
+	}
+
+	// Then continue to the children.
+	return trie.children.walk(&prefix, visitor)
+}
+
+func (trie *Trie) longestCommonPrefixLength(prefix Prefix) (i int) {
+	for ; i < len(prefix) && i < len(trie.prefix) && prefix[i] == trie.prefix[i]; i++ {
+	}
+	return
+}
+
+// Errors ----------------------------------------------------------------------
+
+var (
+	SkipSubtree  = errors.New("Skip this subtree")
+	ErrNilPrefix = errors.New("Nil prefix passed into a method call")
+)

Godeps/_workspace/src/github.com/tchap/go-patricia/patricia/patricia_dense_test.go

@@ -0,0 +1,161 @@
+// Copyright (c) 2014 The go-patricia AUTHORS
+//
+// Use of this source code is governed by The MIT License
+// that can be found in the LICENSE file.
+
+package patricia
+
+import (
+	"testing"
+)
+
+// Tests -----------------------------------------------------------------------
+
+func TestTrie_InsertDense(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"aba", 0, success},
+		{"abb", 1, success},
+		{"abc", 2, success},
+		{"abd", 3, success},
+		{"abe", 4, success},
+		{"abf", 5, success},
+		{"abg", 6, success},
+		{"abh", 7, success},
+		{"abi", 8, success},
+		{"abj", 9, success},
+		{"abk", 0, success},
+		{"abl", 1, success},
+		{"abm", 2, success},
+		{"abn", 3, success},
+		{"abo", 4, success},
+		{"abp", 5, success},
+		{"abq", 6, success},
+		{"abr", 7, success},
+		{"abs", 8, success},
+		{"abt", 9, success},
+		{"abu", 0, success},
+		{"abv", 1, success},
+		{"abw", 2, success},
+		{"abx", 3, success},
+		{"aby", 4, success},
+		{"abz", 5, success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+}
+
+func TestTrie_InsertDensePreceeding(t *testing.T) {
+	trie := NewTrie()
+	start := byte(70)
+	// create a dense node
+	for i := byte(0); i <= DefaultMaxChildrenPerSparseNode; i++ {
+		if !trie.Insert(Prefix([]byte{start + i}), true) {
+			t.Errorf("insert failed, prefix=%v", start+i)
+		}
+	}
+	// insert some preceeding keys
+	for i := byte(1); i < start; i *= i + 1 {
+		if !trie.Insert(Prefix([]byte{start - i}), true) {
+			t.Errorf("insert failed, prefix=%v", start-i)
+		}
+	}
+}
+
+func TestTrie_InsertDenseDuplicatePrefixes(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"aba", 0, success},
+		{"abb", 1, success},
+		{"abc", 2, success},
+		{"abd", 3, success},
+		{"abe", 4, success},
+		{"abf", 5, success},
+		{"abg", 6, success},
+		{"abh", 7, success},
+		{"abi", 8, success},
+		{"abj", 9, success},
+		{"abk", 0, success},
+		{"abl", 1, success},
+		{"abm", 2, success},
+		{"abn", 3, success},
+		{"abo", 4, success},
+		{"abp", 5, success},
+		{"abq", 6, success},
+		{"abr", 7, success},
+		{"abs", 8, success},
+		{"abt", 9, success},
+		{"abu", 0, success},
+		{"abv", 1, success},
+		{"abw", 2, success},
+		{"abx", 3, success},
+		{"aby", 4, success},
+		{"abz", 5, success},
+		{"aba", 0, failure},
+		{"abb", 1, failure},
+		{"abc", 2, failure},
+		{"abd", 3, failure},
+		{"abe", 4, failure},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+}
+
+func TestTrie_DeleteDense(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"aba", 0, success},
+		{"abb", 1, success},
+		{"abc", 2, success},
+		{"abd", 3, success},
+		{"abe", 4, success},
+		{"abf", 5, success},
+		{"abg", 6, success},
+		{"abh", 7, success},
+		{"abi", 8, success},
+		{"abj", 9, success},
+		{"abk", 0, success},
+		{"abl", 1, success},
+		{"abm", 2, success},
+		{"abn", 3, success},
+		{"abo", 4, success},
+		{"abp", 5, success},
+		{"abq", 6, success},
+		{"abr", 7, success},
+		{"abs", 8, success},
+		{"abt", 9, success},
+		{"abu", 0, success},
+		{"abv", 1, success},
+		{"abw", 2, success},
+		{"abx", 3, success},
+		{"aby", 4, success},
+		{"abz", 5, success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	for _, v := range data {
+		t.Logf("DELETE word=%v, success=%v", v.key, v.retVal)
+		if ok := trie.Delete([]byte(v.key)); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+}

Godeps/_workspace/src/github.com/tchap/go-patricia/patricia/patricia_sparse_test.go

@@ -0,0 +1,659 @@
+// Copyright (c) 2014 The go-patricia AUTHORS
+//
+// Use of this source code is governed by The MIT License
+// that can be found in the LICENSE file.
+
+package patricia
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+)
+
+const (
+	success = true
+	failure = false
+)
+
+type testData struct {
+	key    string
+	value  interface{}
+	retVal bool
+}
+
+// Tests -----------------------------------------------------------------------
+
+func TestTrie_InsertDifferentPrefixes(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"Pepaneeeeeeeeeeeeee", "Pepan Zdepan", success},
+		{"Honzooooooooooooooo", "Honza Novak", success},
+		{"Jenikuuuuuuuuuuuuuu", "Jenik Poustevnicek", success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+}
+
+func TestTrie_InsertDuplicatePrefixes(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"Pepan", "Pepan Zdepan", success},
+		{"Pepan", "Pepan Zdepan", failure},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+}
+
+func TestTrie_InsertVariousPrefixes(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"Pepan", "Pepan Zdepan", success},
+		{"Pepin", "Pepin Omacka", success},
+		{"Honza", "Honza Novak", success},
+		{"Jenik", "Jenik Poustevnicek", success},
+		{"Pepan", "Pepan Dupan", failure},
+		{"Karel", "Karel Pekar", success},
+		{"Jenik", "Jenik Poustevnicek", failure},
+		{"Pepanek", "Pepanek Zemlicka", success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+}
+
+func TestTrie_InsertAndMatchPrefix(t *testing.T) {
+	trie := NewTrie()
+	t.Log("INSERT prefix=by week")
+	trie.Insert(Prefix("by week"), 2)
+	t.Log("INSERT prefix=by")
+	trie.Insert(Prefix("by"), 1)
+
+	if !trie.Match(Prefix("by")) {
+		t.Error("MATCH prefix=by, expected=true, got=false")
+	}
+}
+
+func TestTrie_SetGet(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"Pepan", "Pepan Zdepan", success},
+		{"Pepin", "Pepin Omacka", success},
+		{"Honza", "Honza Novak", success},
+		{"Jenik", "Jenik Poustevnicek", success},
+		{"Pepan", "Pepan Dupan", failure},
+		{"Karel", "Karel Pekar", success},
+		{"Jenik", "Jenik Poustevnicek", failure},
+		{"Pepanek", "Pepanek Zemlicka", success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	for _, v := range data {
+		t.Logf("SET %q to 10", v.key)
+		trie.Set(Prefix(v.key), 10)
+	}
+
+	for _, v := range data {
+		value := trie.Get(Prefix(v.key))
+		t.Logf("GET %q => %v", v.key, value)
+		if value.(int) != 10 {
+			t.Errorf("Unexpected return value, %v != 10", value)
+		}
+	}
+
+	if value := trie.Get(Prefix("random crap")); value != nil {
+		t.Errorf("Unexpected return value, %v != <nil>", value)
+	}
+}
+
+func TestTrie_Match(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"Pepan", "Pepan Zdepan", success},
+		{"Pepin", "Pepin Omacka", success},
+		{"Honza", "Honza Novak", success},
+		{"Jenik", "Jenik Poustevnicek", success},
+		{"Pepan", "Pepan Dupan", failure},
+		{"Karel", "Karel Pekar", success},
+		{"Jenik", "Jenik Poustevnicek", failure},
+		{"Pepanek", "Pepanek Zemlicka", success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	for _, v := range data {
+		matched := trie.Match(Prefix(v.key))
+		t.Logf("MATCH %q => %v", v.key, matched)
+		if !matched {
+			t.Errorf("Inserted key %q was not matched", v.key)
+		}
+	}
+
+	if trie.Match(Prefix("random crap")) {
+		t.Errorf("Key that was not inserted matched: %q", "random crap")
+	}
+}
+
+func TestTrie_MatchFalsePositive(t *testing.T) {
+	trie := NewTrie()
+
+	if ok := trie.Insert(Prefix("A"), 1); !ok {
+		t.Fatal("INSERT prefix=A, item=1 not ok")
+	}
+
+	resultMatchSubtree := trie.MatchSubtree(Prefix("A extra"))
+	resultMatch := trie.Match(Prefix("A extra"))
+
+	if resultMatchSubtree != false {
+		t.Error("MatchSubtree returned false positive")
+	}
+
+	if resultMatch != false {
+		t.Error("Match returned false positive")
+	}
+}
+
+func TestTrie_MatchSubtree(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"Pepan", "Pepan Zdepan", success},
+		{"Pepin", "Pepin Omacka", success},
+		{"Honza", "Honza Novak", success},
+		{"Jenik", "Jenik Poustevnicek", success},
+		{"Pepan", "Pepan Dupan", failure},
+		{"Karel", "Karel Pekar", success},
+		{"Jenik", "Jenik Poustevnicek", failure},
+		{"Pepanek", "Pepanek Zemlicka", success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	for _, v := range data {
+		key := Prefix(v.key[:3])
+		matched := trie.MatchSubtree(key)
+		t.Logf("MATCH_SUBTREE %q => %v", key, matched)
+		if !matched {
+			t.Errorf("Subtree %q was not matched", v.key)
+		}
+	}
+}
+
+func TestTrie_Visit(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"Pepa", 0, success},
+		{"Pepa Zdepa", 1, success},
+		{"Pepa Kuchar", 2, success},
+		{"Honza", 3, success},
+		{"Jenik", 4, success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal {
+			t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	if err := trie.Visit(func(prefix Prefix, item Item) error {
+		name := data[item.(int)].key
+		t.Logf("VISITING prefix=%q, item=%v", prefix, item)
+		if !strings.HasPrefix(string(prefix), name) {
+			t.Errorf("Unexpected prefix encountered, %q not a prefix of %q", prefix, name)
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestTrie_VisitSkipSubtree(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"Pepa", 0, success},
+		{"Pepa Zdepa", 1, success},
+		{"Pepa Kuchar", 2, success},
+		{"Honza", 3, success},
+		{"Jenik", 4, success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal {
+			t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	if err := trie.Visit(func(prefix Prefix, item Item) error {
+		t.Logf("VISITING prefix=%q, item=%v", prefix, item)
+		if item.(int) == 0 {
+			t.Logf("SKIP %q", prefix)
+			return SkipSubtree
+		}
+		if strings.HasPrefix(string(prefix), "Pepa") {
+			t.Errorf("Unexpected prefix encountered, %q", prefix)
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestTrie_VisitReturnError(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"Pepa", 0, success},
+		{"Pepa Zdepa", 1, success},
+		{"Pepa Kuchar", 2, success},
+		{"Honza", 3, success},
+		{"Jenik", 4, success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal {
+			t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	someErr := errors.New("Something exploded")
+	if err := trie.Visit(func(prefix Prefix, item Item) error {
+		t.Logf("VISITING prefix=%q, item=%v", prefix, item)
+		if item.(int) == 3 {
+			return someErr
+		}
+		if item.(int) != 3 {
+			t.Errorf("Unexpected prefix encountered, %q", prefix)
+		}
+		return nil
+	}); err != nil && err != someErr {
+		t.Fatal(err)
+	}
+}
+
+func TestTrie_VisitSubtree(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"Pepa", 0, success},
+		{"Pepa Zdepa", 1, success},
+		{"Pepa Kuchar", 2, success},
+		{"Honza", 3, success},
+		{"Jenik", 4, success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal {
+			t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	var counter int
+	subtreePrefix := []byte("Pep")
+	t.Log("VISIT Pep")
+	if err := trie.VisitSubtree(subtreePrefix, func(prefix Prefix, item Item) error {
+		t.Logf("VISITING prefix=%q, item=%v", prefix, item)
+		if !bytes.HasPrefix(prefix, subtreePrefix) {
+			t.Errorf("Unexpected prefix encountered, %q does not extend %q",
+				prefix, subtreePrefix)
+		}
+		if len(prefix) > len(data[item.(int)].key) {
+			t.Fatalf("Something is rather fishy here, prefix=%q", prefix)
+		}
+		counter++
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	if counter != 3 {
+		t.Error("Unexpected number of nodes visited")
+	}
+}
+
+func TestTrie_VisitPrefixes(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"P", 0, success},
+		{"Pe", 1, success},
+		{"Pep", 2, success},
+		{"Pepa", 3, success},
+		{"Pepa Zdepa", 4, success},
+		{"Pepa Kuchar", 5, success},
+		{"Honza", 6, success},
+		{"Jenik", 7, success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal {
+			t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	var counter int
+	word := []byte("Pepa")
+	if err := trie.VisitPrefixes(word, func(prefix Prefix, item Item) error {
+		t.Logf("VISITING prefix=%q, item=%v", prefix, item)
+		if !bytes.HasPrefix(word, prefix) {
+			t.Errorf("Unexpected prefix encountered, %q is not a prefix of %q",
+				prefix, word)
+		}
+		counter++
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	if counter != 4 {
+		t.Error("Unexpected number of nodes visited")
+	}
+}
+
+func TestParticiaTrie_Delete(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"Pepan", "Pepan Zdepan", success},
+		{"Honza", "Honza Novak", success},
+		{"Jenik", "Jenik Poustevnicek", success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal {
+			t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	for _, v := range data {
+		t.Logf("DELETE word=%v, success=%v", v.key, v.retVal)
+		if ok := trie.Delete([]byte(v.key)); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+}
+
+func TestParticiaTrie_DeleteNonExistent(t *testing.T) {
+	trie := NewTrie()
+
+	insertData := []testData{
+		{"Pepan", "Pepan Zdepan", success},
+		{"Honza", "Honza Novak", success},
+		{"Jenik", "Jenik Poustevnicek", success},
+	}
+	deleteData := []testData{
+		{"Pepan", "Pepan Zdepan", success},
+		{"Honza", "Honza Novak", success},
+		{"Pepan", "Pepan Zdepan", failure},
+		{"Jenik", "Jenik Poustevnicek", success},
+		{"Honza", "Honza Novak", failure},
+	}
+
+	for _, v := range insertData {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal {
+			t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	for _, v := range deleteData {
+		t.Logf("DELETE word=%v, success=%v", v.key, v.retVal)
+		if ok := trie.Delete([]byte(v.key)); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+}
+
+func TestParticiaTrie_DeleteSubtree(t *testing.T) {
+	trie := NewTrie()
+
+	insertData := []testData{
+		{"P", 0, success},
+		{"Pe", 1, success},
+		{"Pep", 2, success},
+		{"Pepa", 3, success},
+		{"Pepa Zdepa", 4, success},
+		{"Pepa Kuchar", 5, success},
+		{"Honza", 6, success},
+		{"Jenik", 7, success},
+	}
+	deleteData := []testData{
+		{"Pe", -1, success},
+		{"Pe", -1, failure},
+		{"Honzik", -1, failure},
+		{"Honza", -1, success},
+		{"Honza", -1, failure},
+		{"Pep", -1, failure},
+		{"P", -1, success},
+		{"Nobody", -1, failure},
+		{"", -1, success},
+	}
+
+	for _, v := range insertData {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal {
+			t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	for _, v := range deleteData {
+		t.Logf("DELETE_SUBTREE prefix=%v, success=%v", v.key, v.retVal)
+		if ok := trie.DeleteSubtree([]byte(v.key)); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+}
+
+/*
+func TestTrie_Dump(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"Honda", nil, success},
+		{"Honza", nil, success},
+		{"Jenik", nil, success},
+		{"Pepan", nil, success},
+		{"Pepin", nil, success},
+	}
+
+	for i, v := range data {
+		if _, ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal {
+			t.Logf("INSERT %v %v", v.key, v.value)
+			t.Fatalf("Unexpected return value, expected=%v, got=%v", i, ok)
+		}
+	}
+
+	dump := `
++--+--+ Hon +--+--+ da
+   |           |
+   |           +--+ za
+   |
+   +--+ Jenik
+   |
+   +--+ Pep +--+--+ an
+               |
+               +--+ in
+`
+
+	var buf bytes.Buffer
+	trie.Dump(buf)
+
+	if !bytes.Equal(buf.Bytes(), dump) {
+		t.Logf("DUMP")
+		t.Fatalf("Unexpected dump generated, expected\n\n%v\ngot\n\n%v", dump, buf.String())
+	}
+}
+*/
+
+func TestTrie_compact(t *testing.T) {
+	trie := NewTrie()
+
+	trie.Insert(Prefix("a"), 0)
+	trie.Insert(Prefix("ab"), 0)
+	trie.Insert(Prefix("abc"), 0)
+	trie.Insert(Prefix("abcd"), 0)
+	trie.Insert(Prefix("abcde"), 0)
+	trie.Insert(Prefix("abcdef"), 0)
+	trie.Insert(Prefix("abcdefg"), 0)
+	trie.Insert(Prefix("abcdefgi"), 0)
+	trie.Insert(Prefix("abcdefgij"), 0)
+	trie.Insert(Prefix("abcdefgijk"), 0)
+
+	trie.Delete(Prefix("abcdef"))
+	trie.Delete(Prefix("abcde"))
+	trie.Delete(Prefix("abcdefg"))
+
+	trie.Delete(Prefix("a"))
+	trie.Delete(Prefix("abc"))
+	trie.Delete(Prefix("ab"))
+
+	trie.Visit(func(prefix Prefix, item Item) error {
+		// 97 ~~ 'a',
+		for ch := byte(97); ch <= 107; ch++ {
+			if c := bytes.Count(prefix, []byte{ch}); c > 1 {
+				t.Errorf("%q appeared in %q %v times", ch, prefix, c)
+			}
+		}
+		return nil
+	})
+}
+
+func TestTrie_longestCommonPrefixLenght(t *testing.T) {
+	trie := NewTrie()
+	trie.prefix = []byte("1234567890")
+
+	switch {
+	case trie.longestCommonPrefixLength([]byte("")) != 0:
+		t.Fail()
+	case trie.longestCommonPrefixLength([]byte("12345")) != 5:
+		t.Fail()
+	case trie.longestCommonPrefixLength([]byte("123789")) != 3:
+		t.Fail()
+	case trie.longestCommonPrefixLength([]byte("12345678901")) != 10:
+		t.Fail()
+	}
+}
+
+// Examples --------------------------------------------------------------------
+
+func ExampleTrie() {
+	// Create a new tree.
+	trie := NewTrie()
+
+	// Insert some items.
+	trie.Insert(Prefix("Pepa Novak"), 1)
+	trie.Insert(Prefix("Pepa Sindelar"), 2)
+	trie.Insert(Prefix("Karel Macha"), 3)
+	trie.Insert(Prefix("Karel Hynek Macha"), 4)
+
+	// Just check if some things are present in the tree.
+	key := Prefix("Pepa Novak")
+	fmt.Printf("%q present? %v\n", key, trie.Match(key))
+	key = Prefix("Karel")
+	fmt.Printf("Anybody called %q here? %v\n", key, trie.MatchSubtree(key))
+
+	// Walk the tree.
+	trie.Visit(printItem)
+	// "Karel Hynek Macha": 4
+	// "Karel Macha": 3
+	// "Pepa Novak": 1
+	// "Pepa Sindelar": 2
+
+	// Walk a subtree.
+	trie.VisitSubtree(Prefix("Pepa"), printItem)
+	// "Pepa Novak": 1
+	// "Pepa Sindelar": 2
+
+	// Modify an item, then fetch it from the tree.
+	trie.Set(Prefix("Karel Hynek Macha"), 10)
+	key = Prefix("Karel Hynek Macha")
+	fmt.Printf("%q: %v\n", key, trie.Get(key))
+	// "Karel Hynek Macha": 10
+
+	// Walk prefixes.
+	prefix := Prefix("Karel Hynek Macha je kouzelnik")
+	trie.VisitPrefixes(prefix, printItem)
+	// "Karel Hynek Macha": 10
+
+	// Delete some items.
+	trie.Delete(Prefix("Pepa Novak"))
+	trie.Delete(Prefix("Karel Macha"))
+
+	// Walk again.
+	trie.Visit(printItem)
+	// "Karel Hynek Macha": 10
+	// "Pepa Sindelar": 2
+
+	// Delete a subtree.
+	trie.DeleteSubtree(Prefix("Pepa"))
+
+	// Print what is left.
+	trie.Visit(printItem)
+	// "Karel Hynek Macha": 10
+
+	// Output:
+	// "Pepa Novak" present? true
+	// Anybody called "Karel" here? true
+	// "Karel Hynek Macha": 4
+	// "Karel Macha": 3
+	// "Pepa Novak": 1
+	// "Pepa Sindelar": 2
+	// "Pepa Novak": 1
+	// "Pepa Sindelar": 2
+	// "Karel Hynek Macha": 10
+	// "Karel Hynek Macha": 10
+	// "Karel Hynek Macha": 10
+	// "Pepa Sindelar": 2
+	// "Karel Hynek Macha": 10
+}
+
+// Helpers ---------------------------------------------------------------------
+
+func printItem(prefix Prefix, item Item) error {
+	fmt.Printf("%q: %v\n", prefix, item)
+	return nil
+}

Godeps/_workspace/src/github.com/tchap/go-patricia/patricia/patricia_test.go

@@ -0,0 +1,92 @@
+// Copyright (c) 2014 The go-patricia AUTHORS
+//
+// Use of this source code is governed by The MIT License
+// that can be found in the LICENSE file.
+
+package patricia
+
+import (
+	"crypto/rand"
+	"reflect"
+	"testing"
+)
+
+// Tests -----------------------------------------------------------------------
+
+func TestTrie_ConstructorOptions(t *testing.T) {
+	trie := NewTrie(MaxPrefixPerNode(16), MaxChildrenPerSparseNode(10))
+
+	if trie.maxPrefixPerNode != 16 {
+		t.Errorf("Unexpected trie.maxPrefixPerNode value, expected=%v, got=%v",
+			16, trie.maxPrefixPerNode)
+	}
+
+	if trie.maxChildrenPerSparseNode != 10 {
+		t.Errorf("Unexpected trie.maxChildrenPerSparseNode value, expected=%v, got=%v",
+			10, trie.maxChildrenPerSparseNode)
+	}
+}
+
+func TestTrie_GetNonexistentPrefix(t *testing.T) {
+	trie := NewTrie()
+
+	data := []testData{
+		{"aba", 0, success},
+	}
+
+	for _, v := range data {
+		t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal)
+		if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal {
+			t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok)
+		}
+	}
+
+	t.Logf("GET prefix=baa, expect item=nil")
+	if item := trie.Get(Prefix("baa")); item != nil {
+		t.Errorf("Unexpected return value, expected=<nil>, got=%v", item)
+	}
+}
+
+func TestTrie_RandomKitchenSink(t *testing.T) {
+	if testing.Short() {
+		t.Skip()
+	}
+	const count, size = 750000, 16
+	b := make([]byte, count+size+1)
+	if _, err := rand.Read(b); err != nil {
+		t.Fatal("error generating random bytes", err)
+	}
+	m := make(map[string]string)
+	for i := 0; i < count; i++ {
+		m[string(b[i:i+size])] = string(b[i+1 : i+size+1])
+	}
+	trie := NewTrie()
+	getAndDelete := func(k, v string) {
+		i := trie.Get(Prefix(k))
+		if i == nil {
+			t.Fatalf("item not found, prefix=%v", []byte(k))
+		} else if s, ok := i.(string); !ok {
+			t.Fatalf("unexpected item type, expecting=%v, got=%v", reflect.TypeOf(k), reflect.TypeOf(i))
+		} else if s != v {
+			t.Fatalf("unexpected item, expecting=%v, got=%v", []byte(k), []byte(s))
+		} else if !trie.Delete(Prefix(k)) {
+			t.Fatalf("delete failed, prefix=%v", []byte(k))
+		} else if i = trie.Get(Prefix(k)); i != nil {
+			t.Fatalf("unexpected item, expecting=<nil>, got=%v", i)
+		} else if trie.Delete(Prefix(k)) {
+			t.Fatalf("extra delete succeeded, prefix=%v", []byte(k))
+		}
+	}
+	for k, v := range m {
+		if !trie.Insert(Prefix(k), v) {
+			t.Fatalf("insert failed, prefix=%v", []byte(k))
+		}
+		if byte(k[size/2]) < 128 {
+			getAndDelete(k, v)
+			delete(m, k)
+		}
+	}
+	for k, v := range m {
+		getAndDelete(k, v)
+	}
+}

cp-main.go

@@ -24,7 +24,6 @@ import (
 	"math"
 	"os"
 	"runtime"
-	"strings"
 	"sync"
 
 	"github.com/minio/cli"
@@ -143,7 +142,7 @@ func doPrepareCopyURLs(session *sessionV2, trapCh <-chan bool) {
 
 	// Create a session data file to store the processed URLs.
 	dataFP := session.NewDataWriter()
-	scanBar := scanBarFactory(strings.Join(sourceURLs, " "))
+	scanBar := scanBarFactory("")
 	URLsCh := prepareCopyURLs(sourceURLs, targetURL)
 	done := false
 

diff-main.go

@@ -90,65 +90,12 @@ func runDiffCmd(ctx *cli.Context) {
 		}
 		console.Infoln(diff.message)
 	}
-}
+	console.Println()
-
-func doDiffInRoutine(firstURL, secondURL string, recursive bool, ch chan diff) {
-	defer close(ch)
-	_, firstContent, err := url2Stat(firstURL)
-	if err != nil {
-		ch <- diff{
-			message: "Failed to stat ‘" + firstURL + "’",
-			err:     NewIodine(iodine.New(err, nil)),
-		}
-		return
-	}
-	_, secondContent, err := url2Stat(secondURL)
-	if err != nil {
-		ch <- diff{
-			message: "Failed to stat ‘" + secondURL + "’",
-			err:     NewIodine(iodine.New(err, nil)),
-		}
-		return
-	}
-	if firstContent.Type.IsRegular() {
-		switch {
-		case secondContent.Type.IsDir():
-			newSecondURL, err := urlJoinPath(secondURL, firstURL)
-			if err != nil {
-				ch <- diff{
-					message: "Unable to construct new URL from ‘" + secondURL + "’ using ‘" + firstURL,
-					err:     NewIodine(iodine.New(err, nil)),
-				}
-				return
-			}
-			doDiffObjects(firstURL, newSecondURL, ch)
-		case !secondContent.Type.IsRegular():
-			ch <- diff{
-				message: "‘" + firstURL + "’ and " + "‘" + secondURL + "’ differs in type.",
-				err:     nil,
-			}
-			return
-		case secondContent.Type.IsRegular():
-			doDiffObjects(firstURL, secondURL, ch)
-		}
-	}
-	if firstContent.Type.IsDir() {
-		switch {
-		case !secondContent.Type.IsDir():
-			ch <- diff{
-				message: "‘" + firstURL + "’ and " + "‘" + secondURL + "’ differs in type.",
-				err:     nil,
-			}
-			return
-		default:
-			doDiffDirs(firstURL, secondURL, recursive, ch)
-		}
-	}
 }
 
 // doDiffCmd - Execute the diff command
 func doDiffCmd(firstURL, secondURL string, recursive bool) <-chan diff {
-	ch := make(chan diff)
+	ch := make(chan diff, 10000)
 	go doDiffInRoutine(firstURL, secondURL, recursive, ch)
 	return ch
 }

diff.go

@@ -18,9 +18,12 @@ package main
 
 import (
 	"path/filepath"
+	"strings"
+	"sync"
 
 	"github.com/minio/mc/pkg/client"
 	"github.com/minio/minio/pkg/iodine"
+	"github.com/tchap/go-patricia/patricia"
 )
 
 //
@@ -58,6 +61,60 @@ func urlJoinPath(url1, url2 string) (newURLStr string, err error) {
 	return newURLStr, nil
 }
 
+func doDiffInRoutine(firstURL, secondURL string, recursive bool, ch chan diff) {
+	defer close(ch)
+	firstClnt, firstContent, err := url2Stat(firstURL)
+	if err != nil {
+		ch <- diff{
+			message: "Failed to stat ‘" + firstURL + "’",
+			err:     NewIodine(iodine.New(err, nil)),
+		}
+		return
+	}
+	secondClnt, secondContent, err := url2Stat(secondURL)
+	if err != nil {
+		ch <- diff{
+			message: "Failed to stat ‘" + secondURL + "’",
+			err:     NewIodine(iodine.New(err, nil)),
+		}
+		return
+	}
+	if firstContent.Type.IsRegular() {
+		switch {
+		case secondContent.Type.IsDir():
+			newSecondURL, err := urlJoinPath(secondURL, firstURL)
+			if err != nil {
+				ch <- diff{
+					message: "Unable to construct new URL from ‘" + secondURL + "’ using ‘" + firstURL,
+					err:     NewIodine(iodine.New(err, nil)),
+				}
+				return
+			}
+			doDiffObjects(firstURL, newSecondURL, ch)
+		case !secondContent.Type.IsRegular():
+			ch <- diff{
+				message: "‘" + firstURL + "’ and " + "‘" + secondURL + "’ differs in type.",
+				err:     nil,
+			}
+			return
+		case secondContent.Type.IsRegular():
+			doDiffObjects(firstURL, secondURL, ch)
+		}
+	}
+	if firstContent.Type.IsDir() {
+		switch {
+		case !secondContent.Type.IsDir():
+			ch <- diff{
+				message: "‘" + firstURL + "’ and " + "‘" + secondURL + "’ differs in type.",
+				err:     nil,
+			}
+			return
+		default:
+			doDiffDirs(firstClnt, secondClnt, recursive, ch)
+		}
+	}
+}
+
 // doDiffObjects - Diff two object URLs
 func doDiffObjects(firstURL, secondURL string, ch chan diff) {
 	_, firstContent, errFirst := url2Stat(firstURL)
@@ -104,27 +161,27 @@ func doDiffObjects(firstURL, secondURL string, ch chan diff) {
 	}
 }
 
-func dodiffdirs(firstClnt client.Client, firstURL, secondURL string, recursive bool, ch chan diff) {
+func dodiff(firstClnt, secondClnt client.Client, ch chan diff) {
-	for contentCh := range firstClnt.List(recursive) {
+	for contentCh := range firstClnt.List(false) {
 		if contentCh.Err != nil {
 			ch <- diff{
-				message: "Failed to list ‘" + firstURL + "’",
+				message: "Failed to list ‘" + firstClnt.URL().String() + "’",
 				err:     NewIodine(iodine.New(contentCh.Err, nil)),
 			}
 			return
 		}
-		newFirstURL, err := urlJoinPath(firstURL, contentCh.Content.Name)
+		newFirstURL, err := urlJoinPath(firstClnt.URL().String(), contentCh.Content.Name)
 		if err != nil {
 			ch <- diff{
-				message: "Unable to construct new URL from ‘" + firstURL + "’ using ‘" + contentCh.Content.Name + "’",
+				message: "Unable to construct new URL from ‘" + firstClnt.URL().String() + "’ using ‘" + contentCh.Content.Name + "’",
 				err:     NewIodine(iodine.New(err, nil)),
 			}
 			return
 		}
-		newSecondURL, err := urlJoinPath(secondURL, contentCh.Content.Name)
+		newSecondURL, err := urlJoinPath(secondClnt.URL().String(), contentCh.Content.Name)
 		if err != nil {
 			ch <- diff{
-				message: "Unable to construct new URL from ‘" + secondURL + "’ using ‘" + contentCh.Content.Name + "’",
+				message: "Unable to construct new URL from ‘" + secondClnt.URL().String() + "’ using ‘" + contentCh.Content.Name + "’",
 				err:     NewIodine(iodine.New(err, nil)),
 			}
 			return
@@ -134,13 +191,13 @@ func dodiffdirs(firstClnt client.Client, firstURL, secondURL string, recursive b
 		switch {
 		case errFirst != nil && errSecond == nil:
 			ch <- diff{
-				message: "‘" + newSecondURL + "’ Only in ‘" + secondURL + "’",
+				message: "‘" + newSecondURL + "’ Only in ‘" + secondClnt.URL().String() + "’",
 				err:     nil,
 			}
 			continue
 		case errFirst == nil && errSecond != nil:
 			ch <- diff{
-				message: "‘" + newFirstURL + "’ Only in ‘" + firstURL + "’",
+				message: "‘" + newFirstURL + "’ Only in ‘" + firstClnt.URL().String() + "’",
 				err:     nil,
 			}
 			continue
@@ -168,38 +225,85 @@ func dodiffdirs(firstClnt client.Client, firstURL, secondURL string, recursive b
 	} // End of for-loop
 }
 
-// doDiffDirs - Diff two Dir URLs
+func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan diff) {
-func doDiffDirs(firstURL, secondURL string, recursive bool, ch chan diff) {
+	firstTrie := patricia.NewTrie()
-	firstClnt, firstContent, err := url2Stat(firstURL)
+	secondTrie := patricia.NewTrie()
+	wg := new(sync.WaitGroup)
+
+	wg.Add(1)
+	go func(ch chan<- diff) {
+		defer wg.Done()
+		for firstContentCh := range firstClnt.List(true) {
+			if firstContentCh.Err != nil {
+				ch <- diff{
+					message: "Failed to list ‘" + firstClnt.URL().String() + "’",
+					err:     NewIodine(iodine.New(firstContentCh.Err, nil)),
+				}
+				return
+			}
+			firstURLDelimited := firstClnt.URL().String()[:strings.LastIndex(firstClnt.URL().String(), string(firstClnt.URL().Separator))+1]
+			newFirstURL := firstURLDelimited + firstContentCh.Content.Name
+			newFirstURLParse, err := client.Parse(newFirstURL)
 			if err != nil {
 				ch <- diff{
-			message: "Failed to stat ‘" + firstURL + "’",
+					message: "Unable to construct new URL from ‘" + firstClnt.URL().String() + "’ using ‘" + firstContentCh.Content.Name + "’",
 					err:     NewIodine(iodine.New(err, nil)),
 				}
 				return
 			}
-	_, secondContent, err := url2Stat(secondURL)
+			firstTrie.Insert(patricia.Prefix(newFirstURLParse.String()), struct{}{})
+		}
+	}(ch)
+	wg.Add(1)
+	go func(ch chan<- diff) {
+		defer wg.Done()
+		for secondContentCh := range secondClnt.List(true) {
+			if secondContentCh.Err != nil {
+				ch <- diff{
+					message: "Failed to list ‘" + secondClnt.URL().String() + "’",
+					err:     NewIodine(iodine.New(secondContentCh.Err, nil)),
+				}
+				return
+			}
+			secondURLDelimited := secondClnt.URL().String()[:strings.LastIndex(secondClnt.URL().String(), string(secondClnt.URL().Separator))+1]
+			newSecondURL := secondURLDelimited + secondContentCh.Content.Name
+			newSecondURLParse, err := client.Parse(newSecondURL)
 			if err != nil {
 				ch <- diff{
-			message: "Failed to stat ‘" + secondURL + "’",
+					message: "Unable to construct new URL from ‘" + secondClnt.URL().String() + "’ using ‘" + secondContentCh.Content.Name + "’",
 					err:     NewIodine(iodine.New(err, nil)),
 				}
 				return
 			}
-	switch {
+			secondTrie.Insert(patricia.Prefix(newSecondURLParse.String()), struct{}{})
-	case firstContent.Type.IsDir():
+		}
-		if !secondContent.Type.IsDir() {
+	}(ch)
+	wg.Wait()
+
+	matchURLCh := make(chan string, 10000)
+	go func(matchURLCh chan<- string) {
+		itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
+			matchURLCh <- string(prefix)
+			return nil
+		}
+		firstTrie.Visit(itemFunc)
+		defer close(matchURLCh)
+	}(matchURLCh)
+	for matchURL := range matchURLCh {
+		if !secondTrie.Match(patricia.Prefix(matchURL)) {
 			ch <- diff{
-				message: firstURL + " and " + secondURL + " differs in type.",
+				message: "‘" + matchURL + "’ Only in ‘" + firstClnt.URL().String() + "’",
 				err:     nil,
 			}
 		}
-	default:
-		ch <- diff{
-			message: "‘" + firstURL + "’ is not an object. Please report this bug with ‘--debug’ option.",
-			err:     NewIodine(iodine.New(errNotAnObject{url: firstURL}, nil)),
 	}
+}
+
+// doDiffDirs - Diff two Dir URLs
+func doDiffDirs(firstClnt, secondClnt client.Client, recursive bool, ch chan diff) {
+	if recursive {
+		dodiffRecursive(firstClnt, secondClnt, ch)
 		return
 	}
-	dodiffdirs(firstClnt, firstURL, secondURL, recursive, ch)
+	dodiff(firstClnt, secondClnt, ch)
 }

mirror-main.go

@@ -141,7 +141,7 @@ func doPrepareCastURLs(session *sessionV2, trapCh <-chan bool) {
 	// Create a session data file to store the processed URLs.
 	dataFP := session.NewDataWriter()
 
-	scanBar := scanBarFactory(sourceURL)
+	scanBar := scanBarFactory("")
 	URLsCh := prepareCastURLs(sourceURL, targetURLs)
 	done := false
 	for done == false {

pb.go

@@ -231,6 +231,9 @@ func scanBarFactory(prefix string) scanBarFunc {
 
 	return func(source string) {
 		scanPrefix := fmt.Sprintf("[%s] %s ", humanize.Comma(int64(fileCount)), string(<-cursorCh))
+		if prefix != "" {
+			scanPrefix = fmt.Sprintf("Scanning %s [%s] %s ", prefix, humanize.Comma(int64(fileCount)), string(<-cursorCh))
+		}
 		if prevLineSize != 0 { // erase previous line
 			console.PrintC("\r" + scanPrefix + strings.Repeat(" ", prevLineSize-len([]rune(scanPrefix))))
 		}