From 1a7b3e8e089b5f0b71ab94c87c7e0c760a0696a1 Mon Sep 17 00:00:00 2001 From: evan-gu Date: Sun, 8 Sep 2013 21:14:31 -0400 Subject: [PATCH 1/3] add sorting and its test case --- file_system/event.go | 13 ++++++++ file_system/file_system.go | 6 +++- file_system/file_system_test.go | 54 +++++++++++++++++++++++++-------- store/test.go | 2 +- 4 files changed, 61 insertions(+), 14 deletions(-) diff --git a/file_system/event.go b/file_system/event.go index 1b2a00dbe..16ea275a2 100644 --- a/file_system/event.go +++ b/file_system/event.go @@ -36,6 +36,19 @@ type KeyValuePair struct { KVPairs []KeyValuePair `json:"kvs,omitempty"` } +// interfaces for sort +func (e Event) Len() int { + return len(e.KVPairs) +} + +func (e Event) Less(i, j int) bool { + return e.KVPairs[i].Key < e.KVPairs[j].Key +} + +func (e Event) Swap(i, j int) { + e.KVPairs[i], e.KVPairs[j] = e.KVPairs[j], e.KVPairs[i] +} + func newEvent(action string, key string, index uint64, term uint64) *Event { return &Event{ Action: action, diff --git a/file_system/file_system.go b/file_system/file_system.go index d9e3e20a9..d2735966c 100644 --- a/file_system/file_system.go +++ b/file_system/file_system.go @@ -3,6 +3,7 @@ package fileSystem import ( "fmt" "path" + "sort" "strings" "time" @@ -25,7 +26,7 @@ func New() *FileSystem { } -func (fs *FileSystem) Get(nodePath string, recusive bool, index uint64, term uint64) (*Event, error) { +func (fs *FileSystem) Get(nodePath string, recusive, sorting bool, index uint64, term uint64) (*Event, error) { n, err := fs.InternalGet(nodePath, index, term) if err != nil { @@ -57,6 +58,9 @@ func (fs *FileSystem) Get(nodePath string, recusive bool, index uint64, term uin // eliminate hidden nodes e.KVPairs = e.KVPairs[:i] + if sorting { + sort.Sort(e) + } } else { // node is file e.Value = n.Value } diff --git a/file_system/file_system_test.go b/file_system/file_system_test.go index 4c70b35bb..caa100451 100644 --- a/file_system/file_system_test.go +++ b/file_system/file_system_test.go @@ -1,6 +1,7 @@ package fileSystem import ( + "store" "testing" "time" ) @@ -34,7 +35,7 @@ func TestCreateAndGet(t *testing.T) { t.Fatal("Cannot create /fooDir") } - e, err := fs.Get("/fooDir", false, 3, 1) + e, err := fs.Get("/fooDir", false, false, 3, 1) if err != nil || e.Dir != true { t.Fatal("Cannot create /fooDir ") @@ -64,7 +65,7 @@ func TestUpdateFile(t *testing.T) { t.Fatalf("cannot update %s=barbar [%s]", "/foo/bar", err.Error()) } - e, err := fs.Get("/foo/bar", false, 2, 1) + e, err := fs.Get("/foo/bar", false, false, 2, 1) if err != nil { t.Fatalf("cannot get %s [%s]", "/foo/bar", err.Error()) @@ -106,7 +107,7 @@ func TestUpdateFile(t *testing.T) { // sleep 50ms, it should still reach the node time.Sleep(time.Microsecond * 50) - e, err = fs.Get("/foo/foo", true, 7, 1) + e, err = fs.Get("/foo/foo", true, false, 7, 1) if err != nil || e.Key != "/foo/foo" { t.Fatalf("cannot get dir before expiration [%s]", err.Error()) @@ -126,23 +127,23 @@ func TestUpdateFile(t *testing.T) { // wait for expiration time.Sleep(time.Second * 3) - e, err = fs.Get("/foo/foo", true, 7, 1) + e, err = fs.Get("/foo/foo", true, false, 7, 1) if err == nil { t.Fatal("still can get dir after expiration [%s]") } - _, err = fs.Get("/foo/foo/foo1", true, 7, 1) + _, err = fs.Get("/foo/foo/foo1", true, false, 7, 1) if err == nil { t.Fatal("still can get sub node after expiration [%s]") } - _, err = fs.Get("/foo/foo/foo2", true, 7, 1) + _, err = fs.Get("/foo/foo/foo2", true, false, 7, 1) if err == nil { t.Fatal("still can get sub dir after expiration [%s]") } - _, err = fs.Get("/foo/foo/foo2/boo", true, 7, 1) + _, err = fs.Get("/foo/foo/foo2/boo", true, false, 7, 1) if err == nil { t.Fatalf("still can get sub node of sub dir after expiration [%s]", err.Error()) } @@ -160,7 +161,7 @@ func TestListDirectory(t *testing.T) { // set key-value /foo/fooDir/foo=bar fs.Create("/foo/fooDir/foo", "bar", Permanent, 2, 1) - e, err := fs.Get("/foo", true, 2, 1) + e, err := fs.Get("/foo", true, false, 2, 1) if err != nil { t.Fatalf("%v", err) @@ -187,7 +188,7 @@ func TestListDirectory(t *testing.T) { // set key-value /foo/_hidden/foo -> bar fs.Create("/foo/_hidden/foo", "bar", Permanent, 3, 1) - e, _ = fs.Get("/foo", false, 2, 1) + e, _ = fs.Get("/foo", false, false, 2, 1) if len(e.KVPairs) != 2 { t.Fatalf("hidden node is not hidden! %s", e.KVPairs[2].Key) @@ -204,7 +205,7 @@ func TestRemove(t *testing.T) { t.Fatalf("cannot delete %s [%s]", "/foo", err.Error()) } - _, err = fs.Get("/foo", false, 1, 1) + _, err = fs.Get("/foo", false, false, 1, 1) if err == nil || err.Error() != "Key Not Found" { t.Fatalf("can get the node after deletion") @@ -226,7 +227,7 @@ func TestRemove(t *testing.T) { t.Fatalf("cannot delete %s [%s]", "/foo", err.Error()) } - _, err = fs.Get("/foo", false, 1, 1) + _, err = fs.Get("/foo", false, false, 1, 1) if err == nil || err.Error() != "Key Not Found" { t.Fatalf("can get the node after deletion ") @@ -382,7 +383,7 @@ func createAndGet(fs *FileSystem, path string, t *testing.T) { t.Fatalf("cannot create %s=bar [%s]", path, err.Error()) } - e, err := fs.Get(path, false, 1, 1) + e, err := fs.Get(path, false, false, 1, 1) if err != nil { t.Fatalf("cannot get %s [%s]", path, err.Error()) @@ -402,3 +403,32 @@ func nonblockingRetrive(c <-chan *Event) *Event { return nil } } + +func TestSort(t *testing.T) { + fs := New() + + // simulating random creation + keys := store.GenKeys(100, 5) + + //t.Log(keys) + i := uint64(1) + for _, k := range keys { + _, err := fs.Create(k, "bar", Permanent, i, 1) + if err != nil { + t.Fatalf("create node[%s] failed", k, err.Error()) + } + i++ + } + + e, err := fs.Get("/foo", true, true, i, 1) + if err != nil { + t.Fatalf("get dir nodes failed [%s]", err.Error()) + } + + for i, k := range e.KVPairs[:len(e.KVPairs)-1] { + //t.Log(k) + if k.Key >= e.KVPairs[i+1].Key { + t.Fatalf("sort failed, [%s] should be placed after [%s]", k.Key, e.KVPairs[i+1].Key) + } + } +} diff --git a/store/test.go b/store/test.go index ac23261be..5b922ffcb 100644 --- a/store/test.go +++ b/store/test.go @@ -10,7 +10,7 @@ func GenKeys(num int, depth int) []string { keys := make([]string, num) for i := 0; i < num; i++ { - keys[i] = "/foo/" + keys[i] = "/foo" depth := rand.Intn(depth) + 1 for j := 0; j < depth; j++ { From 38489bd8460a161f80eb401f5d3f8819f026c03f Mon Sep 17 00:00:00 2001 From: evan-gu Date: Sun, 8 Sep 2013 21:21:57 -0400 Subject: [PATCH 2/3] add documentation about sorting, change the argument name from sorting to sorted --- Documentation/etcd-file-system.md | 3 ++- file_system/file_system.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Documentation/etcd-file-system.md b/Documentation/etcd-file-system.md index c48cfecd4..092ecdee6 100644 --- a/Documentation/etcd-file-system.md +++ b/Documentation/etcd-file-system.md @@ -23,12 +23,13 @@ Besides the file and directory difference, all nodes have common attributes and The path of access control list of the node. ### Operation: -- **Get** (path, recursive) +- **Get** (path, recursive, sorted) Get the content of the node - If the node is a file, the data of the file will be returned. - If the node is a directory, the child nodes of the directory will be returned. - If recursive is true, it will recursively get the nodes of the directory. + - If sorted is true, the result will be sorted based on the path. - **Create** (path, value[optional], ttl [optional]) diff --git a/file_system/file_system.go b/file_system/file_system.go index d2735966c..51023ef86 100644 --- a/file_system/file_system.go +++ b/file_system/file_system.go @@ -26,7 +26,7 @@ func New() *FileSystem { } -func (fs *FileSystem) Get(nodePath string, recusive, sorting bool, index uint64, term uint64) (*Event, error) { +func (fs *FileSystem) Get(nodePath string, recusive, sorted bool, index uint64, term uint64) (*Event, error) { n, err := fs.InternalGet(nodePath, index, term) if err != nil { @@ -58,7 +58,7 @@ func (fs *FileSystem) Get(nodePath string, recusive, sorting bool, index uint64, // eliminate hidden nodes e.KVPairs = e.KVPairs[:i] - if sorting { + if sorted { sort.Sort(e) } } else { // node is file From 643a92a4901eca282a299a1228d0afff2eb3f6c8 Mon Sep 17 00:00:00 2001 From: evan-gu Date: Sun, 8 Sep 2013 23:13:28 -0400 Subject: [PATCH 3/3] now it will sort recursively, and the sorting test case is better.... --- file_system/event.go | 14 ++++----- file_system/file_system.go | 14 ++++++--- file_system/file_system_test.go | 54 ++++++++++++++++++++++++++++++--- file_system/node.go | 9 ++++-- store/test.go | 2 +- 5 files changed, 74 insertions(+), 19 deletions(-) diff --git a/file_system/event.go b/file_system/event.go index 16ea275a2..363730bd8 100644 --- a/file_system/event.go +++ b/file_system/event.go @@ -36,17 +36,17 @@ type KeyValuePair struct { KVPairs []KeyValuePair `json:"kvs,omitempty"` } -// interfaces for sort -func (e Event) Len() int { - return len(e.KVPairs) +// interfaces for sorting +func (k KeyValuePair) Len() int { + return len(k.KVPairs) } -func (e Event) Less(i, j int) bool { - return e.KVPairs[i].Key < e.KVPairs[j].Key +func (k KeyValuePair) Less(i, j int) bool { + return k.KVPairs[i].Key < k.KVPairs[j].Key } -func (e Event) Swap(i, j int) { - e.KVPairs[i], e.KVPairs[j] = e.KVPairs[j], e.KVPairs[i] +func (k KeyValuePair) Swap(i, j int) { + k.KVPairs[i], k.KVPairs[j] = k.KVPairs[j], k.KVPairs[i] } func newEvent(action string, key string, index uint64, term uint64) *Event { diff --git a/file_system/file_system.go b/file_system/file_system.go index 51023ef86..c50311d36 100644 --- a/file_system/file_system.go +++ b/file_system/file_system.go @@ -26,7 +26,7 @@ func New() *FileSystem { } -func (fs *FileSystem) Get(nodePath string, recusive, sorted bool, index uint64, term uint64) (*Event, error) { +func (fs *FileSystem) Get(nodePath string, recursive, sorted bool, index uint64, term uint64) (*Event, error) { n, err := fs.InternalGet(nodePath, index, term) if err != nil { @@ -51,16 +51,22 @@ func (fs *FileSystem) Get(nodePath string, recusive, sorted bool, index uint64, continue } - e.KVPairs[i] = child.Pair(recusive) + e.KVPairs[i] = child.Pair(recursive, sorted) i++ } // eliminate hidden nodes e.KVPairs = e.KVPairs[:i] - if sorted { - sort.Sort(e) + + rootPairs := KeyValuePair{ + KVPairs: e.KVPairs, } + + if sorted { + sort.Sort(rootPairs) + } + } else { // node is file e.Value = n.Value } diff --git a/file_system/file_system_test.go b/file_system/file_system_test.go index caa100451..e887fd9da 100644 --- a/file_system/file_system_test.go +++ b/file_system/file_system_test.go @@ -1,7 +1,8 @@ package fileSystem import ( - "store" + "math/rand" + "strconv" "testing" "time" ) @@ -408,16 +409,17 @@ func TestSort(t *testing.T) { fs := New() // simulating random creation - keys := store.GenKeys(100, 5) + keys := GenKeys(80, 4) //t.Log(keys) i := uint64(1) for _, k := range keys { _, err := fs.Create(k, "bar", Permanent, i, 1) if err != nil { - t.Fatalf("create node[%s] failed", k, err.Error()) + //t.Logf("create node[%s] failed %s", k, err.Error()) + } else { + i++ } - i++ } e, err := fs.Get("/foo", true, true, i, 1) @@ -426,9 +428,53 @@ func TestSort(t *testing.T) { } for i, k := range e.KVPairs[:len(e.KVPairs)-1] { + //t.Log("root:") //t.Log(k) if k.Key >= e.KVPairs[i+1].Key { t.Fatalf("sort failed, [%s] should be placed after [%s]", k.Key, e.KVPairs[i+1].Key) } + + if k.Dir { + recursiveTestSort(k, t) + } + + } + + if k := e.KVPairs[len(e.KVPairs)-1]; k.Dir { + recursiveTestSort(k, t) } } + +func recursiveTestSort(k KeyValuePair, t *testing.T) { + //t.Log("recursive in") + //t.Log(k) + for i, v := range k.KVPairs[:len(k.KVPairs)-1] { + if v.Key >= k.KVPairs[i+1].Key { + t.Fatalf("sort failed, [%s] should be placed after [%s]", v.Key, k.KVPairs[i+1].Key) + } + + if v.Dir { + recursiveTestSort(v, t) + } + + } + + if v := k.KVPairs[len(k.KVPairs)-1]; v.Dir { + recursiveTestSort(v, t) + } +} + +// GenKeys randomly generate num of keys with max depth +func GenKeys(num int, depth int) []string { + keys := make([]string, num) + for i := 0; i < num; i++ { + + keys[i] = "/foo" + depth := rand.Intn(depth) + 1 + + for j := 0; j < depth; j++ { + keys[i] += "/" + strconv.Itoa(rand.Int()%20) + } + } + return keys +} diff --git a/file_system/node.go b/file_system/node.go index 362584712..2da22b665 100644 --- a/file_system/node.go +++ b/file_system/node.go @@ -3,6 +3,7 @@ package fileSystem import ( "fmt" "path" + "sort" "sync" "time" @@ -279,7 +280,7 @@ func (n *Node) IsHidden() bool { return false } -func (n *Node) Pair(recurisive bool) KeyValuePair { +func (n *Node) Pair(recurisive, sorted bool) KeyValuePair { if n.IsDir() { pair := KeyValuePair{ @@ -303,14 +304,16 @@ func (n *Node) Pair(recurisive bool) KeyValuePair { continue } - pair.KVPairs[i] = child.Pair(recurisive) + pair.KVPairs[i] = child.Pair(recurisive, sorted) i++ } // eliminate hidden nodes pair.KVPairs = pair.KVPairs[:i] - + if sorted { + sort.Sort(pair) + } return pair } diff --git a/store/test.go b/store/test.go index 5b922ffcb..eaddaa69d 100644 --- a/store/test.go +++ b/store/test.go @@ -14,7 +14,7 @@ func GenKeys(num int, depth int) []string { depth := rand.Intn(depth) + 1 for j := 0; j < depth; j++ { - keys[i] += "/" + strconv.Itoa(rand.Int()) + keys[i] += "/" + strconv.Itoa(rand.Int()%20) } } return keys