diff --git a/Documentation/etcd-file-system.md b/Documentation/etcd-file-system.md index c48cfecd4..092ecdee6 100644 --- a/Documentation/etcd-file-system.md +++ b/Documentation/etcd-file-system.md @@ -23,12 +23,13 @@ Besides the file and directory difference, all nodes have common attributes and The path of access control list of the node. ### Operation: -- **Get** (path, recursive) +- **Get** (path, recursive, sorted) Get the content of the node - If the node is a file, the data of the file will be returned. - If the node is a directory, the child nodes of the directory will be returned. - If recursive is true, it will recursively get the nodes of the directory. + - If sorted is true, the result will be sorted based on the path. - **Create** (path, value[optional], ttl [optional]) diff --git a/file_system/event.go b/file_system/event.go index 8bea67de2..f3d92ebb7 100644 --- a/file_system/event.go +++ b/file_system/event.go @@ -38,6 +38,19 @@ type KeyValuePair struct { KVPairs []KeyValuePair `json:"kvs,omitempty"` } +// interfaces for sorting +func (k KeyValuePair) Len() int { + return len(k.KVPairs) +} + +func (k KeyValuePair) Less(i, j int) bool { + return k.KVPairs[i].Key < k.KVPairs[j].Key +} + +func (k KeyValuePair) Swap(i, j int) { + k.KVPairs[i], k.KVPairs[j] = k.KVPairs[j], k.KVPairs[i] +} + func newEvent(action string, key string, index uint64, term uint64) *Event { return &Event{ Action: action, diff --git a/file_system/file_system.go b/file_system/file_system.go index 0dc9d2ea3..d45c8d1bc 100644 --- a/file_system/file_system.go +++ b/file_system/file_system.go @@ -3,6 +3,7 @@ package fileSystem import ( "fmt" "path" + "sort" "strings" "time" @@ -25,7 +26,7 @@ func New() *FileSystem { } -func (fs *FileSystem) Get(nodePath string, recusive bool, index uint64, term uint64) (*Event, error) { +func (fs *FileSystem) Get(nodePath string, recursive, sorted bool, index uint64, term uint64) (*Event, error) { n, err := fs.InternalGet(nodePath, index, term) if err != nil { @@ -50,13 +51,22 @@ func (fs *FileSystem) Get(nodePath string, recusive bool, index uint64, term uin continue } - e.KVPairs[i] = child.Pair(recusive) + e.KVPairs[i] = child.Pair(recursive, sorted) i++ } // eliminate hidden nodes e.KVPairs = e.KVPairs[:i] + + rootPairs := KeyValuePair{ + KVPairs: e.KVPairs, + } + + if sorted { + sort.Sort(rootPairs) + } + } else { // node is file e.Value = n.Value } diff --git a/file_system/file_system_test.go b/file_system/file_system_test.go index 4c70b35bb..e887fd9da 100644 --- a/file_system/file_system_test.go +++ b/file_system/file_system_test.go @@ -1,6 +1,8 @@ package fileSystem import ( + "math/rand" + "strconv" "testing" "time" ) @@ -34,7 +36,7 @@ func TestCreateAndGet(t *testing.T) { t.Fatal("Cannot create /fooDir") } - e, err := fs.Get("/fooDir", false, 3, 1) + e, err := fs.Get("/fooDir", false, false, 3, 1) if err != nil || e.Dir != true { t.Fatal("Cannot create /fooDir ") @@ -64,7 +66,7 @@ func TestUpdateFile(t *testing.T) { t.Fatalf("cannot update %s=barbar [%s]", "/foo/bar", err.Error()) } - e, err := fs.Get("/foo/bar", false, 2, 1) + e, err := fs.Get("/foo/bar", false, false, 2, 1) if err != nil { t.Fatalf("cannot get %s [%s]", "/foo/bar", err.Error()) @@ -106,7 +108,7 @@ func TestUpdateFile(t *testing.T) { // sleep 50ms, it should still reach the node time.Sleep(time.Microsecond * 50) - e, err = fs.Get("/foo/foo", true, 7, 1) + e, err = fs.Get("/foo/foo", true, false, 7, 1) if err != nil || e.Key != "/foo/foo" { t.Fatalf("cannot get dir before expiration [%s]", err.Error()) @@ -126,23 +128,23 @@ func TestUpdateFile(t *testing.T) { // wait for expiration time.Sleep(time.Second * 3) - e, err = fs.Get("/foo/foo", true, 7, 1) + e, err = fs.Get("/foo/foo", true, false, 7, 1) if err == nil { t.Fatal("still can get dir after expiration [%s]") } - _, err = fs.Get("/foo/foo/foo1", true, 7, 1) + _, err = fs.Get("/foo/foo/foo1", true, false, 7, 1) if err == nil { t.Fatal("still can get sub node after expiration [%s]") } - _, err = fs.Get("/foo/foo/foo2", true, 7, 1) + _, err = fs.Get("/foo/foo/foo2", true, false, 7, 1) if err == nil { t.Fatal("still can get sub dir after expiration [%s]") } - _, err = fs.Get("/foo/foo/foo2/boo", true, 7, 1) + _, err = fs.Get("/foo/foo/foo2/boo", true, false, 7, 1) if err == nil { t.Fatalf("still can get sub node of sub dir after expiration [%s]", err.Error()) } @@ -160,7 +162,7 @@ func TestListDirectory(t *testing.T) { // set key-value /foo/fooDir/foo=bar fs.Create("/foo/fooDir/foo", "bar", Permanent, 2, 1) - e, err := fs.Get("/foo", true, 2, 1) + e, err := fs.Get("/foo", true, false, 2, 1) if err != nil { t.Fatalf("%v", err) @@ -187,7 +189,7 @@ func TestListDirectory(t *testing.T) { // set key-value /foo/_hidden/foo -> bar fs.Create("/foo/_hidden/foo", "bar", Permanent, 3, 1) - e, _ = fs.Get("/foo", false, 2, 1) + e, _ = fs.Get("/foo", false, false, 2, 1) if len(e.KVPairs) != 2 { t.Fatalf("hidden node is not hidden! %s", e.KVPairs[2].Key) @@ -204,7 +206,7 @@ func TestRemove(t *testing.T) { t.Fatalf("cannot delete %s [%s]", "/foo", err.Error()) } - _, err = fs.Get("/foo", false, 1, 1) + _, err = fs.Get("/foo", false, false, 1, 1) if err == nil || err.Error() != "Key Not Found" { t.Fatalf("can get the node after deletion") @@ -226,7 +228,7 @@ func TestRemove(t *testing.T) { t.Fatalf("cannot delete %s [%s]", "/foo", err.Error()) } - _, err = fs.Get("/foo", false, 1, 1) + _, err = fs.Get("/foo", false, false, 1, 1) if err == nil || err.Error() != "Key Not Found" { t.Fatalf("can get the node after deletion ") @@ -382,7 +384,7 @@ func createAndGet(fs *FileSystem, path string, t *testing.T) { t.Fatalf("cannot create %s=bar [%s]", path, err.Error()) } - e, err := fs.Get(path, false, 1, 1) + e, err := fs.Get(path, false, false, 1, 1) if err != nil { t.Fatalf("cannot get %s [%s]", path, err.Error()) @@ -402,3 +404,77 @@ func nonblockingRetrive(c <-chan *Event) *Event { return nil } } + +func TestSort(t *testing.T) { + fs := New() + + // simulating random creation + keys := GenKeys(80, 4) + + //t.Log(keys) + i := uint64(1) + for _, k := range keys { + _, err := fs.Create(k, "bar", Permanent, i, 1) + if err != nil { + //t.Logf("create node[%s] failed %s", k, err.Error()) + } else { + i++ + } + } + + e, err := fs.Get("/foo", true, true, i, 1) + if err != nil { + t.Fatalf("get dir nodes failed [%s]", err.Error()) + } + + for i, k := range e.KVPairs[:len(e.KVPairs)-1] { + //t.Log("root:") + //t.Log(k) + if k.Key >= e.KVPairs[i+1].Key { + t.Fatalf("sort failed, [%s] should be placed after [%s]", k.Key, e.KVPairs[i+1].Key) + } + + if k.Dir { + recursiveTestSort(k, t) + } + + } + + if k := e.KVPairs[len(e.KVPairs)-1]; k.Dir { + recursiveTestSort(k, t) + } +} + +func recursiveTestSort(k KeyValuePair, t *testing.T) { + //t.Log("recursive in") + //t.Log(k) + for i, v := range k.KVPairs[:len(k.KVPairs)-1] { + if v.Key >= k.KVPairs[i+1].Key { + t.Fatalf("sort failed, [%s] should be placed after [%s]", v.Key, k.KVPairs[i+1].Key) + } + + if v.Dir { + recursiveTestSort(v, t) + } + + } + + if v := k.KVPairs[len(k.KVPairs)-1]; v.Dir { + recursiveTestSort(v, t) + } +} + +// GenKeys randomly generate num of keys with max depth +func GenKeys(num int, depth int) []string { + keys := make([]string, num) + for i := 0; i < num; i++ { + + keys[i] = "/foo" + depth := rand.Intn(depth) + 1 + + for j := 0; j < depth; j++ { + keys[i] += "/" + strconv.Itoa(rand.Int()%20) + } + } + return keys +} diff --git a/file_system/node.go b/file_system/node.go index de39c5545..406cc98d6 100644 --- a/file_system/node.go +++ b/file_system/node.go @@ -3,6 +3,7 @@ package fileSystem import ( "fmt" "path" + "sort" "sync" "time" @@ -285,7 +286,7 @@ func (n *Node) IsHidden() bool { return false } -func (n *Node) Pair(recurisive bool) KeyValuePair { +func (n *Node) Pair(recurisive, sorted bool) KeyValuePair { if n.IsDir() { pair := KeyValuePair{ @@ -309,14 +310,16 @@ func (n *Node) Pair(recurisive bool) KeyValuePair { continue } - pair.KVPairs[i] = child.Pair(recurisive) + pair.KVPairs[i] = child.Pair(recurisive, sorted) i++ } // eliminate hidden nodes pair.KVPairs = pair.KVPairs[:i] - + if sorted { + sort.Sort(pair) + } return pair } diff --git a/store/test.go b/store/test.go index ac23261be..eaddaa69d 100644 --- a/store/test.go +++ b/store/test.go @@ -10,11 +10,11 @@ func GenKeys(num int, depth int) []string { keys := make([]string, num) for i := 0; i < num; i++ { - keys[i] = "/foo/" + keys[i] = "/foo" depth := rand.Intn(depth) + 1 for j := 0; j < depth; j++ { - keys[i] += "/" + strconv.Itoa(rand.Int()) + keys[i] += "/" + strconv.Itoa(rand.Int()%20) } } return keys