Merge https://github.com/xiangli-cmu/etcd

store/store.go

@@ -22,9 +22,11 @@ const (
 var PERMANENT = time.Unix(0, 0)
 
 type Store struct {
-	// use the build-in hash map as the key-value store structure
-	Nodes map[string]Node `json:"nodes"`
+	// // use the build-in hash map as the key-value store structure
+	// Nodes map[string]Node `json:"nodes"`
 
+	// use treeMap as the key-value stroe structure
+	Tree *tree
 	// the string channel to send messages to the outside world
 	// now we use it to send changes to the hub of the web service
 	messager *chan string
@@ -77,11 +79,23 @@ type Response struct {
 func CreateStore(max int) *Store {
 	s = new(Store)
 	s.messager = nil
-	s.Nodes = make(map[string]Node)
 	s.ResponseMap = make(map[string]Response)
 	s.ResponseStartIndex = 0
 	s.ResponseMaxSize = max
 	s.ResponseCurrSize = 0
+
+	s.Tree = &tree{ 
+		&treeNode{ 
+			Node {
+				"/",
+				time.Unix(0,0),
+				nil,
+			},
+			true, 
+			make(map[string]*treeNode),
+		},
+	} 
+
 	return s
 }
 
@@ -125,7 +139,7 @@ func Set(key string, value string, expireTime time.Time, index uint64) ([]byte,
 	}
 
 	// get the node
-	node, ok := s.Nodes[key]
+	node, ok := s.Tree.get(key)
 
 	if ok {
 		// if node is not permanent before
@@ -145,7 +159,7 @@ func Set(key string, value string, expireTime time.Time, index uint64) ([]byte,
 		}
 
 		// update the information of the node
-		s.Nodes[key] = Node{value, expireTime, node.update}
+		s.Tree.set(key, Node{value, expireTime, node.update})
 
 		resp := Response{SET, key, node.Value, value, true, expireTime, TTL, index}
 
@@ -168,7 +182,7 @@ func Set(key string, value string, expireTime time.Time, index uint64) ([]byte,
 
 		update := make(chan time.Time)
 
-		s.Nodes[key] = Node{value, expireTime, update}
+		s.Tree.set(key, Node{value, expireTime, update})
 
 		if isExpire {
 			go expire(key, update, expireTime)
@@ -200,12 +214,12 @@ func expire(key string, update chan time.Time, expireTime time.Time) {
 		select {
 		// timeout delete the node
 		case <-time.After(duration):
-			node, ok := s.Nodes[key]
+			node, ok := s.Tree.get(key)
 			if !ok {
 				return
 			} else {
 
-				delete(s.Nodes, key)
+				s.Tree.delete(key)
 
 				resp := Response{DELETE, key, node.Value, "", true, node.ExpireTime, 0, s.Index}
 
@@ -267,7 +281,7 @@ func Get(key string) Response {
 	
 	key = path.Clean(key)
 
-	node, ok := s.Nodes[key]
+	node, ok := s.Tree.get(key)
 
 	if ok {
 		var TTL int64
@@ -298,19 +312,19 @@ func Delete(key string, index uint64) ([]byte, error) {
 
 	key = path.Clean(key)
 
-	node, ok := s.Nodes[key]
+	node, ok := s.Tree.get(key)
 
 	if ok {
 
 		if node.ExpireTime.Equal(PERMANENT) {
 
-			delete(s.Nodes, key)
+			s.Tree.delete(key)
 
 		} else {
 
 			// kill the expire go routine
 			node.update <- PERMANENT
-			delete(s.Nodes, key)
+			s.Tree.delete(key)
 
 		}
 
@@ -361,21 +375,23 @@ func (s *Store) Recovery(state []byte) error {
 
 // clean all expired keys
 func clean() {
-	for key, node := range s.Nodes {
+	s.Tree.traverse(cleanNode, false)
+}
+
+
+func cleanNode(key string, node *Node) {
+
+	if node.ExpireTime.Equal(PERMANENT) {
+		return
+	} else {
+
+		if node.ExpireTime.Sub(time.Now()) >= time.Second {
+			node.update = make(chan time.Time)
+			go expire(key, node.update, node.ExpireTime)
 
-		if node.ExpireTime.Equal(PERMANENT) {
-			continue
 		} else {
-
-			if node.ExpireTime.Sub(time.Now()) >= time.Second {
-				node.update = make(chan time.Time)
-				go expire(key, node.update, node.ExpireTime)
-
-			} else {
-				// we should delete this node
-				delete(s.Nodes, key)
-			}
+			// we should delete this node
+			s.Tree.delete(key)
 		}
-
 	}
 }

store/tree.go

@@ -0,0 +1,208 @@
+package store
+
+import (
+	"path"
+	"strings"
+	"sort"
+	)
+
+type tree struct {
+	Root *treeNode
+}
+
+type treeNode struct {
+
+	Value Node
+
+	Dir bool //for clearity
+
+	NodeMap map[string]*treeNode
+
+}
+
+type tnWithKey struct{
+	key string
+	tn  *treeNode
+}
+
+type tnWithKeySlice []tnWithKey
+
+func (s tnWithKeySlice) Len() int           { return len(s) }
+func (s tnWithKeySlice) Less(i, j int) bool { return s[i].key < s[j].key }
+func (s tnWithKeySlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+
+
+var emptyNode = Node{".", PERMANENT, nil}
+
+// set the key to value, return the old value if the key exists 
+func (s *tree) set(key string, value Node) bool {
+	key = "/" + key
+	key = path.Clean(key)
+
+	nodes := strings.Split(key, "/")
+	nodes = nodes[1:]
+
+	//fmt.Println("TreeStore: Nodes ", nodes, " length: ", len(nodes))
+
+	nodeMap := s.Root.NodeMap
+
+	i := 0
+	newDir := false
+
+	for i = 0; i < len(nodes) - 1; i++ {
+
+		if newDir {
+			node := &treeNode{emptyNode, true, make(map[string]*treeNode)}
+			nodeMap[nodes[i]] = node
+			nodeMap = node.NodeMap
+			continue
+		}
+
+		node, ok := nodeMap[nodes[i]]
+		// add new dir
+		if !ok {
+			//fmt.Println("TreeStore: Add a dir ", nodes[i])
+			newDir = true
+			node := &treeNode{emptyNode, true, make(map[string]*treeNode)}
+			nodeMap[nodes[i]] = node
+			nodeMap = node.NodeMap
+
+		} else if ok && !node.Dir {
+
+			return false
+		} else {
+
+			//fmt.Println("TreeStore: found dir ", nodes[i])
+			nodeMap = node.NodeMap
+		}
+
+	}
+
+	// add the last node and value
+	node, ok := nodeMap[nodes[i]]
+
+	if !ok {
+		node := &treeNode{value, false, nil}
+		nodeMap[nodes[i]] = node
+		//fmt.Println("TreeStore: Add a new Node ", key, "=", value)
+	} else {
+		node.Value = value
+		//fmt.Println("TreeStore: Update a Node ", key, "=", value, "[", oldValue, "]")
+	}
+	return true
+
+}
+
+// get the node of the key
+func (s *tree) get(key string) (Node, bool) {
+	key = "/" + key
+	key = path.Clean(key)
+
+	nodes := strings.Split(key, "/")
+	nodes = nodes[1:]
+
+	//fmt.Println("TreeStore: Nodes ", nodes, " length: ", len(nodes))
+
+	nodeMap := s.Root.NodeMap
+		
+	var i int
+
+	for i = 0; i < len(nodes) - 1; i++ {
+		node, ok := nodeMap[nodes[i]]
+		if !ok || !node.Dir {
+			return emptyNode, false
+		}
+		nodeMap = node.NodeMap
+	}
+
+	treeNode, ok := nodeMap[nodes[i]]
+	if ok {
+		return treeNode.Value, ok
+	} else {
+		return emptyNode, ok
+	}
+
+}
+
+// delete the key, return the old value if the key exists
+func (s *tree) delete(key string) bool {
+	key = "/" + key
+	key = path.Clean(key)
+
+	nodes := strings.Split(key, "/")
+	nodes = nodes[1:]
+
+	//fmt.Println("TreeStore: Nodes ", nodes, " length: ", len(nodes))
+
+	nodeMap := s.Root.NodeMap
+		
+	var i int
+
+	for i = 0; i < len(nodes) - 1; i++ {
+		node, ok := nodeMap[nodes[i]]
+		if !ok || !node.Dir {
+			return false
+		}
+		nodeMap = node.NodeMap
+	}
+
+	node, ok := nodeMap[nodes[i]]
+	if ok && !node.Dir{
+		delete(nodeMap, nodes[i])
+		return true
+	}
+	return false
+}
+
+func (t *tree) traverse(f func(string, *Node), sort bool) {
+	if sort {
+		sortDfs("", t.Root, f)
+	} else {
+		dfs("", t.Root, f)	
+	}
+}
+
+func dfs(key string, t *treeNode, f func(string, *Node)) {
+	// base case
+	if len(t.NodeMap) == 0{
+		f(key, &t.Value)
+
+	// recursion
+	} else {
+		for nodeKey, _treeNode := range t.NodeMap {
+			newKey := key + "/" + nodeKey
+			dfs(newKey, _treeNode, f)
+		}
+	}
+}
+
+func sortDfs(key string, t *treeNode, f func(string, *Node)) {
+	// base case
+	if len(t.NodeMap) == 0{
+		f(key, &t.Value)
+
+	// recursion
+	} else {
+
+		s := make(tnWithKeySlice, len(t.NodeMap))
+		i := 0
+
+		// copy
+		for nodeKey, _treeNode := range t.NodeMap {
+			newKey := key + "/" + nodeKey
+			s[i] = tnWithKey{newKey, _treeNode}
+			i++
+		}
+
+		// sort
+		sort.Sort(s)
+
+		// traverse
+		for i = 0; i < len(t.NodeMap); i++ {
+			sortDfs(s[i].key, s[i].tn, f)
+		}
+	}
+}
+
+

store/tree_store.bak

@@ -1,85 +0,0 @@
-package main
-
-import (
-	"path"
-	"strings"
-	)
-
-type store struct {
-	nodes map[string]node
-}
-
-type node struct {
-	value string
-	dir bool // just for clearity
-	nodes map[string]node
-}
-
-// set the key to value, return the old value if the key exists 
-func (s *store) set(key string, value string) string, error {
-
-	key = path.Clean(key)
-
-	nodeNames := strings.Split(key, "/")
-
-	levelNodes := s.nodes
-	for i = 0; i < len(nodes) - 1; ++i {
-		node, ok := levelNodes[nodeNames[i]]
-		// add new dir
-		if !ok {
-			node := Node{nodeNames[i], true, make(map[string]node)}
-			levelNodes[nodeNames[i]] := node
-		} else if ok && !node.dir {
-			return nil, errors.New("The key is a directory")
-		}
-		else {
-			levelNodes = levelNodes.nodes
-		}
-	}
-	// add the last node and value
-	node, ok := levelNodes[nodeNames[i]]
-
-	if !ok {
-		node := Node{nodeNames[i], false, nil}
-		levelNodes[nodeNames] = node
-		return nil, nil
-	} else {
-		oldValue := node.value
-		node.value = value
-		return oldValue ,nil
-	}
-
-}
-
-// get the node of the key
-func (s *store) get(key string) node {
-	key = path.Clean(key)
-
-	nodeNames := strings.Split(key, "/")
-
-	levelNodes := s.nodes
-	
-	for i = 0; i < len(nodes) - 1; ++i {
-		node, ok := levelNodes[nodeNames[i]]
-		if !ok || !node.dir {
-			return nil
-		}
-		levelNodes = levelNodes.nodes
-	}
-
-	node, ok := levelNodes[nodeNames[i]]
-	if ok {
-		return node
-	}
-	return nil
-
-}
-
-// delete the key, return the old value if the key exists
-func (s *store) delete(key string) string {
-	return nil
-}
-
-func (n *node) Value() string{
-	return n.value
-}

store/tree_store_test.go

@@ -0,0 +1,93 @@
+package store
+
+import (
+	"testing"
+	"math/rand"
+	"strconv"
+	"time"
+	"fmt"
+)
+
+func TestStoreGet(t *testing.T) {
+
+	ts := &tree{ 
+		&treeNode{
+			CreateTestNode("/"), 
+			true, 
+			make(map[string]*treeNode),
+		},
+	} 
+
+	// create key
+	ts.set("/foo", CreateTestNode("bar"))
+	// change value
+	ts.set("/foo", CreateTestNode("barbar"))
+	// create key
+	ts.set("/hello/foo", CreateTestNode("barbarbar"))
+	treeNode, ok := ts.get("/foo")
+
+	if !ok {
+		t.Fatalf("Expect to get node, but not")
+	}
+	if treeNode.Value != "barbar" {
+		t.Fatalf("Expect value barbar, but got %s", treeNode.Value)
+	}
+
+	// create key
+	treeNode, ok = ts.get("/hello/foo")
+	if !ok {
+		t.Fatalf("Expect to get node, but not")
+	}
+	if treeNode.Value != "barbarbar" {
+		t.Fatalf("Expect value barbarbar, but got %s", treeNode.Value)
+	}
+
+	// create a key under other key
+	ok = ts.set("/foo/foo", CreateTestNode("bar"))
+	if ok {
+		t.Fatalf("shoud not add key under a exisiting key")
+	}
+
+	// delete a key
+	ok = ts.delete("/foo") 
+	if !ok {
+		t.Fatalf("cannot delete key")
+	}
+
+	// delete a directory
+	ok = ts.delete("/hello") 
+	if ok {
+		t.Fatalf("Expect cannot delet /hello, but deleted! ")
+	}
+
+
+	// speed test
+	for i:=0; i < 100; i++ {
+		key := "/"
+		depth := rand.Intn(10)
+		for j := 0; j < depth; j++ {
+			key += "/" + strconv.Itoa(rand.Int()%10)
+		}
+		value := strconv.Itoa(rand.Int())
+		ts.set(key, CreateTestNode(value))
+		treeNode, ok := ts.get(key)
+
+		if !ok {
+			continue
+			//t.Fatalf("Expect to get node, but not")
+		}
+		if treeNode.Value != value {
+			t.Fatalf("Expect value %s, but got %s", value, treeNode.Value)
+		}
+
+	}
+	ts.traverse(f, true)
+}
+
+func f (key string, n *Node) {
+	fmt.Println(key, "=", n.Value)
+}
+
+func CreateTestNode(value string) Node{
+	return Node{value, time.Unix(0,0), nil}
+}

web/web.go

@@ -4,10 +4,10 @@ import (
 	"code.google.com/p/go.net/websocket"
 	"fmt"
 	"github.com/xiangli-cmu/go-raft"
-	"github.com/xiangli-cmu/raft-etcd/store"
+	//"github.com/xiangli-cmu/raft-etcd/store"
 	"html/template"
 	"net/http"
-	"time"
+	//"time"
 )
 
 var s *raft.Server
@@ -28,15 +28,15 @@ func handler(w http.ResponseWriter, r *http.Request) {
 
 	fmt.Fprintf(w, "Data\n")
 
-	s := store.GetStore()
+	//s := store.GetStore()
 
-	for key, node := range s.Nodes {
-		if node.ExpireTime.Equal(time.Unix(0, 0)) {
-			fmt.Fprintf(w, "%s %s\n", key, node.Value)
-		} else {
-			fmt.Fprintf(w, "%s %s %s\n", key, node.Value, node.ExpireTime)
-		}
-	}
+	// for key, node := range s.Nodes {
+	// 	if node.ExpireTime.Equal(time.Unix(0, 0)) {
+	// 		fmt.Fprintf(w, "%s %s\n", key, node.Value)
+	// 	} else {
+	// 		fmt.Fprintf(w, "%s %s %s\n", key, node.Value, node.ExpireTime)
+	// 	}
+	// }
 
 }