From 02c52f175f77245e9176a4ce4f223876d59bac75 Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Thu, 12 Feb 2015 19:35:33 -0500 Subject: [PATCH 1/3] migrate: stop deleting etcd --- migrate/snapshot.go | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/migrate/snapshot.go b/migrate/snapshot.go index cadb3f560..77f2ca5f2 100644 --- a/migrate/snapshot.go +++ b/migrate/snapshot.go @@ -63,6 +63,24 @@ type node struct { Children map[string]*node // for directory } +func deepCopyNode(n *node, parent *node) *node { + out := &node{ + Path: n.Path, + CreatedIndex: n.CreatedIndex, + ModifiedIndex: n.ModifiedIndex, + Parent: parent, + ExpireTime: n.ExpireTime, + ACL: n.ACL, + Value: n.Value, + Children: make(map[string]*node), + } + for k, v := range n.Children { + out.Children[k] = deepCopyNode(v, out) + } + + return out +} + func replacePathNames(n *node, s1, s2 string) { n.Path = path.Clean(strings.Replace(n.Path, s1, s2, 1)) for _, c := range n.Children { @@ -142,7 +160,12 @@ func fixEtcd(n *node) { } n.Children["members"].Children[m.ID.String()] = newNode } - delete(n.Children, "machines") + + for k, _ := range n.Children { + if k != "members" { + delete(n.Children, k) + } + } } @@ -157,10 +180,11 @@ func mangleRoot(n *node) *node { } newRoot.Children["1"] = n etcd := n.Children["_etcd"] - delete(n.Children, "_etcd") + newEtcd := deepCopyNode(etcd, newRoot) replacePathNames(n, "/", "/1/") - fixEtcd(etcd) - newRoot.Children["0"] = etcd + fixEtcd(newEtcd) + newRoot.Children["0"] = newEtcd + newRoot.Children["0"].Parent = newRoot return newRoot } From e9f4be498d7dac383274b308a1b4f73be4819cba Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Fri, 13 Feb 2015 15:26:54 -0500 Subject: [PATCH 2/3] migrate: decrease memory usage (only duplicate machines) --- migrate/snapshot.go | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/migrate/snapshot.go b/migrate/snapshot.go index 77f2ca5f2..4029fbb11 100644 --- a/migrate/snapshot.go +++ b/migrate/snapshot.go @@ -105,9 +105,23 @@ func pullNodesFromEtcd(n *node) map[string]uint64 { return out } -func fixEtcd(n *node) { - n.Path = "/0" - machines := n.Children["machines"] +func fixEtcd(etcdref *node) *node { + n := &node{ + Path: "/0", + CreatedIndex: etcdref.CreatedIndex, + ModifiedIndex: etcdref.ModifiedIndex, + ExpireTime: etcdref.ExpireTime, + ACL: etcdref.ACL, + Children: make(map[string]*node), + } + + var machines *node + if machineOrig, ok := etcdref.Children["machines"]; ok { + machines = deepCopyNode(machineOrig, n) + } + if machines == nil { + return n + } n.Children["members"] = &node{ Path: "/0/members", CreatedIndex: machines.CreatedIndex, @@ -160,13 +174,7 @@ func fixEtcd(n *node) { } n.Children["members"].Children[m.ID.String()] = newNode } - - for k, _ := range n.Children { - if k != "members" { - delete(n.Children, k) - } - } - + return n } func mangleRoot(n *node) *node { @@ -180,11 +188,9 @@ func mangleRoot(n *node) *node { } newRoot.Children["1"] = n etcd := n.Children["_etcd"] - newEtcd := deepCopyNode(etcd, newRoot) replacePathNames(n, "/", "/1/") - fixEtcd(newEtcd) - newRoot.Children["0"] = newEtcd - newRoot.Children["0"].Parent = newRoot + newZero := fixEtcd(etcd) + newRoot.Children["0"] = newZero return newRoot } From fdebf2b109988f8d4aac580ce18d666673cad576 Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Fri, 13 Feb 2015 16:54:15 -0500 Subject: [PATCH 3/3] fix parent references --- migrate/snapshot.go | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/migrate/snapshot.go b/migrate/snapshot.go index 4029fbb11..288f375bc 100644 --- a/migrate/snapshot.go +++ b/migrate/snapshot.go @@ -129,6 +129,7 @@ func fixEtcd(etcdref *node) *node { ExpireTime: machines.ExpireTime, ACL: machines.ACL, Children: make(map[string]*node), + Parent: n, } for name, c := range machines.Children { q, err := url.ParseQuery(c.Value) @@ -153,25 +154,29 @@ func fixEtcd(etcdref *node) *node { ModifiedIndex: c.ModifiedIndex, ExpireTime: c.ExpireTime, ACL: c.ACL, - Children: map[string]*node{ - "attributes": &node{ - Path: path.Join("/0/members", m.ID.String(), "attributes"), - CreatedIndex: c.CreatedIndex, - ModifiedIndex: c.ModifiedIndex, - ExpireTime: c.ExpireTime, - ACL: c.ACL, - Value: string(attrBytes), - }, - "raftAttributes": &node{ - Path: path.Join("/0/members", m.ID.String(), "raftAttributes"), - CreatedIndex: c.CreatedIndex, - ModifiedIndex: c.ModifiedIndex, - ExpireTime: c.ExpireTime, - ACL: c.ACL, - Value: string(raftBytes), - }, - }, + Children: make(map[string]*node), + Parent: n.Children["members"], } + attrs := &node{ + Path: path.Join("/0/members", m.ID.String(), "attributes"), + CreatedIndex: c.CreatedIndex, + ModifiedIndex: c.ModifiedIndex, + ExpireTime: c.ExpireTime, + ACL: c.ACL, + Value: string(attrBytes), + Parent: newNode, + } + newNode.Children["attributes"] = attrs + raftAttrs := &node{ + Path: path.Join("/0/members", m.ID.String(), "raftAttributes"), + CreatedIndex: c.CreatedIndex, + ModifiedIndex: c.ModifiedIndex, + ExpireTime: c.ExpireTime, + ACL: c.ACL, + Value: string(raftBytes), + Parent: newNode, + } + newNode.Children["raftAttributes"] = raftAttrs n.Children["members"].Children[m.ID.String()] = newNode } return n @@ -190,6 +195,7 @@ func mangleRoot(n *node) *node { etcd := n.Children["_etcd"] replacePathNames(n, "/", "/1/") newZero := fixEtcd(etcd) + newZero.Parent = newRoot newRoot.Children["0"] = newZero return newRoot }