diff --git a/.travis.yml b/.travis.yml index 94d3c2204..4162c48c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: go -go: 1.1 +go: 1.2 install: - echo "Skip install" diff --git a/server/join_command.go b/server/join_command.go index 55359860d..ed262b1bd 100644 --- a/server/join_command.go +++ b/server/join_command.go @@ -37,11 +37,11 @@ func (c *JoinCommand) CommandName() string { } // Join a server to the cluster -func (c *JoinCommand) Apply(server raft.Server) (interface{}, error) { - ps, _ := server.Context().(*PeerServer) +func (c *JoinCommand) Apply(context raft.Context) (interface{}, error) { + ps, _ := context.Server().Context().(*PeerServer) b := make([]byte, 8) - binary.PutUvarint(b, server.CommitIndex()) + binary.PutUvarint(b, context.CommitIndex()) // Make sure we're not getting a cached value from the registry. ps.registry.Invalidate(c.Name) @@ -54,14 +54,14 @@ func (c *JoinCommand) Apply(server raft.Server) (interface{}, error) { // Check peer number in the cluster if ps.registry.Count() == ps.MaxClusterSize { log.Debug("Reject join request from ", c.Name) - return []byte{0}, etcdErr.NewError(etcdErr.EcodeNoMorePeer, "", server.CommitIndex()) + return []byte{0}, etcdErr.NewError(etcdErr.EcodeNoMorePeer, "", context.CommitIndex()) } // Add to shared peer registry. ps.registry.Register(c.Name, c.RaftURL, c.EtcdURL) // Add peer in raft - err := server.AddPeer(c.Name, "") + err := context.Server().AddPeer(c.Name, "") // Add peer stats if c.Name != ps.RaftServer().Name() { diff --git a/server/peer_server.go b/server/peer_server.go index 00a66e358..dc2de40e6 100644 --- a/server/peer_server.go +++ b/server/peer_server.go @@ -22,6 +22,8 @@ import ( const retryInterval = 10 +const ThresholdMonitorTimeout = 5 * time.Second + type PeerServer struct { raftServer raft.Server server *Server @@ -42,6 +44,9 @@ type PeerServer struct { RetryTimes int HeartbeatTimeout time.Duration ElectionTimeout time.Duration + + closeChan chan bool + timeoutThresholdChan chan interface{} } // TODO: find a good policy to do snapshot @@ -83,6 +88,8 @@ func NewPeerServer(name string, path string, url string, bindAddr string, tlsCon }, HeartbeatTimeout: defaultHeartbeatTimeout, ElectionTimeout: defaultElectionTimeout, + + timeoutThresholdChan: make(chan interface{}, 1), } // Create transporter for raft @@ -95,6 +102,13 @@ func NewPeerServer(name string, path string, url string, bindAddr string, tlsCon } s.raftServer = raftServer + s.raftServer.AddEventListener(raft.StateChangeEventType, s.raftEventLogger) + s.raftServer.AddEventListener(raft.LeaderChangeEventType, s.raftEventLogger) + s.raftServer.AddEventListener(raft.TermChangeEventType, s.raftEventLogger) + s.raftServer.AddEventListener(raft.AddPeerEventType, s.raftEventLogger) + s.raftServer.AddEventListener(raft.RemovePeerEventType, s.raftEventLogger) + s.raftServer.AddEventListener(raft.HeartbeatTimeoutEventType, s.raftEventLogger) + s.raftServer.AddEventListener(raft.ElectionTimeoutThresholdEventType, s.raftEventLogger) return s } @@ -143,7 +157,10 @@ func (s *PeerServer) ListenAndServe(snapshot bool, cluster []string) error { log.Debugf("%s restart as a follower", s.name) } + s.closeChan = make(chan bool) + go s.monitorSync() + go s.monitorTimeoutThreshold(s.closeChan) // open the snapshot if snapshot { @@ -201,6 +218,10 @@ func (s *PeerServer) listenAndServeTLS(certFile, keyFile string) error { // Stops the server. func (s *PeerServer) Close() { + if s.closeChan != nil { + close(s.closeChan) + s.closeChan = nil + } if s.listener != nil { s.listener.Close() s.listener = nil @@ -429,6 +450,43 @@ func (s *PeerServer) PeerStats() []byte { return nil } +// raftEventLogger converts events from the Raft server into log messages. +func (s *PeerServer) raftEventLogger(event raft.Event) { + value := event.Value() + prevValue := event.PrevValue() + if value == nil { + value = "" + } + if prevValue == nil { + prevValue = "" + } + + switch event.Type() { + case raft.StateChangeEventType: + log.Infof("%s: state changed from '%v' to '%v'.", s.name, prevValue, value) + case raft.TermChangeEventType: + log.Infof("%s: term #%v started.", s.name, value) + case raft.LeaderChangeEventType: + log.Infof("%s: leader changed from '%v' to '%v'.", s.name, prevValue, value) + case raft.AddPeerEventType: + log.Infof("%s: peer added: '%v'", s.name, value) + case raft.RemovePeerEventType: + log.Infof("%s: peer removed: '%v'", s.name, value) + case raft.HeartbeatTimeoutEventType: + var name = "" + if peer, ok := value.(*raft.Peer); ok { + name = peer.Name + } + log.Infof("%s: warning: heartbeat timed out: '%v'", s.name, name) + case raft.ElectionTimeoutThresholdEventType: + select { + case s.timeoutThresholdChan <- value: + default: + } + + } +} + func (s *PeerServer) monitorSnapshot() { for { time.Sleep(s.snapConf.checkingInterval) @@ -451,3 +509,18 @@ func (s *PeerServer) monitorSync() { } } } + +// monitorTimeoutThreshold groups timeout threshold events together and prints +// them as a single log line. +func (s *PeerServer) monitorTimeoutThreshold(closeChan chan bool) { + for { + select { + case value := <-s.timeoutThresholdChan: + log.Infof("%s: warning: heartbeat near election timeout: %v", s.name, value) + case <-closeChan: + return + } + + time.Sleep(ThresholdMonitorTimeout) + } +} diff --git a/server/remove_command.go b/server/remove_command.go index ab1dc2650..2057acc4a 100644 --- a/server/remove_command.go +++ b/server/remove_command.go @@ -23,8 +23,8 @@ func (c *RemoveCommand) CommandName() string { } // Remove a server from the cluster -func (c *RemoveCommand) Apply(server raft.Server) (interface{}, error) { - ps, _ := server.Context().(*PeerServer) +func (c *RemoveCommand) Apply(context raft.Context) (interface{}, error) { + ps, _ := context.Server().Context().(*PeerServer) // Remove node from the shared registry. err := ps.registry.Unregister(c.Name) @@ -38,21 +38,21 @@ func (c *RemoveCommand) Apply(server raft.Server) (interface{}, error) { } // Remove peer in raft - err = server.RemovePeer(c.Name) + err = context.Server().RemovePeer(c.Name) if err != nil { log.Debugf("Unable to remove peer: %s (%v)", c.Name, err) return []byte{0}, err } - if c.Name == server.Name() { + if c.Name == context.Server().Name() { // the removed node is this node // if the node is not replaying the previous logs // and the node has sent out a join request in this // start. It is sure that this node received a new remove // command and need to be removed - if server.CommitIndex() > ps.joinIndex && ps.joinIndex != 0 { - log.Debugf("server [%s] is removed", server.Name()) + if context.CommitIndex() > ps.joinIndex && ps.joinIndex != 0 { + log.Debugf("server [%s] is removed", context.Server().Name()) os.Exit(0) } else { // else ignore remove @@ -61,7 +61,7 @@ func (c *RemoveCommand) Apply(server raft.Server) (interface{}, error) { } b := make([]byte, 8) - binary.PutUvarint(b, server.CommitIndex()) + binary.PutUvarint(b, context.CommitIndex()) return b, err } diff --git a/store/v2/compare_and_swap_command.go b/store/v2/compare_and_swap_command.go index 79deec5e4..1af1b5032 100644 --- a/store/v2/compare_and_swap_command.go +++ b/store/v2/compare_and_swap_command.go @@ -27,8 +27,8 @@ func (c *CompareAndSwapCommand) CommandName() string { } // Set the key-value pair if the current value of the key equals to the given prevValue -func (c *CompareAndSwapCommand) Apply(server raft.Server) (interface{}, error) { - s, _ := server.StateMachine().(store.Store) +func (c *CompareAndSwapCommand) Apply(context raft.Context) (interface{}, error) { + s, _ := context.Server().StateMachine().(store.Store) e, err := s.CompareAndSwap(c.Key, c.PrevValue, c.PrevIndex, c.Value, c.ExpireTime) diff --git a/store/v2/create_command.go b/store/v2/create_command.go index d38e51ecf..8c5b4a82c 100644 --- a/store/v2/create_command.go +++ b/store/v2/create_command.go @@ -27,8 +27,8 @@ func (c *CreateCommand) CommandName() string { } // Create node -func (c *CreateCommand) Apply(server raft.Server) (interface{}, error) { - s, _ := server.StateMachine().(store.Store) +func (c *CreateCommand) Apply(context raft.Context) (interface{}, error) { + s, _ := context.Server().StateMachine().(store.Store) e, err := s.Create(c.Key, c.Dir, c.Value, c.Unique, c.ExpireTime) diff --git a/store/v2/delete_command.go b/store/v2/delete_command.go index 12104d094..bbd8ecc70 100644 --- a/store/v2/delete_command.go +++ b/store/v2/delete_command.go @@ -23,8 +23,8 @@ func (c *DeleteCommand) CommandName() string { } // Delete the key -func (c *DeleteCommand) Apply(server raft.Server) (interface{}, error) { - s, _ := server.StateMachine().(store.Store) +func (c *DeleteCommand) Apply(context raft.Context) (interface{}, error) { + s, _ := context.Server().StateMachine().(store.Store) if c.Recursive { // recursive implies dir diff --git a/store/v2/set_command.go b/store/v2/set_command.go index 9c4e51369..bceb3f192 100644 --- a/store/v2/set_command.go +++ b/store/v2/set_command.go @@ -26,8 +26,8 @@ func (c *SetCommand) CommandName() string { } // Create node -func (c *SetCommand) Apply(server raft.Server) (interface{}, error) { - s, _ := server.StateMachine().(store.Store) +func (c *SetCommand) Apply(context raft.Context) (interface{}, error) { + s, _ := context.Server().StateMachine().(store.Store) // create a new node or replace the old node. e, err := s.Set(c.Key, c.Dir, c.Value, c.ExpireTime) diff --git a/store/v2/sync_command.go b/store/v2/sync_command.go index adfbcccd4..f51b3eeaf 100644 --- a/store/v2/sync_command.go +++ b/store/v2/sync_command.go @@ -20,8 +20,8 @@ func (c SyncCommand) CommandName() string { return "etcd:sync" } -func (c SyncCommand) Apply(server raft.Server) (interface{}, error) { - s, _ := server.StateMachine().(store.Store) +func (c SyncCommand) Apply(context raft.Context) (interface{}, error) { + s, _ := context.Server().StateMachine().(store.Store) s.DeleteExpiredKeys(c.Time) return nil, nil diff --git a/store/v2/update_command.go b/store/v2/update_command.go index a677c85a2..a4e7e5990 100644 --- a/store/v2/update_command.go +++ b/store/v2/update_command.go @@ -24,8 +24,8 @@ func (c *UpdateCommand) CommandName() string { } // Create node -func (c *UpdateCommand) Apply(server raft.Server) (interface{}, error) { - s, _ := server.StateMachine().(store.Store) +func (c *UpdateCommand) Apply(context raft.Context) (interface{}, error) { + s, _ := context.Server().StateMachine().(store.Store) e, err := s.Update(c.Key, c.Value, c.ExpireTime) diff --git a/test.sh b/test.sh index ae40d8200..a62568ea9 100755 --- a/test.sh +++ b/test.sh @@ -5,6 +5,10 @@ if [ -z "$PKG" ]; then PKG="./store ./server ./server/v2/tests ./mod/lock/v2/tests" fi +if [ -z "$RUN" ]; then + RUN="." +fi + # Get GOPATH, etc from build . ./build @@ -15,9 +19,9 @@ export GOPATH="${PWD}" for i in $PKG do go test -i $i - go test -v $i + go test -v -test.run=$RUN $i done # Functional tests go test -i ./tests/functional -ETCD_BIN_PATH=$(pwd)/etcd go test -v ./tests/functional +ETCD_BIN_PATH=$(pwd)/etcd go test -v -test.run=$RUN ./tests/functional diff --git a/third_party/code.google.com/p/go.net/.hgignore b/third_party/code.google.com/p/go.net/.hgignore index 571db5fda..c27ee1ef4 100644 --- a/third_party/code.google.com/p/go.net/.hgignore +++ b/third_party/code.google.com/p/go.net/.hgignore @@ -1,2 +1,3 @@ +# Add no patterns to .hgignore except for files generated by the build. syntax:glob last-change diff --git a/third_party/code.google.com/p/go.net/codereview.cfg b/third_party/code.google.com/p/go.net/codereview.cfg index e3eb47ca0..43dbf3ce3 100644 --- a/third_party/code.google.com/p/go.net/codereview.cfg +++ b/third_party/code.google.com/p/go.net/codereview.cfg @@ -1,2 +1,2 @@ -defaultcc: golang-dev@googlegroups.com +defaultcc: golang-codereviews@googlegroups.com contributors: http://go.googlecode.com/hg/CONTRIBUTORS diff --git a/third_party/code.google.com/p/go.net/html/charset/charset.go b/third_party/code.google.com/p/go.net/html/charset/charset.go new file mode 100644 index 000000000..39dc26814 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/charset.go @@ -0,0 +1,227 @@ +// Package charset provides common text encodings for HTML documents. +// +// The mapping from encoding labels to encodings is defined at +// http://encoding.spec.whatwg.org. +package charset + +import ( + "bytes" + "io" + "mime" + "strings" + "unicode/utf8" + + "code.google.com/p/go.net/html" + "code.google.com/p/go.text/encoding" + "code.google.com/p/go.text/encoding/charmap" + "code.google.com/p/go.text/transform" +) + +// Lookup returns the encoding with the specified label, and its canonical +// name. It returns nil and the empty string if label is not one of the +// standard encodings for HTML. Matching is case-insensitive and ignores +// leading and trailing whitespace. +func Lookup(label string) (e encoding.Encoding, name string) { + label = strings.ToLower(strings.Trim(label, "\t\n\r\f ")) + enc := encodings[label] + return enc.e, enc.name +} + +// DetermineEncoding determines the encoding of an HTML document by examining +// up to the first 1024 bytes of content and the declared Content-Type. +// +// See http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#determining-the-character-encoding +func DetermineEncoding(content []byte, contentType string) (e encoding.Encoding, name string, certain bool) { + if len(content) > 1024 { + content = content[:1024] + } + + for _, b := range boms { + if bytes.HasPrefix(content, b.bom) { + e, name = Lookup(b.enc) + return e, name, true + } + } + + if _, params, err := mime.ParseMediaType(contentType); err == nil { + if cs, ok := params["charset"]; ok { + if e, name = Lookup(cs); e != nil { + return e, name, true + } + } + } + + if len(content) > 0 { + e, name = prescan(content) + if e != nil { + return e, name, false + } + } + + // Try to detect UTF-8. + // First eliminate any partial rune at the end. + for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- { + b := content[i] + if b < 0x80 { + break + } + if utf8.RuneStart(b) { + content = content[:i] + break + } + } + hasHighBit := false + for _, c := range content { + if c >= 0x80 { + hasHighBit = true + break + } + } + if hasHighBit && utf8.Valid(content) { + return encoding.Nop, "utf-8", false + } + + // TODO: change default depending on user's locale? + return charmap.Windows1252, "windows-1252", false +} + +// NewReader returns an io.Reader that converts the content of r to UTF-8. +// It calls DetermineEncoding to find out what r's encoding is. +func NewReader(r io.Reader, contentType string) (io.Reader, error) { + preview := make([]byte, 1024) + n, err := io.ReadFull(r, preview) + switch { + case err == io.ErrUnexpectedEOF: + preview = preview[:n] + r = bytes.NewReader(preview) + case err != nil: + return nil, err + default: + r = io.MultiReader(bytes.NewReader(preview), r) + } + + if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop { + r = transform.NewReader(r, e.NewDecoder()) + } + return r, nil +} + +func prescan(content []byte) (e encoding.Encoding, name string) { + z := html.NewTokenizer(bytes.NewReader(content)) + for { + switch z.Next() { + case html.ErrorToken: + return nil, "" + + case html.StartTagToken, html.SelfClosingTagToken: + tagName, hasAttr := z.TagName() + if !bytes.Equal(tagName, []byte("meta")) { + continue + } + attrList := make(map[string]bool) + gotPragma := false + + const ( + dontKnow = iota + doNeedPragma + doNotNeedPragma + ) + needPragma := dontKnow + + name = "" + e = nil + for hasAttr { + var key, val []byte + key, val, hasAttr = z.TagAttr() + ks := string(key) + if attrList[ks] { + continue + } + attrList[ks] = true + for i, c := range val { + if 'A' <= c && c <= 'Z' { + val[i] = c + 0x20 + } + } + + switch ks { + case "http-equiv": + if bytes.Equal(val, []byte("content-type")) { + gotPragma = true + } + + case "content": + if e == nil { + name = fromMetaElement(string(val)) + if name != "" { + e, name = Lookup(name) + if e != nil { + needPragma = doNeedPragma + } + } + } + + case "charset": + e, name = Lookup(string(val)) + needPragma = doNotNeedPragma + } + } + + if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma { + continue + } + + if strings.HasPrefix(name, "utf-16") { + name = "utf-8" + e = encoding.Nop + } + + if e != nil { + return e, name + } + } + } +} + +func fromMetaElement(s string) string { + for s != "" { + csLoc := strings.Index(s, "charset") + if csLoc == -1 { + return "" + } + s = s[csLoc+len("charset"):] + s = strings.TrimLeft(s, " \t\n\f\r") + if !strings.HasPrefix(s, "=") { + continue + } + s = s[1:] + s = strings.TrimLeft(s, " \t\n\f\r") + if s == "" { + return "" + } + if q := s[0]; q == '"' || q == '\'' { + s = s[1:] + closeQuote := strings.IndexRune(s, rune(q)) + if closeQuote == -1 { + return "" + } + return s[:closeQuote] + } + + end := strings.IndexAny(s, "; \t\n\f\r") + if end == -1 { + end = len(s) + } + return s[:end] + } + return "" +} + +var boms = []struct { + bom []byte + enc string +}{ + {[]byte{0xfe, 0xff}, "utf-16be"}, + {[]byte{0xff, 0xfe}, "utf-16le"}, + {[]byte{0xef, 0xbb, 0xbf}, "utf-8"}, +} diff --git a/third_party/code.google.com/p/go.net/html/charset/charset_test.go b/third_party/code.google.com/p/go.net/html/charset/charset_test.go new file mode 100644 index 000000000..a656dd90c --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/charset_test.go @@ -0,0 +1,200 @@ +package charset + +import ( + "bytes" + "io/ioutil" + "strings" + "testing" + + "code.google.com/p/go.text/transform" +) + +func transformString(t transform.Transformer, s string) (string, error) { + r := transform.NewReader(strings.NewReader(s), t) + b, err := ioutil.ReadAll(r) + return string(b), err +} + +var testCases = []struct { + utf8, other, otherEncoding string +}{ + {"Résumé", "Résumé", "utf8"}, + {"Résumé", "R\xe9sum\xe9", "latin1"}, + {"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"}, + {"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"}, + {"Hello, world", "Hello, world", "ASCII"}, + {"Gdańsk", "Gda\xf1sk", "ISO-8859-2"}, + {"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"}, + {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"}, + {"latviešu", "latvie\xf0u", "ISO-8859-13"}, + {"Seònaid", "Se\xf2naid", "ISO-8859-14"}, + {"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"}, + {"românește", "rom\xe2ne\xbate", "ISO-8859-16"}, + {"nutraĵo", "nutra\xbco", "ISO-8859-3"}, + {"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"}, + {"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"}, + {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"}, + {"Kağan", "Ka\xf0an", "ISO-8859-9"}, + {"Résumé", "R\x8esum\x8e", "macintosh"}, + {"Gdańsk", "Gda\xf1sk", "windows-1250"}, + {"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"}, + {"Résumé", "R\xe9sum\xe9", "windows-1252"}, + {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"}, + {"Kağan", "Ka\xf0an", "windows-1254"}, + {"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"}, + {"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"}, + {"latviešu", "latvie\xf0u", "windows-1257"}, + {"Việt", "Vi\xea\xf2t", "windows-1258"}, + {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"}, + {"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"}, + {"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"}, + {"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"}, + {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"}, + {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"}, + {"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"}, + {"㧯", "\x82\x31\x89\x38", "gb18030"}, + {"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"}, + {"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"}, + {"ｲｳｴｵｶ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"}, + {"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"}, + {"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"}, + {"네이트 | 즐거움의 시작, 슈파스(Spaβ) NATE", "\xb3\xd7\xc0\xcc\xc6\xae | \xc1\xf1\xb0\xc5\xbf\xf2\xc0\xc7 \xbd\xc3\xc0\xdb, \xbd\xb4\xc6\xc4\xbd\xba(Spa\xa5\xe2) NATE", "EUC-KR"}, +} + +func TestDecode(t *testing.T) { + for _, tc := range testCases { + e, _ := Lookup(tc.otherEncoding) + if e == nil { + t.Errorf("%s: not found", tc.otherEncoding) + continue + } + s, err := transformString(e.NewDecoder(), tc.other) + if err != nil { + t.Errorf("%s: decode %q: %v", tc.otherEncoding, tc.other, err) + continue + } + if s != tc.utf8 { + t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.utf8) + } + } +} + +func TestEncode(t *testing.T) { + for _, tc := range testCases { + e, _ := Lookup(tc.otherEncoding) + if e == nil { + t.Errorf("%s: not found", tc.otherEncoding) + continue + } + s, err := transformString(e.NewEncoder(), tc.utf8) + if err != nil { + t.Errorf("%s: encode %q: %s", tc.otherEncoding, tc.utf8, err) + continue + } + if s != tc.other { + t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.other) + } + } +} + +// TestNames verifies that you can pass an encoding's name to Lookup and get +// the same encoding back (except for "replacement"). +func TestNames(t *testing.T) { + for _, e := range encodings { + if e.name == "replacement" { + continue + } + _, got := Lookup(e.name) + if got != e.name { + t.Errorf("got %q, want %q", got, e.name) + continue + } + } +} + +var sniffTestCases = []struct { + filename, declared, want string +}{ + {"HTTP-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"}, + {"UTF-16LE-BOM.html", "", "utf-16le"}, + {"UTF-16BE-BOM.html", "", "utf-16be"}, + {"meta-content-attribute.html", "text/html", "iso-8859-15"}, + {"meta-charset-attribute.html", "text/html", "iso-8859-15"}, + {"No-encoding-declaration.html", "text/html", "utf-8"}, + {"HTTP-vs-UTF-8-BOM.html", "text/html; charset=iso-8859-15", "utf-8"}, + {"HTTP-vs-meta-content.html", "text/html; charset=iso-8859-15", "iso-8859-15"}, + {"HTTP-vs-meta-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"}, + {"UTF-8-BOM-vs-meta-content.html", "text/html", "utf-8"}, + {"UTF-8-BOM-vs-meta-charset.html", "text/html", "utf-8"}, +} + +func TestSniff(t *testing.T) { + for _, tc := range sniffTestCases { + content, err := ioutil.ReadFile("testdata/" + tc.filename) + if err != nil { + t.Errorf("%s: error reading file: %v", tc.filename, err) + continue + } + + _, name, _ := DetermineEncoding(content, tc.declared) + if name != tc.want { + t.Errorf("%s: got %q, want %q", tc.filename, name, tc.want) + continue + } + } +} + +func TestReader(t *testing.T) { + for _, tc := range sniffTestCases { + content, err := ioutil.ReadFile("testdata/" + tc.filename) + if err != nil { + t.Errorf("%s: error reading file: %v", tc.filename, err) + continue + } + + r, err := NewReader(bytes.NewReader(content), tc.declared) + if err != nil { + t.Errorf("%s: error creating reader: %v", tc.filename, err) + continue + } + + got, err := ioutil.ReadAll(r) + if err != nil { + t.Errorf("%s: error reading from charset.NewReader: %v", tc.filename, err) + continue + } + + e, _ := Lookup(tc.want) + want, err := ioutil.ReadAll(transform.NewReader(bytes.NewReader(content), e.NewDecoder())) + if err != nil { + t.Errorf("%s: error decoding with hard-coded charset name: %v", tc.filename, err) + continue + } + + if !bytes.Equal(got, want) { + t.Errorf("%s: got %q, want %q", tc.filename, got, want) + continue + } + } +} + +var metaTestCases = []struct { + meta, want string +}{ + {"", ""}, + {"text/html", ""}, + {"text/html; charset utf-8", ""}, + {"text/html; charset=latin-2", "latin-2"}, + {"text/html; charset; charset = utf-8", "utf-8"}, + {`charset="big5"`, "big5"}, + {"charset='shift_jis'", "shift_jis"}, +} + +func TestFromMeta(t *testing.T) { + for _, tc := range metaTestCases { + got := fromMetaElement(tc.meta) + if got != tc.want { + t.Errorf("%q: got %q, want %q", tc.meta, got, tc.want) + } + } +} diff --git a/third_party/code.google.com/p/go.net/html/charset/gen.go b/third_party/code.google.com/p/go.net/html/charset/gen.go new file mode 100644 index 000000000..25a9eb647 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/gen.go @@ -0,0 +1,107 @@ +// +build ignore + +package main + +// Download http://encoding.spec.whatwg.org/encodings.json and use it to +// generate table.go. + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "strings" +) + +type enc struct { + Name string + Labels []string +} + +type group struct { + Encodings []enc + Heading string +} + +const specURL = "http://encoding.spec.whatwg.org/encodings.json" + +func main() { + resp, err := http.Get(specURL) + if err != nil { + log.Fatalf("error fetching %s: %s", specURL, err) + } + if resp.StatusCode != 200 { + log.Fatalf("error fetching %s: HTTP status %s", specURL, resp.Status) + } + defer resp.Body.Close() + + var groups []group + d := json.NewDecoder(resp.Body) + err = d.Decode(&groups) + if err != nil { + log.Fatalf("error reading encodings.json: %s", err) + } + + fmt.Println("// generated by go run gen.go; DO NOT EDIT") + fmt.Println() + fmt.Println("package charset") + fmt.Println() + + fmt.Println("import (") + fmt.Println(`"code.google.com/p/go.text/encoding"`) + for _, pkg := range []string{"charmap", "japanese", "korean", "simplifiedchinese", "traditionalchinese", "unicode"} { + fmt.Printf("\"code.google.com/p/go.text/encoding/%s\"\n", pkg) + } + fmt.Println(")") + fmt.Println() + + fmt.Println("var encodings = map[string]struct{e encoding.Encoding; name string} {") + for _, g := range groups { + for _, e := range g.Encodings { + goName, ok := miscNames[e.Name] + if !ok { + for k, v := range prefixes { + if strings.HasPrefix(e.Name, k) { + goName = v + e.Name[len(k):] + break + } + } + if goName == "" { + log.Fatalf("unrecognized encoding name: %s", e.Name) + } + } + + for _, label := range e.Labels { + fmt.Printf("%q: {%s, %q},\n", label, goName, e.Name) + } + } + } + fmt.Println("}") +} + +var prefixes = map[string]string{ + "iso-8859-": "charmap.ISO8859_", + "windows-": "charmap.Windows", +} + +var miscNames = map[string]string{ + "utf-8": "encoding.Nop", + "ibm866": "charmap.CodePage866", + "iso-8859-8-i": "charmap.ISO8859_8", + "koi8-r": "charmap.KOI8R", + "koi8-u": "charmap.KOI8U", + "macintosh": "charmap.Macintosh", + "x-mac-cyrillic": "charmap.MacintoshCyrillic", + "gbk": "simplifiedchinese.GBK", + "gb18030": "simplifiedchinese.GB18030", + "hz-gb-2312": "simplifiedchinese.HZGB2312", + "big5": "traditionalchinese.Big5", + "euc-jp": "japanese.EUCJP", + "iso-2022-jp": "japanese.ISO2022JP", + "shift_jis": "japanese.ShiftJIS", + "euc-kr": "korean.EUCKR", + "replacement": "encoding.Replacement", + "utf-16be": "unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)", + "utf-16le": "unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM)", + "x-user-defined": "charmap.XUserDefined", +} diff --git a/third_party/code.google.com/p/go.net/html/charset/table.go b/third_party/code.google.com/p/go.net/html/charset/table.go new file mode 100644 index 000000000..66f8af161 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/table.go @@ -0,0 +1,235 @@ +// generated by go run gen.go; DO NOT EDIT + +package charset + +import ( + "code.google.com/p/go.text/encoding" + "code.google.com/p/go.text/encoding/charmap" + "code.google.com/p/go.text/encoding/japanese" + "code.google.com/p/go.text/encoding/korean" + "code.google.com/p/go.text/encoding/simplifiedchinese" + "code.google.com/p/go.text/encoding/traditionalchinese" + "code.google.com/p/go.text/encoding/unicode" +) + +var encodings = map[string]struct { + e encoding.Encoding + name string +}{ + "unicode-1-1-utf-8": {encoding.Nop, "utf-8"}, + "utf-8": {encoding.Nop, "utf-8"}, + "utf8": {encoding.Nop, "utf-8"}, + "866": {charmap.CodePage866, "ibm866"}, + "cp866": {charmap.CodePage866, "ibm866"}, + "csibm866": {charmap.CodePage866, "ibm866"}, + "ibm866": {charmap.CodePage866, "ibm866"}, + "csisolatin2": {charmap.ISO8859_2, "iso-8859-2"}, + "iso-8859-2": {charmap.ISO8859_2, "iso-8859-2"}, + "iso-ir-101": {charmap.ISO8859_2, "iso-8859-2"}, + "iso8859-2": {charmap.ISO8859_2, "iso-8859-2"}, + "iso88592": {charmap.ISO8859_2, "iso-8859-2"}, + "iso_8859-2": {charmap.ISO8859_2, "iso-8859-2"}, + "iso_8859-2:1987": {charmap.ISO8859_2, "iso-8859-2"}, + "l2": {charmap.ISO8859_2, "iso-8859-2"}, + "latin2": {charmap.ISO8859_2, "iso-8859-2"}, + "csisolatin3": {charmap.ISO8859_3, "iso-8859-3"}, + "iso-8859-3": {charmap.ISO8859_3, "iso-8859-3"}, + "iso-ir-109": {charmap.ISO8859_3, "iso-8859-3"}, + "iso8859-3": {charmap.ISO8859_3, "iso-8859-3"}, + "iso88593": {charmap.ISO8859_3, "iso-8859-3"}, + "iso_8859-3": {charmap.ISO8859_3, "iso-8859-3"}, + "iso_8859-3:1988": {charmap.ISO8859_3, "iso-8859-3"}, + "l3": {charmap.ISO8859_3, "iso-8859-3"}, + "latin3": {charmap.ISO8859_3, "iso-8859-3"}, + "csisolatin4": {charmap.ISO8859_4, "iso-8859-4"}, + "iso-8859-4": {charmap.ISO8859_4, "iso-8859-4"}, + "iso-ir-110": {charmap.ISO8859_4, "iso-8859-4"}, + "iso8859-4": {charmap.ISO8859_4, "iso-8859-4"}, + "iso88594": {charmap.ISO8859_4, "iso-8859-4"}, + "iso_8859-4": {charmap.ISO8859_4, "iso-8859-4"}, + "iso_8859-4:1988": {charmap.ISO8859_4, "iso-8859-4"}, + "l4": {charmap.ISO8859_4, "iso-8859-4"}, + "latin4": {charmap.ISO8859_4, "iso-8859-4"}, + "csisolatincyrillic": {charmap.ISO8859_5, "iso-8859-5"}, + "cyrillic": {charmap.ISO8859_5, "iso-8859-5"}, + "iso-8859-5": {charmap.ISO8859_5, "iso-8859-5"}, + "iso-ir-144": {charmap.ISO8859_5, "iso-8859-5"}, + "iso8859-5": {charmap.ISO8859_5, "iso-8859-5"}, + "iso88595": {charmap.ISO8859_5, "iso-8859-5"}, + "iso_8859-5": {charmap.ISO8859_5, "iso-8859-5"}, + "iso_8859-5:1988": {charmap.ISO8859_5, "iso-8859-5"}, + "arabic": {charmap.ISO8859_6, "iso-8859-6"}, + "asmo-708": {charmap.ISO8859_6, "iso-8859-6"}, + "csiso88596e": {charmap.ISO8859_6, "iso-8859-6"}, + "csiso88596i": {charmap.ISO8859_6, "iso-8859-6"}, + "csisolatinarabic": {charmap.ISO8859_6, "iso-8859-6"}, + "ecma-114": {charmap.ISO8859_6, "iso-8859-6"}, + "iso-8859-6": {charmap.ISO8859_6, "iso-8859-6"}, + "iso-8859-6-e": {charmap.ISO8859_6, "iso-8859-6"}, + "iso-8859-6-i": {charmap.ISO8859_6, "iso-8859-6"}, + "iso-ir-127": {charmap.ISO8859_6, "iso-8859-6"}, + "iso8859-6": {charmap.ISO8859_6, "iso-8859-6"}, + "iso88596": {charmap.ISO8859_6, "iso-8859-6"}, + "iso_8859-6": {charmap.ISO8859_6, "iso-8859-6"}, + "iso_8859-6:1987": {charmap.ISO8859_6, "iso-8859-6"}, + "csisolatingreek": {charmap.ISO8859_7, "iso-8859-7"}, + "ecma-118": {charmap.ISO8859_7, "iso-8859-7"}, + "elot_928": {charmap.ISO8859_7, "iso-8859-7"}, + "greek": {charmap.ISO8859_7, "iso-8859-7"}, + "greek8": {charmap.ISO8859_7, "iso-8859-7"}, + "iso-8859-7": {charmap.ISO8859_7, "iso-8859-7"}, + "iso-ir-126": {charmap.ISO8859_7, "iso-8859-7"}, + "iso8859-7": {charmap.ISO8859_7, "iso-8859-7"}, + "iso88597": {charmap.ISO8859_7, "iso-8859-7"}, + "iso_8859-7": {charmap.ISO8859_7, "iso-8859-7"}, + "iso_8859-7:1987": {charmap.ISO8859_7, "iso-8859-7"}, + "sun_eu_greek": {charmap.ISO8859_7, "iso-8859-7"}, + "csiso88598e": {charmap.ISO8859_8, "iso-8859-8"}, + "csisolatinhebrew": {charmap.ISO8859_8, "iso-8859-8"}, + "hebrew": {charmap.ISO8859_8, "iso-8859-8"}, + "iso-8859-8": {charmap.ISO8859_8, "iso-8859-8"}, + "iso-8859-8-e": {charmap.ISO8859_8, "iso-8859-8"}, + "iso-ir-138": {charmap.ISO8859_8, "iso-8859-8"}, + "iso8859-8": {charmap.ISO8859_8, "iso-8859-8"}, + "iso88598": {charmap.ISO8859_8, "iso-8859-8"}, + "iso_8859-8": {charmap.ISO8859_8, "iso-8859-8"}, + "iso_8859-8:1988": {charmap.ISO8859_8, "iso-8859-8"}, + "visual": {charmap.ISO8859_8, "iso-8859-8"}, + "csiso88598i": {charmap.ISO8859_8, "iso-8859-8-i"}, + "iso-8859-8-i": {charmap.ISO8859_8, "iso-8859-8-i"}, + "logical": {charmap.ISO8859_8, "iso-8859-8-i"}, + "csisolatin6": {charmap.ISO8859_10, "iso-8859-10"}, + "iso-8859-10": {charmap.ISO8859_10, "iso-8859-10"}, + "iso-ir-157": {charmap.ISO8859_10, "iso-8859-10"}, + "iso8859-10": {charmap.ISO8859_10, "iso-8859-10"}, + "iso885910": {charmap.ISO8859_10, "iso-8859-10"}, + "l6": {charmap.ISO8859_10, "iso-8859-10"}, + "latin6": {charmap.ISO8859_10, "iso-8859-10"}, + "iso-8859-13": {charmap.ISO8859_13, "iso-8859-13"}, + "iso8859-13": {charmap.ISO8859_13, "iso-8859-13"}, + "iso885913": {charmap.ISO8859_13, "iso-8859-13"}, + "iso-8859-14": {charmap.ISO8859_14, "iso-8859-14"}, + "iso8859-14": {charmap.ISO8859_14, "iso-8859-14"}, + "iso885914": {charmap.ISO8859_14, "iso-8859-14"}, + "csisolatin9": {charmap.ISO8859_15, "iso-8859-15"}, + "iso-8859-15": {charmap.ISO8859_15, "iso-8859-15"}, + "iso8859-15": {charmap.ISO8859_15, "iso-8859-15"}, + "iso885915": {charmap.ISO8859_15, "iso-8859-15"}, + "iso_8859-15": {charmap.ISO8859_15, "iso-8859-15"}, + "l9": {charmap.ISO8859_15, "iso-8859-15"}, + "iso-8859-16": {charmap.ISO8859_16, "iso-8859-16"}, + "cskoi8r": {charmap.KOI8R, "koi8-r"}, + "koi": {charmap.KOI8R, "koi8-r"}, + "koi8": {charmap.KOI8R, "koi8-r"}, + "koi8-r": {charmap.KOI8R, "koi8-r"}, + "koi8_r": {charmap.KOI8R, "koi8-r"}, + "koi8-u": {charmap.KOI8U, "koi8-u"}, + "csmacintosh": {charmap.Macintosh, "macintosh"}, + "mac": {charmap.Macintosh, "macintosh"}, + "macintosh": {charmap.Macintosh, "macintosh"}, + "x-mac-roman": {charmap.Macintosh, "macintosh"}, + "dos-874": {charmap.Windows874, "windows-874"}, + "iso-8859-11": {charmap.Windows874, "windows-874"}, + "iso8859-11": {charmap.Windows874, "windows-874"}, + "iso885911": {charmap.Windows874, "windows-874"}, + "tis-620": {charmap.Windows874, "windows-874"}, + "windows-874": {charmap.Windows874, "windows-874"}, + "cp1250": {charmap.Windows1250, "windows-1250"}, + "windows-1250": {charmap.Windows1250, "windows-1250"}, + "x-cp1250": {charmap.Windows1250, "windows-1250"}, + "cp1251": {charmap.Windows1251, "windows-1251"}, + "windows-1251": {charmap.Windows1251, "windows-1251"}, + "x-cp1251": {charmap.Windows1251, "windows-1251"}, + "ansi_x3.4-1968": {charmap.Windows1252, "windows-1252"}, + "ascii": {charmap.Windows1252, "windows-1252"}, + "cp1252": {charmap.Windows1252, "windows-1252"}, + "cp819": {charmap.Windows1252, "windows-1252"}, + "csisolatin1": {charmap.Windows1252, "windows-1252"}, + "ibm819": {charmap.Windows1252, "windows-1252"}, + "iso-8859-1": {charmap.Windows1252, "windows-1252"}, + "iso-ir-100": {charmap.Windows1252, "windows-1252"}, + "iso8859-1": {charmap.Windows1252, "windows-1252"}, + "iso88591": {charmap.Windows1252, "windows-1252"}, + "iso_8859-1": {charmap.Windows1252, "windows-1252"}, + "iso_8859-1:1987": {charmap.Windows1252, "windows-1252"}, + "l1": {charmap.Windows1252, "windows-1252"}, + "latin1": {charmap.Windows1252, "windows-1252"}, + "us-ascii": {charmap.Windows1252, "windows-1252"}, + "windows-1252": {charmap.Windows1252, "windows-1252"}, + "x-cp1252": {charmap.Windows1252, "windows-1252"}, + "cp1253": {charmap.Windows1253, "windows-1253"}, + "windows-1253": {charmap.Windows1253, "windows-1253"}, + "x-cp1253": {charmap.Windows1253, "windows-1253"}, + "cp1254": {charmap.Windows1254, "windows-1254"}, + "csisolatin5": {charmap.Windows1254, "windows-1254"}, + "iso-8859-9": {charmap.Windows1254, "windows-1254"}, + "iso-ir-148": {charmap.Windows1254, "windows-1254"}, + "iso8859-9": {charmap.Windows1254, "windows-1254"}, + "iso88599": {charmap.Windows1254, "windows-1254"}, + "iso_8859-9": {charmap.Windows1254, "windows-1254"}, + "iso_8859-9:1989": {charmap.Windows1254, "windows-1254"}, + "l5": {charmap.Windows1254, "windows-1254"}, + "latin5": {charmap.Windows1254, "windows-1254"}, + "windows-1254": {charmap.Windows1254, "windows-1254"}, + "x-cp1254": {charmap.Windows1254, "windows-1254"}, + "cp1255": {charmap.Windows1255, "windows-1255"}, + "windows-1255": {charmap.Windows1255, "windows-1255"}, + "x-cp1255": {charmap.Windows1255, "windows-1255"}, + "cp1256": {charmap.Windows1256, "windows-1256"}, + "windows-1256": {charmap.Windows1256, "windows-1256"}, + "x-cp1256": {charmap.Windows1256, "windows-1256"}, + "cp1257": {charmap.Windows1257, "windows-1257"}, + "windows-1257": {charmap.Windows1257, "windows-1257"}, + "x-cp1257": {charmap.Windows1257, "windows-1257"}, + "cp1258": {charmap.Windows1258, "windows-1258"}, + "windows-1258": {charmap.Windows1258, "windows-1258"}, + "x-cp1258": {charmap.Windows1258, "windows-1258"}, + "x-mac-cyrillic": {charmap.MacintoshCyrillic, "x-mac-cyrillic"}, + "x-mac-ukrainian": {charmap.MacintoshCyrillic, "x-mac-cyrillic"}, + "chinese": {simplifiedchinese.GBK, "gbk"}, + "csgb2312": {simplifiedchinese.GBK, "gbk"}, + "csiso58gb231280": {simplifiedchinese.GBK, "gbk"}, + "gb2312": {simplifiedchinese.GBK, "gbk"}, + "gb_2312": {simplifiedchinese.GBK, "gbk"}, + "gb_2312-80": {simplifiedchinese.GBK, "gbk"}, + "gbk": {simplifiedchinese.GBK, "gbk"}, + "iso-ir-58": {simplifiedchinese.GBK, "gbk"}, + "x-gbk": {simplifiedchinese.GBK, "gbk"}, + "gb18030": {simplifiedchinese.GB18030, "gb18030"}, + "hz-gb-2312": {simplifiedchinese.HZGB2312, "hz-gb-2312"}, + "big5": {traditionalchinese.Big5, "big5"}, + "big5-hkscs": {traditionalchinese.Big5, "big5"}, + "cn-big5": {traditionalchinese.Big5, "big5"}, + "csbig5": {traditionalchinese.Big5, "big5"}, + "x-x-big5": {traditionalchinese.Big5, "big5"}, + "cseucpkdfmtjapanese": {japanese.EUCJP, "euc-jp"}, + "euc-jp": {japanese.EUCJP, "euc-jp"}, + "x-euc-jp": {japanese.EUCJP, "euc-jp"}, + "csiso2022jp": {japanese.ISO2022JP, "iso-2022-jp"}, + "iso-2022-jp": {japanese.ISO2022JP, "iso-2022-jp"}, + "csshiftjis": {japanese.ShiftJIS, "shift_jis"}, + "ms_kanji": {japanese.ShiftJIS, "shift_jis"}, + "shift-jis": {japanese.ShiftJIS, "shift_jis"}, + "shift_jis": {japanese.ShiftJIS, "shift_jis"}, + "sjis": {japanese.ShiftJIS, "shift_jis"}, + "windows-31j": {japanese.ShiftJIS, "shift_jis"}, + "x-sjis": {japanese.ShiftJIS, "shift_jis"}, + "cseuckr": {korean.EUCKR, "euc-kr"}, + "csksc56011987": {korean.EUCKR, "euc-kr"}, + "euc-kr": {korean.EUCKR, "euc-kr"}, + "iso-ir-149": {korean.EUCKR, "euc-kr"}, + "korean": {korean.EUCKR, "euc-kr"}, + "ks_c_5601-1987": {korean.EUCKR, "euc-kr"}, + "ks_c_5601-1989": {korean.EUCKR, "euc-kr"}, + "ksc5601": {korean.EUCKR, "euc-kr"}, + "ksc_5601": {korean.EUCKR, "euc-kr"}, + "windows-949": {korean.EUCKR, "euc-kr"}, + "csiso2022kr": {encoding.Replacement, "replacement"}, + "iso-2022-kr": {encoding.Replacement, "replacement"}, + "iso-2022-cn": {encoding.Replacement, "replacement"}, + "iso-2022-cn-ext": {encoding.Replacement, "replacement"}, + "utf-16be": {unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM), "utf-16be"}, + "utf-16": {unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM), "utf-16le"}, + "utf-16le": {unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM), "utf-16le"}, + "x-user-defined": {charmap.XUserDefined, "x-user-defined"}, +} diff --git a/third_party/code.google.com/p/go.net/html/charset/testdata/HTTP-charset.html b/third_party/code.google.com/p/go.net/html/charset/testdata/HTTP-charset.html new file mode 100644 index 000000000..9915fa0ee --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/testdata/HTTP-charset.html @@ -0,0 +1,48 @@ + + + + HTTP charset + + + + + + + + + + + +

HTTP charset

+ + +
+ + +
 
+ + + + + +
+

The character encoding of a page can be set using the HTTP header charset declaration.

+

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.

The only character encoding declaration for this HTML file is in the HTTP header, which sets the encoding to ISO 8859-15.

+
+
+
Next test
HTML5
+

the-input-byte-stream-001
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/third_party/code.google.com/p/go.net/html/charset/testdata/HTTP-vs-UTF-8-BOM.html b/third_party/code.google.com/p/go.net/html/charset/testdata/HTTP-vs-UTF-8-BOM.html new file mode 100644 index 000000000..26e5d8b4e --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/testdata/HTTP-vs-UTF-8-BOM.html @@ -0,0 +1,48 @@ + + + + HTTP vs UTF-8 BOM + + + + + + + + + + + +

HTTP vs UTF-8 BOM

+ + +
+ + +
 
+ + + + + +
+

A character encoding set in the HTTP header has lower precedence than the UTF-8 signature.

+

The HTTP header attempts to set the character encoding to ISO 8859-15. The page starts with a UTF-8 signature.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.

If the test is unsuccessful, the characters  should appear at the top of the page. These represent the bytes that make up the UTF-8 signature when encountered in the ISO 8859-15 encoding.

+
+
+
Next test
HTML5
+

the-input-byte-stream-034
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/third_party/code.google.com/p/go.net/html/charset/testdata/HTTP-vs-meta-charset.html b/third_party/code.google.com/p/go.net/html/charset/testdata/HTTP-vs-meta-charset.html new file mode 100644 index 000000000..2f07e9515 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/testdata/HTTP-vs-meta-charset.html @@ -0,0 +1,49 @@ + + + + HTTP vs meta charset + + + + + + + + + + + +

HTTP vs meta charset

+ + +
+ + +
 
+ + + + + +
+

The HTTP header has a higher precedence than an encoding declaration in a meta charset attribute.

+

The HTTP header attempts to set the character encoding to ISO 8859-15. The page contains an encoding declaration in a meta charset attribute that attempts to set the character encoding to ISO 8859-1.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-018
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/third_party/code.google.com/p/go.net/html/charset/testdata/HTTP-vs-meta-content.html b/third_party/code.google.com/p/go.net/html/charset/testdata/HTTP-vs-meta-content.html new file mode 100644 index 000000000..6853cddec --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/testdata/HTTP-vs-meta-content.html @@ -0,0 +1,49 @@ + + + + HTTP vs meta content + + + + + + + + + + + +

HTTP vs meta content

+ + +
+ + +
 
+ + + + + +
+

The HTTP header has a higher precedence than an encoding declaration in a meta content attribute.

+

The HTTP header attempts to set the character encoding to ISO 8859-15. The page contains an encoding declaration in a meta content attribute that attempts to set the character encoding to ISO 8859-1.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-016
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/third_party/code.google.com/p/go.net/html/charset/testdata/No-encoding-declaration.html b/third_party/code.google.com/p/go.net/html/charset/testdata/No-encoding-declaration.html new file mode 100644 index 000000000..612e26c6c --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/testdata/No-encoding-declaration.html @@ -0,0 +1,47 @@ + + + + No encoding declaration + + + + + + + + + + + +

No encoding declaration

+ + +
+ + +
 
+ + + + + +
+

A page with no encoding information in HTTP, BOM, XML declaration or meta element will be treated as UTF-8.

+

The test on this page contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-015
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/third_party/code.google.com/p/go.net/html/charset/testdata/README b/third_party/code.google.com/p/go.net/html/charset/testdata/README new file mode 100644 index 000000000..a8e1fa471 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/testdata/README @@ -0,0 +1 @@ +These test cases come from http://www.w3.org/International/tests/html5/the-input-byte-stream/results-basics diff --git a/third_party/code.google.com/p/go.net/html/charset/testdata/UTF-16BE-BOM.html b/third_party/code.google.com/p/go.net/html/charset/testdata/UTF-16BE-BOM.html new file mode 100644 index 000000000..3abf7a934 Binary files /dev/null and b/third_party/code.google.com/p/go.net/html/charset/testdata/UTF-16BE-BOM.html differ diff --git a/third_party/code.google.com/p/go.net/html/charset/testdata/UTF-16LE-BOM.html b/third_party/code.google.com/p/go.net/html/charset/testdata/UTF-16LE-BOM.html new file mode 100644 index 000000000..76254c980 Binary files /dev/null and b/third_party/code.google.com/p/go.net/html/charset/testdata/UTF-16LE-BOM.html differ diff --git a/third_party/code.google.com/p/go.net/html/charset/testdata/UTF-8-BOM-vs-meta-charset.html b/third_party/code.google.com/p/go.net/html/charset/testdata/UTF-8-BOM-vs-meta-charset.html new file mode 100644 index 000000000..83de43338 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/testdata/UTF-8-BOM-vs-meta-charset.html @@ -0,0 +1,49 @@ + + + + UTF-8 BOM vs meta charset + + + + + + + + + + + +

UTF-8 BOM vs meta charset

+ + +
+ + +
 
+ + + + + +
+

A page with a UTF-8 BOM will be recognized as UTF-8 even if the meta charset attribute declares a different encoding.

+

The page contains an encoding declaration in a meta charset attribute that attempts to set the character encoding to ISO 8859-15, but the file starts with a UTF-8 signature.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-038
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/third_party/code.google.com/p/go.net/html/charset/testdata/UTF-8-BOM-vs-meta-content.html b/third_party/code.google.com/p/go.net/html/charset/testdata/UTF-8-BOM-vs-meta-content.html new file mode 100644 index 000000000..501aac2d6 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/testdata/UTF-8-BOM-vs-meta-content.html @@ -0,0 +1,48 @@ + + + + UTF-8 BOM vs meta content + + + + + + + + + + + +

UTF-8 BOM vs meta content

+ + +
+ + +
 
+ + + + + +
+

A page with a UTF-8 BOM will be recognized as UTF-8 even if the meta content attribute declares a different encoding.

+

The page contains an encoding declaration in a meta content attribute that attempts to set the character encoding to ISO 8859-15, but the file starts with a UTF-8 signature.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-037
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/third_party/code.google.com/p/go.net/html/charset/testdata/meta-charset-attribute.html b/third_party/code.google.com/p/go.net/html/charset/testdata/meta-charset-attribute.html new file mode 100644 index 000000000..2d7d25aba --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/testdata/meta-charset-attribute.html @@ -0,0 +1,48 @@ + + + + meta charset attribute + + + + + + + + + + + +

meta charset attribute

+ + +
+ + +
 
+ + + + + +
+

The character encoding of the page can be set by a meta element with charset attribute.

+

The only character encoding declaration for this HTML file is in the charset attribute of the meta element, which declares the encoding to be ISO 8859-15.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-009
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/third_party/code.google.com/p/go.net/html/charset/testdata/meta-content-attribute.html b/third_party/code.google.com/p/go.net/html/charset/testdata/meta-content-attribute.html new file mode 100644 index 000000000..1c3f228e7 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/charset/testdata/meta-content-attribute.html @@ -0,0 +1,48 @@ + + + + meta content attribute + + + + + + + + + + + +

meta content attribute

+ + +
+ + +
 
+ + + + + +
+

The character encoding of the page can be set by a meta element with http-equiv and content attributes.

+

The only character encoding declaration for this HTML file is in the content attribute of the meta element, which declares the encoding to be ISO 8859-15.

The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.

+
+
+
Next test
HTML5
+

the-input-byte-stream-007
Result summary & related tests
Detailed results for this test
Link to spec

+
Assumptions:
  • The default encoding for the browser you are testing is not set to ISO 8859-15.
    • +
  • The test is read from a server that supports HTTP.
+
+ + + + + + diff --git a/third_party/code.google.com/p/go.net/html/entity.go b/third_party/code.google.com/p/go.net/html/entity.go index af8a007ed..bd8307523 100644 --- a/third_party/code.google.com/p/go.net/html/entity.go +++ b/third_party/code.google.com/p/go.net/html/entity.go @@ -75,2083 +75,2083 @@ var entity = map[string]rune{ "Copf;": '\U00002102', "Coproduct;": '\U00002210', "CounterClockwiseContourIntegral;": '\U00002233', - "Cross;": '\U00002A2F', - "Cscr;": '\U0001D49E', - "Cup;": '\U000022D3', - "CupCap;": '\U0000224D', - "DD;": '\U00002145', - "DDotrahd;": '\U00002911', - "DJcy;": '\U00000402', - "DScy;": '\U00000405', - "DZcy;": '\U0000040F', - "Dagger;": '\U00002021', - "Darr;": '\U000021A1', - "Dashv;": '\U00002AE4', - "Dcaron;": '\U0000010E', - "Dcy;": '\U00000414', - "Del;": '\U00002207', - "Delta;": '\U00000394', - "Dfr;": '\U0001D507', - "DiacriticalAcute;": '\U000000B4', - "DiacriticalDot;": '\U000002D9', - "DiacriticalDoubleAcute;": '\U000002DD', - "DiacriticalGrave;": '\U00000060', - "DiacriticalTilde;": '\U000002DC', - "Diamond;": '\U000022C4', - "DifferentialD;": '\U00002146', - "Dopf;": '\U0001D53B', - "Dot;": '\U000000A8', - "DotDot;": '\U000020DC', - "DotEqual;": '\U00002250', - "DoubleContourIntegral;": '\U0000222F', - "DoubleDot;": '\U000000A8', - "DoubleDownArrow;": '\U000021D3', - "DoubleLeftArrow;": '\U000021D0', - "DoubleLeftRightArrow;": '\U000021D4', - "DoubleLeftTee;": '\U00002AE4', - "DoubleLongLeftArrow;": '\U000027F8', - "DoubleLongLeftRightArrow;": '\U000027FA', - "DoubleLongRightArrow;": '\U000027F9', - "DoubleRightArrow;": '\U000021D2', - "DoubleRightTee;": '\U000022A8', - "DoubleUpArrow;": '\U000021D1', - "DoubleUpDownArrow;": '\U000021D5', - "DoubleVerticalBar;": '\U00002225', - "DownArrow;": '\U00002193', - "DownArrowBar;": '\U00002913', - "DownArrowUpArrow;": '\U000021F5', - "DownBreve;": '\U00000311', - "DownLeftRightVector;": '\U00002950', - "DownLeftTeeVector;": '\U0000295E', - "DownLeftVector;": '\U000021BD', - "DownLeftVectorBar;": '\U00002956', - "DownRightTeeVector;": '\U0000295F', - "DownRightVector;": '\U000021C1', - "DownRightVectorBar;": '\U00002957', - "DownTee;": '\U000022A4', - "DownTeeArrow;": '\U000021A7', - "Downarrow;": '\U000021D3', - "Dscr;": '\U0001D49F', - "Dstrok;": '\U00000110', - "ENG;": '\U0000014A', - "ETH;": '\U000000D0', - "Eacute;": '\U000000C9', - "Ecaron;": '\U0000011A', - "Ecirc;": '\U000000CA', - "Ecy;": '\U0000042D', - "Edot;": '\U00000116', - "Efr;": '\U0001D508', - "Egrave;": '\U000000C8', - "Element;": '\U00002208', - "Emacr;": '\U00000112', - "EmptySmallSquare;": '\U000025FB', - "EmptyVerySmallSquare;": '\U000025AB', - "Eogon;": '\U00000118', - "Eopf;": '\U0001D53C', - "Epsilon;": '\U00000395', - "Equal;": '\U00002A75', - "EqualTilde;": '\U00002242', - "Equilibrium;": '\U000021CC', - "Escr;": '\U00002130', - "Esim;": '\U00002A73', - "Eta;": '\U00000397', - "Euml;": '\U000000CB', - "Exists;": '\U00002203', - "ExponentialE;": '\U00002147', - "Fcy;": '\U00000424', - "Ffr;": '\U0001D509', - "FilledSmallSquare;": '\U000025FC', - "FilledVerySmallSquare;": '\U000025AA', - "Fopf;": '\U0001D53D', - "ForAll;": '\U00002200', - "Fouriertrf;": '\U00002131', - "Fscr;": '\U00002131', - "GJcy;": '\U00000403', - "GT;": '\U0000003E', - "Gamma;": '\U00000393', - "Gammad;": '\U000003DC', - "Gbreve;": '\U0000011E', - "Gcedil;": '\U00000122', - "Gcirc;": '\U0000011C', - "Gcy;": '\U00000413', - "Gdot;": '\U00000120', - "Gfr;": '\U0001D50A', - "Gg;": '\U000022D9', - "Gopf;": '\U0001D53E', - "GreaterEqual;": '\U00002265', - "GreaterEqualLess;": '\U000022DB', - "GreaterFullEqual;": '\U00002267', - "GreaterGreater;": '\U00002AA2', - "GreaterLess;": '\U00002277', - "GreaterSlantEqual;": '\U00002A7E', - "GreaterTilde;": '\U00002273', - "Gscr;": '\U0001D4A2', - "Gt;": '\U0000226B', - "HARDcy;": '\U0000042A', - "Hacek;": '\U000002C7', - "Hat;": '\U0000005E', - "Hcirc;": '\U00000124', - "Hfr;": '\U0000210C', - "HilbertSpace;": '\U0000210B', - "Hopf;": '\U0000210D', - "HorizontalLine;": '\U00002500', - "Hscr;": '\U0000210B', - "Hstrok;": '\U00000126', - "HumpDownHump;": '\U0000224E', - "HumpEqual;": '\U0000224F', - "IEcy;": '\U00000415', - "IJlig;": '\U00000132', - "IOcy;": '\U00000401', - "Iacute;": '\U000000CD', - "Icirc;": '\U000000CE', - "Icy;": '\U00000418', - "Idot;": '\U00000130', - "Ifr;": '\U00002111', - "Igrave;": '\U000000CC', - "Im;": '\U00002111', - "Imacr;": '\U0000012A', - "ImaginaryI;": '\U00002148', - "Implies;": '\U000021D2', - "Int;": '\U0000222C', - "Integral;": '\U0000222B', - "Intersection;": '\U000022C2', - "InvisibleComma;": '\U00002063', - "InvisibleTimes;": '\U00002062', - "Iogon;": '\U0000012E', - "Iopf;": '\U0001D540', - "Iota;": '\U00000399', - "Iscr;": '\U00002110', - "Itilde;": '\U00000128', - "Iukcy;": '\U00000406', - "Iuml;": '\U000000CF', - "Jcirc;": '\U00000134', - "Jcy;": '\U00000419', - "Jfr;": '\U0001D50D', - "Jopf;": '\U0001D541', - "Jscr;": '\U0001D4A5', - "Jsercy;": '\U00000408', - "Jukcy;": '\U00000404', - "KHcy;": '\U00000425', - "KJcy;": '\U0000040C', - "Kappa;": '\U0000039A', - "Kcedil;": '\U00000136', - "Kcy;": '\U0000041A', - "Kfr;": '\U0001D50E', - "Kopf;": '\U0001D542', - "Kscr;": '\U0001D4A6', - "LJcy;": '\U00000409', - "LT;": '\U0000003C', - "Lacute;": '\U00000139', - "Lambda;": '\U0000039B', - "Lang;": '\U000027EA', - "Laplacetrf;": '\U00002112', - "Larr;": '\U0000219E', - "Lcaron;": '\U0000013D', - "Lcedil;": '\U0000013B', - "Lcy;": '\U0000041B', - "LeftAngleBracket;": '\U000027E8', - "LeftArrow;": '\U00002190', - "LeftArrowBar;": '\U000021E4', - "LeftArrowRightArrow;": '\U000021C6', - "LeftCeiling;": '\U00002308', - "LeftDoubleBracket;": '\U000027E6', - "LeftDownTeeVector;": '\U00002961', - "LeftDownVector;": '\U000021C3', - "LeftDownVectorBar;": '\U00002959', - "LeftFloor;": '\U0000230A', - "LeftRightArrow;": '\U00002194', - "LeftRightVector;": '\U0000294E', - "LeftTee;": '\U000022A3', - "LeftTeeArrow;": '\U000021A4', - "LeftTeeVector;": '\U0000295A', - "LeftTriangle;": '\U000022B2', - "LeftTriangleBar;": '\U000029CF', - "LeftTriangleEqual;": '\U000022B4', - "LeftUpDownVector;": '\U00002951', - "LeftUpTeeVector;": '\U00002960', - "LeftUpVector;": '\U000021BF', - "LeftUpVectorBar;": '\U00002958', - "LeftVector;": '\U000021BC', - "LeftVectorBar;": '\U00002952', - "Leftarrow;": '\U000021D0', - "Leftrightarrow;": '\U000021D4', - "LessEqualGreater;": '\U000022DA', - "LessFullEqual;": '\U00002266', - "LessGreater;": '\U00002276', - "LessLess;": '\U00002AA1', - "LessSlantEqual;": '\U00002A7D', - "LessTilde;": '\U00002272', - "Lfr;": '\U0001D50F', - "Ll;": '\U000022D8', - "Lleftarrow;": '\U000021DA', - "Lmidot;": '\U0000013F', - "LongLeftArrow;": '\U000027F5', - "LongLeftRightArrow;": '\U000027F7', - "LongRightArrow;": '\U000027F6', - "Longleftarrow;": '\U000027F8', - "Longleftrightarrow;": '\U000027FA', - "Longrightarrow;": '\U000027F9', - "Lopf;": '\U0001D543', - "LowerLeftArrow;": '\U00002199', - "LowerRightArrow;": '\U00002198', - "Lscr;": '\U00002112', - "Lsh;": '\U000021B0', - "Lstrok;": '\U00000141', - "Lt;": '\U0000226A', - "Map;": '\U00002905', - "Mcy;": '\U0000041C', - "MediumSpace;": '\U0000205F', - "Mellintrf;": '\U00002133', - "Mfr;": '\U0001D510', - "MinusPlus;": '\U00002213', - "Mopf;": '\U0001D544', - "Mscr;": '\U00002133', - "Mu;": '\U0000039C', - "NJcy;": '\U0000040A', - "Nacute;": '\U00000143', - "Ncaron;": '\U00000147', - "Ncedil;": '\U00000145', - "Ncy;": '\U0000041D', - "NegativeMediumSpace;": '\U0000200B', - "NegativeThickSpace;": '\U0000200B', - "NegativeThinSpace;": '\U0000200B', - "NegativeVeryThinSpace;": '\U0000200B', - "NestedGreaterGreater;": '\U0000226B', - "NestedLessLess;": '\U0000226A', - "NewLine;": '\U0000000A', - "Nfr;": '\U0001D511', - "NoBreak;": '\U00002060', - "NonBreakingSpace;": '\U000000A0', - "Nopf;": '\U00002115', - "Not;": '\U00002AEC', - "NotCongruent;": '\U00002262', - "NotCupCap;": '\U0000226D', - "NotDoubleVerticalBar;": '\U00002226', - "NotElement;": '\U00002209', - "NotEqual;": '\U00002260', - "NotExists;": '\U00002204', - "NotGreater;": '\U0000226F', - "NotGreaterEqual;": '\U00002271', - "NotGreaterLess;": '\U00002279', - "NotGreaterTilde;": '\U00002275', - "NotLeftTriangle;": '\U000022EA', - "NotLeftTriangleEqual;": '\U000022EC', - "NotLess;": '\U0000226E', - "NotLessEqual;": '\U00002270', - "NotLessGreater;": '\U00002278', - "NotLessTilde;": '\U00002274', - "NotPrecedes;": '\U00002280', - "NotPrecedesSlantEqual;": '\U000022E0', - "NotReverseElement;": '\U0000220C', - "NotRightTriangle;": '\U000022EB', - "NotRightTriangleEqual;": '\U000022ED', - "NotSquareSubsetEqual;": '\U000022E2', - "NotSquareSupersetEqual;": '\U000022E3', - "NotSubsetEqual;": '\U00002288', - "NotSucceeds;": '\U00002281', - "NotSucceedsSlantEqual;": '\U000022E1', - "NotSupersetEqual;": '\U00002289', - "NotTilde;": '\U00002241', - "NotTildeEqual;": '\U00002244', - "NotTildeFullEqual;": '\U00002247', - "NotTildeTilde;": '\U00002249', - "NotVerticalBar;": '\U00002224', - "Nscr;": '\U0001D4A9', - "Ntilde;": '\U000000D1', - "Nu;": '\U0000039D', - "OElig;": '\U00000152', - "Oacute;": '\U000000D3', - "Ocirc;": '\U000000D4', - "Ocy;": '\U0000041E', - "Odblac;": '\U00000150', - "Ofr;": '\U0001D512', - "Ograve;": '\U000000D2', - "Omacr;": '\U0000014C', - "Omega;": '\U000003A9', - "Omicron;": '\U0000039F', - "Oopf;": '\U0001D546', - "OpenCurlyDoubleQuote;": '\U0000201C', - "OpenCurlyQuote;": '\U00002018', - "Or;": '\U00002A54', - "Oscr;": '\U0001D4AA', - "Oslash;": '\U000000D8', - "Otilde;": '\U000000D5', - "Otimes;": '\U00002A37', - "Ouml;": '\U000000D6', - "OverBar;": '\U0000203E', - "OverBrace;": '\U000023DE', - "OverBracket;": '\U000023B4', - "OverParenthesis;": '\U000023DC', - "PartialD;": '\U00002202', - "Pcy;": '\U0000041F', - "Pfr;": '\U0001D513', - "Phi;": '\U000003A6', - "Pi;": '\U000003A0', - "PlusMinus;": '\U000000B1', - "Poincareplane;": '\U0000210C', - "Popf;": '\U00002119', - "Pr;": '\U00002ABB', - "Precedes;": '\U0000227A', - "PrecedesEqual;": '\U00002AAF', - "PrecedesSlantEqual;": '\U0000227C', - "PrecedesTilde;": '\U0000227E', - "Prime;": '\U00002033', - "Product;": '\U0000220F', - "Proportion;": '\U00002237', - "Proportional;": '\U0000221D', - "Pscr;": '\U0001D4AB', - "Psi;": '\U000003A8', - "QUOT;": '\U00000022', - "Qfr;": '\U0001D514', - "Qopf;": '\U0000211A', - "Qscr;": '\U0001D4AC', - "RBarr;": '\U00002910', - "REG;": '\U000000AE', - "Racute;": '\U00000154', - "Rang;": '\U000027EB', - "Rarr;": '\U000021A0', - "Rarrtl;": '\U00002916', - "Rcaron;": '\U00000158', - "Rcedil;": '\U00000156', - "Rcy;": '\U00000420', - "Re;": '\U0000211C', - "ReverseElement;": '\U0000220B', - "ReverseEquilibrium;": '\U000021CB', - "ReverseUpEquilibrium;": '\U0000296F', - "Rfr;": '\U0000211C', - "Rho;": '\U000003A1', - "RightAngleBracket;": '\U000027E9', - "RightArrow;": '\U00002192', - "RightArrowBar;": '\U000021E5', - "RightArrowLeftArrow;": '\U000021C4', - "RightCeiling;": '\U00002309', - "RightDoubleBracket;": '\U000027E7', - "RightDownTeeVector;": '\U0000295D', - "RightDownVector;": '\U000021C2', - "RightDownVectorBar;": '\U00002955', - "RightFloor;": '\U0000230B', - "RightTee;": '\U000022A2', - "RightTeeArrow;": '\U000021A6', - "RightTeeVector;": '\U0000295B', - "RightTriangle;": '\U000022B3', - "RightTriangleBar;": '\U000029D0', - "RightTriangleEqual;": '\U000022B5', - "RightUpDownVector;": '\U0000294F', - "RightUpTeeVector;": '\U0000295C', - "RightUpVector;": '\U000021BE', - "RightUpVectorBar;": '\U00002954', - "RightVector;": '\U000021C0', - "RightVectorBar;": '\U00002953', - "Rightarrow;": '\U000021D2', - "Ropf;": '\U0000211D', - "RoundImplies;": '\U00002970', - "Rrightarrow;": '\U000021DB', - "Rscr;": '\U0000211B', - "Rsh;": '\U000021B1', - "RuleDelayed;": '\U000029F4', - "SHCHcy;": '\U00000429', - "SHcy;": '\U00000428', - "SOFTcy;": '\U0000042C', - "Sacute;": '\U0000015A', - "Sc;": '\U00002ABC', - "Scaron;": '\U00000160', - "Scedil;": '\U0000015E', - "Scirc;": '\U0000015C', - "Scy;": '\U00000421', - "Sfr;": '\U0001D516', - "ShortDownArrow;": '\U00002193', - "ShortLeftArrow;": '\U00002190', - "ShortRightArrow;": '\U00002192', - "ShortUpArrow;": '\U00002191', - "Sigma;": '\U000003A3', - "SmallCircle;": '\U00002218', - "Sopf;": '\U0001D54A', - "Sqrt;": '\U0000221A', - "Square;": '\U000025A1', - "SquareIntersection;": '\U00002293', - "SquareSubset;": '\U0000228F', - "SquareSubsetEqual;": '\U00002291', - "SquareSuperset;": '\U00002290', - "SquareSupersetEqual;": '\U00002292', - "SquareUnion;": '\U00002294', - "Sscr;": '\U0001D4AE', - "Star;": '\U000022C6', - "Sub;": '\U000022D0', - "Subset;": '\U000022D0', - "SubsetEqual;": '\U00002286', - "Succeeds;": '\U0000227B', - "SucceedsEqual;": '\U00002AB0', - "SucceedsSlantEqual;": '\U0000227D', - "SucceedsTilde;": '\U0000227F', - "SuchThat;": '\U0000220B', - "Sum;": '\U00002211', - "Sup;": '\U000022D1', - "Superset;": '\U00002283', - "SupersetEqual;": '\U00002287', - "Supset;": '\U000022D1', - "THORN;": '\U000000DE', - "TRADE;": '\U00002122', - "TSHcy;": '\U0000040B', - "TScy;": '\U00000426', - "Tab;": '\U00000009', - "Tau;": '\U000003A4', - "Tcaron;": '\U00000164', - "Tcedil;": '\U00000162', - "Tcy;": '\U00000422', - "Tfr;": '\U0001D517', - "Therefore;": '\U00002234', - "Theta;": '\U00000398', - "ThinSpace;": '\U00002009', - "Tilde;": '\U0000223C', - "TildeEqual;": '\U00002243', - "TildeFullEqual;": '\U00002245', - "TildeTilde;": '\U00002248', - "Topf;": '\U0001D54B', - "TripleDot;": '\U000020DB', - "Tscr;": '\U0001D4AF', - "Tstrok;": '\U00000166', - "Uacute;": '\U000000DA', - "Uarr;": '\U0000219F', - "Uarrocir;": '\U00002949', - "Ubrcy;": '\U0000040E', - "Ubreve;": '\U0000016C', - "Ucirc;": '\U000000DB', - "Ucy;": '\U00000423', - "Udblac;": '\U00000170', - "Ufr;": '\U0001D518', - "Ugrave;": '\U000000D9', - "Umacr;": '\U0000016A', - "UnderBar;": '\U0000005F', - "UnderBrace;": '\U000023DF', - "UnderBracket;": '\U000023B5', - "UnderParenthesis;": '\U000023DD', - "Union;": '\U000022C3', - "UnionPlus;": '\U0000228E', - "Uogon;": '\U00000172', - "Uopf;": '\U0001D54C', - "UpArrow;": '\U00002191', - "UpArrowBar;": '\U00002912', - "UpArrowDownArrow;": '\U000021C5', - "UpDownArrow;": '\U00002195', - "UpEquilibrium;": '\U0000296E', - "UpTee;": '\U000022A5', - "UpTeeArrow;": '\U000021A5', - "Uparrow;": '\U000021D1', - "Updownarrow;": '\U000021D5', - "UpperLeftArrow;": '\U00002196', - "UpperRightArrow;": '\U00002197', - "Upsi;": '\U000003D2', - "Upsilon;": '\U000003A5', - "Uring;": '\U0000016E', - "Uscr;": '\U0001D4B0', - "Utilde;": '\U00000168', - "Uuml;": '\U000000DC', - "VDash;": '\U000022AB', - "Vbar;": '\U00002AEB', - "Vcy;": '\U00000412', - "Vdash;": '\U000022A9', - "Vdashl;": '\U00002AE6', - "Vee;": '\U000022C1', - "Verbar;": '\U00002016', - "Vert;": '\U00002016', - "VerticalBar;": '\U00002223', - "VerticalLine;": '\U0000007C', - "VerticalSeparator;": '\U00002758', - "VerticalTilde;": '\U00002240', - "VeryThinSpace;": '\U0000200A', - "Vfr;": '\U0001D519', - "Vopf;": '\U0001D54D', - "Vscr;": '\U0001D4B1', - "Vvdash;": '\U000022AA', - "Wcirc;": '\U00000174', - "Wedge;": '\U000022C0', - "Wfr;": '\U0001D51A', - "Wopf;": '\U0001D54E', - "Wscr;": '\U0001D4B2', - "Xfr;": '\U0001D51B', - "Xi;": '\U0000039E', - "Xopf;": '\U0001D54F', - "Xscr;": '\U0001D4B3', - "YAcy;": '\U0000042F', - "YIcy;": '\U00000407', - "YUcy;": '\U0000042E', - "Yacute;": '\U000000DD', - "Ycirc;": '\U00000176', - "Ycy;": '\U0000042B', - "Yfr;": '\U0001D51C', - "Yopf;": '\U0001D550', - "Yscr;": '\U0001D4B4', - "Yuml;": '\U00000178', - "ZHcy;": '\U00000416', - "Zacute;": '\U00000179', - "Zcaron;": '\U0000017D', - "Zcy;": '\U00000417', - "Zdot;": '\U0000017B', - "ZeroWidthSpace;": '\U0000200B', - "Zeta;": '\U00000396', - "Zfr;": '\U00002128', - "Zopf;": '\U00002124', - "Zscr;": '\U0001D4B5', - "aacute;": '\U000000E1', - "abreve;": '\U00000103', - "ac;": '\U0000223E', - "acd;": '\U0000223F', - "acirc;": '\U000000E2', - "acute;": '\U000000B4', - "acy;": '\U00000430', - "aelig;": '\U000000E6', - "af;": '\U00002061', - "afr;": '\U0001D51E', - "agrave;": '\U000000E0', - "alefsym;": '\U00002135', - "aleph;": '\U00002135', - "alpha;": '\U000003B1', - "amacr;": '\U00000101', - "amalg;": '\U00002A3F', - "amp;": '\U00000026', - "and;": '\U00002227', - "andand;": '\U00002A55', - "andd;": '\U00002A5C', - "andslope;": '\U00002A58', - "andv;": '\U00002A5A', - "ang;": '\U00002220', - "ange;": '\U000029A4', - "angle;": '\U00002220', - "angmsd;": '\U00002221', - "angmsdaa;": '\U000029A8', - "angmsdab;": '\U000029A9', - "angmsdac;": '\U000029AA', - "angmsdad;": '\U000029AB', - "angmsdae;": '\U000029AC', - "angmsdaf;": '\U000029AD', - "angmsdag;": '\U000029AE', - "angmsdah;": '\U000029AF', - "angrt;": '\U0000221F', - "angrtvb;": '\U000022BE', - "angrtvbd;": '\U0000299D', - "angsph;": '\U00002222', - "angst;": '\U000000C5', - "angzarr;": '\U0000237C', - "aogon;": '\U00000105', - "aopf;": '\U0001D552', - "ap;": '\U00002248', - "apE;": '\U00002A70', - "apacir;": '\U00002A6F', - "ape;": '\U0000224A', - "apid;": '\U0000224B', - "apos;": '\U00000027', - "approx;": '\U00002248', - "approxeq;": '\U0000224A', - "aring;": '\U000000E5', - "ascr;": '\U0001D4B6', - "ast;": '\U0000002A', - "asymp;": '\U00002248', - "asympeq;": '\U0000224D', - "atilde;": '\U000000E3', - "auml;": '\U000000E4', - "awconint;": '\U00002233', - "awint;": '\U00002A11', - "bNot;": '\U00002AED', - "backcong;": '\U0000224C', - "backepsilon;": '\U000003F6', - "backprime;": '\U00002035', - "backsim;": '\U0000223D', - "backsimeq;": '\U000022CD', - "barvee;": '\U000022BD', - "barwed;": '\U00002305', - "barwedge;": '\U00002305', - "bbrk;": '\U000023B5', - "bbrktbrk;": '\U000023B6', - "bcong;": '\U0000224C', - "bcy;": '\U00000431', - "bdquo;": '\U0000201E', - "becaus;": '\U00002235', - "because;": '\U00002235', - "bemptyv;": '\U000029B0', - "bepsi;": '\U000003F6', - "bernou;": '\U0000212C', - "beta;": '\U000003B2', - "beth;": '\U00002136', - "between;": '\U0000226C', - "bfr;": '\U0001D51F', - "bigcap;": '\U000022C2', - "bigcirc;": '\U000025EF', - "bigcup;": '\U000022C3', - "bigodot;": '\U00002A00', - "bigoplus;": '\U00002A01', - "bigotimes;": '\U00002A02', - "bigsqcup;": '\U00002A06', - "bigstar;": '\U00002605', - "bigtriangledown;": '\U000025BD', - "bigtriangleup;": '\U000025B3', - "biguplus;": '\U00002A04', - "bigvee;": '\U000022C1', - "bigwedge;": '\U000022C0', - "bkarow;": '\U0000290D', - "blacklozenge;": '\U000029EB', - "blacksquare;": '\U000025AA', - "blacktriangle;": '\U000025B4', - "blacktriangledown;": '\U000025BE', - "blacktriangleleft;": '\U000025C2', - "blacktriangleright;": '\U000025B8', - "blank;": '\U00002423', - "blk12;": '\U00002592', - "blk14;": '\U00002591', - "blk34;": '\U00002593', - "block;": '\U00002588', - "bnot;": '\U00002310', - "bopf;": '\U0001D553', - "bot;": '\U000022A5', - "bottom;": '\U000022A5', - "bowtie;": '\U000022C8', - "boxDL;": '\U00002557', - "boxDR;": '\U00002554', - "boxDl;": '\U00002556', - "boxDr;": '\U00002553', - "boxH;": '\U00002550', - "boxHD;": '\U00002566', - "boxHU;": '\U00002569', - "boxHd;": '\U00002564', - "boxHu;": '\U00002567', - "boxUL;": '\U0000255D', - "boxUR;": '\U0000255A', - "boxUl;": '\U0000255C', - "boxUr;": '\U00002559', - "boxV;": '\U00002551', - "boxVH;": '\U0000256C', - "boxVL;": '\U00002563', - "boxVR;": '\U00002560', - "boxVh;": '\U0000256B', - "boxVl;": '\U00002562', - "boxVr;": '\U0000255F', - "boxbox;": '\U000029C9', - "boxdL;": '\U00002555', - "boxdR;": '\U00002552', - "boxdl;": '\U00002510', - "boxdr;": '\U0000250C', - "boxh;": '\U00002500', - "boxhD;": '\U00002565', - "boxhU;": '\U00002568', - "boxhd;": '\U0000252C', - "boxhu;": '\U00002534', - "boxminus;": '\U0000229F', - "boxplus;": '\U0000229E', - "boxtimes;": '\U000022A0', - "boxuL;": '\U0000255B', - "boxuR;": '\U00002558', - "boxul;": '\U00002518', - "boxur;": '\U00002514', - "boxv;": '\U00002502', - "boxvH;": '\U0000256A', - "boxvL;": '\U00002561', - "boxvR;": '\U0000255E', - "boxvh;": '\U0000253C', - "boxvl;": '\U00002524', - "boxvr;": '\U0000251C', - "bprime;": '\U00002035', - "breve;": '\U000002D8', - "brvbar;": '\U000000A6', - "bscr;": '\U0001D4B7', - "bsemi;": '\U0000204F', - "bsim;": '\U0000223D', - "bsime;": '\U000022CD', - "bsol;": '\U0000005C', - "bsolb;": '\U000029C5', - "bsolhsub;": '\U000027C8', - "bull;": '\U00002022', - "bullet;": '\U00002022', - "bump;": '\U0000224E', - "bumpE;": '\U00002AAE', - "bumpe;": '\U0000224F', - "bumpeq;": '\U0000224F', - "cacute;": '\U00000107', - "cap;": '\U00002229', - "capand;": '\U00002A44', - "capbrcup;": '\U00002A49', - "capcap;": '\U00002A4B', - "capcup;": '\U00002A47', - "capdot;": '\U00002A40', - "caret;": '\U00002041', - "caron;": '\U000002C7', - "ccaps;": '\U00002A4D', - "ccaron;": '\U0000010D', - "ccedil;": '\U000000E7', - "ccirc;": '\U00000109', - "ccups;": '\U00002A4C', - "ccupssm;": '\U00002A50', - "cdot;": '\U0000010B', - "cedil;": '\U000000B8', - "cemptyv;": '\U000029B2', - "cent;": '\U000000A2', - "centerdot;": '\U000000B7', - "cfr;": '\U0001D520', - "chcy;": '\U00000447', - "check;": '\U00002713', - "checkmark;": '\U00002713', - "chi;": '\U000003C7', - "cir;": '\U000025CB', - "cirE;": '\U000029C3', - "circ;": '\U000002C6', - "circeq;": '\U00002257', - "circlearrowleft;": '\U000021BA', - "circlearrowright;": '\U000021BB', - "circledR;": '\U000000AE', - "circledS;": '\U000024C8', - "circledast;": '\U0000229B', - "circledcirc;": '\U0000229A', - "circleddash;": '\U0000229D', - "cire;": '\U00002257', - "cirfnint;": '\U00002A10', - "cirmid;": '\U00002AEF', - "cirscir;": '\U000029C2', - "clubs;": '\U00002663', - "clubsuit;": '\U00002663', - "colon;": '\U0000003A', - "colone;": '\U00002254', - "coloneq;": '\U00002254', - "comma;": '\U0000002C', - "commat;": '\U00000040', - "comp;": '\U00002201', - "compfn;": '\U00002218', - "complement;": '\U00002201', - "complexes;": '\U00002102', - "cong;": '\U00002245', - "congdot;": '\U00002A6D', - "conint;": '\U0000222E', - "copf;": '\U0001D554', - "coprod;": '\U00002210', - "copy;": '\U000000A9', - "copysr;": '\U00002117', - "crarr;": '\U000021B5', - "cross;": '\U00002717', - "cscr;": '\U0001D4B8', - "csub;": '\U00002ACF', - "csube;": '\U00002AD1', - "csup;": '\U00002AD0', - "csupe;": '\U00002AD2', - "ctdot;": '\U000022EF', - "cudarrl;": '\U00002938', - "cudarrr;": '\U00002935', - "cuepr;": '\U000022DE', - "cuesc;": '\U000022DF', - "cularr;": '\U000021B6', - "cularrp;": '\U0000293D', - "cup;": '\U0000222A', - "cupbrcap;": '\U00002A48', - "cupcap;": '\U00002A46', - "cupcup;": '\U00002A4A', - "cupdot;": '\U0000228D', - "cupor;": '\U00002A45', - "curarr;": '\U000021B7', - "curarrm;": '\U0000293C', - "curlyeqprec;": '\U000022DE', - "curlyeqsucc;": '\U000022DF', - "curlyvee;": '\U000022CE', - "curlywedge;": '\U000022CF', - "curren;": '\U000000A4', - "curvearrowleft;": '\U000021B6', - "curvearrowright;": '\U000021B7', - "cuvee;": '\U000022CE', - "cuwed;": '\U000022CF', - "cwconint;": '\U00002232', - "cwint;": '\U00002231', - "cylcty;": '\U0000232D', - "dArr;": '\U000021D3', - "dHar;": '\U00002965', - "dagger;": '\U00002020', - "daleth;": '\U00002138', - "darr;": '\U00002193', - "dash;": '\U00002010', - "dashv;": '\U000022A3', - "dbkarow;": '\U0000290F', - "dblac;": '\U000002DD', - "dcaron;": '\U0000010F', - "dcy;": '\U00000434', - "dd;": '\U00002146', - "ddagger;": '\U00002021', - "ddarr;": '\U000021CA', - "ddotseq;": '\U00002A77', - "deg;": '\U000000B0', - "delta;": '\U000003B4', - "demptyv;": '\U000029B1', - "dfisht;": '\U0000297F', - "dfr;": '\U0001D521', - "dharl;": '\U000021C3', - "dharr;": '\U000021C2', - "diam;": '\U000022C4', - "diamond;": '\U000022C4', - "diamondsuit;": '\U00002666', - "diams;": '\U00002666', - "die;": '\U000000A8', - "digamma;": '\U000003DD', - "disin;": '\U000022F2', - "div;": '\U000000F7', - "divide;": '\U000000F7', - "divideontimes;": '\U000022C7', - "divonx;": '\U000022C7', - "djcy;": '\U00000452', - "dlcorn;": '\U0000231E', - "dlcrop;": '\U0000230D', - "dollar;": '\U00000024', - "dopf;": '\U0001D555', - "dot;": '\U000002D9', - "doteq;": '\U00002250', - "doteqdot;": '\U00002251', - "dotminus;": '\U00002238', - "dotplus;": '\U00002214', - "dotsquare;": '\U000022A1', - "doublebarwedge;": '\U00002306', - "downarrow;": '\U00002193', - "downdownarrows;": '\U000021CA', - "downharpoonleft;": '\U000021C3', - "downharpoonright;": '\U000021C2', - "drbkarow;": '\U00002910', - "drcorn;": '\U0000231F', - "drcrop;": '\U0000230C', - "dscr;": '\U0001D4B9', - "dscy;": '\U00000455', - "dsol;": '\U000029F6', - "dstrok;": '\U00000111', - "dtdot;": '\U000022F1', - "dtri;": '\U000025BF', - "dtrif;": '\U000025BE', - "duarr;": '\U000021F5', - "duhar;": '\U0000296F', - "dwangle;": '\U000029A6', - "dzcy;": '\U0000045F', - "dzigrarr;": '\U000027FF', - "eDDot;": '\U00002A77', - "eDot;": '\U00002251', - "eacute;": '\U000000E9', - "easter;": '\U00002A6E', - "ecaron;": '\U0000011B', - "ecir;": '\U00002256', - "ecirc;": '\U000000EA', - "ecolon;": '\U00002255', - "ecy;": '\U0000044D', - "edot;": '\U00000117', - "ee;": '\U00002147', - "efDot;": '\U00002252', - "efr;": '\U0001D522', - "eg;": '\U00002A9A', - "egrave;": '\U000000E8', - "egs;": '\U00002A96', - "egsdot;": '\U00002A98', - "el;": '\U00002A99', - "elinters;": '\U000023E7', - "ell;": '\U00002113', - "els;": '\U00002A95', - "elsdot;": '\U00002A97', - "emacr;": '\U00000113', - "empty;": '\U00002205', - "emptyset;": '\U00002205', - "emptyv;": '\U00002205', - "emsp;": '\U00002003', - "emsp13;": '\U00002004', - "emsp14;": '\U00002005', - "eng;": '\U0000014B', - "ensp;": '\U00002002', - "eogon;": '\U00000119', - "eopf;": '\U0001D556', - "epar;": '\U000022D5', - "eparsl;": '\U000029E3', - "eplus;": '\U00002A71', - "epsi;": '\U000003B5', - "epsilon;": '\U000003B5', - "epsiv;": '\U000003F5', - "eqcirc;": '\U00002256', - "eqcolon;": '\U00002255', - "eqsim;": '\U00002242', - "eqslantgtr;": '\U00002A96', - "eqslantless;": '\U00002A95', - "equals;": '\U0000003D', - "equest;": '\U0000225F', - "equiv;": '\U00002261', - "equivDD;": '\U00002A78', - "eqvparsl;": '\U000029E5', - "erDot;": '\U00002253', - "erarr;": '\U00002971', - "escr;": '\U0000212F', - "esdot;": '\U00002250', - "esim;": '\U00002242', - "eta;": '\U000003B7', - "eth;": '\U000000F0', - "euml;": '\U000000EB', - "euro;": '\U000020AC', - "excl;": '\U00000021', - "exist;": '\U00002203', - "expectation;": '\U00002130', - "exponentiale;": '\U00002147', - "fallingdotseq;": '\U00002252', - "fcy;": '\U00000444', - "female;": '\U00002640', - "ffilig;": '\U0000FB03', - "fflig;": '\U0000FB00', - "ffllig;": '\U0000FB04', - "ffr;": '\U0001D523', - "filig;": '\U0000FB01', - "flat;": '\U0000266D', - "fllig;": '\U0000FB02', - "fltns;": '\U000025B1', - "fnof;": '\U00000192', - "fopf;": '\U0001D557', - "forall;": '\U00002200', - "fork;": '\U000022D4', - "forkv;": '\U00002AD9', - "fpartint;": '\U00002A0D', - "frac12;": '\U000000BD', - "frac13;": '\U00002153', - "frac14;": '\U000000BC', - "frac15;": '\U00002155', - "frac16;": '\U00002159', - "frac18;": '\U0000215B', - "frac23;": '\U00002154', - "frac25;": '\U00002156', - "frac34;": '\U000000BE', - "frac35;": '\U00002157', - "frac38;": '\U0000215C', - "frac45;": '\U00002158', - "frac56;": '\U0000215A', - "frac58;": '\U0000215D', - "frac78;": '\U0000215E', - "frasl;": '\U00002044', - "frown;": '\U00002322', - "fscr;": '\U0001D4BB', - "gE;": '\U00002267', - "gEl;": '\U00002A8C', - "gacute;": '\U000001F5', - "gamma;": '\U000003B3', - "gammad;": '\U000003DD', - "gap;": '\U00002A86', - "gbreve;": '\U0000011F', - "gcirc;": '\U0000011D', - "gcy;": '\U00000433', - "gdot;": '\U00000121', - "ge;": '\U00002265', - "gel;": '\U000022DB', - "geq;": '\U00002265', - "geqq;": '\U00002267', - "geqslant;": '\U00002A7E', - "ges;": '\U00002A7E', - "gescc;": '\U00002AA9', - "gesdot;": '\U00002A80', - "gesdoto;": '\U00002A82', - "gesdotol;": '\U00002A84', - "gesles;": '\U00002A94', - "gfr;": '\U0001D524', - "gg;": '\U0000226B', - "ggg;": '\U000022D9', - "gimel;": '\U00002137', - "gjcy;": '\U00000453', - "gl;": '\U00002277', - "glE;": '\U00002A92', - "gla;": '\U00002AA5', - "glj;": '\U00002AA4', - "gnE;": '\U00002269', - "gnap;": '\U00002A8A', - "gnapprox;": '\U00002A8A', - "gne;": '\U00002A88', - "gneq;": '\U00002A88', - "gneqq;": '\U00002269', - "gnsim;": '\U000022E7', - "gopf;": '\U0001D558', - "grave;": '\U00000060', - "gscr;": '\U0000210A', - "gsim;": '\U00002273', - "gsime;": '\U00002A8E', - "gsiml;": '\U00002A90', - "gt;": '\U0000003E', - "gtcc;": '\U00002AA7', - "gtcir;": '\U00002A7A', - "gtdot;": '\U000022D7', - "gtlPar;": '\U00002995', - "gtquest;": '\U00002A7C', - "gtrapprox;": '\U00002A86', - "gtrarr;": '\U00002978', - "gtrdot;": '\U000022D7', - "gtreqless;": '\U000022DB', - "gtreqqless;": '\U00002A8C', - "gtrless;": '\U00002277', - "gtrsim;": '\U00002273', - "hArr;": '\U000021D4', - "hairsp;": '\U0000200A', - "half;": '\U000000BD', - "hamilt;": '\U0000210B', - "hardcy;": '\U0000044A', - "harr;": '\U00002194', - "harrcir;": '\U00002948', - "harrw;": '\U000021AD', - "hbar;": '\U0000210F', - "hcirc;": '\U00000125', - "hearts;": '\U00002665', - "heartsuit;": '\U00002665', - "hellip;": '\U00002026', - "hercon;": '\U000022B9', - "hfr;": '\U0001D525', - "hksearow;": '\U00002925', - "hkswarow;": '\U00002926', - "hoarr;": '\U000021FF', - "homtht;": '\U0000223B', - "hookleftarrow;": '\U000021A9', - "hookrightarrow;": '\U000021AA', - "hopf;": '\U0001D559', - "horbar;": '\U00002015', - "hscr;": '\U0001D4BD', - "hslash;": '\U0000210F', - "hstrok;": '\U00000127', - "hybull;": '\U00002043', - "hyphen;": '\U00002010', - "iacute;": '\U000000ED', - "ic;": '\U00002063', - "icirc;": '\U000000EE', - "icy;": '\U00000438', - "iecy;": '\U00000435', - "iexcl;": '\U000000A1', - "iff;": '\U000021D4', - "ifr;": '\U0001D526', - "igrave;": '\U000000EC', - "ii;": '\U00002148', - "iiiint;": '\U00002A0C', - "iiint;": '\U0000222D', - "iinfin;": '\U000029DC', - "iiota;": '\U00002129', - "ijlig;": '\U00000133', - "imacr;": '\U0000012B', - "image;": '\U00002111', - "imagline;": '\U00002110', - "imagpart;": '\U00002111', - "imath;": '\U00000131', - "imof;": '\U000022B7', - "imped;": '\U000001B5', - "in;": '\U00002208', - "incare;": '\U00002105', - "infin;": '\U0000221E', - "infintie;": '\U000029DD', - "inodot;": '\U00000131', - "int;": '\U0000222B', - "intcal;": '\U000022BA', - "integers;": '\U00002124', - "intercal;": '\U000022BA', - "intlarhk;": '\U00002A17', - "intprod;": '\U00002A3C', - "iocy;": '\U00000451', - "iogon;": '\U0000012F', - "iopf;": '\U0001D55A', - "iota;": '\U000003B9', - "iprod;": '\U00002A3C', - "iquest;": '\U000000BF', - "iscr;": '\U0001D4BE', - "isin;": '\U00002208', - "isinE;": '\U000022F9', - "isindot;": '\U000022F5', - "isins;": '\U000022F4', - "isinsv;": '\U000022F3', - "isinv;": '\U00002208', - "it;": '\U00002062', - "itilde;": '\U00000129', - "iukcy;": '\U00000456', - "iuml;": '\U000000EF', - "jcirc;": '\U00000135', - "jcy;": '\U00000439', - "jfr;": '\U0001D527', - "jmath;": '\U00000237', - "jopf;": '\U0001D55B', - "jscr;": '\U0001D4BF', - "jsercy;": '\U00000458', - "jukcy;": '\U00000454', - "kappa;": '\U000003BA', - "kappav;": '\U000003F0', - "kcedil;": '\U00000137', - "kcy;": '\U0000043A', - "kfr;": '\U0001D528', - "kgreen;": '\U00000138', - "khcy;": '\U00000445', - "kjcy;": '\U0000045C', - "kopf;": '\U0001D55C', - "kscr;": '\U0001D4C0', - "lAarr;": '\U000021DA', - "lArr;": '\U000021D0', - "lAtail;": '\U0000291B', - "lBarr;": '\U0000290E', - "lE;": '\U00002266', - "lEg;": '\U00002A8B', - "lHar;": '\U00002962', - "lacute;": '\U0000013A', - "laemptyv;": '\U000029B4', - "lagran;": '\U00002112', - "lambda;": '\U000003BB', - "lang;": '\U000027E8', - "langd;": '\U00002991', - "langle;": '\U000027E8', - "lap;": '\U00002A85', - "laquo;": '\U000000AB', - "larr;": '\U00002190', - "larrb;": '\U000021E4', - "larrbfs;": '\U0000291F', - "larrfs;": '\U0000291D', - "larrhk;": '\U000021A9', - "larrlp;": '\U000021AB', - "larrpl;": '\U00002939', - "larrsim;": '\U00002973', - "larrtl;": '\U000021A2', - "lat;": '\U00002AAB', - "latail;": '\U00002919', - "late;": '\U00002AAD', - "lbarr;": '\U0000290C', - "lbbrk;": '\U00002772', - "lbrace;": '\U0000007B', - "lbrack;": '\U0000005B', - "lbrke;": '\U0000298B', - "lbrksld;": '\U0000298F', - "lbrkslu;": '\U0000298D', - "lcaron;": '\U0000013E', - "lcedil;": '\U0000013C', - "lceil;": '\U00002308', - "lcub;": '\U0000007B', - "lcy;": '\U0000043B', - "ldca;": '\U00002936', - "ldquo;": '\U0000201C', - "ldquor;": '\U0000201E', - "ldrdhar;": '\U00002967', - "ldrushar;": '\U0000294B', - "ldsh;": '\U000021B2', - "le;": '\U00002264', - "leftarrow;": '\U00002190', - "leftarrowtail;": '\U000021A2', - "leftharpoondown;": '\U000021BD', - "leftharpoonup;": '\U000021BC', - "leftleftarrows;": '\U000021C7', - "leftrightarrow;": '\U00002194', - "leftrightarrows;": '\U000021C6', - "leftrightharpoons;": '\U000021CB', - "leftrightsquigarrow;": '\U000021AD', - "leftthreetimes;": '\U000022CB', - "leg;": '\U000022DA', - "leq;": '\U00002264', - "leqq;": '\U00002266', - "leqslant;": '\U00002A7D', - "les;": '\U00002A7D', - "lescc;": '\U00002AA8', - "lesdot;": '\U00002A7F', - "lesdoto;": '\U00002A81', - "lesdotor;": '\U00002A83', - "lesges;": '\U00002A93', - "lessapprox;": '\U00002A85', - "lessdot;": '\U000022D6', - "lesseqgtr;": '\U000022DA', - "lesseqqgtr;": '\U00002A8B', - "lessgtr;": '\U00002276', - "lesssim;": '\U00002272', - "lfisht;": '\U0000297C', - "lfloor;": '\U0000230A', - "lfr;": '\U0001D529', - "lg;": '\U00002276', - "lgE;": '\U00002A91', - "lhard;": '\U000021BD', - "lharu;": '\U000021BC', - "lharul;": '\U0000296A', - "lhblk;": '\U00002584', - "ljcy;": '\U00000459', - "ll;": '\U0000226A', - "llarr;": '\U000021C7', - "llcorner;": '\U0000231E', - "llhard;": '\U0000296B', - "lltri;": '\U000025FA', - "lmidot;": '\U00000140', - "lmoust;": '\U000023B0', - "lmoustache;": '\U000023B0', - "lnE;": '\U00002268', - "lnap;": '\U00002A89', - "lnapprox;": '\U00002A89', - "lne;": '\U00002A87', - "lneq;": '\U00002A87', - "lneqq;": '\U00002268', - "lnsim;": '\U000022E6', - "loang;": '\U000027EC', - "loarr;": '\U000021FD', - "lobrk;": '\U000027E6', - "longleftarrow;": '\U000027F5', - "longleftrightarrow;": '\U000027F7', - "longmapsto;": '\U000027FC', - "longrightarrow;": '\U000027F6', - "looparrowleft;": '\U000021AB', - "looparrowright;": '\U000021AC', - "lopar;": '\U00002985', - "lopf;": '\U0001D55D', - "loplus;": '\U00002A2D', - "lotimes;": '\U00002A34', - "lowast;": '\U00002217', - "lowbar;": '\U0000005F', - "loz;": '\U000025CA', - "lozenge;": '\U000025CA', - "lozf;": '\U000029EB', - "lpar;": '\U00000028', - "lparlt;": '\U00002993', - "lrarr;": '\U000021C6', - "lrcorner;": '\U0000231F', - "lrhar;": '\U000021CB', - "lrhard;": '\U0000296D', - "lrm;": '\U0000200E', - "lrtri;": '\U000022BF', - "lsaquo;": '\U00002039', - "lscr;": '\U0001D4C1', - "lsh;": '\U000021B0', - "lsim;": '\U00002272', - "lsime;": '\U00002A8D', - "lsimg;": '\U00002A8F', - "lsqb;": '\U0000005B', - "lsquo;": '\U00002018', - "lsquor;": '\U0000201A', - "lstrok;": '\U00000142', - "lt;": '\U0000003C', - "ltcc;": '\U00002AA6', - "ltcir;": '\U00002A79', - "ltdot;": '\U000022D6', - "lthree;": '\U000022CB', - "ltimes;": '\U000022C9', - "ltlarr;": '\U00002976', - "ltquest;": '\U00002A7B', - "ltrPar;": '\U00002996', - "ltri;": '\U000025C3', - "ltrie;": '\U000022B4', - "ltrif;": '\U000025C2', - "lurdshar;": '\U0000294A', - "luruhar;": '\U00002966', - "mDDot;": '\U0000223A', - "macr;": '\U000000AF', - "male;": '\U00002642', - "malt;": '\U00002720', - "maltese;": '\U00002720', - "map;": '\U000021A6', - "mapsto;": '\U000021A6', - "mapstodown;": '\U000021A7', - "mapstoleft;": '\U000021A4', - "mapstoup;": '\U000021A5', - "marker;": '\U000025AE', - "mcomma;": '\U00002A29', - "mcy;": '\U0000043C', - "mdash;": '\U00002014', - "measuredangle;": '\U00002221', - "mfr;": '\U0001D52A', - "mho;": '\U00002127', - "micro;": '\U000000B5', - "mid;": '\U00002223', - "midast;": '\U0000002A', - "midcir;": '\U00002AF0', - "middot;": '\U000000B7', - "minus;": '\U00002212', - "minusb;": '\U0000229F', - "minusd;": '\U00002238', - "minusdu;": '\U00002A2A', - "mlcp;": '\U00002ADB', - "mldr;": '\U00002026', - "mnplus;": '\U00002213', - "models;": '\U000022A7', - "mopf;": '\U0001D55E', - "mp;": '\U00002213', - "mscr;": '\U0001D4C2', - "mstpos;": '\U0000223E', - "mu;": '\U000003BC', - "multimap;": '\U000022B8', - "mumap;": '\U000022B8', - "nLeftarrow;": '\U000021CD', - "nLeftrightarrow;": '\U000021CE', - "nRightarrow;": '\U000021CF', - "nVDash;": '\U000022AF', - "nVdash;": '\U000022AE', - "nabla;": '\U00002207', - "nacute;": '\U00000144', - "nap;": '\U00002249', - "napos;": '\U00000149', - "napprox;": '\U00002249', - "natur;": '\U0000266E', - "natural;": '\U0000266E', - "naturals;": '\U00002115', - "nbsp;": '\U000000A0', - "ncap;": '\U00002A43', - "ncaron;": '\U00000148', - "ncedil;": '\U00000146', - "ncong;": '\U00002247', - "ncup;": '\U00002A42', - "ncy;": '\U0000043D', - "ndash;": '\U00002013', - "ne;": '\U00002260', - "neArr;": '\U000021D7', - "nearhk;": '\U00002924', - "nearr;": '\U00002197', - "nearrow;": '\U00002197', - "nequiv;": '\U00002262', - "nesear;": '\U00002928', - "nexist;": '\U00002204', - "nexists;": '\U00002204', - "nfr;": '\U0001D52B', - "nge;": '\U00002271', - "ngeq;": '\U00002271', - "ngsim;": '\U00002275', - "ngt;": '\U0000226F', - "ngtr;": '\U0000226F', - "nhArr;": '\U000021CE', - "nharr;": '\U000021AE', - "nhpar;": '\U00002AF2', - "ni;": '\U0000220B', - "nis;": '\U000022FC', - "nisd;": '\U000022FA', - "niv;": '\U0000220B', - "njcy;": '\U0000045A', - "nlArr;": '\U000021CD', - "nlarr;": '\U0000219A', - "nldr;": '\U00002025', - "nle;": '\U00002270', - "nleftarrow;": '\U0000219A', - "nleftrightarrow;": '\U000021AE', - "nleq;": '\U00002270', - "nless;": '\U0000226E', - "nlsim;": '\U00002274', - "nlt;": '\U0000226E', - "nltri;": '\U000022EA', - "nltrie;": '\U000022EC', - "nmid;": '\U00002224', - "nopf;": '\U0001D55F', - "not;": '\U000000AC', - "notin;": '\U00002209', - "notinva;": '\U00002209', - "notinvb;": '\U000022F7', - "notinvc;": '\U000022F6', - "notni;": '\U0000220C', - "notniva;": '\U0000220C', - "notnivb;": '\U000022FE', - "notnivc;": '\U000022FD', - "npar;": '\U00002226', - "nparallel;": '\U00002226', - "npolint;": '\U00002A14', - "npr;": '\U00002280', - "nprcue;": '\U000022E0', - "nprec;": '\U00002280', - "nrArr;": '\U000021CF', - "nrarr;": '\U0000219B', - "nrightarrow;": '\U0000219B', - "nrtri;": '\U000022EB', - "nrtrie;": '\U000022ED', - "nsc;": '\U00002281', - "nsccue;": '\U000022E1', - "nscr;": '\U0001D4C3', - "nshortmid;": '\U00002224', - "nshortparallel;": '\U00002226', - "nsim;": '\U00002241', - "nsime;": '\U00002244', - "nsimeq;": '\U00002244', - "nsmid;": '\U00002224', - "nspar;": '\U00002226', - "nsqsube;": '\U000022E2', - "nsqsupe;": '\U000022E3', - "nsub;": '\U00002284', - "nsube;": '\U00002288', - "nsubseteq;": '\U00002288', - "nsucc;": '\U00002281', - "nsup;": '\U00002285', - "nsupe;": '\U00002289', - "nsupseteq;": '\U00002289', - "ntgl;": '\U00002279', - "ntilde;": '\U000000F1', - "ntlg;": '\U00002278', - "ntriangleleft;": '\U000022EA', - "ntrianglelefteq;": '\U000022EC', - "ntriangleright;": '\U000022EB', - "ntrianglerighteq;": '\U000022ED', - "nu;": '\U000003BD', - "num;": '\U00000023', - "numero;": '\U00002116', - "numsp;": '\U00002007', - "nvDash;": '\U000022AD', - "nvHarr;": '\U00002904', - "nvdash;": '\U000022AC', - "nvinfin;": '\U000029DE', - "nvlArr;": '\U00002902', - "nvrArr;": '\U00002903', - "nwArr;": '\U000021D6', - "nwarhk;": '\U00002923', - "nwarr;": '\U00002196', - "nwarrow;": '\U00002196', - "nwnear;": '\U00002927', - "oS;": '\U000024C8', - "oacute;": '\U000000F3', - "oast;": '\U0000229B', - "ocir;": '\U0000229A', - "ocirc;": '\U000000F4', - "ocy;": '\U0000043E', - "odash;": '\U0000229D', - "odblac;": '\U00000151', - "odiv;": '\U00002A38', - "odot;": '\U00002299', - "odsold;": '\U000029BC', - "oelig;": '\U00000153', - "ofcir;": '\U000029BF', - "ofr;": '\U0001D52C', - "ogon;": '\U000002DB', - "ograve;": '\U000000F2', - "ogt;": '\U000029C1', - "ohbar;": '\U000029B5', - "ohm;": '\U000003A9', - "oint;": '\U0000222E', - "olarr;": '\U000021BA', - "olcir;": '\U000029BE', - "olcross;": '\U000029BB', - "oline;": '\U0000203E', - "olt;": '\U000029C0', - "omacr;": '\U0000014D', - "omega;": '\U000003C9', - "omicron;": '\U000003BF', - "omid;": '\U000029B6', - "ominus;": '\U00002296', - "oopf;": '\U0001D560', - "opar;": '\U000029B7', - "operp;": '\U000029B9', - "oplus;": '\U00002295', - "or;": '\U00002228', - "orarr;": '\U000021BB', - "ord;": '\U00002A5D', - "order;": '\U00002134', - "orderof;": '\U00002134', - "ordf;": '\U000000AA', - "ordm;": '\U000000BA', - "origof;": '\U000022B6', - "oror;": '\U00002A56', - "orslope;": '\U00002A57', - "orv;": '\U00002A5B', - "oscr;": '\U00002134', - "oslash;": '\U000000F8', - "osol;": '\U00002298', - "otilde;": '\U000000F5', - "otimes;": '\U00002297', - "otimesas;": '\U00002A36', - "ouml;": '\U000000F6', - "ovbar;": '\U0000233D', - "par;": '\U00002225', - "para;": '\U000000B6', - "parallel;": '\U00002225', - "parsim;": '\U00002AF3', - "parsl;": '\U00002AFD', - "part;": '\U00002202', - "pcy;": '\U0000043F', - "percnt;": '\U00000025', - "period;": '\U0000002E', - "permil;": '\U00002030', - "perp;": '\U000022A5', - "pertenk;": '\U00002031', - "pfr;": '\U0001D52D', - "phi;": '\U000003C6', - "phiv;": '\U000003D5', - "phmmat;": '\U00002133', - "phone;": '\U0000260E', - "pi;": '\U000003C0', - "pitchfork;": '\U000022D4', - "piv;": '\U000003D6', - "planck;": '\U0000210F', - "planckh;": '\U0000210E', - "plankv;": '\U0000210F', - "plus;": '\U0000002B', - "plusacir;": '\U00002A23', - "plusb;": '\U0000229E', - "pluscir;": '\U00002A22', - "plusdo;": '\U00002214', - "plusdu;": '\U00002A25', - "pluse;": '\U00002A72', - "plusmn;": '\U000000B1', - "plussim;": '\U00002A26', - "plustwo;": '\U00002A27', - "pm;": '\U000000B1', - "pointint;": '\U00002A15', - "popf;": '\U0001D561', - "pound;": '\U000000A3', - "pr;": '\U0000227A', - "prE;": '\U00002AB3', - "prap;": '\U00002AB7', - "prcue;": '\U0000227C', - "pre;": '\U00002AAF', - "prec;": '\U0000227A', - "precapprox;": '\U00002AB7', - "preccurlyeq;": '\U0000227C', - "preceq;": '\U00002AAF', - "precnapprox;": '\U00002AB9', - "precneqq;": '\U00002AB5', - "precnsim;": '\U000022E8', - "precsim;": '\U0000227E', - "prime;": '\U00002032', - "primes;": '\U00002119', - "prnE;": '\U00002AB5', - "prnap;": '\U00002AB9', - "prnsim;": '\U000022E8', - "prod;": '\U0000220F', - "profalar;": '\U0000232E', - "profline;": '\U00002312', - "profsurf;": '\U00002313', - "prop;": '\U0000221D', - "propto;": '\U0000221D', - "prsim;": '\U0000227E', - "prurel;": '\U000022B0', - "pscr;": '\U0001D4C5', - "psi;": '\U000003C8', - "puncsp;": '\U00002008', - "qfr;": '\U0001D52E', - "qint;": '\U00002A0C', - "qopf;": '\U0001D562', - "qprime;": '\U00002057', - "qscr;": '\U0001D4C6', - "quaternions;": '\U0000210D', - "quatint;": '\U00002A16', - "quest;": '\U0000003F', - "questeq;": '\U0000225F', - "quot;": '\U00000022', - "rAarr;": '\U000021DB', - "rArr;": '\U000021D2', - "rAtail;": '\U0000291C', - "rBarr;": '\U0000290F', - "rHar;": '\U00002964', - "racute;": '\U00000155', - "radic;": '\U0000221A', - "raemptyv;": '\U000029B3', - "rang;": '\U000027E9', - "rangd;": '\U00002992', - "range;": '\U000029A5', - "rangle;": '\U000027E9', - "raquo;": '\U000000BB', - "rarr;": '\U00002192', - "rarrap;": '\U00002975', - "rarrb;": '\U000021E5', - "rarrbfs;": '\U00002920', - "rarrc;": '\U00002933', - "rarrfs;": '\U0000291E', - "rarrhk;": '\U000021AA', - "rarrlp;": '\U000021AC', - "rarrpl;": '\U00002945', - "rarrsim;": '\U00002974', - "rarrtl;": '\U000021A3', - "rarrw;": '\U0000219D', - "ratail;": '\U0000291A', - "ratio;": '\U00002236', - "rationals;": '\U0000211A', - "rbarr;": '\U0000290D', - "rbbrk;": '\U00002773', - "rbrace;": '\U0000007D', - "rbrack;": '\U0000005D', - "rbrke;": '\U0000298C', - "rbrksld;": '\U0000298E', - "rbrkslu;": '\U00002990', - "rcaron;": '\U00000159', - "rcedil;": '\U00000157', - "rceil;": '\U00002309', - "rcub;": '\U0000007D', - "rcy;": '\U00000440', - "rdca;": '\U00002937', - "rdldhar;": '\U00002969', - "rdquo;": '\U0000201D', - "rdquor;": '\U0000201D', - "rdsh;": '\U000021B3', - "real;": '\U0000211C', - "realine;": '\U0000211B', - "realpart;": '\U0000211C', - "reals;": '\U0000211D', - "rect;": '\U000025AD', - "reg;": '\U000000AE', - "rfisht;": '\U0000297D', - "rfloor;": '\U0000230B', - "rfr;": '\U0001D52F', - "rhard;": '\U000021C1', - "rharu;": '\U000021C0', - "rharul;": '\U0000296C', - "rho;": '\U000003C1', - "rhov;": '\U000003F1', - "rightarrow;": '\U00002192', - "rightarrowtail;": '\U000021A3', - "rightharpoondown;": '\U000021C1', - "rightharpoonup;": '\U000021C0', - "rightleftarrows;": '\U000021C4', - "rightleftharpoons;": '\U000021CC', - "rightrightarrows;": '\U000021C9', - "rightsquigarrow;": '\U0000219D', - "rightthreetimes;": '\U000022CC', - "ring;": '\U000002DA', - "risingdotseq;": '\U00002253', - "rlarr;": '\U000021C4', - "rlhar;": '\U000021CC', - "rlm;": '\U0000200F', - "rmoust;": '\U000023B1', - "rmoustache;": '\U000023B1', - "rnmid;": '\U00002AEE', - "roang;": '\U000027ED', - "roarr;": '\U000021FE', - "robrk;": '\U000027E7', - "ropar;": '\U00002986', - "ropf;": '\U0001D563', - "roplus;": '\U00002A2E', - "rotimes;": '\U00002A35', - "rpar;": '\U00000029', - "rpargt;": '\U00002994', - "rppolint;": '\U00002A12', - "rrarr;": '\U000021C9', - "rsaquo;": '\U0000203A', - "rscr;": '\U0001D4C7', - "rsh;": '\U000021B1', - "rsqb;": '\U0000005D', - "rsquo;": '\U00002019', - "rsquor;": '\U00002019', - "rthree;": '\U000022CC', - "rtimes;": '\U000022CA', - "rtri;": '\U000025B9', - "rtrie;": '\U000022B5', - "rtrif;": '\U000025B8', - "rtriltri;": '\U000029CE', - "ruluhar;": '\U00002968', - "rx;": '\U0000211E', - "sacute;": '\U0000015B', - "sbquo;": '\U0000201A', - "sc;": '\U0000227B', - "scE;": '\U00002AB4', - "scap;": '\U00002AB8', - "scaron;": '\U00000161', - "sccue;": '\U0000227D', - "sce;": '\U00002AB0', - "scedil;": '\U0000015F', - "scirc;": '\U0000015D', - "scnE;": '\U00002AB6', - "scnap;": '\U00002ABA', - "scnsim;": '\U000022E9', - "scpolint;": '\U00002A13', - "scsim;": '\U0000227F', - "scy;": '\U00000441', - "sdot;": '\U000022C5', - "sdotb;": '\U000022A1', - "sdote;": '\U00002A66', - "seArr;": '\U000021D8', - "searhk;": '\U00002925', - "searr;": '\U00002198', - "searrow;": '\U00002198', - "sect;": '\U000000A7', - "semi;": '\U0000003B', - "seswar;": '\U00002929', - "setminus;": '\U00002216', - "setmn;": '\U00002216', - "sext;": '\U00002736', - "sfr;": '\U0001D530', - "sfrown;": '\U00002322', - "sharp;": '\U0000266F', - "shchcy;": '\U00000449', - "shcy;": '\U00000448', - "shortmid;": '\U00002223', - "shortparallel;": '\U00002225', - "shy;": '\U000000AD', - "sigma;": '\U000003C3', - "sigmaf;": '\U000003C2', - "sigmav;": '\U000003C2', - "sim;": '\U0000223C', - "simdot;": '\U00002A6A', - "sime;": '\U00002243', - "simeq;": '\U00002243', - "simg;": '\U00002A9E', - "simgE;": '\U00002AA0', - "siml;": '\U00002A9D', - "simlE;": '\U00002A9F', - "simne;": '\U00002246', - "simplus;": '\U00002A24', - "simrarr;": '\U00002972', - "slarr;": '\U00002190', - "smallsetminus;": '\U00002216', - "smashp;": '\U00002A33', - "smeparsl;": '\U000029E4', - "smid;": '\U00002223', - "smile;": '\U00002323', - "smt;": '\U00002AAA', - "smte;": '\U00002AAC', - "softcy;": '\U0000044C', - "sol;": '\U0000002F', - "solb;": '\U000029C4', - "solbar;": '\U0000233F', - "sopf;": '\U0001D564', - "spades;": '\U00002660', - "spadesuit;": '\U00002660', - "spar;": '\U00002225', - "sqcap;": '\U00002293', - "sqcup;": '\U00002294', - "sqsub;": '\U0000228F', - "sqsube;": '\U00002291', - "sqsubset;": '\U0000228F', - "sqsubseteq;": '\U00002291', - "sqsup;": '\U00002290', - "sqsupe;": '\U00002292', - "sqsupset;": '\U00002290', - "sqsupseteq;": '\U00002292', - "squ;": '\U000025A1', - "square;": '\U000025A1', - "squarf;": '\U000025AA', - "squf;": '\U000025AA', - "srarr;": '\U00002192', - "sscr;": '\U0001D4C8', - "ssetmn;": '\U00002216', - "ssmile;": '\U00002323', - "sstarf;": '\U000022C6', - "star;": '\U00002606', - "starf;": '\U00002605', - "straightepsilon;": '\U000003F5', - "straightphi;": '\U000003D5', - "strns;": '\U000000AF', - "sub;": '\U00002282', - "subE;": '\U00002AC5', - "subdot;": '\U00002ABD', - "sube;": '\U00002286', - "subedot;": '\U00002AC3', - "submult;": '\U00002AC1', - "subnE;": '\U00002ACB', - "subne;": '\U0000228A', - "subplus;": '\U00002ABF', - "subrarr;": '\U00002979', - "subset;": '\U00002282', - "subseteq;": '\U00002286', - "subseteqq;": '\U00002AC5', - "subsetneq;": '\U0000228A', - "subsetneqq;": '\U00002ACB', - "subsim;": '\U00002AC7', - "subsub;": '\U00002AD5', - "subsup;": '\U00002AD3', - "succ;": '\U0000227B', - "succapprox;": '\U00002AB8', - "succcurlyeq;": '\U0000227D', - "succeq;": '\U00002AB0', - "succnapprox;": '\U00002ABA', - "succneqq;": '\U00002AB6', - "succnsim;": '\U000022E9', - "succsim;": '\U0000227F', - "sum;": '\U00002211', - "sung;": '\U0000266A', - "sup;": '\U00002283', - "sup1;": '\U000000B9', - "sup2;": '\U000000B2', - "sup3;": '\U000000B3', - "supE;": '\U00002AC6', - "supdot;": '\U00002ABE', - "supdsub;": '\U00002AD8', - "supe;": '\U00002287', - "supedot;": '\U00002AC4', - "suphsol;": '\U000027C9', - "suphsub;": '\U00002AD7', - "suplarr;": '\U0000297B', - "supmult;": '\U00002AC2', - "supnE;": '\U00002ACC', - "supne;": '\U0000228B', - "supplus;": '\U00002AC0', - "supset;": '\U00002283', - "supseteq;": '\U00002287', - "supseteqq;": '\U00002AC6', - "supsetneq;": '\U0000228B', - "supsetneqq;": '\U00002ACC', - "supsim;": '\U00002AC8', - "supsub;": '\U00002AD4', - "supsup;": '\U00002AD6', - "swArr;": '\U000021D9', - "swarhk;": '\U00002926', - "swarr;": '\U00002199', - "swarrow;": '\U00002199', - "swnwar;": '\U0000292A', - "szlig;": '\U000000DF', - "target;": '\U00002316', - "tau;": '\U000003C4', - "tbrk;": '\U000023B4', - "tcaron;": '\U00000165', - "tcedil;": '\U00000163', - "tcy;": '\U00000442', - "tdot;": '\U000020DB', - "telrec;": '\U00002315', - "tfr;": '\U0001D531', - "there4;": '\U00002234', - "therefore;": '\U00002234', - "theta;": '\U000003B8', - "thetasym;": '\U000003D1', - "thetav;": '\U000003D1', - "thickapprox;": '\U00002248', - "thicksim;": '\U0000223C', - "thinsp;": '\U00002009', - "thkap;": '\U00002248', - "thksim;": '\U0000223C', - "thorn;": '\U000000FE', - "tilde;": '\U000002DC', - "times;": '\U000000D7', - "timesb;": '\U000022A0', - "timesbar;": '\U00002A31', - "timesd;": '\U00002A30', - "tint;": '\U0000222D', - "toea;": '\U00002928', - "top;": '\U000022A4', - "topbot;": '\U00002336', - "topcir;": '\U00002AF1', - "topf;": '\U0001D565', - "topfork;": '\U00002ADA', - "tosa;": '\U00002929', - "tprime;": '\U00002034', - "trade;": '\U00002122', - "triangle;": '\U000025B5', - "triangledown;": '\U000025BF', - "triangleleft;": '\U000025C3', - "trianglelefteq;": '\U000022B4', - "triangleq;": '\U0000225C', - "triangleright;": '\U000025B9', - "trianglerighteq;": '\U000022B5', - "tridot;": '\U000025EC', - "trie;": '\U0000225C', - "triminus;": '\U00002A3A', - "triplus;": '\U00002A39', - "trisb;": '\U000029CD', - "tritime;": '\U00002A3B', - "trpezium;": '\U000023E2', - "tscr;": '\U0001D4C9', - "tscy;": '\U00000446', - "tshcy;": '\U0000045B', - "tstrok;": '\U00000167', - "twixt;": '\U0000226C', - "twoheadleftarrow;": '\U0000219E', - "twoheadrightarrow;": '\U000021A0', - "uArr;": '\U000021D1', - "uHar;": '\U00002963', - "uacute;": '\U000000FA', - "uarr;": '\U00002191', - "ubrcy;": '\U0000045E', - "ubreve;": '\U0000016D', - "ucirc;": '\U000000FB', - "ucy;": '\U00000443', - "udarr;": '\U000021C5', - "udblac;": '\U00000171', - "udhar;": '\U0000296E', - "ufisht;": '\U0000297E', - "ufr;": '\U0001D532', - "ugrave;": '\U000000F9', - "uharl;": '\U000021BF', - "uharr;": '\U000021BE', - "uhblk;": '\U00002580', - "ulcorn;": '\U0000231C', - "ulcorner;": '\U0000231C', - "ulcrop;": '\U0000230F', - "ultri;": '\U000025F8', - "umacr;": '\U0000016B', - "uml;": '\U000000A8', - "uogon;": '\U00000173', - "uopf;": '\U0001D566', - "uparrow;": '\U00002191', - "updownarrow;": '\U00002195', - "upharpoonleft;": '\U000021BF', - "upharpoonright;": '\U000021BE', - "uplus;": '\U0000228E', - "upsi;": '\U000003C5', - "upsih;": '\U000003D2', - "upsilon;": '\U000003C5', - "upuparrows;": '\U000021C8', - "urcorn;": '\U0000231D', - "urcorner;": '\U0000231D', - "urcrop;": '\U0000230E', - "uring;": '\U0000016F', - "urtri;": '\U000025F9', - "uscr;": '\U0001D4CA', - "utdot;": '\U000022F0', - "utilde;": '\U00000169', - "utri;": '\U000025B5', - "utrif;": '\U000025B4', - "uuarr;": '\U000021C8', - "uuml;": '\U000000FC', - "uwangle;": '\U000029A7', - "vArr;": '\U000021D5', - "vBar;": '\U00002AE8', - "vBarv;": '\U00002AE9', - "vDash;": '\U000022A8', - "vangrt;": '\U0000299C', - "varepsilon;": '\U000003F5', - "varkappa;": '\U000003F0', - "varnothing;": '\U00002205', - "varphi;": '\U000003D5', - "varpi;": '\U000003D6', - "varpropto;": '\U0000221D', - "varr;": '\U00002195', - "varrho;": '\U000003F1', - "varsigma;": '\U000003C2', - "vartheta;": '\U000003D1', - "vartriangleleft;": '\U000022B2', - "vartriangleright;": '\U000022B3', - "vcy;": '\U00000432', - "vdash;": '\U000022A2', - "vee;": '\U00002228', - "veebar;": '\U000022BB', - "veeeq;": '\U0000225A', - "vellip;": '\U000022EE', - "verbar;": '\U0000007C', - "vert;": '\U0000007C', - "vfr;": '\U0001D533', - "vltri;": '\U000022B2', - "vopf;": '\U0001D567', - "vprop;": '\U0000221D', - "vrtri;": '\U000022B3', - "vscr;": '\U0001D4CB', - "vzigzag;": '\U0000299A', - "wcirc;": '\U00000175', - "wedbar;": '\U00002A5F', - "wedge;": '\U00002227', - "wedgeq;": '\U00002259', - "weierp;": '\U00002118', - "wfr;": '\U0001D534', - "wopf;": '\U0001D568', - "wp;": '\U00002118', - "wr;": '\U00002240', - "wreath;": '\U00002240', - "wscr;": '\U0001D4CC', - "xcap;": '\U000022C2', - "xcirc;": '\U000025EF', - "xcup;": '\U000022C3', - "xdtri;": '\U000025BD', - "xfr;": '\U0001D535', - "xhArr;": '\U000027FA', - "xharr;": '\U000027F7', - "xi;": '\U000003BE', - "xlArr;": '\U000027F8', - "xlarr;": '\U000027F5', - "xmap;": '\U000027FC', - "xnis;": '\U000022FB', - "xodot;": '\U00002A00', - "xopf;": '\U0001D569', - "xoplus;": '\U00002A01', - "xotime;": '\U00002A02', - "xrArr;": '\U000027F9', - "xrarr;": '\U000027F6', - "xscr;": '\U0001D4CD', - "xsqcup;": '\U00002A06', - "xuplus;": '\U00002A04', - "xutri;": '\U000025B3', - "xvee;": '\U000022C1', - "xwedge;": '\U000022C0', - "yacute;": '\U000000FD', - "yacy;": '\U0000044F', - "ycirc;": '\U00000177', - "ycy;": '\U0000044B', - "yen;": '\U000000A5', - "yfr;": '\U0001D536', - "yicy;": '\U00000457', - "yopf;": '\U0001D56A', - "yscr;": '\U0001D4CE', - "yucy;": '\U0000044E', - "yuml;": '\U000000FF', - "zacute;": '\U0000017A', - "zcaron;": '\U0000017E', - "zcy;": '\U00000437', - "zdot;": '\U0000017C', - "zeetrf;": '\U00002128', - "zeta;": '\U000003B6', - "zfr;": '\U0001D537', - "zhcy;": '\U00000436', - "zigrarr;": '\U000021DD', - "zopf;": '\U0001D56B', - "zscr;": '\U0001D4CF', - "zwj;": '\U0000200D', - "zwnj;": '\U0000200C', - "AElig": '\U000000C6', - "AMP": '\U00000026', - "Aacute": '\U000000C1', - "Acirc": '\U000000C2', - "Agrave": '\U000000C0', - "Aring": '\U000000C5', - "Atilde": '\U000000C3', - "Auml": '\U000000C4', - "COPY": '\U000000A9', - "Ccedil": '\U000000C7', - "ETH": '\U000000D0', - "Eacute": '\U000000C9', - "Ecirc": '\U000000CA', - "Egrave": '\U000000C8', - "Euml": '\U000000CB', - "GT": '\U0000003E', - "Iacute": '\U000000CD', - "Icirc": '\U000000CE', - "Igrave": '\U000000CC', - "Iuml": '\U000000CF', - "LT": '\U0000003C', - "Ntilde": '\U000000D1', - "Oacute": '\U000000D3', - "Ocirc": '\U000000D4', - "Ograve": '\U000000D2', - "Oslash": '\U000000D8', - "Otilde": '\U000000D5', - "Ouml": '\U000000D6', - "QUOT": '\U00000022', - "REG": '\U000000AE', - "THORN": '\U000000DE', - "Uacute": '\U000000DA', - "Ucirc": '\U000000DB', - "Ugrave": '\U000000D9', - "Uuml": '\U000000DC', - "Yacute": '\U000000DD', - "aacute": '\U000000E1', - "acirc": '\U000000E2', - "acute": '\U000000B4', - "aelig": '\U000000E6', - "agrave": '\U000000E0', - "amp": '\U00000026', - "aring": '\U000000E5', - "atilde": '\U000000E3', - "auml": '\U000000E4', - "brvbar": '\U000000A6', - "ccedil": '\U000000E7', - "cedil": '\U000000B8', - "cent": '\U000000A2', - "copy": '\U000000A9', - "curren": '\U000000A4', - "deg": '\U000000B0', - "divide": '\U000000F7', - "eacute": '\U000000E9', - "ecirc": '\U000000EA', - "egrave": '\U000000E8', - "eth": '\U000000F0', - "euml": '\U000000EB', - "frac12": '\U000000BD', - "frac14": '\U000000BC', - "frac34": '\U000000BE', - "gt": '\U0000003E', - "iacute": '\U000000ED', - "icirc": '\U000000EE', - "iexcl": '\U000000A1', - "igrave": '\U000000EC', - "iquest": '\U000000BF', - "iuml": '\U000000EF', - "laquo": '\U000000AB', - "lt": '\U0000003C', - "macr": '\U000000AF', - "micro": '\U000000B5', - "middot": '\U000000B7', - "nbsp": '\U000000A0', - "not": '\U000000AC', - "ntilde": '\U000000F1', - "oacute": '\U000000F3', - "ocirc": '\U000000F4', - "ograve": '\U000000F2', - "ordf": '\U000000AA', - "ordm": '\U000000BA', - "oslash": '\U000000F8', - "otilde": '\U000000F5', - "ouml": '\U000000F6', - "para": '\U000000B6', - "plusmn": '\U000000B1', - "pound": '\U000000A3', - "quot": '\U00000022', - "raquo": '\U000000BB', - "reg": '\U000000AE', - "sect": '\U000000A7', - "shy": '\U000000AD', - "sup1": '\U000000B9', - "sup2": '\U000000B2', - "sup3": '\U000000B3', - "szlig": '\U000000DF', - "thorn": '\U000000FE', - "times": '\U000000D7', - "uacute": '\U000000FA', - "ucirc": '\U000000FB', - "ugrave": '\U000000F9', - "uml": '\U000000A8', - "uuml": '\U000000FC', - "yacute": '\U000000FD', - "yen": '\U000000A5', - "yuml": '\U000000FF', + "Cross;": '\U00002A2F', + "Cscr;": '\U0001D49E', + "Cup;": '\U000022D3', + "CupCap;": '\U0000224D', + "DD;": '\U00002145', + "DDotrahd;": '\U00002911', + "DJcy;": '\U00000402', + "DScy;": '\U00000405', + "DZcy;": '\U0000040F', + "Dagger;": '\U00002021', + "Darr;": '\U000021A1', + "Dashv;": '\U00002AE4', + "Dcaron;": '\U0000010E', + "Dcy;": '\U00000414', + "Del;": '\U00002207', + "Delta;": '\U00000394', + "Dfr;": '\U0001D507', + "DiacriticalAcute;": '\U000000B4', + "DiacriticalDot;": '\U000002D9', + "DiacriticalDoubleAcute;": '\U000002DD', + "DiacriticalGrave;": '\U00000060', + "DiacriticalTilde;": '\U000002DC', + "Diamond;": '\U000022C4', + "DifferentialD;": '\U00002146', + "Dopf;": '\U0001D53B', + "Dot;": '\U000000A8', + "DotDot;": '\U000020DC', + "DotEqual;": '\U00002250', + "DoubleContourIntegral;": '\U0000222F', + "DoubleDot;": '\U000000A8', + "DoubleDownArrow;": '\U000021D3', + "DoubleLeftArrow;": '\U000021D0', + "DoubleLeftRightArrow;": '\U000021D4', + "DoubleLeftTee;": '\U00002AE4', + "DoubleLongLeftArrow;": '\U000027F8', + "DoubleLongLeftRightArrow;": '\U000027FA', + "DoubleLongRightArrow;": '\U000027F9', + "DoubleRightArrow;": '\U000021D2', + "DoubleRightTee;": '\U000022A8', + "DoubleUpArrow;": '\U000021D1', + "DoubleUpDownArrow;": '\U000021D5', + "DoubleVerticalBar;": '\U00002225', + "DownArrow;": '\U00002193', + "DownArrowBar;": '\U00002913', + "DownArrowUpArrow;": '\U000021F5', + "DownBreve;": '\U00000311', + "DownLeftRightVector;": '\U00002950', + "DownLeftTeeVector;": '\U0000295E', + "DownLeftVector;": '\U000021BD', + "DownLeftVectorBar;": '\U00002956', + "DownRightTeeVector;": '\U0000295F', + "DownRightVector;": '\U000021C1', + "DownRightVectorBar;": '\U00002957', + "DownTee;": '\U000022A4', + "DownTeeArrow;": '\U000021A7', + "Downarrow;": '\U000021D3', + "Dscr;": '\U0001D49F', + "Dstrok;": '\U00000110', + "ENG;": '\U0000014A', + "ETH;": '\U000000D0', + "Eacute;": '\U000000C9', + "Ecaron;": '\U0000011A', + "Ecirc;": '\U000000CA', + "Ecy;": '\U0000042D', + "Edot;": '\U00000116', + "Efr;": '\U0001D508', + "Egrave;": '\U000000C8', + "Element;": '\U00002208', + "Emacr;": '\U00000112', + "EmptySmallSquare;": '\U000025FB', + "EmptyVerySmallSquare;": '\U000025AB', + "Eogon;": '\U00000118', + "Eopf;": '\U0001D53C', + "Epsilon;": '\U00000395', + "Equal;": '\U00002A75', + "EqualTilde;": '\U00002242', + "Equilibrium;": '\U000021CC', + "Escr;": '\U00002130', + "Esim;": '\U00002A73', + "Eta;": '\U00000397', + "Euml;": '\U000000CB', + "Exists;": '\U00002203', + "ExponentialE;": '\U00002147', + "Fcy;": '\U00000424', + "Ffr;": '\U0001D509', + "FilledSmallSquare;": '\U000025FC', + "FilledVerySmallSquare;": '\U000025AA', + "Fopf;": '\U0001D53D', + "ForAll;": '\U00002200', + "Fouriertrf;": '\U00002131', + "Fscr;": '\U00002131', + "GJcy;": '\U00000403', + "GT;": '\U0000003E', + "Gamma;": '\U00000393', + "Gammad;": '\U000003DC', + "Gbreve;": '\U0000011E', + "Gcedil;": '\U00000122', + "Gcirc;": '\U0000011C', + "Gcy;": '\U00000413', + "Gdot;": '\U00000120', + "Gfr;": '\U0001D50A', + "Gg;": '\U000022D9', + "Gopf;": '\U0001D53E', + "GreaterEqual;": '\U00002265', + "GreaterEqualLess;": '\U000022DB', + "GreaterFullEqual;": '\U00002267', + "GreaterGreater;": '\U00002AA2', + "GreaterLess;": '\U00002277', + "GreaterSlantEqual;": '\U00002A7E', + "GreaterTilde;": '\U00002273', + "Gscr;": '\U0001D4A2', + "Gt;": '\U0000226B', + "HARDcy;": '\U0000042A', + "Hacek;": '\U000002C7', + "Hat;": '\U0000005E', + "Hcirc;": '\U00000124', + "Hfr;": '\U0000210C', + "HilbertSpace;": '\U0000210B', + "Hopf;": '\U0000210D', + "HorizontalLine;": '\U00002500', + "Hscr;": '\U0000210B', + "Hstrok;": '\U00000126', + "HumpDownHump;": '\U0000224E', + "HumpEqual;": '\U0000224F', + "IEcy;": '\U00000415', + "IJlig;": '\U00000132', + "IOcy;": '\U00000401', + "Iacute;": '\U000000CD', + "Icirc;": '\U000000CE', + "Icy;": '\U00000418', + "Idot;": '\U00000130', + "Ifr;": '\U00002111', + "Igrave;": '\U000000CC', + "Im;": '\U00002111', + "Imacr;": '\U0000012A', + "ImaginaryI;": '\U00002148', + "Implies;": '\U000021D2', + "Int;": '\U0000222C', + "Integral;": '\U0000222B', + "Intersection;": '\U000022C2', + "InvisibleComma;": '\U00002063', + "InvisibleTimes;": '\U00002062', + "Iogon;": '\U0000012E', + "Iopf;": '\U0001D540', + "Iota;": '\U00000399', + "Iscr;": '\U00002110', + "Itilde;": '\U00000128', + "Iukcy;": '\U00000406', + "Iuml;": '\U000000CF', + "Jcirc;": '\U00000134', + "Jcy;": '\U00000419', + "Jfr;": '\U0001D50D', + "Jopf;": '\U0001D541', + "Jscr;": '\U0001D4A5', + "Jsercy;": '\U00000408', + "Jukcy;": '\U00000404', + "KHcy;": '\U00000425', + "KJcy;": '\U0000040C', + "Kappa;": '\U0000039A', + "Kcedil;": '\U00000136', + "Kcy;": '\U0000041A', + "Kfr;": '\U0001D50E', + "Kopf;": '\U0001D542', + "Kscr;": '\U0001D4A6', + "LJcy;": '\U00000409', + "LT;": '\U0000003C', + "Lacute;": '\U00000139', + "Lambda;": '\U0000039B', + "Lang;": '\U000027EA', + "Laplacetrf;": '\U00002112', + "Larr;": '\U0000219E', + "Lcaron;": '\U0000013D', + "Lcedil;": '\U0000013B', + "Lcy;": '\U0000041B', + "LeftAngleBracket;": '\U000027E8', + "LeftArrow;": '\U00002190', + "LeftArrowBar;": '\U000021E4', + "LeftArrowRightArrow;": '\U000021C6', + "LeftCeiling;": '\U00002308', + "LeftDoubleBracket;": '\U000027E6', + "LeftDownTeeVector;": '\U00002961', + "LeftDownVector;": '\U000021C3', + "LeftDownVectorBar;": '\U00002959', + "LeftFloor;": '\U0000230A', + "LeftRightArrow;": '\U00002194', + "LeftRightVector;": '\U0000294E', + "LeftTee;": '\U000022A3', + "LeftTeeArrow;": '\U000021A4', + "LeftTeeVector;": '\U0000295A', + "LeftTriangle;": '\U000022B2', + "LeftTriangleBar;": '\U000029CF', + "LeftTriangleEqual;": '\U000022B4', + "LeftUpDownVector;": '\U00002951', + "LeftUpTeeVector;": '\U00002960', + "LeftUpVector;": '\U000021BF', + "LeftUpVectorBar;": '\U00002958', + "LeftVector;": '\U000021BC', + "LeftVectorBar;": '\U00002952', + "Leftarrow;": '\U000021D0', + "Leftrightarrow;": '\U000021D4', + "LessEqualGreater;": '\U000022DA', + "LessFullEqual;": '\U00002266', + "LessGreater;": '\U00002276', + "LessLess;": '\U00002AA1', + "LessSlantEqual;": '\U00002A7D', + "LessTilde;": '\U00002272', + "Lfr;": '\U0001D50F', + "Ll;": '\U000022D8', + "Lleftarrow;": '\U000021DA', + "Lmidot;": '\U0000013F', + "LongLeftArrow;": '\U000027F5', + "LongLeftRightArrow;": '\U000027F7', + "LongRightArrow;": '\U000027F6', + "Longleftarrow;": '\U000027F8', + "Longleftrightarrow;": '\U000027FA', + "Longrightarrow;": '\U000027F9', + "Lopf;": '\U0001D543', + "LowerLeftArrow;": '\U00002199', + "LowerRightArrow;": '\U00002198', + "Lscr;": '\U00002112', + "Lsh;": '\U000021B0', + "Lstrok;": '\U00000141', + "Lt;": '\U0000226A', + "Map;": '\U00002905', + "Mcy;": '\U0000041C', + "MediumSpace;": '\U0000205F', + "Mellintrf;": '\U00002133', + "Mfr;": '\U0001D510', + "MinusPlus;": '\U00002213', + "Mopf;": '\U0001D544', + "Mscr;": '\U00002133', + "Mu;": '\U0000039C', + "NJcy;": '\U0000040A', + "Nacute;": '\U00000143', + "Ncaron;": '\U00000147', + "Ncedil;": '\U00000145', + "Ncy;": '\U0000041D', + "NegativeMediumSpace;": '\U0000200B', + "NegativeThickSpace;": '\U0000200B', + "NegativeThinSpace;": '\U0000200B', + "NegativeVeryThinSpace;": '\U0000200B', + "NestedGreaterGreater;": '\U0000226B', + "NestedLessLess;": '\U0000226A', + "NewLine;": '\U0000000A', + "Nfr;": '\U0001D511', + "NoBreak;": '\U00002060', + "NonBreakingSpace;": '\U000000A0', + "Nopf;": '\U00002115', + "Not;": '\U00002AEC', + "NotCongruent;": '\U00002262', + "NotCupCap;": '\U0000226D', + "NotDoubleVerticalBar;": '\U00002226', + "NotElement;": '\U00002209', + "NotEqual;": '\U00002260', + "NotExists;": '\U00002204', + "NotGreater;": '\U0000226F', + "NotGreaterEqual;": '\U00002271', + "NotGreaterLess;": '\U00002279', + "NotGreaterTilde;": '\U00002275', + "NotLeftTriangle;": '\U000022EA', + "NotLeftTriangleEqual;": '\U000022EC', + "NotLess;": '\U0000226E', + "NotLessEqual;": '\U00002270', + "NotLessGreater;": '\U00002278', + "NotLessTilde;": '\U00002274', + "NotPrecedes;": '\U00002280', + "NotPrecedesSlantEqual;": '\U000022E0', + "NotReverseElement;": '\U0000220C', + "NotRightTriangle;": '\U000022EB', + "NotRightTriangleEqual;": '\U000022ED', + "NotSquareSubsetEqual;": '\U000022E2', + "NotSquareSupersetEqual;": '\U000022E3', + "NotSubsetEqual;": '\U00002288', + "NotSucceeds;": '\U00002281', + "NotSucceedsSlantEqual;": '\U000022E1', + "NotSupersetEqual;": '\U00002289', + "NotTilde;": '\U00002241', + "NotTildeEqual;": '\U00002244', + "NotTildeFullEqual;": '\U00002247', + "NotTildeTilde;": '\U00002249', + "NotVerticalBar;": '\U00002224', + "Nscr;": '\U0001D4A9', + "Ntilde;": '\U000000D1', + "Nu;": '\U0000039D', + "OElig;": '\U00000152', + "Oacute;": '\U000000D3', + "Ocirc;": '\U000000D4', + "Ocy;": '\U0000041E', + "Odblac;": '\U00000150', + "Ofr;": '\U0001D512', + "Ograve;": '\U000000D2', + "Omacr;": '\U0000014C', + "Omega;": '\U000003A9', + "Omicron;": '\U0000039F', + "Oopf;": '\U0001D546', + "OpenCurlyDoubleQuote;": '\U0000201C', + "OpenCurlyQuote;": '\U00002018', + "Or;": '\U00002A54', + "Oscr;": '\U0001D4AA', + "Oslash;": '\U000000D8', + "Otilde;": '\U000000D5', + "Otimes;": '\U00002A37', + "Ouml;": '\U000000D6', + "OverBar;": '\U0000203E', + "OverBrace;": '\U000023DE', + "OverBracket;": '\U000023B4', + "OverParenthesis;": '\U000023DC', + "PartialD;": '\U00002202', + "Pcy;": '\U0000041F', + "Pfr;": '\U0001D513', + "Phi;": '\U000003A6', + "Pi;": '\U000003A0', + "PlusMinus;": '\U000000B1', + "Poincareplane;": '\U0000210C', + "Popf;": '\U00002119', + "Pr;": '\U00002ABB', + "Precedes;": '\U0000227A', + "PrecedesEqual;": '\U00002AAF', + "PrecedesSlantEqual;": '\U0000227C', + "PrecedesTilde;": '\U0000227E', + "Prime;": '\U00002033', + "Product;": '\U0000220F', + "Proportion;": '\U00002237', + "Proportional;": '\U0000221D', + "Pscr;": '\U0001D4AB', + "Psi;": '\U000003A8', + "QUOT;": '\U00000022', + "Qfr;": '\U0001D514', + "Qopf;": '\U0000211A', + "Qscr;": '\U0001D4AC', + "RBarr;": '\U00002910', + "REG;": '\U000000AE', + "Racute;": '\U00000154', + "Rang;": '\U000027EB', + "Rarr;": '\U000021A0', + "Rarrtl;": '\U00002916', + "Rcaron;": '\U00000158', + "Rcedil;": '\U00000156', + "Rcy;": '\U00000420', + "Re;": '\U0000211C', + "ReverseElement;": '\U0000220B', + "ReverseEquilibrium;": '\U000021CB', + "ReverseUpEquilibrium;": '\U0000296F', + "Rfr;": '\U0000211C', + "Rho;": '\U000003A1', + "RightAngleBracket;": '\U000027E9', + "RightArrow;": '\U00002192', + "RightArrowBar;": '\U000021E5', + "RightArrowLeftArrow;": '\U000021C4', + "RightCeiling;": '\U00002309', + "RightDoubleBracket;": '\U000027E7', + "RightDownTeeVector;": '\U0000295D', + "RightDownVector;": '\U000021C2', + "RightDownVectorBar;": '\U00002955', + "RightFloor;": '\U0000230B', + "RightTee;": '\U000022A2', + "RightTeeArrow;": '\U000021A6', + "RightTeeVector;": '\U0000295B', + "RightTriangle;": '\U000022B3', + "RightTriangleBar;": '\U000029D0', + "RightTriangleEqual;": '\U000022B5', + "RightUpDownVector;": '\U0000294F', + "RightUpTeeVector;": '\U0000295C', + "RightUpVector;": '\U000021BE', + "RightUpVectorBar;": '\U00002954', + "RightVector;": '\U000021C0', + "RightVectorBar;": '\U00002953', + "Rightarrow;": '\U000021D2', + "Ropf;": '\U0000211D', + "RoundImplies;": '\U00002970', + "Rrightarrow;": '\U000021DB', + "Rscr;": '\U0000211B', + "Rsh;": '\U000021B1', + "RuleDelayed;": '\U000029F4', + "SHCHcy;": '\U00000429', + "SHcy;": '\U00000428', + "SOFTcy;": '\U0000042C', + "Sacute;": '\U0000015A', + "Sc;": '\U00002ABC', + "Scaron;": '\U00000160', + "Scedil;": '\U0000015E', + "Scirc;": '\U0000015C', + "Scy;": '\U00000421', + "Sfr;": '\U0001D516', + "ShortDownArrow;": '\U00002193', + "ShortLeftArrow;": '\U00002190', + "ShortRightArrow;": '\U00002192', + "ShortUpArrow;": '\U00002191', + "Sigma;": '\U000003A3', + "SmallCircle;": '\U00002218', + "Sopf;": '\U0001D54A', + "Sqrt;": '\U0000221A', + "Square;": '\U000025A1', + "SquareIntersection;": '\U00002293', + "SquareSubset;": '\U0000228F', + "SquareSubsetEqual;": '\U00002291', + "SquareSuperset;": '\U00002290', + "SquareSupersetEqual;": '\U00002292', + "SquareUnion;": '\U00002294', + "Sscr;": '\U0001D4AE', + "Star;": '\U000022C6', + "Sub;": '\U000022D0', + "Subset;": '\U000022D0', + "SubsetEqual;": '\U00002286', + "Succeeds;": '\U0000227B', + "SucceedsEqual;": '\U00002AB0', + "SucceedsSlantEqual;": '\U0000227D', + "SucceedsTilde;": '\U0000227F', + "SuchThat;": '\U0000220B', + "Sum;": '\U00002211', + "Sup;": '\U000022D1', + "Superset;": '\U00002283', + "SupersetEqual;": '\U00002287', + "Supset;": '\U000022D1', + "THORN;": '\U000000DE', + "TRADE;": '\U00002122', + "TSHcy;": '\U0000040B', + "TScy;": '\U00000426', + "Tab;": '\U00000009', + "Tau;": '\U000003A4', + "Tcaron;": '\U00000164', + "Tcedil;": '\U00000162', + "Tcy;": '\U00000422', + "Tfr;": '\U0001D517', + "Therefore;": '\U00002234', + "Theta;": '\U00000398', + "ThinSpace;": '\U00002009', + "Tilde;": '\U0000223C', + "TildeEqual;": '\U00002243', + "TildeFullEqual;": '\U00002245', + "TildeTilde;": '\U00002248', + "Topf;": '\U0001D54B', + "TripleDot;": '\U000020DB', + "Tscr;": '\U0001D4AF', + "Tstrok;": '\U00000166', + "Uacute;": '\U000000DA', + "Uarr;": '\U0000219F', + "Uarrocir;": '\U00002949', + "Ubrcy;": '\U0000040E', + "Ubreve;": '\U0000016C', + "Ucirc;": '\U000000DB', + "Ucy;": '\U00000423', + "Udblac;": '\U00000170', + "Ufr;": '\U0001D518', + "Ugrave;": '\U000000D9', + "Umacr;": '\U0000016A', + "UnderBar;": '\U0000005F', + "UnderBrace;": '\U000023DF', + "UnderBracket;": '\U000023B5', + "UnderParenthesis;": '\U000023DD', + "Union;": '\U000022C3', + "UnionPlus;": '\U0000228E', + "Uogon;": '\U00000172', + "Uopf;": '\U0001D54C', + "UpArrow;": '\U00002191', + "UpArrowBar;": '\U00002912', + "UpArrowDownArrow;": '\U000021C5', + "UpDownArrow;": '\U00002195', + "UpEquilibrium;": '\U0000296E', + "UpTee;": '\U000022A5', + "UpTeeArrow;": '\U000021A5', + "Uparrow;": '\U000021D1', + "Updownarrow;": '\U000021D5', + "UpperLeftArrow;": '\U00002196', + "UpperRightArrow;": '\U00002197', + "Upsi;": '\U000003D2', + "Upsilon;": '\U000003A5', + "Uring;": '\U0000016E', + "Uscr;": '\U0001D4B0', + "Utilde;": '\U00000168', + "Uuml;": '\U000000DC', + "VDash;": '\U000022AB', + "Vbar;": '\U00002AEB', + "Vcy;": '\U00000412', + "Vdash;": '\U000022A9', + "Vdashl;": '\U00002AE6', + "Vee;": '\U000022C1', + "Verbar;": '\U00002016', + "Vert;": '\U00002016', + "VerticalBar;": '\U00002223', + "VerticalLine;": '\U0000007C', + "VerticalSeparator;": '\U00002758', + "VerticalTilde;": '\U00002240', + "VeryThinSpace;": '\U0000200A', + "Vfr;": '\U0001D519', + "Vopf;": '\U0001D54D', + "Vscr;": '\U0001D4B1', + "Vvdash;": '\U000022AA', + "Wcirc;": '\U00000174', + "Wedge;": '\U000022C0', + "Wfr;": '\U0001D51A', + "Wopf;": '\U0001D54E', + "Wscr;": '\U0001D4B2', + "Xfr;": '\U0001D51B', + "Xi;": '\U0000039E', + "Xopf;": '\U0001D54F', + "Xscr;": '\U0001D4B3', + "YAcy;": '\U0000042F', + "YIcy;": '\U00000407', + "YUcy;": '\U0000042E', + "Yacute;": '\U000000DD', + "Ycirc;": '\U00000176', + "Ycy;": '\U0000042B', + "Yfr;": '\U0001D51C', + "Yopf;": '\U0001D550', + "Yscr;": '\U0001D4B4', + "Yuml;": '\U00000178', + "ZHcy;": '\U00000416', + "Zacute;": '\U00000179', + "Zcaron;": '\U0000017D', + "Zcy;": '\U00000417', + "Zdot;": '\U0000017B', + "ZeroWidthSpace;": '\U0000200B', + "Zeta;": '\U00000396', + "Zfr;": '\U00002128', + "Zopf;": '\U00002124', + "Zscr;": '\U0001D4B5', + "aacute;": '\U000000E1', + "abreve;": '\U00000103', + "ac;": '\U0000223E', + "acd;": '\U0000223F', + "acirc;": '\U000000E2', + "acute;": '\U000000B4', + "acy;": '\U00000430', + "aelig;": '\U000000E6', + "af;": '\U00002061', + "afr;": '\U0001D51E', + "agrave;": '\U000000E0', + "alefsym;": '\U00002135', + "aleph;": '\U00002135', + "alpha;": '\U000003B1', + "amacr;": '\U00000101', + "amalg;": '\U00002A3F', + "amp;": '\U00000026', + "and;": '\U00002227', + "andand;": '\U00002A55', + "andd;": '\U00002A5C', + "andslope;": '\U00002A58', + "andv;": '\U00002A5A', + "ang;": '\U00002220', + "ange;": '\U000029A4', + "angle;": '\U00002220', + "angmsd;": '\U00002221', + "angmsdaa;": '\U000029A8', + "angmsdab;": '\U000029A9', + "angmsdac;": '\U000029AA', + "angmsdad;": '\U000029AB', + "angmsdae;": '\U000029AC', + "angmsdaf;": '\U000029AD', + "angmsdag;": '\U000029AE', + "angmsdah;": '\U000029AF', + "angrt;": '\U0000221F', + "angrtvb;": '\U000022BE', + "angrtvbd;": '\U0000299D', + "angsph;": '\U00002222', + "angst;": '\U000000C5', + "angzarr;": '\U0000237C', + "aogon;": '\U00000105', + "aopf;": '\U0001D552', + "ap;": '\U00002248', + "apE;": '\U00002A70', + "apacir;": '\U00002A6F', + "ape;": '\U0000224A', + "apid;": '\U0000224B', + "apos;": '\U00000027', + "approx;": '\U00002248', + "approxeq;": '\U0000224A', + "aring;": '\U000000E5', + "ascr;": '\U0001D4B6', + "ast;": '\U0000002A', + "asymp;": '\U00002248', + "asympeq;": '\U0000224D', + "atilde;": '\U000000E3', + "auml;": '\U000000E4', + "awconint;": '\U00002233', + "awint;": '\U00002A11', + "bNot;": '\U00002AED', + "backcong;": '\U0000224C', + "backepsilon;": '\U000003F6', + "backprime;": '\U00002035', + "backsim;": '\U0000223D', + "backsimeq;": '\U000022CD', + "barvee;": '\U000022BD', + "barwed;": '\U00002305', + "barwedge;": '\U00002305', + "bbrk;": '\U000023B5', + "bbrktbrk;": '\U000023B6', + "bcong;": '\U0000224C', + "bcy;": '\U00000431', + "bdquo;": '\U0000201E', + "becaus;": '\U00002235', + "because;": '\U00002235', + "bemptyv;": '\U000029B0', + "bepsi;": '\U000003F6', + "bernou;": '\U0000212C', + "beta;": '\U000003B2', + "beth;": '\U00002136', + "between;": '\U0000226C', + "bfr;": '\U0001D51F', + "bigcap;": '\U000022C2', + "bigcirc;": '\U000025EF', + "bigcup;": '\U000022C3', + "bigodot;": '\U00002A00', + "bigoplus;": '\U00002A01', + "bigotimes;": '\U00002A02', + "bigsqcup;": '\U00002A06', + "bigstar;": '\U00002605', + "bigtriangledown;": '\U000025BD', + "bigtriangleup;": '\U000025B3', + "biguplus;": '\U00002A04', + "bigvee;": '\U000022C1', + "bigwedge;": '\U000022C0', + "bkarow;": '\U0000290D', + "blacklozenge;": '\U000029EB', + "blacksquare;": '\U000025AA', + "blacktriangle;": '\U000025B4', + "blacktriangledown;": '\U000025BE', + "blacktriangleleft;": '\U000025C2', + "blacktriangleright;": '\U000025B8', + "blank;": '\U00002423', + "blk12;": '\U00002592', + "blk14;": '\U00002591', + "blk34;": '\U00002593', + "block;": '\U00002588', + "bnot;": '\U00002310', + "bopf;": '\U0001D553', + "bot;": '\U000022A5', + "bottom;": '\U000022A5', + "bowtie;": '\U000022C8', + "boxDL;": '\U00002557', + "boxDR;": '\U00002554', + "boxDl;": '\U00002556', + "boxDr;": '\U00002553', + "boxH;": '\U00002550', + "boxHD;": '\U00002566', + "boxHU;": '\U00002569', + "boxHd;": '\U00002564', + "boxHu;": '\U00002567', + "boxUL;": '\U0000255D', + "boxUR;": '\U0000255A', + "boxUl;": '\U0000255C', + "boxUr;": '\U00002559', + "boxV;": '\U00002551', + "boxVH;": '\U0000256C', + "boxVL;": '\U00002563', + "boxVR;": '\U00002560', + "boxVh;": '\U0000256B', + "boxVl;": '\U00002562', + "boxVr;": '\U0000255F', + "boxbox;": '\U000029C9', + "boxdL;": '\U00002555', + "boxdR;": '\U00002552', + "boxdl;": '\U00002510', + "boxdr;": '\U0000250C', + "boxh;": '\U00002500', + "boxhD;": '\U00002565', + "boxhU;": '\U00002568', + "boxhd;": '\U0000252C', + "boxhu;": '\U00002534', + "boxminus;": '\U0000229F', + "boxplus;": '\U0000229E', + "boxtimes;": '\U000022A0', + "boxuL;": '\U0000255B', + "boxuR;": '\U00002558', + "boxul;": '\U00002518', + "boxur;": '\U00002514', + "boxv;": '\U00002502', + "boxvH;": '\U0000256A', + "boxvL;": '\U00002561', + "boxvR;": '\U0000255E', + "boxvh;": '\U0000253C', + "boxvl;": '\U00002524', + "boxvr;": '\U0000251C', + "bprime;": '\U00002035', + "breve;": '\U000002D8', + "brvbar;": '\U000000A6', + "bscr;": '\U0001D4B7', + "bsemi;": '\U0000204F', + "bsim;": '\U0000223D', + "bsime;": '\U000022CD', + "bsol;": '\U0000005C', + "bsolb;": '\U000029C5', + "bsolhsub;": '\U000027C8', + "bull;": '\U00002022', + "bullet;": '\U00002022', + "bump;": '\U0000224E', + "bumpE;": '\U00002AAE', + "bumpe;": '\U0000224F', + "bumpeq;": '\U0000224F', + "cacute;": '\U00000107', + "cap;": '\U00002229', + "capand;": '\U00002A44', + "capbrcup;": '\U00002A49', + "capcap;": '\U00002A4B', + "capcup;": '\U00002A47', + "capdot;": '\U00002A40', + "caret;": '\U00002041', + "caron;": '\U000002C7', + "ccaps;": '\U00002A4D', + "ccaron;": '\U0000010D', + "ccedil;": '\U000000E7', + "ccirc;": '\U00000109', + "ccups;": '\U00002A4C', + "ccupssm;": '\U00002A50', + "cdot;": '\U0000010B', + "cedil;": '\U000000B8', + "cemptyv;": '\U000029B2', + "cent;": '\U000000A2', + "centerdot;": '\U000000B7', + "cfr;": '\U0001D520', + "chcy;": '\U00000447', + "check;": '\U00002713', + "checkmark;": '\U00002713', + "chi;": '\U000003C7', + "cir;": '\U000025CB', + "cirE;": '\U000029C3', + "circ;": '\U000002C6', + "circeq;": '\U00002257', + "circlearrowleft;": '\U000021BA', + "circlearrowright;": '\U000021BB', + "circledR;": '\U000000AE', + "circledS;": '\U000024C8', + "circledast;": '\U0000229B', + "circledcirc;": '\U0000229A', + "circleddash;": '\U0000229D', + "cire;": '\U00002257', + "cirfnint;": '\U00002A10', + "cirmid;": '\U00002AEF', + "cirscir;": '\U000029C2', + "clubs;": '\U00002663', + "clubsuit;": '\U00002663', + "colon;": '\U0000003A', + "colone;": '\U00002254', + "coloneq;": '\U00002254', + "comma;": '\U0000002C', + "commat;": '\U00000040', + "comp;": '\U00002201', + "compfn;": '\U00002218', + "complement;": '\U00002201', + "complexes;": '\U00002102', + "cong;": '\U00002245', + "congdot;": '\U00002A6D', + "conint;": '\U0000222E', + "copf;": '\U0001D554', + "coprod;": '\U00002210', + "copy;": '\U000000A9', + "copysr;": '\U00002117', + "crarr;": '\U000021B5', + "cross;": '\U00002717', + "cscr;": '\U0001D4B8', + "csub;": '\U00002ACF', + "csube;": '\U00002AD1', + "csup;": '\U00002AD0', + "csupe;": '\U00002AD2', + "ctdot;": '\U000022EF', + "cudarrl;": '\U00002938', + "cudarrr;": '\U00002935', + "cuepr;": '\U000022DE', + "cuesc;": '\U000022DF', + "cularr;": '\U000021B6', + "cularrp;": '\U0000293D', + "cup;": '\U0000222A', + "cupbrcap;": '\U00002A48', + "cupcap;": '\U00002A46', + "cupcup;": '\U00002A4A', + "cupdot;": '\U0000228D', + "cupor;": '\U00002A45', + "curarr;": '\U000021B7', + "curarrm;": '\U0000293C', + "curlyeqprec;": '\U000022DE', + "curlyeqsucc;": '\U000022DF', + "curlyvee;": '\U000022CE', + "curlywedge;": '\U000022CF', + "curren;": '\U000000A4', + "curvearrowleft;": '\U000021B6', + "curvearrowright;": '\U000021B7', + "cuvee;": '\U000022CE', + "cuwed;": '\U000022CF', + "cwconint;": '\U00002232', + "cwint;": '\U00002231', + "cylcty;": '\U0000232D', + "dArr;": '\U000021D3', + "dHar;": '\U00002965', + "dagger;": '\U00002020', + "daleth;": '\U00002138', + "darr;": '\U00002193', + "dash;": '\U00002010', + "dashv;": '\U000022A3', + "dbkarow;": '\U0000290F', + "dblac;": '\U000002DD', + "dcaron;": '\U0000010F', + "dcy;": '\U00000434', + "dd;": '\U00002146', + "ddagger;": '\U00002021', + "ddarr;": '\U000021CA', + "ddotseq;": '\U00002A77', + "deg;": '\U000000B0', + "delta;": '\U000003B4', + "demptyv;": '\U000029B1', + "dfisht;": '\U0000297F', + "dfr;": '\U0001D521', + "dharl;": '\U000021C3', + "dharr;": '\U000021C2', + "diam;": '\U000022C4', + "diamond;": '\U000022C4', + "diamondsuit;": '\U00002666', + "diams;": '\U00002666', + "die;": '\U000000A8', + "digamma;": '\U000003DD', + "disin;": '\U000022F2', + "div;": '\U000000F7', + "divide;": '\U000000F7', + "divideontimes;": '\U000022C7', + "divonx;": '\U000022C7', + "djcy;": '\U00000452', + "dlcorn;": '\U0000231E', + "dlcrop;": '\U0000230D', + "dollar;": '\U00000024', + "dopf;": '\U0001D555', + "dot;": '\U000002D9', + "doteq;": '\U00002250', + "doteqdot;": '\U00002251', + "dotminus;": '\U00002238', + "dotplus;": '\U00002214', + "dotsquare;": '\U000022A1', + "doublebarwedge;": '\U00002306', + "downarrow;": '\U00002193', + "downdownarrows;": '\U000021CA', + "downharpoonleft;": '\U000021C3', + "downharpoonright;": '\U000021C2', + "drbkarow;": '\U00002910', + "drcorn;": '\U0000231F', + "drcrop;": '\U0000230C', + "dscr;": '\U0001D4B9', + "dscy;": '\U00000455', + "dsol;": '\U000029F6', + "dstrok;": '\U00000111', + "dtdot;": '\U000022F1', + "dtri;": '\U000025BF', + "dtrif;": '\U000025BE', + "duarr;": '\U000021F5', + "duhar;": '\U0000296F', + "dwangle;": '\U000029A6', + "dzcy;": '\U0000045F', + "dzigrarr;": '\U000027FF', + "eDDot;": '\U00002A77', + "eDot;": '\U00002251', + "eacute;": '\U000000E9', + "easter;": '\U00002A6E', + "ecaron;": '\U0000011B', + "ecir;": '\U00002256', + "ecirc;": '\U000000EA', + "ecolon;": '\U00002255', + "ecy;": '\U0000044D', + "edot;": '\U00000117', + "ee;": '\U00002147', + "efDot;": '\U00002252', + "efr;": '\U0001D522', + "eg;": '\U00002A9A', + "egrave;": '\U000000E8', + "egs;": '\U00002A96', + "egsdot;": '\U00002A98', + "el;": '\U00002A99', + "elinters;": '\U000023E7', + "ell;": '\U00002113', + "els;": '\U00002A95', + "elsdot;": '\U00002A97', + "emacr;": '\U00000113', + "empty;": '\U00002205', + "emptyset;": '\U00002205', + "emptyv;": '\U00002205', + "emsp;": '\U00002003', + "emsp13;": '\U00002004', + "emsp14;": '\U00002005', + "eng;": '\U0000014B', + "ensp;": '\U00002002', + "eogon;": '\U00000119', + "eopf;": '\U0001D556', + "epar;": '\U000022D5', + "eparsl;": '\U000029E3', + "eplus;": '\U00002A71', + "epsi;": '\U000003B5', + "epsilon;": '\U000003B5', + "epsiv;": '\U000003F5', + "eqcirc;": '\U00002256', + "eqcolon;": '\U00002255', + "eqsim;": '\U00002242', + "eqslantgtr;": '\U00002A96', + "eqslantless;": '\U00002A95', + "equals;": '\U0000003D', + "equest;": '\U0000225F', + "equiv;": '\U00002261', + "equivDD;": '\U00002A78', + "eqvparsl;": '\U000029E5', + "erDot;": '\U00002253', + "erarr;": '\U00002971', + "escr;": '\U0000212F', + "esdot;": '\U00002250', + "esim;": '\U00002242', + "eta;": '\U000003B7', + "eth;": '\U000000F0', + "euml;": '\U000000EB', + "euro;": '\U000020AC', + "excl;": '\U00000021', + "exist;": '\U00002203', + "expectation;": '\U00002130', + "exponentiale;": '\U00002147', + "fallingdotseq;": '\U00002252', + "fcy;": '\U00000444', + "female;": '\U00002640', + "ffilig;": '\U0000FB03', + "fflig;": '\U0000FB00', + "ffllig;": '\U0000FB04', + "ffr;": '\U0001D523', + "filig;": '\U0000FB01', + "flat;": '\U0000266D', + "fllig;": '\U0000FB02', + "fltns;": '\U000025B1', + "fnof;": '\U00000192', + "fopf;": '\U0001D557', + "forall;": '\U00002200', + "fork;": '\U000022D4', + "forkv;": '\U00002AD9', + "fpartint;": '\U00002A0D', + "frac12;": '\U000000BD', + "frac13;": '\U00002153', + "frac14;": '\U000000BC', + "frac15;": '\U00002155', + "frac16;": '\U00002159', + "frac18;": '\U0000215B', + "frac23;": '\U00002154', + "frac25;": '\U00002156', + "frac34;": '\U000000BE', + "frac35;": '\U00002157', + "frac38;": '\U0000215C', + "frac45;": '\U00002158', + "frac56;": '\U0000215A', + "frac58;": '\U0000215D', + "frac78;": '\U0000215E', + "frasl;": '\U00002044', + "frown;": '\U00002322', + "fscr;": '\U0001D4BB', + "gE;": '\U00002267', + "gEl;": '\U00002A8C', + "gacute;": '\U000001F5', + "gamma;": '\U000003B3', + "gammad;": '\U000003DD', + "gap;": '\U00002A86', + "gbreve;": '\U0000011F', + "gcirc;": '\U0000011D', + "gcy;": '\U00000433', + "gdot;": '\U00000121', + "ge;": '\U00002265', + "gel;": '\U000022DB', + "geq;": '\U00002265', + "geqq;": '\U00002267', + "geqslant;": '\U00002A7E', + "ges;": '\U00002A7E', + "gescc;": '\U00002AA9', + "gesdot;": '\U00002A80', + "gesdoto;": '\U00002A82', + "gesdotol;": '\U00002A84', + "gesles;": '\U00002A94', + "gfr;": '\U0001D524', + "gg;": '\U0000226B', + "ggg;": '\U000022D9', + "gimel;": '\U00002137', + "gjcy;": '\U00000453', + "gl;": '\U00002277', + "glE;": '\U00002A92', + "gla;": '\U00002AA5', + "glj;": '\U00002AA4', + "gnE;": '\U00002269', + "gnap;": '\U00002A8A', + "gnapprox;": '\U00002A8A', + "gne;": '\U00002A88', + "gneq;": '\U00002A88', + "gneqq;": '\U00002269', + "gnsim;": '\U000022E7', + "gopf;": '\U0001D558', + "grave;": '\U00000060', + "gscr;": '\U0000210A', + "gsim;": '\U00002273', + "gsime;": '\U00002A8E', + "gsiml;": '\U00002A90', + "gt;": '\U0000003E', + "gtcc;": '\U00002AA7', + "gtcir;": '\U00002A7A', + "gtdot;": '\U000022D7', + "gtlPar;": '\U00002995', + "gtquest;": '\U00002A7C', + "gtrapprox;": '\U00002A86', + "gtrarr;": '\U00002978', + "gtrdot;": '\U000022D7', + "gtreqless;": '\U000022DB', + "gtreqqless;": '\U00002A8C', + "gtrless;": '\U00002277', + "gtrsim;": '\U00002273', + "hArr;": '\U000021D4', + "hairsp;": '\U0000200A', + "half;": '\U000000BD', + "hamilt;": '\U0000210B', + "hardcy;": '\U0000044A', + "harr;": '\U00002194', + "harrcir;": '\U00002948', + "harrw;": '\U000021AD', + "hbar;": '\U0000210F', + "hcirc;": '\U00000125', + "hearts;": '\U00002665', + "heartsuit;": '\U00002665', + "hellip;": '\U00002026', + "hercon;": '\U000022B9', + "hfr;": '\U0001D525', + "hksearow;": '\U00002925', + "hkswarow;": '\U00002926', + "hoarr;": '\U000021FF', + "homtht;": '\U0000223B', + "hookleftarrow;": '\U000021A9', + "hookrightarrow;": '\U000021AA', + "hopf;": '\U0001D559', + "horbar;": '\U00002015', + "hscr;": '\U0001D4BD', + "hslash;": '\U0000210F', + "hstrok;": '\U00000127', + "hybull;": '\U00002043', + "hyphen;": '\U00002010', + "iacute;": '\U000000ED', + "ic;": '\U00002063', + "icirc;": '\U000000EE', + "icy;": '\U00000438', + "iecy;": '\U00000435', + "iexcl;": '\U000000A1', + "iff;": '\U000021D4', + "ifr;": '\U0001D526', + "igrave;": '\U000000EC', + "ii;": '\U00002148', + "iiiint;": '\U00002A0C', + "iiint;": '\U0000222D', + "iinfin;": '\U000029DC', + "iiota;": '\U00002129', + "ijlig;": '\U00000133', + "imacr;": '\U0000012B', + "image;": '\U00002111', + "imagline;": '\U00002110', + "imagpart;": '\U00002111', + "imath;": '\U00000131', + "imof;": '\U000022B7', + "imped;": '\U000001B5', + "in;": '\U00002208', + "incare;": '\U00002105', + "infin;": '\U0000221E', + "infintie;": '\U000029DD', + "inodot;": '\U00000131', + "int;": '\U0000222B', + "intcal;": '\U000022BA', + "integers;": '\U00002124', + "intercal;": '\U000022BA', + "intlarhk;": '\U00002A17', + "intprod;": '\U00002A3C', + "iocy;": '\U00000451', + "iogon;": '\U0000012F', + "iopf;": '\U0001D55A', + "iota;": '\U000003B9', + "iprod;": '\U00002A3C', + "iquest;": '\U000000BF', + "iscr;": '\U0001D4BE', + "isin;": '\U00002208', + "isinE;": '\U000022F9', + "isindot;": '\U000022F5', + "isins;": '\U000022F4', + "isinsv;": '\U000022F3', + "isinv;": '\U00002208', + "it;": '\U00002062', + "itilde;": '\U00000129', + "iukcy;": '\U00000456', + "iuml;": '\U000000EF', + "jcirc;": '\U00000135', + "jcy;": '\U00000439', + "jfr;": '\U0001D527', + "jmath;": '\U00000237', + "jopf;": '\U0001D55B', + "jscr;": '\U0001D4BF', + "jsercy;": '\U00000458', + "jukcy;": '\U00000454', + "kappa;": '\U000003BA', + "kappav;": '\U000003F0', + "kcedil;": '\U00000137', + "kcy;": '\U0000043A', + "kfr;": '\U0001D528', + "kgreen;": '\U00000138', + "khcy;": '\U00000445', + "kjcy;": '\U0000045C', + "kopf;": '\U0001D55C', + "kscr;": '\U0001D4C0', + "lAarr;": '\U000021DA', + "lArr;": '\U000021D0', + "lAtail;": '\U0000291B', + "lBarr;": '\U0000290E', + "lE;": '\U00002266', + "lEg;": '\U00002A8B', + "lHar;": '\U00002962', + "lacute;": '\U0000013A', + "laemptyv;": '\U000029B4', + "lagran;": '\U00002112', + "lambda;": '\U000003BB', + "lang;": '\U000027E8', + "langd;": '\U00002991', + "langle;": '\U000027E8', + "lap;": '\U00002A85', + "laquo;": '\U000000AB', + "larr;": '\U00002190', + "larrb;": '\U000021E4', + "larrbfs;": '\U0000291F', + "larrfs;": '\U0000291D', + "larrhk;": '\U000021A9', + "larrlp;": '\U000021AB', + "larrpl;": '\U00002939', + "larrsim;": '\U00002973', + "larrtl;": '\U000021A2', + "lat;": '\U00002AAB', + "latail;": '\U00002919', + "late;": '\U00002AAD', + "lbarr;": '\U0000290C', + "lbbrk;": '\U00002772', + "lbrace;": '\U0000007B', + "lbrack;": '\U0000005B', + "lbrke;": '\U0000298B', + "lbrksld;": '\U0000298F', + "lbrkslu;": '\U0000298D', + "lcaron;": '\U0000013E', + "lcedil;": '\U0000013C', + "lceil;": '\U00002308', + "lcub;": '\U0000007B', + "lcy;": '\U0000043B', + "ldca;": '\U00002936', + "ldquo;": '\U0000201C', + "ldquor;": '\U0000201E', + "ldrdhar;": '\U00002967', + "ldrushar;": '\U0000294B', + "ldsh;": '\U000021B2', + "le;": '\U00002264', + "leftarrow;": '\U00002190', + "leftarrowtail;": '\U000021A2', + "leftharpoondown;": '\U000021BD', + "leftharpoonup;": '\U000021BC', + "leftleftarrows;": '\U000021C7', + "leftrightarrow;": '\U00002194', + "leftrightarrows;": '\U000021C6', + "leftrightharpoons;": '\U000021CB', + "leftrightsquigarrow;": '\U000021AD', + "leftthreetimes;": '\U000022CB', + "leg;": '\U000022DA', + "leq;": '\U00002264', + "leqq;": '\U00002266', + "leqslant;": '\U00002A7D', + "les;": '\U00002A7D', + "lescc;": '\U00002AA8', + "lesdot;": '\U00002A7F', + "lesdoto;": '\U00002A81', + "lesdotor;": '\U00002A83', + "lesges;": '\U00002A93', + "lessapprox;": '\U00002A85', + "lessdot;": '\U000022D6', + "lesseqgtr;": '\U000022DA', + "lesseqqgtr;": '\U00002A8B', + "lessgtr;": '\U00002276', + "lesssim;": '\U00002272', + "lfisht;": '\U0000297C', + "lfloor;": '\U0000230A', + "lfr;": '\U0001D529', + "lg;": '\U00002276', + "lgE;": '\U00002A91', + "lhard;": '\U000021BD', + "lharu;": '\U000021BC', + "lharul;": '\U0000296A', + "lhblk;": '\U00002584', + "ljcy;": '\U00000459', + "ll;": '\U0000226A', + "llarr;": '\U000021C7', + "llcorner;": '\U0000231E', + "llhard;": '\U0000296B', + "lltri;": '\U000025FA', + "lmidot;": '\U00000140', + "lmoust;": '\U000023B0', + "lmoustache;": '\U000023B0', + "lnE;": '\U00002268', + "lnap;": '\U00002A89', + "lnapprox;": '\U00002A89', + "lne;": '\U00002A87', + "lneq;": '\U00002A87', + "lneqq;": '\U00002268', + "lnsim;": '\U000022E6', + "loang;": '\U000027EC', + "loarr;": '\U000021FD', + "lobrk;": '\U000027E6', + "longleftarrow;": '\U000027F5', + "longleftrightarrow;": '\U000027F7', + "longmapsto;": '\U000027FC', + "longrightarrow;": '\U000027F6', + "looparrowleft;": '\U000021AB', + "looparrowright;": '\U000021AC', + "lopar;": '\U00002985', + "lopf;": '\U0001D55D', + "loplus;": '\U00002A2D', + "lotimes;": '\U00002A34', + "lowast;": '\U00002217', + "lowbar;": '\U0000005F', + "loz;": '\U000025CA', + "lozenge;": '\U000025CA', + "lozf;": '\U000029EB', + "lpar;": '\U00000028', + "lparlt;": '\U00002993', + "lrarr;": '\U000021C6', + "lrcorner;": '\U0000231F', + "lrhar;": '\U000021CB', + "lrhard;": '\U0000296D', + "lrm;": '\U0000200E', + "lrtri;": '\U000022BF', + "lsaquo;": '\U00002039', + "lscr;": '\U0001D4C1', + "lsh;": '\U000021B0', + "lsim;": '\U00002272', + "lsime;": '\U00002A8D', + "lsimg;": '\U00002A8F', + "lsqb;": '\U0000005B', + "lsquo;": '\U00002018', + "lsquor;": '\U0000201A', + "lstrok;": '\U00000142', + "lt;": '\U0000003C', + "ltcc;": '\U00002AA6', + "ltcir;": '\U00002A79', + "ltdot;": '\U000022D6', + "lthree;": '\U000022CB', + "ltimes;": '\U000022C9', + "ltlarr;": '\U00002976', + "ltquest;": '\U00002A7B', + "ltrPar;": '\U00002996', + "ltri;": '\U000025C3', + "ltrie;": '\U000022B4', + "ltrif;": '\U000025C2', + "lurdshar;": '\U0000294A', + "luruhar;": '\U00002966', + "mDDot;": '\U0000223A', + "macr;": '\U000000AF', + "male;": '\U00002642', + "malt;": '\U00002720', + "maltese;": '\U00002720', + "map;": '\U000021A6', + "mapsto;": '\U000021A6', + "mapstodown;": '\U000021A7', + "mapstoleft;": '\U000021A4', + "mapstoup;": '\U000021A5', + "marker;": '\U000025AE', + "mcomma;": '\U00002A29', + "mcy;": '\U0000043C', + "mdash;": '\U00002014', + "measuredangle;": '\U00002221', + "mfr;": '\U0001D52A', + "mho;": '\U00002127', + "micro;": '\U000000B5', + "mid;": '\U00002223', + "midast;": '\U0000002A', + "midcir;": '\U00002AF0', + "middot;": '\U000000B7', + "minus;": '\U00002212', + "minusb;": '\U0000229F', + "minusd;": '\U00002238', + "minusdu;": '\U00002A2A', + "mlcp;": '\U00002ADB', + "mldr;": '\U00002026', + "mnplus;": '\U00002213', + "models;": '\U000022A7', + "mopf;": '\U0001D55E', + "mp;": '\U00002213', + "mscr;": '\U0001D4C2', + "mstpos;": '\U0000223E', + "mu;": '\U000003BC', + "multimap;": '\U000022B8', + "mumap;": '\U000022B8', + "nLeftarrow;": '\U000021CD', + "nLeftrightarrow;": '\U000021CE', + "nRightarrow;": '\U000021CF', + "nVDash;": '\U000022AF', + "nVdash;": '\U000022AE', + "nabla;": '\U00002207', + "nacute;": '\U00000144', + "nap;": '\U00002249', + "napos;": '\U00000149', + "napprox;": '\U00002249', + "natur;": '\U0000266E', + "natural;": '\U0000266E', + "naturals;": '\U00002115', + "nbsp;": '\U000000A0', + "ncap;": '\U00002A43', + "ncaron;": '\U00000148', + "ncedil;": '\U00000146', + "ncong;": '\U00002247', + "ncup;": '\U00002A42', + "ncy;": '\U0000043D', + "ndash;": '\U00002013', + "ne;": '\U00002260', + "neArr;": '\U000021D7', + "nearhk;": '\U00002924', + "nearr;": '\U00002197', + "nearrow;": '\U00002197', + "nequiv;": '\U00002262', + "nesear;": '\U00002928', + "nexist;": '\U00002204', + "nexists;": '\U00002204', + "nfr;": '\U0001D52B', + "nge;": '\U00002271', + "ngeq;": '\U00002271', + "ngsim;": '\U00002275', + "ngt;": '\U0000226F', + "ngtr;": '\U0000226F', + "nhArr;": '\U000021CE', + "nharr;": '\U000021AE', + "nhpar;": '\U00002AF2', + "ni;": '\U0000220B', + "nis;": '\U000022FC', + "nisd;": '\U000022FA', + "niv;": '\U0000220B', + "njcy;": '\U0000045A', + "nlArr;": '\U000021CD', + "nlarr;": '\U0000219A', + "nldr;": '\U00002025', + "nle;": '\U00002270', + "nleftarrow;": '\U0000219A', + "nleftrightarrow;": '\U000021AE', + "nleq;": '\U00002270', + "nless;": '\U0000226E', + "nlsim;": '\U00002274', + "nlt;": '\U0000226E', + "nltri;": '\U000022EA', + "nltrie;": '\U000022EC', + "nmid;": '\U00002224', + "nopf;": '\U0001D55F', + "not;": '\U000000AC', + "notin;": '\U00002209', + "notinva;": '\U00002209', + "notinvb;": '\U000022F7', + "notinvc;": '\U000022F6', + "notni;": '\U0000220C', + "notniva;": '\U0000220C', + "notnivb;": '\U000022FE', + "notnivc;": '\U000022FD', + "npar;": '\U00002226', + "nparallel;": '\U00002226', + "npolint;": '\U00002A14', + "npr;": '\U00002280', + "nprcue;": '\U000022E0', + "nprec;": '\U00002280', + "nrArr;": '\U000021CF', + "nrarr;": '\U0000219B', + "nrightarrow;": '\U0000219B', + "nrtri;": '\U000022EB', + "nrtrie;": '\U000022ED', + "nsc;": '\U00002281', + "nsccue;": '\U000022E1', + "nscr;": '\U0001D4C3', + "nshortmid;": '\U00002224', + "nshortparallel;": '\U00002226', + "nsim;": '\U00002241', + "nsime;": '\U00002244', + "nsimeq;": '\U00002244', + "nsmid;": '\U00002224', + "nspar;": '\U00002226', + "nsqsube;": '\U000022E2', + "nsqsupe;": '\U000022E3', + "nsub;": '\U00002284', + "nsube;": '\U00002288', + "nsubseteq;": '\U00002288', + "nsucc;": '\U00002281', + "nsup;": '\U00002285', + "nsupe;": '\U00002289', + "nsupseteq;": '\U00002289', + "ntgl;": '\U00002279', + "ntilde;": '\U000000F1', + "ntlg;": '\U00002278', + "ntriangleleft;": '\U000022EA', + "ntrianglelefteq;": '\U000022EC', + "ntriangleright;": '\U000022EB', + "ntrianglerighteq;": '\U000022ED', + "nu;": '\U000003BD', + "num;": '\U00000023', + "numero;": '\U00002116', + "numsp;": '\U00002007', + "nvDash;": '\U000022AD', + "nvHarr;": '\U00002904', + "nvdash;": '\U000022AC', + "nvinfin;": '\U000029DE', + "nvlArr;": '\U00002902', + "nvrArr;": '\U00002903', + "nwArr;": '\U000021D6', + "nwarhk;": '\U00002923', + "nwarr;": '\U00002196', + "nwarrow;": '\U00002196', + "nwnear;": '\U00002927', + "oS;": '\U000024C8', + "oacute;": '\U000000F3', + "oast;": '\U0000229B', + "ocir;": '\U0000229A', + "ocirc;": '\U000000F4', + "ocy;": '\U0000043E', + "odash;": '\U0000229D', + "odblac;": '\U00000151', + "odiv;": '\U00002A38', + "odot;": '\U00002299', + "odsold;": '\U000029BC', + "oelig;": '\U00000153', + "ofcir;": '\U000029BF', + "ofr;": '\U0001D52C', + "ogon;": '\U000002DB', + "ograve;": '\U000000F2', + "ogt;": '\U000029C1', + "ohbar;": '\U000029B5', + "ohm;": '\U000003A9', + "oint;": '\U0000222E', + "olarr;": '\U000021BA', + "olcir;": '\U000029BE', + "olcross;": '\U000029BB', + "oline;": '\U0000203E', + "olt;": '\U000029C0', + "omacr;": '\U0000014D', + "omega;": '\U000003C9', + "omicron;": '\U000003BF', + "omid;": '\U000029B6', + "ominus;": '\U00002296', + "oopf;": '\U0001D560', + "opar;": '\U000029B7', + "operp;": '\U000029B9', + "oplus;": '\U00002295', + "or;": '\U00002228', + "orarr;": '\U000021BB', + "ord;": '\U00002A5D', + "order;": '\U00002134', + "orderof;": '\U00002134', + "ordf;": '\U000000AA', + "ordm;": '\U000000BA', + "origof;": '\U000022B6', + "oror;": '\U00002A56', + "orslope;": '\U00002A57', + "orv;": '\U00002A5B', + "oscr;": '\U00002134', + "oslash;": '\U000000F8', + "osol;": '\U00002298', + "otilde;": '\U000000F5', + "otimes;": '\U00002297', + "otimesas;": '\U00002A36', + "ouml;": '\U000000F6', + "ovbar;": '\U0000233D', + "par;": '\U00002225', + "para;": '\U000000B6', + "parallel;": '\U00002225', + "parsim;": '\U00002AF3', + "parsl;": '\U00002AFD', + "part;": '\U00002202', + "pcy;": '\U0000043F', + "percnt;": '\U00000025', + "period;": '\U0000002E', + "permil;": '\U00002030', + "perp;": '\U000022A5', + "pertenk;": '\U00002031', + "pfr;": '\U0001D52D', + "phi;": '\U000003C6', + "phiv;": '\U000003D5', + "phmmat;": '\U00002133', + "phone;": '\U0000260E', + "pi;": '\U000003C0', + "pitchfork;": '\U000022D4', + "piv;": '\U000003D6', + "planck;": '\U0000210F', + "planckh;": '\U0000210E', + "plankv;": '\U0000210F', + "plus;": '\U0000002B', + "plusacir;": '\U00002A23', + "plusb;": '\U0000229E', + "pluscir;": '\U00002A22', + "plusdo;": '\U00002214', + "plusdu;": '\U00002A25', + "pluse;": '\U00002A72', + "plusmn;": '\U000000B1', + "plussim;": '\U00002A26', + "plustwo;": '\U00002A27', + "pm;": '\U000000B1', + "pointint;": '\U00002A15', + "popf;": '\U0001D561', + "pound;": '\U000000A3', + "pr;": '\U0000227A', + "prE;": '\U00002AB3', + "prap;": '\U00002AB7', + "prcue;": '\U0000227C', + "pre;": '\U00002AAF', + "prec;": '\U0000227A', + "precapprox;": '\U00002AB7', + "preccurlyeq;": '\U0000227C', + "preceq;": '\U00002AAF', + "precnapprox;": '\U00002AB9', + "precneqq;": '\U00002AB5', + "precnsim;": '\U000022E8', + "precsim;": '\U0000227E', + "prime;": '\U00002032', + "primes;": '\U00002119', + "prnE;": '\U00002AB5', + "prnap;": '\U00002AB9', + "prnsim;": '\U000022E8', + "prod;": '\U0000220F', + "profalar;": '\U0000232E', + "profline;": '\U00002312', + "profsurf;": '\U00002313', + "prop;": '\U0000221D', + "propto;": '\U0000221D', + "prsim;": '\U0000227E', + "prurel;": '\U000022B0', + "pscr;": '\U0001D4C5', + "psi;": '\U000003C8', + "puncsp;": '\U00002008', + "qfr;": '\U0001D52E', + "qint;": '\U00002A0C', + "qopf;": '\U0001D562', + "qprime;": '\U00002057', + "qscr;": '\U0001D4C6', + "quaternions;": '\U0000210D', + "quatint;": '\U00002A16', + "quest;": '\U0000003F', + "questeq;": '\U0000225F', + "quot;": '\U00000022', + "rAarr;": '\U000021DB', + "rArr;": '\U000021D2', + "rAtail;": '\U0000291C', + "rBarr;": '\U0000290F', + "rHar;": '\U00002964', + "racute;": '\U00000155', + "radic;": '\U0000221A', + "raemptyv;": '\U000029B3', + "rang;": '\U000027E9', + "rangd;": '\U00002992', + "range;": '\U000029A5', + "rangle;": '\U000027E9', + "raquo;": '\U000000BB', + "rarr;": '\U00002192', + "rarrap;": '\U00002975', + "rarrb;": '\U000021E5', + "rarrbfs;": '\U00002920', + "rarrc;": '\U00002933', + "rarrfs;": '\U0000291E', + "rarrhk;": '\U000021AA', + "rarrlp;": '\U000021AC', + "rarrpl;": '\U00002945', + "rarrsim;": '\U00002974', + "rarrtl;": '\U000021A3', + "rarrw;": '\U0000219D', + "ratail;": '\U0000291A', + "ratio;": '\U00002236', + "rationals;": '\U0000211A', + "rbarr;": '\U0000290D', + "rbbrk;": '\U00002773', + "rbrace;": '\U0000007D', + "rbrack;": '\U0000005D', + "rbrke;": '\U0000298C', + "rbrksld;": '\U0000298E', + "rbrkslu;": '\U00002990', + "rcaron;": '\U00000159', + "rcedil;": '\U00000157', + "rceil;": '\U00002309', + "rcub;": '\U0000007D', + "rcy;": '\U00000440', + "rdca;": '\U00002937', + "rdldhar;": '\U00002969', + "rdquo;": '\U0000201D', + "rdquor;": '\U0000201D', + "rdsh;": '\U000021B3', + "real;": '\U0000211C', + "realine;": '\U0000211B', + "realpart;": '\U0000211C', + "reals;": '\U0000211D', + "rect;": '\U000025AD', + "reg;": '\U000000AE', + "rfisht;": '\U0000297D', + "rfloor;": '\U0000230B', + "rfr;": '\U0001D52F', + "rhard;": '\U000021C1', + "rharu;": '\U000021C0', + "rharul;": '\U0000296C', + "rho;": '\U000003C1', + "rhov;": '\U000003F1', + "rightarrow;": '\U00002192', + "rightarrowtail;": '\U000021A3', + "rightharpoondown;": '\U000021C1', + "rightharpoonup;": '\U000021C0', + "rightleftarrows;": '\U000021C4', + "rightleftharpoons;": '\U000021CC', + "rightrightarrows;": '\U000021C9', + "rightsquigarrow;": '\U0000219D', + "rightthreetimes;": '\U000022CC', + "ring;": '\U000002DA', + "risingdotseq;": '\U00002253', + "rlarr;": '\U000021C4', + "rlhar;": '\U000021CC', + "rlm;": '\U0000200F', + "rmoust;": '\U000023B1', + "rmoustache;": '\U000023B1', + "rnmid;": '\U00002AEE', + "roang;": '\U000027ED', + "roarr;": '\U000021FE', + "robrk;": '\U000027E7', + "ropar;": '\U00002986', + "ropf;": '\U0001D563', + "roplus;": '\U00002A2E', + "rotimes;": '\U00002A35', + "rpar;": '\U00000029', + "rpargt;": '\U00002994', + "rppolint;": '\U00002A12', + "rrarr;": '\U000021C9', + "rsaquo;": '\U0000203A', + "rscr;": '\U0001D4C7', + "rsh;": '\U000021B1', + "rsqb;": '\U0000005D', + "rsquo;": '\U00002019', + "rsquor;": '\U00002019', + "rthree;": '\U000022CC', + "rtimes;": '\U000022CA', + "rtri;": '\U000025B9', + "rtrie;": '\U000022B5', + "rtrif;": '\U000025B8', + "rtriltri;": '\U000029CE', + "ruluhar;": '\U00002968', + "rx;": '\U0000211E', + "sacute;": '\U0000015B', + "sbquo;": '\U0000201A', + "sc;": '\U0000227B', + "scE;": '\U00002AB4', + "scap;": '\U00002AB8', + "scaron;": '\U00000161', + "sccue;": '\U0000227D', + "sce;": '\U00002AB0', + "scedil;": '\U0000015F', + "scirc;": '\U0000015D', + "scnE;": '\U00002AB6', + "scnap;": '\U00002ABA', + "scnsim;": '\U000022E9', + "scpolint;": '\U00002A13', + "scsim;": '\U0000227F', + "scy;": '\U00000441', + "sdot;": '\U000022C5', + "sdotb;": '\U000022A1', + "sdote;": '\U00002A66', + "seArr;": '\U000021D8', + "searhk;": '\U00002925', + "searr;": '\U00002198', + "searrow;": '\U00002198', + "sect;": '\U000000A7', + "semi;": '\U0000003B', + "seswar;": '\U00002929', + "setminus;": '\U00002216', + "setmn;": '\U00002216', + "sext;": '\U00002736', + "sfr;": '\U0001D530', + "sfrown;": '\U00002322', + "sharp;": '\U0000266F', + "shchcy;": '\U00000449', + "shcy;": '\U00000448', + "shortmid;": '\U00002223', + "shortparallel;": '\U00002225', + "shy;": '\U000000AD', + "sigma;": '\U000003C3', + "sigmaf;": '\U000003C2', + "sigmav;": '\U000003C2', + "sim;": '\U0000223C', + "simdot;": '\U00002A6A', + "sime;": '\U00002243', + "simeq;": '\U00002243', + "simg;": '\U00002A9E', + "simgE;": '\U00002AA0', + "siml;": '\U00002A9D', + "simlE;": '\U00002A9F', + "simne;": '\U00002246', + "simplus;": '\U00002A24', + "simrarr;": '\U00002972', + "slarr;": '\U00002190', + "smallsetminus;": '\U00002216', + "smashp;": '\U00002A33', + "smeparsl;": '\U000029E4', + "smid;": '\U00002223', + "smile;": '\U00002323', + "smt;": '\U00002AAA', + "smte;": '\U00002AAC', + "softcy;": '\U0000044C', + "sol;": '\U0000002F', + "solb;": '\U000029C4', + "solbar;": '\U0000233F', + "sopf;": '\U0001D564', + "spades;": '\U00002660', + "spadesuit;": '\U00002660', + "spar;": '\U00002225', + "sqcap;": '\U00002293', + "sqcup;": '\U00002294', + "sqsub;": '\U0000228F', + "sqsube;": '\U00002291', + "sqsubset;": '\U0000228F', + "sqsubseteq;": '\U00002291', + "sqsup;": '\U00002290', + "sqsupe;": '\U00002292', + "sqsupset;": '\U00002290', + "sqsupseteq;": '\U00002292', + "squ;": '\U000025A1', + "square;": '\U000025A1', + "squarf;": '\U000025AA', + "squf;": '\U000025AA', + "srarr;": '\U00002192', + "sscr;": '\U0001D4C8', + "ssetmn;": '\U00002216', + "ssmile;": '\U00002323', + "sstarf;": '\U000022C6', + "star;": '\U00002606', + "starf;": '\U00002605', + "straightepsilon;": '\U000003F5', + "straightphi;": '\U000003D5', + "strns;": '\U000000AF', + "sub;": '\U00002282', + "subE;": '\U00002AC5', + "subdot;": '\U00002ABD', + "sube;": '\U00002286', + "subedot;": '\U00002AC3', + "submult;": '\U00002AC1', + "subnE;": '\U00002ACB', + "subne;": '\U0000228A', + "subplus;": '\U00002ABF', + "subrarr;": '\U00002979', + "subset;": '\U00002282', + "subseteq;": '\U00002286', + "subseteqq;": '\U00002AC5', + "subsetneq;": '\U0000228A', + "subsetneqq;": '\U00002ACB', + "subsim;": '\U00002AC7', + "subsub;": '\U00002AD5', + "subsup;": '\U00002AD3', + "succ;": '\U0000227B', + "succapprox;": '\U00002AB8', + "succcurlyeq;": '\U0000227D', + "succeq;": '\U00002AB0', + "succnapprox;": '\U00002ABA', + "succneqq;": '\U00002AB6', + "succnsim;": '\U000022E9', + "succsim;": '\U0000227F', + "sum;": '\U00002211', + "sung;": '\U0000266A', + "sup;": '\U00002283', + "sup1;": '\U000000B9', + "sup2;": '\U000000B2', + "sup3;": '\U000000B3', + "supE;": '\U00002AC6', + "supdot;": '\U00002ABE', + "supdsub;": '\U00002AD8', + "supe;": '\U00002287', + "supedot;": '\U00002AC4', + "suphsol;": '\U000027C9', + "suphsub;": '\U00002AD7', + "suplarr;": '\U0000297B', + "supmult;": '\U00002AC2', + "supnE;": '\U00002ACC', + "supne;": '\U0000228B', + "supplus;": '\U00002AC0', + "supset;": '\U00002283', + "supseteq;": '\U00002287', + "supseteqq;": '\U00002AC6', + "supsetneq;": '\U0000228B', + "supsetneqq;": '\U00002ACC', + "supsim;": '\U00002AC8', + "supsub;": '\U00002AD4', + "supsup;": '\U00002AD6', + "swArr;": '\U000021D9', + "swarhk;": '\U00002926', + "swarr;": '\U00002199', + "swarrow;": '\U00002199', + "swnwar;": '\U0000292A', + "szlig;": '\U000000DF', + "target;": '\U00002316', + "tau;": '\U000003C4', + "tbrk;": '\U000023B4', + "tcaron;": '\U00000165', + "tcedil;": '\U00000163', + "tcy;": '\U00000442', + "tdot;": '\U000020DB', + "telrec;": '\U00002315', + "tfr;": '\U0001D531', + "there4;": '\U00002234', + "therefore;": '\U00002234', + "theta;": '\U000003B8', + "thetasym;": '\U000003D1', + "thetav;": '\U000003D1', + "thickapprox;": '\U00002248', + "thicksim;": '\U0000223C', + "thinsp;": '\U00002009', + "thkap;": '\U00002248', + "thksim;": '\U0000223C', + "thorn;": '\U000000FE', + "tilde;": '\U000002DC', + "times;": '\U000000D7', + "timesb;": '\U000022A0', + "timesbar;": '\U00002A31', + "timesd;": '\U00002A30', + "tint;": '\U0000222D', + "toea;": '\U00002928', + "top;": '\U000022A4', + "topbot;": '\U00002336', + "topcir;": '\U00002AF1', + "topf;": '\U0001D565', + "topfork;": '\U00002ADA', + "tosa;": '\U00002929', + "tprime;": '\U00002034', + "trade;": '\U00002122', + "triangle;": '\U000025B5', + "triangledown;": '\U000025BF', + "triangleleft;": '\U000025C3', + "trianglelefteq;": '\U000022B4', + "triangleq;": '\U0000225C', + "triangleright;": '\U000025B9', + "trianglerighteq;": '\U000022B5', + "tridot;": '\U000025EC', + "trie;": '\U0000225C', + "triminus;": '\U00002A3A', + "triplus;": '\U00002A39', + "trisb;": '\U000029CD', + "tritime;": '\U00002A3B', + "trpezium;": '\U000023E2', + "tscr;": '\U0001D4C9', + "tscy;": '\U00000446', + "tshcy;": '\U0000045B', + "tstrok;": '\U00000167', + "twixt;": '\U0000226C', + "twoheadleftarrow;": '\U0000219E', + "twoheadrightarrow;": '\U000021A0', + "uArr;": '\U000021D1', + "uHar;": '\U00002963', + "uacute;": '\U000000FA', + "uarr;": '\U00002191', + "ubrcy;": '\U0000045E', + "ubreve;": '\U0000016D', + "ucirc;": '\U000000FB', + "ucy;": '\U00000443', + "udarr;": '\U000021C5', + "udblac;": '\U00000171', + "udhar;": '\U0000296E', + "ufisht;": '\U0000297E', + "ufr;": '\U0001D532', + "ugrave;": '\U000000F9', + "uharl;": '\U000021BF', + "uharr;": '\U000021BE', + "uhblk;": '\U00002580', + "ulcorn;": '\U0000231C', + "ulcorner;": '\U0000231C', + "ulcrop;": '\U0000230F', + "ultri;": '\U000025F8', + "umacr;": '\U0000016B', + "uml;": '\U000000A8', + "uogon;": '\U00000173', + "uopf;": '\U0001D566', + "uparrow;": '\U00002191', + "updownarrow;": '\U00002195', + "upharpoonleft;": '\U000021BF', + "upharpoonright;": '\U000021BE', + "uplus;": '\U0000228E', + "upsi;": '\U000003C5', + "upsih;": '\U000003D2', + "upsilon;": '\U000003C5', + "upuparrows;": '\U000021C8', + "urcorn;": '\U0000231D', + "urcorner;": '\U0000231D', + "urcrop;": '\U0000230E', + "uring;": '\U0000016F', + "urtri;": '\U000025F9', + "uscr;": '\U0001D4CA', + "utdot;": '\U000022F0', + "utilde;": '\U00000169', + "utri;": '\U000025B5', + "utrif;": '\U000025B4', + "uuarr;": '\U000021C8', + "uuml;": '\U000000FC', + "uwangle;": '\U000029A7', + "vArr;": '\U000021D5', + "vBar;": '\U00002AE8', + "vBarv;": '\U00002AE9', + "vDash;": '\U000022A8', + "vangrt;": '\U0000299C', + "varepsilon;": '\U000003F5', + "varkappa;": '\U000003F0', + "varnothing;": '\U00002205', + "varphi;": '\U000003D5', + "varpi;": '\U000003D6', + "varpropto;": '\U0000221D', + "varr;": '\U00002195', + "varrho;": '\U000003F1', + "varsigma;": '\U000003C2', + "vartheta;": '\U000003D1', + "vartriangleleft;": '\U000022B2', + "vartriangleright;": '\U000022B3', + "vcy;": '\U00000432', + "vdash;": '\U000022A2', + "vee;": '\U00002228', + "veebar;": '\U000022BB', + "veeeq;": '\U0000225A', + "vellip;": '\U000022EE', + "verbar;": '\U0000007C', + "vert;": '\U0000007C', + "vfr;": '\U0001D533', + "vltri;": '\U000022B2', + "vopf;": '\U0001D567', + "vprop;": '\U0000221D', + "vrtri;": '\U000022B3', + "vscr;": '\U0001D4CB', + "vzigzag;": '\U0000299A', + "wcirc;": '\U00000175', + "wedbar;": '\U00002A5F', + "wedge;": '\U00002227', + "wedgeq;": '\U00002259', + "weierp;": '\U00002118', + "wfr;": '\U0001D534', + "wopf;": '\U0001D568', + "wp;": '\U00002118', + "wr;": '\U00002240', + "wreath;": '\U00002240', + "wscr;": '\U0001D4CC', + "xcap;": '\U000022C2', + "xcirc;": '\U000025EF', + "xcup;": '\U000022C3', + "xdtri;": '\U000025BD', + "xfr;": '\U0001D535', + "xhArr;": '\U000027FA', + "xharr;": '\U000027F7', + "xi;": '\U000003BE', + "xlArr;": '\U000027F8', + "xlarr;": '\U000027F5', + "xmap;": '\U000027FC', + "xnis;": '\U000022FB', + "xodot;": '\U00002A00', + "xopf;": '\U0001D569', + "xoplus;": '\U00002A01', + "xotime;": '\U00002A02', + "xrArr;": '\U000027F9', + "xrarr;": '\U000027F6', + "xscr;": '\U0001D4CD', + "xsqcup;": '\U00002A06', + "xuplus;": '\U00002A04', + "xutri;": '\U000025B3', + "xvee;": '\U000022C1', + "xwedge;": '\U000022C0', + "yacute;": '\U000000FD', + "yacy;": '\U0000044F', + "ycirc;": '\U00000177', + "ycy;": '\U0000044B', + "yen;": '\U000000A5', + "yfr;": '\U0001D536', + "yicy;": '\U00000457', + "yopf;": '\U0001D56A', + "yscr;": '\U0001D4CE', + "yucy;": '\U0000044E', + "yuml;": '\U000000FF', + "zacute;": '\U0000017A', + "zcaron;": '\U0000017E', + "zcy;": '\U00000437', + "zdot;": '\U0000017C', + "zeetrf;": '\U00002128', + "zeta;": '\U000003B6', + "zfr;": '\U0001D537', + "zhcy;": '\U00000436', + "zigrarr;": '\U000021DD', + "zopf;": '\U0001D56B', + "zscr;": '\U0001D4CF', + "zwj;": '\U0000200D', + "zwnj;": '\U0000200C', + "AElig": '\U000000C6', + "AMP": '\U00000026', + "Aacute": '\U000000C1', + "Acirc": '\U000000C2', + "Agrave": '\U000000C0', + "Aring": '\U000000C5', + "Atilde": '\U000000C3', + "Auml": '\U000000C4', + "COPY": '\U000000A9', + "Ccedil": '\U000000C7', + "ETH": '\U000000D0', + "Eacute": '\U000000C9', + "Ecirc": '\U000000CA', + "Egrave": '\U000000C8', + "Euml": '\U000000CB', + "GT": '\U0000003E', + "Iacute": '\U000000CD', + "Icirc": '\U000000CE', + "Igrave": '\U000000CC', + "Iuml": '\U000000CF', + "LT": '\U0000003C', + "Ntilde": '\U000000D1', + "Oacute": '\U000000D3', + "Ocirc": '\U000000D4', + "Ograve": '\U000000D2', + "Oslash": '\U000000D8', + "Otilde": '\U000000D5', + "Ouml": '\U000000D6', + "QUOT": '\U00000022', + "REG": '\U000000AE', + "THORN": '\U000000DE', + "Uacute": '\U000000DA', + "Ucirc": '\U000000DB', + "Ugrave": '\U000000D9', + "Uuml": '\U000000DC', + "Yacute": '\U000000DD', + "aacute": '\U000000E1', + "acirc": '\U000000E2', + "acute": '\U000000B4', + "aelig": '\U000000E6', + "agrave": '\U000000E0', + "amp": '\U00000026', + "aring": '\U000000E5', + "atilde": '\U000000E3', + "auml": '\U000000E4', + "brvbar": '\U000000A6', + "ccedil": '\U000000E7', + "cedil": '\U000000B8', + "cent": '\U000000A2', + "copy": '\U000000A9', + "curren": '\U000000A4', + "deg": '\U000000B0', + "divide": '\U000000F7', + "eacute": '\U000000E9', + "ecirc": '\U000000EA', + "egrave": '\U000000E8', + "eth": '\U000000F0', + "euml": '\U000000EB', + "frac12": '\U000000BD', + "frac14": '\U000000BC', + "frac34": '\U000000BE', + "gt": '\U0000003E', + "iacute": '\U000000ED', + "icirc": '\U000000EE', + "iexcl": '\U000000A1', + "igrave": '\U000000EC', + "iquest": '\U000000BF', + "iuml": '\U000000EF', + "laquo": '\U000000AB', + "lt": '\U0000003C', + "macr": '\U000000AF', + "micro": '\U000000B5', + "middot": '\U000000B7', + "nbsp": '\U000000A0', + "not": '\U000000AC', + "ntilde": '\U000000F1', + "oacute": '\U000000F3', + "ocirc": '\U000000F4', + "ograve": '\U000000F2', + "ordf": '\U000000AA', + "ordm": '\U000000BA', + "oslash": '\U000000F8', + "otilde": '\U000000F5', + "ouml": '\U000000F6', + "para": '\U000000B6', + "plusmn": '\U000000B1', + "pound": '\U000000A3', + "quot": '\U00000022', + "raquo": '\U000000BB', + "reg": '\U000000AE', + "sect": '\U000000A7', + "shy": '\U000000AD', + "sup1": '\U000000B9', + "sup2": '\U000000B2', + "sup3": '\U000000B3', + "szlig": '\U000000DF', + "thorn": '\U000000FE', + "times": '\U000000D7', + "uacute": '\U000000FA', + "ucirc": '\U000000FB', + "ugrave": '\U000000F9', + "uml": '\U000000A8', + "uuml": '\U000000FC', + "yacute": '\U000000FD', + "yen": '\U000000A5', + "yuml": '\U000000FF', } // HTML entities that are two unicode codepoints. diff --git a/third_party/code.google.com/p/go.net/html/token.go b/third_party/code.google.com/p/go.net/html/token.go index 3a1ee7ccb..c43debbcf 100644 --- a/third_party/code.google.com/p/go.net/html/token.go +++ b/third_party/code.google.com/p/go.net/html/token.go @@ -734,7 +734,6 @@ func (z *Tokenizer) readCDATA() bool { brackets = 0 } } - panic("unreachable") } // startTagIn returns whether the start tag in z.buf[z.data.start:z.data.end] @@ -934,13 +933,13 @@ func (z *Tokenizer) readTagAttrVal() { // Next scans the next token and returns its type. func (z *Tokenizer) Next() TokenType { + z.raw.start = z.raw.end + z.data.start = z.raw.end + z.data.end = z.raw.end if z.err != nil { z.tt = ErrorToken return z.tt } - z.raw.start = z.raw.end - z.data.start = z.raw.end - z.data.end = z.raw.end if z.rawTag != "" { if z.rawTag == "plaintext" { // Read everything up to EOF. @@ -1010,12 +1009,11 @@ loop: break loop } if c == '>' { - // "" does not generate a token at all. + // "" does not generate a token at all. Generate an empty comment + // to allow passthrough clients to pick up the data using Raw. // Reset the tokenizer state and start again. - z.raw.start = z.raw.end - z.data.start = z.raw.end - z.data.end = z.raw.end - continue loop + z.tt = CommentToken + return z.tt } if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' { z.readTag(false) diff --git a/third_party/code.google.com/p/go.net/html/token_test.go b/third_party/code.google.com/p/go.net/html/token_test.go index ca408da40..e7235cda5 100644 --- a/third_party/code.google.com/p/go.net/html/token_test.go +++ b/third_party/code.google.com/p/go.net/html/token_test.go @@ -63,12 +63,12 @@ var tokenTests = []tokenTest{ { "not a tag #2", "", - "", + "", }, { "not a tag #3", "ab", - "a$b", + "a$$b", }, { "not a tag #4", @@ -469,6 +469,25 @@ loop: } } +func TestPassthrough(t *testing.T) { + // Accumulating the raw output for each parse event should reconstruct the + // original input. + for _, test := range tokenTests { + z := NewTokenizer(strings.NewReader(test.html)) + var parsed bytes.Buffer + for { + tt := z.Next() + parsed.Write(z.Raw()) + if tt == ErrorToken { + break + } + } + if got, want := parsed.String(), test.html; got != want { + t.Errorf("%s: parsed output:\n got: %q\nwant: %q", test.desc, got, want) + } + } +} + func TestBufAPI(t *testing.T) { s := "0123456789" z := NewTokenizer(bytes.NewBufferString(s)) @@ -510,20 +529,20 @@ func TestConvertNewlines(t *testing.T) { "Mac\rDOS\r\nUnix\n": "Mac\nDOS\nUnix\n", "Unix\nMac\rDOS\r\n": "Unix\nMac\nDOS\n", "DOS\r\nDOS\r\nDOS\r\n": "DOS\nDOS\nDOS\n", - "": "", - "\n": "\n", - "\n\r": "\n\n", - "\r": "\n", - "\r\n": "\n", - "\r\n\n": "\n\n", - "\r\n\r": "\n\n", - "\r\n\r\n": "\n\n", - "\r\r": "\n\n", - "\r\r\n": "\n\n", - "\r\r\n\n": "\n\n\n", - "\r\r\r\n": "\n\n\n", - "\r \n": "\n \n", - "xyz": "xyz", + "": "", + "\n": "\n", + "\n\r": "\n\n", + "\r": "\n", + "\r\n": "\n", + "\r\n\n": "\n\n", + "\r\n\r": "\n\n", + "\r\n\r\n": "\n\n", + "\r\r": "\n\n", + "\r\r\n": "\n\n", + "\r\r\n\n": "\n\n\n", + "\r\r\r\n": "\n\n\n", + "\r \n": "\n \n", + "xyz": "xyz", } for in, want := range testCases { if got := string(convertNewlines([]byte(in))); got != want { diff --git a/third_party/code.google.com/p/go.net/ipv6/control_rfc2292_unix.go b/third_party/code.google.com/p/go.net/ipv6/control_rfc2292_unix.go new file mode 100644 index 000000000..47196c58f --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/control_rfc2292_unix.go @@ -0,0 +1,151 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin + +package ipv6 + +import ( + "net" + "os" + "syscall" + "unsafe" +) + +const pktinfo = FlagDst | FlagInterface + +func setControlMessage(fd int, opt *rawOpt, cf ControlFlags, on bool) error { + opt.Lock() + defer opt.Unlock() + if cf&FlagHopLimit != 0 { + if err := setIPv6ReceiveHopLimit(fd, on); err != nil { + return err + } + if on { + opt.set(FlagHopLimit) + } else { + opt.clear(FlagHopLimit) + } + } + if cf&pktinfo != 0 { + if err := setIPv6ReceivePacketInfo(fd, on); err != nil { + return err + } + if on { + opt.set(cf & pktinfo) + } else { + opt.clear(cf & pktinfo) + } + } + return nil +} + +func newControlMessage(opt *rawOpt) (oob []byte) { + opt.Lock() + defer opt.Unlock() + l, off := 0, 0 + if opt.isset(FlagHopLimit) { + l += syscall.CmsgSpace(4) + } + if opt.isset(pktinfo) { + l += syscall.CmsgSpace(sysSizeofPacketInfo) + } + if l > 0 { + oob = make([]byte, l) + if opt.isset(FlagHopLimit) { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockopt2292HopLimit + m.SetLen(syscall.CmsgLen(4)) + off += syscall.CmsgSpace(4) + } + if opt.isset(pktinfo) { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockopt2292PacketInfo + m.SetLen(syscall.CmsgLen(sysSizeofPacketInfo)) + off += syscall.CmsgSpace(sysSizeofPacketInfo) + } + } + return +} + +func parseControlMessage(b []byte) (*ControlMessage, error) { + if len(b) == 0 { + return nil, nil + } + cmsgs, err := syscall.ParseSocketControlMessage(b) + if err != nil { + return nil, os.NewSyscallError("parse socket control message", err) + } + cm := &ControlMessage{} + for _, m := range cmsgs { + if m.Header.Level != ianaProtocolIPv6 { + continue + } + switch m.Header.Type { + case sysSockopt2292HopLimit: + cm.HopLimit = int(*(*byte)(unsafe.Pointer(&m.Data[:1][0]))) + case sysSockopt2292PacketInfo: + pi := (*sysPacketInfo)(unsafe.Pointer(&m.Data[0])) + cm.IfIndex = int(pi.IfIndex) + cm.Dst = pi.IP[:] + } + } + return cm, nil +} + +func marshalControlMessage(cm *ControlMessage) (oob []byte) { + if cm == nil { + return + } + l, off := 0, 0 + if cm.HopLimit > 0 { + l += syscall.CmsgSpace(4) + } + pion := false + if cm.Src.To4() == nil && cm.Src.To16() != nil || cm.IfIndex != 0 { + pion = true + l += syscall.CmsgSpace(sysSizeofPacketInfo) + } + if len(cm.NextHop) == net.IPv6len { + l += syscall.CmsgSpace(syscall.SizeofSockaddrInet6) + } + if l > 0 { + oob = make([]byte, l) + if cm.HopLimit > 0 { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockopt2292HopLimit + m.SetLen(syscall.CmsgLen(4)) + data := oob[off+syscall.CmsgLen(0):] + *(*byte)(unsafe.Pointer(&data[:1][0])) = byte(cm.HopLimit) + off += syscall.CmsgSpace(4) + } + if pion { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockopt2292PacketInfo + m.SetLen(syscall.CmsgLen(sysSizeofPacketInfo)) + pi := (*sysPacketInfo)(unsafe.Pointer(&oob[off+syscall.CmsgLen(0)])) + if ip := cm.Src.To16(); ip != nil && ip.To4() == nil { + copy(pi.IP[:], ip) + } + if cm.IfIndex != 0 { + pi.IfIndex = uint32(cm.IfIndex) + } + off += syscall.CmsgSpace(sysSizeofPacketInfo) + } + if len(cm.NextHop) == net.IPv6len { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockopt2292NextHop + m.SetLen(syscall.CmsgLen(syscall.SizeofSockaddrInet6)) + sa := (*syscall.RawSockaddrInet6)(unsafe.Pointer(&oob[off+syscall.CmsgLen(0)])) + setSockaddr(sa, cm.NextHop, cm.IfIndex) + off += syscall.CmsgSpace(syscall.SizeofSockaddrInet6) + } + } + return +} diff --git a/third_party/code.google.com/p/go.net/ipv6/control_rfc3542_unix.go b/third_party/code.google.com/p/go.net/ipv6/control_rfc3542_unix.go new file mode 100644 index 000000000..882a77be3 --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/control_rfc3542_unix.go @@ -0,0 +1,210 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd linux netbsd openbsd + +package ipv6 + +import ( + "net" + "os" + "syscall" + "unsafe" +) + +const pktinfo = FlagDst | FlagInterface + +func setControlMessage(fd int, opt *rawOpt, cf ControlFlags, on bool) error { + opt.Lock() + defer opt.Unlock() + if cf&FlagTrafficClass != 0 { + if err := setIPv6ReceiveTrafficClass(fd, on); err != nil { + return err + } + if on { + opt.set(FlagTrafficClass) + } else { + opt.clear(FlagTrafficClass) + } + } + if cf&FlagHopLimit != 0 { + if err := setIPv6ReceiveHopLimit(fd, on); err != nil { + return err + } + if on { + opt.set(FlagHopLimit) + } else { + opt.clear(FlagHopLimit) + } + } + if cf&pktinfo != 0 { + if err := setIPv6ReceivePacketInfo(fd, on); err != nil { + return err + } + if on { + opt.set(cf & pktinfo) + } else { + opt.clear(cf & pktinfo) + } + } + if cf&FlagPathMTU != 0 { + if err := setIPv6ReceivePathMTU(fd, on); err != nil { + return err + } + if on { + opt.set(FlagPathMTU) + } else { + opt.clear(FlagPathMTU) + } + } + return nil +} + +func newControlMessage(opt *rawOpt) (oob []byte) { + opt.Lock() + defer opt.Unlock() + l, off := 0, 0 + if opt.isset(FlagTrafficClass) { + l += syscall.CmsgSpace(4) + } + if opt.isset(FlagHopLimit) { + l += syscall.CmsgSpace(4) + } + if opt.isset(pktinfo) { + l += syscall.CmsgSpace(sysSizeofPacketInfo) + } + if opt.isset(FlagPathMTU) { + l += syscall.CmsgSpace(sysSizeofMTUInfo) + } + if l > 0 { + oob = make([]byte, l) + if opt.isset(FlagTrafficClass) { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockoptReceiveTrafficClass + m.SetLen(syscall.CmsgLen(4)) + off += syscall.CmsgSpace(4) + } + if opt.isset(FlagHopLimit) { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockoptReceiveHopLimit + m.SetLen(syscall.CmsgLen(4)) + off += syscall.CmsgSpace(4) + } + if opt.isset(pktinfo) { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockoptReceivePacketInfo + m.SetLen(syscall.CmsgLen(sysSizeofPacketInfo)) + off += syscall.CmsgSpace(sysSizeofPacketInfo) + } + if opt.isset(FlagPathMTU) { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockoptReceivePathMTU + m.SetLen(syscall.CmsgLen(sysSizeofMTUInfo)) + off += syscall.CmsgSpace(sysSizeofMTUInfo) + } + } + return +} + +func parseControlMessage(b []byte) (*ControlMessage, error) { + if len(b) == 0 { + return nil, nil + } + cmsgs, err := syscall.ParseSocketControlMessage(b) + if err != nil { + return nil, os.NewSyscallError("parse socket control message", err) + } + cm := &ControlMessage{} + for _, m := range cmsgs { + if m.Header.Level != ianaProtocolIPv6 { + continue + } + switch m.Header.Type { + case sysSockoptTrafficClass: + cm.TrafficClass = int(*(*byte)(unsafe.Pointer(&m.Data[:1][0]))) + case sysSockoptHopLimit: + cm.HopLimit = int(*(*byte)(unsafe.Pointer(&m.Data[:1][0]))) + case sysSockoptPacketInfo: + pi := (*sysPacketInfo)(unsafe.Pointer(&m.Data[0])) + cm.Dst = pi.IP[:] + cm.IfIndex = int(pi.IfIndex) + case sysSockoptPathMTU: + mi := (*sysMTUInfo)(unsafe.Pointer(&m.Data[0])) + cm.Dst = mi.Addr.Addr[:] + cm.IfIndex = int(mi.Addr.Scope_id) + cm.MTU = int(mi.MTU) + } + } + return cm, nil +} + +func marshalControlMessage(cm *ControlMessage) (oob []byte) { + if cm == nil { + return + } + l, off := 0, 0 + if cm.TrafficClass > 0 { + l += syscall.CmsgSpace(4) + } + if cm.HopLimit > 0 { + l += syscall.CmsgSpace(4) + } + pion := false + if cm.Src.To4() == nil && cm.Src.To16() != nil || cm.IfIndex != 0 { + pion = true + l += syscall.CmsgSpace(sysSizeofPacketInfo) + } + if len(cm.NextHop) == net.IPv6len { + l += syscall.CmsgSpace(syscall.SizeofSockaddrInet6) + } + if l > 0 { + oob = make([]byte, l) + if cm.TrafficClass > 0 { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockoptTrafficClass + m.SetLen(syscall.CmsgLen(4)) + data := oob[off+syscall.CmsgLen(0):] + *(*byte)(unsafe.Pointer(&data[:1][0])) = byte(cm.TrafficClass) + off += syscall.CmsgSpace(4) + } + if cm.HopLimit > 0 { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockoptHopLimit + m.SetLen(syscall.CmsgLen(4)) + data := oob[off+syscall.CmsgLen(0):] + *(*byte)(unsafe.Pointer(&data[:1][0])) = byte(cm.HopLimit) + off += syscall.CmsgSpace(4) + } + if pion { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockoptPacketInfo + m.SetLen(syscall.CmsgLen(sysSizeofPacketInfo)) + pi := (*sysPacketInfo)(unsafe.Pointer(&oob[off+syscall.CmsgLen(0)])) + if ip := cm.Src.To16(); ip != nil && ip.To4() == nil { + copy(pi.IP[:], ip) + } + if cm.IfIndex != 0 { + pi.IfIndex = uint32(cm.IfIndex) + } + off += syscall.CmsgSpace(sysSizeofPacketInfo) + } + if len(cm.NextHop) == net.IPv6len { + m := (*syscall.Cmsghdr)(unsafe.Pointer(&oob[off])) + m.Level = ianaProtocolIPv6 + m.Type = sysSockoptNextHop + m.SetLen(syscall.CmsgLen(syscall.SizeofSockaddrInet6)) + sa := (*syscall.RawSockaddrInet6)(unsafe.Pointer(&oob[off+syscall.CmsgLen(0)])) + setSockaddr(sa, cm.NextHop, cm.IfIndex) + off += syscall.CmsgSpace(syscall.SizeofSockaddrInet6) + } + } + return +} diff --git a/third_party/code.google.com/p/go.net/ipv6/icmp_test.go b/third_party/code.google.com/p/go.net/ipv6/icmp_test.go index a7833dd16..40050361d 100644 --- a/third_party/code.google.com/p/go.net/ipv6/icmp_test.go +++ b/third_party/code.google.com/p/go.net/ipv6/icmp_test.go @@ -14,6 +14,24 @@ import ( "testing" ) +var icmpStringTests = []struct { + in ipv6.ICMPType + out string +}{ + {ipv6.ICMPTypeDestinationUnreachable, "destination unreachable"}, + + {256, ""}, +} + +func TestICMPString(t *testing.T) { + for _, tt := range icmpStringTests { + s := tt.in.String() + if s != tt.out { + t.Errorf("got %s; expected %s", s, tt.out) + } + } +} + func TestICMPFilter(t *testing.T) { switch runtime.GOOS { case "plan9", "windows": diff --git a/third_party/code.google.com/p/go.net/ipv6/mocktransponder_test.go b/third_party/code.google.com/p/go.net/ipv6/mocktransponder_test.go index 2da546412..708670395 100644 --- a/third_party/code.google.com/p/go.net/ipv6/mocktransponder_test.go +++ b/third_party/code.google.com/p/go.net/ipv6/mocktransponder_test.go @@ -86,25 +86,3 @@ func acceptor(t *testing.T, ln net.Listener, done chan<- bool) { } c.Close() } - -func transponder(t *testing.T, ln net.Listener, done chan<- bool) { - defer func() { done <- true }() - - c, err := ln.Accept() - if err != nil { - t.Errorf("net.Listener.Accept failed: %v", err) - return - } - defer c.Close() - - b := make([]byte, 128) - n, err := c.Read(b) - if err != nil { - t.Errorf("net.Conn.Read failed: %v", err) - return - } - if _, err := c.Write(b[:n]); err != nil { - t.Errorf("net.Conn.Write failed: %v", err) - return - } -} diff --git a/third_party/code.google.com/p/go.net/ipv6/multicast_test.go b/third_party/code.google.com/p/go.net/ipv6/multicast_test.go index 79568f46b..cdf597f18 100644 --- a/third_party/code.google.com/p/go.net/ipv6/multicast_test.go +++ b/third_party/code.google.com/p/go.net/ipv6/multicast_test.go @@ -5,6 +5,7 @@ package ipv6_test import ( + "bytes" "code.google.com/p/go.net/ipv6" "net" "os" @@ -64,24 +65,30 @@ func TestPacketConnReadWriteMulticastUDP(t *testing.T) { cm := ipv6.ControlMessage{ TrafficClass: DiffServAF11 | CongestionExperienced, + Src: net.IPv6loopback, IfIndex: ifi.Index, } - cf := ipv6.FlagTrafficClass | ipv6.FlagHopLimit | ipv6.FlagInterface | ipv6.FlagPathMTU + cf := ipv6.FlagTrafficClass | ipv6.FlagHopLimit | ipv6.FlagSrc | ipv6.FlagDst | ipv6.FlagInterface | ipv6.FlagPathMTU + wb := []byte("HELLO-R-U-THERE") for i, toggle := range []bool{true, false, true} { if err := p.SetControlMessage(cf, toggle); err != nil { t.Fatalf("ipv6.PacketConn.SetControlMessage failed: %v", err) } - if err := p.SetDeadline(time.Now().Add(time.Millisecond * 200)); err != nil { + if err := p.SetDeadline(time.Now().Add(200 * time.Millisecond)); err != nil { t.Fatalf("ipv6.PacketConn.SetDeadline failed: %v", err) } cm.HopLimit = i + 1 - if _, err := p.WriteTo([]byte("HELLO-R-U-THERE"), &cm, dst); err != nil { + if n, err := p.WriteTo(wb, &cm, dst); err != nil { t.Fatalf("ipv6.PacketConn.WriteTo failed: %v", err) + } else if n != len(wb) { + t.Fatalf("ipv6.PacketConn.WriteTo failed: short write: %v", n) } - b := make([]byte, 128) - if _, cm, _, err := p.ReadFrom(b); err != nil { + rb := make([]byte, 128) + if n, cm, _, err := p.ReadFrom(rb); err != nil { t.Fatalf("ipv6.PacketConn.ReadFrom failed: %v", err) + } else if !bytes.Equal(rb[:n], wb) { + t.Fatalf("got %v; expected %v", rb[:n], wb) } else { t.Logf("rcvd cmsg: %v", cm) } @@ -136,9 +143,10 @@ func TestPacketConnReadWriteMulticastICMP(t *testing.T) { cm := ipv6.ControlMessage{ TrafficClass: DiffServAF11 | CongestionExperienced, + Src: net.IPv6loopback, IfIndex: ifi.Index, } - cf := ipv6.FlagTrafficClass | ipv6.FlagHopLimit | ipv6.FlagInterface | ipv6.FlagPathMTU + cf := ipv6.FlagTrafficClass | ipv6.FlagHopLimit | ipv6.FlagSrc | ipv6.FlagDst | ipv6.FlagInterface | ipv6.FlagPathMTU var f ipv6.ICMPFilter f.SetAll(true) @@ -173,19 +181,21 @@ func TestPacketConnReadWriteMulticastICMP(t *testing.T) { if err := p.SetControlMessage(cf, toggle); err != nil { t.Fatalf("ipv6.PacketConn.SetControlMessage failed: %v", err) } - if err := p.SetDeadline(time.Now().Add(time.Millisecond * 200)); err != nil { + if err := p.SetDeadline(time.Now().Add(200 * time.Millisecond)); err != nil { t.Fatalf("ipv6.PacketConn.SetDeadline failed: %v", err) } cm.HopLimit = i + 1 - if _, err := p.WriteTo(wb, &cm, dst); err != nil { + if n, err := p.WriteTo(wb, &cm, dst); err != nil { t.Fatalf("ipv6.PacketConn.WriteTo failed: %v", err) + } else if n != len(wb) { + t.Fatalf("ipv6.PacketConn.WriteTo failed: short write: %v", n) } - b := make([]byte, 128) - if n, cm, _, err := p.ReadFrom(b); err != nil { + rb := make([]byte, 128) + if n, cm, _, err := p.ReadFrom(rb); err != nil { t.Fatalf("ipv6.PacketConn.ReadFrom failed: %v", err) } else { t.Logf("rcvd cmsg: %v", cm) - if m, err := parseICMPMessage(b[:n]); err != nil { + if m, err := parseICMPMessage(rb[:n]); err != nil { t.Fatalf("parseICMPMessage failed: %v", err) } else if m.Type != ipv6.ICMPTypeEchoReply || m.Code != 0 { t.Fatalf("got type=%v, code=%v; expected type=%v, code=%v", m.Type, m.Code, ipv6.ICMPTypeEchoReply, 0) diff --git a/third_party/code.google.com/p/go.net/ipv6/readwrite_test.go b/third_party/code.google.com/p/go.net/ipv6/readwrite_test.go new file mode 100644 index 000000000..f0d09b331 --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/readwrite_test.go @@ -0,0 +1,168 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6_test + +import ( + "bytes" + "code.google.com/p/go.net/ipv6" + "net" + "runtime" + "sync" + "testing" +) + +func benchmarkUDPListener() (net.PacketConn, net.Addr, error) { + c, err := net.ListenPacket("udp6", "[::1]:0") + if err != nil { + return nil, nil, err + } + dst, err := net.ResolveUDPAddr("udp6", c.LocalAddr().String()) + if err != nil { + c.Close() + return nil, nil, err + } + return c, dst, nil +} + +func BenchmarkReadWriteNetUDP(b *testing.B) { + c, dst, err := benchmarkUDPListener() + if err != nil { + b.Fatalf("benchmarkUDPListener failed: %v", err) + } + defer c.Close() + + wb, rb := []byte("HELLO-R-U-THERE"), make([]byte, 128) + b.ResetTimer() + for i := 0; i < b.N; i++ { + benchmarkReadWriteNetUDP(b, c, wb, rb, dst) + } +} + +func benchmarkReadWriteNetUDP(b *testing.B, c net.PacketConn, wb, rb []byte, dst net.Addr) { + if _, err := c.WriteTo(wb, dst); err != nil { + b.Fatalf("net.PacketConn.WriteTo failed: %v", err) + } + if _, _, err := c.ReadFrom(rb); err != nil { + b.Fatalf("net.PacketConn.ReadFrom failed: %v", err) + } +} + +func BenchmarkReadWriteIPv6UDP(b *testing.B) { + c, dst, err := benchmarkUDPListener() + if err != nil { + b.Fatalf("benchmarkUDPListener failed: %v", err) + } + defer c.Close() + + p := ipv6.NewPacketConn(c) + cf := ipv6.FlagTrafficClass | ipv6.FlagHopLimit | ipv6.FlagInterface | ipv6.FlagPathMTU + if err := p.SetControlMessage(cf, true); err != nil { + b.Fatalf("ipv6.PacketConn.SetControlMessage failed: %v", err) + } + ifi := loopbackInterface() + + wb, rb := []byte("HELLO-R-U-THERE"), make([]byte, 128) + b.ResetTimer() + for i := 0; i < b.N; i++ { + benchmarkReadWriteIPv6UDP(b, p, wb, rb, dst, ifi) + } +} + +func benchmarkReadWriteIPv6UDP(b *testing.B, p *ipv6.PacketConn, wb, rb []byte, dst net.Addr, ifi *net.Interface) { + cm := ipv6.ControlMessage{ + TrafficClass: DiffServAF11 | CongestionExperienced, + HopLimit: 1, + } + if ifi != nil { + cm.IfIndex = ifi.Index + } + if n, err := p.WriteTo(wb, &cm, dst); err != nil { + b.Fatalf("ipv6.PacketConn.WriteTo failed: %v", err) + } else if n != len(wb) { + b.Fatalf("ipv6.PacketConn.WriteTo failed: short write: %v", n) + } + if _, _, _, err := p.ReadFrom(rb); err != nil { + b.Fatalf("ipv6.PacketConn.ReadFrom failed: %v", err) + } +} + +func TestPacketConnConcurrentReadWriteUnicastUDP(t *testing.T) { + switch runtime.GOOS { + case "plan9", "windows": + t.Skipf("not supported on %q", runtime.GOOS) + } + if !supportsIPv6 { + t.Skip("ipv6 is not supported") + } + + c, err := net.ListenPacket("udp6", "[::1]:0") + if err != nil { + t.Fatalf("net.ListenPacket failed: %v", err) + } + defer c.Close() + p := ipv6.NewPacketConn(c) + defer p.Close() + + dst, err := net.ResolveUDPAddr("udp6", c.LocalAddr().String()) + if err != nil { + t.Fatalf("net.ResolveUDPAddr failed: %v", err) + } + + ifi := loopbackInterface() + cf := ipv6.FlagTrafficClass | ipv6.FlagHopLimit | ipv6.FlagSrc | ipv6.FlagDst | ipv6.FlagInterface | ipv6.FlagPathMTU + wb := []byte("HELLO-R-U-THERE") + + var wg sync.WaitGroup + reader := func() { + defer wg.Done() + rb := make([]byte, 128) + if n, cm, _, err := p.ReadFrom(rb); err != nil { + t.Errorf("ipv6.PacketConn.ReadFrom failed: %v", err) + return + } else if !bytes.Equal(rb[:n], wb) { + t.Errorf("got %v; expected %v", rb[:n], wb) + return + } else { + t.Logf("rcvd cmsg: %v", cm) + } + } + writer := func(toggle bool) { + defer wg.Done() + cm := ipv6.ControlMessage{ + TrafficClass: DiffServAF11 | CongestionExperienced, + Src: net.IPv6loopback, + Dst: net.IPv6loopback, + } + if ifi != nil { + cm.IfIndex = ifi.Index + } + if err := p.SetControlMessage(cf, toggle); err != nil { + t.Errorf("ipv6.PacketConn.SetControlMessage failed: %v", err) + return + } + if n, err := p.WriteTo(wb, &cm, dst); err != nil { + t.Errorf("ipv6.PacketConn.WriteTo failed: %v", err) + return + } else if n != len(wb) { + t.Errorf("ipv6.PacketConn.WriteTo failed: short write: %v", n) + return + } + } + + const N = 10 + wg.Add(N) + for i := 0; i < N; i++ { + go reader() + } + wg.Add(2 * N) + for i := 0; i < 2*N; i++ { + go writer(i%2 != 0) + } + wg.Add(N) + for i := 0; i < N; i++ { + go reader() + } + wg.Wait() +} diff --git a/third_party/code.google.com/p/go.net/ipv6/sockopt_rfc2292_unix.go b/third_party/code.google.com/p/go.net/ipv6/sockopt_rfc2292_unix.go new file mode 100644 index 000000000..827e943ff --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/sockopt_rfc2292_unix.go @@ -0,0 +1,73 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin + +package ipv6 + +import ( + "os" + "unsafe" +) + +func ipv6ReceiveTrafficClass(fd int) (bool, error) { + return false, errNotSupported +} + +func setIPv6ReceiveTrafficClass(fd int, v bool) error { + return errNotSupported +} + +func ipv6ReceiveHopLimit(fd int) (bool, error) { + var v int32 + l := sysSockoptLen(4) + if err := getsockopt(fd, ianaProtocolIPv6, sysSockopt2292HopLimit, uintptr(unsafe.Pointer(&v)), &l); err != nil { + return false, os.NewSyscallError("getsockopt", err) + } + return v == 1, nil +} + +func setIPv6ReceiveHopLimit(fd int, v bool) error { + vv := int32(boolint(v)) + return os.NewSyscallError("setsockopt", setsockopt(fd, ianaProtocolIPv6, sysSockopt2292HopLimit, uintptr(unsafe.Pointer(&vv)), 4)) +} + +func ipv6ReceivePacketInfo(fd int) (bool, error) { + var v int32 + l := sysSockoptLen(4) + if err := getsockopt(fd, ianaProtocolIPv6, sysSockopt2292PacketInfo, uintptr(unsafe.Pointer(&v)), &l); err != nil { + return false, os.NewSyscallError("getsockopt", err) + } + return v == 1, nil +} + +func setIPv6ReceivePacketInfo(fd int, v bool) error { + vv := int32(boolint(v)) + return os.NewSyscallError("setsockopt", setsockopt(fd, ianaProtocolIPv6, sysSockopt2292PacketInfo, uintptr(unsafe.Pointer(&vv)), 4)) +} + +func ipv6PathMTU(fd int) (int, error) { + return 0, errNotSupported +} + +func ipv6ReceivePathMTU(fd int) (bool, error) { + return false, errNotSupported +} + +func setIPv6ReceivePathMTU(fd int, v bool) error { + return errNotSupported +} + +func ipv6ICMPFilter(fd int) (*ICMPFilter, error) { + var v ICMPFilter + l := sysSockoptLen(sysSizeofICMPFilter) + if err := getsockopt(fd, ianaProtocolIPv6ICMP, sysSockoptICMPFilter, uintptr(unsafe.Pointer(&v.sysICMPFilter)), &l); err != nil { + return nil, os.NewSyscallError("getsockopt", err) + } + return &v, nil +} + +func setIPv6ICMPFilter(fd int, f *ICMPFilter) error { + return os.NewSyscallError("setsockopt", setsockopt(fd, ianaProtocolIPv6ICMP, sysSockoptICMPFilter, uintptr(unsafe.Pointer(&f.sysICMPFilter)), sysSizeofICMPFilter)) +} diff --git a/third_party/code.google.com/p/go.net/ipv6/sys.go b/third_party/code.google.com/p/go.net/ipv6/sys.go new file mode 100644 index 000000000..18b1acacd --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/sys.go @@ -0,0 +1,23 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +type sysSockoptLen uint32 + +const ( + sysSizeofPacketInfo = 0x14 + sysSizeofMulticastReq = 0x14 + sysSizeofICMPFilter = 0x20 +) + +type sysPacketInfo struct { + IP [16]byte + IfIndex uint32 +} + +type sysMulticastReq struct { + IP [16]byte + IfIndex uint32 +} diff --git a/third_party/code.google.com/p/go.net/ipv6/sys_bsd.go b/third_party/code.google.com/p/go.net/ipv6/sys_bsd.go new file mode 100644 index 000000000..4a08217fe --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/sys_bsd.go @@ -0,0 +1,48 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd netbsd openbsd + +package ipv6 + +import ( + "net" + "syscall" +) + +// RFC 3493 options +const ( + // See /usr/include/netinet6/in6.h. + sysSockoptUnicastHopLimit = 0x4 + sysSockoptMulticastHopLimit = 0xa + sysSockoptMulticastInterface = 0x9 + sysSockoptMulticastLoopback = 0xb + sysSockoptJoinGroup = 0xc + sysSockoptLeaveGroup = 0xd +) + +// RFC 3542 options +const ( + // See /usr/include/netinet6/in6.h. + sysSockoptReceiveTrafficClass = 0x39 + sysSockoptTrafficClass = 0x3d + sysSockoptReceiveHopLimit = 0x25 + sysSockoptHopLimit = 0x2f + sysSockoptReceivePacketInfo = 0x24 + sysSockoptPacketInfo = 0x2e + sysSockoptReceivePathMTU = 0x2b + sysSockoptPathMTU = 0x2c + sysSockoptNextHop = 0x30 + sysSockoptChecksum = 0x1a + + // See /usr/include/netinet6/in6.h. + sysSockoptICMPFilter = 0x12 +) + +func setSockaddr(sa *syscall.RawSockaddrInet6, ip net.IP, ifindex int) { + sa.Len = syscall.SizeofSockaddrInet6 + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], ip) + sa.Scope_id = uint32(ifindex) +} diff --git a/third_party/code.google.com/p/go.net/ipv6/sys_darwin.go b/third_party/code.google.com/p/go.net/ipv6/sys_darwin.go new file mode 100644 index 000000000..3d07dff57 --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/sys_darwin.go @@ -0,0 +1,54 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "net" + "syscall" +) + +// RFC 2292 options +const ( + // See /usr/include/netinet6/in6.h. + sysSockopt2292HopLimit = 0x14 + sysSockopt2292PacketInfo = 0x13 + sysSockopt2292NextHop = 0x15 +) + +// RFC 3493 options +const ( + // See /usr/include/netinet6/in6.h. + sysSockoptUnicastHopLimit = 0x4 + sysSockoptMulticastHopLimit = 0xa + sysSockoptMulticastInterface = 0x9 + sysSockoptMulticastLoopback = 0xb + sysSockoptJoinGroup = 0xc + sysSockoptLeaveGroup = 0xd +) + +// RFC 3542 options +const ( + // See /usr/include/netinet6/in6.h. + sysSockoptReceiveTrafficClass = 0x23 + sysSockoptTrafficClass = 0x24 + sysSockoptReceiveHopLimit = 0x25 + sysSockoptHopLimit = 0x2f + sysSockoptReceivePacketInfo = 0x3d + sysSockoptPacketInfo = 0x2e + sysSockoptReceivePathMTU = 0x2b + sysSockoptPathMTU = 0x2c + sysSockoptNextHop = 0x30 + sysSockoptChecksum = 0x1a + + // See /usr/include/netinet6/in6.h. + sysSockoptICMPFilter = 0x12 +) + +func setSockaddr(sa *syscall.RawSockaddrInet6, ip net.IP, ifindex int) { + sa.Len = syscall.SizeofSockaddrInet6 + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], ip) + sa.Scope_id = uint32(ifindex) +} diff --git a/third_party/code.google.com/p/go.net/ipv6/sys_linux.go b/third_party/code.google.com/p/go.net/ipv6/sys_linux.go new file mode 100644 index 000000000..d90c8cb8a --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/sys_linux.go @@ -0,0 +1,45 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "net" + "syscall" +) + +// RFC 3493 options +const ( + // See /usr/include/linux/in6.h. + sysSockoptUnicastHopLimit = 0x10 + sysSockoptMulticastHopLimit = 0x12 + sysSockoptMulticastInterface = 0x11 + sysSockoptMulticastLoopback = 0x13 + sysSockoptJoinGroup = 0x14 + sysSockoptLeaveGroup = 0x15 +) + +// RFC 3542 options +const ( + // See /usr/include/linux/ipv6.h,in6.h. + sysSockoptReceiveTrafficClass = 0x42 + sysSockoptTrafficClass = 0x43 + sysSockoptReceiveHopLimit = 0x33 + sysSockoptHopLimit = 0x34 + sysSockoptReceivePacketInfo = 0x31 + sysSockoptPacketInfo = 0x32 + sysSockoptReceivePathMTU = 0x3c + sysSockoptPathMTU = 0x3d + sysSockoptNextHop = 0x9 + sysSockoptChecksum = 0x7 + + // See /usr/include/linux/icmpv6.h. + sysSockoptICMPFilter = 0x1 +) + +func setSockaddr(sa *syscall.RawSockaddrInet6, ip net.IP, ifindex int) { + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], ip) + sa.Scope_id = uint32(ifindex) +} diff --git a/third_party/code.google.com/p/go.net/ipv6/sys_unix.go b/third_party/code.google.com/p/go.net/ipv6/sys_unix.go new file mode 100644 index 000000000..40cdfc4d8 --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/sys_unix.go @@ -0,0 +1,16 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin freebsd linux netbsd openbsd + +package ipv6 + +import "syscall" + +const sysSizeofMTUInfo = 0x20 + +type sysMTUInfo struct { + Addr syscall.RawSockaddrInet6 + MTU uint32 +} diff --git a/third_party/code.google.com/p/go.net/ipv6/sys_windows.go b/third_party/code.google.com/p/go.net/ipv6/sys_windows.go new file mode 100644 index 000000000..c09672f28 --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/sys_windows.go @@ -0,0 +1,33 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "net" + "syscall" +) + +// RFC 3493 options +const ( + // See ws2tcpip.h. + sysSockoptUnicastHopLimit = 0x4 + sysSockoptMulticastHopLimit = 0xa + sysSockoptMulticastInterface = 0x9 + sysSockoptMulticastLoopback = 0xb + sysSockoptJoinGroup = 0xc + sysSockoptLeaveGroup = 0xd +) + +// RFC 3542 options +const ( + // See ws2tcpip.h. + sysSockoptPacketInfo = 0x13 +) + +func setSockaddr(sa *syscall.RawSockaddrInet6, ip net.IP, ifindex int) { + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], ip) + sa.Scope_id = uint32(ifindex) +} diff --git a/third_party/code.google.com/p/go.net/ipv6/syscall_linux_386.go b/third_party/code.google.com/p/go.net/ipv6/syscall_linux_386.go new file mode 100644 index 000000000..a3866362e --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/syscall_linux_386.go @@ -0,0 +1,42 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This code is a duplicate of syscall/syscall_linux_386.go with small +// modifications. + +package ipv6 + +import ( + "syscall" + "unsafe" +) + +// On x86 Linux, all the socket calls go through an extra indirection, +// I think because the 5-register system call interface can't handle +// the 6-argument calls like sendto and recvfrom. Instead the +// arguments to the underlying system call are the number below and a +// pointer to an array of uintptr. We hide the pointer in the +// socketcall assembly to avoid allocation on every system call. + +const ( + // See /usr/include/linux/net.h. + _SETSOCKOPT = 14 + _GETSOCKOPT = 15 +) + +var socketcall func(call int, a0, a1, a2, a3, a4, a5 uintptr) (int, syscall.Errno) + +func getsockopt(fd int, level int, name int, v uintptr, l *sysSockoptLen) error { + if _, errno := socketcall(_GETSOCKOPT, uintptr(fd), uintptr(level), uintptr(name), uintptr(v), uintptr(unsafe.Pointer(l)), 0); errno != 0 { + return error(errno) + } + return nil +} + +func setsockopt(fd int, level int, name int, v uintptr, l uintptr) error { + if _, errno := socketcall(_SETSOCKOPT, uintptr(fd), uintptr(level), uintptr(name), v, l, 0); errno != 0 { + return error(errno) + } + return nil +} diff --git a/third_party/code.google.com/p/go.net/ipv6/syscall_linux_386.s b/third_party/code.google.com/p/go.net/ipv6/syscall_linux_386.s new file mode 100644 index 000000000..34c0457fd --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/syscall_linux_386.s @@ -0,0 +1,56 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This code is a duplicate of syscall/syscall_linux_386.s with small +// modifications. + +#define SYS_SOCKETCALL 102 // from zsysnum_linux_386.go + +// func socketcallnosplit7(call int, a0, a1, a2, a3, a4, a5 uintptr) (n int, errno int) +// Kernel interface gets call sub-number and pointer to a0 for Go 1.1. +TEXT ·socketcallnosplit7(SB),7,$0 + CALL runtime·entersyscall(SB) + MOVL $SYS_SOCKETCALL, AX // syscall entry + MOVL 4(SP), BX // socket call number + LEAL 8(SP), CX // pointer to call arguments + MOVL $0, DX + MOVL $0, SI + MOVL $0, DI + CALL *runtime·_vdso(SB) + CMPL AX, $0xfffff001 + JLS ok1 + MOVL $-1, 32(SP) // n + NEGL AX + MOVL AX, 36(SP) // errno + CALL runtime·exitsyscall(SB) + RET +ok1: + MOVL AX, 32(SP) // n + MOVL $0, 36(SP) // errno + CALL runtime·exitsyscall(SB) + RET + +// func socketcallnosplit4(call int, a0, a1, a2, a3, a4, a5 uintptr) (n int, errno int) +// Kernel interface gets call sub-number and pointer to a0 for Go 1.2. +TEXT ·socketcallnosplit4(SB),4,$0-40 + CALL runtime·entersyscall(SB) + MOVL $SYS_SOCKETCALL, AX // syscall entry + MOVL 4(SP), BX // socket call number + LEAL 8(SP), CX // pointer to call arguments + MOVL $0, DX + MOVL $0, SI + MOVL $0, DI + CALL *runtime·_vdso(SB) + CMPL AX, $0xfffff001 + JLS ok2 + MOVL $-1, 32(SP) // n + NEGL AX + MOVL AX, 36(SP) // errno + CALL runtime·exitsyscall(SB) + RET +ok2: + MOVL AX, 32(SP) // n + MOVL $0, 36(SP) // errno + CALL runtime·exitsyscall(SB) + RET diff --git a/third_party/code.google.com/p/go.net/ipv6/syscall_nosplit4_linux_386.go b/third_party/code.google.com/p/go.net/ipv6/syscall_nosplit4_linux_386.go new file mode 100644 index 000000000..6d4ac0960 --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/syscall_nosplit4_linux_386.go @@ -0,0 +1,15 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.2 + +package ipv6 + +import "syscall" + +func socketcallnosplit4(call int, a0, a1, a2, a3, a4, a5 uintptr) (int, syscall.Errno) + +func init() { + socketcall = socketcallnosplit4 +} diff --git a/third_party/code.google.com/p/go.net/ipv6/syscall_nosplit7_linux_386.go b/third_party/code.google.com/p/go.net/ipv6/syscall_nosplit7_linux_386.go new file mode 100644 index 000000000..2e4da7b53 --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/syscall_nosplit7_linux_386.go @@ -0,0 +1,15 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.1,!go1.2 + +package ipv6 + +import "syscall" + +func socketcallnosplit7(call int, a0, a1, a2, a3, a4, a5 uintptr) (int, syscall.Errno) + +func init() { + socketcall = socketcallnosplit7 +} diff --git a/third_party/code.google.com/p/go.net/ipv6/syscall_unix.go b/third_party/code.google.com/p/go.net/ipv6/syscall_unix.go new file mode 100644 index 000000000..d88dab52c --- /dev/null +++ b/third_party/code.google.com/p/go.net/ipv6/syscall_unix.go @@ -0,0 +1,26 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin freebsd linux,amd64 linux,arm netbsd openbsd + +package ipv6 + +import ( + "syscall" + "unsafe" +) + +func getsockopt(fd int, level, name int, v uintptr, l *sysSockoptLen) error { + if _, _, errno := syscall.Syscall6(syscall.SYS_GETSOCKOPT, uintptr(fd), uintptr(level), uintptr(name), uintptr(v), uintptr(unsafe.Pointer(l)), 0); errno != 0 { + return error(errno) + } + return nil +} + +func setsockopt(fd int, level int, name int, v uintptr, l uintptr) error { + if _, _, errno := syscall.Syscall6(syscall.SYS_SETSOCKOPT, uintptr(fd), uintptr(level), uintptr(name), uintptr(v), uintptr(l), 0); errno != 0 { + return error(errno) + } + return nil +} diff --git a/third_party/code.google.com/p/go.net/ipv6/unicast_test.go b/third_party/code.google.com/p/go.net/ipv6/unicast_test.go index fed8a69da..954255034 100644 --- a/third_party/code.google.com/p/go.net/ipv6/unicast_test.go +++ b/third_party/code.google.com/p/go.net/ipv6/unicast_test.go @@ -5,6 +5,7 @@ package ipv6_test import ( + "bytes" "code.google.com/p/go.net/ipv6" "net" "os" @@ -13,79 +14,6 @@ import ( "time" ) -func benchmarkUDPListener() (net.PacketConn, net.Addr, error) { - c, err := net.ListenPacket("udp6", "[::1]:0") - if err != nil { - return nil, nil, err - } - dst, err := net.ResolveUDPAddr("udp6", c.LocalAddr().String()) - if err != nil { - c.Close() - return nil, nil, err - } - return c, dst, nil -} - -func BenchmarkReadWriteNetUDP(b *testing.B) { - c, dst, err := benchmarkUDPListener() - if err != nil { - b.Fatalf("benchmarkUDPListener failed: %v", err) - } - defer c.Close() - - wb, rb := []byte("HELLO-R-U-THERE"), make([]byte, 128) - b.ResetTimer() - for i := 0; i < b.N; i++ { - benchmarkReadWriteNetUDP(b, c, wb, rb, dst) - } -} - -func benchmarkReadWriteNetUDP(b *testing.B, c net.PacketConn, wb, rb []byte, dst net.Addr) { - if _, err := c.WriteTo(wb, dst); err != nil { - b.Fatalf("net.PacketConn.WriteTo failed: %v", err) - } - if _, _, err := c.ReadFrom(rb); err != nil { - b.Fatalf("net.PacketConn.ReadFrom failed: %v", err) - } -} - -func BenchmarkReadWriteIPv6UDP(b *testing.B) { - c, dst, err := benchmarkUDPListener() - if err != nil { - b.Fatalf("benchmarkUDPListener failed: %v", err) - } - defer c.Close() - - p := ipv6.NewPacketConn(c) - cf := ipv6.FlagTrafficClass | ipv6.FlagHopLimit | ipv6.FlagInterface | ipv6.FlagPathMTU - if err := p.SetControlMessage(cf, true); err != nil { - b.Fatalf("ipv6.PacketConn.SetControlMessage failed: %v", err) - } - ifi := loopbackInterface() - - wb, rb := []byte("HELLO-R-U-THERE"), make([]byte, 128) - b.ResetTimer() - for i := 0; i < b.N; i++ { - benchmarkReadWriteIPv6UDP(b, p, wb, rb, dst, ifi) - } -} - -func benchmarkReadWriteIPv6UDP(b *testing.B, p *ipv6.PacketConn, wb, rb []byte, dst net.Addr, ifi *net.Interface) { - cm := ipv6.ControlMessage{ - TrafficClass: DiffServAF11 | CongestionExperienced, - HopLimit: 1, - } - if ifi != nil { - cm.IfIndex = ifi.Index - } - if _, err := p.WriteTo(wb, &cm, dst); err != nil { - b.Fatalf("ipv6.PacketConn.WriteTo failed: %v", err) - } - if _, _, _, err := p.ReadFrom(rb); err != nil { - b.Fatalf("ipv6.PacketConn.ReadFrom failed: %v", err) - } -} - func TestPacketConnReadWriteUnicastUDP(t *testing.T) { switch runtime.GOOS { case "plan9", "windows": @@ -100,40 +28,47 @@ func TestPacketConnReadWriteUnicastUDP(t *testing.T) { t.Fatalf("net.ListenPacket failed: %v", err) } defer c.Close() + p := ipv6.NewPacketConn(c) + defer p.Close() dst, err := net.ResolveUDPAddr("udp6", c.LocalAddr().String()) if err != nil { t.Fatalf("net.ResolveUDPAddr failed: %v", err) } - p := ipv6.NewPacketConn(c) - defer p.Close() cm := ipv6.ControlMessage{ TrafficClass: DiffServAF11 | CongestionExperienced, + Src: net.IPv6loopback, + Dst: net.IPv6loopback, } - cf := ipv6.FlagTrafficClass | ipv6.FlagHopLimit | ipv6.FlagInterface | ipv6.FlagPathMTU + cf := ipv6.FlagTrafficClass | ipv6.FlagHopLimit | ipv6.FlagSrc | ipv6.FlagDst | ipv6.FlagInterface | ipv6.FlagPathMTU ifi := loopbackInterface() if ifi != nil { cm.IfIndex = ifi.Index } + wb := []byte("HELLO-R-U-THERE") for i, toggle := range []bool{true, false, true} { if err := p.SetControlMessage(cf, toggle); err != nil { t.Fatalf("ipv6.PacketConn.SetControlMessage failed: %v", err) } cm.HopLimit = i + 1 - if err := p.SetWriteDeadline(time.Now().Add(time.Millisecond * 100)); err != nil { + if err := p.SetWriteDeadline(time.Now().Add(100 * time.Millisecond)); err != nil { t.Fatalf("ipv6.PacketConn.SetWriteDeadline failed: %v", err) } - if _, err := p.WriteTo([]byte("HELLO-R-U-THERE"), &cm, dst); err != nil { + if n, err := p.WriteTo(wb, &cm, dst); err != nil { t.Fatalf("ipv6.PacketConn.WriteTo failed: %v", err) + } else if n != len(wb) { + t.Fatalf("ipv6.PacketConn.WriteTo failed: short write: %v", n) } - b := make([]byte, 128) - if err := p.SetReadDeadline(time.Now().Add(time.Millisecond * 100)); err != nil { + rb := make([]byte, 128) + if err := p.SetReadDeadline(time.Now().Add(100 * time.Millisecond)); err != nil { t.Fatalf("ipv6.PacketConn.SetReadDeadline failed: %v", err) } - if _, cm, _, err := p.ReadFrom(b); err != nil { + if n, cm, _, err := p.ReadFrom(rb); err != nil { t.Fatalf("ipv6.PacketConn.ReadFrom failed: %v", err) + } else if !bytes.Equal(rb[:n], wb) { + t.Fatalf("got %v; expected %v", rb[:n], wb) } else { t.Logf("rcvd cmsg: %v", cm) } @@ -157,6 +92,8 @@ func TestPacketConnReadWriteUnicastICMP(t *testing.T) { t.Fatalf("net.ListenPacket failed: %v", err) } defer c.Close() + p := ipv6.NewPacketConn(c) + defer p.Close() dst, err := net.ResolveIPAddr("ip6", "::1") if err != nil { @@ -164,10 +101,12 @@ func TestPacketConnReadWriteUnicastICMP(t *testing.T) { } pshicmp := ipv6PseudoHeader(c.LocalAddr().(*net.IPAddr).IP, dst.IP, ianaProtocolIPv6ICMP) - p := ipv6.NewPacketConn(c) - defer p.Close() - cm := ipv6.ControlMessage{TrafficClass: DiffServAF11 | CongestionExperienced} - cf := ipv6.FlagTrafficClass | ipv6.FlagHopLimit | ipv6.FlagInterface | ipv6.FlagPathMTU + cm := ipv6.ControlMessage{ + TrafficClass: DiffServAF11 | CongestionExperienced, + Src: net.IPv6loopback, + Dst: net.IPv6loopback, + } + cf := ipv6.FlagTrafficClass | ipv6.FlagHopLimit | ipv6.FlagSrc | ipv6.FlagDst | ipv6.FlagInterface | ipv6.FlagPathMTU ifi := loopbackInterface() if ifi != nil { cm.IfIndex = ifi.Index @@ -207,21 +146,23 @@ func TestPacketConnReadWriteUnicastICMP(t *testing.T) { t.Fatalf("ipv6.PacketConn.SetControlMessage failed: %v", err) } cm.HopLimit = i + 1 - if err := p.SetWriteDeadline(time.Now().Add(time.Millisecond * 100)); err != nil { + if err := p.SetWriteDeadline(time.Now().Add(100 * time.Millisecond)); err != nil { t.Fatalf("ipv6.PacketConn.SetWriteDeadline failed: %v", err) } - if _, err := p.WriteTo(wb, &cm, dst); err != nil { + if n, err := p.WriteTo(wb, &cm, dst); err != nil { t.Fatalf("ipv6.PacketConn.WriteTo failed: %v", err) + } else if n != len(wb) { + t.Fatalf("ipv6.PacketConn.WriteTo failed: short write: %v", n) } - b := make([]byte, 128) - if err := p.SetReadDeadline(time.Now().Add(time.Millisecond * 100)); err != nil { + rb := make([]byte, 128) + if err := p.SetReadDeadline(time.Now().Add(100 * time.Millisecond)); err != nil { t.Fatalf("ipv6.PacketConn.SetReadDeadline failed: %v", err) } - if n, cm, _, err := p.ReadFrom(b); err != nil { + if n, cm, _, err := p.ReadFrom(rb); err != nil { t.Fatalf("ipv6.PacketConn.ReadFrom failed: %v", err) } else { t.Logf("rcvd cmsg: %v", cm) - if m, err := parseICMPMessage(b[:n]); err != nil { + if m, err := parseICMPMessage(rb[:n]); err != nil { t.Fatalf("parseICMPMessage failed: %v", err) } else if m.Type != ipv6.ICMPTypeEchoReply || m.Code != 0 { t.Fatalf("got type=%v, code=%v; expected type=%v, code=%v", m.Type, m.Code, ipv6.ICMPTypeEchoReply, 0) diff --git a/third_party/code.google.com/p/go.net/spdy/spdy_test.go b/third_party/code.google.com/p/go.net/spdy/spdy_test.go index ce581f1d0..8ab4382ce 100644 --- a/third_party/code.google.com/p/go.net/spdy/spdy_test.go +++ b/third_party/code.google.com/p/go.net/spdy/spdy_test.go @@ -39,9 +39,9 @@ func TestCreateParseSynStreamFrameCompressionDisable(t *testing.T) { // Fixture framer for no compression test. framer := &Framer{ headerCompressionDisabled: true, - w: buffer, - headerBuf: new(bytes.Buffer), - r: buffer, + w: buffer, + headerBuf: new(bytes.Buffer), + r: buffer, } synStreamFrame := SynStreamFrame{ CFHeader: ControlFrameHeader{ @@ -101,9 +101,9 @@ func TestCreateParseSynReplyFrameCompressionDisable(t *testing.T) { buffer := new(bytes.Buffer) framer := &Framer{ headerCompressionDisabled: true, - w: buffer, - headerBuf: new(bytes.Buffer), - r: buffer, + w: buffer, + headerBuf: new(bytes.Buffer), + r: buffer, } synReplyFrame := SynReplyFrame{ CFHeader: ControlFrameHeader{ @@ -302,9 +302,9 @@ func TestCreateParseHeadersFrame(t *testing.T) { buffer := new(bytes.Buffer) framer := &Framer{ headerCompressionDisabled: true, - w: buffer, - headerBuf: new(bytes.Buffer), - r: buffer, + w: buffer, + headerBuf: new(bytes.Buffer), + r: buffer, } headersFrame := HeadersFrame{ CFHeader: ControlFrameHeader{ diff --git a/third_party/code.google.com/p/goprotobuf/README b/third_party/code.google.com/p/goprotobuf/README index f9bd2507d..97dbc2b66 100644 --- a/third_party/code.google.com/p/goprotobuf/README +++ b/third_party/code.google.com/p/goprotobuf/README @@ -2,6 +2,8 @@ Go support for Protocol Buffers - Google's data interchange format Copyright 2010 The Go Authors. http://code.google.com/p/goprotobuf/ +This package and the code it generates requires at least Go 1.1. + This software implements Go bindings for protocol buffers. For information about protocol buffers themselves, see http://code.google.com/apis/protocolbuffers/ diff --git a/third_party/code.google.com/p/goprotobuf/lib/codereview/codereview.cfg b/third_party/code.google.com/p/goprotobuf/lib/codereview/codereview.cfg index e3eb47ca0..43dbf3ce3 100644 --- a/third_party/code.google.com/p/goprotobuf/lib/codereview/codereview.cfg +++ b/third_party/code.google.com/p/goprotobuf/lib/codereview/codereview.cfg @@ -1,2 +1,2 @@ -defaultcc: golang-dev@googlegroups.com +defaultcc: golang-codereviews@googlegroups.com contributors: http://go.googlecode.com/hg/CONTRIBUTORS diff --git a/third_party/github.com/BurntSushi/toml/README.md b/third_party/github.com/BurntSushi/toml/README.md index f8c847d66..49f43f34e 100644 --- a/third_party/github.com/BurntSushi/toml/README.md +++ b/third_party/github.com/BurntSushi/toml/README.md @@ -1,6 +1,10 @@ -# TOML parser for Go with reflection +# TOML parser and encoder for Go with reflection -TOML stands for Tom's Obvious, Minimal Language. +TOML stands for Tom's Obvious, Minimal Language. This Go package provides a +reflection interface similar to Go's standard library `json` and `xml` +packages. This package also supports the `encoding.TextUnmarshaler` and +`encoding.TextMarshaler` interfaces so that you can define custom data +representations. (There is an example of this below.) Spec: https://github.com/mojombo/toml @@ -26,7 +30,8 @@ tomlv some-toml-file.toml ## Testing This package passes all tests in -[toml-test](https://github.com/BurntSushi/toml-test). +[toml-test](https://github.com/BurntSushi/toml-test) for both the decoder +and the encoder. ## Examples @@ -78,6 +83,56 @@ type TOML struct { } ``` +## Using the `encoding.TextUnmarshaler` interface + +Here's an example that automatically parses duration strings into +`time.Duration` values: + +```toml +[[song]] +name = "Thunder Road" +duration = "4m49s" + +[[song]] +name = "Stairway to Heaven" +duration = "8m03s" +``` + +Which can be decoded with: + +```go +type song struct { + Name string + Duration duration +} +type songs struct { + Song []song +} +var favorites songs +if _, err := Decode(blob, &favorites); err != nil { + log.Fatal(err) +} + +for _, s := range favorites.Song { + fmt.Printf("%s (%s)\n", s.Name, s.Duration) +} +``` + +And you'll also need a `duration` type that satisfies the +`encoding.TextUnmarshaler` interface: + +```go +type duration struct { + time.Duration +} + +func (d *duration) UnmarshalText(text []byte) error { + var err error + d.Duration, err = time.ParseDuration(string(text)) + return err +} +``` + ## More complex usage Here's an example of how to load the example from the official spec page: diff --git a/third_party/github.com/BurntSushi/toml/decode.go b/third_party/github.com/BurntSushi/toml/decode.go index ddb26cf2b..a106f3670 100644 --- a/third_party/github.com/BurntSushi/toml/decode.go +++ b/third_party/github.com/BurntSushi/toml/decode.go @@ -1,6 +1,7 @@ package toml import ( + "encoding" "fmt" "io" "io/ioutil" @@ -43,11 +44,62 @@ func PrimitiveDecode(primValue Primitive, v interface{}) error { // TOML hashes correspond to Go structs or maps. (Dealer's choice. They can be // used interchangeably.) // +// TOML arrays of tables correspond to either a slice of structs or a slice +// of maps. +// // TOML datetimes correspond to Go `time.Time` values. // // All other TOML types (float, string, int, bool and array) correspond // to the obvious Go types. // +// An exception to the above rules is if a type implements the +// encoding.TextUnmarshaler interface. In this case, any primitive TOML value +// (floats, strings, integers, booleans and datetimes) will be converted to +// a byte string and given to the value's UnmarshalText method. Here's an +// example for parsing durations: +// +// type duration struct { +// time.Duration +// } +// +// func (d *duration) UnmarshalText(text []byte) error { +// var err error +// d.Duration, err = time.ParseDuration(string(text)) +// return err +// } +// +// func ExampleUnmarshaler() { +// blob := ` +// [[song]] +// name = "Thunder Road" +// duration = "4m49s" +// +// [[song]] +// name = "Stairway to Heaven" +// duration = "8m03s" +// ` +// type song struct { +// Name string +// Duration duration +// } +// type songs struct { +// Song []song +// } +// var favorites songs +// if _, err := Decode(blob, &favorites); err != nil { +// log.Fatal(err) +// } +// +// for _, s := range favorites.Song { +// fmt.Printf("%s (%s)\n", s.Name, s.Duration) +// } +// // Output: +// // Thunder Road (4m49s) +// // Stairway to Heaven (8m3s) +// } +// +// Key mapping +// // TOML keys can map to either keys in a Go map or field names in a Go // struct. The special `toml` struct tag may be used to map TOML keys to // struct fields that don't match the key name exactly. (See the example.) @@ -100,11 +152,17 @@ func unify(data interface{}, rv reflect.Value) error { return unifyAnything(data, rv) } - // Special case. Go's `time.Time` is a struct, which we don't want - // to confuse with a user struct. - if rv.Type().AssignableTo(rvalue(time.Time{}).Type()) { - return unifyDatetime(data, rv) + // Special case. Look for a value satisfying the TextUnmarshaler interface. + if v, ok := rv.Interface().(encoding.TextUnmarshaler); ok { + return unifyText(data, v) } + // BUG(burntsushi) + // The behavior here is incorrect whenever a Go type satisfies the + // encoding.TextUnmarshaler interface but also corresponds to a TOML + // hash or array. In particular, the unmarshaler should only be applied + // to primitive TOML values. But at this point, it will be applied to + // all kinds of values and produce an incorrect error whenever those values + // are hashes or arrays (including arrays of tables). k := rv.Kind() @@ -151,29 +209,41 @@ func unifyStruct(mapping interface{}, rv reflect.Value) error { return mismatch(rv, "map", mapping) } - rt := rv.Type() - for i := 0; i < rt.NumField(); i++ { - // A little tricky. We want to use the special `toml` name in the - // struct tag if it exists. In particular, we need to make sure that - // this struct field is in the current map before trying to unify it. - sft := rt.Field(i) - kname := sft.Tag.Get("toml") - if len(kname) == 0 { - kname = sft.Name + for key, datum := range tmap { + var f *field + fields := cachedTypeFields(rv.Type()) + for i := range fields { + ff := &fields[i] + if ff.name == key { + f = ff + break + } + if f == nil && strings.EqualFold(ff.name, key) { + f = ff + } } - if datum, ok := insensitiveGet(tmap, kname); ok { - sf := indirect(rv.Field(i)) + if f != nil { + subv := rv + for _, i := range f.index { + if subv.Kind() == reflect.Ptr { + if subv.IsNil() { + subv.Set(reflect.New(subv.Type().Elem())) + } + subv = subv.Elem() + } + subv = subv.Field(i) + } + sf := indirect(subv) - // Don't try to mess with unexported types and other such things. - if sf.CanSet() { + if isUnifiable(sf) { if err := unify(datum, sf); err != nil { return e("Type mismatch for '%s.%s': %s", - rt.String(), sft.Name, err) + rv.Type().String(), f.name, err) } - } else if len(sft.Tag.Get("toml")) > 0 { + } else if f.name != "" { // Bad user! No soup for you! return e("Field '%s.%s' is unexported, and therefore cannot "+ - "be loaded with reflection.", rt.String(), sft.Name) + "be loaded with reflection.", rv.Type().String(), f.name) } } } @@ -287,7 +357,7 @@ func unifyBool(data interface{}, rv reflect.Value) error { rv.SetBool(b) return nil } - return badtype("integer", data) + return badtype("boolean", data) } func unifyAnything(data interface{}, rv reflect.Value) error { @@ -296,6 +366,34 @@ func unifyAnything(data interface{}, rv reflect.Value) error { return nil } +func unifyText(data interface{}, v encoding.TextUnmarshaler) error { + var s string + switch sdata := data.(type) { + case encoding.TextMarshaler: + text, err := sdata.MarshalText() + if err != nil { + return err + } + s = string(text) + case fmt.Stringer: + s = sdata.String() + case string: + s = sdata + case bool: + s = fmt.Sprintf("%v", sdata) + case int64: + s = fmt.Sprintf("%d", sdata) + case float64: + s = fmt.Sprintf("%f", sdata) + default: + return badtype("primitive (string-like)", data) + } + if err := v.UnmarshalText([]byte(s)); err != nil { + return err + } + return nil +} + // rvalue returns a reflect.Value of `v`. All pointers are resolved. func rvalue(v interface{}) reflect.Value { return indirect(reflect.ValueOf(v)) @@ -304,8 +402,17 @@ func rvalue(v interface{}) reflect.Value { // indirect returns the value pointed to by a pointer. // Pointers are followed until the value is not a pointer. // New values are allocated for each nil pointer. +// +// An exception to this rule is if the value satisfies an interface of +// interest to us (like encoding.TextUnmarshaler). func indirect(v reflect.Value) reflect.Value { if v.Kind() != reflect.Ptr { + if v.CanAddr() { + pv := v.Addr() + if _, ok := pv.Interface().(encoding.TextUnmarshaler); ok { + return pv + } + } return v } if v.IsNil() { @@ -314,6 +421,16 @@ func indirect(v reflect.Value) reflect.Value { return indirect(reflect.Indirect(v)) } +func isUnifiable(rv reflect.Value) bool { + if rv.CanSet() { + return true + } + if _, ok := rv.Interface().(encoding.TextUnmarshaler); ok { + return true + } + return false +} + func tstring(rv reflect.Value) string { return rv.Type().String() } @@ -344,8 +461,6 @@ func insensitiveGet( // MetaData allows access to meta information about TOML data that may not // be inferrable via reflection. In particular, whether a key has been defined // and the TOML type of a key. -// -// (XXX: If TOML gets NULL values, that information will be added here too.) type MetaData struct { mapping map[string]interface{} types map[string]tomlType diff --git a/third_party/github.com/BurntSushi/toml/decode_test.go b/third_party/github.com/BurntSushi/toml/decode_test.go index ad80adee3..c9cb64edf 100644 --- a/third_party/github.com/BurntSushi/toml/decode_test.go +++ b/third_party/github.com/BurntSushi/toml/decode_test.go @@ -1,6 +1,7 @@ package toml import ( + "encoding/json" "fmt" "log" "reflect" @@ -63,6 +64,53 @@ func TestDecode(t *testing.T) { testf("%v\n", val) } +func TestDecodeEmbedded(t *testing.T) { + type Dog struct{ Name string } + + tests := map[string]struct { + input string + decodeInto interface{} + wantDecoded interface{} + }{ + "embedded struct": { + input: `Name = "milton"`, + decodeInto: &struct{ Dog }{}, + wantDecoded: &struct{ Dog }{Dog{"milton"}}, + }, + "embedded non-nil pointer to struct": { + input: `Name = "milton"`, + decodeInto: &struct{ *Dog }{}, + wantDecoded: &struct{ *Dog }{&Dog{"milton"}}, + }, + "embedded nil pointer to struct": { + input: ``, + decodeInto: &struct{ *Dog }{}, + wantDecoded: &struct{ *Dog }{nil}, + }, + } + + for label, test := range tests { + _, err := Decode(test.input, test.decodeInto) + if err != nil { + t.Fatal(err) + } + + want, got := jsonstr(test.wantDecoded), jsonstr(test.decodeInto) + if want != got { + t.Errorf("%s: want decoded == %+v, got %+v", label, want, got) + } + } +} + +// jsonstr allows comparison of deeply nested structs with pointer members. +func jsonstr(o interface{}) string { + s, err := json.MarshalIndent(o, "", " ") + if err != nil { + panic(err.Error()) + } + return string(s) +} + var tomlTableArrays = ` [[albums]] name = "Born to Run" @@ -124,8 +172,6 @@ tOpdate = 2006-01-02T15:04:05Z tOparray = [ "array" ] Match = "i should be in Match only" MatcH = "i should be in MatcH only" -Field = "neat" -FielD = "messy" once = "just once" [nEst.eD] nEstedString = "another string" @@ -140,7 +186,6 @@ type Insensitive struct { TopArray []string Match string MatcH string - Field string Once string OncE string Nest InsensitiveNest @@ -168,9 +213,8 @@ func TestCase(t *testing.T) { TopArray: []string{"array"}, MatcH: "i should be in MatcH only", Match: "i should be in Match only", - Field: "neat", // encoding/json would store "messy" here Once: "just once", - OncE: "just once", // wait, what? + OncE: "", Nest: InsensitiveNest{ Ed: InsensitiveEd{NestedString: "another string"}, }, @@ -335,9 +379,50 @@ ip = "10.0.0.2" fmt.Printf("Ports: %v\n", s.Config.Ports) } - // // Output: + // Output: // Server: alpha (ip: 10.0.0.1) in Toronto created on 1987-07-05 // Ports: [8001 8002] // Server: beta (ip: 10.0.0.2) in New Jersey created on 1887-01-05 // Ports: [9001 9002] } + +type duration struct { + time.Duration +} + +func (d *duration) UnmarshalText(text []byte) error { + var err error + d.Duration, err = time.ParseDuration(string(text)) + return err +} + +// Example Unmarshaler blah blah. +func ExampleUnmarshaler() { + blob := ` +[[song]] +name = "Thunder Road" +duration = "4m49s" + +[[song]] +name = "Stairway to Heaven" +duration = "8m03s" +` + type song struct { + Name string + Duration duration + } + type songs struct { + Song []song + } + var favorites songs + if _, err := Decode(blob, &favorites); err != nil { + log.Fatal(err) + } + + for _, s := range favorites.Song { + fmt.Printf("%s (%s)\n", s.Name, s.Duration) + } + // Output: + // Thunder Road (4m49s) + // Stairway to Heaven (8m3s) +} diff --git a/third_party/github.com/BurntSushi/toml/encode.go b/third_party/github.com/BurntSushi/toml/encode.go index 7b080f414..b3d748743 100644 --- a/third_party/github.com/BurntSushi/toml/encode.go +++ b/third_party/github.com/BurntSushi/toml/encode.go @@ -15,27 +15,41 @@ package toml import ( "bufio" + "encoding" + "errors" "fmt" "io" "reflect" + "sort" + "strconv" "strings" ) -type encoder struct { +var ( + ErrArrayMixedElementTypes = errors.New( + "can't encode array with mixed element types") + ErrArrayNilElement = errors.New( + "can't encode array with nil element") +) + +type Encoder struct { // A single indentation level. By default it is two spaces. Indent string w *bufio.Writer + + // hasWritten is whether we have written any output to w yet. + hasWritten bool } -func newEncoder(w io.Writer) *encoder { - return &encoder{ +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ w: bufio.NewWriter(w), Indent: " ", } } -func (enc *encoder) Encode(v interface{}) error { +func (enc *Encoder) Encode(v interface{}) error { rv := eindirect(reflect.ValueOf(v)) if err := enc.encode(Key([]string{}), rv); err != nil { return err @@ -43,49 +57,466 @@ func (enc *encoder) Encode(v interface{}) error { return enc.w.Flush() } -func (enc *encoder) encode(key Key, rv reflect.Value) error { +func (enc *Encoder) encode(key Key, rv reflect.Value) error { + // Special case. If we can marshal the type to text, then we used that. + if _, ok := rv.Interface().(encoding.TextMarshaler); ok { + err := enc.eKeyEq(key) + if err != nil { + return err + } + return enc.eElement(rv) + } + k := rv.Kind() switch k { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, + reflect.Uint64, + reflect.Float32, reflect.Float64, + reflect.String, reflect.Bool: + err := enc.eKeyEq(key) + if err != nil { + return err + } + return enc.eElement(rv) + case reflect.Array, reflect.Slice: + return enc.eArrayOrSlice(key, rv) + case reflect.Interface: + if rv.IsNil() { + return nil + } + return enc.encode(key, rv.Elem()) + case reflect.Map: + if rv.IsNil() { + return nil + } + return enc.eTable(key, rv) + case reflect.Ptr: + if rv.IsNil() { + return nil + } + return enc.encode(key, rv.Elem()) case reflect.Struct: - return enc.eStruct(key, rv) - case reflect.String: - return enc.eString(key, rv) + return enc.eTable(key, rv) } return e("Unsupported type for key '%s': %s", key, k) } -func (enc *encoder) eStruct(key Key, rv reflect.Value) error { - rt := rv.Type() - for i := 0; i < rt.NumField(); i++ { - sft := rt.Field(i) - sf := rv.Field(i) - if err := enc.encode(key.add(sft.Name), sf); err != nil { +// eElement encodes any value that can be an array element (primitives and +// arrays). +func (enc *Encoder) eElement(rv reflect.Value) error { + ws := func(s string) error { + _, err := io.WriteString(enc.w, s) + return err + } + // By the TOML spec, all floats must have a decimal with at least one + // number on either side. + floatAddDecimal := func(fstr string) string { + if !strings.Contains(fstr, ".") { + return fstr + ".0" + } + return fstr + } + + // Special case. Use text marshaler if it's available for this value. + if v, ok := rv.Interface().(encoding.TextMarshaler); ok { + s, err := v.MarshalText() + if err != nil { return err } + return ws(string(s)) } - return nil + + var err error + k := rv.Kind() + switch k { + case reflect.Bool: + err = ws(strconv.FormatBool(rv.Bool())) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + err = ws(strconv.FormatInt(rv.Int(), 10)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, + reflect.Uint32, reflect.Uint64: + err = ws(strconv.FormatUint(rv.Uint(), 10)) + case reflect.Float32: + err = ws(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 32))) + case reflect.Float64: + err = ws(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 64))) + case reflect.Array, reflect.Slice: + return enc.eArrayOrSliceElement(rv) + case reflect.Interface: + return enc.eElement(rv.Elem()) + case reflect.String: + s := rv.String() + s = strings.NewReplacer( + "\t", "\\t", + "\n", "\\n", + "\r", "\\r", + "\"", "\\\"", + "\\", "\\\\", + ).Replace(s) + err = ws("\"" + s + "\"") + default: + return e("Unexpected primitive type: %s", k) + } + return err } -func (enc *encoder) eString(key Key, rv reflect.Value) error { - s := rv.String() - s = strings.NewReplacer( - "\t", "\\t", - "\n", "\\n", - "\r", "\\r", - "\"", "\\\"", - "\\", "\\\\", - ).Replace(s) - s = "\"" + s + "\"" - if err := enc.eKeyVal(key, s); err != nil { +func (enc *Encoder) eArrayOrSlice(key Key, rv reflect.Value) error { + // Determine whether this is an array of tables or of primitives. + elemV := reflect.ValueOf(nil) + if rv.Len() > 0 { + elemV = rv.Index(0) + } + isTableType, err := isTOMLTableType(rv.Type().Elem(), elemV) + if err != nil { + return err + } + + if len(key) > 0 && isTableType { + return enc.eArrayOfTables(key, rv) + } + + err = enc.eKeyEq(key) + if err != nil { + return err + } + return enc.eArrayOrSliceElement(rv) +} + +func (enc *Encoder) eArrayOrSliceElement(rv reflect.Value) error { + if _, err := enc.w.Write([]byte{'['}); err != nil { + return err + } + + length := rv.Len() + if length > 0 { + arrayElemType, isNil := tomlTypeName(rv.Index(0)) + if isNil { + return ErrArrayNilElement + } + + for i := 0; i < length; i++ { + elem := rv.Index(i) + + // Ensure that the array's elements each have the same TOML type. + elemType, isNil := tomlTypeName(elem) + if isNil { + return ErrArrayNilElement + } + if elemType != arrayElemType { + return ErrArrayMixedElementTypes + } + + if err := enc.eElement(elem); err != nil { + return err + } + if i != length-1 { + if _, err := enc.w.Write([]byte(", ")); err != nil { + return err + } + } + } + } + + if _, err := enc.w.Write([]byte{']'}); err != nil { return err } return nil } -func (enc *encoder) eKeyVal(key Key, value string) error { - out := fmt.Sprintf("%s%s = %s", - strings.Repeat(enc.Indent, len(key)-1), key[len(key)-1], value) - if _, err := fmt.Fprintln(enc.w, out); err != nil { +func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) error { + if enc.hasWritten { + _, err := enc.w.Write([]byte{'\n'}) + if err != nil { + return err + } + } + + for i := 0; i < rv.Len(); i++ { + trv := rv.Index(i) + if isNil(trv) { + continue + } + + _, err := fmt.Fprintf(enc.w, "%s[[%s]]\n", + strings.Repeat(enc.Indent, len(key)-1), key.String()) + if err != nil { + return err + } + + err = enc.eMapOrStruct(key, trv) + if err != nil { + return err + } + + if i != rv.Len()-1 { + if _, err := enc.w.Write([]byte("\n\n")); err != nil { + return err + } + } + enc.hasWritten = true + } + return nil +} + +func isStructOrMap(rv reflect.Value) bool { + switch rv.Kind() { + case reflect.Interface, reflect.Ptr: + return isStructOrMap(rv.Elem()) + case reflect.Map, reflect.Struct: + return true + default: + return false + } +} + +func (enc *Encoder) eTable(key Key, rv reflect.Value) error { + if enc.hasWritten { + _, err := enc.w.Write([]byte{'\n'}) + if err != nil { + return err + } + } + if len(key) > 0 { + _, err := fmt.Fprintf(enc.w, "%s[%s]\n", + strings.Repeat(enc.Indent, len(key)-1), key.String()) + if err != nil { + return err + } + } + return enc.eMapOrStruct(key, rv) +} + +func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value) error { + switch rv.Kind() { + case reflect.Map: + return enc.eMap(key, rv) + case reflect.Struct: + return enc.eStruct(key, rv) + case reflect.Ptr, reflect.Interface: + return enc.eMapOrStruct(key, rv.Elem()) + default: + panic("eTable: unhandled reflect.Value Kind: " + rv.Kind().String()) + } +} + +func (enc *Encoder) eMap(key Key, rv reflect.Value) error { + rt := rv.Type() + if rt.Key().Kind() != reflect.String { + return errors.New("can't encode a map with non-string key type") + } + + // Sort keys so that we have deterministic output. And write keys directly + // underneath this key first, before writing sub-structs or sub-maps. + var mapKeysDirect, mapKeysSub []string + for _, mapKey := range rv.MapKeys() { + k := mapKey.String() + mrv := rv.MapIndex(mapKey) + if isStructOrMap(mrv) { + mapKeysSub = append(mapKeysSub, k) + } else { + mapKeysDirect = append(mapKeysDirect, k) + } + } + + var writeMapKeys = func(mapKeys []string) error { + sort.Strings(mapKeys) + for i, mapKey := range mapKeys { + mrv := rv.MapIndex(reflect.ValueOf(mapKey)) + if isNil(mrv) { + // Don't write anything for nil fields. + continue + } + if err := enc.encode(key.add(mapKey), mrv); err != nil { + return err + } + + if i != len(mapKeys)-1 { + if _, err := enc.w.Write([]byte{'\n'}); err != nil { + return err + } + } + enc.hasWritten = true + } + + return nil + } + + err := writeMapKeys(mapKeysDirect) + if err != nil { + return err + } + err = writeMapKeys(mapKeysSub) + if err != nil { + return err + } + return nil +} + +func (enc *Encoder) eStruct(key Key, rv reflect.Value) error { + // Write keys for fields directly under this key first, because if we write + // a field that creates a new table, then all keys under it will be in that + // table (not the one we're writing here). + rt := rv.Type() + var fieldsDirect, fieldsSub [][]int + var addFields func(rt reflect.Type, rv reflect.Value, start []int) + addFields = func(rt reflect.Type, rv reflect.Value, start []int) { + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + frv := rv.Field(i) + if f.Anonymous { + t := frv.Type() + if t.Kind() == reflect.Ptr { + t = t.Elem() + frv = frv.Elem() + } + addFields(t, frv, f.Index) + } else if isStructOrMap(frv) { + fieldsSub = append(fieldsSub, append(start, f.Index...)) + } else { + fieldsDirect = append(fieldsDirect, append(start, f.Index...)) + } + } + } + addFields(rt, rv, nil) + + var writeFields = func(fields [][]int) error { + for i, fieldIndex := range fields { + sft := rt.FieldByIndex(fieldIndex) + sf := rv.FieldByIndex(fieldIndex) + if isNil(sf) { + // Don't write anything for nil fields. + continue + } + + keyName := sft.Tag.Get("toml") + if keyName == "-" { + continue + } + if keyName == "" { + keyName = sft.Name + } + + if err := enc.encode(key.add(keyName), sf); err != nil { + return err + } + + if i != len(fields)-1 { + if _, err := enc.w.Write([]byte{'\n'}); err != nil { + return err + } + } + enc.hasWritten = true + } + return nil + } + + err := writeFields(fieldsDirect) + if err != nil { + return err + } + if len(fieldsDirect) > 0 && len(fieldsSub) > 0 { + _, err = enc.w.Write([]byte{'\n'}) + if err != nil { + return err + } + } + err = writeFields(fieldsSub) + if err != nil { + return err + } + return nil +} + +// tomlTypeName returns the TOML type name of the Go value's type. It is used to +// determine whether the types of array elements are mixed (which is forbidden). +// If the Go value is nil, then it is illegal for it to be an array element, and +// valueIsNil is returned as true. +func tomlTypeName(rv reflect.Value) (typeName string, valueIsNil bool) { + if isNil(rv) { + return "", true + } + k := rv.Kind() + switch k { + case reflect.Bool: + return "bool", false + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, + reflect.Uint64: + return "integer", false + case reflect.Float32, reflect.Float64: + return "float", false + case reflect.Array, reflect.Slice: + return "array", false + case reflect.Ptr, reflect.Interface: + return tomlTypeName(rv.Elem()) + case reflect.String: + return "string", false + case reflect.Map, reflect.Struct: + return "table", false + default: + panic("unexpected reflect.Kind: " + k.String()) + } +} + +// isTOMLTableType returns whether this type and value represents a TOML table +// type (true) or element type (false). Both rt and rv are needed to determine +// this, in case the Go type is interface{} or in case rv is nil. If there is +// some other impossible situation detected, an error is returned. +func isTOMLTableType(rt reflect.Type, rv reflect.Value) (bool, error) { + k := rt.Kind() + switch k { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, + reflect.Uint64, + reflect.Float32, reflect.Float64, + reflect.String, reflect.Bool: + return false, nil + case reflect.Array, reflect.Slice: + // Make sure that these eventually contain an underlying non-table type + // element. + elemV := reflect.ValueOf(nil) + if rv.Len() > 0 { + elemV = rv.Index(0) + } + hasUnderlyingTableType, err := isTOMLTableType(rt.Elem(), elemV) + if err != nil { + return false, err + } + if hasUnderlyingTableType { + return true, errors.New("TOML array element can't contain a table") + } + return false, nil + case reflect.Ptr: + return isTOMLTableType(rt.Elem(), rv.Elem()) + case reflect.Interface: + if rv.Kind() == reflect.Interface { + return false, nil + } + return isTOMLTableType(rv.Type(), rv) + case reflect.Map, reflect.Struct: + return true, nil + default: + panic("unexpected reflect.Kind: " + k.String()) + } +} + +func isNil(rv reflect.Value) bool { + switch rv.Kind() { + case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return rv.IsNil() + default: + return false + } +} + +func (enc *Encoder) eKeyEq(key Key) error { + _, err := io.WriteString(enc.w, strings.Repeat(enc.Indent, len(key)-1)) + if err != nil { + return err + } + _, err = io.WriteString(enc.w, key[len(key)-1]+" = ") + if err != nil { return err } return nil diff --git a/third_party/github.com/BurntSushi/toml/encode_test.go b/third_party/github.com/BurntSushi/toml/encode_test.go index 3f7fad8b4..3e2b13b2e 100644 --- a/third_party/github.com/BurntSushi/toml/encode_test.go +++ b/third_party/github.com/BurntSushi/toml/encode_test.go @@ -5,21 +5,278 @@ import ( "testing" ) -type encodeSimple struct { - Location string - // Ages []int - // DOB time.Time -} - +// XXX(burntsushi) +// I think these tests probably should be removed. They are good, but they +// ought to be obsolete by toml-test. func TestEncode(t *testing.T) { - v := encodeSimple{ - Location: "Westborough, MA", + tests := map[string]struct { + input interface{} + wantOutput string + wantError error + }{ + "bool field": { + input: struct { + BoolTrue bool + BoolFalse bool + }{true, false}, + wantOutput: "BoolTrue = true\nBoolFalse = false", + }, + "int fields": { + input: struct { + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + }{1, 2, 3, 4, 5}, + wantOutput: "Int = 1\nInt8 = 2\nInt16 = 3\nInt32 = 4\nInt64 = 5", + }, + "uint fields": { + input: struct { + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + }{1, 2, 3, 4, 5}, + wantOutput: "Uint = 1\nUint8 = 2\nUint16 = 3\nUint32 = 4" + + "\nUint64 = 5", + }, + "float fields": { + input: struct { + Float32 float32 + Float64 float64 + }{1.5, 2.5}, + wantOutput: "Float32 = 1.5\nFloat64 = 2.5", + }, + "string field": { + input: struct{ String string }{"foo"}, + wantOutput: `String = "foo"`, + }, + "array fields": { + input: struct { + IntArray0 [0]int + IntArray3 [3]int + }{[0]int{}, [3]int{1, 2, 3}}, + wantOutput: "IntArray0 = []\nIntArray3 = [1, 2, 3]", + }, + "slice fields": { + input: struct{ IntSliceNil, IntSlice0, IntSlice3 []int }{ + nil, []int{}, []int{1, 2, 3}, + }, + wantOutput: "IntSlice0 = []\nIntSlice3 = [1, 2, 3]", + }, + "nested arrays and slices": { + input: struct { + SliceOfArrays [][2]int + ArrayOfSlices [2][]int + SliceOfArraysOfSlices [][2][]int + ArrayOfSlicesOfArrays [2][][2]int + SliceOfMixedArrays [][2]interface{} + ArrayOfMixedSlices [2][]interface{} + }{ + [][2]int{[2]int{1, 2}, [2]int{3, 4}}, + [2][]int{[]int{1, 2}, []int{3, 4}}, + [][2][]int{ + [2][]int{ + []int{1, 2}, []int{3, 4}, + }, + [2][]int{ + []int{5, 6}, []int{7, 8}, + }, + }, + [2][][2]int{ + [][2]int{ + [2]int{1, 2}, [2]int{3, 4}, + }, + [][2]int{ + [2]int{5, 6}, [2]int{7, 8}, + }, + }, + [][2]interface{}{ + [2]interface{}{1, 2}, [2]interface{}{"a", "b"}, + }, + [2][]interface{}{ + []interface{}{1, 2}, []interface{}{"a", "b"}, + }, + }, + wantOutput: `SliceOfArrays = [[1, 2], [3, 4]] +ArrayOfSlices = [[1, 2], [3, 4]] +SliceOfArraysOfSlices = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] +ArrayOfSlicesOfArrays = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] +SliceOfMixedArrays = [[1, 2], ["a", "b"]] +ArrayOfMixedSlices = [[1, 2], ["a", "b"]]`, + }, + "(error) slice with element type mismatch (string and integer)": { + input: struct{ Mixed []interface{} }{[]interface{}{1, "a"}}, + wantError: ErrArrayMixedElementTypes, + }, + "(error) slice with element type mismatch (integer and float)": { + input: struct{ Mixed []interface{} }{[]interface{}{1, 2.5}}, + wantError: ErrArrayMixedElementTypes, + }, + "slice with elems of differing Go types, same TOML types": { + input: struct { + MixedInts []interface{} + MixedFloats []interface{} + }{ + []interface{}{ + int(1), int8(2), int16(3), int32(4), int64(5), + uint(1), uint8(2), uint16(3), uint32(4), uint64(5), + }, + []interface{}{float32(1.5), float64(2.5)}, + }, + wantOutput: "MixedInts = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]\n" + + "MixedFloats = [1.5, 2.5]", + }, + "(error) slice w/ element type mismatch (one is nested array)": { + input: struct{ Mixed []interface{} }{ + []interface{}{1, []interface{}{2}}, + }, + wantError: ErrArrayMixedElementTypes, + }, + "(error) slice with 1 nil element": { + input: struct{ NilElement1 []interface{} }{[]interface{}{nil}}, + wantError: ErrArrayNilElement, + }, + "(error) slice with 1 nil element (and other non-nil elements)": { + input: struct{ NilElement []interface{} }{ + []interface{}{1, nil}, + }, + wantError: ErrArrayNilElement, + }, + "simple map": { + input: map[string]int{"a": 1, "b": 2}, + wantOutput: "a = 1\nb = 2", + }, + "map with interface{} value type": { + input: map[string]interface{}{"a": 1, "b": "c"}, + wantOutput: "a = 1\nb = \"c\"", + }, + "map with interface{} value type, some of which are structs": { + input: map[string]interface{}{ + "a": struct{ Int int }{2}, + "b": 1, + }, + wantOutput: "b = 1\n[a]\n Int = 2", + }, + "nested map": { + input: map[string]map[string]int{ + "a": map[string]int{"b": 1}, + "c": map[string]int{"d": 2}, + }, + wantOutput: "[a]\n b = 1\n\n[c]\n d = 2", + }, + "nested struct": { + input: struct{ Struct struct{ Int int } }{ + struct{ Int int }{1}, + }, + wantOutput: "[Struct]\n Int = 1", + }, + "nested struct and non-struct field": { + input: struct { + Struct struct{ Int int } + Bool bool + }{struct{ Int int }{1}, true}, + wantOutput: "Bool = true\n\n[Struct]\n Int = 1", + }, + "2 nested structs": { + input: struct{ Struct1, Struct2 struct{ Int int } }{ + struct{ Int int }{1}, struct{ Int int }{2}, + }, + wantOutput: "[Struct1]\n Int = 1\n\n[Struct2]\n Int = 2", + }, + "deeply nested structs": { + input: struct { + Struct1, Struct2 struct{ Struct3 *struct{ Int int } } + }{ + struct{ Struct3 *struct{ Int int } }{&struct{ Int int }{1}}, + struct{ Struct3 *struct{ Int int } }{nil}, + }, + wantOutput: "[Struct1]\n [Struct1.Struct3]\n Int = 1" + + "\n\n[Struct2]\n", + }, + "nested struct with nil struct elem": { + input: struct { + Struct struct{ Inner *struct{ Int int } } + }{ + struct{ Inner *struct{ Int int } }{nil}, + }, + wantOutput: "[Struct]\n", + }, + "nested struct with no fields": { + input: struct { + Struct struct{ Inner struct{} } + }{ + struct{ Inner struct{} }{struct{}{}}, + }, + wantOutput: "[Struct]\n [Struct.Inner]\n", + }, + "struct with tags": { + input: struct { + Struct struct { + Int int `toml:"_int"` + } `toml:"_struct"` + Bool bool `toml:"_bool"` + }{ + struct { + Int int `toml:"_int"` + }{1}, true, + }, + wantOutput: "_bool = true\n\n[_struct]\n _int = 1", + }, + "embedded struct": { + input: struct{ Embedded }{Embedded{1}}, + wantOutput: "_int = 1", + }, + "embedded *struct": { + input: struct{ *Embedded }{&Embedded{1}}, + wantOutput: "_int = 1", + }, + "nested embedded struct": { + input: struct { + Struct struct{ Embedded } `toml:"_struct"` + }{struct{ Embedded }{Embedded{1}}}, + wantOutput: "[_struct]\n _int = 1", + }, + "nested embedded *struct": { + input: struct { + Struct struct{ *Embedded } `toml:"_struct"` + }{struct{ *Embedded }{&Embedded{1}}}, + wantOutput: "[_struct]\n _int = 1", + }, + "array of tables": { + input: struct { + Structs []*struct{ Int int } `toml:"struct"` + }{ + []*struct{ Int int }{ + {1}, nil, {3}, + }, + }, + wantOutput: "[[struct]]\n Int = 1\n\n[[struct]]\n Int = 3", + }, } - - buf := new(bytes.Buffer) - e := newEncoder(buf) - if err := e.Encode(v); err != nil { - t.Fatal(err) + for label, test := range tests { + var buf bytes.Buffer + e := NewEncoder(&buf) + err := e.Encode(test.input) + if err != test.wantError { + if test.wantError != nil { + t.Errorf("%s: want Encode error %v, got %v", + label, test.wantError, err) + } else { + t.Errorf("%s: Encode failed: %s", label, err) + } + } + if err != nil { + continue + } + if got := buf.String(); test.wantOutput != got { + t.Errorf("%s: want %q, got %q", label, test.wantOutput, got) + } } - testf(buf.String()) +} + +type Embedded struct { + Int int `toml:"_int"` } diff --git a/third_party/github.com/BurntSushi/toml/parse.go b/third_party/github.com/BurntSushi/toml/parse.go index 57e4a70a9..2abb173f8 100644 --- a/third_party/github.com/BurntSushi/toml/parse.go +++ b/third_party/github.com/BurntSushi/toml/parse.go @@ -159,6 +159,8 @@ func (p *parser) value(it item) (interface{}, tomlType) { case itemInteger: num, err := strconv.ParseInt(it.val, 10, 64) if err != nil { + // See comment below for floats describing why we make a + // distinction between a bug and a user error. if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { @@ -172,6 +174,13 @@ func (p *parser) value(it item) (interface{}, tomlType) { case itemFloat: num, err := strconv.ParseFloat(it.val, 64) if err != nil { + // Distinguish float values. Normally, it'd be a bug if the lexer + // provides an invalid float, but it's possible that the float is + // out of range of valid values (which the lexer cannot determine). + // So mark the former as a bug but the latter as a legitimate user + // error. + // + // This is also true for integers. if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { @@ -209,7 +218,8 @@ func (p *parser) value(it item) (interface{}, tomlType) { } // establishContext sets the current context of the parser, -// where the context is the hash currently in scope. +// where the context is either a hash or an array of hashes. Which one is +// set depends on the value of the `array` parameter. // // Establishing the context also makes sure that the key isn't a duplicate, and // will create implicit hashes automatically. @@ -248,10 +258,15 @@ func (p *parser) establishContext(key Key, array bool) { p.context = keyContext if array { + // If this is the first element for this array, then allocate a new + // list of tables for it. k := key[len(key)-1] if _, ok := hashContext[k]; !ok { hashContext[k] = make([]map[string]interface{}, 0, 5) } + + // Add a new table. But make sure the key hasn't already been used + // for something else. if hash, ok := hashContext[k].([]map[string]interface{}); ok { hashContext[k] = append(hash, make(map[string]interface{})) } else { @@ -280,6 +295,8 @@ func (p *parser) setValue(key string, value interface{}) { } switch t := tmpHash.(type) { case []map[string]interface{}: + // The context is a table of hashes. Pick the most recent table + // defined as the current hash. hash = t[len(t)-1] case map[string]interface{}: hash = t @@ -291,9 +308,17 @@ func (p *parser) setValue(key string, value interface{}) { keyContext = append(keyContext, key) if _, ok := hash[key]; ok { - // We need to do some fancy footwork here. If `hash[key]` was implcitly - // created AND `value` is a hash, then let this go through and stop - // tagging this table as implicit. + // Typically, if the given key has already been set, then we have + // to raise an error since duplicate keys are disallowed. However, + // it's possible that a key was previously defined implicitly. In this + // case, it is allowed to be redefined concretely. (See the + // `tests/valid/implicit-and-explicit-after.toml` test in `toml-test`.) + // + // But we have to make sure to stop marking it as an implicit. (So that + // another redefinition provokes an error.) + // + // Note that since it has already been defined (as a hash), we don't + // want to overwrite it. So our business is done. if p.isImplicit(keyContext) { p.removeImplicit(keyContext) return @@ -308,6 +333,9 @@ func (p *parser) setValue(key string, value interface{}) { // setType sets the type of a particular value at a given key. // It should be called immediately AFTER setValue. +// +// Note that if `key` is empty, then the type given will be applied to the +// current context (which is either a table or an array of tables). func (p *parser) setType(key string, typ tomlType) { keyContext := make(Key, 0, len(p.context)+1) for _, k := range p.context { @@ -377,9 +405,10 @@ func (p *parser) asciiEscapeToUnicode(s string) string { "lexer claims it's OK: %s", s, err) } - // I honestly don't understand how this works. I can't seem to find - // a way to make this fail. I figured this would fail on invalid UTF-8 - // characters like U+DCFF, but it doesn't. + // BUG(burntsushi) + // I honestly don't understand how this works. I can't seem + // to find a way to make this fail. I figured this would fail on invalid + // UTF-8 characters like U+DCFF, but it doesn't. r := string(rune(hex)) if !utf8.ValidString(r) { p.panic("Escaped character '\\u%s' is not valid UTF-8.", s) diff --git a/third_party/github.com/BurntSushi/toml/toml-test-encoder/COPYING b/third_party/github.com/BurntSushi/toml/toml-test-encoder/COPYING new file mode 100644 index 000000000..5a8e33254 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/toml-test-encoder/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/third_party/github.com/BurntSushi/toml/toml-test-encoder/README.md b/third_party/github.com/BurntSushi/toml/toml-test-encoder/README.md new file mode 100644 index 000000000..45a603f29 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/toml-test-encoder/README.md @@ -0,0 +1,14 @@ +# Implements the TOML test suite interface for TOML encoders + +This is an implementation of the interface expected by +[toml-test](https://github.com/BurntSushi/toml-test) for the +[TOML encoder](https://github.com/BurntSushi/toml). +In particular, it maps JSON data on `stdin` to a TOML format on `stdout`. + + +Compatible with TOML version +[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md) + +Compatible with `toml-test` version +[v0.2.0](https://github.com/BurntSushi/toml-test/tree/v0.2.0) + diff --git a/third_party/github.com/BurntSushi/toml/toml-test-encoder/main.go b/third_party/github.com/BurntSushi/toml/toml-test-encoder/main.go new file mode 100644 index 000000000..03864f8ae --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/toml-test-encoder/main.go @@ -0,0 +1,129 @@ +package main + +import ( + "encoding/json" + "flag" + "log" + "os" + "path" + "strconv" + "time" + + "github.com/BurntSushi/toml" +) + +func init() { + log.SetFlags(0) + + flag.Usage = usage + flag.Parse() +} + +func usage() { + log.Printf("Usage: %s < json-file\n", path.Base(os.Args[0])) + flag.PrintDefaults() + + os.Exit(1) +} + +func main() { + if flag.NArg() != 0 { + flag.Usage() + } + + var tmp interface{} + if err := json.NewDecoder(os.Stdin).Decode(&tmp); err != nil { + log.Fatalf("Error decoding JSON: %s", err) + } + + tomlData := translate(tmp) + if err := toml.NewEncoder(os.Stdout).Encode(tomlData); err != nil { + log.Fatalf("Error encoding TOML: %s", err) + } +} + +func translate(typedJson interface{}) interface{} { + switch v := typedJson.(type) { + case map[string]interface{}: + if len(v) == 2 && in("type", v) && in("value", v) { + return untag(v) + } + m := make(map[string]interface{}, len(v)) + for k, v2 := range v { + m[k] = translate(v2) + } + return m + case []interface{}: + tabArray := make([]map[string]interface{}, len(v)) + for i := range v { + if m, ok := translate(v[i]).(map[string]interface{}); ok { + tabArray[i] = m + } else { + log.Fatalf("JSON arrays may only contain objects. This "+ + "corresponds to only tables being allowed in "+ + "TOML table arrays.") + } + } + return tabArray + } + log.Fatalf("Unrecognized JSON format '%T'.", typedJson) + panic("unreachable") +} + +func untag(typed map[string]interface{}) interface{} { + t := typed["type"].(string) + v := typed["value"] + switch t { + case "string": + return v.(string) + case "integer": + v := v.(string) + n, err := strconv.Atoi(v) + if err != nil { + log.Fatalf("Could not parse '%s' as integer: %s", v, err) + } + return n + case "float": + v := v.(string) + f, err := strconv.ParseFloat(v, 64) + if err != nil { + log.Fatalf("Could not parse '%s' as float64: %s", v, err) + } + return f + case "datetime": + v := v.(string) + t, err := time.Parse("2006-01-02T15:04:05Z", v) + if err != nil { + log.Fatalf("Could not parse '%s' as a datetime: %s", v, err) + } + return t + case "bool": + v := v.(string) + switch v { + case "true": + return true + case "false": + return false + } + log.Fatalf("Could not parse '%s' as a boolean.", v) + case "array": + v := v.([]interface{}) + array := make([]interface{}, len(v)) + for i := range v { + if m, ok := v[i].(map[string]interface{}); ok { + array[i] = untag(m) + } else { + log.Fatalf("Arrays may only contain other arrays or "+ + "primitive values, but found a '%T'.", m) + } + } + return array + } + log.Fatalf("Unrecognized tag type '%s'.", t) + panic("unreachable") +} + +func in(key string, m map[string]interface{}) bool { + _, ok := m[key] + return ok +} diff --git a/third_party/github.com/BurntSushi/toml/toml-test-go/main.go b/third_party/github.com/BurntSushi/toml/toml-test-go/main.go index 79a3c212f..eb0f95b8d 100644 --- a/third_party/github.com/BurntSushi/toml/toml-test-go/main.go +++ b/third_party/github.com/BurntSushi/toml/toml-test-go/main.go @@ -43,7 +43,6 @@ func main() { } func translate(tomlData interface{}) interface{} { - switch orig := tomlData.(type) { case map[string]interface{}: typed := make(map[string]interface{}, len(orig)) diff --git a/third_party/github.com/BurntSushi/toml/type-check.go b/third_party/github.com/BurntSushi/toml/type_check.go similarity index 100% rename from third_party/github.com/BurntSushi/toml/type-check.go rename to third_party/github.com/BurntSushi/toml/type_check.go diff --git a/third_party/github.com/BurntSushi/toml/type_fields.go b/third_party/github.com/BurntSushi/toml/type_fields.go new file mode 100644 index 000000000..138fc0037 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/type_fields.go @@ -0,0 +1,241 @@ +package toml + +// Struct field handling is adapted from code in encoding/json: +// +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +import ( + "reflect" + "sort" + "sync" +) + +// A field represents a single field found in a struct. +type field struct { + name string // the name of the field (`toml` tag included) + tag bool // whether field has a `toml` tag + index []int // represents the depth of an anonymous field + typ reflect.Type // the type of the field +} + +// byName sorts field by name, breaking ties with depth, +// then breaking ties with "name came from toml tag", then +// breaking ties with index sequence. +type byName []field + +func (x byName) Len() int { return len(x) } + +func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byName) Less(i, j int) bool { + if x[i].name != x[j].name { + return x[i].name < x[j].name + } + if len(x[i].index) != len(x[j].index) { + return len(x[i].index) < len(x[j].index) + } + if x[i].tag != x[j].tag { + return x[i].tag + } + return byIndex(x).Less(i, j) +} + +// byIndex sorts field by index sequence. +type byIndex []field + +func (x byIndex) Len() int { return len(x) } + +func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byIndex) Less(i, j int) bool { + for k, xik := range x[i].index { + if k >= len(x[j].index) { + return false + } + if xik != x[j].index[k] { + return xik < x[j].index[k] + } + } + return len(x[i].index) < len(x[j].index) +} + +// typeFields returns a list of fields that TOML should recognize for the given +// type. The algorithm is breadth-first search over the set of structs to +// include - the top struct and then any reachable anonymous structs. +func typeFields(t reflect.Type) []field { + // Anonymous fields to explore at the current level and the next. + current := []field{} + next := []field{{typ: t}} + + // Count of queued names for current level and the next. + count := map[reflect.Type]int{} + nextCount := map[reflect.Type]int{} + + // Types already visited at an earlier level. + visited := map[reflect.Type]bool{} + + // Fields found. + var fields []field + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, map[reflect.Type]int{} + + for _, f := range current { + if visited[f.typ] { + continue + } + visited[f.typ] = true + + // Scan f.typ for fields to include. + for i := 0; i < f.typ.NumField(); i++ { + sf := f.typ.Field(i) + if sf.PkgPath != "" { // unexported + continue + } + name := sf.Tag.Get("toml") + if name == "-" { + continue + } + index := make([]int, len(f.index)+1) + copy(index, f.index) + index[len(f.index)] = i + + ft := sf.Type + if ft.Name() == "" && ft.Kind() == reflect.Ptr { + // Follow pointer. + ft = ft.Elem() + } + + // Record found field and index sequence. + if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := name != "" + if name == "" { + name = sf.Name + } + fields = append(fields, field{name, tagged, index, ft}) + if count[f.typ] > 1 { + // If there were multiple instances, add a second, + // so that the annihilation code will see a duplicate. + // It only cares about the distinction between 1 or 2, + // so don't bother generating any more copies. + fields = append(fields, fields[len(fields)-1]) + } + continue + } + + // Record new anonymous struct to explore in next round. + nextCount[ft]++ + if nextCount[ft] == 1 { + f := field{name: ft.Name(), index: index, typ: ft} + next = append(next, f) + } + } + } + } + + sort.Sort(byName(fields)) + + // Delete all fields that are hidden by the Go rules for embedded fields, + // except that fields with TOML tags are promoted. + + // The fields are sorted in primary order of name, secondary order + // of field index length. Loop over names; for each name, delete + // hidden fields by choosing the one dominant field that survives. + out := fields[:0] + for advance, i := 0, 0; i < len(fields); i += advance { + // One iteration per name. + // Find the sequence of fields with the name of this first field. + fi := fields[i] + name := fi.name + for advance = 1; i+advance < len(fields); advance++ { + fj := fields[i+advance] + if fj.name != name { + break + } + } + if advance == 1 { // Only one field with this name + out = append(out, fi) + continue + } + dominant, ok := dominantField(fields[i : i+advance]) + if ok { + out = append(out, dominant) + } + } + + fields = out + sort.Sort(byIndex(fields)) + + return fields +} + +// dominantField looks through the fields, all of which are known to +// have the same name, to find the single field that dominates the +// others using Go's embedding rules, modified by the presence of +// TOML tags. If there are multiple top-level fields, the boolean +// will be false: This condition is an error in Go and we skip all +// the fields. +func dominantField(fields []field) (field, bool) { + // The fields are sorted in increasing index-length order. The winner + // must therefore be one with the shortest index length. Drop all + // longer entries, which is easy: just truncate the slice. + length := len(fields[0].index) + tagged := -1 // Index of first tagged field. + for i, f := range fields { + if len(f.index) > length { + fields = fields[:i] + break + } + if f.tag { + if tagged >= 0 { + // Multiple tagged fields at the same level: conflict. + // Return no field. + return field{}, false + } + tagged = i + } + } + if tagged >= 0 { + return fields[tagged], true + } + // All remaining fields have the same length. If there's more than one, + // we have a conflict (two fields named "X" at the same level) and we + // return no field. + if len(fields) > 1 { + return field{}, false + } + return fields[0], true +} + +var fieldCache struct { + sync.RWMutex + m map[reflect.Type][]field +} + +// cachedTypeFields is like typeFields but uses a cache to avoid repeated work. +func cachedTypeFields(t reflect.Type) []field { + fieldCache.RLock() + f := fieldCache.m[t] + fieldCache.RUnlock() + if f != nil { + return f + } + + // Compute fields without lock. + // Might duplicate effort but won't hold other computations back. + f = typeFields(t) + if f == nil { + f = []field{} + } + + fieldCache.Lock() + if fieldCache.m == nil { + fieldCache.m = map[reflect.Type][]field{} + } + fieldCache.m[t] = f + fieldCache.Unlock() + return f +} diff --git a/third_party/github.com/coreos/go-etcd/etcd/add_child_test.go b/third_party/github.com/coreos/go-etcd/etcd/add_child_test.go index b5a5bd971..26223ff1c 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/add_child_test.go +++ b/third_party/github.com/coreos/go-etcd/etcd/add_child_test.go @@ -9,7 +9,7 @@ func TestAddChild(t *testing.T) { c.Delete("nonexistentDir", true) }() - c.SetDir("fooDir", 5) + c.CreateDir("fooDir", 5) _, err := c.AddChild("fooDir", "v0", 5) if err != nil { @@ -44,7 +44,7 @@ func TestAddChildDir(t *testing.T) { c.Delete("nonexistentDir", true) }() - c.SetDir("fooDir", 5) + c.CreateDir("fooDir", 5) _, err := c.AddChildDir("fooDir", 5) if err != nil { diff --git a/third_party/github.com/coreos/go-etcd/etcd/client.go b/third_party/github.com/coreos/go-etcd/etcd/client.go index e03719e72..caa5fe2be 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/client.go +++ b/third_party/github.com/coreos/go-etcd/etcd/client.go @@ -2,6 +2,7 @@ package etcd import ( "crypto/tls" + "crypto/x509" "encoding/json" "errors" "io" @@ -41,6 +42,7 @@ type Cluster struct { type Config struct { CertFile string `json:"certFile"` KeyFile string `json:"keyFile"` + CaCertFile string `json:"caCertFile"` Scheme string `json:"scheme"` Timeout time.Duration `json:"timeout"` Consistency string `json: "consistency"` @@ -131,7 +133,7 @@ func NewClientReader(reader io.Reader) (*Client, error) { func setupHttpClient(client *Client) error { if client.config.CertFile != "" && client.config.KeyFile != "" { - err := client.SetCertAndKey(client.config.CertFile, client.config.KeyFile) + err := client.SetCertAndKey(client.config.CertFile, client.config.KeyFile, client.config.CaCertFile) if err != nil { return err } @@ -229,7 +231,7 @@ func (c *Client) saveConfig() error { return nil } -func (c *Client) SetCertAndKey(cert string, key string) error { +func (c *Client) SetCertAndKey(cert string, key string, caCert string) error { if cert != "" && key != "" { tlsCert, err := tls.LoadX509KeyPair(cert, key) @@ -237,12 +239,30 @@ func (c *Client) SetCertAndKey(cert string, key string) error { return err } + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + } + + if caCert != "" { + caCertPool := x509.NewCertPool() + + certBytes, err := ioutil.ReadFile(caCert) + if err != nil { + return err + } + + if !caCertPool.AppendCertsFromPEM(certBytes) { + return errors.New("Unable to load caCert") + } + + tlsConfig.RootCAs = caCertPool + } else { + tlsConfig.InsecureSkipVerify = true + } + tr := &http.Transport{ - TLSClientConfig: &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - InsecureSkipVerify: true, - }, - Dial: dialTimeout, + TLSClientConfig: tlsConfig, + Dial: dialTimeout, } c.httpClient = &http.Client{Transport: tr} diff --git a/third_party/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go b/third_party/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go index 56f8e4a8e..14a1b00f5 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go +++ b/third_party/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go @@ -17,11 +17,14 @@ func TestCompareAndSwap(t *testing.T) { if err != nil { t.Fatal(err) } - if !(resp.Node.Value == "bar2" && resp.Node.PrevValue == "bar" && - resp.Node.Key == "/foo" && resp.Node.TTL == 5) { + if !(resp.Node.Value == "bar2" && resp.Node.Key == "/foo" && resp.Node.TTL == 5) { t.Fatalf("CompareAndSwap 1 failed: %#v", resp) } + if !(resp.PrevNode.Value == "bar" && resp.PrevNode.Key == "/foo" && resp.PrevNode.TTL == 5) { + t.Fatalf("CompareAndSwap 1 prevNode failed: %#v", resp) + } + // This should fail because it gives an incorrect prevValue resp, err = c.CompareAndSwap("foo", "bar3", 5, "xxx", 0) if err == nil { @@ -38,14 +41,17 @@ func TestCompareAndSwap(t *testing.T) { if err != nil { t.Fatal(err) } - if !(resp.Node.Value == "bar2" && resp.Node.PrevValue == "bar" && - resp.Node.Key == "/foo" && resp.Node.TTL == 5) { - t.Fatalf("CompareAndSwap 1 failed: %#v", resp) + if !(resp.Node.Value == "bar2" && resp.Node.Key == "/foo" && resp.Node.TTL == 5) { + t.Fatalf("CompareAndSwap 3 failed: %#v", resp) + } + + if !(resp.PrevNode.Value == "bar" && resp.PrevNode.Key == "/foo" && resp.PrevNode.TTL == 5) { + t.Fatalf("CompareAndSwap 3 prevNode failed: %#v", resp) } // This should fail because it gives an incorrect prevIndex resp, err = c.CompareAndSwap("foo", "bar3", 5, "", 29817514) if err == nil { - t.Fatalf("CompareAndSwap 2 should have failed. The response is: %#v", resp) + t.Fatalf("CompareAndSwap 4 should have failed. The response is: %#v", resp) } } diff --git a/third_party/github.com/coreos/go-etcd/etcd/delete.go b/third_party/github.com/coreos/go-etcd/etcd/delete.go index 869b88e34..51d1c8546 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/delete.go +++ b/third_party/github.com/coreos/go-etcd/etcd/delete.go @@ -1,14 +1,16 @@ package etcd // Delete deletes the given key. -// When recursive set to false If the key points to a -// directory, the method will fail. +// +// When recursive set to false, if the key points to a +// directory the method will fail. +// // When recursive set to true, if the key points to a file, -// the file will be deleted. If the key points -// to a directory, then everything under the directory, including -// all child directories, will be deleted. +// the file will be deleted; if the key points to a directory, +// then everything under the directory (including all child directories) +// will be deleted. func (c *Client) Delete(key string, recursive bool) (*Response, error) { - raw, err := c.DeleteRaw(key, recursive) + raw, err := c.DeleteRaw(key, recursive, false) if err != nil { return nil, err @@ -17,9 +19,21 @@ func (c *Client) Delete(key string, recursive bool) (*Response, error) { return raw.toResponse() } -func (c *Client) DeleteRaw(key string, recursive bool) (*RawResponse, error) { +// DeleteDir deletes an empty directory or a key value pair +func (c *Client) DeleteDir(key string) (*Response, error) { + raw, err := c.DeleteRaw(key, false, true) + + if err != nil { + return nil, err + } + + return raw.toResponse() +} + +func (c *Client) DeleteRaw(key string, recursive bool, dir bool) (*RawResponse, error) { ops := options{ "recursive": recursive, + "dir": dir, } return c.delete(key, ops) diff --git a/third_party/github.com/coreos/go-etcd/etcd/delete_test.go b/third_party/github.com/coreos/go-etcd/etcd/delete_test.go index 30089aac4..590497155 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/delete_test.go +++ b/third_party/github.com/coreos/go-etcd/etcd/delete_test.go @@ -16,9 +16,12 @@ func TestDelete(t *testing.T) { t.Fatal(err) } - if !(resp.Node.PrevValue == "bar" && resp.Node.Value == "") { - t.Fatalf("Delete failed with %s %s", resp.Node.PrevValue, - resp.Node.Value) + if !(resp.Node.Value == "") { + t.Fatalf("Delete failed with %s", resp.Node.Value) + } + + if !(resp.PrevNode.Value == "bar") { + t.Fatalf("Delete PrevNode failed with %s", resp.Node.Value) } resp, err = c.Delete("foo", false) @@ -35,27 +38,41 @@ func TestDeleteAll(t *testing.T) { c.Delete("fooDir", true) }() - c.Set("foo", "bar", 5) - resp, err := c.Delete("foo", true) + c.SetDir("foo", 5) + // test delete an empty dir + resp, err := c.DeleteDir("foo") if err != nil { t.Fatal(err) } - if !(resp.Node.PrevValue == "bar" && resp.Node.Value == "") { + if !(resp.Node.Value == "") { t.Fatalf("DeleteAll 1 failed: %#v", resp) } - c.SetDir("fooDir", 5) + if !(resp.PrevNode.Dir == true && resp.PrevNode.Value == "") { + t.Fatalf("DeleteAll 1 PrevNode failed: %#v", resp) + } + + c.CreateDir("fooDir", 5) c.Set("fooDir/foo", "bar", 5) + _, err = c.DeleteDir("fooDir") + if err == nil { + t.Fatal("should not able to delete a non-empty dir with deletedir") + } + resp, err = c.Delete("fooDir", true) if err != nil { t.Fatal(err) } - if !(resp.Node.PrevValue == "" && resp.Node.Value == "") { + if !(resp.Node.Value == "") { t.Fatalf("DeleteAll 2 failed: %#v", resp) } + if !(resp.PrevNode.Dir == true && resp.PrevNode.Value == "") { + t.Fatalf("DeleteAll 2 PrevNode failed: %#v", resp) + } + resp, err = c.Delete("foo", true) if err == nil { t.Fatalf("DeleteAll should have failed because the key foo did not exist. "+ diff --git a/third_party/github.com/coreos/go-etcd/etcd/error.go b/third_party/github.com/coreos/go-etcd/etcd/error.go index 9a3268d60..9a33d1665 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/error.go +++ b/third_party/github.com/coreos/go-etcd/etcd/error.go @@ -5,14 +5,34 @@ import ( "fmt" ) +const ( + ErrCodeEtcdNotReachable = 501 +) + +var ( + errorMap = map[int]string{ + ErrCodeEtcdNotReachable: "All the given peers are not reachable", + } +) + type EtcdError struct { ErrorCode int `json:"errorCode"` Message string `json:"message"` Cause string `json:"cause,omitempty"` + Index uint64 `json:"index"` } func (e EtcdError) Error() string { - return fmt.Sprintf("%d: %s (%s)", e.ErrorCode, e.Message, e.Cause) + return fmt.Sprintf("%v: %v (%v) [%v]", e.ErrorCode, e.Message, e.Cause, e.Index) +} + +func newError(errorCode int, cause string, index uint64) *EtcdError { + return &EtcdError{ + ErrorCode: errorCode, + Message: errorMap[errorCode], + Cause: cause, + Index: index, + } } func handleError(b []byte) error { diff --git a/third_party/github.com/coreos/go-etcd/etcd/get_test.go b/third_party/github.com/coreos/go-etcd/etcd/get_test.go index 1f56d4ab5..06755e556 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/get_test.go +++ b/third_party/github.com/coreos/go-etcd/etcd/get_test.go @@ -35,7 +35,7 @@ func TestGetAll(t *testing.T) { c.Delete("fooDir", true) }() - c.SetDir("fooDir", 5) + c.CreateDir("fooDir", 5) c.Set("fooDir/k0", "v0", 5) c.Set("fooDir/k1", "v1", 5) @@ -73,7 +73,7 @@ func TestGetAll(t *testing.T) { } // Test the `recursive` option - c.SetDir("fooDir/childDir", 5) + c.CreateDir("fooDir/childDir", 5) c.Set("fooDir/childDir/k2", "v2", 5) // Return kv-pairs in sorted order @@ -83,6 +83,11 @@ func TestGetAll(t *testing.T) { result.Node.Expiration = nil for i, _ := range result.Node.Nodes { result.Node.Nodes[i].Expiration = nil + if result.Node.Nodes[i].Nodes != nil { + for j, _ := range result.Node.Nodes[i].Nodes { + result.Node.Nodes[i].Nodes[j].Expiration = nil + } + } } if err != nil { @@ -95,22 +100,30 @@ func TestGetAll(t *testing.T) { Dir: true, Nodes: Nodes{ Node{ - Key: "/fooDir/childDir/k2", - Value: "v2", - TTL: 5, + Key: "/fooDir/childDir/k2", + Value: "v2", + TTL: 5, + ModifiedIndex: 34, + CreatedIndex: 34, }, }, - TTL: 5, + TTL: 5, + ModifiedIndex: 33, + CreatedIndex: 33, }, Node{ - Key: "/fooDir/k0", - Value: "v0", - TTL: 5, + Key: "/fooDir/k0", + Value: "v0", + TTL: 5, + ModifiedIndex: 31, + CreatedIndex: 31, }, Node{ - Key: "/fooDir/k1", - Value: "v1", - TTL: 5, + Key: "/fooDir/k1", + Value: "v1", + TTL: 5, + ModifiedIndex: 32, + CreatedIndex: 32, }, } diff --git a/third_party/github.com/coreos/go-etcd/etcd/options.go b/third_party/github.com/coreos/go-etcd/etcd/options.go index 31fe80288..93efdca57 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/options.go +++ b/third_party/github.com/coreos/go-etcd/etcd/options.go @@ -28,12 +28,14 @@ var ( "prevValue": reflect.String, "prevIndex": reflect.Uint64, "prevExist": reflect.Bool, + "dir": reflect.Bool, } VALID_POST_OPTIONS = validOptions{} VALID_DELETE_OPTIONS = validOptions{ "recursive": reflect.Bool, + "dir": reflect.Bool, } ) diff --git a/third_party/github.com/coreos/go-etcd/etcd/requests.go b/third_party/github.com/coreos/go-etcd/etcd/requests.go index 7385bfacd..511c51aac 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/requests.go +++ b/third_party/github.com/coreos/go-etcd/etcd/requests.go @@ -14,8 +14,8 @@ import ( // get issues a GET request func (c *Client) get(key string, options options) (*RawResponse, error) { logger.Debugf("get %s [%s]", key, c.cluster.Leader) + p := keyToPath(key) - p := path.Join("keys", key) // If consistency level is set to STRONG, append // the `consistent` query string. if c.config.Consistency == STRONG_CONSISTENCY { @@ -42,7 +42,7 @@ func (c *Client) put(key string, value string, ttl uint64, options options) (*RawResponse, error) { logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader) - p := path.Join("keys", key) + p := keyToPath(key) str, err := options.toParameters(VALID_PUT_OPTIONS) if err != nil { @@ -62,7 +62,7 @@ func (c *Client) put(key string, value string, ttl uint64, // post issues a POST request func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) { logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader) - p := path.Join("keys", key) + p := keyToPath(key) resp, err := c.sendRequest("POST", p, buildValues(value, ttl)) @@ -76,8 +76,7 @@ func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error // delete issues a DELETE request func (c *Client) delete(key string, options options) (*RawResponse, error) { logger.Debugf("delete %s [%s]", key, c.cluster.Leader) - - p := path.Join("keys", key) + p := keyToPath(key) str, err := options.toParameters(VALID_DELETE_OPTIONS) if err != nil { @@ -111,7 +110,8 @@ func (c *Client) sendRequest(method string, relativePath string, trial++ logger.Debug("begin trail ", trial) if trial > 2*len(c.cluster.Machines) { - return nil, fmt.Errorf("Cannot reach servers after %v time", trial) + return nil, newError(ErrCodeEtcdNotReachable, + "Tried to connect to each peer twice and failed", 0) } if method == "GET" && c.config.Consistency == WEAK_CONSISTENCY { @@ -203,9 +203,7 @@ func (c *Client) handleResp(resp *http.Response) (bool, []byte) { } else if code == http.StatusInternalServerError { time.Sleep(time.Millisecond * 200) - } else if code == http.StatusOK || - code == http.StatusCreated || - code == http.StatusBadRequest { + } else if validHttpStatusCode[code] { b, err := ioutil.ReadAll(resp.Body) if err != nil { @@ -249,3 +247,19 @@ func buildValues(value string, ttl uint64) url.Values { return v } + +// convert key string to http path exclude version +// for example: key[foo] -> path[keys/foo] +// key[/] -> path[keys/] +func keyToPath(key string) string { + p := path.Join("keys", key) + + // corner case: if key is "/" or "//" ect + // path join will clear the tailing "/" + // we need to add it back + if p == "keys" { + p = "keys/" + } + + return p +} diff --git a/third_party/github.com/coreos/go-etcd/etcd/response.go b/third_party/github.com/coreos/go-etcd/etcd/response.go index 746a7c6e5..a72c2d8d7 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/response.go +++ b/third_party/github.com/coreos/go-etcd/etcd/response.go @@ -3,6 +3,7 @@ package etcd import ( "encoding/json" "net/http" + "strconv" "time" ) @@ -19,8 +20,19 @@ type RawResponse struct { Header http.Header } +var ( + validHttpStatusCode = map[int]bool{ + http.StatusCreated: true, + http.StatusOK: true, + http.StatusBadRequest: true, + http.StatusNotFound: true, + http.StatusPreconditionFailed: true, + http.StatusForbidden: true, + } +) + func (rr *RawResponse) toResponse() (*Response, error) { - if rr.StatusCode == http.StatusBadRequest { + if rr.StatusCode != http.StatusOK && rr.StatusCode != http.StatusCreated { return nil, handleError(rr.Body) } @@ -32,17 +44,25 @@ func (rr *RawResponse) toResponse() (*Response, error) { return nil, err } + // attach index and term to response + resp.EtcdIndex, _ = strconv.ParseUint(rr.Header.Get("X-Etcd-Index"), 10, 64) + resp.RaftIndex, _ = strconv.ParseUint(rr.Header.Get("X-Raft-Index"), 10, 64) + resp.RaftTerm, _ = strconv.ParseUint(rr.Header.Get("X-Raft-Term"), 10, 64) + return resp, nil } type Response struct { - Action string `json:"action"` - Node *Node `json:"node,omitempty"` + Action string `json:"action"` + Node *Node `json:"node"` + PrevNode *Node `json:"prevNode,omitempty"` + EtcdIndex uint64 `json:"etcdIndex"` + RaftIndex uint64 `json:"raftIndex"` + RaftTerm uint64 `json:"raftTerm"` } type Node struct { Key string `json:"key, omitempty"` - PrevValue string `json:"prevValue,omitempty"` Value string `json:"value,omitempty"` Dir bool `json:"dir,omitempty"` Expiration *time.Time `json:"expiration,omitempty"` diff --git a/third_party/github.com/coreos/go-etcd/etcd/set_update_create.go b/third_party/github.com/coreos/go-etcd/etcd/set_update_create.go index 8558db1ac..ab420d8fe 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/set_update_create.go +++ b/third_party/github.com/coreos/go-etcd/etcd/set_update_create.go @@ -1,6 +1,21 @@ package etcd -// SetDir sets the given key to a directory. +// Set sets the given key to the given value. +// It will create a new key value pair or replace the old one. +// It will not replace a existing directory. +func (c *Client) Set(key string, value string, ttl uint64) (*Response, error) { + raw, err := c.RawSet(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.toResponse() +} + +// Set sets the given key to a directory. +// It will create a new directory or replace the old key value pair by a directory. +// It will not replace a existing directory. func (c *Client) SetDir(key string, ttl uint64) (*Response, error) { raw, err := c.RawSetDir(key, ttl) @@ -11,19 +26,7 @@ func (c *Client) SetDir(key string, ttl uint64) (*Response, error) { return raw.toResponse() } -// UpdateDir updates the given key to a directory. It succeeds only if the -// given key already exists. -func (c *Client) UpdateDir(key string, ttl uint64) (*Response, error) { - raw, err := c.RawUpdateDir(key, ttl) - - if err != nil { - return nil, err - } - - return raw.toResponse() -} - -// UpdateDir creates a directory under the given key. It succeeds only if +// CreateDir creates a directory. It succeeds only if // the given key does not yet exist. func (c *Client) CreateDir(key string, ttl uint64) (*Response, error) { raw, err := c.RawCreateDir(key, ttl) @@ -35,21 +38,10 @@ func (c *Client) CreateDir(key string, ttl uint64) (*Response, error) { return raw.toResponse() } -// Set sets the given key to the given value. -func (c *Client) Set(key string, value string, ttl uint64) (*Response, error) { - raw, err := c.RawSet(key, value, ttl) - - if err != nil { - return nil, err - } - - return raw.toResponse() -} - -// Update updates the given key to the given value. It succeeds only if the +// UpdateDir updates the given directory. It succeeds only if the // given key already exists. -func (c *Client) Update(key string, value string, ttl uint64) (*Response, error) { - raw, err := c.RawUpdate(key, value, ttl) +func (c *Client) UpdateDir(key string, ttl uint64) (*Response, error) { + raw, err := c.RawUpdateDir(key, ttl) if err != nil { return nil, err @@ -70,13 +62,22 @@ func (c *Client) Create(key string, value string, ttl uint64) (*Response, error) return raw.toResponse() } -func (c *Client) RawSetDir(key string, ttl uint64) (*RawResponse, error) { - return c.put(key, "", ttl, nil) +// Update updates the given key to the given value. It succeeds only if the +// given key already exists. +func (c *Client) Update(key string, value string, ttl uint64) (*Response, error) { + raw, err := c.RawUpdate(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.toResponse() } func (c *Client) RawUpdateDir(key string, ttl uint64) (*RawResponse, error) { ops := options{ "prevExist": true, + "dir": true, } return c.put(key, "", ttl, ops) @@ -85,6 +86,7 @@ func (c *Client) RawUpdateDir(key string, ttl uint64) (*RawResponse, error) { func (c *Client) RawCreateDir(key string, ttl uint64) (*RawResponse, error) { ops := options{ "prevExist": false, + "dir": true, } return c.put(key, "", ttl, ops) @@ -94,6 +96,14 @@ func (c *Client) RawSet(key string, value string, ttl uint64) (*RawResponse, err return c.put(key, value, ttl, nil) } +func (c *Client) RawSetDir(key string, ttl uint64) (*RawResponse, error) { + ops := options{ + "dir": true, + } + + return c.put(key, "", ttl, ops) +} + func (c *Client) RawUpdate(key string, value string, ttl uint64) (*RawResponse, error) { ops := options{ "prevExist": true, diff --git a/third_party/github.com/coreos/go-etcd/etcd/set_update_create_test.go b/third_party/github.com/coreos/go-etcd/etcd/set_update_create_test.go index 215622b9e..2003c59bb 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/set_update_create_test.go +++ b/third_party/github.com/coreos/go-etcd/etcd/set_update_create_test.go @@ -17,15 +17,20 @@ func TestSet(t *testing.T) { if resp.Node.Key != "/foo" || resp.Node.Value != "bar" || resp.Node.TTL != 5 { t.Fatalf("Set 1 failed: %#v", resp) } + if resp.PrevNode != nil { + t.Fatalf("Set 1 PrevNode failed: %#v", resp) + } resp, err = c.Set("foo", "bar2", 5) if err != nil { t.Fatal(err) } - if !(resp.Node.Key == "/foo" && resp.Node.Value == "bar2" && - resp.Node.PrevValue == "bar" && resp.Node.TTL == 5) { + if !(resp.Node.Key == "/foo" && resp.Node.Value == "bar2" && resp.Node.TTL == 5) { t.Fatalf("Set 2 failed: %#v", resp) } + if resp.PrevNode.Key != "/foo" || resp.PrevNode.Value != "bar" || resp.Node.TTL != 5 { + t.Fatalf("Set 2 PrevNode failed: %#v", resp) + } } func TestUpdate(t *testing.T) { @@ -47,10 +52,12 @@ func TestUpdate(t *testing.T) { t.Fatal(err) } - if !(resp.Action == "update" && resp.Node.Key == "/foo" && - resp.Node.PrevValue == "bar" && resp.Node.TTL == 5) { + if !(resp.Action == "update" && resp.Node.Key == "/foo" && resp.Node.TTL == 5) { t.Fatalf("Update 1 failed: %#v", resp) } + if !(resp.PrevNode.Key == "/foo" && resp.PrevNode.Value == "bar" && resp.Node.TTL == 5) { + t.Fatalf("Update 1 prevValue failed: %#v", resp) + } // This should fail because the key does not exist. resp, err = c.Update("nonexistent", "whatever", 5) @@ -76,9 +83,12 @@ func TestCreate(t *testing.T) { } if !(resp.Action == "create" && resp.Node.Key == newKey && - resp.Node.Value == newValue && resp.Node.PrevValue == "" && resp.Node.TTL == 5) { + resp.Node.Value == newValue && resp.Node.TTL == 5) { t.Fatalf("Create 1 failed: %#v", resp) } + if resp.PrevNode != nil { + t.Fatalf("Create 1 PrevNode failed: %#v", resp) + } // This should fail, because the key is already there resp, err = c.Create(newKey, newValue, 5) @@ -95,16 +105,19 @@ func TestSetDir(t *testing.T) { c.Delete("fooDir", true) }() - resp, err := c.SetDir("fooDir", 5) + resp, err := c.CreateDir("fooDir", 5) if err != nil { t.Fatal(err) } if !(resp.Node.Key == "/fooDir" && resp.Node.Value == "" && resp.Node.TTL == 5) { t.Fatalf("SetDir 1 failed: %#v", resp) } + if resp.PrevNode != nil { + t.Fatalf("SetDir 1 PrevNode failed: %#v", resp) + } // This should fail because /fooDir already points to a directory - resp, err = c.SetDir("/fooDir", 5) + resp, err = c.CreateDir("/fooDir", 5) if err == nil { t.Fatalf("fooDir already points to a directory, so SetDir should have failed."+ "The response was: %#v", resp) @@ -116,12 +129,15 @@ func TestSetDir(t *testing.T) { } // This should succeed + // It should replace the key resp, err = c.SetDir("foo", 5) if err != nil { t.Fatal(err) } - if !(resp.Node.Key == "/foo" && resp.Node.Value == "" && - resp.Node.PrevValue == "bar" && resp.Node.TTL == 5) { + if !(resp.Node.Key == "/foo" && resp.Node.Value == "" && resp.Node.TTL == 5) { + t.Fatalf("SetDir 2 failed: %#v", resp) + } + if !(resp.PrevNode.Key == "/foo" && resp.PrevNode.Value == "bar" && resp.PrevNode.TTL == 5) { t.Fatalf("SetDir 2 failed: %#v", resp) } } @@ -132,7 +148,7 @@ func TestUpdateDir(t *testing.T) { c.Delete("fooDir", true) }() - resp, err := c.SetDir("fooDir", 5) + resp, err := c.CreateDir("fooDir", 5) if err != nil { t.Fatal(err) } @@ -144,9 +160,12 @@ func TestUpdateDir(t *testing.T) { } if !(resp.Action == "update" && resp.Node.Key == "/fooDir" && - resp.Node.Value == "" && resp.Node.PrevValue == "" && resp.Node.TTL == 5) { + resp.Node.Value == "" && resp.Node.TTL == 5) { t.Fatalf("UpdateDir 1 failed: %#v", resp) } + if !(resp.PrevNode.Key == "/fooDir" && resp.PrevNode.Dir == true && resp.PrevNode.TTL == 5) { + t.Fatalf("UpdateDir 1 PrevNode failed: %#v", resp) + } // This should fail because the key does not exist. resp, err = c.UpdateDir("nonexistentDir", 5) @@ -169,9 +188,12 @@ func TestCreateDir(t *testing.T) { } if !(resp.Action == "create" && resp.Node.Key == "/fooDir" && - resp.Node.Value == "" && resp.Node.PrevValue == "" && resp.Node.TTL == 5) { + resp.Node.Value == "" && resp.Node.TTL == 5) { t.Fatalf("CreateDir 1 failed: %#v", resp) } + if resp.PrevNode != nil { + t.Fatalf("CreateDir 1 PrevNode failed: %#v", resp) + } // This should fail, because the key is already there resp, err = c.CreateDir("fooDir", 5) diff --git a/third_party/github.com/coreos/go-log/log/commands.go b/third_party/github.com/coreos/go-log/log/commands.go index 94dc9e152..f39fdef97 100644 --- a/third_party/github.com/coreos/go-log/log/commands.go +++ b/third_party/github.com/coreos/go-log/log/commands.go @@ -1,4 +1,5 @@ package log + // Copyright 2013, CoreOS, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -42,7 +43,6 @@ func (logger *Logger) Logf(priority Priority, format string, v ...interface{}) { logger.Log(priority, fmt.Sprintf(format, v...)) } - func (logger *Logger) Emergency(v ...interface{}) { logger.Log(PriEmerg, v...) } @@ -99,7 +99,6 @@ func (logger *Logger) Debugf(format string, v ...interface{}) { logger.Log(PriDebug, fmt.Sprintf(format, v...)) } - func Emergency(v ...interface{}) { defaultLogger.Log(PriEmerg, v...) } @@ -158,57 +157,56 @@ func Debugf(format string, v ...interface{}) { // Standard library log functions -func (logger *Logger)Fatalln (v ...interface{}) { +func (logger *Logger) Fatalln(v ...interface{}) { logger.Log(PriCrit, v...) os.Exit(1) } -func (logger *Logger)Fatalf (format string, v ...interface{}) { +func (logger *Logger) Fatalf(format string, v ...interface{}) { logger.Logf(PriCrit, format, v...) os.Exit(1) } -func (logger *Logger)Panicln (v ...interface{}) { +func (logger *Logger) Panicln(v ...interface{}) { s := fmt.Sprint(v...) logger.Log(PriErr, s) panic(s) } -func (logger *Logger)Panicf (format string, v ...interface{}) { +func (logger *Logger) Panicf(format string, v ...interface{}) { s := fmt.Sprintf(format, v...) logger.Log(PriErr, s) panic(s) } -func (logger *Logger)Println (v ...interface{}) { +func (logger *Logger) Println(v ...interface{}) { logger.Log(PriInfo, v...) } -func (logger *Logger)Printf (format string, v ...interface{}) { +func (logger *Logger) Printf(format string, v ...interface{}) { logger.Logf(PriInfo, format, v...) } - -func Fatalln (v ...interface{}) { +func Fatalln(v ...interface{}) { defaultLogger.Log(PriCrit, v...) os.Exit(1) } -func Fatalf (format string, v ...interface{}) { +func Fatalf(format string, v ...interface{}) { defaultLogger.Logf(PriCrit, format, v...) os.Exit(1) } -func Panicln (v ...interface{}) { +func Panicln(v ...interface{}) { s := fmt.Sprint(v...) defaultLogger.Log(PriErr, s) panic(s) } -func Panicf (format string, v ...interface{}) { +func Panicf(format string, v ...interface{}) { s := fmt.Sprintf(format, v...) defaultLogger.Log(PriErr, s) panic(s) } -func Println (v ...interface{}) { +func Println(v ...interface{}) { defaultLogger.Log(PriInfo, v...) } -func Printf (format string, v ...interface{}) { +func Printf(format string, v ...interface{}) { defaultLogger.Logf(PriInfo, format, v...) } diff --git a/third_party/github.com/coreos/go-log/log/fields.go b/third_party/github.com/coreos/go-log/log/fields.go index e8d9698a0..b04edc8eb 100644 --- a/third_party/github.com/coreos/go-log/log/fields.go +++ b/third_party/github.com/coreos/go-log/log/fields.go @@ -1,4 +1,5 @@ package log + // Copyright 2013, CoreOS, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/third_party/github.com/coreos/go-log/log/logger.go b/third_party/github.com/coreos/go-log/log/logger.go index 2089a11f8..8c3b86c1d 100644 --- a/third_party/github.com/coreos/go-log/log/logger.go +++ b/third_party/github.com/coreos/go-log/log/logger.go @@ -1,4 +1,5 @@ package log + // Copyright 2013, CoreOS, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/third_party/github.com/coreos/go-log/log/priority.go b/third_party/github.com/coreos/go-log/log/priority.go index ac73fc8a4..c169d6869 100644 --- a/third_party/github.com/coreos/go-log/log/priority.go +++ b/third_party/github.com/coreos/go-log/log/priority.go @@ -1,4 +1,5 @@ package log + // Copyright 2013, CoreOS, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/third_party/github.com/coreos/go-log/log/sinks.go b/third_party/github.com/coreos/go-log/log/sinks.go index a41f3365d..bdf1e41f1 100644 --- a/third_party/github.com/coreos/go-log/log/sinks.go +++ b/third_party/github.com/coreos/go-log/log/sinks.go @@ -1,4 +1,5 @@ package log + // Copyright 2013, CoreOS, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/third_party/github.com/coreos/go-systemd/README.md b/third_party/github.com/coreos/go-systemd/README.md index 9b8a2f83f..2adbac515 100644 --- a/third_party/github.com/coreos/go-systemd/README.md +++ b/third_party/github.com/coreos/go-systemd/README.md @@ -1,3 +1,8 @@ # go-systemd Go bindings to systemd socket activation, journal and D-BUS APIs. + +## Socket Activation + +See an example in `examples/activation/httpserver.go`. For easy debugging use +`/usr/lib/systemd/systemd-activate` diff --git a/third_party/github.com/coreos/go-systemd/activation/files.go b/third_party/github.com/coreos/go-systemd/activation/files.go index a05dbcc91..3f323d820 100644 --- a/third_party/github.com/coreos/go-systemd/activation/files.go +++ b/third_party/github.com/coreos/go-systemd/activation/files.go @@ -29,7 +29,7 @@ func Files(unsetEnv bool) []*os.File { if err != nil || nfds == 0 { return nil } - files := []*os.File(nil) + var files []*os.File for fd := listenFdsStart; fd < listenFdsStart+nfds; fd++ { syscall.CloseOnExec(fd) files = append(files, os.NewFile(uintptr(fd), "LISTEN_FD_"+strconv.Itoa(fd))) diff --git a/third_party/github.com/coreos/go-systemd/activation/files_test.go b/third_party/github.com/coreos/go-systemd/activation/files_test.go new file mode 100644 index 000000000..cee919e76 --- /dev/null +++ b/third_party/github.com/coreos/go-systemd/activation/files_test.go @@ -0,0 +1,68 @@ +package activation + +import ( + "bytes" + "io" + "os" + "os/exec" + "testing" +) + +// correctStringWritten fails the text if the correct string wasn't written +// to the other side of the pipe. +func correctStringWritten(t *testing.T, r *os.File, expected string) bool { + bytes := make([]byte, len(expected)) + io.ReadAtLeast(r, bytes, len(expected)) + + if string(bytes) != expected { + t.Fatalf("Unexpected string %s", string(bytes)) + } + + return true +} + +// TestActivation forks out a copy of activation.go example and reads back two +// strings from the pipes that are passed in. +func TestActivation(t *testing.T) { + cmd := exec.Command("go", "run", "../examples/activation/activation.go") + + r1, w1, _ := os.Pipe() + r2, w2, _ := os.Pipe() + cmd.ExtraFiles = []*os.File{ + w1, + w2, + } + + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "LISTEN_FDS=2", "FIX_LISTEN_PID=1") + + err := cmd.Run() + if err != nil { + t.Fatalf(err.Error()) + } + + correctStringWritten(t, r1, "Hello world") + correctStringWritten(t, r2, "Goodbye world") +} + +func TestActivationNoFix(t *testing.T) { + cmd := exec.Command("go", "run", "../examples/activation/activation.go") + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "LISTEN_FDS=2") + + out, _ := cmd.CombinedOutput() + if bytes.Contains(out, []byte("No files")) == false { + t.Fatalf("Child didn't error out as expected") + } +} + +func TestActivationNoFiles(t *testing.T) { + cmd := exec.Command("go", "run", "../examples/activation/activation.go") + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "LISTEN_FDS=0", "FIX_LISTEN_PID=1") + + out, _ := cmd.CombinedOutput() + if bytes.Contains(out, []byte("No files")) == false { + t.Fatalf("Child didn't error out as expected") + } +} diff --git a/third_party/github.com/coreos/go-systemd/examples/activation/activation.go b/third_party/github.com/coreos/go-systemd/examples/activation/activation.go new file mode 100644 index 000000000..b3cf70ed8 --- /dev/null +++ b/third_party/github.com/coreos/go-systemd/examples/activation/activation.go @@ -0,0 +1,44 @@ +// Activation example used by the activation unit tests. +package main + +import ( + "fmt" + "os" + + "github.com/coreos/go-systemd/activation" +) + +func fixListenPid() { + if os.Getenv("FIX_LISTEN_PID") != "" { + // HACK: real systemd would set LISTEN_PID before exec'ing but + // this is too difficult in golang for the purpose of a test. + // Do not do this in real code. + os.Setenv("LISTEN_PID", fmt.Sprintf("%d", os.Getpid())) + } +} + +func main() { + fixListenPid() + + files := activation.Files(false) + + if len(files) == 0 { + panic("No files") + } + + if os.Getenv("LISTEN_PID") == "" || os.Getenv("LISTEN_FDS") == "" { + panic("Should not unset envs") + } + + files = activation.Files(true) + + if os.Getenv("LISTEN_PID") != "" || os.Getenv("LISTEN_FDS") != "" { + panic("Can not unset envs") + } + + // Write out the expected strings to the two pipes + files[0].Write([]byte("Hello world")) + files[1].Write([]byte("Goodbye world")) + + return +} diff --git a/third_party/github.com/coreos/go-systemd/examples/activation/httpserver/README.md b/third_party/github.com/coreos/go-systemd/examples/activation/httpserver/README.md new file mode 100644 index 000000000..91c7cbf13 --- /dev/null +++ b/third_party/github.com/coreos/go-systemd/examples/activation/httpserver/README.md @@ -0,0 +1 @@ +Example of using socket activation with systemd to serve a simple HTTP server on http://127.0.0.1:8076 diff --git a/third_party/github.com/coreos/go-systemd/examples/activation/httpserver/hello.service b/third_party/github.com/coreos/go-systemd/examples/activation/httpserver/hello.service new file mode 100644 index 000000000..c8dea0f6b --- /dev/null +++ b/third_party/github.com/coreos/go-systemd/examples/activation/httpserver/hello.service @@ -0,0 +1,11 @@ +[Unit] +Description=Hello World HTTP +Requires=network.target +After=multi-user.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/httpserver + +[Install] +WantedBy=multi-user.target diff --git a/third_party/github.com/coreos/go-systemd/examples/activation/httpserver/hello.socket b/third_party/github.com/coreos/go-systemd/examples/activation/httpserver/hello.socket new file mode 100644 index 000000000..723ed7ed9 --- /dev/null +++ b/third_party/github.com/coreos/go-systemd/examples/activation/httpserver/hello.socket @@ -0,0 +1,5 @@ +[Socket] +ListenStream=127.0.0.1:8076 + +[Install] +WantedBy=sockets.target diff --git a/third_party/github.com/coreos/go-systemd/examples/activation/httpserver/httpserver.go b/third_party/github.com/coreos/go-systemd/examples/activation/httpserver/httpserver.go new file mode 100644 index 000000000..db12ecf61 --- /dev/null +++ b/third_party/github.com/coreos/go-systemd/examples/activation/httpserver/httpserver.go @@ -0,0 +1,29 @@ +package main + +import ( + "io" + "net" + "net/http" + + "github.com/coreos/go-systemd/activation" +) + +func HelloServer(w http.ResponseWriter, req *http.Request) { + io.WriteString(w, "hello socket activated world!\n") +} + +func main() { + files := activation.Files(true) + + if len(files) != 1 { + panic("Unexpected number of socket activation fds") + } + + l, err := net.FileListener(files[0]) + if err != nil { + panic(err) + } + + http.HandleFunc("/", HelloServer) + http.Serve(l, nil) +} diff --git a/third_party/github.com/coreos/go-systemd/test b/third_party/github.com/coreos/go-systemd/test new file mode 100755 index 000000000..ce0d9a6c3 --- /dev/null +++ b/third_party/github.com/coreos/go-systemd/test @@ -0,0 +1,9 @@ +#!/bin/sh -e + +PKG="github.com/coreos/go-systemd" + +rm -R src +mkdir -p src/$(dirname $PKG) +ln -s ../../../ src/$PKG + +go test -v ${PKG}/activation diff --git a/third_party/github.com/coreos/raft/.travis.yml b/third_party/github.com/coreos/raft/.travis.yml index 5f70bdf4c..4e10f5ffe 100644 --- a/third_party/github.com/coreos/raft/.travis.yml +++ b/third_party/github.com/coreos/raft/.travis.yml @@ -1,8 +1,10 @@ language: go go: - - 1.1 + - 1.1.2 + - 1.2 install: + - go get github.com/stretchr/testify/assert - make dependencies diff --git a/third_party/github.com/coreos/raft/Makefile b/third_party/github.com/coreos/raft/Makefile index a5014509e..9847fe2de 100644 --- a/third_party/github.com/coreos/raft/Makefile +++ b/third_party/github.com/coreos/raft/Makefile @@ -8,6 +8,7 @@ dependencies: go get -d . test: + go test -i ./... go test -v ./... .PHONY: coverage dependencies test diff --git a/third_party/github.com/coreos/raft/README.md b/third_party/github.com/coreos/raft/README.md index 9d189ff6d..5486cc569 100644 --- a/third_party/github.com/coreos/raft/README.md +++ b/third_party/github.com/coreos/raft/README.md @@ -34,11 +34,27 @@ go-raft is under the MIT license. These projects are built on go-raft: -- [coreos/etcd](https://github.com/coreos/etcd) - A highly-available key value store for shared configuration and service discovery -- [benbjohnson/raftd](https://github.com/benbjohnson/raftd) - A reference implementation for using the go-raft library for distributed consensus. +- [coreos/etcd](https://github.com/coreos/etcd) - A highly-available key value store for shared configuration and service discovery. +- [goraft/raftd](https://github.com/goraft/raftd) - A reference implementation for using the go-raft library for distributed consensus. +- [skynetservices/skydns](https://github.com/skynetservices/skydns) - DNS for skynet or any other service discovery. +- [influxdb/influxdb](https://github.com/influxdb/influxdb) - An open-source, distributed, time series, events, and metrics database. If you have a project that you're using go-raft in, please add it to this README so others can see implementation examples. +## Contact and Resources + +- [raft-dev][raft-dev] is a mailing list for discussion about best practices + and implementation of Raft. Not goraft specific but helpful if you have + questions. +- [Slides from Ben's talk][bens-talk] which includes easy to understand + diagrams of leader election and replication +- The [Raft Consensus homepage][raft-home] has links to additional raft + implementations, slides to talks on Raft and general information + +[raft-home]: http://raftconsensus.github.io/ +[raft-dev]: https://groups.google.com/forum/#!forum/raft-dev +[bens-talk]: https://speakerdeck.com/benbjohnson/raft-the-understandable-distributed-consensus-protocol + ## The Raft Protocol This section provides a summary of the Raft protocol from a high level. @@ -83,6 +99,38 @@ By ensuring that this log is replicated identically between all the nodes in the Replicating the log under normal conditions is done by sending an `AppendEntries` RPC from the leader to each of the other servers in the cluster (called Peers). Each peer will append the entries from the leader through a 2-phase commit process which ensure that a majority of servers in the cluster have entries written to log. + +## Raft in Practice + +### Optimal Cluster Size + +The primary consideration when choosing the node count in your Raft cluster is the number of nodes that can simultaneously fail. +Because Raft requires a majority of nodes to be available to make progress, the number of node failures the cluster can tolerate is `(n / 2) - 1`. + +This means that a 3-node cluster can tolerate 1 node failure. +If 2 nodes fail then the cluster cannot commit entries or elect a new leader so progress stops. +A 5-node cluster can tolerate 2 node failures. A 9-node cluster can tolerate 4 node failures. +It is unlikely that 4 nodes will simultaneously fail so clusters larger than 9 nodes are not common. + +Another consideration is performance. +The leader must replicate log entries for each follower node so CPU and networking resources can quickly be bottlenecked under stress in a large cluster. + + +### Scaling Raft + +Once you grow beyond the maximum size of your cluster there are a few options for scaling Raft: + +1. *Core nodes with dumb replication.* + This option requires you to maintain a small cluster (e.g. 5 nodes) that is involved in the Raft process and then replicate only committed log entries to the remaining nodes in the cluster. + This works well if you have reads in your system that can be stale. + +2. *Sharding.* + This option requires that you segment your data into different clusters. + This option works well if you need very strong consistency and therefore need to read and write heavily from the leader. + +If you have a very large cluster that you need to replicate to using Option 1 then you may want to look at performing hierarchical replication so that nodes can better share the load. + + ## History Ben Johnson started this library for use in his behavioral analytics database called [Sky](https://github.com/skydb/sky). diff --git a/third_party/github.com/coreos/raft/append_entries_request_test.go b/third_party/github.com/coreos/raft/append_entries_request_test.go index d8cbce735..34a827f67 100644 --- a/third_party/github.com/coreos/raft/append_entries_request_test.go +++ b/third_party/github.com/coreos/raft/append_entries_request_test.go @@ -28,7 +28,7 @@ func createTestAppendEntriesRequest(entryCount int) (*AppendEntriesRequest, []by entries := make([]*LogEntry, 0) for i := 0; i < entryCount; i++ { command := &DefaultJoinCommand{Name: "localhost:1000"} - entry, _ := newLogEntry(nil, 1, 2, command) + entry, _ := newLogEntry(nil, nil, 1, 2, command) entries = append(entries, entry) } req := newAppendEntriesRequest(1, 1, 1, 1, "leader", entries) diff --git a/third_party/github.com/coreos/raft/command.go b/third_party/github.com/coreos/raft/command.go index 9188881b8..14341fa21 100644 --- a/third_party/github.com/coreos/raft/command.go +++ b/third_party/github.com/coreos/raft/command.go @@ -8,28 +8,25 @@ import ( "reflect" ) -//------------------------------------------------------------------------------ -// -// Globals -// -//------------------------------------------------------------------------------ - var commandTypes map[string]Command func init() { commandTypes = map[string]Command{} } -//------------------------------------------------------------------------------ -// -// Typedefs -// -//------------------------------------------------------------------------------ - -// A command represents an action to be taken on the replicated state machine. +// Command represents an action to be taken on the replicated state machine. type Command interface { CommandName() string - Apply(server Server) (interface{}, error) +} + +// CommandApply represents the interface to apply a command to the server. +type CommandApply interface { + Apply(Context) (interface{}, error) +} + +// deprecatedCommandApply represents the old interface to apply a command to the server. +type deprecatedCommandApply interface { + Apply(Server) (interface{}, error) } type CommandEncoder interface { @@ -37,16 +34,6 @@ type CommandEncoder interface { Decode(r io.Reader) error } -//------------------------------------------------------------------------------ -// -// Functions -// -//------------------------------------------------------------------------------ - -//-------------------------------------- -// Instantiation -//-------------------------------------- - // Creates a new instance of a command by name. func newCommand(name string, data []byte) (Command, error) { // Find the registered command. @@ -76,10 +63,6 @@ func newCommand(name string, data []byte) (Command, error) { return copy, nil } -//-------------------------------------- -// Registration -//-------------------------------------- - // Registers a command by storing a reference to an instance of it. func RegisterCommand(command Command) { if command == nil { diff --git a/third_party/github.com/coreos/raft/context.go b/third_party/github.com/coreos/raft/context.go new file mode 100644 index 000000000..bea7aafb7 --- /dev/null +++ b/third_party/github.com/coreos/raft/context.go @@ -0,0 +1,39 @@ +package raft + +// Context represents the current state of the server. It is passed into +// a command when the command is being applied since the server methods +// are locked. +type Context interface { + Server() Server + CurrentTerm() uint64 + CurrentIndex() uint64 + CommitIndex() uint64 +} + +// context is the concrete implementation of Context. +type context struct { + server Server + currentIndex uint64 + currentTerm uint64 + commitIndex uint64 +} + +// Server returns a reference to the server. +func (c *context) Server() Server { + return c.server +} + +// CurrentTerm returns current term the server is in. +func (c *context) CurrentTerm() uint64 { + return c.currentTerm +} + +// CurrentIndex returns current index the server is at. +func (c *context) CurrentIndex() uint64 { + return c.currentIndex +} + +// CommitIndex returns last commit index the server is at. +func (c *context) CommitIndex() uint64 { + return c.commitIndex +} diff --git a/third_party/github.com/coreos/raft/event.go b/third_party/github.com/coreos/raft/event.go new file mode 100644 index 000000000..649a1158c --- /dev/null +++ b/third_party/github.com/coreos/raft/event.go @@ -0,0 +1,58 @@ +package raft + +const ( + StateChangeEventType = "stateChange" + LeaderChangeEventType = "leaderChange" + TermChangeEventType = "termChange" + AddPeerEventType = "addPeer" + RemovePeerEventType = "removePeer" + + HeartbeatTimeoutEventType = "heartbeatTimeout" + ElectionTimeoutThresholdEventType = "electionTimeoutThreshold" +) + +// Event represents an action that occurred within the Raft library. +// Listeners can subscribe to event types by using the Server.AddEventListener() function. +type Event interface { + Type() string + Source() interface{} + Value() interface{} + PrevValue() interface{} +} + +// event is the concrete implementation of the Event interface. +type event struct { + typ string + source interface{} + value interface{} + prevValue interface{} +} + +// newEvent creates a new event. +func newEvent(typ string, value interface{}, prevValue interface{}) *event { + return &event{ + typ: typ, + value: value, + prevValue: prevValue, + } +} + +// Type returns the type of event that occurred. +func (e *event) Type() string { + return e.typ +} + +// Source returns the object that dispatched the event. +func (e *event) Source() interface{} { + return e.source +} + +// Value returns the current value associated with the event, if applicable. +func (e *event) Value() interface{} { + return e.value +} + +// PrevValue returns the previous value associated with the event, if applicable. +func (e *event) PrevValue() interface{} { + return e.prevValue +} diff --git a/third_party/github.com/coreos/raft/event_dispatcher.go b/third_party/github.com/coreos/raft/event_dispatcher.go new file mode 100644 index 000000000..e985a99fc --- /dev/null +++ b/third_party/github.com/coreos/raft/event_dispatcher.go @@ -0,0 +1,50 @@ +package raft + +import ( + "sync" +) + +// eventDispatcher is responsible for managing listeners for named events +// and dispatching event notifications to those listeners. +type eventDispatcher struct { + sync.RWMutex + source interface{} + listeners map[string]eventListeners +} + +// EventListener is a function that can receive event notifications. +type EventListener func(Event) + +// EventListeners represents a collection of individual listeners. +type eventListeners []EventListener + +// newEventDispatcher creates a new eventDispatcher instance. +func newEventDispatcher(source interface{}) *eventDispatcher { + return &eventDispatcher{ + source: source, + listeners: make(map[string]eventListeners), + } +} + +// AddEventListener adds a listener function for a given event type. +func (d *eventDispatcher) AddEventListener(typ string, listener EventListener) { + d.Lock() + defer d.Unlock() + d.listeners[typ] = append(d.listeners[typ], listener) +} + +// DispatchEvent dispatches an event. +func (d *eventDispatcher) DispatchEvent(e Event) { + d.RLock() + defer d.RUnlock() + + // Automatically set the event source. + if e, ok := e.(*event); ok { + e.source = d.source + } + + // Dispatch the event to all listeners. + for _, l := range d.listeners[e.Type()] { + l(e) + } +} diff --git a/third_party/github.com/coreos/raft/event_dispatcher_test.go b/third_party/github.com/coreos/raft/event_dispatcher_test.go new file mode 100644 index 000000000..affdaaab9 --- /dev/null +++ b/third_party/github.com/coreos/raft/event_dispatcher_test.go @@ -0,0 +1,45 @@ +package raft + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Ensure that we can listen and dispatch events. +func TestDispatchEvent(t *testing.T) { + var count int + dispatcher := newEventDispatcher(nil) + dispatcher.AddEventListener("foo", func(e Event) { + count += 1 + }) + dispatcher.AddEventListener("foo", func(e Event) { + count += 10 + }) + dispatcher.AddEventListener("bar", func(e Event) { + count += 100 + }) + dispatcher.DispatchEvent(&event{typ: "foo", value: nil, prevValue: nil}) + assert.Equal(t, 11, count) +} + +// Ensure that event is properly passed to listener. +func TestEventListener(t *testing.T) { + dispatcher := newEventDispatcher("X") + dispatcher.AddEventListener("foo", func(e Event) { + assert.Equal(t, "foo", e.Type()) + assert.Equal(t, "X", e.Source()) + assert.Equal(t, 10, e.Value()) + assert.Equal(t, 20, e.PrevValue()) + }) + dispatcher.DispatchEvent(&event{typ: "foo", value: 10, prevValue: 20}) +} + +// Benchmark the performance of event dispatch. +func BenchmarkEventDispatch(b *testing.B) { + dispatcher := newEventDispatcher(nil) + dispatcher.AddEventListener("xxx", func(e Event) {}) + for i := 0; i < b.N; i++ { + dispatcher.DispatchEvent(&event{typ: "foo", value: 10, prevValue: 20}) + } +} diff --git a/third_party/github.com/coreos/raft/http_transporter.go b/third_party/github.com/coreos/raft/http_transporter.go index 80aa3e74e..a74a5d628 100644 --- a/third_party/github.com/coreos/raft/http_transporter.go +++ b/third_party/github.com/coreos/raft/http_transporter.go @@ -23,6 +23,8 @@ type HTTPTransporter struct { prefix string appendEntriesPath string requestVotePath string + httpClient http.Client + Transport *http.Transport } type HTTPMuxer interface { @@ -37,12 +39,15 @@ type HTTPMuxer interface { // Creates a new HTTP transporter with the given path prefix. func NewHTTPTransporter(prefix string) *HTTPTransporter { - return &HTTPTransporter{ + t := &HTTPTransporter{ DisableKeepAlives: false, prefix: prefix, appendEntriesPath: fmt.Sprintf("%s%s", prefix, "/appendEntries"), requestVotePath: fmt.Sprintf("%s%s", prefix, "/requestVote"), + Transport: &http.Transport{DisableKeepAlives: false}, } + t.httpClient.Transport = t.Transport + return t } //------------------------------------------------------------------------------ @@ -97,8 +102,8 @@ func (t *HTTPTransporter) SendAppendEntriesRequest(server Server, peer *Peer, re url := fmt.Sprintf("%s%s", peer.ConnectionString, t.AppendEntriesPath()) traceln(server.Name(), "POST", url) - client := &http.Client{Transport: &http.Transport{DisableKeepAlives: t.DisableKeepAlives}} - httpResp, err := client.Post(url, "application/protobuf", &b) + t.Transport.ResponseHeaderTimeout = server.ElectionTimeout() + httpResp, err := t.httpClient.Post(url, "application/protobuf", &b) if httpResp == nil || err != nil { traceln("transporter.ae.response.error:", err) return nil @@ -125,8 +130,7 @@ func (t *HTTPTransporter) SendVoteRequest(server Server, peer *Peer, req *Reques url := fmt.Sprintf("%s%s", peer.ConnectionString, t.RequestVotePath()) traceln(server.Name(), "POST", url) - client := &http.Client{Transport: &http.Transport{DisableKeepAlives: t.DisableKeepAlives}} - httpResp, err := client.Post(url, "application/protobuf", &b) + httpResp, err := t.httpClient.Post(url, "application/protobuf", &b) if httpResp == nil || err != nil { traceln("transporter.rv.response.error:", err) return nil diff --git a/third_party/github.com/coreos/raft/join_command.go b/third_party/github.com/coreos/raft/join_command.go index 1a43db2c3..aa761b473 100644 --- a/third_party/github.com/coreos/raft/join_command.go +++ b/third_party/github.com/coreos/raft/join_command.go @@ -2,8 +2,7 @@ package raft // Join command interface type JoinCommand interface { - CommandName() string - Apply(server Server) (interface{}, error) + Command NodeName() string } diff --git a/third_party/github.com/coreos/raft/leave_command.go b/third_party/github.com/coreos/raft/leave_command.go index ca2f148cf..37ec46395 100644 --- a/third_party/github.com/coreos/raft/leave_command.go +++ b/third_party/github.com/coreos/raft/leave_command.go @@ -2,8 +2,7 @@ package raft // Leave command interface type LeaveCommand interface { - CommandName() string - Apply(server Server) (interface{}, error) + Command NodeName() string } diff --git a/third_party/github.com/coreos/raft/log.go b/third_party/github.com/coreos/raft/log.go index 757e4301f..f137ecf98 100644 --- a/third_party/github.com/coreos/raft/log.go +++ b/third_party/github.com/coreos/raft/log.go @@ -23,7 +23,6 @@ type Log struct { file *os.File path string entries []*LogEntry - results []*logResult commitIndex uint64 mutex sync.RWMutex startIndex uint64 // the index before the first entry in the Log entries @@ -74,11 +73,7 @@ func (l *Log) CommitIndex() uint64 { func (l *Log) currentIndex() uint64 { l.mutex.RLock() defer l.mutex.RUnlock() - - if len(l.entries) == 0 { - return l.startIndex - } - return l.entries[len(l.entries)-1].Index + return l.internalCurrentIndex() } // The current index in the log without locking @@ -166,7 +161,7 @@ func (l *Log) open(path string) error { // Read the file and decode entries. for { // Instantiate log entry and decode into it. - entry, _ := newLogEntry(l, 0, 0, nil) + entry, _ := newLogEntry(l, nil, 0, 0, nil) entry.Position, _ = l.file.Seek(0, os.SEEK_CUR) n, err := entry.decode(l.file) @@ -195,8 +190,6 @@ func (l *Log) open(path string) error { readBytes += int64(n) } - l.results = make([]*logResult, len(l.entries)) - debugln("open.log.recovery number of log ", len(l.entries)) return nil } @@ -211,7 +204,6 @@ func (l *Log) close() { l.file = nil } l.entries = make([]*LogEntry, 0) - l.results = make([]*logResult, 0) } //-------------------------------------- @@ -219,8 +211,8 @@ func (l *Log) close() { //-------------------------------------- // Creates a log entry associated with this log. -func (l *Log) createEntry(term uint64, command Command) (*LogEntry, error) { - return newLogEntry(l, l.nextIndex(), term, command) +func (l *Log) createEntry(term uint64, command Command, e *ev) (*LogEntry, error) { + return newLogEntry(l, e, l.nextIndex(), term, command) } // Retrieves an entry from the log. If the entry has been eliminated because @@ -280,35 +272,6 @@ func (l *Log) getEntriesAfter(index uint64, maxLogEntriesPerRequest uint64) ([]* } } -// Retrieves the return value and error for an entry. The result can only exist -// after the entry has been committed. -func (l *Log) getEntryResult(entry *LogEntry, clear bool) (interface{}, error) { - l.mutex.RLock() - defer l.mutex.RUnlock() - - if entry == nil { - panic("raft: Log entry required for error retrieval") - } - debugln("getEntryResult.result index: ", entry.Index-l.startIndex-1) - // If a result exists for the entry then return it with its error. - if entry.Index > l.startIndex && entry.Index <= l.startIndex+uint64(len(l.results)) { - if result := l.results[entry.Index-l.startIndex-1]; result != nil { - - // keep the records before remove it - returnValue, err := result.returnValue, result.err - - // Remove reference to result if it's being cleared after retrieval. - if clear { - result.returnValue = nil - } - - return returnValue, err - } - } - - return nil, nil -} - //-------------------------------------- // Commit //-------------------------------------- @@ -333,7 +296,7 @@ func (l *Log) commitInfo() (index uint64, term uint64) { return entry.Index, entry.Term } -// Retrieves the last index and term that has been committed to the log. +// Retrieves the last index and term that has been appended to the log. func (l *Log) lastInfo() (index uint64, term uint64) { l.mutex.RLock() defer l.mutex.RUnlock() @@ -366,8 +329,7 @@ func (l *Log) setCommitIndex(index uint64) error { // this is not error any more after limited the number of sending entries // commit up to what we already have if index > l.startIndex+uint64(len(l.entries)) { - debugln("raft.StartIndex", l.startIndex) - debugln("raft.Log: Commit index", index, "set back to ", l.startIndex+uint64(len(l.entries))) + debugln("raft.Log: Commit index", index, "set back to ", len(l.entries)) index = l.startIndex + uint64(len(l.entries)) } @@ -387,7 +349,6 @@ func (l *Log) setCommitIndex(index uint64) error { // follower 2 should reply success and let leader 3 update the committed index to 80 if index < l.commitIndex { - debugln("raft.Log: index", index, "committedIndex", l.commitIndex) return nil } @@ -408,7 +369,10 @@ func (l *Log) setCommitIndex(index uint64) error { // Apply the changes to the state machine and store the error code. returnValue, err := l.ApplyFunc(command) debugln("setCommitIndex.set.result index: ", entryIndex) - l.results[entryIndex] = &logResult{returnValue: returnValue, err: err} + if entry.event != nil { + entry.event.returnValue = returnValue + entry.event.c <- err + } } return nil } @@ -449,6 +413,14 @@ func (l *Log) truncate(index uint64, term uint64) error { debugln("log.truncate.clear") l.file.Truncate(0) l.file.Seek(0, os.SEEK_SET) + + // notify clients if this node is the previous leader + for _, entry := range l.entries { + if entry.event != nil { + entry.event.c <- errors.New("command failed to be committed due to node failure") + } + } + l.entries = []*LogEntry{} } else { // Do not truncate if the entry at index does not have the matching term. @@ -464,6 +436,15 @@ func (l *Log) truncate(index uint64, term uint64) error { position := l.entries[index-l.startIndex].Position l.file.Truncate(position) l.file.Seek(position, os.SEEK_SET) + + // notify clients if this node is the previous leader + for i := index - l.startIndex; i < uint64(len(l.entries)); i++ { + entry := l.entries[i] + if entry.event != nil { + entry.event.c <- errors.New("command failed to be committed due to node failure") + } + } + l.entries = l.entries[0 : index-l.startIndex] } } @@ -475,8 +456,7 @@ func (l *Log) truncate(index uint64, term uint64) error { // Append //-------------------------------------- -// Appends a series of entries to the log. These entries are not written to -// disk until setCommitIndex() is called. +// Appends a series of entries to the log. func (l *Log) appendEntries(entries []*LogEntry) error { l.mutex.Lock() defer l.mutex.Unlock() @@ -497,14 +477,20 @@ func (l *Log) appendEntries(entries []*LogEntry) error { startPosition += size } w.Flush() + err = l.file.Sync() + + if err != nil { + panic(err) + } return nil } -// Writes a single log entry to the end of the log. This function does not -// obtain a lock and should only be used internally. Use AppendEntries() and -// AppendEntry() to use it externally. +// Writes a single log entry to the end of the log. func (l *Log) appendEntry(entry *LogEntry) error { + l.mutex.Lock() + defer l.mutex.Unlock() + if l.file == nil { return errors.New("raft.Log: Log is not open") } @@ -530,7 +516,6 @@ func (l *Log) appendEntry(entry *LogEntry) error { // Append to entries list if stored on disk. l.entries = append(l.entries, entry) - l.results = append(l.results, nil) return nil } @@ -559,7 +544,6 @@ func (l *Log) writeEntry(entry *LogEntry, w io.Writer) (int64, error) { // Append to entries list if stored on disk. l.entries = append(l.entries, entry) - l.results = append(l.results, nil) return int64(size), nil } @@ -571,7 +555,6 @@ func (l *Log) writeEntry(entry *LogEntry, w io.Writer) (int64, error) { // compact the log before index (including index) func (l *Log) compact(index uint64, term uint64) error { var entries []*LogEntry - var results []*logResult l.mutex.Lock() defer l.mutex.Unlock() @@ -584,11 +567,9 @@ func (l *Log) compact(index uint64, term uint64) error { // we just recovery from on snapshot if index >= l.internalCurrentIndex() { entries = make([]*LogEntry, 0) - results = make([]*logResult, 0) } else { // get all log entries after index entries = l.entries[index-l.startIndex:] - results = l.results[index-l.startIndex:] } // create a new log file and add all the entries @@ -622,7 +603,6 @@ func (l *Log) compact(index uint64, term uint64) error { // compaction the in memory log l.entries = entries - l.results = results l.startIndex = index l.startTerm = term return nil diff --git a/third_party/github.com/coreos/raft/log_entry.go b/third_party/github.com/coreos/raft/log_entry.go index ff344d048..94cfb94e2 100644 --- a/third_party/github.com/coreos/raft/log_entry.go +++ b/third_party/github.com/coreos/raft/log_entry.go @@ -17,11 +17,11 @@ type LogEntry struct { CommandName string Command []byte Position int64 // position in the log file - commit chan bool + event *ev } // Creates a new log entry associated with a log. -func newLogEntry(log *Log, index uint64, term uint64, command Command) (*LogEntry, error) { +func newLogEntry(log *Log, event *ev, index uint64, term uint64, command Command) (*LogEntry, error) { var buf bytes.Buffer var commandName string if command != nil { @@ -41,7 +41,7 @@ func newLogEntry(log *Log, index uint64, term uint64, command Command) (*LogEntr Term: term, CommandName: commandName, Command: buf.Bytes(), - commit: make(chan bool, 5), + event: event, } return e, nil diff --git a/third_party/github.com/coreos/raft/log_test.go b/third_party/github.com/coreos/raft/log_test.go index e890090c3..e2b53c846 100644 --- a/third_party/github.com/coreos/raft/log_test.go +++ b/third_party/github.com/coreos/raft/log_test.go @@ -30,15 +30,15 @@ func TestLogNewLog(t *testing.T) { defer log.close() defer os.Remove(path) - e, _ := newLogEntry(log, 1, 1, &testCommand1{Val: "foo", I: 20}) + e, _ := newLogEntry(log, nil, 1, 1, &testCommand1{Val: "foo", I: 20}) if err := log.appendEntry(e); err != nil { t.Fatalf("Unable to append: %v", err) } - e, _ = newLogEntry(log, 2, 1, &testCommand2{X: 100}) + e, _ = newLogEntry(log, nil, 2, 1, &testCommand2{X: 100}) if err := log.appendEntry(e); err != nil { t.Fatalf("Unable to append: %v", err) } - e, _ = newLogEntry(log, 3, 2, &testCommand1{Val: "bar", I: 0}) + e, _ = newLogEntry(log, nil, 3, 2, &testCommand1{Val: "bar", I: 0}) if err := log.appendEntry(e); err != nil { t.Fatalf("Unable to append: %v", err) } @@ -63,9 +63,9 @@ func TestLogNewLog(t *testing.T) { // Ensure that we can decode and encode to an existing log. func TestLogExistingLog(t *testing.T) { tmpLog := newLog() - e0, _ := newLogEntry(tmpLog, 1, 1, &testCommand1{Val: "foo", I: 20}) - e1, _ := newLogEntry(tmpLog, 2, 1, &testCommand2{X: 100}) - e2, _ := newLogEntry(tmpLog, 3, 2, &testCommand1{Val: "bar", I: 0}) + e0, _ := newLogEntry(tmpLog, nil, 1, 1, &testCommand1{Val: "foo", I: 20}) + e1, _ := newLogEntry(tmpLog, nil, 2, 1, &testCommand2{X: 100}) + e2, _ := newLogEntry(tmpLog, nil, 3, 2, &testCommand1{Val: "bar", I: 0}) log, path := setupLog([]*LogEntry{e0, e1, e2}) defer log.close() defer os.Remove(path) @@ -88,9 +88,9 @@ func TestLogExistingLog(t *testing.T) { // Ensure that we can check the contents of the log by index/term. func TestLogContainsEntries(t *testing.T) { tmpLog := newLog() - e0, _ := newLogEntry(tmpLog, 1, 1, &testCommand1{Val: "foo", I: 20}) - e1, _ := newLogEntry(tmpLog, 2, 1, &testCommand2{X: 100}) - e2, _ := newLogEntry(tmpLog, 3, 2, &testCommand1{Val: "bar", I: 0}) + e0, _ := newLogEntry(tmpLog, nil, 1, 1, &testCommand1{Val: "foo", I: 20}) + e1, _ := newLogEntry(tmpLog, nil, 2, 1, &testCommand2{X: 100}) + e2, _ := newLogEntry(tmpLog, nil, 3, 2, &testCommand1{Val: "bar", I: 0}) log, path := setupLog([]*LogEntry{e0, e1, e2}) defer log.close() defer os.Remove(path) @@ -115,8 +115,8 @@ func TestLogContainsEntries(t *testing.T) { // Ensure that we can recover from an incomplete/corrupt log and continue logging. func TestLogRecovery(t *testing.T) { tmpLog := newLog() - e0, _ := newLogEntry(tmpLog, 1, 1, &testCommand1{Val: "foo", I: 20}) - e1, _ := newLogEntry(tmpLog, 2, 1, &testCommand2{X: 100}) + e0, _ := newLogEntry(tmpLog, nil, 1, 1, &testCommand1{Val: "foo", I: 20}) + e1, _ := newLogEntry(tmpLog, nil, 2, 1, &testCommand2{X: 100}) f, _ := ioutil.TempFile("", "raft-log-") e0.encode(f) @@ -134,7 +134,7 @@ func TestLogRecovery(t *testing.T) { defer log.close() defer os.Remove(f.Name()) - e, _ := newLogEntry(log, 3, 2, &testCommand1{Val: "bat", I: -5}) + e, _ := newLogEntry(log, nil, 3, 2, &testCommand1{Val: "bat", I: -5}) if err := log.appendEntry(e); err != nil { t.Fatalf("Unable to append: %v", err) } @@ -167,15 +167,15 @@ func TestLogTruncate(t *testing.T) { defer os.Remove(path) - entry1, _ := newLogEntry(log, 1, 1, &testCommand1{Val: "foo", I: 20}) + entry1, _ := newLogEntry(log, nil, 1, 1, &testCommand1{Val: "foo", I: 20}) if err := log.appendEntry(entry1); err != nil { t.Fatalf("Unable to append: %v", err) } - entry2, _ := newLogEntry(log, 2, 1, &testCommand2{X: 100}) + entry2, _ := newLogEntry(log, nil, 2, 1, &testCommand2{X: 100}) if err := log.appendEntry(entry2); err != nil { t.Fatalf("Unable to append: %v", err) } - entry3, _ := newLogEntry(log, 3, 2, &testCommand1{Val: "bar", I: 0}) + entry3, _ := newLogEntry(log, nil, 3, 2, &testCommand1{Val: "bar", I: 0}) if err := log.appendEntry(entry3); err != nil { t.Fatalf("Unable to append: %v", err) } diff --git a/third_party/github.com/coreos/raft/peer.go b/third_party/github.com/coreos/raft/peer.go index 516b535e4..1a0f20583 100644 --- a/third_party/github.com/coreos/raft/peer.go +++ b/third_party/github.com/coreos/raft/peer.go @@ -126,6 +126,8 @@ func (p *Peer) heartbeat(c chan bool) { c <- true + ticker := time.Tick(p.heartbeatTimeout) + debugln("peer.heartbeat: ", p.Name, p.heartbeatTimeout) for { @@ -142,7 +144,7 @@ func (p *Peer) heartbeat(c chan bool) { return } - case <-time.After(p.heartbeatTimeout): + case <-ticker: p.flush() } } @@ -174,6 +176,7 @@ func (p *Peer) sendAppendEntriesRequest(req *AppendEntriesRequest) { resp := p.server.Transporter().SendAppendEntriesRequest(p.server, p, req) if resp == nil { + p.server.DispatchEvent(newEvent(HeartbeatTimeoutEventType, p, nil)) debugln("peer.flush.timeout: ", p.server.Name(), "->", p.Name) return } diff --git a/third_party/github.com/coreos/raft/server.go b/third_party/github.com/coreos/raft/server.go index d7dec5b23..4513196da 100644 --- a/third_party/github.com/coreos/raft/server.go +++ b/third_party/github.com/coreos/raft/server.go @@ -37,6 +37,11 @@ const ( DefaultElectionTimeout = 150 * time.Millisecond ) +// ElectionTimeoutThresholdPercent specifies the threshold at which the server +// will dispatch warning events that the heartbeat RTT is too close to the +// election timeout. +const ElectionTimeoutThresholdPercent = 0.8 + var stopValue interface{} //------------------------------------------------------------------------------ @@ -94,9 +99,12 @@ type Server interface { Do(command Command) (interface{}, error) TakeSnapshot() error LoadSnapshot() error + AddEventListener(string, EventListener) } type server struct { + *eventDispatcher + name string path string state string @@ -111,7 +119,7 @@ type server struct { mutex sync.RWMutex syncedPeer map[string]bool - c chan *event + c chan *ev electionTimeout time.Duration heartbeatTimeout time.Duration @@ -123,8 +131,8 @@ type server struct { connectionString string } -// An event to be processed by the server's event loop. -type event struct { +// An internal event to be processed by the server's event loop. +type ev struct { target interface{} returnValue interface{} c chan error @@ -136,8 +144,12 @@ type event struct { // //------------------------------------------------------------------------------ -// Creates a new server with a log at the given path. -func NewServer(name string, path string, transporter Transporter, stateMachine StateMachine, context interface{}, connectionString string) (Server, error) { +// Creates a new server with a log at the given path. transporter must +// not be nil. stateMachine can be nil if snapshotting and log +// compaction is to be disabled. context can be anything (including nil) +// and is not used by the raft package except returned by +// Server.Context(). connectionString can be anything. +func NewServer(name string, path string, transporter Transporter, stateMachine StateMachine, ctx interface{}, connectionString string) (Server, error) { if name == "" { return nil, errors.New("raft.Server: Name cannot be blank") } @@ -150,21 +162,33 @@ func NewServer(name string, path string, transporter Transporter, stateMachine S path: path, transporter: transporter, stateMachine: stateMachine, - context: context, + context: ctx, state: Stopped, peers: make(map[string]*Peer), log: newLog(), - c: make(chan *event, 256), + c: make(chan *ev, 256), electionTimeout: DefaultElectionTimeout, heartbeatTimeout: DefaultHeartbeatTimeout, maxLogEntriesPerRequest: MaxLogEntriesPerRequest, connectionString: connectionString, } + s.eventDispatcher = newEventDispatcher(s) // Setup apply function. s.log.ApplyFunc = func(c Command) (interface{}, error) { - result, err := c.Apply(s) - return result, err + switch c := c.(type) { + case CommandApply: + return c.Apply(&context{ + server: s, + currentTerm: s.currentTerm, + currentIndex: s.log.internalCurrentIndex(), + commitIndex: s.log.commitIndex, + }) + case deprecatedCommandApply: + return c.Apply(s) + default: + return nil, fmt.Errorf("Command does not implement Apply()") + } } return s, nil @@ -246,19 +270,37 @@ func (s *server) State() string { func (s *server) setState(state string) { s.mutex.Lock() defer s.mutex.Unlock() + + // Temporarily store previous values. + prevState := s.state + prevLeader := s.leader + + // Update state and leader. s.state = state if state == Leader { s.leader = s.Name() } + + // Dispatch state and leader change events. + if prevState != state { + s.DispatchEvent(newEvent(StateChangeEventType, s.state, prevState)) + } + if prevLeader != s.leader { + s.DispatchEvent(newEvent(LeaderChangeEventType, s.leader, prevLeader)) + } } // Retrieves the current term of the server. func (s *server) Term() uint64 { + s.mutex.RLock() + defer s.mutex.RUnlock() return s.currentTerm } // Retrieves the current commit index of the server. func (s *server) CommitIndex() uint64 { + s.log.mutex.RLock() + defer s.log.mutex.RUnlock() return s.log.commitIndex } @@ -375,7 +417,7 @@ func init() { func (s *server) Start() error { // Exit if the server is already running. - if s.state != Stopped { + if s.State() != Stopped { return errors.New("raft.Server: Server already running") } @@ -443,22 +485,34 @@ func (s *server) setCurrentTerm(term uint64, leaderName string, append bool) { s.mutex.Lock() defer s.mutex.Unlock() - // update the term and clear vote for + // Store previous values temporarily. + prevState := s.state + prevTerm := s.currentTerm + prevLeader := s.leader + if term > s.currentTerm { + // update the term and clear vote for s.state = Follower s.currentTerm = term s.leader = leaderName s.votedFor = "" - return - } - - // discover new leader when candidate - // save leader name when follower - if term == s.currentTerm && s.state != Leader && append { + } else if term == s.currentTerm && s.state != Leader && append { + // discover new leader when candidate + // save leader name when follower s.state = Follower s.leader = leaderName } + // Dispatch change events. + if prevState != s.state { + s.DispatchEvent(newEvent(StateChangeEventType, s.state, prevState)) + } + if prevLeader != s.leader { + s.DispatchEvent(newEvent(LeaderChangeEventType, s.leader, prevLeader)) + } + if prevTerm != s.currentTerm { + s.DispatchEvent(newEvent(TermChangeEventType, s.currentTerm, prevTerm)) + } } //-------------------------------------- @@ -512,8 +566,8 @@ func (s *server) send(value interface{}) (interface{}, error) { return event.returnValue, err } -func (s *server) sendAsync(value interface{}) *event { - event := &event{target: value, c: make(chan error, 1)} +func (s *server) sendAsync(value interface{}) *ev { + event := &ev{target: value, c: make(chan error, 1)} s.c <- event return event } @@ -526,6 +580,8 @@ func (s *server) sendAsync(value interface{}) *event { func (s *server) followerLoop() { s.setState(Follower) + since := time.Now() + electionTimeout := s.ElectionTimeout() timeoutChan := afterBetween(s.ElectionTimeout(), s.ElectionTimeout()*2) for { @@ -548,6 +604,11 @@ func (s *server) followerLoop() { err = NotLeaderError } case *AppendEntriesRequest: + // If heartbeats get too close to the election timeout then send an event. + elapsedTime := time.Now().Sub(since) + if elapsedTime > time.Duration(float64(electionTimeout)*ElectionTimeoutThresholdPercent) { + s.DispatchEvent(newEvent(ElectionTimeoutThresholdEventType, elapsedTime, nil)) + } e.returnValue, update = s.processAppendEntriesRequest(req) case *RequestVoteRequest: e.returnValue, update = s.processRequestVoteRequest(req) @@ -575,6 +636,7 @@ func (s *server) followerLoop() { // 1.Receiving valid AppendEntries RPC, or // 2.Granting vote to candidate if update { + since = time.Now() timeoutChan = afterBetween(s.ElectionTimeout(), s.ElectionTimeout()*2) } @@ -588,7 +650,13 @@ func (s *server) followerLoop() { // The event loop that is run when the server is in a Candidate state. func (s *server) candidateLoop() { lastLogIndex, lastLogTerm := s.log.lastInfo() + + // Clear leader value. + prevLeader := s.leader s.leader = "" + if prevLeader != s.leader { + s.DispatchEvent(newEvent(LeaderChangeEventType, s.leader, prevLeader)) + } for { // Increment current term, vote for self. @@ -765,11 +833,11 @@ func (s *server) Do(command Command) (interface{}, error) { } // Processes a command. -func (s *server) processCommand(command Command, e *event) { +func (s *server) processCommand(command Command, e *ev) { s.debugln("server.command.process") // Create an entry for the command in the log. - entry, err := s.log.createEntry(s.currentTerm, command) + entry, err := s.log.createEntry(s.currentTerm, command, e) if err != nil { s.debugln("server.command.log.entry.error:", err) @@ -783,21 +851,6 @@ func (s *server) processCommand(command Command, e *event) { return } - // Issue a callback for the entry once it's committed. - go func() { - // Wait for the entry to be committed. - select { - case <-entry.commit: - var err error - s.debugln("server.command.commit") - e.returnValue, err = s.log.getEntryResult(entry, true) - e.c <- err - case <-time.After(time.Second): - s.debugln("server.command.timeout") - e.c <- CommandTimeoutError - } - }() - // Issue an append entries response for the server. resp := newAppendEntriesResponse(s.currentTerm, true, s.log.currentIndex(), s.log.CommitIndex()) resp.append = true @@ -866,7 +919,7 @@ func (s *server) processAppendEntriesRequest(req *AppendEntriesRequest) (*Append func (s *server) processAppendEntriesResponse(resp *AppendEntriesResponse) { // If we find a higher term then change to a follower and exit. - if resp.Term > s.currentTerm { + if resp.Term > s.Term() { s.setCurrentTerm(resp.Term, "", false) return } @@ -902,21 +955,6 @@ func (s *server) processAppendEntriesResponse(resp *AppendEntriesResponse) { if commitIndex > committedIndex { s.log.setCommitIndex(commitIndex) s.debugln("commit index ", commitIndex) - for i := committedIndex; i < commitIndex; i++ { - if entry := s.log.getEntry(i + 1); entry != nil { - // if the leader is a new one and the entry came from the - // old leader, the commit channel will be nil and no go routine - // is waiting from this channel - // if we try to send to it, the new leader will get stuck - if entry.commit != nil { - select { - case entry.commit <- true: - default: - panic("server unable to send signal to commit channel") - } - } - } - } } } @@ -937,7 +975,7 @@ func (s *server) RequestVote(req *RequestVoteRequest) *RequestVoteResponse { func (s *server) processRequestVoteRequest(req *RequestVoteRequest) (*RequestVoteResponse, bool) { // If the request is coming from an old term then reject it. - if req.Term < s.currentTerm { + if req.Term < s.Term() { s.debugln("server.rv.error: stale term") return newRequestVoteResponse(s.currentTerm, false), false } @@ -989,6 +1027,8 @@ func (s *server) AddPeer(name string, connectiongString string) error { } s.peers[peer.Name] = peer + + s.DispatchEvent(newEvent(AddPeerEventType, name, nil)) } // Write the configuration to file. @@ -1015,6 +1055,8 @@ func (s *server) RemovePeer(name string) error { } delete(s.peers, name) + + s.DispatchEvent(newEvent(RemovePeerEventType, name, nil)) } // Write the configuration to file. @@ -1317,7 +1359,7 @@ func (s *server) readConf() error { //-------------------------------------- func (s *server) debugln(v ...interface{}) { - debugf("[%s Term:%d] %s", s.name, s.currentTerm, fmt.Sprintln(v...)) + debugf("[%s Term:%d] %s", s.name, s.Term(), fmt.Sprintln(v...)) } func (s *server) traceln(v ...interface{}) { diff --git a/third_party/github.com/coreos/raft/server_test.go b/third_party/github.com/coreos/raft/server_test.go index 792ed90f3..3ac36811b 100644 --- a/third_party/github.com/coreos/raft/server_test.go +++ b/third_party/github.com/coreos/raft/server_test.go @@ -1,6 +1,7 @@ package raft import ( + "encoding/json" "fmt" "reflect" "strconv" @@ -44,7 +45,10 @@ func TestServerRequestVoteDeniedForStaleTerm(t *testing.T) { t.Fatalf("Server %s unable to join: %v", s.Name(), err) } + s.(*server).mutex.Lock() s.(*server).currentTerm = 2 + s.(*server).mutex.Unlock() + defer s.Stop() resp := s.RequestVote(newRequestVoteRequest(1, "foo", 1, 0)) if resp.Term != 2 || resp.VoteGranted { @@ -64,7 +68,9 @@ func TestServerRequestVoteDeniedIfAlreadyVoted(t *testing.T) { t.Fatalf("Server %s unable to join: %v", s.Name(), err) } + s.(*server).mutex.Lock() s.(*server).currentTerm = 2 + s.(*server).mutex.Unlock() defer s.Stop() resp := s.RequestVote(newRequestVoteRequest(2, "foo", 1, 0)) if resp.Term != 2 || !resp.VoteGranted { @@ -87,7 +93,9 @@ func TestServerRequestVoteApprovedIfAlreadyVotedInOlderTerm(t *testing.T) { time.Sleep(time.Millisecond * 100) + s.(*server).mutex.Lock() s.(*server).currentTerm = 2 + s.(*server).mutex.Unlock() defer s.Stop() resp := s.RequestVote(newRequestVoteRequest(2, "foo", 2, 1)) if resp.Term != 2 || !resp.VoteGranted || s.VotedFor() != "foo" { @@ -103,9 +111,9 @@ func TestServerRequestVoteApprovedIfAlreadyVotedInOlderTerm(t *testing.T) { // Ensure that a vote request is denied if the log is out of date. func TestServerRequestVoteDenyIfCandidateLogIsBehind(t *testing.T) { tmpLog := newLog() - e0, _ := newLogEntry(tmpLog, 1, 1, &testCommand1{Val: "foo", I: 20}) - e1, _ := newLogEntry(tmpLog, 2, 1, &testCommand2{X: 100}) - e2, _ := newLogEntry(tmpLog, 3, 2, &testCommand1{Val: "bar", I: 0}) + e0, _ := newLogEntry(tmpLog, nil, 1, 1, &testCommand1{Val: "foo", I: 20}) + e1, _ := newLogEntry(tmpLog, nil, 2, 1, &testCommand2{X: 100}) + e2, _ := newLogEntry(tmpLog, nil, 3, 2, &testCommand1{Val: "bar", I: 0}) s := newTestServerWithLog("1", &testTransporter{}, []*LogEntry{e0, e1, e2}) // start as a follower with term 2 and index 3 @@ -143,7 +151,7 @@ func TestServerRequestVoteDenyIfCandidateLogIsBehind(t *testing.T) { // // Ensure that we can self-promote a server to candidate, obtain votes and become a fearless leader. func TestServerPromoteSelf(t *testing.T) { - e0, _ := newLogEntry(newLog(), 1, 1, &testCommand1{Val: "foo", I: 20}) + e0, _ := newLogEntry(newLog(), nil, 1, 1, &testCommand1{Val: "foo", I: 20}) s := newTestServerWithLog("1", &testTransporter{}, []*LogEntry{e0}) // start as a follower @@ -196,7 +204,7 @@ func TestServerAppendEntries(t *testing.T) { defer s.Stop() // Append single entry. - e, _ := newLogEntry(nil, 1, 1, &testCommand1{Val: "foo", I: 10}) + e, _ := newLogEntry(nil, nil, 1, 1, &testCommand1{Val: "foo", I: 10}) entries := []*LogEntry{e} resp := s.AppendEntries(newAppendEntriesRequest(1, 0, 0, 0, "ldr", entries)) if resp.Term != 1 || !resp.Success { @@ -207,8 +215,8 @@ func TestServerAppendEntries(t *testing.T) { } // Append multiple entries + commit the last one. - e1, _ := newLogEntry(nil, 2, 1, &testCommand1{Val: "bar", I: 20}) - e2, _ := newLogEntry(nil, 3, 1, &testCommand1{Val: "baz", I: 30}) + e1, _ := newLogEntry(nil, nil, 2, 1, &testCommand1{Val: "bar", I: 20}) + e2, _ := newLogEntry(nil, nil, 3, 1, &testCommand1{Val: "baz", I: 30}) entries = []*LogEntry{e1, e2} resp = s.AppendEntries(newAppendEntriesRequest(1, 1, 1, 1, "ldr", entries)) if resp.Term != 1 || !resp.Success { @@ -235,10 +243,12 @@ func TestServerAppendEntriesWithStaleTermsAreRejected(t *testing.T) { s.Start() defer s.Stop() + s.(*server).mutex.Lock() s.(*server).currentTerm = 2 + s.(*server).mutex.Unlock() // Append single entry. - e, _ := newLogEntry(nil, 1, 1, &testCommand1{Val: "foo", I: 10}) + e, _ := newLogEntry(nil, nil, 1, 1, &testCommand1{Val: "foo", I: 10}) entries := []*LogEntry{e} resp := s.AppendEntries(newAppendEntriesRequest(1, 0, 0, 0, "ldr", entries)) if resp.Term != 2 || resp.Success { @@ -256,8 +266,8 @@ func TestServerAppendEntriesRejectedIfAlreadyCommitted(t *testing.T) { defer s.Stop() // Append single entry + commit. - e1, _ := newLogEntry(nil, 1, 1, &testCommand1{Val: "foo", I: 10}) - e2, _ := newLogEntry(nil, 2, 1, &testCommand1{Val: "foo", I: 15}) + e1, _ := newLogEntry(nil, nil, 1, 1, &testCommand1{Val: "foo", I: 10}) + e2, _ := newLogEntry(nil, nil, 2, 1, &testCommand1{Val: "foo", I: 15}) entries := []*LogEntry{e1, e2} resp := s.AppendEntries(newAppendEntriesRequest(1, 0, 0, 2, "ldr", entries)) if resp.Term != 1 || !resp.Success { @@ -265,7 +275,7 @@ func TestServerAppendEntriesRejectedIfAlreadyCommitted(t *testing.T) { } // Append entry again (post-commit). - e, _ := newLogEntry(nil, 2, 1, &testCommand1{Val: "bar", I: 20}) + e, _ := newLogEntry(nil, nil, 2, 1, &testCommand1{Val: "bar", I: 20}) entries = []*LogEntry{e} resp = s.AppendEntries(newAppendEntriesRequest(1, 2, 1, 1, "ldr", entries)) if resp.Term != 1 || resp.Success { @@ -279,9 +289,9 @@ func TestServerAppendEntriesOverwritesUncommittedEntries(t *testing.T) { s.Start() defer s.Stop() - entry1, _ := newLogEntry(nil, 1, 1, &testCommand1{Val: "foo", I: 10}) - entry2, _ := newLogEntry(nil, 2, 1, &testCommand1{Val: "foo", I: 15}) - entry3, _ := newLogEntry(nil, 2, 2, &testCommand1{Val: "bar", I: 20}) + entry1, _ := newLogEntry(nil, nil, 1, 1, &testCommand1{Val: "foo", I: 10}) + entry2, _ := newLogEntry(nil, nil, 2, 1, &testCommand1{Val: "foo", I: 15}) + entry3, _ := newLogEntry(nil, nil, 2, 2, &testCommand1{Val: "bar", I: 20}) // Append single entry + commit. entries := []*LogEntry{entry1, entry2} @@ -328,13 +338,23 @@ func TestServerRecoverFromPreviousLogAndConf(t *testing.T) { mutex.RLock() target := servers[peer.Name] mutex.RUnlock() - return target.RequestVote(req) + + b, _ := json.Marshal(req) + clonedReq := &RequestVoteRequest{} + json.Unmarshal(b, clonedReq) + + return target.RequestVote(clonedReq) } transporter.sendAppendEntriesRequestFunc = func(s Server, peer *Peer, req *AppendEntriesRequest) *AppendEntriesResponse { mutex.RLock() target := servers[peer.Name] mutex.RUnlock() - return target.AppendEntries(req) + + b, _ := json.Marshal(req) + clonedReq := &AppendEntriesRequest{} + json.Unmarshal(b, clonedReq) + + return target.AppendEntries(clonedReq) } disTransporter := &testTransporter{} @@ -359,7 +379,9 @@ func TestServerRecoverFromPreviousLogAndConf(t *testing.T) { for _, name := range names { s := newTestServer(name, transporter) + mutex.Lock() servers[name] = s + mutex.Unlock() paths[name] = s.Path() if name == "1" { @@ -474,13 +496,23 @@ func TestServerMultiNode(t *testing.T) { mutex.RLock() target := servers[peer.Name] mutex.RUnlock() - return target.RequestVote(req) + + b, _ := json.Marshal(req) + clonedReq := &RequestVoteRequest{} + json.Unmarshal(b, clonedReq) + + return target.RequestVote(clonedReq) } transporter.sendAppendEntriesRequestFunc = func(s Server, peer *Peer, req *AppendEntriesRequest) *AppendEntriesResponse { mutex.RLock() target := servers[peer.Name] mutex.RUnlock() - return target.AppendEntries(req) + + b, _ := json.Marshal(req) + clonedReq := &AppendEntriesRequest{} + json.Unmarshal(b, clonedReq) + + return target.AppendEntries(clonedReq) } disTransporter := &testTransporter{} diff --git a/third_party/github.com/coreos/raft/statemachine.go b/third_party/github.com/coreos/raft/statemachine.go index e59036cef..a0a22e8a3 100644 --- a/third_party/github.com/coreos/raft/statemachine.go +++ b/third_party/github.com/coreos/raft/statemachine.go @@ -7,7 +7,8 @@ package raft //------------------------------------------------------------------------------ // StateMachine is the interface for allowing the host application to save and -// recovery the state machine +// recovery the state machine. This makes it possible to make snapshots +// and compact the log. type StateMachine interface { Save() ([]byte, error) Recovery([]byte) error diff --git a/third_party/github.com/coreos/raft/test.go b/third_party/github.com/coreos/raft/test.go index 5b323f749..3ea9c7e3b 100644 --- a/third_party/github.com/coreos/raft/test.go +++ b/third_party/github.com/coreos/raft/test.go @@ -12,6 +12,10 @@ const ( testElectionTimeout = 200 * time.Millisecond ) +const ( + testListenerLoggerEnabled = false +) + func init() { RegisterCommand(&testCommand1{}) RegisterCommand(&testCommand2{}) @@ -66,6 +70,15 @@ func newTestServer(name string, transporter Transporter) Server { panic(err.Error()) } server, _ := NewServer(name, p, transporter, nil, nil, "") + if testListenerLoggerEnabled { + fn := func(e Event) { + server := e.Source().(Server) + warnf("[%s] %s %v -> %v\n", server.Name(), e.Type(), e.PrevValue(), e.Value()) + } + server.AddEventListener(StateChangeEventType, fn) + server.AddEventListener(LeaderChangeEventType, fn) + server.AddEventListener(TermChangeEventType, fn) + } return server } @@ -90,7 +103,7 @@ func newTestServerWithLog(name string, transporter Transporter, entries []*LogEn func newTestCluster(names []string, transporter Transporter, lookup map[string]Server) []Server { servers := []Server{} - e0, _ := newLogEntry(newLog(), 1, 1, &testCommand1{Val: "foo", I: 20}) + e0, _ := newLogEntry(newLog(), nil, 1, 1, &testCommand1{Val: "foo", I: 20}) for _, name := range names { if lookup[name] != nil { diff --git a/third_party/github.com/gorilla/context/.travis.yml b/third_party/github.com/gorilla/context/.travis.yml new file mode 100644 index 000000000..d87d46576 --- /dev/null +++ b/third_party/github.com/gorilla/context/.travis.yml @@ -0,0 +1,7 @@ +language: go + +go: + - 1.0 + - 1.1 + - 1.2 + - tip diff --git a/third_party/github.com/gorilla/mux/.travis.yml b/third_party/github.com/gorilla/mux/.travis.yml new file mode 100644 index 000000000..d87d46576 --- /dev/null +++ b/third_party/github.com/gorilla/mux/.travis.yml @@ -0,0 +1,7 @@ +language: go + +go: + - 1.0 + - 1.1 + - 1.2 + - tip diff --git a/third_party/github.com/gorilla/mux/mux.go b/third_party/github.com/gorilla/mux/mux.go index ca51a011d..f410b237f 100644 --- a/third_party/github.com/gorilla/mux/mux.go +++ b/third_party/github.com/gorilla/mux/mux.go @@ -166,7 +166,7 @@ func (r *Router) Handle(path string, handler http.Handler) *Route { // HandleFunc registers a new route with a matcher for the URL path. // See Route.Path() and Route.HandlerFunc(). func (r *Router) HandleFunc(path string, f func(http.ResponseWriter, - *http.Request)) *Route { + *http.Request),) *Route { return r.NewRoute().Path(path).HandlerFunc(f) } diff --git a/third_party/github.com/gorilla/mux/mux_test.go b/third_party/github.com/gorilla/mux/mux_test.go index 1a2a092df..292776d30 100644 --- a/third_party/github.com/gorilla/mux/mux_test.go +++ b/third_party/github.com/gorilla/mux/mux_test.go @@ -22,7 +22,6 @@ type routeTest struct { shouldMatch bool // whether the request is expected to match the route at all } - func TestHost(t *testing.T) { // newRequestHost a new request with a method, url, and host header newRequestHost := func(method, url, host string) *http.Request { @@ -673,7 +672,7 @@ func testRoute(t *testing.T, test routeTest) { func TestKeepContext(t *testing.T) { func1 := func(w http.ResponseWriter, r *http.Request) {} - r:= NewRouter() + r := NewRouter() r.HandleFunc("/", func1).Name("func1") req, _ := http.NewRequest("GET", "http://localhost/", nil) @@ -698,21 +697,20 @@ func TestKeepContext(t *testing.T) { } - type TestA301ResponseWriter struct { - hh http.Header - status int + hh http.Header + status int } func (ho TestA301ResponseWriter) Header() http.Header { return http.Header(ho.hh) } -func (ho TestA301ResponseWriter) Write( b []byte) (int, error) { +func (ho TestA301ResponseWriter) Write(b []byte) (int, error) { return 0, nil } -func (ho TestA301ResponseWriter) WriteHeader( code int ) { +func (ho TestA301ResponseWriter) WriteHeader(code int) { ho.status = code } @@ -722,16 +720,16 @@ func Test301Redirect(t *testing.T) { func1 := func(w http.ResponseWriter, r *http.Request) {} func2 := func(w http.ResponseWriter, r *http.Request) {} - r:= NewRouter() + r := NewRouter() r.HandleFunc("/api/", func2).Name("func2") r.HandleFunc("/", func1).Name("func1") req, _ := http.NewRequest("GET", "http://localhost//api/?abc=def", nil) res := TestA301ResponseWriter{ - hh: m, - status : 0, - } + hh: m, + status: 0, + } r.ServeHTTP(&res, req) if "http://localhost/api/?abc=def" != res.hh["Location"][0] { diff --git a/third_party/github.com/stretchr/testify/mock/mock.go b/third_party/github.com/stretchr/testify/mock/mock.go index b1bfc3b3e..333ea529c 100644 --- a/third_party/github.com/stretchr/testify/mock/mock.go +++ b/third_party/github.com/stretchr/testify/mock/mock.go @@ -94,23 +94,23 @@ func (m *Mock) Return(returnArguments ...interface{}) *Mock { // Once indicates that that the mock should only return the value once. // // Mock.On("MyMethod", arg1, arg2).Return(returnArg1, returnArg2).Once() -func (m* Mock) Once() { - m.ExpectedCalls[len(m.ExpectedCalls) - 1].Repeatability = 1 +func (m *Mock) Once() { + m.ExpectedCalls[len(m.ExpectedCalls)-1].Repeatability = 1 } // Twice indicates that that the mock should only return the value twice. // // Mock.On("MyMethod", arg1, arg2).Return(returnArg1, returnArg2).Twice() -func (m* Mock) Twice() { - m.ExpectedCalls[len(m.ExpectedCalls) - 1].Repeatability = 2 +func (m *Mock) Twice() { + m.ExpectedCalls[len(m.ExpectedCalls)-1].Repeatability = 2 } // Times indicates that that the mock should only return the indicated number // of times. // // Mock.On("MyMethod", arg1, arg2).Return(returnArg1, returnArg2).Times(5) -func (m* Mock) Times(i int) { - m.ExpectedCalls[len(m.ExpectedCalls) - 1].Repeatability = i +func (m *Mock) Times(i int) { + m.ExpectedCalls[len(m.ExpectedCalls)-1].Repeatability = i } /* @@ -369,22 +369,22 @@ func (args Arguments) Diff(objects []interface{}) (string, int) { for i := 0; i < maxArgCount; i++ { var actual, expected interface{} - if len(args) <= i { + if len(objects) <= i { actual = "(Missing)" } else { - actual = args[i] + actual = objects[i] } - if len(objects) <= i { + if len(args) <= i { expected = "(Missing)" } else { - expected = objects[i] + expected = args[i] } if reflect.TypeOf(expected) == reflect.TypeOf((*AnythingOfTypeArgument)(nil)).Elem() { // type checking - if reflect.TypeOf(actual).Name() != string(expected.(AnythingOfTypeArgument)) { + if reflect.TypeOf(actual).Name() != string(expected.(AnythingOfTypeArgument)) && reflect.TypeOf(actual).String() != string(expected.(AnythingOfTypeArgument)) { // not match differences++ output = fmt.Sprintf("%s\t%d: \u274C type %s != type %s - %s\n", output, i, expected, reflect.TypeOf(actual).Name(), actual) @@ -396,11 +396,11 @@ func (args Arguments) Diff(objects []interface{}) (string, int) { if assert.ObjectsAreEqual(expected, Anything) || assert.ObjectsAreEqual(actual, Anything) || assert.ObjectsAreEqual(actual, expected) { // match - output = fmt.Sprintf("%s\t%d: \u2705 %s == %s\n", output, i, expected, actual) + output = fmt.Sprintf("%s\t%d: \u2705 %s == %s\n", output, i, actual, expected) } else { // not match differences++ - output = fmt.Sprintf("%s\t%d: \u274C %s != %s\n", output, i, expected, actual) + output = fmt.Sprintf("%s\t%d: \u274C %s != %s\n", output, i, actual, expected) } } diff --git a/third_party/github.com/stretchr/testify/mock/mock_test.go b/third_party/github.com/stretchr/testify/mock/mock_test.go index 1ed126623..0531284f4 100644 --- a/third_party/github.com/stretchr/testify/mock/mock_test.go +++ b/third_party/github.com/stretchr/testify/mock/mock_test.go @@ -542,9 +542,9 @@ func Test_Arguments_Diff_WithAnythingArgument_InActualToo(t *testing.T) { func Test_Arguments_Diff_WithAnythingOfTypeArgument(t *testing.T) { - var args Arguments = []interface{}{"string", 123, true} + var args Arguments = []interface{}{"string", AnythingOfType("int"), true} var count int - _, count = args.Diff([]interface{}{"string", AnythingOfType("int"), true}) + _, count = args.Diff([]interface{}{"string", 123, true}) assert.Equal(t, 0, count) @@ -552,10 +552,10 @@ func Test_Arguments_Diff_WithAnythingOfTypeArgument(t *testing.T) { func Test_Arguments_Diff_WithAnythingOfTypeArgument_Failing(t *testing.T) { - var args Arguments = []interface{}{"string", 123, true} + var args Arguments = []interface{}{"string", AnythingOfType("string"), true} var count int var diff string - diff, count = args.Diff([]interface{}{"string", AnythingOfType("string"), true}) + diff, count = args.Diff([]interface{}{"string", 123, true}) assert.Equal(t, 1, count) assert.Contains(t, diff, `string != type int - %!s(int=123)`)