From aac485d67b341948e1509584e2b73b566e8c7266 Mon Sep 17 00:00:00 2001 From: Brandon Philips Date: Tue, 6 Aug 2013 10:50:04 -0700 Subject: [PATCH 01/10] bump(github.com/coreos/go-raft): 75c9644bbc43e5ef7805480c13deacec02195706 --- .../github.com/coreos/go-raft/.gitignore | 24 + .../github.com/coreos/go-raft/.travis.yml | 8 + third_party/github.com/coreos/go-raft/LICENSE | 20 + .../github.com/coreos/go-raft/Makefile | 13 + .../github.com/coreos/go-raft/README.md | 68 + .../coreos/go-raft/append_entries_request.go | 98 ++ .../go-raft/append_entries_request_test.go | 40 + .../coreos/go-raft/append_entries_response.go | 70 + .../go-raft/append_entries_response_test.go | 34 + .../github.com/coreos/go-raft/command.go | 92 ++ .../github.com/coreos/go-raft/debug.go | 116 ++ .../coreos/go-raft/http_transporter.go | 195 +++ .../coreos/go-raft/http_transporter_test.go | 153 ++ .../github.com/coreos/go-raft/join_command.go | 28 + .../coreos/go-raft/leave_command.go | 27 + third_party/github.com/coreos/go-raft/log.go | 610 ++++++++ .../github.com/coreos/go-raft/log_entry.go | 99 ++ .../github.com/coreos/go-raft/log_test.go | 232 +++ .../github.com/coreos/go-raft/nop_command.go | 26 + third_party/github.com/coreos/go-raft/peer.go | 271 ++++ .../protobuf/append_entries_request.pb.go | 115 ++ .../protobuf/append_entries_request.proto | 18 + .../protobuf/append_entries_responses.pb.go | 57 + .../protobuf/append_entries_responses.proto | 8 + .../coreos/go-raft/protobuf/log_entry.pb.go | 57 + .../coreos/go-raft/protobuf/log_entry.proto | 8 + .../protobuf/request_vote_request.pb.go | 57 + .../protobuf/request_vote_request.proto | 8 + .../protobuf/request_vote_responses.pb.go | 41 + .../protobuf/request_vote_responses.proto | 6 + .../protobuf/snapshot_recovery_request.pb.go | 65 + .../protobuf/snapshot_recovery_request.proto | 9 + .../protobuf/snapshot_recovery_response.pb.go | 49 + .../protobuf/snapshot_recovery_response.proto | 7 + .../go-raft/protobuf/snapshot_request.pb.go | 49 + .../go-raft/protobuf/snapshot_request.proto | 7 + .../go-raft/protobuf/snapshot_response.pb.go | 33 + .../go-raft/protobuf/snapshot_response.proto | 5 + .../coreos/go-raft/request_vote_request.go | 68 + .../coreos/go-raft/request_vote_response.go | 61 + .../github.com/coreos/go-raft/server.go | 1260 +++++++++++++++++ .../github.com/coreos/go-raft/server_test.go | 504 +++++++ .../github.com/coreos/go-raft/snapshot.go | 65 + .../go-raft/snapshot_recovery_request.go | 77 + .../go-raft/snapshot_recovery_response.go | 69 + .../coreos/go-raft/snapshot_request.go | 70 + .../coreos/go-raft/snapshot_response.go | 61 + third_party/github.com/coreos/go-raft/sort.go | 23 + .../github.com/coreos/go-raft/statemachine.go | 14 + third_party/github.com/coreos/go-raft/test.go | 179 +++ third_party/github.com/coreos/go-raft/time.go | 17 + .../github.com/coreos/go-raft/timer.go | 170 +++ .../github.com/coreos/go-raft/timer_test.go | 86 ++ .../github.com/coreos/go-raft/transporter.go | 16 + .../github.com/coreos/go-raft/z_test.go | 13 + 55 files changed, 5576 insertions(+) create mode 100644 third_party/github.com/coreos/go-raft/.gitignore create mode 100644 third_party/github.com/coreos/go-raft/.travis.yml create mode 100644 third_party/github.com/coreos/go-raft/LICENSE create mode 100644 third_party/github.com/coreos/go-raft/Makefile create mode 100644 third_party/github.com/coreos/go-raft/README.md create mode 100644 third_party/github.com/coreos/go-raft/append_entries_request.go create mode 100644 third_party/github.com/coreos/go-raft/append_entries_request_test.go create mode 100644 third_party/github.com/coreos/go-raft/append_entries_response.go create mode 100644 third_party/github.com/coreos/go-raft/append_entries_response_test.go create mode 100644 third_party/github.com/coreos/go-raft/command.go create mode 100644 third_party/github.com/coreos/go-raft/debug.go create mode 100644 third_party/github.com/coreos/go-raft/http_transporter.go create mode 100644 third_party/github.com/coreos/go-raft/http_transporter_test.go create mode 100644 third_party/github.com/coreos/go-raft/join_command.go create mode 100644 third_party/github.com/coreos/go-raft/leave_command.go create mode 100644 third_party/github.com/coreos/go-raft/log.go create mode 100644 third_party/github.com/coreos/go-raft/log_entry.go create mode 100644 third_party/github.com/coreos/go-raft/log_test.go create mode 100644 third_party/github.com/coreos/go-raft/nop_command.go create mode 100644 third_party/github.com/coreos/go-raft/peer.go create mode 100644 third_party/github.com/coreos/go-raft/protobuf/append_entries_request.pb.go create mode 100644 third_party/github.com/coreos/go-raft/protobuf/append_entries_request.proto create mode 100644 third_party/github.com/coreos/go-raft/protobuf/append_entries_responses.pb.go create mode 100644 third_party/github.com/coreos/go-raft/protobuf/append_entries_responses.proto create mode 100644 third_party/github.com/coreos/go-raft/protobuf/log_entry.pb.go create mode 100644 third_party/github.com/coreos/go-raft/protobuf/log_entry.proto create mode 100644 third_party/github.com/coreos/go-raft/protobuf/request_vote_request.pb.go create mode 100644 third_party/github.com/coreos/go-raft/protobuf/request_vote_request.proto create mode 100644 third_party/github.com/coreos/go-raft/protobuf/request_vote_responses.pb.go create mode 100644 third_party/github.com/coreos/go-raft/protobuf/request_vote_responses.proto create mode 100644 third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_request.pb.go create mode 100644 third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_request.proto create mode 100644 third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_response.pb.go create mode 100644 third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_response.proto create mode 100644 third_party/github.com/coreos/go-raft/protobuf/snapshot_request.pb.go create mode 100644 third_party/github.com/coreos/go-raft/protobuf/snapshot_request.proto create mode 100644 third_party/github.com/coreos/go-raft/protobuf/snapshot_response.pb.go create mode 100644 third_party/github.com/coreos/go-raft/protobuf/snapshot_response.proto create mode 100644 third_party/github.com/coreos/go-raft/request_vote_request.go create mode 100644 third_party/github.com/coreos/go-raft/request_vote_response.go create mode 100644 third_party/github.com/coreos/go-raft/server.go create mode 100644 third_party/github.com/coreos/go-raft/server_test.go create mode 100644 third_party/github.com/coreos/go-raft/snapshot.go create mode 100644 third_party/github.com/coreos/go-raft/snapshot_recovery_request.go create mode 100644 third_party/github.com/coreos/go-raft/snapshot_recovery_response.go create mode 100644 third_party/github.com/coreos/go-raft/snapshot_request.go create mode 100644 third_party/github.com/coreos/go-raft/snapshot_response.go create mode 100644 third_party/github.com/coreos/go-raft/sort.go create mode 100644 third_party/github.com/coreos/go-raft/statemachine.go create mode 100644 third_party/github.com/coreos/go-raft/test.go create mode 100644 third_party/github.com/coreos/go-raft/time.go create mode 100644 third_party/github.com/coreos/go-raft/timer.go create mode 100644 third_party/github.com/coreos/go-raft/timer_test.go create mode 100644 third_party/github.com/coreos/go-raft/transporter.go create mode 100644 third_party/github.com/coreos/go-raft/z_test.go diff --git a/third_party/github.com/coreos/go-raft/.gitignore b/third_party/github.com/coreos/go-raft/.gitignore new file mode 100644 index 000000000..56a5e9893 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe + +coverage.html diff --git a/third_party/github.com/coreos/go-raft/.travis.yml b/third_party/github.com/coreos/go-raft/.travis.yml new file mode 100644 index 000000000..5f70bdf4c --- /dev/null +++ b/third_party/github.com/coreos/go-raft/.travis.yml @@ -0,0 +1,8 @@ +language: go + +go: + - 1.1 + +install: + - make dependencies + diff --git a/third_party/github.com/coreos/go-raft/LICENSE b/third_party/github.com/coreos/go-raft/LICENSE new file mode 100644 index 000000000..ee7f22228 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/LICENSE @@ -0,0 +1,20 @@ +Copyright 2013 go-raft contributors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/third_party/github.com/coreos/go-raft/Makefile b/third_party/github.com/coreos/go-raft/Makefile new file mode 100644 index 000000000..afbbf63c7 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/Makefile @@ -0,0 +1,13 @@ +all: test + +coverage: + gocov test github.com/benbjohnson/go-raft | gocov-html > coverage.html + open coverage.html + +dependencies: + go get -d . + +test: + go test -v ./... + +.PHONY: coverage dependencies test diff --git a/third_party/github.com/coreos/go-raft/README.md b/third_party/github.com/coreos/go-raft/README.md new file mode 100644 index 000000000..a41b251df --- /dev/null +++ b/third_party/github.com/coreos/go-raft/README.md @@ -0,0 +1,68 @@ +[![Stories in Ready](http://badge.waffle.io/benbjohnson/go-raft.png)](http://waffle.io/benbjohnson/go-raft) +go-raft +======= + +## Overview + +This is an Go implementation of the Raft distributed consensus protocol. +Raft is a protocol by which a cluster of nodes can maintain a replicated state machine. +The state machine is kept in sync through the use of a replicated log. + +For more details on Raft, you can read [In Search of an Understandable Consensus Algorithm](https://ramcloud.stanford.edu/wiki/download/attachments/11370504/raft.pdf) by Diego Ongaro and John Ousterhout. + + +## The Raft Protocol + +### Overview + +Maintaining state in a single process on a single server is easy. +Your process is a single point of authority so there are no conflicts when reading and writing state. +Even multi-threaded processes can rely on locks or coroutines to serialize access to the data. + +However, in a distributed system there is no single point of authority. +Servers can crash or the network between two machines can become unavailable or any number of other problems can occur. + +A distributed consensus protocol is used for maintaining a consistent state across multiple servers in a cluster. +Many distributed systems are built upon the Paxos protocol but Paxos can be difficult to understand and there are many gaps between Paxos and real world implementation. + +An alternative is the [Raft distributed consensus protocol](https://ramcloud.stanford.edu/wiki/download/attachments/11370504/raft.pdf) by Diego Ongaro and John Ousterhout. +Raft is a protocol built with understandability as a primary tenant and it centers around two things: + +1. Leader Election +2. Replicated Log + +With these two constructs, you can build a system that can maintain state across multiple servers -- even in the event of multiple failures. + +### Leader Election + +The Raft protocol effectively works as a master-slave system whereby state changes are written to a single server in the cluster and are distributed out to the rest of the servers in the cluster. +This simplifies the protocol since there is only one data authority and conflicts will not have to be resolved. + +Raft ensures that there is only one leader at a time. +It does this by performing elections among the nodes in the cluster and requiring that a node must receive a majority of the votes in order to become leader. +For example, if you have 3 nodes in your cluster then a single node would need 2 votes in order to become the leader. +For a 5 node cluster, a server would need 3 votes to become leader. + +### Replicated Log + +To maintain state, a log of commands is maintained. +Each command makes a change to the state of the server and the command is deterministic. +By ensuring that this log is replicated identically between all the nodes in the cluster we can replicate the state at any point in time in the log by running each command sequentially. + +Replicating the log under normal conditions is done by sending an `AppendEntries` RPC from the leader to each of the other servers in the cluster (called Peers). +Each peer will append the entries from the leader through a 2-phase commit process which ensure that a majority of servers in the cluster have entries written to log. + +For a more detailed explanation on the failover process and election terms please see the full paper describing the protocol: [In Search of an Understandable Consensus Algorithm](https://ramcloud.stanford.edu/wiki/download/attachments/11370504/raft.pdf) + + +## Project Status + +The go-raft library is feature complete but in alpha. +There is a reference implementation called [raftd](https://github.com/benbjohnson/raftd) that demonstrates how to use the library + +The library will be considered experimental until it has significant production usage. +I'm writing the library for the purpose of including distributed processing in my behavioral analytics database called [Sky](https://github.com/skydb/sky). +However, I hope other projects can benefit from having a distributed consensus protocol so the go-raft library is available under MIT license. + +If you have a project that you're using go-raft in, please add it to this README and send a pull request so others can see implementation examples. +If you have any questions on implementing go-raft in your project, feel free to contact me on [GitHub](https://github.com/benbjohnson), [Twitter](https://twitter.com/benbjohnson) or by e-mail at [ben@skylandlabs.com](mailto:ben@skylandlabs.com). diff --git a/third_party/github.com/coreos/go-raft/append_entries_request.go b/third_party/github.com/coreos/go-raft/append_entries_request.go new file mode 100644 index 000000000..0d22ef9fd --- /dev/null +++ b/third_party/github.com/coreos/go-raft/append_entries_request.go @@ -0,0 +1,98 @@ +package raft + +import ( + "code.google.com/p/goprotobuf/proto" + "github.com/benbjohnson/go-raft/protobuf" + "io" + "io/ioutil" +) + +// The request sent to a server to append entries to the log. +type AppendEntriesRequest struct { + Term uint64 + PrevLogIndex uint64 + PrevLogTerm uint64 + CommitIndex uint64 + LeaderName string + Entries []*LogEntry +} + +// Creates a new AppendEntries request. +func newAppendEntriesRequest(term uint64, prevLogIndex uint64, prevLogTerm uint64, commitIndex uint64, leaderName string, entries []*LogEntry) *AppendEntriesRequest { + return &AppendEntriesRequest{ + Term: term, + PrevLogIndex: prevLogIndex, + PrevLogTerm: prevLogTerm, + CommitIndex: commitIndex, + LeaderName: leaderName, + Entries: entries, + } +} + +// Encodes the AppendEntriesRequest to a buffer. Returns the number of bytes +// written and any error that may have occurred. +func (req *AppendEntriesRequest) encode(w io.Writer) (int, error) { + + protoEntries := make([]*protobuf.ProtoAppendEntriesRequest_ProtoLogEntry, len(req.Entries)) + + for i, entry := range req.Entries { + protoEntries[i] = &protobuf.ProtoAppendEntriesRequest_ProtoLogEntry{ + Index: proto.Uint64(entry.Index), + Term: proto.Uint64(entry.Term), + CommandName: proto.String(entry.CommandName), + Command: entry.Command, + } + } + + pb := &protobuf.ProtoAppendEntriesRequest{ + Term: proto.Uint64(req.Term), + PrevLogIndex: proto.Uint64(req.PrevLogIndex), + PrevLogTerm: proto.Uint64(req.PrevLogTerm), + CommitIndex: proto.Uint64(req.CommitIndex), + LeaderName: proto.String(req.LeaderName), + Entries: protoEntries, + } + + p, err := proto.Marshal(pb) + if err != nil { + return -1, err + } + + return w.Write(p) +} + +// Decodes the AppendEntriesRequest from a buffer. Returns the number of bytes read and +// any error that occurs. +func (req *AppendEntriesRequest) decode(r io.Reader) (int, error) { + data, err := ioutil.ReadAll(r) + + if err != nil { + return -1, err + } + + totalBytes := len(data) + + pb := &protobuf.ProtoAppendEntriesRequest{} + if err := proto.Unmarshal(data, pb); err != nil { + return -1, err + } + + req.Term = pb.GetTerm() + req.PrevLogIndex = pb.GetPrevLogIndex() + req.PrevLogTerm = pb.GetPrevLogTerm() + req.CommitIndex = pb.GetCommitIndex() + req.LeaderName = pb.GetLeaderName() + + req.Entries = make([]*LogEntry, len(pb.Entries)) + + for i, entry := range pb.Entries { + req.Entries[i] = &LogEntry{ + Index: entry.GetIndex(), + Term: entry.GetTerm(), + CommandName: entry.GetCommandName(), + Command: entry.Command, + } + } + + return totalBytes, nil +} diff --git a/third_party/github.com/coreos/go-raft/append_entries_request_test.go b/third_party/github.com/coreos/go-raft/append_entries_request_test.go new file mode 100644 index 000000000..ef6732fc4 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/append_entries_request_test.go @@ -0,0 +1,40 @@ +package raft + +import ( + "bytes" + "testing" +) + +func BenchmarkAppendEntriesRequestEncoding(b *testing.B) { + req, tmp := createTestAppendEntriesRequest(2000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + var buf bytes.Buffer + req.encode(&buf) + } + b.SetBytes(int64(len(tmp))) +} + +func BenchmarkAppendEntriesRequestDecoding(b *testing.B) { + req, buf := createTestAppendEntriesRequest(2000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + req.decode(bytes.NewReader(buf)) + } + b.SetBytes(int64(len(buf))) +} + +func createTestAppendEntriesRequest(entryCount int) (*AppendEntriesRequest, []byte) { + entries := make([]*LogEntry, 0) + for i := 0; i < entryCount; i++ { + command := &DefaultJoinCommand{Name: "localhost:1000"} + entry, _ := newLogEntry(nil, 1, 2, command) + entries = append(entries, entry) + } + req := newAppendEntriesRequest(1, 1, 1, 1, "leader", entries) + + var buf bytes.Buffer + req.encode(&buf) + + return req, buf.Bytes() +} diff --git a/third_party/github.com/coreos/go-raft/append_entries_response.go b/third_party/github.com/coreos/go-raft/append_entries_response.go new file mode 100644 index 000000000..ed0c29e24 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/append_entries_response.go @@ -0,0 +1,70 @@ +package raft + +import ( + "code.google.com/p/goprotobuf/proto" + "github.com/benbjohnson/go-raft/protobuf" + "io" + "io/ioutil" +) + +// The response returned from a server appending entries to the log. +type AppendEntriesResponse struct { + Term uint64 + // the current index of the server + Index uint64 + Success bool + CommitIndex uint64 + peer string + append bool +} + +// Creates a new AppendEntries response. +func newAppendEntriesResponse(term uint64, success bool, index uint64, commitIndex uint64) *AppendEntriesResponse { + return &AppendEntriesResponse{ + Term: term, + Success: success, + Index: index, + CommitIndex: commitIndex, + } +} + +// Encodes the AppendEntriesResponse to a buffer. Returns the number of bytes +// written and any error that may have occurred. +func (resp *AppendEntriesResponse) encode(w io.Writer) (int, error) { + pb := &protobuf.ProtoAppendEntriesResponse{ + Term: proto.Uint64(resp.Term), + Index: proto.Uint64(resp.Index), + CommitIndex: proto.Uint64(resp.CommitIndex), + Success: proto.Bool(resp.Success), + } + p, err := proto.Marshal(pb) + if err != nil { + return -1, err + } + + return w.Write(p) +} + +// Decodes the AppendEntriesResponse from a buffer. Returns the number of bytes read and +// any error that occurs. +func (resp *AppendEntriesResponse) decode(r io.Reader) (int, error) { + data, err := ioutil.ReadAll(r) + + if err != nil { + return -1, err + } + + totalBytes := len(data) + + pb := &protobuf.ProtoAppendEntriesResponse{} + if err := proto.Unmarshal(data, pb); err != nil { + return -1, err + } + + resp.Term = pb.GetTerm() + resp.Index = pb.GetIndex() + resp.CommitIndex = pb.GetCommitIndex() + resp.Success = pb.GetSuccess() + + return totalBytes, nil +} diff --git a/third_party/github.com/coreos/go-raft/append_entries_response_test.go b/third_party/github.com/coreos/go-raft/append_entries_response_test.go new file mode 100644 index 000000000..038dcda76 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/append_entries_response_test.go @@ -0,0 +1,34 @@ +package raft + +import ( + "bytes" + "testing" +) + +func BenchmarkAppendEntriesResponseEncoding(b *testing.B) { + req, tmp := createTestAppendEntriesResponse(2000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + var buf bytes.Buffer + req.encode(&buf) + } + b.SetBytes(int64(len(tmp))) +} + +func BenchmarkAppendEntriesResponseDecoding(b *testing.B) { + req, buf := createTestAppendEntriesResponse(2000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + req.decode(bytes.NewReader(buf)) + } + b.SetBytes(int64(len(buf))) +} + +func createTestAppendEntriesResponse(entryCount int) (*AppendEntriesResponse, []byte) { + resp := newAppendEntriesResponse(1, true, 1, 1) + + var buf bytes.Buffer + resp.encode(&buf) + + return resp, buf.Bytes() +} diff --git a/third_party/github.com/coreos/go-raft/command.go b/third_party/github.com/coreos/go-raft/command.go new file mode 100644 index 000000000..2c0495171 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/command.go @@ -0,0 +1,92 @@ +package raft + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "reflect" +) + +//------------------------------------------------------------------------------ +// +// Globals +// +//------------------------------------------------------------------------------ + +var commandTypes map[string]Command + +func init() { + commandTypes = map[string]Command{} +} + +//------------------------------------------------------------------------------ +// +// Typedefs +// +//------------------------------------------------------------------------------ + +// A command represents an action to be taken on the replicated state machine. +type Command interface { + CommandName() string + Apply(server *Server) (interface{}, error) +} + +type CommandEncoder interface { + Encode(w io.Writer) error + Decode(r io.Reader) error +} + +//------------------------------------------------------------------------------ +// +// Functions +// +//------------------------------------------------------------------------------ + +//-------------------------------------- +// Instantiation +//-------------------------------------- + +// Creates a new instance of a command by name. +func newCommand(name string, data []byte) (Command, error) { + // Find the registered command. + command := commandTypes[name] + if command == nil { + return nil, fmt.Errorf("raft.Command: Unregistered command type: %s", name) + } + + // Make a copy of the command. + v := reflect.New(reflect.Indirect(reflect.ValueOf(command)).Type()).Interface() + copy, ok := v.(Command) + if !ok { + panic(fmt.Sprintf("raft: Unable to copy command: %s (%v)", command.CommandName(), reflect.ValueOf(v).Kind().String())) + } + + // If data for the command was passed in the decode it. + if data != nil { + if encoder, ok := copy.(CommandEncoder); ok { + if err := encoder.Decode(bytes.NewReader(data)); err != nil { + return nil, err + } + } else { + json.NewDecoder(bytes.NewReader(data)).Decode(copy) + } + } + + return copy, nil +} + +//-------------------------------------- +// Registration +//-------------------------------------- + +// Registers a command by storing a reference to an instance of it. +func RegisterCommand(command Command) { + if command == nil { + panic(fmt.Sprintf("raft: Cannot register nil")) + } else if commandTypes[command.CommandName()] != nil { + panic(fmt.Sprintf("raft: Duplicate registration: %s", command.CommandName())) + return + } + commandTypes[command.CommandName()] = command +} diff --git a/third_party/github.com/coreos/go-raft/debug.go b/third_party/github.com/coreos/go-raft/debug.go new file mode 100644 index 000000000..97e2bc772 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/debug.go @@ -0,0 +1,116 @@ +package raft + +import ( + "log" + "os" +) + +//------------------------------------------------------------------------------ +// +// Variables +// +//------------------------------------------------------------------------------ + +const ( + Debug = 1 + Trace = 2 +) + +var logLevel int = 0 +var logger *log.Logger + +func init() { + logger = log.New(os.Stdout, "[raft]", log.Lmicroseconds) +} + +//------------------------------------------------------------------------------ +// +// Functions +// +//------------------------------------------------------------------------------ + +func LogLevel() int { + return logLevel +} + +func SetLogLevel(level int) { + logLevel = level +} + +//-------------------------------------- +// Warnings +//-------------------------------------- + +// Prints to the standard logger. Arguments are handled in the manner of +// fmt.Print. +func warn(v ...interface{}) { + logger.Print(v...) +} + +// Prints to the standard logger. Arguments are handled in the manner of +// fmt.Printf. +func warnf(format string, v ...interface{}) { + logger.Printf(format, v...) +} + +// Prints to the standard logger. Arguments are handled in the manner of +// fmt.Println. +func warnln(v ...interface{}) { + logger.Println(v...) +} + +//-------------------------------------- +// Basic debugging +//-------------------------------------- + +// Prints to the standard logger if debug mode is enabled. Arguments +// are handled in the manner of fmt.Print. +func debug(v ...interface{}) { + if logLevel >= Debug { + logger.Print(v...) + } +} + +// Prints to the standard logger if debug mode is enabled. Arguments +// are handled in the manner of fmt.Printf. +func debugf(format string, v ...interface{}) { + if logLevel >= Debug { + logger.Printf(format, v...) + } +} + +// Prints to the standard logger if debug mode is enabled. Arguments +// are handled in the manner of fmt.Println. +func debugln(v ...interface{}) { + if logLevel >= Debug { + logger.Println(v...) + } +} + +//-------------------------------------- +// Trace-level debugging +//-------------------------------------- + +// Prints to the standard logger if trace debugging is enabled. Arguments +// are handled in the manner of fmt.Print. +func trace(v ...interface{}) { + if logLevel >= Trace { + logger.Print(v...) + } +} + +// Prints to the standard logger if trace debugging is enabled. Arguments +// are handled in the manner of fmt.Printf. +func tracef(format string, v ...interface{}) { + if logLevel >= Trace { + logger.Printf(format, v...) + } +} + +// Prints to the standard logger if trace debugging is enabled. Arguments +// are handled in the manner of debugln. +func traceln(v ...interface{}) { + if logLevel >= Trace { + logger.Println(v...) + } +} diff --git a/third_party/github.com/coreos/go-raft/http_transporter.go b/third_party/github.com/coreos/go-raft/http_transporter.go new file mode 100644 index 000000000..1125f91f5 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/http_transporter.go @@ -0,0 +1,195 @@ +package raft + +import ( + "bytes" + "fmt" + "io" + "net/http" +) + +// Parts from this transporter were heavily influenced by Peter Bougon's +// raft implementation: https://github.com/peterbourgon/raft + +//------------------------------------------------------------------------------ +// +// Typedefs +// +//------------------------------------------------------------------------------ + +// An HTTPTransporter is a default transport layer used to communicate between +// multiple servers. +type HTTPTransporter struct { + DisableKeepAlives bool + prefix string + appendEntriesPath string + requestVotePath string +} + +type HTTPMuxer interface { + HandleFunc(string, func(http.ResponseWriter, *http.Request)) +} + +//------------------------------------------------------------------------------ +// +// Constructor +// +//------------------------------------------------------------------------------ + +// Creates a new HTTP transporter with the given path prefix. +func NewHTTPTransporter(prefix string) *HTTPTransporter { + return &HTTPTransporter{ + DisableKeepAlives: false, + prefix: prefix, + appendEntriesPath: fmt.Sprintf("%s%s", prefix, "/appendEntries"), + requestVotePath: fmt.Sprintf("%s%s", prefix, "/requestVote"), + } +} + +//------------------------------------------------------------------------------ +// +// Accessors +// +//------------------------------------------------------------------------------ + +// Retrieves the path prefix used by the transporter. +func (t *HTTPTransporter) Prefix() string { + return t.prefix +} + +// Retrieves the AppendEntries path. +func (t *HTTPTransporter) AppendEntriesPath() string { + return t.appendEntriesPath +} + +// Retrieves the RequestVote path. +func (t *HTTPTransporter) RequestVotePath() string { + return t.requestVotePath +} + +//------------------------------------------------------------------------------ +// +// Methods +// +//------------------------------------------------------------------------------ + +//-------------------------------------- +// Installation +//-------------------------------------- + +// Applies Raft routes to an HTTP router for a given server. +func (t *HTTPTransporter) Install(server *Server, mux HTTPMuxer) { + mux.HandleFunc(t.AppendEntriesPath(), t.appendEntriesHandler(server)) + mux.HandleFunc(t.RequestVotePath(), t.requestVoteHandler(server)) +} + +//-------------------------------------- +// Outgoing +//-------------------------------------- + +// Sends an AppendEntries RPC to a peer. +func (t *HTTPTransporter) SendAppendEntriesRequest(server *Server, peer *Peer, req *AppendEntriesRequest) *AppendEntriesResponse { + var b bytes.Buffer + if _, err := req.encode(&b); err != nil { + traceln("transporter.ae.encoding.error:", err) + return nil + } + + url := fmt.Sprintf("http://%s%s", peer.Name(), t.AppendEntriesPath()) + traceln(server.Name(), "POST", url) + + client := &http.Client{Transport: &http.Transport{DisableKeepAlives: t.DisableKeepAlives}} + httpResp, err := client.Post(url, "application/protobuf", &b) + if httpResp == nil || err != nil { + traceln("transporter.ae.response.error:", err) + return nil + } + defer httpResp.Body.Close() + + resp := &AppendEntriesResponse{} + if _, err = resp.decode(httpResp.Body); err != nil && err != io.EOF { + traceln("transporter.ae.decoding.error:", err) + return nil + } + + return resp +} + +// Sends a RequestVote RPC to a peer. +func (t *HTTPTransporter) SendVoteRequest(server *Server, peer *Peer, req *RequestVoteRequest) *RequestVoteResponse { + var b bytes.Buffer + if _, err := req.encode(&b); err != nil { + traceln("transporter.rv.encoding.error:", err) + return nil + } + + url := fmt.Sprintf("http://%s%s", peer.Name(), t.RequestVotePath()) + traceln(server.Name(), "POST", url) + + client := &http.Client{Transport: &http.Transport{DisableKeepAlives: t.DisableKeepAlives}} + httpResp, err := client.Post(url, "application/protobuf", &b) + if httpResp == nil || err != nil { + traceln("transporter.rv.response.error:", err) + return nil + } + defer httpResp.Body.Close() + + resp := &RequestVoteResponse{} + if _, err = resp.decode(httpResp.Body); err != nil && err != io.EOF { + traceln("transporter.rv.decoding.error:", err) + return nil + } + + return resp +} + +// Sends a SnapshotRequest RPC to a peer. +func (t *HTTPTransporter) SendSnapshotRequest(server *Server, peer *Peer, req *SnapshotRequest) *SnapshotResponse { + return nil +} + +// Sends a SnapshotRequest RPC to a peer. +func (t *HTTPTransporter) SendSnapshotRecoveryRequest(server *Server, peer *Peer, req *SnapshotRecoveryRequest) *SnapshotRecoveryResponse { + return nil +} + +//-------------------------------------- +// Incoming +//-------------------------------------- + +// Handles incoming AppendEntries requests. +func (t *HTTPTransporter) appendEntriesHandler(server *Server) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + traceln(server.Name(), "RECV /appendEntries") + + req := &AppendEntriesRequest{} + if _, err := req.decode(r.Body); err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + + resp := server.AppendEntries(req) + if _, err := resp.encode(w); err != nil { + http.Error(w, "", http.StatusInternalServerError) + return + } + } +} + +// Handles incoming RequestVote requests. +func (t *HTTPTransporter) requestVoteHandler(server *Server) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + traceln(server.Name(), "RECV /requestVote") + + req := &RequestVoteRequest{} + if _, err := req.decode(r.Body); err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + + resp := server.RequestVote(req) + if _, err := resp.encode(w); err != nil { + http.Error(w, "", http.StatusInternalServerError) + return + } + } +} diff --git a/third_party/github.com/coreos/go-raft/http_transporter_test.go b/third_party/github.com/coreos/go-raft/http_transporter_test.go new file mode 100644 index 000000000..3bd4a6d74 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/http_transporter_test.go @@ -0,0 +1,153 @@ +package raft + +import ( + "fmt" + "net" + "net/http" + "sync" + "testing" + "time" +) + +// Ensure that we can start several servers and have them communicate. +func TestHTTPTransporter(t *testing.T) { + transporter := NewHTTPTransporter("/raft") + transporter.DisableKeepAlives = true + + servers := []*Server{} + f0 := func(server *Server, httpServer *http.Server) { + // Stop the leader and wait for an election. + server.Stop() + time.Sleep(testElectionTimeout * 2) + + if servers[1].State() != Leader && servers[2].State() != Leader { + t.Fatal("Expected re-election:", servers[1].State(), servers[2].State()) + } + server.Start() + } + f1 := func(server *Server, httpServer *http.Server) { + } + f2 := func(server *Server, httpServer *http.Server) { + } + runTestHttpServers(t, &servers, transporter, f0, f1, f2) +} + +// Starts multiple independent Raft servers wrapped with HTTP servers. +func runTestHttpServers(t *testing.T, servers *[]*Server, transporter *HTTPTransporter, callbacks ...func(*Server, *http.Server)) { + var wg sync.WaitGroup + httpServers := []*http.Server{} + listeners := []net.Listener{} + for i := range callbacks { + wg.Add(1) + port := 9000 + i + + // Create raft server. + server := newTestServer(fmt.Sprintf("localhost:%d", port), transporter) + server.SetHeartbeatTimeout(testHeartbeatTimeout) + server.SetElectionTimeout(testElectionTimeout) + server.Start() + + defer server.Stop() + *servers = append(*servers, server) + + // Create listener for HTTP server and start it. + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + panic(err) + } + defer listener.Close() + listeners = append(listeners, listener) + + // Create wrapping HTTP server. + mux := http.NewServeMux() + transporter.Install(server, mux) + httpServer := &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: mux} + httpServers = append(httpServers, httpServer) + go func() { httpServer.Serve(listener) }() + } + + // Setup configuration. + for _, server := range *servers { + if _, err := (*servers)[0].Do(&DefaultJoinCommand{Name: server.Name()}); err != nil { + t.Fatalf("Server %s unable to join: %v", server.Name(), err) + } + } + + // Wait for configuration to propagate. + time.Sleep(testHeartbeatTimeout * 2) + + // Execute all the callbacks at the same time. + for _i, _f := range callbacks { + i, f := _i, _f + go func() { + defer wg.Done() + f((*servers)[i], httpServers[i]) + }() + } + + // Wait until everything is done. + wg.Wait() +} + +func BenchmarkSpeed(b *testing.B) { + + transporter := NewHTTPTransporter("/raft") + transporter.DisableKeepAlives = true + + servers := []*Server{} + + for i := 0; i < 3; i++ { + port := 9000 + i + + // Create raft server. + server := newTestServer(fmt.Sprintf("localhost:%d", port), transporter) + server.SetHeartbeatTimeout(testHeartbeatTimeout) + server.SetElectionTimeout(testElectionTimeout) + server.Start() + + defer server.Stop() + servers = append(servers, server) + + // Create listener for HTTP server and start it. + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + panic(err) + } + defer listener.Close() + + // Create wrapping HTTP server. + mux := http.NewServeMux() + transporter.Install(server, mux) + httpServer := &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: mux} + + go func() { httpServer.Serve(listener) }() + } + + // Setup configuration. + for _, server := range servers { + (servers)[0].Do(&DefaultJoinCommand{Name: server.Name()}) + } + + c := make(chan bool) + + // Wait for configuration to propagate. + time.Sleep(testHeartbeatTimeout * 2) + + b.ResetTimer() + for n := 0; n < b.N; n++ { + for i := 0; i < 1000; i++ { + go send(c, servers[0]) + } + + for i := 0; i < 1000; i++ { + <-c + } + } +} + +func send(c chan bool, s *Server) { + for i := 0; i < 20; i++ { + s.Do(&NOPCommand{}) + } + c <- true +} diff --git a/third_party/github.com/coreos/go-raft/join_command.go b/third_party/github.com/coreos/go-raft/join_command.go new file mode 100644 index 000000000..74e14239d --- /dev/null +++ b/third_party/github.com/coreos/go-raft/join_command.go @@ -0,0 +1,28 @@ +package raft + +// Join command interface +type JoinCommand interface { + CommandName() string + Apply(server *Server) (interface{}, error) + NodeName() string +} + +// Join command +type DefaultJoinCommand struct { + Name string `json:"name"` +} + +// The name of the Join command in the log +func (c *DefaultJoinCommand) CommandName() string { + return "raft:join" +} + +func (c *DefaultJoinCommand) Apply(server *Server) (interface{}, error) { + err := server.AddPeer(c.Name) + + return []byte("join"), err +} + +func (c *DefaultJoinCommand) NodeName() string { + return c.Name +} diff --git a/third_party/github.com/coreos/go-raft/leave_command.go b/third_party/github.com/coreos/go-raft/leave_command.go new file mode 100644 index 000000000..c2a4923a0 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/leave_command.go @@ -0,0 +1,27 @@ +package raft + +// Leave command interface +type LeaveCommand interface { + CommandName() string + Apply(server *Server) (interface{}, error) + NodeName() string +} + +// Leave command +type DefaultLeaveCommand struct { + Name string `json:"name"` +} + +// The name of the Leave command in the log +func (c *DefaultLeaveCommand) CommandName() string { + return "raft:leave" +} + +func (c *DefaultLeaveCommand) Apply(server *Server) (interface{}, error) { + err := server.RemovePeer(c.Name) + + return []byte("leave"), err +} +func (c *DefaultLeaveCommand) NodeName() string { + return c.Name +} diff --git a/third_party/github.com/coreos/go-raft/log.go b/third_party/github.com/coreos/go-raft/log.go new file mode 100644 index 000000000..4033e92f9 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/log.go @@ -0,0 +1,610 @@ +package raft + +import ( + "bufio" + "code.google.com/p/goprotobuf/proto" + "errors" + "fmt" + "github.com/benbjohnson/go-raft/protobuf" + "io" + "os" + "sync" +) + +//------------------------------------------------------------------------------ +// +// Typedefs +// +//------------------------------------------------------------------------------ + +// A log is a collection of log entries that are persisted to durable storage. +type Log struct { + ApplyFunc func(Command) (interface{}, error) + file *os.File + path string + entries []*LogEntry + results []*logResult + commitIndex uint64 + mutex sync.RWMutex + startIndex uint64 // the index before the first entry in the Log entries + startTerm uint64 + pBuffer *proto.Buffer + pLogEntry *protobuf.ProtoLogEntry +} + +// The results of the applying a log entry. +type logResult struct { + returnValue interface{} + err error +} + +//------------------------------------------------------------------------------ +// +// Constructor +// +//------------------------------------------------------------------------------ + +// Creates a new log. +func newLog() *Log { + return &Log{ + entries: make([]*LogEntry, 0), + pBuffer: proto.NewBuffer(nil), + pLogEntry: &protobuf.ProtoLogEntry{}, + } +} + +//------------------------------------------------------------------------------ +// +// Accessors +// +//------------------------------------------------------------------------------ + +//-------------------------------------- +// Log Indices +//-------------------------------------- + +// The last committed index in the log. +func (l *Log) CommitIndex() uint64 { + l.mutex.RLock() + defer l.mutex.RUnlock() + return l.commitIndex +} + +// The current index in the log. +func (l *Log) currentIndex() uint64 { + l.mutex.RLock() + defer l.mutex.RUnlock() + + if len(l.entries) == 0 { + return l.startIndex + } + return l.entries[len(l.entries)-1].Index +} + +// The current index in the log without locking +func (l *Log) internalCurrentIndex() uint64 { + if len(l.entries) == 0 { + return l.startIndex + } + return l.entries[len(l.entries)-1].Index +} + +// The next index in the log. +func (l *Log) nextIndex() uint64 { + return l.currentIndex() + 1 +} + +// Determines if the log contains zero entries. +func (l *Log) isEmpty() bool { + l.mutex.RLock() + defer l.mutex.RUnlock() + return (len(l.entries) == 0) && (l.startIndex == 0) +} + +// The name of the last command in the log. +func (l *Log) lastCommandName() string { + l.mutex.RLock() + defer l.mutex.RUnlock() + if len(l.entries) > 0 { + if entry := l.entries[len(l.entries)-1]; entry != nil { + return entry.CommandName + } + } + return "" +} + +//-------------------------------------- +// Log Terms +//-------------------------------------- + +// The current term in the log. +func (l *Log) currentTerm() uint64 { + l.mutex.RLock() + defer l.mutex.RUnlock() + + if len(l.entries) == 0 { + return l.startTerm + } + return l.entries[len(l.entries)-1].Term +} + +//------------------------------------------------------------------------------ +// +// Methods +// +//------------------------------------------------------------------------------ + +//-------------------------------------- +// State +//-------------------------------------- + +// Opens the log file and reads existing entries. The log can remain open and +// continue to append entries to the end of the log. +func (l *Log) open(path string) error { + l.mutex.Lock() + defer l.mutex.Unlock() + + // Read all the entries from the log if one exists. + var readBytes int64 + + var err error + debugln("log.open.open ", path) + // open log file + l.file, err = os.OpenFile(path, os.O_RDWR, 0600) + l.path = path + + if err != nil { + // if the log file does not exist before + // we create the log file and set commitIndex to 0 + if os.IsNotExist(err) { + l.file, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0600) + debugln("log.open.create ", path) + + return err + } + return err + } + debugln("log.open.exist ", path) + + // Read the file and decode entries. + for { + + // Instantiate log entry and decode into it. + entry, _ := newLogEntry(l, 0, 0, nil) + entry.Position, _ = l.file.Seek(0, os.SEEK_CUR) + + n, err := entry.decode(l.file) + if err != nil { + if err == io.EOF { + debugln("open.log.append: finish ") + } else { + if err = os.Truncate(path, readBytes); err != nil { + return fmt.Errorf("raft.Log: Unable to recover: %v", err) + } + } + break + } + + // Append entry. + l.entries = append(l.entries, entry) + debugln("open.log.append log index ", entry.Index) + + readBytes += int64(n) + } + l.results = make([]*logResult, len(l.entries)) + debugln("open.log.recovery number of log ", len(l.entries)) + return nil +} + +// Closes the log file. +func (l *Log) close() { + l.mutex.Lock() + defer l.mutex.Unlock() + + if l.file != nil { + l.file.Close() + l.file = nil + } + l.entries = make([]*LogEntry, 0) + l.results = make([]*logResult, 0) +} + +//-------------------------------------- +// Entries +//-------------------------------------- + +// Creates a log entry associated with this log. +func (l *Log) createEntry(term uint64, command Command) (*LogEntry, error) { + return newLogEntry(l, l.nextIndex(), term, command) +} + +// Retrieves an entry from the log. If the entry has been eliminated because +// of a snapshot then nil is returned. +func (l *Log) getEntry(index uint64) *LogEntry { + l.mutex.RLock() + defer l.mutex.RUnlock() + + if index <= l.startIndex || index > (l.startIndex+uint64(len(l.entries))) { + return nil + } + return l.entries[index-l.startIndex-1] +} + +// Checks if the log contains a given index/term combination. +func (l *Log) containsEntry(index uint64, term uint64) bool { + entry := l.getEntry(index) + return (entry != nil && entry.Term == term) +} + +// Retrieves a list of entries after a given index as well as the term of the +// index provided. A nil list of entries is returned if the index no longer +// exists because a snapshot was made. +func (l *Log) getEntriesAfter(index uint64, maxLogEntriesPerRequest uint64) ([]*LogEntry, uint64) { + l.mutex.Lock() + defer l.mutex.Unlock() + + // Return nil if index is before the start of the log. + if index < l.startIndex { + traceln("log.entriesAfter.before: ", index, " ", l.startIndex) + return nil, 0 + } + + // Return an error if the index doesn't exist. + if index > (uint64(len(l.entries)) + l.startIndex) { + panic(fmt.Sprintf("raft: Index is beyond end of log: %v %v", len(l.entries), index)) + } + + // If we're going from the beginning of the log then return the whole log. + if index == l.startIndex { + traceln("log.entriesAfter.beginning: ", index, " ", l.startIndex) + return l.entries, l.startTerm + } + + traceln("log.entriesAfter.partial: ", index, " ", l.entries[len(l.entries)-1].Index) + + entries := l.entries[index-l.startIndex:] + length := len(entries) + + if uint64(length) < maxLogEntriesPerRequest { + // Determine the term at the given entry and return a subslice. + return entries, l.entries[index-1-l.startIndex].Term + } else { + return entries[:maxLogEntriesPerRequest], l.entries[index-1-l.startIndex].Term + } +} + +// Retrieves the return value and error for an entry. The result can only exist +// after the entry has been committed. +func (l *Log) getEntryResult(entry *LogEntry, clear bool) (interface{}, error) { + l.mutex.RLock() + defer l.mutex.RUnlock() + + if entry == nil { + panic("raft: Log entry required for error retrieval") + } + + // If a result exists for the entry then return it with its error. + if entry.Index > l.startIndex && entry.Index <= uint64(len(l.results)) { + if result := l.results[entry.Index-l.startIndex-1]; result != nil { + + // keep the records before remove it + returnValue, err := result.returnValue, result.err + + // Remove reference to result if it's being cleared after retrieval. + if clear { + result.returnValue = nil + } + + return returnValue, err + } + } + + return nil, nil +} + +//-------------------------------------- +// Commit +//-------------------------------------- + +// Retrieves the last index and term that has been committed to the log. +func (l *Log) commitInfo() (index uint64, term uint64) { + l.mutex.RLock() + defer l.mutex.RUnlock() + + // If we don't have any entries then just return zeros. + if l.commitIndex == 0 { + return 0, 0 + } + + // No new commit log after snapshot + if l.commitIndex == l.startIndex { + return l.startIndex, l.startTerm + } + + // Return the last index & term from the last committed entry. + entry := l.entries[l.commitIndex-1-l.startIndex] + return entry.Index, entry.Term +} + +// Retrieves the last index and term that has been committed to the log. +func (l *Log) lastInfo() (index uint64, term uint64) { + l.mutex.RLock() + defer l.mutex.RUnlock() + + // If we don't have any entries then just return zeros. + if len(l.entries) == 0 { + return l.startIndex, l.startTerm + } + + // Return the last index & term + entry := l.entries[len(l.entries)-1] + return entry.Index, entry.Term +} + +// Updates the commit index +func (l *Log) updateCommitIndex(index uint64) { + l.mutex.Lock() + defer l.mutex.Unlock() + l.commitIndex = index +} + +// Updates the commit index and writes entries after that index to the stable storage. +func (l *Log) setCommitIndex(index uint64) error { + l.mutex.Lock() + defer l.mutex.Unlock() + + // this is not error any more after limited the number of sending entries + // commit up to what we already have + if index > l.startIndex+uint64(len(l.entries)) { + debugln("raft.Log: Commit index", index, "set back to ", len(l.entries)) + index = l.startIndex + uint64(len(l.entries)) + } + + // Do not allow previous indices to be committed again. + + // This could happens, since the guarantee is that the new leader has up-to-dated + // log entires rather than has most up-to-dated committed index + + // For example, Leader 1 send log 80 to follower 2 and follower 3 + // follower 2 and follow 3 all got the new entries and reply + // leader 1 committed entry 80 and send reply to follower 2 and follower3 + // follower 2 receive the new committed index and update committed index to 80 + // leader 1 fail to send the committed index to follower 3 + // follower 3 promote to leader (server 1 and server 2 will vote, since leader 3 + // has up-to-dated the entries) + // when new leader 3 send heartbeat with committed index = 0 to follower 2, + // follower 2 should reply success and let leader 3 update the committed index to 80 + + if index < l.commitIndex { + return nil + } + + // Find all entries whose index is between the previous index and the current index. + for i := l.commitIndex + 1; i <= index; i++ { + entryIndex := i - 1 - l.startIndex + entry := l.entries[entryIndex] + + // Update commit index. + l.commitIndex = entry.Index + + // Decode the command. + command, err := newCommand(entry.CommandName, entry.Command) + if err != nil { + return err + } + + // Apply the changes to the state machine and store the error code. + returnValue, err := l.ApplyFunc(command) + l.results[entryIndex] = &logResult{returnValue: returnValue, err: err} + } + return nil +} + +// Set the commitIndex at the head of the log file to the current +// commit Index. This should be called after obtained a log lock +func (l *Log) flushCommitIndex() { + l.file.Seek(0, os.SEEK_SET) + fmt.Fprintf(l.file, "%8x\n", l.commitIndex) + l.file.Seek(0, os.SEEK_END) +} + +//-------------------------------------- +// Truncation +//-------------------------------------- + +// Truncates the log to the given index and term. This only works if the log +// at the index has not been committed. +func (l *Log) truncate(index uint64, term uint64) error { + l.mutex.Lock() + defer l.mutex.Unlock() + debugln("log.truncate: ", index) + + // Do not allow committed entries to be truncated. + if index < l.commitIndex { + debugln("log.truncate.before") + return fmt.Errorf("raft.Log: Index is already committed (%v): (IDX=%v, TERM=%v)", l.commitIndex, index, term) + } + + // Do not truncate past end of entries. + if index > l.startIndex+uint64(len(l.entries)) { + debugln("log.truncate.after") + return fmt.Errorf("raft.Log: Entry index does not exist (MAX=%v): (IDX=%v, TERM=%v)", len(l.entries), index, term) + } + + // If we're truncating everything then just clear the entries. + if index == l.startIndex { + debugln("log.truncate.clear") + l.file.Truncate(0) + l.file.Seek(0, os.SEEK_SET) + l.entries = []*LogEntry{} + } else { + // Do not truncate if the entry at index does not have the matching term. + entry := l.entries[index-l.startIndex-1] + if len(l.entries) > 0 && entry.Term != term { + debugln("log.truncate.termMismatch") + return fmt.Errorf("raft.Log: Entry at index does not have matching term (%v): (IDX=%v, TERM=%v)", entry.Term, index, term) + } + + // Otherwise truncate up to the desired entry. + if index < l.startIndex+uint64(len(l.entries)) { + debugln("log.truncate.finish") + position := l.entries[index-l.startIndex].Position + l.file.Truncate(position) + l.file.Seek(position, os.SEEK_SET) + l.entries = l.entries[0 : index-l.startIndex] + } + } + + return nil +} + +//-------------------------------------- +// Append +//-------------------------------------- + +// Appends a series of entries to the log. These entries are not written to +// disk until setCommitIndex() is called. +func (l *Log) appendEntries(entries []*LogEntry) error { + l.mutex.Lock() + defer l.mutex.Unlock() + + startPosition, _ := l.file.Seek(0, os.SEEK_CUR) + + w := bufio.NewWriter(l.file) + + var size int64 + var err error + // Append each entry but exit if we hit an error. + for _, entry := range entries { + entry.log = l + if size, err = l.writeEntry(entry, w); err != nil { + return err + } + entry.Position = startPosition + startPosition += size + } + w.Flush() + + return nil +} + +// Writes a single log entry to the end of the log. This function does not +// obtain a lock and should only be used internally. Use AppendEntries() and +// AppendEntry() to use it externally. +func (l *Log) appendEntry(entry *LogEntry) error { + if l.file == nil { + return errors.New("raft.Log: Log is not open") + } + + // Make sure the term and index are greater than the previous. + if len(l.entries) > 0 { + lastEntry := l.entries[len(l.entries)-1] + if entry.Term < lastEntry.Term { + return fmt.Errorf("raft.Log: Cannot append entry with earlier term (%x:%x <= %x:%x)", entry.Term, entry.Index, lastEntry.Term, lastEntry.Index) + } else if entry.Term == lastEntry.Term && entry.Index <= lastEntry.Index { + return fmt.Errorf("raft.Log: Cannot append entry with earlier index in the same term (%x:%x <= %x:%x)", entry.Term, entry.Index, lastEntry.Term, lastEntry.Index) + } + } + + position, _ := l.file.Seek(0, os.SEEK_CUR) + + entry.Position = position + + // Write to storage. + if _, err := entry.encode(l.file); err != nil { + return err + } + + // Append to entries list if stored on disk. + l.entries = append(l.entries, entry) + l.results = append(l.results, nil) + + return nil +} + +// appendEntry with Buffered io +func (l *Log) writeEntry(entry *LogEntry, w io.Writer) (int64, error) { + if l.file == nil { + return -1, errors.New("raft.Log: Log is not open") + } + + // Make sure the term and index are greater than the previous. + if len(l.entries) > 0 { + lastEntry := l.entries[len(l.entries)-1] + if entry.Term < lastEntry.Term { + return -1, fmt.Errorf("raft.Log: Cannot append entry with earlier term (%x:%x <= %x:%x)", entry.Term, entry.Index, lastEntry.Term, lastEntry.Index) + } else if entry.Term == lastEntry.Term && entry.Index <= lastEntry.Index { + return -1, fmt.Errorf("raft.Log: Cannot append entry with earlier index in the same term (%x:%x <= %x:%x)", entry.Term, entry.Index, lastEntry.Term, lastEntry.Index) + } + } + + // Write to storage. + size, err := entry.encode(w) + if err != nil { + return -1, err + } + + // Append to entries list if stored on disk. + l.entries = append(l.entries, entry) + l.results = append(l.results, nil) + + return int64(size), nil +} + +//-------------------------------------- +// Log compaction +//-------------------------------------- + +// compaction the log before index +func (l *Log) compact(index uint64, term uint64) error { + var entries []*LogEntry + + l.mutex.Lock() + defer l.mutex.Unlock() + + // nothing to compaction + // the index may be greater than the current index if + // we just recovery from on snapshot + if index >= l.internalCurrentIndex() { + entries = make([]*LogEntry, 0) + } else { + + // get all log entries after index + entries = l.entries[index-l.startIndex:] + } + + // create a new log file and add all the entries + file, err := os.OpenFile(l.path+".new", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return err + } + for _, entry := range entries { + position, _ := l.file.Seek(0, os.SEEK_CUR) + entry.Position = position + + if _, err = entry.encode(file); err != nil { + return err + } + } + // close the current log file + l.file.Close() + + // remove the current log file to .bak + err = os.Remove(l.path) + if err != nil { + return err + } + + // rename the new log file + err = os.Rename(l.path+".new", l.path) + if err != nil { + return err + } + l.file = file + + // compaction the in memory log + l.entries = entries + l.startIndex = index + l.startTerm = term + return nil +} diff --git a/third_party/github.com/coreos/go-raft/log_entry.go b/third_party/github.com/coreos/go-raft/log_entry.go new file mode 100644 index 000000000..a1a505c7d --- /dev/null +++ b/third_party/github.com/coreos/go-raft/log_entry.go @@ -0,0 +1,99 @@ +package raft + +import ( + "bytes" + "code.google.com/p/goprotobuf/proto" + "encoding/json" + "fmt" + "github.com/benbjohnson/go-raft/protobuf" + "io" +) + +// A log entry stores a single item in the log. +type LogEntry struct { + log *Log + Index uint64 + Term uint64 + CommandName string + Command []byte + Position int64 // position in the log file + commit chan bool +} + +// Creates a new log entry associated with a log. +func newLogEntry(log *Log, index uint64, term uint64, command Command) (*LogEntry, error) { + var buf bytes.Buffer + var commandName string + if command != nil { + commandName = command.CommandName() + if encoder, ok := command.(CommandEncoder); ok { + if err := encoder.Encode(&buf); err != nil { + return nil, err + } + } else { + json.NewEncoder(&buf).Encode(command) + } + } + + e := &LogEntry{ + log: log, + Index: index, + Term: term, + CommandName: commandName, + Command: buf.Bytes(), + commit: make(chan bool, 5), + } + + return e, nil +} + +// Encodes the log entry to a buffer. Returns the number of bytes +// written and any error that may have occurred. +func (e *LogEntry) encode(w io.Writer) (int, error) { + defer e.log.pBuffer.Reset() + e.log.pLogEntry.Index = proto.Uint64(e.Index) + e.log.pLogEntry.Term = proto.Uint64(e.Term) + e.log.pLogEntry.CommandName = proto.String(e.CommandName) + e.log.pLogEntry.Command = e.Command + + err := e.log.pBuffer.Marshal(e.log.pLogEntry) + if err != nil { + return -1, err + } + + if _, err = fmt.Fprintf(w, "%8x\n", len(e.log.pBuffer.Bytes())); err != nil { + return -1, err + } + + return w.Write(e.log.pBuffer.Bytes()) +} + +// Decodes the log entry from a buffer. Returns the number of bytes read and +// any error that occurs. +func (e *LogEntry) decode(r io.Reader) (int, error) { + + var length int + _, err := fmt.Fscanf(r, "%8x\n", &length) + if err != nil { + return -1, err + } + + data := make([]byte, length) + _, err = r.Read(data) + + if err != nil { + return -1, err + } + + pb := &protobuf.ProtoLogEntry{} + if err = proto.Unmarshal(data, pb); err != nil { + return -1, err + } + + e.Term = pb.GetTerm() + e.Index = pb.GetIndex() + e.CommandName = pb.GetCommandName() + e.Command = pb.Command + + return length, nil +} diff --git a/third_party/github.com/coreos/go-raft/log_test.go b/third_party/github.com/coreos/go-raft/log_test.go new file mode 100644 index 000000000..e890090c3 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/log_test.go @@ -0,0 +1,232 @@ +package raft + +import ( + "io/ioutil" + "os" + "reflect" + "testing" +) + +//------------------------------------------------------------------------------ +// +// Tests +// +//------------------------------------------------------------------------------ + +//-------------------------------------- +// Append +//-------------------------------------- + +// Ensure that we can append to a new log. +func TestLogNewLog(t *testing.T) { + path := getLogPath() + log := newLog() + log.ApplyFunc = func(c Command) (interface{}, error) { + return nil, nil + } + if err := log.open(path); err != nil { + t.Fatalf("Unable to open log: %v", err) + } + defer log.close() + defer os.Remove(path) + + e, _ := newLogEntry(log, 1, 1, &testCommand1{Val: "foo", I: 20}) + if err := log.appendEntry(e); err != nil { + t.Fatalf("Unable to append: %v", err) + } + e, _ = newLogEntry(log, 2, 1, &testCommand2{X: 100}) + if err := log.appendEntry(e); err != nil { + t.Fatalf("Unable to append: %v", err) + } + e, _ = newLogEntry(log, 3, 2, &testCommand1{Val: "bar", I: 0}) + if err := log.appendEntry(e); err != nil { + t.Fatalf("Unable to append: %v", err) + } + + // Partial commit. + if err := log.setCommitIndex(2); err != nil { + t.Fatalf("Unable to partially commit: %v", err) + } + if index, term := log.commitInfo(); index != 2 || term != 1 { + t.Fatalf("Invalid commit info [IDX=%v, TERM=%v]", index, term) + } + + // Full commit. + if err := log.setCommitIndex(3); err != nil { + t.Fatalf("Unable to commit: %v", err) + } + if index, term := log.commitInfo(); index != 3 || term != 2 { + t.Fatalf("Invalid commit info [IDX=%v, TERM=%v]", index, term) + } +} + +// Ensure that we can decode and encode to an existing log. +func TestLogExistingLog(t *testing.T) { + tmpLog := newLog() + e0, _ := newLogEntry(tmpLog, 1, 1, &testCommand1{Val: "foo", I: 20}) + e1, _ := newLogEntry(tmpLog, 2, 1, &testCommand2{X: 100}) + e2, _ := newLogEntry(tmpLog, 3, 2, &testCommand1{Val: "bar", I: 0}) + log, path := setupLog([]*LogEntry{e0, e1, e2}) + defer log.close() + defer os.Remove(path) + + // Validate existing log entries. + if len(log.entries) != 3 { + t.Fatalf("Expected 3 entries, got %d", len(log.entries)) + } + if log.entries[0].Index != 1 || log.entries[0].Term != 1 { + t.Fatalf("Unexpected entry[0]: %v", log.entries[0]) + } + if log.entries[1].Index != 2 || log.entries[1].Term != 1 { + t.Fatalf("Unexpected entry[1]: %v", log.entries[1]) + } + if log.entries[2].Index != 3 || log.entries[2].Term != 2 { + t.Fatalf("Unexpected entry[2]: %v", log.entries[2]) + } +} + +// Ensure that we can check the contents of the log by index/term. +func TestLogContainsEntries(t *testing.T) { + tmpLog := newLog() + e0, _ := newLogEntry(tmpLog, 1, 1, &testCommand1{Val: "foo", I: 20}) + e1, _ := newLogEntry(tmpLog, 2, 1, &testCommand2{X: 100}) + e2, _ := newLogEntry(tmpLog, 3, 2, &testCommand1{Val: "bar", I: 0}) + log, path := setupLog([]*LogEntry{e0, e1, e2}) + defer log.close() + defer os.Remove(path) + + if log.containsEntry(0, 0) { + t.Fatalf("Zero-index entry should not exist in log.") + } + if log.containsEntry(1, 0) { + t.Fatalf("Entry with mismatched term should not exist") + } + if log.containsEntry(4, 0) { + t.Fatalf("Out-of-range entry should not exist") + } + if !log.containsEntry(2, 1) { + t.Fatalf("Entry 2/1 should exist") + } + if !log.containsEntry(3, 2) { + t.Fatalf("Entry 2/1 should exist") + } +} + +// Ensure that we can recover from an incomplete/corrupt log and continue logging. +func TestLogRecovery(t *testing.T) { + tmpLog := newLog() + e0, _ := newLogEntry(tmpLog, 1, 1, &testCommand1{Val: "foo", I: 20}) + e1, _ := newLogEntry(tmpLog, 2, 1, &testCommand2{X: 100}) + f, _ := ioutil.TempFile("", "raft-log-") + + e0.encode(f) + e1.encode(f) + f.WriteString("CORRUPT!") + f.Close() + + log := newLog() + log.ApplyFunc = func(c Command) (interface{}, error) { + return nil, nil + } + if err := log.open(f.Name()); err != nil { + t.Fatalf("Unable to open log: %v", err) + } + defer log.close() + defer os.Remove(f.Name()) + + e, _ := newLogEntry(log, 3, 2, &testCommand1{Val: "bat", I: -5}) + if err := log.appendEntry(e); err != nil { + t.Fatalf("Unable to append: %v", err) + } + + // Validate existing log entries. + if len(log.entries) != 3 { + t.Fatalf("Expected 3 entries, got %d", len(log.entries)) + } + if log.entries[0].Index != 1 || log.entries[0].Term != 1 { + t.Fatalf("Unexpected entry[0]: %v", log.entries[0]) + } + if log.entries[1].Index != 2 || log.entries[1].Term != 1 { + t.Fatalf("Unexpected entry[1]: %v", log.entries[1]) + } + if log.entries[2].Index != 3 || log.entries[2].Term != 2 { + t.Fatalf("Unexpected entry[2]: %v", log.entries[2]) + } +} + +//-------------------------------------- +// Append +//-------------------------------------- + +// Ensure that we can truncate uncommitted entries in the log. +func TestLogTruncate(t *testing.T) { + log, path := setupLog(nil) + if err := log.open(path); err != nil { + t.Fatalf("Unable to open log: %v", err) + } + + defer os.Remove(path) + + entry1, _ := newLogEntry(log, 1, 1, &testCommand1{Val: "foo", I: 20}) + if err := log.appendEntry(entry1); err != nil { + t.Fatalf("Unable to append: %v", err) + } + entry2, _ := newLogEntry(log, 2, 1, &testCommand2{X: 100}) + if err := log.appendEntry(entry2); err != nil { + t.Fatalf("Unable to append: %v", err) + } + entry3, _ := newLogEntry(log, 3, 2, &testCommand1{Val: "bar", I: 0}) + if err := log.appendEntry(entry3); err != nil { + t.Fatalf("Unable to append: %v", err) + } + if err := log.setCommitIndex(2); err != nil { + t.Fatalf("Unable to partially commit: %v", err) + } + + // Truncate committed entry. + if err := log.truncate(1, 1); err == nil || err.Error() != "raft.Log: Index is already committed (2): (IDX=1, TERM=1)" { + t.Fatalf("Truncating committed entries shouldn't work: %v", err) + } + // Truncate past end of log. + if err := log.truncate(4, 2); err == nil || err.Error() != "raft.Log: Entry index does not exist (MAX=3): (IDX=4, TERM=2)" { + t.Fatalf("Truncating past end-of-log shouldn't work: %v", err) + } + // Truncate entry with mismatched term. + if err := log.truncate(2, 2); err == nil || err.Error() != "raft.Log: Entry at index does not have matching term (1): (IDX=2, TERM=2)" { + t.Fatalf("Truncating mismatched entries shouldn't work: %v", err) + } + // Truncate end of log. + if err := log.truncate(3, 2); !(err == nil && reflect.DeepEqual(log.entries, []*LogEntry{entry1, entry2, entry3})) { + t.Fatalf("Truncating end of log should work: %v\n\nEntries:\nActual: %v\nExpected: %v", err, log.entries, []*LogEntry{entry1, entry2, entry3}) + } + // Truncate at last commit. + if err := log.truncate(2, 1); !(err == nil && reflect.DeepEqual(log.entries, []*LogEntry{entry1, entry2})) { + t.Fatalf("Truncating at last commit should work: %v\n\nEntries:\nActual: %v\nExpected: %v", err, log.entries, []*LogEntry{entry1, entry2}) + } + + // Append after truncate + if err := log.appendEntry(entry3); err != nil { + t.Fatalf("Unable to append after truncate: %v", err) + } + + log.close() + + // Recovery the truncated log + log = newLog() + if err := log.open(path); err != nil { + t.Fatalf("Unable to open log: %v", err) + } + // Validate existing log entries. + if len(log.entries) != 3 { + t.Fatalf("Expected 3 entries, got %d", len(log.entries)) + } + if log.entries[0].Index != 1 || log.entries[0].Term != 1 { + t.Fatalf("Unexpected entry[0]: %v", log.entries[0]) + } + if log.entries[1].Index != 2 || log.entries[1].Term != 1 { + t.Fatalf("Unexpected entry[1]: %v", log.entries[1]) + } + if log.entries[2].Index != 3 || log.entries[2].Term != 2 { + t.Fatalf("Unexpected entry[2]: %v", log.entries[2]) + } +} diff --git a/third_party/github.com/coreos/go-raft/nop_command.go b/third_party/github.com/coreos/go-raft/nop_command.go new file mode 100644 index 000000000..e3183cdd8 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/nop_command.go @@ -0,0 +1,26 @@ +package raft + +import ( + "io" +) + +// NOP command +type NOPCommand struct { +} + +// The name of the NOP command in the log +func (c NOPCommand) CommandName() string { + return "raft:nop" +} + +func (c NOPCommand) Apply(server *Server) (interface{}, error) { + return nil, nil +} + +func (c NOPCommand) Encode(w io.Writer) error { + return nil +} + +func (c NOPCommand) Decode(r io.Reader) error { + return nil +} diff --git a/third_party/github.com/coreos/go-raft/peer.go b/third_party/github.com/coreos/go-raft/peer.go new file mode 100644 index 000000000..e7761dd97 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/peer.go @@ -0,0 +1,271 @@ +package raft + +import ( + "sync" + "time" +) + +//------------------------------------------------------------------------------ +// +// Typedefs +// +//------------------------------------------------------------------------------ + +// A peer is a reference to another server involved in the consensus protocol. +type Peer struct { + server *Server + name string + prevLogIndex uint64 + mutex sync.RWMutex + stopChan chan bool + heartbeatTimeout time.Duration +} + +//------------------------------------------------------------------------------ +// +// Constructor +// +//------------------------------------------------------------------------------ + +// Creates a new peer. +func newPeer(server *Server, name string, heartbeatTimeout time.Duration) *Peer { + return &Peer{ + server: server, + name: name, + heartbeatTimeout: heartbeatTimeout, + } +} + +//------------------------------------------------------------------------------ +// +// Accessors +// +//------------------------------------------------------------------------------ + +// Retrieves the name of the peer. +func (p *Peer) Name() string { + return p.name +} + +// Sets the heartbeat timeout. +func (p *Peer) setHeartbeatTimeout(duration time.Duration) { + p.heartbeatTimeout = duration +} + +//-------------------------------------- +// Prev log index +//-------------------------------------- + +// Retrieves the previous log index. +func (p *Peer) getPrevLogIndex() uint64 { + p.mutex.RLock() + defer p.mutex.RUnlock() + return p.prevLogIndex +} + +// Sets the previous log index. +func (p *Peer) setPrevLogIndex(value uint64) { + p.mutex.Lock() + defer p.mutex.Unlock() + p.prevLogIndex = value +} + +//------------------------------------------------------------------------------ +// +// Methods +// +//------------------------------------------------------------------------------ + +//-------------------------------------- +// Heartbeat +//-------------------------------------- + +// Starts the peer heartbeat. +func (p *Peer) startHeartbeat() { + p.stopChan = make(chan bool, 1) + c := make(chan bool) + go p.heartbeat(c) + <-c +} + +// Stops the peer heartbeat. +func (p *Peer) stopHeartbeat() { + // here is a problem + // the previous stop is no buffer leader may get blocked + // when heartbeat returns at line 132 + // I make the channel with 1 buffer + // and try to panic here + select { + case p.stopChan <- true: + + default: + panic("[" + p.server.Name() + "] cannot stop [" + p.Name() + "] heartbeat") + } +} + +//-------------------------------------- +// Copying +//-------------------------------------- + +// Clones the state of the peer. The clone is not attached to a server and +// the heartbeat timer will not exist. +func (p *Peer) clone() *Peer { + p.mutex.Lock() + defer p.mutex.Unlock() + return &Peer{ + name: p.name, + prevLogIndex: p.prevLogIndex, + } +} + +//-------------------------------------- +// Heartbeat +//-------------------------------------- + +// Listens to the heartbeat timeout and flushes an AppendEntries RPC. +func (p *Peer) heartbeat(c chan bool) { + stopChan := p.stopChan + + c <- true + + debugln("peer.heartbeat: ", p.Name(), p.heartbeatTimeout) + + for { + select { + case <-stopChan: + debugln("peer.heartbeat.stop: ", p.Name()) + return + + case <-time.After(p.heartbeatTimeout): + debugln("peer.heartbeat.run: ", p.Name()) + prevLogIndex := p.getPrevLogIndex() + entries, prevLogTerm := p.server.log.getEntriesAfter(prevLogIndex, p.server.maxLogEntriesPerRequest) + + if p.server.State() != Leader { + return + } + + if entries != nil { + p.sendAppendEntriesRequest(newAppendEntriesRequest(p.server.currentTerm, prevLogIndex, prevLogTerm, p.server.log.CommitIndex(), p.server.name, entries)) + } else { + p.sendSnapshotRequest(newSnapshotRequest(p.server.name, p.server.lastSnapshot)) + } + } + } +} + +//-------------------------------------- +// Append Entries +//-------------------------------------- + +// Sends an AppendEntries request to the peer through the transport. +func (p *Peer) sendAppendEntriesRequest(req *AppendEntriesRequest) { + traceln("peer.flush.send: ", p.server.Name(), "->", p.Name(), " ", len(req.Entries)) + + resp := p.server.Transporter().SendAppendEntriesRequest(p.server, p, req) + if resp == nil { + debugln("peer.flush.timeout: ", p.server.Name(), "->", p.Name()) + return + } + traceln("peer.flush.recv: ", p.Name()) + + // If successful then update the previous log index. + p.mutex.Lock() + if resp.Success { + if len(req.Entries) > 0 { + p.prevLogIndex = req.Entries[len(req.Entries)-1].Index + + // if peer append a log entry from the current term + // we set append to true + if req.Entries[len(req.Entries)-1].Term == p.server.currentTerm { + resp.append = true + } + } + traceln("peer.flush.success: ", p.server.Name(), "->", p.Name(), "; idx =", p.prevLogIndex) + + // If it was unsuccessful then decrement the previous log index and + // we'll try again next time. + } else { + if resp.CommitIndex >= p.prevLogIndex { + + // we may miss a response from peer + // so maybe the peer has commited the logs we sent + // but we did not receive the success reply and did not increase + // the prevLogIndex + + p.prevLogIndex = resp.CommitIndex + + debugln("peer.flush.commitIndex: ", p.server.Name(), "->", p.Name(), " idx =", p.prevLogIndex) + } else if p.prevLogIndex > 0 { + // Decrement the previous log index down until we find a match. Don't + // let it go below where the peer's commit index is though. That's a + // problem. + p.prevLogIndex-- + // if it not enough, we directly decrease to the index of the + if p.prevLogIndex > resp.Index { + p.prevLogIndex = resp.Index + } + + debugln("peer.flush.decrement: ", p.server.Name(), "->", p.Name(), " idx =", p.prevLogIndex) + } + } + p.mutex.Unlock() + + // Attach the peer to resp, thus server can know where it comes from + resp.peer = p.Name() + // Send response to server for processing. + p.server.send(resp) +} + +// Sends an Snapshot request to the peer through the transport. +func (p *Peer) sendSnapshotRequest(req *SnapshotRequest) { + debugln("peer.snap.send: ", p.name) + + resp := p.server.Transporter().SendSnapshotRequest(p.server, p, req) + if resp == nil { + debugln("peer.snap.timeout: ", p.name) + return + } + + debugln("peer.snap.recv: ", p.name) + + // If successful, the peer should have been to snapshot state + // Send it the snapshot! + if resp.Success { + p.sendSnapshotRecoveryRequest() + } else { + debugln("peer.snap.failed: ", p.name) + return + } + +} + +// Sends an Snapshot Recovery request to the peer through the transport. +func (p *Peer) sendSnapshotRecoveryRequest() { + req := newSnapshotRecoveryRequest(p.server.name, p.server.lastSnapshot) + debugln("peer.snap.recovery.send: ", p.name) + resp := p.server.Transporter().SendSnapshotRecoveryRequest(p.server, p, req) + if resp.Success { + p.prevLogIndex = req.LastIndex + } else { + debugln("peer.snap.recovery.failed: ", p.name) + return + } + // Send response to server for processing. + p.server.send(&AppendEntriesResponse{Term: resp.Term, Success: resp.Success, append: (resp.Term == p.server.currentTerm)}) +} + +//-------------------------------------- +// Vote Requests +//-------------------------------------- + +// send VoteRequest Request +func (p *Peer) sendVoteRequest(req *RequestVoteRequest, c chan *RequestVoteResponse) { + debugln("peer.vote: ", p.server.Name(), "->", p.Name()) + req.peer = p + if resp := p.server.Transporter().SendVoteRequest(p.server, p, req); resp != nil { + debugln("peer.vote: recv", p.server.Name(), "<-", p.Name()) + resp.peer = p + c <- resp + } +} diff --git a/third_party/github.com/coreos/go-raft/protobuf/append_entries_request.pb.go b/third_party/github.com/coreos/go-raft/protobuf/append_entries_request.pb.go new file mode 100644 index 000000000..f7ef595d8 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/append_entries_request.pb.go @@ -0,0 +1,115 @@ +// Code generated by protoc-gen-go. +// source: append_entries_request.proto +// DO NOT EDIT! + +package protobuf + +import proto "code.google.com/p/goprotobuf/proto" +import json "encoding/json" +import math "math" + +// Reference proto, json, and math imports to suppress error if they are not otherwise used. +var _ = proto.Marshal +var _ = &json.SyntaxError{} +var _ = math.Inf + +type ProtoAppendEntriesRequest struct { + Term *uint64 `protobuf:"varint,1,req" json:"Term,omitempty"` + PrevLogIndex *uint64 `protobuf:"varint,2,req" json:"PrevLogIndex,omitempty"` + PrevLogTerm *uint64 `protobuf:"varint,3,req" json:"PrevLogTerm,omitempty"` + CommitIndex *uint64 `protobuf:"varint,4,req" json:"CommitIndex,omitempty"` + LeaderName *string `protobuf:"bytes,5,req" json:"LeaderName,omitempty"` + Entries []*ProtoAppendEntriesRequest_ProtoLogEntry `protobuf:"bytes,6,rep" json:"Entries,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ProtoAppendEntriesRequest) Reset() { *m = ProtoAppendEntriesRequest{} } +func (m *ProtoAppendEntriesRequest) String() string { return proto.CompactTextString(m) } +func (*ProtoAppendEntriesRequest) ProtoMessage() {} + +func (m *ProtoAppendEntriesRequest) GetTerm() uint64 { + if m != nil && m.Term != nil { + return *m.Term + } + return 0 +} + +func (m *ProtoAppendEntriesRequest) GetPrevLogIndex() uint64 { + if m != nil && m.PrevLogIndex != nil { + return *m.PrevLogIndex + } + return 0 +} + +func (m *ProtoAppendEntriesRequest) GetPrevLogTerm() uint64 { + if m != nil && m.PrevLogTerm != nil { + return *m.PrevLogTerm + } + return 0 +} + +func (m *ProtoAppendEntriesRequest) GetCommitIndex() uint64 { + if m != nil && m.CommitIndex != nil { + return *m.CommitIndex + } + return 0 +} + +func (m *ProtoAppendEntriesRequest) GetLeaderName() string { + if m != nil && m.LeaderName != nil { + return *m.LeaderName + } + return "" +} + +func (m *ProtoAppendEntriesRequest) GetEntries() []*ProtoAppendEntriesRequest_ProtoLogEntry { + if m != nil { + return m.Entries + } + return nil +} + +type ProtoAppendEntriesRequest_ProtoLogEntry struct { + Index *uint64 `protobuf:"varint,1,req" json:"Index,omitempty"` + Term *uint64 `protobuf:"varint,2,req" json:"Term,omitempty"` + CommandName *string `protobuf:"bytes,3,req" json:"CommandName,omitempty"` + Command []byte `protobuf:"bytes,4,opt" json:"Command,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ProtoAppendEntriesRequest_ProtoLogEntry) Reset() { + *m = ProtoAppendEntriesRequest_ProtoLogEntry{} +} +func (m *ProtoAppendEntriesRequest_ProtoLogEntry) String() string { return proto.CompactTextString(m) } +func (*ProtoAppendEntriesRequest_ProtoLogEntry) ProtoMessage() {} + +func (m *ProtoAppendEntriesRequest_ProtoLogEntry) GetIndex() uint64 { + if m != nil && m.Index != nil { + return *m.Index + } + return 0 +} + +func (m *ProtoAppendEntriesRequest_ProtoLogEntry) GetTerm() uint64 { + if m != nil && m.Term != nil { + return *m.Term + } + return 0 +} + +func (m *ProtoAppendEntriesRequest_ProtoLogEntry) GetCommandName() string { + if m != nil && m.CommandName != nil { + return *m.CommandName + } + return "" +} + +func (m *ProtoAppendEntriesRequest_ProtoLogEntry) GetCommand() []byte { + if m != nil { + return m.Command + } + return nil +} + +func init() { +} diff --git a/third_party/github.com/coreos/go-raft/protobuf/append_entries_request.proto b/third_party/github.com/coreos/go-raft/protobuf/append_entries_request.proto new file mode 100644 index 000000000..90790d13a --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/append_entries_request.proto @@ -0,0 +1,18 @@ +package protobuf; + +message ProtoAppendEntriesRequest { + required uint64 Term=1; + required uint64 PrevLogIndex=2; + required uint64 PrevLogTerm=3; + required uint64 CommitIndex=4; + required string LeaderName=5; + + message ProtoLogEntry { + required uint64 Index=1; + required uint64 Term=2; + required string CommandName=3; + optional bytes Command=4; + } + + repeated ProtoLogEntry Entries=6; +} \ No newline at end of file diff --git a/third_party/github.com/coreos/go-raft/protobuf/append_entries_responses.pb.go b/third_party/github.com/coreos/go-raft/protobuf/append_entries_responses.pb.go new file mode 100644 index 000000000..30a990d5e --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/append_entries_responses.pb.go @@ -0,0 +1,57 @@ +// Code generated by protoc-gen-go. +// source: append_entries_responses.proto +// DO NOT EDIT! + +package protobuf + +import proto "code.google.com/p/goprotobuf/proto" +import json "encoding/json" +import math "math" + +// Reference proto, json, and math imports to suppress error if they are not otherwise used. +var _ = proto.Marshal +var _ = &json.SyntaxError{} +var _ = math.Inf + +type ProtoAppendEntriesResponse struct { + Term *uint64 `protobuf:"varint,1,req" json:"Term,omitempty"` + Index *uint64 `protobuf:"varint,2,req" json:"Index,omitempty"` + CommitIndex *uint64 `protobuf:"varint,3,req" json:"CommitIndex,omitempty"` + Success *bool `protobuf:"varint,4,req" json:"Success,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ProtoAppendEntriesResponse) Reset() { *m = ProtoAppendEntriesResponse{} } +func (m *ProtoAppendEntriesResponse) String() string { return proto.CompactTextString(m) } +func (*ProtoAppendEntriesResponse) ProtoMessage() {} + +func (m *ProtoAppendEntriesResponse) GetTerm() uint64 { + if m != nil && m.Term != nil { + return *m.Term + } + return 0 +} + +func (m *ProtoAppendEntriesResponse) GetIndex() uint64 { + if m != nil && m.Index != nil { + return *m.Index + } + return 0 +} + +func (m *ProtoAppendEntriesResponse) GetCommitIndex() uint64 { + if m != nil && m.CommitIndex != nil { + return *m.CommitIndex + } + return 0 +} + +func (m *ProtoAppendEntriesResponse) GetSuccess() bool { + if m != nil && m.Success != nil { + return *m.Success + } + return false +} + +func init() { +} diff --git a/third_party/github.com/coreos/go-raft/protobuf/append_entries_responses.proto b/third_party/github.com/coreos/go-raft/protobuf/append_entries_responses.proto new file mode 100644 index 000000000..b6f793249 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/append_entries_responses.proto @@ -0,0 +1,8 @@ +package protobuf; + +message ProtoAppendEntriesResponse { + required uint64 Term=1; + required uint64 Index=2; + required uint64 CommitIndex=3; + required bool Success=4; +} \ No newline at end of file diff --git a/third_party/github.com/coreos/go-raft/protobuf/log_entry.pb.go b/third_party/github.com/coreos/go-raft/protobuf/log_entry.pb.go new file mode 100644 index 000000000..631928e8f --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/log_entry.pb.go @@ -0,0 +1,57 @@ +// Code generated by protoc-gen-go. +// source: log_entry.proto +// DO NOT EDIT! + +package protobuf + +import proto "code.google.com/p/goprotobuf/proto" +import json "encoding/json" +import math "math" + +// Reference proto, json, and math imports to suppress error if they are not otherwise used. +var _ = proto.Marshal +var _ = &json.SyntaxError{} +var _ = math.Inf + +type ProtoLogEntry struct { + Index *uint64 `protobuf:"varint,1,req" json:"Index,omitempty"` + Term *uint64 `protobuf:"varint,2,req" json:"Term,omitempty"` + CommandName *string `protobuf:"bytes,3,req" json:"CommandName,omitempty"` + Command []byte `protobuf:"bytes,4,opt" json:"Command,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ProtoLogEntry) Reset() { *m = ProtoLogEntry{} } +func (m *ProtoLogEntry) String() string { return proto.CompactTextString(m) } +func (*ProtoLogEntry) ProtoMessage() {} + +func (m *ProtoLogEntry) GetIndex() uint64 { + if m != nil && m.Index != nil { + return *m.Index + } + return 0 +} + +func (m *ProtoLogEntry) GetTerm() uint64 { + if m != nil && m.Term != nil { + return *m.Term + } + return 0 +} + +func (m *ProtoLogEntry) GetCommandName() string { + if m != nil && m.CommandName != nil { + return *m.CommandName + } + return "" +} + +func (m *ProtoLogEntry) GetCommand() []byte { + if m != nil { + return m.Command + } + return nil +} + +func init() { +} diff --git a/third_party/github.com/coreos/go-raft/protobuf/log_entry.proto b/third_party/github.com/coreos/go-raft/protobuf/log_entry.proto new file mode 100644 index 000000000..c63d86912 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/log_entry.proto @@ -0,0 +1,8 @@ +package protobuf; + +message ProtoLogEntry { + required uint64 Index=1; + required uint64 Term=2; + required string CommandName=3; + optional bytes Command=4; // for nop-command +} \ No newline at end of file diff --git a/third_party/github.com/coreos/go-raft/protobuf/request_vote_request.pb.go b/third_party/github.com/coreos/go-raft/protobuf/request_vote_request.pb.go new file mode 100644 index 000000000..dc5a2ee9a --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/request_vote_request.pb.go @@ -0,0 +1,57 @@ +// Code generated by protoc-gen-go. +// source: request_vote_request.proto +// DO NOT EDIT! + +package protobuf + +import proto "code.google.com/p/goprotobuf/proto" +import json "encoding/json" +import math "math" + +// Reference proto, json, and math imports to suppress error if they are not otherwise used. +var _ = proto.Marshal +var _ = &json.SyntaxError{} +var _ = math.Inf + +type ProtoRequestVoteRequest struct { + Term *uint64 `protobuf:"varint,1,req" json:"Term,omitempty"` + LastLogIndex *uint64 `protobuf:"varint,2,req" json:"LastLogIndex,omitempty"` + LastLogTerm *uint64 `protobuf:"varint,3,req" json:"LastLogTerm,omitempty"` + CandidateName *string `protobuf:"bytes,4,req" json:"CandidateName,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ProtoRequestVoteRequest) Reset() { *m = ProtoRequestVoteRequest{} } +func (m *ProtoRequestVoteRequest) String() string { return proto.CompactTextString(m) } +func (*ProtoRequestVoteRequest) ProtoMessage() {} + +func (m *ProtoRequestVoteRequest) GetTerm() uint64 { + if m != nil && m.Term != nil { + return *m.Term + } + return 0 +} + +func (m *ProtoRequestVoteRequest) GetLastLogIndex() uint64 { + if m != nil && m.LastLogIndex != nil { + return *m.LastLogIndex + } + return 0 +} + +func (m *ProtoRequestVoteRequest) GetLastLogTerm() uint64 { + if m != nil && m.LastLogTerm != nil { + return *m.LastLogTerm + } + return 0 +} + +func (m *ProtoRequestVoteRequest) GetCandidateName() string { + if m != nil && m.CandidateName != nil { + return *m.CandidateName + } + return "" +} + +func init() { +} diff --git a/third_party/github.com/coreos/go-raft/protobuf/request_vote_request.proto b/third_party/github.com/coreos/go-raft/protobuf/request_vote_request.proto new file mode 100644 index 000000000..e729926ee --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/request_vote_request.proto @@ -0,0 +1,8 @@ +package protobuf; + +message ProtoRequestVoteRequest { + required uint64 Term=1; + required uint64 LastLogIndex=2; + required uint64 LastLogTerm=3; + required string CandidateName=4; +} \ No newline at end of file diff --git a/third_party/github.com/coreos/go-raft/protobuf/request_vote_responses.pb.go b/third_party/github.com/coreos/go-raft/protobuf/request_vote_responses.pb.go new file mode 100644 index 000000000..16e0e582a --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/request_vote_responses.pb.go @@ -0,0 +1,41 @@ +// Code generated by protoc-gen-go. +// source: request_vote_responses.proto +// DO NOT EDIT! + +package protobuf + +import proto "code.google.com/p/goprotobuf/proto" +import json "encoding/json" +import math "math" + +// Reference proto, json, and math imports to suppress error if they are not otherwise used. +var _ = proto.Marshal +var _ = &json.SyntaxError{} +var _ = math.Inf + +type ProtoRequestVoteResponse struct { + Term *uint64 `protobuf:"varint,1,req" json:"Term,omitempty"` + VoteGranted *bool `protobuf:"varint,2,req" json:"VoteGranted,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ProtoRequestVoteResponse) Reset() { *m = ProtoRequestVoteResponse{} } +func (m *ProtoRequestVoteResponse) String() string { return proto.CompactTextString(m) } +func (*ProtoRequestVoteResponse) ProtoMessage() {} + +func (m *ProtoRequestVoteResponse) GetTerm() uint64 { + if m != nil && m.Term != nil { + return *m.Term + } + return 0 +} + +func (m *ProtoRequestVoteResponse) GetVoteGranted() bool { + if m != nil && m.VoteGranted != nil { + return *m.VoteGranted + } + return false +} + +func init() { +} diff --git a/third_party/github.com/coreos/go-raft/protobuf/request_vote_responses.proto b/third_party/github.com/coreos/go-raft/protobuf/request_vote_responses.proto new file mode 100644 index 000000000..577491b61 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/request_vote_responses.proto @@ -0,0 +1,6 @@ +package protobuf; + +message ProtoRequestVoteResponse { + required uint64 Term=1; + required bool VoteGranted=2; +} \ No newline at end of file diff --git a/third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_request.pb.go b/third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_request.pb.go new file mode 100644 index 000000000..f580de6ab --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_request.pb.go @@ -0,0 +1,65 @@ +// Code generated by protoc-gen-go. +// source: snapshot_recovery_request.proto +// DO NOT EDIT! + +package protobuf + +import proto "code.google.com/p/goprotobuf/proto" +import json "encoding/json" +import math "math" + +// Reference proto, json, and math imports to suppress error if they are not otherwise used. +var _ = proto.Marshal +var _ = &json.SyntaxError{} +var _ = math.Inf + +type ProtoSnapshotRecoveryRequest struct { + LeaderName *string `protobuf:"bytes,1,req" json:"LeaderName,omitempty"` + LastIndex *uint64 `protobuf:"varint,2,req" json:"LastIndex,omitempty"` + LastTerm *uint64 `protobuf:"varint,3,req" json:"LastTerm,omitempty"` + Peers []string `protobuf:"bytes,4,rep" json:"Peers,omitempty"` + State []byte `protobuf:"bytes,5,req" json:"State,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ProtoSnapshotRecoveryRequest) Reset() { *m = ProtoSnapshotRecoveryRequest{} } +func (m *ProtoSnapshotRecoveryRequest) String() string { return proto.CompactTextString(m) } +func (*ProtoSnapshotRecoveryRequest) ProtoMessage() {} + +func (m *ProtoSnapshotRecoveryRequest) GetLeaderName() string { + if m != nil && m.LeaderName != nil { + return *m.LeaderName + } + return "" +} + +func (m *ProtoSnapshotRecoveryRequest) GetLastIndex() uint64 { + if m != nil && m.LastIndex != nil { + return *m.LastIndex + } + return 0 +} + +func (m *ProtoSnapshotRecoveryRequest) GetLastTerm() uint64 { + if m != nil && m.LastTerm != nil { + return *m.LastTerm + } + return 0 +} + +func (m *ProtoSnapshotRecoveryRequest) GetPeers() []string { + if m != nil { + return m.Peers + } + return nil +} + +func (m *ProtoSnapshotRecoveryRequest) GetState() []byte { + if m != nil { + return m.State + } + return nil +} + +func init() { +} diff --git a/third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_request.proto b/third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_request.proto new file mode 100644 index 000000000..000c54d48 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_request.proto @@ -0,0 +1,9 @@ +package protobuf; + +message ProtoSnapshotRecoveryRequest { + required string LeaderName=1; + required uint64 LastIndex=2; + required uint64 LastTerm=3; + repeated string Peers=4; + required bytes State=5; +} \ No newline at end of file diff --git a/third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_response.pb.go b/third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_response.pb.go new file mode 100644 index 000000000..62081f5c1 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_response.pb.go @@ -0,0 +1,49 @@ +// Code generated by protoc-gen-go. +// source: snapshot_recovery_response.proto +// DO NOT EDIT! + +package protobuf + +import proto "code.google.com/p/goprotobuf/proto" +import json "encoding/json" +import math "math" + +// Reference proto, json, and math imports to suppress error if they are not otherwise used. +var _ = proto.Marshal +var _ = &json.SyntaxError{} +var _ = math.Inf + +type ProtoSnapshotRecoveryResponse struct { + Term *uint64 `protobuf:"varint,1,req" json:"Term,omitempty"` + Success *bool `protobuf:"varint,2,req" json:"Success,omitempty"` + CommitIndex *uint64 `protobuf:"varint,3,req" json:"CommitIndex,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ProtoSnapshotRecoveryResponse) Reset() { *m = ProtoSnapshotRecoveryResponse{} } +func (m *ProtoSnapshotRecoveryResponse) String() string { return proto.CompactTextString(m) } +func (*ProtoSnapshotRecoveryResponse) ProtoMessage() {} + +func (m *ProtoSnapshotRecoveryResponse) GetTerm() uint64 { + if m != nil && m.Term != nil { + return *m.Term + } + return 0 +} + +func (m *ProtoSnapshotRecoveryResponse) GetSuccess() bool { + if m != nil && m.Success != nil { + return *m.Success + } + return false +} + +func (m *ProtoSnapshotRecoveryResponse) GetCommitIndex() uint64 { + if m != nil && m.CommitIndex != nil { + return *m.CommitIndex + } + return 0 +} + +func init() { +} diff --git a/third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_response.proto b/third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_response.proto new file mode 100644 index 000000000..41ff83d25 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/snapshot_recovery_response.proto @@ -0,0 +1,7 @@ +package protobuf; + +message ProtoSnapshotRecoveryResponse { + required uint64 Term=1; + required bool Success=2; + required uint64 CommitIndex=3; +} \ No newline at end of file diff --git a/third_party/github.com/coreos/go-raft/protobuf/snapshot_request.pb.go b/third_party/github.com/coreos/go-raft/protobuf/snapshot_request.pb.go new file mode 100644 index 000000000..510145748 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/snapshot_request.pb.go @@ -0,0 +1,49 @@ +// Code generated by protoc-gen-go. +// source: snapshot_request.proto +// DO NOT EDIT! + +package protobuf + +import proto "code.google.com/p/goprotobuf/proto" +import json "encoding/json" +import math "math" + +// Reference proto, json, and math imports to suppress error if they are not otherwise used. +var _ = proto.Marshal +var _ = &json.SyntaxError{} +var _ = math.Inf + +type ProtoSnapshotRequest struct { + LeaderName *string `protobuf:"bytes,1,req" json:"LeaderName,omitempty"` + LastIndex *uint64 `protobuf:"varint,2,req" json:"LastIndex,omitempty"` + LastTerm *uint64 `protobuf:"varint,3,req" json:"LastTerm,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ProtoSnapshotRequest) Reset() { *m = ProtoSnapshotRequest{} } +func (m *ProtoSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*ProtoSnapshotRequest) ProtoMessage() {} + +func (m *ProtoSnapshotRequest) GetLeaderName() string { + if m != nil && m.LeaderName != nil { + return *m.LeaderName + } + return "" +} + +func (m *ProtoSnapshotRequest) GetLastIndex() uint64 { + if m != nil && m.LastIndex != nil { + return *m.LastIndex + } + return 0 +} + +func (m *ProtoSnapshotRequest) GetLastTerm() uint64 { + if m != nil && m.LastTerm != nil { + return *m.LastTerm + } + return 0 +} + +func init() { +} diff --git a/third_party/github.com/coreos/go-raft/protobuf/snapshot_request.proto b/third_party/github.com/coreos/go-raft/protobuf/snapshot_request.proto new file mode 100644 index 000000000..2b7c3850f --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/snapshot_request.proto @@ -0,0 +1,7 @@ +package protobuf; + +message ProtoSnapshotRequest { + required string LeaderName=1; + required uint64 LastIndex=2; + required uint64 LastTerm=3; +} \ No newline at end of file diff --git a/third_party/github.com/coreos/go-raft/protobuf/snapshot_response.pb.go b/third_party/github.com/coreos/go-raft/protobuf/snapshot_response.pb.go new file mode 100644 index 000000000..43c05dc61 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/snapshot_response.pb.go @@ -0,0 +1,33 @@ +// Code generated by protoc-gen-go. +// source: snapshot_response.proto +// DO NOT EDIT! + +package protobuf + +import proto "code.google.com/p/goprotobuf/proto" +import json "encoding/json" +import math "math" + +// Reference proto, json, and math imports to suppress error if they are not otherwise used. +var _ = proto.Marshal +var _ = &json.SyntaxError{} +var _ = math.Inf + +type ProtoSnapshotResponse struct { + Success *bool `protobuf:"varint,1,req" json:"Success,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ProtoSnapshotResponse) Reset() { *m = ProtoSnapshotResponse{} } +func (m *ProtoSnapshotResponse) String() string { return proto.CompactTextString(m) } +func (*ProtoSnapshotResponse) ProtoMessage() {} + +func (m *ProtoSnapshotResponse) GetSuccess() bool { + if m != nil && m.Success != nil { + return *m.Success + } + return false +} + +func init() { +} diff --git a/third_party/github.com/coreos/go-raft/protobuf/snapshot_response.proto b/third_party/github.com/coreos/go-raft/protobuf/snapshot_response.proto new file mode 100644 index 000000000..225c19208 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/protobuf/snapshot_response.proto @@ -0,0 +1,5 @@ +package protobuf; + +message ProtoSnapshotResponse { + required bool Success=1; +} \ No newline at end of file diff --git a/third_party/github.com/coreos/go-raft/request_vote_request.go b/third_party/github.com/coreos/go-raft/request_vote_request.go new file mode 100644 index 000000000..c928f5f28 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/request_vote_request.go @@ -0,0 +1,68 @@ +package raft + +import ( + "code.google.com/p/goprotobuf/proto" + "github.com/benbjohnson/go-raft/protobuf" + "io" + "io/ioutil" +) + +// The request sent to a server to vote for a candidate to become a leader. +type RequestVoteRequest struct { + peer *Peer + Term uint64 + LastLogIndex uint64 + LastLogTerm uint64 + CandidateName string +} + +// Creates a new RequestVote request. +func newRequestVoteRequest(term uint64, candidateName string, lastLogIndex uint64, lastLogTerm uint64) *RequestVoteRequest { + return &RequestVoteRequest{ + Term: term, + LastLogIndex: lastLogIndex, + LastLogTerm: lastLogTerm, + CandidateName: candidateName, + } +} + +// Encodes the RequestVoteRequest to a buffer. Returns the number of bytes +// written and any error that may have occurred. +func (req *RequestVoteRequest) encode(w io.Writer) (int, error) { + pb := &protobuf.ProtoRequestVoteRequest{ + Term: proto.Uint64(req.Term), + LastLogIndex: proto.Uint64(req.LastLogIndex), + LastLogTerm: proto.Uint64(req.LastLogTerm), + CandidateName: proto.String(req.CandidateName), + } + p, err := proto.Marshal(pb) + if err != nil { + return -1, err + } + + return w.Write(p) +} + +// Decodes the RequestVoteRequest from a buffer. Returns the number of bytes read and +// any error that occurs. +func (req *RequestVoteRequest) decode(r io.Reader) (int, error) { + data, err := ioutil.ReadAll(r) + + if err != nil { + return -1, err + } + + totalBytes := len(data) + + pb := &protobuf.ProtoRequestVoteRequest{} + if err = proto.Unmarshal(data, pb); err != nil { + return -1, err + } + + req.Term = pb.GetTerm() + req.LastLogIndex = pb.GetLastLogIndex() + req.LastLogTerm = pb.GetLastLogTerm() + req.CandidateName = pb.GetCandidateName() + + return totalBytes, nil +} diff --git a/third_party/github.com/coreos/go-raft/request_vote_response.go b/third_party/github.com/coreos/go-raft/request_vote_response.go new file mode 100644 index 000000000..d12004430 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/request_vote_response.go @@ -0,0 +1,61 @@ +package raft + +import ( + "code.google.com/p/goprotobuf/proto" + "github.com/benbjohnson/go-raft/protobuf" + "io" + "io/ioutil" +) + +// The response returned from a server after a vote for a candidate to become a leader. +type RequestVoteResponse struct { + peer *Peer + Term uint64 + VoteGranted bool +} + +// Creates a new RequestVote response. +func newRequestVoteResponse(term uint64, voteGranted bool) *RequestVoteResponse { + return &RequestVoteResponse{ + Term: term, + VoteGranted: voteGranted, + } +} + +// Encodes the RequestVoteResponse to a buffer. Returns the number of bytes +// written and any error that may have occurred. +func (resp *RequestVoteResponse) encode(w io.Writer) (int, error) { + pb := &protobuf.ProtoRequestVoteResponse{ + Term: proto.Uint64(resp.Term), + VoteGranted: proto.Bool(resp.VoteGranted), + } + + p, err := proto.Marshal(pb) + if err != nil { + return -1, err + } + + return w.Write(p) +} + +// Decodes the RequestVoteResponse from a buffer. Returns the number of bytes read and +// any error that occurs. +func (resp *RequestVoteResponse) decode(r io.Reader) (int, error) { + data, err := ioutil.ReadAll(r) + + if err != nil { + return 0, err + } + + totalBytes := len(data) + + pb := &protobuf.ProtoRequestVoteResponse{} + if err = proto.Unmarshal(data, pb); err != nil { + return -1, err + } + + resp.Term = pb.GetTerm() + resp.VoteGranted = pb.GetVoteGranted() + + return totalBytes, nil +} diff --git a/third_party/github.com/coreos/go-raft/server.go b/third_party/github.com/coreos/go-raft/server.go new file mode 100644 index 000000000..074ca6f26 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/server.go @@ -0,0 +1,1260 @@ +package raft + +import ( + "encoding/json" + "errors" + "fmt" + "hash/crc32" + "io" + "io/ioutil" + "os" + "path" + "sort" + "sync" + "time" +) + +//------------------------------------------------------------------------------ +// +// Constants +// +//------------------------------------------------------------------------------ + +const ( + Stopped = "stopped" + Follower = "follower" + Candidate = "candidate" + Leader = "leader" + Snapshotting = "snapshotting" +) + +const ( + MaxLogEntriesPerRequest = 2000 + NumberOfLogEntriesAfterSnapshot = 200 +) + +const ( + DefaultHeartbeatTimeout = 50 * time.Millisecond + DefaultElectionTimeout = 150 * time.Millisecond +) + +var stopValue interface{} + +//------------------------------------------------------------------------------ +// +// Errors +// +//------------------------------------------------------------------------------ + +var NotLeaderError = errors.New("raft.Server: Not current leader") +var DuplicatePeerError = errors.New("raft.Server: Duplicate peer") +var CommandTimeoutError = errors.New("raft: Command timeout") + +//------------------------------------------------------------------------------ +// +// Typedefs +// +//------------------------------------------------------------------------------ + +// A server is involved in the consensus protocol and can act as a follower, +// candidate or a leader. +type Server struct { + name string + path string + state string + transporter Transporter + context interface{} + currentTerm uint64 + + votedFor string + log *Log + leader string + peers map[string]*Peer + mutex sync.RWMutex + syncedPeer map[string]bool + + c chan *event + electionTimeout time.Duration + heartbeatTimeout time.Duration + + currentSnapshot *Snapshot + lastSnapshot *Snapshot + stateMachine StateMachine + maxLogEntriesPerRequest uint64 + + confFile *os.File +} + +// An event to be processed by the server's event loop. +type event struct { + target interface{} + returnValue interface{} + c chan error +} + +//------------------------------------------------------------------------------ +// +// Constructor +// +//------------------------------------------------------------------------------ + +// Creates a new server with a log at the given path. +func NewServer(name string, path string, transporter Transporter, stateMachine StateMachine, context interface{}) (*Server, error) { + if name == "" { + return nil, errors.New("raft.Server: Name cannot be blank") + } + if transporter == nil { + panic("raft: Transporter required") + } + + s := &Server{ + name: name, + path: path, + transporter: transporter, + stateMachine: stateMachine, + context: context, + state: Stopped, + peers: make(map[string]*Peer), + log: newLog(), + c: make(chan *event, 256), + electionTimeout: DefaultElectionTimeout, + heartbeatTimeout: DefaultHeartbeatTimeout, + maxLogEntriesPerRequest: MaxLogEntriesPerRequest, + } + + // Setup apply function. + s.log.ApplyFunc = func(c Command) (interface{}, error) { + result, err := c.Apply(s) + return result, err + } + + return s, nil +} + +//------------------------------------------------------------------------------ +// +// Accessors +// +//------------------------------------------------------------------------------ + +//-------------------------------------- +// General +//-------------------------------------- + +// Retrieves the name of the server. +func (s *Server) Name() string { + return s.name +} + +// Retrieves the storage path for the server. +func (s *Server) Path() string { + return s.path +} + +// The name of the current leader. +func (s *Server) Leader() string { + return s.leader +} + +// Retrieves a copy of the peer data. +func (s *Server) Peers() map[string]*Peer { + s.mutex.Lock() + defer s.mutex.Unlock() + + peers := make(map[string]*Peer) + for name, peer := range s.peers { + peers[name] = peer.clone() + } + return peers +} + +// Retrieves the object that transports requests. +func (s *Server) Transporter() Transporter { + s.mutex.RLock() + defer s.mutex.RUnlock() + return s.transporter +} + +func (s *Server) SetTransporter(t Transporter) { + s.mutex.Lock() + defer s.mutex.Unlock() + s.transporter = t +} + +// Retrieves the context passed into the constructor. +func (s *Server) Context() interface{} { + return s.context +} + +// Retrieves the log path for the server. +func (s *Server) LogPath() string { + return path.Join(s.path, "log") +} + +// Retrieves the current state of the server. +func (s *Server) State() string { + s.mutex.RLock() + defer s.mutex.RUnlock() + return s.state +} + +// Sets the state of the server. +func (s *Server) setState(state string) { + s.mutex.Lock() + defer s.mutex.Unlock() + s.state = state + if state == Leader { + s.leader = s.Name() + } +} + +// Retrieves the current term of the server. +func (s *Server) Term() uint64 { + return s.currentTerm +} + +// Retrieves the current commit index of the server. +func (s *Server) CommitIndex() uint64 { + return s.log.commitIndex +} + +// Retrieves the name of the candidate this server voted for in this term. +func (s *Server) VotedFor() string { + return s.votedFor +} + +// Retrieves whether the server's log has no entries. +func (s *Server) IsLogEmpty() bool { + return s.log.isEmpty() +} + +// A list of all the log entries. This should only be used for debugging purposes. +func (s *Server) LogEntries() []*LogEntry { + return s.log.entries +} + +// A reference to the command name of the last entry. +func (s *Server) LastCommandName() string { + return s.log.lastCommandName() +} + +// Get the state of the server for debugging +func (s *Server) GetState() string { + s.mutex.RLock() + defer s.mutex.RUnlock() + return fmt.Sprintf("Name: %s, State: %s, Term: %v, Index: %v ", s.name, s.state, s.currentTerm, s.log.commitIndex) +} + +// Check if the server is promotable +func (s *Server) promotable() bool { + return s.log.currentIndex() > 0 +} + +//-------------------------------------- +// Membership +//-------------------------------------- + +// Retrieves the number of member servers in the consensus. +func (s *Server) MemberCount() int { + s.mutex.Lock() + defer s.mutex.Unlock() + return len(s.peers) + 1 +} + +// Retrieves the number of servers required to make a quorum. +func (s *Server) QuorumSize() int { + return (s.MemberCount() / 2) + 1 +} + +//-------------------------------------- +// Election timeout +//-------------------------------------- + +// Retrieves the election timeout. +func (s *Server) ElectionTimeout() time.Duration { + return s.electionTimeout +} + +// Sets the election timeout. +func (s *Server) SetElectionTimeout(duration time.Duration) { + s.electionTimeout = duration +} + +//-------------------------------------- +// Heartbeat timeout +//-------------------------------------- + +// Retrieves the heartbeat timeout. +func (s *Server) HeartbeatTimeout() time.Duration { + return s.heartbeatTimeout +} + +// Sets the heartbeat timeout. +func (s *Server) SetHeartbeatTimeout(duration time.Duration) { + s.mutex.Lock() + defer s.mutex.Unlock() + + s.heartbeatTimeout = duration + for _, peer := range s.peers { + peer.setHeartbeatTimeout(duration) + } +} + +//------------------------------------------------------------------------------ +// +// Methods +// +//------------------------------------------------------------------------------ + +//-------------------------------------- +// Initialization +//-------------------------------------- + +// Reg the NOPCommand +func init() { + RegisterCommand(&NOPCommand{}) + RegisterCommand(&DefaultJoinCommand{}) + RegisterCommand(&DefaultLeaveCommand{}) +} + +// Start as follow +// If log entries exist then allow promotion to candidate if no AEs received. +// If no log entries exist then wait for AEs from another node. +// If no log entries exist and a self-join command is issued then +// immediately become leader and commit entry. + +func (s *Server) Start() error { + // Exit if the server is already running. + if s.state != Stopped { + return errors.New("raft.Server: Server already running") + } + + // Create snapshot directory if not exist + os.Mkdir(path.Join(s.path, "snapshot"), 0700) + + // Initialize the log and load it up. + if err := s.log.open(s.LogPath()); err != nil { + s.debugln("raft: Log error: ", err) + return fmt.Errorf("raft: Initialization error: %s", err) + } + + if err := s.readConf(); err != nil { + s.debugln("raft: Conf file error: ", err) + return fmt.Errorf("raft: Initialization error: %s", err) + } + + // Update the term to the last term in the log. + _, s.currentTerm = s.log.lastInfo() + + s.setState(Follower) + + // If no log entries exist then + // 1. wait for AEs from another node + // 2. wait for self-join command + // to set itself promotable + if !s.promotable() { + s.debugln("start as a new raft server") + + // If log entries exist then allow promotion to candidate + // if no AEs received. + } else { + s.debugln("start from previous saved state") + } + + go s.loop() + + return nil +} + +// Read the configuration for the server. +func (s *Server) readConf() error { + var err error + confPath := path.Join(s.path, "conf") + s.debugln("readConf.open ", confPath) + // open conf file + s.confFile, err = os.OpenFile(confPath, os.O_RDWR, 0600) + + if err != nil { + if os.IsNotExist(err) { + s.confFile, err = os.OpenFile(confPath, os.O_WRONLY|os.O_CREATE, 0600) + debugln("readConf.create ", confPath) + if err != nil { + return err + } + } + return err + } + + for { + var peerName string + _, err = fmt.Fscanf(s.confFile, "%s\n", &peerName) + + if err != nil { + if err == io.EOF { + s.debugln("server.peer.conf: finish") + return nil + } + return err + } + s.debugln("server.peer.conf.read: ", peerName) + + peer := newPeer(s, peerName, s.heartbeatTimeout) + + s.peers[peer.name] = peer + + } + + return nil +} + +// Shuts down the server. +func (s *Server) Stop() { + s.send(&stopValue) + s.mutex.Lock() + s.log.close() + s.mutex.Unlock() +} + +// Checks if the server is currently running. +func (s *Server) Running() bool { + s.mutex.RLock() + defer s.mutex.RUnlock() + return s.state != Stopped +} + +//-------------------------------------- +// Term +//-------------------------------------- + +// Sets the current term for the server. This is only used when an external +// current term is found. +func (s *Server) setCurrentTerm(term uint64, leaderName string, append bool) { + s.mutex.Lock() + defer s.mutex.Unlock() + + // update the term and clear vote for + if term > s.currentTerm { + s.state = Follower + s.currentTerm = term + s.leader = leaderName + s.votedFor = "" + return + } + + // discover new leader when candidate + // save leader name when follower + if term == s.currentTerm && s.state != Leader && append { + s.state = Follower + s.leader = leaderName + } + +} + +//-------------------------------------- +// Event Loop +//-------------------------------------- + +// ________ +// --|Snapshot| timeout +// | -------- ______ +// recover | ^ | | +// snapshot / | |snapshot | | +// higher | | v | recv majority votes +// term | -------- timeout ----------- ----------- +// |-> |Follower| ----------> | Candidate |--------------------> | Leader | +// -------- ----------- ----------- +// ^ higher term/ | higher term | +// | new leader | | +// |_______________________|____________________________________ | +// The main event loop for the server +func (s *Server) loop() { + defer s.debugln("server.loop.end") + + for { + state := s.State() + + s.debugln("server.loop.run ", state) + switch state { + case Follower: + s.followerLoop() + + case Candidate: + s.candidateLoop() + + case Leader: + s.leaderLoop() + + case Snapshotting: + s.snapshotLoop() + + case Stopped: + return + } + } +} + +// Sends an event to the event loop to be processed. The function will wait +// until the event is actually processed before returning. +func (s *Server) send(value interface{}) (interface{}, error) { + event := s.sendAsync(value) + err := <-event.c + return event.returnValue, err +} + +func (s *Server) sendAsync(value interface{}) *event { + event := &event{target: value, c: make(chan error, 1)} + s.c <- event + return event +} + +// The event loop that is run when the server is in a Follower state. +// Responds to RPCs from candidates and leaders. +// Converts to candidate if election timeout elapses without either: +// 1.Receiving valid AppendEntries RPC, or +// 2.Granting vote to candidate +func (s *Server) followerLoop() { + + s.setState(Follower) + timeoutChan := afterBetween(s.ElectionTimeout(), s.ElectionTimeout()*2) + + for { + var err error + update := false + select { + case e := <-s.c: + if e.target == &stopValue { + s.setState(Stopped) + } else if command, ok := e.target.(JoinCommand); ok { + //If no log entries exist and a self-join command is issued + //then immediately become leader and commit entry. + if s.log.currentIndex() == 0 && command.NodeName() == s.Name() { + s.debugln("selfjoin and promote to leader") + s.setState(Leader) + s.processCommand(command, e) + } else { + err = NotLeaderError + } + } else if req, ok := e.target.(*AppendEntriesRequest); ok { + e.returnValue, update = s.processAppendEntriesRequest(req) + } else if req, ok := e.target.(*RequestVoteRequest); ok { + e.returnValue, update = s.processRequestVoteRequest(req) + } else if req, ok := e.target.(*SnapshotRequest); ok { + e.returnValue = s.processSnapshotRequest(req) + } else { + err = NotLeaderError + } + + // Callback to event. + e.c <- err + + case <-timeoutChan: + + // only allow synced follower to promote to candidate + if s.promotable() { + s.setState(Candidate) + } else { + update = true + } + } + + // Converts to candidate if election timeout elapses without either: + // 1.Receiving valid AppendEntries RPC, or + // 2.Granting vote to candidate + if update { + timeoutChan = afterBetween(s.ElectionTimeout(), s.ElectionTimeout()*2) + } + + // Exit loop on state change. + if s.State() != Follower { + break + } + } +} + +// The event loop that is run when the server is in a Candidate state. +func (s *Server) candidateLoop() { + lastLogIndex, lastLogTerm := s.log.lastInfo() + s.leader = "" + + for { + // Increment current term, vote for self. + s.currentTerm++ + s.votedFor = s.name + + // Send RequestVote RPCs to all other servers. + respChan := make(chan *RequestVoteResponse, len(s.peers)) + for _, peer := range s.peers { + go peer.sendVoteRequest(newRequestVoteRequest(s.currentTerm, s.name, lastLogIndex, lastLogTerm), respChan) + } + + // Wait for either: + // * Votes received from majority of servers: become leader + // * AppendEntries RPC received from new leader: step down. + // * Election timeout elapses without election resolution: increment term, start new election + // * Discover higher term: step down (§5.1) + votesGranted := 1 + timeoutChan := afterBetween(s.ElectionTimeout(), s.ElectionTimeout()*2) + timeout := false + + for { + // If we received enough votes then stop waiting for more votes. + s.debugln("server.candidate.votes: ", votesGranted, " quorum:", s.QuorumSize()) + if votesGranted >= s.QuorumSize() { + s.setState(Leader) + break + } + + // Collect votes from peers. + select { + case resp := <-respChan: + if resp.VoteGranted { + s.debugln("server.candidate.vote.granted: ", votesGranted) + votesGranted++ + } else if resp.Term > s.currentTerm { + s.debugln("server.candidate.vote.failed") + s.setCurrentTerm(resp.Term, "", false) + } else { + s.debugln("server.candidate.vote: denied") + } + + case e := <-s.c: + var err error + if e.target == &stopValue { + s.setState(Stopped) + } else if _, ok := e.target.(Command); ok { + err = NotLeaderError + } else if req, ok := e.target.(*AppendEntriesRequest); ok { + e.returnValue, _ = s.processAppendEntriesRequest(req) + } else if req, ok := e.target.(*RequestVoteRequest); ok { + e.returnValue, _ = s.processRequestVoteRequest(req) + } + + // Callback to event. + e.c <- err + + case <-timeoutChan: + timeout = true + } + + // both process AER and RVR can make the server to follower + // also break when timeout happens + if s.State() != Candidate || timeout { + break + } + } + + // break when we are not candidate + if s.State() != Candidate { + break + } + + // continue when timeout happened + } +} + +// The event loop that is run when the server is in a Candidate state. +func (s *Server) leaderLoop() { + s.setState(Leader) + s.syncedPeer = make(map[string]bool) + logIndex, _ := s.log.lastInfo() + + // Update the peers prevLogIndex to leader's lastLogIndex and start heartbeat. + s.debugln("leaderLoop.set.PrevIndex to ", logIndex) + for _, peer := range s.peers { + peer.setPrevLogIndex(logIndex) + peer.startHeartbeat() + } + + go s.Do(NOPCommand{}) + + // Begin to collect response from followers + for { + var err error + select { + case e := <-s.c: + if e.target == &stopValue { + s.setState(Stopped) + } else if command, ok := e.target.(Command); ok { + s.processCommand(command, e) + continue + } else if req, ok := e.target.(*AppendEntriesRequest); ok { + e.returnValue, _ = s.processAppendEntriesRequest(req) + } else if resp, ok := e.target.(*AppendEntriesResponse); ok { + s.processAppendEntriesResponse(resp) + } else if req, ok := e.target.(*RequestVoteRequest); ok { + e.returnValue, _ = s.processRequestVoteRequest(req) + } + + // Callback to event. + e.c <- err + } + + // Exit loop on state change. + if s.State() != Leader { + break + } + } + + // Stop all peers. + for _, peer := range s.peers { + peer.stopHeartbeat() + } + s.syncedPeer = nil +} + +func (s *Server) snapshotLoop() { + s.setState(Snapshotting) + + for { + var err error + + e := <-s.c + + if e.target == &stopValue { + s.setState(Stopped) + } else if _, ok := e.target.(Command); ok { + err = NotLeaderError + } else if req, ok := e.target.(*AppendEntriesRequest); ok { + e.returnValue, _ = s.processAppendEntriesRequest(req) + } else if req, ok := e.target.(*RequestVoteRequest); ok { + e.returnValue, _ = s.processRequestVoteRequest(req) + } else if req, ok := e.target.(*SnapshotRecoveryRequest); ok { + e.returnValue = s.processSnapshotRecoveryRequest(req) + } + + // Callback to event. + e.c <- err + + // Exit loop on state change. + if s.State() != Snapshotting { + break + } + } +} + +//-------------------------------------- +// Commands +//-------------------------------------- + +// Attempts to execute a command and replicate it. The function will return +// when the command has been successfully committed or an error has occurred. + +func (s *Server) Do(command Command) (interface{}, error) { + return s.send(command) +} + +// Processes a command. +func (s *Server) processCommand(command Command, e *event) { + s.debugln("server.command.process") + + // Create an entry for the command in the log. + entry, err := s.log.createEntry(s.currentTerm, command) + + if err != nil { + s.debugln("server.command.log.entry.error:", err) + e.c <- err + return + } + + if err := s.log.appendEntry(entry); err != nil { + s.debugln("server.command.log.error:", err) + e.c <- err + return + } + + // Issue a callback for the entry once it's committed. + go func() { + // Wait for the entry to be committed. + select { + case <-entry.commit: + var err error + s.debugln("server.command.commit") + e.returnValue, err = s.log.getEntryResult(entry, true) + e.c <- err + case <-time.After(time.Second): + s.debugln("server.command.timeout") + e.c <- CommandTimeoutError + } + }() + + // Issue an append entries response for the server. + resp := newAppendEntriesResponse(s.currentTerm, true, s.log.currentIndex(), s.log.CommitIndex()) + resp.append = true + resp.peer = s.Name() + + // this must be async + // sendAsync is not really async every time + // when the sending speed of the user is larger than + // the processing speed of the server, the buffered channel + // will be full. Then sendAsync will become sync, which will + // cause deadlock here. + // so we use a goroutine to avoid the deadlock + go s.sendAsync(resp) +} + +//-------------------------------------- +// Append Entries +//-------------------------------------- + +// Appends zero or more log entry from the leader to this server. +func (s *Server) AppendEntries(req *AppendEntriesRequest) *AppendEntriesResponse { + ret, _ := s.send(req) + resp, _ := ret.(*AppendEntriesResponse) + return resp +} + +// Processes the "append entries" request. +func (s *Server) processAppendEntriesRequest(req *AppendEntriesRequest) (*AppendEntriesResponse, bool) { + + s.traceln("server.ae.process") + + if req.Term < s.currentTerm { + s.debugln("server.ae.error: stale term") + return newAppendEntriesResponse(s.currentTerm, false, s.log.currentIndex(), s.log.CommitIndex()), false + } + + // Update term and leader. + s.setCurrentTerm(req.Term, req.LeaderName, true) + + // Reject if log doesn't contain a matching previous entry. + if err := s.log.truncate(req.PrevLogIndex, req.PrevLogTerm); err != nil { + s.debugln("server.ae.truncate.error: ", err) + return newAppendEntriesResponse(s.currentTerm, false, s.log.currentIndex(), s.log.CommitIndex()), true + } + + // Append entries to the log. + if err := s.log.appendEntries(req.Entries); err != nil { + s.debugln("server.ae.append.error: ", err) + return newAppendEntriesResponse(s.currentTerm, false, s.log.currentIndex(), s.log.CommitIndex()), true + } + + // Commit up to the commit index. + if err := s.log.setCommitIndex(req.CommitIndex); err != nil { + s.debugln("server.ae.commit.error: ", err) + return newAppendEntriesResponse(s.currentTerm, false, s.log.currentIndex(), s.log.CommitIndex()), true + } + + // once the server appended and commited all the log entries from the leader + + return newAppendEntriesResponse(s.currentTerm, true, s.log.currentIndex(), s.log.CommitIndex()), true +} + +// Processes the "append entries" response from the peer. This is only +// processed when the server is a leader. Responses received during other +// states are dropped. +func (s *Server) processAppendEntriesResponse(resp *AppendEntriesResponse) { + + // If we find a higher term then change to a follower and exit. + if resp.Term > s.currentTerm { + s.setCurrentTerm(resp.Term, "", false) + return + } + + // panic response if it's not successful. + if !resp.Success { + return + } + + // if one peer successfully append a log from the leader term, + // we add it to the synced list + if resp.append == true { + s.syncedPeer[resp.peer] = true + } + + // Increment the commit count to make sure we have a quorum before committing. + if len(s.syncedPeer) < s.QuorumSize() { + return + } + + // Determine the committed index that a majority has. + var indices []uint64 + indices = append(indices, s.log.currentIndex()) + for _, peer := range s.peers { + indices = append(indices, peer.getPrevLogIndex()) + } + sort.Sort(uint64Slice(indices)) + + // We can commit up to the index which the majority of the members have appended. + commitIndex := indices[s.QuorumSize()-1] + committedIndex := s.log.commitIndex + + if commitIndex > committedIndex { + s.log.setCommitIndex(commitIndex) + s.debugln("commit index ", commitIndex) + for i := committedIndex; i < commitIndex; i++ { + if entry := s.log.getEntry(i + 1); entry != nil { + // if the leader is a new one and the entry came from the + // old leader, the commit channel will be nil and no go routine + // is waiting from this channel + // if we try to send to it, the new leader will get stuck + if entry.commit != nil { + select { + case entry.commit <- true: + default: + panic("server unable to send signal to commit channel") + } + } + } + } + } +} + +//-------------------------------------- +// Request Vote +//-------------------------------------- + +// Requests a vote from a server. A vote can be obtained if the vote's term is +// at the server's current term and the server has not made a vote yet. A vote +// can also be obtained if the term is greater than the server's current term. +func (s *Server) RequestVote(req *RequestVoteRequest) *RequestVoteResponse { + ret, _ := s.send(req) + resp, _ := ret.(*RequestVoteResponse) + return resp +} + +// Processes a "request vote" request. +func (s *Server) processRequestVoteRequest(req *RequestVoteRequest) (*RequestVoteResponse, bool) { + + // If the request is coming from an old term then reject it. + if req.Term < s.currentTerm { + s.debugln("server.rv.error: stale term") + return newRequestVoteResponse(s.currentTerm, false), false + } + + s.setCurrentTerm(req.Term, "", false) + + // If we've already voted for a different candidate then don't vote for this candidate. + if s.votedFor != "" && s.votedFor != req.CandidateName { + s.debugln("server.rv.error: duplicate vote: ", req.CandidateName, + " already vote for ", s.votedFor) + return newRequestVoteResponse(s.currentTerm, false), false + } + + // If the candidate's log is not at least as up-to-date as our last log then don't vote. + lastIndex, lastTerm := s.log.lastInfo() + if lastIndex > req.LastLogIndex || lastTerm > req.LastLogTerm { + s.debugln("server.rv.error: out of date log: ", req.CandidateName, + "Index :[", lastIndex, "]", " [", req.LastLogIndex, "]", + "Term :[", lastTerm, "]", " [", req.LastLogTerm, "]") + return newRequestVoteResponse(s.currentTerm, false), false + } + + // If we made it this far then cast a vote and reset our election time out. + s.debugln("server.rv.vote: ", s.name, " votes for", req.CandidateName, "at term", req.Term) + s.votedFor = req.CandidateName + + return newRequestVoteResponse(s.currentTerm, true), true +} + +//-------------------------------------- +// Membership +//-------------------------------------- + +// Adds a peer to the server. +func (s *Server) AddPeer(name string) error { + s.debugln("server.peer.add: ", name, len(s.peers)) + + // Do not allow peers to be added twice. + if s.peers[name] != nil { + return nil + } + + // Only add the peer if it doesn't have the same name. + if s.name != name { + _, err := fmt.Fprintln(s.confFile, name) + s.debugln("server.peer.conf.write: ", name) + if err != nil { + return err + } + peer := newPeer(s, name, s.heartbeatTimeout) + if s.State() == Leader { + peer.startHeartbeat() + } + s.peers[peer.name] = peer + } + + return nil +} + +// Removes a peer from the server. +func (s *Server) RemovePeer(name string) error { + s.debugln("server.peer.remove: ", name, len(s.peers)) + + // Ignore removal of the server itself. + if s.name == name { + return nil + } + // Return error if peer doesn't exist. + peer := s.peers[name] + if peer == nil { + return fmt.Errorf("raft: Peer not found: %s", name) + } + + // TODO: Flush entries to the peer first. + + // Stop peer and remove it. + peer.stopHeartbeat() + + delete(s.peers, name) + + s.confFile.Truncate(0) + s.confFile.Seek(0, os.SEEK_SET) + + for peer := range s.peers { + _, err := fmt.Fprintln(s.confFile, peer) + if err != nil { + return err + } + } + + return nil +} + +//-------------------------------------- +// Log compaction +//-------------------------------------- + +// The background snapshot function +func (s *Server) Snapshot() { + for { + // TODO: change this... to something reasonable + time.Sleep(1 * time.Second) + + s.takeSnapshot() + } +} + +func (s *Server) takeSnapshot() error { + //TODO put a snapshot mutex + s.debugln("take Snapshot") + if s.currentSnapshot != nil { + return errors.New("handling snapshot") + } + + lastIndex, lastTerm := s.log.commitInfo() + + if lastIndex == 0 || lastTerm == 0 { + return errors.New("No logs") + } + + path := s.SnapshotPath(lastIndex, lastTerm) + + var state []byte + var err error + + if s.stateMachine != nil { + state, err = s.stateMachine.Save() + + if err != nil { + return err + } + + } else { + state = []byte{0} + } + + var peerNames []string + + for _, peer := range s.peers { + peerNames = append(peerNames, peer.Name()) + } + peerNames = append(peerNames, s.Name()) + + s.currentSnapshot = &Snapshot{lastIndex, lastTerm, peerNames, state, path} + + s.saveSnapshot() + + // We keep some log entries after the snapshot + // We do not want to send the whole snapshot + // to the slightly slow machines + if lastIndex-s.log.startIndex > NumberOfLogEntriesAfterSnapshot { + compactIndex := lastIndex - NumberOfLogEntriesAfterSnapshot + compactTerm := s.log.getEntry(compactIndex).Term + s.log.compact(compactIndex, compactTerm) + } + + return nil +} + +// Retrieves the log path for the server. +func (s *Server) saveSnapshot() error { + + if s.currentSnapshot == nil { + return errors.New("no snapshot to save") + } + + err := s.currentSnapshot.save() + + if err != nil { + return err + } + + tmp := s.lastSnapshot + s.lastSnapshot = s.currentSnapshot + + // delete the previous snapshot if there is any change + if tmp != nil && !(tmp.LastIndex == s.lastSnapshot.LastIndex && tmp.LastTerm == s.lastSnapshot.LastTerm) { + tmp.remove() + } + s.currentSnapshot = nil + return nil +} + +// Retrieves the log path for the server. +func (s *Server) SnapshotPath(lastIndex uint64, lastTerm uint64) string { + return path.Join(s.path, "snapshot", fmt.Sprintf("%v_%v.ss", lastTerm, lastIndex)) +} + +func (s *Server) RequestSnapshot(req *SnapshotRequest) *SnapshotResponse { + ret, _ := s.send(req) + resp, _ := ret.(*SnapshotResponse) + return resp +} + +func (s *Server) processSnapshotRequest(req *SnapshotRequest) *SnapshotResponse { + + // If the follower’s log contains an entry at the snapshot’s last index with a term + // that matches the snapshot’s last term + // Then the follower already has all the information found in the snapshot + // and can reply false + + entry := s.log.getEntry(req.LastIndex) + + if entry != nil && entry.Term == req.LastTerm { + return newSnapshotResponse(false) + } + + s.setState(Snapshotting) + + return newSnapshotResponse(true) +} + +func (s *Server) SnapshotRecoveryRequest(req *SnapshotRecoveryRequest) *SnapshotRecoveryResponse { + ret, _ := s.send(req) + resp, _ := ret.(*SnapshotRecoveryResponse) + return resp +} + +func (s *Server) processSnapshotRecoveryRequest(req *SnapshotRecoveryRequest) *SnapshotRecoveryResponse { + + s.stateMachine.Recovery(req.State) + + // clear the peer map + s.peers = make(map[string]*Peer) + + // recovery the cluster configuration + for _, peerName := range req.Peers { + s.AddPeer(peerName) + } + + //update term and index + s.currentTerm = req.LastTerm + + s.log.updateCommitIndex(req.LastIndex) + + snapshotPath := s.SnapshotPath(req.LastIndex, req.LastTerm) + + s.currentSnapshot = &Snapshot{req.LastIndex, req.LastTerm, req.Peers, req.State, snapshotPath} + + s.saveSnapshot() + + // clear the previous log entries + s.log.compact(req.LastIndex, req.LastTerm) + + return newSnapshotRecoveryResponse(req.LastTerm, true, req.LastIndex) + +} + +// Load a snapshot at restart +func (s *Server) LoadSnapshot() error { + dir, err := os.OpenFile(path.Join(s.path, "snapshot"), os.O_RDONLY, 0) + if err != nil { + + return err + } + + filenames, err := dir.Readdirnames(-1) + + if err != nil { + dir.Close() + panic(err) + } + + dir.Close() + if len(filenames) == 0 { + return errors.New("no snapshot") + } + + // not sure how many snapshot we should keep + sort.Strings(filenames) + snapshotPath := path.Join(s.path, "snapshot", filenames[len(filenames)-1]) + + // should not fail + file, err := os.OpenFile(snapshotPath, os.O_RDONLY, 0) + defer file.Close() + if err != nil { + panic(err) + } + + // TODO check checksum first + + var snapshotBytes []byte + var checksum uint32 + + n, err := fmt.Fscanf(file, "%08x\n", &checksum) + + if err != nil { + return err + } + + if n != 1 { + return errors.New("Bad snapshot file") + } + + snapshotBytes, _ = ioutil.ReadAll(file) + s.debugln(string(snapshotBytes)) + + // Generate checksum. + byteChecksum := crc32.ChecksumIEEE(snapshotBytes) + + if uint32(checksum) != byteChecksum { + s.debugln(checksum, " ", byteChecksum) + return errors.New("bad snapshot file") + } + + err = json.Unmarshal(snapshotBytes, &s.lastSnapshot) + + if err != nil { + s.debugln("unmarshal error: ", err) + return err + } + + err = s.stateMachine.Recovery(s.lastSnapshot.State) + + if err != nil { + s.debugln("recovery error: ", err) + return err + } + + for _, peerName := range s.lastSnapshot.Peers { + s.AddPeer(peerName) + } + + s.log.startTerm = s.lastSnapshot.LastTerm + s.log.startIndex = s.lastSnapshot.LastIndex + s.log.updateCommitIndex(s.lastSnapshot.LastIndex) + + return err +} + +//-------------------------------------- +// Debugging +//-------------------------------------- + +func (s *Server) debugln(v ...interface{}) { + debugf("[%s Term:%d] %s", s.name, s.currentTerm, fmt.Sprintln(v...)) +} + +func (s *Server) traceln(v ...interface{}) { + tracef("[%s] %s", s.name, fmt.Sprintln(v...)) +} diff --git a/third_party/github.com/coreos/go-raft/server_test.go b/third_party/github.com/coreos/go-raft/server_test.go new file mode 100644 index 000000000..0410846a2 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/server_test.go @@ -0,0 +1,504 @@ +package raft + +import ( + "fmt" + "reflect" + "strconv" + "sync" + "testing" + "time" +) + +//------------------------------------------------------------------------------ +// +// Tests +// +//------------------------------------------------------------------------------ + +//-------------------------------------- +// Request Vote +//-------------------------------------- + +// Ensure that we can request a vote from a server that has not voted. +func TestServerRequestVote(t *testing.T) { + server := newTestServer("1", &testTransporter{}) + + server.Start() + if _, err := server.Do(&DefaultJoinCommand{Name: server.Name()}); err != nil { + t.Fatalf("Server %s unable to join: %v", server.Name(), err) + } + + defer server.Stop() + resp := server.RequestVote(newRequestVoteRequest(1, "foo", 1, 0)) + if resp.Term != 1 || !resp.VoteGranted { + t.Fatalf("Invalid request vote response: %v/%v", resp.Term, resp.VoteGranted) + } +} + +// // Ensure that a vote request is denied if it comes from an old term. +func TestServerRequestVoteDeniedForStaleTerm(t *testing.T) { + server := newTestServer("1", &testTransporter{}) + + server.Start() + if _, err := server.Do(&DefaultJoinCommand{Name: server.Name()}); err != nil { + t.Fatalf("Server %s unable to join: %v", server.Name(), err) + } + + server.currentTerm = 2 + defer server.Stop() + resp := server.RequestVote(newRequestVoteRequest(1, "foo", 1, 0)) + if resp.Term != 2 || resp.VoteGranted { + t.Fatalf("Invalid request vote response: %v/%v", resp.Term, resp.VoteGranted) + } + if server.currentTerm != 2 && server.State() != Follower { + t.Fatalf("Server did not update term and demote: %v / %v", server.currentTerm, server.State()) + } +} + +// Ensure that a vote request is denied if we've already voted for a different candidate. +func TestServerRequestVoteDeniedIfAlreadyVoted(t *testing.T) { + server := newTestServer("1", &testTransporter{}) + + server.Start() + if _, err := server.Do(&DefaultJoinCommand{Name: server.Name()}); err != nil { + t.Fatalf("Server %s unable to join: %v", server.Name(), err) + } + + server.currentTerm = 2 + defer server.Stop() + resp := server.RequestVote(newRequestVoteRequest(2, "foo", 1, 0)) + if resp.Term != 2 || !resp.VoteGranted { + t.Fatalf("First vote should not have been denied") + } + resp = server.RequestVote(newRequestVoteRequest(2, "bar", 1, 0)) + if resp.Term != 2 || resp.VoteGranted { + t.Fatalf("Second vote should have been denied") + } +} + +// Ensure that a vote request is approved if vote occurs in a new term. +func TestServerRequestVoteApprovedIfAlreadyVotedInOlderTerm(t *testing.T) { + server := newTestServer("1", &testTransporter{}) + + server.Start() + if _, err := server.Do(&DefaultJoinCommand{Name: server.Name()}); err != nil { + t.Fatalf("Server %s unable to join: %v", server.Name(), err) + } + + time.Sleep(time.Millisecond * 100) + + server.currentTerm = 2 + defer server.Stop() + resp := server.RequestVote(newRequestVoteRequest(2, "foo", 2, 1)) + if resp.Term != 2 || !resp.VoteGranted || server.VotedFor() != "foo" { + t.Fatalf("First vote should not have been denied") + } + resp = server.RequestVote(newRequestVoteRequest(3, "bar", 2, 1)) + + if resp.Term != 3 || !resp.VoteGranted || server.VotedFor() != "bar" { + t.Fatalf("Second vote should have been approved") + } +} + +// Ensure that a vote request is denied if the log is out of date. +func TestServerRequestVoteDenyIfCandidateLogIsBehind(t *testing.T) { + tmpLog := newLog() + e0, _ := newLogEntry(tmpLog, 1, 1, &testCommand1{Val: "foo", I: 20}) + e1, _ := newLogEntry(tmpLog, 2, 1, &testCommand2{X: 100}) + e2, _ := newLogEntry(tmpLog, 3, 2, &testCommand1{Val: "bar", I: 0}) + server := newTestServerWithLog("1", &testTransporter{}, []*LogEntry{e0, e1, e2}) + + // start as a follower with term 2 and index 3 + server.Start() + + defer server.Stop() + + // request vote from term 3 with last log entry 2, 2 + resp := server.RequestVote(newRequestVoteRequest(3, "foo", 2, 2)) + if resp.Term != 3 || resp.VoteGranted { + t.Fatalf("Stale index vote should have been denied [%v/%v]", resp.Term, resp.VoteGranted) + } + + // request vote from term 2 with last log entry 2, 3 + resp = server.RequestVote(newRequestVoteRequest(2, "foo", 3, 2)) + if resp.Term != 3 || resp.VoteGranted { + t.Fatalf("Stale term vote should have been denied [%v/%v]", resp.Term, resp.VoteGranted) + } + + // request vote from term 3 with last log entry 2, 3 + resp = server.RequestVote(newRequestVoteRequest(3, "foo", 3, 2)) + if resp.Term != 3 || !resp.VoteGranted { + t.Fatalf("Matching log vote should have been granted") + } + + // request vote from term 3 with last log entry 2, 4 + resp = server.RequestVote(newRequestVoteRequest(3, "foo", 4, 2)) + if resp.Term != 3 || !resp.VoteGranted { + t.Fatalf("Ahead-of-log vote should have been granted") + } +} + +// //-------------------------------------- +// // Promotion +// //-------------------------------------- + +// // Ensure that we can self-promote a server to candidate, obtain votes and become a fearless leader. +func TestServerPromoteSelf(t *testing.T) { + e0, _ := newLogEntry(newLog(), 1, 1, &testCommand1{Val: "foo", I: 20}) + server := newTestServerWithLog("1", &testTransporter{}, []*LogEntry{e0}) + + // start as a follower + server.Start() + + defer server.Stop() + + time.Sleep(2 * testElectionTimeout) + + if server.State() != Leader { + t.Fatalf("Server self-promotion failed: %v", server.State()) + } +} + +//Ensure that we can promote a server within a cluster to a leader. +func TestServerPromote(t *testing.T) { + lookup := map[string]*Server{} + transporter := &testTransporter{} + transporter.sendVoteRequestFunc = func(server *Server, peer *Peer, req *RequestVoteRequest) *RequestVoteResponse { + return lookup[peer.Name()].RequestVote(req) + } + transporter.sendAppendEntriesRequestFunc = func(server *Server, peer *Peer, req *AppendEntriesRequest) *AppendEntriesResponse { + return lookup[peer.Name()].AppendEntries(req) + } + servers := newTestCluster([]string{"1", "2", "3"}, transporter, lookup) + + servers[0].Start() + servers[1].Start() + servers[2].Start() + + time.Sleep(2 * testElectionTimeout) + + if servers[0].State() != Leader && servers[1].State() != Leader && servers[2].State() != Leader { + t.Fatalf("No leader elected: (%s, %s, %s)", servers[0].State(), servers[1].State(), servers[2].State()) + } + for _, server := range servers { + server.Stop() + } +} + +//-------------------------------------- +// Append Entries +//-------------------------------------- + +// Ensure we can append entries to a server. +func TestServerAppendEntries(t *testing.T) { + server := newTestServer("1", &testTransporter{}) + + server.SetHeartbeatTimeout(time.Second * 10) + server.Start() + defer server.Stop() + + // Append single entry. + e, _ := newLogEntry(nil, 1, 1, &testCommand1{Val: "foo", I: 10}) + entries := []*LogEntry{e} + resp := server.AppendEntries(newAppendEntriesRequest(1, 0, 0, 0, "ldr", entries)) + if resp.Term != 1 || !resp.Success { + t.Fatalf("AppendEntries failed: %v/%v", resp.Term, resp.Success) + } + if index, term := server.log.commitInfo(); index != 0 || term != 0 { + t.Fatalf("Invalid commit info [IDX=%v, TERM=%v]", index, term) + } + + // Append multiple entries + commit the last one. + e1, _ := newLogEntry(nil, 2, 1, &testCommand1{Val: "bar", I: 20}) + e2, _ := newLogEntry(nil, 3, 1, &testCommand1{Val: "baz", I: 30}) + entries = []*LogEntry{e1, e2} + resp = server.AppendEntries(newAppendEntriesRequest(1, 1, 1, 1, "ldr", entries)) + if resp.Term != 1 || !resp.Success { + t.Fatalf("AppendEntries failed: %v/%v", resp.Term, resp.Success) + } + if index, term := server.log.commitInfo(); index != 1 || term != 1 { + t.Fatalf("Invalid commit info [IDX=%v, TERM=%v]", index, term) + } + + // Send zero entries and commit everything. + resp = server.AppendEntries(newAppendEntriesRequest(2, 3, 1, 3, "ldr", []*LogEntry{})) + if resp.Term != 2 || !resp.Success { + t.Fatalf("AppendEntries failed: %v/%v", resp.Term, resp.Success) + } + if index, term := server.log.commitInfo(); index != 3 || term != 1 { + t.Fatalf("Invalid commit info [IDX=%v, TERM=%v]", index, term) + } +} + +//Ensure that entries with stale terms are rejected. +func TestServerAppendEntriesWithStaleTermsAreRejected(t *testing.T) { + server := newTestServer("1", &testTransporter{}) + + server.Start() + + defer server.Stop() + server.currentTerm = 2 + + // Append single entry. + e, _ := newLogEntry(nil, 1, 1, &testCommand1{Val: "foo", I: 10}) + entries := []*LogEntry{e} + resp := server.AppendEntries(newAppendEntriesRequest(1, 0, 0, 0, "ldr", entries)) + if resp.Term != 2 || resp.Success { + t.Fatalf("AppendEntries should have failed: %v/%v", resp.Term, resp.Success) + } + if index, term := server.log.commitInfo(); index != 0 || term != 0 { + t.Fatalf("Invalid commit info [IDX=%v, TERM=%v]", index, term) + } +} + +// Ensure that we reject entries if the commit log is different. +func TestServerAppendEntriesRejectedIfAlreadyCommitted(t *testing.T) { + server := newTestServer("1", &testTransporter{}) + server.Start() + + defer server.Stop() + + // Append single entry + commit. + e1, _ := newLogEntry(nil, 1, 1, &testCommand1{Val: "foo", I: 10}) + e2, _ := newLogEntry(nil, 2, 1, &testCommand1{Val: "foo", I: 15}) + entries := []*LogEntry{e1, e2} + resp := server.AppendEntries(newAppendEntriesRequest(1, 0, 0, 2, "ldr", entries)) + if resp.Term != 1 || !resp.Success { + t.Fatalf("AppendEntries failed: %v/%v", resp.Term, resp.Success) + } + + // Append entry again (post-commit). + e, _ := newLogEntry(nil, 2, 1, &testCommand1{Val: "bar", I: 20}) + entries = []*LogEntry{e} + resp = server.AppendEntries(newAppendEntriesRequest(1, 2, 1, 1, "ldr", entries)) + if resp.Term != 1 || resp.Success { + t.Fatalf("AppendEntries should have failed: %v/%v", resp.Term, resp.Success) + } +} + +// Ensure that we uncommitted entries are rolled back if new entries overwrite them. +func TestServerAppendEntriesOverwritesUncommittedEntries(t *testing.T) { + server := newTestServer("1", &testTransporter{}) + server.Start() + defer server.Stop() + + entry1, _ := newLogEntry(nil, 1, 1, &testCommand1{Val: "foo", I: 10}) + entry2, _ := newLogEntry(nil, 2, 1, &testCommand1{Val: "foo", I: 15}) + entry3, _ := newLogEntry(nil, 2, 2, &testCommand1{Val: "bar", I: 20}) + + // Append single entry + commit. + entries := []*LogEntry{entry1, entry2} + resp := server.AppendEntries(newAppendEntriesRequest(1, 0, 0, 1, "ldr", entries)) + if resp.Term != 1 || !resp.Success || server.log.commitIndex != 1 || !reflect.DeepEqual(server.log.entries, []*LogEntry{entry1, entry2}) { + t.Fatalf("AppendEntries failed: %v/%v", resp.Term, resp.Success) + } + + // Append entry that overwrites the second (uncommitted) entry. + entries = []*LogEntry{entry3} + resp = server.AppendEntries(newAppendEntriesRequest(2, 1, 1, 2, "ldr", entries)) + if resp.Term != 2 || !resp.Success || server.log.commitIndex != 2 || !reflect.DeepEqual(server.log.entries, []*LogEntry{entry1, entry3}) { + t.Fatalf("AppendEntries should have succeeded: %v/%v", resp.Term, resp.Success) + } +} + +//-------------------------------------- +// Command Execution +//-------------------------------------- + +// Ensure that a follower cannot execute a command. +func TestServerDenyCommandExecutionWhenFollower(t *testing.T) { + server := newTestServer("1", &testTransporter{}) + server.Start() + defer server.Stop() + var err error + if _, err = server.Do(&testCommand1{Val: "foo", I: 10}); err != NotLeaderError { + t.Fatalf("Expected error: %v, got: %v", NotLeaderError, err) + } +} + +//-------------------------------------- +// Membership +//-------------------------------------- + +// Ensure that we can start a single server and append to its log. +func TestServerSingleNode(t *testing.T) { + server := newTestServer("1", &testTransporter{}) + if server.State() != Stopped { + t.Fatalf("Unexpected server state: %v", server.State()) + } + + server.Start() + + time.Sleep(testHeartbeatTimeout) + + // Join the server to itself. + if _, err := server.Do(&DefaultJoinCommand{Name: "1"}); err != nil { + t.Fatalf("Unable to join: %v", err) + } + debugln("finish command") + + if server.State() != Leader { + t.Fatalf("Unexpected server state: %v", server.State()) + } + + server.Stop() + + if server.State() != Stopped { + t.Fatalf("Unexpected server state: %v", server.State()) + } +} + +// Ensure that we can start multiple servers and determine a leader. +func TestServerMultiNode(t *testing.T) { + // Initialize the servers. + var mutex sync.RWMutex + servers := map[string]*Server{} + + transporter := &testTransporter{} + transporter.sendVoteRequestFunc = func(server *Server, peer *Peer, req *RequestVoteRequest) *RequestVoteResponse { + mutex.RLock() + s := servers[peer.name] + mutex.RUnlock() + return s.RequestVote(req) + } + transporter.sendAppendEntriesRequestFunc = func(server *Server, peer *Peer, req *AppendEntriesRequest) *AppendEntriesResponse { + mutex.RLock() + s := servers[peer.name] + mutex.RUnlock() + return s.AppendEntries(req) + } + + disTransporter := &testTransporter{} + disTransporter.sendVoteRequestFunc = func(server *Server, peer *Peer, req *RequestVoteRequest) *RequestVoteResponse { + return nil + } + disTransporter.sendAppendEntriesRequestFunc = func(server *Server, peer *Peer, req *AppendEntriesRequest) *AppendEntriesResponse { + return nil + } + + var names []string + + n := 5 + + // add n servers + for i := 1; i <= n; i++ { + names = append(names, strconv.Itoa(i)) + } + + var leader *Server + for _, name := range names { + server := newTestServer(name, transporter) + defer server.Stop() + + mutex.Lock() + servers[name] = server + mutex.Unlock() + + if name == "1" { + leader = server + server.SetHeartbeatTimeout(testHeartbeatTimeout) + server.Start() + time.Sleep(testHeartbeatTimeout) + } else { + server.SetElectionTimeout(testElectionTimeout) + server.SetHeartbeatTimeout(testHeartbeatTimeout) + server.Start() + time.Sleep(testHeartbeatTimeout) + } + if _, err := leader.Do(&DefaultJoinCommand{Name: name}); err != nil { + t.Fatalf("Unable to join server[%s]: %v", name, err) + } + + } + time.Sleep(2 * testElectionTimeout) + + // Check that two peers exist on leader. + mutex.RLock() + if leader.MemberCount() != n { + t.Fatalf("Expected member count to be %v, got %v", n, leader.MemberCount()) + } + if servers["2"].State() == Leader || servers["3"].State() == Leader { + t.Fatalf("Expected leader should be 1: 2=%v, 3=%v\n", servers["2"].state, servers["3"].state) + } + mutex.RUnlock() + + for i := 0; i < 20; i++ { + retry := 0 + fmt.Println("Round ", i) + + num := strconv.Itoa(i%(len(servers)) + 1) + num_1 := strconv.Itoa((i+3)%(len(servers)) + 1) + toStop := servers[num] + toStop_1 := servers[num_1] + + // Stop the first server and wait for a re-election. + time.Sleep(2 * testElectionTimeout) + debugln("Disconnect ", toStop.Name()) + debugln("disconnect ", num, " ", num_1) + toStop.SetTransporter(disTransporter) + toStop_1.SetTransporter(disTransporter) + time.Sleep(2 * testElectionTimeout) + // Check that either server 2 or 3 is the leader now. + //mutex.Lock() + + leader := 0 + + for key, value := range servers { + debugln("Play begin") + if key != num && key != num_1 { + if value.State() == Leader { + debugln("Found leader") + for i := 0; i < 10; i++ { + debugln("[Test] do ", value.Name()) + if _, err := value.Do(&testCommand2{X: 1}); err != nil { + break + } + debugln("[Test] Done") + } + debugln("Leader is ", value.Name(), " Index ", value.log.commitIndex) + } + debugln("Not Found leader") + } + } + for { + for key, value := range servers { + if key != num && key != num_1 { + if value.State() == Leader { + leader++ + } + debugln(value.Name(), " ", value.currentTerm, " ", value.state) + } + } + + if leader > 1 { + if retry < 300 { + debugln("retry") + retry++ + leader = 0 + time.Sleep(2 * testElectionTimeout) + continue + } + t.Fatalf("wrong leader number %v", leader) + } + if leader == 0 { + if retry < 300 { + retry++ + fmt.Println("retry 0") + leader = 0 + time.Sleep(2 * testElectionTimeout) + continue + } + t.Fatalf("wrong leader number %v", leader) + } + if leader == 1 { + break + } + } + + //mutex.Unlock() + + toStop.SetTransporter(transporter) + toStop_1.SetTransporter(transporter) + } + +} diff --git a/third_party/github.com/coreos/go-raft/snapshot.go b/third_party/github.com/coreos/go-raft/snapshot.go new file mode 100644 index 000000000..d35474f8a --- /dev/null +++ b/third_party/github.com/coreos/go-raft/snapshot.go @@ -0,0 +1,65 @@ +package raft + +import ( + //"bytes" + "encoding/json" + "fmt" + "hash/crc32" + "os" + "syscall" +) + +//------------------------------------------------------------------------------ +// +// Typedefs +// +//------------------------------------------------------------------------------ + +// the in memory SnapShot struct +// TODO add cluster configuration +type Snapshot struct { + LastIndex uint64 `json:"lastIndex"` + LastTerm uint64 `json:"lastTerm"` + // cluster configuration. + Peers []string `json: "peers"` + State []byte `json: "state"` + Path string `json: "path"` +} + +// Save the snapshot to a file +func (ss *Snapshot) save() error { + // Write machine state to temporary buffer. + + // open file + file, err := os.OpenFile(ss.Path, os.O_CREATE|os.O_WRONLY, 0600) + + if err != nil { + return err + } + + defer file.Close() + + b, err := json.Marshal(ss) + + // Generate checksum. + checksum := crc32.ChecksumIEEE(b) + + // Write snapshot with checksum. + if _, err = fmt.Fprintf(file, "%08x\n", checksum); err != nil { + return err + } + + if _, err = file.Write(b); err != nil { + return err + } + + // force the change writting to disk + syscall.Fsync(int(file.Fd())) + return err +} + +// remove the file of the snapshot +func (ss *Snapshot) remove() error { + err := os.Remove(ss.Path) + return err +} diff --git a/third_party/github.com/coreos/go-raft/snapshot_recovery_request.go b/third_party/github.com/coreos/go-raft/snapshot_recovery_request.go new file mode 100644 index 000000000..2aa0c12e5 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/snapshot_recovery_request.go @@ -0,0 +1,77 @@ +package raft + +import ( + "code.google.com/p/goprotobuf/proto" + "github.com/benbjohnson/go-raft/protobuf" + "io" + "io/ioutil" +) + +// The request sent to a server to start from the snapshot. +type SnapshotRecoveryRequest struct { + LeaderName string + LastIndex uint64 + LastTerm uint64 + Peers []string + State []byte +} + +//------------------------------------------------------------------------------ +// +// Constructors +// +//------------------------------------------------------------------------------ + +// Creates a new Snapshot request. +func newSnapshotRecoveryRequest(leaderName string, snapshot *Snapshot) *SnapshotRecoveryRequest { + return &SnapshotRecoveryRequest{ + LeaderName: leaderName, + LastIndex: snapshot.LastIndex, + LastTerm: snapshot.LastTerm, + Peers: snapshot.Peers, + State: snapshot.State, + } +} + +// Encodes the SnapshotRecoveryRequest to a buffer. Returns the number of bytes +// written and any error that may have occurred. +func (req *SnapshotRecoveryRequest) encode(w io.Writer) (int, error) { + pb := &protobuf.ProtoSnapshotRecoveryRequest{ + LeaderName: proto.String(req.LeaderName), + LastIndex: proto.Uint64(req.LastIndex), + LastTerm: proto.Uint64(req.LastTerm), + Peers: req.Peers, + State: req.State, + } + p, err := proto.Marshal(pb) + if err != nil { + return -1, err + } + + return w.Write(p) +} + +// Decodes the SnapshotRecoveryRequest from a buffer. Returns the number of bytes read and +// any error that occurs. +func (req *SnapshotRecoveryRequest) decode(r io.Reader) (int, error) { + data, err := ioutil.ReadAll(r) + + if err != nil { + return 0, err + } + + totalBytes := len(data) + + pb := &protobuf.ProtoSnapshotRequest{} + if err = proto.Unmarshal(data, pb); err != nil { + return -1, err + } + + req.LeaderName = pb.GetLeaderName() + req.LastIndex = pb.GetLastIndex() + req.LastTerm = pb.GetLastTerm() + req.Peers = req.Peers + req.State = req.State + + return totalBytes, nil +} diff --git a/third_party/github.com/coreos/go-raft/snapshot_recovery_response.go b/third_party/github.com/coreos/go-raft/snapshot_recovery_response.go new file mode 100644 index 000000000..14f8e0450 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/snapshot_recovery_response.go @@ -0,0 +1,69 @@ +package raft + +import ( + "code.google.com/p/goprotobuf/proto" + "github.com/benbjohnson/go-raft/protobuf" + "io" + "io/ioutil" +) + +// The response returned from a server appending entries to the log. +type SnapshotRecoveryResponse struct { + Term uint64 + Success bool + CommitIndex uint64 +} + +//------------------------------------------------------------------------------ +// +// Constructors +// +//------------------------------------------------------------------------------ + +// Creates a new Snapshot response. +func newSnapshotRecoveryResponse(term uint64, success bool, commitIndex uint64) *SnapshotRecoveryResponse { + return &SnapshotRecoveryResponse{ + Term: term, + Success: success, + CommitIndex: commitIndex, + } +} + +// Encodes the SnapshotRecoveryResponse to a buffer. Returns the number of bytes +// written and any error that may have occurred. +func (req *SnapshotRecoveryResponse) encode(w io.Writer) (int, error) { + pb := &protobuf.ProtoSnapshotRecoveryResponse{ + Term: proto.Uint64(req.Term), + Success: proto.Bool(req.Success), + CommitIndex: proto.Uint64(req.CommitIndex), + } + p, err := proto.Marshal(pb) + if err != nil { + return -1, err + } + + return w.Write(p) +} + +// Decodes the SnapshotRecoveryResponse from a buffer. Returns the number of bytes read and +// any error that occurs. +func (req *SnapshotRecoveryResponse) decode(r io.Reader) (int, error) { + data, err := ioutil.ReadAll(r) + + if err != nil { + return 0, err + } + + totalBytes := len(data) + + pb := &protobuf.ProtoSnapshotRecoveryResponse{} + if err := proto.Unmarshal(data, pb); err != nil { + return -1, err + } + + req.Term = pb.GetTerm() + req.Success = pb.GetSuccess() + req.CommitIndex = pb.GetCommitIndex() + + return totalBytes, nil +} diff --git a/third_party/github.com/coreos/go-raft/snapshot_request.go b/third_party/github.com/coreos/go-raft/snapshot_request.go new file mode 100644 index 000000000..5d37b4ed8 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/snapshot_request.go @@ -0,0 +1,70 @@ +package raft + +import ( + "code.google.com/p/goprotobuf/proto" + "github.com/benbjohnson/go-raft/protobuf" + "io" + "io/ioutil" +) + +// The request sent to a server to start from the snapshot. +type SnapshotRequest struct { + LeaderName string + LastIndex uint64 + LastTerm uint64 +} + +//------------------------------------------------------------------------------ +// +// Constructors +// +//------------------------------------------------------------------------------ + +// Creates a new Snapshot request. +func newSnapshotRequest(leaderName string, snapshot *Snapshot) *SnapshotRequest { + return &SnapshotRequest{ + LeaderName: leaderName, + LastIndex: snapshot.LastIndex, + LastTerm: snapshot.LastTerm, + } +} + +// Encodes the SnapshotRequest to a buffer. Returns the number of bytes +// written and any error that may have occurred. +func (req *SnapshotRequest) encode(w io.Writer) (int, error) { + pb := &protobuf.ProtoSnapshotRequest{ + LeaderName: proto.String(req.LeaderName), + LastIndex: proto.Uint64(req.LastIndex), + LastTerm: proto.Uint64(req.LastTerm), + } + p, err := proto.Marshal(pb) + if err != nil { + return -1, err + } + + return w.Write(p) +} + +// Decodes the SnapshotRequest from a buffer. Returns the number of bytes read and +// any error that occurs. +func (req *SnapshotRequest) decode(r io.Reader) (int, error) { + data, err := ioutil.ReadAll(r) + + if err != nil { + return 0, err + } + + totalBytes := len(data) + + pb := &protobuf.ProtoSnapshotRequest{} + + if err := proto.Unmarshal(data, pb); err != nil { + return -1, err + } + + req.LeaderName = pb.GetLeaderName() + req.LastIndex = pb.GetLastIndex() + req.LastTerm = pb.GetLastTerm() + + return totalBytes, nil +} diff --git a/third_party/github.com/coreos/go-raft/snapshot_response.go b/third_party/github.com/coreos/go-raft/snapshot_response.go new file mode 100644 index 000000000..c3b9ae40d --- /dev/null +++ b/third_party/github.com/coreos/go-raft/snapshot_response.go @@ -0,0 +1,61 @@ +package raft + +import ( + "code.google.com/p/goprotobuf/proto" + "github.com/benbjohnson/go-raft/protobuf" + "io" + "io/ioutil" +) + +// The response returned if the follower entered snapshot state +type SnapshotResponse struct { + Success bool `json:"success"` +} + +//------------------------------------------------------------------------------ +// +// Constructors +// +//------------------------------------------------------------------------------ + +// Creates a new Snapshot response. +func newSnapshotResponse(success bool) *SnapshotResponse { + return &SnapshotResponse{ + Success: success, + } +} + +// Encodes the SnapshotResponse to a buffer. Returns the number of bytes +// written and any error that may have occurred. +func (resp *SnapshotResponse) encode(w io.Writer) (int, error) { + pb := &protobuf.ProtoSnapshotResponse{ + Success: proto.Bool(resp.Success), + } + p, err := proto.Marshal(pb) + if err != nil { + return -1, err + } + + return w.Write(p) +} + +// Decodes the SnapshotResponse from a buffer. Returns the number of bytes read and +// any error that occurs. +func (resp *SnapshotResponse) decode(r io.Reader) (int, error) { + data, err := ioutil.ReadAll(r) + + if err != nil { + return 0, err + } + + totalBytes := len(data) + + pb := &protobuf.ProtoSnapshotResponse{} + if err := proto.Unmarshal(data, pb); err != nil { + return -1, err + } + + resp.Success = pb.GetSuccess() + + return totalBytes, nil +} diff --git a/third_party/github.com/coreos/go-raft/sort.go b/third_party/github.com/coreos/go-raft/sort.go new file mode 100644 index 000000000..bf4c303af --- /dev/null +++ b/third_party/github.com/coreos/go-raft/sort.go @@ -0,0 +1,23 @@ +package raft + +//------------------------------------------------------------------------------ +// +// Typedefs +// +//------------------------------------------------------------------------------ + +type uint64Slice []uint64 + +//------------------------------------------------------------------------------ +// +// Functions +// +//------------------------------------------------------------------------------ + +//-------------------------------------- +// uint64 +//-------------------------------------- + +func (p uint64Slice) Len() int { return len(p) } +func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/third_party/github.com/coreos/go-raft/statemachine.go b/third_party/github.com/coreos/go-raft/statemachine.go new file mode 100644 index 000000000..e59036cef --- /dev/null +++ b/third_party/github.com/coreos/go-raft/statemachine.go @@ -0,0 +1,14 @@ +package raft + +//------------------------------------------------------------------------------ +// +// Typedefs +// +//------------------------------------------------------------------------------ + +// StateMachine is the interface for allowing the host application to save and +// recovery the state machine +type StateMachine interface { + Save() ([]byte, error) + Recovery([]byte) error +} diff --git a/third_party/github.com/coreos/go-raft/test.go b/third_party/github.com/coreos/go-raft/test.go new file mode 100644 index 000000000..606594bf7 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/test.go @@ -0,0 +1,179 @@ +package raft + +import ( + "fmt" + "io/ioutil" + "os" + "time" +) + +const ( + testHeartbeatTimeout = 50 * time.Millisecond + testElectionTimeout = 200 * time.Millisecond +) + +func init() { + RegisterCommand(&testCommand1{}) + RegisterCommand(&testCommand2{}) +} + +//------------------------------------------------------------------------------ +// +// Helpers +// +//------------------------------------------------------------------------------ + +//-------------------------------------- +// Logs +//-------------------------------------- + +func getLogPath() string { + f, _ := ioutil.TempFile("", "raft-log-") + f.Close() + os.Remove(f.Name()) + return f.Name() +} + +func setupLog(entries []*LogEntry) (*Log, string) { + f, _ := ioutil.TempFile("", "raft-log-") + + for _, entry := range entries { + entry.encode(f) + } + err := f.Close() + + if err != nil { + panic(err) + } + + log := newLog() + log.ApplyFunc = func(c Command) (interface{}, error) { + return nil, nil + } + if err := log.open(f.Name()); err != nil { + panic(err) + } + return log, f.Name() +} + +//-------------------------------------- +// Servers +//-------------------------------------- + +func newTestServer(name string, transporter Transporter) *Server { + p, _ := ioutil.TempDir("", "raft-server-") + if err := os.MkdirAll(p, 0644); err != nil { + panic(err.Error()) + } + server, _ := NewServer(name, p, transporter, nil, nil) + return server +} + +func newTestServerWithLog(name string, transporter Transporter, entries []*LogEntry) *Server { + server := newTestServer(name, transporter) + f, err := os.Create(server.LogPath()) + if err != nil { + panic(err) + } + + for _, entry := range entries { + entry.encode(f) + } + f.Close() + return server +} + +func newTestCluster(names []string, transporter Transporter, lookup map[string]*Server) []*Server { + servers := []*Server{} + e0, _ := newLogEntry(newLog(), 1, 1, &testCommand1{Val: "foo", I: 20}) + + for _, name := range names { + if lookup[name] != nil { + panic(fmt.Sprintf("raft: Duplicate server in test cluster! %v", name)) + } + server := newTestServerWithLog("1", transporter, []*LogEntry{e0}) + server.SetElectionTimeout(testElectionTimeout) + servers = append(servers, server) + lookup[name] = server + } + for _, server := range servers { + server.SetHeartbeatTimeout(testHeartbeatTimeout) + server.Start() + for _, peer := range servers { + server.AddPeer(peer.Name()) + } + } + return servers +} + +//-------------------------------------- +// Transporter +//-------------------------------------- + +type testTransporter struct { + sendVoteRequestFunc func(server *Server, peer *Peer, req *RequestVoteRequest) *RequestVoteResponse + sendAppendEntriesRequestFunc func(server *Server, peer *Peer, req *AppendEntriesRequest) *AppendEntriesResponse + sendSnapshotRequestFunc func(server *Server, peer *Peer, req *SnapshotRequest) *SnapshotResponse +} + +func (t *testTransporter) SendVoteRequest(server *Server, peer *Peer, req *RequestVoteRequest) *RequestVoteResponse { + return t.sendVoteRequestFunc(server, peer, req) +} + +func (t *testTransporter) SendAppendEntriesRequest(server *Server, peer *Peer, req *AppendEntriesRequest) *AppendEntriesResponse { + return t.sendAppendEntriesRequestFunc(server, peer, req) +} + +func (t *testTransporter) SendSnapshotRequest(server *Server, peer *Peer, req *SnapshotRequest) *SnapshotResponse { + return t.sendSnapshotRequestFunc(server, peer, req) +} + +func (t *testTransporter) SendSnapshotRecoveryRequest(server *Server, peer *Peer, req *SnapshotRecoveryRequest) *SnapshotRecoveryResponse { + return t.SendSnapshotRecoveryRequest(server, peer, req) +} + +type testStateMachine struct { + saveFunc func() ([]byte, error) + recoveryFunc func([]byte) error +} + +func (sm *testStateMachine) Save() ([]byte, error) { + return sm.saveFunc() +} + +func (sm *testStateMachine) Recovery(state []byte) error { + return sm.recoveryFunc(state) +} + +//-------------------------------------- +// Command1 +//-------------------------------------- + +type testCommand1 struct { + Val string `json:"val"` + I int `json:"i"` +} + +func (c *testCommand1) CommandName() string { + return "cmd_1" +} + +func (c *testCommand1) Apply(server *Server) (interface{}, error) { + return nil, nil +} + +//-------------------------------------- +// Command2 +//-------------------------------------- + +type testCommand2 struct { + X int `json:"x"` +} + +func (c *testCommand2) CommandName() string { + return "cmd_2" +} + +func (c *testCommand2) Apply(server *Server) (interface{}, error) { + return nil, nil +} diff --git a/third_party/github.com/coreos/go-raft/time.go b/third_party/github.com/coreos/go-raft/time.go new file mode 100644 index 000000000..cae863ccf --- /dev/null +++ b/third_party/github.com/coreos/go-raft/time.go @@ -0,0 +1,17 @@ +package raft + +import ( + "math/rand" + "time" +) + +// Waits for a random time between two durations and sends the current time on +// the returned channel. +func afterBetween(min time.Duration, max time.Duration) <-chan time.Time { + rand := rand.New(rand.NewSource(time.Now().UnixNano())) + d, delta := min, (max - min) + if delta > 0 { + d += time.Duration(rand.Int63n(int64(delta))) + } + return time.After(d) +} diff --git a/third_party/github.com/coreos/go-raft/timer.go b/third_party/github.com/coreos/go-raft/timer.go new file mode 100644 index 000000000..d0c258a3e --- /dev/null +++ b/third_party/github.com/coreos/go-raft/timer.go @@ -0,0 +1,170 @@ +package raft + +import ( + "math/rand" + "sync" + "time" +) + +//------------------------------------------------------------------------------ +// +// Typedefs +// +//------------------------------------------------------------------------------ + +type timer struct { + fireChan chan time.Time + stopChan chan bool + state int + + rand *rand.Rand + minDuration time.Duration + maxDuration time.Duration + internalTimer *time.Timer + + mutex sync.Mutex +} + +const ( + STOPPED = iota + READY + RUNNING +) + +//------------------------------------------------------------------------------ +// +// Constructors +// +//------------------------------------------------------------------------------ + +// Creates a new timer. Panics if a non-positive duration is used. +func newTimer(minDuration time.Duration, maxDuration time.Duration) *timer { + if minDuration <= 0 { + panic("raft: Non-positive minimum duration not allowed") + } else if maxDuration <= 0 { + panic("raft: Non-positive maximum duration not allowed") + } else if minDuration > maxDuration { + panic("raft: Minimum duration cannot be greater than maximum duration") + } + + return &timer{ + minDuration: minDuration, + maxDuration: maxDuration, + state: READY, + rand: rand.New(rand.NewSource(time.Now().UnixNano())), + stopChan: make(chan bool, 1), + fireChan: make(chan time.Time), + } +} + +//------------------------------------------------------------------------------ +// +// Accessors +// +//------------------------------------------------------------------------------ + +// Sets the minimum and maximum duration of the timer. +func (t *timer) setDuration(duration time.Duration) { + t.minDuration = duration + t.maxDuration = duration +} + +//------------------------------------------------------------------------------ +// +// Methods +// +//------------------------------------------------------------------------------ + +// Checks if the timer is currently running. +func (t *timer) running() bool { + return t.state == RUNNING +} + +// Stops the timer and closes the channel. +func (t *timer) stop() { + t.mutex.Lock() + defer t.mutex.Unlock() + + if t.internalTimer != nil { + t.internalTimer.Stop() + } + + if t.state != STOPPED { + t.state = STOPPED + + // non-blocking buffer + t.stopChan <- true + } +} + +// Change the state of timer to ready +func (t *timer) ready() { + t.mutex.Lock() + defer t.mutex.Unlock() + + if t.state == RUNNING { + panic("Timer is already running") + } + t.state = READY + t.stopChan = make(chan bool, 1) + t.fireChan = make(chan time.Time) +} + +// Fire at the timer +func (t *timer) fire() { + select { + case t.fireChan <- time.Now(): + return + default: + return + } +} + +// Start the timer, this func will be blocked until the timer: +// (1) times out +// (2) stopped +// (3) fired +// Return false if stopped. +// Make sure the start func will not restart the stopped timer. +func (t *timer) start() bool { + t.mutex.Lock() + + if t.state != READY { + t.mutex.Unlock() + return false + } + t.state = RUNNING + + d := t.minDuration + + if t.maxDuration > t.minDuration { + d += time.Duration(t.rand.Int63n(int64(t.maxDuration - t.minDuration))) + } + + t.internalTimer = time.NewTimer(d) + internalTimer := t.internalTimer + + t.mutex.Unlock() + + // Wait for the timer channel, stop channel or fire channel. + stopped := false + select { + case <-internalTimer.C: + case <-t.fireChan: + case <-t.stopChan: + stopped = true + } + + // Clean up timer and state. + t.mutex.Lock() + t.internalTimer.Stop() + t.internalTimer = nil + if stopped { + t.state = STOPPED + } else if t.state == RUNNING { + t.state = READY + } + t.mutex.Unlock() + + return !stopped +} diff --git a/third_party/github.com/coreos/go-raft/timer_test.go b/third_party/github.com/coreos/go-raft/timer_test.go new file mode 100644 index 000000000..60cd746b2 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/timer_test.go @@ -0,0 +1,86 @@ +package raft + +import ( + "testing" + "time" +) + +//------------------------------------------------------------------------------ +// +// Tests +// +//------------------------------------------------------------------------------ + +// Ensure that we can start an election timer and it will go off in the specified duration. +func TestTimer(t *testing.T) { + timer := newTimer(5*time.Millisecond, 10*time.Millisecond) + + // test timer start + for i := 0; i < 10; i++ { + start := time.Now() + timer.start() + + duration := time.Now().Sub(start) + if duration > 12*time.Millisecond || duration < 5*time.Millisecond { + t.Fatal("Duration Error! ", duration) + } + } + + // test timer stop + for i := 0; i < 100; i++ { + start := time.Now() + go stop(timer) + timer.start() + + duration := time.Now().Sub(start) + if duration > 3*time.Millisecond { + t.Fatal("Duration Error! ", duration) + } + + // ready the timer after stop it + timer.ready() + } + + // test timer fire + for i := 0; i < 100; i++ { + start := time.Now() + go fire(timer) + timer.start() + + duration := time.Now().Sub(start) + if duration > 3*time.Millisecond { + t.Fatal("Fire Duration Error! ", duration) + } + } + + resp := make(chan bool) + + // play with start and stop + // make sure we can stop timer + // in all the possible seq of start and stop + for i := 0; i < 100; i++ { + go stop(timer) + go start(timer, resp) + ret := <-resp + if ret != false { + t.Fatal("cannot stop timer!") + } + timer.ready() + } + +} + +func stop(t *timer) { + time.Sleep(time.Millisecond) + t.stop() +} + +func start(t *timer, resp chan bool) { + time.Sleep(time.Millisecond) + resp <- t.start() +} + +func fire(t *timer) { + time.Sleep(time.Millisecond) + t.fire() +} diff --git a/third_party/github.com/coreos/go-raft/transporter.go b/third_party/github.com/coreos/go-raft/transporter.go new file mode 100644 index 000000000..f7d51e527 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/transporter.go @@ -0,0 +1,16 @@ +package raft + +//------------------------------------------------------------------------------ +// +// Typedefs +// +//------------------------------------------------------------------------------ + +// Transporter is the interface for allowing the host application to transport +// requests to other nodes. +type Transporter interface { + SendVoteRequest(server *Server, peer *Peer, req *RequestVoteRequest) *RequestVoteResponse + SendAppendEntriesRequest(server *Server, peer *Peer, req *AppendEntriesRequest) *AppendEntriesResponse + SendSnapshotRequest(server *Server, peer *Peer, req *SnapshotRequest) *SnapshotResponse + SendSnapshotRecoveryRequest(server *Server, peer *Peer, req *SnapshotRecoveryRequest) *SnapshotRecoveryResponse +} diff --git a/third_party/github.com/coreos/go-raft/z_test.go b/third_party/github.com/coreos/go-raft/z_test.go new file mode 100644 index 000000000..cafdf8905 --- /dev/null +++ b/third_party/github.com/coreos/go-raft/z_test.go @@ -0,0 +1,13 @@ +package raft + +/* +import ( + "testing" + "time" +) + +func TestGC(t *testing.T) { + <-time.After(500 * time.Millisecond) + panic("Oh god no!") +} +*/ From 7992171974079942aeffb753da5e7cd848e071ac Mon Sep 17 00:00:00 2001 From: Brandon Philips Date: Tue, 6 Aug 2013 10:50:08 -0700 Subject: [PATCH 02/10] bump(github.com/ccding/go-logging): d4e747a24b2af160872a886a430a16ac65f76456 --- .../github.com/ccding/go-logging/.gitignore | 22 ++ .../github.com/ccding/go-logging/LICENSE | 191 +++++++++++++ .../github.com/ccding/go-logging/README.md | 226 +++++++++++++++ .../github.com/ccding/go-logging/example.conf | 3 + .../github.com/ccding/go-logging/example.go | 45 +++ .../ccding/go-logging/logging/commands.go | 98 +++++++ .../ccding/go-logging/logging/fields.go | 236 ++++++++++++++++ .../ccding/go-logging/logging/fields_test.go | 60 ++++ .../ccding/go-logging/logging/formater.go | 62 +++++ .../ccding/go-logging/logging/get_go_id.c | 25 ++ .../ccding/go-logging/logging/level.go | 68 +++++ .../ccding/go-logging/logging/logging.go | 260 ++++++++++++++++++ .../ccding/go-logging/logging/logging_test.go | 71 +++++ .../ccding/go-logging/logging/request.go | 24 ++ .../ccding/go-logging/logging/writer.go | 130 +++++++++ 15 files changed, 1521 insertions(+) create mode 100644 third_party/github.com/ccding/go-logging/.gitignore create mode 100644 third_party/github.com/ccding/go-logging/LICENSE create mode 100644 third_party/github.com/ccding/go-logging/README.md create mode 100644 third_party/github.com/ccding/go-logging/example.conf create mode 100644 third_party/github.com/ccding/go-logging/example.go create mode 100644 third_party/github.com/ccding/go-logging/logging/commands.go create mode 100644 third_party/github.com/ccding/go-logging/logging/fields.go create mode 100644 third_party/github.com/ccding/go-logging/logging/fields_test.go create mode 100644 third_party/github.com/ccding/go-logging/logging/formater.go create mode 100644 third_party/github.com/ccding/go-logging/logging/get_go_id.c create mode 100644 third_party/github.com/ccding/go-logging/logging/level.go create mode 100644 third_party/github.com/ccding/go-logging/logging/logging.go create mode 100644 third_party/github.com/ccding/go-logging/logging/logging_test.go create mode 100644 third_party/github.com/ccding/go-logging/logging/request.go create mode 100644 third_party/github.com/ccding/go-logging/logging/writer.go diff --git a/third_party/github.com/ccding/go-logging/.gitignore b/third_party/github.com/ccding/go-logging/.gitignore new file mode 100644 index 000000000..00268614f --- /dev/null +++ b/third_party/github.com/ccding/go-logging/.gitignore @@ -0,0 +1,22 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe diff --git a/third_party/github.com/ccding/go-logging/LICENSE b/third_party/github.com/ccding/go-logging/LICENSE new file mode 100644 index 000000000..37ec93a14 --- /dev/null +++ b/third_party/github.com/ccding/go-logging/LICENSE @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/github.com/ccding/go-logging/README.md b/third_party/github.com/ccding/go-logging/README.md new file mode 100644 index 000000000..dbc8187cb --- /dev/null +++ b/third_party/github.com/ccding/go-logging/README.md @@ -0,0 +1,226 @@ +#go-logging +go-logging is a high-performance logging library for golang. +* Simple: It supports only necessary operations and easy to get start. +* Fast: Asynchronous logging without runtime-related fields has an extremely + low delay of about 800 nano-seconds. + +## Getting Started +The stable version is under the `stable` branch, which does never revert and +is fully tested. The tags in `stable` branch indicate the version numbers. + +However, `master` branch is unstable version, and `dev` branch is development +branch. `master` branch merges `dev` branch periodically. + +Btw, all the pull request should be sent to the `dev` branch. + +### Installation +The step below will download the library source code to +`${GOPATH}/src/github.com/ccding/go-logging`. +```bash +go get github.com/ccding/go-logging/logging +``` + +Given the source code downloaded, it makes you be able to run the examples, +tests, and benchmarks. +```bash +cd ${GOPATH}/src/github.com/ccding/go-logging/logging +go get +go run ../example.go +go test -v -bench . +``` + +### Example +go-logging is used like any other Go libraries. You can simply use the library +in this way. +```go +import "github.com/ccding/go-logging/logging" +``` + +Here is a simple example. +```go +package main + +import ( + "github.com/ccding/go-logging/logging" +) + +func main() { + logger, _ := logging.SimpleLogger("main") + logger.SetLevel(logging.DEBUG) + logger.Error("this is a test from error") + logger.Destroy() +} +``` + +### Configuration +#### Construction Functions +It has the following functions to create a logger. +```go +// with BasicFormat and writing to stdout +SimpleLogger(name string) (*Logger, error) +// with BasicFormat and writing to DefaultFileName +BasicLogger(name string) (*Logger, error) +// with RichFormatand writing to DefaultFileName +RichLogger(name string) (*Logger, error) +// with detailed configuration and writing to file +FileLogger(name string, level Level, format string, timeFormat string, file string, sync bool) (*Logger, error) +// with detailed configuration and writing to a writer +WriterLogger(name string, level Level, format string, timeFormat string, out io.Writer, sync bool) (*Logger, error) +// read configurations from a config file +ConfigLogger(filename string) (*Logger, error) +``` +The meanings of these fields are +```go +name string // logger name +level Level // record level higher than this will be printed +format string // format configuration +timeFormat string // format for time +file string // file name for logging +out io.Writer // writer for logging +sync bool // use sync or async way to record logs +``` +The detailed description of these fields will be presented later. + +#### Logging Functions +It supports the following functions for logging. All of these functions are +thread-safe. +```go +(*Logger) Logf(level Level, format string, v ...interface{}) +(*Logger) Log(level Level, v ...interface{}) +(*Logger) Criticalf(format string, v ...interface{}) +(*Logger) Critical(v ...interface{}) +(*Logger) Fatalf(format string, v ...interface{}) +(*Logger) Fatal(v ...interface{}) +(*Logger) Errorf(format string, v ...interface{}) +(*Logger) Error(v ...interface{}) +(*Logger) Warningf(format string, v ...interface{}) +(*Logger) Warning(v ...interface{}) +(*Logger) Warnf(format string, v ...interface{}) +(*Logger) Warn(v ...interface{}) +(*Logger) Infof(format string, v ...interface{}) +(*Logger) Info(v ...interface{}) +(*Logger) Debugf(format string, v ...interface{}) +(*Logger) Debug(v ...interface{}) +(*Logger) Notsetf(format string, v ...interface{}) +(*Logger) Notset(v ...interface{}) +``` + +#### Logger Operations +The logger supports the following operations. In these functions, `SetWriter` +and `Destroy` are not thread-safe, while others are. All these functions are +running in a synchronous way. +```go +// Getter functions +(*Logger) Name() string // get name +(*Logger) TimeFormat() string // get time format +(*Logger) Level() Level // get level [this function is thread safe] +(*Logger) RecordFormat() string // get the first part of the format +(*Logger) RecordArgs() []string // get the second part of the format +(*Logger) Writer() io.Writer // get writer +(*Logger) Sync() bool // get sync or async + +// Setter functions +(*Logger) SetLevel(level Level) // set level [this function is thread safe] +(*Logger) SetWriter(out ...io.Writer) // set multiple writers + +// Other functions +(*Logger) Flush() // flush the writer +(*Logger) Destroy() // destroy the logger +``` + +#### Fields Description + +##### Name +Name field is a string, which can be written to the logging and used to +separate multiple loggers. It allows two logger having the same name. There +is not any default value for name. + +##### Logging Levels +There are these levels in logging. +```go +CRITICAL 50 +FATAL CRITICAL +ERROR 40 +WARNING 30 +WARN WARNING +INFO 20 +DEBUG 10 +NOTSET 0 +``` + +##### Record Format +The record format is described by a string, which has two parts separated by +`\n`. The first part describes the format of the log, and the second part +lists all the fields to be shown in the log. In other word, the first part is +the first parameter `format` of `fmt.Printf(format string, v ...interface{})`, +and the second part describes the second parameter `v` of it. It is not +allowed to have `\n` in the first part. The fields in the second part are +separated by comma `,`, while extra blank spaces are allowed. An example of +the format string is +```go +const BasicFormat = "%s [%6s] %30s - %s\n name,levelname,time,message" +``` +which is the pre-defined `BasicFormat` used by `BasicLogger()` and +`SimpleLogger()`. + +It supports the following fields for the second part of the format. +```go +"name" string %s // name of the logger +"seqid" uint64 %d // sequence number +"levelno" int32 %d // level number +"levelname" string %s // level name +"created" int64 %d // starting time of the logger +"nsecs" int64 %d // nanosecond of the starting time +"time" string %s // record created time +"timestamp" int64 %d // timestamp of record +"rtime" int64 %d // relative time since started +"filename" string %s // source filename of the caller +"pathname" string %s // filename with path +"module" string %s // executable filename +"lineno" int %d // line number in source code +"funcname" string %s // function name of the caller +"thread" int32 %d // thread id +"process" int %d // process id +"message" string %d // logger message +``` +The following runtime-related fields is extremely expensive and slow, please +be careful when using them. +```go +"filename" string %s // source filename of the caller +"pathname" string %s // filename with path +"lineno" int %d // line number in source code +"funcname" string %s // function name of the caller +"thread" int32 %d // thread id +``` + +There are a few pre-defined values for record format. +```go +BasicFormat = "%s [%6s] %30s - %s\n name,levelname,time,message" +RichFormat = "%s [%6s] %d %30s - %d - %s:%s:%d - %s\n name, levelname, seqid, time, thread, filename, funcname, lineno, message" +``` + +##### Time Format +We use the same time format as golang. The default time format is +```go +DefaultTimeFormat = "2006-01-02 15:04:05.999999999" // default time format +``` + +##### File Name, Writer, and Sync +The meaning of these fields are obvious. Filename is used to create writer. +We also allow the user create a writer by herself and pass it to the logger. +Sync describes whether the user would like to use synchronous or asynchronous +method to write logs. `true` value means synchronous method, and `false` value +means asynchronous way. We suggest you use asynchronous way because it causes +extremely low extra delay by the logging functions. + +## Contributors +In alphabetical order +* Cong Ding ([ccding][ccding]) +* Xiang Li ([xiangli-cmu][xiangli]) +* Zifei Tong ([5kg][5kg]) +[ccding]: //github.com/ccding +[xiangli]: //github.com/xiangli-cmu +[5kg]: //github.com/5kg + +## TODO List +1. logging server diff --git a/third_party/github.com/ccding/go-logging/example.conf b/third_party/github.com/ccding/go-logging/example.conf new file mode 100644 index 000000000..ecc9a860e --- /dev/null +++ b/third_party/github.com/ccding/go-logging/example.conf @@ -0,0 +1,3 @@ +name = example +sync = 0 + diff --git a/third_party/github.com/ccding/go-logging/example.go b/third_party/github.com/ccding/go-logging/example.go new file mode 100644 index 000000000..5d9fc6397 --- /dev/null +++ b/third_party/github.com/ccding/go-logging/example.go @@ -0,0 +1,45 @@ +// Copyright 2013, Cong Ding. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: Cong Ding +// +package main + +import ( + "github.com/ccding/go-logging/logging" + "time" +) + +func main() { + logger1, _ := logging.SimpleLogger("main") + logger1.SetLevel(logging.NOTSET) + logger1.Error("this is a test from error") + logger1.Debug("this is a test from debug") + logger1.Notset("orz", time.Now().UnixNano()) + logger1.Destroy() + + logger2, _ := logging.RichLogger("main") + logger2.SetLevel(logging.DEBUG) + logger2.Error("this is a test from error") + logger2.Debug("this is a test from debug") + logger2.Notset("orz", time.Now().UnixNano()) + logger2.Destroy() + + logger3, _ := logging.ConfigLogger("example.conf") + logger3.SetLevel(logging.DEBUG) + logger3.Error("this is a test from error") + logger3.Debug("this is a test from debug") + logger3.Notset("orz", time.Now().UnixNano()) + logger3.Destroy() +} diff --git a/third_party/github.com/ccding/go-logging/logging/commands.go b/third_party/github.com/ccding/go-logging/logging/commands.go new file mode 100644 index 000000000..6cf2bb4a3 --- /dev/null +++ b/third_party/github.com/ccding/go-logging/logging/commands.go @@ -0,0 +1,98 @@ +// Copyright 2013, Cong Ding. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: Cong Ding +// +package logging + +// Logln receives log request from the client. The request includes a set of +// variables. +func (logger *Logger) Log(level Level, v ...interface{}) { + // Don't delete this calling. The calling is used to keep the same + // calldepth for all the logging functions. The calldepth is used to + // get runtime information such as line number, function name, etc. + logger.log(level, v...) +} + +// Logf receives log request from the client. The request has a string +// parameter to describe the format of output. +func (logger *Logger) Logf(level Level, format string, v ...interface{}) { + logger.logf(level, format, v...) +} + +// Other quick commands for different level + +func (logger *Logger) Critical(v ...interface{}) { + logger.log(CRITICAL, v...) +} + +func (logger *Logger) Fatal(v ...interface{}) { + logger.log(CRITICAL, v...) +} + +func (logger *Logger) Error(v ...interface{}) { + logger.log(ERROR, v...) +} + +func (logger *Logger) Warn(v ...interface{}) { + logger.log(WARNING, v...) +} + +func (logger *Logger) Warning(v ...interface{}) { + logger.log(WARNING, v...) +} + +func (logger *Logger) Info(v ...interface{}) { + logger.log(INFO, v...) +} + +func (logger *Logger) Debug(v ...interface{}) { + logger.log(DEBUG, v...) +} + +func (logger *Logger) Notset(v ...interface{}) { + logger.log(NOTSET, v...) +} + +func (logger *Logger) Criticalf(format string, v ...interface{}) { + logger.logf(CRITICAL, format, v...) +} + +func (logger *Logger) Fatalf(format string, v ...interface{}) { + logger.logf(CRITICAL, format, v...) +} + +func (logger *Logger) Errorf(format string, v ...interface{}) { + logger.logf(ERROR, format, v...) +} + +func (logger *Logger) Warnf(format string, v ...interface{}) { + logger.logf(WARNING, format, v...) +} + +func (logger *Logger) Warningf(format string, v ...interface{}) { + logger.logf(WARNING, format, v...) +} + +func (logger *Logger) Infof(format string, v ...interface{}) { + logger.logf(INFO, format, v...) +} + +func (logger *Logger) Debugf(format string, v ...interface{}) { + logger.logf(DEBUG, format, v...) +} + +func (logger *Logger) Notsetf(format string, v ...interface{}) { + logger.logf(NOTSET, format, v...) +} diff --git a/third_party/github.com/ccding/go-logging/logging/fields.go b/third_party/github.com/ccding/go-logging/logging/fields.go new file mode 100644 index 000000000..74c05b190 --- /dev/null +++ b/third_party/github.com/ccding/go-logging/logging/fields.go @@ -0,0 +1,236 @@ +// Copyright 2013, Cong Ding. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: Cong Ding +// +package logging + +import ( + "bitbucket.org/kardianos/osext" + "os" + "path" + "runtime" + "sync/atomic" + "time" +) + +// The struct for each log record +type record struct { + level Level + seqid uint64 + pathname string + filename string + module string + lineno int + funcname string + thread int + process int + message string + time time.Time +} + +// This variable maps fields in recordArgs to relavent function signatures +var fields = map[string]func(*Logger, *record) interface{}{ + "name": (*Logger).lname, // name of the logger + "seqid": (*Logger).nextSeqid, // sequence number + "levelno": (*Logger).levelno, // level number + "levelname": (*Logger).levelname, // level name + "created": (*Logger).created, // starting time of the logger + "nsecs": (*Logger).nsecs, // nanosecond of the starting time + "time": (*Logger).time, // record created time + "timestamp": (*Logger).timestamp, // timestamp of record + "rtime": (*Logger).rtime, // relative time since started + "filename": (*Logger).filename, // source filename of the caller + "pathname": (*Logger).pathname, // filename with path + "module": (*Logger).module, // executable filename + "lineno": (*Logger).lineno, // line number in source code + "funcname": (*Logger).funcname, // function name of the caller + "thread": (*Logger).thread, // thread id + "process": (*Logger).process, // process id + "message": (*Logger).message, // logger message +} + +var runtimeFields = map[string]bool{ + "name": false, + "seqid": false, + "levelno": false, + "levelname": false, + "created": false, + "nsecs": false, + "time": false, + "timestamp": false, + "rtime": false, + "filename": true, + "pathname": true, + "module": false, + "lineno": true, + "funcname": true, + "thread": true, + "process": false, + "message": false, +} + +// If it fails to get some fields with string type, these fields are set to +// errString value. +const errString = "???" + +// GetGoID returns the id of goroutine, which is defined in ./get_go_id.c +func GetGoID() int32 + +// genRuntime generates the runtime information, including pathname, function +// name, filename, line number. +func genRuntime(r *record) { + calldepth := 5 + pc, file, line, ok := runtime.Caller(calldepth) + if ok { + // Generate short function name + fname := runtime.FuncForPC(pc).Name() + fshort := fname + for i := len(fname) - 1; i > 0; i-- { + if fname[i] == '.' { + fshort = fname[i+1:] + break + } + } + + r.pathname = file + r.funcname = fshort + r.filename = path.Base(file) + r.lineno = line + } else { + r.pathname = errString + r.funcname = errString + r.filename = errString + // Here we uses -1 rather than 0, because the default value in + // golang is 0 and we should know the value is uninitialized + // or failed to get + r.lineno = -1 + } +} + +// Logger name +func (logger *Logger) lname(r *record) interface{} { + return logger.name +} + +// Next sequence number +func (logger *Logger) nextSeqid(r *record) interface{} { + if r.seqid == 0 { + r.seqid = atomic.AddUint64(&(logger.seqid), 1) + } + return r.seqid +} + +// Log level number +func (logger *Logger) levelno(r *record) interface{} { + return int32(r.level) +} + +// Log level name +func (logger *Logger) levelname(r *record) interface{} { + return levelNames[r.level] +} + +// File name of calling logger, with whole path +func (logger *Logger) pathname(r *record) interface{} { + if r.pathname == "" { + genRuntime(r) + } + return r.pathname +} + +// File name of calling logger +func (logger *Logger) filename(r *record) interface{} { + if r.filename == "" { + genRuntime(r) + } + return r.filename +} + +// module name +func (logger *Logger) module(r *record) interface{} { + module, _ := osext.Executable() + return path.Base(module) +} + +// Line number +func (logger *Logger) lineno(r *record) interface{} { + if r.lineno == 0 { + genRuntime(r) + } + return r.lineno +} + +// Function name +func (logger *Logger) funcname(r *record) interface{} { + if r.funcname == "" { + genRuntime(r) + } + return r.funcname +} + +// Timestamp of starting time +func (logger *Logger) created(r *record) interface{} { + return logger.startTime.UnixNano() +} + +// RFC3339Nano time +func (logger *Logger) time(r *record) interface{} { + if r.time.IsZero() { + r.time = time.Now() + } + return r.time.Format(logger.timeFormat) +} + +// Nanosecond of starting time +func (logger *Logger) nsecs(r *record) interface{} { + return logger.startTime.Nanosecond() +} + +// Nanosecond timestamp +func (logger *Logger) timestamp(r *record) interface{} { + if r.time.IsZero() { + r.time = time.Now() + } + return r.time.UnixNano() +} + +// Nanoseconds since logger created +func (logger *Logger) rtime(r *record) interface{} { + if r.time.IsZero() { + r.time = time.Now() + } + return r.time.Sub(logger.startTime).Nanoseconds() +} + +// Thread ID +func (logger *Logger) thread(r *record) interface{} { + if r.thread == 0 { + r.thread = int(GetGoID()) + } + return r.thread +} + +// Process ID +func (logger *Logger) process(r *record) interface{} { + if r.process == 0 { + r.process = os.Getpid() + } + return r.process +} + +// The log message +func (logger *Logger) message(r *record) interface{} { + return r.message +} diff --git a/third_party/github.com/ccding/go-logging/logging/fields_test.go b/third_party/github.com/ccding/go-logging/logging/fields_test.go new file mode 100644 index 000000000..8433850d9 --- /dev/null +++ b/third_party/github.com/ccding/go-logging/logging/fields_test.go @@ -0,0 +1,60 @@ +// Copyright 2013, Cong Ding. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: Cong Ding +// +package logging + +import ( + "fmt" + "strconv" + "testing" +) + +func empty() { +} + +func TestGetGoID(t *testing.T) { + for i := 0; i < 1000; i++ { + goid := int(GetGoID()) + go empty() + goid2 := int(GetGoID()) + if goid != goid2 { + t.Errorf("%v, %v\n", goid, goid2) + } + } +} + +func TestSeqid(t *testing.T) { + logger, _ := BasicLogger("test") + for i := 0; i < 1000; i++ { + r := new(record) + name := strconv.Itoa(i + 1) + seq := logger.nextSeqid(r) + if fmt.Sprintf("%d", seq) != name { + t.Errorf("%v, %v\n", seq, name) + } + } + logger.Destroy() +} + +func TestName(t *testing.T) { + name := "test" + logger, _ := BasicLogger(name) + r := new(record) + if logger.lname(r) != name { + t.Errorf("%v, %v\n", logger.lname(r), name) + } + logger.Destroy() +} diff --git a/third_party/github.com/ccding/go-logging/logging/formater.go b/third_party/github.com/ccding/go-logging/logging/formater.go new file mode 100644 index 000000000..c704d57ae --- /dev/null +++ b/third_party/github.com/ccding/go-logging/logging/formater.go @@ -0,0 +1,62 @@ +// Copyright 2013, Cong Ding. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: Cong Ding +// +package logging + +import ( + "errors" + "fmt" + "strings" +) + +// pre-defined formats +const ( + BasicFormat = "%s [%6s] %30s - %s\n name,levelname,time,message" + RichFormat = "%s [%6s] %d %30s - %d - %s:%s:%d - %s\n name, levelname, seqid, time, thread, filename, funcname, lineno, message" +) + +// genLog generates log string from the format setting. +func (logger *Logger) genLog(level Level, message string) string { + fs := make([]interface{}, len(logger.recordArgs)) + r := new(record) + r.message = message + r.level = level + for k, v := range logger.recordArgs { + fs[k] = fields[v](logger, r) + } + return fmt.Sprintf(logger.recordFormat, fs...) +} + +// parseFormat checks the legality of format and parses it to recordFormat and recordArgs +func (logger *Logger) parseFormat(format string) error { + logger.runtime = false + fts := strings.Split(format, "\n") + if len(fts) != 2 { + return errors.New("logging format error") + } + logger.recordFormat = fts[0] + logger.recordArgs = strings.Split(fts[1], ",") + for k, v := range logger.recordArgs { + tv := strings.TrimSpace(v) + _, ok := fields[tv] + if ok == false { + return errors.New("logging format error") + } + logger.recordArgs[k] = tv + logger.runtime = logger.runtime || runtimeFields[tv] + } + return nil +} diff --git a/third_party/github.com/ccding/go-logging/logging/get_go_id.c b/third_party/github.com/ccding/go-logging/logging/get_go_id.c new file mode 100644 index 000000000..400848cf9 --- /dev/null +++ b/third_party/github.com/ccding/go-logging/logging/get_go_id.c @@ -0,0 +1,25 @@ +// Copyright 2013, Cong Ding. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: Cong Ding +// +// This file defines GetGoId function, which is used to get the id of the +// current goroutine. More details about this function are availeble in the +// runtime.c file of golang source code. +#include + +void ·GetGoID(int32 ret) { + ret = g->goid; + USED(&ret); +} diff --git a/third_party/github.com/ccding/go-logging/logging/level.go b/third_party/github.com/ccding/go-logging/logging/level.go new file mode 100644 index 000000000..e640fa626 --- /dev/null +++ b/third_party/github.com/ccding/go-logging/logging/level.go @@ -0,0 +1,68 @@ +// Copyright 2013, Cong Ding. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: Cong Ding +// +package logging + +// Level is the type of level. +type Level int32 + +// Values of level +const ( + CRITICAL Level = 50 + FATAL Level = CRITICAL + ERROR Level = 40 + WARNING Level = 30 + WARN Level = WARNING + INFO Level = 20 + DEBUG Level = 10 + NOTSET Level = 0 +) + +// The mapping from level to level name +var levelNames = map[Level]string{ + CRITICAL: "CRITICAL", + ERROR: "ERROR", + WARNING: "WARNING", + INFO: "INFO", + DEBUG: "DEBUG", + NOTSET: "NOTSET", +} + +// The mapping from level name to level +var levelValues = map[string]Level{ + "CRITICAL": CRITICAL, + "ERROR": ERROR, + "WARN": WARNING, + "WARNING": WARNING, + "INFO": INFO, + "DEBUG": DEBUG, + "NOTSET": NOTSET, +} + +// String function casts level value to string +func (level *Level) String() string { + return levelNames[*level] +} + +// GetLevelName lets users be able to get level name from level value. +func GetLevelName(levelValue Level) string { + return levelNames[levelValue] +} + +// GetLevelValue lets users be able to get level value from level name. +func GetLevelValue(levelName string) Level { + return levelValues[levelName] +} diff --git a/third_party/github.com/ccding/go-logging/logging/logging.go b/third_party/github.com/ccding/go-logging/logging/logging.go new file mode 100644 index 000000000..1981c9a61 --- /dev/null +++ b/third_party/github.com/ccding/go-logging/logging/logging.go @@ -0,0 +1,260 @@ +// Copyright 2013, Cong Ding. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: Cong Ding + +// Package logging implements log library for other applications. It provides +// functions Debug, Info, Warning, Error, Critical, and formatting version +// Logf. +// +// Example: +// +// logger := logging.SimpleLogger("main") +// logger.SetLevel(logging.WARNING) +// logger.Error("test for error") +// logger.Warning("test for warning", "second parameter") +// logger.Debug("test for debug") +// +package logging + +import ( + "github.com/ccding/go-config-reader/config" + "io" + "os" + "strconv" + "sync" + "sync/atomic" + "time" +) + +// Pre-defined formats +const ( + DefaultFileName = "logging.log" // default logging filename + DefaultConfigFile = "logging.conf" // default logging configuration file + DefaultTimeFormat = "2006-01-02 15:04:05.999999999" // defaulttime format + bufSize = 1000 // buffer size for writer + queueSize = 10000 // chan queue size in async logging + reqSize = 10000 // chan queue size in async logging +) + +// Logger is the logging struct. +type Logger struct { + + // Be careful of the alignment issue of the variable seqid because it + // uses the sync/atomic.AddUint64() operation. If the alignment is + // wrong, it will cause a panic. To solve the alignment issue in an + // easy way, we put seqid to the beginning of the structure. + // seqid is only visiable internally. + seqid uint64 // last used sequence number in record + + // These variables can be configured by users. + name string // logger name + level Level // record level higher than this will be printed + recordFormat string // format of the record + recordArgs []string // arguments to be used in the recordFormat + out io.Writer // writer + sync bool // use sync or async way to record logs + timeFormat string // format for time + + // These variables are visible to users. + startTime time.Time // start time of the logger + + // Internally used variables, which don't have get and set functions. + wlock sync.Mutex // writer lock + queue chan string // queue used in async logging + request chan request // queue used in non-runtime logging + flush chan bool // flush signal for the watcher to write + quit chan bool // quit signal for the watcher to quit + fd *os.File // file handler, used to close the file on destroy + runtime bool // with runtime operation or not +} + +// SimpleLogger creates a new logger with simple configuration. +func SimpleLogger(name string) (*Logger, error) { + return createLogger(name, WARNING, BasicFormat, DefaultTimeFormat, os.Stdout, false) +} + +// BasicLogger creates a new logger with basic configuration. +func BasicLogger(name string) (*Logger, error) { + return FileLogger(name, WARNING, BasicFormat, DefaultTimeFormat, DefaultFileName, false) +} + +// RichLogger creates a new logger with simple configuration. +func RichLogger(name string) (*Logger, error) { + return FileLogger(name, NOTSET, RichFormat, DefaultTimeFormat, DefaultFileName, false) +} + +// FileLogger creates a new logger with file output. +func FileLogger(name string, level Level, format string, timeFormat string, file string, sync bool) (*Logger, error) { + out, err := os.Create(file) + if err != nil { + return new(Logger), err + } + logger, err := createLogger(name, level, format, timeFormat, out, sync) + if err == nil { + logger.fd = out + } + return logger, err +} + +// WriterLogger creates a new logger with a writer +func WriterLogger(name string, level Level, format string, timeFormat string, out io.Writer, sync bool) (*Logger, error) { + return createLogger(name, level, format, timeFormat, out, sync) +} + +// WriterLogger creates a new logger from a configuration file +func ConfigLogger(filename string) (*Logger, error) { + conf, err := config.Read(filename) + if err != nil { + return new(Logger), err + } + ok := true + name, ok := conf["name"] + if !ok { + name = "" + } + slevel, ok := conf["level"] + if !ok { + slevel = "0" + } + l, err := strconv.Atoi(slevel) + if err != nil { + return new(Logger), err + } + level := Level(l) + format, ok := conf["format"] + if !ok { + format = BasicFormat + } + timeFormat, ok := conf["timeFormat"] + if !ok { + timeFormat = DefaultTimeFormat + } + ssync, ok := conf["sync"] + if !ok { + ssync = "0" + } + file, ok := conf["file"] + if !ok { + file = DefaultFileName + } + sync := true + if ssync == "0" { + sync = false + } else if ssync == "1" { + sync = true + } else { + return new(Logger), err + } + return FileLogger(name, level, format, timeFormat, file, sync) +} + +// createLogger create a new logger +func createLogger(name string, level Level, format string, timeFormat string, out io.Writer, sync bool) (*Logger, error) { + logger := new(Logger) + + err := logger.parseFormat(format) + if err != nil { + return logger, err + } + + // asign values to logger + logger.name = name + logger.level = level + logger.out = out + logger.seqid = 0 + logger.sync = sync + logger.queue = make(chan string, queueSize) + logger.request = make(chan request, reqSize) + logger.flush = make(chan bool) + logger.quit = make(chan bool) + logger.startTime = time.Now() + logger.fd = nil + logger.timeFormat = timeFormat + + // start watcher to write logs if it is async or no runtime field + if !logger.sync { + go logger.watcher() + } + + return logger, nil +} + +// Destroy sends quit signal to watcher and releases all the resources. +func (logger *Logger) Destroy() { + if !logger.sync { + // quit watcher + logger.quit <- true + // wait for watcher quit + <-logger.quit + } + // clean up + if logger.fd != nil { + logger.fd.Close() + } +} + +// Flush the writer +func (logger *Logger) Flush() { + if !logger.sync { + // send flush signal + logger.flush <- true + // wait for flush finish + <-logger.flush + } +} + +// Getter functions + +func (logger *Logger) Name() string { + return logger.name +} + +func (logger *Logger) StartTime() int64 { + return logger.startTime.UnixNano() +} + +func (logger *Logger) TimeFormat() string { + return logger.timeFormat +} + +func (logger *Logger) Level() Level { + return Level(atomic.LoadInt32((*int32)(&logger.level))) +} + +func (logger *Logger) RecordFormat() string { + return logger.recordFormat +} + +func (logger *Logger) RecordArgs() []string { + return logger.recordArgs +} + +func (logger *Logger) Writer() io.Writer { + return logger.out +} + +func (logger *Logger) Sync() bool { + return logger.sync +} + +// Setter functions + +func (logger *Logger) SetLevel(level Level) { + atomic.StoreInt32((*int32)(&logger.level), int32(level)) +} + +func (logger *Logger) SetWriter(out ...io.Writer) { + logger.out = io.MultiWriter(out...) +} diff --git a/third_party/github.com/ccding/go-logging/logging/logging_test.go b/third_party/github.com/ccding/go-logging/logging/logging_test.go new file mode 100644 index 000000000..b6e0fa502 --- /dev/null +++ b/third_party/github.com/ccding/go-logging/logging/logging_test.go @@ -0,0 +1,71 @@ +// Copyright 2013, Cong Ding. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: Cong Ding +// +package logging + +import ( + "fmt" + "os" + "testing" +) + +func BenchmarkSync(b *testing.B) { + logger, _ := RichLogger("main") + logger.SetLevel(NOTSET) + for i := 0; i < b.N; i++ { + logger.Error("this is a test from error") + } + logger.Flush() + logger.Destroy() +} + +func BenchmarkAsync(b *testing.B) { + logger, _ := RichLogger("main") + logger.SetLevel(NOTSET) + for i := 0; i < b.N; i++ { + logger.Error("this is a test from error") + } + logger.Flush() + logger.Destroy() +} + +func BenchmarkBasicSync(b *testing.B) { + logger, _ := BasicLogger("main") + logger.SetLevel(NOTSET) + for i := 0; i < b.N; i++ { + logger.Error("this is a test from error") + } + logger.Flush() + logger.Destroy() +} + +func BenchmarkBasicAsync(b *testing.B) { + logger, _ := BasicLogger("main") + logger.SetLevel(NOTSET) + for i := 0; i < b.N; i++ { + logger.Error("this is a test from error") + } + logger.Flush() + logger.Destroy() +} + +func BenchmarkPrintln(b *testing.B) { + out, _ := os.Create("logging.log") + for i := 0; i < b.N; i++ { + fmt.Fprintln(out, "this is a test from error") + } + out.Close() +} diff --git a/third_party/github.com/ccding/go-logging/logging/request.go b/third_party/github.com/ccding/go-logging/logging/request.go new file mode 100644 index 000000000..e823fa719 --- /dev/null +++ b/third_party/github.com/ccding/go-logging/logging/request.go @@ -0,0 +1,24 @@ +// Copyright 2013, Cong Ding. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: Cong Ding +// +package logging + +// request struct stores the logger request +type request struct { + level Level + format string + v []interface{} +} diff --git a/third_party/github.com/ccding/go-logging/logging/writer.go b/third_party/github.com/ccding/go-logging/logging/writer.go new file mode 100644 index 000000000..12d17901a --- /dev/null +++ b/third_party/github.com/ccding/go-logging/logging/writer.go @@ -0,0 +1,130 @@ +// Copyright 2013, Cong Ding. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: Cong Ding +// +package logging + +import ( + "bytes" + "fmt" + "sync/atomic" + "time" +) + +// watcher watches the logger.queue channel, and writes the logs to output +func (logger *Logger) watcher() { + var buf bytes.Buffer + for { + timeout := time.After(time.Second / 10) + + for i := 0; i < bufSize; i++ { + select { + case msg := <-logger.queue: + fmt.Fprintln(&buf, msg) + case req := <-logger.request: + logger.flushReq(&buf, &req) + case <-timeout: + i = bufSize + case <-logger.flush: + logger.flushBuf(&buf) + logger.flush <- true + i = bufSize + case <-logger.quit: + // If quit signal received, cleans the channel + // and writes all of them to io.Writer. + for { + select { + case msg := <-logger.queue: + fmt.Fprintln(&buf, msg) + case req := <-logger.request: + logger.flushReq(&buf, &req) + case <-logger.flush: + // do nothing + default: + logger.flushBuf(&buf) + logger.quit <- true + return + } + } + } + } + logger.flushBuf(&buf) + } +} + +// flushBuf flushes the content of buffer to out and reset the buffer +func (logger *Logger) flushBuf(b *bytes.Buffer) { + if len(b.Bytes()) > 0 { + logger.out.Write(b.Bytes()) + b.Reset() + } +} + +// flushReq handles the request and writes the result to writer +func (logger *Logger) flushReq(b *bytes.Buffer, req *request) { + if req.format == "" { + msg := fmt.Sprint(req.v...) + msg = logger.genLog(req.level, msg) + fmt.Fprintln(b, msg) + } else { + msg := fmt.Sprintf(req.format, req.v...) + msg = logger.genLog(req.level, msg) + fmt.Fprintln(b, msg) + } +} + +// flushMsg is to print log to file, stdout, or others. +func (logger *Logger) flushMsg(message string) { + if logger.sync { + logger.wlock.Lock() + defer logger.wlock.Unlock() + fmt.Fprintln(logger.out, message) + } else { + logger.queue <- message + } +} + +// log records log v... with level `level'. +func (logger *Logger) log(level Level, v ...interface{}) { + if int32(level) >= atomic.LoadInt32((*int32)(&logger.level)) { + if logger.runtime || logger.sync { + message := fmt.Sprint(v...) + message = logger.genLog(level, message) + logger.flushMsg(message) + } else { + r := new(request) + r.level = level + r.v = v + logger.request <- *r + } + } +} + +// logf records log v... with level `level'. +func (logger *Logger) logf(level Level, format string, v ...interface{}) { + if int32(level) >= atomic.LoadInt32((*int32)(&logger.level)) { + if logger.runtime || logger.sync { + message := fmt.Sprintf(format, v...) + message = logger.genLog(level, message) + logger.flushMsg(message) + } else { + r := new(request) + r.level = level + r.format = format + r.v = v + logger.request <- *r + } + } +} From 20833eb63d0f765fdfbf09fe4fb864f490c3c3b3 Mon Sep 17 00:00:00 2001 From: Brandon Philips Date: Tue, 6 Aug 2013 10:50:11 -0700 Subject: [PATCH 03/10] bump(bitbucket.org/kardianos/osext): 84d8cbacbe13 --- .../bitbucket.org/kardianos/osext/LICENSE | 20 +++++ .../bitbucket.org/kardianos/osext/osext.go | 32 ++++++++ .../kardianos/osext/osext_plan9.go | 16 ++++ .../kardianos/osext/osext_procfs.go | 25 ++++++ .../kardianos/osext/osext_sysctl.go | 64 +++++++++++++++ .../kardianos/osext/osext_test.go | 79 +++++++++++++++++++ .../kardianos/osext/osext_windows.go | 34 ++++++++ 7 files changed, 270 insertions(+) create mode 100644 third_party/bitbucket.org/kardianos/osext/LICENSE create mode 100644 third_party/bitbucket.org/kardianos/osext/osext.go create mode 100644 third_party/bitbucket.org/kardianos/osext/osext_plan9.go create mode 100644 third_party/bitbucket.org/kardianos/osext/osext_procfs.go create mode 100644 third_party/bitbucket.org/kardianos/osext/osext_sysctl.go create mode 100644 third_party/bitbucket.org/kardianos/osext/osext_test.go create mode 100644 third_party/bitbucket.org/kardianos/osext/osext_windows.go diff --git a/third_party/bitbucket.org/kardianos/osext/LICENSE b/third_party/bitbucket.org/kardianos/osext/LICENSE new file mode 100644 index 000000000..18527a28f --- /dev/null +++ b/third_party/bitbucket.org/kardianos/osext/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2012 Daniel Theophanes + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. diff --git a/third_party/bitbucket.org/kardianos/osext/osext.go b/third_party/bitbucket.org/kardianos/osext/osext.go new file mode 100644 index 000000000..37efbb221 --- /dev/null +++ b/third_party/bitbucket.org/kardianos/osext/osext.go @@ -0,0 +1,32 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Extensions to the standard "os" package. +package osext + +import "path/filepath" + +// Executable returns an absolute path that can be used to +// re-invoke the current program. +// It may not be valid after the current program exits. +func Executable() (string, error) { + p, err := executable() + return filepath.Clean(p), err +} + +// Returns same path as Executable, returns just the folder +// path. Excludes the executable name. +func ExecutableFolder() (string, error) { + p, err := Executable() + if err != nil { + return "", err + } + folder, _ := filepath.Split(p) + return folder, nil +} + +// Depricated. Same as Executable(). +func GetExePath() (exePath string, err error) { + return Executable() +} diff --git a/third_party/bitbucket.org/kardianos/osext/osext_plan9.go b/third_party/bitbucket.org/kardianos/osext/osext_plan9.go new file mode 100644 index 000000000..e88c1e093 --- /dev/null +++ b/third_party/bitbucket.org/kardianos/osext/osext_plan9.go @@ -0,0 +1,16 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package osext + +import "syscall" + +func executable() (string, error) { + f, err := Open("/proc/" + itoa(Getpid()) + "/text") + if err != nil { + return "", err + } + defer f.Close() + return syscall.Fd2path(int(f.Fd())) +} diff --git a/third_party/bitbucket.org/kardianos/osext/osext_procfs.go b/third_party/bitbucket.org/kardianos/osext/osext_procfs.go new file mode 100644 index 000000000..546fec915 --- /dev/null +++ b/third_party/bitbucket.org/kardianos/osext/osext_procfs.go @@ -0,0 +1,25 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux netbsd openbsd + +package osext + +import ( + "errors" + "os" + "runtime" +) + +func executable() (string, error) { + switch runtime.GOOS { + case "linux": + return os.Readlink("/proc/self/exe") + case "netbsd": + return os.Readlink("/proc/curproc/exe") + case "openbsd": + return os.Readlink("/proc/curproc/file") + } + return "", errors.New("ExecPath not implemented for " + runtime.GOOS) +} diff --git a/third_party/bitbucket.org/kardianos/osext/osext_sysctl.go b/third_party/bitbucket.org/kardianos/osext/osext_sysctl.go new file mode 100644 index 000000000..d76464628 --- /dev/null +++ b/third_party/bitbucket.org/kardianos/osext/osext_sysctl.go @@ -0,0 +1,64 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin freebsd + +package osext + +import ( + "os" + "runtime" + "syscall" + "unsafe" +) + +var startUpcwd, getwdError = os.Getwd() + +func executable() (string, error) { + var mib [4]int32 + switch runtime.GOOS { + case "freebsd": + mib = [4]int32{1 /* CTL_KERN */, 14 /* KERN_PROC */, 12 /* KERN_PROC_PATHNAME */, -1} + case "darwin": + mib = [4]int32{1 /* CTL_KERN */, 38 /* KERN_PROCARGS */, int32(os.Getpid()), -1} + } + + n := uintptr(0) + // get length + _, _, err := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0) + if err != 0 { + return "", err + } + if n == 0 { // shouldn't happen + return "", nil + } + buf := make([]byte, n) + _, _, err = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0) + if err != 0 { + return "", err + } + if n == 0 { // shouldn't happen + return "", nil + } + for i, v := range buf { + if v == 0 { + buf = buf[:i] + break + } + } + if buf[0] != '/' { + if getwdError != nil { + return string(buf), getwdError + } else { + if buf[0] == '.' { + buf = buf[1:] + } + if startUpcwd[len(startUpcwd)-1] != '/' { + return startUpcwd + "/" + string(buf), nil + } + return startUpcwd + string(buf), nil + } + } + return string(buf), nil +} diff --git a/third_party/bitbucket.org/kardianos/osext/osext_test.go b/third_party/bitbucket.org/kardianos/osext/osext_test.go new file mode 100644 index 000000000..dc661dbc2 --- /dev/null +++ b/third_party/bitbucket.org/kardianos/osext/osext_test.go @@ -0,0 +1,79 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin linux freebsd netbsd windows + +package osext + +import ( + "fmt" + "os" + oexec "os/exec" + "path/filepath" + "runtime" + "testing" +) + +const execPath_EnvVar = "OSTEST_OUTPUT_EXECPATH" + +func TestExecPath(t *testing.T) { + ep, err := Executable() + if err != nil { + t.Fatalf("ExecPath failed: %v", err) + } + // we want fn to be of the form "dir/prog" + dir := filepath.Dir(filepath.Dir(ep)) + fn, err := filepath.Rel(dir, ep) + if err != nil { + t.Fatalf("filepath.Rel: %v", err) + } + cmd := &oexec.Cmd{} + // make child start with a relative program path + cmd.Dir = dir + cmd.Path = fn + // forge argv[0] for child, so that we can verify we could correctly + // get real path of the executable without influenced by argv[0]. + cmd.Args = []string{"-", "-test.run=XXXX"} + cmd.Env = []string{fmt.Sprintf("%s=1", execPath_EnvVar)} + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("exec(self) failed: %v", err) + } + outs := string(out) + if !filepath.IsAbs(outs) { + t.Fatalf("Child returned %q, want an absolute path", out) + } + if !sameFile(outs, ep) { + t.Fatalf("Child returned %q, not the same file as %q", out, ep) + } +} + +func sameFile(fn1, fn2 string) bool { + fi1, err := os.Stat(fn1) + if err != nil { + return false + } + fi2, err := os.Stat(fn2) + if err != nil { + return false + } + return os.SameFile(fi1, fi2) +} + +func init() { + if e := os.Getenv(execPath_EnvVar); e != "" { + // first chdir to another path + dir := "/" + if runtime.GOOS == "windows" { + dir = filepath.VolumeName(".") + } + os.Chdir(dir) + if ep, err := Executable(); err != nil { + fmt.Fprint(os.Stderr, "ERROR: ", err) + } else { + fmt.Fprint(os.Stderr, ep) + } + os.Exit(0) + } +} diff --git a/third_party/bitbucket.org/kardianos/osext/osext_windows.go b/third_party/bitbucket.org/kardianos/osext/osext_windows.go new file mode 100644 index 000000000..72d282cf8 --- /dev/null +++ b/third_party/bitbucket.org/kardianos/osext/osext_windows.go @@ -0,0 +1,34 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package osext + +import ( + "syscall" + "unicode/utf16" + "unsafe" +) + +var ( + kernel = syscall.MustLoadDLL("kernel32.dll") + getModuleFileNameProc = kernel.MustFindProc("GetModuleFileNameW") +) + +// GetModuleFileName() with hModule = NULL +func executable() (exePath string, err error) { + return getModuleFileName() +} + +func getModuleFileName() (string, error) { + var n uint32 + b := make([]uint16, syscall.MAX_PATH) + size := uint32(len(b)) + + r0, _, e1 := getModuleFileNameProc.Call(0, uintptr(unsafe.Pointer(&b[0])), uintptr(size)) + n = uint32(r0) + if n == 0 { + return "", e1 + } + return string(utf16.Decode(b[0:n])), nil +} From 46bce9cc4969c1b1d426491cf2c626301e1b2245 Mon Sep 17 00:00:00 2001 From: Brandon Philips Date: Tue, 6 Aug 2013 10:50:12 -0700 Subject: [PATCH 04/10] bump(code.google.com/p/go.net): ca4c826193c2 --- .../code.google.com/p/go.net/.hgignore | 2 + third_party/code.google.com/p/go.net/AUTHORS | 3 + .../code.google.com/p/go.net/CONTRIBUTORS | 3 + third_party/code.google.com/p/go.net/LICENSE | 27 + third_party/code.google.com/p/go.net/PATENTS | 22 + third_party/code.google.com/p/go.net/README | 3 + .../code.google.com/p/go.net/codereview.cfg | 2 + .../code.google.com/p/go.net/dict/dict.go | 210 + .../p/go.net/html/atom/atom.go | 78 + .../p/go.net/html/atom/atom_test.go | 109 + .../code.google.com/p/go.net/html/atom/gen.go | 636 + .../p/go.net/html/atom/table.go | 694 + .../p/go.net/html/atom/table_test.go | 341 + .../code.google.com/p/go.net/html/const.go | 100 + .../code.google.com/p/go.net/html/doc.go | 106 + .../code.google.com/p/go.net/html/doctype.go | 156 + .../code.google.com/p/go.net/html/entity.go | 2253 +++ .../p/go.net/html/entity_test.go | 29 + .../code.google.com/p/go.net/html/escape.go | 258 + .../p/go.net/html/escape_test.go | 97 + .../p/go.net/html/example_test.go | 40 + .../code.google.com/p/go.net/html/foreign.go | 226 + .../code.google.com/p/go.net/html/node.go | 193 + .../p/go.net/html/node_test.go | 146 + .../code.google.com/p/go.net/html/parse.go | 2092 +++ .../p/go.net/html/parse_test.go | 388 + .../code.google.com/p/go.net/html/render.go | 271 + .../p/go.net/html/render_test.go | 156 + .../p/go.net/html/testdata/go1.html | 2237 +++ .../p/go.net/html/testdata/webkit/README | 28 + .../html/testdata/webkit/adoption01.dat | 194 + .../html/testdata/webkit/adoption02.dat | 31 + .../html/testdata/webkit/comments01.dat | 135 + .../go.net/html/testdata/webkit/doctype01.dat | 370 + .../html/testdata/webkit/entities01.dat | 603 + .../html/testdata/webkit/entities02.dat | 249 + .../html/testdata/webkit/html5test-com.dat | 246 + .../go.net/html/testdata/webkit/inbody01.dat | 43 + .../p/go.net/html/testdata/webkit/isindex.dat | 40 + ...pending-spec-changes-plain-text-unsafe.dat | Bin 0 -> 115 bytes .../testdata/webkit/pending-spec-changes.dat | 52 + .../testdata/webkit/plain-text-unsafe.dat | Bin 0 -> 4166 bytes .../html/testdata/webkit/scriptdata01.dat | 308 + .../testdata/webkit/scripted/adoption01.dat | 15 + .../testdata/webkit/scripted/webkit01.dat | 28 + .../go.net/html/testdata/webkit/tables01.dat | 212 + .../p/go.net/html/testdata/webkit/tests1.dat | 1952 +++ .../p/go.net/html/testdata/webkit/tests10.dat | 799 + .../p/go.net/html/testdata/webkit/tests11.dat | 482 + .../p/go.net/html/testdata/webkit/tests12.dat | 62 + .../p/go.net/html/testdata/webkit/tests14.dat | 74 + .../p/go.net/html/testdata/webkit/tests15.dat | 208 + .../p/go.net/html/testdata/webkit/tests16.dat | 2299 +++ .../p/go.net/html/testdata/webkit/tests17.dat | 153 + .../p/go.net/html/testdata/webkit/tests18.dat | 269 + .../p/go.net/html/testdata/webkit/tests19.dat | 1237 ++ .../p/go.net/html/testdata/webkit/tests2.dat | 763 + .../p/go.net/html/testdata/webkit/tests20.dat | 455 + .../p/go.net/html/testdata/webkit/tests21.dat | 221 + .../p/go.net/html/testdata/webkit/tests22.dat | 157 + .../p/go.net/html/testdata/webkit/tests23.dat | 155 + .../p/go.net/html/testdata/webkit/tests24.dat | 79 + .../p/go.net/html/testdata/webkit/tests25.dat | 219 + .../p/go.net/html/testdata/webkit/tests26.dat | 313 + .../p/go.net/html/testdata/webkit/tests3.dat | 305 + .../p/go.net/html/testdata/webkit/tests4.dat | 59 + .../p/go.net/html/testdata/webkit/tests5.dat | 191 + .../p/go.net/html/testdata/webkit/tests6.dat | 663 + .../p/go.net/html/testdata/webkit/tests7.dat | 390 + .../p/go.net/html/testdata/webkit/tests8.dat | 148 + .../p/go.net/html/testdata/webkit/tests9.dat | 457 + .../testdata/webkit/tests_innerHTML_1.dat | 741 + .../go.net/html/testdata/webkit/tricky01.dat | 261 + .../go.net/html/testdata/webkit/webkit01.dat | 610 + .../go.net/html/testdata/webkit/webkit02.dat | 159 + .../code.google.com/p/go.net/html/token.go | 1174 ++ .../p/go.net/html/token_test.go | 589 + .../code.google.com/p/go.net/idna/idna.go | 68 + .../p/go.net/idna/idna_test.go | 43 + .../code.google.com/p/go.net/idna/punycode.go | 200 + .../p/go.net/idna/punycode_test.go | 198 + .../code.google.com/p/go.net/ipv4/control.go | 52 + .../p/go.net/ipv4/control_bsd.go | 121 + .../p/go.net/ipv4/control_linux.go | 126 + .../p/go.net/ipv4/control_plan9.go | 27 + .../p/go.net/ipv4/control_windows.go | 27 + .../p/go.net/ipv4/dgramopt_plan9.go | 50 + .../p/go.net/ipv4/dgramopt_posix.go | 125 + .../code.google.com/p/go.net/ipv4/doc.go | 196 + .../code.google.com/p/go.net/ipv4/endpoint.go | 181 + .../p/go.net/ipv4/example_test.go | 283 + .../code.google.com/p/go.net/ipv4/gen.go | 252 + .../p/go.net/ipv4/genericopt_plan9.go | 29 + .../p/go.net/ipv4/genericopt_posix.go | 61 + .../code.google.com/p/go.net/ipv4/gentest.go | 196 + .../code.google.com/p/go.net/ipv4/header.go | 149 + .../p/go.net/ipv4/header_test.go | 95 + .../code.google.com/p/go.net/ipv4/helper.go | 85 + .../p/go.net/ipv4/helper_plan9.go | 27 + .../p/go.net/ipv4/helper_posix.go | 42 + .../p/go.net/ipv4/helper_unix.go | 50 + .../p/go.net/ipv4/helper_windows.go | 49 + .../code.google.com/p/go.net/ipv4/iana.go | 179 + .../p/go.net/ipv4/iana_test.go | 38 + .../code.google.com/p/go.net/ipv4/icmp.go | 16 + .../p/go.net/ipv4/mockicmp_test.go | 111 + .../p/go.net/ipv4/mocktransponder_test.go | 124 + .../p/go.net/ipv4/multicast_test.go | 168 + .../p/go.net/ipv4/multicastlistener_test.go | 237 + .../p/go.net/ipv4/multicastsockopt_test.go | 126 + .../code.google.com/p/go.net/ipv4/packet.go | 97 + .../code.google.com/p/go.net/ipv4/payload.go | 81 + .../p/go.net/ipv4/sockopt_bsd.go | 95 + .../p/go.net/ipv4/sockopt_freebsd.go | 22 + .../p/go.net/ipv4/sockopt_linux.go | 94 + .../p/go.net/ipv4/sockopt_plan9.go | 17 + .../p/go.net/ipv4/sockopt_unix.go | 60 + .../p/go.net/ipv4/sockopt_windows.go | 146 + .../p/go.net/ipv4/unicast_test.go | 192 + .../p/go.net/ipv4/unicastsockopt_test.go | 145 + .../code.google.com/p/go.net/ipv6/control.go | 84 + .../p/go.net/ipv6/control_rfc2292_darwin.go | 151 + .../p/go.net/ipv6/control_rfc3542_bsd.go | 213 + .../p/go.net/ipv6/control_rfc3542_linux.go | 217 + .../p/go.net/ipv6/control_rfc3542_plan9.go | 27 + .../p/go.net/ipv6/control_rfc3542_windows.go | 27 + .../p/go.net/ipv6/control_test.go | 42 + .../p/go.net/ipv6/dgramopt_plan9.go | 96 + .../p/go.net/ipv6/dgramopt_posix.go | 178 + .../code.google.com/p/go.net/ipv6/doc.go | 193 + .../code.google.com/p/go.net/ipv6/endpoint.go | 119 + .../code.google.com/p/go.net/ipv6/gen.go | 247 + .../p/go.net/ipv6/genericopt_plan9.go | 34 + .../p/go.net/ipv6/genericopt_posix.go | 60 + .../code.google.com/p/go.net/ipv6/gentest.go | 196 + .../code.google.com/p/go.net/ipv6/helper.go | 28 + .../p/go.net/ipv6/helper_plan9.go | 22 + .../p/go.net/ipv6/helper_unix.go | 46 + .../p/go.net/ipv6/helper_windows.go | 45 + .../code.google.com/p/go.net/ipv6/iana.go | 224 + .../p/go.net/ipv6/iana_test.go | 38 + .../code.google.com/p/go.net/ipv6/icmp.go | 46 + .../code.google.com/p/go.net/ipv6/icmp_bsd.go | 35 + .../p/go.net/ipv6/icmp_linux.go | 33 + .../p/go.net/ipv6/icmp_plan9.go | 22 + .../p/go.net/ipv6/icmp_test.go | 84 + .../p/go.net/ipv6/icmp_windows.go | 22 + .../p/go.net/ipv6/mockicmp_test.go | 112 + .../p/go.net/ipv6/mocktransponder_test.go | 110 + .../p/go.net/ipv6/multicast_test.go | 161 + .../p/go.net/ipv6/multicastlistener_test.go | 197 + .../p/go.net/ipv6/multicastsockopt_test.go | 76 + .../code.google.com/p/go.net/ipv6/payload.go | 15 + .../p/go.net/ipv6/payload_cmsg.go | 70 + .../p/go.net/ipv6/payload_noncmsg.go | 41 + .../p/go.net/ipv6/sockopt_rfc2292_darwin.go | 66 + .../p/go.net/ipv6/sockopt_rfc3493_bsd.go | 19 + .../p/go.net/ipv6/sockopt_rfc3493_linux.go | 17 + .../p/go.net/ipv6/sockopt_rfc3493_unix.go | 114 + .../p/go.net/ipv6/sockopt_rfc3493_windows.go | 116 + .../p/go.net/ipv6/sockopt_rfc3542_bsd.go | 44 + .../p/go.net/ipv6/sockopt_rfc3542_linux.go | 42 + .../p/go.net/ipv6/sockopt_rfc3542_plan9.go | 12 + .../p/go.net/ipv6/sockopt_rfc3542_unix.go | 48 + .../p/go.net/ipv6/sockopt_rfc3542_windows.go | 62 + .../p/go.net/ipv6/sockopt_test.go | 136 + .../p/go.net/ipv6/unicast_test.go | 203 + .../p/go.net/ipv6/unicastsockopt_test.go | 101 + .../code.google.com/p/go.net/proxy/direct.go | 18 + .../p/go.net/proxy/per_host.go | 140 + .../p/go.net/proxy/per_host_test.go | 55 + .../code.google.com/p/go.net/proxy/proxy.go | 94 + .../p/go.net/proxy/proxy_test.go | 142 + .../code.google.com/p/go.net/proxy/socks5.go | 207 + .../p/go.net/publicsuffix/gen.go | 570 + .../p/go.net/publicsuffix/list.go | 133 + .../p/go.net/publicsuffix/list_test.go | 404 + .../p/go.net/publicsuffix/table.go | 6877 +++++++++ .../p/go.net/publicsuffix/table_test.go | 12226 ++++++++++++++++ .../p/go.net/spdy/dictionary.go | 187 + .../code.google.com/p/go.net/spdy/read.go | 348 + .../p/go.net/spdy/spdy_test.go | 644 + .../code.google.com/p/go.net/spdy/types.go | 275 + .../code.google.com/p/go.net/spdy/write.go | 318 + .../p/go.net/websocket/client.go | 112 + .../p/go.net/websocket/exampledial_test.go | 31 + .../p/go.net/websocket/examplehandler_test.go | 26 + .../p/go.net/websocket/hixie.go | 695 + .../p/go.net/websocket/hixie_test.go | 201 + .../p/go.net/websocket/hybi.go | 580 + .../p/go.net/websocket/hybi_test.go | 698 + .../p/go.net/websocket/server.go | 122 + .../p/go.net/websocket/websocket.go | 415 + .../p/go.net/websocket/websocket_test.go | 327 + 194 files changed, 63892 insertions(+) create mode 100644 third_party/code.google.com/p/go.net/.hgignore create mode 100644 third_party/code.google.com/p/go.net/AUTHORS create mode 100644 third_party/code.google.com/p/go.net/CONTRIBUTORS create mode 100644 third_party/code.google.com/p/go.net/LICENSE create mode 100644 third_party/code.google.com/p/go.net/PATENTS create mode 100644 third_party/code.google.com/p/go.net/README create mode 100644 third_party/code.google.com/p/go.net/codereview.cfg create mode 100644 third_party/code.google.com/p/go.net/dict/dict.go create mode 100644 third_party/code.google.com/p/go.net/html/atom/atom.go create mode 100644 third_party/code.google.com/p/go.net/html/atom/atom_test.go create mode 100644 third_party/code.google.com/p/go.net/html/atom/gen.go create mode 100644 third_party/code.google.com/p/go.net/html/atom/table.go create mode 100644 third_party/code.google.com/p/go.net/html/atom/table_test.go create mode 100644 third_party/code.google.com/p/go.net/html/const.go create mode 100644 third_party/code.google.com/p/go.net/html/doc.go create mode 100644 third_party/code.google.com/p/go.net/html/doctype.go create mode 100644 third_party/code.google.com/p/go.net/html/entity.go create mode 100644 third_party/code.google.com/p/go.net/html/entity_test.go create mode 100644 third_party/code.google.com/p/go.net/html/escape.go create mode 100644 third_party/code.google.com/p/go.net/html/escape_test.go create mode 100644 third_party/code.google.com/p/go.net/html/example_test.go create mode 100644 third_party/code.google.com/p/go.net/html/foreign.go create mode 100644 third_party/code.google.com/p/go.net/html/node.go create mode 100644 third_party/code.google.com/p/go.net/html/node_test.go create mode 100644 third_party/code.google.com/p/go.net/html/parse.go create mode 100644 third_party/code.google.com/p/go.net/html/parse_test.go create mode 100644 third_party/code.google.com/p/go.net/html/render.go create mode 100644 third_party/code.google.com/p/go.net/html/render_test.go create mode 100644 third_party/code.google.com/p/go.net/html/testdata/go1.html create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/README create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/adoption01.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/adoption02.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/comments01.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/doctype01.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/entities01.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/entities02.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/html5test-com.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/inbody01.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/isindex.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/pending-spec-changes-plain-text-unsafe.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/pending-spec-changes.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/plain-text-unsafe.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/scriptdata01.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/scripted/adoption01.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/scripted/webkit01.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tables01.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests1.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests10.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests11.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests12.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests14.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests15.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests16.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests17.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests18.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests19.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests2.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests20.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests21.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests22.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests23.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests24.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests25.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests26.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests3.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests4.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests5.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests6.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests7.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests8.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests9.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tests_innerHTML_1.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/tricky01.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/webkit01.dat create mode 100644 third_party/code.google.com/p/go.net/html/testdata/webkit/webkit02.dat create mode 100644 third_party/code.google.com/p/go.net/html/token.go create mode 100644 third_party/code.google.com/p/go.net/html/token_test.go create mode 100644 third_party/code.google.com/p/go.net/idna/idna.go create mode 100644 third_party/code.google.com/p/go.net/idna/idna_test.go create mode 100644 third_party/code.google.com/p/go.net/idna/punycode.go create mode 100644 third_party/code.google.com/p/go.net/idna/punycode_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/control.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/control_bsd.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/control_linux.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/control_plan9.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/control_windows.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/dgramopt_plan9.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/dgramopt_posix.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/doc.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/endpoint.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/example_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/gen.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/genericopt_plan9.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/genericopt_posix.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/gentest.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/header.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/header_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/helper.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/helper_plan9.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/helper_posix.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/helper_unix.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/helper_windows.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/iana.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/iana_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/icmp.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/mockicmp_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/mocktransponder_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/multicast_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/multicastlistener_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/multicastsockopt_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/packet.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/payload.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/sockopt_bsd.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/sockopt_freebsd.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/sockopt_linux.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/sockopt_plan9.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/sockopt_unix.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/sockopt_windows.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/unicast_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv4/unicastsockopt_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/control.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/control_rfc2292_darwin.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/control_rfc3542_bsd.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/control_rfc3542_linux.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/control_rfc3542_plan9.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/control_rfc3542_windows.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/control_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/dgramopt_plan9.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/dgramopt_posix.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/doc.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/endpoint.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/gen.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/genericopt_plan9.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/genericopt_posix.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/gentest.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/helper.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/helper_plan9.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/helper_unix.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/helper_windows.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/iana.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/iana_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/icmp.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/icmp_bsd.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/icmp_linux.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/icmp_plan9.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/icmp_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/icmp_windows.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/mockicmp_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/mocktransponder_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/multicast_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/multicastlistener_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/multicastsockopt_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/payload.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/payload_cmsg.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/payload_noncmsg.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/sockopt_rfc2292_darwin.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/sockopt_rfc3493_bsd.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/sockopt_rfc3493_linux.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/sockopt_rfc3493_unix.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/sockopt_rfc3493_windows.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/sockopt_rfc3542_bsd.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/sockopt_rfc3542_linux.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/sockopt_rfc3542_plan9.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/sockopt_rfc3542_unix.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/sockopt_rfc3542_windows.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/sockopt_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/unicast_test.go create mode 100644 third_party/code.google.com/p/go.net/ipv6/unicastsockopt_test.go create mode 100644 third_party/code.google.com/p/go.net/proxy/direct.go create mode 100644 third_party/code.google.com/p/go.net/proxy/per_host.go create mode 100644 third_party/code.google.com/p/go.net/proxy/per_host_test.go create mode 100644 third_party/code.google.com/p/go.net/proxy/proxy.go create mode 100644 third_party/code.google.com/p/go.net/proxy/proxy_test.go create mode 100644 third_party/code.google.com/p/go.net/proxy/socks5.go create mode 100644 third_party/code.google.com/p/go.net/publicsuffix/gen.go create mode 100644 third_party/code.google.com/p/go.net/publicsuffix/list.go create mode 100644 third_party/code.google.com/p/go.net/publicsuffix/list_test.go create mode 100644 third_party/code.google.com/p/go.net/publicsuffix/table.go create mode 100644 third_party/code.google.com/p/go.net/publicsuffix/table_test.go create mode 100644 third_party/code.google.com/p/go.net/spdy/dictionary.go create mode 100644 third_party/code.google.com/p/go.net/spdy/read.go create mode 100644 third_party/code.google.com/p/go.net/spdy/spdy_test.go create mode 100644 third_party/code.google.com/p/go.net/spdy/types.go create mode 100644 third_party/code.google.com/p/go.net/spdy/write.go create mode 100644 third_party/code.google.com/p/go.net/websocket/client.go create mode 100644 third_party/code.google.com/p/go.net/websocket/exampledial_test.go create mode 100644 third_party/code.google.com/p/go.net/websocket/examplehandler_test.go create mode 100644 third_party/code.google.com/p/go.net/websocket/hixie.go create mode 100644 third_party/code.google.com/p/go.net/websocket/hixie_test.go create mode 100644 third_party/code.google.com/p/go.net/websocket/hybi.go create mode 100644 third_party/code.google.com/p/go.net/websocket/hybi_test.go create mode 100644 third_party/code.google.com/p/go.net/websocket/server.go create mode 100644 third_party/code.google.com/p/go.net/websocket/websocket.go create mode 100644 third_party/code.google.com/p/go.net/websocket/websocket_test.go diff --git a/third_party/code.google.com/p/go.net/.hgignore b/third_party/code.google.com/p/go.net/.hgignore new file mode 100644 index 000000000..571db5fda --- /dev/null +++ b/third_party/code.google.com/p/go.net/.hgignore @@ -0,0 +1,2 @@ +syntax:glob +last-change diff --git a/third_party/code.google.com/p/go.net/AUTHORS b/third_party/code.google.com/p/go.net/AUTHORS new file mode 100644 index 000000000..15167cd74 --- /dev/null +++ b/third_party/code.google.com/p/go.net/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/third_party/code.google.com/p/go.net/CONTRIBUTORS b/third_party/code.google.com/p/go.net/CONTRIBUTORS new file mode 100644 index 000000000..1c4577e96 --- /dev/null +++ b/third_party/code.google.com/p/go.net/CONTRIBUTORS @@ -0,0 +1,3 @@ +# This source code was written by the Go contributors. +# The master list of contributors is in the main Go distribution, +# visible at http://tip.golang.org/CONTRIBUTORS. diff --git a/third_party/code.google.com/p/go.net/LICENSE b/third_party/code.google.com/p/go.net/LICENSE new file mode 100644 index 000000000..6a66aea5e --- /dev/null +++ b/third_party/code.google.com/p/go.net/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/code.google.com/p/go.net/PATENTS b/third_party/code.google.com/p/go.net/PATENTS new file mode 100644 index 000000000..733099041 --- /dev/null +++ b/third_party/code.google.com/p/go.net/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/third_party/code.google.com/p/go.net/README b/third_party/code.google.com/p/go.net/README new file mode 100644 index 000000000..6b13d8e50 --- /dev/null +++ b/third_party/code.google.com/p/go.net/README @@ -0,0 +1,3 @@ +This repository holds supplementary Go networking libraries. + +To submit changes to this repository, see http://golang.org/doc/contribute.html. diff --git a/third_party/code.google.com/p/go.net/codereview.cfg b/third_party/code.google.com/p/go.net/codereview.cfg new file mode 100644 index 000000000..e3eb47ca0 --- /dev/null +++ b/third_party/code.google.com/p/go.net/codereview.cfg @@ -0,0 +1,2 @@ +defaultcc: golang-dev@googlegroups.com +contributors: http://go.googlecode.com/hg/CONTRIBUTORS diff --git a/third_party/code.google.com/p/go.net/dict/dict.go b/third_party/code.google.com/p/go.net/dict/dict.go new file mode 100644 index 000000000..e7f5290f5 --- /dev/null +++ b/third_party/code.google.com/p/go.net/dict/dict.go @@ -0,0 +1,210 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package dict implements the Dictionary Server Protocol +// as defined in RFC 2229. +package dict + +import ( + "net/textproto" + "strconv" + "strings" +) + +// A Client represents a client connection to a dictionary server. +type Client struct { + text *textproto.Conn +} + +// Dial returns a new client connected to a dictionary server at +// addr on the given network. +func Dial(network, addr string) (*Client, error) { + text, err := textproto.Dial(network, addr) + if err != nil { + return nil, err + } + _, _, err = text.ReadCodeLine(220) + if err != nil { + text.Close() + return nil, err + } + return &Client{text: text}, nil +} + +// Close closes the connection to the dictionary server. +func (c *Client) Close() error { + return c.text.Close() +} + +// A Dict represents a dictionary available on the server. +type Dict struct { + Name string // short name of dictionary + Desc string // long description +} + +// Dicts returns a list of the dictionaries available on the server. +func (c *Client) Dicts() ([]Dict, error) { + id, err := c.text.Cmd("SHOW DB") + if err != nil { + return nil, err + } + + c.text.StartResponse(id) + defer c.text.EndResponse(id) + + _, _, err = c.text.ReadCodeLine(110) + if err != nil { + return nil, err + } + lines, err := c.text.ReadDotLines() + if err != nil { + return nil, err + } + _, _, err = c.text.ReadCodeLine(250) + + dicts := make([]Dict, len(lines)) + for i := range dicts { + d := &dicts[i] + a, _ := fields(lines[i]) + if len(a) < 2 { + return nil, textproto.ProtocolError("invalid dictionary: " + lines[i]) + } + d.Name = a[0] + d.Desc = a[1] + } + return dicts, err +} + +// A Defn represents a definition. +type Defn struct { + Dict Dict // Dict where definition was found + Word string // Word being defined + Text []byte // Definition text, typically multiple lines +} + +// Define requests the definition of the given word. +// The argument dict names the dictionary to use, +// the Name field of a Dict returned by Dicts. +// +// The special dictionary name "*" means to look in all the +// server's dictionaries. +// The special dictionary name "!" means to look in all the +// server's dictionaries in turn, stopping after finding the word +// in one of them. +func (c *Client) Define(dict, word string) ([]*Defn, error) { + id, err := c.text.Cmd("DEFINE %s %q", dict, word) + if err != nil { + return nil, err + } + + c.text.StartResponse(id) + defer c.text.EndResponse(id) + + _, line, err := c.text.ReadCodeLine(150) + if err != nil { + return nil, err + } + a, _ := fields(line) + if len(a) < 1 { + return nil, textproto.ProtocolError("malformed response: " + line) + } + n, err := strconv.Atoi(a[0]) + if err != nil { + return nil, textproto.ProtocolError("invalid definition count: " + a[0]) + } + def := make([]*Defn, n) + for i := 0; i < n; i++ { + _, line, err = c.text.ReadCodeLine(151) + if err != nil { + return nil, err + } + a, _ := fields(line) + if len(a) < 3 { + // skip it, to keep protocol in sync + i-- + n-- + def = def[0:n] + continue + } + d := &Defn{Word: a[0], Dict: Dict{a[1], a[2]}} + d.Text, err = c.text.ReadDotBytes() + if err != nil { + return nil, err + } + def[i] = d + } + _, _, err = c.text.ReadCodeLine(250) + return def, err +} + +// Fields returns the fields in s. +// Fields are space separated unquoted words +// or quoted with single or double quote. +func fields(s string) ([]string, error) { + var v []string + i := 0 + for { + for i < len(s) && (s[i] == ' ' || s[i] == '\t') { + i++ + } + if i >= len(s) { + break + } + if s[i] == '"' || s[i] == '\'' { + q := s[i] + // quoted string + var j int + for j = i + 1; ; j++ { + if j >= len(s) { + return nil, textproto.ProtocolError("malformed quoted string") + } + if s[j] == '\\' { + j++ + continue + } + if s[j] == q { + j++ + break + } + } + v = append(v, unquote(s[i+1:j-1])) + i = j + } else { + // atom + var j int + for j = i; j < len(s); j++ { + if s[j] == ' ' || s[j] == '\t' || s[j] == '\\' || s[j] == '"' || s[j] == '\'' { + break + } + } + v = append(v, s[i:j]) + i = j + } + if i < len(s) { + c := s[i] + if c != ' ' && c != '\t' { + return nil, textproto.ProtocolError("quotes not on word boundaries") + } + } + } + return v, nil +} + +func unquote(s string) string { + if strings.Index(s, "\\") < 0 { + return s + } + b := []byte(s) + w := 0 + for r := 0; r < len(b); r++ { + c := b[r] + if c == '\\' { + r++ + c = b[r] + } + b[w] = c + w++ + } + return string(b[0:w]) +} diff --git a/third_party/code.google.com/p/go.net/html/atom/atom.go b/third_party/code.google.com/p/go.net/html/atom/atom.go new file mode 100644 index 000000000..227404bda --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/atom/atom.go @@ -0,0 +1,78 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package atom provides integer codes (also known as atoms) for a fixed set of +// frequently occurring HTML strings: tag names and attribute keys such as "p" +// and "id". +// +// Sharing an atom's name between all elements with the same tag can result in +// fewer string allocations when tokenizing and parsing HTML. Integer +// comparisons are also generally faster than string comparisons. +// +// The value of an atom's particular code is not guaranteed to stay the same +// between versions of this package. Neither is any ordering guaranteed: +// whether atom.H1 < atom.H2 may also change. The codes are not guaranteed to +// be dense. The only guarantees are that e.g. looking up "div" will yield +// atom.Div, calling atom.Div.String will return "div", and atom.Div != 0. +package atom + +// Atom is an integer code for a string. The zero value maps to "". +type Atom uint32 + +// String returns the atom's name. +func (a Atom) String() string { + start := uint32(a >> 8) + n := uint32(a & 0xff) + if start+n > uint32(len(atomText)) { + return "" + } + return atomText[start : start+n] +} + +func (a Atom) string() string { + return atomText[a>>8 : a>>8+a&0xff] +} + +// fnv computes the FNV hash with an arbitrary starting value h. +func fnv(h uint32, s []byte) uint32 { + for i := range s { + h ^= uint32(s[i]) + h *= 16777619 + } + return h +} + +func match(s string, t []byte) bool { + for i, c := range t { + if s[i] != c { + return false + } + } + return true +} + +// Lookup returns the atom whose name is s. It returns zero if there is no +// such atom. The lookup is case sensitive. +func Lookup(s []byte) Atom { + if len(s) == 0 || len(s) > maxAtomLen { + return 0 + } + h := fnv(hash0, s) + if a := table[h&uint32(len(table)-1)]; int(a&0xff) == len(s) && match(a.string(), s) { + return a + } + if a := table[(h>>16)&uint32(len(table)-1)]; int(a&0xff) == len(s) && match(a.string(), s) { + return a + } + return 0 +} + +// String returns a string whose contents are equal to s. In that sense, it is +// equivalent to string(s) but may be more efficient. +func String(s []byte) string { + if a := Lookup(s); a != 0 { + return a.String() + } + return string(s) +} diff --git a/third_party/code.google.com/p/go.net/html/atom/atom_test.go b/third_party/code.google.com/p/go.net/html/atom/atom_test.go new file mode 100644 index 000000000..6e33704dd --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/atom/atom_test.go @@ -0,0 +1,109 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atom + +import ( + "sort" + "testing" +) + +func TestKnown(t *testing.T) { + for _, s := range testAtomList { + if atom := Lookup([]byte(s)); atom.String() != s { + t.Errorf("Lookup(%q) = %#x (%q)", s, uint32(atom), atom.String()) + } + } +} + +func TestHits(t *testing.T) { + for _, a := range table { + if a == 0 { + continue + } + got := Lookup([]byte(a.String())) + if got != a { + t.Errorf("Lookup(%q) = %#x, want %#x", a.String(), uint32(got), uint32(a)) + } + } +} + +func TestMisses(t *testing.T) { + testCases := []string{ + "", + "\x00", + "\xff", + "A", + "DIV", + "Div", + "dIV", + "aa", + "a\x00", + "ab", + "abb", + "abbr0", + "abbr ", + " abbr", + " a", + "acceptcharset", + "acceptCharset", + "accept_charset", + "h0", + "h1h2", + "h7", + "onClick", + "λ", + // The following string has the same hash (0xa1d7fab7) as "onmouseover". + "\x00\x00\x00\x00\x00\x50\x18\xae\x38\xd0\xb7", + } + for _, tc := range testCases { + got := Lookup([]byte(tc)) + if got != 0 { + t.Errorf("Lookup(%q): got %d, want 0", tc, got) + } + } +} + +func TestForeignObject(t *testing.T) { + const ( + afo = Foreignobject + afO = ForeignObject + sfo = "foreignobject" + sfO = "foreignObject" + ) + if got := Lookup([]byte(sfo)); got != afo { + t.Errorf("Lookup(%q): got %#v, want %#v", sfo, got, afo) + } + if got := Lookup([]byte(sfO)); got != afO { + t.Errorf("Lookup(%q): got %#v, want %#v", sfO, got, afO) + } + if got := afo.String(); got != sfo { + t.Errorf("Atom(%#v).String(): got %q, want %q", afo, got, sfo) + } + if got := afO.String(); got != sfO { + t.Errorf("Atom(%#v).String(): got %q, want %q", afO, got, sfO) + } +} + +func BenchmarkLookup(b *testing.B) { + sortedTable := make([]string, 0, len(table)) + for _, a := range table { + if a != 0 { + sortedTable = append(sortedTable, a.String()) + } + } + sort.Strings(sortedTable) + + x := make([][]byte, 1000) + for i := range x { + x[i] = []byte(sortedTable[i%len(sortedTable)]) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, s := range x { + Lookup(s) + } + } +} diff --git a/third_party/code.google.com/p/go.net/html/atom/gen.go b/third_party/code.google.com/p/go.net/html/atom/gen.go new file mode 100644 index 000000000..9958a7188 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/atom/gen.go @@ -0,0 +1,636 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This program generates table.go and table_test.go. +// Invoke as +// +// go run gen.go |gofmt >table.go +// go run gen.go -test |gofmt >table_test.go + +import ( + "flag" + "fmt" + "math/rand" + "os" + "sort" + "strings" +) + +// identifier converts s to a Go exported identifier. +// It converts "div" to "Div" and "accept-charset" to "AcceptCharset". +func identifier(s string) string { + b := make([]byte, 0, len(s)) + cap := true + for _, c := range s { + if c == '-' { + cap = true + continue + } + if cap && 'a' <= c && c <= 'z' { + c -= 'a' - 'A' + } + cap = false + b = append(b, byte(c)) + } + return string(b) +} + +var test = flag.Bool("test", false, "generate table_test.go") + +func main() { + flag.Parse() + + var all []string + all = append(all, elements...) + all = append(all, attributes...) + all = append(all, eventHandlers...) + all = append(all, extra...) + sort.Strings(all) + + if *test { + fmt.Printf("// generated by go run gen.go -test; DO NOT EDIT\n\n") + fmt.Printf("package atom\n\n") + fmt.Printf("var testAtomList = []string{\n") + for _, s := range all { + fmt.Printf("\t%q,\n", s) + } + fmt.Printf("}\n") + return + } + + // uniq - lists have dups + // compute max len too + maxLen := 0 + w := 0 + for _, s := range all { + if w == 0 || all[w-1] != s { + if maxLen < len(s) { + maxLen = len(s) + } + all[w] = s + w++ + } + } + all = all[:w] + + // Find hash that minimizes table size. + var best *table + for i := 0; i < 1000000; i++ { + if best != nil && 1<<(best.k-1) < len(all) { + break + } + h := rand.Uint32() + for k := uint(0); k <= 16; k++ { + if best != nil && k >= best.k { + break + } + var t table + if t.init(h, k, all) { + best = &t + break + } + } + } + if best == nil { + fmt.Fprintf(os.Stderr, "failed to construct string table\n") + os.Exit(1) + } + + // Lay out strings, using overlaps when possible. + layout := append([]string{}, all...) + + // Remove strings that are substrings of other strings + for changed := true; changed; { + changed = false + for i, s := range layout { + if s == "" { + continue + } + for j, t := range layout { + if i != j && t != "" && strings.Contains(s, t) { + changed = true + layout[j] = "" + } + } + } + } + + // Join strings where one suffix matches another prefix. + for { + // Find best i, j, k such that layout[i][len-k:] == layout[j][:k], + // maximizing overlap length k. + besti := -1 + bestj := -1 + bestk := 0 + for i, s := range layout { + if s == "" { + continue + } + for j, t := range layout { + if i == j { + continue + } + for k := bestk + 1; k <= len(s) && k <= len(t); k++ { + if s[len(s)-k:] == t[:k] { + besti = i + bestj = j + bestk = k + } + } + } + } + if bestk > 0 { + layout[besti] += layout[bestj][bestk:] + layout[bestj] = "" + continue + } + break + } + + text := strings.Join(layout, "") + + atom := map[string]uint32{} + for _, s := range all { + off := strings.Index(text, s) + if off < 0 { + panic("lost string " + s) + } + atom[s] = uint32(off<<8 | len(s)) + } + + // Generate the Go code. + fmt.Printf("// generated by go run gen.go; DO NOT EDIT\n\n") + fmt.Printf("package atom\n\nconst (\n") + for _, s := range all { + fmt.Printf("\t%s Atom = %#x\n", identifier(s), atom[s]) + } + fmt.Printf(")\n\n") + + fmt.Printf("const hash0 = %#x\n\n", best.h0) + fmt.Printf("const maxAtomLen = %d\n\n", maxLen) + + fmt.Printf("var table = [1<<%d]Atom{\n", best.k) + for i, s := range best.tab { + if s == "" { + continue + } + fmt.Printf("\t%#x: %#x, // %s\n", i, atom[s], s) + } + fmt.Printf("}\n") + datasize := (1 << best.k) * 4 + + fmt.Printf("const atomText =\n") + textsize := len(text) + for len(text) > 60 { + fmt.Printf("\t%q +\n", text[:60]) + text = text[60:] + } + fmt.Printf("\t%q\n\n", text) + + fmt.Fprintf(os.Stderr, "%d atoms; %d string bytes + %d tables = %d total data\n", len(all), textsize, datasize, textsize+datasize) +} + +type byLen []string + +func (x byLen) Less(i, j int) bool { return len(x[i]) > len(x[j]) } +func (x byLen) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x byLen) Len() int { return len(x) } + +// fnv computes the FNV hash with an arbitrary starting value h. +func fnv(h uint32, s string) uint32 { + for i := 0; i < len(s); i++ { + h ^= uint32(s[i]) + h *= 16777619 + } + return h +} + +// A table represents an attempt at constructing the lookup table. +// The lookup table uses cuckoo hashing, meaning that each string +// can be found in one of two positions. +type table struct { + h0 uint32 + k uint + mask uint32 + tab []string +} + +// hash returns the two hashes for s. +func (t *table) hash(s string) (h1, h2 uint32) { + h := fnv(t.h0, s) + h1 = h & t.mask + h2 = (h >> 16) & t.mask + return +} + +// init initializes the table with the given parameters. +// h0 is the initial hash value, +// k is the number of bits of hash value to use, and +// x is the list of strings to store in the table. +// init returns false if the table cannot be constructed. +func (t *table) init(h0 uint32, k uint, x []string) bool { + t.h0 = h0 + t.k = k + t.tab = make([]string, 1< len(t.tab) { + return false + } + s := t.tab[i] + h1, h2 := t.hash(s) + j := h1 + h2 - i + if t.tab[j] != "" && !t.push(j, depth+1) { + return false + } + t.tab[j] = s + return true +} + +// The lists of element names and attribute keys were taken from +// http://www.whatwg.org/specs/web-apps/current-work/multipage/section-index.html +// as of the "HTML Living Standard - Last Updated 30 May 2012" version. + +var elements = []string{ + "a", + "abbr", + "address", + "area", + "article", + "aside", + "audio", + "b", + "base", + "bdi", + "bdo", + "blockquote", + "body", + "br", + "button", + "canvas", + "caption", + "cite", + "code", + "col", + "colgroup", + "command", + "data", + "datalist", + "dd", + "del", + "details", + "dfn", + "dialog", + "div", + "dl", + "dt", + "em", + "embed", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "head", + "header", + "hgroup", + "hr", + "html", + "i", + "iframe", + "img", + "input", + "ins", + "kbd", + "keygen", + "label", + "legend", + "li", + "link", + "map", + "mark", + "menu", + "meta", + "meter", + "nav", + "noscript", + "object", + "ol", + "optgroup", + "option", + "output", + "p", + "param", + "pre", + "progress", + "q", + "rp", + "rt", + "ruby", + "s", + "samp", + "script", + "section", + "select", + "small", + "source", + "span", + "strong", + "style", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "textarea", + "tfoot", + "th", + "thead", + "time", + "title", + "tr", + "track", + "u", + "ul", + "var", + "video", + "wbr", +} + +var attributes = []string{ + "accept", + "accept-charset", + "accesskey", + "action", + "alt", + "async", + "autocomplete", + "autofocus", + "autoplay", + "border", + "challenge", + "charset", + "checked", + "cite", + "class", + "cols", + "colspan", + "command", + "content", + "contenteditable", + "contextmenu", + "controls", + "coords", + "crossorigin", + "data", + "datetime", + "default", + "defer", + "dir", + "dirname", + "disabled", + "download", + "draggable", + "dropzone", + "enctype", + "for", + "form", + "formaction", + "formenctype", + "formmethod", + "formnovalidate", + "formtarget", + "headers", + "height", + "hidden", + "high", + "href", + "hreflang", + "http-equiv", + "icon", + "id", + "inert", + "ismap", + "itemid", + "itemprop", + "itemref", + "itemscope", + "itemtype", + "keytype", + "kind", + "label", + "lang", + "list", + "loop", + "low", + "manifest", + "max", + "maxlength", + "media", + "mediagroup", + "method", + "min", + "multiple", + "muted", + "name", + "novalidate", + "open", + "optimum", + "pattern", + "ping", + "placeholder", + "poster", + "preload", + "radiogroup", + "readonly", + "rel", + "required", + "reversed", + "rows", + "rowspan", + "sandbox", + "spellcheck", + "scope", + "scoped", + "seamless", + "selected", + "shape", + "size", + "sizes", + "span", + "src", + "srcdoc", + "srclang", + "start", + "step", + "style", + "tabindex", + "target", + "title", + "translate", + "type", + "typemustmatch", + "usemap", + "value", + "width", + "wrap", +} + +var eventHandlers = []string{ + "onabort", + "onafterprint", + "onbeforeprint", + "onbeforeunload", + "onblur", + "oncancel", + "oncanplay", + "oncanplaythrough", + "onchange", + "onclick", + "onclose", + "oncontextmenu", + "oncuechange", + "ondblclick", + "ondrag", + "ondragend", + "ondragenter", + "ondragleave", + "ondragover", + "ondragstart", + "ondrop", + "ondurationchange", + "onemptied", + "onended", + "onerror", + "onfocus", + "onhashchange", + "oninput", + "oninvalid", + "onkeydown", + "onkeypress", + "onkeyup", + "onload", + "onloadeddata", + "onloadedmetadata", + "onloadstart", + "onmessage", + "onmousedown", + "onmousemove", + "onmouseout", + "onmouseover", + "onmouseup", + "onmousewheel", + "onoffline", + "ononline", + "onpagehide", + "onpageshow", + "onpause", + "onplay", + "onplaying", + "onpopstate", + "onprogress", + "onratechange", + "onreset", + "onresize", + "onscroll", + "onseeked", + "onseeking", + "onselect", + "onshow", + "onstalled", + "onstorage", + "onsubmit", + "onsuspend", + "ontimeupdate", + "onunload", + "onvolumechange", + "onwaiting", +} + +// extra are ad-hoc values not covered by any of the lists above. +var extra = []string{ + "align", + "annotation", + "annotation-xml", + "applet", + "basefont", + "bgsound", + "big", + "blink", + "center", + "color", + "desc", + "face", + "font", + "foreignObject", // HTML is case-insensitive, but SVG-embedded-in-HTML is case-sensitive. + "foreignobject", + "frame", + "frameset", + "image", + "isindex", + "listing", + "malignmark", + "marquee", + "math", + "mglyph", + "mi", + "mn", + "mo", + "ms", + "mtext", + "nobr", + "noembed", + "noframes", + "plaintext", + "prompt", + "public", + "spacer", + "strike", + "svg", + "system", + "tt", + "xmp", +} diff --git a/third_party/code.google.com/p/go.net/html/atom/table.go b/third_party/code.google.com/p/go.net/html/atom/table.go new file mode 100644 index 000000000..20b8b8a59 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/atom/table.go @@ -0,0 +1,694 @@ +// generated by go run gen.go; DO NOT EDIT + +package atom + +const ( + A Atom = 0x1 + Abbr Atom = 0x4 + Accept Atom = 0x2106 + AcceptCharset Atom = 0x210e + Accesskey Atom = 0x3309 + Action Atom = 0x21b06 + Address Atom = 0x5d507 + Align Atom = 0x1105 + Alt Atom = 0x4503 + Annotation Atom = 0x18d0a + AnnotationXml Atom = 0x18d0e + Applet Atom = 0x2d106 + Area Atom = 0x31804 + Article Atom = 0x39907 + Aside Atom = 0x4f05 + Async Atom = 0x9305 + Audio Atom = 0xaf05 + Autocomplete Atom = 0xd50c + Autofocus Atom = 0xe109 + Autoplay Atom = 0x10c08 + B Atom = 0x101 + Base Atom = 0x11404 + Basefont Atom = 0x11408 + Bdi Atom = 0x1a03 + Bdo Atom = 0x12503 + Bgsound Atom = 0x13807 + Big Atom = 0x14403 + Blink Atom = 0x14705 + Blockquote Atom = 0x14c0a + Body Atom = 0x2f04 + Border Atom = 0x15606 + Br Atom = 0x202 + Button Atom = 0x15c06 + Canvas Atom = 0x4b06 + Caption Atom = 0x1e007 + Center Atom = 0x2df06 + Challenge Atom = 0x23e09 + Charset Atom = 0x2807 + Checked Atom = 0x33f07 + Cite Atom = 0x9704 + Class Atom = 0x3d905 + Code Atom = 0x16f04 + Col Atom = 0x17603 + Colgroup Atom = 0x17608 + Color Atom = 0x18305 + Cols Atom = 0x18804 + Colspan Atom = 0x18807 + Command Atom = 0x19b07 + Content Atom = 0x42c07 + Contenteditable Atom = 0x42c0f + Contextmenu Atom = 0x3480b + Controls Atom = 0x1ae08 + Coords Atom = 0x1ba06 + Crossorigin Atom = 0x1c40b + Data Atom = 0x44304 + Datalist Atom = 0x44308 + Datetime Atom = 0x25b08 + Dd Atom = 0x28802 + Default Atom = 0x5207 + Defer Atom = 0x17105 + Del Atom = 0x4d603 + Desc Atom = 0x4804 + Details Atom = 0x6507 + Dfn Atom = 0x8303 + Dialog Atom = 0x1b06 + Dir Atom = 0x9d03 + Dirname Atom = 0x9d07 + Disabled Atom = 0x10008 + Div Atom = 0x10703 + Dl Atom = 0x13e02 + Download Atom = 0x40908 + Draggable Atom = 0x1a109 + Dropzone Atom = 0x3a208 + Dt Atom = 0x4e402 + Em Atom = 0x7f02 + Embed Atom = 0x7f05 + Enctype Atom = 0x23007 + Face Atom = 0x2dd04 + Fieldset Atom = 0x1d508 + Figcaption Atom = 0x1dd0a + Figure Atom = 0x1f106 + Font Atom = 0x11804 + Footer Atom = 0x5906 + For Atom = 0x1fd03 + ForeignObject Atom = 0x1fd0d + Foreignobject Atom = 0x20a0d + Form Atom = 0x21704 + Formaction Atom = 0x2170a + Formenctype Atom = 0x22c0b + Formmethod Atom = 0x2470a + Formnovalidate Atom = 0x2510e + Formtarget Atom = 0x2660a + Frame Atom = 0x8705 + Frameset Atom = 0x8708 + H1 Atom = 0x13602 + H2 Atom = 0x29602 + H3 Atom = 0x2c502 + H4 Atom = 0x30e02 + H5 Atom = 0x4e602 + H6 Atom = 0x27002 + Head Atom = 0x2fa04 + Header Atom = 0x2fa06 + Headers Atom = 0x2fa07 + Height Atom = 0x27206 + Hgroup Atom = 0x27a06 + Hidden Atom = 0x28606 + High Atom = 0x29304 + Hr Atom = 0x13102 + Href Atom = 0x29804 + Hreflang Atom = 0x29808 + Html Atom = 0x27604 + HttpEquiv Atom = 0x2a00a + I Atom = 0x601 + Icon Atom = 0x42b04 + Id Atom = 0x5102 + Iframe Atom = 0x2b406 + Image Atom = 0x2ba05 + Img Atom = 0x2bf03 + Inert Atom = 0x4c105 + Input Atom = 0x3f605 + Ins Atom = 0x1cd03 + Isindex Atom = 0x2c707 + Ismap Atom = 0x2ce05 + Itemid Atom = 0x9806 + Itemprop Atom = 0x57e08 + Itemref Atom = 0x2d707 + Itemscope Atom = 0x2e509 + Itemtype Atom = 0x2ef08 + Kbd Atom = 0x1903 + Keygen Atom = 0x3906 + Keytype Atom = 0x51207 + Kind Atom = 0xfd04 + Label Atom = 0xba05 + Lang Atom = 0x29c04 + Legend Atom = 0x1a806 + Li Atom = 0x1202 + Link Atom = 0x14804 + List Atom = 0x44704 + Listing Atom = 0x44707 + Loop Atom = 0xbe04 + Low Atom = 0x13f03 + Malignmark Atom = 0x100a + Manifest Atom = 0x5b608 + Map Atom = 0x2d003 + Mark Atom = 0x1604 + Marquee Atom = 0x5f207 + Math Atom = 0x2f704 + Max Atom = 0x30603 + Maxlength Atom = 0x30609 + Media Atom = 0xa205 + Mediagroup Atom = 0xa20a + Menu Atom = 0x34f04 + Meta Atom = 0x45604 + Meter Atom = 0x26105 + Method Atom = 0x24b06 + Mglyph Atom = 0x2c006 + Mi Atom = 0x9b02 + Min Atom = 0x31003 + Mn Atom = 0x25402 + Mo Atom = 0x47a02 + Ms Atom = 0x2e802 + Mtext Atom = 0x31305 + Multiple Atom = 0x32108 + Muted Atom = 0x32905 + Name Atom = 0xa004 + Nav Atom = 0x3e03 + Nobr Atom = 0x7404 + Noembed Atom = 0x7d07 + Noframes Atom = 0x8508 + Noscript Atom = 0x28b08 + Novalidate Atom = 0x2550a + Object Atom = 0x21106 + Ol Atom = 0xcd02 + Onabort Atom = 0x16007 + Onafterprint Atom = 0x1e50c + Onbeforeprint Atom = 0x21f0d + Onbeforeunload Atom = 0x5c90e + Onblur Atom = 0x3e206 + Oncancel Atom = 0xb308 + Oncanplay Atom = 0x12709 + Oncanplaythrough Atom = 0x12710 + Onchange Atom = 0x3b808 + Onclick Atom = 0x2ad07 + Onclose Atom = 0x32e07 + Oncontextmenu Atom = 0x3460d + Oncuechange Atom = 0x3530b + Ondblclick Atom = 0x35e0a + Ondrag Atom = 0x36806 + Ondragend Atom = 0x36809 + Ondragenter Atom = 0x3710b + Ondragleave Atom = 0x37c0b + Ondragover Atom = 0x3870a + Ondragstart Atom = 0x3910b + Ondrop Atom = 0x3a006 + Ondurationchange Atom = 0x3b010 + Onemptied Atom = 0x3a709 + Onended Atom = 0x3c007 + Onerror Atom = 0x3c707 + Onfocus Atom = 0x3ce07 + Onhashchange Atom = 0x3e80c + Oninput Atom = 0x3f407 + Oninvalid Atom = 0x3fb09 + Onkeydown Atom = 0x40409 + Onkeypress Atom = 0x4110a + Onkeyup Atom = 0x42107 + Onload Atom = 0x43b06 + Onloadeddata Atom = 0x43b0c + Onloadedmetadata Atom = 0x44e10 + Onloadstart Atom = 0x4640b + Onmessage Atom = 0x46f09 + Onmousedown Atom = 0x4780b + Onmousemove Atom = 0x4830b + Onmouseout Atom = 0x48e0a + Onmouseover Atom = 0x49b0b + Onmouseup Atom = 0x4a609 + Onmousewheel Atom = 0x4af0c + Onoffline Atom = 0x4bb09 + Ononline Atom = 0x4c608 + Onpagehide Atom = 0x4ce0a + Onpageshow Atom = 0x4d90a + Onpause Atom = 0x4e807 + Onplay Atom = 0x4f206 + Onplaying Atom = 0x4f209 + Onpopstate Atom = 0x4fb0a + Onprogress Atom = 0x5050a + Onratechange Atom = 0x5190c + Onreset Atom = 0x52507 + Onresize Atom = 0x52c08 + Onscroll Atom = 0x53a08 + Onseeked Atom = 0x54208 + Onseeking Atom = 0x54a09 + Onselect Atom = 0x55308 + Onshow Atom = 0x55d06 + Onstalled Atom = 0x56609 + Onstorage Atom = 0x56f09 + Onsubmit Atom = 0x57808 + Onsuspend Atom = 0x58809 + Ontimeupdate Atom = 0x1190c + Onunload Atom = 0x59108 + Onvolumechange Atom = 0x5990e + Onwaiting Atom = 0x5a709 + Open Atom = 0x58404 + Optgroup Atom = 0xc008 + Optimum Atom = 0x5b007 + Option Atom = 0x5c506 + Output Atom = 0x49506 + P Atom = 0xc01 + Param Atom = 0xc05 + Pattern Atom = 0x6e07 + Ping Atom = 0xab04 + Placeholder Atom = 0xc70b + Plaintext Atom = 0xf109 + Poster Atom = 0x17d06 + Pre Atom = 0x27f03 + Preload Atom = 0x27f07 + Progress Atom = 0x50708 + Prompt Atom = 0x5bf06 + Public Atom = 0x42706 + Q Atom = 0x15101 + Radiogroup Atom = 0x30a + Readonly Atom = 0x31908 + Rel Atom = 0x28003 + Required Atom = 0x1f508 + Reversed Atom = 0x5e08 + Rows Atom = 0x7704 + Rowspan Atom = 0x7707 + Rp Atom = 0x1eb02 + Rt Atom = 0x16502 + Ruby Atom = 0xd104 + S Atom = 0x2c01 + Samp Atom = 0x6b04 + Sandbox Atom = 0xe907 + Scope Atom = 0x2e905 + Scoped Atom = 0x2e906 + Script Atom = 0x28d06 + Seamless Atom = 0x33308 + Section Atom = 0x3dd07 + Select Atom = 0x55506 + Selected Atom = 0x55508 + Shape Atom = 0x1b505 + Size Atom = 0x53004 + Sizes Atom = 0x53005 + Small Atom = 0x1bf05 + Source Atom = 0x1cf06 + Spacer Atom = 0x30006 + Span Atom = 0x7a04 + Spellcheck Atom = 0x33a0a + Src Atom = 0x3d403 + Srcdoc Atom = 0x3d406 + Srclang Atom = 0x41a07 + Start Atom = 0x39705 + Step Atom = 0x5bc04 + Strike Atom = 0x50e06 + Strong Atom = 0x53406 + Style Atom = 0x5db05 + Sub Atom = 0x57a03 + Summary Atom = 0x5e007 + Sup Atom = 0x5e703 + Svg Atom = 0x5ea03 + System Atom = 0x5ed06 + Tabindex Atom = 0x45c08 + Table Atom = 0x43605 + Target Atom = 0x26a06 + Tbody Atom = 0x2e05 + Td Atom = 0x4702 + Textarea Atom = 0x31408 + Tfoot Atom = 0x5805 + Th Atom = 0x13002 + Thead Atom = 0x2f905 + Time Atom = 0x11b04 + Title Atom = 0x8e05 + Tr Atom = 0xf902 + Track Atom = 0xf905 + Translate Atom = 0x16609 + Tt Atom = 0x7002 + Type Atom = 0x23304 + Typemustmatch Atom = 0x2330d + U Atom = 0xb01 + Ul Atom = 0x5602 + Usemap Atom = 0x4ec06 + Value Atom = 0x4005 + Var Atom = 0x10903 + Video Atom = 0x2a905 + Wbr Atom = 0x14103 + Width Atom = 0x4e205 + Wrap Atom = 0x56204 + Xmp Atom = 0xef03 +) + +const hash0 = 0xc17da63e + +const maxAtomLen = 16 + +var table = [1 << 9]Atom{ + 0x1: 0x4830b, // onmousemove + 0x2: 0x5a709, // onwaiting + 0x4: 0x5bf06, // prompt + 0x7: 0x5b007, // optimum + 0x8: 0x1604, // mark + 0xa: 0x2d707, // itemref + 0xb: 0x4d90a, // onpageshow + 0xc: 0x55506, // select + 0xd: 0x1a109, // draggable + 0xe: 0x3e03, // nav + 0xf: 0x19b07, // command + 0x11: 0xb01, // u + 0x14: 0x2fa07, // headers + 0x15: 0x44308, // datalist + 0x17: 0x6b04, // samp + 0x1a: 0x40409, // onkeydown + 0x1b: 0x53a08, // onscroll + 0x1c: 0x17603, // col + 0x20: 0x57e08, // itemprop + 0x21: 0x2a00a, // http-equiv + 0x22: 0x5e703, // sup + 0x24: 0x1f508, // required + 0x2b: 0x27f07, // preload + 0x2c: 0x21f0d, // onbeforeprint + 0x2d: 0x3710b, // ondragenter + 0x2e: 0x4e402, // dt + 0x2f: 0x57808, // onsubmit + 0x30: 0x13102, // hr + 0x31: 0x3460d, // oncontextmenu + 0x33: 0x2ba05, // image + 0x34: 0x4e807, // onpause + 0x35: 0x27a06, // hgroup + 0x36: 0xab04, // ping + 0x37: 0x55308, // onselect + 0x3a: 0x10703, // div + 0x40: 0x9b02, // mi + 0x41: 0x33308, // seamless + 0x42: 0x2807, // charset + 0x43: 0x5102, // id + 0x44: 0x4fb0a, // onpopstate + 0x45: 0x4d603, // del + 0x46: 0x5f207, // marquee + 0x47: 0x3309, // accesskey + 0x49: 0x5906, // footer + 0x4a: 0x2d106, // applet + 0x4b: 0x2ce05, // ismap + 0x51: 0x34f04, // menu + 0x52: 0x2f04, // body + 0x55: 0x8708, // frameset + 0x56: 0x52507, // onreset + 0x57: 0x14705, // blink + 0x58: 0x8e05, // title + 0x59: 0x39907, // article + 0x5b: 0x13002, // th + 0x5d: 0x15101, // q + 0x5e: 0x58404, // open + 0x5f: 0x31804, // area + 0x61: 0x43b06, // onload + 0x62: 0x3f605, // input + 0x63: 0x11404, // base + 0x64: 0x18807, // colspan + 0x65: 0x51207, // keytype + 0x66: 0x13e02, // dl + 0x68: 0x1d508, // fieldset + 0x6a: 0x31003, // min + 0x6b: 0x10903, // var + 0x6f: 0x2fa06, // header + 0x70: 0x16502, // rt + 0x71: 0x17608, // colgroup + 0x72: 0x25402, // mn + 0x74: 0x16007, // onabort + 0x75: 0x3906, // keygen + 0x76: 0x4bb09, // onoffline + 0x77: 0x23e09, // challenge + 0x78: 0x2d003, // map + 0x7a: 0x30e02, // h4 + 0x7b: 0x3c707, // onerror + 0x7c: 0x30609, // maxlength + 0x7d: 0x31305, // mtext + 0x7e: 0x5805, // tfoot + 0x7f: 0x11804, // font + 0x80: 0x100a, // malignmark + 0x81: 0x45604, // meta + 0x82: 0x9305, // async + 0x83: 0x2c502, // h3 + 0x84: 0x28802, // dd + 0x85: 0x29804, // href + 0x86: 0xa20a, // mediagroup + 0x87: 0x1ba06, // coords + 0x88: 0x41a07, // srclang + 0x89: 0x35e0a, // ondblclick + 0x8a: 0x4005, // value + 0x8c: 0xb308, // oncancel + 0x8e: 0x33a0a, // spellcheck + 0x8f: 0x8705, // frame + 0x91: 0x14403, // big + 0x94: 0x21b06, // action + 0x95: 0x9d03, // dir + 0x97: 0x31908, // readonly + 0x99: 0x43605, // table + 0x9a: 0x5e007, // summary + 0x9b: 0x14103, // wbr + 0x9c: 0x30a, // radiogroup + 0x9d: 0xa004, // name + 0x9f: 0x5ed06, // system + 0xa1: 0x18305, // color + 0xa2: 0x4b06, // canvas + 0xa3: 0x27604, // html + 0xa5: 0x54a09, // onseeking + 0xac: 0x1b505, // shape + 0xad: 0x28003, // rel + 0xae: 0x12710, // oncanplaythrough + 0xaf: 0x3870a, // ondragover + 0xb1: 0x1fd0d, // foreignObject + 0xb3: 0x7704, // rows + 0xb6: 0x44707, // listing + 0xb7: 0x49506, // output + 0xb9: 0x3480b, // contextmenu + 0xbb: 0x13f03, // low + 0xbc: 0x1eb02, // rp + 0xbd: 0x58809, // onsuspend + 0xbe: 0x15c06, // button + 0xbf: 0x4804, // desc + 0xc1: 0x3dd07, // section + 0xc2: 0x5050a, // onprogress + 0xc3: 0x56f09, // onstorage + 0xc4: 0x2f704, // math + 0xc5: 0x4f206, // onplay + 0xc7: 0x5602, // ul + 0xc8: 0x6e07, // pattern + 0xc9: 0x4af0c, // onmousewheel + 0xca: 0x36809, // ondragend + 0xcb: 0xd104, // ruby + 0xcc: 0xc01, // p + 0xcd: 0x32e07, // onclose + 0xce: 0x26105, // meter + 0xcf: 0x13807, // bgsound + 0xd2: 0x27206, // height + 0xd4: 0x101, // b + 0xd5: 0x2ef08, // itemtype + 0xd8: 0x1e007, // caption + 0xd9: 0x10008, // disabled + 0xdc: 0x5ea03, // svg + 0xdd: 0x1bf05, // small + 0xde: 0x44304, // data + 0xe0: 0x4c608, // ononline + 0xe1: 0x2c006, // mglyph + 0xe3: 0x7f05, // embed + 0xe4: 0xf902, // tr + 0xe5: 0x4640b, // onloadstart + 0xe7: 0x3b010, // ondurationchange + 0xed: 0x12503, // bdo + 0xee: 0x4702, // td + 0xef: 0x4f05, // aside + 0xf0: 0x29602, // h2 + 0xf1: 0x50708, // progress + 0xf2: 0x14c0a, // blockquote + 0xf4: 0xba05, // label + 0xf5: 0x601, // i + 0xf7: 0x7707, // rowspan + 0xfb: 0x4f209, // onplaying + 0xfd: 0x2bf03, // img + 0xfe: 0xc008, // optgroup + 0xff: 0x42c07, // content + 0x101: 0x5190c, // onratechange + 0x103: 0x3e80c, // onhashchange + 0x104: 0x6507, // details + 0x106: 0x40908, // download + 0x109: 0xe907, // sandbox + 0x10b: 0x42c0f, // contenteditable + 0x10d: 0x37c0b, // ondragleave + 0x10e: 0x2106, // accept + 0x10f: 0x55508, // selected + 0x112: 0x2170a, // formaction + 0x113: 0x2df06, // center + 0x115: 0x44e10, // onloadedmetadata + 0x116: 0x14804, // link + 0x117: 0x11b04, // time + 0x118: 0x1c40b, // crossorigin + 0x119: 0x3ce07, // onfocus + 0x11a: 0x56204, // wrap + 0x11b: 0x42b04, // icon + 0x11d: 0x2a905, // video + 0x11e: 0x3d905, // class + 0x121: 0x5990e, // onvolumechange + 0x122: 0x3e206, // onblur + 0x123: 0x2e509, // itemscope + 0x124: 0x5db05, // style + 0x127: 0x42706, // public + 0x129: 0x2510e, // formnovalidate + 0x12a: 0x55d06, // onshow + 0x12c: 0x16609, // translate + 0x12d: 0x9704, // cite + 0x12e: 0x2e802, // ms + 0x12f: 0x1190c, // ontimeupdate + 0x130: 0xfd04, // kind + 0x131: 0x2660a, // formtarget + 0x135: 0x3c007, // onended + 0x136: 0x28606, // hidden + 0x137: 0x2c01, // s + 0x139: 0x2470a, // formmethod + 0x13a: 0x44704, // list + 0x13c: 0x27002, // h6 + 0x13d: 0xcd02, // ol + 0x13e: 0x3530b, // oncuechange + 0x13f: 0x20a0d, // foreignobject + 0x143: 0x5c90e, // onbeforeunload + 0x145: 0x3a709, // onemptied + 0x146: 0x17105, // defer + 0x147: 0xef03, // xmp + 0x148: 0xaf05, // audio + 0x149: 0x1903, // kbd + 0x14c: 0x46f09, // onmessage + 0x14d: 0x5c506, // option + 0x14e: 0x4503, // alt + 0x14f: 0x33f07, // checked + 0x150: 0x10c08, // autoplay + 0x152: 0x202, // br + 0x153: 0x2550a, // novalidate + 0x156: 0x7d07, // noembed + 0x159: 0x2ad07, // onclick + 0x15a: 0x4780b, // onmousedown + 0x15b: 0x3b808, // onchange + 0x15e: 0x3fb09, // oninvalid + 0x15f: 0x2e906, // scoped + 0x160: 0x1ae08, // controls + 0x161: 0x32905, // muted + 0x163: 0x4ec06, // usemap + 0x164: 0x1dd0a, // figcaption + 0x165: 0x36806, // ondrag + 0x166: 0x29304, // high + 0x168: 0x3d403, // src + 0x169: 0x17d06, // poster + 0x16b: 0x18d0e, // annotation-xml + 0x16c: 0x5bc04, // step + 0x16d: 0x4, // abbr + 0x16e: 0x1b06, // dialog + 0x170: 0x1202, // li + 0x172: 0x47a02, // mo + 0x175: 0x1fd03, // for + 0x176: 0x1cd03, // ins + 0x178: 0x53004, // size + 0x17a: 0x5207, // default + 0x17b: 0x1a03, // bdi + 0x17c: 0x4ce0a, // onpagehide + 0x17d: 0x9d07, // dirname + 0x17e: 0x23304, // type + 0x17f: 0x21704, // form + 0x180: 0x4c105, // inert + 0x181: 0x12709, // oncanplay + 0x182: 0x8303, // dfn + 0x183: 0x45c08, // tabindex + 0x186: 0x7f02, // em + 0x187: 0x29c04, // lang + 0x189: 0x3a208, // dropzone + 0x18a: 0x4110a, // onkeypress + 0x18b: 0x25b08, // datetime + 0x18c: 0x18804, // cols + 0x18d: 0x1, // a + 0x18e: 0x43b0c, // onloadeddata + 0x191: 0x15606, // border + 0x192: 0x2e05, // tbody + 0x193: 0x24b06, // method + 0x195: 0xbe04, // loop + 0x196: 0x2b406, // iframe + 0x198: 0x2fa04, // head + 0x19e: 0x5b608, // manifest + 0x19f: 0xe109, // autofocus + 0x1a0: 0x16f04, // code + 0x1a1: 0x53406, // strong + 0x1a2: 0x32108, // multiple + 0x1a3: 0xc05, // param + 0x1a6: 0x23007, // enctype + 0x1a7: 0x2dd04, // face + 0x1a8: 0xf109, // plaintext + 0x1a9: 0x13602, // h1 + 0x1aa: 0x56609, // onstalled + 0x1ad: 0x28d06, // script + 0x1ae: 0x30006, // spacer + 0x1af: 0x52c08, // onresize + 0x1b0: 0x49b0b, // onmouseover + 0x1b1: 0x59108, // onunload + 0x1b2: 0x54208, // onseeked + 0x1b4: 0x2330d, // typemustmatch + 0x1b5: 0x1f106, // figure + 0x1b6: 0x48e0a, // onmouseout + 0x1b7: 0x27f03, // pre + 0x1b8: 0x4e205, // width + 0x1bb: 0x7404, // nobr + 0x1be: 0x7002, // tt + 0x1bf: 0x1105, // align + 0x1c0: 0x3f407, // oninput + 0x1c3: 0x42107, // onkeyup + 0x1c6: 0x1e50c, // onafterprint + 0x1c7: 0x210e, // accept-charset + 0x1c8: 0x9806, // itemid + 0x1cb: 0x50e06, // strike + 0x1cc: 0x57a03, // sub + 0x1cd: 0xf905, // track + 0x1ce: 0x39705, // start + 0x1d0: 0x11408, // basefont + 0x1d6: 0x1cf06, // source + 0x1d7: 0x1a806, // legend + 0x1d8: 0x2f905, // thead + 0x1da: 0x2e905, // scope + 0x1dd: 0x21106, // object + 0x1de: 0xa205, // media + 0x1df: 0x18d0a, // annotation + 0x1e0: 0x22c0b, // formenctype + 0x1e2: 0x28b08, // noscript + 0x1e4: 0x53005, // sizes + 0x1e5: 0xd50c, // autocomplete + 0x1e6: 0x7a04, // span + 0x1e7: 0x8508, // noframes + 0x1e8: 0x26a06, // target + 0x1e9: 0x3a006, // ondrop + 0x1ea: 0x3d406, // srcdoc + 0x1ec: 0x5e08, // reversed + 0x1f0: 0x2c707, // isindex + 0x1f3: 0x29808, // hreflang + 0x1f5: 0x4e602, // h5 + 0x1f6: 0x5d507, // address + 0x1fa: 0x30603, // max + 0x1fb: 0xc70b, // placeholder + 0x1fc: 0x31408, // textarea + 0x1fe: 0x4a609, // onmouseup + 0x1ff: 0x3910b, // ondragstart +} + +const atomText = "abbradiogrouparamalignmarkbdialogaccept-charsetbodyaccesskey" + + "genavaluealtdescanvasidefaultfootereversedetailsampatternobr" + + "owspanoembedfnoframesetitleasyncitemidirnamediagroupingaudio" + + "ncancelabelooptgrouplaceholderubyautocompleteautofocusandbox" + + "mplaintextrackindisabledivarautoplaybasefontimeupdatebdoncan" + + "playthrough1bgsoundlowbrbigblinkblockquoteborderbuttonabortr" + + "anslatecodefercolgroupostercolorcolspannotation-xmlcommandra" + + "ggablegendcontrolshapecoordsmallcrossoriginsourcefieldsetfig" + + "captionafterprintfigurequiredforeignObjectforeignobjectforma" + + "ctionbeforeprintformenctypemustmatchallengeformmethodformnov" + + "alidatetimeterformtargeth6heightmlhgroupreloadhiddenoscripth" + + "igh2hreflanghttp-equivideonclickiframeimageimglyph3isindexis" + + "mappletitemrefacenteritemscopeditemtypematheaderspacermaxlen" + + "gth4minmtextareadonlymultiplemutedoncloseamlesspellcheckedon" + + "contextmenuoncuechangeondblclickondragendondragenterondragle" + + "aveondragoverondragstarticleondropzonemptiedondurationchange" + + "onendedonerroronfocusrcdoclassectionbluronhashchangeoninputo" + + "ninvalidonkeydownloadonkeypressrclangonkeyupublicontentedita" + + "bleonloadeddatalistingonloadedmetadatabindexonloadstartonmes" + + "sageonmousedownonmousemoveonmouseoutputonmouseoveronmouseupo" + + "nmousewheelonofflinertononlineonpagehidelonpageshowidth5onpa" + + "usemaponplayingonpopstateonprogresstrikeytypeonratechangeonr" + + "esetonresizestrongonscrollonseekedonseekingonselectedonshowr" + + "aponstalledonstorageonsubmitempropenonsuspendonunloadonvolum" + + "echangeonwaitingoptimumanifestepromptoptionbeforeunloaddress" + + "tylesummarysupsvgsystemarquee" diff --git a/third_party/code.google.com/p/go.net/html/atom/table_test.go b/third_party/code.google.com/p/go.net/html/atom/table_test.go new file mode 100644 index 000000000..db016a1c0 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/atom/table_test.go @@ -0,0 +1,341 @@ +// generated by go run gen.go -test; DO NOT EDIT + +package atom + +var testAtomList = []string{ + "a", + "abbr", + "accept", + "accept-charset", + "accesskey", + "action", + "address", + "align", + "alt", + "annotation", + "annotation-xml", + "applet", + "area", + "article", + "aside", + "async", + "audio", + "autocomplete", + "autofocus", + "autoplay", + "b", + "base", + "basefont", + "bdi", + "bdo", + "bgsound", + "big", + "blink", + "blockquote", + "body", + "border", + "br", + "button", + "canvas", + "caption", + "center", + "challenge", + "charset", + "checked", + "cite", + "cite", + "class", + "code", + "col", + "colgroup", + "color", + "cols", + "colspan", + "command", + "command", + "content", + "contenteditable", + "contextmenu", + "controls", + "coords", + "crossorigin", + "data", + "data", + "datalist", + "datetime", + "dd", + "default", + "defer", + "del", + "desc", + "details", + "dfn", + "dialog", + "dir", + "dirname", + "disabled", + "div", + "dl", + "download", + "draggable", + "dropzone", + "dt", + "em", + "embed", + "enctype", + "face", + "fieldset", + "figcaption", + "figure", + "font", + "footer", + "for", + "foreignObject", + "foreignobject", + "form", + "form", + "formaction", + "formenctype", + "formmethod", + "formnovalidate", + "formtarget", + "frame", + "frameset", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "head", + "header", + "headers", + "height", + "hgroup", + "hidden", + "high", + "hr", + "href", + "hreflang", + "html", + "http-equiv", + "i", + "icon", + "id", + "iframe", + "image", + "img", + "inert", + "input", + "ins", + "isindex", + "ismap", + "itemid", + "itemprop", + "itemref", + "itemscope", + "itemtype", + "kbd", + "keygen", + "keytype", + "kind", + "label", + "label", + "lang", + "legend", + "li", + "link", + "list", + "listing", + "loop", + "low", + "malignmark", + "manifest", + "map", + "mark", + "marquee", + "math", + "max", + "maxlength", + "media", + "mediagroup", + "menu", + "meta", + "meter", + "method", + "mglyph", + "mi", + "min", + "mn", + "mo", + "ms", + "mtext", + "multiple", + "muted", + "name", + "nav", + "nobr", + "noembed", + "noframes", + "noscript", + "novalidate", + "object", + "ol", + "onabort", + "onafterprint", + "onbeforeprint", + "onbeforeunload", + "onblur", + "oncancel", + "oncanplay", + "oncanplaythrough", + "onchange", + "onclick", + "onclose", + "oncontextmenu", + "oncuechange", + "ondblclick", + "ondrag", + "ondragend", + "ondragenter", + "ondragleave", + "ondragover", + "ondragstart", + "ondrop", + "ondurationchange", + "onemptied", + "onended", + "onerror", + "onfocus", + "onhashchange", + "oninput", + "oninvalid", + "onkeydown", + "onkeypress", + "onkeyup", + "onload", + "onloadeddata", + "onloadedmetadata", + "onloadstart", + "onmessage", + "onmousedown", + "onmousemove", + "onmouseout", + "onmouseover", + "onmouseup", + "onmousewheel", + "onoffline", + "ononline", + "onpagehide", + "onpageshow", + "onpause", + "onplay", + "onplaying", + "onpopstate", + "onprogress", + "onratechange", + "onreset", + "onresize", + "onscroll", + "onseeked", + "onseeking", + "onselect", + "onshow", + "onstalled", + "onstorage", + "onsubmit", + "onsuspend", + "ontimeupdate", + "onunload", + "onvolumechange", + "onwaiting", + "open", + "optgroup", + "optimum", + "option", + "output", + "p", + "param", + "pattern", + "ping", + "placeholder", + "plaintext", + "poster", + "pre", + "preload", + "progress", + "prompt", + "public", + "q", + "radiogroup", + "readonly", + "rel", + "required", + "reversed", + "rows", + "rowspan", + "rp", + "rt", + "ruby", + "s", + "samp", + "sandbox", + "scope", + "scoped", + "script", + "seamless", + "section", + "select", + "selected", + "shape", + "size", + "sizes", + "small", + "source", + "spacer", + "span", + "span", + "spellcheck", + "src", + "srcdoc", + "srclang", + "start", + "step", + "strike", + "strong", + "style", + "style", + "sub", + "summary", + "sup", + "svg", + "system", + "tabindex", + "table", + "target", + "tbody", + "td", + "textarea", + "tfoot", + "th", + "thead", + "time", + "title", + "title", + "tr", + "track", + "translate", + "tt", + "type", + "typemustmatch", + "u", + "ul", + "usemap", + "value", + "var", + "video", + "wbr", + "width", + "wrap", + "xmp", +} diff --git a/third_party/code.google.com/p/go.net/html/const.go b/third_party/code.google.com/p/go.net/html/const.go new file mode 100644 index 000000000..d7cc8bb9a --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/const.go @@ -0,0 +1,100 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +// Section 12.2.3.2 of the HTML5 specification says "The following elements +// have varying levels of special parsing rules". +// http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#the-stack-of-open-elements +var isSpecialElementMap = map[string]bool{ + "address": true, + "applet": true, + "area": true, + "article": true, + "aside": true, + "base": true, + "basefont": true, + "bgsound": true, + "blockquote": true, + "body": true, + "br": true, + "button": true, + "caption": true, + "center": true, + "col": true, + "colgroup": true, + "command": true, + "dd": true, + "details": true, + "dir": true, + "div": true, + "dl": true, + "dt": true, + "embed": true, + "fieldset": true, + "figcaption": true, + "figure": true, + "footer": true, + "form": true, + "frame": true, + "frameset": true, + "h1": true, + "h2": true, + "h3": true, + "h4": true, + "h5": true, + "h6": true, + "head": true, + "header": true, + "hgroup": true, + "hr": true, + "html": true, + "iframe": true, + "img": true, + "input": true, + "isindex": true, + "li": true, + "link": true, + "listing": true, + "marquee": true, + "menu": true, + "meta": true, + "nav": true, + "noembed": true, + "noframes": true, + "noscript": true, + "object": true, + "ol": true, + "p": true, + "param": true, + "plaintext": true, + "pre": true, + "script": true, + "section": true, + "select": true, + "style": true, + "summary": true, + "table": true, + "tbody": true, + "td": true, + "textarea": true, + "tfoot": true, + "th": true, + "thead": true, + "title": true, + "tr": true, + "ul": true, + "wbr": true, + "xmp": true, +} + +func isSpecialElement(element *Node) bool { + switch element.Namespace { + case "", "html": + return isSpecialElementMap[element.Data] + case "svg": + return element.Data == "foreignObject" + } + return false +} diff --git a/third_party/code.google.com/p/go.net/html/doc.go b/third_party/code.google.com/p/go.net/html/doc.go new file mode 100644 index 000000000..fac0f54e7 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/doc.go @@ -0,0 +1,106 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package html implements an HTML5-compliant tokenizer and parser. + +Tokenization is done by creating a Tokenizer for an io.Reader r. It is the +caller's responsibility to ensure that r provides UTF-8 encoded HTML. + + z := html.NewTokenizer(r) + +Given a Tokenizer z, the HTML is tokenized by repeatedly calling z.Next(), +which parses the next token and returns its type, or an error: + + for { + tt := z.Next() + if tt == html.ErrorToken { + // ... + return ... + } + // Process the current token. + } + +There are two APIs for retrieving the current token. The high-level API is to +call Token; the low-level API is to call Text or TagName / TagAttr. Both APIs +allow optionally calling Raw after Next but before Token, Text, TagName, or +TagAttr. In EBNF notation, the valid call sequence per token is: + + Next {Raw} [ Token | Text | TagName {TagAttr} ] + +Token returns an independent data structure that completely describes a token. +Entities (such as "<") are unescaped, tag names and attribute keys are +lower-cased, and attributes are collected into a []Attribute. For example: + + for { + if z.Next() == html.ErrorToken { + // Returning io.EOF indicates success. + return z.Err() + } + emitToken(z.Token()) + } + +The low-level API performs fewer allocations and copies, but the contents of +the []byte values returned by Text, TagName and TagAttr may change on the next +call to Next. For example, to extract an HTML page's anchor text: + + depth := 0 + for { + tt := z.Next() + switch tt { + case ErrorToken: + return z.Err() + case TextToken: + if depth > 0 { + // emitBytes should copy the []byte it receives, + // if it doesn't process it immediately. + emitBytes(z.Text()) + } + case StartTagToken, EndTagToken: + tn, _ := z.TagName() + if len(tn) == 1 && tn[0] == 'a' { + if tt == StartTagToken { + depth++ + } else { + depth-- + } + } + } + } + +Parsing is done by calling Parse with an io.Reader, which returns the root of +the parse tree (the document element) as a *Node. It is the caller's +responsibility to ensure that the Reader provides UTF-8 encoded HTML. For +example, to process each anchor node in depth-first order: + + doc, err := html.Parse(r) + if err != nil { + // ... + } + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + // Do something with n... + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(doc) + +The relevant specifications include: +http://www.whatwg.org/specs/web-apps/current-work/multipage/syntax.html and +http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html +*/ +package html + +// The tokenization algorithm implemented by this package is not a line-by-line +// transliteration of the relatively verbose state-machine in the WHATWG +// specification. A more direct approach is used instead, where the program +// counter implies the state, such as whether it is tokenizing a tag or a text +// node. Specification compliance is verified by checking expected and actual +// outputs over a test suite rather than aiming for algorithmic fidelity. + +// TODO(nigeltao): Does a DOM API belong in this package or a separate one? +// TODO(nigeltao): How does parsing interact with a JavaScript engine? diff --git a/third_party/code.google.com/p/go.net/html/doctype.go b/third_party/code.google.com/p/go.net/html/doctype.go new file mode 100644 index 000000000..c484e5a94 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/doctype.go @@ -0,0 +1,156 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "strings" +) + +// parseDoctype parses the data from a DoctypeToken into a name, +// public identifier, and system identifier. It returns a Node whose Type +// is DoctypeNode, whose Data is the name, and which has attributes +// named "system" and "public" for the two identifiers if they were present. +// quirks is whether the document should be parsed in "quirks mode". +func parseDoctype(s string) (n *Node, quirks bool) { + n = &Node{Type: DoctypeNode} + + // Find the name. + space := strings.IndexAny(s, whitespace) + if space == -1 { + space = len(s) + } + n.Data = s[:space] + // The comparison to "html" is case-sensitive. + if n.Data != "html" { + quirks = true + } + n.Data = strings.ToLower(n.Data) + s = strings.TrimLeft(s[space:], whitespace) + + if len(s) < 6 { + // It can't start with "PUBLIC" or "SYSTEM". + // Ignore the rest of the string. + return n, quirks || s != "" + } + + key := strings.ToLower(s[:6]) + s = s[6:] + for key == "public" || key == "system" { + s = strings.TrimLeft(s, whitespace) + if s == "" { + break + } + quote := s[0] + if quote != '"' && quote != '\'' { + break + } + s = s[1:] + q := strings.IndexRune(s, rune(quote)) + var id string + if q == -1 { + id = s + s = "" + } else { + id = s[:q] + s = s[q+1:] + } + n.Attr = append(n.Attr, Attribute{Key: key, Val: id}) + if key == "public" { + key = "system" + } else { + key = "" + } + } + + if key != "" || s != "" { + quirks = true + } else if len(n.Attr) > 0 { + if n.Attr[0].Key == "public" { + public := strings.ToLower(n.Attr[0].Val) + switch public { + case "-//w3o//dtd w3 html strict 3.0//en//", "-/w3d/dtd html 4.0 transitional/en", "html": + quirks = true + default: + for _, q := range quirkyIDs { + if strings.HasPrefix(public, q) { + quirks = true + break + } + } + } + // The following two public IDs only cause quirks mode if there is no system ID. + if len(n.Attr) == 1 && (strings.HasPrefix(public, "-//w3c//dtd html 4.01 frameset//") || + strings.HasPrefix(public, "-//w3c//dtd html 4.01 transitional//")) { + quirks = true + } + } + if lastAttr := n.Attr[len(n.Attr)-1]; lastAttr.Key == "system" && + strings.ToLower(lastAttr.Val) == "http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd" { + quirks = true + } + } + + return n, quirks +} + +// quirkyIDs is a list of public doctype identifiers that cause a document +// to be interpreted in quirks mode. The identifiers should be in lower case. +var quirkyIDs = []string{ + "+//silmaril//dtd html pro v0r11 19970101//", + "-//advasoft ltd//dtd html 3.0 aswedit + extensions//", + "-//as//dtd html 3.0 aswedit + extensions//", + "-//ietf//dtd html 2.0 level 1//", + "-//ietf//dtd html 2.0 level 2//", + "-//ietf//dtd html 2.0 strict level 1//", + "-//ietf//dtd html 2.0 strict level 2//", + "-//ietf//dtd html 2.0 strict//", + "-//ietf//dtd html 2.0//", + "-//ietf//dtd html 2.1e//", + "-//ietf//dtd html 3.0//", + "-//ietf//dtd html 3.2 final//", + "-//ietf//dtd html 3.2//", + "-//ietf//dtd html 3//", + "-//ietf//dtd html level 0//", + "-//ietf//dtd html level 1//", + "-//ietf//dtd html level 2//", + "-//ietf//dtd html level 3//", + "-//ietf//dtd html strict level 0//", + "-//ietf//dtd html strict level 1//", + "-//ietf//dtd html strict level 2//", + "-//ietf//dtd html strict level 3//", + "-//ietf//dtd html strict//", + "-//ietf//dtd html//", + "-//metrius//dtd metrius presentational//", + "-//microsoft//dtd internet explorer 2.0 html strict//", + "-//microsoft//dtd internet explorer 2.0 html//", + "-//microsoft//dtd internet explorer 2.0 tables//", + "-//microsoft//dtd internet explorer 3.0 html strict//", + "-//microsoft//dtd internet explorer 3.0 html//", + "-//microsoft//dtd internet explorer 3.0 tables//", + "-//netscape comm. corp.//dtd html//", + "-//netscape comm. corp.//dtd strict html//", + "-//o'reilly and associates//dtd html 2.0//", + "-//o'reilly and associates//dtd html extended 1.0//", + "-//o'reilly and associates//dtd html extended relaxed 1.0//", + "-//softquad software//dtd hotmetal pro 6.0::19990601::extensions to html 4.0//", + "-//softquad//dtd hotmetal pro 4.0::19971010::extensions to html 4.0//", + "-//spyglass//dtd html 2.0 extended//", + "-//sq//dtd html 2.0 hotmetal + extensions//", + "-//sun microsystems corp.//dtd hotjava html//", + "-//sun microsystems corp.//dtd hotjava strict html//", + "-//w3c//dtd html 3 1995-03-24//", + "-//w3c//dtd html 3.2 draft//", + "-//w3c//dtd html 3.2 final//", + "-//w3c//dtd html 3.2//", + "-//w3c//dtd html 3.2s draft//", + "-//w3c//dtd html 4.0 frameset//", + "-//w3c//dtd html 4.0 transitional//", + "-//w3c//dtd html experimental 19960712//", + "-//w3c//dtd html experimental 970421//", + "-//w3c//dtd w3 html//", + "-//w3o//dtd w3 html 3.0//", + "-//webtechs//dtd mozilla html 2.0//", + "-//webtechs//dtd mozilla html//", +} diff --git a/third_party/code.google.com/p/go.net/html/entity.go b/third_party/code.google.com/p/go.net/html/entity.go new file mode 100644 index 000000000..af8a007ed --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/entity.go @@ -0,0 +1,2253 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +// All entities that do not end with ';' are 6 or fewer bytes long. +const longestEntityWithoutSemicolon = 6 + +// entity is a map from HTML entity names to their values. The semicolon matters: +// http://www.whatwg.org/specs/web-apps/current-work/multipage/named-character-references.html +// lists both "amp" and "amp;" as two separate entries. +// +// Note that the HTML5 list is larger than the HTML4 list at +// http://www.w3.org/TR/html4/sgml/entities.html +var entity = map[string]rune{ + "AElig;": '\U000000C6', + "AMP;": '\U00000026', + "Aacute;": '\U000000C1', + "Abreve;": '\U00000102', + "Acirc;": '\U000000C2', + "Acy;": '\U00000410', + "Afr;": '\U0001D504', + "Agrave;": '\U000000C0', + "Alpha;": '\U00000391', + "Amacr;": '\U00000100', + "And;": '\U00002A53', + "Aogon;": '\U00000104', + "Aopf;": '\U0001D538', + "ApplyFunction;": '\U00002061', + "Aring;": '\U000000C5', + "Ascr;": '\U0001D49C', + "Assign;": '\U00002254', + "Atilde;": '\U000000C3', + "Auml;": '\U000000C4', + "Backslash;": '\U00002216', + "Barv;": '\U00002AE7', + "Barwed;": '\U00002306', + "Bcy;": '\U00000411', + "Because;": '\U00002235', + "Bernoullis;": '\U0000212C', + "Beta;": '\U00000392', + "Bfr;": '\U0001D505', + "Bopf;": '\U0001D539', + "Breve;": '\U000002D8', + "Bscr;": '\U0000212C', + "Bumpeq;": '\U0000224E', + "CHcy;": '\U00000427', + "COPY;": '\U000000A9', + "Cacute;": '\U00000106', + "Cap;": '\U000022D2', + "CapitalDifferentialD;": '\U00002145', + "Cayleys;": '\U0000212D', + "Ccaron;": '\U0000010C', + "Ccedil;": '\U000000C7', + "Ccirc;": '\U00000108', + "Cconint;": '\U00002230', + "Cdot;": '\U0000010A', + "Cedilla;": '\U000000B8', + "CenterDot;": '\U000000B7', + "Cfr;": '\U0000212D', + "Chi;": '\U000003A7', + "CircleDot;": '\U00002299', + "CircleMinus;": '\U00002296', + "CirclePlus;": '\U00002295', + "CircleTimes;": '\U00002297', + "ClockwiseContourIntegral;": '\U00002232', + "CloseCurlyDoubleQuote;": '\U0000201D', + "CloseCurlyQuote;": '\U00002019', + "Colon;": '\U00002237', + "Colone;": '\U00002A74', + "Congruent;": '\U00002261', + "Conint;": '\U0000222F', + "ContourIntegral;": '\U0000222E', + "Copf;": '\U00002102', + "Coproduct;": '\U00002210', + "CounterClockwiseContourIntegral;": '\U00002233', + "Cross;": '\U00002A2F', + "Cscr;": '\U0001D49E', + "Cup;": '\U000022D3', + "CupCap;": '\U0000224D', + "DD;": '\U00002145', + "DDotrahd;": '\U00002911', + "DJcy;": '\U00000402', + "DScy;": '\U00000405', + "DZcy;": '\U0000040F', + "Dagger;": '\U00002021', + "Darr;": '\U000021A1', + "Dashv;": '\U00002AE4', + "Dcaron;": '\U0000010E', + "Dcy;": '\U00000414', + "Del;": '\U00002207', + "Delta;": '\U00000394', + "Dfr;": '\U0001D507', + "DiacriticalAcute;": '\U000000B4', + "DiacriticalDot;": '\U000002D9', + "DiacriticalDoubleAcute;": '\U000002DD', + "DiacriticalGrave;": '\U00000060', + "DiacriticalTilde;": '\U000002DC', + "Diamond;": '\U000022C4', + "DifferentialD;": '\U00002146', + "Dopf;": '\U0001D53B', + "Dot;": '\U000000A8', + "DotDot;": '\U000020DC', + "DotEqual;": '\U00002250', + "DoubleContourIntegral;": '\U0000222F', + "DoubleDot;": '\U000000A8', + "DoubleDownArrow;": '\U000021D3', + "DoubleLeftArrow;": '\U000021D0', + "DoubleLeftRightArrow;": '\U000021D4', + "DoubleLeftTee;": '\U00002AE4', + "DoubleLongLeftArrow;": '\U000027F8', + "DoubleLongLeftRightArrow;": '\U000027FA', + "DoubleLongRightArrow;": '\U000027F9', + "DoubleRightArrow;": '\U000021D2', + "DoubleRightTee;": '\U000022A8', + "DoubleUpArrow;": '\U000021D1', + "DoubleUpDownArrow;": '\U000021D5', + "DoubleVerticalBar;": '\U00002225', + "DownArrow;": '\U00002193', + "DownArrowBar;": '\U00002913', + "DownArrowUpArrow;": '\U000021F5', + "DownBreve;": '\U00000311', + "DownLeftRightVector;": '\U00002950', + "DownLeftTeeVector;": '\U0000295E', + "DownLeftVector;": '\U000021BD', + "DownLeftVectorBar;": '\U00002956', + "DownRightTeeVector;": '\U0000295F', + "DownRightVector;": '\U000021C1', + "DownRightVectorBar;": '\U00002957', + "DownTee;": '\U000022A4', + "DownTeeArrow;": '\U000021A7', + "Downarrow;": '\U000021D3', + "Dscr;": '\U0001D49F', + "Dstrok;": '\U00000110', + "ENG;": '\U0000014A', + "ETH;": '\U000000D0', + "Eacute;": '\U000000C9', + "Ecaron;": '\U0000011A', + "Ecirc;": '\U000000CA', + "Ecy;": '\U0000042D', + "Edot;": '\U00000116', + "Efr;": '\U0001D508', + "Egrave;": '\U000000C8', + "Element;": '\U00002208', + "Emacr;": '\U00000112', + "EmptySmallSquare;": '\U000025FB', + "EmptyVerySmallSquare;": '\U000025AB', + "Eogon;": '\U00000118', + "Eopf;": '\U0001D53C', + "Epsilon;": '\U00000395', + "Equal;": '\U00002A75', + "EqualTilde;": '\U00002242', + "Equilibrium;": '\U000021CC', + "Escr;": '\U00002130', + "Esim;": '\U00002A73', + "Eta;": '\U00000397', + "Euml;": '\U000000CB', + "Exists;": '\U00002203', + "ExponentialE;": '\U00002147', + "Fcy;": '\U00000424', + "Ffr;": '\U0001D509', + "FilledSmallSquare;": '\U000025FC', + "FilledVerySmallSquare;": '\U000025AA', + "Fopf;": '\U0001D53D', + "ForAll;": '\U00002200', + "Fouriertrf;": '\U00002131', + "Fscr;": '\U00002131', + "GJcy;": '\U00000403', + "GT;": '\U0000003E', + "Gamma;": '\U00000393', + "Gammad;": '\U000003DC', + "Gbreve;": '\U0000011E', + "Gcedil;": '\U00000122', + "Gcirc;": '\U0000011C', + "Gcy;": '\U00000413', + "Gdot;": '\U00000120', + "Gfr;": '\U0001D50A', + "Gg;": '\U000022D9', + "Gopf;": '\U0001D53E', + "GreaterEqual;": '\U00002265', + "GreaterEqualLess;": '\U000022DB', + "GreaterFullEqual;": '\U00002267', + "GreaterGreater;": '\U00002AA2', + "GreaterLess;": '\U00002277', + "GreaterSlantEqual;": '\U00002A7E', + "GreaterTilde;": '\U00002273', + "Gscr;": '\U0001D4A2', + "Gt;": '\U0000226B', + "HARDcy;": '\U0000042A', + "Hacek;": '\U000002C7', + "Hat;": '\U0000005E', + "Hcirc;": '\U00000124', + "Hfr;": '\U0000210C', + "HilbertSpace;": '\U0000210B', + "Hopf;": '\U0000210D', + "HorizontalLine;": '\U00002500', + "Hscr;": '\U0000210B', + "Hstrok;": '\U00000126', + "HumpDownHump;": '\U0000224E', + "HumpEqual;": '\U0000224F', + "IEcy;": '\U00000415', + "IJlig;": '\U00000132', + "IOcy;": '\U00000401', + "Iacute;": '\U000000CD', + "Icirc;": '\U000000CE', + "Icy;": '\U00000418', + "Idot;": '\U00000130', + "Ifr;": '\U00002111', + "Igrave;": '\U000000CC', + "Im;": '\U00002111', + "Imacr;": '\U0000012A', + "ImaginaryI;": '\U00002148', + "Implies;": '\U000021D2', + "Int;": '\U0000222C', + "Integral;": '\U0000222B', + "Intersection;": '\U000022C2', + "InvisibleComma;": '\U00002063', + "InvisibleTimes;": '\U00002062', + "Iogon;": '\U0000012E', + "Iopf;": '\U0001D540', + "Iota;": '\U00000399', + "Iscr;": '\U00002110', + "Itilde;": '\U00000128', + "Iukcy;": '\U00000406', + "Iuml;": '\U000000CF', + "Jcirc;": '\U00000134', + "Jcy;": '\U00000419', + "Jfr;": '\U0001D50D', + "Jopf;": '\U0001D541', + "Jscr;": '\U0001D4A5', + "Jsercy;": '\U00000408', + "Jukcy;": '\U00000404', + "KHcy;": '\U00000425', + "KJcy;": '\U0000040C', + "Kappa;": '\U0000039A', + "Kcedil;": '\U00000136', + "Kcy;": '\U0000041A', + "Kfr;": '\U0001D50E', + "Kopf;": '\U0001D542', + "Kscr;": '\U0001D4A6', + "LJcy;": '\U00000409', + "LT;": '\U0000003C', + "Lacute;": '\U00000139', + "Lambda;": '\U0000039B', + "Lang;": '\U000027EA', + "Laplacetrf;": '\U00002112', + "Larr;": '\U0000219E', + "Lcaron;": '\U0000013D', + "Lcedil;": '\U0000013B', + "Lcy;": '\U0000041B', + "LeftAngleBracket;": '\U000027E8', + "LeftArrow;": '\U00002190', + "LeftArrowBar;": '\U000021E4', + "LeftArrowRightArrow;": '\U000021C6', + "LeftCeiling;": '\U00002308', + "LeftDoubleBracket;": '\U000027E6', + "LeftDownTeeVector;": '\U00002961', + "LeftDownVector;": '\U000021C3', + "LeftDownVectorBar;": '\U00002959', + "LeftFloor;": '\U0000230A', + "LeftRightArrow;": '\U00002194', + "LeftRightVector;": '\U0000294E', + "LeftTee;": '\U000022A3', + "LeftTeeArrow;": '\U000021A4', + "LeftTeeVector;": '\U0000295A', + "LeftTriangle;": '\U000022B2', + "LeftTriangleBar;": '\U000029CF', + "LeftTriangleEqual;": '\U000022B4', + "LeftUpDownVector;": '\U00002951', + "LeftUpTeeVector;": '\U00002960', + "LeftUpVector;": '\U000021BF', + "LeftUpVectorBar;": '\U00002958', + "LeftVector;": '\U000021BC', + "LeftVectorBar;": '\U00002952', + "Leftarrow;": '\U000021D0', + "Leftrightarrow;": '\U000021D4', + "LessEqualGreater;": '\U000022DA', + "LessFullEqual;": '\U00002266', + "LessGreater;": '\U00002276', + "LessLess;": '\U00002AA1', + "LessSlantEqual;": '\U00002A7D', + "LessTilde;": '\U00002272', + "Lfr;": '\U0001D50F', + "Ll;": '\U000022D8', + "Lleftarrow;": '\U000021DA', + "Lmidot;": '\U0000013F', + "LongLeftArrow;": '\U000027F5', + "LongLeftRightArrow;": '\U000027F7', + "LongRightArrow;": '\U000027F6', + "Longleftarrow;": '\U000027F8', + "Longleftrightarrow;": '\U000027FA', + "Longrightarrow;": '\U000027F9', + "Lopf;": '\U0001D543', + "LowerLeftArrow;": '\U00002199', + "LowerRightArrow;": '\U00002198', + "Lscr;": '\U00002112', + "Lsh;": '\U000021B0', + "Lstrok;": '\U00000141', + "Lt;": '\U0000226A', + "Map;": '\U00002905', + "Mcy;": '\U0000041C', + "MediumSpace;": '\U0000205F', + "Mellintrf;": '\U00002133', + "Mfr;": '\U0001D510', + "MinusPlus;": '\U00002213', + "Mopf;": '\U0001D544', + "Mscr;": '\U00002133', + "Mu;": '\U0000039C', + "NJcy;": '\U0000040A', + "Nacute;": '\U00000143', + "Ncaron;": '\U00000147', + "Ncedil;": '\U00000145', + "Ncy;": '\U0000041D', + "NegativeMediumSpace;": '\U0000200B', + "NegativeThickSpace;": '\U0000200B', + "NegativeThinSpace;": '\U0000200B', + "NegativeVeryThinSpace;": '\U0000200B', + "NestedGreaterGreater;": '\U0000226B', + "NestedLessLess;": '\U0000226A', + "NewLine;": '\U0000000A', + "Nfr;": '\U0001D511', + "NoBreak;": '\U00002060', + "NonBreakingSpace;": '\U000000A0', + "Nopf;": '\U00002115', + "Not;": '\U00002AEC', + "NotCongruent;": '\U00002262', + "NotCupCap;": '\U0000226D', + "NotDoubleVerticalBar;": '\U00002226', + "NotElement;": '\U00002209', + "NotEqual;": '\U00002260', + "NotExists;": '\U00002204', + "NotGreater;": '\U0000226F', + "NotGreaterEqual;": '\U00002271', + "NotGreaterLess;": '\U00002279', + "NotGreaterTilde;": '\U00002275', + "NotLeftTriangle;": '\U000022EA', + "NotLeftTriangleEqual;": '\U000022EC', + "NotLess;": '\U0000226E', + "NotLessEqual;": '\U00002270', + "NotLessGreater;": '\U00002278', + "NotLessTilde;": '\U00002274', + "NotPrecedes;": '\U00002280', + "NotPrecedesSlantEqual;": '\U000022E0', + "NotReverseElement;": '\U0000220C', + "NotRightTriangle;": '\U000022EB', + "NotRightTriangleEqual;": '\U000022ED', + "NotSquareSubsetEqual;": '\U000022E2', + "NotSquareSupersetEqual;": '\U000022E3', + "NotSubsetEqual;": '\U00002288', + "NotSucceeds;": '\U00002281', + "NotSucceedsSlantEqual;": '\U000022E1', + "NotSupersetEqual;": '\U00002289', + "NotTilde;": '\U00002241', + "NotTildeEqual;": '\U00002244', + "NotTildeFullEqual;": '\U00002247', + "NotTildeTilde;": '\U00002249', + "NotVerticalBar;": '\U00002224', + "Nscr;": '\U0001D4A9', + "Ntilde;": '\U000000D1', + "Nu;": '\U0000039D', + "OElig;": '\U00000152', + "Oacute;": '\U000000D3', + "Ocirc;": '\U000000D4', + "Ocy;": '\U0000041E', + "Odblac;": '\U00000150', + "Ofr;": '\U0001D512', + "Ograve;": '\U000000D2', + "Omacr;": '\U0000014C', + "Omega;": '\U000003A9', + "Omicron;": '\U0000039F', + "Oopf;": '\U0001D546', + "OpenCurlyDoubleQuote;": '\U0000201C', + "OpenCurlyQuote;": '\U00002018', + "Or;": '\U00002A54', + "Oscr;": '\U0001D4AA', + "Oslash;": '\U000000D8', + "Otilde;": '\U000000D5', + "Otimes;": '\U00002A37', + "Ouml;": '\U000000D6', + "OverBar;": '\U0000203E', + "OverBrace;": '\U000023DE', + "OverBracket;": '\U000023B4', + "OverParenthesis;": '\U000023DC', + "PartialD;": '\U00002202', + "Pcy;": '\U0000041F', + "Pfr;": '\U0001D513', + "Phi;": '\U000003A6', + "Pi;": '\U000003A0', + "PlusMinus;": '\U000000B1', + "Poincareplane;": '\U0000210C', + "Popf;": '\U00002119', + "Pr;": '\U00002ABB', + "Precedes;": '\U0000227A', + "PrecedesEqual;": '\U00002AAF', + "PrecedesSlantEqual;": '\U0000227C', + "PrecedesTilde;": '\U0000227E', + "Prime;": '\U00002033', + "Product;": '\U0000220F', + "Proportion;": '\U00002237', + "Proportional;": '\U0000221D', + "Pscr;": '\U0001D4AB', + "Psi;": '\U000003A8', + "QUOT;": '\U00000022', + "Qfr;": '\U0001D514', + "Qopf;": '\U0000211A', + "Qscr;": '\U0001D4AC', + "RBarr;": '\U00002910', + "REG;": '\U000000AE', + "Racute;": '\U00000154', + "Rang;": '\U000027EB', + "Rarr;": '\U000021A0', + "Rarrtl;": '\U00002916', + "Rcaron;": '\U00000158', + "Rcedil;": '\U00000156', + "Rcy;": '\U00000420', + "Re;": '\U0000211C', + "ReverseElement;": '\U0000220B', + "ReverseEquilibrium;": '\U000021CB', + "ReverseUpEquilibrium;": '\U0000296F', + "Rfr;": '\U0000211C', + "Rho;": '\U000003A1', + "RightAngleBracket;": '\U000027E9', + "RightArrow;": '\U00002192', + "RightArrowBar;": '\U000021E5', + "RightArrowLeftArrow;": '\U000021C4', + "RightCeiling;": '\U00002309', + "RightDoubleBracket;": '\U000027E7', + "RightDownTeeVector;": '\U0000295D', + "RightDownVector;": '\U000021C2', + "RightDownVectorBar;": '\U00002955', + "RightFloor;": '\U0000230B', + "RightTee;": '\U000022A2', + "RightTeeArrow;": '\U000021A6', + "RightTeeVector;": '\U0000295B', + "RightTriangle;": '\U000022B3', + "RightTriangleBar;": '\U000029D0', + "RightTriangleEqual;": '\U000022B5', + "RightUpDownVector;": '\U0000294F', + "RightUpTeeVector;": '\U0000295C', + "RightUpVector;": '\U000021BE', + "RightUpVectorBar;": '\U00002954', + "RightVector;": '\U000021C0', + "RightVectorBar;": '\U00002953', + "Rightarrow;": '\U000021D2', + "Ropf;": '\U0000211D', + "RoundImplies;": '\U00002970', + "Rrightarrow;": '\U000021DB', + "Rscr;": '\U0000211B', + "Rsh;": '\U000021B1', + "RuleDelayed;": '\U000029F4', + "SHCHcy;": '\U00000429', + "SHcy;": '\U00000428', + "SOFTcy;": '\U0000042C', + "Sacute;": '\U0000015A', + "Sc;": '\U00002ABC', + "Scaron;": '\U00000160', + "Scedil;": '\U0000015E', + "Scirc;": '\U0000015C', + "Scy;": '\U00000421', + "Sfr;": '\U0001D516', + "ShortDownArrow;": '\U00002193', + "ShortLeftArrow;": '\U00002190', + "ShortRightArrow;": '\U00002192', + "ShortUpArrow;": '\U00002191', + "Sigma;": '\U000003A3', + "SmallCircle;": '\U00002218', + "Sopf;": '\U0001D54A', + "Sqrt;": '\U0000221A', + "Square;": '\U000025A1', + "SquareIntersection;": '\U00002293', + "SquareSubset;": '\U0000228F', + "SquareSubsetEqual;": '\U00002291', + "SquareSuperset;": '\U00002290', + "SquareSupersetEqual;": '\U00002292', + "SquareUnion;": '\U00002294', + "Sscr;": '\U0001D4AE', + "Star;": '\U000022C6', + "Sub;": '\U000022D0', + "Subset;": '\U000022D0', + "SubsetEqual;": '\U00002286', + "Succeeds;": '\U0000227B', + "SucceedsEqual;": '\U00002AB0', + "SucceedsSlantEqual;": '\U0000227D', + "SucceedsTilde;": '\U0000227F', + "SuchThat;": '\U0000220B', + "Sum;": '\U00002211', + "Sup;": '\U000022D1', + "Superset;": '\U00002283', + "SupersetEqual;": '\U00002287', + "Supset;": '\U000022D1', + "THORN;": '\U000000DE', + "TRADE;": '\U00002122', + "TSHcy;": '\U0000040B', + "TScy;": '\U00000426', + "Tab;": '\U00000009', + "Tau;": '\U000003A4', + "Tcaron;": '\U00000164', + "Tcedil;": '\U00000162', + "Tcy;": '\U00000422', + "Tfr;": '\U0001D517', + "Therefore;": '\U00002234', + "Theta;": '\U00000398', + "ThinSpace;": '\U00002009', + "Tilde;": '\U0000223C', + "TildeEqual;": '\U00002243', + "TildeFullEqual;": '\U00002245', + "TildeTilde;": '\U00002248', + "Topf;": '\U0001D54B', + "TripleDot;": '\U000020DB', + "Tscr;": '\U0001D4AF', + "Tstrok;": '\U00000166', + "Uacute;": '\U000000DA', + "Uarr;": '\U0000219F', + "Uarrocir;": '\U00002949', + "Ubrcy;": '\U0000040E', + "Ubreve;": '\U0000016C', + "Ucirc;": '\U000000DB', + "Ucy;": '\U00000423', + "Udblac;": '\U00000170', + "Ufr;": '\U0001D518', + "Ugrave;": '\U000000D9', + "Umacr;": '\U0000016A', + "UnderBar;": '\U0000005F', + "UnderBrace;": '\U000023DF', + "UnderBracket;": '\U000023B5', + "UnderParenthesis;": '\U000023DD', + "Union;": '\U000022C3', + "UnionPlus;": '\U0000228E', + "Uogon;": '\U00000172', + "Uopf;": '\U0001D54C', + "UpArrow;": '\U00002191', + "UpArrowBar;": '\U00002912', + "UpArrowDownArrow;": '\U000021C5', + "UpDownArrow;": '\U00002195', + "UpEquilibrium;": '\U0000296E', + "UpTee;": '\U000022A5', + "UpTeeArrow;": '\U000021A5', + "Uparrow;": '\U000021D1', + "Updownarrow;": '\U000021D5', + "UpperLeftArrow;": '\U00002196', + "UpperRightArrow;": '\U00002197', + "Upsi;": '\U000003D2', + "Upsilon;": '\U000003A5', + "Uring;": '\U0000016E', + "Uscr;": '\U0001D4B0', + "Utilde;": '\U00000168', + "Uuml;": '\U000000DC', + "VDash;": '\U000022AB', + "Vbar;": '\U00002AEB', + "Vcy;": '\U00000412', + "Vdash;": '\U000022A9', + "Vdashl;": '\U00002AE6', + "Vee;": '\U000022C1', + "Verbar;": '\U00002016', + "Vert;": '\U00002016', + "VerticalBar;": '\U00002223', + "VerticalLine;": '\U0000007C', + "VerticalSeparator;": '\U00002758', + "VerticalTilde;": '\U00002240', + "VeryThinSpace;": '\U0000200A', + "Vfr;": '\U0001D519', + "Vopf;": '\U0001D54D', + "Vscr;": '\U0001D4B1', + "Vvdash;": '\U000022AA', + "Wcirc;": '\U00000174', + "Wedge;": '\U000022C0', + "Wfr;": '\U0001D51A', + "Wopf;": '\U0001D54E', + "Wscr;": '\U0001D4B2', + "Xfr;": '\U0001D51B', + "Xi;": '\U0000039E', + "Xopf;": '\U0001D54F', + "Xscr;": '\U0001D4B3', + "YAcy;": '\U0000042F', + "YIcy;": '\U00000407', + "YUcy;": '\U0000042E', + "Yacute;": '\U000000DD', + "Ycirc;": '\U00000176', + "Ycy;": '\U0000042B', + "Yfr;": '\U0001D51C', + "Yopf;": '\U0001D550', + "Yscr;": '\U0001D4B4', + "Yuml;": '\U00000178', + "ZHcy;": '\U00000416', + "Zacute;": '\U00000179', + "Zcaron;": '\U0000017D', + "Zcy;": '\U00000417', + "Zdot;": '\U0000017B', + "ZeroWidthSpace;": '\U0000200B', + "Zeta;": '\U00000396', + "Zfr;": '\U00002128', + "Zopf;": '\U00002124', + "Zscr;": '\U0001D4B5', + "aacute;": '\U000000E1', + "abreve;": '\U00000103', + "ac;": '\U0000223E', + "acd;": '\U0000223F', + "acirc;": '\U000000E2', + "acute;": '\U000000B4', + "acy;": '\U00000430', + "aelig;": '\U000000E6', + "af;": '\U00002061', + "afr;": '\U0001D51E', + "agrave;": '\U000000E0', + "alefsym;": '\U00002135', + "aleph;": '\U00002135', + "alpha;": '\U000003B1', + "amacr;": '\U00000101', + "amalg;": '\U00002A3F', + "amp;": '\U00000026', + "and;": '\U00002227', + "andand;": '\U00002A55', + "andd;": '\U00002A5C', + "andslope;": '\U00002A58', + "andv;": '\U00002A5A', + "ang;": '\U00002220', + "ange;": '\U000029A4', + "angle;": '\U00002220', + "angmsd;": '\U00002221', + "angmsdaa;": '\U000029A8', + "angmsdab;": '\U000029A9', + "angmsdac;": '\U000029AA', + "angmsdad;": '\U000029AB', + "angmsdae;": '\U000029AC', + "angmsdaf;": '\U000029AD', + "angmsdag;": '\U000029AE', + "angmsdah;": '\U000029AF', + "angrt;": '\U0000221F', + "angrtvb;": '\U000022BE', + "angrtvbd;": '\U0000299D', + "angsph;": '\U00002222', + "angst;": '\U000000C5', + "angzarr;": '\U0000237C', + "aogon;": '\U00000105', + "aopf;": '\U0001D552', + "ap;": '\U00002248', + "apE;": '\U00002A70', + "apacir;": '\U00002A6F', + "ape;": '\U0000224A', + "apid;": '\U0000224B', + "apos;": '\U00000027', + "approx;": '\U00002248', + "approxeq;": '\U0000224A', + "aring;": '\U000000E5', + "ascr;": '\U0001D4B6', + "ast;": '\U0000002A', + "asymp;": '\U00002248', + "asympeq;": '\U0000224D', + "atilde;": '\U000000E3', + "auml;": '\U000000E4', + "awconint;": '\U00002233', + "awint;": '\U00002A11', + "bNot;": '\U00002AED', + "backcong;": '\U0000224C', + "backepsilon;": '\U000003F6', + "backprime;": '\U00002035', + "backsim;": '\U0000223D', + "backsimeq;": '\U000022CD', + "barvee;": '\U000022BD', + "barwed;": '\U00002305', + "barwedge;": '\U00002305', + "bbrk;": '\U000023B5', + "bbrktbrk;": '\U000023B6', + "bcong;": '\U0000224C', + "bcy;": '\U00000431', + "bdquo;": '\U0000201E', + "becaus;": '\U00002235', + "because;": '\U00002235', + "bemptyv;": '\U000029B0', + "bepsi;": '\U000003F6', + "bernou;": '\U0000212C', + "beta;": '\U000003B2', + "beth;": '\U00002136', + "between;": '\U0000226C', + "bfr;": '\U0001D51F', + "bigcap;": '\U000022C2', + "bigcirc;": '\U000025EF', + "bigcup;": '\U000022C3', + "bigodot;": '\U00002A00', + "bigoplus;": '\U00002A01', + "bigotimes;": '\U00002A02', + "bigsqcup;": '\U00002A06', + "bigstar;": '\U00002605', + "bigtriangledown;": '\U000025BD', + "bigtriangleup;": '\U000025B3', + "biguplus;": '\U00002A04', + "bigvee;": '\U000022C1', + "bigwedge;": '\U000022C0', + "bkarow;": '\U0000290D', + "blacklozenge;": '\U000029EB', + "blacksquare;": '\U000025AA', + "blacktriangle;": '\U000025B4', + "blacktriangledown;": '\U000025BE', + "blacktriangleleft;": '\U000025C2', + "blacktriangleright;": '\U000025B8', + "blank;": '\U00002423', + "blk12;": '\U00002592', + "blk14;": '\U00002591', + "blk34;": '\U00002593', + "block;": '\U00002588', + "bnot;": '\U00002310', + "bopf;": '\U0001D553', + "bot;": '\U000022A5', + "bottom;": '\U000022A5', + "bowtie;": '\U000022C8', + "boxDL;": '\U00002557', + "boxDR;": '\U00002554', + "boxDl;": '\U00002556', + "boxDr;": '\U00002553', + "boxH;": '\U00002550', + "boxHD;": '\U00002566', + "boxHU;": '\U00002569', + "boxHd;": '\U00002564', + "boxHu;": '\U00002567', + "boxUL;": '\U0000255D', + "boxUR;": '\U0000255A', + "boxUl;": '\U0000255C', + "boxUr;": '\U00002559', + "boxV;": '\U00002551', + "boxVH;": '\U0000256C', + "boxVL;": '\U00002563', + "boxVR;": '\U00002560', + "boxVh;": '\U0000256B', + "boxVl;": '\U00002562', + "boxVr;": '\U0000255F', + "boxbox;": '\U000029C9', + "boxdL;": '\U00002555', + "boxdR;": '\U00002552', + "boxdl;": '\U00002510', + "boxdr;": '\U0000250C', + "boxh;": '\U00002500', + "boxhD;": '\U00002565', + "boxhU;": '\U00002568', + "boxhd;": '\U0000252C', + "boxhu;": '\U00002534', + "boxminus;": '\U0000229F', + "boxplus;": '\U0000229E', + "boxtimes;": '\U000022A0', + "boxuL;": '\U0000255B', + "boxuR;": '\U00002558', + "boxul;": '\U00002518', + "boxur;": '\U00002514', + "boxv;": '\U00002502', + "boxvH;": '\U0000256A', + "boxvL;": '\U00002561', + "boxvR;": '\U0000255E', + "boxvh;": '\U0000253C', + "boxvl;": '\U00002524', + "boxvr;": '\U0000251C', + "bprime;": '\U00002035', + "breve;": '\U000002D8', + "brvbar;": '\U000000A6', + "bscr;": '\U0001D4B7', + "bsemi;": '\U0000204F', + "bsim;": '\U0000223D', + "bsime;": '\U000022CD', + "bsol;": '\U0000005C', + "bsolb;": '\U000029C5', + "bsolhsub;": '\U000027C8', + "bull;": '\U00002022', + "bullet;": '\U00002022', + "bump;": '\U0000224E', + "bumpE;": '\U00002AAE', + "bumpe;": '\U0000224F', + "bumpeq;": '\U0000224F', + "cacute;": '\U00000107', + "cap;": '\U00002229', + "capand;": '\U00002A44', + "capbrcup;": '\U00002A49', + "capcap;": '\U00002A4B', + "capcup;": '\U00002A47', + "capdot;": '\U00002A40', + "caret;": '\U00002041', + "caron;": '\U000002C7', + "ccaps;": '\U00002A4D', + "ccaron;": '\U0000010D', + "ccedil;": '\U000000E7', + "ccirc;": '\U00000109', + "ccups;": '\U00002A4C', + "ccupssm;": '\U00002A50', + "cdot;": '\U0000010B', + "cedil;": '\U000000B8', + "cemptyv;": '\U000029B2', + "cent;": '\U000000A2', + "centerdot;": '\U000000B7', + "cfr;": '\U0001D520', + "chcy;": '\U00000447', + "check;": '\U00002713', + "checkmark;": '\U00002713', + "chi;": '\U000003C7', + "cir;": '\U000025CB', + "cirE;": '\U000029C3', + "circ;": '\U000002C6', + "circeq;": '\U00002257', + "circlearrowleft;": '\U000021BA', + "circlearrowright;": '\U000021BB', + "circledR;": '\U000000AE', + "circledS;": '\U000024C8', + "circledast;": '\U0000229B', + "circledcirc;": '\U0000229A', + "circleddash;": '\U0000229D', + "cire;": '\U00002257', + "cirfnint;": '\U00002A10', + "cirmid;": '\U00002AEF', + "cirscir;": '\U000029C2', + "clubs;": '\U00002663', + "clubsuit;": '\U00002663', + "colon;": '\U0000003A', + "colone;": '\U00002254', + "coloneq;": '\U00002254', + "comma;": '\U0000002C', + "commat;": '\U00000040', + "comp;": '\U00002201', + "compfn;": '\U00002218', + "complement;": '\U00002201', + "complexes;": '\U00002102', + "cong;": '\U00002245', + "congdot;": '\U00002A6D', + "conint;": '\U0000222E', + "copf;": '\U0001D554', + "coprod;": '\U00002210', + "copy;": '\U000000A9', + "copysr;": '\U00002117', + "crarr;": '\U000021B5', + "cross;": '\U00002717', + "cscr;": '\U0001D4B8', + "csub;": '\U00002ACF', + "csube;": '\U00002AD1', + "csup;": '\U00002AD0', + "csupe;": '\U00002AD2', + "ctdot;": '\U000022EF', + "cudarrl;": '\U00002938', + "cudarrr;": '\U00002935', + "cuepr;": '\U000022DE', + "cuesc;": '\U000022DF', + "cularr;": '\U000021B6', + "cularrp;": '\U0000293D', + "cup;": '\U0000222A', + "cupbrcap;": '\U00002A48', + "cupcap;": '\U00002A46', + "cupcup;": '\U00002A4A', + "cupdot;": '\U0000228D', + "cupor;": '\U00002A45', + "curarr;": '\U000021B7', + "curarrm;": '\U0000293C', + "curlyeqprec;": '\U000022DE', + "curlyeqsucc;": '\U000022DF', + "curlyvee;": '\U000022CE', + "curlywedge;": '\U000022CF', + "curren;": '\U000000A4', + "curvearrowleft;": '\U000021B6', + "curvearrowright;": '\U000021B7', + "cuvee;": '\U000022CE', + "cuwed;": '\U000022CF', + "cwconint;": '\U00002232', + "cwint;": '\U00002231', + "cylcty;": '\U0000232D', + "dArr;": '\U000021D3', + "dHar;": '\U00002965', + "dagger;": '\U00002020', + "daleth;": '\U00002138', + "darr;": '\U00002193', + "dash;": '\U00002010', + "dashv;": '\U000022A3', + "dbkarow;": '\U0000290F', + "dblac;": '\U000002DD', + "dcaron;": '\U0000010F', + "dcy;": '\U00000434', + "dd;": '\U00002146', + "ddagger;": '\U00002021', + "ddarr;": '\U000021CA', + "ddotseq;": '\U00002A77', + "deg;": '\U000000B0', + "delta;": '\U000003B4', + "demptyv;": '\U000029B1', + "dfisht;": '\U0000297F', + "dfr;": '\U0001D521', + "dharl;": '\U000021C3', + "dharr;": '\U000021C2', + "diam;": '\U000022C4', + "diamond;": '\U000022C4', + "diamondsuit;": '\U00002666', + "diams;": '\U00002666', + "die;": '\U000000A8', + "digamma;": '\U000003DD', + "disin;": '\U000022F2', + "div;": '\U000000F7', + "divide;": '\U000000F7', + "divideontimes;": '\U000022C7', + "divonx;": '\U000022C7', + "djcy;": '\U00000452', + "dlcorn;": '\U0000231E', + "dlcrop;": '\U0000230D', + "dollar;": '\U00000024', + "dopf;": '\U0001D555', + "dot;": '\U000002D9', + "doteq;": '\U00002250', + "doteqdot;": '\U00002251', + "dotminus;": '\U00002238', + "dotplus;": '\U00002214', + "dotsquare;": '\U000022A1', + "doublebarwedge;": '\U00002306', + "downarrow;": '\U00002193', + "downdownarrows;": '\U000021CA', + "downharpoonleft;": '\U000021C3', + "downharpoonright;": '\U000021C2', + "drbkarow;": '\U00002910', + "drcorn;": '\U0000231F', + "drcrop;": '\U0000230C', + "dscr;": '\U0001D4B9', + "dscy;": '\U00000455', + "dsol;": '\U000029F6', + "dstrok;": '\U00000111', + "dtdot;": '\U000022F1', + "dtri;": '\U000025BF', + "dtrif;": '\U000025BE', + "duarr;": '\U000021F5', + "duhar;": '\U0000296F', + "dwangle;": '\U000029A6', + "dzcy;": '\U0000045F', + "dzigrarr;": '\U000027FF', + "eDDot;": '\U00002A77', + "eDot;": '\U00002251', + "eacute;": '\U000000E9', + "easter;": '\U00002A6E', + "ecaron;": '\U0000011B', + "ecir;": '\U00002256', + "ecirc;": '\U000000EA', + "ecolon;": '\U00002255', + "ecy;": '\U0000044D', + "edot;": '\U00000117', + "ee;": '\U00002147', + "efDot;": '\U00002252', + "efr;": '\U0001D522', + "eg;": '\U00002A9A', + "egrave;": '\U000000E8', + "egs;": '\U00002A96', + "egsdot;": '\U00002A98', + "el;": '\U00002A99', + "elinters;": '\U000023E7', + "ell;": '\U00002113', + "els;": '\U00002A95', + "elsdot;": '\U00002A97', + "emacr;": '\U00000113', + "empty;": '\U00002205', + "emptyset;": '\U00002205', + "emptyv;": '\U00002205', + "emsp;": '\U00002003', + "emsp13;": '\U00002004', + "emsp14;": '\U00002005', + "eng;": '\U0000014B', + "ensp;": '\U00002002', + "eogon;": '\U00000119', + "eopf;": '\U0001D556', + "epar;": '\U000022D5', + "eparsl;": '\U000029E3', + "eplus;": '\U00002A71', + "epsi;": '\U000003B5', + "epsilon;": '\U000003B5', + "epsiv;": '\U000003F5', + "eqcirc;": '\U00002256', + "eqcolon;": '\U00002255', + "eqsim;": '\U00002242', + "eqslantgtr;": '\U00002A96', + "eqslantless;": '\U00002A95', + "equals;": '\U0000003D', + "equest;": '\U0000225F', + "equiv;": '\U00002261', + "equivDD;": '\U00002A78', + "eqvparsl;": '\U000029E5', + "erDot;": '\U00002253', + "erarr;": '\U00002971', + "escr;": '\U0000212F', + "esdot;": '\U00002250', + "esim;": '\U00002242', + "eta;": '\U000003B7', + "eth;": '\U000000F0', + "euml;": '\U000000EB', + "euro;": '\U000020AC', + "excl;": '\U00000021', + "exist;": '\U00002203', + "expectation;": '\U00002130', + "exponentiale;": '\U00002147', + "fallingdotseq;": '\U00002252', + "fcy;": '\U00000444', + "female;": '\U00002640', + "ffilig;": '\U0000FB03', + "fflig;": '\U0000FB00', + "ffllig;": '\U0000FB04', + "ffr;": '\U0001D523', + "filig;": '\U0000FB01', + "flat;": '\U0000266D', + "fllig;": '\U0000FB02', + "fltns;": '\U000025B1', + "fnof;": '\U00000192', + "fopf;": '\U0001D557', + "forall;": '\U00002200', + "fork;": '\U000022D4', + "forkv;": '\U00002AD9', + "fpartint;": '\U00002A0D', + "frac12;": '\U000000BD', + "frac13;": '\U00002153', + "frac14;": '\U000000BC', + "frac15;": '\U00002155', + "frac16;": '\U00002159', + "frac18;": '\U0000215B', + "frac23;": '\U00002154', + "frac25;": '\U00002156', + "frac34;": '\U000000BE', + "frac35;": '\U00002157', + "frac38;": '\U0000215C', + "frac45;": '\U00002158', + "frac56;": '\U0000215A', + "frac58;": '\U0000215D', + "frac78;": '\U0000215E', + "frasl;": '\U00002044', + "frown;": '\U00002322', + "fscr;": '\U0001D4BB', + "gE;": '\U00002267', + "gEl;": '\U00002A8C', + "gacute;": '\U000001F5', + "gamma;": '\U000003B3', + "gammad;": '\U000003DD', + "gap;": '\U00002A86', + "gbreve;": '\U0000011F', + "gcirc;": '\U0000011D', + "gcy;": '\U00000433', + "gdot;": '\U00000121', + "ge;": '\U00002265', + "gel;": '\U000022DB', + "geq;": '\U00002265', + "geqq;": '\U00002267', + "geqslant;": '\U00002A7E', + "ges;": '\U00002A7E', + "gescc;": '\U00002AA9', + "gesdot;": '\U00002A80', + "gesdoto;": '\U00002A82', + "gesdotol;": '\U00002A84', + "gesles;": '\U00002A94', + "gfr;": '\U0001D524', + "gg;": '\U0000226B', + "ggg;": '\U000022D9', + "gimel;": '\U00002137', + "gjcy;": '\U00000453', + "gl;": '\U00002277', + "glE;": '\U00002A92', + "gla;": '\U00002AA5', + "glj;": '\U00002AA4', + "gnE;": '\U00002269', + "gnap;": '\U00002A8A', + "gnapprox;": '\U00002A8A', + "gne;": '\U00002A88', + "gneq;": '\U00002A88', + "gneqq;": '\U00002269', + "gnsim;": '\U000022E7', + "gopf;": '\U0001D558', + "grave;": '\U00000060', + "gscr;": '\U0000210A', + "gsim;": '\U00002273', + "gsime;": '\U00002A8E', + "gsiml;": '\U00002A90', + "gt;": '\U0000003E', + "gtcc;": '\U00002AA7', + "gtcir;": '\U00002A7A', + "gtdot;": '\U000022D7', + "gtlPar;": '\U00002995', + "gtquest;": '\U00002A7C', + "gtrapprox;": '\U00002A86', + "gtrarr;": '\U00002978', + "gtrdot;": '\U000022D7', + "gtreqless;": '\U000022DB', + "gtreqqless;": '\U00002A8C', + "gtrless;": '\U00002277', + "gtrsim;": '\U00002273', + "hArr;": '\U000021D4', + "hairsp;": '\U0000200A', + "half;": '\U000000BD', + "hamilt;": '\U0000210B', + "hardcy;": '\U0000044A', + "harr;": '\U00002194', + "harrcir;": '\U00002948', + "harrw;": '\U000021AD', + "hbar;": '\U0000210F', + "hcirc;": '\U00000125', + "hearts;": '\U00002665', + "heartsuit;": '\U00002665', + "hellip;": '\U00002026', + "hercon;": '\U000022B9', + "hfr;": '\U0001D525', + "hksearow;": '\U00002925', + "hkswarow;": '\U00002926', + "hoarr;": '\U000021FF', + "homtht;": '\U0000223B', + "hookleftarrow;": '\U000021A9', + "hookrightarrow;": '\U000021AA', + "hopf;": '\U0001D559', + "horbar;": '\U00002015', + "hscr;": '\U0001D4BD', + "hslash;": '\U0000210F', + "hstrok;": '\U00000127', + "hybull;": '\U00002043', + "hyphen;": '\U00002010', + "iacute;": '\U000000ED', + "ic;": '\U00002063', + "icirc;": '\U000000EE', + "icy;": '\U00000438', + "iecy;": '\U00000435', + "iexcl;": '\U000000A1', + "iff;": '\U000021D4', + "ifr;": '\U0001D526', + "igrave;": '\U000000EC', + "ii;": '\U00002148', + "iiiint;": '\U00002A0C', + "iiint;": '\U0000222D', + "iinfin;": '\U000029DC', + "iiota;": '\U00002129', + "ijlig;": '\U00000133', + "imacr;": '\U0000012B', + "image;": '\U00002111', + "imagline;": '\U00002110', + "imagpart;": '\U00002111', + "imath;": '\U00000131', + "imof;": '\U000022B7', + "imped;": '\U000001B5', + "in;": '\U00002208', + "incare;": '\U00002105', + "infin;": '\U0000221E', + "infintie;": '\U000029DD', + "inodot;": '\U00000131', + "int;": '\U0000222B', + "intcal;": '\U000022BA', + "integers;": '\U00002124', + "intercal;": '\U000022BA', + "intlarhk;": '\U00002A17', + "intprod;": '\U00002A3C', + "iocy;": '\U00000451', + "iogon;": '\U0000012F', + "iopf;": '\U0001D55A', + "iota;": '\U000003B9', + "iprod;": '\U00002A3C', + "iquest;": '\U000000BF', + "iscr;": '\U0001D4BE', + "isin;": '\U00002208', + "isinE;": '\U000022F9', + "isindot;": '\U000022F5', + "isins;": '\U000022F4', + "isinsv;": '\U000022F3', + "isinv;": '\U00002208', + "it;": '\U00002062', + "itilde;": '\U00000129', + "iukcy;": '\U00000456', + "iuml;": '\U000000EF', + "jcirc;": '\U00000135', + "jcy;": '\U00000439', + "jfr;": '\U0001D527', + "jmath;": '\U00000237', + "jopf;": '\U0001D55B', + "jscr;": '\U0001D4BF', + "jsercy;": '\U00000458', + "jukcy;": '\U00000454', + "kappa;": '\U000003BA', + "kappav;": '\U000003F0', + "kcedil;": '\U00000137', + "kcy;": '\U0000043A', + "kfr;": '\U0001D528', + "kgreen;": '\U00000138', + "khcy;": '\U00000445', + "kjcy;": '\U0000045C', + "kopf;": '\U0001D55C', + "kscr;": '\U0001D4C0', + "lAarr;": '\U000021DA', + "lArr;": '\U000021D0', + "lAtail;": '\U0000291B', + "lBarr;": '\U0000290E', + "lE;": '\U00002266', + "lEg;": '\U00002A8B', + "lHar;": '\U00002962', + "lacute;": '\U0000013A', + "laemptyv;": '\U000029B4', + "lagran;": '\U00002112', + "lambda;": '\U000003BB', + "lang;": '\U000027E8', + "langd;": '\U00002991', + "langle;": '\U000027E8', + "lap;": '\U00002A85', + "laquo;": '\U000000AB', + "larr;": '\U00002190', + "larrb;": '\U000021E4', + "larrbfs;": '\U0000291F', + "larrfs;": '\U0000291D', + "larrhk;": '\U000021A9', + "larrlp;": '\U000021AB', + "larrpl;": '\U00002939', + "larrsim;": '\U00002973', + "larrtl;": '\U000021A2', + "lat;": '\U00002AAB', + "latail;": '\U00002919', + "late;": '\U00002AAD', + "lbarr;": '\U0000290C', + "lbbrk;": '\U00002772', + "lbrace;": '\U0000007B', + "lbrack;": '\U0000005B', + "lbrke;": '\U0000298B', + "lbrksld;": '\U0000298F', + "lbrkslu;": '\U0000298D', + "lcaron;": '\U0000013E', + "lcedil;": '\U0000013C', + "lceil;": '\U00002308', + "lcub;": '\U0000007B', + "lcy;": '\U0000043B', + "ldca;": '\U00002936', + "ldquo;": '\U0000201C', + "ldquor;": '\U0000201E', + "ldrdhar;": '\U00002967', + "ldrushar;": '\U0000294B', + "ldsh;": '\U000021B2', + "le;": '\U00002264', + "leftarrow;": '\U00002190', + "leftarrowtail;": '\U000021A2', + "leftharpoondown;": '\U000021BD', + "leftharpoonup;": '\U000021BC', + "leftleftarrows;": '\U000021C7', + "leftrightarrow;": '\U00002194', + "leftrightarrows;": '\U000021C6', + "leftrightharpoons;": '\U000021CB', + "leftrightsquigarrow;": '\U000021AD', + "leftthreetimes;": '\U000022CB', + "leg;": '\U000022DA', + "leq;": '\U00002264', + "leqq;": '\U00002266', + "leqslant;": '\U00002A7D', + "les;": '\U00002A7D', + "lescc;": '\U00002AA8', + "lesdot;": '\U00002A7F', + "lesdoto;": '\U00002A81', + "lesdotor;": '\U00002A83', + "lesges;": '\U00002A93', + "lessapprox;": '\U00002A85', + "lessdot;": '\U000022D6', + "lesseqgtr;": '\U000022DA', + "lesseqqgtr;": '\U00002A8B', + "lessgtr;": '\U00002276', + "lesssim;": '\U00002272', + "lfisht;": '\U0000297C', + "lfloor;": '\U0000230A', + "lfr;": '\U0001D529', + "lg;": '\U00002276', + "lgE;": '\U00002A91', + "lhard;": '\U000021BD', + "lharu;": '\U000021BC', + "lharul;": '\U0000296A', + "lhblk;": '\U00002584', + "ljcy;": '\U00000459', + "ll;": '\U0000226A', + "llarr;": '\U000021C7', + "llcorner;": '\U0000231E', + "llhard;": '\U0000296B', + "lltri;": '\U000025FA', + "lmidot;": '\U00000140', + "lmoust;": '\U000023B0', + "lmoustache;": '\U000023B0', + "lnE;": '\U00002268', + "lnap;": '\U00002A89', + "lnapprox;": '\U00002A89', + "lne;": '\U00002A87', + "lneq;": '\U00002A87', + "lneqq;": '\U00002268', + "lnsim;": '\U000022E6', + "loang;": '\U000027EC', + "loarr;": '\U000021FD', + "lobrk;": '\U000027E6', + "longleftarrow;": '\U000027F5', + "longleftrightarrow;": '\U000027F7', + "longmapsto;": '\U000027FC', + "longrightarrow;": '\U000027F6', + "looparrowleft;": '\U000021AB', + "looparrowright;": '\U000021AC', + "lopar;": '\U00002985', + "lopf;": '\U0001D55D', + "loplus;": '\U00002A2D', + "lotimes;": '\U00002A34', + "lowast;": '\U00002217', + "lowbar;": '\U0000005F', + "loz;": '\U000025CA', + "lozenge;": '\U000025CA', + "lozf;": '\U000029EB', + "lpar;": '\U00000028', + "lparlt;": '\U00002993', + "lrarr;": '\U000021C6', + "lrcorner;": '\U0000231F', + "lrhar;": '\U000021CB', + "lrhard;": '\U0000296D', + "lrm;": '\U0000200E', + "lrtri;": '\U000022BF', + "lsaquo;": '\U00002039', + "lscr;": '\U0001D4C1', + "lsh;": '\U000021B0', + "lsim;": '\U00002272', + "lsime;": '\U00002A8D', + "lsimg;": '\U00002A8F', + "lsqb;": '\U0000005B', + "lsquo;": '\U00002018', + "lsquor;": '\U0000201A', + "lstrok;": '\U00000142', + "lt;": '\U0000003C', + "ltcc;": '\U00002AA6', + "ltcir;": '\U00002A79', + "ltdot;": '\U000022D6', + "lthree;": '\U000022CB', + "ltimes;": '\U000022C9', + "ltlarr;": '\U00002976', + "ltquest;": '\U00002A7B', + "ltrPar;": '\U00002996', + "ltri;": '\U000025C3', + "ltrie;": '\U000022B4', + "ltrif;": '\U000025C2', + "lurdshar;": '\U0000294A', + "luruhar;": '\U00002966', + "mDDot;": '\U0000223A', + "macr;": '\U000000AF', + "male;": '\U00002642', + "malt;": '\U00002720', + "maltese;": '\U00002720', + "map;": '\U000021A6', + "mapsto;": '\U000021A6', + "mapstodown;": '\U000021A7', + "mapstoleft;": '\U000021A4', + "mapstoup;": '\U000021A5', + "marker;": '\U000025AE', + "mcomma;": '\U00002A29', + "mcy;": '\U0000043C', + "mdash;": '\U00002014', + "measuredangle;": '\U00002221', + "mfr;": '\U0001D52A', + "mho;": '\U00002127', + "micro;": '\U000000B5', + "mid;": '\U00002223', + "midast;": '\U0000002A', + "midcir;": '\U00002AF0', + "middot;": '\U000000B7', + "minus;": '\U00002212', + "minusb;": '\U0000229F', + "minusd;": '\U00002238', + "minusdu;": '\U00002A2A', + "mlcp;": '\U00002ADB', + "mldr;": '\U00002026', + "mnplus;": '\U00002213', + "models;": '\U000022A7', + "mopf;": '\U0001D55E', + "mp;": '\U00002213', + "mscr;": '\U0001D4C2', + "mstpos;": '\U0000223E', + "mu;": '\U000003BC', + "multimap;": '\U000022B8', + "mumap;": '\U000022B8', + "nLeftarrow;": '\U000021CD', + "nLeftrightarrow;": '\U000021CE', + "nRightarrow;": '\U000021CF', + "nVDash;": '\U000022AF', + "nVdash;": '\U000022AE', + "nabla;": '\U00002207', + "nacute;": '\U00000144', + "nap;": '\U00002249', + "napos;": '\U00000149', + "napprox;": '\U00002249', + "natur;": '\U0000266E', + "natural;": '\U0000266E', + "naturals;": '\U00002115', + "nbsp;": '\U000000A0', + "ncap;": '\U00002A43', + "ncaron;": '\U00000148', + "ncedil;": '\U00000146', + "ncong;": '\U00002247', + "ncup;": '\U00002A42', + "ncy;": '\U0000043D', + "ndash;": '\U00002013', + "ne;": '\U00002260', + "neArr;": '\U000021D7', + "nearhk;": '\U00002924', + "nearr;": '\U00002197', + "nearrow;": '\U00002197', + "nequiv;": '\U00002262', + "nesear;": '\U00002928', + "nexist;": '\U00002204', + "nexists;": '\U00002204', + "nfr;": '\U0001D52B', + "nge;": '\U00002271', + "ngeq;": '\U00002271', + "ngsim;": '\U00002275', + "ngt;": '\U0000226F', + "ngtr;": '\U0000226F', + "nhArr;": '\U000021CE', + "nharr;": '\U000021AE', + "nhpar;": '\U00002AF2', + "ni;": '\U0000220B', + "nis;": '\U000022FC', + "nisd;": '\U000022FA', + "niv;": '\U0000220B', + "njcy;": '\U0000045A', + "nlArr;": '\U000021CD', + "nlarr;": '\U0000219A', + "nldr;": '\U00002025', + "nle;": '\U00002270', + "nleftarrow;": '\U0000219A', + "nleftrightarrow;": '\U000021AE', + "nleq;": '\U00002270', + "nless;": '\U0000226E', + "nlsim;": '\U00002274', + "nlt;": '\U0000226E', + "nltri;": '\U000022EA', + "nltrie;": '\U000022EC', + "nmid;": '\U00002224', + "nopf;": '\U0001D55F', + "not;": '\U000000AC', + "notin;": '\U00002209', + "notinva;": '\U00002209', + "notinvb;": '\U000022F7', + "notinvc;": '\U000022F6', + "notni;": '\U0000220C', + "notniva;": '\U0000220C', + "notnivb;": '\U000022FE', + "notnivc;": '\U000022FD', + "npar;": '\U00002226', + "nparallel;": '\U00002226', + "npolint;": '\U00002A14', + "npr;": '\U00002280', + "nprcue;": '\U000022E0', + "nprec;": '\U00002280', + "nrArr;": '\U000021CF', + "nrarr;": '\U0000219B', + "nrightarrow;": '\U0000219B', + "nrtri;": '\U000022EB', + "nrtrie;": '\U000022ED', + "nsc;": '\U00002281', + "nsccue;": '\U000022E1', + "nscr;": '\U0001D4C3', + "nshortmid;": '\U00002224', + "nshortparallel;": '\U00002226', + "nsim;": '\U00002241', + "nsime;": '\U00002244', + "nsimeq;": '\U00002244', + "nsmid;": '\U00002224', + "nspar;": '\U00002226', + "nsqsube;": '\U000022E2', + "nsqsupe;": '\U000022E3', + "nsub;": '\U00002284', + "nsube;": '\U00002288', + "nsubseteq;": '\U00002288', + "nsucc;": '\U00002281', + "nsup;": '\U00002285', + "nsupe;": '\U00002289', + "nsupseteq;": '\U00002289', + "ntgl;": '\U00002279', + "ntilde;": '\U000000F1', + "ntlg;": '\U00002278', + "ntriangleleft;": '\U000022EA', + "ntrianglelefteq;": '\U000022EC', + "ntriangleright;": '\U000022EB', + "ntrianglerighteq;": '\U000022ED', + "nu;": '\U000003BD', + "num;": '\U00000023', + "numero;": '\U00002116', + "numsp;": '\U00002007', + "nvDash;": '\U000022AD', + "nvHarr;": '\U00002904', + "nvdash;": '\U000022AC', + "nvinfin;": '\U000029DE', + "nvlArr;": '\U00002902', + "nvrArr;": '\U00002903', + "nwArr;": '\U000021D6', + "nwarhk;": '\U00002923', + "nwarr;": '\U00002196', + "nwarrow;": '\U00002196', + "nwnear;": '\U00002927', + "oS;": '\U000024C8', + "oacute;": '\U000000F3', + "oast;": '\U0000229B', + "ocir;": '\U0000229A', + "ocirc;": '\U000000F4', + "ocy;": '\U0000043E', + "odash;": '\U0000229D', + "odblac;": '\U00000151', + "odiv;": '\U00002A38', + "odot;": '\U00002299', + "odsold;": '\U000029BC', + "oelig;": '\U00000153', + "ofcir;": '\U000029BF', + "ofr;": '\U0001D52C', + "ogon;": '\U000002DB', + "ograve;": '\U000000F2', + "ogt;": '\U000029C1', + "ohbar;": '\U000029B5', + "ohm;": '\U000003A9', + "oint;": '\U0000222E', + "olarr;": '\U000021BA', + "olcir;": '\U000029BE', + "olcross;": '\U000029BB', + "oline;": '\U0000203E', + "olt;": '\U000029C0', + "omacr;": '\U0000014D', + "omega;": '\U000003C9', + "omicron;": '\U000003BF', + "omid;": '\U000029B6', + "ominus;": '\U00002296', + "oopf;": '\U0001D560', + "opar;": '\U000029B7', + "operp;": '\U000029B9', + "oplus;": '\U00002295', + "or;": '\U00002228', + "orarr;": '\U000021BB', + "ord;": '\U00002A5D', + "order;": '\U00002134', + "orderof;": '\U00002134', + "ordf;": '\U000000AA', + "ordm;": '\U000000BA', + "origof;": '\U000022B6', + "oror;": '\U00002A56', + "orslope;": '\U00002A57', + "orv;": '\U00002A5B', + "oscr;": '\U00002134', + "oslash;": '\U000000F8', + "osol;": '\U00002298', + "otilde;": '\U000000F5', + "otimes;": '\U00002297', + "otimesas;": '\U00002A36', + "ouml;": '\U000000F6', + "ovbar;": '\U0000233D', + "par;": '\U00002225', + "para;": '\U000000B6', + "parallel;": '\U00002225', + "parsim;": '\U00002AF3', + "parsl;": '\U00002AFD', + "part;": '\U00002202', + "pcy;": '\U0000043F', + "percnt;": '\U00000025', + "period;": '\U0000002E', + "permil;": '\U00002030', + "perp;": '\U000022A5', + "pertenk;": '\U00002031', + "pfr;": '\U0001D52D', + "phi;": '\U000003C6', + "phiv;": '\U000003D5', + "phmmat;": '\U00002133', + "phone;": '\U0000260E', + "pi;": '\U000003C0', + "pitchfork;": '\U000022D4', + "piv;": '\U000003D6', + "planck;": '\U0000210F', + "planckh;": '\U0000210E', + "plankv;": '\U0000210F', + "plus;": '\U0000002B', + "plusacir;": '\U00002A23', + "plusb;": '\U0000229E', + "pluscir;": '\U00002A22', + "plusdo;": '\U00002214', + "plusdu;": '\U00002A25', + "pluse;": '\U00002A72', + "plusmn;": '\U000000B1', + "plussim;": '\U00002A26', + "plustwo;": '\U00002A27', + "pm;": '\U000000B1', + "pointint;": '\U00002A15', + "popf;": '\U0001D561', + "pound;": '\U000000A3', + "pr;": '\U0000227A', + "prE;": '\U00002AB3', + "prap;": '\U00002AB7', + "prcue;": '\U0000227C', + "pre;": '\U00002AAF', + "prec;": '\U0000227A', + "precapprox;": '\U00002AB7', + "preccurlyeq;": '\U0000227C', + "preceq;": '\U00002AAF', + "precnapprox;": '\U00002AB9', + "precneqq;": '\U00002AB5', + "precnsim;": '\U000022E8', + "precsim;": '\U0000227E', + "prime;": '\U00002032', + "primes;": '\U00002119', + "prnE;": '\U00002AB5', + "prnap;": '\U00002AB9', + "prnsim;": '\U000022E8', + "prod;": '\U0000220F', + "profalar;": '\U0000232E', + "profline;": '\U00002312', + "profsurf;": '\U00002313', + "prop;": '\U0000221D', + "propto;": '\U0000221D', + "prsim;": '\U0000227E', + "prurel;": '\U000022B0', + "pscr;": '\U0001D4C5', + "psi;": '\U000003C8', + "puncsp;": '\U00002008', + "qfr;": '\U0001D52E', + "qint;": '\U00002A0C', + "qopf;": '\U0001D562', + "qprime;": '\U00002057', + "qscr;": '\U0001D4C6', + "quaternions;": '\U0000210D', + "quatint;": '\U00002A16', + "quest;": '\U0000003F', + "questeq;": '\U0000225F', + "quot;": '\U00000022', + "rAarr;": '\U000021DB', + "rArr;": '\U000021D2', + "rAtail;": '\U0000291C', + "rBarr;": '\U0000290F', + "rHar;": '\U00002964', + "racute;": '\U00000155', + "radic;": '\U0000221A', + "raemptyv;": '\U000029B3', + "rang;": '\U000027E9', + "rangd;": '\U00002992', + "range;": '\U000029A5', + "rangle;": '\U000027E9', + "raquo;": '\U000000BB', + "rarr;": '\U00002192', + "rarrap;": '\U00002975', + "rarrb;": '\U000021E5', + "rarrbfs;": '\U00002920', + "rarrc;": '\U00002933', + "rarrfs;": '\U0000291E', + "rarrhk;": '\U000021AA', + "rarrlp;": '\U000021AC', + "rarrpl;": '\U00002945', + "rarrsim;": '\U00002974', + "rarrtl;": '\U000021A3', + "rarrw;": '\U0000219D', + "ratail;": '\U0000291A', + "ratio;": '\U00002236', + "rationals;": '\U0000211A', + "rbarr;": '\U0000290D', + "rbbrk;": '\U00002773', + "rbrace;": '\U0000007D', + "rbrack;": '\U0000005D', + "rbrke;": '\U0000298C', + "rbrksld;": '\U0000298E', + "rbrkslu;": '\U00002990', + "rcaron;": '\U00000159', + "rcedil;": '\U00000157', + "rceil;": '\U00002309', + "rcub;": '\U0000007D', + "rcy;": '\U00000440', + "rdca;": '\U00002937', + "rdldhar;": '\U00002969', + "rdquo;": '\U0000201D', + "rdquor;": '\U0000201D', + "rdsh;": '\U000021B3', + "real;": '\U0000211C', + "realine;": '\U0000211B', + "realpart;": '\U0000211C', + "reals;": '\U0000211D', + "rect;": '\U000025AD', + "reg;": '\U000000AE', + "rfisht;": '\U0000297D', + "rfloor;": '\U0000230B', + "rfr;": '\U0001D52F', + "rhard;": '\U000021C1', + "rharu;": '\U000021C0', + "rharul;": '\U0000296C', + "rho;": '\U000003C1', + "rhov;": '\U000003F1', + "rightarrow;": '\U00002192', + "rightarrowtail;": '\U000021A3', + "rightharpoondown;": '\U000021C1', + "rightharpoonup;": '\U000021C0', + "rightleftarrows;": '\U000021C4', + "rightleftharpoons;": '\U000021CC', + "rightrightarrows;": '\U000021C9', + "rightsquigarrow;": '\U0000219D', + "rightthreetimes;": '\U000022CC', + "ring;": '\U000002DA', + "risingdotseq;": '\U00002253', + "rlarr;": '\U000021C4', + "rlhar;": '\U000021CC', + "rlm;": '\U0000200F', + "rmoust;": '\U000023B1', + "rmoustache;": '\U000023B1', + "rnmid;": '\U00002AEE', + "roang;": '\U000027ED', + "roarr;": '\U000021FE', + "robrk;": '\U000027E7', + "ropar;": '\U00002986', + "ropf;": '\U0001D563', + "roplus;": '\U00002A2E', + "rotimes;": '\U00002A35', + "rpar;": '\U00000029', + "rpargt;": '\U00002994', + "rppolint;": '\U00002A12', + "rrarr;": '\U000021C9', + "rsaquo;": '\U0000203A', + "rscr;": '\U0001D4C7', + "rsh;": '\U000021B1', + "rsqb;": '\U0000005D', + "rsquo;": '\U00002019', + "rsquor;": '\U00002019', + "rthree;": '\U000022CC', + "rtimes;": '\U000022CA', + "rtri;": '\U000025B9', + "rtrie;": '\U000022B5', + "rtrif;": '\U000025B8', + "rtriltri;": '\U000029CE', + "ruluhar;": '\U00002968', + "rx;": '\U0000211E', + "sacute;": '\U0000015B', + "sbquo;": '\U0000201A', + "sc;": '\U0000227B', + "scE;": '\U00002AB4', + "scap;": '\U00002AB8', + "scaron;": '\U00000161', + "sccue;": '\U0000227D', + "sce;": '\U00002AB0', + "scedil;": '\U0000015F', + "scirc;": '\U0000015D', + "scnE;": '\U00002AB6', + "scnap;": '\U00002ABA', + "scnsim;": '\U000022E9', + "scpolint;": '\U00002A13', + "scsim;": '\U0000227F', + "scy;": '\U00000441', + "sdot;": '\U000022C5', + "sdotb;": '\U000022A1', + "sdote;": '\U00002A66', + "seArr;": '\U000021D8', + "searhk;": '\U00002925', + "searr;": '\U00002198', + "searrow;": '\U00002198', + "sect;": '\U000000A7', + "semi;": '\U0000003B', + "seswar;": '\U00002929', + "setminus;": '\U00002216', + "setmn;": '\U00002216', + "sext;": '\U00002736', + "sfr;": '\U0001D530', + "sfrown;": '\U00002322', + "sharp;": '\U0000266F', + "shchcy;": '\U00000449', + "shcy;": '\U00000448', + "shortmid;": '\U00002223', + "shortparallel;": '\U00002225', + "shy;": '\U000000AD', + "sigma;": '\U000003C3', + "sigmaf;": '\U000003C2', + "sigmav;": '\U000003C2', + "sim;": '\U0000223C', + "simdot;": '\U00002A6A', + "sime;": '\U00002243', + "simeq;": '\U00002243', + "simg;": '\U00002A9E', + "simgE;": '\U00002AA0', + "siml;": '\U00002A9D', + "simlE;": '\U00002A9F', + "simne;": '\U00002246', + "simplus;": '\U00002A24', + "simrarr;": '\U00002972', + "slarr;": '\U00002190', + "smallsetminus;": '\U00002216', + "smashp;": '\U00002A33', + "smeparsl;": '\U000029E4', + "smid;": '\U00002223', + "smile;": '\U00002323', + "smt;": '\U00002AAA', + "smte;": '\U00002AAC', + "softcy;": '\U0000044C', + "sol;": '\U0000002F', + "solb;": '\U000029C4', + "solbar;": '\U0000233F', + "sopf;": '\U0001D564', + "spades;": '\U00002660', + "spadesuit;": '\U00002660', + "spar;": '\U00002225', + "sqcap;": '\U00002293', + "sqcup;": '\U00002294', + "sqsub;": '\U0000228F', + "sqsube;": '\U00002291', + "sqsubset;": '\U0000228F', + "sqsubseteq;": '\U00002291', + "sqsup;": '\U00002290', + "sqsupe;": '\U00002292', + "sqsupset;": '\U00002290', + "sqsupseteq;": '\U00002292', + "squ;": '\U000025A1', + "square;": '\U000025A1', + "squarf;": '\U000025AA', + "squf;": '\U000025AA', + "srarr;": '\U00002192', + "sscr;": '\U0001D4C8', + "ssetmn;": '\U00002216', + "ssmile;": '\U00002323', + "sstarf;": '\U000022C6', + "star;": '\U00002606', + "starf;": '\U00002605', + "straightepsilon;": '\U000003F5', + "straightphi;": '\U000003D5', + "strns;": '\U000000AF', + "sub;": '\U00002282', + "subE;": '\U00002AC5', + "subdot;": '\U00002ABD', + "sube;": '\U00002286', + "subedot;": '\U00002AC3', + "submult;": '\U00002AC1', + "subnE;": '\U00002ACB', + "subne;": '\U0000228A', + "subplus;": '\U00002ABF', + "subrarr;": '\U00002979', + "subset;": '\U00002282', + "subseteq;": '\U00002286', + "subseteqq;": '\U00002AC5', + "subsetneq;": '\U0000228A', + "subsetneqq;": '\U00002ACB', + "subsim;": '\U00002AC7', + "subsub;": '\U00002AD5', + "subsup;": '\U00002AD3', + "succ;": '\U0000227B', + "succapprox;": '\U00002AB8', + "succcurlyeq;": '\U0000227D', + "succeq;": '\U00002AB0', + "succnapprox;": '\U00002ABA', + "succneqq;": '\U00002AB6', + "succnsim;": '\U000022E9', + "succsim;": '\U0000227F', + "sum;": '\U00002211', + "sung;": '\U0000266A', + "sup;": '\U00002283', + "sup1;": '\U000000B9', + "sup2;": '\U000000B2', + "sup3;": '\U000000B3', + "supE;": '\U00002AC6', + "supdot;": '\U00002ABE', + "supdsub;": '\U00002AD8', + "supe;": '\U00002287', + "supedot;": '\U00002AC4', + "suphsol;": '\U000027C9', + "suphsub;": '\U00002AD7', + "suplarr;": '\U0000297B', + "supmult;": '\U00002AC2', + "supnE;": '\U00002ACC', + "supne;": '\U0000228B', + "supplus;": '\U00002AC0', + "supset;": '\U00002283', + "supseteq;": '\U00002287', + "supseteqq;": '\U00002AC6', + "supsetneq;": '\U0000228B', + "supsetneqq;": '\U00002ACC', + "supsim;": '\U00002AC8', + "supsub;": '\U00002AD4', + "supsup;": '\U00002AD6', + "swArr;": '\U000021D9', + "swarhk;": '\U00002926', + "swarr;": '\U00002199', + "swarrow;": '\U00002199', + "swnwar;": '\U0000292A', + "szlig;": '\U000000DF', + "target;": '\U00002316', + "tau;": '\U000003C4', + "tbrk;": '\U000023B4', + "tcaron;": '\U00000165', + "tcedil;": '\U00000163', + "tcy;": '\U00000442', + "tdot;": '\U000020DB', + "telrec;": '\U00002315', + "tfr;": '\U0001D531', + "there4;": '\U00002234', + "therefore;": '\U00002234', + "theta;": '\U000003B8', + "thetasym;": '\U000003D1', + "thetav;": '\U000003D1', + "thickapprox;": '\U00002248', + "thicksim;": '\U0000223C', + "thinsp;": '\U00002009', + "thkap;": '\U00002248', + "thksim;": '\U0000223C', + "thorn;": '\U000000FE', + "tilde;": '\U000002DC', + "times;": '\U000000D7', + "timesb;": '\U000022A0', + "timesbar;": '\U00002A31', + "timesd;": '\U00002A30', + "tint;": '\U0000222D', + "toea;": '\U00002928', + "top;": '\U000022A4', + "topbot;": '\U00002336', + "topcir;": '\U00002AF1', + "topf;": '\U0001D565', + "topfork;": '\U00002ADA', + "tosa;": '\U00002929', + "tprime;": '\U00002034', + "trade;": '\U00002122', + "triangle;": '\U000025B5', + "triangledown;": '\U000025BF', + "triangleleft;": '\U000025C3', + "trianglelefteq;": '\U000022B4', + "triangleq;": '\U0000225C', + "triangleright;": '\U000025B9', + "trianglerighteq;": '\U000022B5', + "tridot;": '\U000025EC', + "trie;": '\U0000225C', + "triminus;": '\U00002A3A', + "triplus;": '\U00002A39', + "trisb;": '\U000029CD', + "tritime;": '\U00002A3B', + "trpezium;": '\U000023E2', + "tscr;": '\U0001D4C9', + "tscy;": '\U00000446', + "tshcy;": '\U0000045B', + "tstrok;": '\U00000167', + "twixt;": '\U0000226C', + "twoheadleftarrow;": '\U0000219E', + "twoheadrightarrow;": '\U000021A0', + "uArr;": '\U000021D1', + "uHar;": '\U00002963', + "uacute;": '\U000000FA', + "uarr;": '\U00002191', + "ubrcy;": '\U0000045E', + "ubreve;": '\U0000016D', + "ucirc;": '\U000000FB', + "ucy;": '\U00000443', + "udarr;": '\U000021C5', + "udblac;": '\U00000171', + "udhar;": '\U0000296E', + "ufisht;": '\U0000297E', + "ufr;": '\U0001D532', + "ugrave;": '\U000000F9', + "uharl;": '\U000021BF', + "uharr;": '\U000021BE', + "uhblk;": '\U00002580', + "ulcorn;": '\U0000231C', + "ulcorner;": '\U0000231C', + "ulcrop;": '\U0000230F', + "ultri;": '\U000025F8', + "umacr;": '\U0000016B', + "uml;": '\U000000A8', + "uogon;": '\U00000173', + "uopf;": '\U0001D566', + "uparrow;": '\U00002191', + "updownarrow;": '\U00002195', + "upharpoonleft;": '\U000021BF', + "upharpoonright;": '\U000021BE', + "uplus;": '\U0000228E', + "upsi;": '\U000003C5', + "upsih;": '\U000003D2', + "upsilon;": '\U000003C5', + "upuparrows;": '\U000021C8', + "urcorn;": '\U0000231D', + "urcorner;": '\U0000231D', + "urcrop;": '\U0000230E', + "uring;": '\U0000016F', + "urtri;": '\U000025F9', + "uscr;": '\U0001D4CA', + "utdot;": '\U000022F0', + "utilde;": '\U00000169', + "utri;": '\U000025B5', + "utrif;": '\U000025B4', + "uuarr;": '\U000021C8', + "uuml;": '\U000000FC', + "uwangle;": '\U000029A7', + "vArr;": '\U000021D5', + "vBar;": '\U00002AE8', + "vBarv;": '\U00002AE9', + "vDash;": '\U000022A8', + "vangrt;": '\U0000299C', + "varepsilon;": '\U000003F5', + "varkappa;": '\U000003F0', + "varnothing;": '\U00002205', + "varphi;": '\U000003D5', + "varpi;": '\U000003D6', + "varpropto;": '\U0000221D', + "varr;": '\U00002195', + "varrho;": '\U000003F1', + "varsigma;": '\U000003C2', + "vartheta;": '\U000003D1', + "vartriangleleft;": '\U000022B2', + "vartriangleright;": '\U000022B3', + "vcy;": '\U00000432', + "vdash;": '\U000022A2', + "vee;": '\U00002228', + "veebar;": '\U000022BB', + "veeeq;": '\U0000225A', + "vellip;": '\U000022EE', + "verbar;": '\U0000007C', + "vert;": '\U0000007C', + "vfr;": '\U0001D533', + "vltri;": '\U000022B2', + "vopf;": '\U0001D567', + "vprop;": '\U0000221D', + "vrtri;": '\U000022B3', + "vscr;": '\U0001D4CB', + "vzigzag;": '\U0000299A', + "wcirc;": '\U00000175', + "wedbar;": '\U00002A5F', + "wedge;": '\U00002227', + "wedgeq;": '\U00002259', + "weierp;": '\U00002118', + "wfr;": '\U0001D534', + "wopf;": '\U0001D568', + "wp;": '\U00002118', + "wr;": '\U00002240', + "wreath;": '\U00002240', + "wscr;": '\U0001D4CC', + "xcap;": '\U000022C2', + "xcirc;": '\U000025EF', + "xcup;": '\U000022C3', + "xdtri;": '\U000025BD', + "xfr;": '\U0001D535', + "xhArr;": '\U000027FA', + "xharr;": '\U000027F7', + "xi;": '\U000003BE', + "xlArr;": '\U000027F8', + "xlarr;": '\U000027F5', + "xmap;": '\U000027FC', + "xnis;": '\U000022FB', + "xodot;": '\U00002A00', + "xopf;": '\U0001D569', + "xoplus;": '\U00002A01', + "xotime;": '\U00002A02', + "xrArr;": '\U000027F9', + "xrarr;": '\U000027F6', + "xscr;": '\U0001D4CD', + "xsqcup;": '\U00002A06', + "xuplus;": '\U00002A04', + "xutri;": '\U000025B3', + "xvee;": '\U000022C1', + "xwedge;": '\U000022C0', + "yacute;": '\U000000FD', + "yacy;": '\U0000044F', + "ycirc;": '\U00000177', + "ycy;": '\U0000044B', + "yen;": '\U000000A5', + "yfr;": '\U0001D536', + "yicy;": '\U00000457', + "yopf;": '\U0001D56A', + "yscr;": '\U0001D4CE', + "yucy;": '\U0000044E', + "yuml;": '\U000000FF', + "zacute;": '\U0000017A', + "zcaron;": '\U0000017E', + "zcy;": '\U00000437', + "zdot;": '\U0000017C', + "zeetrf;": '\U00002128', + "zeta;": '\U000003B6', + "zfr;": '\U0001D537', + "zhcy;": '\U00000436', + "zigrarr;": '\U000021DD', + "zopf;": '\U0001D56B', + "zscr;": '\U0001D4CF', + "zwj;": '\U0000200D', + "zwnj;": '\U0000200C', + "AElig": '\U000000C6', + "AMP": '\U00000026', + "Aacute": '\U000000C1', + "Acirc": '\U000000C2', + "Agrave": '\U000000C0', + "Aring": '\U000000C5', + "Atilde": '\U000000C3', + "Auml": '\U000000C4', + "COPY": '\U000000A9', + "Ccedil": '\U000000C7', + "ETH": '\U000000D0', + "Eacute": '\U000000C9', + "Ecirc": '\U000000CA', + "Egrave": '\U000000C8', + "Euml": '\U000000CB', + "GT": '\U0000003E', + "Iacute": '\U000000CD', + "Icirc": '\U000000CE', + "Igrave": '\U000000CC', + "Iuml": '\U000000CF', + "LT": '\U0000003C', + "Ntilde": '\U000000D1', + "Oacute": '\U000000D3', + "Ocirc": '\U000000D4', + "Ograve": '\U000000D2', + "Oslash": '\U000000D8', + "Otilde": '\U000000D5', + "Ouml": '\U000000D6', + "QUOT": '\U00000022', + "REG": '\U000000AE', + "THORN": '\U000000DE', + "Uacute": '\U000000DA', + "Ucirc": '\U000000DB', + "Ugrave": '\U000000D9', + "Uuml": '\U000000DC', + "Yacute": '\U000000DD', + "aacute": '\U000000E1', + "acirc": '\U000000E2', + "acute": '\U000000B4', + "aelig": '\U000000E6', + "agrave": '\U000000E0', + "amp": '\U00000026', + "aring": '\U000000E5', + "atilde": '\U000000E3', + "auml": '\U000000E4', + "brvbar": '\U000000A6', + "ccedil": '\U000000E7', + "cedil": '\U000000B8', + "cent": '\U000000A2', + "copy": '\U000000A9', + "curren": '\U000000A4', + "deg": '\U000000B0', + "divide": '\U000000F7', + "eacute": '\U000000E9', + "ecirc": '\U000000EA', + "egrave": '\U000000E8', + "eth": '\U000000F0', + "euml": '\U000000EB', + "frac12": '\U000000BD', + "frac14": '\U000000BC', + "frac34": '\U000000BE', + "gt": '\U0000003E', + "iacute": '\U000000ED', + "icirc": '\U000000EE', + "iexcl": '\U000000A1', + "igrave": '\U000000EC', + "iquest": '\U000000BF', + "iuml": '\U000000EF', + "laquo": '\U000000AB', + "lt": '\U0000003C', + "macr": '\U000000AF', + "micro": '\U000000B5', + "middot": '\U000000B7', + "nbsp": '\U000000A0', + "not": '\U000000AC', + "ntilde": '\U000000F1', + "oacute": '\U000000F3', + "ocirc": '\U000000F4', + "ograve": '\U000000F2', + "ordf": '\U000000AA', + "ordm": '\U000000BA', + "oslash": '\U000000F8', + "otilde": '\U000000F5', + "ouml": '\U000000F6', + "para": '\U000000B6', + "plusmn": '\U000000B1', + "pound": '\U000000A3', + "quot": '\U00000022', + "raquo": '\U000000BB', + "reg": '\U000000AE', + "sect": '\U000000A7', + "shy": '\U000000AD', + "sup1": '\U000000B9', + "sup2": '\U000000B2', + "sup3": '\U000000B3', + "szlig": '\U000000DF', + "thorn": '\U000000FE', + "times": '\U000000D7', + "uacute": '\U000000FA', + "ucirc": '\U000000FB', + "ugrave": '\U000000F9', + "uml": '\U000000A8', + "uuml": '\U000000FC', + "yacute": '\U000000FD', + "yen": '\U000000A5', + "yuml": '\U000000FF', +} + +// HTML entities that are two unicode codepoints. +var entity2 = map[string][2]rune{ + // TODO(nigeltao): Handle replacements that are wider than their names. + // "nLt;": {'\u226A', '\u20D2'}, + // "nGt;": {'\u226B', '\u20D2'}, + "NotEqualTilde;": {'\u2242', '\u0338'}, + "NotGreaterFullEqual;": {'\u2267', '\u0338'}, + "NotGreaterGreater;": {'\u226B', '\u0338'}, + "NotGreaterSlantEqual;": {'\u2A7E', '\u0338'}, + "NotHumpDownHump;": {'\u224E', '\u0338'}, + "NotHumpEqual;": {'\u224F', '\u0338'}, + "NotLeftTriangleBar;": {'\u29CF', '\u0338'}, + "NotLessLess;": {'\u226A', '\u0338'}, + "NotLessSlantEqual;": {'\u2A7D', '\u0338'}, + "NotNestedGreaterGreater;": {'\u2AA2', '\u0338'}, + "NotNestedLessLess;": {'\u2AA1', '\u0338'}, + "NotPrecedesEqual;": {'\u2AAF', '\u0338'}, + "NotRightTriangleBar;": {'\u29D0', '\u0338'}, + "NotSquareSubset;": {'\u228F', '\u0338'}, + "NotSquareSuperset;": {'\u2290', '\u0338'}, + "NotSubset;": {'\u2282', '\u20D2'}, + "NotSucceedsEqual;": {'\u2AB0', '\u0338'}, + "NotSucceedsTilde;": {'\u227F', '\u0338'}, + "NotSuperset;": {'\u2283', '\u20D2'}, + "ThickSpace;": {'\u205F', '\u200A'}, + "acE;": {'\u223E', '\u0333'}, + "bne;": {'\u003D', '\u20E5'}, + "bnequiv;": {'\u2261', '\u20E5'}, + "caps;": {'\u2229', '\uFE00'}, + "cups;": {'\u222A', '\uFE00'}, + "fjlig;": {'\u0066', '\u006A'}, + "gesl;": {'\u22DB', '\uFE00'}, + "gvertneqq;": {'\u2269', '\uFE00'}, + "gvnE;": {'\u2269', '\uFE00'}, + "lates;": {'\u2AAD', '\uFE00'}, + "lesg;": {'\u22DA', '\uFE00'}, + "lvertneqq;": {'\u2268', '\uFE00'}, + "lvnE;": {'\u2268', '\uFE00'}, + "nGg;": {'\u22D9', '\u0338'}, + "nGtv;": {'\u226B', '\u0338'}, + "nLl;": {'\u22D8', '\u0338'}, + "nLtv;": {'\u226A', '\u0338'}, + "nang;": {'\u2220', '\u20D2'}, + "napE;": {'\u2A70', '\u0338'}, + "napid;": {'\u224B', '\u0338'}, + "nbump;": {'\u224E', '\u0338'}, + "nbumpe;": {'\u224F', '\u0338'}, + "ncongdot;": {'\u2A6D', '\u0338'}, + "nedot;": {'\u2250', '\u0338'}, + "nesim;": {'\u2242', '\u0338'}, + "ngE;": {'\u2267', '\u0338'}, + "ngeqq;": {'\u2267', '\u0338'}, + "ngeqslant;": {'\u2A7E', '\u0338'}, + "nges;": {'\u2A7E', '\u0338'}, + "nlE;": {'\u2266', '\u0338'}, + "nleqq;": {'\u2266', '\u0338'}, + "nleqslant;": {'\u2A7D', '\u0338'}, + "nles;": {'\u2A7D', '\u0338'}, + "notinE;": {'\u22F9', '\u0338'}, + "notindot;": {'\u22F5', '\u0338'}, + "nparsl;": {'\u2AFD', '\u20E5'}, + "npart;": {'\u2202', '\u0338'}, + "npre;": {'\u2AAF', '\u0338'}, + "npreceq;": {'\u2AAF', '\u0338'}, + "nrarrc;": {'\u2933', '\u0338'}, + "nrarrw;": {'\u219D', '\u0338'}, + "nsce;": {'\u2AB0', '\u0338'}, + "nsubE;": {'\u2AC5', '\u0338'}, + "nsubset;": {'\u2282', '\u20D2'}, + "nsubseteqq;": {'\u2AC5', '\u0338'}, + "nsucceq;": {'\u2AB0', '\u0338'}, + "nsupE;": {'\u2AC6', '\u0338'}, + "nsupset;": {'\u2283', '\u20D2'}, + "nsupseteqq;": {'\u2AC6', '\u0338'}, + "nvap;": {'\u224D', '\u20D2'}, + "nvge;": {'\u2265', '\u20D2'}, + "nvgt;": {'\u003E', '\u20D2'}, + "nvle;": {'\u2264', '\u20D2'}, + "nvlt;": {'\u003C', '\u20D2'}, + "nvltrie;": {'\u22B4', '\u20D2'}, + "nvrtrie;": {'\u22B5', '\u20D2'}, + "nvsim;": {'\u223C', '\u20D2'}, + "race;": {'\u223D', '\u0331'}, + "smtes;": {'\u2AAC', '\uFE00'}, + "sqcaps;": {'\u2293', '\uFE00'}, + "sqcups;": {'\u2294', '\uFE00'}, + "varsubsetneq;": {'\u228A', '\uFE00'}, + "varsubsetneqq;": {'\u2ACB', '\uFE00'}, + "varsupsetneq;": {'\u228B', '\uFE00'}, + "varsupsetneqq;": {'\u2ACC', '\uFE00'}, + "vnsub;": {'\u2282', '\u20D2'}, + "vnsup;": {'\u2283', '\u20D2'}, + "vsubnE;": {'\u2ACB', '\uFE00'}, + "vsubne;": {'\u228A', '\uFE00'}, + "vsupnE;": {'\u2ACC', '\uFE00'}, + "vsupne;": {'\u228B', '\uFE00'}, +} diff --git a/third_party/code.google.com/p/go.net/html/entity_test.go b/third_party/code.google.com/p/go.net/html/entity_test.go new file mode 100644 index 000000000..b53f866fa --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/entity_test.go @@ -0,0 +1,29 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "testing" + "unicode/utf8" +) + +func TestEntityLength(t *testing.T) { + // We verify that the length of UTF-8 encoding of each value is <= 1 + len(key). + // The +1 comes from the leading "&". This property implies that the length of + // unescaped text is <= the length of escaped text. + for k, v := range entity { + if 1+len(k) < utf8.RuneLen(v) { + t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v)) + } + if len(k) > longestEntityWithoutSemicolon && k[len(k)-1] != ';' { + t.Errorf("entity name %s is %d characters, but longestEntityWithoutSemicolon=%d", k, len(k), longestEntityWithoutSemicolon) + } + } + for k, v := range entity2 { + if 1+len(k) < utf8.RuneLen(v[0])+utf8.RuneLen(v[1]) { + t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v[0]) + string(v[1])) + } + } +} diff --git a/third_party/code.google.com/p/go.net/html/escape.go b/third_party/code.google.com/p/go.net/html/escape.go new file mode 100644 index 000000000..75bddff09 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/escape.go @@ -0,0 +1,258 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "strings" + "unicode/utf8" +) + +// These replacements permit compatibility with old numeric entities that +// assumed Windows-1252 encoding. +// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference +var replacementTable = [...]rune{ + '\u20AC', // First entry is what 0x80 should be replaced with. + '\u0081', + '\u201A', + '\u0192', + '\u201E', + '\u2026', + '\u2020', + '\u2021', + '\u02C6', + '\u2030', + '\u0160', + '\u2039', + '\u0152', + '\u008D', + '\u017D', + '\u008F', + '\u0090', + '\u2018', + '\u2019', + '\u201C', + '\u201D', + '\u2022', + '\u2013', + '\u2014', + '\u02DC', + '\u2122', + '\u0161', + '\u203A', + '\u0153', + '\u009D', + '\u017E', + '\u0178', // Last entry is 0x9F. + // 0x00->'\uFFFD' is handled programmatically. + // 0x0D->'\u000D' is a no-op. +} + +// unescapeEntity reads an entity like "<" from b[src:] and writes the +// corresponding "<" to b[dst:], returning the incremented dst and src cursors. +// Precondition: b[src] == '&' && dst <= src. +// attribute should be true if parsing an attribute value. +func unescapeEntity(b []byte, dst, src int, attribute bool) (dst1, src1 int) { + // http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference + + // i starts at 1 because we already know that s[0] == '&'. + i, s := 1, b[src:] + + if len(s) <= 1 { + b[dst] = b[src] + return dst + 1, src + 1 + } + + if s[i] == '#' { + if len(s) <= 3 { // We need to have at least "&#.". + b[dst] = b[src] + return dst + 1, src + 1 + } + i++ + c := s[i] + hex := false + if c == 'x' || c == 'X' { + hex = true + i++ + } + + x := '\x00' + for i < len(s) { + c = s[i] + i++ + if hex { + if '0' <= c && c <= '9' { + x = 16*x + rune(c) - '0' + continue + } else if 'a' <= c && c <= 'f' { + x = 16*x + rune(c) - 'a' + 10 + continue + } else if 'A' <= c && c <= 'F' { + x = 16*x + rune(c) - 'A' + 10 + continue + } + } else if '0' <= c && c <= '9' { + x = 10*x + rune(c) - '0' + continue + } + if c != ';' { + i-- + } + break + } + + if i <= 3 { // No characters matched. + b[dst] = b[src] + return dst + 1, src + 1 + } + + if 0x80 <= x && x <= 0x9F { + // Replace characters from Windows-1252 with UTF-8 equivalents. + x = replacementTable[x-0x80] + } else if x == 0 || (0xD800 <= x && x <= 0xDFFF) || x > 0x10FFFF { + // Replace invalid characters with the replacement character. + x = '\uFFFD' + } + + return dst + utf8.EncodeRune(b[dst:], x), src + i + } + + // Consume the maximum number of characters possible, with the + // consumed characters matching one of the named references. + + for i < len(s) { + c := s[i] + i++ + // Lower-cased characters are more common in entities, so we check for them first. + if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' { + continue + } + if c != ';' { + i-- + } + break + } + + entityName := string(s[1:i]) + if entityName == "" { + // No-op. + } else if attribute && entityName[len(entityName)-1] != ';' && len(s) > i && s[i] == '=' { + // No-op. + } else if x := entity[entityName]; x != 0 { + return dst + utf8.EncodeRune(b[dst:], x), src + i + } else if x := entity2[entityName]; x[0] != 0 { + dst1 := dst + utf8.EncodeRune(b[dst:], x[0]) + return dst1 + utf8.EncodeRune(b[dst1:], x[1]), src + i + } else if !attribute { + maxLen := len(entityName) - 1 + if maxLen > longestEntityWithoutSemicolon { + maxLen = longestEntityWithoutSemicolon + } + for j := maxLen; j > 1; j-- { + if x := entity[entityName[:j]]; x != 0 { + return dst + utf8.EncodeRune(b[dst:], x), src + j + 1 + } + } + } + + dst1, src1 = dst+i, src+i + copy(b[dst:dst1], b[src:src1]) + return dst1, src1 +} + +// unescape unescapes b's entities in-place, so that "a<b" becomes "a': + esc = ">" + case '"': + // """ is shorter than """. + esc = """ + case '\r': + esc = " " + default: + panic("unrecognized escape character") + } + s = s[i+1:] + if _, err := w.WriteString(esc); err != nil { + return err + } + i = strings.IndexAny(s, escapedChars) + } + _, err := w.WriteString(s) + return err +} + +// EscapeString escapes special characters like "<" to become "<". It +// escapes only five such characters: <, >, &, ' and ". +// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't +// always true. +func EscapeString(s string) string { + if strings.IndexAny(s, escapedChars) == -1 { + return s + } + var buf bytes.Buffer + escape(&buf, s) + return buf.String() +} + +// UnescapeString unescapes entities like "<" to become "<". It unescapes a +// larger range of entities than EscapeString escapes. For example, "á" +// unescapes to "á", as does "á" and "&xE1;". +// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't +// always true. +func UnescapeString(s string) string { + for _, c := range s { + if c == '&' { + return string(unescape([]byte(s), false)) + } + } + return s +} diff --git a/third_party/code.google.com/p/go.net/html/escape_test.go b/third_party/code.google.com/p/go.net/html/escape_test.go new file mode 100644 index 000000000..b405d4b4a --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/escape_test.go @@ -0,0 +1,97 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import "testing" + +type unescapeTest struct { + // A short description of the test case. + desc string + // The HTML text. + html string + // The unescaped text. + unescaped string +} + +var unescapeTests = []unescapeTest{ + // Handle no entities. + { + "copy", + "A\ttext\nstring", + "A\ttext\nstring", + }, + // Handle simple named entities. + { + "simple", + "& > <", + "& > <", + }, + // Handle hitting the end of the string. + { + "stringEnd", + "& &", + "& &", + }, + // Handle entities with two codepoints. + { + "multiCodepoint", + "text ⋛︀ blah", + "text \u22db\ufe00 blah", + }, + // Handle decimal numeric entities. + { + "decimalEntity", + "Delta = Δ ", + "Delta = Δ ", + }, + // Handle hexadecimal numeric entities. + { + "hexadecimalEntity", + "Lambda = λ = λ ", + "Lambda = λ = λ ", + }, + // Handle numeric early termination. + { + "numericEnds", + "&# &#x €43 © = ©f = ©", + "&# &#x €43 © = ©f = ©", + }, + // Handle numeric ISO-8859-1 entity replacements. + { + "numericReplacements", + "Footnote‡", + "Footnote‡", + }, +} + +func TestUnescape(t *testing.T) { + for _, tt := range unescapeTests { + unescaped := UnescapeString(tt.html) + if unescaped != tt.unescaped { + t.Errorf("TestUnescape %s: want %q, got %q", tt.desc, tt.unescaped, unescaped) + } + } +} + +func TestUnescapeEscape(t *testing.T) { + ss := []string{ + ``, + `abc def`, + `a & b`, + `a&b`, + `a & b`, + `"`, + `"`, + `"<&>"`, + `"<&>"`, + `3&5==1 && 0<1, "0<1", a+acute=á`, + `The special characters are: <, >, &, ' and "`, + } + for _, s := range ss { + if got := UnescapeString(EscapeString(s)); got != s { + t.Errorf("got %q want %q", got, s) + } + } +} diff --git a/third_party/code.google.com/p/go.net/html/example_test.go b/third_party/code.google.com/p/go.net/html/example_test.go new file mode 100644 index 000000000..47341f020 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/example_test.go @@ -0,0 +1,40 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This example demonstrates parsing HTML data and walking the resulting tree. +package html_test + +import ( + "fmt" + "log" + "strings" + + "code.google.com/p/go.net/html" +) + +func ExampleParse() { + s := `

Links:

` + doc, err := html.Parse(strings.NewReader(s)) + if err != nil { + log.Fatal(err) + } + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + for _, a := range n.Attr { + if a.Key == "href" { + fmt.Println(a.Val) + break + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(doc) + // Output: + // foo + // /bar/baz +} diff --git a/third_party/code.google.com/p/go.net/html/foreign.go b/third_party/code.google.com/p/go.net/html/foreign.go new file mode 100644 index 000000000..d3b384409 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/foreign.go @@ -0,0 +1,226 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "strings" +) + +func adjustAttributeNames(aa []Attribute, nameMap map[string]string) { + for i := range aa { + if newName, ok := nameMap[aa[i].Key]; ok { + aa[i].Key = newName + } + } +} + +func adjustForeignAttributes(aa []Attribute) { + for i, a := range aa { + if a.Key == "" || a.Key[0] != 'x' { + continue + } + switch a.Key { + case "xlink:actuate", "xlink:arcrole", "xlink:href", "xlink:role", "xlink:show", + "xlink:title", "xlink:type", "xml:base", "xml:lang", "xml:space", "xmlns:xlink": + j := strings.Index(a.Key, ":") + aa[i].Namespace = a.Key[:j] + aa[i].Key = a.Key[j+1:] + } + } +} + +func htmlIntegrationPoint(n *Node) bool { + if n.Type != ElementNode { + return false + } + switch n.Namespace { + case "math": + if n.Data == "annotation-xml" { + for _, a := range n.Attr { + if a.Key == "encoding" { + val := strings.ToLower(a.Val) + if val == "text/html" || val == "application/xhtml+xml" { + return true + } + } + } + } + case "svg": + switch n.Data { + case "desc", "foreignObject", "title": + return true + } + } + return false +} + +func mathMLTextIntegrationPoint(n *Node) bool { + if n.Namespace != "math" { + return false + } + switch n.Data { + case "mi", "mo", "mn", "ms", "mtext": + return true + } + return false +} + +// Section 12.2.5.5. +var breakout = map[string]bool{ + "b": true, + "big": true, + "blockquote": true, + "body": true, + "br": true, + "center": true, + "code": true, + "dd": true, + "div": true, + "dl": true, + "dt": true, + "em": true, + "embed": true, + "h1": true, + "h2": true, + "h3": true, + "h4": true, + "h5": true, + "h6": true, + "head": true, + "hr": true, + "i": true, + "img": true, + "li": true, + "listing": true, + "menu": true, + "meta": true, + "nobr": true, + "ol": true, + "p": true, + "pre": true, + "ruby": true, + "s": true, + "small": true, + "span": true, + "strong": true, + "strike": true, + "sub": true, + "sup": true, + "table": true, + "tt": true, + "u": true, + "ul": true, + "var": true, +} + +// Section 12.2.5.5. +var svgTagNameAdjustments = map[string]string{ + "altglyph": "altGlyph", + "altglyphdef": "altGlyphDef", + "altglyphitem": "altGlyphItem", + "animatecolor": "animateColor", + "animatemotion": "animateMotion", + "animatetransform": "animateTransform", + "clippath": "clipPath", + "feblend": "feBlend", + "fecolormatrix": "feColorMatrix", + "fecomponenttransfer": "feComponentTransfer", + "fecomposite": "feComposite", + "feconvolvematrix": "feConvolveMatrix", + "fediffuselighting": "feDiffuseLighting", + "fedisplacementmap": "feDisplacementMap", + "fedistantlight": "feDistantLight", + "feflood": "feFlood", + "fefunca": "feFuncA", + "fefuncb": "feFuncB", + "fefuncg": "feFuncG", + "fefuncr": "feFuncR", + "fegaussianblur": "feGaussianBlur", + "feimage": "feImage", + "femerge": "feMerge", + "femergenode": "feMergeNode", + "femorphology": "feMorphology", + "feoffset": "feOffset", + "fepointlight": "fePointLight", + "fespecularlighting": "feSpecularLighting", + "fespotlight": "feSpotLight", + "fetile": "feTile", + "feturbulence": "feTurbulence", + "foreignobject": "foreignObject", + "glyphref": "glyphRef", + "lineargradient": "linearGradient", + "radialgradient": "radialGradient", + "textpath": "textPath", +} + +// Section 12.2.5.1 +var mathMLAttributeAdjustments = map[string]string{ + "definitionurl": "definitionURL", +} + +var svgAttributeAdjustments = map[string]string{ + "attributename": "attributeName", + "attributetype": "attributeType", + "basefrequency": "baseFrequency", + "baseprofile": "baseProfile", + "calcmode": "calcMode", + "clippathunits": "clipPathUnits", + "contentscripttype": "contentScriptType", + "contentstyletype": "contentStyleType", + "diffuseconstant": "diffuseConstant", + "edgemode": "edgeMode", + "externalresourcesrequired": "externalResourcesRequired", + "filterres": "filterRes", + "filterunits": "filterUnits", + "glyphref": "glyphRef", + "gradienttransform": "gradientTransform", + "gradientunits": "gradientUnits", + "kernelmatrix": "kernelMatrix", + "kernelunitlength": "kernelUnitLength", + "keypoints": "keyPoints", + "keysplines": "keySplines", + "keytimes": "keyTimes", + "lengthadjust": "lengthAdjust", + "limitingconeangle": "limitingConeAngle", + "markerheight": "markerHeight", + "markerunits": "markerUnits", + "markerwidth": "markerWidth", + "maskcontentunits": "maskContentUnits", + "maskunits": "maskUnits", + "numoctaves": "numOctaves", + "pathlength": "pathLength", + "patterncontentunits": "patternContentUnits", + "patterntransform": "patternTransform", + "patternunits": "patternUnits", + "pointsatx": "pointsAtX", + "pointsaty": "pointsAtY", + "pointsatz": "pointsAtZ", + "preservealpha": "preserveAlpha", + "preserveaspectratio": "preserveAspectRatio", + "primitiveunits": "primitiveUnits", + "refx": "refX", + "refy": "refY", + "repeatcount": "repeatCount", + "repeatdur": "repeatDur", + "requiredextensions": "requiredExtensions", + "requiredfeatures": "requiredFeatures", + "specularconstant": "specularConstant", + "specularexponent": "specularExponent", + "spreadmethod": "spreadMethod", + "startoffset": "startOffset", + "stddeviation": "stdDeviation", + "stitchtiles": "stitchTiles", + "surfacescale": "surfaceScale", + "systemlanguage": "systemLanguage", + "tablevalues": "tableValues", + "targetx": "targetX", + "targety": "targetY", + "textlength": "textLength", + "viewbox": "viewBox", + "viewtarget": "viewTarget", + "xchannelselector": "xChannelSelector", + "ychannelselector": "yChannelSelector", + "zoomandpan": "zoomAndPan", +} diff --git a/third_party/code.google.com/p/go.net/html/node.go b/third_party/code.google.com/p/go.net/html/node.go new file mode 100644 index 000000000..e7b4e50a0 --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/node.go @@ -0,0 +1,193 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "code.google.com/p/go.net/html/atom" +) + +// A NodeType is the type of a Node. +type NodeType uint32 + +const ( + ErrorNode NodeType = iota + TextNode + DocumentNode + ElementNode + CommentNode + DoctypeNode + scopeMarkerNode +) + +// Section 12.2.3.3 says "scope markers are inserted when entering applet +// elements, buttons, object elements, marquees, table cells, and table +// captions, and are used to prevent formatting from 'leaking'". +var scopeMarker = Node{Type: scopeMarkerNode} + +// A Node consists of a NodeType and some Data (tag name for element nodes, +// content for text) and are part of a tree of Nodes. Element nodes may also +// have a Namespace and contain a slice of Attributes. Data is unescaped, so +// that it looks like "a 0 { + return (*s)[i-1] + } + return nil +} + +// index returns the index of the top-most occurrence of n in the stack, or -1 +// if n is not present. +func (s *nodeStack) index(n *Node) int { + for i := len(*s) - 1; i >= 0; i-- { + if (*s)[i] == n { + return i + } + } + return -1 +} + +// insert inserts a node at the given index. +func (s *nodeStack) insert(i int, n *Node) { + (*s) = append(*s, nil) + copy((*s)[i+1:], (*s)[i:]) + (*s)[i] = n +} + +// remove removes a node from the stack. It is a no-op if n is not present. +func (s *nodeStack) remove(n *Node) { + i := s.index(n) + if i == -1 { + return + } + copy((*s)[i:], (*s)[i+1:]) + j := len(*s) - 1 + (*s)[j] = nil + *s = (*s)[:j] +} diff --git a/third_party/code.google.com/p/go.net/html/node_test.go b/third_party/code.google.com/p/go.net/html/node_test.go new file mode 100644 index 000000000..471102f3a --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/node_test.go @@ -0,0 +1,146 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "fmt" +) + +// checkTreeConsistency checks that a node and its descendants are all +// consistent in their parent/child/sibling relationships. +func checkTreeConsistency(n *Node) error { + return checkTreeConsistency1(n, 0) +} + +func checkTreeConsistency1(n *Node, depth int) error { + if depth == 1e4 { + return fmt.Errorf("html: tree looks like it contains a cycle") + } + if err := checkNodeConsistency(n); err != nil { + return err + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if err := checkTreeConsistency1(c, depth+1); err != nil { + return err + } + } + return nil +} + +// checkNodeConsistency checks that a node's parent/child/sibling relationships +// are consistent. +func checkNodeConsistency(n *Node) error { + if n == nil { + return nil + } + + nParent := 0 + for p := n.Parent; p != nil; p = p.Parent { + nParent++ + if nParent == 1e4 { + return fmt.Errorf("html: parent list looks like an infinite loop") + } + } + + nForward := 0 + for c := n.FirstChild; c != nil; c = c.NextSibling { + nForward++ + if nForward == 1e6 { + return fmt.Errorf("html: forward list of children looks like an infinite loop") + } + if c.Parent != n { + return fmt.Errorf("html: inconsistent child/parent relationship") + } + } + + nBackward := 0 + for c := n.LastChild; c != nil; c = c.PrevSibling { + nBackward++ + if nBackward == 1e6 { + return fmt.Errorf("html: backward list of children looks like an infinite loop") + } + if c.Parent != n { + return fmt.Errorf("html: inconsistent child/parent relationship") + } + } + + if n.Parent != nil { + if n.Parent == n { + return fmt.Errorf("html: inconsistent parent relationship") + } + if n.Parent == n.FirstChild { + return fmt.Errorf("html: inconsistent parent/first relationship") + } + if n.Parent == n.LastChild { + return fmt.Errorf("html: inconsistent parent/last relationship") + } + if n.Parent == n.PrevSibling { + return fmt.Errorf("html: inconsistent parent/prev relationship") + } + if n.Parent == n.NextSibling { + return fmt.Errorf("html: inconsistent parent/next relationship") + } + + parentHasNAsAChild := false + for c := n.Parent.FirstChild; c != nil; c = c.NextSibling { + if c == n { + parentHasNAsAChild = true + break + } + } + if !parentHasNAsAChild { + return fmt.Errorf("html: inconsistent parent/child relationship") + } + } + + if n.PrevSibling != nil && n.PrevSibling.NextSibling != n { + return fmt.Errorf("html: inconsistent prev/next relationship") + } + if n.NextSibling != nil && n.NextSibling.PrevSibling != n { + return fmt.Errorf("html: inconsistent next/prev relationship") + } + + if (n.FirstChild == nil) != (n.LastChild == nil) { + return fmt.Errorf("html: inconsistent first/last relationship") + } + if n.FirstChild != nil && n.FirstChild == n.LastChild { + // We have a sole child. + if n.FirstChild.PrevSibling != nil || n.FirstChild.NextSibling != nil { + return fmt.Errorf("html: inconsistent sole child's sibling relationship") + } + } + + seen := map[*Node]bool{} + + var last *Node + for c := n.FirstChild; c != nil; c = c.NextSibling { + if seen[c] { + return fmt.Errorf("html: inconsistent repeated child") + } + seen[c] = true + last = c + } + if last != n.LastChild { + return fmt.Errorf("html: inconsistent last relationship") + } + + var first *Node + for c := n.LastChild; c != nil; c = c.PrevSibling { + if !seen[c] { + return fmt.Errorf("html: inconsistent missing child") + } + delete(seen, c) + first = c + } + if first != n.FirstChild { + return fmt.Errorf("html: inconsistent first relationship") + } + + if len(seen) != 0 { + return fmt.Errorf("html: inconsistent forwards/backwards child list") + } + + return nil +} diff --git a/third_party/code.google.com/p/go.net/html/parse.go b/third_party/code.google.com/p/go.net/html/parse.go new file mode 100644 index 000000000..bf99ec6ab --- /dev/null +++ b/third_party/code.google.com/p/go.net/html/parse.go @@ -0,0 +1,2092 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "errors" + "fmt" + "io" + "strings" + + a "code.google.com/p/go.net/html/atom" +) + +// A parser implements the HTML5 parsing algorithm: +// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#tree-construction +type parser struct { + // tokenizer provides the tokens for the parser. + tokenizer *Tokenizer + // tok is the most recently read token. + tok Token + // Self-closing tags like
are treated as start tags, except that + // hasSelfClosingToken is set while they are being processed. + hasSelfClosingToken bool + // doc is the document root element. + doc *Node + // The stack of open elements (section 12.2.3.2) and active formatting + // elements (section 12.2.3.3). + oe, afe nodeStack + // Element pointers (section 12.2.3.4). + head, form *Node + // Other parsing state flags (section 12.2.3.5). + scripting, framesetOK bool + // im is the current insertion mode. + im insertionMode + // originalIM is the insertion mode to go back to after completing a text + // or inTableText insertion mode. + originalIM insertionMode + // fosterParenting is whether new elements should be inserted according to + // the foster parenting rules (section 12.2.5.3). + fosterParenting bool + // quirks is whether the parser is operating in "quirks mode." + quirks bool + // fragment is whether the parser is parsing an HTML fragment. + fragment bool + // context is the context element when parsing an HTML fragment + // (section 12.4). + context *Node +} + +func (p *parser) top() *Node { + if n := p.oe.top(); n != nil { + return n + } + return p.doc +} + +// Stop tags for use in popUntil. These come from section 12.2.3.2. +var ( + defaultScopeStopTags = map[string][]a.Atom{ + "": {a.Applet, a.Caption, a.Html, a.Table, a.Td, a.Th, a.Marquee, a.Object}, + "math": {a.AnnotationXml, a.Mi, a.Mn, a.Mo, a.Ms, a.Mtext}, + "svg": {a.Desc, a.ForeignObject, a.Title}, + } +) + +type scope int + +const ( + defaultScope scope = iota + listItemScope + buttonScope + tableScope + tableRowScope + tableBodyScope + selectScope +) + +// popUntil pops the stack of open elements at the highest element whose tag +// is in matchTags, provided there is no higher element in the scope's stop +// tags (as defined in section 12.2.3.2). It returns whether or not there was +// such an element. If there was not, popUntil leaves the stack unchanged. +// +// For example, the set of stop tags for table scope is: "html", "table". If +// the stack was: +// ["html", "body", "font", "table", "b", "i", "u"] +// then popUntil(tableScope, "font") would return false, but +// popUntil(tableScope, "i") would return true and the stack would become: +// ["html", "body", "font", "table", "b"] +// +// If an element's tag is in both the stop tags and matchTags, then the stack +// will be popped and the function returns true (provided, of course, there was +// no higher element in the stack that was also in the stop tags). For example, +// popUntil(tableScope, "table") returns true and leaves: +// ["html", "body", "font"] +func (p *parser) popUntil(s scope, matchTags ...a.Atom) bool { + if i := p.indexOfElementInScope(s, matchTags...); i != -1 { + p.oe = p.oe[:i] + return true + } + return false +} + +// indexOfElementInScope returns the index in p.oe of the highest element whose +// tag is in matchTags that is in scope. If no matching element is in scope, it +// returns -1. +func (p *parser) indexOfElementInScope(s scope, matchTags ...a.Atom) int { + for i := len(p.oe) - 1; i >= 0; i-- { + tagAtom := p.oe[i].DataAtom + if p.oe[i].Namespace == "" { + for _, t := range matchTags { + if t == tagAtom { + return i + } + } + switch s { + case defaultScope: + // No-op. + case listItemScope: + if tagAtom == a.Ol || tagAtom == a.Ul { + return -1 + } + case buttonScope: + if tagAtom == a.Button { + return -1 + } + case tableScope: + if tagAtom == a.Html || tagAtom == a.Table { + return -1 + } + case selectScope: + if tagAtom != a.Optgroup && tagAtom != a.Option { + return -1 + } + default: + panic("unreachable") + } + } + switch s { + case defaultScope, listItemScope, buttonScope: + for _, t := range defaultScopeStopTags[p.oe[i].Namespace] { + if t == tagAtom { + return -1 + } + } + } + } + return -1 +} + +// elementInScope is like popUntil, except that it doesn't modify the stack of +// open elements. +func (p *parser) elementInScope(s scope, matchTags ...a.Atom) bool { + return p.indexOfElementInScope(s, matchTags...) != -1 +} + +// clearStackToContext pops elements off the stack of open elements until a +// scope-defined element is found. +func (p *parser) clearStackToContext(s scope) { + for i := len(p.oe) - 1; i >= 0; i-- { + tagAtom := p.oe[i].DataAtom + switch s { + case tableScope: + if tagAtom == a.Html || tagAtom == a.Table { + p.oe = p.oe[:i+1] + return + } + case tableRowScope: + if tagAtom == a.Html || tagAtom == a.Tr { + p.oe = p.oe[:i+1] + return + } + case tableBodyScope: + if tagAtom == a.Html || tagAtom == a.Tbody || tagAtom == a.Tfoot || tagAtom == a.Thead { + p.oe = p.oe[:i+1] + return + } + default: + panic("unreachable") + } + } +} + +// generateImpliedEndTags pops nodes off the stack of open elements as long as +// the top node has a tag name of dd, dt, li, option, optgroup, p, rp, or rt. +// If exceptions are specified, nodes with that name will not be popped off. +func (p *parser) generateImpliedEndTags(exceptions ...string) { + var i int +loop: + for i = len(p.oe) - 1; i >= 0; i-- { + n := p.oe[i] + if n.Type == ElementNode { + switch n.DataAtom { + case a.Dd, a.Dt, a.Li, a.Option, a.Optgroup, a.P, a.Rp, a.Rt: + for _, except := range exceptions { + if n.Data == except { + break loop + } + } + continue + } + } + break + } + + p.oe = p.oe[:i+1] +} + +// addChild adds a child node n to the top element, and pushes n onto the stack +// of open elements if it is an element node. +func (p *parser) addChild(n *Node) { + if p.shouldFosterParent() { + p.fosterParent(n) + } else { + p.top().AppendChild(n) + } + + if n.Type == ElementNode { + p.oe = append(p.oe, n) + } +} + +// shouldFosterParent returns whether the next node to be added should be +// foster parented. +func (p *parser) shouldFosterParent() bool { + if p.fosterParenting { + switch p.top().DataAtom { + case a.Table, a.Tbody, a.Tfoot, a.Thead, a.Tr: + return true + } + } + return false +} + +// fosterParent adds a child node according to the foster parenting rules. +// Section 12.2.5.3, "foster parenting". +func (p *parser) fosterParent(n *Node) { + var table, parent, prev *Node + var i int + for i = len(p.oe) - 1; i >= 0; i-- { + if p.oe[i].DataAtom == a.Table { + table = p.oe[i] + break + } + } + + if table == nil { + // The foster parent is the html element. + parent = p.oe[0] + } else { + parent = table.Parent + } + if parent == nil { + parent = p.oe[i-1] + } + + if table != nil { + prev = table.PrevSibling + } else { + prev = parent.LastChild + } + if prev != nil && prev.Type == TextNode && n.Type == TextNode { + prev.Data += n.Data + return + } + + parent.InsertBefore(n, table) +} + +// addText adds text to the preceding node if it is a text node, or else it +// calls addChild with a new text node. +func (p *parser) addText(text string) { + if text == "" { + return + } + + if p.shouldFosterParent() { + p.fosterParent(&Node{ + Type: TextNode, + Data: text, + }) + return + } + + t := p.top() + if n := t.LastChild; n != nil && n.Type == TextNode { + n.Data += text + return + } + p.addChild(&Node{ + Type: TextNode, + Data: text, + }) +} + +// addElement adds a child element based on the current token. +func (p *parser) addElement() { + p.addChild(&Node{ + Type: ElementNode, + DataAtom: p.tok.DataAtom, + Data: p.tok.Data, + Attr: p.tok.Attr, + }) +} + +// Section 12.2.3.3. +func (p *parser) addFormattingElement() { + tagAtom, attr := p.tok.DataAtom, p.tok.Attr + p.addElement() + + // Implement the Noah's Ark clause, but with three per family instead of two. + identicalElements := 0 +findIdenticalElements: + for i := len(p.afe) - 1; i >= 0; i-- { + n := p.afe[i] + if n.Type == scopeMarkerNode { + break + } + if n.Type != ElementNode { + continue + } + if n.Namespace != "" { + continue + } + if n.DataAtom != tagAtom { + continue + } + if len(n.Attr) != len(attr) { + continue + } + compareAttributes: + for _, t0 := range n.Attr { + for _, t1 := range attr { + if t0.Key == t1.Key && t0.Namespace == t1.Namespace && t0.Val == t1.Val { + // Found a match for this attribute, continue with the next attribute. + continue compareAttributes + } + } + // If we get here, there is no attribute that matches a. + // Therefore the element is not identical to the new one. + continue findIdenticalElements + } + + identicalElements++ + if identicalElements >= 3 { + p.afe.remove(n) + } + } + + p.afe = append(p.afe, p.top()) +} + +// Section 12.2.3.3. +func (p *parser) clearActiveFormattingElements() { + for { + n := p.afe.pop() + if len(p.afe) == 0 || n.Type == scopeMarkerNode { + return + } + } +} + +// Section 12.2.3.3. +func (p *parser) reconstructActiveFormattingElements() { + n := p.afe.top() + if n == nil { + return + } + if n.Type == scopeMarkerNode || p.oe.index(n) != -1 { + return + } + i := len(p.afe) - 1 + for n.Type != scopeMarkerNode && p.oe.index(n) == -1 { + if i == 0 { + i = -1 + break + } + i-- + n = p.afe[i] + } + for { + i++ + clone := p.afe[i].clone() + p.addChild(clone) + p.afe[i] = clone + if i == len(p.afe)-1 { + break + } + } +} + +// Section 12.2.4. +func (p *parser) acknowledgeSelfClosingTag() { + p.hasSelfClosingToken = false +} + +// An insertion mode (section 12.2.3.1) is the state transition function from +// a particular state in the HTML5 parser's state machine. It updates the +// parser's fields depending on parser.tok (where ErrorToken means EOF). +// It returns whether the token was consumed. +type insertionMode func(*parser) bool + +// setOriginalIM sets the insertion mode to return to after completing a text or +// inTableText insertion mode. +// Section 12.2.3.1, "using the rules for". +func (p *parser) setOriginalIM() { + if p.originalIM != nil { + panic("html: bad parser state: originalIM was set twice") + } + p.originalIM = p.im +} + +// Section 12.2.3.1, "reset the insertion mode". +func (p *parser) resetInsertionMode() { + for i := len(p.oe) - 1; i >= 0; i-- { + n := p.oe[i] + if i == 0 && p.context != nil { + n = p.context + } + + switch n.DataAtom { + case a.Select: + p.im = inSelectIM + case a.Td, a.Th: + p.im = inCellIM + case a.Tr: + p.im = inRowIM + case a.Tbody, a.Thead, a.Tfoot: + p.im = inTableBodyIM + case a.Caption: + p.im = inCaptionIM + case a.Colgroup: + p.im = inColumnGroupIM + case a.Table: + p.im = inTableIM + case a.Head: + p.im = inBodyIM + case a.Body: + p.im = inBodyIM + case a.Frameset: + p.im = inFramesetIM + case a.Html: + p.im = beforeHeadIM + default: + continue + } + return + } + p.im = inBodyIM +} + +const whitespace = " \t\r\n\f" + +// Section 12.2.5.4.1. +func initialIM(p *parser) bool { + switch p.tok.Type { + case TextToken: + p.tok.Data = strings.TrimLeft(p.tok.Data, whitespace) + if len(p.tok.Data) == 0 { + // It was all whitespace, so ignore it. + return true + } + case CommentToken: + p.doc.AppendChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return true + case DoctypeToken: + n, quirks := parseDoctype(p.tok.Data) + p.doc.AppendChild(n) + p.quirks = quirks + p.im = beforeHTMLIM + return true + } + p.quirks = true + p.im = beforeHTMLIM + return false +} + +// Section 12.2.5.4.2. +func beforeHTMLIM(p *parser) bool { + switch p.tok.Type { + case DoctypeToken: + // Ignore the token. + return true + case TextToken: + p.tok.Data = strings.TrimLeft(p.tok.Data, whitespace) + if len(p.tok.Data) == 0 { + // It was all whitespace, so ignore it. + return true + } + case StartTagToken: + if p.tok.DataAtom == a.Html { + p.addElement() + p.im = beforeHeadIM + return true + } + case EndTagToken: + switch p.tok.DataAtom { + case a.Head, a.Body, a.Html, a.Br: + p.parseImpliedToken(StartTagToken, a.Html, a.Html.String()) + return false + default: + // Ignore the token. + return true + } + case CommentToken: + p.doc.AppendChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return true + } + p.parseImpliedToken(StartTagToken, a.Html, a.Html.String()) + return false +} + +// Section 12.2.5.4.3. +func beforeHeadIM(p *parser) bool { + switch p.tok.Type { + case TextToken: + p.tok.Data = strings.TrimLeft(p.tok.Data, whitespace) + if len(p.tok.Data) == 0 { + // It was all whitespace, so ignore it. + return true + } + case StartTagToken: + switch p.tok.DataAtom { + case a.Head: + p.addElement() + p.head = p.top() + p.im = inHeadIM + return true + case a.Html: + return inBodyIM(p) + } + case EndTagToken: + switch p.tok.DataAtom { + case a.Head, a.Body, a.Html, a.Br: + p.parseImpliedToken(StartTagToken, a.Head, a.Head.String()) + return false + default: + // Ignore the token. + return true + } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return true + case DoctypeToken: + // Ignore the token. + return true + } + + p.parseImpliedToken(StartTagToken, a.Head, a.Head.String()) + return false +} + +// Section 12.2.5.4.4. +func inHeadIM(p *parser) bool { + switch p.tok.Type { + case TextToken: + s := strings.TrimLeft(p.tok.Data, whitespace) + if len(s) < len(p.tok.Data) { + // Add the initial whitespace to the current node. + p.addText(p.tok.Data[:len(p.tok.Data)-len(s)]) + if s == "" { + return true + } + p.tok.Data = s + } + case StartTagToken: + switch p.tok.DataAtom { + case a.Html: + return inBodyIM(p) + case a.Base, a.Basefont, a.Bgsound, a.Command, a.Link, a.Meta: + p.addElement() + p.oe.pop() + p.acknowledgeSelfClosingTag() + return true + case a.Script, a.Title, a.Noscript, a.Noframes, a.Style: + p.addElement() + p.setOriginalIM() + p.im = textIM + return true + case a.Head: + // Ignore the token. + return true + } + case EndTagToken: + switch p.tok.DataAtom { + case a.Head: + n := p.oe.pop() + if n.DataAtom != a.Head { + panic("html: bad parser state: element not found, in the in-head insertion mode") + } + p.im = afterHeadIM + return true + case a.Body, a.Html, a.Br: + p.parseImpliedToken(EndTagToken, a.Head, a.Head.String()) + return false + default: + // Ignore the token. + return true + } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return true + case DoctypeToken: + // Ignore the token. + return true + } + + p.parseImpliedToken(EndTagToken, a.Head, a.Head.String()) + return false +} + +// Section 12.2.5.4.6. +func afterHeadIM(p *parser) bool { + switch p.tok.Type { + case TextToken: + s := strings.TrimLeft(p.tok.Data, whitespace) + if len(s) < len(p.tok.Data) { + // Add the initial whitespace to the current node. + p.addText(p.tok.Data[:len(p.tok.Data)-len(s)]) + if s == "" { + return true + } + p.tok.Data = s + } + case StartTagToken: + switch p.tok.DataAtom { + case a.Html: + return inBodyIM(p) + case a.Body: + p.addElement() + p.framesetOK = false + p.im = inBodyIM + return true + case a.Frameset: + p.addElement() + p.im = inFramesetIM + return true + case a.Base, a.Basefont, a.Bgsound, a.Link, a.Meta, a.Noframes, a.Script, a.Style, a.Title: + p.oe = append(p.oe, p.head) + defer p.oe.remove(p.head) + return inHeadIM(p) + case a.Head: + // Ignore the token. + return true + } + case EndTagToken: + switch p.tok.DataAtom { + case a.Body, a.Html, a.Br: + // Drop down to creating an implied tag. + default: + // Ignore the token. + return true + } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return true + case DoctypeToken: + // Ignore the token. + return true + } + + p.parseImpliedToken(StartTagToken, a.Body, a.Body.String()) + p.framesetOK = true + return false +} + +// copyAttributes copies attributes of src not found on dst to dst. +func copyAttributes(dst *Node, src Token) { + if len(src.Attr) == 0 { + return + } + attr := map[string]string{} + for _, t := range dst.Attr { + attr[t.Key] = t.Val + } + for _, t := range src.Attr { + if _, ok := attr[t.Key]; !ok { + dst.Attr = append(dst.Attr, t) + attr[t.Key] = t.Val + } + } +} + +// Section 12.2.5.4.7. +func inBodyIM(p *parser) bool { + switch p.tok.Type { + case TextToken: + d := p.tok.Data + switch n := p.oe.top(); n.DataAtom { + case a.Pre, a.Listing: + if n.FirstChild == nil { + // Ignore a newline at the start of a 
 block.
+				if d != "" && d[0] == '\r' {
+					d = d[1:]
+				}
+				if d != "" && d[0] == '\n' {
+					d = d[1:]
+				}
+			}
+		}
+		d = strings.Replace(d, "\x00", "", -1)
+		if d == "" {
+			return true
+		}
+		p.reconstructActiveFormattingElements()
+		p.addText(d)
+		if p.framesetOK && strings.TrimLeft(d, whitespace) != "" {
+			// There were non-whitespace characters inserted.
+			p.framesetOK = false
+		}
+	case StartTagToken:
+		switch p.tok.DataAtom {
+		case a.Html:
+			copyAttributes(p.oe[0], p.tok)
+		case a.Base, a.Basefont, a.Bgsound, a.Command, a.Link, a.Meta, a.Noframes, a.Script, a.Style, a.Title:
+			return inHeadIM(p)
+		case a.Body:
+			if len(p.oe) >= 2 {
+				body := p.oe[1]
+				if body.Type == ElementNode && body.DataAtom == a.Body {
+					p.framesetOK = false
+					copyAttributes(body, p.tok)
+				}
+			}
+		case a.Frameset:
+			if !p.framesetOK || len(p.oe) < 2 || p.oe[1].DataAtom != a.Body {
+				// Ignore the token.
+				return true
+			}
+			body := p.oe[1]
+			if body.Parent != nil {
+				body.Parent.RemoveChild(body)
+			}
+			p.oe = p.oe[:1]
+			p.addElement()
+			p.im = inFramesetIM
+			return true
+		case a.Address, a.Article, a.Aside, a.Blockquote, a.Center, a.Details, a.Dir, a.Div, a.Dl, a.Fieldset, a.Figcaption, a.Figure, a.Footer, a.Header, a.Hgroup, a.Menu, a.Nav, a.Ol, a.P, a.Section, a.Summary, a.Ul:
+			p.popUntil(buttonScope, a.P)
+			p.addElement()
+		case a.H1, a.H2, a.H3, a.H4, a.H5, a.H6:
+			p.popUntil(buttonScope, a.P)
+			switch n := p.top(); n.DataAtom {
+			case a.H1, a.H2, a.H3, a.H4, a.H5, a.H6:
+				p.oe.pop()
+			}
+			p.addElement()
+		case a.Pre, a.Listing:
+			p.popUntil(buttonScope, a.P)
+			p.addElement()
+			// The newline, if any, will be dealt with by the TextToken case.
+			p.framesetOK = false
+		case a.Form:
+			if p.form == nil {
+				p.popUntil(buttonScope, a.P)
+				p.addElement()
+				p.form = p.top()
+			}
+		case a.Li:
+			p.framesetOK = false
+			for i := len(p.oe) - 1; i >= 0; i-- {
+				node := p.oe[i]
+				switch node.DataAtom {
+				case a.Li:
+					p.oe = p.oe[:i]
+				case a.Address, a.Div, a.P:
+					continue
+				default:
+					if !isSpecialElement(node) {
+						continue
+					}
+				}
+				break
+			}
+			p.popUntil(buttonScope, a.P)
+			p.addElement()
+		case a.Dd, a.Dt:
+			p.framesetOK = false
+			for i := len(p.oe) - 1; i >= 0; i-- {
+				node := p.oe[i]
+				switch node.DataAtom {
+				case a.Dd, a.Dt:
+					p.oe = p.oe[:i]
+				case a.Address, a.Div, a.P:
+					continue
+				default:
+					if !isSpecialElement(node) {
+						continue
+					}
+				}
+				break
+			}
+			p.popUntil(buttonScope, a.P)
+			p.addElement()
+		case a.Plaintext:
+			p.popUntil(buttonScope, a.P)
+			p.addElement()
+		case a.Button:
+			p.popUntil(defaultScope, a.Button)
+			p.reconstructActiveFormattingElements()
+			p.addElement()
+			p.framesetOK = false
+		case a.A:
+			for i := len(p.afe) - 1; i >= 0 && p.afe[i].Type != scopeMarkerNode; i-- {
+				if n := p.afe[i]; n.Type == ElementNode && n.DataAtom == a.A {
+					p.inBodyEndTagFormatting(a.A)
+					p.oe.remove(n)
+					p.afe.remove(n)
+					break
+				}
+			}
+			p.reconstructActiveFormattingElements()
+			p.addFormattingElement()
+		case a.B, a.Big, a.Code, a.Em, a.Font, a.I, a.S, a.Small, a.Strike, a.Strong, a.Tt, a.U:
+			p.reconstructActiveFormattingElements()
+			p.addFormattingElement()
+		case a.Nobr:
+			p.reconstructActiveFormattingElements()
+			if p.elementInScope(defaultScope, a.Nobr) {
+				p.inBodyEndTagFormatting(a.Nobr)
+				p.reconstructActiveFormattingElements()
+			}
+			p.addFormattingElement()
+		case a.Applet, a.Marquee, a.Object:
+			p.reconstructActiveFormattingElements()
+			p.addElement()
+			p.afe = append(p.afe, &scopeMarker)
+			p.framesetOK = false
+		case a.Table:
+			if !p.quirks {
+				p.popUntil(buttonScope, a.P)
+			}
+			p.addElement()
+			p.framesetOK = false
+			p.im = inTableIM
+			return true
+		case a.Area, a.Br, a.Embed, a.Img, a.Input, a.Keygen, a.Wbr:
+			p.reconstructActiveFormattingElements()
+			p.addElement()
+			p.oe.pop()
+			p.acknowledgeSelfClosingTag()
+			if p.tok.DataAtom == a.Input {
+				for _, t := range p.tok.Attr {
+					if t.Key == "type" {
+						if strings.ToLower(t.Val) == "hidden" {
+							// Skip setting framesetOK = false
+							return true
+						}
+					}
+				}
+			}
+			p.framesetOK = false
+		case a.Param, a.Source, a.Track:
+			p.addElement()
+			p.oe.pop()
+			p.acknowledgeSelfClosingTag()
+		case a.Hr:
+			p.popUntil(buttonScope, a.P)
+			p.addElement()
+			p.oe.pop()
+			p.acknowledgeSelfClosingTag()
+			p.framesetOK = false
+		case a.Image:
+			p.tok.DataAtom = a.Img
+			p.tok.Data = a.Img.String()
+			return false
+		case a.Isindex:
+			if p.form != nil {
+				// Ignore the token.
+				return true
+			}
+			action := ""
+			prompt := "This is a searchable index. Enter search keywords: "
+			attr := []Attribute{{Key: "name", Val: "isindex"}}
+			for _, t := range p.tok.Attr {
+				switch t.Key {
+				case "action":
+					action = t.Val
+				case "name":
+					// Ignore the attribute.
+				case "prompt":
+					prompt = t.Val
+				default:
+					attr = append(attr, t)
+				}
+			}
+			p.acknowledgeSelfClosingTag()
+			p.popUntil(buttonScope, a.P)
+			p.parseImpliedToken(StartTagToken, a.Form, a.Form.String())
+			if action != "" {
+				p.form.Attr = []Attribute{{Key: "action", Val: action}}
+			}
+			p.parseImpliedToken(StartTagToken, a.Hr, a.Hr.String())
+			p.parseImpliedToken(StartTagToken, a.Label, a.Label.String())
+			p.addText(prompt)
+			p.addChild(&Node{
+				Type:     ElementNode,
+				DataAtom: a.Input,
+				Data:     a.Input.String(),
+				Attr:     attr,
+			})
+			p.oe.pop()
+			p.parseImpliedToken(EndTagToken, a.Label, a.Label.String())
+			p.parseImpliedToken(StartTagToken, a.Hr, a.Hr.String())
+			p.parseImpliedToken(EndTagToken, a.Form, a.Form.String())
+		case a.Textarea:
+			p.addElement()
+			p.setOriginalIM()
+			p.framesetOK = false
+			p.im = textIM
+		case a.Xmp:
+			p.popUntil(buttonScope, a.P)
+			p.reconstructActiveFormattingElements()
+			p.framesetOK = false
+			p.addElement()
+			p.setOriginalIM()
+			p.im = textIM
+		case a.Iframe:
+			p.framesetOK = false
+			p.addElement()
+			p.setOriginalIM()
+			p.im = textIM
+		case a.Noembed, a.Noscript:
+			p.addElement()
+			p.setOriginalIM()
+			p.im = textIM
+		case a.Select:
+			p.reconstructActiveFormattingElements()
+			p.addElement()
+			p.framesetOK = false
+			p.im = inSelectIM
+			return true
+		case a.Optgroup, a.Option:
+			if p.top().DataAtom == a.Option {
+				p.oe.pop()
+			}
+			p.reconstructActiveFormattingElements()
+			p.addElement()
+		case a.Rp, a.Rt:
+			if p.elementInScope(defaultScope, a.Ruby) {
+				p.generateImpliedEndTags()
+			}
+			p.addElement()
+		case a.Math, a.Svg:
+			p.reconstructActiveFormattingElements()
+			if p.tok.DataAtom == a.Math {
+				adjustAttributeNames(p.tok.Attr, mathMLAttributeAdjustments)
+			} else {
+				adjustAttributeNames(p.tok.Attr, svgAttributeAdjustments)
+			}
+			adjustForeignAttributes(p.tok.Attr)
+			p.addElement()
+			p.top().Namespace = p.tok.Data
+			if p.hasSelfClosingToken {
+				p.oe.pop()
+				p.acknowledgeSelfClosingTag()
+			}
+			return true
+		case a.Caption, a.Col, a.Colgroup, a.Frame, a.Head, a.Tbody, a.Td, a.Tfoot, a.Th, a.Thead, a.Tr:
+			// Ignore the token.
+		default:
+			p.reconstructActiveFormattingElements()
+			p.addElement()
+		}
+	case EndTagToken:
+		switch p.tok.DataAtom {
+		case a.Body:
+			if p.elementInScope(defaultScope, a.Body) {
+				p.im = afterBodyIM
+			}
+		case a.Html:
+			if p.elementInScope(defaultScope, a.Body) {
+				p.parseImpliedToken(EndTagToken, a.Body, a.Body.String())
+				return false
+			}
+			return true
+		case a.Address, a.Article, a.Aside, a.Blockquote, a.Button, a.Center, a.Details, a.Dir, a.Div, a.Dl, a.Fieldset, a.Figcaption, a.Figure, a.Footer, a.Header, a.Hgroup, a.Listing, a.Menu, a.Nav, a.Ol, a.Pre, a.Section, a.Summary, a.Ul:
+			p.popUntil(defaultScope, p.tok.DataAtom)
+		case a.Form:
+			node := p.form
+			p.form = nil
+			i := p.indexOfElementInScope(defaultScope, a.Form)
+			if node == nil || i == -1 || p.oe[i] != node {
+				// Ignore the token.
+				return true
+			}
+			p.generateImpliedEndTags()
+			p.oe.remove(node)
+		case a.P:
+			if !p.elementInScope(buttonScope, a.P) {
+				p.parseImpliedToken(StartTagToken, a.P, a.P.String())
+			}
+			p.popUntil(buttonScope, a.P)
+		case a.Li:
+			p.popUntil(listItemScope, a.Li)
+		case a.Dd, a.Dt:
+			p.popUntil(defaultScope, p.tok.DataAtom)
+		case a.H1, a.H2, a.H3, a.H4, a.H5, a.H6:
+			p.popUntil(defaultScope, a.H1, a.H2, a.H3, a.H4, a.H5, a.H6)
+		case a.A, a.B, a.Big, a.Code, a.Em, a.Font, a.I, a.Nobr, a.S, a.Small, a.Strike, a.Strong, a.Tt, a.U:
+			p.inBodyEndTagFormatting(p.tok.DataAtom)
+		case a.Applet, a.Marquee, a.Object:
+			if p.popUntil(defaultScope, p.tok.DataAtom) {
+				p.clearActiveFormattingElements()
+			}
+		case a.Br:
+			p.tok.Type = StartTagToken
+			return false
+		default:
+			p.inBodyEndTagOther(p.tok.DataAtom)
+		}
+	case CommentToken:
+		p.addChild(&Node{
+			Type: CommentNode,
+			Data: p.tok.Data,
+		})
+	}
+
+	return true
+}
+
+func (p *parser) inBodyEndTagFormatting(tagAtom a.Atom) {
+	// This is the "adoption agency" algorithm, described at
+	// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#adoptionAgency
+
+	// TODO: this is a fairly literal line-by-line translation of that algorithm.
+	// Once the code successfully parses the comprehensive test suite, we should
+	// refactor this code to be more idiomatic.
+
+	// Steps 1-3. The outer loop.
+	for i := 0; i < 8; i++ {
+		// Step 4. Find the formatting element.
+		var formattingElement *Node
+		for j := len(p.afe) - 1; j >= 0; j-- {
+			if p.afe[j].Type == scopeMarkerNode {
+				break
+			}
+			if p.afe[j].DataAtom == tagAtom {
+				formattingElement = p.afe[j]
+				break
+			}
+		}
+		if formattingElement == nil {
+			p.inBodyEndTagOther(tagAtom)
+			return
+		}
+		feIndex := p.oe.index(formattingElement)
+		if feIndex == -1 {
+			p.afe.remove(formattingElement)
+			return
+		}
+		if !p.elementInScope(defaultScope, tagAtom) {
+			// Ignore the tag.
+			return
+		}
+
+		// Steps 5-6. Find the furthest block.
+		var furthestBlock *Node
+		for _, e := range p.oe[feIndex:] {
+			if isSpecialElement(e) {
+				furthestBlock = e
+				break
+			}
+		}
+		if furthestBlock == nil {
+			e := p.oe.pop()
+			for e != formattingElement {
+				e = p.oe.pop()
+			}
+			p.afe.remove(e)
+			return
+		}
+
+		// Steps 7-8. Find the common ancestor and bookmark node.
+		commonAncestor := p.oe[feIndex-1]
+		bookmark := p.afe.index(formattingElement)
+
+		// Step 9. The inner loop. Find the lastNode to reparent.
+		lastNode := furthestBlock
+		node := furthestBlock
+		x := p.oe.index(node)
+		// Steps 9.1-9.3.
+		for j := 0; j < 3; j++ {
+			// Step 9.4.
+			x--
+			node = p.oe[x]
+			// Step 9.5.
+			if p.afe.index(node) == -1 {
+				p.oe.remove(node)
+				continue
+			}
+			// Step 9.6.
+			if node == formattingElement {
+				break
+			}
+			// Step 9.7.
+			clone := node.clone()
+			p.afe[p.afe.index(node)] = clone
+			p.oe[p.oe.index(node)] = clone
+			node = clone
+			// Step 9.8.
+			if lastNode == furthestBlock {
+				bookmark = p.afe.index(node) + 1
+			}
+			// Step 9.9.
+			if lastNode.Parent != nil {
+				lastNode.Parent.RemoveChild(lastNode)
+			}
+			node.AppendChild(lastNode)
+			// Step 9.10.
+			lastNode = node
+		}
+
+		// Step 10. Reparent lastNode to the common ancestor,
+		// or for misnested table nodes, to the foster parent.
+		if lastNode.Parent != nil {
+			lastNode.Parent.RemoveChild(lastNode)
+		}
+		switch commonAncestor.DataAtom {
+		case a.Table, a.Tbody, a.Tfoot, a.Thead, a.Tr:
+			p.fosterParent(lastNode)
+		default:
+			commonAncestor.AppendChild(lastNode)
+		}
+
+		// Steps 11-13. Reparent nodes from the furthest block's children
+		// to a clone of the formatting element.
+		clone := formattingElement.clone()
+		reparentChildren(clone, furthestBlock)
+		furthestBlock.AppendChild(clone)
+
+		// Step 14. Fix up the list of active formatting elements.
+		if oldLoc := p.afe.index(formattingElement); oldLoc != -1 && oldLoc < bookmark {
+			// Move the bookmark with the rest of the list.
+			bookmark--
+		}
+		p.afe.remove(formattingElement)
+		p.afe.insert(bookmark, clone)
+
+		// Step 15. Fix up the stack of open elements.
+		p.oe.remove(formattingElement)
+		p.oe.insert(p.oe.index(furthestBlock)+1, clone)
+	}
+}
+
+// inBodyEndTagOther performs the "any other end tag" algorithm for inBodyIM.
+func (p *parser) inBodyEndTagOther(tagAtom a.Atom) {
+	for i := len(p.oe) - 1; i >= 0; i-- {
+		if p.oe[i].DataAtom == tagAtom {
+			p.oe = p.oe[:i]
+			break
+		}
+		if isSpecialElement(p.oe[i]) {
+			break
+		}
+	}
+}
+
+// Section 12.2.5.4.8.
+func textIM(p *parser) bool {
+	switch p.tok.Type {
+	case ErrorToken:
+		p.oe.pop()
+	case TextToken:
+		d := p.tok.Data
+		if n := p.oe.top(); n.DataAtom == a.Textarea && n.FirstChild == nil {
+			// Ignore a newline at the start of a -->
+#errors
+#document
+| 
+|   
+|   
+|     -->
+#errors
+#document
+| 
+|   
+|   
+|     
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+#document
+| 
+|   
+|   
+|     
+#errors
+Line: 1 Col: 9 Unexpected end tag (strong). Expected DOCTYPE.
+Line: 1 Col: 9 Unexpected end tag (strong) after the (implied) root element.
+Line: 1 Col: 13 Unexpected end tag (b) after the (implied) root element.
+Line: 1 Col: 18 Unexpected end tag (em) after the (implied) root element.
+Line: 1 Col: 22 Unexpected end tag (i) after the (implied) root element.
+Line: 1 Col: 26 Unexpected end tag (u) after the (implied) root element.
+Line: 1 Col: 35 Unexpected end tag (strike) after the (implied) root element.
+Line: 1 Col: 39 Unexpected end tag (s) after the (implied) root element.
+Line: 1 Col: 47 Unexpected end tag (blink) after the (implied) root element.
+Line: 1 Col: 52 Unexpected end tag (tt) after the (implied) root element.
+Line: 1 Col: 58 Unexpected end tag (pre) after the (implied) root element.
+Line: 1 Col: 64 Unexpected end tag (big) after the (implied) root element.
+Line: 1 Col: 72 Unexpected end tag (small) after the (implied) root element.
+Line: 1 Col: 79 Unexpected end tag (font) after the (implied) root element.
+Line: 1 Col: 88 Unexpected end tag (select) after the (implied) root element.
+Line: 1 Col: 93 Unexpected end tag (h1) after the (implied) root element.
+Line: 1 Col: 98 Unexpected end tag (h2) after the (implied) root element.
+Line: 1 Col: 103 Unexpected end tag (h3) after the (implied) root element.
+Line: 1 Col: 108 Unexpected end tag (h4) after the (implied) root element.
+Line: 1 Col: 113 Unexpected end tag (h5) after the (implied) root element.
+Line: 1 Col: 118 Unexpected end tag (h6) after the (implied) root element.
+Line: 1 Col: 125 Unexpected end tag (body) after the (implied) root element.
+Line: 1 Col: 130 Unexpected end tag (br). Treated as br element.
+Line: 1 Col: 134 End tag (a) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 140 This element (img) has no end tag.
+Line: 1 Col: 148 Unexpected end tag (title). Ignored.
+Line: 1 Col: 155 Unexpected end tag (span). Ignored.
+Line: 1 Col: 163 Unexpected end tag (style). Ignored.
+Line: 1 Col: 172 Unexpected end tag (script). Ignored.
+Line: 1 Col: 180 Unexpected end tag (table). Ignored.
+Line: 1 Col: 185 Unexpected end tag (th). Ignored.
+Line: 1 Col: 190 Unexpected end tag (td). Ignored.
+Line: 1 Col: 195 Unexpected end tag (tr). Ignored.
+Line: 1 Col: 203 This element (frame) has no end tag.
+Line: 1 Col: 210 This element (area) has no end tag.
+Line: 1 Col: 217 Unexpected end tag (link). Ignored.
+Line: 1 Col: 225 This element (param) has no end tag.
+Line: 1 Col: 230 This element (hr) has no end tag.
+Line: 1 Col: 238 This element (input) has no end tag.
+Line: 1 Col: 244 Unexpected end tag (col). Ignored.
+Line: 1 Col: 251 Unexpected end tag (base). Ignored.
+Line: 1 Col: 258 Unexpected end tag (meta). Ignored.
+Line: 1 Col: 269 This element (basefont) has no end tag.
+Line: 1 Col: 279 This element (bgsound) has no end tag.
+Line: 1 Col: 287 This element (embed) has no end tag.
+Line: 1 Col: 296 This element (spacer) has no end tag.
+Line: 1 Col: 300 Unexpected end tag (p). Ignored.
+Line: 1 Col: 305 End tag (dd) seen too early. Expected other end tag.
+Line: 1 Col: 310 End tag (dt) seen too early. Expected other end tag.
+Line: 1 Col: 320 Unexpected end tag (caption). Ignored.
+Line: 1 Col: 331 Unexpected end tag (colgroup). Ignored.
+Line: 1 Col: 339 Unexpected end tag (tbody). Ignored.
+Line: 1 Col: 347 Unexpected end tag (tfoot). Ignored.
+Line: 1 Col: 355 Unexpected end tag (thead). Ignored.
+Line: 1 Col: 365 End tag (address) seen too early. Expected other end tag.
+Line: 1 Col: 378 End tag (blockquote) seen too early. Expected other end tag.
+Line: 1 Col: 387 End tag (center) seen too early. Expected other end tag.
+Line: 1 Col: 393 Unexpected end tag (dir). Ignored.
+Line: 1 Col: 399 End tag (div) seen too early. Expected other end tag.
+Line: 1 Col: 404 End tag (dl) seen too early. Expected other end tag.
+Line: 1 Col: 415 End tag (fieldset) seen too early. Expected other end tag.
+Line: 1 Col: 425 End tag (listing) seen too early. Expected other end tag.
+Line: 1 Col: 432 End tag (menu) seen too early. Expected other end tag.
+Line: 1 Col: 437 End tag (ol) seen too early. Expected other end tag.
+Line: 1 Col: 442 End tag (ul) seen too early. Expected other end tag.
+Line: 1 Col: 447 End tag (li) seen too early. Expected other end tag.
+Line: 1 Col: 454 End tag (nobr) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 460 This element (wbr) has no end tag.
+Line: 1 Col: 476 End tag (button) seen too early. Expected other end tag.
+Line: 1 Col: 486 End tag (marquee) seen too early. Expected other end tag.
+Line: 1 Col: 495 End tag (object) seen too early. Expected other end tag.
+Line: 1 Col: 513 Unexpected end tag (html). Ignored.
+Line: 1 Col: 513 Unexpected end tag (frameset). Ignored.
+Line: 1 Col: 520 Unexpected end tag (head). Ignored.
+Line: 1 Col: 529 Unexpected end tag (iframe). Ignored.
+Line: 1 Col: 537 This element (image) has no end tag.
+Line: 1 Col: 547 This element (isindex) has no end tag.
+Line: 1 Col: 557 Unexpected end tag (noembed). Ignored.
+Line: 1 Col: 568 Unexpected end tag (noframes). Ignored.
+Line: 1 Col: 579 Unexpected end tag (noscript). Ignored.
+Line: 1 Col: 590 Unexpected end tag (optgroup). Ignored.
+Line: 1 Col: 599 Unexpected end tag (option). Ignored.
+Line: 1 Col: 611 Unexpected end tag (plaintext). Ignored.
+Line: 1 Col: 622 Unexpected end tag (textarea). Ignored.
+#document
+| 
+|   
+|   
+|     

+|     

+
+#data
+

+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 20 Unexpected end tag (strong) in table context caused voodoo mode.
+Line: 1 Col: 20 End tag (strong) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 24 Unexpected end tag (b) in table context caused voodoo mode.
+Line: 1 Col: 24 End tag (b) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 29 Unexpected end tag (em) in table context caused voodoo mode.
+Line: 1 Col: 29 End tag (em) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 33 Unexpected end tag (i) in table context caused voodoo mode.
+Line: 1 Col: 33 End tag (i) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 37 Unexpected end tag (u) in table context caused voodoo mode.
+Line: 1 Col: 37 End tag (u) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 46 Unexpected end tag (strike) in table context caused voodoo mode.
+Line: 1 Col: 46 End tag (strike) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 50 Unexpected end tag (s) in table context caused voodoo mode.
+Line: 1 Col: 50 End tag (s) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 58 Unexpected end tag (blink) in table context caused voodoo mode.
+Line: 1 Col: 58 Unexpected end tag (blink). Ignored.
+Line: 1 Col: 63 Unexpected end tag (tt) in table context caused voodoo mode.
+Line: 1 Col: 63 End tag (tt) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 69 Unexpected end tag (pre) in table context caused voodoo mode.
+Line: 1 Col: 69 End tag (pre) seen too early. Expected other end tag.
+Line: 1 Col: 75 Unexpected end tag (big) in table context caused voodoo mode.
+Line: 1 Col: 75 End tag (big) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 83 Unexpected end tag (small) in table context caused voodoo mode.
+Line: 1 Col: 83 End tag (small) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 90 Unexpected end tag (font) in table context caused voodoo mode.
+Line: 1 Col: 90 End tag (font) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 99 Unexpected end tag (select) in table context caused voodoo mode.
+Line: 1 Col: 99 Unexpected end tag (select). Ignored.
+Line: 1 Col: 104 Unexpected end tag (h1) in table context caused voodoo mode.
+Line: 1 Col: 104 End tag (h1) seen too early. Expected other end tag.
+Line: 1 Col: 109 Unexpected end tag (h2) in table context caused voodoo mode.
+Line: 1 Col: 109 End tag (h2) seen too early. Expected other end tag.
+Line: 1 Col: 114 Unexpected end tag (h3) in table context caused voodoo mode.
+Line: 1 Col: 114 End tag (h3) seen too early. Expected other end tag.
+Line: 1 Col: 119 Unexpected end tag (h4) in table context caused voodoo mode.
+Line: 1 Col: 119 End tag (h4) seen too early. Expected other end tag.
+Line: 1 Col: 124 Unexpected end tag (h5) in table context caused voodoo mode.
+Line: 1 Col: 124 End tag (h5) seen too early. Expected other end tag.
+Line: 1 Col: 129 Unexpected end tag (h6) in table context caused voodoo mode.
+Line: 1 Col: 129 End tag (h6) seen too early. Expected other end tag.
+Line: 1 Col: 136 Unexpected end tag (body) in the table row phase. Ignored.
+Line: 1 Col: 141 Unexpected end tag (br) in table context caused voodoo mode.
+Line: 1 Col: 141 Unexpected end tag (br). Treated as br element.
+Line: 1 Col: 145 Unexpected end tag (a) in table context caused voodoo mode.
+Line: 1 Col: 145 End tag (a) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 151 Unexpected end tag (img) in table context caused voodoo mode.
+Line: 1 Col: 151 This element (img) has no end tag.
+Line: 1 Col: 159 Unexpected end tag (title) in table context caused voodoo mode.
+Line: 1 Col: 159 Unexpected end tag (title). Ignored.
+Line: 1 Col: 166 Unexpected end tag (span) in table context caused voodoo mode.
+Line: 1 Col: 166 Unexpected end tag (span). Ignored.
+Line: 1 Col: 174 Unexpected end tag (style) in table context caused voodoo mode.
+Line: 1 Col: 174 Unexpected end tag (style). Ignored.
+Line: 1 Col: 183 Unexpected end tag (script) in table context caused voodoo mode.
+Line: 1 Col: 183 Unexpected end tag (script). Ignored.
+Line: 1 Col: 196 Unexpected end tag (th). Ignored.
+Line: 1 Col: 201 Unexpected end tag (td). Ignored.
+Line: 1 Col: 206 Unexpected end tag (tr). Ignored.
+Line: 1 Col: 214 This element (frame) has no end tag.
+Line: 1 Col: 221 This element (area) has no end tag.
+Line: 1 Col: 228 Unexpected end tag (link). Ignored.
+Line: 1 Col: 236 This element (param) has no end tag.
+Line: 1 Col: 241 This element (hr) has no end tag.
+Line: 1 Col: 249 This element (input) has no end tag.
+Line: 1 Col: 255 Unexpected end tag (col). Ignored.
+Line: 1 Col: 262 Unexpected end tag (base). Ignored.
+Line: 1 Col: 269 Unexpected end tag (meta). Ignored.
+Line: 1 Col: 280 This element (basefont) has no end tag.
+Line: 1 Col: 290 This element (bgsound) has no end tag.
+Line: 1 Col: 298 This element (embed) has no end tag.
+Line: 1 Col: 307 This element (spacer) has no end tag.
+Line: 1 Col: 311 Unexpected end tag (p). Ignored.
+Line: 1 Col: 316 End tag (dd) seen too early. Expected other end tag.
+Line: 1 Col: 321 End tag (dt) seen too early. Expected other end tag.
+Line: 1 Col: 331 Unexpected end tag (caption). Ignored.
+Line: 1 Col: 342 Unexpected end tag (colgroup). Ignored.
+Line: 1 Col: 350 Unexpected end tag (tbody). Ignored.
+Line: 1 Col: 358 Unexpected end tag (tfoot). Ignored.
+Line: 1 Col: 366 Unexpected end tag (thead). Ignored.
+Line: 1 Col: 376 End tag (address) seen too early. Expected other end tag.
+Line: 1 Col: 389 End tag (blockquote) seen too early. Expected other end tag.
+Line: 1 Col: 398 End tag (center) seen too early. Expected other end tag.
+Line: 1 Col: 404 Unexpected end tag (dir). Ignored.
+Line: 1 Col: 410 End tag (div) seen too early. Expected other end tag.
+Line: 1 Col: 415 End tag (dl) seen too early. Expected other end tag.
+Line: 1 Col: 426 End tag (fieldset) seen too early. Expected other end tag.
+Line: 1 Col: 436 End tag (listing) seen too early. Expected other end tag.
+Line: 1 Col: 443 End tag (menu) seen too early. Expected other end tag.
+Line: 1 Col: 448 End tag (ol) seen too early. Expected other end tag.
+Line: 1 Col: 453 End tag (ul) seen too early. Expected other end tag.
+Line: 1 Col: 458 End tag (li) seen too early. Expected other end tag.
+Line: 1 Col: 465 End tag (nobr) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 471 This element (wbr) has no end tag.
+Line: 1 Col: 487 End tag (button) seen too early. Expected other end tag.
+Line: 1 Col: 497 End tag (marquee) seen too early. Expected other end tag.
+Line: 1 Col: 506 End tag (object) seen too early. Expected other end tag.
+Line: 1 Col: 524 Unexpected end tag (html). Ignored.
+Line: 1 Col: 524 Unexpected end tag (frameset). Ignored.
+Line: 1 Col: 531 Unexpected end tag (head). Ignored.
+Line: 1 Col: 540 Unexpected end tag (iframe). Ignored.
+Line: 1 Col: 548 This element (image) has no end tag.
+Line: 1 Col: 558 This element (isindex) has no end tag.
+Line: 1 Col: 568 Unexpected end tag (noembed). Ignored.
+Line: 1 Col: 579 Unexpected end tag (noframes). Ignored.
+Line: 1 Col: 590 Unexpected end tag (noscript). Ignored.
+Line: 1 Col: 601 Unexpected end tag (optgroup). Ignored.
+Line: 1 Col: 610 Unexpected end tag (option). Ignored.
+Line: 1 Col: 622 Unexpected end tag (plaintext). Ignored.
+Line: 1 Col: 633 Unexpected end tag (textarea). Ignored.
+#document
+| 
+|   
+|   
+|     

+|     
+|       
+|         
+|     

+
+#data
+
+#errors
+Line: 1 Col: 10 Unexpected start tag (frameset). Expected DOCTYPE.
+Line: 1 Col: 10 Expected closing tag. Unexpected end of file.
+#document
+| 
+|   
+|   
diff --git a/third_party/code.google.com/p/go.net/html/testdata/webkit/tests10.dat b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests10.dat
new file mode 100644
index 000000000..4f8df86f2
--- /dev/null
+++ b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests10.dat
@@ -0,0 +1,799 @@
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+a
+#errors
+29: Bogus comment
+#document
+| 
+| 
+|   
+|   
+|     
+|     
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+35: Stray “svg” start tag.
+42: Stray end tag “svg”
+#document
+| 
+| 
+|   
+|   
+|     
+#errors
+43: Stray “svg” start tag.
+50: Stray end tag “svg”
+#document
+| 
+| 
+|   
+|   
+|     

+#errors
+34: Start tag “svg” seen in “table”.
+41: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|     
+
+#data
+
foo

+#errors
+34: Start tag “svg” seen in “table”.
+46: Stray end tag “g”.
+53: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|     
+
+#data
+
foobar

+#errors
+34: Start tag “svg” seen in “table”.
+46: Stray end tag “g”.
+58: Stray end tag “g”.
+65: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     
+
+#data
+
foobar

+#errors
+41: Start tag “svg” seen in “table”.
+53: Stray end tag “g”.
+65: Stray end tag “g”.
+72: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     
+|       
+
+#data
+
foobar

+#errors
+45: Start tag “svg” seen in “table”.
+57: Stray end tag “g”.
+69: Stray end tag “g”.
+76: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     
+|       
+|         
+
+#data
+
foobar

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|           

+|             
+|               
+|                 "foo"
+|               
+|                 "bar"
+
+#data
+
foobar
baz

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|           

+|             
+|               
+|                 "foo"
+|               
+|                 "bar"
+|             

+|               "baz"
+
+#data
+
foobar
baz

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       

+|         
+|           
+|             "foo"
+|           
+|             "bar"
+|         

+|           "baz"
+
+#data
+
foobar
baz
quux
+#errors
+70: HTML start tag “p” in a foreign namespace context.
+81: “table” closed but “caption” was still open.
+#document
+| 
+| 
+|   
+|   
+|     
+|       

+|         
+|           
+|             "foo"
+|           
+|             "bar"
+|         

+|           "baz"
+|     

+|       "quux"
+
+#data
+
foobarbaz
quux
+#errors
+78: “table” closed but “caption” was still open.
+78: Unclosed elements on stack.
+#document
+| 
+| 
+|   
+|   
+|     
+|       

+|         
+|           
+|             "foo"
+|           
+|             "bar"
+|           "baz"
+|     

+|       "quux"
+
+#data
+foobar
baz
quux
+#errors
+44: Start tag “svg” seen in “table”.
+56: Stray end tag “g”.
+68: Stray end tag “g”.
+71: HTML start tag “p” in a foreign namespace context.
+71: Start tag “p” seen in “table”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     

+|       "baz"
+|     
+|       +|     

+|       "quux"
+
+#data
+


quux
+#errors
+50: Stray “svg” start tag.
+54: Stray “g” start tag.
+62: Stray end tag “g”
+66: Stray “g” start tag.
+74: Stray end tag “g”
+77: Stray “p” start tag.
+88: “table” end tag with “select” open.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|           

+|             
quux
+#errors
+36: Start tag “select” seen in “table”.
+42: Stray “svg” start tag.
+46: Stray “g” start tag.
+54: Stray end tag “g”
+58: Stray “g” start tag.
+66: Stray end tag “g”
+69: Stray “p” start tag.
+80: “table” end tag with “select” open.
+#document
+| 
+| 
+|   
+|   
+|     
+|     

+|       "quux"
+
+#data
+foobar
baz
+#errors
+41: Stray “svg” start tag.
+68: HTML start tag “p” in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     

+|       "baz"
+
+#data
+foobar
baz
+#errors
+34: Stray “svg” start tag.
+61: HTML start tag “p” in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     

+|       "baz"
+
+#data
+

+#errors
+31: Stray “svg” start tag.
+35: Stray “g” start tag.
+40: Stray end tag “g”
+44: Stray “g” start tag.
+49: Stray end tag “g”
+52: Stray “p” start tag.
+58: Stray “span” start tag.
+58: End of file seen and there were open elements.
+#document
+| 
+| 
+|   
+|   
+
+#data
+

+#errors
+42: Stray “svg” start tag.
+46: Stray “g” start tag.
+51: Stray end tag “g”
+55: Stray “g” start tag.
+60: Stray end tag “g”
+63: Stray “p” start tag.
+69: Stray “span” start tag.
+#document
+| 
+| 
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     
+|       xlink href="foo"
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     xml:lang="en"
+|     
+|       
+|         xlink href="foo"
+|         xml lang="en"
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     xml:lang="en"
+|     
+|       
+|         xlink href="foo"
+|         xml lang="en"
+
+#data
+bar
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     xml:lang="en"
+|     
+|       
+|         xlink href="foo"
+|         xml lang="en"
+|       "bar"
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+
+#data
+
a
+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|     "a"
+
+#data
+
a
+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|     "a"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|       
+
+#data
+
a
+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|             
+|               "a"
+
+#data
+
a
+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|             

+|               "a"
+
+#data
+
    a
+#errors
+40: HTML start tag “ul” in a foreign namespace context.
+41: End of file in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
    
+|           
+|           
      
+|             "a"
+
+#data
+
        a
+#errors
+35: HTML start tag “ul” in a foreign namespace context.
+36: End of file in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|         
          
+|           "a"
+
+#data
+
          
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
          
+|       
+|         
+|           
          
+
+#data
+
          
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
          
+|       
+|         
+|           
          
+
+#data
+
          
+#errors
+#document
+| 
+|   
+|   
+|     
          
+|       
+|         
+|           
+|             
          
+|             
          
+
+#data
+
          
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
          
+|           
+|             
          
+|               
+|       
+
+#data
+
          
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+|           
+|             
          
+|               
          
+|       
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|       
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     
+|     

+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+|       
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+|           
+|             

+|               
+|                 
+|               
+|           
+|       
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+|           
+|             
+|               
+|                 
+|               
+|             
+|           
+|       
diff --git a/third_party/code.google.com/p/go.net/html/testdata/webkit/tests11.dat b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests11.dat
new file mode 100644
index 000000000..638cde479
--- /dev/null
+++ b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests11.dat
@@ -0,0 +1,482 @@
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributeName=""
+|       attributeType=""
+|       baseFrequency=""
+|       baseProfile=""
+|       calcMode=""
+|       clipPathUnits=""
+|       contentScriptType=""
+|       contentStyleType=""
+|       diffuseConstant=""
+|       edgeMode=""
+|       externalResourcesRequired=""
+|       filterRes=""
+|       filterUnits=""
+|       glyphRef=""
+|       gradientTransform=""
+|       gradientUnits=""
+|       kernelMatrix=""
+|       kernelUnitLength=""
+|       keyPoints=""
+|       keySplines=""
+|       keyTimes=""
+|       lengthAdjust=""
+|       limitingConeAngle=""
+|       markerHeight=""
+|       markerUnits=""
+|       markerWidth=""
+|       maskContentUnits=""
+|       maskUnits=""
+|       numOctaves=""
+|       pathLength=""
+|       patternContentUnits=""
+|       patternTransform=""
+|       patternUnits=""
+|       pointsAtX=""
+|       pointsAtY=""
+|       pointsAtZ=""
+|       preserveAlpha=""
+|       preserveAspectRatio=""
+|       primitiveUnits=""
+|       refX=""
+|       refY=""
+|       repeatCount=""
+|       repeatDur=""
+|       requiredExtensions=""
+|       requiredFeatures=""
+|       specularConstant=""
+|       specularExponent=""
+|       spreadMethod=""
+|       startOffset=""
+|       stdDeviation=""
+|       stitchTiles=""
+|       surfaceScale=""
+|       systemLanguage=""
+|       tableValues=""
+|       targetX=""
+|       targetY=""
+|       textLength=""
+|       viewBox=""
+|       viewTarget=""
+|       xChannelSelector=""
+|       yChannelSelector=""
+|       zoomAndPan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributeName=""
+|       attributeType=""
+|       baseFrequency=""
+|       baseProfile=""
+|       calcMode=""
+|       clipPathUnits=""
+|       contentScriptType=""
+|       contentStyleType=""
+|       diffuseConstant=""
+|       edgeMode=""
+|       externalResourcesRequired=""
+|       filterRes=""
+|       filterUnits=""
+|       glyphRef=""
+|       gradientTransform=""
+|       gradientUnits=""
+|       kernelMatrix=""
+|       kernelUnitLength=""
+|       keyPoints=""
+|       keySplines=""
+|       keyTimes=""
+|       lengthAdjust=""
+|       limitingConeAngle=""
+|       markerHeight=""
+|       markerUnits=""
+|       markerWidth=""
+|       maskContentUnits=""
+|       maskUnits=""
+|       numOctaves=""
+|       pathLength=""
+|       patternContentUnits=""
+|       patternTransform=""
+|       patternUnits=""
+|       pointsAtX=""
+|       pointsAtY=""
+|       pointsAtZ=""
+|       preserveAlpha=""
+|       preserveAspectRatio=""
+|       primitiveUnits=""
+|       refX=""
+|       refY=""
+|       repeatCount=""
+|       repeatDur=""
+|       requiredExtensions=""
+|       requiredFeatures=""
+|       specularConstant=""
+|       specularExponent=""
+|       spreadMethod=""
+|       startOffset=""
+|       stdDeviation=""
+|       stitchTiles=""
+|       surfaceScale=""
+|       systemLanguage=""
+|       tableValues=""
+|       targetX=""
+|       targetY=""
+|       textLength=""
+|       viewBox=""
+|       viewTarget=""
+|       xChannelSelector=""
+|       yChannelSelector=""
+|       zoomAndPan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributeName=""
+|       attributeType=""
+|       baseFrequency=""
+|       baseProfile=""
+|       calcMode=""
+|       clipPathUnits=""
+|       contentScriptType=""
+|       contentStyleType=""
+|       diffuseConstant=""
+|       edgeMode=""
+|       externalResourcesRequired=""
+|       filterRes=""
+|       filterUnits=""
+|       glyphRef=""
+|       gradientTransform=""
+|       gradientUnits=""
+|       kernelMatrix=""
+|       kernelUnitLength=""
+|       keyPoints=""
+|       keySplines=""
+|       keyTimes=""
+|       lengthAdjust=""
+|       limitingConeAngle=""
+|       markerHeight=""
+|       markerUnits=""
+|       markerWidth=""
+|       maskContentUnits=""
+|       maskUnits=""
+|       numOctaves=""
+|       pathLength=""
+|       patternContentUnits=""
+|       patternTransform=""
+|       patternUnits=""
+|       pointsAtX=""
+|       pointsAtY=""
+|       pointsAtZ=""
+|       preserveAlpha=""
+|       preserveAspectRatio=""
+|       primitiveUnits=""
+|       refX=""
+|       refY=""
+|       repeatCount=""
+|       repeatDur=""
+|       requiredExtensions=""
+|       requiredFeatures=""
+|       specularConstant=""
+|       specularExponent=""
+|       spreadMethod=""
+|       startOffset=""
+|       stdDeviation=""
+|       stitchTiles=""
+|       surfaceScale=""
+|       systemLanguage=""
+|       tableValues=""
+|       targetX=""
+|       targetY=""
+|       textLength=""
+|       viewBox=""
+|       viewTarget=""
+|       xChannelSelector=""
+|       yChannelSelector=""
+|       zoomAndPan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributename=""
+|       attributetype=""
+|       basefrequency=""
+|       baseprofile=""
+|       calcmode=""
+|       clippathunits=""
+|       contentscripttype=""
+|       contentstyletype=""
+|       diffuseconstant=""
+|       edgemode=""
+|       externalresourcesrequired=""
+|       filterres=""
+|       filterunits=""
+|       glyphref=""
+|       gradienttransform=""
+|       gradientunits=""
+|       kernelmatrix=""
+|       kernelunitlength=""
+|       keypoints=""
+|       keysplines=""
+|       keytimes=""
+|       lengthadjust=""
+|       limitingconeangle=""
+|       markerheight=""
+|       markerunits=""
+|       markerwidth=""
+|       maskcontentunits=""
+|       maskunits=""
+|       numoctaves=""
+|       pathlength=""
+|       patterncontentunits=""
+|       patterntransform=""
+|       patternunits=""
+|       pointsatx=""
+|       pointsaty=""
+|       pointsatz=""
+|       preservealpha=""
+|       preserveaspectratio=""
+|       primitiveunits=""
+|       refx=""
+|       refy=""
+|       repeatcount=""
+|       repeatdur=""
+|       requiredextensions=""
+|       requiredfeatures=""
+|       specularconstant=""
+|       specularexponent=""
+|       spreadmethod=""
+|       startoffset=""
+|       stddeviation=""
+|       stitchtiles=""
+|       surfacescale=""
+|       systemlanguage=""
+|       tablevalues=""
+|       targetx=""
+|       targety=""
+|       textlength=""
+|       viewbox=""
+|       viewtarget=""
+|       xchannelselector=""
+|       ychannelselector=""
+|       zoomandpan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
diff --git a/third_party/code.google.com/p/go.net/html/testdata/webkit/tests12.dat b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests12.dat
new file mode 100644
index 000000000..63107d277
--- /dev/null
+++ b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests12.dat
@@ -0,0 +1,62 @@
+#data
+
foobazeggs
spam
quuxbar
+#errors
+#document
+| 
+| 
+|   
+|   
+|     

+|       "foo"
+|       
+|         
+|           
+|             "baz"
+|         
+|           
+|             
+|               
+|                 "eggs"
+|             
+|               
+|                 

+|                   "spam"
+|                 
+|                   
+|                     
+|                       

+|                         
+|             
+|               "quux"
+|       "bar"
+
+#data
+foobazeggs
spam
quuxbar
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "foo"
+|     
+|       
+|         
+|           "baz"
+|       
+|         
+|           
+|             
+|               "eggs"
+|           
+|             
+|               

+|                 "spam"
+|               
+|                 
+|                   
+|                     

+|                       
+|           
+|             "quux"
+|     "bar"
diff --git a/third_party/code.google.com/p/go.net/html/testdata/webkit/tests14.dat b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests14.dat
new file mode 100644
index 000000000..b8713f885
--- /dev/null
+++ b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests14.dat
@@ -0,0 +1,74 @@
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|     
+
+#data
+
+#errors
+15: Unexpected start tag html
+#document
+| 
+| 
+|   abc:def="gh"
+|   
+|   
+|     
+
+#data
+
+#errors
+15: Unexpected start tag html
+#document
+| 
+| 
+|   xml:lang="bar"
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   123="456"
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   123="456"
+|   789="012"
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     789="012"
diff --git a/third_party/code.google.com/p/go.net/html/testdata/webkit/tests15.dat b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests15.dat
new file mode 100644
index 000000000..6ce1c0d16
--- /dev/null
+++ b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests15.dat
@@ -0,0 +1,208 @@
+#data
+
 
X
+#errors
+Line: 1 Col: 31 Unexpected end tag (p). Ignored.
+Line: 1 Col: 36 Expected closing tag. Unexpected end of file.
+#document
+| 
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|     
+|       
+|         
+|           " "
+|           

+|             "X"
+
+#data
+

+
X
+#errors
+Line: 1 Col: 3 Unexpected start tag (p). Expected DOCTYPE.
+Line: 1 Col: 16 Unexpected end tag (p). Ignored.
+Line: 2 Col: 4 Expected closing tag. Unexpected end of file.
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|     
+|       
+|         
+|           "
+"
+|           

+|             "X"
+
+#data
+ 
+#errors
+Line: 1 Col: 22 Unexpected end tag (html) after the (implied) root element.
+#document
+| 
+| 
+|   
+|   
+|     " "
+
+#data
+
+#errors
+Line: 1 Col: 22 Unexpected end tag (body) after the (implied) root element.
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+Line: 1 Col: 6 Unexpected start tag (html). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end tag (html) after the (implied) root element.
+#document
+| 
+|   
+|   
+| 
+
+#data
+X
+#errors
+Line: 1 Col: 22 Unexpected end tag (body) after the (implied) root element.
+#document
+| 
+| 
+|   
+|   
+|     
+|       "X"
+
+#data
+<!doctype html><table> X<meta></table>
+#errors
+Line: 1 Col: 24 Unexpected non-space characters in table context caused voodoo mode.
+Line: 1 Col: 30 Unexpected start tag (meta) in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " X"
+|     <meta>
+|     <table>
+
+#data
+<!doctype html><table> x</table>
+#errors
+Line: 1 Col: 24 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " x"
+|     <table>
+
+#data
+<!doctype html><table> x </table>
+#errors
+Line: 1 Col: 25 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " x "
+|     <table>
+
+#data
+<!doctype html><table><tr> x</table>
+#errors
+Line: 1 Col: 28 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " x"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table>X<style> <tr>x </style> </table>
+#errors
+Line: 1 Col: 23 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "X"
+|     <table>
+|       <style>
+|         " <tr>x "
+|       " "
+
+#data
+<!doctype html><div><table><a>foo</a> <tr><td>bar</td> </tr></table></div>
+#errors
+Line: 1 Col: 30 Unexpected start tag (a) in table context caused voodoo mode.
+Line: 1 Col: 37 Unexpected end tag (a) in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <div>
+|       <a>
+|         "foo"
+|       <table>
+|         " "
+|         <tbody>
+|           <tr>
+|             <td>
+|               "bar"
+|             " "
+
+#data
+<frame></frame></frame><frameset><frame><frameset><frame></frameset><noframes></frameset><noframes>
+#errors
+6: Start tag seen without seeing a doctype first. Expected “<!DOCTYPE html>”.
+13: Stray start tag “frame”.
+21: Stray end tag “frame”.
+29: Stray end tag “frame”.
+39: “frameset” start tag after “body” already open.
+105: End of file seen inside an [R]CDATA element.
+105: End of file seen and there were open elements.
+XXX: These errors are wrong, please fix me!
+#document
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+|     <frameset>
+|       <frame>
+|     <noframes>
+|       "</frameset><noframes>"
+
+#data
+<!DOCTYPE html><object></html>
+#errors
+1: Expected closing tag. Unexpected end of file
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <object>
diff --git a/third_party/code.google.com/p/go.net/html/testdata/webkit/tests16.dat b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests16.dat
new file mode 100644
index 000000000..c8ef66f0e
--- /dev/null
+++ b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests16.dat
@@ -0,0 +1,2299 @@
+#data
+<!doctype html><script>
+#errors
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<!doctype html><script>a
+#errors
+Line: 1 Col: 24 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "a"
+|   <body>
+
+#data
+<!doctype html><script><
+#errors
+Line: 1 Col: 24 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<"
+|   <body>
+
+#data
+<!doctype html><script></
+#errors
+Line: 1 Col: 25 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</"
+|   <body>
+
+#data
+<!doctype html><script></S
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</S"
+|   <body>
+
+#data
+<!doctype html><script></SC
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SC"
+|   <body>
+
+#data
+<!doctype html><script></SCR
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCR"
+|   <body>
+
+#data
+<!doctype html><script></SCRI
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCRI"
+|   <body>
+
+#data
+<!doctype html><script></SCRIP
+#errors
+Line: 1 Col: 30 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIP"
+|   <body>
+
+#data
+<!doctype html><script></SCRIPT
+#errors
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIPT"
+|   <body>
+
+#data
+<!doctype html><script></SCRIPT 
+#errors
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<!doctype html><script></s
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</s"
+|   <body>
+
+#data
+<!doctype html><script></sc
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</sc"
+|   <body>
+
+#data
+<!doctype html><script></scr
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</scr"
+|   <body>
+
+#data
+<!doctype html><script></scri
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</scri"
+|   <body>
+
+#data
+<!doctype html><script></scrip
+#errors
+Line: 1 Col: 30 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</scrip"
+|   <body>
+
+#data
+<!doctype html><script></script
+#errors
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</script"
+|   <body>
+
+#data
+<!doctype html><script></script 
+#errors
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<!doctype html><script><!
+#errors
+Line: 1 Col: 25 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!"
+|   <body>
+
+#data
+<!doctype html><script><!a
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!a"
+|   <body>
+
+#data
+<!doctype html><script><!-
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!-"
+|   <body>
+
+#data
+<!doctype html><script><!-a
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!-a"
+|   <body>
+
+#data
+<!doctype html><script><!--
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<!doctype html><script><!--a
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--a"
+|   <body>
+
+#data
+<!doctype html><script><!--<
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<"
+|   <body>
+
+#data
+<!doctype html><script><!--<a
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<a"
+|   <body>
+
+#data
+<!doctype html><script><!--</
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--</"
+|   <body>
+
+#data
+<!doctype html><script><!--</script
+#errors
+Line: 1 Col: 35 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--</script"
+|   <body>
+
+#data
+<!doctype html><script><!--</script 
+#errors
+Line: 1 Col: 36 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<!doctype html><script><!--<s
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<s"
+|   <body>
+
+#data
+<!doctype html><script><!--<script
+#errors
+Line: 1 Col: 34 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script 
+#errors
+Line: 1 Col: 35 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script <
+#errors
+Line: 1 Col: 36 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <"
+|   <body>
+
+#data
+<!doctype html><script><!--<script <a
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </s
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </s"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script
+#errors
+Line: 1 Col: 43 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </scripta
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </scripta"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script 
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script>
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script>"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script/
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script/"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script <
+#errors
+Line: 1 Col: 45 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script <a
+#errors
+Line: 1 Col: 46 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </
+#errors
+Line: 1 Col: 46 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script
+#errors
+Line: 1 Col: 52 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script 
+#errors
+Line: 1 Col: 53 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script/
+#errors
+Line: 1 Col: 53 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script -
+#errors
+Line: 1 Col: 36 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -"
+|   <body>
+
+#data
+<!doctype html><script><!--<script -a
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script -<
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -<"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --a
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --<
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --<"
+|   <body>
+
+#data
+<!doctype html><script><!--<script -->
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --><
+#errors
+Line: 1 Col: 39 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --><"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></
+#errors
+Line: 1 Col: 40 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script
+#errors
+Line: 1 Col: 46 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script 
+#errors
+Line: 1 Col: 47 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script/
+#errors
+Line: 1 Col: 47 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script><\/script>--></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script><\/script>-->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></scr'+'ipt>--></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt>-->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>--><!--</script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>--><!--"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>-- ></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>-- >"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>- -></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- ->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>- - ></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- - >"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>-></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script>--!></script>X
+#errors
+Line: 1 Col: 49 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script>--!></script>X"
+|   <body>
+
+#data
+<!doctype html><script><!--<scr'+'ipt></script>--></script>
+#errors
+Line: 1 Col: 59 Unexpected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<scr'+'ipt>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><script><!--<script></scr'+'ipt></script>X
+#errors
+Line: 1 Col: 57 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt></script>X"
+|   <body>
+
+#data
+<!doctype html><style><!--<style></style>--></style>
+#errors
+Line: 1 Col: 52 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--<style>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><style><!--</style>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--"
+|   <body>
+|     "X"
+
+#data
+<!doctype html><style><!--...</style>...--></style>
+#errors
+Line: 1 Col: 51 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|   <body>
+|     "...-->"
+
+#data
+<!doctype html><style><!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style></style>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style>"
+|   <body>
+|     "X"
+
+#data
+<!doctype html><style><!--...<style><!--...--!></style>--></style>
+#errors
+Line: 1 Col: 66 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--...<style><!--...--!>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><style><!--...</style><!-- --><style>@import ...</style>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|     <!--   -->
+|     <style>
+|       "@import ..."
+|   <body>
+
+#data
+<!doctype html><style>...<style><!--...</style><!-- --></style>
+#errors
+Line: 1 Col: 63 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "...<style><!--..."
+|     <!--   -->
+|   <body>
+
+#data
+<!doctype html><style>...<!--[if IE]><style>...</style>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "...<!--[if IE]><style>..."
+|   <body>
+|     "X"
+
+#data
+<!doctype html><title><!--<title>-->
+#errors
+Line: 1 Col: 52 Unexpected end tag (title).
+#document
+| 
+| 
+|   
+|     
+|       "<!--<title>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><title></title>
+#errors
+#document
+| 
+| 
+|   
+|     
+|       ""
+|   
+
+#data
+foo/title><link></head><body>X
+#errors
+Line: 1 Col: 52 Unexpected end of file. Expected end tag (title).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <title>
+|       "foo/title><link></head><body>X"
+|   <body>
+
+#data
+<!doctype html><noscript><!--<noscript></noscript>--></noscript>
+#errors
+Line: 1 Col: 64 Unexpected end tag (noscript).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--<noscript>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><noscript><!--</noscript>X<noscript>--></noscript>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--"
+|   <body>
+|     "X"
+|     <noscript>
+|       "-->"
+
+#data
+<!doctype html><noscript><iframe></noscript>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noscript>
+|       "<iframe>"
+|   <body>
+|     "X"
+
+#data
+<!doctype html><noframes><!--<noframes></noframes>--></noframes>
+#errors
+Line: 1 Col: 64 Unexpected end tag (noframes).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noframes>
+|       "<!--<noframes>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><noframes><body><script><!--...</script></body></noframes></html>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noframes>
+|       "<body><script><!--...</script></body>"
+|   <body>
+
+#data
+<!doctype html><textarea><!--<textarea></textarea>--></textarea>
+#errors
+Line: 1 Col: 64 Unexpected end tag (textarea).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "<!--<textarea>"
+|     "-->"
+
+#data
+<!doctype html><textarea></textarea></textarea>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "</textarea>"
+
+#data
+<!doctype html><textarea><</textarea>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "<"
+
+#data
+<!doctype html><textarea>a<b</textarea>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "a<b"
+
+#data
+<!doctype html><iframe><!--<iframe></iframe>--></iframe>
+#errors
+Line: 1 Col: 56 Unexpected end tag (iframe).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "<!--<iframe>"
+|     "-->"
+
+#data
+<!doctype html><iframe>...<!--X->...<!--/X->...</iframe>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "...<!--X->...<!--/X->..."
+
+#data
+<!doctype html><xmp><!--<xmp></xmp>--></xmp>
+#errors
+Line: 1 Col: 44 Unexpected end tag (xmp).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <xmp>
+|       "<!--<xmp>"
+|     "-->"
+
+#data
+<!doctype html><noembed><!--<noembed></noembed>--></noembed>
+#errors
+Line: 1 Col: 60 Unexpected end tag (noembed).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <noembed>
+|       "<!--<noembed>"
+|     "-->"
+
+#data
+<script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 8 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<script>a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 9 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "a"
+|   <body>
+
+#data
+<script><
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 9 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<"
+|   <body>
+
+#data
+<script></
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 10 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</"
+|   <body>
+
+#data
+<script></S
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</S"
+|   <body>
+
+#data
+<script></SC
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SC"
+|   <body>
+
+#data
+<script></SCR
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCR"
+|   <body>
+
+#data
+<script></SCRI
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCRI"
+|   <body>
+
+#data
+<script></SCRIP
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 15 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIP"
+|   <body>
+
+#data
+<script></SCRIPT
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 16 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIPT"
+|   <body>
+
+#data
+<script></SCRIPT 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 17 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<script></s
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</s"
+|   <body>
+
+#data
+<script></sc
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</sc"
+|   <body>
+
+#data
+<script></scr
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</scr"
+|   <body>
+
+#data
+<script></scri
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</scri"
+|   <body>
+
+#data
+<script></scrip
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 15 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</scrip"
+|   <body>
+
+#data
+<script></script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 16 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</script"
+|   <body>
+
+#data
+<script></script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 17 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<script><!
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 10 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!"
+|   <body>
+
+#data
+<script><!a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!a"
+|   <body>
+
+#data
+<script><!-
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!-"
+|   <body>
+
+#data
+<script><!-a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!-a"
+|   <body>
+
+#data
+<script><!--
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<script><!--a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--a"
+|   <body>
+
+#data
+<script><!--<
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<"
+|   <body>
+
+#data
+<script><!--<a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<a"
+|   <body>
+
+#data
+<script><!--</
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--</"
+|   <body>
+
+#data
+<script><!--</script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 20 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--</script"
+|   <body>
+
+#data
+<script><!--</script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<script><!--<s
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<s"
+|   <body>
+
+#data
+<script><!--<script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 19 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script"
+|   <body>
+
+#data
+<script><!--<script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 20 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script "
+|   <body>
+
+#data
+<script><!--<script <
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <"
+|   <body>
+
+#data
+<script><!--<script <a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <a"
+|   <body>
+
+#data
+<script><!--<script </
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </"
+|   <body>
+
+#data
+<script><!--<script </s
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </s"
+|   <body>
+
+#data
+<script><!--<script </script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script"
+|   <body>
+
+#data
+<script><!--<script </scripta
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </scripta"
+|   <body>
+
+#data
+<script><!--<script </script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script </script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script>"
+|   <body>
+
+#data
+<script><!--<script </script/
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script/"
+|   <body>
+
+#data
+<script><!--<script </script <
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 30 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <"
+|   <body>
+
+#data
+<script><!--<script </script <a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <a"
+|   <body>
+
+#data
+<script><!--<script </script </
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </"
+|   <body>
+
+#data
+<script><!--<script </script </script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </script"
+|   <body>
+
+#data
+<script><!--<script </script </script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script </script </script/
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script </script </script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script -
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -"
+|   <body>
+
+#data
+<script><!--<script -a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -a"
+|   <body>
+
+#data
+<script><!--<script --
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --"
+|   <body>
+
+#data
+<script><!--<script --a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --a"
+|   <body>
+
+#data
+<script><!--<script -->
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script --><
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 24 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --><"
+|   <body>
+
+#data
+<script><!--<script --></
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 25 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></"
+|   <body>
+
+#data
+<script><!--<script --></script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></script"
+|   <body>
+
+#data
+<script><!--<script --></script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script --></script/
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script --></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script><\/script>--></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script><\/script>-->"
+|   <body>
+
+#data
+<script><!--<script></scr'+'ipt>--></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt>-->"
+|   <body>
+
+#data
+<script><!--<script></script><script></script></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>--><!--</script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>--><!--"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>-- ></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>-- >"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>- -></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- ->"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>- - ></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- - >"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>-></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>->"
+|   <body>
+
+#data
+<script><!--<script>--!></script>X
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 34 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script>--!></script>X"
+|   <body>
+
+#data
+<script><!--<scr'+'ipt></script>--></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 44 Unexpected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<scr'+'ipt>"
+|   <body>
+|     "-->"
+
+#data
+<script><!--<script></scr'+'ipt></script>X
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 42 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt></script>X"
+|   <body>
+
+#data
+<style><!--<style></style>--></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 37 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--<style>"
+|   <body>
+|     "-->"
+
+#data
+<style><!--</style>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--"
+|   <body>
+|     "X"
+
+#data
+<style><!--...</style>...--></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 36 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|   <body>
+|     "...-->"
+
+#data
+<style><!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style></style>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style>"
+|   <body>
+|     "X"
+
+#data
+<style><!--...<style><!--...--!></style>--></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 51 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--...<style><!--...--!>"
+|   <body>
+|     "-->"
+
+#data
+<style><!--...</style><!-- --><style>@import ...</style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|     <!--   -->
+|     <style>
+|       "@import ..."
+|   <body>
+
+#data
+<style>...<style><!--...</style><!-- --></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 48 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "...<style><!--..."
+|     <!--   -->
+|   <body>
+
+#data
+<style>...<!--[if IE]><style>...</style>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "...<!--[if IE]><style>..."
+|   <body>
+|     "X"
+
+#data
+<title><!--<title>-->
+#errors
+Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE.
+Line: 1 Col: 37 Unexpected end tag (title).
+#document
+| 
+|   
+|     
+|       "<!--<title>"
+|   <body>
+|     "-->"
+
+#data
+<title></title>
+#errors
+Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE.
+#document
+| 
+|   
+|     
+|       ""
+|   
+
+#data
+foo/title><link></head><body>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE.
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (title).
+#document
+| <html>
+|   <head>
+|     <title>
+|       "foo/title><link></head><body>X"
+|   <body>
+
+#data
+<noscript><!--<noscript></noscript>--></noscript>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE.
+Line: 1 Col: 49 Unexpected end tag (noscript).
+#document
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--<noscript>"
+|   <body>
+|     "-->"
+
+#data
+<noscript><!--</noscript>X<noscript>--></noscript>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--"
+|   <body>
+|     "X"
+|     <noscript>
+|       "-->"
+
+#data
+<noscript><iframe></noscript>X
+#errors
+Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <noscript>
+|       "<iframe>"
+|   <body>
+|     "X"
+
+#data
+<noframes><!--<noframes></noframes>--></noframes>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noframes). Expected DOCTYPE.
+Line: 1 Col: 49 Unexpected end tag (noframes).
+#document
+| <html>
+|   <head>
+|     <noframes>
+|       "<!--<noframes>"
+|   <body>
+|     "-->"
+
+#data
+<noframes><body><script><!--...</script></body></noframes></html>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noframes). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <noframes>
+|       "<body><script><!--...</script></body>"
+|   <body>
+
+#data
+<textarea><!--<textarea></textarea>--></textarea>
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+Line: 1 Col: 49 Unexpected end tag (textarea).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "<!--<textarea>"
+|     "-->"
+
+#data
+<textarea></textarea></textarea>
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "</textarea>"
+
+#data
+<iframe><!--<iframe></iframe>--></iframe>
+#errors
+Line: 1 Col: 8 Unexpected start tag (iframe). Expected DOCTYPE.
+Line: 1 Col: 41 Unexpected end tag (iframe).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "<!--<iframe>"
+|     "-->"
+
+#data
+<iframe>...<!--X->...<!--/X->...</iframe>
+#errors
+Line: 1 Col: 8 Unexpected start tag (iframe). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "...<!--X->...<!--/X->..."
+
+#data
+<xmp><!--<xmp></xmp>--></xmp>
+#errors
+Line: 1 Col: 5 Unexpected start tag (xmp). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end tag (xmp).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <xmp>
+|       "<!--<xmp>"
+|     "-->"
+
+#data
+<noembed><!--<noembed></noembed>--></noembed>
+#errors
+Line: 1 Col: 9 Unexpected start tag (noembed). Expected DOCTYPE.
+Line: 1 Col: 45 Unexpected end tag (noembed).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <noembed>
+|       "<!--<noembed>"
+|     "-->"
+
+#data
+<!doctype html><table>
+
+#errors
+Line 2 Col 0 Unexpected end of file. Expected table content.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       "
+"
+
+#data
+<!doctype html><table><td><span><font></span><span>
+#errors
+Line 1 Col 26 Unexpected table cell start tag (td) in the table body phase.
+Line 1 Col 45 Unexpected end tag (span).
+Line 1 Col 51 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <span>
+|               <font>
+|             <font>
+|               <span>
+
+#data
+<!doctype html><form><table></form><form></table></form>
+#errors
+35: Stray end tag “form”.
+41: Start tag “form” seen in “table”.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <table>
+|         <form>
diff --git a/third_party/code.google.com/p/go.net/html/testdata/webkit/tests17.dat b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests17.dat
new file mode 100644
index 000000000..7b555f888
--- /dev/null
+++ b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests17.dat
@@ -0,0 +1,153 @@
+#data
+<!doctype html><table><tbody><select><tr>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table><tr><select><td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+
+#data
+<!doctype html><table><tr><td><select><td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <select>
+|           <td>
+
+#data
+<!doctype html><table><tr><th><select><td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <th>
+|             <select>
+|           <td>
+
+#data
+<!doctype html><table><caption><select><tr>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         <select>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><select><tr>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><th>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><tbody>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><thead>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><tfoot>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><caption>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><table><tr></table>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|     "a"
diff --git a/third_party/code.google.com/p/go.net/html/testdata/webkit/tests18.dat b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests18.dat
new file mode 100644
index 000000000..680e1f068
--- /dev/null
+++ b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests18.dat
@@ -0,0 +1,269 @@
+#data
+<!doctype html><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+
+#data
+<!doctype html><table><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+|     <table>
+
+#data
+<!doctype html><table><tbody><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+|     <table>
+|       <tbody>
+
+#data
+<!doctype html><table><tbody><tr><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table><tbody><tr><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table><td><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <plaintext>
+|               "</plaintext>"
+
+#data
+<!doctype html><table><caption><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         <plaintext>
+|           "</plaintext>"
+
+#data
+<!doctype html><table><tr><style></script></style>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "abc"
+|     <table>
+|       <tbody>
+|         <tr>
+|           <style>
+|             "</script>"
+
+#data
+<!doctype html><table><tr><script></style></script>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "abc"
+|     <table>
+|       <tbody>
+|         <tr>
+|           <script>
+|             "</style>"
+
+#data
+<!doctype html><table><caption><style></script></style>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         <style>
+|           "</script>"
+|         "abc"
+
+#data
+<!doctype html><table><td><style></script></style>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <style>
+|               "</script>"
+|             "abc"
+
+#data
+<!doctype html><select><script></style></script>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <script>
+|         "</style>"
+|       "abc"
+
+#data
+<!doctype html><table><select><script></style></script>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <script>
+|         "</style>"
+|       "abc"
+|     <table>
+
+#data
+<!doctype html><table><tr><select><script></style></script>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <script>
+|         "</style>"
+|       "abc"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><frameset></frameset><noframes>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   <noframes>
+|     "abc"
+
+#data
+<!doctype html><frameset></frameset><noframes>abc</noframes><!--abc-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   <noframes>
+|     "abc"
+|   <!-- abc -->
+
+#data
+<!doctype html><frameset></frameset></html><noframes>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   <noframes>
+|     "abc"
+
+#data
+<!doctype html><frameset></frameset></html><noframes>abc</noframes><!--abc-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   <noframes>
+|     "abc"
+| <!-- abc -->
+
+#data
+<!doctype html><table><tr></tbody><tfoot>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|       <tfoot>
+
+#data
+<!doctype html><table><td><svg></svg>abc<td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <svg svg>
+|             "abc"
+|           <td>
diff --git a/third_party/code.google.com/p/go.net/html/testdata/webkit/tests19.dat b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests19.dat
new file mode 100644
index 000000000..0d62f5a5b
--- /dev/null
+++ b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests19.dat
@@ -0,0 +1,1237 @@
+#data
+<!doctype html><math><mn DefinitionUrl="foo">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <math math>
+|       <math mn>
+|         definitionURL="foo"
+
+#data
+<!doctype html><html></p><!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <!-- foo -->
+|   <head>
+|   <body>
+
+#data
+<!doctype html><head></head></p><!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <!-- foo -->
+|   <body>
+
+#data
+<!doctype html><body><p><pre>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <pre>
+
+#data
+<!doctype html><body><p><listing>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <listing>
+
+#data
+<!doctype html><p><plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <plaintext>
+
+#data
+<!doctype html><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <h1>
+
+#data
+<!doctype html><form><isindex>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+
+#data
+<!doctype html><isindex action="POST">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       action="POST"
+|       <hr>
+|       <label>
+|         "This is a searchable index. Enter search keywords: "
+|         <input>
+|           name="isindex"
+|       <hr>
+
+#data
+<!doctype html><isindex prompt="this is isindex">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "this is isindex"
+|         <input>
+|           name="isindex"
+|       <hr>
+
+#data
+<!doctype html><isindex type="hidden">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "This is a searchable index. Enter search keywords: "
+|         <input>
+|           name="isindex"
+|           type="hidden"
+|       <hr>
+
+#data
+<!doctype html><isindex name="foo">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "This is a searchable index. Enter search keywords: "
+|         <input>
+|           name="isindex"
+|       <hr>
+
+#data
+<!doctype html><ruby><p><rp>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <p>
+|       <rp>
+
+#data
+<!doctype html><ruby><div><span><rp>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <div>
+|         <span>
+|           <rp>
+
+#data
+<!doctype html><ruby><div><p><rp>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <div>
+|         <p>
+|         <rp>
+
+#data
+<!doctype html><ruby><p><rt>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <p>
+|       <rt>
+
+#data
+<!doctype html><ruby><div><span><rt>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <div>
+|         <span>
+|           <rt>
+
+#data
+<!doctype html><ruby><div><p><rt>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <div>
+|         <p>
+|         <rt>
+
+#data
+<!doctype html><math/><foo>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <math math>
+|     <foo>
+
+#data
+<!doctype html><svg/><foo>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <svg svg>
+|     <foo>
+
+#data
+<!doctype html><div></body><!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <div>
+|   <!-- foo -->
+
+#data
+<!doctype html><h1><div><h3><span></h1>foo
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <h1>
+|       <div>
+|         <h3>
+|           <span>
+|         "foo"
+
+#data
+<!doctype html><p></h3>foo
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       "foo"
+
+#data
+<!doctype html><h3><li>abc</h2>foo
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <h3>
+|       <li>
+|         "abc"
+|     "foo"
+
+#data
+<!doctype html><table>abc<!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "abc"
+|     <table>
+|       <!-- foo -->
+
+#data
+<!doctype html><table>  <!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       "  "
+|       <!-- foo -->
+
+#data
+<!doctype html><table> b <!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " b "
+|     <table>
+|       <!-- foo -->
+
+#data
+<!doctype html><select><option><option>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+|       <option>
+
+#data
+<!doctype html><select><option></optgroup>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+
+#data
+<!doctype html><select><option></optgroup>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+
+#data
+<!doctype html><p><math><mi><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mi>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><p><math><mo><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mo>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><p><math><mn><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mn>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><p><math><ms><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math ms>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><p><math><mtext><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mtext>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><frameset></noframes>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><html c=d><body></html><html a=b>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   a="b"
+|   c="d"
+|   <head>
+|   <body>
+
+#data
+<!doctype html><html c=d><frameset></frameset></html><html a=b>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   a="b"
+|   c="d"
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><html><frameset></frameset></html><!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+| <!-- foo -->
+
+#data
+<!doctype html><html><frameset></frameset></html>  
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   "  "
+
+#data
+<!doctype html><html><frameset></frameset></html>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><html><frameset></frameset></html><p>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><html><frameset></frameset></html></p>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<html><frameset></frameset></html><!doctype html>
+#errors
+#document
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><body><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!doctype html><p><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><p>a<frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       "a"
+
+#data
+<!doctype html><p> <frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><pre><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <pre>
+
+#data
+<!doctype html><listing><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <listing>
+
+#data
+<!doctype html><li><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <li>
+
+#data
+<!doctype html><dd><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <dd>
+
+#data
+<!doctype html><dt><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <dt>
+
+#data
+<!doctype html><button><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <button>
+
+#data
+<!doctype html><applet><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <applet>
+
+#data
+<!doctype html><marquee><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <marquee>
+
+#data
+<!doctype html><object><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <object>
+
+#data
+<!doctype html><table><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+
+#data
+<!doctype html><area><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <area>
+
+#data
+<!doctype html><basefont><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <basefont>
+|   <frameset>
+
+#data
+<!doctype html><bgsound><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <bgsound>
+|   <frameset>
+
+#data
+<!doctype html><br><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <br>
+
+#data
+<!doctype html><embed><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <embed>
+
+#data
+<!doctype html><img><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <img>
+
+#data
+<!doctype html><input><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <input>
+
+#data
+<!doctype html><keygen><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <keygen>
+
+#data
+<!doctype html><wbr><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <wbr>
+
+#data
+<!doctype html><hr><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <hr>
+
+#data
+<!doctype html><textarea></textarea><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+
+#data
+<!doctype html><xmp></xmp><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <xmp>
+
+#data
+<!doctype html><iframe></iframe><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+
+#data
+<!doctype html><select></select><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><svg></svg><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><math></math><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><svg><foreignObject><div> <frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><svg>a</svg><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <svg svg>
+|       "a"
+
+#data
+<!doctype html><svg> </svg><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<html>aaa<frameset></frameset>
+#errors
+#document
+| <html>
+|   <head>
+|   <body>
+|     "aaa"
+
+#data
+<html> a <frameset></frameset>
+#errors
+#document
+| <html>
+|   <head>
+|   <body>
+|     "a "
+
+#data
+<!doctype html><div><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><div><body><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <div>
+
+#data
+<!doctype html><p><math></p>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|     "a"
+
+#data
+<!doctype html><p><math><mn><span></p>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mn>
+|           <span>
+|             <p>
+|             "a"
+
+#data
+<!doctype html><math></html>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <math math>
+
+#data
+<!doctype html><meta charset="ascii">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <meta>
+|       charset="ascii"
+|   <body>
+
+#data
+<!doctype html><meta http-equiv="content-type" content="text/html;charset=ascii">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <meta>
+|       content="text/html;charset=ascii"
+|       http-equiv="content-type"
+|   <body>
+
+#data
+<!doctype html><head><!--aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa--><meta charset="utf8">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <!-- aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -->
+|     <meta>
+|       charset="utf8"
+|   <body>
+
+#data
+<!doctype html><html a=b><head></head><html c=d>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   a="b"
+|   c="d"
+|   <head>
+|   <body>
+
+#data
+<!doctype html><image/>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <img>
+
+#data
+<!doctype html>a<i>b<table>c<b>d</i>e</b>f
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "a"
+|     <i>
+|       "bc"
+|       <b>
+|         "de"
+|       "f"
+|       <table>
+
+#data
+<!doctype html><table><i>a<b>b<div>c<a>d</i>e</b>f
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <b>
+|         "b"
+|     <b>
+|     <div>
+|       <b>
+|         <i>
+|           "c"
+|           <a>
+|             "d"
+|         <a>
+|           "e"
+|       <a>
+|         "f"
+|     <table>
+
+#data
+<!doctype html><i>a<b>b<div>c<a>d</i>e</b>f
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <b>
+|         "b"
+|     <b>
+|     <div>
+|       <b>
+|         <i>
+|           "c"
+|           <a>
+|             "d"
+|         <a>
+|           "e"
+|       <a>
+|         "f"
+
+#data
+<!doctype html><table><i>a<b>b<div>c</i>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <b>
+|         "b"
+|     <b>
+|       <div>
+|         <i>
+|           "c"
+|     <table>
+
+#data
+<!doctype html><table><i>a<b>b<div>c<a>d</i>e</b>f
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <b>
+|         "b"
+|     <b>
+|     <div>
+|       <b>
+|         <i>
+|           "c"
+|           <a>
+|             "d"
+|         <a>
+|           "e"
+|       <a>
+|         "f"
+|     <table>
+
+#data
+<!doctype html><table><i>a<div>b<tr>c<b>d</i>e
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <div>
+|         "b"
+|     <i>
+|       "c"
+|       <b>
+|         "d"
+|     <b>
+|       "e"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table><td><table><i>a<div>b<b>c</i>d
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <i>
+|               "a"
+|             <div>
+|               <i>
+|                 "b"
+|                 <b>
+|                   "c"
+|               <b>
+|                 "d"
+|             <table>
+
+#data
+<!doctype html><body><bgsound>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <bgsound>
+
+#data
+<!doctype html><body><basefont>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <basefont>
+
+#data
+<!doctype html><a><b></a><basefont>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <a>
+|       <b>
+|     <basefont>
+
+#data
+<!doctype html><a><b></a><bgsound>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <a>
+|       <b>
+|     <bgsound>
+
+#data
+<!doctype html><figcaption><article></figcaption>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <figcaption>
+|       <article>
+|     "a"
+
+#data
+<!doctype html><summary><article></summary>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <summary>
+|       <article>
+|     "a"
+
+#data
+<!doctype html><p><a><plaintext>b
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <a>
+|     <plaintext>
+|       <a>
+|         "b"
+
+#data
+<!DOCTYPE html><div>a<a></div>b<p>c</p>d
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <div>
+|       "a"
+|       <a>
+|     <a>
+|       "b"
+|       <p>
+|         "c"
+|       "d"
diff --git a/third_party/code.google.com/p/go.net/html/testdata/webkit/tests2.dat b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests2.dat
new file mode 100644
index 000000000..60d859221
--- /dev/null
+++ b/third_party/code.google.com/p/go.net/html/testdata/webkit/tests2.dat
@@ -0,0 +1,763 @@
+#data
+<!DOCTYPE html>Test
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "Test"
+
+#data
+<textarea>test</div>test
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+Line: 1 Col: 24 Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "test</div>test"
+
+#data
+<table><td>
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected table cell start tag (td) in the table body phase.
+Line: 1 Col: 11 Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+
+#data
+<table><td>test</tbody></table>
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected table cell start tag (td) in the table body phase.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             "test"
+
+#data
+<frame>test
+#errors
+Line: 1 Col: 7 Unexpected start tag (frame). Expected DOCTYPE.
+Line: 1 Col: 7 Unexpected start tag frame. Ignored.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "test"
+
+#data
+<!DOCTYPE html><frameset>test
+#errors
+Line: 1 Col: 29 Unepxected characters in the frameset phase. Characters ignored.
+Line: 1 Col: 29 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!DOCTYPE html><frameset><!DOCTYPE html>
+#errors
+Line: 1 Col: 40 Unexpected DOCTYPE. Ignored.
+Line: 1 Col: 40 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!DOCTYPE html><font><p><b>test</font>
+#errors
+Line: 1 Col: 38 End tag (font) violates step 1, paragraph 3 of the adoption agency algorithm.
+Line: 1 Col: 38 End tag (font) violates step 1, paragraph 3 of the adoption agency algorithm.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <font>
+|     <p>
+|       <font>
+|         <b>
+|           "test"
+
+#data
+<!DOCTYPE html><dt><div><dd>
+#errors
+Line: 1 Col: 28 Missing end tag (div, dt).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <dt>
+|       <div>
+|     <dd>
+
+#data
+<script></x
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</x"
+|   <body>
+
+#data
+<table><plaintext><td>
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 18 Unexpected start tag (plaintext) in table context caused voodoo mode.
+Line: 1 Col: 22 Unexpected end of file. Expected table content.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "<td>"
+|     <table>
+
+#data
+<plaintext></plaintext>
+#errors
+Line: 1 Col: 11 Unexpected start tag (plaintext). Expected DOCTYPE.
+Line: 1 Col: 23 Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+
+#data
+<!DOCTYPE html><table><tr>TEST
+#errors
+Line: 1 Col: 30 Unexpected non-space characters in table context caused voodoo mode.
+Line: 1 Col: 30 Unexpected end of file. Expected table content.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "TEST"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!DOCTYPE html><body t1=1><body t2=2><body t3=3 t4=4>
+#errors
+Line: 1 Col: 37 Unexpected start tag (body).
+Line: 1 Col: 53 Unexpected start tag (body).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     t1="1"
+|     t2="2"
+|     t3="3"
+|     t4="4"
+
+#data
+</b test
+#errors
+Line: 1 Col: 8 Unexpected end of file in attribute name.
+Line: 1 Col: 8 End tag contains unexpected attributes.
+Line: 1 Col: 8 Unexpected end tag (b). Expected DOCTYPE.
+Line: 1 Col: 8 Unexpected end tag (b) after the (implied) root element.
+#document
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html></b test<b &=&>X
+#errors
+Line: 1 Col: 32 Named entity didn't end with ';'.
+Line: 1 Col: 33 End tag contains unexpected attributes.
+Line: 1 Col: 33 Unexpected end tag (b) after the (implied) root element.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "X"
+
+#data
+<!doctypehtml><scrIPt type=text/x-foobar;baz>X</SCRipt
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+Line: 1 Col: 54 Unexpected end of file in the tag name.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       type="text/x-foobar;baz"
+|       "X</SCRipt"
+|   <body>
+
+#data
+&
+#errors
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&"
+
+#data
+&#
+#errors
+Line: 1 Col: 1 Numeric entity expected. Got end of file instead.
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&#"
+
+#data
+&#X
+#errors
+Line: 1 Col: 3 Numeric entity expected but none found.
+Line: 1 Col: 3 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&#X"
+
+#data
+&#x
+#errors
+Line: 1 Col: 3 Numeric entity expected but none found.
+Line: 1 Col: 3 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&#x"
+
+#data
+-
+#errors
+Line: 1 Col: 4 Numeric entity didn't end with ';'.
+Line: 1 Col: 4 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "-"
+
+#data
+&x-test
+#errors
+Line: 1 Col: 1 Named entity expected. Got none.
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&x-test"
+
+#data
+<!doctypehtml><p><li>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <li>
+
+#data
+<!doctypehtml><p><dt>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <dt>
+
+#data
+<!doctypehtml><p><dd>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <dd>
+
+#data
+<!doctypehtml><p><form>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+Line: 1 Col: 23 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <form>
+
+#data
+<!DOCTYPE html><p></P>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     "X"
+
+#data
+&
+#errors
+Line: 1 Col: 4 Named entity didn't end with ';'.
+Line: 1 Col: 4 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&"
+
+#data
+&AMp;
+#errors
+Line: 1 Col: 1 Named entity expected. Got none.
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&AMp;"
+
+#data
+<!DOCTYPE html><html><head></head><body><thisISasillyTESTelementNameToMakeSureCrazyTagNamesArePARSEDcorrectLY>
+#errors
+Line: 1 Col: 110 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <thisisasillytestelementnametomakesurecrazytagnamesareparsedcorrectly>
+
+#data
+<!DOCTYPE html>X</body>X
+#errors
+Line: 1 Col: 24 Unexpected non-space characters in the after body phase.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "XX"
+
+#data
+<!DOCTYPE html><!-- X
+#errors
+Line: 1 Col: 21 Unexpected end of file in comment.
+#document
+| <!DOCTYPE html>
+| <!--  X -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><table><caption>test TEST</caption><td>test
+#errors
+Line: 1 Col: 54 Unexpected table cell start tag (td) in the table body phase.
+Line: 1 Col: 58 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         "test TEST"
+|       <tbody>
+|         <tr>
+|           <td>
+|             "test"
+
+#data
+<!DOCTYPE html><select><option><optgroup>
+#errors
+Line: 1 Col: 41 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+|       <optgroup>
+
+#data
+<!DOCTYPE html><select><optgroup><option></optgroup><option><select><option>
+#errors
+Line: 1 Col: 68 Unexpected select start tag in the select phase treated as select end tag.
+Line: 1 Col: 76 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <optgroup>
+|         <option>
+|       <option>
+|     <option>
+
+#data
+<!DOCTYPE html><select><optgroup><option><optgroup>
+#errors
+Line: 1 Col: 51 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <optgroup>
+|         <option>
+|       <optgroup>
+
+#data
+<!DOCTYPE html><datalist><option>foo</datalist>bar
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <datalist>
+|       <option>
+|         "foo"
+|     "bar"
+
+#data
+<!DOCTYPE html><font><input><input></font>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <font>
+|       <input>
+|       <input>
+
+#data
+<!DOCTYPE html><!-- XXX - XXX -->
+#errors
+#document
+| <!DOCTYPE html>
+| <!--  XXX - XXX  -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><!-- XXX - XXX
+#errors
+Line: 1 Col: 29 Unexpected end of file in comment (-)
+#document
+| <!DOCTYPE html>
+| <!--  XXX - XXX -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><!-- XXX - XXX - XXX -->
+#errors
+#document
+| <!DOCTYPE html>
+| <!--  XXX - XXX - XXX  -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<isindex test=x name=x>
+#errors
+Line: 1 Col: 23 Unexpected start tag (isindex). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected start tag isindex. Don't use it!
+#document
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "This is a searchable index. Enter search keywords: "
+|         <input>
+|           name="isindex"
+|           test="x"
+|       <hr>
+
+#data
+test
+test
+#errors
+Line: 2 Col: 4 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "test
+test"
+
+#data
+<!DOCTYPE html><body><title>test</body>
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       "test</body>"
+
+#data
+<!DOCTYPE html><body><title>X
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       "X"
+|     <meta>
+|       name="z"
+|     <link>
+|       rel="foo"
+|     <style>
+|       "
+x { content:"</style" } "
+
+#data
+<!DOCTYPE html><select><optgroup></optgroup></select>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <optgroup>
+
+#data
+ 
+ 
+#errors
+Line: 2 Col: 1 Unexpected End of file. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html>  <html>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><script>
+</script>  <title>x  
+#errors
+#document
+| 
+| 
+|   
+|     
+#errors
+Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected start tag (script) that can be in head. Moved.
+#document
+| 
+|   
+|     
+#errors
+Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE.
+Line: 1 Col: 28 Unexpected start tag (style) that can be in head. Moved.
+#document
+| 
+|   
+|     
+#errors
+Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE.
+#document
+| 
+|   
+|   
+|   
+|     "x"
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (style).
+#document
+| 
+|   
+|      --> x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|      

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       ddd
+#errors
+#document
+| 
+|   
+|     
+#errors
+#document
+| 
+|   
+|   
+|     
+|     
  • 
+|       
+|         ",
+		"