implement rest of tokenizer states

[peach-html5-editor.git] / parse-html.coffee

diff --git a/parse-html.coffee b/parse-html.coffee
index c16f690..f64c734 100644 (file)
--- a/parse-html.coffee
+++ b/parse-html.coffee
@@ -47,6 +47,12 @@
 #   0: a "end of the list", "current node", "bottommost", "last"
 
 
 #   0: a "end of the list", "current node", "bottommost", "last"
 
 

+# browser
+# note: to get this to run outside a browser, you'll have to write a native
+# implementation of decode_named_char_ref()
+unless module?.exports?
+       window.wheic = {}
+       module = exports: window.wheic

 
 # Each node is an obect of the Node class. Here are the Node types:
 TYPE_TAG = 0 # name, {attributes}, [children]
 
 # Each node is an obect of the Node class. Here are the Node types:
 TYPE_TAG = 0 # name, {attributes}, [children]

@@ -85,6 +91,7 @@ class Node
                @children = args.children ? []
                @namespace = args.namespace ? NS_HTML
                @parent = args.parent ? null
                @children = args.children ? []
                @namespace = args.namespace ? NS_HTML
                @parent = args.parent ? null

+               @token = args.token ? null

                if args.id?
                        @id = "#{args.id}+"
                else
                if args.id?
                        @id = "#{args.id}+"
                else

@@ -93,9 +100,12 @@ class Node
                # WARNING this doesn't work right on open tags that are still being parsed
                attrs = {}
                attrs[k] = v for k, v of @attrs
                # WARNING this doesn't work right on open tags that are still being parsed
                attrs = {}
                attrs[k] = v for k, v of @attrs

-               return new Node @type, name: @name, text: @text, attrs: attrs, namespace: @namespace, id: @id
+               return new Node @type, name: @name, text: @text, attrs: attrs, namespace: @namespace, id: @id, token: @token

        acknowledge_self_closing: ->
        acknowledge_self_closing: ->

-               @flag 'did_self_close', true
+               if @token?
+                       @token.flag 'did_self_close'
+               else
+                       @flag 'did_self_close', true

        flag: ->
                # fixfull
        serialize: (shallow = false, show_ids = false) -> # for unit tests
        flag: ->
                # fixfull
        serialize: (shallow = false, show_ids = false) -> # for unit tests

@@ -154,8 +164,10 @@ new_element = (name) ->
 new_text_node = (txt) ->
        return new Node TYPE_TEXT, text: txt
 new_character_token = new_text_node
 new_text_node = (txt) ->
        return new Node TYPE_TEXT, text: txt
 new_character_token = new_text_node

-new_comment_node = (txt) ->
+new_comment_token = (txt) ->

        return new Node TYPE_COMMENT, text: txt
        return new Node TYPE_COMMENT, text: txt

+new_doctype_token = (name) ->
+       return new Node TYPE_DOCTYPE, name: name

 new_eof_token = ->
        return new Node TYPE_EOF
 new_afe_marker = ->
 new_eof_token = ->
        return new Node TYPE_EOF
 new_afe_marker = ->

@@ -169,11 +181,29 @@ digits = "0123456789"
 alnum = lc_alpha + uc_alpha + digits
 hex_chars = digits + "abcdefABCDEF"
 
 alnum = lc_alpha + uc_alpha + digits
 hex_chars = digits + "abcdefABCDEF"
 

+is_uc_alpha = (str) ->
+       return str.length is 1 and uc_alpha.indexOf(str) > -1
+is_lc_alpha = (str) ->
+       return str.length is 1 and lc_alpha.indexOf(str) > -1
+

 # some SVG elements have dashes in them
 tag_name_chars = alnum + "-"
 
 # http://www.w3.org/TR/html5/infrastructure.html#space-character
 space_chars = "\u0009\u000a\u000c\u000d\u0020"
 # some SVG elements have dashes in them
 tag_name_chars = alnum + "-"
 
 # http://www.w3.org/TR/html5/infrastructure.html#space-character
 space_chars = "\u0009\u000a\u000c\u000d\u0020"

+is_space = (txt) ->
+       return txt.length is 1 and space_chars.indexOf(txt) > -1
+is_space_tok = (t) ->
+       return t.type is TYPE_TEXT && t.text.length is 1 and space_chars.indexOf(t.text) > -1
+
+is_input_hidden_tok = (t) ->
+       return unless t.type is TYPE_START_TAG
+       for a of t.attrs_a
+               if a[0] is 'type'
+                       if a[1].toLowerCase() is 'hidden'
+                               return true
+                       return false
+       return false

 
 # https://en.wikipedia.org/wiki/Whitespace_character#Unicode
 whitespace_chars = "\u0009\u000a\u000b\u000c\u000d\u0020\u0085\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000"
 
 # https://en.wikipedia.org/wiki/Whitespace_character#Unicode
 whitespace_chars = "\u0009\u000a\u000b\u000c\u000d\u0020\u0085\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000"

@@ -315,9 +345,7 @@ end_tag_implied = {
 }
 
 el_is_special = (e) ->
 }
 
 el_is_special = (e) ->

-       return special_elements[e.name]?
-       # FIXME it should really be:
-       #return special_elements[e.name] is e.namespace
+       return special_elements[e.name] is e.namespace

 
 # decode_named_char_ref()
 #
 
 # decode_named_char_ref()
 #

@@ -343,8 +371,8 @@ decode_named_char_ref = (txt) ->
 
 parse_html = (txt, parse_error_cb = null) ->
        cur = 0 # index of next char in txt to be parsed
 
 parse_html = (txt, parse_error_cb = null) ->
        cur = 0 # index of next char in txt to be parsed

-       # declare tree and tokenizer variables so they're in scope below
-       tree = null
+       # declare doc and tokenizer variables so they're in scope below
+       doc = null

        open_els = null # stack of open elements
        afe = null # active formatting elements
        template_insertion_modes = null
        open_els = null # stack of open elements
        afe = null # active formatting elements
        template_insertion_modes = null

@@ -359,6 +387,12 @@ parse_html = (txt, parse_error_cb = null) ->
        form_element_pointer = null
        temporary_buffer = null
        pending_table_character_tokens = null
        form_element_pointer = null
        temporary_buffer = null
        pending_table_character_tokens = null

+       head_element_pointer = null
+       flag_fragment_parsing = null
+       context_element = null
+
+       stop_parsing = ->
+               flag_parsing = false

 
        parse_error = ->
                if parse_error_cb?
 
        parse_error = ->
                if parse_error_cb?

@@ -430,7 +464,7 @@ parse_html = (txt, parse_error_cb = null) ->
                for t in open_els
                        if t.name is tag_name and (namespace is null or namespace is t.namespace)
                                return true
                for t in open_els
                        if t.name is tag_name and (namespace is null or namespace is t.namespace)
                                return true

-                       if t.ns isnt NS_HTML t.name isnt 'optgroup' and t.name isnt 'option'
+                       if t.ns isnt NS_HTML and t.name isnt 'optgroup' and t.name isnt 'option'

                                return false
                return false
        # this checks for a particular element, not by name
                                return false
                return false
        # this checks for a particular element, not by name

@@ -442,6 +476,49 @@ parse_html = (txt, parse_error_cb = null) ->
                                return false
                return false
 
                                return false
                return false
 

+       clear_to_table_stopers = {
+               'table': true
+               'template': true
+               'html': true
+       }
+       clear_stack_to_table_context = ->
+               loop
+                       if clear_to_table_stopers[open_els[0].name]?
+                               break
+                       open_els.shift()
+               return
+       clear_to_table_body_stopers = {
+               'tbody': true
+               'tfoot': true
+               'thead': true
+               'template': true
+               'html': true
+       }
+       clear_stack_to_table_body_context = ->
+               loop
+                       if clear_to_table_body_stopers[open_els[0].name]?
+                               break
+                       open_els.shift()
+               return
+       clear_to_table_row_stopers = {
+               'tr': true
+               'template': true
+               'html': true
+       }
+       clear_stack_to_table_row_context = ->
+               loop
+                       if clear_to_table_row_stopers[open_els[0].name]?
+                               break
+                       open_els.shift()
+               return
+       clear_afe_to_marker = ->
+               loop
+                       return unless afe.length > 0 # this happens in fragment case, ?spec error
+                       el = afe.shift()
+                       if el.type is TYPE_AFE_MARKER
+                               return
+               return
+

        # 8.2.3.1 ...
        # http://www.w3.org/TR/html5/syntax.html#reset-the-insertion-mode-appropriately
        reset_insertion_mode = ->
        # 8.2.3.1 ...
        # http://www.w3.org/TR/html5/syntax.html#reset-the-insertion-mode-appropriately
        reset_insertion_mode = ->

@@ -565,6 +642,14 @@ parse_html = (txt, parse_error_cb = null) ->
                        node = open_els[node_i]
                        # 19. Return to the step labeled loop.
 
                        node = open_els[node_i]
                        # 19. Return to the step labeled loop.
 

+       # 8.2.3.2
+
+       # http://www.w3.org/TR/html5/syntax.html#adjusted-current-node
+       adjusted_current_node = ->
+               if open_els.length is 1 and flag_fragment_parsing
+                       return context_element
+               return open_els[0]
+

        # http://www.w3.org/TR/html5/syntax.html#reconstruct-the-active-formatting-elements
        # this implementation is structured (mostly) as described at the link above.
        # capitalized comments are the "labels" described at the link above.
        # http://www.w3.org/TR/html5/syntax.html#reconstruct-the-active-formatting-elements
        # this implementation is structured (mostly) as described at the link above.
        # capitalized comments are the "labels" described at the link above.

@@ -597,7 +682,7 @@ parse_html = (txt, parse_error_cb = null) ->
        #   http://www.w3.org/TR/html5/syntax.html#unclosed-formatting-elements
        adoption_agency = (subject) ->
                debug_log "adoption_agency()"
        #   http://www.w3.org/TR/html5/syntax.html#unclosed-formatting-elements
        adoption_agency = (subject) ->
                debug_log "adoption_agency()"

-               debug_log "tree: #{serialize_els tree.children, false, true}"
+               debug_log "tree: #{serialize_els doc.children, false, true}"

                debug_log "open_els: #{serialize_els open_els, true, true}"
                debug_log "afe: #{serialize_els afe, true, true}"
                if open_els[0].name is subject
                debug_log "open_els: #{serialize_els open_els, true, true}"
                debug_log "afe: #{serialize_els afe, true, true}"
                if open_els[0].name is subject

@@ -709,7 +794,7 @@ parse_html = (txt, parse_error_cb = null) ->
                                                break
                                node = node_next ? node_above
                                debug_log "inner loop #{inner}"
                                                break
                                node = node_next ? node_above
                                debug_log "inner loop #{inner}"

-                               debug_log "tree: #{serialize_els tree.children, false, true}"
+                               debug_log "tree: #{serialize_els doc.children, false, true}"

                                debug_log "open_els: #{serialize_els open_els, true, true}"
                                debug_log "afe: #{serialize_els afe, true, true}"
                                debug_log "ca: #{ca.name}##{ca.id} children: #{serialize_els ca.children, true, true}"
                                debug_log "open_els: #{serialize_els open_els, true, true}"
                                debug_log "afe: #{serialize_els afe, true, true}"
                                debug_log "ca: #{ca.name}##{ca.id} children: #{serialize_els ca.children, true, true}"

@@ -803,7 +888,7 @@ parse_html = (txt, parse_error_cb = null) ->
                        # at the appropriate place for inserting a node, but using common
                        # ancestor as the override target.
 
                        # at the appropriate place for inserting a node, but using common
                        # ancestor as the override target.
 

-                       # JASON: In the case where fe is immediately followed by fb:
+                       # In the case where fe is immediately followed by fb:

                        #   * inner loop exits out early (node==fe)
                        #   * last_node is fb
                        #   * last_node is still in the tree (not a duplicate)
                        #   * inner loop exits out early (node==fe)
                        #   * last_node is fb
                        #   * last_node is still in the tree (not a duplicate)

@@ -820,7 +905,7 @@ parse_html = (txt, parse_error_cb = null) ->
                        debug_log "fe: #{fe.name}##{fe.id} children: #{serialize_els fe.children, true, true}"
                        debug_log "fb: #{fb.name}##{fb.id} children: #{serialize_els fb.children, true, true}"
                        debug_log "last_node: #{last_node.name}##{last_node.id} children: #{serialize_els last_node.children, true, true}"
                        debug_log "fe: #{fe.name}##{fe.id} children: #{serialize_els fe.children, true, true}"
                        debug_log "fb: #{fb.name}##{fb.id} children: #{serialize_els fb.children, true, true}"
                        debug_log "last_node: #{last_node.name}##{last_node.id} children: #{serialize_els last_node.children, true, true}"

-                       debug_log "tree: #{serialize_els tree.children, false, true}"
+                       debug_log "tree: #{serialize_els doc.children, false, true}"

 
                        debug_log "insert"
 
 
                        debug_log "insert"
 

@@ -836,7 +921,7 @@ parse_html = (txt, parse_error_cb = null) ->
                        debug_log "fe: #{fe.name}##{fe.id} children: #{serialize_els fe.children, true, true}"
                        debug_log "fb: #{fb.name}##{fb.id} children: #{serialize_els fb.children, true, true}"
                        debug_log "last_node: #{last_node.name}##{last_node.id} children: #{serialize_els last_node.children, true, true}"
                        debug_log "fe: #{fe.name}##{fe.id} children: #{serialize_els fe.children, true, true}"
                        debug_log "fb: #{fb.name}##{fb.id} children: #{serialize_els fb.children, true, true}"
                        debug_log "last_node: #{last_node.name}##{last_node.id} children: #{serialize_els last_node.children, true, true}"

-                       debug_log "tree: #{serialize_els tree.children, false, true}"
+                       debug_log "tree: #{serialize_els doc.children, false, true}"

 
                        # 15. Create an element for the token for which formatting element
                        # was created, in the HTML namespace, with furthest block as the
 
                        # 15. Create an element for the token for which formatting element
                        # was created, in the HTML namespace, with furthest block as the

@@ -876,7 +961,7 @@ parse_html = (txt, parse_error_cb = null) ->
                                        break
                        # 20. Jump back to the step labeled outer loop.
                        debug_log "done wrapping fb's children. new_element: #{new_element.name}##{new_element.id}"
                                        break
                        # 20. Jump back to the step labeled outer loop.
                        debug_log "done wrapping fb's children. new_element: #{new_element.name}##{new_element.id}"

-                       debug_log "tree: #{serialize_els tree.children, false, true}"
+                       debug_log "tree: #{serialize_els doc.children, false, true}"

                        debug_log "open_els: #{serialize_els open_els, true, true}"
                        debug_log "afe: #{serialize_els afe, true, true}"
                debug_log "AAA DONE"
                        debug_log "open_els: #{serialize_els open_els, true, true}"
                        debug_log "afe: #{serialize_els afe, true, true}"
                debug_log "AAA DONE"

@@ -949,7 +1034,7 @@ parse_html = (txt, parse_error_cb = null) ->
                                # last template's template contents, after its last child (if
                                # any), and abort these substeps.
                                if last_template and (last_table is null or last_template_i < last_table_i)
                                # last template's template contents, after its last child (if
                                # any), and abort these substeps.
                                if last_template and (last_table is null or last_template_i < last_table_i)

-                                       target = template # fixfull should be it's contents
+                                       target = last_template # fixfull should be it's contents

                                        target_i = target.children.length
                                        break
                                # 4. If there is no last table, then let adjusted insertion
                                        target_i = target.children.length
                                        break
                                # 4. If there is no last table, then let adjusted insertion

@@ -1007,7 +1092,7 @@ parse_html = (txt, parse_error_cb = null) ->
                while t.attrs_a.length
                        a = t.attrs_a.pop()
                        attrs[a[0]] = a[1] # TODO check what to do with dupilcate attrs
                while t.attrs_a.length
                        a = t.attrs_a.pop()
                        attrs[a[0]] = a[1] # TODO check what to do with dupilcate attrs

-               el = new Node TYPE_TAG, name: t.name, namespace: namespace, attrs: attrs
+               el = new Node TYPE_TAG, name: t.name, namespace: namespace, attrs: attrs, token: t

 
                # TODO 2. If the newly created element has an xmlns attribute in the
                # XMLNS namespace whose value is not exactly the same as the element's
 
                # TODO 2. If the newly created element has an xmlns attribute in the
                # XMLNS namespace whose value is not exactly the same as the element's

@@ -1079,6 +1164,93 @@ parse_html = (txt, parse_error_cb = null) ->
                while end_tag_implied[open_els[0].name] and open_els[0].name isnt except
                        open_els.shift()
 
                while end_tag_implied[open_els[0].name] and open_els[0].name isnt except
                        open_els.shift()
 

+       # 8.2.5.4 The rules for parsing tokens in HTML content
+       # http://www.w3.org/TR/html5/syntax.html#parsing-main-inhtml
+
+       # 8.2.5.4.1 The "initial" insertion mode
+       # http://www.w3.org/TR/html5/syntax.html#the-initial-insertion-mode
+       ins_mode_initial = (t) ->
+               if is_space_tok t
+                       return
+               if t.type is TYPE_COMMENT
+                       # ?fixfull
+                       doc.children.push t
+                       return
+               if t.type is TYPE_DOCTYPE
+                       # FIXME check identifiers, set quirks, etc
+                       # fixfull
+                       doc.children.push t
+                       insertion_mode = ins_mode_before_html
+                       return
+               # Anything else
+               #fixfull (iframe, quirks)
+               insertion_mode = ins_mode_before_html
+               insertion_mode t # reprocess the token
+               return
+
+       # 8.2.5.4.2 http://www.w3.org/TR/html5/syntax.html#the-before-html-insertion-mode
+       ins_mode_before_html = (t) ->
+               if t.type is TYPE_DOCTYPE
+                       parse_error()
+                       return
+               if t.type is TYPE_COMMENT
+                       doc.children.push t
+                       return
+               if is_space_tok t
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'html'
+                       el = token_to_element t, NS_HTML, doc
+                       doc.children.push el
+                       open_els.unshift(el)
+                       # fixfull (big paragraph in spec about manifest, fragment, urls, etc)
+                       insertion_mode = ins_mode_before_head
+                       return
+               if t.type is TYPE_END_TAG
+                       if t.name is 'head' or t.name is 'body' or t.name is 'html' or t.name is 'br'
+                               # fall through to "anything else"
+                       else
+                               parse_error()
+                               return
+               # Anything else
+               html_tok = new_open_tag 'html'
+               el = token_to_element html_tok, NS_HTML, doc
+               doc.children.push el
+               open_els.unshift el
+               # ?fixfull browsing context
+               insertion_mode = ins_mode_before_head
+               insertion_mode t
+               return
+
+       # 8.2.5.4.3 http://www.w3.org/TR/html5/syntax.html#the-before-head-insertion-mode
+       ins_mode_before_head = (t) ->
+               if is_space_tok t
+                       return
+               if t.type is TYPE_COMMENT
+                       insert_comment t
+                       return
+               if t.type is TYPE_DOCTYPE
+                       parse_error()
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'html'
+                       ins_mode_in_body t
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'head'
+                       el = insert_html_element t
+                       head_element_pointer = el
+                       insertion_mode = ins_mode_in_head
+               if t.type is TYPE_END_TAG
+                       if t.name is 'head' or t.name is 'body' or t.name is 'html' or t.name is 'br'
+                               # fall through to Anything else below
+                       else
+                               parse_error()
+                               return
+               # Anything else
+               head_tok = new_open_tag 'head'
+               el = insert_html_element head_tok
+               head_element_pointer = el
+               insertion_mode = ins_mode_in_head
+               insertion_mode t # reprocess current token
+

        # 8.2.5.4.4 http://www.w3.org/TR/html5/syntax.html#parsing-main-inhead
        ins_mode_in_head_else = (t) -> # factored out for same-as-spec flow control
                open_els.shift() # spec says this will be a 'head' node
        # 8.2.5.4.4 http://www.w3.org/TR/html5/syntax.html#parsing-main-inhead
        ins_mode_in_head_else = (t) -> # factored out for same-as-spec flow control
                open_els.shift() # spec says this will be a 'head' node

@@ -1100,28 +1272,28 @@ parse_html = (txt, parse_error_cb = null) ->
                if t.type is TYPE_START_TAG and (t.name is 'base' or t.name is 'basefont' or t.name is 'bgsound' or t.name is 'link')
                        el = insert_html_element t
                        open_els.shift()
                if t.type is TYPE_START_TAG and (t.name is 'base' or t.name is 'basefont' or t.name is 'bgsound' or t.name is 'link')
                        el = insert_html_element t
                        open_els.shift()

-                       el.acknowledge_self_closing()
+                       t.acknowledge_self_closing()

                        return
                if t.type is TYPE_START_TAG and t.name is 'meta'
                        el = insert_html_element t
                        open_els.shift()
                        return
                if t.type is TYPE_START_TAG and t.name is 'meta'
                        el = insert_html_element t
                        open_els.shift()

-                       el.acknowledge_self_closing()
+                       t.acknowledge_self_closing()

                        # fixfull encoding stuff
                        return
                if t.type is TYPE_START_TAG and t.name is 'title'
                        # fixfull encoding stuff
                        return
                if t.type is TYPE_START_TAG and t.name is 'title'

-                       parse_generic_rcdata_element t
+                       parse_generic_rcdata_text t

                        return
                if t.type is TYPE_START_TAG and ((t.name is 'noscript' and flag_scripting) or (t.name is 'noframes' or t.name is 'style'))
                        parse_generic_raw_text t
                        return
                if t.type is TYPE_START_TAG and t.name is 'noscript' and flag_scripting is false
                        insert_html_element t
                        return
                if t.type is TYPE_START_TAG and ((t.name is 'noscript' and flag_scripting) or (t.name is 'noframes' or t.name is 'style'))
                        parse_generic_raw_text t
                        return
                if t.type is TYPE_START_TAG and t.name is 'noscript' and flag_scripting is false
                        insert_html_element t

-                       insertion_mode = in_head_noscript # FIXME implement
+                       insertion_mode = ins_mode_in_head_noscript # FIXME implement

                        return
                if t.type is TYPE_START_TAG and t.name is 'script'
                        ail = adjusted_insertion_location()
                        el = token_to_element t, NS_HTML, ail
                        return
                if t.type is TYPE_START_TAG and t.name is 'script'
                        ail = adjusted_insertion_location()
                        el = token_to_element t, NS_HTML, ail

-                       el.flag_parser_inserted true # FIXME implement
+                       el.flag 'parser-inserted', true # FIXME implement

                        # fixfull frament case
                        ail[0].children.splice ail[1], 0, el
                        open_els.unshift el
                        # fixfull frament case
                        ail[0].children.splice ail[1], 0, el
                        open_els.unshift el

@@ -1158,11 +1330,67 @@ parse_html = (txt, parse_error_cb = null) ->
                        else
                                parse_error()
                        return
                        else
                                parse_error()
                        return

-               if (t.type is TYPE_OPEN_TAG and t.name is 'head') or t.type is TYPE_END_TAG
+               if (t.type is TYPE_START_TAG and t.name is 'head') or t.type is TYPE_END_TAG

                        parse_error()
                        return
                ins_mode_in_head_else t
 
                        parse_error()
                        return
                ins_mode_in_head_else t
 

+       # 8.2.5.4.5 http://www.w3.org/TR/html5/syntax.html#parsing-main-inheadnoscript
+       ins_mode_in_head_noscript = (t) ->
+               # FIXME ?fixfull
+               console.log "ins_mode_in_head_noscript unimplemented"
+
+       # 8.2.5.4.6 http://www.w3.org/TR/html5/syntax.html#the-after-head-insertion-mode
+       ins_mode_after_head_else = (t) ->
+               body_tok = new_open_tag 'body'
+               insert_html_element body_tok
+               insertion_mode = ins_mode_in_body
+               insertion_mode t # reprocess token
+               return
+       ins_mode_after_head = (t) ->
+               if is_space_tok t
+                       insert_character t
+                       return
+               if t.type is TYPE_COMMENT
+                       insert_comment t
+                       return
+               if t.type is TYPE_DOCTYPE
+                       parse_error()
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'html'
+                       ins_mode_in_body t
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'body'
+                       insert_html_element t
+                       flag_frameset_ok = false
+                       insertion_mode = ins_mode_in_body
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'frameset'
+                       insert_html_element t
+                       insertion_mode = ins_mode_in_frameset
+                       return
+               if t.type is TYPE_START_TAG and (t.name is 'base' or t.name is 'basefont' or t.name is 'bgsound' or t.name is 'link' or t.name is 'meta' or t.name is 'noframes' or t.name is 'script' or t.name is 'style' or t.name is 'template' or t.name is 'title')
+                       parse_error()
+                       open_els.unshift head_element_pointer
+                       ins_mode_in_head t
+                       for el, i of open_els
+                               if el is head_element_pointer
+                                       open_els.splice i, 1
+                                       return
+                       console.log "warning: 23904 couldn't find head element in open_els"
+                       return
+               if t.type is TYPE_END_TAG and t.name is 'template'
+                       ins_mode_in_head t
+                       return
+               if t.type is TYPE_END_TAG and (t.name is 'body' or t.name is 'html' or t.name is 'br')
+                       ins_mode_after_head_else t
+                       return
+               if (t.type is TYPE_START_TAG and t.name is 'head') or t.type is TYPE_END_TAG
+                       parse_error()
+                       return
+               # Anything else
+               ins_mode_after_head_else t
+

        # 8.2.5.4.7 http://www.w3.org/TR/html5/syntax.html#parsing-main-inbody
        in_body_any_other_end_tag = (name) -> # factored out because adoption agency calls it
                for node, i in open_els
        # 8.2.5.4.7 http://www.w3.org/TR/html5/syntax.html#parsing-main-inbody
        in_body_any_other_end_tag = (name) -> # factored out because adoption agency calls it
                for node, i in open_els

@@ -1272,7 +1500,7 @@ parse_html = (txt, parse_error_cb = null) ->
                                                parse_error()
                                                break
                                # TODO stack of template insertion modes thing
                                                parse_error()
                                                break
                                # TODO stack of template insertion modes thing

-                               flag_parsing = false # stop parsing
+                               stop_parsing()

                        when TYPE_END_TAG
                                switch t.name
                                        when 'body'
                        when TYPE_END_TAG
                                switch t.name
                                        when 'body'

@@ -1315,53 +1543,13 @@ parse_html = (txt, parse_error_cb = null) ->
                flag_foster_parenting = true # FIXME
                ins_mode_in_body t
                flag_foster_parenting = false
                flag_foster_parenting = true # FIXME
                ins_mode_in_body t
                flag_foster_parenting = false

-       can_in_table = {
-               'table': true
-               'tbody': true
-               'tfoot': true
-               'thead': true
-               'tr': true
-       }
-       clear_to_table_stopers = {
+       can_in_table = { # FIXME do this inline like everywhere else

                'table': true
                'table': true

-               'template': true
-               'html': true
-       }
-       clear_stack_to_table_context = ->
-               loop
-                       if clear_to_table_stopers[open_els[0].name]?
-                               break
-                       open_els.shift()
-               return
-       clear_to_table_body_stopers = {

                'tbody': true
                'tfoot': true
                'thead': true
                'tbody': true
                'tfoot': true
                'thead': true

-               'template': true
-               'html': true
-       }
-       clear_stack_to_table_body_context = ->
-               loop
-                       if clear_to_table_body_stopers[open_els[0].name]?
-                               break
-                       open_els.shift()
-               return
-       clear_to_table_row_stopers = {

                'tr': true
                'tr': true

-               'template': true
-               'html': true

        }
        }

-       clear_stack_to_table_row_context = ->
-               loop
-                       if clear_to_table_row_stopers[open_els[0].name]?
-                               break
-                       open_els.shift()
-               return
-       clear_afe_to_marker = ->
-               loop
-                       el = afe.shift()
-                       if el.type is TYPE_AFE_MARKER
-                               return

 
        # 8.2.5.4.8 http://www.w3.org/TR/html5/syntax.html#parsing-main-incdata
        ins_mode_text = (t) ->
 
        # 8.2.5.4.8 http://www.w3.org/TR/html5/syntax.html#parsing-main-incdata
        ins_mode_text = (t) ->

@@ -1442,13 +1630,13 @@ parse_html = (txt, parse_error_cb = null) ->
                                        when 'style', 'script', 'template'
                                                ins_mode_in_head t
                                        when 'input'
                                        when 'style', 'script', 'template'
                                                ins_mode_in_head t
                                        when 'input'

-                                               if token_is_input_hidden t
+                                               if is_input_hidden_tok t

                                                        ins_mode_in_table_else t
                                                else
                                                        parse_error()
                                                        el = insert_html_element t
                                                        open_els.shift()
                                                        ins_mode_in_table_else t
                                                else
                                                        parse_error()
                                                        el = insert_html_element t
                                                        open_els.shift()

-                                                       el.acknowledge_self_closing()
+                                                       t.acknowledge_self_closing()

                                        when 'form'
                                                parse_error()
                                                if form_element_pointer?
                                        when 'form'
                                                parse_error()
                                                if form_element_pointer?

@@ -1494,7 +1682,7 @@ parse_html = (txt, parse_error_cb = null) ->
                # Anything else
                all_space = true
                for old in pending_table_character_tokens
                # Anything else
                all_space = true
                for old in pending_table_character_tokens

-                       unless space_chars.indexOf(old.text) > -1
+                       unless is_space_tok old

                                all_space = false
                                break
                if all_space
                                all_space = false
                                break
                if all_space

@@ -1519,7 +1707,7 @@ parse_html = (txt, parse_error_cb = null) ->
                                        if el.name is 'caption'
                                                break
                                clear_afe_to_marker()
                                        if el.name is 'caption'
                                                break
                                clear_afe_to_marker()

-                               insertion_mode = in_table
+                               insertion_mode = ins_mode_in_table

                        else
                                parse_error()
                                # fragment case
                        else
                                parse_error()
                                # fragment case

@@ -1532,7 +1720,7 @@ parse_html = (txt, parse_error_cb = null) ->
                                        if el.name is 'caption'
                                                break
                                clear_afe_to_marker()
                                        if el.name is 'caption'
                                                break
                                clear_afe_to_marker()

-                               insertion_mode = in_table
+                               insertion_mode = ins_mode_in_table

                                insertion_mode t
                        # else fragment case
                        return
                                insertion_mode t
                        # else fragment case
                        return

@@ -1544,7 +1732,7 @@ parse_html = (txt, parse_error_cb = null) ->
 
        # 8.2.5.4.12 http://www.w3.org/TR/html5/syntax.html#parsing-main-incolgroup
        ins_mode_in_column_group = (t) ->
 
        # 8.2.5.4.12 http://www.w3.org/TR/html5/syntax.html#parsing-main-incolgroup
        ins_mode_in_column_group = (t) ->

-               if t.type is TYPE_TEXT and space_chars.indexOf(t.text) > -1
+               if is_space_tok t

                        insert_character t
                        return
                if t.type is TYPE_COMMENT
                        insert_character t
                        return
                if t.type is TYPE_COMMENT

@@ -1559,11 +1747,11 @@ parse_html = (txt, parse_error_cb = null) ->
                if t.type is TYPE_START_TAG and t.name is 'col'
                        el = insert_html_element t
                        open_els.shift()
                if t.type is TYPE_START_TAG and t.name is 'col'
                        el = insert_html_element t
                        open_els.shift()

-                       el.acknowledge_self_closing()
+                       t.acknowledge_self_closing()

                        return
                if t.type is TYPE_END_TAG and t.name is 'colgroup'
                        if open_els[0].name is 'colgroup'
                        return
                if t.type is TYPE_END_TAG and t.name is 'colgroup'
                        if open_els[0].name is 'colgroup'

-                               open_els[0].shift()
+                               open_els.shift()

                                insertion_mode = ins_mode_in_table
                        else
                                parse_error()
                                insertion_mode = ins_mode_in_table
                        else
                                parse_error()

@@ -1726,73 +1914,373 @@ parse_html = (txt, parse_error_cb = null) ->
                # Anything Else
                ins_mode_in_body t
 
                # Anything Else
                ins_mode_in_body t
 

-       # 8.2.4.1 http://www.w3.org/TR/html5/syntax.html#data-state
-       tok_state_data = ->
-               switch c = txt.charAt(cur++)
-                       when '&'
-                               return new_text_node parse_character_reference()
-                       when '<'
-                               tok_state = tok_state_tag_open
-                       when "\u0000"
-                               parse_error()
-                               return new_text_node c
-                       when '' # EOF
-                               return new_eof_token()
+       # 8.2.5.4.16 http://www.w3.org/TR/html5/syntax.html#parsing-main-inselect
+       ins_mode_in_select = (t) ->
+               if t.type is TYPE_TEXT and t.text is "\u0000"
+                       parse_error()
+                       return
+               if t.type is TYPE_TEXT
+                       insert_character t
+                       return
+               if t.type is TYPE_COMMENT
+                       insert_comment t
+                       return
+               if t.type is TYPE_DOCTYPE
+                       parse_error()
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'html'
+                       ins_mode_in_body t
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'option'
+                       if open_els[0].name is 'option'
+                               open_els.shift()
+                       insert_html_element t
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'optgroup'
+                       if open_els[0].name is 'option'
+                               open_els.shift()
+                       if open_els[0].name is 'optgroup'
+                               open_els.shift()
+                       insert_html_element t
+                       return
+               if t.type is TYPE_END_TAG and t.name is 'optgroup'
+                       if open_els[0].name is 'option' and open_els[1].name is 'optgroup'
+                               open_els.shift()
+                       if open_els[0].name is 'optgroup'
+                               open_els.shift()

                        else
                        else

-                               return new_text_node c
-               return null
-
-       # 8.2.4.2 http://www.w3.org/TR/html5/syntax.html#character-reference-in-data-state
-       # not needed: tok_state_character_reference_in_data = ->
-       # just call parse_character_reference()
-
-       # 8.2.4.3 http://www.w3.org/TR/html5/syntax.html#rcdata-state
-       tok_state_rcdata = ->
-               switch c = txt.charAt(cur++)
-                       when '&'
-                               return new_text_node parse_character_reference()
-                       when '<'
-                               tok_state = tok_state_rcdata_less_than_sign
-                       when "\u0000"

                                parse_error()
                                parse_error()

-                               return new_character_token "\ufffd"
-                       when '' # EOF
-                               return new_eof_token()
+                       return
+               if t.type is TYPE_END_TAG and t.name is 'option'
+                       if open_els[0].name is 'option'
+                               open_els.shift()

                        else
                        else

-                               return new_character_token c
-               return null
-
-       # 8.2.4.4 http://www.w3.org/TR/html5/syntax.html#character-reference-in-rcdata-state
-       # not needed: tok_state_character_reference_in_rcdata = ->
-       # just call parse_character_reference()
-
-       # 8.2.4.5 http://www.w3.org/TR/html5/syntax.html#rawtext-state
-       tok_state_rawtext = ->
-               switch c = txt.charAt(cur++)
-                       when '<'
-                               tok_state = tok_state_rawtext_less_than_sign
-                       when "\u0000"

                                parse_error()
                                parse_error()

-                               return new_character_token "\ufffd"
-                       when '' # EOF
-                               return new_eof_token()
+                       return
+               if t.type is TYPE_END_TAG and t.name is 'select'
+                       if is_in_select_scope 'select'
+                               loop
+                                       el = open_els.shift()
+                                       if el.name is 'select'
+                                               break
+                               reset_insertion_mode()

                        else
                        else

-                               return new_character_token c
-               return null
-
-       # 8.2.4.6 http://www.w3.org/TR/html5/syntax.html#script-data-state
-       tok_state_script_data = ->
-               switch c = txt.charAt(cur++)
-                       when '<'
-                               tok_state = tok_state_script_data_less_than_sign
-                       when "\u0000"

                                parse_error()
                                parse_error()

-                               return new_character_token "\ufffd"
-                       when '' # EOF
-                               return new_eof_token()
-                       else
-                               return new_character_token c
-               return null
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'select'
+                       parse_error()
+                       loop
+                               el = open_els.shift()
+                               if el.name is 'select'
+                                       break
+                       reset_insertion_mode()
+                       # spec says that this is the same as </select> but it doesn't say
+                       # to check scope first
+                       return
+               if t.type is TYPE_START_TAG and (t.name is 'input' or t.name is 'keygen' or t.name is 'textarea')
+                       parse_error()
+                       if is_in_select_scope 'select'
+                               return
+                       loop
+                               el = open_els.shift()
+                               if el.name is 'select'
+                                       break
+                       reset_insertion_mode()
+                       insertion_mode t
+                       return
+               if t.type is TYPE_START_TAG and (t.name is 'script' or t.name is 'template')
+                       ins_mode_in_head t
+                       return
+               if t.type is TYPE_EOF
+                       ins_mode_in_body t
+                       return
+               # Anything else
+               parse_error()
+               return
+
+       # 8.2.5.4.17 http://www.w3.org/TR/html5/syntax.html#parsing-main-inselectintable
+       ins_mode_in_select_in_table = (t) ->
+               if t.type is TYPE_START_TAG and (t.name is 'caption' or t.name is 'table' or t.name is 'tbody' or t.name is 'tfoot' or t.name is 'thead' or t.name is 'tr' or t.name is 'td' or t.name is 'th')
+                       parse_error()
+                       loop
+                               el = open_els.shift()
+                               if el.name is 'select'
+                                       break
+                       reset_insertion_mode()
+                       insertion_mode t
+                       return
+               if t.type is TYPE_END_TAG and (t.name is 'caption' or t.name is 'table' or t.name is 'tbody' or t.name is 'tfoot' or t.name is 'thead' or t.name is 'tr' or t.name is 'td' or t.name is 'th')
+                       parse_error()
+                       unless is_in_table_scope t.name, NS_HTML
+                               return
+                       loop
+                               el = open_els.shift()
+                               if el.name is 'select'
+                                       break
+                       reset_insertion_mode()
+                       insertion_mode t
+                       return
+               # Anything else
+               ins_mode_in_select t
+               return
+
+       # 8.2.5.4.18 http://www.w3.org/TR/html5/syntax.html#parsing-main-intemplate
+       ins_mode_in_template = (t) ->
+               if t.type is TYPE_TEXT or t.type is TYPE_COMMENT or t.type is TYPE_DOCTYPE
+                       ins_mode_in_body t
+                       return
+               if (t.type is TYPE_START_TAG and (t.name is 'base' or t.name is 'basefont' or t.name is 'bgsound' or t.name is 'link' or t.name is 'meta' or t.name is 'noframes' or t.name is 'script' or t.name is 'style' or t.name is 'template' or t.name is 'title')) or (t.type is TYPE_END_TAG and t.name is 'template')
+                       ins_mode_in_head t
+                       return
+               if t.type is TYPE_START_TAG and (t.name is 'caption' or t.name is 'colgroup' or t.name is 'tbody' or t.name is 'tfoot' or t.name is 'thead')
+                       template_insertion_modes.shift()
+                       template_insertion_modes.unshift ins_mode_in_table
+                       insertion_mode = ins_mode_in_table
+                       insertion_mode t
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'col'
+                       template_insertion_modes.shift()
+                       template_insertion_modes.unshift ins_mode_in_column_group
+                       insertion_mode = ins_mode_in_column_group
+                       insertion_mode t
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'tr'
+                       template_insertion_modes.shift()
+                       template_insertion_modes.unshift ins_mode_in_table_body
+                       insertion_mode = ins_mode_in_table_body
+                       insertion_mode t
+                       return
+               if t.type is TYPE_START_TAG and (t.name is 'td' or t.name is 'th')
+                       template_insertion_modes.shift()
+                       template_insertion_modes.unshift ins_mode_in_row
+                       insertion_mode = ins_mode_in_row
+                       insertion_mode t
+                       return
+               if t.type is TYPE_START_TAG
+                       template_insertion_modes.shift()
+                       template_insertion_modes.unshift ins_mode_in_body
+                       insertion_mode = ins_mode_in_body
+                       insertion_mode t
+                       return
+               if t.type is TYPE_END_TAG
+                       parse_error()
+                       return
+               if t.type is TYPE_EOF
+                       unless template_tag_is_open()
+                               stop_parsing()
+                               return
+                       parse_error()
+                       loop
+                               el = open_els.shift()
+                               if el.name is 'template' # fixfull check namespace
+                                       break
+                       clear_afe_to_marker()
+                       template_insertion_modes.shift()
+                       reset_insertion_mode()
+                       insertion_mode t
+
+       # 8.2.5.4.19 http://www.w3.org/TR/html5/syntax.html#parsing-main-afterbody
+       ins_mode_after_body = (t) ->
+               if is_space_tok t
+                       ins_mode_in_body t
+                       return
+               if t.type is TYPE_COMMENT
+                       insert_comment t, [open_els[0], open_els[0].children.length]
+                       return
+               if t.type is TYPE_DOCTYPE
+                       parse_error()
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'html'
+                       ins_mode_in_body t
+                       return
+               if t.type is TYPE_END_TAG and t.name is 'html'
+                       # fixfull fragment case
+                       insertion_mode = ins_mode_after_after_body
+                       return
+               if t.type is TYPE_EOF
+                       stop_parsing()
+                       return
+               # Anything ELse
+               parse_error()
+               insertion_mode = ins_mode_in_body
+               insertion_mode t
+
+       # 8.2.5.4.20 http://www.w3.org/TR/html5/syntax.html#parsing-main-inframeset
+       ins_mode_in_frameset = (t) ->
+               if is_space_tok t
+                       insert_character t
+                       return
+               if t.type is TYPE_COMMENT
+                       insert_comment t
+                       return
+               if t.type is TYPE_DOCTYPE
+                       parse_error()
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'html'
+                       ins_mode_in_body t
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'frameset'
+                       insert_html_element t
+                       return
+               if t.type is TYPE_END_TAG and t.name is 'frameset'
+                       # TODO ?correct for: "if the current node is the root html element"
+                       if open_els.length is 1
+                               parse_error()
+                               return # fragment case
+                       open_els.shift()
+                       if flag_fragment_parsing is false and open_els[0].name isnt 'frameset'
+                               insertion_mode = ins_mode_after_frameset
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'frame'
+                       insert_html_element t
+                       open_els.shift()
+                       t.acknowledge_self_closing()
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'noframes'
+                       ins_mode_in_head t
+                       return
+               if t.type is TYPE_EOF
+                       # TODO ?correct for: "if the current node is not the root html element"
+                       if open_els.length isnt 1
+                               parse_error()
+                       stop_parsing()
+                       return
+               # Anything else
+               parse_error()
+               return
+
+       # 8.2.5.4.21 http://www.w3.org/TR/html5/syntax.html#parsing-main-afterframeset
+       ins_mode_after_frameset = (t) ->
+               if is_space_tok t
+                       insert_character t
+                       return
+               if t.type is TYPE_COMMENT
+                       insert_comment t
+                       return
+               if t.type is TYPE_DOCTYPE
+                       parse_error()
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'html'
+                       ins_mode_in_body t
+                       return
+               if t.type is TYPE_END_TAG and t.name is 'html'
+                       insert_mode = ins_mode_after_after_frameset
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'noframes'
+                       ins_mode_in_head t
+                       return
+               if t.type is TYPE_EOF
+                       stop_parsing()
+                       return
+               # Anything else
+               parse_error()
+               return
+
+       # 8.2.5.4.22 http://www.w3.org/TR/html5/syntax.html#the-after-after-body-insertion-mode
+       ins_mode_after_after_body = (t) ->
+               if t.type is TYPE_COMMENT
+                       insert_comment t, [doc, doc.children.length]
+                       return
+               if t.type is TYPE_DOCTYPE or is_space_tok(t) or (t.type is TYPE_START_TAG and t.name is 'html')
+                       ins_mode_in_body t
+                       return
+               if t.type is TYPE_EOF
+                       stop_parsing()
+                       return
+               # Anything else
+               parse_error()
+               insertion_mode = ins_mode_in_body
+               return
+
+       # 8.2.5.4.23 http://www.w3.org/TR/html5/syntax.html#the-after-after-frameset-insertion-mode
+       ins_mode_after_after_frameset = (t) ->
+               if t.type is TYPE_COMMENT
+                       insert_comment t, [doc, doc.children.length]
+                       return
+               if t.type is TYPE_DOCTYPE or is_space_tok(t) or (t.type is TYPE_START_TAG and t.name is 'html')
+                       ins_mode_in_body t
+                       return
+               if t.type is TYPE_EOF
+                       stop_parsing()
+                       return
+               if t.type is TYPE_START_TAG and t.name is 'noframes'
+                       ins_mode_in_head t
+                       return
+               # Anything else
+               parse_error()
+               return
+
+
+
+
+
+       # 8.2.4.1 http://www.w3.org/TR/html5/syntax.html#data-state
+       tok_state_data = ->
+               switch c = txt.charAt(cur++)
+                       when '&'
+                               return new_text_node parse_character_reference()
+                       when '<'
+                               tok_state = tok_state_tag_open
+                       when "\u0000"
+                               parse_error()
+                               return new_text_node c
+                       when '' # EOF
+                               return new_eof_token()
+                       else
+                               return new_text_node c
+               return null
+
+       # 8.2.4.2 http://www.w3.org/TR/html5/syntax.html#character-reference-in-data-state
+       # not needed: tok_state_character_reference_in_data = ->
+       # just call parse_character_reference()
+
+       # 8.2.4.3 http://www.w3.org/TR/html5/syntax.html#rcdata-state
+       tok_state_rcdata = ->
+               switch c = txt.charAt(cur++)
+                       when '&'
+                               return new_text_node parse_character_reference()
+                       when '<'
+                               tok_state = tok_state_rcdata_less_than_sign
+                       when "\u0000"
+                               parse_error()
+                               return new_character_token "\ufffd"
+                       when '' # EOF
+                               return new_eof_token()
+                       else
+                               return new_character_token c
+               return null
+
+       # 8.2.4.4 http://www.w3.org/TR/html5/syntax.html#character-reference-in-rcdata-state
+       # not needed: tok_state_character_reference_in_rcdata = ->
+       # just call parse_character_reference()
+
+       # 8.2.4.5 http://www.w3.org/TR/html5/syntax.html#rawtext-state
+       tok_state_rawtext = ->
+               switch c = txt.charAt(cur++)
+                       when '<'
+                               tok_state = tok_state_rawtext_less_than_sign
+                       when "\u0000"
+                               parse_error()
+                               return new_character_token "\ufffd"
+                       when '' # EOF
+                               return new_eof_token()
+                       else
+                               return new_character_token c
+               return null
+
+       # 8.2.4.6 http://www.w3.org/TR/html5/syntax.html#script-data-state
+       tok_state_script_data = ->
+               switch c = txt.charAt(cur++)
+                       when '<'
+                               tok_state = tok_state_script_data_less_than_sign
+                       when "\u0000"
+                               parse_error()
+                               return new_character_token "\ufffd"
+                       when '' # EOF
+                               return new_eof_token()
+                       else
+                               return new_character_token c
+               return null

 
        # 8.2.4.7 http://www.w3.org/TR/html5/syntax.html#plaintext-state
        tok_state_plaintext = ->
 
        # 8.2.4.7 http://www.w3.org/TR/html5/syntax.html#plaintext-state
        tok_state_plaintext = ->

@@ -1816,12 +2304,13 @@ parse_html = (txt, parse_error_cb = null) ->
                                tok_state = tok_state_end_tag_open
                        when '?'
                                parse_error()
                                tok_state = tok_state_end_tag_open
                        when '?'
                                parse_error()

+                               tok_cur_tag = new_comment_token '?'

                                tok_state = tok_state_bogus_comment
                        else
                                tok_state = tok_state_bogus_comment
                        else

-                               if lc_alpha.indexOf(c) > -1
+                               if is_lc_alpha(c)

                                        tok_cur_tag = new_open_tag c
                                        tok_state = tok_state_tag_name
                                        tok_cur_tag = new_open_tag c
                                        tok_state = tok_state_tag_name

-                               else if uc_alpha.indexOf(c) > -1
+                               else if is_uc_alpha(c)

                                        tok_cur_tag = new_open_tag c.toLowerCase()
                                        tok_state = tok_state_tag_name
                                else
                                        tok_cur_tag = new_open_tag c.toLowerCase()
                                        tok_state = tok_state_tag_name
                                else

@@ -1842,14 +2331,15 @@ parse_html = (txt, parse_error_cb = null) ->
                                tok_state = tok_state_data
                                return new_text_node '</'
                        else
                                tok_state = tok_state_data
                                return new_text_node '</'
                        else

-                               if uc_alpha.indexOf(c) > -1
+                               if is_uc_alpha(c)

                                        tok_cur_tag = new_end_tag c.toLowerCase()
                                        tok_state = tok_state_tag_name
                                        tok_cur_tag = new_end_tag c.toLowerCase()
                                        tok_state = tok_state_tag_name

-                               else if lc_alpha.indexOf(c) > -1
+                               else if is_lc_alpha(c)

                                        tok_cur_tag = new_end_tag c
                                        tok_state = tok_state_tag_name
                                else
                                        parse_error()
                                        tok_cur_tag = new_end_tag c
                                        tok_state = tok_state_tag_name
                                else
                                        parse_error()

+                                       tok_cur_tag = new_comment_token '/'

                                        tok_state = tok_state_bogus_comment
                return null
 
                                        tok_state = tok_state_bogus_comment
                return null
 

@@ -1872,7 +2362,7 @@ parse_html = (txt, parse_error_cb = null) ->
                                parse_error()
                                tok_state = tok_state_data
                        else
                                parse_error()
                                tok_state = tok_state_data
                        else

-                               if uc_alpha.indexOf(c) > -1
+                               if is_uc_alpha(c)

                                        tok_cur_tag.name += c.toLowerCase()
                                else
                                        tok_cur_tag.name += c
                                        tok_cur_tag.name += c.toLowerCase()
                                else
                                        tok_cur_tag.name += c

@@ -1893,12 +2383,12 @@ parse_html = (txt, parse_error_cb = null) ->
        # 8.2.4.12 http://www.w3.org/TR/html5/syntax.html#rcdata-end-tag-open-state
        tok_state_rcdata_end_tag_open = ->
                c = txt.charAt(cur++)
        # 8.2.4.12 http://www.w3.org/TR/html5/syntax.html#rcdata-end-tag-open-state
        tok_state_rcdata_end_tag_open = ->
                c = txt.charAt(cur++)

-               if uc_alpha.indexOf(c) > -1
+               if is_uc_alpha(c)

                        tok_cur_tag = new_end_tag c.toLowerCase()
                        temporary_buffer += c
                        tok_state = tok_state_rcdata_end_tag_name
                        return null
                        tok_cur_tag = new_end_tag c.toLowerCase()
                        temporary_buffer += c
                        tok_state = tok_state_rcdata_end_tag_name
                        return null

-               if lc_alpha.indexOf(c) > -1
+               if is_lc_alpha(c)

                        tok_cur_tag = new_end_tag c
                        temporary_buffer += c
                        tok_state = tok_state_rcdata_end_tag_name
                        tok_cur_tag = new_end_tag c
                        temporary_buffer += c
                        tok_state = tok_state_rcdata_end_tag_name

@@ -1936,11 +2426,11 @@ parse_html = (txt, parse_error_cb = null) ->
                                tok_state = tok_state_data
                                return tok_cur_tag
                        # else fall through to "Anything else"
                                tok_state = tok_state_data
                                return tok_cur_tag
                        # else fall through to "Anything else"

-               if uc_alpha.indexOf(c) > -1
+               if is_uc_alpha(c)

                        tok_cur_tag.name += c.toLowerCase()
                        temporary_buffer += c
                        return null
                        tok_cur_tag.name += c.toLowerCase()
                        temporary_buffer += c
                        return null

-               if lc_alpha.indexOf(c) > -1
+               if is_lc_alpha(c)

                        tok_cur_tag.name += c
                        temporary_buffer += c
                        return null
                        tok_cur_tag.name += c
                        temporary_buffer += c
                        return null

@@ -1964,12 +2454,12 @@ parse_html = (txt, parse_error_cb = null) ->
        # 8.2.4.15 http://www.w3.org/TR/html5/syntax.html#rawtext-end-tag-open-state
        tok_state_rawtext_end_tag_open = ->
                c = txt.charAt(cur++)
        # 8.2.4.15 http://www.w3.org/TR/html5/syntax.html#rawtext-end-tag-open-state
        tok_state_rawtext_end_tag_open = ->
                c = txt.charAt(cur++)

-               if uc_alpha.indexOf(c) > -1
+               if is_uc_alpha(c)

                        tok_cur_tag = new_end_tag c.toLowerCase()
                        temporary_buffer += c
                        tok_state = tok_state_rawtext_end_tag_name
                        return null
                        tok_cur_tag = new_end_tag c.toLowerCase()
                        temporary_buffer += c
                        tok_state = tok_state_rawtext_end_tag_name
                        return null

-               if lc_alpha.indexOf(c) > -1
+               if is_lc_alpha(c)

                        tok_cur_tag = new_end_tag c
                        temporary_buffer += c
                        tok_state = tok_state_rawtext_end_tag_name
                        tok_cur_tag = new_end_tag c
                        temporary_buffer += c
                        tok_state = tok_state_rawtext_end_tag_name

@@ -1997,11 +2487,11 @@ parse_html = (txt, parse_error_cb = null) ->
                                tok_state = tok_state_data
                                return tok_cur_tag
                        # else fall through to "Anything else"
                                tok_state = tok_state_data
                                return tok_cur_tag
                        # else fall through to "Anything else"

-               if uc_alpha.indexOf(c) > -1
+               if is_uc_alpha(c)

                        tok_cur_tag.name += c.toLowerCase()
                        temporary_buffer += c
                        return null
                        tok_cur_tag.name += c.toLowerCase()
                        temporary_buffer += c
                        return null

-               if lc_alpha.indexOf(c) > -1
+               if is_lc_alpha(c)

                        tok_cur_tag.name += c
                        temporary_buffer += c
                        return null
                        tok_cur_tag.name += c
                        temporary_buffer += c
                        return null

@@ -2010,24 +2500,351 @@ parse_html = (txt, parse_error_cb = null) ->
                cur -= 1 # reconsume the input character
                return new_character_token '</' + temporary_buffer # fixfull separate these
 
                cur -= 1 # reconsume the input character
                return new_character_token '</' + temporary_buffer # fixfull separate these
 

-       # TODO _all_ of the missing states here (17-33) are for parsing script tags
+       # 8.2.4.17 http://www.w3.org/TR/html5/syntax.html#script-data-less-than-sign-state
+       tok_state_script_data_less_than_sign = ->
+               c = txt.charAt(cur++)
+               if c is '/'
+                       temporary_buffer = ''
+                       tok_state = tok_state_script_data_end_tag_open
+                       return
+               if c is '!'
+                       tok_state = tok_state_script_data_escape_start
+                       return new_character_token '<!' # fixfull split
+               # Anything else
+               tok_state = tok_state_script_data
+               cur -= 1 # Reconsume
+               return new_character_token '<'
+
+       # 8.2.4.18 http://www.w3.org/TR/html5/syntax.html#script-data-end-tag-open-state
+       tok_state_script_data_end_tag_open = ->
+               c = txt.charAt(cur++)
+               if is_uc_alpha(c)
+                       tok_cur_tag = new_end_tag c.toLowerCase()
+                       temporary_buffer += c
+                       tok_state = tok_state_script_data_end_tag_name
+                       return
+               if is_lc_alpha(c)
+                       tok_cur_tag = new_end_tag c
+                       temporary_buffer += c
+                       tok_state = tok_state_script_data_end_tag_name
+                       return
+               # Anything else
+               tok_state = tok_state_script_data
+               cur -= 1 # Reconsume
+               return new_character_token '</'

 
 

-       # 8.2.4.34 http://www.w3.org/TR/html5/syntax.html#before-attribute-name-state
-       tok_state_before_attribute_name = ->
-               attr_name = null
-               switch c = txt.charAt(cur++)
-                       when "\t", "\n", "\u000c", ' '
-                               return null
-                       when '/'
+       # 8.2.4.19 http://www.w3.org/TR/html5/syntax.html#script-data-end-tag-open-state
+       tok_state_script_data_end_tag_name = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\n" or c is "\u000c" or c is ' '
+                       if is_appropriate_end_tag tok_cur_tag
+                               tok_state = tok_state_before_attribute_name
+                               return
+                       # fall through
+               if c is '/'
+                       if is_appropriate_end_tag tok_cur_tag

                                tok_state = tok_state_self_closing_start_tag
                                tok_state = tok_state_self_closing_start_tag

-                               return null
-                       when '>'
-                               tok_state = tok_state_data
-                               tmp = tok_cur_tag
-                               tok_cur_tag = null
-                               return tmp
-                       when "\u0000"
-                               parse_error()
+                               return
+                       # fall through
+               if is_uc_alpha(c)
+                       tok_cur_tag.name += c.toLowerCase()
+                       temporary_buffer += c
+                       return
+               if is_lc_alpha(c)
+                       tok_cur_tag.name += c
+                       temporary_buffer += c
+                       return
+               # Anything else
+               tok_state = tok_state_script_data
+               cur -= 1 # Reconsume
+               return new_character_token "</#{temporary_buffer}" # fixfull split
+
+       # 8.2.4.20 http://www.w3.org/TR/html5/syntax.html#script-data-escape-start-state
+       tok_state_script_data_escape_start = ->
+               c = txt.charAt(cur++)
+               if c is '-'
+                       tok_state = tok_state_script_data_escape_start_dash
+                       return new_character_token '-'
+               # Anything else
+               tok_state = tok_state_script_data
+               cur -= 1 # Reconsume
+               return
+
+       # 8.2.4.21 http://www.w3.org/TR/html5/syntax.html#script-data-escape-start-dash-state
+       tok_state_script_data_escape_start_dash = ->
+               c = txt.charAt(cur++)
+               if c is '-'
+                       tok_state = tok_state_script_data_escaped_dash_dash
+                       return new_character_token '-'
+               # Anything else
+               tok_state = tok_state_script_data
+               cur -= 1 # Reconsume
+               return
+
+       # 8.2.4.22 http://www.w3.org/TR/html5/syntax.html#script-data-escaped-state
+       tok_state_script_data_escaped = ->
+               c = txt.charAt(cur++)
+               if c is '-'
+                       tok_state = tok_state_script_data_escaped_dash
+                       return new_character_token '-'
+               if c is '<'
+                       tok_state = tok_state_script_data_escaped_less_than_sign
+                       return
+               if c is "\u0000"
+                       parse_error()
+                       return new_character_token "\ufffd"
+               if c is '' # EOF
+                       tok_state = tok_state_data
+                       parse_error()
+                       cur -= 1 # Reconsume
+                       return
+               # Anything else
+               return new_character_token c
+
+       # 8.2.4.23 http://www.w3.org/TR/html5/syntax.html#script-data-escaped-dash-state
+       tok_state_script_data_escaped_dash = ->
+               c = txt.charAt(cur++)
+               if c is '-'
+                       tok_state = tok_state_script_data_escaped_dash_dash
+                       return new_character_token '-'
+               if c is '<'
+                       tok_state = tok_state_script_data_escaped_less_than_sign
+                       return
+               if c is "\u0000"
+                       parse_error()
+                       tok_state = tok_state_script_data_escaped
+                       return new_character_token "\ufffd"
+               if c is '' # EOF
+                       tok_state = tok_state_data
+                       parse_error()
+                       cur -= 1 # Reconsume
+                       return
+               # Anything else
+               tok_state = tok_state_script_data_escaped
+               return new_character_token c
+
+       # 8.2.4.24 http://www.w3.org/TR/html5/syntax.html#script-data-escaped-dash-dash-state
+       tok_state_script_data_escaped_dash_dash = ->
+               c = txt.charAt(cur++)
+               if c is '-'
+                       return new_character_token '-'
+               if c is '<'
+                       tok_state = tok_state_script_data_escaped_less_than_sign
+                       return
+               if c is '>'
+                       tok_state = tok_state_script_data
+                       return new_character_token '>'
+               if c is "\u0000"
+                       parse_error()
+                       tok_state = tok_state_script_data_escaped
+                       return new_character_token "\ufffd"
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       cur -= 1 # Reconsume
+                       return
+               # Anything else
+               tok_state = tok_state_script_data_escaped
+               return new_character_token c
+
+       # 8.2.4.25 http://www.w3.org/TR/html5/syntax.html#script-data-escaped-less-than-sign-state
+       tok_state_script_data_escaped_less_than_sign = ->
+               c = txt.charAt(cur++)
+               if c is '/'
+                       temporary_buffer = ''
+                       tok_state = tok_state_script_data_escaped_end_tag_open
+                       return
+               if is_uc_alpha(c)
+                       temporary_buffer = c.toLowerCase() # yes, really
+                       tok_state = tok_state_script_data_double_escape_start
+                       return new_character_token "<#{c}" # fixfull split
+               if is_lc_alpha(c)
+                       temporary_buffer = c
+                       tok_state = tok_state_script_data_double_escape_start
+                       return new_character_token "<#{c}" # fixfull split
+               # Anything else
+               tok_state = tok_state_script_data_escaped
+               cur -= 1 # Reconsume
+               return new_character_token c
+
+       # 8.2.4.26 http://www.w3.org/TR/html5/syntax.html#script-data-escaped-end-tag-open-state
+       tok_state_script_data_escaped_end_tag_open = ->
+               c = txt.charAt(cur++)
+               if is_uc_alpha(c)
+                       tok_cur_tag = new_end_tag c.toLowerCase()
+                       temporary_buffer += c
+                       tok_state = tok_state_script_data_escaped_end_tag_name
+                       return
+               if is_lc_alpha(c)
+                       tok_cur_tag = new_end_tag c
+                       temporary_buffer += c
+                       tok_state = tok_state_script_data_escaped_end_tag_name
+                       return
+               # Anything else
+               tok_state = tok_state_script_data_escaped
+               cur -= 1 # Reconsume
+               return new_character_token '</' # fixfull split
+
+       # 8.2.4.27 http://www.w3.org/TR/html5/syntax.html#script-data-escaped-end-tag-name-state
+       tok_state_script_data_escaped_end_tag_name = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' '
+                       if is_appropriate_end_tag tok_cur_tag
+                               tok_state = tok_state_before_attribute_name
+                               return
+                       # fall through
+               if c is '/'
+                       if is_appropriate_end_tag tok_cur_tag
+                               tok_state = tok_state_self_closing_start_tag
+                               return
+                       # fall through
+               if is_uc_alpha(c)
+                       tok_cur_tag.name += c.toLowerCase()
+                       temporary_buffer += c.toLowerCase()
+                       return
+               if is_lc_alpha(c)
+                       tok_cur_tag.name += c
+                       temporary_buffer += c.toLowerCase()
+                       return
+               # Anything else
+               tok_state = tok_state_script_data_escaped
+               cur -= 1 # Reconsume
+               return new_character_token "</#{temporary_buffer}" # fixfull split
+
+       # 8.2.4.28 http://www.w3.org/TR/html5/syntax.html#script-data-double-escape-start-state
+       tok_state_script_data_double_escape_start = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' ' or c is '/' or c is '>'
+                       if temporary_buffer is 'script'
+                               tok_state = tok_state_script_data_double_escaped
+                       else
+                               tok_state = tok_state_script_data_escaped
+                       return new_character_token c
+               if is_uc_alpha(c)
+                       temporary_buffer += c.toLowerCase() # yes, really lowercase
+                       return new_character_token c
+               if is_lc_alpha(c)
+                       temporary_buffer += c
+                       return new_character_token c
+               # Anything else
+               tok_state = tok_state_script_data_escaped
+               cur -= 1 # Reconsume
+               return
+
+       # 8.2.4.29 http://www.w3.org/TR/html5/syntax.html#script-data-double-escaped-state
+       tok_state_script_data_double_escaped = ->
+               c = txt.charAt(cur++)
+               if c is '-'
+                       tok_state = tok_state_script_data_double_escaped_dash
+                       return new_character_token '-'
+               if c is '<'
+                       tok_state = tok_state_script_data_double_escaped_less_than_sign
+                       return new_character_token '<'
+               if c is "\u0000"
+                       parse_error()
+                       return new_character_token "\ufffd"
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       cur -= 1 # Reconsume
+                       return
+               # Anything else
+               return new_character_token c
+
+       # 8.2.4.30 http://www.w3.org/TR/html5/syntax.html#script-data-double-escaped-dash-state
+       tok_state_script_data_double_escaped_dash = ->
+               c = txt.charAt(cur++)
+               if c is '-'
+                       tok_state = tok_state_script_data_double_escaped_dash_dash
+                       return new_character_token '-'
+               if c is '<'
+                       tok_state = tok_state_script_data_double_escaped_less_than_sign
+                       return new_character_token '<'
+               if c is "\u0000"
+                       parse_error()
+                       tok_state = tok_state_script_data_double_escaped
+                       return new_character_token "\ufffd"
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       cur -= 1 # Reconsume
+                       return
+               # Anything else
+               tok_state = tok_state_script_data_double_escaped
+               return new_character_token c
+
+       # 8.2.4.31 http://www.w3.org/TR/html5/syntax.html#script-data-double-escaped-dash-dash-state
+       tok_state_script_data_double_escaped_dash_dash = ->
+               c = txt.charAt(cur++)
+               if c is '-'
+                       return new_character_token '-'
+               if c is '<'
+                       tok_state = tok_state_script_data_double_escaped_less_than_sign
+                       return new_character_token '<'
+               if c is '>'
+                       tok_state = tok_state_script_data
+                       return new_character_token '>'
+               if c is "\u0000"
+                       parse_error()
+                       tok_state = tok_state_script_data_double_escaped
+                       return new_character_token "\ufffd"
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       cur -= 1 # Reconsume
+                       return
+               # Anything else
+               tok_state = tok_state_script_data_double_escaped
+               return new_character_token c
+
+       # 8.2.4.32 http://www.w3.org/TR/html5/syntax.html#script-data-double-escaped-less-than-sign-state
+       tok_state_script_data_double_escaped_less_than_sign = ->
+               c = txt.charAt(cur++)
+               if c is '/'
+                       temporary_buffer = ''
+                       tok_state = tok_state_script_data_double_escape_end
+                       return new_character_token '/'
+               # Anything else
+               tok_state = tok_state_script_data_double_escaped
+               cur -= 1 # Reconsume
+               return
+
+       # 8.2.4.33 http://www.w3.org/TR/html5/syntax.html#script-data-double-escape-end-state
+       tok_state_script_data_double_escape_end = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' ' or c is '/' or c is '>'
+                       if temporary_buffer is 'script'
+                               tok_state = tok_state_script_data_escaped
+                       else
+                               tok_state = tok_state_script_data_double_escaped
+                       return new_character_token c
+               if is_uc_alpha(c)
+                       temporary_buffer += c.toLowerCase() # yes, really lowercase
+                       return new_character_token c
+               if is_lc_alpha(c)
+                       temporary_buffer += c
+                       return new_character_token c
+               # Anything else
+               tok_state = tok_state_script_data_double_escaped
+               cur -= 1 # Reconsume
+               return
+
+       # 8.2.4.34 http://www.w3.org/TR/html5/syntax.html#before-attribute-name-state
+       tok_state_before_attribute_name = ->
+               attr_name = null
+               switch c = txt.charAt(cur++)
+                       when "\t", "\n", "\u000c", ' '
+                               return null
+                       when '/'
+                               tok_state = tok_state_self_closing_start_tag
+                               return null
+                       when '>'
+                               tok_state = tok_state_data
+                               tmp = tok_cur_tag
+                               tok_cur_tag = null
+                               return tmp
+                       when "\u0000"
+                               parse_error()

                                attr_name = "\ufffd"
                        when '"', "'", '<', '='
                                parse_error()
                                attr_name = "\ufffd"
                        when '"', "'", '<', '='
                                parse_error()

@@ -2036,7 +2853,7 @@ parse_html = (txt, parse_error_cb = null) ->
                                parse_error()
                                tok_state = tok_state_data
                        else
                                parse_error()
                                tok_state = tok_state_data
                        else

-                               if uc_alpha.indexOf(c) > -1
+                               if is_uc_alpha(c)

                                        attr_name = c.toLowerCase()
                                else
                                        attr_name = c
                                        attr_name = c.toLowerCase()
                                else
                                        attr_name = c

@@ -2069,7 +2886,7 @@ parse_html = (txt, parse_error_cb = null) ->
                                parse_error()
                                tok_state = tok_state_data
                        else
                                parse_error()
                                tok_state = tok_state_data
                        else

-                               if uc_alpha.indexOf(c) > -1
+                               if is_uc_alpha(c)

                                        tok_cur_tag.attrs_a[0][0] = c.toLowerCase()
                                else
                                        tok_cur_tag.attrs_a[0][0] += c
                                        tok_cur_tag.attrs_a[0][0] = c.toLowerCase()
                                else
                                        tok_cur_tag.attrs_a[0][0] += c

@@ -2089,7 +2906,7 @@ parse_html = (txt, parse_error_cb = null) ->
                if c is '>'
                        tok_state = tok_state_data
                        return
                if c is '>'
                        tok_state = tok_state_data
                        return

-               if uc_alpha.indexOf(c) > -1
+               if is_uc_alpha(c)

                        tok_cur_tag.attrs_a.unshift [c.toLowerCase(), '']
                        tok_state = tok_state_attribute_name
                        return
                        tok_cur_tag.attrs_a.unshift [c.toLowerCase(), '']
                        tok_state = tok_state_attribute_name
                        return

@@ -2217,6 +3034,619 @@ parse_html = (txt, parse_error_cb = null) ->
                                cur -= 1 # we didn't handle that char
                return null
 
                                cur -= 1 # we didn't handle that char
                return null
 

+       # 8.2.4.43 http://www.w3.org/TR/html5/syntax.html#self-closing-start-tag-state
+       tok_state_self_closing_start_tag = ->
+               c = txt.charAt(cur++)
+               if c is '>'
+                       tok_cur_tag.flag 'self-closing'
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is ''
+                       parse_error()
+                       tok_state = tok_state_data
+                       cur -= 1 # Reconsume
+                       return
+               # Anything else
+               parse_error()
+               tok_state = tok_state_before_attribute_name
+               cur -= 1 # Reconsume
+               return
+
+       # 8.2.4.44 http://www.w3.org/TR/html5/syntax.html#bogus-comment-state
+       # WARNING: put a comment token in tok_cur_tag before setting this state
+       tok_state_bogus_comment = ->
+               next_gt = txt.indexOf '>', cur
+               if next_gt is -1
+                       val = txt.substr cur
+                       cur = txt.length
+               else
+                       val = txt.substr cur, (next_gt - cur)
+                       cur = next_gt + 1
+               val = val.replace "\u0000", "\ufffd"
+               tok_cur_tag.text += val
+               tok_state = tok_state_data
+               return tok_cur_tag
+
+       # 8.2.4.45 http://www.w3.org/TR/html5/syntax.html#markup-declaration-open-state
+       tok_state_markup_declaration_open = ->
+               if txt.substr(cur, 2) is '--'
+                       cur += 2
+                       tok_cur_tag = new_comment_token ''
+                       tok_state = tok_state_comment_start
+                       return
+               if txt.substr(cur, 7).toLowerCase() is 'doctype'
+                       cur += 7
+                       tok_state = tok_state_doctype
+                       return
+               acn = adjusted_current_node()
+               if acn and acn.namespace isnt NS_HTML and txt.substr(cur, 7) is '[CDATA['
+                       cur += 7
+                       tok_state = tok_state_cdata_section
+                       return
+               # Otherwise
+               parse_error()
+               tok_cur_tag = new_comment_token '!' # TODO test ("!" right?)
+               tok_state = tok_state_bogus_comment
+               return
+
+       # 8.2.4.46 http://www.w3.org/TR/html5/syntax.html#comment-start-state
+       tok_state_comment_start = ->
+               switch c = txt.charAt(cur++)
+                       when '-'
+                               tok_state = tok_state_comment_start_dash
+                       when "\u0000"
+                               parse_error()
+                               return new_character_token "\ufffd"
+                       when '>'
+                               parse_error()
+                               tok_state = tok_state_data
+                               return tok_cur_tag
+                       when '' # EOF
+                               parse_error()
+                               tok_state = tok_state_data
+                               cur -= 1 # Reconsume
+                               return tok_cur_tag
+                       else
+                               tok_cur_tag.text += c
+               return null
+
+       # 8.2.4.47 http://www.w3.org/TR/html5/syntax.html#comment-start-dash-state
+       tok_state_comment_start_dash = ->
+               switch c = txt.charAt(cur++)
+                       when '-'
+                               tok_state = tok_state_comment_end
+                       when "\u0000"
+                               parse_error()
+                               tok_cur_tag.text += "-\ufffd"
+                               tok_state = tok_state_comment
+                       when '>'
+                               parse_error()
+                               tok_state = tok_state_data
+                               return tok_cur_tag
+                       when '' # EOF
+                               parse_error()
+                               tok_state = tok_state_data
+                               cur -= 1 # Reconsume
+                               return tok_cur_tag
+                       else
+                               tok_cur_tag.text += "-#{c}"
+                               tok_state = tok_state_comment
+               return null
+
+       # 8.2.4.48 http://www.w3.org/TR/html5/syntax.html#comment-state
+       tok_state_comment = ->
+               switch c = txt.charAt(cur++)
+                       when '-'
+                               tok_state = tok_state_comment_end_dash
+                       when "\u0000"
+                               parse_error()
+                               tok_cur_tag.text += "\ufffd"
+                       when '' # EOF
+                               parse_error()
+                               tok_state = tok_state_data
+                               cur -= 1 # Reconsume
+                               return tok_cur_tag
+                       else
+                               tok_cur_tag.text += c
+               return null
+
+       # 8.2.4.49 http://www.w3.org/TR/html5/syntax.html#comment-end-dash-state
+       tok_state_comment_end_dash = ->
+               switch c = txt.charAt(cur++)
+                       when '-'
+                               tok_state = tok_state_comment_end
+                       when "\u0000"
+                               parse_error()
+                               tok_cur_tag.text += "-\ufffd"
+                               tok_state = tok_state_comment
+                       when '' # EOF
+                               parse_error()
+                               tok_state = tok_state_data
+                               cur -= 1 # Reconsume
+                               return tok_cur_tag
+                       else
+                               tok_cur_tag.text += "-#{c}"
+                               tok_state = tok_state_comment
+               return null
+
+       # 8.2.4.50 http://www.w3.org/TR/html5/syntax.html#comment-end-state
+       tok_state_comment_end = ->
+               switch c = txt.charAt(cur++)
+                       when '>'
+                               tok_state = tok_state_data
+                               return tok_cur_tag
+                       when "\u0000"
+                               parse_error()
+                               tok_cur_tag.text += "--\ufffd"
+                               tok_state = tok_state_comment
+                       when '!'
+                               parse_error()
+                               tok_state = tok_state_comment_end_bang
+                       when '-'
+                               parse_error()
+                               tok_cur_tag.text += '-'
+                       when '' # EOF
+                               parse_error()
+                               tok_state = tok_state_data
+                               cur -= 1 # Reconsume
+                               return tok_cur_tag
+                       else
+                               parse_error()
+                               tok_cur_tag.text += "--#{c}"
+                               tok_state = tok_state_comment
+               return null
+
+       # 8.2.4.51 http://www.w3.org/TR/html5/syntax.html#comment-end-bang-state
+       tok_state_comment_end_bang = ->
+               switch c = txt.charAt(cur++)
+                       when '-'
+                               tok_cur_tag.text += "--!#{c}"
+                               tok_state = tok_state_comment_end_dash
+                       when '>'
+                               tok_state = tok_state_data
+                               return tok_cur_tag
+                       when "\u0000"
+                               parse_error()
+                               tok_cur_tag.text += "--!\ufffd"
+                               tok_state = tok_state_comment
+                       when '' # EOF
+                               parse_error()
+                               tok_state = tok_state_data
+                               cur -= 1 # Reconsume
+                               return tok_cur_tag
+                       else
+                               tok_cur_tag.text += "--!#{c}"
+                               tok_state = tok_state_comment
+               return null
+
+       # 8.2.4.52 http://www.w3.org/TR/html5/syntax.html#doctype-state
+       tok_state_doctype = ->
+               switch c = txt.charAt(cur++)
+                       when "\t", "\u000a", "\u000c", ' '
+                               tok_state = tok_state_before_doctype_name
+                       when '' # EOF
+                               parse_error()
+                               tok_state = tok_state_data
+                               el = new_doctype_token ''
+                               el.flag 'force-quirks', true
+                               cur -= 1 # Reconsume
+                               return el
+                       else
+                               parse_error()
+                               tok_state = tok_state_before_doctype_name
+                               cur -= 1 # Reconsume
+               return null
+
+       # 8.2.4.52 http://www.w3.org/TR/html5/syntax.html#doctype-state
+       tok_state_before_doctype_name = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' '
+                       return
+               if is_uc_alpha(c)
+                       tok_cur_tag = new_doctype_token c.toLowerCase()
+                       tok_state = tok_state_doctype_name
+                       return
+               if c is "\u0000"
+                       parse_error()
+                       tok_cur_tag = new_doctype_token "\ufffd"
+                       tok_state = tok_state_doctype_name
+                       return
+               if c is '>'
+                       parse_error()
+                       el = new_doctype_token ''
+                       el.flag 'force-quirks', true
+                       tok_state = tok_state_data
+                       return el
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       el = new_doctype_token ''
+                       el.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return el
+               # Anything else
+               tok_cur_tag = new_doctype_token c
+               tok_state = tok_state_doctype_name
+               return null
+
+       # 8.2.4.54 http://www.w3.org/TR/html5/syntax.html#doctype-name-state
+       tok_state_doctype_name = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' '
+                       tok_state = tok_state_after_doctype_name
+                       return
+               if c is '>'
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if is_uc_alpha(c)
+                       tok_cur_tag.name += c.toLowerCase()
+                       return
+               if c is "\u0000"
+                       parse_error()
+                       tok_cur_tag.name += "\ufffd"
+                       return
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               tok_cur_tag.name += c
+               return null
+
+       # 8.2.4.55 http://www.w3.org/TR/html5/syntax.html#after-doctype-name-state
+       tok_state_after_doctype_name = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' '
+                       return
+               if c is '>'
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               if txt.substr(cur - 1, 6).toLowerCase() is 'public'
+                       cur += 5
+                       tok_state = tok_state_after_doctype_public_keyword
+                       return
+               if txt.substr(cur - 1, 6).toLowerCase() is 'system'
+                       cur += 5
+                       tok_state = tok_state_after_doctype_system_keyword
+                       return
+               parse_error()
+               tok_cur_tag.flag 'force-quirks', true
+               tok_state = tok_state_bogus_doctype
+               return null
+
+       # 8.2.4.56 http://www.w3.org/TR/html5/syntax.html#after-doctype-public-keyword-state
+       tok_state_after_doctype_public_keyword = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' '
+                       tok_state = tok_state_before_doctype_public_identifier
+                       return
+               if c is '"'
+                       parse_error()
+                       tok_cur_tag.public_identifier = '' # FIXME should this go in @attrs or @text?
+                       tok_state = tok_state_doctype_public_identifier_double_quoted
+                       return
+               if c is "'"
+                       parse_error()
+                       tok_cur_tag.public_identifier = '' # FIXME should this go in @attrs or @text?
+                       tok_state = tok_state_doctype_public_identifier_single_quoted
+                       return
+               if c is '>'
+                       parse_error()
+                       tok_cur_tag.flag 'force-quirks', true
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               parse_error()
+               tok_cur_tag.flag 'force-quirks', true
+               tok_state = tok_state_bogus_doctype
+               return null
+
+       # 8.2.4.57 http://www.w3.org/TR/html5/syntax.html#before-doctype-public-identifier-state
+       tok_state_before_doctype_public_identifier = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' '
+                       return
+               if c is '"'
+                       parse_error()
+                       tok_cur_tag.public_identifier = '' # FIXME should this go in @attrs or @text?
+                       tok_state = tok_state_doctype_public_identifier_double_quoted
+                       return
+               if c is "'"
+                       parse_error()
+                       tok_cur_tag.public_identifier = '' # FIXME should this go in @attrs or @text?
+                       tok_state = tok_state_doctype_public_identifier_single_quoted
+                       return
+               if c is '>'
+                       parse_error()
+                       tok_cur_tag.flag 'force-quirks', true
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               parse_error()
+               tok_cur_tag.flag 'force-quirks', true
+               tok_state = tok_state_bogus_doctype
+               return null
+
+
+       # 8.2.4.58 http://www.w3.org/TR/html5/syntax.html#doctype-public-identifier-(double-quoted)-state
+       tok_state_doctype_public_identifier_double_quoted = ->
+               c = txt.charAt(cur++)
+               if c is '"'
+                       tok_state = tok_state_after_doctype_public_identifier
+                       return
+               if c is "\u0000"
+                       parse_error()
+                       tok_cur_tag.public_identifier += "\ufffd"
+                       return
+               if c is '>'
+                       parse_error()
+                       tok_cur_tag.flag 'force-quirks', true
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               tok_cur_tag.public_identifier += c
+               return null
+
+       # 8.2.4.59 http://www.w3.org/TR/html5/syntax.html#doctype-public-identifier-(single-quoted)-state
+       tok_state_doctype_public_identifier_single_quoted = ->
+               c = txt.charAt(cur++)
+               if c is "'"
+                       tok_state = tok_state_after_doctype_public_identifier
+                       return
+               if c is "\u0000"
+                       parse_error()
+                       tok_cur_tag.public_identifier += "\ufffd"
+                       return
+               if c is '>'
+                       parse_error()
+                       tok_cur_tag.flag 'force-quirks', true
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               tok_cur_tag.public_identifier += c
+               return null
+
+       # 8.2.4.60 http://www.w3.org/TR/html5/syntax.html#after-doctype-public-identifier-state
+       tok_state_after_doctype_public_identifier = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' '
+                       tok_state = tok_state_between_doctype_public_and_system_identifiers
+                       return
+               if c is '>'
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '"'
+                       parse_error()
+                       tok_cur_tag.system_identifier = ''
+                       tok_state = tok_state_doctype_system_identifier_double_quoted
+                       return
+               if c is "'"
+                       parse_error()
+                       tok_cur_tag.system_identifier = ''
+                       tok_state = tok_state_doctype_system_identifier_single_quoted
+                       return
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               parse_error()
+               tok_cur_tag.flag 'force-quirks', true
+               tok_state = tok_state_bogus_doctype
+               return null
+
+       # 8.2.4.61 http://www.w3.org/TR/html5/syntax.html#between-doctype-public-and-system-identifiers-state
+       tok_state_between_doctype_public_and_system_identifiers = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' '
+                       return
+               if c is '>'
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '"'
+                       parse_error()
+                       tok_cur_tag.system_identifier = ''
+                       tok_state = tok_state_doctype_system_identifier_double_quoted
+                       return
+               if c is "'"
+                       parse_error()
+                       tok_cur_tag.system_identifier = ''
+                       tok_state = tok_state_doctype_system_identifier_single_quoted
+                       return
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               parse_error()
+               tok_cur_tag.flag 'force-quirks', true
+               tok_state = tok_state_bogus_doctype
+               return null
+
+       # 8.2.4.62 http://www.w3.org/TR/html5/syntax.html#after-doctype-system-keyword-state
+       tok_state_after_doctype_system_keyword = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' '
+                       tok_state = tok_state_before_doctype_system_identifier
+                       return
+               if c is '"'
+                       parse_error()
+                       tok_cur_tag.system_identifier = ''
+                       tok_state = tok_state_doctype_system_identifier_double_quoted
+                       return
+               if c is "'"
+                       parse_error()
+                       tok_cur_tag.system_identifier = ''
+                       tok_state = tok_state_doctype_system_identifier_single_quoted
+                       return
+               if c is '>'
+                       parse_error()
+                       tok_cur_tag.flag 'force-quirks', true
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               parse_error()
+               tok_cur_tag.flag 'force-quirks', true
+               tok_state = tok_state_bogus_doctype
+               return null
+
+       # 8.2.4.63 http://www.w3.org/TR/html5/syntax.html#before-doctype-system-identifier-state
+       tok_state_before_doctype_system_identifier = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' '
+                       return
+               if c is '"'
+                       tok_cur_tag.system_identifier = ''
+                       tok_state = tok_state_doctype_system_identifier_double_quoted
+                       return
+               if c is "'"
+                       tok_cur_tag.system_identifier = ''
+                       tok_state = tok_state_doctype_system_identifier_single_quoted
+                       return
+               if c is '>'
+                       parse_error()
+                       tok_cur_tag.flag 'force-quirks', true
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               parse_error()
+               tok_cur_tag.flag 'force-quirks', true
+               tok_state = tok_state_bogus_doctype
+               return null
+
+       # 8.2.4.64 http://www.w3.org/TR/html5/syntax.html#doctype-system-identifier-(double-quoted)-state
+       tok_state_doctype_system_identifier_double_quoted = ->
+               c = txt.charAt(cur++)
+               if c is '"'
+                       tok_state = tok_state_after_doctype_system_identifier
+                       return
+               if c is "\u0000"
+                       parse_error()
+                       tok_cur_tag.system_identifier += "\ufffd"
+                       return
+               if c is '>'
+                       parse_error()
+                       tok_cur_tag.flag 'force-quirks', true
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               tok_cur_tag.system_identifier += c
+               return null
+
+       # 8.2.4.65 http://www.w3.org/TR/html5/syntax.html#doctype-system-identifier-(single-quoted)-state
+       tok_state_doctype_system_identifier_single_quoted = ->
+               c = txt.charAt(cur++)
+               if c is "'"
+                       tok_state = tok_state_after_doctype_system_identifier
+                       return
+               if c is "\u0000"
+                       parse_error()
+                       tok_cur_tag.system_identifier += "\ufffd"
+                       return
+               if c is '>'
+                       parse_error()
+                       tok_cur_tag.flag 'force-quirks', true
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               tok_cur_tag.system_identifier += c
+               return null
+
+       # 8.2.4.66 http://www.w3.org/TR/html5/syntax.html#after-doctype-system-identifier-state
+       tok_state_after_doctype_system_identifier = ->
+               c = txt.charAt(cur++)
+               if c is "\t" or c is "\u000a" or c is "\u000c" or c is ' '
+                       return
+               if c is '>'
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '' # EOF
+                       parse_error()
+                       tok_state = tok_state_data
+                       tok_cur_tag.flag 'force-quirks', true
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               parse_error()
+               # do _not_ tok_cur_tag.flag 'force-quirks', true
+               tok_state = tok_state_bogus_doctype
+               return null
+
+       # 8.2.4.67 http://www.w3.org/TR/html5/syntax.html#bogus-doctype-state
+       tok_state_bogus_doctype = ->
+               c = txt.charAt(cur++)
+               if c is '>'
+                       tok_state = tok_state_data
+                       return tok_cur_tag
+               if c is '' # EOF
+                       tok_state = tok_state_data
+                       cur -= 1 # Reconsume
+                       return tok_cur_tag
+               # Anything else
+               return null
+
+

        # 8.2.4.69 http://www.w3.org/TR/html5/syntax.html#consume-a-character-reference
        # Don't set this as a state, just call it
        # returns a string (NOT a text node)
        # 8.2.4.69 http://www.w3.org/TR/html5/syntax.html#consume-a-character-reference
        # Don't set this as a state, just call it
        # returns a string (NOT a text node)

@@ -2295,11 +3725,11 @@ parse_html = (txt, parse_error_cb = null) ->
 
        # tree constructor initialization
        # see comments on TYPE_TAG/etc for the structure of this data
 
        # tree constructor initialization
        # see comments on TYPE_TAG/etc for the structure of this data

-       tree = new Node TYPE_TAG, name: 'html', namespace: NS_HTML
-       open_els = [tree]
+       doc = new Node TYPE_TAG, name: 'html', namespace: NS_HTML
+       open_els = []

        afe = [] # active formatting elements
        template_insertion_modes = []
        afe = [] # active formatting elements
        template_insertion_modes = []

-       insertion_mode = ins_mode_in_body
+       insertion_mode = ins_mode_initial

        original_insertion_mode = insertion_mode # TODO check spec
        flag_scripting = true # TODO might need an extra flag to get <noscript> to parse correctly
        flag_frameset_ok = true
        original_insertion_mode = insertion_mode # TODO check spec
        flag_scripting = true # TODO might need an extra flag to get <noscript> to parse correctly
        flag_frameset_ok = true

@@ -2308,6 +3738,9 @@ parse_html = (txt, parse_error_cb = null) ->
        form_element_pointer = null
        temporary_buffer = null
        pending_table_character_tokens = []
        form_element_pointer = null
        temporary_buffer = null
        pending_table_character_tokens = []

+       head_element_pointer = null
+       flag_fragment_parsing = false # parser originally created as part of the html fragment parsing algorithm (fragment case)
+       context_element = null # FIXME initialize from args.fragment http://www.w3.org/TR/html5/syntax.html#parsing-html-fragments

 
        # tokenizer initialization
        tok_state = tok_state_data
 
        # tokenizer initialization
        tok_state = tok_state_data

@@ -2317,16 +3750,9 @@ parse_html = (txt, parse_error_cb = null) ->
                t = tok_state()
                if t?
                        insertion_mode t
                t = tok_state()
                if t?
                        insertion_mode t

-       return tree.children
-
-# everything below is tests on the above
-test_equals = (description, output, expected_output) ->
-       if output is expected_output
-               console.log "passed." # don't say name, so smart consoles can merge all of these
-       else
-               console.log "FAILED: \"#{description}\""
-               console.log "   Expected: #{expected_output}"
-               console.log "     Actual: #{output}"
+                       # fixfull parse error if has self-closing flag, but it wasn't acknolwedged
+       return doc.children
+

 serialize_els = (els, shallow, show_ids) ->
        serialized = ''
        sep = ''
 serialize_els = (els, shallow, show_ids) ->
        serialized = ''
        sep = ''

@@ -2335,198 +3761,12 @@ serialize_els = (els, shallow, show_ids) ->
                sep = ','
                serialized += t.serialize shallow, show_ids
        return serialized
                sep = ','
                serialized += t.serialize shallow, show_ids
        return serialized

-test_parser = (args) ->
-       debug_log_reset()
-       parse_errors = []
-       errors_cb = (i) ->
-               parse_errors.push i
-       prev_node_id = 0 # reset counter
-       parsed = parse_html args.html, errors_cb
-       serialized = serialize_els parsed, false, false
-       if serialized isnt args.expected
-               debug_log_each (str) ->
-                       console.log str
-               console.log "FAILED: \"#{args.name}\""
-               console.log "      Input: #{args.html}"
-               console.log "    Correct: #{args.expected}"
-               console.log "     Output: #{serialized}"
-               if parse_errors.length > 0
-                       console.log " parse errs: #{JSON.stringify parse_errors}"
-               else
-                       console.log "   No parse errors"
-       else
-               console.log "passed \"#{args.name}\""
-
-test_parser name: "empty", \
-       html: "",
-       expected: ''
-test_parser name: "just text", \
-       html: "abc",
-       expected: 'text:"abc"'
-test_parser name: "named entity", \
-       html: "a&amp;1234",
-       expected: 'text:"a&1234"'
-test_parser name: "broken named character references", \
-       html: "1&amp2&&amp;3&aabbcc;",
-       expected: 'text:"1&2&&3&aabbcc;"'
-test_parser name: "numbered entity overrides", \
-       html: "1&#X80&#x80; &#x83",
-       expected: 'text:"1€€ ƒ"'
-test_parser name: "open tag", \
-       html: "foo<span>bar",
-       expected: 'text:"foo",tag:"span",{},[text:"bar"]'
-test_parser name: "open tag with attributes", \
-       html: "foo<span style=\"foo: bar\" title=\"hi\">bar",
-       expected: 'text:"foo",tag:"span",{"style":"foo: bar","title":"hi"},[text:"bar"]'
-test_parser name: "open tag with attributes of various quotings", \
-       html: "foo<span abc=\"def\" g=hij klm='nopqrstuv\"' autofocus>bar",
-       expected: 'text:"foo",tag:"span",{"abc":"def","autofocus":"","g":"hij","klm":"nopqrstuv\\""},[text:"bar"]'
-test_parser name: "attribute entity exceptions dq", \
-       html: "foo<a href=\"foo?t=1&amp=2&ampo=3&amp;lt=foo\">bar",
-       expected: 'text:"foo",tag:"a",{"href":"foo?t=1&amp=2&ampo=3&lt=foo"},[text:"bar"]'
-test_parser name: "attribute entity exceptions sq", \
-       html: "foo<a href='foo?t=1&amp=2&ampo=3&amp;lt=foo'>bar",
-       expected: 'text:"foo",tag:"a",{"href":"foo?t=1&amp=2&ampo=3&lt=foo"},[text:"bar"]'
-test_parser name: "attribute entity exceptions uq", \
-       html: "foo<a href=foo?t=1&amp=2&ampo=3&amp;lt=foo>bar",
-       expected: 'text:"foo",tag:"a",{"href":"foo?t=1&amp=2&ampo=3&lt=foo"},[text:"bar"]'
-test_parser name: "matching closing tags", \
-       html: "foo<a href=\"hi\">hi</a><div>1<div>foo</div>2</div>bar",
-       expected: 'text:"foo",tag:"a",{"href":"hi"},[text:"hi"],tag:"div",{},[text:"1",tag:"div",{},[text:"foo"],text:"2"],text:"bar"'
-test_parser name: "missing closing tag inside", \
-       html: "foo<div>bar<span>baz</div>qux",
-       expected: 'text:"foo",tag:"div",{},[text:"bar",tag:"span",{},[text:"baz"]],text:"qux"'
-test_parser name: "mis-matched closing tags", \
-       html: "<span>12<div>34</span>56</div>78",
-       expected: 'tag:"span",{},[text:"12",tag:"div",{},[text:"3456"],text:"78"]'
-test_parser name: "mis-matched formatting elements", \
-       html: "12<b>34<i>56</b>78</i>90",
-       expected: 'text:"12",tag:"b",{},[text:"34",tag:"i",{},[text:"56"]],tag:"i",{},[text:"78"],text:"90"'
-test_parser name: "8.2.8.1 Misnested tags: <b><i></b></i>", \
-       html: '<p>1<b>2<i>3</b>4</i>5</p>',
-       expected: 'tag:"p",{},[text:"1",tag:"b",{},[text:"2",tag:"i",{},[text:"3"]],tag:"i",{},[text:"4"],text:"5"]'
-test_parser name: "8.2.8.2 Misnested tags: <b><p></b></p>", \
-       html: '<b>1<p>2</b>3</p>',
-       expected: 'tag:"b",{},[text:"1"],tag:"p",{},[tag:"b",{},[text:"2"],text:"3"]'
-test_parser name: "crazy formatting elements test", \
-       html: "<b><i><a><s><tt><div></b>first</b></div></tt></s></a>second</i>",
-       # chrome does this: expected: 'tag:"b",{},[tag:"i",{},[tag:"a",{},[tag:"s",{},[tag:"tt",{},[]]],text:"second"]],tag:"a",{},[tag:"s",{},[tag:"tt",{},[tag:"div",{},[tag:"b",{},[],text:"first"]]]]'
-       # firefox does this:
-       expected: 'tag:"b",{},[tag:"i",{},[tag:"a",{},[tag:"s",{},[tag:"tt",{},[]]]]],tag:"a",{},[tag:"s",{},[tag:"tt",{},[tag:"div",{},[tag:"b",{},[],text:"first"]]]],text:"second"'
-# tests from https://github.com/html5lib/html5lib-tests/blob/master/tree-construction/adoption01.dat
-test_parser name: "html5lib aaa 1", \
-       html: '<a><p></a></p>',
-       expected: 'tag:"a",{},[],tag:"p",{},[tag:"a",{},[]]'
-test_parser name: "html5lib aaa 2", \
-       html: '<a>1<p>2</a>3</p>',
-       expected: 'tag:"a",{},[text:"1"],tag:"p",{},[tag:"a",{},[text:"2"],text:"3"]'
-test_parser name: "html5lib aaa 3", \
-       html: '<a>1<button>2</a>3</button>',
-       expected: 'tag:"a",{},[text:"1"],tag:"button",{},[tag:"a",{},[text:"2"],text:"3"]'
-test_parser name: "html5lib aaa 4", \
-       html: '<a>1<b>2</a>3</b>',
-       expected: 'tag:"a",{},[text:"1",tag:"b",{},[text:"2"]],tag:"b",{},[text:"3"]'
-test_parser name: "html5lib aaa 5 (two divs deep)", \
-       html: '<a>1<div>2<div>3</a>4</div>5</div>',
-       expected: 'tag:"a",{},[text:"1"],tag:"div",{},[tag:"a",{},[text:"2"],tag:"div",{},[tag:"a",{},[text:"3"],text:"4"],text:"5"]'
-test_parser name: "html5lib aaa 6 (foster parenting)", \
-       html: '<table><a>1<p>2</a>3</p>',
-       expected: 'tag:"a",{},[text:"1"],tag:"p",{},[tag:"a",{},[text:"2"],text:"3"],tag:"table",{},[]'
-test_parser name: "html5lib aaa 7 (aaa, eof) 1", \
-       html: '<b><b><a><p></a>',
-       expected: 'tag:"b",{},[tag:"b",{},[tag:"a",{},[],tag:"p",{},[tag:"a",{},[]]]]'
-test_parser name: "html5lib aaa 8 (aaa, eof) 2", \
-       html: '<b><a><b><p></a>',
-       expected: 'tag:"b",{},[tag:"a",{},[tag:"b",{},[]],tag:"b",{},[tag:"p",{},[tag:"a",{},[]]]]'
-test_parser name: "html5lib aaa 9 (aaa, eof) 3", \
-       html: '<a><b><b><p></a>',
-       expected: 'tag:"a",{},[tag:"b",{},[tag:"b",{},[]]],tag:"b",{},[tag:"b",{},[tag:"p",{},[tag:"a",{},[]]]]'
-test_parser name: "html5lib aaa 10 (formatting, nesting, attrs, aaa)", \
-       html: '<p>1<s id="A">2<b id="B">3</p>4</s>5</b>',
-       expected: 'tag:"p",{},[text:"1",tag:"s",{"id":"A"},[text:"2",tag:"b",{"id":"B"},[text:"3"]]],tag:"s",{"id":"A"},[tag:"b",{"id":"B"},[text:"4"]],tag:"b",{"id":"B"},[text:"5"]'
-test_parser name: "html5lib aaa 11 (table with foster parenting, formatting el and td)", \
-       html: '<table><a>1<td>2</td>3</table>',
-       expected: 'tag:"a",{},[text:"1"],tag:"a",{},[text:"3"],tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[tag:"td",{},[text:"2"]]]]'
-test_parser name: "html5lib aaa 12 (table with foster parenting, split text)", \
-       html: '<table>A<td>B</td>C</table>',
-       expected: 'text:"AC",tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[tag:"td",{},[text:"B"]]]]'
-# TODO implement svg and namespacing
-#test_parser name: "html5lib aaa 13 (svg tr input)", \
-#      html: '<a><svg><tr><input></a>',
-#      expected: 'tag:"a",{},[svg:"svg",{},[svg:"tr",{},[svg:"input"]]]'
-test_parser name: "html5lib aaa 14 (deep ?outer aaa)", \
-       html: '<div><a><b><div><div><div><div><div><div><div><div><div><div></a>',
-       expected: 'tag:"div",{},[tag:"a",{},[tag:"b",{},[]],tag:"b",{},[tag:"div",{},[tag:"a",{},[],tag:"div",{},[tag:"a",{},[],tag:"div",{},[tag:"a",{},[],tag:"div",{},[tag:"a",{},[],tag:"div",{},[tag:"a",{},[],tag:"div",{},[tag:"a",{},[],tag:"div",{},[tag:"a",{},[],tag:"div",{},[tag:"a",{},[tag:"div",{},[tag:"div",{},[]]]]]]]]]]]]]'
-test_parser name: "html5lib aaa 15 (deep ?inner aaa)", \
-       html: '<div><a><b><u><i><code><div></a>',
-       expected: 'tag:"div",{},[tag:"a",{},[tag:"b",{},[tag:"u",{},[tag:"i",{},[tag:"code",{},[]]]]],tag:"u",{},[tag:"i",{},[tag:"code",{},[tag:"div",{},[tag:"a",{},[]]]]]]'
-test_parser name: "html5lib aaa 16 (correctly nested 4b)", \
-       html: '<b><b><b><b>x</b></b></b></b>y',
-       expected: 'tag:"b",{},[tag:"b",{},[tag:"b",{},[tag:"b",{},[text:"x"]]]],text:"y"'
-test_parser name: "html5lib aaa 17 (formatting, implied /p, noah's ark)", \
-       html: '<p><b><b><b><b><p>x',
-       expected: 'tag:"p",{},[tag:"b",{},[tag:"b",{},[tag:"b",{},[tag:"b",{},[]]]]],tag:"p",{},[tag:"b",{},[tag:"b",{},[tag:"b",{},[text:"x"]]]]'
-test_parser name: "variation on html5lib aaa 17 (with attributes in various orders)", \
-       html: '<p><b c="d" e="f"><b e="f" c="d"><b e="f" c="d"><b c="d" e="f"><p>x',
-       expected: 'tag:"p",{},[tag:"b",{"c":"d","e":"f"},[tag:"b",{"c":"d","e":"f"},[tag:"b",{"c":"d","e":"f"},[tag:"b",{"c":"d","e":"f"},[]]]]],tag:"p",{},[tag:"b",{"c":"d","e":"f"},[tag:"b",{"c":"d","e":"f"},[tag:"b",{"c":"d","e":"f"},[text:"x"]]]]'
-test_parser name: "junk after attribute close-quote", \
-       html: '<p><b c="d", e="f">foo<p>x',
-       expected: 'tag:"p",{},[tag:"b",{",":"","c":"d","e":"f"},[text:"foo"]],tag:"p",{},[tag:"b",{",":"","c":"d","e":"f"},[text:"x"]]'
-test_parser name: "html5lib aaa02 1", \
-       html: '<b>1<i>2<p>3</b>4',
-       expected: 'tag:"b",{},[text:"1",tag:"i",{},[text:"2"]],tag:"i",{},[tag:"p",{},[tag:"b",{},[text:"3"],text:"4"]]'
-test_parser name: "html5lib aaa02 2", \
-       html: '<a><div><style></style><address><a>',
-       expected: 'tag:"a",{},[],tag:"div",{},[tag:"a",{},[tag:"style",{},[]],tag:"address",{},[tag:"a",{},[],tag:"a",{},[]]]'
-test_parser name: "html5lib tables 1", \
-       html: '<table><th>',
-       expected: 'tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[tag:"th",{},[]]]]'
-test_parser name: "html5lib tables 2", \
-       html: '<table><td>',
-       expected: 'tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[tag:"td",{},[]]]]'
-test_parser name: "html5lib tables 3", \
-       html: "<table><col foo='bar'>",
-       expected: 'tag:"table",{},[tag:"colgroup",{},[tag:"col",{"foo":"bar"},[]]]'
-test_parser name: "html5lib tables 4", \
-       html: '<table><colgroup></html>foo',
-       expected: 'text:"foo",tag:"table",{},[tag:"colgroup",{},[]]'
-test_parser name: "html5lib tables 5", \
-       html: '<table></table><p>foo',
-       expected: 'tag:"table",{},[],tag:"p",{},[text:"foo"]'
-test_parser name: "html5lib tables 6", \
-       html: '<table></body></caption></col></colgroup></html></tbody></td></tfoot></th></thead></tr><td>',
-       expected: 'tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[tag:"td",{},[]]]]'
-test_parser name: "html5lib tables 7", \
-       html: '<table><select><option>3</select></table>',
-       expected: 'tag:"select",{},[tag:"option",{},[text:"3"]],tag:"table",{},[]'
-test_parser name: "html5lib tables 8", \
-       html: '<table><select><table></table></select></table>',
-       expected: 'tag:"select",{},[],tag:"table",{},[],tag:"table",{},[]'
-test_parser name: "html5lib tables 9", \
-       html: '<table><select></table>',
-       expected: 'tag:"select",{},[],tag:"table",{},[]'
-test_parser name: "html5lib tables 10", \
-       html: '<table><select><option>A<tr><td>B</td></tr></table>',
-       expected: 'tag:"select",{},[tag:"option",{},[text:"A"]],tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[tag:"td",{},[text:"B"]]]]'
-test_parser name: "html5lib tables 11", \
-       html: '<table><td></body></caption></col></colgroup></html>foo',
-       expected: 'tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[tag:"td",{},[text:"foo"]]]]'
-test_parser name: "html5lib tables 12", \
-       html: '<table><td>A</table>B',
-       expected: 'tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[tag:"td",{},[text:"A"]]]],text:"B"'
-test_parser name: "html5lib tables 13", \
-       html: '<table><tr><caption>',
-       expected: 'tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[]],tag:"caption",{},[]]'
-test_parser name: "html5lib tables 14", \
-       html: '<table><tr></body></caption></col></colgroup></html></td></th><td>foo',
-       expected: 'tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[tag:"td",{},[text:"foo"]]]]'
-test_parser name: "html5lib tables 15", \
-       html: '<table><td><tr>',
-       expected: 'tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[tag:"td",{},[]],tag:"tr",{},[]]]'
-test_parser name: "html5lib tables 16", \
-       html: '<table><td><button><td>',
-       expected: 'tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[tag:"td",{},[tag:"button",{},[]],tag:"td",{},[]]]]'
-# TODO implement svg parsing
-#test_parser name: "html5lib tables 17", \
-#      html: '<table><tr><td><svg><desc><td>',
-#      expected: 'tag:"table",{},[tag:"tbody",{},[tag:"tr",{},[tag:"td",{},[svg:"svg",{},[svg:"desc",{},[]]],tag:"td",{},[]]]]'
+
+# TODO export TYPE_*
+module.exports.parse_html = parse_html
+module.exports.debug_log_reset = debug_log_reset
+module.exports.debug_log_each = debug_log_each
+module.exports.TYPE_TAG = TYPE_TAG
+module.exports.TYPE_TEXT = TYPE_TEXT
+module.exports.TYPE_COMMENT = TYPE_COMMENT
+module.exports.TYPE_DOCTYPE = TYPE_DOCTYPE