1 # HTML parser meant to run in a browser, in support of WYSIWYG editor
2 # Copyright 2015 Jason Woofenden
4 # This program is free software: you can redistribute it and/or modify it under
5 # the terms of the GNU Affero General Public License as published by the Free
6 # Software Foundation, either version 3 of the License, or (at your option) any
9 # This program is distributed in the hope that it will be useful, but WITHOUT
10 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
11 # FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18 # This file implements a parser for html snippets, meant to be used by a
19 # WYSIWYG editor. Hence it does not attempt to parse doctypes, <html>, <head>
20 # or <body> tags, nor does it produce the top level "document" node in the dom
21 # tree, nor nodes for html, head or body. Comments containing "fixfull"
22 # indicate places where additional code is needed for full HTML document
25 # Instead, the data structure produced by this parser is an array of nodes.
27 # Each node is an obect of the Node class. Here are the Node types:
28 TYPE_TAG = 0 # name, {attributes}, [children]
29 TYPE_TEXT = 1 # "text"
32 # the following types are emited by the tokenizer, but shouldn't end up in the tree:
33 TYPE_OPEN_TAG = 4 # name, [attributes ([key,value]...) in reverse order], [children]
34 TYPE_END_TAG = 5 # name
36 TYPE_MARKER = 7 # http://www.w3.org/TR/html5/syntax.html#reconstruct-the-active-formatting-elements
37 TYPE_AAA_BOOKMARK = 8 # http://www.w3.org/TR/html5/syntax.html#adoption-agency-algorithm
45 constructor: (type, args = {}) ->
46 @type = type # one of the TYPE_* constants above
47 @name = args.name ? '' # tag name
48 @text = args.text ? '' # contents for text/comment nodes
49 @attrs = args.attrs ? {}
50 @attrs_a = args.attr_k ? [] # attrs in progress, TYPE_OPEN_TAG only
51 @children = args.children ? []
52 @namespace = args.namespace ? NS_HTML
53 @parent = args.parent ? null
54 shallow_clone: -> # return a new node that's the same except without the children or parent
55 # WARNING this doesn't work right on open tags that are still being parsed
57 attrs[k] = v for k, v of @attrs
58 return new Node @type, name: @name, text: @text, attrs: attrs, namespace: @namespace
59 serialize: -> # for unit tests
64 ret += JSON.stringify @name
66 ret += JSON.stringify @attrs
76 ret += JSON.stringify @text
79 ret += JSON.stringify @text
85 console.log "unknown: #{JSON.stringify @}" # backtrace is just as well
88 # helpers: (only take args that are normally known when parser creates nodes)
89 new_open_tag = (name) ->
90 return new Node TYPE_OPEN_TAG, name: name
91 new_end_tag = (name) ->
92 return new Node TYPE_END_TAG, name: name
93 new_text_node = (txt) ->
94 return new Node TYPE_TEXT, text: txt
95 new_comment_node = (txt) ->
96 return new Node TYPE_COMMENT, text: txt
98 return new Node TYPE_EOF
100 return new Node TYPE_AAA_BOOKMARK
102 lc_alpha = "abcdefghijklmnopqrstuvwxqz"
103 uc_alpha = "ABCDEFGHIJKLMNOPQRSTUVWXQZ"
104 digits = "0123456789"
105 alnum = lc_alpha + uc_alpha + digits
106 hex_chars = digits + "abcdefABCDEF"
108 # some SVG elements have dashes in them
109 tag_name_chars = alnum + "-"
111 # http://www.w3.org/TR/html5/infrastructure.html#space-character
112 space_chars = "\u0009\u000a\u000c\u000d\u0020"
114 # https://en.wikipedia.org/wiki/Whitespace_character#Unicode
115 whitespace_chars = "\u0009\u000a\u000b\u000c\u000d\u0020\u0085\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000"
117 # These are the character references that don't need a terminating semicolon
118 # min length: 2, max: 6, none are a prefix of any other.
120 Aacute: 'Á', aacute: 'á', Acirc: 'Â', acirc: 'â', acute: '´', AElig: 'Æ',
121 aelig: 'æ', Agrave: 'À', agrave: 'à', AMP: '&', amp: '&', Aring: 'Å',
122 aring: 'å', Atilde: 'Ã', atilde: 'ã', Auml: 'Ä', auml: 'ä', brvbar: '¦',
123 Ccedil: 'Ç', ccedil: 'ç', cedil: '¸', cent: '¢', COPY: '©', copy: '©',
124 curren: '¤', deg: '°', divide: '÷', Eacute: 'É', eacute: 'é', Ecirc: 'Ê',
125 ecirc: 'ê', Egrave: 'È', egrave: 'è', ETH: 'Ð', eth: 'ð', Euml: 'Ë',
126 euml: 'ë', frac12: '½', frac14: '¼', frac34: '¾', GT: '>', gt: '>',
127 Iacute: 'Í', iacute: 'í', Icirc: 'Î', icirc: 'î', iexcl: '¡', Igrave: 'Ì',
128 igrave: 'ì', iquest: '¿', Iuml: 'Ï', iuml: 'ï', laquo: '«', LT: '<',
129 lt: '<', macr: '¯', micro: 'µ', middot: '·', nbsp: "\u00a0", not: '¬',
130 Ntilde: 'Ñ', ntilde: 'ñ', Oacute: 'Ó', oacute: 'ó', Ocirc: 'Ô', ocirc: 'ô',
131 Ograve: 'Ò', ograve: 'ò', ordf: 'ª', ordm: 'º', Oslash: 'Ø', oslash: 'ø',
132 Otilde: 'Õ', otilde: 'õ', Ouml: 'Ö', ouml: 'ö', para: '¶', plusmn: '±',
133 pound: '£', QUOT: '"', quot: '"', raquo: '»', REG: '®', reg: '®', sect: '§',
134 shy: '', sup1: '¹', sup2: '²', sup3: '³', szlig: 'ß', THORN: 'Þ', thorn: 'þ',
135 times: '×', Uacute: 'Ú', uacute: 'ú', Ucirc: 'Û', ucirc: 'û', Ugrave: 'Ù',
136 ugrave: 'ù', uml: '¨', Uuml: 'Ü', uuml: 'ü', Yacute: 'Ý', yacute: 'ý',
140 void_elements = ['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'meta', 'param', 'source', 'track', 'wbr']
141 raw_text_elements = ['script', 'style']
142 escapable_raw_text_elements = ['textarea', 'title']
143 # http://www.w3.org/TR/SVG/ 1.1 (Second Edition)
145 'a', 'altGlyph', 'altGlyphDef', 'altGlyphItem', 'animate', 'animateColor',
146 'animateMotion', 'animateTransform', 'circle', 'clipPath', 'color-profile',
147 'cursor', 'defs', 'desc', 'ellipse', 'feBlend', 'feColorMatrix',
148 'feComponentTransfer', 'feComposite', 'feConvolveMatrix',
149 'feDiffuseLighting', 'feDisplacementMap', 'feDistantLight', 'feFlood',
150 'feFuncA', 'feFuncB', 'feFuncG', 'feFuncR', 'feGaussianBlur', 'feImage',
151 'feMerge', 'feMergeNode', 'feMorphology', 'feOffset', 'fePointLight',
152 'feSpecularLighting', 'feSpotLight', 'feTile', 'feTurbulence', 'filter',
153 'font', 'font-face', 'font-face-format', 'font-face-name', 'font-face-src',
154 'font-face-uri', 'foreignObject', 'g', 'glyph', 'glyphRef', 'hkern',
155 'image', 'line', 'linearGradient', 'marker', 'mask', 'metadata',
156 'missing-glyph', 'mpath', 'path', 'pattern', 'polygon', 'polyline',
157 'radialGradient', 'rect', 'script', 'set', 'stop', 'style', 'svg',
158 'switch', 'symbol', 'text', 'textPath', 'title', 'tref', 'tspan', 'use',
162 # http://www.w3.org/TR/MathML/ Version 3.0 2nd Edition
164 'abs', 'and', 'annotation', 'annotation-xml', 'apply', 'approx', 'arccos',
165 'arccosh', 'arccot', 'arccoth', 'arccsc', 'arccsch', 'arcsec', 'arcsech',
166 'arcsin', 'arcsinh', 'arctan', 'arctanh', 'arg', 'bind', 'bvar', 'card',
167 'cartesianproduct', 'cbytes', 'ceiling', 'cerror', 'ci', 'cn', 'codomain',
168 'complexes', 'compose', 'condition', 'conjugate', 'cos', 'cosh', 'cot',
169 'coth', 'cs', 'csc', 'csch', 'csymbol', 'curl', 'declare', 'degree',
170 'determinant', 'diff', 'divergence', 'divide', 'domain',
171 'domainofapplication', 'emptyset', 'eq', 'equivalent', 'eulergamma',
172 'exists', 'exp', 'exponentiale', 'factorial', 'factorof', 'false', 'floor',
173 'fn', 'forall', 'gcd', 'geq', 'grad', 'gt', 'ident', 'image', 'imaginary',
174 'imaginaryi', 'implies', 'in', 'infinity', 'int', 'integers', 'intersect',
175 'interval', 'inverse', 'lambda', 'laplacian', 'lcm', 'leq', 'limit',
176 'list', 'ln', 'log', 'logbase', 'lowlimit', 'lt', 'maction', 'maligngroup',
177 'malignmark', 'math', 'matrix', 'matrixrow', 'max', 'mean', 'median',
178 'menclose', 'merror', 'mfenced', 'mfrac', 'mglyph', 'mi', 'mi', 'min',
179 'minus', 'mlabeledtr', 'mlongdiv', 'mmultiscripts', 'mn', 'mo', 'mode',
180 'moment', 'momentabout', 'mover', 'mpadded', 'mphantom', 'mprescripts',
181 'mroot', 'mrow', 'ms', 'mscarries', 'mscarry', 'msgroup', 'msline',
182 'mspace', 'msqrt', 'msrow', 'mstack', 'mstyle', 'msub', 'msubsup', 'msup',
183 'mtable', 'mtd', 'mtext', 'mtr', 'munder', 'munderover', 'naturalnumbers',
184 'neq', 'none', 'not', 'notanumber', 'notin', 'notprsubset', 'notsubset',
185 'or', 'otherwise', 'outerproduct', 'partialdiff', 'pi', 'piece',
186 'piecewise', 'plus', 'power', 'primes', 'product', 'prsubset', 'quotient',
187 'rationals', 'real', 'reals', 'reln', 'rem', 'root', 'scalarproduct',
188 'sdev', 'sec', 'sech', 'selector', 'semantics', 'sep', 'set', 'setdiff',
189 'share', 'sin', 'sinh', 'span', 'subset', 'sum', 'tan', 'tanh', 'tendsto',
190 'times', 'transpose', 'true', 'union', 'uplimit', 'variance', 'vector',
191 'vectorproduct', 'xor'
193 # foreign_elements = [svg_elements..., mathml_elements...]
194 #normal_elements = All other allowed HTML elements are normal elements.
198 address:NS_HTML, applet:NS_HTML, area:NS_HTML, article:NS_HTML,
199 aside:NS_HTML, base:NS_HTML, basefont:NS_HTML, bgsound:NS_HTML,
200 blockquote:NS_HTML, body:NS_HTML, br:NS_HTML, button:NS_HTML,
201 caption:NS_HTML, center:NS_HTML, col:NS_HTML, colgroup:NS_HTML, dd:NS_HTML,
202 details:NS_HTML, dir:NS_HTML, div:NS_HTML, dl:NS_HTML, dt:NS_HTML,
203 embed:NS_HTML, fieldset:NS_HTML, figcaption:NS_HTML, figure:NS_HTML,
204 footer:NS_HTML, form:NS_HTML, frame:NS_HTML, frameset:NS_HTML, h1:NS_HTML,
205 h2:NS_HTML, h3:NS_HTML, h4:NS_HTML, h5:NS_HTML, h6:NS_HTML, head:NS_HTML,
206 header:NS_HTML, hgroup:NS_HTML, hr:NS_HTML, html:NS_HTML, iframe:NS_HTML,
207 img:NS_HTML, input:NS_HTML, isindex:NS_HTML, li:NS_HTML, link:NS_HTML,
208 listing:NS_HTML, main:NS_HTML, marquee:NS_HTML, meta:NS_HTML, nav:NS_HTML,
209 noembed:NS_HTML, noframes:NS_HTML, noscript:NS_HTML, object:NS_HTML,
210 ol:NS_HTML, p:NS_HTML, param:NS_HTML, plaintext:NS_HTML, pre:NS_HTML,
211 script:NS_HTML, section:NS_HTML, select:NS_HTML, source:NS_HTML,
212 style:NS_HTML, summary:NS_HTML, table:NS_HTML, tbody:NS_HTML, td:NS_HTML,
213 template:NS_HTML, textarea:NS_HTML, tfoot:NS_HTML, th:NS_HTML,
214 thead:NS_HTML, title:NS_HTML, tr:NS_HTML, track:NS_HTML, ul:NS_HTML,
215 wbr:NS_HTML, xmp:NS_HTML,
218 mi:NS_MATHML, mo:NS_MATHML, mn:NS_MATHML, ms:NS_MATHML, mtext:NS_MATHML,
219 'annotation-xml':NS_MATHML,
222 foreignObject:NS_SVG, desc:NS_SVG, title:NS_SVG
225 formatting_elements = {
226 a: true, b: true, big: true, code: true, em: true, font: true, i: true,
227 nobr: true, s: true, small: true, strike: true, strong: true, tt: true,
231 el_is_special = (e) ->
232 return special_elements[e] is e.namespace
234 # decode_named_char_ref()
236 # The list of named character references is _huge_ so ask the browser to decode
237 # for us instead of wasting bandwidth/space on including the table here.
239 # Pass without the "&" but with the ";" examples:
240 # for "&" pass "amp;"
241 # for "′" pass "x2032;"
244 textarea: document.createElement('textarea')
246 # TODO test this in IE8
247 decode_named_char_ref = (txt) ->
249 decoded = g_dncr.cache[txt]
250 return decoded if decoded?
251 g_dncr.textarea.innerHTML = txt
252 decoded = g_dncr.textarea.value
253 return null if decoded is txt
254 return g_dncr.cache[txt] = decoded
256 parse_html = (txt, parse_error_cb = null) ->
257 cur = 0 # index of next char in txt to be parsed
258 # declare tree and tokenizer variables so they're in scope below
260 open_els = [] # stack of open elements
263 tok_cur_tag = null # partially parsed tag
264 flag_frameset_ok = null
266 flag_foster_parenting = null
267 afe = [] # active formatting elements
273 console.log "Parse error at character #{cur} of #{txt.length}"
276 # the functions below impliment the Tree Contstruction algorithm
277 # http://www.w3.org/TR/html5/syntax.html#tree-construction
279 # But first... the helpers
280 template_tag_is_open = ->
282 if t.type is TYPE_TAG and t.name is 'template'
285 is_in_scope_x = (tag_name, scope) ->
287 if t.name is tag_name
292 is_in_scope_x_y = (tag_name, scope, scope2) ->
294 if t.name is tag_name
301 standard_scopers = { # FIXME these are supposed to be namespace specific
302 'applet': true, 'caption': true, 'html': true, 'table': true, 'td': true,
303 'th': true, 'marquee': true, 'object': true, 'template': true, 'mi': true,
304 'mo': true, 'mn': true, 'ms': true, 'mtext': true, 'annotation-xml': true,
305 'foreignObject': true, 'desc': true, 'title'
307 button_scopers = button: true
308 li_scopers = ol: true, ul: true
309 table_scopers = html: true, table: true, template: true
310 is_in_scope = (tag_name) ->
311 return is_in_scope_x tag_name, standard_scopers
312 is_in_button_scope = (tag_name) ->
313 return is_in_scope_x_y tag_name, standard_scopers, button_scopers
314 is_in_table_scope = (tag_name) ->
315 return is_in_scope_x tag_name, table_scopers
316 is_in_select_scope = (tag_name) ->
318 if t.name is tag_name
320 if t.name isnt 'optgroup' and t.name isnt 'option'
323 # this checks for a particular element, not by name
324 el_is_in_scope = (el) ->
328 if t.name of standard_scopers
332 # http://www.w3.org/TR/html5/syntax.html#reconstruct-the-active-formatting-elements
333 # this implementation is structured (mostly) as described at the link above.
334 # capitalized comments are the "labels" described at the link above.
335 reconstruct_active_formatting_elements = ->
336 return if afe.length is 0
337 if afe[0].type is TYPE_MARKER or afe[0] in open_els
342 if i is afe.length - 1
345 if afe[i].type is TYPE_MARKER or afe[i] in open_els
350 el = afe[i].shallow_clone()
351 tree_insert_element el
356 # http://www.w3.org/TR/html5/syntax.html#adoption-agency-algorithm
357 # adoption agency algorithm
358 adoption_agency = (subject) ->
359 if open_els[0].name is subject
362 # remove it from the list of active formatting elements (if found)
374 for t, fe_index in afe
375 if t.type is TYPE_MARKER
381 in_body_any_other_end_tag subject
390 # "remove it from the list" must mean afe, since it's not in open_els
391 afe.splice fe_index, 1
393 unless el_is_in_scope fe
396 unless open_els[0] is fe
411 afe.splice fe_index, 1
413 ca = open_els[fe_index + 1] # common ancestor
414 node_above = open_els[fb_index + 1] # next node if node isn't in open_els anymore
415 # 12. Let a bookmark note the position of formatting element in the list of active formatting elements relative to the elements on either side of it in the list.
416 bookmark = new_aaa_bookmark()
419 afe.splice i, 0, bookmark
420 node = last_node = fb
427 node_next = open_els[i + 1]
429 node = node_next ? node_above
430 # TODO make sure node_above gets re-set if/when node is removed from open_els
444 node_above = open_els[i + 1]
448 # 7. reate an element for the token for which the element node
449 # was created, in the HTML namespace, with common ancestor as
450 # the intended parent; replace the entry for node in the list
451 # of active formatting elements with an entry for the new
452 # element, replace the entry for node in the stack of open
453 # elements with an entry for the new element, and let node be
455 new_node = node.shallow_clone()
462 open_els[i] = new_node
465 # 8. If last node is furthest block, then move the
466 # aforementioned bookmark to be immediately after the new node
467 # in the list of active formatting elements.
474 # TODO test: position i gets you "after"?
475 afe.splice i, 0, new_aaa_bookmark()
476 # 9. Insert last node into node, first removing it from its
477 # previous parent node if any.
479 for c, i of last_node.parent.children
481 last_node.parent.children.splice i, 1
482 node.children.push last_node
483 last_node.parent = node
484 # 10. Let last node be node.
486 # 11. Return to the step labeled inner loop.
487 # 14. Insert whatever last node ended up being in the previous step
488 # at the appropriate place for inserting a node, but using common
489 # ancestor as the override target.
490 tree_insert_element last_node, ca
491 # 15. Create an element for the token for which formatting element
492 # was created, in the HTML namespace, with furthest block as the
494 new_element = fe.shallow_clone()
495 # 16. Take all of the child nodes of furthest block and append them
496 # to the element created in the last step.
497 while fb.children.length
498 t = fb.children.shift()
499 t.parent = new_element
500 new_element.children.push t
501 # 17. Append that new element to furthest block.
502 new_element.parent = fb
503 fb.children.push new_element
504 # 18. Remove formatting element from the list of active formatting
505 # elements, and insert the new element into the list of active
506 # formatting elements at the position of the aforementioned
516 # 19. Remove formatting element from the stack of open elements,
517 # and insert the new element into the stack of open elements
518 # immediately below the position of furthest block in that stack.
525 open_els.splice i, 0, new_element
527 # 20. Jump back to the step labeled outer loop.
529 # http://www.w3.org/TR/html5/syntax.html#close-a-p-element
530 # FIXME implement this
531 close_p_if_in_button_scope = ->
532 if open_els[0].name is 'p'
535 #p = find_button_scope 'p'
537 # TODO generate_implied_end_tags except for p tags
538 # TODO parse_error unless open_els[0].name is 'p'
539 # TODO pop stack until 'p' popped
541 # http://www.w3.org/TR/html5/syntax.html#insert-a-character
542 tree_insert_text = (t) ->
543 dest = adjusted_insertion_location()
545 prev = dest[0].children[dest[1] - 1]
546 if prev.type is TYPE_TEXT
549 dest[0].children.splice dest[1], 0, t
552 # http://www.w3.org/TR/html5/syntax.html#creating-and-inserting-nodes
553 # http://www.w3.org/TR/html5/syntax.html#appropriate-place-for-inserting-a-node
554 adjusted_insertion_location = (override_target = null) ->
555 # 1. If there was an override target specified, then let target be the
558 target = override_target
559 else # Otherwise, let target be the current node.
561 # 2. Determine the adjusted insertion location using the first matching
562 # steps from the following list:
564 # If foster parenting is enabled and target is a table, tbody, tfoot,
565 # thead, or tr element Foster parenting happens when content is
566 # misnested in tables.
567 if flag_foster_parenting and target.name in foster_parenting_targets
568 console.log "foster parenting isn't implemented yet" # TODO
569 # 1. Let last template be the last template element in the stack of
570 # open elements, if any.
571 # 2. Let last table be the last table element in the stack of open
574 # 3. If there is a last template and either there is no last table,
575 # or there is one, but last template is lower (more recently added)
576 # than last table in the stack of open elements, then: let adjusted
577 # insertion location be inside last template's template contents,
578 # after its last child (if any), and abort these substeps.
580 # 4. If there is no last table, then let adjusted insertion
581 # location be inside the first element in the stack of open
582 # elements (the html element), after its last child (if any), and
583 # abort these substeps. (fragment case)
585 # 5. If last table has a parent element, then let adjusted
586 # insertion location be inside last table's parent element,
587 # immediately before last table, and abort these substeps.
589 # 6. Let previous element be the element immediately above last
590 # table in the stack of open elements.
592 # 7. Let adjusted insertion location be inside previous element,
593 # after its last child (if any).
595 # Note: These steps are involved in part because it's possible for
596 # elements, the table element in this case in particular, to have
597 # been moved by a script around in the DOM, or indeed removed from
598 # the DOM entirely, after the element was inserted by the parser.
600 # Otherwise Let adjusted insertion location be inside target, after
601 # its last child (if any).
602 target_i = target.children.length
604 # 3. If the adjusted insertion location is inside a template element,
605 # let it instead be inside the template element's template contents,
606 # after its last child (if any). TODO
608 # 4. Return the adjusted insertion location.
609 return [target, target_i]
611 # http://www.w3.org/TR/html5/syntax.html#create-an-element-for-the-token
612 # aka create_an_element_for_token
613 token_to_element = (t, namespace, intended_parent) ->
614 t.type = TYPE_TAG # not TYPE_OPEN_TAG
615 # convert attributes into a hash
617 while t.attrs_a.length
619 attrs[a[0]] = a[1] # TODO check what to do with dupilcate attrs
620 el = new Node TYPE_TAG, name: t.name, namespace: namespace, attrs: attrs
622 # TODO 2. If the newly created element has an xmlns attribute in the
623 # XMLNS namespace whose value is not exactly the same as the element's
624 # namespace, that is a parse error. Similarly, if the newly created
625 # element has an xmlns:xlink attribute in the XMLNS namespace whose
626 # value is not the XLink Namespace, that is a parse error.
628 # fixfull: the spec says stuff about form pointers and ownerDocument
632 # FIXME read implement "foster parenting" part
633 # FIXME read spec, do this right
634 # FIXME implement the override target thing
635 # note: this assumes it's an open tag
636 # TODO tree_insert_html_element = (t, ...
637 tree_insert_element = (el, override_target = null, namespace = null) ->
638 dest = adjusted_insertion_location override_target
639 if el.type is TYPE_OPEN_TAG # means it's a "token"
640 el = token_to_element el, namespace, dest[0]
641 # fixfull: Document nodes sometimes can't accept more chidren
642 dest[0].children.splice dest[1], 0, el
647 # http://www.w3.org/TR/html5/syntax.html#insert-a-comment
648 tree_insert_a_comment = (t) ->
649 # FIXME read spec for "adjusted insertion location, etc, this might be wrong
650 open_els[0].children.push t
652 # 8.2.5.4 http://www.w3.org/TR/html5/syntax.html#parsing-main-inbody
653 in_body_any_other_end_tag = (name) -> # factored out because adoption agency calls it
654 for node, i in open_els
656 # FIXME generate implied end tags except those with name==name
657 parse_error() unless i is 0
663 if special_elements[node.name]?
666 tree_in_body = (t) ->
672 when "\t", "\u000a", "\u000c", "\u000d", ' '
673 reconstruct_active_formatting_elements()
676 reconstruct_active_formatting_elements()
678 flag_frameset_ok = false
680 tree_insert_a_comment t
687 return if template_tag_is_open()
688 root_attrs = open_els[open_els.length - 1].children
690 root_attrs[k] = v unless root_attrs[k]?
691 when 'base', 'basefont', 'bgsound', 'link', 'meta', 'noframes', 'script', 'style', 'template', 'title'
692 # FIXME also do this for </template> (end tag)
693 return tree_in_head t
700 when 'address', 'article', 'aside', 'blockquote', 'center', 'details', 'dialog', 'dir', 'div', 'dl', 'fieldset', 'figcaption', 'figure', 'footer', 'header', 'hgroup', 'main', 'nav', 'ol', 'p', 'section', 'summary', 'ul'
701 close_p_if_in_button_scope()
702 tree_insert_element t
703 when 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'
704 close_p_if_in_button_scope()
705 if open_els[0].name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']
708 tree_insert_element t
709 # TODO lots more to implement here
710 when 'b', 'big', 'code', 'em', 'font', 'i', 's', 'small', 'strike', 'strong', 'tt', 'u'
711 reconstruct_active_formatting_elements()
712 el = tree_insert_element t
714 # TODO lots more to implement here
715 else # any other start tag
716 reconstruct_active_formatting_elements()
717 tree_insert_element t
720 dd: true, dt: true, li: true, p: true, tbody: true, td: true,
721 tfoot: true, th: true, thead: true, tr: true, body: true, html: true,
724 unless ok_tags[t.name]?
727 # TODO stack of template insertion modes thing
728 flag_parsing = false # stop parsing
732 unless is_in_scope 'body'
735 # TODO implement parse error and move to tree_after_body
737 unless is_in_scope 'body' # weird, but it's what the spec says
740 # TODO implement parse error and move to tree_after_body, reprocess
741 # TODO lots more close tags to implement here
742 when 'a', 'b', 'big', 'code', 'em', 'font', 'i', 'nobr', 's', 'small', 'strike', 'strong', 'tt', 'u'
743 adoption_agency t.name
744 # TODO lots more close tags to implement here
746 in_body_any_other_end_tag t.name
750 # the functions below implement the tokenizer stats described here:
751 # http://www.w3.org/TR/html5/syntax.html#tokenization
753 # 8.2.4.1 http://www.w3.org/TR/html5/syntax.html#data-state
755 switch c = txt.charAt(cur++)
757 return new_text_node tokenize_character_reference()
759 tok_state = tok_state_tag_open
762 return new_text_node c
764 return new_eof_token()
766 return new_text_node c
769 # 8.2.4.2 http://www.w3.org/TR/html5/syntax.html#character-reference-in-data-state
770 # not needed: tok_state_character_reference_in_data = ->
771 # just call tok_state_character_reference_in_data()
773 # 8.2.4.8 http://www.w3.org/TR/html5/syntax.html#tag-open-state
774 tok_state_tag_open = ->
775 switch c = txt.charAt(cur++)
777 tok_state = tok_state_markup_declaration_open
779 tok_state = tok_state_end_tag_open
782 tok_state = tok_state_bogus_comment
784 if lc_alpha.indexOf(c) > -1
785 tok_cur_tag = new_open_tag c
786 tok_state = tok_state_tag_name
787 else if uc_alpha.indexOf(c) > -1
788 tok_cur_tag = new_open_tag c.toLowerCase()
789 tok_state = tok_state_tag_name
792 tok_state = tok_state_data
793 cur -= 1 # we didn't parse/handle the char after <
794 return new_text_node '<'
797 # 8.2.4.9 http://www.w3.org/TR/html5/syntax.html#end-tag-open-state
798 tok_state_end_tag_open = ->
799 switch c = txt.charAt(cur++)
802 tok_state = tok_state_data
805 tok_state = tok_state_data
806 return new_text_node '</'
808 if uc_alpha.indexOf(c) > -1
809 tok_cur_tag = new_end_tag c.toLowerCase()
810 tok_state = tok_state_tag_name
811 else if lc_alpha.indexOf(c) > -1
812 tok_cur_tag = new_end_tag c
813 tok_state = tok_state_tag_name
816 tok_state = tok_state_bogus_comment
819 # 8.2.4.10 http://www.w3.org/TR/html5/syntax.html#tag-name-state
820 tok_state_tag_name = ->
821 switch c = txt.charAt(cur++)
822 when "\t", "\n", "\u000c", ' '
823 tok_state = tok_state_before_attribute_name
825 tok_state = tok_state_self_closing_start_tag
827 tok_state = tok_state_data
833 tok_cur_tag.name += "\ufffd"
836 tok_state = tok_state_data
838 if uc_alpha.indexOf(c) > -1
839 tok_cur_tag.name += c.toLowerCase()
841 tok_cur_tag.name += c
844 # 8.2.4.34 http://www.w3.org/TR/html5/syntax.html#before-attribute-name-state
845 tok_state_before_attribute_name = ->
847 switch c = txt.charAt(cur++)
848 when "\t", "\n", "\u000c", ' '
851 tok_state = tok_state_self_closing_start_tag
854 tok_state = tok_state_data
861 when '"', "'", '<', '='
866 tok_state = tok_state_data
868 if uc_alpha.indexOf(c) > -1
869 attr_name = c.toLowerCase()
873 tok_cur_tag.attrs_a.unshift [attr_name, '']
874 tok_state = tok_state_attribute_name
877 # 8.2.4.35 http://www.w3.org/TR/html5/syntax.html#attribute-name-state
878 tok_state_attribute_name = ->
879 switch c = txt.charAt(cur++)
880 when "\t", "\n", "\u000c", ' '
881 tok_state = tok_state_after_attribute_name
883 tok_state = tok_state_self_closing_start_tag
885 tok_state = tok_state_before_attribute_value
887 tok_state = tok_state_data
893 tok_cur_tag.attrs_a[0][0] = "\ufffd"
896 tok_cur_tag.attrs_a[0][0] = c
899 tok_state = tok_state_data
901 if uc_alpha.indexOf(c) > -1
902 tok_cur_tag.attrs_a[0][0] = c.toLowerCase()
904 tok_cur_tag.attrs_a[0][0] += c
907 # 8.2.4.37 http://www.w3.org/TR/html5/syntax.html#before-attribute-value-state
908 tok_state_before_attribute_value = ->
909 switch c = txt.charAt(cur++)
910 when "\t", "\n", "\u000c", ' '
913 tok_state = tok_state_attribute_value_double_quoted
915 tok_state = tok_state_attribute_value_unquoted
918 tok_state = tok_state_attribute_value_single_quoted
921 tok_cur_tag.attrs_a[0][1] += "\ufffd"
922 tok_state = tok_state_attribute_value_unquoted
925 tok_state = tok_state_data
931 tok_state = tok_state_data
933 tok_cur_tag.attrs_a[0][1] += c
934 tok_state = tok_state_attribute_value_unquoted
937 # 8.2.4.38 http://www.w3.org/TR/html5/syntax.html#attribute-value-(double-quoted)-state
938 tok_state_attribute_value_double_quoted = ->
939 switch c = txt.charAt(cur++)
941 tok_state = tok_state_after_attribute_value_quoted
943 tok_cur_tag.attrs_a[0][1] += tokenize_character_reference '"', true
946 tok_cur_tag.attrs_a[0][1] += "\ufffd"
949 tok_state = tok_state_data
951 tok_cur_tag.attrs_a[0][1] += c
954 # 8.2.4.39 http://www.w3.org/TR/html5/syntax.html#attribute-value-(single-quoted)-state
955 tok_state_attribute_value_single_quoted = ->
956 switch c = txt.charAt(cur++)
958 tok_state = tok_state_after_attribute_value_quoted
960 tok_cur_tag.attrs_a[0][1] += tokenize_character_reference "'", true
963 tok_cur_tag.attrs_a[0][1] += "\ufffd"
966 tok_state = tok_state_data
968 tok_cur_tag.attrs_a[0][1] += c
971 # 8.2.4.40 http://www.w3.org/TR/html5/syntax.html#attribute-value-(unquoted)-state
972 tok_state_attribute_value_unquoted = ->
973 switch c = txt.charAt(cur++)
974 when "\t", "\n", "\u000c", ' '
975 tok_state = tok_state_before_attribute_name
977 tok_cur_tag.attrs_a[0][1] += tokenize_character_reference '>', true
979 tok_state = tok_state_data
984 tok_cur_tag.attrs_a[0][1] += "\ufffd"
987 tok_state = tok_state_data
989 # Parse Error if ', <, = or ` (backtick)
990 tok_cur_tag.attrs_a[0][1] += c
993 # 8.2.4.42 http://www.w3.org/TR/html5/syntax.html#after-attribute-value-(quoted)-state
994 tok_state_after_attribute_value_quoted = ->
995 switch c = txt.charAt(cur++)
996 when "\t", "\n", "\u000c", ' '
997 tok_state = tok_state_before_attribute_name
999 tok_state = tok_state_self_closing_start_tag
1001 tok_state = tok_state_data
1007 tok_state = tok_state_data
1010 tok_state = tok_state_before_attribute_name
1011 cur -= 1 # we didn't handle that char
1014 # 8.2.4.69 http://www.w3.org/TR/html5/syntax.html#consume-a-character-reference
1015 # Don't set this as a state, just call it
1016 # returns a string (NOT a text node)
1017 tokenize_character_reference = (allowed_char = null, in_attr = false) ->
1018 if cur >= txt.length
1020 switch c = txt.charAt(cur)
1021 when "\t", "\n", "\u000c", ' ', '<', '&', '', allowed_char
1022 # explicitly not a parse error
1025 # there has to be "one or more" alnums between & and ; to be a parse error
1028 if cur + 1 >= txt.length
1030 if txt.charAt(cur + 1).toLowerCase() is 'x'
1039 while start + i < txt.length and charset.indexOf(txt.charAt(start + i)) > -1
1043 if txt.charAt(start + i) is ';'
1045 # FIXME This is supposed to generate parse errors for some chars
1046 decoded = decode_named_char_ref(prefix + txt.substr(start, i).toLowerCase())
1053 if alnum.indexOf(txt.charAt(cur + i)) is -1
1056 # exit early, because parse_error() below needs at least one alnum
1058 if txt.charAt(cur + i) is ';'
1059 i += 1 # include ';' terminator in value
1060 decoded = decode_named_char_ref txt.substr(cur, i)
1067 # no ';' terminator (only legacy char refs)
1069 for i in [2..max] # no prefix matches, so ok to check shortest first
1070 c = legacy_char_refs[txt.substr(cur, i)]
1073 if txt.charAt(cur + i) is '='
1074 # "because some legacy user agents will
1075 # misinterpret the markup in those cases"
1078 if alnum.indexOf(txt.charAt(cur + i)) > -1
1079 # this makes attributes forgiving about url args
1081 # ok, and besides the weird exceptions for attributes...
1082 # return the matching char
1083 cur += i # consume entity chars
1084 parse_error() # because no terminating ";"
1088 return # never reached
1090 # tree constructor initialization
1091 # see comments on TYPE_TAG/etc for the structure of this data
1092 tree = new Node TYPE_TAG, name: 'html'
1094 tree_state = tree_in_body
1095 flag_frameset_ok = true
1097 flag_foster_parenting = false
1098 afe = [] # active formatting elements
1100 # tokenizer initialization
1101 tok_state = tok_state_data
1108 return tree.children
1110 # everything below is tests on the above
1111 test_equals = (description, output, expected_output) ->
1112 if output is expected_output
1113 console.log "passed." # don't say name, so smart consoles can merge all of these
1115 console.log "FAILED: \"#{description}\""
1116 console.log " Expected: #{expected_output}"
1117 console.log " Actual: #{output}"
1118 test_parser = (args) ->
1122 parsed = parse_html args.html, errors_cb
1128 serialized += t.serialize()
1129 if serialized isnt args.expected or parse_errors.length isnt args.errors
1130 console.log "FAILED: \"#{args.name}\""
1132 console.log "passed \"#{args.name}\""
1133 if serialized isnt args.expected
1134 console.log " Input: #{args.html}"
1135 console.log " Correct: #{args.expected}"
1136 console.log " Output: #{serialized}"
1137 if parse_errors.length isnt args.errors
1138 console.log " Expected #{args.errors} parse errors, but got these: #{JSON.stringify parse_errors}"
1140 test_parser name: "empty", \
1144 test_parser name: "just text", \
1146 expected: 'text:"abc"',
1148 test_parser name: "named entity", \
1150 expected: 'text:"a&1234"',
1152 test_parser name: "broken named character references", \
1153 html: "1&2&&3&aabbcc;",
1154 expected: 'text:"1&2&&3&aabbcc;"',
1156 test_parser name: "numbered entity overrides", \
1157 html: "1€€ ƒ",
1158 expected: 'text:"1€€ ƒ"',
1160 test_parser name: "open tag", \
1161 html: "foo<span>bar",
1162 expected: 'text:"foo",tag:"span",{},[text:"bar"]',
1163 errors: 1 # no close tag
1164 test_parser name: "open tag with attributes", \
1165 html: "foo<span style=\"foo: bar\" title=\"hi\">bar",
1166 expected: 'text:"foo",tag:"span",{"style":"foo: bar","title":"hi"},[text:"bar"]',
1167 errors: 1 # no close tag
1168 test_parser name: "open tag with attributes of various quotings", \
1169 html: "foo<span abc=\"def\" g=hij klm='nopqrstuv\"' autofocus>bar",
1170 expected: 'text:"foo",tag:"span",{"abc":"def","g":"hij","klm":"nopqrstuv\\"","autofocus":""},[text:"bar"]',
1171 errors: 1 # no close tag
1172 test_parser name: "attribute entity exceptions dq", \
1173 html: "foo<a href=\"foo?t=1&=2&o=3&lt=foo\">bar",
1174 expected: 'text:"foo",tag:"a",{"href":"foo?t=1&=2&o=3<=foo"},[text:"bar"]',
1175 errors: 2 # no close tag, &= in attr
1176 test_parser name: "attribute entity exceptions sq", \
1177 html: "foo<a href='foo?t=1&=2&o=3&lt=foo'>bar",
1178 expected: 'text:"foo",tag:"a",{"href":"foo?t=1&=2&o=3<=foo"},[text:"bar"]',
1179 errors: 2 # no close tag, &= in attr
1180 test_parser name: "attribute entity exceptions uq", \
1181 html: "foo<a href=foo?t=1&=2&o=3&lt=foo>bar",
1182 expected: 'text:"foo",tag:"a",{"href":"foo?t=1&=2&o=3<=foo"},[text:"bar"]',
1183 errors: 2 # no close tag, &= in attr
1184 test_parser name: "matching closing tags", \
1185 html: "foo<a href=\"hi\">hi</a><div>1<div>foo</div>2</div>bar",
1186 expected: 'text:"foo",tag:"a",{"href":"hi"},[text:"hi"],tag:"div",{},[text:"1",tag:"div",{},[text:"foo"],text:"2"],text:"bar"',
1188 test_parser name: "missing closing tag inside", \
1189 html: "foo<div>bar<span>baz</div>qux",
1190 expected: 'text:"foo",tag:"div",{},[text:"bar",tag:"span",{},[text:"baz"]],text:"qux"',
1191 errors: 1 # close tag mismatch
1192 test_parser name: "mis-matched closing tags", \
1193 html: "<span>12<div>34</span>56</div>78",
1194 expected: 'tag:"span",{},[text:"12",tag:"div",{},[text:"3456"],text:"78"]',
1195 errors: 2 # misplaced </span>, no </span> at the end
1196 test_parser name: "mis-matched formatting elements", \
1197 html: "12<b>34<i>56</b>78</i>90",
1198 expected: 'text:"12",tag:"b",{},[text:"34",tag:"i",{},[text:"56"]],tag:"i",{},[text:"78"],text:"90"',
1199 errors: 1 # no idea how many their should be
1200 test_parser name: "crazy formatting elements test", \
1201 html: "<b><i><a><s><tt><div></b>first</b></div></tt></s></a>second</i>",
1202 # chrome does this: expected: 'tag:"b",{},[tag:"i",{},[tag:"a",{},[tag:"s",{},[tag:"tt",{},[]]],text:"second"]],tag:"a",{},[tag:"s",{},[tag:"tt",{},[tag:"div",{},[tag:"b",{},[],text:"first"]]]]'
1203 # firefox does this:
1204 expected: 'tag:"b",{},[tag:"i",{},[tag:"a",{},[tag:"s",{},[tag:"tt",{},[]]]]],tag:"a",{},[tag:"s",{},[tag:"tt",{},[tag:"div",{},[tag:"b",{},[],text:"first"]]]],text:"second"',
1205 errors: 6 # no idea how many there should be
1206 # tests from https://github.com/html5lib/html5lib-tests/blob/master/tree-construction/adoption01.dat
1207 test_parser name: "html5lib aaa 1", \
1208 html: '<a><p></a></p>',
1209 expected: 'tag:"a",{},[],tag:"p",{},[tag:"a",{},[]]',
1211 test_parser name: "html5lib aaa 2", \
1212 html: '<a>1<p>2</a>3</p>',
1213 expected: 'tag:"a",{},[text:"1"],tag:"p",{},[tag:"a",{},[text:"2"],text:"3"]',
1215 test_parser name: "html5lib aaa 3", \
1216 html: '<a>1<button>2</a>3</button>',
1217 expected: 'tag:"a",{},[text:"1"],tag:"button",{},[tag:"a",{},[text:"2"],text:"3"]',
1219 test_parser name: "html5lib aaa 4", \
1220 html: '<a>1<b>2</a>3</b>',
1221 expected: 'tag:"a",{},[text:"1",tag:"b",{},[text:"2"]],tag:"b",{},[text:"3"]',
1223 test_parser name: "html5lib aaa 5", \
1224 html: '<a>1<div>2<div>3</a>4</div>5</div>',
1225 expected: 'tag:"a",{},[text:"1"],tag:"div",{},[tag:"a",{},[text:"2"],tag:"div",{},[tag:"a",{},[text:"3"],text:"4"]text:"5"]',