# This file implements a parser for html snippets, meant to be used by a
-# WYSIWYG editor. Hence it does not attempt to parse doctypes, <html>, <head>
-# or <body> tags, nor does it produce the top level "document" node in the dom
-# tree, nor nodes for html, head or body. Comments containing "fixfull"
-# indicate places where additional code is needed for full HTML document
-# parsing.
+# WYSIWYG editor.
+
+# The implementation is a pretty direct implementation of the parsing algorithm
+# described here:
+# http://www.w3.org/TR/html5/syntax.html#preprocessing-the-input-stream
+#
+# Deviations from that spec:
#
-# Instead, the data structure produced by this parser is an array of Nodes.
+# Purposeful: search this file for "WTAG"
+#
+# Not finished yet: search this file for "fixfull", "TODO" and "FIXME"
# stacks/lists
window.wheic = {}
module = exports: window.wheic
+from_code_point = (x) ->
+ if String.fromCodePoint?
+ return String.fromCodePoint x
+ else
+ if x <= 0xffff
+ return String.fromCharCode x
+ x -= 0x10000
+ return String.fromCharCode((x >> 10) + 0xd800, (x % 0x400) + 0xdc00)
+
# Each node is an obect of the Node class. Here are the Node types:
TYPE_TAG = 0 # name, {attributes}, [children]
TYPE_TEXT = 1 # "text"
@id = "#{++prev_node_id}"
acknowledge_self_closing: ->
if @token?
- @token.flag 'did_self_close'
+ @token.flag 'did_self_close', true
else
@flag 'did_self_close', true
flag: (key, value = null) ->
# https://en.wikipedia.org/wiki/Whitespace_character#Unicode
whitespace_chars = "\u0009\u000a\u000b\u000c\u000d\u0020\u0085\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000"
+unicode_fixes = {}
+unicode_fixes[0x00] = "\uFFFD"
+unicode_fixes[0x80] = "\u20AC"
+unicode_fixes[0x82] = "\u201A"
+unicode_fixes[0x83] = "\u0192"
+unicode_fixes[0x84] = "\u201E"
+unicode_fixes[0x85] = "\u2026"
+unicode_fixes[0x86] = "\u2020"
+unicode_fixes[0x87] = "\u2021"
+unicode_fixes[0x88] = "\u02C6"
+unicode_fixes[0x89] = "\u2030"
+unicode_fixes[0x8A] = "\u0160"
+unicode_fixes[0x8B] = "\u2039"
+unicode_fixes[0x8C] = "\u0152"
+unicode_fixes[0x8E] = "\u017D"
+unicode_fixes[0x91] = "\u2018"
+unicode_fixes[0x92] = "\u2019"
+unicode_fixes[0x93] = "\u201C"
+unicode_fixes[0x94] = "\u201D"
+unicode_fixes[0x95] = "\u2022"
+unicode_fixes[0x96] = "\u2013"
+unicode_fixes[0x97] = "\u2014"
+unicode_fixes[0x98] = "\u02DC"
+unicode_fixes[0x99] = "\u2122"
+unicode_fixes[0x9A] = "\u0161"
+unicode_fixes[0x9B] = "\u203A"
+unicode_fixes[0x9C] = "\u0153"
+unicode_fixes[0x9E] = "\u017E"
+unicode_fixes[0x9F] = "\u0178"
+
# These are the character references that don't need a terminating semicolon
# min length: 2, max: 6, none are a prefix of any other.
legacy_char_refs = {
h2:NS_HTML, h3:NS_HTML, h4:NS_HTML, h5:NS_HTML, h6:NS_HTML, head:NS_HTML,
header:NS_HTML, hgroup:NS_HTML, hr:NS_HTML, html:NS_HTML, iframe:NS_HTML,
img:NS_HTML, input:NS_HTML, isindex:NS_HTML, li:NS_HTML, link:NS_HTML,
- listing:NS_HTML, main:NS_HTML, marquee:NS_HTML, meta:NS_HTML, nav:NS_HTML,
- noembed:NS_HTML, noframes:NS_HTML, noscript:NS_HTML, object:NS_HTML,
- ol:NS_HTML, p:NS_HTML, param:NS_HTML, plaintext:NS_HTML, pre:NS_HTML,
- script:NS_HTML, section:NS_HTML, select:NS_HTML, source:NS_HTML,
- style:NS_HTML, summary:NS_HTML, table:NS_HTML, tbody:NS_HTML, td:NS_HTML,
- template:NS_HTML, textarea:NS_HTML, tfoot:NS_HTML, th:NS_HTML,
- thead:NS_HTML, title:NS_HTML, tr:NS_HTML, track:NS_HTML, ul:NS_HTML,
- wbr:NS_HTML, xmp:NS_HTML,
+ listing:NS_HTML, main:NS_HTML, marquee:NS_HTML,
+
+ menu:NS_HTML,menuitem:NS_HTML, # WATWG adds these
+
+ meta:NS_HTML, nav:NS_HTML, noembed:NS_HTML, noframes:NS_HTML,
+ noscript:NS_HTML, object:NS_HTML, ol:NS_HTML, p:NS_HTML, param:NS_HTML,
+ plaintext:NS_HTML, pre:NS_HTML, script:NS_HTML, section:NS_HTML,
+ select:NS_HTML, source:NS_HTML, style:NS_HTML, summary:NS_HTML,
+ table:NS_HTML, tbody:NS_HTML, td:NS_HTML, template:NS_HTML,
+ textarea:NS_HTML, tfoot:NS_HTML, th:NS_HTML, thead:NS_HTML, title:NS_HTML,
+ tr:NS_HTML, track:NS_HTML, ul:NS_HTML, wbr:NS_HTML, xmp:NS_HTML,
# MathML:
mi:NS_MATHML, mo:NS_MATHML, mn:NS_MATHML, ms:NS_MATHML, mtext:NS_MATHML,
ychannelselector: 'yChannelSelector'
zoomandpan: 'zoomAndPan'
}
+foreign_attr_fixes = {
+ 'xlink:actuate': 'xlink actuate'
+ 'xlink:arcrole': 'xlink arcrole'
+ 'xlink:href': 'xlink href'
+ 'xlink:role': 'xlink role'
+ 'xlink:show': 'xlink show'
+ 'xlink:title': 'xlink title'
+ 'xlink:type': 'xlink type'
+ 'xml:base': 'xml base'
+ 'xml:lang': 'xml lang'
+ 'xml:space': 'xml space'
+ 'xmlns': 'xmlns'
+ 'xmlns:xlink': 'xmlns xlink'
+}
adjust_mathml_attributes = (t) ->
for a in t.attrs_a
if a[0] is 'definitionurl'
return
adjust_foreign_attributes = (t) ->
# fixfull
+ for a in t.attrs_a
+ if foreign_attr_fixes[a[0]]?
+ a[0] = foreign_attr_fixes[a[0]]
return
# decode_named_char_ref()
standard_scopers = {
applet: NS_HTML, caption: NS_HTML, html: NS_HTML, table: NS_HTML,
td: NS_HTML, th: NS_HTML, marquee: NS_HTML, object: NS_HTML,
- template: NS_HTML, mi: NS_MATHML,
+ template: NS_HTML,
- mo: NS_MATHML, mn: NS_MATHML, ms: NS_MATHML, mtext: NS_MATHML,
- 'annotation-xml': NS_MATHML,
+ mi: NS_MATHML, mo: NS_MATHML, mn: NS_MATHML, ms: NS_MATHML,
+ mtext: NS_MATHML, 'annotation-xml': NS_MATHML,
foreignObject: NS_SVG, desc: NS_SVG, title: NS_SVG
}
for t in open_els
if t.name is tag_name and (namespace is null or namespace is t.namespace)
return true
- if t.ns isnt NS_HTML and t.name isnt 'optgroup' and t.name isnt 'option'
+ if t.namespace isnt NS_HTML and t.name isnt 'optgroup' and t.name isnt 'option'
return false
return false
# this checks for a particular element, not by name
# fixfull (fragment case)
# 4. If node is a select element, run these substeps:
- if node.name is 'select'
+ if node.name is 'select' and node.namespace is NS_HTML
# 1. If last is true, jump to the step below labeled done.
unless last
# 2. Let ancestor be node.
ancestor = open_els[ancestor_i]
# 5. If ancestor is a template node, jump to the step below
# labeled done.
- if ancestor.name is 'template'
+ if ancestor.name is 'template' and ancestor.namespace is NS_HTML
break
# 6. If ancestor is a table node, switch the insertion mode
# to "in select in table" and abort these steps.
- if ancestor.name is 'table'
+ if ancestor.name is 'table' and ancestor.namespace is NS_HTML
ins_mode = ins_mode_in_select_in_table
return
# 7. Jump back to the step labeled loop.
return
# 5. If node is a td or th element and last is false, then switch
# the insertion mode to "in cell" and abort these steps.
- if (node.name is 'td' or node.name is 'th') and last is false
+ if (node.name is 'td' or node.name is 'th') and node.namespace is NS_HTML and last is false
ins_mode = ins_mode_in_cell
return
# 6. If node is a tr element, then switch the insertion mode to "in
# row" and abort these steps.
- if node.name is 'tr'
+ if node.name is 'tr' and node.namespace is NS_HTML
ins_mode = ins_mode_in_row
return
# 7. If node is a tbody, thead, or tfoot element, then switch the
# insertion mode to "in table body" and abort these steps.
- if node.name is 'tbody' or node.name is 'thead' or node.name is 'tfoot'
+ if (node.name is 'tbody' or node.name is 'thead' or node.name is 'tfoot') and node.namespace is NS_HTML
ins_mode = ins_mode_in_table_body
return
# 8. If node is a caption element, then switch the insertion mode
# to "in caption" and abort these steps.
- if node.name is 'caption'
+ if node.name is 'caption' and node.namespace is NS_HTML
ins_mode = ins_mode_in_caption
return
# 9. If node is a colgroup element, then switch the insertion mode
# to "in column group" and abort these steps.
- if node.name is 'colgroup'
+ if node.name is 'colgroup' and node.namespace is NS_HTML
ins_mode = ins_mode_in_column_group
return
# 10. If node is a table element, then switch the insertion mode to
# "in table" and abort these steps.
- if node.name is 'table'
+ if node.name is 'table' and node.namespace is NS_HTML
ins_mode = ins_mode_in_table
return
# 11. If node is a template element, then switch the insertion mode
# to the current template insertion mode and abort these steps.
- # fixfull (template insertion mode stack)
-
+ if node.name is 'template' and node.namespace is NS_HTML
+ ins_mode = template_ins_modes[0]
+ return
# 12. If node is a head element and last is true, then switch the
# insertion mode to "in body" ("in body"! not "in head"!) and abort
# these steps. (fragment case)
- if node.name is 'head' and last
+ if node.name is 'head' and node.namespace is NS_HTML and last
ins_mode = ins_mode_in_body
return
# 13. If node is a head element and last is false, then switch the
# insertion mode to "in head" and abort these steps.
- if node.name is 'head' and last is false
+ if node.name is 'head' and node.namespace is NS_HTML and last is false
ins_mode = ins_mode_in_head
return
# 14. If node is a body element, then switch the insertion mode to
# "in body" and abort these steps.
- if node.name is 'body'
+ if node.name is 'body' and node.namespace is NS_HTML
ins_mode = ins_mode_in_body
return
# 15. If node is a frameset element, then switch the insertion mode
# to "in frameset" and abort these steps. (fragment case)
- if node.name is 'frameset'
+ if node.name is 'frameset' and node.namespace is NS_HTML
ins_mode = ins_mode_in_frameset
return
# 16. If node is an html element, run these substeps:
- if node.name is 'html'
+ if node.name is 'html' and node.namespace is NS_HTML
# 1. If the head element pointer is null, switch the insertion
# mode to "before head" and abort these steps. (fragment case)
if head_element_pointer is null
ins_mode t
return
if is_mathml_text_integration_point(acn)
- if t.type is TYPE_START_TAG and (t.name is 'mglyph' or t.name is 'malignmark')
+ if t.type is TYPE_START_TAG and not (t.name is 'mglyph' or t.name is 'malignmark')
ins_mode t
return
if t.type is TYPE_TEXT
parse_error()
return if open_els.length < 2
second = open_els[open_els.length - 2]
- return unless second.ns is NS_HTML
+ return unless second.namespace is NS_HTML
return unless second.name is 'body'
return if template_tag_is_open()
- frameset_ok_flag = false
+ flag_frameset_ok = false
for a of t.attrs_a
second.attrs[a[0]] = a[1] unless second.attrs[a[0]]?
return
return if open_els.length < 2
second_i = open_els.length - 2
second = open_els[second_i]
- return unless second.ns is NS_HTML
+ return unless second.namespace is NS_HTML
return unless second.name is 'body'
- flag_frameset_ok = false
+ if flag_frameset_ok is false
+ return
if second.parent?
for el, i in second.parent.children
if el is second
reconstruct_afe()
insert_html_element t
return
- if t.type is TYPE_START_TAG and (t.name is 'rb' or t.name is 'rp' or t.name is 'rtc')
+# this comment block implements the W3C spec
+# if t.type is TYPE_START_TAG and (t.name is 'rb' or t.name is 'rp' or t.name is 'rtc')
+# if is_in_scope 'ruby', NS_HTML
+# generate_implied_end_tags()
+# unless open_els[0].name is 'ruby' and open_els[0].namespace is NS_HTML
+# parse_error()
+# insert_html_element t
+# return
+# if t.type is TYPE_START_TAG and t.name is 'rt'
+# if is_in_scope 'ruby', NS_HTML
+# generate_implied_end_tags 'rtc' # arg is exception
+# unless (open_els[0].name is 'ruby' or open_els[0].name is 'rtc') and open_els[0].namespace is NS_HTML
+# parse_error()
+# insert_html_element t
+# return
+# below implements the WATWG spec https://html.spec.whatwg.org/multipage/syntax.html#parsing-main-inbody
+ if t.type is TYPE_START_TAG and (t.name is 'rb' or t.name is 'rtc')
if is_in_scope 'ruby', NS_HTML
generate_implied_end_tags()
unless open_els[0].name is 'ruby' and open_els[0].namespace is NS_HTML
parse_error()
insert_html_element t
return
- if t.type is TYPE_START_TAG and t.name is 'rt'
+ if t.type is TYPE_START_TAG and (t.name is 'rp' or t.name is 'rt')
if is_in_scope 'ruby', NS_HTML
- generate_implied_end_tags 'rtc' # arg is exception
+ generate_implied_end_tags 'rtc'
unless (open_els[0].name is 'ruby' or open_els[0].name is 'rtc') and open_els[0].namespace is NS_HTML
parse_error()
insert_html_element t
return
+# end WATWG chunk
if t.type is TYPE_START_TAG and t.name is 'math'
reconstruct_afe()
adjust_mathml_attributes t
ins_mode_in_table = (t) ->
switch t.type
when TYPE_TEXT
- if t.name is 'table' or t.name is 'tbody' or t.name is 'tfoot' or t.name is 'thead' or t.name is 'tr'
+ if (open_els[0].name is 'table' or open_els[0].name is 'tbody' or open_els[0].name is 'tfoot' or open_els[0].name is 'thead' or open_els[0].name is 'tr') and open_els[0].namespace is NS_HTML
+ pending_table_character_tokens = []
original_ins_mode = ins_mode
ins_mode = ins_mode_in_table_text
process_token t
# 8.2.5.4.10 http://www.w3.org/TR/html5/syntax.html#parsing-main-intabletext
ins_mode_in_table_text = (t) ->
if t.type is TYPE_TEXT and t.text is "\u0000"
- # huh? I thought the tokenizer didn't emit these
+ # from javascript?
parse_error()
return
if t.type is TYPE_TEXT
insert_character old
else
for old in pending_table_character_tokens
- ins_mode_table_else old
- pending_table_character_tokens = [] # FIXME test (spec doesn't say this)
+ ins_mode_in_table_else old
+ pending_table_character_tokens = []
ins_mode = original_ins_mode
process_token t
ins_mode_in_body t
return
if t.type is TYPE_END_TAG and t.name is 'html'
- # fixfull fragment case
+ if flag_fragment_parsing
+ parse_error()
+ return
ins_mode = ins_mode_after_after_body
return
if t.type is TYPE_EOF
# Anything else
parse_error()
ins_mode = ins_mode_in_body
+ process_token t
return
# 8.2.5.4.23 http://www.w3.org/TR/html5/syntax.html#the-after-after-frameset-insertion-mode
if t.name is 'script'
t.acknowledge_self_closing()
in_foreign_content_end_script()
+ # fixfull
else
open_els.shift()
t.acknowledge_self_closing()
return
loop # is this safe?
open_els.shift()
- cn = open_els[0]
- if is_mathml_text_integration_point(cn) or is_html_integration(cn) or cn.namespace is NS_HTML
+ if is_mathml_text_integration_point(open_els[0]) or is_html_integration(open_els[0]) or open_els[0].namespace is NS_HTML
break
process_token t
return
in_foreign_content_end_script()
return
if t.type is TYPE_END_TAG
- if open_els[0].name.toLowerCase() isnt t.name
+ i = 0
+ node = open_els[i]
+ if node.name.toLowerCase() isnt t.name
parse_error()
- for node in open_els
+ loop
if node is open_els[open_els.length - 1]
return
if node.name.toLowerCase() is t.name
el = open_els.shift()
if el is node
return
+ i += 1
+ node = open_els[i]
if node.namespace is NS_HTML
break
ins_mode t # explicitly call HTML insertion mode
tok_state = tok_state_tag_open
when "\u0000"
parse_error()
- return new_text_node c
+ return new_text_node "\ufffd"
when '' # EOF
return new_eof_token()
else
tok_state_self_closing_start_tag = ->
c = txt.charAt(cur++)
if c is '>'
- tok_cur_tag.flag 'self-closing'
+ tok_cur_tag.flag 'self-closing', true
tok_state = tok_state_data
return tok_cur_tag
if c is ''
else
val = txt.substr cur, (next_gt - cur)
cur = next_gt + 1
- val = val.replace "\u0000", "\ufffd"
+ val = val.replace(new RegExp("\u0000", 'g'), "\ufffd")
tok_cur_tag.text += val
tok_state = tok_state_data
return tok_cur_tag
else
val = txt.substr cur, (next_gt - cur)
cur = next_gt + 3
- val = val.replace(new RegExp("\u0000", 'g'), "\ufffd") # fixfull spec doesn't say this
- val = val.replace(new RegExp("\r\n", 'g'), "\n") # fixfull spec doesn't say this
- val = val.replace(new RegExp("\r", 'g'), "\n") # fixfull spec doesn't say this
return new_character_token val # fixfull split
# 8.2.4.69 http://www.w3.org/TR/html5/syntax.html#consume-a-character-reference
if cur + 1 >= txt.length
return '&'
if txt.charAt(cur + 1).toLowerCase() is 'x'
- prefix = '#x'
+ base = 16
charset = hex_chars
start = cur + 2
else
charset = digits
start = cur + 1
- prefix = '#'
+ base = 10
i = 0
while start + i < txt.length and charset.indexOf(txt.charAt(start + i)) > -1
i += 1
if i is 0
return '&'
+ cur = start + i
if txt.charAt(start + i) is ';'
- i += 1
- # FIXME This is supposed to generate parse errors for some chars
- decoded = decode_named_char_ref(prefix + txt.substr(start, i).toLowerCase())
- if decoded?
- cur = start + i
- return decoded
- return '&'
+ cur += 1
+ else
+ parse_error()
+ code_point = txt.substr(start, i)
+ while code_point.charAt(0) is '0' and code_point.length > 1
+ code_point = code_point.substr 1
+ code_point = parseInt(code_point, base)
+ if unicode_fixes[code_point]?
+ parse_error()
+ return unicode_fixes[code_point]
+ else
+ if (code_point >= 0xd800 and code_point <= 0xdfff) or code_point > 0x10ffff
+ parse_error()
+ return "\ufffd"
+ else
+ if (code_point >= 0x0001 and code_point <= 0x0008) or (code_point >= 0x000D and code_point <= 0x001F) or (code_point >= 0x007F and code_point <= 0x009F) or (code_point >= 0xFDD0 and code_point <= 0xFDEF) or code_point is 0x000B or code_point is 0xFFFE or code_point is 0xFFFF or code_point is 0x1FFFE or code_point is 0x1FFFF or code_point is 0x2FFFE or code_point is 0x2FFFF or code_point is 0x3FFFE or code_point is 0x3FFFF or code_point is 0x4FFFE or code_point is 0x4FFFF or code_point is 0x5FFFE or code_point is 0x5FFFF or code_point is 0x6FFFE or code_point is 0x6FFFF or code_point is 0x7FFFE or code_point is 0x7FFFF or code_point is 0x8FFFE or code_point is 0x8FFFF or code_point is 0x9FFFE or code_point is 0x9FFFF or code_point is 0xAFFFE or code_point is 0xAFFFF or code_point is 0xBFFFE or code_point is 0xBFFFF or code_point is 0xCFFFE or code_point is 0xCFFFF or code_point is 0xDFFFE or code_point is 0xDFFFF or code_point is 0xEFFFE or code_point is 0xEFFFF or code_point is 0xFFFFE or code_point is 0xFFFFF or code_point is 0x10FFFE or code_point is 0x10FFFF
+ parse_error()
+ return from_code_point code_point
+ return
else
for i in [0...31]
if alnum.indexOf(txt.charAt(cur + i)) is -1
# tokenizer initialization
tok_state = tok_state_data
- if args.name is "namespace-sensitivity.dat #1"
+ # text pre-processing
+ # FIXME http://www.w3.org/TR/html5/syntax.html#preprocessing-the-input-stream
+ txt = txt.replace(new RegExp("\u0000", 'g'), "\ufffd") # fixfull spec doesn't say this
+ txt = txt.replace(new RegExp("\r\n", 'g'), "\n") # fixfull spec doesn't say this
+ txt = txt.replace(new RegExp("\r", 'g'), "\n") # fixfull spec doesn't say this
+
+ if args.name is "plain-text-unsafe.dat #4"
console.log "hi"
# proccess input
# http://www.w3.org/TR/html5/syntax.html#tree-construction
Got questions, comments, patches, etc.? Contact Jason Woofenden