24: def scan_tokens encoder, options
25:
26: match = nil
27: code = ''
28:
29: until eos?
30:
31: if bol?
32: if match = scan(/!!!.*/)
33: encoder.text_token match, :doctype
34: next
35: end
36:
37: if match = scan(/(?>( *)(\/(?!\[if)|-\#|:javascript|:ruby|:\w+) *)(?=\n)/)
38: encoder.text_token match, :comment
39:
40: code = self[2]
41: if match = scan(/(?:\n+#{self[1]} .*)+/)
42: case code
43: when '/', '-#'
44: encoder.text_token match, :comment
45: when ':javascript'
46:
47: @java_script_scanner ||= CodeRay.scanner :java_script, :tokens => @tokens, :keep_tokens => true
48: @java_script_scanner.tokenize match, :tokens => encoder
49: when ':ruby'
50: @ruby_scanner.tokenize match, :tokens => encoder
51: when /:\w+/
52: encoder.text_token match, :comment
53: else
54: raise 'else-case reached: %p' % [code]
55: end
56: end
57: end
58:
59: if match = scan(/ +/)
60: encoder.text_token match, :space
61: end
62:
63: if match = scan(/\/.*/)
64: encoder.text_token match, :comment
65: next
66: end
67:
68: if match = scan(/\\/)
69: encoder.text_token match, :plain
70: if match = scan(/.+/)
71: @html_scanner.tokenize match, :tokens => encoder
72: end
73: next
74: end
75:
76: tag = false
77:
78: if match = scan(/%[\w:]+\/?/)
79: encoder.text_token match, :tag
80:
81:
82:
83:
84: tag = true
85: end
86:
87: while match = scan(/([.#])[-\w]*\w/)
88: encoder.text_token match, self[1] == '#' ? :constant : :class
89: tag = true
90: end
91:
92: if tag && match = scan(/(\()([^)]+)?(\))?/)
93:
94: encoder.text_token self[1], :plain
95: @html_scanner.tokenize self[2], :tokens => encoder, :state => :attribute if self[2]
96: encoder.text_token self[3], :plain if self[3]
97: end
98:
99: if tag && match = scan(/\{/)
100: encoder.text_token match, :plain
101:
102: code = ''
103: level = 1
104: while true
105: code << scan(/([^\{\},\n]|, *\n?)*/)
106: case match = getch
107: when '{'
108: level += 1
109: code << match
110: when '}'
111: level -= 1
112: if level > 0
113: code << match
114: else
115: break
116: end
117: when "\n", ",", nil
118: break
119: end
120: end
121: @ruby_scanner.tokenize code, :tokens => encoder unless code.empty?
122:
123: encoder.text_token match, :plain if match
124: end
125:
126: if tag && match = scan(/(\[)([^\]\n]+)?(\])?/)
127: encoder.text_token self[1], :plain
128: @ruby_scanner.tokenize self[2], :tokens => encoder if self[2]
129: encoder.text_token self[3], :plain if self[3]
130: end
131:
132: if tag && match = scan(/\//)
133: encoder.text_token match, :tag
134: end
135:
136: if scan(/(>?<?[-=]|[&!]=|(& |!)|~)( *)([^,\n\|]+(?:(, *|\|(?=.|\n.*\|$))\n?[^,\n\|]*)*)?/)
137: encoder.text_token self[1] + self[3], :plain
138: if self[4]
139: if self[2]
140: @embedded_ruby_scanner.tokenize self[4], :tokens => encoder
141: else
142: @ruby_scanner.tokenize self[4], :tokens => encoder
143: end
144: end
145: elsif match = scan(/((?:<|><?)(?![!?\/\w]))?(.+)?/)
146: encoder.text_token self[1], :plain if self[1]
147:
148: @html_scanner.tokenize self[2], :tokens => encoder if self[2]
149: end
150:
151: elsif match = scan(/.+/)
152: @html_scanner.tokenize match, :tokens => encoder
153:
154: end
155:
156: if match = scan(/\n/)
157: encoder.text_token match, :space
158: end
159: end
160:
161: encoder
162:
163: end