58: def scan_tokens encoder, options
59: states = Array(options[:state] || @state)
60: value_expected = @value_expected
61:
62: until eos?
63:
64: if match = scan(/\s+/)
65: encoder.text_token match, :space
66:
67: elsif case states.last
68: when :initial, :media
69: if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox)
70: encoder.text_token match, :type
71: next
72: elsif match = scan(RE::Class)
73: encoder.text_token match, :class
74: next
75: elsif match = scan(RE::Id)
76: encoder.text_token match, :constant
77: next
78: elsif match = scan(RE::PseudoClass)
79: encoder.text_token match, :pseudo_class
80: next
81: elsif match = scan(RE::AttributeSelector)
82:
83: encoder.text_token match[0,1], :operator
84: encoder.text_token match[1..-2], :attribute_name if match.size > 2
85: encoder.text_token match[-1,1], :operator if match[-1] == ?]
86: next
87: elsif match = scan(/@media/)
88: encoder.text_token match, :directive
89: states.push :media_before_name
90: next
91: end
92:
93: when :block
94: if match = scan(/(?>#{RE::Ident})(?!\()/ox)
95: if value_expected
96: encoder.text_token match, :value
97: else
98: encoder.text_token match, :key
99: end
100: next
101: end
102:
103: when :media_before_name
104: if match = scan(RE::Ident)
105: encoder.text_token match, :type
106: states[-1] = :media_after_name
107: next
108: end
109:
110: when :media_after_name
111: if match = scan(/\{/)
112: encoder.text_token match, :operator
113: states[-1] = :media
114: next
115: end
116:
117: else
118:
119: raise_inspect 'Unknown state', encoder
120:
121:
122: end
123:
124: elsif match = scan(/\/\*(?:.*?\*\/|\z)/m)
125: encoder.text_token match, :comment
126:
127: elsif match = scan(/\{/)
128: value_expected = false
129: encoder.text_token match, :operator
130: states.push :block
131:
132: elsif match = scan(/\}/)
133: value_expected = false
134: encoder.text_token match, :operator
135: if states.last == :block || states.last == :media
136: states.pop
137: end
138:
139: elsif match = scan(/#{RE::String}/o)
140: encoder.begin_group :string
141: encoder.text_token match[0, 1], :delimiter
142: encoder.text_token match[1..-2], :content if match.size > 2
143: encoder.text_token match[-1, 1], :delimiter if match.size >= 2
144: encoder.end_group :string
145:
146: elsif match = scan(/#{RE::Function}/o)
147: encoder.begin_group :string
148: start = match[/^\w+\(/]
149: encoder.text_token start, :delimiter
150: if match[-1] == ?)
151: encoder.text_token match[start.size..-2], :content
152: encoder.text_token ')', :delimiter
153: else
154: encoder.text_token match[start.size..-1], :content
155: end
156: encoder.end_group :string
157:
158: elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
159: encoder.text_token match, :float
160:
161: elsif match = scan(/#{RE::Color}/o)
162: encoder.text_token match, :color
163:
164: elsif match = scan(/! *important/)
165: encoder.text_token match, :important
166:
167: elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/)
168: encoder.text_token match, :color
169:
170: elsif match = scan(RE::AtKeyword)
171: encoder.text_token match, :directive
172:
173: elsif match = scan(/ [+>:;,.=()\/] /x)
174: if match == ':'
175: value_expected = true
176: elsif match == ';'
177: value_expected = false
178: end
179: encoder.text_token match, :operator
180:
181: else
182: encoder.text_token getch, :error
183:
184: end
185:
186: end
187:
188: if options[:keep_state]
189: @state = states
190: @value_expected = value_expected
191: end
192:
193: encoder
194: end