45: def scan_tokens encoder, options
46:
47: state = :initial
48: last_token = ''
49:
50: until eos?
51:
52: if state == :initial
53:
54: if match = scan(/ \s+ /x)
55: encoder.text_token match, :space
56: next
57:
58: elsif match = scan(%r! \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !mx)
59: encoder.text_token match, :preprocessor
60: next
61:
62: elsif match = scan(%r! // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !mx)
63: encoder.text_token match, :comment
64: next
65:
66: elsif match = scan(/ <[>=]? | >=? | :=? | [-+=*\/;,@\^|\(\)\[\]] | \.\. /x)
67: encoder.text_token match, :operator
68:
69: elsif match = scan(/\./)
70: encoder.text_token match, :operator
71: next if last_token == 'end'
72:
73: elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
74: encoder.text_token match, NAME_FOLLOWS[last_token] ? :ident : IDENT_KIND[match]
75:
76: elsif match = skip(/ ' ( [^\n']|'' ) (?:'|$) /x)
77: encoder.begin_group :char
78: encoder.text_token "'", :delimiter
79: encoder.text_token self[1], :content
80: encoder.text_token "'", :delimiter
81: encoder.end_group :char
82: next
83:
84: elsif match = scan(/ ' /x)
85: encoder.begin_group :string
86: encoder.text_token match, :delimiter
87: state = :string
88:
89: elsif match = scan(/ \# (?: \d+ | \$[0-9A-Fa-f]+ ) /x)
90: encoder.text_token match, :char
91:
92: elsif match = scan(/ \$ [0-9A-Fa-f]+ /x)
93: encoder.text_token match, :hex
94:
95: elsif match = scan(/ (?: \d+ ) (?![eE]|\.[^.]) /x)
96: encoder.text_token match, :integer
97:
98: elsif match = scan(/ \d+ (?: \.\d+ (?: [eE][+-]? \d+ )? | [eE][+-]? \d+ ) /x)
99: encoder.text_token match, :float
100:
101: else
102: encoder.text_token getch, :error
103: next
104:
105: end
106:
107: elsif state == :string
108: if match = scan(/[^\n']+/)
109: encoder.text_token match, :content
110: elsif match = scan(/''/)
111: encoder.text_token match, :char
112: elsif match = scan(/'/)
113: encoder.text_token match, :delimiter
114: encoder.end_group :string
115: state = :initial
116: next
117: elsif match = scan(/\n/)
118: encoder.end_group :string
119: encoder.text_token match, :space
120: state = :initial
121: else
122: raise "else case \' reached; %p not handled." % peek(1), encoder
123: end
124:
125: else
126: raise 'else-case reached', encoder
127:
128: end
129:
130: last_token = match
131:
132: end
133:
134: if state == :string
135: encoder.end_group state
136: end
137:
138: encoder
139: end