16: def scan_tokens encoder, options
17:
18: state = :initial
19: key_indent = string_indent = 0
20:
21: until eos?
22:
23: key_indent = nil if bol?
24:
25: if match = scan(/ +[\t ]*/)
26: encoder.text_token match, :space
27:
28: elsif match = scan(/\n+/)
29: encoder.text_token match, :space
30: state = :initial if match.index(?\n)
31:
32: elsif match = scan(/#.*/)
33: encoder.text_token match, :comment
34:
35: elsif bol? and case
36: when match = scan(/---|\.\.\./)
37: encoder.begin_group :head
38: encoder.text_token match, :head
39: encoder.end_group :head
40: next
41: when match = scan(/%.*/)
42: encoder.text_token match, :doctype
43: next
44: end
45:
46: elsif state == :value and case
47: when !check(/(?:"[^"]*")(?=: |:$)/) && match = scan(/"/)
48: encoder.begin_group :string
49: encoder.text_token match, :delimiter
50: encoder.text_token match, :content if match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)
51: encoder.text_token match, :delimiter if match = scan(/"/)
52: encoder.end_group :string
53: next
54: when match = scan(/[|>][-+]?/)
55: encoder.begin_group :string
56: encoder.text_token match, :delimiter
57: string_indent = key_indent || column(pos - match.size) - 1
58: encoder.text_token matched, :content if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
59: encoder.end_group :string
60: next
61: when match = scan(/(?![!"*&]).+?(?=$|\s+#)/)
62: encoder.begin_group :string
63: encoder.text_token match, :content
64: string_indent = key_indent || column(pos - match.size) - 1
65: encoder.text_token matched, :content if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
66: encoder.end_group :string
67: next
68: end
69:
70: elsif case
71: when match = scan(/[-:](?= |$)/)
72: state = :value if state == :colon && (match == ':' || match == '-')
73: state = :value if state == :initial && match == '-'
74: encoder.text_token match, :operator
75: next
76: when match = scan(/[,{}\[\]]/)
77: encoder.text_token match, :operator
78: next
79: when state == :initial && match = scan(/[-\w.()\/ ]*\S(?= *:(?: |$))/)
80: encoder.text_token match, :key
81: key_indent = column(pos - match.size) - 1
82: state = :colon
83: next
84: when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/)
85: encoder.begin_group :key
86: encoder.text_token match[0,1], :delimiter
87: encoder.text_token match[1..-2], :content
88: encoder.text_token match[-1,1], :delimiter
89: encoder.end_group :key
90: key_indent = column(pos - match.size) - 1
91: state = :colon
92: next
93: when match = scan(/(![\w\/]+)(:([\w:]+))?/)
94: encoder.text_token self[1], :type
95: if self[2]
96: encoder.text_token ':', :operator
97: encoder.text_token self[3], :class
98: end
99: next
100: when match = scan(/&\S+/)
101: encoder.text_token match, :variable
102: next
103: when match = scan(/\*\w+/)
104: encoder.text_token match, :global_variable
105: next
106: when match = scan(/<</)
107: encoder.text_token match, :class_variable
108: next
109: when match = scan(/\d\d:\d\d:\d\d/)
110: encoder.text_token match, :octal
111: next
112: when match = scan(/\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d(\.\d+)? [-+]\d\d:\d\d/)
113: encoder.text_token match, :octal
114: next
115: when match = scan(/:\w+/)
116: encoder.text_token match, :symbol
117: next
118: when match = scan(/[^:\s]+(:(?! |$)[^:\s]*)* .*/)
119: encoder.text_token match, :error
120: next
121: when match = scan(/[^:\s]+(:(?! |$)[^:\s]*)*/)
122: encoder.text_token match, :error
123: next
124: end
125:
126: else
127: raise if eos?
128: encoder.text_token getch, :error
129:
130: end
131:
132: end
133:
134: encoder
135: end