9: def scan_tokens tokens, options
10:
11: line_kind = nil
12: state = :initial
13:
14: until eos?
15: kind = match = nil
16:
17: if match = scan(/\n/)
18: if line_kind
19: tokens << [:end_line, line_kind]
20: line_kind = nil
21: end
22: tokens << [match, :space]
23: next
24: end
25:
26: case state
27:
28: when :initial
29: if match = scan(/--- |\+\+\+ |=+|_+/)
30: tokens << [:begin_line, line_kind = :head]
31: tokens << [match, :head]
32: next unless match = scan(/.+/)
33: kind = :plain
34: elsif match = scan(/Index: |Property changes on: /)
35: tokens << [:begin_line, line_kind = :head]
36: tokens << [match, :head]
37: next unless match = scan(/.+/)
38: kind = :plain
39: elsif match = scan(/Added: /)
40: tokens << [:begin_line, line_kind = :head]
41: tokens << [match, :head]
42: next unless match = scan(/.+/)
43: kind = :plain
44: state = :added
45: elsif match = scan(/\\ /)
46: tokens << [:begin_line, line_kind = :change]
47: tokens << [match, :change]
48: next unless match = scan(/.+/)
49: kind = :plain
50: elsif match = scan(/@@(?>[^@\n]*)@@/)
51: if check(/\n|$/)
52: tokens << [:begin_line, line_kind = :change]
53: else
54: tokens << [:open, :change]
55: end
56: tokens << [match[0,2], :change]
57: tokens << [match[2...-2], :plain]
58: tokens << [match[-2,2], :change]
59: tokens << [:close, :change] unless line_kind
60: next unless match = scan(/.+/)
61: kind = :plain
62: elsif match = scan(/\+/)
63: tokens << [:begin_line, line_kind = :insert]
64: tokens << [match, :insert]
65: next unless match = scan(/.+/)
66: kind = :plain
67: elsif match = scan(/-/)
68: tokens << [:begin_line, line_kind = :delete]
69: tokens << [match, :delete]
70: next unless match = scan(/.+/)
71: kind = :plain
72: elsif scan(/ .*/)
73: kind = :comment
74: elsif scan(/.+/)
75: tokens << [:begin_line, line_kind = :comment]
76: kind = :plain
77: else
78: raise_inspect 'else case rached'
79: end
80:
81: when :added
82: if match = scan(/ \+/)
83: tokens << [:begin_line, line_kind = :insert]
84: tokens << [match, :insert]
85: next unless match = scan(/.+/)
86: kind = :plain
87: else
88: state = :initial
89: next
90: end
91: end
92:
93: match ||= matched
94: if $CODERAY_DEBUG and not kind
95: raise_inspect 'Error token %p in line %d' %
96: [[match, kind], line], tokens
97: end
98: raise_inspect 'Empty token', tokens unless match
99:
100: tokens << [match, kind]
101: end
102:
103: tokens << [:end_line, line_kind] if line_kind
104: tokens
105: end