class Gem::RequestSet::Lockfile::Tokenizer
Public Class Methods
from_file(file)
click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 5 def self.from_file file new File.read(file), file end
new(input, filename = nil, line = 0, pos = 0)
click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 9 def initialize input, filename = nil, line = 0, pos = 0 @line = line @line_pos = pos @tokens = [] @filename = filename tokenize input end
Public Instance Methods
empty?()
click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 37 def empty? @tokens.empty? end
make_parser(set, platforms)
click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 17 def make_parser set, platforms Gem::RequestSet::Lockfile::Parser.new self, set, platforms, @filename end
next_token()
click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 45 def next_token @tokens.shift end
Also aliased as: shift
peek()
click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 50 def peek @tokens.first || [:EOF] end
skip(type)
click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 25 def skip type @tokens.shift while not @tokens.empty? and peek.first == type end
to_a()
click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 21 def to_a @tokens end
unshift(token)
click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 41 def unshift token @tokens.unshift token end
Private Instance Methods
tokenize(input)
click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 56 def tokenize input s = StringScanner.new input until s.eos? do pos = s.pos pos = s.pos if leading_whitespace = s.scan(/ +/) if s.scan(/[<|=>]{7}/) then message = "your #{@filename} contains merge conflict markers" column, line = token_pos pos raise Gem::RequestSet::Lockfile::ParseError.new message, column, line, @filename end @tokens << case when s.scan(/\r?\n/) then token = [:newline, nil, *token_pos(pos)] @line_pos = s.pos @line += 1 token when s.scan(/[A-Z]+/) then if leading_whitespace then text = s.matched text += s.scan(/[^\s)]*/).to_s # in case of no match [:text, text, *token_pos(pos)] else [:section, s.matched, *token_pos(pos)] end when s.scan(/([a-z]+):\s/) then s.pos -= 1 # rewind for possible newline [:entry, s[1], *token_pos(pos)] when s.scan(/\(/) then [:l_paren, nil, *token_pos(pos)] when s.scan(/\)/) then [:r_paren, nil, *token_pos(pos)] when s.scan(/<=|>=|=|~>|<|>|!=/) then [:requirement, s.matched, *token_pos(pos)] when s.scan(/,/) then [:comma, nil, *token_pos(pos)] when s.scan(/!/) then [:bang, nil, *token_pos(pos)] when s.scan(/[^\s),!]*/) then [:text, s.matched, *token_pos(pos)] else raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}" end end @tokens end