# File lib/hocon/impl/tokenizer.rb, line 23 def self.as_string(codepoint) if codepoint == "\n" "newline" elsif codepoint == "\t" "tab" elsif codepoint == -1 "end of file" elsif codepoint =~ /[[:cntrl:]]/ "control character 0x%x" % codepoint else "%c" % codepoint end end
# File lib/hocon/impl/tokenizer.rb, line 43 def self.render(tokens) rendered_text = "" while (t = tokens.next) rendered_text << t.token_text end rendered_text end
Tokenizes a Reader. Does not close the reader; you have to arrange to do that after you're done with the returned iterator.
# File lib/hocon/impl/tokenizer.rb, line 39 def self.tokenize(origin, input, syntax) TokenIterator.new(origin, input, syntax != Hocon::ConfigSyntax::JSON) end