D7net
Home
Console
Upload
information
Create File
Create Folder
About
Tools
:
/
proc
/
self
/
root
/
opt
/
alt
/
ruby24
/
lib64
/
ruby
/
2.4.0
/
rdoc
/
Filename :
ruby_token.rb
back
Copy
# frozen_string_literal: false #-- # irb/ruby-token.rb - ruby tokens # $Release Version: 0.9.5$ # $Revision: 11708 $ # $Date: 2007-02-12 15:01:19 -0800 (Mon, 12 Feb 2007) $ # by Keiju ISHITSUKA(keiju@ruby-lang.org) #++ # Definitions of all tokens involved in the lexical analysis. # # This class is not documented because it is so deep in the internals. module RDoc::RubyToken # :stopdoc: EXPR_BEG = :EXPR_BEG EXPR_MID = :EXPR_MID EXPR_END = :EXPR_END EXPR_ARG = :EXPR_ARG EXPR_FNAME = :EXPR_FNAME EXPR_DOT = :EXPR_DOT EXPR_CLASS = :EXPR_CLASS # for ruby 1.4X if !defined?(Symbol) Symbol = Integer end def set_token_position(line, char) @prev_line_no = line @prev_char_no = char end class Token def initialize(seek, line_no, char_no, text = nil) @seek = seek @line_no = line_no @char_no = char_no @text = text end attr_reader :seek attr_reader :line_no attr_reader :char_no attr_accessor :text def ==(other) self.class == other.class and other.line_no == @line_no and other.char_no == @char_no and other.text == @text end ## # Because we're used in contexts that expect to return a token, we set the # text string and then return ourselves def set_text(text) @text = text self end def inspect # :nodoc: klass = self.class.name.split('::').last "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @text] end end class TkNode < Token def initialize(seek, line_no, char_no, node = nil) super seek, line_no, char_no @node = node end attr_reader:node def ==(other) self.class == other.class and other.line_no == @line_no and other.char_no == @char_no and other.node == @node end def set_text text @node = text self end alias text node def inspect # :nodoc: klass = self.class.name.split('::').last "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @node] end end class TkId < Token def initialize(seek, line_no, char_no, name) super(seek, line_no, char_no) @name = name end attr_reader:name def ==(other) self.class == other.class and other.line_no == @line_no and other.char_no == @char_no and other.name == @name end def set_text text @name = text self end alias text name def inspect # :nodoc: klass = self.class.name.split('::').last "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name] end end class TkKW < TkId end class TkVal < Token def initialize(seek, line_no, char_no, value = nil) super(seek, line_no, char_no) @value = value end attr_accessor :value def ==(other) self.class == other.class and other.line_no == @line_no and other.char_no == @char_no and other.value == @value end def set_text text @value = text self end alias text value def inspect # :nodoc: klass = self.class.name.split('::').last "{%s %s, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @value] end end class TkOp < Token def initialize(seek, line_no, char_no, name = nil) super seek, line_no, char_no @name = name end attr_accessor :name def ==(other) self.class == other.class and other.line_no == @line_no and other.char_no == @char_no and other.name == @name end def set_text text @name = text self end alias text name def inspect # :nodoc: klass = self.class.name.split('::').last "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name] end end class TkOPASGN < TkOp def initialize(seek, line_no, char_no, op) super(seek, line_no, char_no) op = TkReading2Token[op][0] unless op.kind_of?(Symbol) @op = op @text = nil end attr_reader:op def ==(other) self.class == other.class and other.line_no == @line_no and other.char_no == @char_no and other.op == @op end def text @text ||= "#{TkToken2Reading[op]}=" end def inspect # :nodoc: klass = self.class.name.split('::').last "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @op] end end class TkUnknownChar < Token def initialize(seek, line_no, char_no, name) super(seek, line_no, char_no) @name = name end attr_reader:name def ==(other) self.class == other.class and other.line_no == @line_no and other.char_no == @char_no and other.name == @name end def set_text text @name = text self end alias text name def inspect # :nodoc: klass = self.class.name.split('::').last "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name] end end class TkError < Token end def Token(token, value = nil) value ||= TkToken2Reading[token] case token when String if (tk = TkReading2Token[token]).nil? IRB.fail TkReading2TokenNoKey, token end tk = Token(tk[0], value) if tk.kind_of?(TkOp) then tk.name = token end when Symbol if (tk = TkSymbol2Token[token]).nil? IRB.fail TkSymbol2TokenNoKey, token end tk = Token(tk[0], value) else if token.instance_method(:initialize).arity == 3 then tk = token.new(@prev_seek, @prev_line_no, @prev_char_no) tk.set_text value else tk = token.new(@prev_seek, @prev_line_no, @prev_char_no, value) end end tk end TokenDefinitions = [ [:TkCLASS, TkKW, "class", :EXPR_CLASS], [:TkMODULE, TkKW, "module", :EXPR_BEG], [:TkDEF, TkKW, "def", :EXPR_FNAME], [:TkUNDEF, TkKW, "undef", :EXPR_FNAME], [:TkBEGIN, TkKW, "begin", :EXPR_BEG], [:TkRESCUE, TkKW, "rescue", :EXPR_MID], [:TkENSURE, TkKW, "ensure", :EXPR_BEG], [:TkEND, TkKW, "end", :EXPR_END], [:TkIF, TkKW, "if", :EXPR_BEG, :TkIF_MOD], [:TkUNLESS, TkKW, "unless", :EXPR_BEG, :TkUNLESS_MOD], [:TkTHEN, TkKW, "then", :EXPR_BEG], [:TkELSIF, TkKW, "elsif", :EXPR_BEG], [:TkELSE, TkKW, "else", :EXPR_BEG], [:TkCASE, TkKW, "case", :EXPR_BEG], [:TkWHEN, TkKW, "when", :EXPR_BEG], [:TkWHILE, TkKW, "while", :EXPR_BEG, :TkWHILE_MOD], [:TkUNTIL, TkKW, "until", :EXPR_BEG, :TkUNTIL_MOD], [:TkFOR, TkKW, "for", :EXPR_BEG], [:TkBREAK, TkKW, "break", :EXPR_MID], [:TkNEXT, TkKW, "next", :EXPR_END], [:TkREDO, TkKW, "redo", :EXPR_END], [:TkRETRY, TkKW, "retry", :EXPR_END], [:TkIN, TkKW, "in", :EXPR_BEG], [:TkDO, TkKW, "do", :EXPR_BEG], [:TkRETURN, TkKW, "return", :EXPR_MID], [:TkYIELD, TkKW, "yield", :EXPR_END], [:TkSUPER, TkKW, "super", :EXPR_END], [:TkSELF, TkKW, "self", :EXPR_END], [:TkNIL, TkKW, "nil", :EXPR_END], [:TkTRUE, TkKW, "true", :EXPR_END], [:TkFALSE, TkKW, "false", :EXPR_END], [:TkAND, TkKW, "and", :EXPR_BEG], [:TkOR, TkKW, "or", :EXPR_BEG], [:TkNOT, TkKW, "not", :EXPR_BEG], [:TkIF_MOD, TkKW], [:TkUNLESS_MOD, TkKW], [:TkWHILE_MOD, TkKW], [:TkUNTIL_MOD, TkKW], [:TkALIAS, TkKW, "alias", :EXPR_FNAME], [:TkDEFINED, TkKW, "defined?", :EXPR_END], [:TklBEGIN, TkKW, "BEGIN", :EXPR_END], [:TklEND, TkKW, "END", :EXPR_END], [:Tk__LINE__, TkKW, "__LINE__", :EXPR_END], [:Tk__FILE__, TkKW, "__FILE__", :EXPR_END], [:TkIDENTIFIER, TkId], [:TkFID, TkId], [:TkGVAR, TkId], [:TkCVAR, TkId], [:TkIVAR, TkId], [:TkCONSTANT, TkId], [:TkINTEGER, TkVal], [:TkFLOAT, TkVal], [:TkSTRING, TkVal], [:TkHEREDOC, TkVal], [:TkXSTRING, TkVal], [:TkREGEXP, TkVal], [:TkSYMBOL, TkVal], [:TkCHAR, TkVal], [:TkDSTRING, TkNode], [:TkDXSTRING, TkNode], [:TkDREGEXP, TkNode], [:TkNTH_REF, TkNode], [:TkBACK_REF, TkNode], [:TkUPLUS, TkOp, "+@"], [:TkUMINUS, TkOp, "-@"], [:TkPOW, TkOp, "**"], [:TkCMP, TkOp, "<=>"], [:TkEQ, TkOp, "=="], [:TkEQQ, TkOp, "==="], [:TkNEQ, TkOp, "!="], [:TkGEQ, TkOp, ">="], [:TkLEQ, TkOp, "<="], [:TkANDOP, TkOp, "&&"], [:TkOROP, TkOp, "||"], [:TkMATCH, TkOp, "=~"], [:TkNMATCH, TkOp, "!~"], [:TkDOT2, TkOp, ".."], [:TkDOT3, TkOp, "..."], [:TkAREF, TkOp, "[]"], [:TkASET, TkOp, "[]="], [:TkLSHFT, TkOp, "<<"], [:TkRSHFT, TkOp, ">>"], [:TkCOLON2, TkOp, '::'], [:TkCOLON3, TkOp, '::'], #[:OPASGN, TkOp], # +=, -= etc. # [:TkASSOC, TkOp, "=>"], [:TkQUESTION, TkOp, "?"], #? [:TkCOLON, TkOp, ":"], #: [:TkfLPAREN, Token, "("], # func( # [:TkfLBRACK, Token, "["], # func[ # [:TkfLBRACE, Token, "{"], # func{ # [:TkSYMBEG, Token, ":"], # :SYMBOL [:TkAMPER, TkOp, "&"], [:TkGT, TkOp, ">"], [:TkLT, TkOp, "<"], [:TkPLUS, TkOp, "+"], [:TkSTAR, TkOp, "*"], [:TkMINUS, TkOp, "-"], [:TkMULT, TkOp, "*"], [:TkDIV, TkOp, "/"], [:TkMOD, TkOp, "%"], [:TkBITOR, TkOp, "|"], [:TkBITXOR, TkOp, "^"], [:TkBITAND, TkOp, "&"], [:TkBITNOT, TkOp, "~"], [:TkNOTOP, TkOp, "!"], [:TkBACKQUOTE, TkOp, "`"], [:TkASSIGN, Token, "="], [:TkDOT, Token, "."], [:TkLPAREN, Token, "("], #(exp) [:TkLBRACK, Token, "["], #[arry] [:TkLBRACE, Token, "{"], #{hash} [:TkRPAREN, Token, ")"], [:TkRBRACK, Token, "]"], [:TkRBRACE, Token, "}"], [:TkCOMMA, Token, ","], [:TkSEMICOLON, Token, ";"], [:TkCOMMENT, TkVal], [:TkSPACE, Token, " "], [:TkNL, Token, "\n"], [:TkEND_OF_SCRIPT], [:TkBACKSLASH, TkUnknownChar, "\\"], [:TkAT, TkUnknownChar, "@"], [:TkDOLLAR, TkUnknownChar, "$"], ] # {reading => token_class} # {reading => [token_class, *opt]} TkReading2Token = {} TkToken2Reading = {} TkSymbol2Token = {} def self.def_token(token_n, super_token = Token, reading = nil, *opts) token_n = token_n.id2name if token_n.kind_of?(Symbol) if const_defined?(token_n) IRB.fail AlreadyDefinedToken, token_n end token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}") if reading TkToken2Reading[token_c] = reading return if TkReading2Token[reading] if opts.empty? TkReading2Token[reading] = [token_c] else TkReading2Token[reading] = [token_c].concat(opts) end end TkSymbol2Token[token_n.intern] = token_c end for defs in TokenDefinitions def_token(*defs) end def_token :TkRD_COMMENT, TkCOMMENT NEWLINE_TOKEN = TkNL.new 0, 0, 0, "\n" class TkSYMBOL def to_sym @sym ||= text[1..-1].intern end end # :startdoc: end