+ Added tentative 3.4 support.

+ Restructured comparison rake tasks and normalization to deal with lrama changes.
+ Updated compare versions and added 3.4.

[git-p4: depot-paths = "//src/ruby_parser/dev/": change = 14472]

Author: zenspider

Manifest.txt

@@ -30,6 +30,7 @@ lib/ruby_parser30.rb
 lib/ruby_parser31.rb
 lib/ruby_parser32.rb
 lib/ruby_parser33.rb
+lib/ruby_parser34.rb
 lib/ruby_parser_extras.rb
 test/test_ruby_lexer.rb
 test/test_ruby_parser.rb

Rakefile

@@ -14,7 +14,7 @@ Hoe.add_include_dirs "../../oedipus_lex/dev/lib"
 Hoe.add_include_dirs "../../ruby2ruby/dev/lib"
 
 V2   = %w[20 21 22 23 24 25 26 27]
-V3   = %w[30 31 32 33]
+V3   = %w[30 31 32 33 34]
 
 VERS = V2 + V3
 
@@ -164,7 +164,7 @@ def ruby_parse version
   desc "fetch all tarballs"
   task :fetch => tarball
 
-  file parse_y => tarball do
+  file ruby_dir => tarball do
     extract_glob = case
                    when version > "3.3" then
                      "{id.h,parse.y,tool/{id2token.rb,lrama},defs/id.def}"
@@ -176,15 +176,20 @@ def ruby_parse version
                      "{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
                    end
     system "tar xf #{tarball} -C compare #{File.basename ruby_dir}/#{extract_glob}"
+  end
+
+  file parse_y => ruby_dir do
+    # env -u RUBYOPT rake compare/parse33.y
+    warn "Warning: RUBYOPT is set! Use 'env -u RUBYOPT rake'" if ENV["RUBYOPT"]
 
     # Debugging a new parse build system:
     #
     # Unpack the ruby tarball in question, configure, and run the following:
     #
-    # % touch parse.y; make -n parse.c
+    # % [ -e Makefile ] || ./configure ; make -n -W parse.y parse.c
     # ...
     # echo generating parse.c
-    # /Users/ryan/.rubies.current/bin/ruby --disable=gems ./tool/id2token.rb parse.y | \
+    # ruby --disable=gems ./tool/id2token.rb parse.y | \
     #       ruby ./tool/lrama/exe/lrama -oparse.c -Hparse.h - parse.y
     #
     # Then integrate these commands into the mess below:
@@ -197,12 +202,6 @@ def ruby_parse version
           end
 
     sh cmd
-
-    if File.exist? "#{d}/tool/lrama" then # UGH: this is dumb
-      rm_rf "compare/lrama"
-      sh "mv #{d}/tool/lrama compare"
-    end
-    sh "rm -rf #{d}"
   end
 
   bison = Dir["/opt/homebrew/opt/bison/bin/bison",
@@ -211,8 +210,9 @@ def ruby_parse version
              ].first
 
   file mri_txt => [parse_y, normalize] do
+    d = ruby_dir
     if version > "3.3" then
-      sh "./compare/lrama/exe/lrama -r all -ocompare/parse#{v}.tab.c #{parse_y}"
+      sh "./#{d}/tool/lrama/exe/lrama -r states --report-file=compare/parse#{v}.output -ocompare/parse#{v}.tab.c #{parse_y}"
     else
       sh "#{bison} -r all #{parse_y}"
       mv Dir["parse#{v}.*"], "compare"
@@ -247,7 +247,7 @@ def ruby_parse version
   end
 
   task :clean do
-    rm_f Dir[mri_txt, rp_txt]
+    rm_f Dir[mri_txt, rp_txt, ruby_dir]
   end
 
   task :realclean do
@@ -296,9 +296,10 @@ ruby_parse "2.5.9"
 ruby_parse "2.6.10"
 ruby_parse "2.7.8"
 ruby_parse "3.0.6"
-ruby_parse "3.1.4"
-ruby_parse "3.2.2"
-ruby_parse "3.3.0"
+ruby_parse "3.1.7"
+ruby_parse "3.2.8"
+ruby_parse "3.3.7"
+ruby_parse "3.4.2"
 
 task :debug => :isolate do
   ENV["V"] ||= VERS.last

compare/normalize.rb

@@ -153,13 +153,21 @@ def munge s
 
              '"do (for condition)"', "kDO_COND",
              '"do (for lambda)"',    "kDO_LAMBDA",
-             '"do (for block)"',     "kDO_BLOCK",
+             %("'do' for block"),    "kDO_BLOCK",    # 3.4
+             %("'do' for lambda"),   "kDO_LAMBDA",   # 3.4
+             %("'do' for condition"),"kDO_COND",     # 3.4
+             %q("#{"),               "tSTRING_DBEG", # 3.4
+             '"do (for block)"',     "kDO_BLOCK",    # 3.4
 
+             /\"'(\w+)' \(?modifier\)?\"/, proc { |x| "k#{$1.upcase}_MOD" }, # 3.4
              /\"(\w+) \(?modifier\)?\"/, proc { |x| "k#{$1.upcase}_MOD" },
-             /\"(\w+)\"/,                proc { |x| "k#{$1.upcase}" },
+             /\"((?!k)\w+)\"/,             proc { |x| "k#{$1.upcase}" },
 
              /\$?@(\d+)(\s+|$)/,    "", # newer bison
 
+             # 3.4(ish?) changes:
+             "option_tNL",              "opt_nl", # ruby 3.4
+
              # TODO: remove for 3.0 work:
              "lex_ctxt ", "" # 3.0 production that's mostly noise right now
             ]
@@ -178,12 +186,16 @@ def munge s
 ARGF.each_line do |line|
   next unless good or line =~ /^-* ?Grammar|\$accept : /
 
-  case line.strip
+  case line.strip # TODO: .delete %q["'()]
   when /^$/ then
   when /^(\d+) (\$?[@\w]+): (.*)/ then    # yacc
     rule = $2
     order << rule unless rules.has_key? rule
     rules[rule] << munge($3)
+  when /^(\d+) (\$?[@\w]+'(?: |\\n)'): (.*)/ then # munges both sides
+    rule = $2
+    order << rule unless rules.has_key? rule
+    rules[munge(rule)] << munge($3)
   when /^(\d+) \s+\| (.*)/ then        # yacc
     rules[rule] << munge($2)
   when /^(\d+) (@\d+): (.*)/ then      # yacc

lib/ruby_parser.rb

@@ -83,10 +83,12 @@ class SyntaxError < RuntimeError; end
 require "ruby_parser31"
 require "ruby_parser32"
 require "ruby_parser33"
+require "ruby_parser34"
 
 class RubyParser # HACK
   VERSIONS.clear # also a HACK caused by racc namespace issues
 
+  class V34 < ::Ruby33Parser; end
   class V33 < ::Ruby33Parser; end
   class V32 < ::Ruby32Parser; end
   class V31 < ::Ruby31Parser; end

lib/ruby_parser3.yy

@@ -8,6 +8,8 @@ class Ruby31Parser
 class Ruby32Parser
 #elif V == 33
 class Ruby33Parser
+#elif V == 34
+class Ruby34Parser
 #else
 fail "version not specified or supported on code generation"
 #endif

test/test_ruby_parser.rb

@@ -5681,6 +5681,10 @@ module TestRubyParserShared33Plus
   include TestRubyParserShared32Plus
 end
 
+module TestRubyParserShared34Plus
+  include TestRubyParserShared33Plus
+end
+
 class Minitest::Test
   def skip s = "blah"
     warn "ignoring skip for %s: %s" % [name, s]
@@ -6039,6 +6043,16 @@ def setup
   end
 end
 
+class TestRubyParserV34 < RubyParserTestCase
+  include TestRubyParserShared34Plus
+
+  def setup
+    super
+
+    self.processor = RubyParser::V34.new
+  end
+end
+
 RubyParser::VERSIONS.each do |klass|
   v = klass.version
   describe "block args arity #{v}" do