cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r173:aa6e7ff3942c` public

parent | child

Description:

merge from algo-bm

Commit status:

[Not Reviewed]

References:

merge java

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r173:aa6e7ff3942c - Sun, 28 Sep 2014 02:42:25 - 9 files changed: 124 inserted, 34 deleted

grader ¶ +18 -2

               end
               def display_manual
                 puts <<USAGE
               Grader.
               using: (1) grader
-            -        (2) grader environment [mode]
+            +        (2) grader environment [mode] [options]
                      (3) grader stop [all|pids-list]
                      (4) grader --help
               (1) call grader with environment = 'exam', mode = 'queue'
               (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
+            +   queue: repeatedly check the task queue and grade any available tasks
+            +
+            +   prob:  re-grade every user latest submission of the specific problem.
+            +          the problem name must be specified by the next argument.
+            +
+            +          additional options:
+            +
+            +             --all-sub    re-grade every submissions instead of just the latest submission of each user.
+            +   sub:   re-grader the specified submission.
+            +          The submission ID to be re-graded must be specified by the next argument.
+            +
               (3) create stop-file to stop running grader in queue mode
               (4) You are here.
               USAGE
               end
               def process_options_and_stop_file
                 # Process mode and environment option
                 if ARGV.length >= 1
                   options[:environment] = ARGV.shift
                   if ARGV.length >=1
                     options[:mode] = ARGV.shift
                   end
+            +   else
+            +     puts 'no argument specified, using default mode and environment.'
                 end
                 options[:dry_run] = (ARGV.delete('--dry') != nil)
                 if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
                   puts "Dry run currently works only for 'prob' or 'contest' modes."
                   exit(0)
                 options[:report] = (ARGV.delete('--report') != nil)
                 if options[:report] and (not ['prob','contest','autonew'].include? options[:mode])
                   puts "Report currently works only for 'prob' or 'contest' modes."
                   exit(0)
                 end
+            +   options[:all_sub] = (ARGV.delete('--all-sub') != nil)
+            +
                 return options
               end
               class ResultCollector
                 def initialize
                   @results = {}
                 ARGV.each do |prob_name|
                   prob = Problem.find_by_name(prob_name)
                   if prob==nil
                     puts "cannot find problem: #{prob_name}"
                   else
-            -       runner.grade_problem(prob)
+            +       runner.grade_problem(prob,options)
                   end
                 end
                 if options[:report]
                   result_collector.print_report_by_user
                 end
               options = process_options_and_stop_file
               GRADER_ENV = options[:environment]
               grader_mode = options[:mode]
               dry_run = options[:dry_run]
               puts "environment: #{GRADER_ENV}"
+            + puts "grader mode: #{grader_mode}"
               require File.join(File.dirname(__FILE__),'config/environment')
               # add grader_mode to config
               # this is needed because method log needs it.  TODO: clean this up
               class << config
                 attr_accessor :grader_mode

lib/runner.rb ¶ +10 -5

                       @grader_process.report_inactive(task) if @grader_process!=nil
                     end
                     return task
                   end
                   def grade_problem(problem, options={})
-            -       users = User.find(:all)
+            +       User.find_each do |u|
-            -       users.each do |u|
                       puts "user: #{u.login}"
                       if options[:user_conditions]!=nil
                         con_proc = options[:user_conditions]
                         next if not con_proc.call(u)
                       end
-            -         last_sub = Submission.find_last_by_user_and_problem(u.id,problem.id)
+            +         if options[:all_sub]
-            -         if last_sub!=nil
+            +           Submission.where(user_id: u.id,problem_id: problem.id).find_each do |sub|
-            -           @engine.grade(last_sub)
+            +             @engine.grade(sub)
+            +           end
+            +         else
+            +           last_sub = Submission.find_last_by_user_and_problem(u.id,problem.id)
+            +           if last_sub!=nil
+            +             @engine.grade(last_sub)
+            +           end
                       end
                     end
                   end
                   def grade_submission(submission)
                     puts "Submission: #{submission.id} by #{submission.user.full_name}"

lib/submission_helper.rb ¶ +22 -4

                       cmp_file.close
                     else
                       cmp_msg = ""
                     end
                     result_fname = "#{test_result_dir}/result"
-            -       comment_fname = "#{test_result_dir}/comment"
+            +       comment_fname = "#{test_result_dir}/comment"
+            +       runstat_fname = "#{test_result_dir}/run_stat"
                     if FileTest.exist?(result_fname)
                       comment = ""
                       begin
                         result_file = File.open(result_fname)
                         result = result_file.readline.to_i
                         result_file.close
                         comment += comment_file.readline.chomp
                         comment_file.close
                       rescue
                         comment += ""
                       end
-            -         return {:points => result,
+            +         begin
-            -           :comment => comment,
+            +           runstat_file = File.open(runstat_fname)
-            -           :cmp_msg => cmp_msg}
+            +           max_runtime = runstat_file.readline.to_f
+            +           peak_memory = runstat_file.readline.to_i
+            +         rescue
+            +           max_runtime = -1
+            +           peak_memory = -1
+            +         end
+            +
+            +
+            +         return {points: result,
+            +           comment: comment,
+            +           cmp_msg: cmp_msg,
+            +           max_runtime: max_runtime,
+            +           peak_memory: peak_memory
+            +         }
                     else
                       if FileTest.exist?("#{test_result_dir}/a.out")
                         return {:points => 0,
                           :comment => 'error during grading',
                           :cmp_msg => cmp_msg}
                       else
                     problem = submission.problem
                     submission.graded_at = Time.now.gmtime
                     points = result[:points]
                     submission.points = points
                     comment = @config.report_comment(result[:comment])
+            +       submission.peak_memory = result[:peak_memory]
+            +       submission.max_runtime =  result[:max_runtime]
+            +       submission.effective_code_length =submission.source.length
+            +
                     #
                     # TODO: FIX THIS MESSAGE
                     #
                     if problem == nil
                       submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
                     elsif points == problem.full_score

lib/test_request_helper.rb ¶ +1 -1

                     else
                       seconds = nil
                       time_stat = "Time used: n/a sec."
                     end
                     # extract memory usage
-            -       if res = /s(.*)m/.match(running_stat_line)
+            +       if res = /s(.*)kbytes/.match(running_stat_line)
                       memory_used = res[1].to_i
                     else
                       memory_used = -1
                     end
                     return {

std-script/box64-new.c ¶ +2 -7

               			double system_time,
               			int mem_usage)
               {
                 //total is user
                 //wall is wall
                 //
-            -   fprintf(stderr,"%.4lfr%.4lfu%.4lfs%dm\n",
+            +   fprintf(stderr,"%.4lfr%.4lfu%.4lfs%dkbytes\n",
               	  wall_time, user_time, system_time, mem_usage);
               }
               static void
               final_stats(struct rusage *rus)
               {
                     }
                   }
                 print_running_stat(
                       (double)wall_ms/1000,
                       (double)total_ms/1000,
                       (double)sys_ms/1000,
-            -         (mem_peak_kb + 1023) / 1024);
+            +         mem_peak_kb);
                 meta_close();
                 exit(rc);
               }
               static void
               flush_line(void)
               	  if (timeout && total_ms > timeout)
               	    err("TO: Time limit exceeded");
               	  if (wall_timeout && wall_ms > wall_timeout)
               	    err("TO: Time limit exceeded (wall clock)");
               	  flush_line();
                   fprintf(stderr,"OK\n");
-            -         print_running_stat(
-            -               (double)wall_ms/1000,
-            -               (double)total_ms/1000,
-            -               (double)sys_ms/1000,
-            -               (mem_peak_kb + 1023) / 1024);
               	  box_exit(0);
               	}
                     if (WIFSIGNALED(stat))
               	{
               	  box_pid = 0;
               	  meta_printf("exitsig:%d\n", WTERMSIG(stat));

std-script/compile ¶ +25 -3

               CPLUSPLUS_COMPILER = "/usr/bin/g++"
               PASCAL_COMPILER = "/usr/bin/fpc"
               JAVA_COMPILER = "/usr/bin/javac"
               RUBY_INTERPRETER = "/usr/bin/ruby"
               PYTHON_INTERPRETER = "/usr/bin/python"
               PYTHON_CHECKER = "/usr/bin/pyflakes"
+            + PHP_INTERPRETER = "/usr/bin/php"
               C_OPTIONS = "-O2 -s -static -std=c99 -DCONTEST -lm -Wall"
-            - CPLUSPLUS_OPTIONS = "-O2 -s -static -DCONTEST -lm -Wall"
+            + CPLUSPLUS_OPTIONS = "-O2 -s -std=c++11 -static -DCONTEST -lm -Wall"
               PASCAL_OPTIONS = "-O1 -XS -dCONTEST"
               JAVA_OPTIONS = ""
               PYTHON_OPTIONS = ""
+            + PHP_OPTIONS = "-l"
               # Check for the correct number of arguments. Otherwise, print usage.
               if ARGV.length == 0 or ARGV.length > 4
                 puts "Usage: compile <language> [<source-file>] [<output-file>] [<message-file>]"
                 puts
                 puts "<source-file>  is defaulted to \"source\"."
               when "java"
                 #rename the file to the public class name
                 #get the class name
                 classname = 'DUMMY'
+            +   source = Array.new
                 File.foreach(params[:source_file]) do |line|
                   md = /\s*public\s*class\s*(\w*)/.match(line)
                   classname=md[1] if md
+            +     source << line unless line =~ /\s*package\s*\w+\s*\;/
                 end
-            -   system("cp #{params[:source_file]} #{classname}.java")
+            +   File.open("#{classname}.java","w") do |file|
+            +     source.each do |s|
+            +       file.puts s
+            +     end
+            +   end
+            +   #system("cp #{params[:source_file]} #{classname}.java")
                 command = "#{JAVA_COMPILER} #{classname}.java 2> #{params[:message_file]}"
                 system(command)
                 if File.exists?(classname + ".class")
                   File.open(params[:output_file],"w") {|file| file.write("#{classname}")}
                 end
                 if classname == 'DUMMY'
                   puts "pwd: " + Dir.pwd
                   Dir.new('.').each {|file| puts file}
                   File.open(params[:output_file],"w") do |out_file|
                     out_file.puts "#!#{PYTHON_INTERPRETER} #{params[:source_file]}c"
                   end
                   File.chmod(0755, params[:output_file])
+            +     FileUtils.cp("#{params[:source_file]}c",params[:output_file])
                 end
+            +
+            + when "php"
+            +   command = "#{PHP_INTERPRETER} #{PHP_OPTIONS} #{params[:source_file]} 2> #{params[:message_file]}"
+            +   if system(command)
+            +     File.open(params[:output_file],"w") do |out_file|
+            +       out_file.puts "#!#{PHP_INTERPRETER}"
+            +       File.open(params[:source_file],"r").each do |line|
+            +         out_file.print line
+            +       end
+            +     end
+            +     File.chmod(0755, params[:output_file])
+            +   end
+            +
               else
                 talk("ERROR: Invalid language specified!")
                 open(params[:message_file],"w") do |f|
                   f.puts "ERROR: Invalid language specified!"
                 end
                 exit(127)

std-script/grade ¶ +27 -2

                   res[1]
                 else
                   RUN_ERROR_MARK  # these are run time errors
                 end
               end
+            + def extract_time(t)
+            +   #puts "TIME: #{t}"
+            +   if (result=/^(.*)r(.*)u(.*)s(.*)kbytes/.match(t))
+            +     {:real => result[1], :user => result[2], :sys => result[3], :mem => result[4]}
+            +   else
+            +     #{:real => 0, :user => 0, :sys => 0}
+            +     #puts "ERROR READING RUNNING TIME: #{t}"
+            +     raise "Error reading running time: #{t}"
+            +   end
+            + end
+            +
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               if problem.well_formed? == false
                 log "The problem specification is not well formed."
                 exit(127)
               end
               all_score = 0
               all_comment = ''
+            + peak_memory = -1
+            + max_runtime = -1
               (1..(problem.runs.length-1)).each do |k|
                 log "grade run #{k}"
                 run = problem.runs[k]
                 run_score = nil
                 run_comment = ''
                 run_comment_short = ''
                     run_comment += "result file for test #{test_num} not found\n"
                     run_comment_short += RUN_ERROR_MARK
                     log "Cannot find the file #{test_num}/result!"
                   else
                     result_file = File.new(result_file_name, "r")
                     result_file_lines = result_file.readlines
-            -       if result_file_lines.length>=2
+            +       if result_file_lines.length>=3
                       current_run_score = result_file_lines[1].to_i
                       run_comment += result_file_lines[0]
                       run_comment_short += char_comment(result_file_lines[0].chomp)
+            +
+            +         #update max runtime & memory
+            +         run_stat = extract_time result_file_lines[2]
+            +         peak_memory = [peak_memory,run_stat[:mem].to_i].max
+            +         max_runtime = [max_runtime,run_stat[:user].to_f + run_stat[:sys].to_f].max
                     else
                       current_run_score = 0
                       run_comment += "result file for test #{test_num} error\n"
                       run_comment_short += RUN_ERROR_MARK
                       log "Error in #{test_num}/result!"
                     end
               result_file.close
               comment_file = File.new("comment", "w")
               comment_file.write "#{all_comment}\n"
               comment_file.close
-            - log "score = #{all_score} comment = #{all_comment}"
+            + File.open("run_stat","w") do |file|
+            +   file.puts max_runtime
+            +   file.puts peak_memory
+            + end
+            +
+            + log "score = #{all_score}\ncomment = #{all_comment}"
+            + log "max_runtime = #{max_runtime}\npeak_memory = #{peak_memory}"

std-script/judge ¶ +3 -2

                 puts "  <test-result-directory> is defaulted to ./test-result"
                 puts "WARNING: The judge script will forcefully create the (implicitly and explicitly) specified directories and remove anything inside it."
                 exit(127)
               end
               language = ARGV[0]
-            - if language != "c" && language != "c++" && language != "pas" && language != "java" && language != "ruby" && language != "python"
+            + if language != "c" && language != "c++" && language != "pas" && language != "java" && language != "ruby" && language != "python" && language != "php"
                 log "You specified a language that is not supported: #{language}."
                 exit(127)
               end
               source_file = ARGV[1]
+            + ENV['SOURCE_NAME'] = source_file
               if File.exist?(source_file) == false
                 log "The source file does not exist."
                 exit(127)
               end
               log "Making test result and sandbox directories..."
                   FileUtils.cp("#{test_result_dir}/a.out", sandbox_dir, :preserve => true)
                   if language == "java" then Dir["#{test_result_dir}/*.class"].each { |file| FileUtils.cp(file,sandbox_dir)} end
                   if language == "python" then Dir["#{test_result_dir}/*.pyc"].each { |file| FileUtils.cp(file,sandbox_dir)} end
                 }
                 begin
-            -     execute("#{problem_home}/script/run #{language} #{test_num}", "Error occured during execution of the run script")
+            +     execute("#{problem_home}/script/run #{language} #{test_num} ", "Error occured during execution of the run script")
                 rescue
                   # do nothing
                 end
                 call_and_log("Cannot create directory #{test_result_dir}/#{test_num}") {
                   FileUtils.mkdir "#{test_result_dir}/#{test_num}"

std-script/run ¶ +16 -8

                 program_name = ARGV[2]
               else
                 program_name = "a.out"
               end
               problem_home = ENV['PROBLEM_HOME']
+            + source_name = ENV['SOURCE_NAME']
               require "#{problem_home}/script/test_dsl.rb"
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
+            + sandbox_dir = Dir.getwd
+            +
               if problem.well_formed? == false
                 log "The problem specification is not well formed."
                 exit(127)
               end
               # Check if the test number is okay.
                 compile_box("#{problem_home}/script/box.cc",
                             "#{problem_home}/script/box")
               end
               # Hide PROBLEM_HOME
               ENV['PROBLEM_HOME'] = nil
+            + ENV['SOURCE_NAME'] = nil
               # Run the program.
               #run_command = "/usr/bin/time -f \"#{time_output_format}\" 2>run_result #{problem_home}/script/box_new -a 2 -f -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name}"
               #
+            + JAVA_OPTION = "-s set_robust_list -s futex -s clone -s getppid -s clone -s wait4 -p /usr/bin/ -p ./"
+            + RUBY_OPTION   = "-p /usr/lib64/ -p /usr/local/lib64/ -p /usr/local/lib/ -p /lib64/ -p /dev/urandom -p #{sandbox_dir}/#{program_name} -p #{sandbox_dir}/ -s set_robust_list -s sched_getaffinity -s clock_gettime -s sigaltstack -s pipe2 -s clone -s futex -s openat -s pipe"
+            + PYTHON_OPTION = "-p /usr/lib64/ -p /usr/local/lib64/ -p /usr/local/lib/ -p /usr/bin/ -p /lib64/ -p #{sandbox_dir}/#{program_name} -p ./#{program_name} -p #{sandbox_dir}/#{source_name} -s set_robust_list -s openat -s recvmsg -s connect -s socket -s sendto -s futex -E PYTHONNOUSERSITE=yes"
+            + PHP_OPTION = "-p /usr/lib64/ -p/lib64/ -p /usr/bin/ -p #{sandbox_dir}/#{program_name} -p ./#{program_name} -p /usr/share/ -s setfsuid -s setfsgid -s openat -s set_robust_list -s futex -s clone -s socket -s connect"
               case language
                 when "java"
                   # for java, extract the classname
                   # wne have to add additional systemcall and we don't check the mem limit (dunno how to fix...)
                   classname = 'DUMMY'
                   File.open(program_name,"r").each do |line|
                     classname = line
                   end
-            -     run_command = "#{problem_home}/script/box -T -t #{time_limit} -s getppid -s clone -s wait4 -p /usr/bin/ -p ./ -i #{input_file_name} -o output.txt /usr/bin/java #{classname} 2>run_result"
+            +     #for java, we cannot really check the memory limit...
+            +     run_command = "#{problem_home}/script/box -a 3 -f -T -t #{time_limit} #{JAVA_OPTION} -i #{input_file_name} -o output.txt /usr/bin/java #{classname} 2>run_result"
                 when "ruby"
-            -     run_command = "#{problem_home}/script/box -T -t #{time_limit} -s getppid -s wait4 -s clone -s set_robust_list -s futex -s sigaltstack -p /dev/urandom -p ./ -p /home/dae/.rvm/rubies/ruby-1.9.2-p320/ -p #{problem_home}/ -i #{input_file_name} -o output.txt #{program_name} 2>run_result"
+            +     run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit*=2} -m #{mem_limit} #{RUBY_OPTION} -i #{input_file_name} -o output.txt /usr/bin/ruby #{program_name} 2>run_result"
                 when "python"
-            -     #this code just run without any checking
+            +     run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit*=2} -m #{mem_limit} #{PYTHON_OPTION} -i #{input_file_name} -o output.txt /usr/bin/python #{program_name} 2>run_result"
-            -     run_command = "#{problem_home}/script/box -T -t #{time_limit} -p #{problem_home}/ -i #{input_file_name} -o output.txt #{program_name} 2>run_result"
+            +   when "php"
+            +     run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit*=2} #{PHP_OPTION} -i #{input_file_name} -o output.txt /usr/bin/php #{program_name} 2>run_result"
                 else # for c++, pascal, we do the normal checking
                   run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name} 2>run_result"
               end
               log "Running test #{test_num}..."
               log run_command
-            - log
+            + log
               system(run_command)
               # Restore PROBLEM_HOME
               ENV['PROBLEM_HOME'] = problem_home
               # Create the result file.
                 comment_file.close
                 log "Done!"
                 exit(0)
               }
+            +
               if run_result[0][0,2] != "OK"
                 log "There was a runtime error."
                 report.call(run_result[0], 0, "No comment.\n")
               end
               if running_time[:user].to_f > time_limit

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r173:aa6e7ff3942c - Sun, 28 Sep 2014 02:42:25 - 9 files changed: 124 inserted, 34 deleted

Sign in to your account

Commit r173:aa6e7ff3942c public

Author

r173:aa6e7ff3942c - Sun, 28 Sep 2014 02:42:25 - 9 files changed: 124 inserted, 34 deleted

Commit `r173:aa6e7ff3942c` public