cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r181:5c6f0b732e70` public

parent | child

Description:

handle the case when problem id or submission id is null. Grader will simply skip such request. Add more report on console (for command line grading) (mercurial grafted from d233105d3965c5368c9b33125f390e39b25f910e)

Commit status:

[Not Reviewed]

References:

default

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r181:5c6f0b732e70 - Tue, 12 Aug 2014 08:44:55 - 2 files changed: 30 inserted, 30 deleted

grader ¶ +1 -1

               #!/usr/bin/env ruby
               def stop_grader(id)
                 if id==:all
                   File.open(File.dirname(__FILE__) + "/stop.all",'w').close
                 else
                   File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
                 end
               end
               def check_stopfile
                 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
                   FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
               end
               def clear_stopfile
                 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
                   File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
                 end
               end
               def config
                 Grader::Configuration.get_instance
               end
               def log_file_name
                 if !File.exists?(config.log_dir)
                   raise "Log directory does not exist: #{config.log_dir}"
                 end
                 config.log_dir +
                   "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
               end
               def log(str)
                 if config.talkative
                   puts str
                 end
                 if config.logging
                   fp = File.open(log_file_name,"a")
                   fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
                   fp.close
                 end
               end
               def display_manual
                 puts <<USAGE
               Grader.
               using: (1) grader
                      (2) grader environment [mode]
                      (3) grader stop [all|pids-list]
                      (4) grader --help
               (1) call grader with environment = 'exam', mode = 'queue'
               (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
               (3) create stop-file to stop running grader in queue mode
               (4) You are here.
               USAGE
               end
               def process_options_and_stop_file
                 # The list of options are:
                 #   - stop [all|process ids]
                 #   -
                 # Process 'help' option
                 if (ARGV.length==1) and (/help/.match(ARGV[0]))
                   display_manual
                   exit(0)
                 end
                 # Process 'stop' option.
                 if (ARGV.length >= 1) and (ARGV[0]=='stop')
                   if ARGV.length==1
                     puts "you should specify pid-list or 'all'"
                     display_manual
                   elsif (ARGV.length==2) and (ARGV[1]=='all')
                     stop_grader(:all)
                     puts "A global stop file ('stop.all') created."
                     puts "You should remove it manually later."
                   else
                     (1..ARGV.length-1).each do |i|
                       stop_grader(ARGV[i])
                     end
                     puts "stop file(s) created"
                   end
                   exit(0)
                 end
                 # Check stop file.
                 if check_stopfile
                   puts "Stop file exists.  Terminated."
                   clear_stopfile
                   exit(0)
                 end
                 #default options
                 options = {
                   :mode => 'queue',
                   :environment => 'exam',
                   :dry_run => false,
                 }
                 # Process mode and environment option
                 if ARGV.length >= 1
                   options[:environment] = ARGV.shift
                   if ARGV.length >=1
                     options[:mode] = ARGV.shift
                   end
                 end
                 options[:dry_run] = (ARGV.delete('--dry') != nil)
                 if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
                   puts "Dry run currently works only for 'prob' or 'contest' modes."
                   exit(0)
                 end
                 options[:report] = (ARGV.delete('--report') != nil)
                 if options[:report] and (not ['prob','contest','autonew'].include? options[:mode])
                   puts "Report currently works only for 'prob' or 'contest' modes."
                   exit(0)
                 end
                 return options
               end
               class ResultCollector
                 def initialize
                   @results = {}
                   @problems = {}
                   @users = {}
                 end
                 def after_save_hook(submission, grading_result)
                 end
                 def save(submission, grading_result)
                   user = submission.user
                   problem = submission.problem
                   if not @problems.has_key? problem.id
                     @problems[problem.id] = problem
                   end
                   if not @users.has_key? user.id
                     @users[user.id] = user
                   end
                   @results[[user.id, problem.id]] = grading_result
                   after_save_hook(submission, grading_result)
                 end
                 def print_report_by_user
                   puts "---------------------"
                   puts "       REPORT"
                   puts "---------------------"
                   print "login,email"
                   @problems.each_value do |problem|
                     print ",#{problem.name}"
                   end
                   print "\n"
                   @users.each_value do |user|
                     print "#{user.login},#{user.email}"
                     @problems.each_value do |problem|
                       if @results.has_key? [user.id, problem.id]
                         print ",#{@results[[user.id,problem.id]][:points]}"
                       else
                         print ","
                       end
                     end
                     print "\n"
                   end
                 end
               end
               def grader_general_loop(engine, grader_proc, options)
                 runner = Grader::Runner.new(engine, grader_proc)
                 while true
                   if check_stopfile    # created by calling grader stop
                     clear_stopfile
                     log "stopped (with stop file)"
                     break
                   end
                   task = yield(runner)
                   if task==nil
                     sleep(1)
                   end
                 end
               end
               def grader_queue_loop(grader_proc, options)
                 log "Grader: queue"
                 engine = Grader::Engine.new
                 grader_general_loop(engine, grader_proc, options) do |runner|
                   runner.grade_oldest_task
                 end
               end
               def grader_test_request_loop(grader_proc, options)
                 log "Grader: test_request"
                 engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
                                             :reporter => Grader::TestRequestReporter.new)
                 grader_general_loop(engine, grader_proc, options) do |runner|
                   runner.grade_oldest_test_request
                 end
               end
               def grader_autonew_loop(grader_proc, options)
                 log "Grader: autonew"
                 if options[:report]
                   result_collector = ResultCollector.new
                 else
                   result_collector = nil
                 end
                 if options[:dry_run]
                   puts "Running in dry mode"
                 end
                 prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
                                                                :result_collector => result_collector)
                 engine = Grader::Engine.new(:reporter => prob_reporter)
                 runner = Grader::Runner.new(engine, grader_proc)
                 grader_proc.report_active if grader_proc!=nil
                 latest_submitted_at = nil
                 graded_submission_ids = {}
                 while true
                   if check_stopfile    # created by calling grader stop
                     clear_stopfile
                     log "stopped (with stop file)"
                     break
                   end
                   if latest_submitted_at==nil
                     submissions = Submission.all
                   else
                     submissions = Submission.all(:conditions => ["submitted_at >= :latest",
                                                                  {:latest => latest_submitted_at}])
                   end
                   graded_any = false
                   if submissions.length != 0
                     submissions.each do |submission|
                       if (submission.problem == nil) or (!submission.problem.available)
                         next
                       end
                       if ! graded_submission_ids[submission.id]
                         runner.grade_submission(submission)
                         graded_submission_ids[submission.id] = true
                         if (!latest_submitted_at or
                             latest_submitted_at < submission.submitted_at)
                           latest_submitted_at = submission.submitted_at
                         end
                         puts "graded: #{submission.id}"
                         puts "latest: #{latest_submitted_at}"
                         graded_any = true
                       end
                     end
                   end
                   if ! graded_any
                     sleep(1)
                   end
                 end
               end
               def grader_grade_problems(grader_proc, options)
                 if options[:report]
                   result_collector = ResultCollector.new
                 else
                   result_collector = nil
                 end
                 if options[:dry_run]
                   puts "Running in dry mode"
                 end
                 prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
                                                                :result_collector => result_collector)
                 engine = Grader::Engine.new(:reporter => prob_reporter)
                 runner = Grader::Runner.new(engine, grader_proc)
                 grader_proc.report_active if grader_proc!=nil
                 ARGV.each do |prob_name|
                   prob = Problem.find_by_name(prob_name)
                   if prob==nil
                     puts "cannot find problem: #{prob_name}"
                   else
                     runner.grade_problem(prob)
                   end
                 end
                 if options[:report]
                   result_collector.print_report_by_user
                 end
               end
               def grader_grade_contests(grader_proc, options)
                 # always use dry run when grading during contest
                 dry_run = options[:dry_run] = true
                 contest_name = ARGV.shift
                 contest = Contest.find_by_name(contest_name)
                 if contest==nil
                   puts "cannot find contest: #{contest_name}"
                   exit(0)
                 end
                 if options[:report]
                   result_collector = ResultCollector.new
                 else
                   result_collector = nil
                 end
                 if options[:dry_run]
                   puts "Running in dry mode"
                 end
                 prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
                                                                :result_collector => result_collector)
                 engine = Grader::Engine.new(:reporter => prob_reporter)
                 runner = Grader::Runner.new(engine, grader_proc)
                 grader_proc.report_active if grader_proc!=nil
                 contest.problems.each do |problem|
                   puts "Grading: #{problem.name}"
                   runner.grade_problem(problem,
                                        :user_conditions => lambda do |u|
                                          u.contest_finished? and
                                            u.contest_ids.include?(contest.id)
                                        end)
                 end
                 if options[:report]
                   result_collector.print_report_by_user
                 end
               end
               def grader_grade_submissions(grader_proc, options)
                 engine = Grader::Engine.new
                 runner = Grader::Runner.new(engine, grader_proc)
                 grader_proc.report_active if grader_proc!=nil
                 ARGV.each do |sub_id|
                   puts "Grading #{sub_id}"
                   begin
                     submission = Submission.find(sub_id.to_i)
                   rescue ActiveRecord::RecordNotFound
-            -       puts "Record not found"
+            +       puts "Submission #{sub_id} not found"
                     submission = nil
                   end
                   if submission!=nil
                     runner.grade_submission(submission)
                   end
                 end
               end
               #########################################
               # main program
               #########################################
               options = process_options_and_stop_file
               GRADER_ENV = options[:environment]
               grader_mode = options[:mode]
               dry_run = options[:dry_run]
               puts "environment: #{GRADER_ENV}"
               require File.join(File.dirname(__FILE__),'config/environment')
               # add grader_mode to config
               # this is needed because method log needs it.  TODO: clean this up
               class << config
                 attr_accessor :grader_mode
               end
               config.grader_mode = grader_mode
               # reading rails environment
               log 'Reading rails environment'
               RAILS_ENV = config.rails_env
               require RAILS_ROOT + '/config/environment'
               # register grader process
               if config.report_grader
                 grader_proc = GraderProcess.register(config.grader_hostname,
                                                      Process.pid,
                                                      grader_mode)
               else
                 grader_proc = nil
               end
               #set loggin environment
               ENV['GRADER_LOGGING'] = log_file_name
               # register exit handler to report inactive, and terminated
               at_exit do
                 if grader_proc!=nil
                   grader_proc.report_inactive
                   grader_proc.terminate
                 end
               end
               #
               # MAIN LOOP
               #
               case grader_mode
               when "queue"
                 grader_queue_loop(grader_proc, options)
               when "test_request"
                 grader_test_request_loop(grader_proc, options)
               when "prob"
                 grader_grade_problems(grader_proc, options)
               when "contest"
                 grader_grade_contests(grader_proc, options)
               when "sub"
                 grader_grade_submissions(grader_proc, options)
               when "autonew"
                 grader_autonew_loop(grader_proc, options)
               else
                 display_manual
                 exit(0)
               end

lib/engine.rb ¶ +29 -29

               require 'fileutils'
               require File.join(File.dirname(__FILE__),'dir_init')
               module Grader
                 #
                 # A grader engine grades a submission, against anything: a test
                 # data, or a user submitted test data.  It uses two helpers objects:
                 # room_maker and reporter.
                 #
                 class Engine
                   attr_writer :room_maker
                   attr_writer :reporter
                   def initialize(options={})
                     # default options
                     if not options.include? :room_maker
                       options[:room_maker] = Grader::SubmissionRoomMaker.new
                     end
                     if not options.include? :reporter
                       options[:reporter] =  Grader::SubmissionReporter.new
                     end
                     @config = Grader::Configuration.get_instance
                     @room_maker = options[:room_maker]
                     @reporter = options[:reporter]
                   end
                   # takes a submission, asks room_maker to produce grading directories,
                   # calls grader scripts, and asks reporter to save the result
                   def grade(submission)
                     current_dir = FileUtils.pwd
                     user = submission.user
                     problem = submission.problem
-            -       # TODO: will have to create real exception for this
+            +       begin
-            -       if user==nil or problem == nil
+            +         # TODO: will have to create real exception for this
-            -         @reporter.report_error(submission,"Grading error: problem with submission")
+            +         if user==nil or problem == nil
-            -         #raise "engine: user or problem is nil"
+            +           @reporter.report_error(submission,"Grading error: problem with submission")
-            -       end
+            +           raise "engine: user or problem is nil"
+            +         end
-            -       # TODO: this is another hack so that output only task can be judged
-            -       if submission.language!=nil
-            -         language = submission.language.name
-            -         lang_ext = submission.language.ext
-            -       else
-            -         language = 'c'
-            -         lang_ext = 'c'
-            -       end
-            -       # This is needed because older version of std-scripts/compile
+            +         # TODO: this is another hack so that output only task can be judged
-            -       # only look for c++.
+            +         if submission.language!=nil
-            -       if language == 'cpp'
+            +           language = submission.language.name
-            -         language = 'c++'
+            +           lang_ext = submission.language.ext
-            -       end
+            +         else
+            +           language = 'c'
+            +           lang_ext = 'c'
+            +         end
-            -       # COMMENT: should it be only source.ext?
+            +         # This is needed because older version of std-scripts/compile
-            -       if problem!=nil
+            +         # only look for c++.
-            -         source_name = "#{problem.name}.#{lang_ext}"
+            +         if language == 'cpp'
-            -       else
+            +           language = 'c++'
-            -         source_name = "source.#{lang_ext}"
+            +         end
-            -       end
-            -       begin
+            +         # COMMENT: should it be only source.ext?
+            +         if problem!=nil
+            +           source_name = "#{problem.name}.#{lang_ext}"
+            +         else
+            +           source_name = "source.#{lang_ext}"
+            +         end
+            +
                       grading_dir = @room_maker.produce_grading_room(submission)
                       @room_maker.save_source(submission,source_name)
                       problem_home = @room_maker.find_problem_home(submission)
                       # puts "GRADING DIR: #{grading_dir}"
                       # puts "PROBLEM DIR: #{problem_home}"
                       if !FileTest.exist?(problem_home)
                         puts "PROBLEM DIR: #{problem_home}"
-            -           puts "No test data. (check problem dir)"
+            +           raise "engine: No test data."
-            -           raise "No test data."
                       end
                       dinit = DirInit::Manager.new(problem_home)
                       dinit.setup do
                         copy_log = copy_script(problem_home)
                         save_copy_log(problem_home,copy_log)
                       end
                       call_judge(problem_home,language,grading_dir,source_name)
                       @reporter.report(submission,"#{grading_dir}/test-result")
                       dinit.teardown do
                         copy_log = load_copy_log(problem_home)
                         clear_copy_log(problem_home)
                         clear_script(copy_log,problem_home)
                       end
                     rescue RuntimeError => msg
                       @reporter.report_error(submission, msg)
+            +         puts "ERROR: #{msg}"
                     ensure
                       @room_maker.clean_up(submission)
                       Dir.chdir(current_dir)   # this is really important
                     end
                   end
                   protected
                   def talk(str)
                     if @config.talkative
                       puts str
                     end
                   end
                   def call_judge(problem_home,language,grading_dir,fname)
                     ENV['PROBLEM_HOME'] = problem_home
                     ENV['RUBYOPT'] = ''
                     talk grading_dir
                     Dir.chdir grading_dir
                     cmd = "#{problem_home}/script/judge #{language} #{fname}"
                     talk "CMD: #{cmd}"
                     system(cmd)
                   end
                   def get_std_script_dir
                     GRADER_ROOT + '/std-script'
                   end
                   def copy_script(problem_home)
                     script_dir = "#{problem_home}/script"
                     std_script_dir = get_std_script_dir
-            -       raise "std-script directory not found" if !FileTest.exist?(std_script_dir)
+            +       raise "engine: std-script directory not found" if !FileTest.exist?(std_script_dir)
                     scripts = Dir[std_script_dir + '/*']
                     copied = []
                     scripts.each do |s|
                       fname = File.basename(s)
                       next if FileTest.directory?(s)
                       if !FileTest.exist?("#{script_dir}/#{fname}")
                         copied << fname
                         FileUtils.cp(s, "#{script_dir}", :preserve => true)
                       end
                     end
                     return copied
                   end
                   def copy_log_filename(problem_home)
                     return File.join(problem_home, '.scripts_copied')
                   end
                   def save_copy_log(problem_home, log)
                     f = File.new(copy_log_filename(problem_home),"w")
                     log.each do |fname|
                       f.write("#{fname}\n")
                     end
                     f.close
                   end
                   def load_copy_log(problem_home)
                     f = File.new(copy_log_filename(problem_home),"r")
                     log = []
                     f.readlines.each do |line|
                       log << line.strip
                     end
                     f.close
                     log
                   end
                   def clear_copy_log(problem_home)
                     File.delete(copy_log_filename(problem_home))
                   end
                   def clear_script(log,problem_home)
                     log.each do |s|
                       FileUtils.rm("#{problem_home}/script/#{s}")
                     end
                   end
                   def mkdir_if_does_not_exist(dirname)
                     Dir.mkdir(dirname) if !FileTest.exist?(dirname)
                   end
                 end
               end

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r181:5c6f0b732e70 - Tue, 12 Aug 2014 08:44:55 - 2 files changed: 30 inserted, 30 deleted

Sign in to your account

Commit r181:5c6f0b732e70 public

Author

r181:5c6f0b732e70 - Tue, 12 Aug 2014 08:44:55 - 2 files changed: 30 inserted, 30 deleted

Commit `r181:5c6f0b732e70` public