cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r92:274af5f15b1f` public

parent | child

Description:

added grading report

Commit status:

[Not Reviewed]

References:

default

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r92:274af5f15b1f - Thu, 11 Mar 2010 04:28:53 - 4 files changed: 95 inserted, 21 deleted

grader ¶ +71 -13

               #!/usr/bin/ruby
               def stop_grader(id)
                 if id==:all
                   File.open(File.dirname(__FILE__) + "/stop.all",'w').close
                 else
                   File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
                 end
               end
               def check_stopfile
                 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
                   FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
               end
               def clear_stopfile
                 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
                   system("rm " + File.dirname(__FILE__) + "/stop.#{Process.pid}")
                 end
               end
               def config
                 Grader::Configuration.get_instance
               end
               def log_file_name
                 if !File.exists?(config.log_dir)
                   raise "Log directory does not exist: #{config.log_dir}"
                 end
                 config.log_dir +
                   "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
               end
               def log(str)
                 if config.talkative
                   puts str
                 end
                 if config.logging
                   fp = File.open(log_file_name,"a")
                   fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
                   fp.close
                 end
               end
               def display_manual
                 puts <<USAGE
               Grader.
               using: (1) grader
                      (2) grader environment [mode]
                      (3) grader stop [all|pids-list]
                      (4) grader --help
               (1) call grader with environment = 'exam', mode = 'queue'
               (2) possible modes are: 'queue', 'prob', 'test_request'
               (3) create stop-file to stop running grader in queue mode
               (4) You are here.
               USAGE
               end
-            - #########################################
-            - # main program
-            - #########################################
-            - # Reading environment and options.
               def process_options_and_stop_file
                 # The list of options are:
                 #   - stop [all|process ids]
                 #   -
                 # Process 'help' option
                 if (ARGV.length==1) and (/help/.match(ARGV[0]))
                   display_manual
                   exit(0)
                 end
                 # Process 'stop' option.
                 if (ARGV.length >= 1) and (ARGV[0]=='stop')
                   if ARGV.length==1
                     puts "you should specify pid-list or 'all'"
                     display_manual
                   elsif (ARGV.length==2) and (ARGV[1]=='all')
                     stop_grader(:all)
                     puts "A global stop file ('stop.all') created."
                     puts "You should remove it manually later."
                   else
                     (1..ARGV.length-1).each do |i|
                       stop_grader(ARGV[i])
                     end
                     puts "stop file(s) created"
                   end
                   exit(0)
                 end
                 # Check stop file.
                 if check_stopfile
                   puts "Stop file exists.  Terminated."
                   clear_stopfile
                   exit(0)
                 end
                 #default options
                 options = {
                   :mode => 'queue',
                   :environment => 'exam',
                   :dry_run => false,
                 }
                 # Process mode and environment option
                 if ARGV.length >= 1
                   options[:environment] = ARGV.shift
                   if ARGV.length >=1
                     options[:mode] = ARGV.shift
                   end
                 end
                 options[:dry_run] = (ARGV.delete('--dry') != nil)
                 if options[:dry_run] and (not options[:mode] == 'prob')
-            -     puts "Dry run currently works for 'prob' mode."
+            +     puts "Dry run currently works only for 'prob' mode."
+            +     exit(0)
+            +   end
+            +
+            +   options[:report] = (ARGV.delete('--report') != nil)
+            +   if options[:report] and (not options[:mode] == 'prob')
+            +     puts "Report currently works only for 'prob' mode."
                   exit(0)
                 end
                 return options
               end
-            - # ======= Main ========
+            + class ResultCollector
+            +   def initialize
+            +     @results = {}
+            +     @problems = {}
+            +     @users = {}
+            +   end
+            +
+            +   def save(user, problem, grading_result)
+            +     if not @problems.has_key? problem.id
+            +       @problems[problem.id] = problem
+            +     end
+            +     if not @users.has_key? user.id
+            +       @users[user.id] = user
+            +     end
+            +     @results[[user.id, problem.id]] = grading_result
+            +   end
+            +
+            +   def print_report_by_user
+            +     puts "---------------------"
+            +     puts "       REPORT"
+            +     puts "---------------------"
+            +
+            +     print "login"
+            +     @problems.each_value do |problem|
+            +       print ",#{problem.name}"
+            +     end
+            +     print "\n"
+            +
+            +     @users.each_value do |user|
+            +       print "#{user.login}"
+            +       @problems.each_value do |problem|
+            +         if @results.has_key? [user.id, problem.id]
+            +           print ",#{@results[[user.id,problem.id]][:points]}"
+            +         else
+            +           print ","
+            +         end
+            +       end
+            +       print "\n"
+            +     end
+            +   end
+            + end
+            +
+            + #########################################
+            + # main program
+            + #########################################
+            +
               options = process_options_and_stop_file
               GRADER_ENV = options[:environment]
               grader_mode = options[:mode]
               dry_run = options[:dry_run]
               puts "environment: #{GRADER_ENV}"
               require File.join(File.dirname(__FILE__),'config/environment')
               # add grader_mode to config
               # this is needed because method log needs it.  TODO: clean this up
               class << config
                 attr_accessor :grader_mode
               end
               config.grader_mode = grader_mode
               # reading rails environment
               log 'Reading rails environment'
               RAILS_ENV = config.rails_env
               require RAILS_ROOT + '/config/environment'
               # register grader process
               if config.report_grader
                 grader_proc = GraderProcess.register(config.grader_hostname,
                                                      Process.pid,
                                                      grader_mode)
               else
                 grader_proc = nil
               end
               #set loggin environment
               ENV['GRADER_LOGGING'] = log_file_name
               # register exit handler to report inactive, and terminated
               at_exit do
                 if grader_proc!=nil
                   grader_proc.report_inactive
                   grader_proc.terminate
                 end
               end
               #
               # MAIN LOOP
               #
               case grader_mode
               when "queue", "test_request"
                 log "Grader: #{grader_mode}"
                 if grader_mode=="queue"
                   engine = Grader::Engine.new
                 else
-            -     engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
+            +     engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
-            -                                 Grader::TestRequestReporter.new)
+            +                                 :reporter => Grader::TestRequestReporter.new)
                 end
                 runner = Grader::Runner.new(engine, grader_proc)
                 while true
                   if check_stopfile    # created by calling grader stop
                     clear_stopfile
                     log "stopped (with stop file)"
                     break
                   end
                   if grader_mode=="queue"
                     task = runner.grade_oldest_task
                   else
                     task = runner.grade_oldest_test_request
                   end
                   if task==nil
                     sleep(1)
                   end
                 end
               when "prob"
-            -   engine = Grader::Engine.new(nil, Grader::SubmissionReporter.new(dry_run))
+            +   if options[:report]
+            +     result_collector = ResultCollector.new
+            +   else
+            +     result_collector = nil
+            +   end
+            +
+            +   engine = (Grader::Engine.
+            +             new(:reporter =>
+            +                 Grader::SubmissionReporter.new(:dry_run => dry_run,
+            +                                                :result_collector => result_collector)))
                 runner = Grader::Runner.new(engine, grader_proc)
+            +
                 grader_proc.report_active if grader_proc!=nil
+            +
                 ARGV.each do |prob_name|
                   prob = Problem.find_by_name(prob_name)
                   if prob==nil
                     puts "cannot find problem: #{prob_name}"
                   else
                     runner.grade_problem(prob)
                   end
                 end
+            +   if options[:report]
+            +     result_collector.print_report_by_user
+            +   end
+            +
               when "sub"
                 engine = Grader::Engine.new
                 runner = Grader::Runner.new(engine, grader_proc)
                 grader_proc.report_active if grader_proc!=nil
                 ARGV.each do |sub_id|
                   puts "Grading #{sub_id}"
                   begin
                     submission = Submission.find(sub_id.to_i)
                   rescue ActiveRecord::RecordNotFound
                     puts "Record not found"
                     submission = nil
                   end
                   if submission!=nil
                     runner.grade_submission(submission)
                   end
                 end
               else
                 display_manual
                 exit(0)
               end

lib/engine.rb ¶ +11 -3

               require 'fileutils'
               require 'ftools'
               require File.join(File.dirname(__FILE__),'dir_init')
               module Grader
                 #
                 # A grader engine grades a submission, against anything: a test
                 # data, or a user submitted test data.  It uses two helpers objects:
                 # room_maker and reporter.
                 #
                 class Engine
                   attr_writer :room_maker
                   attr_writer :reporter
-            -     def initialize(room_maker=nil, reporter=nil)
+            +     def initialize(options={})
+            +       # default options
+            +       if not options.include? :room_maker
+            +         options[:room_maker] = Grader::SubmissionRoomMaker.new
+            +       end
+            +       if not options.include? :reporter
+            +         options[:reporter] =  Grader::SubmissionReporter.new
+            +       end
+            +
                     @config = Grader::Configuration.get_instance
-            -       @room_maker = room_maker || Grader::SubmissionRoomMaker.new
+            +       @room_maker = options[:room_maker]
-            -       @reporter = reporter || Grader::SubmissionReporter.new
+            +       @reporter = options[:reporter]
                   end
                   # takes a submission, asks room_maker to produce grading directories,
                   # calls grader scripts, and asks reporter to save the result
                   def grade(submission)
                     current_dir = `pwd`.chomp
                     user = submission.user
                     problem = submission.problem
                     # TODO: will have to create real exception for this
                     if user==nil or problem == nil
                       @reporter.report_error(submission,"Grading error: problem with submission")
                       #raise "engine: user or problem is nil"
                     end
                     # TODO: this is another hack so that output only task can be judged
                     if submission.language!=nil
                       language = submission.language.name
                       lang_ext = submission.language.ext
                     else
                       language = 'c'
                       lang_ext = 'c'
                     end
                     # FIX THIS
                     talk 'some hack on language'
                     if language == 'cpp'
                       language = 'c++'
                     end
                     # COMMENT: should it be only source.ext?
                     if problem!=nil
                       source_name = "#{problem.name}.#{lang_ext}"
                     else
                       source_name = "source.#{lang_ext}"
                     end
                     begin
                       grading_dir = @room_maker.produce_grading_room(submission)
                       @room_maker.save_source(submission,source_name)
                       problem_home = @room_maker.find_problem_home(submission)
                       # puts "GRADING DIR: #{grading_dir}"
                       # puts "PROBLEM DIR: #{problem_home}"
                       if !FileTest.exist?(problem_home)
                         raise "No test data."
                       end
                       dinit = DirInit::Manager.new(problem_home)
                       dinit.setup do
                         copy_log = copy_script(problem_home)
                         save_copy_log(problem_home,copy_log)
                       end
                       call_judge(problem_home,language,grading_dir,source_name)
                       @reporter.report(submission,"#{grading_dir}/test-result")
                       dinit.teardown do
                         copy_log = load_copy_log(problem_home)
                         clear_copy_log(problem_home)
                         clear_script(copy_log,problem_home)
                       end
                     rescue RuntimeError => msg
                       @reporter.report_error(submission, msg)
                     ensure
                       @room_maker.clean_up(submission)
                       Dir.chdir(current_dir)   # this is really important
                     end
                   end
                   protected
                   def talk(str)
                     if @config.talkative
                       puts str
                     end
                   end
                   def call_judge(problem_home,language,grading_dir,fname)
                     ENV['PROBLEM_HOME'] = problem_home
                     talk grading_dir
                     Dir.chdir grading_dir
                     cmd = "#{problem_home}/script/judge #{language} #{fname}"
                     talk "CMD: #{cmd}"
                     system(cmd)
                   end
                   def get_std_script_dir
                     GRADER_ROOT + '/std-script'

lib/submission_helper.rb ¶ +11 -3

               module Grader
                 class SubmissionRoomMaker
                   def initialize
                     @config = Grader::Configuration.get_instance
                   end
                   def produce_grading_room(submission)
                     user = submission.user
                     problem = submission.problem
                     grading_room = "#{@config.user_result_dir}/" +
                       "#{user.login}/#{problem.name}/#{submission.id}"
                     FileUtils.mkdir_p(grading_room)
                     grading_room
                   end
                   def find_problem_home(submission)
                     problem = submission.problem
                     "#{@config.problems_dir}/#{problem.name}"
                   end
                   def save_source(submission,source_name)
                     dir = self.produce_grading_room(submission)
                     f = File.open("#{dir}/#{source_name}","w")
                     f.write(submission.source)
                     f.close
                   end
                   def clean_up(submission)
                   end
                 end
                 class SubmissionReporter
-            -     def initialize(dry_run=false)
+            +     def initialize(options={})
+            +       options = {:dry_run => false, :result_collector => nil}.merge(options)
                     @config = Grader::Configuration.get_instance
-            -       @dry_run = dry_run
+            +       @dry_run = options[:dry_run]
+            +       @result_collector = options[:result_collector]
                   end
                   def report(sub,test_result_dir)
-            -       save_result(sub,read_result(test_result_dir))
+            +       result = read_result(test_result_dir)
+            +       if @result_collector
+            +         @result_collector.save(sub.user,
+            +                                sub.problem,
+            +                                result)
+            +       end
+            +       save_result(sub,result)
                   end
                   def report_error(sub,msg)
                     save_result(sub,{:points => 0,
                                   :comment => "Grading error: #{msg}" })
                   end
                   protected
                   def read_result(test_result_dir)
                     cmp_msg_fname = "#{test_result_dir}/compiler_message"
                     if FileTest.exist?(cmp_msg_fname)
                       cmp_file = File.open(cmp_msg_fname)
                       cmp_msg = cmp_file.read
                       cmp_file.close
                     else
                       cmp_msg = ""
                     end
                     result_fname = "#{test_result_dir}/result"
                     comment_fname = "#{test_result_dir}/comment"
                     if FileTest.exist?(result_fname)
                       comment = ""
                       begin
                         result_file = File.open(result_fname)
                         result = result_file.readline.to_i
                         result_file.close
                       rescue
                         result = 0
                         comment = "error reading result file."
                       end
                       begin
                         comment_file = File.open(comment_fname)
                         comment += comment_file.readline.chomp
                         comment_file.close
                       rescue
                         comment += ""
                       end
                       return {:points => result,
                         :comment => comment,
                         :cmp_msg => cmp_msg}
                     else
                       if FileTest.exist?("#{test_result_dir}/a.out")
                         return {:points => 0,
                           :comment => 'error during grading',
                           :cmp_msg => cmp_msg}
                       else
                         return {:points => 0,
                           :comment => 'compilation error',
                           :cmp_msg => cmp_msg}
                       end
                     end
                   end
                   def save_result(submission,result)
                     problem = submission.problem
                     submission.graded_at = Time.now.gmtime
                     points = result[:points]
                     submission.points = points
                     comment = @config.report_comment(result[:comment])
                     #
                     # TODO: FIX THIS MESSAGE
                     #
                     if problem == nil
                       submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
                     elsif points == problem.full_score
                       #submission.grader_comment = 'PASSED: ' + comment
                       submission.grader_comment = comment
                     elsif result[:comment].chomp =~ /^[\[\]P]+$/
                       submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
                     else
                       #submission.grader_comment = 'FAILED: ' + comment
                       submission.grader_comment = comment
                     end
                     submission.compiler_message = result[:cmp_msg] or ''
                     if not @dry_run
                       submission.save
                     end
                   end
                 end
               end

test/engine_spec.rb ¶ +2 -2

                   @problem_test_memory = stub(Problem,
                                               :id => 1, :name => 'test_memory',
                                               :full_score => 20)
                   grader_should(:grade => "add_too_much_memory_static.c",
                                 :on => @problem_test_memory,
                                 :and_report => {
                                   :score => 10,
                                   :comment => /[^P]P/})
                 end
                 it "should not allow submission to allocate too much dynamic memory" do
                   @problem_test_memory = stub(Problem,
                                               :id => 1, :name => 'test_memory',
                                               :full_score => 20)
                   grader_should(:grade => "add_too_much_memory_dynamic.c",
                                 :on => @problem_test_memory,
                                 :and_report => {
                                   :score => 10,
                                   :comment => /[^P]P/})
                 end
                 it "should score test runs correctly when submission fails in some test case" do
                   grader_should(:grade => "add_fail_test_case_1.c",
                                 :on => @problem_test_normal,
                                 :and_report => {
                                   :score => 105,
                                   :comment => /^.*(-|x).*$/})
                 end
                 it "should fail submission with non-zero exit status" do
                   grader_should(:grade => "add_nonzero_exit_status.c",
                                 :on => @problem_test_normal,
                                 :and_report => {
                                   :score => 0,
                                   :comment => /^(\[|x|\])+$/})
                 end
                 it "should not allow malicious submission to see PROBLEM_HOME" do
                   problem_test_yesno =  stub(Problem,
                                               :id => 1, :name => 'test_yesno',
                                               :full_score => 10)
                   grader_should(:grade => "yesno_access_problem_home.c",
                                 :on => problem_test_yesno,
                                 :and_report => {
                                   :score => 0,
                                   :comment => /(-|x)/})
                 end
                 it "should not allow malicious submission to open files" do
                   problem_test_yesno =  stub(Problem,
                                               :id => 1, :name => 'test_yesno',
                                               :full_score => 10)
                   grader_should(:grade => "yesno_open_file.c",
                                 :on => problem_test_yesno,
                                 :and_report => {
                                   :score => 0,
                                   :comment => /(-|x)/})
                 end
                 def grader_should(args)
                   @user1 = stub(User,
                                 :id => 1, :login => 'user1')
                   submission =
                     create_submission_from_file(1, @user1, args[:on], args[:grade])
                   submission.should_receive(:graded_at=)
                   expected_score = args[:and_report][:score]
                   expected_comment = args[:and_report][:comment]
                   if args[:and_report][:compiler_message]!=nil
                     expected_compiler_message = args[:and_report][:compiler_message]
                   else
                     expected_compiler_message = ''
                   end
                   submission.should_receive(:points=).with(expected_score)
                   submission.should_receive(:grader_comment=).with(expected_comment)
                   submission.should_receive(:compiler_message=).with(expected_compiler_message)
                   submission.should_receive(:save)
                   @engine.grade(submission)
                 end
                 protected
                 def create_normal_submission_mock_from_file(source_fname)
                   create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
                 end
               end
               describe "A grader engine, when grading test requests" do
                 include GraderEngineHelperMethods
                 before(:each) do
                   @config = Grader::Configuration.get_instance
-            -     @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
+            +     @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
-            -                                  Grader::TestRequestReporter.new)
+            +                                  :reporter => Grader::TestRequestReporter.new)
                   init_sandbox
                 end
                 it "should report error if there is no problem template" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'nothing')
                   grader_should(:grade => 'test1_correct.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => /template not found/,
                                   :running_time= => nil,
                                   :exit_status= => nil,
                                   :memory_usage= => nil,
                                   :save => nil})
                 end
                 it "should run test request and produce output file" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test1_correct.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => /0.0\d* sec./,
                                   :output_file_name= => lambda { |fname|
                                     File.exists?(fname).should be_true
                                   },
                                   :running_time= => nil,
                                   :exit_status= => nil,
                                   :memory_usage= => nil,
                                   :save => nil})
                 end
                 it "should clean up problem directory after running test request" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test1_correct.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => nil,
                                   :output_file_name= => nil,
                                   :running_time= => nil,
                                   :exit_status= => nil,
                                   :memory_usage= => nil,
                                   :save => nil})
                   File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
                 end
                 it "should compile test request with error and report compilation error" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test1_compile_error.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => /.+/,
                                   :grader_comment= => /[Cc]ompil.*error/,
                                   :running_stat= => '',
                                   :save => nil})
                 end
                 it "should report exit status" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'add_nonzero_exit_status.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
                                   :output_file_name= => lambda { |fname|
                                     File.exists?(fname).should be_true
                                   },
                                   :running_time= => nil,
                                   :exit_status= => /10/,
                                   :memory_usage= => nil,
                                   :save => nil})
                 end
                 it "should produce running statistics for normal submission" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r92:274af5f15b1f - Thu, 11 Mar 2010 04:28:53 - 4 files changed: 95 inserted, 21 deleted

Sign in to your account

Commit r92:274af5f15b1f public

Author

r92:274af5f15b1f - Thu, 11 Mar 2010 04:28:53 - 4 files changed: 95 inserted, 21 deleted

Commit `r92:274af5f15b1f` public