cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r37:effd16a52c43` public

parent | child

Description:

[grader] fixed bug #6 and #9 git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@139 6386c4cd-e34a-4fa8-8920-d93eb39b512e

Commit status:

[Not Reviewed]

References:

default

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r37:effd16a52c43 - Fri, 21 Mar 2008 16:34:23 - 5 files changed: 13 inserted, 3 deleted

lib/configuration.rb ¶ +3 -1

               module Grader
                 # This singleton class holds basic configurations for grader.  When
                 # running in each mode, grader uses resources from different
                 # directories and outputs differently.  Usually the attributes name
                 # are descriptive; below we explain more on each attributes.
                 class Configuration
                   # Rails' environment: "development", "production"
                   attr_accessor :rails_env
                   # Grader looks for problem [prob] in problem_dir/[prob], and store
                   # execution results for submission [x] of user [u] in directory
                   # user_result_dir/[u]/[x]
                   attr_accessor :problems_dir
                   attr_accessor :user_result_dir
                   # If report_grader=true, the grader would add a row in model
                   # GraderProcess.  It would report itself with grader_hostname and
                   # process id.
                   attr_accessor :report_grader
                   attr_accessor :grader_hostname
                   # If talkative=true, grader would report status to console.  If
                   # logging=true, grader would report status to a log file located
                   # in log_dir, in a file name mode.options.pid.  TODO: defined
                   # log file naming.
                   attr_accessor :talkative
                   attr_accessor :logging
                   attr_accessor :log_dir
                   # These are directories related to the test interface.
                   attr_accessor :test_request_input_base_dir
                   attr_accessor :test_request_output_base_dir
                   attr_accessor :test_request_problem_templates_dir
                   # Comment received from the grading script will be filtered
                   # through Configuration#report_comment.  How this method behave
                   # depends on this option; right now only two formats, :short and
                   # :long
                   attr_accessor :comment_report_style
                   def report_comment(comment)
                     case comment_report_style
                     when :short
-            -         if comment.chomp =~ /^P+$/    # all P's
+            +         if comment.chomp =~ /^[\[\]P]+$/    # all P's
                         'passed'
+            +         elsif comment.chomp =~ /[Cc]ompil.*[Ee]rror/
+            +           'compilation error'
                       else
                         'failed'
                       end
                     when :full
                       comment.chomp
                     end
                   end
                   # Codes for singleton
                   private_class_method :new
                   @@instance = nil
                   def self.get_instance
                     if @@instance==nil
                       @@instance = new
                     end
                     @@instance
                   end
                   private
                   def initialize
                     @talkative = false
                     @log_file = nil
                     @report_grader = false
                     @grader_hostname = `hostname`.chomp
                     @rails_env = 'development'
                     @comment_report_style = :full
                   end
                 end
               end

lib/submission_helper.rb ¶ +3 -1

               module Grader
                 class SubmissionRoomMaker
                   def initialize
                     @config = Grader::Configuration.get_instance
                   end
                   def produce_grading_room(submission)
                     user = submission.user
                     problem = submission.problem
                     grading_room = "#{@config.user_result_dir}/" +
                       "#{user.login}/#{problem.name}/#{submission.id}"
                     FileUtils.mkdir_p(grading_room)
                     grading_room
                   end
                   def find_problem_home(submission)
                     problem = submission.problem
                     "#{@config.problems_dir}/#{problem.name}"
                   end
                   def save_source(submission,source_name)
                     dir = self.produce_grading_room(submission)
                     f = File.open("#{dir}/#{source_name}","w")
                     f.write(submission.source)
                     f.close
                   end
                   def clean_up(submission)
                   end
                 end
                 class SubmissionReporter
                   def initialize
                     @config = Grader::Configuration.get_instance
                   end
                   def report(sub,test_result_dir)
                     save_result(sub,read_result(test_result_dir))
                   end
                   def report_error(sub,msg)
                     save_result(sub,{:points => 0,
                                   :comment => "Grading error: #{msg}" })
                   end
                   protected
                   def read_result(test_result_dir)
                     cmp_msg_fname = "#{test_result_dir}/compiler_message"
                     if FileTest.exist?(cmp_msg_fname)
                       cmp_file = File.open(cmp_msg_fname)
                       cmp_msg = cmp_file.read
                       cmp_file.close
                     else
                       cmp_msg = ""
                     end
                     result_fname = "#{test_result_dir}/result"
                     comment_fname = "#{test_result_dir}/comment"
                     if FileTest.exist?(result_fname)
                       comment = ""
                       begin
                         result_file = File.open(result_fname)
                         result = result_file.readline.to_i
                         result_file.close
                       rescue
                         result = 0
                         comment = "error reading result file."
                       end
                       begin
                         comment_file = File.open(comment_fname)
                         comment += comment_file.readline.chomp
                         comment_file.close
                       rescue
                         comment += ""
                       end
                       return {:points => result,
                         :comment => comment,
                         :cmp_msg => cmp_msg}
                     else
                       if FileTest.exist?("#{test_result_dir}/a.out")
                         return {:points => 0,
                           :comment => 'error during grading',
                           :cmp_msg => cmp_msg}
                       else
                         return {:points => 0,
-            -             :comment => 'compile error',
+            +             :comment => 'compilation error',
                           :cmp_msg => cmp_msg}
                       end
                     end
                   end
                   def save_result(submission,result)
                     problem = submission.problem
                     submission.graded_at = Time.now
                     points = result[:points]
                     submission.points = points
                     comment = @config.report_comment(result[:comment])
                     if problem == nil
                       submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
                     elsif points == problem.full_score
                       submission.grader_comment = 'PASSED: ' + comment
+            +       elsif result[:comment].chomp =~ /^[\[\]P]+$/
+            +         submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
                     else
                       submission.grader_comment = 'FAILED: ' + comment
                     end
                     submission.compiler_message = result[:cmp_msg] or ''
                     submission.save
                   end
                 end
               end

test/data/test1_compile_error.c ¶ +3

+            + /*
+            + LANG: C
+            + */
               #include <stdio.h>
               int main()
               {
                 int a,
                 scanf("%d %d",&a,&b);
                 printf("%d\n",a+b);
                 return 0;
               }

test/data/test1_correct.c ¶ +3

+            + /*
+            + LANG: C
+            + */
               #include <stdio.h>
               int main()
               {
                 int a,b;
                 scanf("%d %d",&a,&b);
                 printf("%d\n",a+b);
                 return 0;
               }

test/engine_spec.rb ¶ +1 -1

               require File.join(File.dirname(__FILE__),'spec_helper')
               require File.join(File.dirname(__FILE__),'engine_spec_helper')
               describe "A grader engine, when grading submissions" do
                 include GraderEngineHelperMethods
                 before(:each) do
                   @config = Grader::Configuration.get_instance
                   # this test is from Pong
                   @problem_test_normal = stub(Problem,
                                               :id => 1, :name => 'test_normal',
                                               :full_score => 135)
                   @user_user1 = stub(User,
                                      :id => 1, :login => 'user1')
                   @engine = Grader::Engine.new
                   init_sandbox
                 end
                 it "should grade normal submission" do
                   grader_should(:grade => "test1_correct.c",
                                 :on => @problem_test_normal,
                                 :and_report => {
                                   :score => 135,
                                   :comment => /^PASSED/})
                 end
                 it "should produce error message when submission cannot compile" do
                   grader_should(:grade => "test1_compile_error.c",
                                 :on => @problem_test_normal,
                                 :and_report => {
                                   :score => 0,
-            -                     :comment => 'FAILED: compile error',
+            +                     :comment => 'FAILED: compilation error',
                                   :compiler_message => /[Ee]rror/})
                 end
                 it "should produce timeout error when submission runs forever" do
                   @problem_test_timeout = stub(Problem,
                                                :id => 1, :name => 'test_timeout',
                                                :full_score => 10)
                   grader_should(:grade => "test2_timeout.c",
                                 :on => @problem_test_timeout,
                                 :and_report => {
                                   :score => 0,
                                   :comment => 'FAILED: TT'})
                 end
                 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
                   @problem_test_timeout = stub(Problem,
                                                :id => 1, :name => 'test_timeout',
                                                :full_score => 20)
                   grader_should(:grade => "test2_1-5sec.c",
                                 :on => @problem_test_timeout,
                                 :and_report => {
                                   :score => 10,
                                   :comment => 'FAILED: TP'})
                 end
                 it "should produce runtime error when submission uses too much static memory" do
                   @problem_test_memory = stub(Problem,
                                               :id => 1, :name => 'test_memory',
                                               :full_score => 20)
                   grader_should(:grade => "add_too_much_memory_static.c",
                                 :on => @problem_test_memory,
                                 :and_report => {
                                   :score => 10,
                                   :comment => /FAILED: [^P]P/})
                 end
                 it "should not allow submission to allocate too much dynamic memory" do
                   @problem_test_memory = stub(Problem,
                                               :id => 1, :name => 'test_memory',
                                               :full_score => 20)
                   grader_should(:grade => "add_too_much_memory_dynamic.c",
                                 :on => @problem_test_memory,
                                 :and_report => {
                                   :score => 10,
                                   :comment => /FAILED: [^P]P/})
                 end
                 it "should score test runs correctly when submission fails in some test case" do
                   grader_should(:grade => "add_fail_test_case_1.c",
                                 :on => @problem_test_normal,
                                 :and_report => {
                                   :score => 105,
                                   :comment => /^FAILED:/})
                 end
                 it "should fail submission with non-zero exit status" do
                   grader_should(:grade => "add_nonzero_exit_status.c",
                                 :on => @problem_test_normal,
                                 :and_report => {
                                   :score => 0,
                                   :comment => /^FAILED:/})
                 end
                 it "should not allow malicious submission to see PROBLEM_HOME" do
                   problem_test_yesno =  stub(Problem,
                                               :id => 1, :name => 'test_yesno',
                                               :full_score => 10)
                   grader_should(:grade => "yesno_access_problem_home.c",
                                 :on => problem_test_yesno,
                                 :and_report => {
                                   :score => 0,
                                   :comment => /^FAILED:/})
                 end
                 it "should not allow malicious submission to open files" do
                   problem_test_yesno =  stub(Problem,
                                               :id => 1, :name => 'test_yesno',
                                               :full_score => 10)
                   grader_should(:grade => "yesno_open_file.c",
                                 :on => problem_test_yesno,
                                 :and_report => {
                                   :score => 0,
                                   :comment => /^FAILED:/})
                 end
                 def grader_should(args)
                   @user1 = stub(User,
                                 :id => 1, :login => 'user1')
                   submission =
                     create_submission_from_file(1, @user1, args[:on], args[:grade])
                   submission.should_receive(:graded_at=)
                   expected_score = args[:and_report][:score]
                   expected_comment = args[:and_report][:comment]
                   if args[:and_report][:compiler_message]!=nil
                     expected_compiler_message = args[:and_report][:compiler_message]
                   else
                     expected_compiler_message = ''
                   end
                   submission.should_receive(:points=).with(expected_score)
                   submission.should_receive(:grader_comment=).with(expected_comment)
                   submission.should_receive(:compiler_message=).with(expected_compiler_message)
                   submission.should_receive(:save)
                   @engine.grade(submission)
                 end
                 protected
                 def create_normal_submission_mock_from_file(source_fname)
                   create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
                 end
               end
               describe "A grader engine, when grading test requests" do
                 include GraderEngineHelperMethods
                 before(:each) do
                   @config = Grader::Configuration.get_instance
                   @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
                                                Grader::TestRequestReporter.new)
                   init_sandbox
                 end
                 it "should report error if there is no problem template" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'nothing')
                   grader_should(:grade => 'test1_correct.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => /template not found/,
                                   :save => nil})
                 end
                 it "should run test request and produce output file" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test1_correct.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => /0.0 sec./,
                                   :output_file_name= => lambda { |fname|
                                     File.exists?(fname).should be_true
                                   },
                                   :save => nil})
                 end
                 it "should clean up problem directory after running test request" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test1_correct.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => nil,
                                   :output_file_name= => nil,
                                   :save => nil})
                   File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
                 end
                 it "should compile test request with error and report compilation error" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test1_compile_error.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => /.+/,
                                   :grader_comment= => /[Cc]ompil.*error/,
                                   :running_stat= => '',
                                   :save => nil})
                 end
                 it "should report exit status" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'add_nonzero_exit_status.c',

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r37:effd16a52c43 - Fri, 21 Mar 2008 16:34:23 - 5 files changed: 13 inserted, 3 deleted

Sign in to your account

Commit r37:effd16a52c43 public

Author

r37:effd16a52c43 - Fri, 21 Mar 2008 16:34:23 - 5 files changed: 13 inserted, 3 deleted

Commit `r37:effd16a52c43` public