cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r32:f9158ac7d119` public

parent | child

Description:

fix test run scoring bug git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@126 6386c4cd-e34a-4fa8-8920-d93eb39b512e

Commit status:

[Not Reviewed]

References:

default

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r32:f9158ac7d119 - Mon, 17 Mar 2008 05:19:01 - 3 files changed: 35 inserted, 0 deleted

test/data/add_fail_test_case_1.c created 644 ¶ +13

@@ -0,0 +1,13
	1	+ #include <stdio.h>
	2	+
	3	+ int main()
	4	+ {
	5	+ int a,b;
	6	+ scanf("%d %d",&a,&b);
	7	+ if((a==1) && (b==1))
	8	+ printf("100\n");
	9	+ else
	10	+ printf("%d\n",a+b);
	11	+ return 0;
	12	+ }
	13	+

std-script/grade ¶ +14

               end
               def char_comment(comment)
                 if comment =~ /[iI]ncorrect/
                   INCORRECT_MARK
                 elsif comment =~ /[Cc]orrect/
                   CORRECT_MARK
                 elsif comment =~ /[Tt]ime/
                   TIMEOUT_MARK
                 else
                   RUN_ERROR_MARK  # these are run time errors
                 end
               end
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               if problem.well_formed? == false
                 log "The problem specification is not well formed."
                 exit(127)
               end
               all_score = 0
               all_comment = ''
               (1..(problem.runs.length-1)).each do |k|
                 log "grade run #{k}"
                 run = problem.runs[k]
                 run_score = 0
                 run_comment = ''
                 run_comment_short = ''
                 run.tests.each do |test_num|
                   result_file_name = "#{test_num}/result"
                   if not File.exists?(result_file_name)
                     run_comment += "result file for test #{test_num} not found\n"
                     run_comment_short += RUN_ERROR_MARK
                     log "Cannot find the file #{test_num}/result!"
                   else
                     result_file = File.new(result_file_name, "r")
                     result_file_lines = result_file.readlines
                     run_score = run_score + result_file_lines[1].to_i
                     run_comment += result_file_lines[0]
                     run_comment_short += char_comment(result_file_lines[0])
                     result_file.close
                   end
                 end
+            +   # find total score for this run
+            +   run_total_score = 0
+            +   problem = Problem.get_instance
+            +   run.tests.each { |test_num| run_total_score += problem.get_score(test_num) }
+            +
+            +   if run_total_score!=run_score   # fail in some test cases, fail the run
+            +     run_score = 0
+            +   end
+            +
                 run_result_file = File.new("result-#{k}", "w")
                 run_result_file.write run_score
                 run_result_file.write "\n"
                 run_result_file.close
                 run_comment_file = File.new("comment-#{k}", "w")
                 run_comment_file.write "#{run_comment}\n"
                 run_comment_file.close
                 all_score = all_score + run_score
+            +
+            +   # append comment for test run with many test cases
+            +   if run.tests.length > 1
+            +     run_comment_short = '[' + run_comment_short + ']'
+            +   end
                 all_comment += run_comment_short
               end
               result_file = File.new("result", "w")
               result_file.write all_score
               result_file.write "\n"
               result_file.close
               comment_file = File.new("comment", "w")
               comment_file.write "#{all_comment}\n"
               comment_file.close

test/engine_spec.rb ¶ +8

                                   :comment => 'FAILED: compile error',
                                   :compiler_message => /[Ee]rror/})
                 end
                 it "should produce timeout error when submission runs forever" do
                   @problem_test_timeout = stub(Problem,
                                                :id => 1, :name => 'test_timeout',
                                                :full_score => 10)
                   grader_should(:grade => "test2_timeout.c",
                                 :on => @problem_test_timeout,
                                 :and_report => {
                                   :score => 0,
                                   :comment => 'FAILED: TT'})
                 end
                 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
                   @problem_test_timeout = stub(Problem,
                                                :id => 1, :name => 'test_timeout',
                                                :full_score => 20)
                   grader_should(:grade => "test2_1-5sec.c",
                                 :on => @problem_test_timeout,
                                 :and_report => {
                                   :score => 10,
                                   :comment => 'FAILED: TP'})
                 end
                 it "should produce runtime error when submission uses too much static memory" do
                   @problem_test_memory = stub(Problem,
                                               :id => 1, :name => 'test_memory',
                                               :full_score => 20)
                   grader_should(:grade => "add_too_much_memory_static.c",
                                 :on => @problem_test_memory,
                                 :and_report => {
                                   :score => 10,
                                   :comment => /FAILED: [^P]P/})
                 end
                 it "should not allow submission to allocate too much dynamic memory" do
                   @problem_test_memory = stub(Problem,
                                               :id => 1, :name => 'test_memory',
                                               :full_score => 20)
                   grader_should(:grade => "add_too_much_memory_dynamic.c",
                                 :on => @problem_test_memory,
                                 :and_report => {
                                   :score => 10,
                                   :comment => /FAILED: [^P]P/})
                 end
+            +   it "should score test runs correctly when submission fails in some test case" do
+            +     grader_should(:grade => "add_fail_test_case_1.c",
+            +                   :on => @problem_test_normal,
+            +                   :and_report => {
+            +                     :score => 105,
+            +                     :comment => /^FAILED:/})
+            +   end
+            +
                 it "should fail submission with non-zero exit status" do
                   violated("has not been implemented")
                 end
                 def grader_should(args)
                   @user1 = stub(User,
                                 :id => 1, :login => 'user1')
                   submission =
                     create_submission_from_file(1, @user1, args[:on], args[:grade])
                   submission.should_receive(:graded_at=)
                   expected_score = args[:and_report][:score]
                   expected_comment = args[:and_report][:comment]
                   if args[:and_report][:compiler_message]!=nil
                     expected_compiler_message = args[:and_report][:compiler_message]
                   else
                     expected_compiler_message = ''
                   end
                   submission.should_receive(:points=).with(expected_score)
                   submission.should_receive(:grader_comment=).with(expected_comment)
                   submission.should_receive(:compiler_message=).with(expected_compiler_message)
                   submission.should_receive(:save)
                   @engine.grade(submission)
                 end
                 protected
                 def create_normal_submission_mock_from_file(source_fname)
                   create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
                 end
               end
               describe "A grader engine, when grading test requests" do
                 include GraderEngineHelperMethods
                 before(:each) do
                   @config = Grader::Configuration.get_instance
                   @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
                                                Grader::TestRequestReporter.new)
                   init_sandbox
                 end
                 it "should report error if there is no problem template" do
                   problem = stub(Problem,

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r32:f9158ac7d119 - Mon, 17 Mar 2008 05:19:01 - 3 files changed: 35 inserted, 0 deleted

Sign in to your account

Commit r32:f9158ac7d119 public

Author

r32:f9158ac7d119 - Mon, 17 Mar 2008 05:19:01 - 3 files changed: 35 inserted, 0 deleted

Commit `r32:f9158ac7d119` public