cafe-grader-judge-scripts

Commit r32:f9158ac7d119 public

parent | child
Description:
fix test run scoring bug git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@126 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r32:f9158ac7d119 - - 3 files changed: 35 inserted, 0 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -0,0 +1,13
1 + #include <stdio.h>
2 +
3 + int main()
4 + {
5 + int a,b;
6 + scanf("%d %d",&a,&b);
7 + if((a==1) && (b==1))
8 + printf("100\n");
9 + else
10 + printf("%d\n",a+b);
11 + return 0;
12 + }
13 +
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -17,70 +17,84
17 17 end
18 18 end
19 19
20 20 def char_comment(comment)
21 21 if comment =~ /[iI]ncorrect/
22 22 INCORRECT_MARK
23 23 elsif comment =~ /[Cc]orrect/
24 24 CORRECT_MARK
25 25 elsif comment =~ /[Tt]ime/
26 26 TIMEOUT_MARK
27 27 else
28 28 RUN_ERROR_MARK # these are run time errors
29 29 end
30 30 end
31 31
32 32 problem_home = ENV['PROBLEM_HOME']
33 33 require "#{problem_home}/script/test_dsl.rb"
34 34 load "#{problem_home}/test_cases/all_tests.cfg"
35 35 problem = Problem.get_instance
36 36
37 37 if problem.well_formed? == false
38 38 log "The problem specification is not well formed."
39 39 exit(127)
40 40 end
41 41
42 42 all_score = 0
43 43 all_comment = ''
44 44 (1..(problem.runs.length-1)).each do |k|
45 45 log "grade run #{k}"
46 46 run = problem.runs[k]
47 47 run_score = 0
48 48 run_comment = ''
49 49 run_comment_short = ''
50 50 run.tests.each do |test_num|
51 51 result_file_name = "#{test_num}/result"
52 52 if not File.exists?(result_file_name)
53 53 run_comment += "result file for test #{test_num} not found\n"
54 54 run_comment_short += RUN_ERROR_MARK
55 55 log "Cannot find the file #{test_num}/result!"
56 56 else
57 57 result_file = File.new(result_file_name, "r")
58 58 result_file_lines = result_file.readlines
59 59 run_score = run_score + result_file_lines[1].to_i
60 60 run_comment += result_file_lines[0]
61 61 run_comment_short += char_comment(result_file_lines[0])
62 62 result_file.close
63 63 end
64 64 end
65 +
66 + # find total score for this run
67 + run_total_score = 0
68 + problem = Problem.get_instance
69 + run.tests.each { |test_num| run_total_score += problem.get_score(test_num) }
70 +
71 + if run_total_score!=run_score # fail in some test cases, fail the run
72 + run_score = 0
73 + end
65 74
66 75 run_result_file = File.new("result-#{k}", "w")
67 76 run_result_file.write run_score
68 77 run_result_file.write "\n"
69 78 run_result_file.close
70 79
71 80 run_comment_file = File.new("comment-#{k}", "w")
72 81 run_comment_file.write "#{run_comment}\n"
73 82 run_comment_file.close
74 83
75 84 all_score = all_score + run_score
85 +
86 + # append comment for test run with many test cases
87 + if run.tests.length > 1
88 + run_comment_short = '[' + run_comment_short + ']'
89 + end
76 90 all_comment += run_comment_short
77 91 end
78 92
79 93 result_file = File.new("result", "w")
80 94 result_file.write all_score
81 95 result_file.write "\n"
82 96 result_file.close
83 97
84 98 comment_file = File.new("comment", "w")
85 99 comment_file.write "#{all_comment}\n"
86 100 comment_file.close
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -36,96 +36,104
36 36 :comment => 'FAILED: compile error',
37 37 :compiler_message => /[Ee]rror/})
38 38 end
39 39
40 40 it "should produce timeout error when submission runs forever" do
41 41 @problem_test_timeout = stub(Problem,
42 42 :id => 1, :name => 'test_timeout',
43 43 :full_score => 10)
44 44 grader_should(:grade => "test2_timeout.c",
45 45 :on => @problem_test_timeout,
46 46 :and_report => {
47 47 :score => 0,
48 48 :comment => 'FAILED: TT'})
49 49 end
50 50
51 51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
52 52 @problem_test_timeout = stub(Problem,
53 53 :id => 1, :name => 'test_timeout',
54 54 :full_score => 20)
55 55 grader_should(:grade => "test2_1-5sec.c",
56 56 :on => @problem_test_timeout,
57 57 :and_report => {
58 58 :score => 10,
59 59 :comment => 'FAILED: TP'})
60 60 end
61 61
62 62 it "should produce runtime error when submission uses too much static memory" do
63 63 @problem_test_memory = stub(Problem,
64 64 :id => 1, :name => 'test_memory',
65 65 :full_score => 20)
66 66 grader_should(:grade => "add_too_much_memory_static.c",
67 67 :on => @problem_test_memory,
68 68 :and_report => {
69 69 :score => 10,
70 70 :comment => /FAILED: [^P]P/})
71 71 end
72 72
73 73 it "should not allow submission to allocate too much dynamic memory" do
74 74 @problem_test_memory = stub(Problem,
75 75 :id => 1, :name => 'test_memory',
76 76 :full_score => 20)
77 77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 78 :on => @problem_test_memory,
79 79 :and_report => {
80 80 :score => 10,
81 81 :comment => /FAILED: [^P]P/})
82 82 end
83 83
84 + it "should score test runs correctly when submission fails in some test case" do
85 + grader_should(:grade => "add_fail_test_case_1.c",
86 + :on => @problem_test_normal,
87 + :and_report => {
88 + :score => 105,
89 + :comment => /^FAILED:/})
90 + end
91 +
84 92 it "should fail submission with non-zero exit status" do
85 93 violated("has not been implemented")
86 94 end
87 95
88 96 def grader_should(args)
89 97 @user1 = stub(User,
90 98 :id => 1, :login => 'user1')
91 99 submission =
92 100 create_submission_from_file(1, @user1, args[:on], args[:grade])
93 101 submission.should_receive(:graded_at=)
94 102
95 103 expected_score = args[:and_report][:score]
96 104 expected_comment = args[:and_report][:comment]
97 105 if args[:and_report][:compiler_message]!=nil
98 106 expected_compiler_message = args[:and_report][:compiler_message]
99 107 else
100 108 expected_compiler_message = ''
101 109 end
102 110
103 111 submission.should_receive(:points=).with(expected_score)
104 112 submission.should_receive(:grader_comment=).with(expected_comment)
105 113 submission.should_receive(:compiler_message=).with(expected_compiler_message)
106 114 submission.should_receive(:save)
107 115
108 116 @engine.grade(submission)
109 117 end
110 118
111 119 protected
112 120
113 121 def create_normal_submission_mock_from_file(source_fname)
114 122 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
115 123 end
116 124
117 125 end
118 126
119 127 describe "A grader engine, when grading test requests" do
120 128
121 129 include GraderEngineHelperMethods
122 130
123 131 before(:each) do
124 132 @config = Grader::Configuration.get_instance
125 133 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
126 134 Grader::TestRequestReporter.new)
127 135 init_sandbox
128 136 end
129 137
130 138 it "should report error if there is no problem template" do
131 139 problem = stub(Problem,
You need to be logged in to leave comments. Login now