cafe-grader-judge-scripts

Commit r32:f9158ac7d119 public

parent | child
Description:
fix test run scoring bug git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@126 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r32:f9158ac7d119 - - 3 files changed: 35 inserted, 0 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -0,0 +1,13
1 + #include <stdio.h>
2 +
3 + int main()
4 + {
5 + int a,b;
6 + scanf("%d %d",&a,&b);
7 + if((a==1) && (b==1))
8 + printf("100\n");
9 + else
10 + printf("%d\n",a+b);
11 + return 0;
12 + }
13 +
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,86 +1,100
1 #!/usr/bin/ruby
1 #!/usr/bin/ruby
2
2
3 CORRECT_MARK = 'P'
3 CORRECT_MARK = 'P'
4 INCORRECT_MARK = '-'
4 INCORRECT_MARK = '-'
5 TIMEOUT_MARK = 'T'
5 TIMEOUT_MARK = 'T'
6 RUN_ERROR_MARK = 'x'
6 RUN_ERROR_MARK = 'x'
7
7
8 def log(str='')
8 def log(str='')
9 if ENV['TALKATIVE']!=nil
9 if ENV['TALKATIVE']!=nil
10 puts str
10 puts str
11 end
11 end
12 if ENV['GRADER_LOGGING']!=nil
12 if ENV['GRADER_LOGGING']!=nil
13 log_fname = ENV['GRADER_LOGGING']
13 log_fname = ENV['GRADER_LOGGING']
14 fp = File.open(log_fname,"a")
14 fp = File.open(log_fname,"a")
15 fp.puts("grade: #{Time.new.strftime("%H:%M")} #{str}")
15 fp.puts("grade: #{Time.new.strftime("%H:%M")} #{str}")
16 fp.close
16 fp.close
17 end
17 end
18 end
18 end
19
19
20 def char_comment(comment)
20 def char_comment(comment)
21 if comment =~ /[iI]ncorrect/
21 if comment =~ /[iI]ncorrect/
22 INCORRECT_MARK
22 INCORRECT_MARK
23 elsif comment =~ /[Cc]orrect/
23 elsif comment =~ /[Cc]orrect/
24 CORRECT_MARK
24 CORRECT_MARK
25 elsif comment =~ /[Tt]ime/
25 elsif comment =~ /[Tt]ime/
26 TIMEOUT_MARK
26 TIMEOUT_MARK
27 else
27 else
28 RUN_ERROR_MARK # these are run time errors
28 RUN_ERROR_MARK # these are run time errors
29 end
29 end
30 end
30 end
31
31
32 problem_home = ENV['PROBLEM_HOME']
32 problem_home = ENV['PROBLEM_HOME']
33 require "#{problem_home}/script/test_dsl.rb"
33 require "#{problem_home}/script/test_dsl.rb"
34 load "#{problem_home}/test_cases/all_tests.cfg"
34 load "#{problem_home}/test_cases/all_tests.cfg"
35 problem = Problem.get_instance
35 problem = Problem.get_instance
36
36
37 if problem.well_formed? == false
37 if problem.well_formed? == false
38 log "The problem specification is not well formed."
38 log "The problem specification is not well formed."
39 exit(127)
39 exit(127)
40 end
40 end
41
41
42 all_score = 0
42 all_score = 0
43 all_comment = ''
43 all_comment = ''
44 (1..(problem.runs.length-1)).each do |k|
44 (1..(problem.runs.length-1)).each do |k|
45 log "grade run #{k}"
45 log "grade run #{k}"
46 run = problem.runs[k]
46 run = problem.runs[k]
47 run_score = 0
47 run_score = 0
48 run_comment = ''
48 run_comment = ''
49 run_comment_short = ''
49 run_comment_short = ''
50 run.tests.each do |test_num|
50 run.tests.each do |test_num|
51 result_file_name = "#{test_num}/result"
51 result_file_name = "#{test_num}/result"
52 if not File.exists?(result_file_name)
52 if not File.exists?(result_file_name)
53 run_comment += "result file for test #{test_num} not found\n"
53 run_comment += "result file for test #{test_num} not found\n"
54 run_comment_short += RUN_ERROR_MARK
54 run_comment_short += RUN_ERROR_MARK
55 log "Cannot find the file #{test_num}/result!"
55 log "Cannot find the file #{test_num}/result!"
56 else
56 else
57 result_file = File.new(result_file_name, "r")
57 result_file = File.new(result_file_name, "r")
58 result_file_lines = result_file.readlines
58 result_file_lines = result_file.readlines
59 run_score = run_score + result_file_lines[1].to_i
59 run_score = run_score + result_file_lines[1].to_i
60 run_comment += result_file_lines[0]
60 run_comment += result_file_lines[0]
61 run_comment_short += char_comment(result_file_lines[0])
61 run_comment_short += char_comment(result_file_lines[0])
62 result_file.close
62 result_file.close
63 end
63 end
64 end
64 end
65
65
66 + # find total score for this run
67 + run_total_score = 0
68 + problem = Problem.get_instance
69 + run.tests.each { |test_num| run_total_score += problem.get_score(test_num) }
70 +
71 + if run_total_score!=run_score # fail in some test cases, fail the run
72 + run_score = 0
73 + end
74 +
66 run_result_file = File.new("result-#{k}", "w")
75 run_result_file = File.new("result-#{k}", "w")
67 run_result_file.write run_score
76 run_result_file.write run_score
68 run_result_file.write "\n"
77 run_result_file.write "\n"
69 run_result_file.close
78 run_result_file.close
70
79
71 run_comment_file = File.new("comment-#{k}", "w")
80 run_comment_file = File.new("comment-#{k}", "w")
72 run_comment_file.write "#{run_comment}\n"
81 run_comment_file.write "#{run_comment}\n"
73 run_comment_file.close
82 run_comment_file.close
74
83
75 all_score = all_score + run_score
84 all_score = all_score + run_score
85 +
86 + # append comment for test run with many test cases
87 + if run.tests.length > 1
88 + run_comment_short = '[' + run_comment_short + ']'
89 + end
76 all_comment += run_comment_short
90 all_comment += run_comment_short
77 end
91 end
78
92
79 result_file = File.new("result", "w")
93 result_file = File.new("result", "w")
80 result_file.write all_score
94 result_file.write all_score
81 result_file.write "\n"
95 result_file.write "\n"
82 result_file.close
96 result_file.close
83
97
84 comment_file = File.new("comment", "w")
98 comment_file = File.new("comment", "w")
85 comment_file.write "#{all_comment}\n"
99 comment_file.write "#{all_comment}\n"
86 comment_file.close
100 comment_file.close
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,246 +1,254
1 require File.join(File.dirname(__FILE__),'spec_helper')
1 require File.join(File.dirname(__FILE__),'spec_helper')
2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3
3
4 describe "A grader engine, when grading submissions" do
4 describe "A grader engine, when grading submissions" do
5
5
6 include GraderEngineHelperMethods
6 include GraderEngineHelperMethods
7
7
8 before(:each) do
8 before(:each) do
9 @config = Grader::Configuration.get_instance
9 @config = Grader::Configuration.get_instance
10
10
11 # this test is from Pong
11 # this test is from Pong
12 @problem_test_normal = stub(Problem,
12 @problem_test_normal = stub(Problem,
13 :id => 1, :name => 'test_normal',
13 :id => 1, :name => 'test_normal',
14 :full_score => 135)
14 :full_score => 135)
15 @user_user1 = stub(User,
15 @user_user1 = stub(User,
16 :id => 1, :login => 'user1')
16 :id => 1, :login => 'user1')
17
17
18 @engine = Grader::Engine.new
18 @engine = Grader::Engine.new
19 init_sandbox
19 init_sandbox
20 end
20 end
21
21
22 it "should grade normal submission" do
22 it "should grade normal submission" do
23 grader_should(:grade => "test1_correct.c",
23 grader_should(:grade => "test1_correct.c",
24 :on => @problem_test_normal,
24 :on => @problem_test_normal,
25 :and_report => {
25 :and_report => {
26 :score => 135,
26 :score => 135,
27 :comment => /^PASSED/})
27 :comment => /^PASSED/})
28 end
28 end
29
29
30
30
31 it "should produce error message when submission cannot compile" do
31 it "should produce error message when submission cannot compile" do
32 grader_should(:grade => "test1_compile_error.c",
32 grader_should(:grade => "test1_compile_error.c",
33 :on => @problem_test_normal,
33 :on => @problem_test_normal,
34 :and_report => {
34 :and_report => {
35 :score => 0,
35 :score => 0,
36 :comment => 'FAILED: compile error',
36 :comment => 'FAILED: compile error',
37 :compiler_message => /[Ee]rror/})
37 :compiler_message => /[Ee]rror/})
38 end
38 end
39
39
40 it "should produce timeout error when submission runs forever" do
40 it "should produce timeout error when submission runs forever" do
41 @problem_test_timeout = stub(Problem,
41 @problem_test_timeout = stub(Problem,
42 :id => 1, :name => 'test_timeout',
42 :id => 1, :name => 'test_timeout',
43 :full_score => 10)
43 :full_score => 10)
44 grader_should(:grade => "test2_timeout.c",
44 grader_should(:grade => "test2_timeout.c",
45 :on => @problem_test_timeout,
45 :on => @problem_test_timeout,
46 :and_report => {
46 :and_report => {
47 :score => 0,
47 :score => 0,
48 :comment => 'FAILED: TT'})
48 :comment => 'FAILED: TT'})
49 end
49 end
50
50
51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
52 @problem_test_timeout = stub(Problem,
52 @problem_test_timeout = stub(Problem,
53 :id => 1, :name => 'test_timeout',
53 :id => 1, :name => 'test_timeout',
54 :full_score => 20)
54 :full_score => 20)
55 grader_should(:grade => "test2_1-5sec.c",
55 grader_should(:grade => "test2_1-5sec.c",
56 :on => @problem_test_timeout,
56 :on => @problem_test_timeout,
57 :and_report => {
57 :and_report => {
58 :score => 10,
58 :score => 10,
59 :comment => 'FAILED: TP'})
59 :comment => 'FAILED: TP'})
60 end
60 end
61
61
62 it "should produce runtime error when submission uses too much static memory" do
62 it "should produce runtime error when submission uses too much static memory" do
63 @problem_test_memory = stub(Problem,
63 @problem_test_memory = stub(Problem,
64 :id => 1, :name => 'test_memory',
64 :id => 1, :name => 'test_memory',
65 :full_score => 20)
65 :full_score => 20)
66 grader_should(:grade => "add_too_much_memory_static.c",
66 grader_should(:grade => "add_too_much_memory_static.c",
67 :on => @problem_test_memory,
67 :on => @problem_test_memory,
68 :and_report => {
68 :and_report => {
69 :score => 10,
69 :score => 10,
70 :comment => /FAILED: [^P]P/})
70 :comment => /FAILED: [^P]P/})
71 end
71 end
72
72
73 it "should not allow submission to allocate too much dynamic memory" do
73 it "should not allow submission to allocate too much dynamic memory" do
74 @problem_test_memory = stub(Problem,
74 @problem_test_memory = stub(Problem,
75 :id => 1, :name => 'test_memory',
75 :id => 1, :name => 'test_memory',
76 :full_score => 20)
76 :full_score => 20)
77 grader_should(:grade => "add_too_much_memory_dynamic.c",
77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 :on => @problem_test_memory,
78 :on => @problem_test_memory,
79 :and_report => {
79 :and_report => {
80 :score => 10,
80 :score => 10,
81 :comment => /FAILED: [^P]P/})
81 :comment => /FAILED: [^P]P/})
82 end
82 end
83
83
84 + it "should score test runs correctly when submission fails in some test case" do
85 + grader_should(:grade => "add_fail_test_case_1.c",
86 + :on => @problem_test_normal,
87 + :and_report => {
88 + :score => 105,
89 + :comment => /^FAILED:/})
90 + end
91 +
84 it "should fail submission with non-zero exit status" do
92 it "should fail submission with non-zero exit status" do
85 violated("has not been implemented")
93 violated("has not been implemented")
86 end
94 end
87
95
88 def grader_should(args)
96 def grader_should(args)
89 @user1 = stub(User,
97 @user1 = stub(User,
90 :id => 1, :login => 'user1')
98 :id => 1, :login => 'user1')
91 submission =
99 submission =
92 create_submission_from_file(1, @user1, args[:on], args[:grade])
100 create_submission_from_file(1, @user1, args[:on], args[:grade])
93 submission.should_receive(:graded_at=)
101 submission.should_receive(:graded_at=)
94
102
95 expected_score = args[:and_report][:score]
103 expected_score = args[:and_report][:score]
96 expected_comment = args[:and_report][:comment]
104 expected_comment = args[:and_report][:comment]
97 if args[:and_report][:compiler_message]!=nil
105 if args[:and_report][:compiler_message]!=nil
98 expected_compiler_message = args[:and_report][:compiler_message]
106 expected_compiler_message = args[:and_report][:compiler_message]
99 else
107 else
100 expected_compiler_message = ''
108 expected_compiler_message = ''
101 end
109 end
102
110
103 submission.should_receive(:points=).with(expected_score)
111 submission.should_receive(:points=).with(expected_score)
104 submission.should_receive(:grader_comment=).with(expected_comment)
112 submission.should_receive(:grader_comment=).with(expected_comment)
105 submission.should_receive(:compiler_message=).with(expected_compiler_message)
113 submission.should_receive(:compiler_message=).with(expected_compiler_message)
106 submission.should_receive(:save)
114 submission.should_receive(:save)
107
115
108 @engine.grade(submission)
116 @engine.grade(submission)
109 end
117 end
110
118
111 protected
119 protected
112
120
113 def create_normal_submission_mock_from_file(source_fname)
121 def create_normal_submission_mock_from_file(source_fname)
114 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
122 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
115 end
123 end
116
124
117 end
125 end
118
126
119 describe "A grader engine, when grading test requests" do
127 describe "A grader engine, when grading test requests" do
120
128
121 include GraderEngineHelperMethods
129 include GraderEngineHelperMethods
122
130
123 before(:each) do
131 before(:each) do
124 @config = Grader::Configuration.get_instance
132 @config = Grader::Configuration.get_instance
125 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
133 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
126 Grader::TestRequestReporter.new)
134 Grader::TestRequestReporter.new)
127 init_sandbox
135 init_sandbox
128 end
136 end
129
137
130 it "should report error if there is no problem template" do
138 it "should report error if there is no problem template" do
131 problem = stub(Problem,
139 problem = stub(Problem,
132 :id => 1, :name => 'nothing')
140 :id => 1, :name => 'nothing')
133 grader_should(:grade => 'test1_correct.c',
141 grader_should(:grade => 'test1_correct.c',
134 :on => problem,
142 :on => problem,
135 :with => 'in1.txt',
143 :with => 'in1.txt',
136 :and_report => {
144 :and_report => {
137 :graded_at= => nil,
145 :graded_at= => nil,
138 :compiler_message= => '',
146 :compiler_message= => '',
139 :grader_comment= => '',
147 :grader_comment= => '',
140 :running_stat= => /template not found/,
148 :running_stat= => /template not found/,
141 :save => nil})
149 :save => nil})
142 end
150 end
143
151
144 it "should run test request and produce output file" do
152 it "should run test request and produce output file" do
145 problem = stub(Problem,
153 problem = stub(Problem,
146 :id => 1, :name => 'test_normal')
154 :id => 1, :name => 'test_normal')
147 grader_should(:grade => 'test1_correct.c',
155 grader_should(:grade => 'test1_correct.c',
148 :on => problem,
156 :on => problem,
149 :with => 'in1.txt',
157 :with => 'in1.txt',
150 :and_report => {
158 :and_report => {
151 :graded_at= => nil,
159 :graded_at= => nil,
152 :compiler_message= => '',
160 :compiler_message= => '',
153 :grader_comment= => '',
161 :grader_comment= => '',
154 :running_stat= => /0.0 sec./,
162 :running_stat= => /0.0 sec./,
155 :output_file_name= => lambda { |fname|
163 :output_file_name= => lambda { |fname|
156 File.exists?(fname).should be_true
164 File.exists?(fname).should be_true
157 },
165 },
158 :save => nil})
166 :save => nil})
159 end
167 end
160
168
161 it "should clean up problem directory after running test request" do
169 it "should clean up problem directory after running test request" do
162 problem = stub(Problem,
170 problem = stub(Problem,
163 :id => 1, :name => 'test_normal')
171 :id => 1, :name => 'test_normal')
164 grader_should(:grade => 'test1_correct.c',
172 grader_should(:grade => 'test1_correct.c',
165 :on => problem,
173 :on => problem,
166 :with => 'in1.txt',
174 :with => 'in1.txt',
167 :and_report => {
175 :and_report => {
168 :graded_at= => nil,
176 :graded_at= => nil,
169 :compiler_message= => '',
177 :compiler_message= => '',
170 :grader_comment= => '',
178 :grader_comment= => '',
171 :running_stat= => nil,
179 :running_stat= => nil,
172 :output_file_name= => nil,
180 :output_file_name= => nil,
173 :save => nil})
181 :save => nil})
174 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
182 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
175 end
183 end
176
184
177 it "should compile test request with error and report compilation error" do
185 it "should compile test request with error and report compilation error" do
178 problem = stub(Problem,
186 problem = stub(Problem,
179 :id => 1, :name => 'test_normal')
187 :id => 1, :name => 'test_normal')
180 grader_should(:grade => 'test1_compile_error.c',
188 grader_should(:grade => 'test1_compile_error.c',
181 :on => problem,
189 :on => problem,
182 :with => 'in1.txt',
190 :with => 'in1.txt',
183 :and_report => {
191 :and_report => {
184 :graded_at= => nil,
192 :graded_at= => nil,
185 :compiler_message= => /.+/,
193 :compiler_message= => /.+/,
186 :grader_comment= => /[Cc]ompil.*error/,
194 :grader_comment= => /[Cc]ompil.*error/,
187 :running_stat= => '',
195 :running_stat= => '',
188 :save => nil})
196 :save => nil})
189 end
197 end
190
198
191 it "should report exit status" do
199 it "should report exit status" do
192 problem = stub(Problem,
200 problem = stub(Problem,
193 :id => 1, :name => 'test_normal')
201 :id => 1, :name => 'test_normal')
194 grader_should(:grade => 'add_nonzero_exit_status.c',
202 grader_should(:grade => 'add_nonzero_exit_status.c',
195 :on => problem,
203 :on => problem,
196 :with => 'in1.txt',
204 :with => 'in1.txt',
197 :and_report => {
205 :and_report => {
198 :graded_at= => nil,
206 :graded_at= => nil,
199 :compiler_message= => '',
207 :compiler_message= => '',
200 :grader_comment= => '',
208 :grader_comment= => '',
201 :running_stat= => /[Ee]xit.*status.*10.*0.0 sec./,
209 :running_stat= => /[Ee]xit.*status.*10.*0.0 sec./,
202 :output_file_name= => lambda { |fname|
210 :output_file_name= => lambda { |fname|
203 File.exists?(fname).should be_true
211 File.exists?(fname).should be_true
204 },
212 },
205 :save => nil})
213 :save => nil})
206 end
214 end
207
215
208 protected
216 protected
209 def grader_should(args)
217 def grader_should(args)
210 @user1 = stub(User,
218 @user1 = stub(User,
211 :id => 1, :login => 'user1')
219 :id => 1, :login => 'user1')
212
220
213 problem = args[:on]
221 problem = args[:on]
214 input_file = @config.test_request_input_base_dir + "/" + args[:with]
222 input_file = @config.test_request_input_base_dir + "/" + args[:with]
215
223
216 submission =
224 submission =
217 create_submission_from_file(1, @user1, args[:on], args[:grade])
225 create_submission_from_file(1, @user1, args[:on], args[:grade])
218
226
219 test_request = stub(TestRequest,
227 test_request = stub(TestRequest,
220 :id => 1,
228 :id => 1,
221 :user => @user1,
229 :user => @user1,
222 :problem => problem,
230 :problem => problem,
223 :submission => submission,
231 :submission => submission,
224 :input_file_name => input_file,
232 :input_file_name => input_file,
225 :language => submission.language,
233 :language => submission.language,
226 :problem_name => problem.name)
234 :problem_name => problem.name)
227
235
228 expectations = args[:and_report]
236 expectations = args[:and_report]
229
237
230 expectations.each do |key,val|
238 expectations.each do |key,val|
231 if val==nil
239 if val==nil
232 test_request.should_receive(key)
240 test_request.should_receive(key)
233 elsif val.class == Proc
241 elsif val.class == Proc
234 test_request.should_receive(key) { |fname|
242 test_request.should_receive(key) { |fname|
235 val.call(fname)
243 val.call(fname)
236 }
244 }
237 else
245 else
238 test_request.should_receive(key).with(val)
246 test_request.should_receive(key).with(val)
239 end
247 end
240 end
248 end
241
249
242 @engine.grade(test_request)
250 @engine.grade(test_request)
243 end
251 end
244
252
245 end
253 end
246
254
You need to be logged in to leave comments. Login now