cafe-grader-judge-scripts

Commit r44:a07593cb07c4 public

parent | child
Description:
[grader] small tweak on one test case git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@168 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r44:a07593cb07c4 - - 1 file changed: 1 inserted, 1 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,316 +1,316
1 require File.join(File.dirname(__FILE__),'spec_helper')
1 require File.join(File.dirname(__FILE__),'spec_helper')
2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3
3
4 describe "A grader engine, when grading submissions" do
4 describe "A grader engine, when grading submissions" do
5
5
6 include GraderEngineHelperMethods
6 include GraderEngineHelperMethods
7
7
8 before(:each) do
8 before(:each) do
9 @config = Grader::Configuration.get_instance
9 @config = Grader::Configuration.get_instance
10
10
11 # this test is from Pong
11 # this test is from Pong
12 @problem_test_normal = stub(Problem,
12 @problem_test_normal = stub(Problem,
13 :id => 1, :name => 'test_normal',
13 :id => 1, :name => 'test_normal',
14 :full_score => 135)
14 :full_score => 135)
15 @user_user1 = stub(User,
15 @user_user1 = stub(User,
16 :id => 1, :login => 'user1')
16 :id => 1, :login => 'user1')
17
17
18 @engine = Grader::Engine.new
18 @engine = Grader::Engine.new
19 init_sandbox
19 init_sandbox
20 end
20 end
21
21
22 it "should grade normal submission" do
22 it "should grade normal submission" do
23 grader_should(:grade => "test1_correct.c",
23 grader_should(:grade => "test1_correct.c",
24 :on => @problem_test_normal,
24 :on => @problem_test_normal,
25 :and_report => {
25 :and_report => {
26 :score => 135,
26 :score => 135,
27 :comment => /^PASSED/})
27 :comment => /^PASSED/})
28 end
28 end
29
29
30
30
31 it "should produce error message when submission cannot compile" do
31 it "should produce error message when submission cannot compile" do
32 grader_should(:grade => "test1_compile_error.c",
32 grader_should(:grade => "test1_compile_error.c",
33 :on => @problem_test_normal,
33 :on => @problem_test_normal,
34 :and_report => {
34 :and_report => {
35 :score => 0,
35 :score => 0,
36 :comment => 'FAILED: compilation error',
36 :comment => 'FAILED: compilation error',
37 :compiler_message => /[Ee]rror/})
37 :compiler_message => /[Ee]rror/})
38 end
38 end
39
39
40 it "should produce timeout error when submission runs forever" do
40 it "should produce timeout error when submission runs forever" do
41 @problem_test_timeout = stub(Problem,
41 @problem_test_timeout = stub(Problem,
42 :id => 1, :name => 'test_timeout',
42 :id => 1, :name => 'test_timeout',
43 :full_score => 10)
43 :full_score => 10)
44 grader_should(:grade => "test2_timeout.c",
44 grader_should(:grade => "test2_timeout.c",
45 :on => @problem_test_timeout,
45 :on => @problem_test_timeout,
46 :and_report => {
46 :and_report => {
47 :score => 0,
47 :score => 0,
48 :comment => 'FAILED: TT'})
48 :comment => 'FAILED: TT'})
49 end
49 end
50
50
51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
52 @problem_test_timeout = stub(Problem,
52 @problem_test_timeout = stub(Problem,
53 :id => 1, :name => 'test_timeout',
53 :id => 1, :name => 'test_timeout',
54 :full_score => 20)
54 :full_score => 20)
55 grader_should(:grade => "test2_1-5sec.c",
55 grader_should(:grade => "test2_1-5sec.c",
56 :on => @problem_test_timeout,
56 :on => @problem_test_timeout,
57 :and_report => {
57 :and_report => {
58 :score => 10,
58 :score => 10,
59 :comment => 'FAILED: TP'})
59 :comment => 'FAILED: TP'})
60 end
60 end
61
61
62 it "should produce runtime error when submission uses too much static memory" do
62 it "should produce runtime error when submission uses too much static memory" do
63 @problem_test_memory = stub(Problem,
63 @problem_test_memory = stub(Problem,
64 :id => 1, :name => 'test_memory',
64 :id => 1, :name => 'test_memory',
65 :full_score => 20)
65 :full_score => 20)
66 grader_should(:grade => "add_too_much_memory_static.c",
66 grader_should(:grade => "add_too_much_memory_static.c",
67 :on => @problem_test_memory,
67 :on => @problem_test_memory,
68 :and_report => {
68 :and_report => {
69 :score => 10,
69 :score => 10,
70 :comment => /FAILED: [^P]P/})
70 :comment => /FAILED: [^P]P/})
71 end
71 end
72
72
73 it "should not allow submission to allocate too much dynamic memory" do
73 it "should not allow submission to allocate too much dynamic memory" do
74 @problem_test_memory = stub(Problem,
74 @problem_test_memory = stub(Problem,
75 :id => 1, :name => 'test_memory',
75 :id => 1, :name => 'test_memory',
76 :full_score => 20)
76 :full_score => 20)
77 grader_should(:grade => "add_too_much_memory_dynamic.c",
77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 :on => @problem_test_memory,
78 :on => @problem_test_memory,
79 :and_report => {
79 :and_report => {
80 :score => 10,
80 :score => 10,
81 :comment => /FAILED: [^P]P/})
81 :comment => /FAILED: [^P]P/})
82 end
82 end
83
83
84 it "should score test runs correctly when submission fails in some test case" do
84 it "should score test runs correctly when submission fails in some test case" do
85 grader_should(:grade => "add_fail_test_case_1.c",
85 grader_should(:grade => "add_fail_test_case_1.c",
86 :on => @problem_test_normal,
86 :on => @problem_test_normal,
87 :and_report => {
87 :and_report => {
88 :score => 105,
88 :score => 105,
89 :comment => /^FAILED:/})
89 :comment => /^FAILED:/})
90 end
90 end
91
91
92 it "should fail submission with non-zero exit status" do
92 it "should fail submission with non-zero exit status" do
93 grader_should(:grade => "add_nonzero_exit_status.c",
93 grader_should(:grade => "add_nonzero_exit_status.c",
94 :on => @problem_test_normal,
94 :on => @problem_test_normal,
95 :and_report => {
95 :and_report => {
96 :score => 0,
96 :score => 0,
97 :comment => /^FAILED:/})
97 :comment => /^FAILED:/})
98 end
98 end
99
99
100 it "should not allow malicious submission to see PROBLEM_HOME" do
100 it "should not allow malicious submission to see PROBLEM_HOME" do
101 problem_test_yesno = stub(Problem,
101 problem_test_yesno = stub(Problem,
102 :id => 1, :name => 'test_yesno',
102 :id => 1, :name => 'test_yesno',
103 :full_score => 10)
103 :full_score => 10)
104 grader_should(:grade => "yesno_access_problem_home.c",
104 grader_should(:grade => "yesno_access_problem_home.c",
105 :on => problem_test_yesno,
105 :on => problem_test_yesno,
106 :and_report => {
106 :and_report => {
107 :score => 0,
107 :score => 0,
108 :comment => /^FAILED:/})
108 :comment => /^FAILED:/})
109 end
109 end
110
110
111 it "should not allow malicious submission to open files" do
111 it "should not allow malicious submission to open files" do
112 problem_test_yesno = stub(Problem,
112 problem_test_yesno = stub(Problem,
113 :id => 1, :name => 'test_yesno',
113 :id => 1, :name => 'test_yesno',
114 :full_score => 10)
114 :full_score => 10)
115 grader_should(:grade => "yesno_open_file.c",
115 grader_should(:grade => "yesno_open_file.c",
116 :on => problem_test_yesno,
116 :on => problem_test_yesno,
117 :and_report => {
117 :and_report => {
118 :score => 0,
118 :score => 0,
119 :comment => /^FAILED:/})
119 :comment => /^FAILED:/})
120 end
120 end
121
121
122 def grader_should(args)
122 def grader_should(args)
123 @user1 = stub(User,
123 @user1 = stub(User,
124 :id => 1, :login => 'user1')
124 :id => 1, :login => 'user1')
125 submission =
125 submission =
126 create_submission_from_file(1, @user1, args[:on], args[:grade])
126 create_submission_from_file(1, @user1, args[:on], args[:grade])
127 submission.should_receive(:graded_at=)
127 submission.should_receive(:graded_at=)
128
128
129 expected_score = args[:and_report][:score]
129 expected_score = args[:and_report][:score]
130 expected_comment = args[:and_report][:comment]
130 expected_comment = args[:and_report][:comment]
131 if args[:and_report][:compiler_message]!=nil
131 if args[:and_report][:compiler_message]!=nil
132 expected_compiler_message = args[:and_report][:compiler_message]
132 expected_compiler_message = args[:and_report][:compiler_message]
133 else
133 else
134 expected_compiler_message = ''
134 expected_compiler_message = ''
135 end
135 end
136
136
137 submission.should_receive(:points=).with(expected_score)
137 submission.should_receive(:points=).with(expected_score)
138 submission.should_receive(:grader_comment=).with(expected_comment)
138 submission.should_receive(:grader_comment=).with(expected_comment)
139 submission.should_receive(:compiler_message=).with(expected_compiler_message)
139 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 submission.should_receive(:save)
140 submission.should_receive(:save)
141
141
142 @engine.grade(submission)
142 @engine.grade(submission)
143 end
143 end
144
144
145 protected
145 protected
146
146
147 def create_normal_submission_mock_from_file(source_fname)
147 def create_normal_submission_mock_from_file(source_fname)
148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 end
149 end
150
150
151 end
151 end
152
152
153 describe "A grader engine, when grading test requests" do
153 describe "A grader engine, when grading test requests" do
154
154
155 include GraderEngineHelperMethods
155 include GraderEngineHelperMethods
156
156
157 before(:each) do
157 before(:each) do
158 @config = Grader::Configuration.get_instance
158 @config = Grader::Configuration.get_instance
159 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
159 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
160 Grader::TestRequestReporter.new)
160 Grader::TestRequestReporter.new)
161 init_sandbox
161 init_sandbox
162 end
162 end
163
163
164 it "should report error if there is no problem template" do
164 it "should report error if there is no problem template" do
165 problem = stub(Problem,
165 problem = stub(Problem,
166 :id => 1, :name => 'nothing')
166 :id => 1, :name => 'nothing')
167 grader_should(:grade => 'test1_correct.c',
167 grader_should(:grade => 'test1_correct.c',
168 :on => problem,
168 :on => problem,
169 :with => 'in1.txt',
169 :with => 'in1.txt',
170 :and_report => {
170 :and_report => {
171 :graded_at= => nil,
171 :graded_at= => nil,
172 :compiler_message= => '',
172 :compiler_message= => '',
173 :grader_comment= => '',
173 :grader_comment= => '',
174 :running_stat= => /template not found/,
174 :running_stat= => /template not found/,
175 :running_time= => nil,
175 :running_time= => nil,
176 :exit_status= => nil,
176 :exit_status= => nil,
177 :memory_usage= => nil,
177 :memory_usage= => nil,
178 :save => nil})
178 :save => nil})
179 end
179 end
180
180
181 it "should run test request and produce output file" do
181 it "should run test request and produce output file" do
182 problem = stub(Problem,
182 problem = stub(Problem,
183 :id => 1, :name => 'test_normal')
183 :id => 1, :name => 'test_normal')
184 grader_should(:grade => 'test1_correct.c',
184 grader_should(:grade => 'test1_correct.c',
185 :on => problem,
185 :on => problem,
186 :with => 'in1.txt',
186 :with => 'in1.txt',
187 :and_report => {
187 :and_report => {
188 :graded_at= => nil,
188 :graded_at= => nil,
189 :compiler_message= => '',
189 :compiler_message= => '',
190 :grader_comment= => '',
190 :grader_comment= => '',
191 - :running_stat= => /0.0 sec./,
191 + :running_stat= => /0.0\d* sec./,
192 :output_file_name= => lambda { |fname|
192 :output_file_name= => lambda { |fname|
193 File.exists?(fname).should be_true
193 File.exists?(fname).should be_true
194 },
194 },
195 :running_time= => nil,
195 :running_time= => nil,
196 :exit_status= => nil,
196 :exit_status= => nil,
197 :memory_usage= => nil,
197 :memory_usage= => nil,
198 :save => nil})
198 :save => nil})
199 end
199 end
200
200
201 it "should clean up problem directory after running test request" do
201 it "should clean up problem directory after running test request" do
202 problem = stub(Problem,
202 problem = stub(Problem,
203 :id => 1, :name => 'test_normal')
203 :id => 1, :name => 'test_normal')
204 grader_should(:grade => 'test1_correct.c',
204 grader_should(:grade => 'test1_correct.c',
205 :on => problem,
205 :on => problem,
206 :with => 'in1.txt',
206 :with => 'in1.txt',
207 :and_report => {
207 :and_report => {
208 :graded_at= => nil,
208 :graded_at= => nil,
209 :compiler_message= => '',
209 :compiler_message= => '',
210 :grader_comment= => '',
210 :grader_comment= => '',
211 :running_stat= => nil,
211 :running_stat= => nil,
212 :output_file_name= => nil,
212 :output_file_name= => nil,
213 :running_time= => nil,
213 :running_time= => nil,
214 :exit_status= => nil,
214 :exit_status= => nil,
215 :memory_usage= => nil,
215 :memory_usage= => nil,
216 :save => nil})
216 :save => nil})
217 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
217 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
218 end
218 end
219
219
220 it "should compile test request with error and report compilation error" do
220 it "should compile test request with error and report compilation error" do
221 problem = stub(Problem,
221 problem = stub(Problem,
222 :id => 1, :name => 'test_normal')
222 :id => 1, :name => 'test_normal')
223 grader_should(:grade => 'test1_compile_error.c',
223 grader_should(:grade => 'test1_compile_error.c',
224 :on => problem,
224 :on => problem,
225 :with => 'in1.txt',
225 :with => 'in1.txt',
226 :and_report => {
226 :and_report => {
227 :graded_at= => nil,
227 :graded_at= => nil,
228 :compiler_message= => /.+/,
228 :compiler_message= => /.+/,
229 :grader_comment= => /[Cc]ompil.*error/,
229 :grader_comment= => /[Cc]ompil.*error/,
230 :running_stat= => '',
230 :running_stat= => '',
231 :save => nil})
231 :save => nil})
232 end
232 end
233
233
234 it "should report exit status" do
234 it "should report exit status" do
235 problem = stub(Problem,
235 problem = stub(Problem,
236 :id => 1, :name => 'test_normal')
236 :id => 1, :name => 'test_normal')
237 grader_should(:grade => 'add_nonzero_exit_status.c',
237 grader_should(:grade => 'add_nonzero_exit_status.c',
238 :on => problem,
238 :on => problem,
239 :with => 'in1.txt',
239 :with => 'in1.txt',
240 :and_report => {
240 :and_report => {
241 :graded_at= => nil,
241 :graded_at= => nil,
242 :compiler_message= => '',
242 :compiler_message= => '',
243 :grader_comment= => '',
243 :grader_comment= => '',
244 :running_stat= => /[Ee]xit.*status.*10.*0\.0 sec/m,
244 :running_stat= => /[Ee]xit.*status.*10.*0\.0 sec/m,
245 :output_file_name= => lambda { |fname|
245 :output_file_name= => lambda { |fname|
246 File.exists?(fname).should be_true
246 File.exists?(fname).should be_true
247 },
247 },
248 :running_time= => nil,
248 :running_time= => nil,
249 :exit_status= => /10/,
249 :exit_status= => /10/,
250 :memory_usage= => nil,
250 :memory_usage= => nil,
251 :save => nil})
251 :save => nil})
252 end
252 end
253
253
254 it "should produce running statistics for normal submission" do
254 it "should produce running statistics for normal submission" do
255 problem = stub(Problem,
255 problem = stub(Problem,
256 :id => 1, :name => 'test_normal')
256 :id => 1, :name => 'test_normal')
257 grader_should(:grade => 'test_run_stat.c',
257 grader_should(:grade => 'test_run_stat.c',
258 :on => problem,
258 :on => problem,
259 :with => 'in1.txt',
259 :with => 'in1.txt',
260 :and_report => {
260 :and_report => {
261 :graded_at= => nil,
261 :graded_at= => nil,
262 :compiler_message= => '',
262 :compiler_message= => '',
263 :grader_comment= => '',
263 :grader_comment= => '',
264 :running_stat= => nil,
264 :running_stat= => nil,
265 :output_file_name= => lambda { |fname|
265 :output_file_name= => lambda { |fname|
266 File.exists?(fname).should be_true
266 File.exists?(fname).should be_true
267 },
267 },
268 :running_time= => lambda { |t|
268 :running_time= => lambda { |t|
269 (t>=0.14) and (t<=0.16)
269 (t>=0.14) and (t<=0.16)
270 },
270 },
271 :exit_status= => nil,
271 :exit_status= => nil,
272 :memory_usage= => lambda { |s|
272 :memory_usage= => lambda { |s|
273 (s>=500) and (s<=1000)
273 (s>=500) and (s<=1000)
274 },
274 },
275 :save => nil})
275 :save => nil})
276 end
276 end
277
277
278 protected
278 protected
279 def grader_should(args)
279 def grader_should(args)
280 @user1 = stub(User,
280 @user1 = stub(User,
281 :id => 1, :login => 'user1')
281 :id => 1, :login => 'user1')
282
282
283 problem = args[:on]
283 problem = args[:on]
284 input_file = @config.test_request_input_base_dir + "/" + args[:with]
284 input_file = @config.test_request_input_base_dir + "/" + args[:with]
285
285
286 submission =
286 submission =
287 create_submission_from_file(1, @user1, args[:on], args[:grade])
287 create_submission_from_file(1, @user1, args[:on], args[:grade])
288
288
289 test_request = stub(TestRequest,
289 test_request = stub(TestRequest,
290 :id => 1,
290 :id => 1,
291 :user => @user1,
291 :user => @user1,
292 :problem => problem,
292 :problem => problem,
293 :submission => submission,
293 :submission => submission,
294 :input_file_name => input_file,
294 :input_file_name => input_file,
295 :language => submission.language,
295 :language => submission.language,
296 :problem_name => problem.name)
296 :problem_name => problem.name)
297
297
298 expectations = args[:and_report]
298 expectations = args[:and_report]
299
299
300 expectations.each do |key,val|
300 expectations.each do |key,val|
301 if val==nil
301 if val==nil
302 test_request.should_receive(key)
302 test_request.should_receive(key)
303 elsif val.class == Proc
303 elsif val.class == Proc
304 test_request.should_receive(key) { |fname|
304 test_request.should_receive(key) { |fname|
305 val.call(fname)
305 val.call(fname)
306 }
306 }
307 else
307 else
308 test_request.should_receive(key).with(val)
308 test_request.should_receive(key).with(val)
309 end
309 end
310 end
310 end
311
311
312 @engine.grade(test_request)
312 @engine.grade(test_request)
313 end
313 end
314
314
315 end
315 end
316
316
You need to be logged in to leave comments. Login now