cafe-grader-judge-scripts

Commit r205:825a5364bee6 public

parent | child
Description:
Updated `be_true` and `be_false` to `be true` and `be false` respectively
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r205:825a5364bee6 - - 1 file changed: 4 inserted, 4 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,328 +1,328
1 require File.join(File.dirname(__FILE__),'spec_helper')
1 require File.join(File.dirname(__FILE__),'spec_helper')
2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3
3
4 describe "A grader engine, when grading submissions" do
4 describe "A grader engine, when grading submissions" do
5
5
6 include GraderEngineHelperMethods
6 include GraderEngineHelperMethods
7
7
8 before(:each) do
8 before(:each) do
9 @config = Grader::Configuration.get_instance
9 @config = Grader::Configuration.get_instance
10
10
11 # this test is from Pong
11 # this test is from Pong
12 @problem_test_normal = stub(Problem,
12 @problem_test_normal = stub(Problem,
13 :id => 1, :name => 'test_normal',
13 :id => 1, :name => 'test_normal',
14 :full_score => 135)
14 :full_score => 135)
15 @user_user1 = stub(User,
15 @user_user1 = stub(User,
16 :id => 1, :login => 'user1')
16 :id => 1, :login => 'user1')
17
17
18 @engine = Grader::Engine.new
18 @engine = Grader::Engine.new
19 init_sandbox
19 init_sandbox
20 end
20 end
21
21
22 it "should grade normal submission" do
22 it "should grade normal submission" do
23 grader_should(:grade => "test1_correct.c",
23 grader_should(:grade => "test1_correct.c",
24 :on => @problem_test_normal,
24 :on => @problem_test_normal,
25 :and_report => {
25 :and_report => {
26 :score => 135,
26 :score => 135,
27 :comment => /^(\[|P|\])+/})
27 :comment => /^(\[|P|\])+/})
28 end
28 end
29
29
30
30
31 it "should grade normal submission in C++" do
31 it "should grade normal submission in C++" do
32 cpp_lang = stub(Language, :name => 'cpp', :ext => 'cpp')
32 cpp_lang = stub(Language, :name => 'cpp', :ext => 'cpp')
33
33
34 grader_should(:grade => "test1_correct_cc.cc",
34 grader_should(:grade => "test1_correct_cc.cc",
35 :language => cpp_lang,
35 :language => cpp_lang,
36 :on => @problem_test_normal,
36 :on => @problem_test_normal,
37 :and_report => {
37 :and_report => {
38 :score => 135,
38 :score => 135,
39 :comment => /^(\[|P|\])+/})
39 :comment => /^(\[|P|\])+/})
40 end
40 end
41
41
42 it "should produce error message when submission cannot compile" do
42 it "should produce error message when submission cannot compile" do
43 grader_should(:grade => "test1_compile_error.c",
43 grader_should(:grade => "test1_compile_error.c",
44 :on => @problem_test_normal,
44 :on => @problem_test_normal,
45 :and_report => {
45 :and_report => {
46 :score => 0,
46 :score => 0,
47 :comment => 'compilation error',
47 :comment => 'compilation error',
48 :compiler_message => /[Ee]rror/})
48 :compiler_message => /[Ee]rror/})
49 end
49 end
50
50
51 it "should produce timeout error when submission runs forever" do
51 it "should produce timeout error when submission runs forever" do
52 @problem_test_timeout = stub(Problem,
52 @problem_test_timeout = stub(Problem,
53 :id => 1, :name => 'test_timeout',
53 :id => 1, :name => 'test_timeout',
54 :full_score => 10)
54 :full_score => 10)
55 grader_should(:grade => "test2_timeout.c",
55 grader_should(:grade => "test2_timeout.c",
56 :on => @problem_test_timeout,
56 :on => @problem_test_timeout,
57 :and_report => {
57 :and_report => {
58 :score => 0,
58 :score => 0,
59 :comment => 'TT'})
59 :comment => 'TT'})
60 end
60 end
61
61
62 it "should produce timeout error correctly with fractional running time and fractional time limits" do
62 it "should produce timeout error correctly with fractional running time and fractional time limits" do
63 @problem_test_timeout = stub(Problem,
63 @problem_test_timeout = stub(Problem,
64 :id => 1, :name => 'test_timeout',
64 :id => 1, :name => 'test_timeout',
65 :full_score => 20)
65 :full_score => 20)
66 grader_should(:grade => "test2_05sec.c",
66 grader_should(:grade => "test2_05sec.c",
67 :on => @problem_test_timeout,
67 :on => @problem_test_timeout,
68 :and_report => {
68 :and_report => {
69 :score => 10,
69 :score => 10,
70 :comment => 'TP'})
70 :comment => 'TP'})
71 end
71 end
72
72
73 it "should produce runtime error when submission uses too much static memory" do
73 it "should produce runtime error when submission uses too much static memory" do
74 @problem_test_memory = stub(Problem,
74 @problem_test_memory = stub(Problem,
75 :id => 1, :name => 'test_memory',
75 :id => 1, :name => 'test_memory',
76 :full_score => 20)
76 :full_score => 20)
77 grader_should(:grade => "add_too_much_memory_static.c",
77 grader_should(:grade => "add_too_much_memory_static.c",
78 :on => @problem_test_memory,
78 :on => @problem_test_memory,
79 :and_report => {
79 :and_report => {
80 :score => 10,
80 :score => 10,
81 :comment => /[^P]P/})
81 :comment => /[^P]P/})
82 end
82 end
83
83
84 it "should not allow submission to allocate too much dynamic memory" do
84 it "should not allow submission to allocate too much dynamic memory" do
85 @problem_test_memory = stub(Problem,
85 @problem_test_memory = stub(Problem,
86 :id => 1, :name => 'test_memory',
86 :id => 1, :name => 'test_memory',
87 :full_score => 20)
87 :full_score => 20)
88 grader_should(:grade => "add_too_much_memory_dynamic.c",
88 grader_should(:grade => "add_too_much_memory_dynamic.c",
89 :on => @problem_test_memory,
89 :on => @problem_test_memory,
90 :and_report => {
90 :and_report => {
91 :score => 10,
91 :score => 10,
92 :comment => /[^P]P/})
92 :comment => /[^P]P/})
93 end
93 end
94
94
95 it "should score test runs correctly when submission fails in some test case" do
95 it "should score test runs correctly when submission fails in some test case" do
96 grader_should(:grade => "add_fail_test_case_1.c",
96 grader_should(:grade => "add_fail_test_case_1.c",
97 :on => @problem_test_normal,
97 :on => @problem_test_normal,
98 :and_report => {
98 :and_report => {
99 :score => 105,
99 :score => 105,
100 :comment => /^.*(-|x).*$/})
100 :comment => /^.*(-|x).*$/})
101 end
101 end
102
102
103 it "should fail submission with non-zero exit status" do
103 it "should fail submission with non-zero exit status" do
104 grader_should(:grade => "add_nonzero_exit_status.c",
104 grader_should(:grade => "add_nonzero_exit_status.c",
105 :on => @problem_test_normal,
105 :on => @problem_test_normal,
106 :and_report => {
106 :and_report => {
107 :score => 0,
107 :score => 0,
108 :comment => /^(\[|x|\])+$/})
108 :comment => /^(\[|x|\])+$/})
109 end
109 end
110
110
111 it "should not allow malicious submission to see PROBLEM_HOME" do
111 it "should not allow malicious submission to see PROBLEM_HOME" do
112 problem_test_yesno = stub(Problem,
112 problem_test_yesno = stub(Problem,
113 :id => 1, :name => 'test_yesno',
113 :id => 1, :name => 'test_yesno',
114 :full_score => 10)
114 :full_score => 10)
115 grader_should(:grade => "yesno_access_problem_home.c",
115 grader_should(:grade => "yesno_access_problem_home.c",
116 :on => problem_test_yesno,
116 :on => problem_test_yesno,
117 :and_report => {
117 :and_report => {
118 :score => 0,
118 :score => 0,
119 :comment => /(-|x)/})
119 :comment => /(-|x)/})
120 end
120 end
121
121
122 it "should not allow malicious submission to open files" do
122 it "should not allow malicious submission to open files" do
123 problem_test_yesno = stub(Problem,
123 problem_test_yesno = stub(Problem,
124 :id => 1, :name => 'test_yesno',
124 :id => 1, :name => 'test_yesno',
125 :full_score => 10)
125 :full_score => 10)
126 grader_should(:grade => "yesno_open_file.c",
126 grader_should(:grade => "yesno_open_file.c",
127 :on => problem_test_yesno,
127 :on => problem_test_yesno,
128 :and_report => {
128 :and_report => {
129 :score => 0,
129 :score => 0,
130 :comment => /(-|x)/})
130 :comment => /(-|x)/})
131 end
131 end
132
132
133 def grader_should(args)
133 def grader_should(args)
134 @user1 = stub(User,
134 @user1 = stub(User,
135 :id => 1, :login => 'user1')
135 :id => 1, :login => 'user1')
136
136
137 submission =
137 submission =
138 create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language])
138 create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language])
139 submission.should_receive(:graded_at=)
139 submission.should_receive(:graded_at=)
140
140
141 expected_score = args[:and_report][:score]
141 expected_score = args[:and_report][:score]
142 expected_comment = args[:and_report][:comment]
142 expected_comment = args[:and_report][:comment]
143 if args[:and_report][:compiler_message]!=nil
143 if args[:and_report][:compiler_message]!=nil
144 expected_compiler_message = args[:and_report][:compiler_message]
144 expected_compiler_message = args[:and_report][:compiler_message]
145 else
145 else
146 expected_compiler_message = ''
146 expected_compiler_message = ''
147 end
147 end
148
148
149 submission.should_receive(:points=).with(expected_score)
149 submission.should_receive(:points=).with(expected_score)
150 submission.should_receive(:grader_comment=).with(expected_comment)
150 submission.should_receive(:grader_comment=).with(expected_comment)
151 submission.should_receive(:compiler_message=).with(expected_compiler_message)
151 submission.should_receive(:compiler_message=).with(expected_compiler_message)
152 submission.should_receive(:save)
152 submission.should_receive(:save)
153
153
154 @engine.grade(submission)
154 @engine.grade(submission)
155 end
155 end
156
156
157 protected
157 protected
158
158
159 def create_normal_submission_mock_from_file(source_fname)
159 def create_normal_submission_mock_from_file(source_fname)
160 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
160 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
161 end
161 end
162
162
163 end
163 end
164
164
165 describe "A grader engine, when grading test requests" do
165 describe "A grader engine, when grading test requests" do
166
166
167 include GraderEngineHelperMethods
167 include GraderEngineHelperMethods
168
168
169 before(:each) do
169 before(:each) do
170 @config = Grader::Configuration.get_instance
170 @config = Grader::Configuration.get_instance
171 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
171 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
172 :reporter => Grader::TestRequestReporter.new)
172 :reporter => Grader::TestRequestReporter.new)
173 init_sandbox
173 init_sandbox
174 end
174 end
175
175
176 it "should report error if there is no problem template" do
176 it "should report error if there is no problem template" do
177 problem = stub(Problem,
177 problem = stub(Problem,
178 :id => 1, :name => 'nothing')
178 :id => 1, :name => 'nothing')
179 grader_should(:grade => 'test1_correct.c',
179 grader_should(:grade => 'test1_correct.c',
180 :on => problem,
180 :on => problem,
181 :with => 'in1.txt',
181 :with => 'in1.txt',
182 :and_report => {
182 :and_report => {
183 :graded_at= => nil,
183 :graded_at= => nil,
184 :compiler_message= => '',
184 :compiler_message= => '',
185 :grader_comment= => '',
185 :grader_comment= => '',
186 :running_stat= => /template not found/,
186 :running_stat= => /template not found/,
187 :running_time= => nil,
187 :running_time= => nil,
188 :exit_status= => nil,
188 :exit_status= => nil,
189 :memory_usage= => nil,
189 :memory_usage= => nil,
190 :save => nil})
190 :save => nil})
191 end
191 end
192
192
193 it "should run test request and produce output file" do
193 it "should run test request and produce output file" do
194 problem = stub(Problem,
194 problem = stub(Problem,
195 :id => 1, :name => 'test_normal')
195 :id => 1, :name => 'test_normal')
196 grader_should(:grade => 'test1_correct.c',
196 grader_should(:grade => 'test1_correct.c',
197 :on => problem,
197 :on => problem,
198 :with => 'in1.txt',
198 :with => 'in1.txt',
199 :and_report => {
199 :and_report => {
200 :graded_at= => nil,
200 :graded_at= => nil,
201 :compiler_message= => '',
201 :compiler_message= => '',
202 :grader_comment= => '',
202 :grader_comment= => '',
203 :running_stat= => /0.0\d* sec./,
203 :running_stat= => /0.0\d* sec./,
204 :output_file_name= => lambda { |fname|
204 :output_file_name= => lambda { |fname|
205 - File.exists?(fname).should be_true
205 + File.exists?(fname).should be true
206 },
206 },
207 :running_time= => nil,
207 :running_time= => nil,
208 :exit_status= => nil,
208 :exit_status= => nil,
209 :memory_usage= => nil,
209 :memory_usage= => nil,
210 :save => nil})
210 :save => nil})
211 end
211 end
212
212
213 it "should clean up problem directory after running test request" do
213 it "should clean up problem directory after running test request" do
214 problem = stub(Problem,
214 problem = stub(Problem,
215 :id => 1, :name => 'test_normal')
215 :id => 1, :name => 'test_normal')
216 grader_should(:grade => 'test1_correct.c',
216 grader_should(:grade => 'test1_correct.c',
217 :on => problem,
217 :on => problem,
218 :with => 'in1.txt',
218 :with => 'in1.txt',
219 :and_report => {
219 :and_report => {
220 :graded_at= => nil,
220 :graded_at= => nil,
221 :compiler_message= => '',
221 :compiler_message= => '',
222 :grader_comment= => '',
222 :grader_comment= => '',
223 :running_stat= => nil,
223 :running_stat= => nil,
224 :output_file_name= => nil,
224 :output_file_name= => nil,
225 :running_time= => nil,
225 :running_time= => nil,
226 :exit_status= => nil,
226 :exit_status= => nil,
227 :memory_usage= => nil,
227 :memory_usage= => nil,
228 :save => nil})
228 :save => nil})
229 - File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
229 + File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be false
230 end
230 end
231
231
232 it "should compile test request with error and report compilation error" do
232 it "should compile test request with error and report compilation error" do
233 problem = stub(Problem,
233 problem = stub(Problem,
234 :id => 1, :name => 'test_normal')
234 :id => 1, :name => 'test_normal')
235 grader_should(:grade => 'test1_compile_error.c',
235 grader_should(:grade => 'test1_compile_error.c',
236 :on => problem,
236 :on => problem,
237 :with => 'in1.txt',
237 :with => 'in1.txt',
238 :and_report => {
238 :and_report => {
239 :graded_at= => nil,
239 :graded_at= => nil,
240 :compiler_message= => /.+/,
240 :compiler_message= => /.+/,
241 :grader_comment= => /[Cc]ompil.*error/,
241 :grader_comment= => /[Cc]ompil.*error/,
242 :running_stat= => '',
242 :running_stat= => '',
243 :save => nil})
243 :save => nil})
244 end
244 end
245
245
246 it "should report exit status" do
246 it "should report exit status" do
247 problem = stub(Problem,
247 problem = stub(Problem,
248 :id => 1, :name => 'test_normal')
248 :id => 1, :name => 'test_normal')
249 grader_should(:grade => 'add_nonzero_exit_status.c',
249 grader_should(:grade => 'add_nonzero_exit_status.c',
250 :on => problem,
250 :on => problem,
251 :with => 'in1.txt',
251 :with => 'in1.txt',
252 :and_report => {
252 :and_report => {
253 :graded_at= => nil,
253 :graded_at= => nil,
254 :compiler_message= => '',
254 :compiler_message= => '',
255 :grader_comment= => '',
255 :grader_comment= => '',
256 :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
256 :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
257 :output_file_name= => lambda { |fname|
257 :output_file_name= => lambda { |fname|
258 - File.exists?(fname).should be_true
258 + File.exists?(fname).should be true
259 },
259 },
260 :running_time= => nil,
260 :running_time= => nil,
261 :exit_status= => /10/,
261 :exit_status= => /10/,
262 :memory_usage= => nil,
262 :memory_usage= => nil,
263 :save => nil})
263 :save => nil})
264 end
264 end
265
265
266 it "should produce running statistics for normal submission" do
266 it "should produce running statistics for normal submission" do
267 problem = stub(Problem,
267 problem = stub(Problem,
268 :id => 1, :name => 'test_normal')
268 :id => 1, :name => 'test_normal')
269 grader_should(:grade => 'test_run_stat.c',
269 grader_should(:grade => 'test_run_stat.c',
270 :on => problem,
270 :on => problem,
271 :with => 'in1.txt',
271 :with => 'in1.txt',
272 :and_report => {
272 :and_report => {
273 :graded_at= => nil,
273 :graded_at= => nil,
274 :compiler_message= => '',
274 :compiler_message= => '',
275 :grader_comment= => '',
275 :grader_comment= => '',
276 :running_stat= => nil,
276 :running_stat= => nil,
277 :output_file_name= => lambda { |fname|
277 :output_file_name= => lambda { |fname|
278 - File.exists?(fname).should be_true
278 + File.exists?(fname).should be true
279 },
279 },
280 :running_time= => lambda { |t|
280 :running_time= => lambda { |t|
281 (t>=0.14) and (t<=0.16)
281 (t>=0.14) and (t<=0.16)
282 },
282 },
283 :exit_status= => nil,
283 :exit_status= => nil,
284 :memory_usage= => lambda { |s|
284 :memory_usage= => lambda { |s|
285 (s>=500) and (s<=1000)
285 (s>=500) and (s<=1000)
286 },
286 },
287 :save => nil})
287 :save => nil})
288 end
288 end
289
289
290 protected
290 protected
291 def grader_should(args)
291 def grader_should(args)
292 @user1 = stub(User,
292 @user1 = stub(User,
293 :id => 1, :login => 'user1')
293 :id => 1, :login => 'user1')
294
294
295 problem = args[:on]
295 problem = args[:on]
296 input_file = @config.test_request_input_base_dir + "/" + args[:with]
296 input_file = @config.test_request_input_base_dir + "/" + args[:with]
297
297
298 submission =
298 submission =
299 create_submission_from_file(1, @user1, args[:on], args[:grade])
299 create_submission_from_file(1, @user1, args[:on], args[:grade])
300
300
301 test_request = stub(TestRequest,
301 test_request = stub(TestRequest,
302 :id => 1,
302 :id => 1,
303 :user => @user1,
303 :user => @user1,
304 :problem => problem,
304 :problem => problem,
305 :submission => submission,
305 :submission => submission,
306 :input_file_name => input_file,
306 :input_file_name => input_file,
307 :language => submission.language,
307 :language => submission.language,
308 :problem_name => problem.name)
308 :problem_name => problem.name)
309
309
310 expectations = args[:and_report]
310 expectations = args[:and_report]
311
311
312 expectations.each do |key,val|
312 expectations.each do |key,val|
313 if val==nil
313 if val==nil
314 test_request.should_receive(key)
314 test_request.should_receive(key)
315 elsif val.class == Proc
315 elsif val.class == Proc
316 test_request.should_receive(key) { |fname|
316 test_request.should_receive(key) { |fname|
317 val.call(fname)
317 val.call(fname)
318 }
318 }
319 else
319 else
320 test_request.should_receive(key).with(val)
320 test_request.should_receive(key).with(val)
321 end
321 end
322 end
322 end
323
323
324 @engine.grade(test_request)
324 @engine.grade(test_request)
325 end
325 end
326
326
327 end
327 end
328
328
You need to be logged in to leave comments. Login now