cafe-grader-judge-scripts

Commit r126:be832795a805 public

parent | child
Description:
Merge branch 'master' into windows
Commit status:
[Not Reviewed]
References:
merge default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r126:be832795a805 - - 2 files changed: 26 inserted, 1 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -0,0 +1,13
1 + /*
2 + LANG: C++
3 + */
4 + #include <cstdio>
5 +
6 + int main()
7 + {
8 + int a,b,r;
9 + r = scanf("%d %d",&a,&b);
10 + printf("%d\n",a+b);
11 + return 0;
12 + }
13 +
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,316 +1,328
1 require File.join(File.dirname(__FILE__),'spec_helper')
1 require File.join(File.dirname(__FILE__),'spec_helper')
2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3
3
4 describe "A grader engine, when grading submissions" do
4 describe "A grader engine, when grading submissions" do
5
5
6 include GraderEngineHelperMethods
6 include GraderEngineHelperMethods
7
7
8 before(:each) do
8 before(:each) do
9 @config = Grader::Configuration.get_instance
9 @config = Grader::Configuration.get_instance
10
10
11 # this test is from Pong
11 # this test is from Pong
12 @problem_test_normal = stub(Problem,
12 @problem_test_normal = stub(Problem,
13 :id => 1, :name => 'test_normal',
13 :id => 1, :name => 'test_normal',
14 :full_score => 135)
14 :full_score => 135)
15 @user_user1 = stub(User,
15 @user_user1 = stub(User,
16 :id => 1, :login => 'user1')
16 :id => 1, :login => 'user1')
17
17
18 @engine = Grader::Engine.new
18 @engine = Grader::Engine.new
19 init_sandbox
19 init_sandbox
20 end
20 end
21
21
22 it "should grade normal submission" do
22 it "should grade normal submission" do
23 grader_should(:grade => "test1_correct.c",
23 grader_should(:grade => "test1_correct.c",
24 :on => @problem_test_normal,
24 :on => @problem_test_normal,
25 :and_report => {
25 :and_report => {
26 :score => 135,
26 :score => 135,
27 :comment => /^(\[|P|\])+/})
27 :comment => /^(\[|P|\])+/})
28 end
28 end
29
29
30
30
31 + it "should grade normal submission in C++" do
32 + cpp_lang = stub(Language, :name => 'cpp', :ext => 'cpp')
33 +
34 + grader_should(:grade => "test1_correct_cc.cc",
35 + :language => cpp_lang,
36 + :on => @problem_test_normal,
37 + :and_report => {
38 + :score => 135,
39 + :comment => /^(\[|P|\])+/})
40 + end
41 +
31 it "should produce error message when submission cannot compile" do
42 it "should produce error message when submission cannot compile" do
32 grader_should(:grade => "test1_compile_error.c",
43 grader_should(:grade => "test1_compile_error.c",
33 :on => @problem_test_normal,
44 :on => @problem_test_normal,
34 :and_report => {
45 :and_report => {
35 :score => 0,
46 :score => 0,
36 :comment => 'compilation error',
47 :comment => 'compilation error',
37 :compiler_message => /[Ee]rror/})
48 :compiler_message => /[Ee]rror/})
38 end
49 end
39
50
40 it "should produce timeout error when submission runs forever" do
51 it "should produce timeout error when submission runs forever" do
41 @problem_test_timeout = stub(Problem,
52 @problem_test_timeout = stub(Problem,
42 :id => 1, :name => 'test_timeout',
53 :id => 1, :name => 'test_timeout',
43 :full_score => 10)
54 :full_score => 10)
44 grader_should(:grade => "test2_timeout.c",
55 grader_should(:grade => "test2_timeout.c",
45 :on => @problem_test_timeout,
56 :on => @problem_test_timeout,
46 :and_report => {
57 :and_report => {
47 :score => 0,
58 :score => 0,
48 :comment => 'TT'})
59 :comment => 'TT'})
49 end
60 end
50
61
51 it "should produce timeout error correctly with fractional running time and fractional time limits" do
62 it "should produce timeout error correctly with fractional running time and fractional time limits" do
52 @problem_test_timeout = stub(Problem,
63 @problem_test_timeout = stub(Problem,
53 :id => 1, :name => 'test_timeout',
64 :id => 1, :name => 'test_timeout',
54 :full_score => 20)
65 :full_score => 20)
55 grader_should(:grade => "test2_05sec.c",
66 grader_should(:grade => "test2_05sec.c",
56 :on => @problem_test_timeout,
67 :on => @problem_test_timeout,
57 :and_report => {
68 :and_report => {
58 :score => 10,
69 :score => 10,
59 :comment => 'TP'})
70 :comment => 'TP'})
60 end
71 end
61
72
62 it "should produce runtime error when submission uses too much static memory" do
73 it "should produce runtime error when submission uses too much static memory" do
63 @problem_test_memory = stub(Problem,
74 @problem_test_memory = stub(Problem,
64 :id => 1, :name => 'test_memory',
75 :id => 1, :name => 'test_memory',
65 :full_score => 20)
76 :full_score => 20)
66 grader_should(:grade => "add_too_much_memory_static.c",
77 grader_should(:grade => "add_too_much_memory_static.c",
67 :on => @problem_test_memory,
78 :on => @problem_test_memory,
68 :and_report => {
79 :and_report => {
69 :score => 10,
80 :score => 10,
70 :comment => /[^P]P/})
81 :comment => /[^P]P/})
71 end
82 end
72
83
73 it "should not allow submission to allocate too much dynamic memory" do
84 it "should not allow submission to allocate too much dynamic memory" do
74 @problem_test_memory = stub(Problem,
85 @problem_test_memory = stub(Problem,
75 :id => 1, :name => 'test_memory',
86 :id => 1, :name => 'test_memory',
76 :full_score => 20)
87 :full_score => 20)
77 grader_should(:grade => "add_too_much_memory_dynamic.c",
88 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 :on => @problem_test_memory,
89 :on => @problem_test_memory,
79 :and_report => {
90 :and_report => {
80 :score => 10,
91 :score => 10,
81 :comment => /[^P]P/})
92 :comment => /[^P]P/})
82 end
93 end
83
94
84 it "should score test runs correctly when submission fails in some test case" do
95 it "should score test runs correctly when submission fails in some test case" do
85 grader_should(:grade => "add_fail_test_case_1.c",
96 grader_should(:grade => "add_fail_test_case_1.c",
86 :on => @problem_test_normal,
97 :on => @problem_test_normal,
87 :and_report => {
98 :and_report => {
88 :score => 105,
99 :score => 105,
89 :comment => /^.*(-|x).*$/})
100 :comment => /^.*(-|x).*$/})
90 end
101 end
91
102
92 it "should fail submission with non-zero exit status" do
103 it "should fail submission with non-zero exit status" do
93 grader_should(:grade => "add_nonzero_exit_status.c",
104 grader_should(:grade => "add_nonzero_exit_status.c",
94 :on => @problem_test_normal,
105 :on => @problem_test_normal,
95 :and_report => {
106 :and_report => {
96 :score => 0,
107 :score => 0,
97 :comment => /^(\[|x|\])+$/})
108 :comment => /^(\[|x|\])+$/})
98 end
109 end
99
110
100 it "should not allow malicious submission to see PROBLEM_HOME" do
111 it "should not allow malicious submission to see PROBLEM_HOME" do
101 problem_test_yesno = stub(Problem,
112 problem_test_yesno = stub(Problem,
102 :id => 1, :name => 'test_yesno',
113 :id => 1, :name => 'test_yesno',
103 :full_score => 10)
114 :full_score => 10)
104 grader_should(:grade => "yesno_access_problem_home.c",
115 grader_should(:grade => "yesno_access_problem_home.c",
105 :on => problem_test_yesno,
116 :on => problem_test_yesno,
106 :and_report => {
117 :and_report => {
107 :score => 0,
118 :score => 0,
108 :comment => /(-|x)/})
119 :comment => /(-|x)/})
109 end
120 end
110
121
111 it "should not allow malicious submission to open files" do
122 it "should not allow malicious submission to open files" do
112 problem_test_yesno = stub(Problem,
123 problem_test_yesno = stub(Problem,
113 :id => 1, :name => 'test_yesno',
124 :id => 1, :name => 'test_yesno',
114 :full_score => 10)
125 :full_score => 10)
115 grader_should(:grade => "yesno_open_file.c",
126 grader_should(:grade => "yesno_open_file.c",
116 :on => problem_test_yesno,
127 :on => problem_test_yesno,
117 :and_report => {
128 :and_report => {
118 :score => 0,
129 :score => 0,
119 :comment => /(-|x)/})
130 :comment => /(-|x)/})
120 end
131 end
121
132
122 def grader_should(args)
133 def grader_should(args)
123 @user1 = stub(User,
134 @user1 = stub(User,
124 :id => 1, :login => 'user1')
135 :id => 1, :login => 'user1')
136 +
125 submission =
137 submission =
126 - create_submission_from_file(1, @user1, args[:on], args[:grade])
138 + create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language])
127 submission.should_receive(:graded_at=)
139 submission.should_receive(:graded_at=)
128
140
129 expected_score = args[:and_report][:score]
141 expected_score = args[:and_report][:score]
130 expected_comment = args[:and_report][:comment]
142 expected_comment = args[:and_report][:comment]
131 if args[:and_report][:compiler_message]!=nil
143 if args[:and_report][:compiler_message]!=nil
132 expected_compiler_message = args[:and_report][:compiler_message]
144 expected_compiler_message = args[:and_report][:compiler_message]
133 else
145 else
134 expected_compiler_message = ''
146 expected_compiler_message = ''
135 end
147 end
136
148
137 submission.should_receive(:points=).with(expected_score)
149 submission.should_receive(:points=).with(expected_score)
138 submission.should_receive(:grader_comment=).with(expected_comment)
150 submission.should_receive(:grader_comment=).with(expected_comment)
139 submission.should_receive(:compiler_message=).with(expected_compiler_message)
151 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 submission.should_receive(:save)
152 submission.should_receive(:save)
141
153
142 @engine.grade(submission)
154 @engine.grade(submission)
143 end
155 end
144
156
145 protected
157 protected
146
158
147 def create_normal_submission_mock_from_file(source_fname)
159 def create_normal_submission_mock_from_file(source_fname)
148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
160 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 end
161 end
150
162
151 end
163 end
152
164
153 describe "A grader engine, when grading test requests" do
165 describe "A grader engine, when grading test requests" do
154
166
155 include GraderEngineHelperMethods
167 include GraderEngineHelperMethods
156
168
157 before(:each) do
169 before(:each) do
158 @config = Grader::Configuration.get_instance
170 @config = Grader::Configuration.get_instance
159 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
171 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
160 :reporter => Grader::TestRequestReporter.new)
172 :reporter => Grader::TestRequestReporter.new)
161 init_sandbox
173 init_sandbox
162 end
174 end
163
175
164 it "should report error if there is no problem template" do
176 it "should report error if there is no problem template" do
165 problem = stub(Problem,
177 problem = stub(Problem,
166 :id => 1, :name => 'nothing')
178 :id => 1, :name => 'nothing')
167 grader_should(:grade => 'test1_correct.c',
179 grader_should(:grade => 'test1_correct.c',
168 :on => problem,
180 :on => problem,
169 :with => 'in1.txt',
181 :with => 'in1.txt',
170 :and_report => {
182 :and_report => {
171 :graded_at= => nil,
183 :graded_at= => nil,
172 :compiler_message= => '',
184 :compiler_message= => '',
173 :grader_comment= => '',
185 :grader_comment= => '',
174 :running_stat= => /template not found/,
186 :running_stat= => /template not found/,
175 :running_time= => nil,
187 :running_time= => nil,
176 :exit_status= => nil,
188 :exit_status= => nil,
177 :memory_usage= => nil,
189 :memory_usage= => nil,
178 :save => nil})
190 :save => nil})
179 end
191 end
180
192
181 it "should run test request and produce output file" do
193 it "should run test request and produce output file" do
182 problem = stub(Problem,
194 problem = stub(Problem,
183 :id => 1, :name => 'test_normal')
195 :id => 1, :name => 'test_normal')
184 grader_should(:grade => 'test1_correct.c',
196 grader_should(:grade => 'test1_correct.c',
185 :on => problem,
197 :on => problem,
186 :with => 'in1.txt',
198 :with => 'in1.txt',
187 :and_report => {
199 :and_report => {
188 :graded_at= => nil,
200 :graded_at= => nil,
189 :compiler_message= => '',
201 :compiler_message= => '',
190 :grader_comment= => '',
202 :grader_comment= => '',
191 :running_stat= => /0.0\d* sec./,
203 :running_stat= => /0.0\d* sec./,
192 :output_file_name= => lambda { |fname|
204 :output_file_name= => lambda { |fname|
193 File.exists?(fname).should be_true
205 File.exists?(fname).should be_true
194 },
206 },
195 :running_time= => nil,
207 :running_time= => nil,
196 :exit_status= => nil,
208 :exit_status= => nil,
197 :memory_usage= => nil,
209 :memory_usage= => nil,
198 :save => nil})
210 :save => nil})
199 end
211 end
200
212
201 it "should clean up problem directory after running test request" do
213 it "should clean up problem directory after running test request" do
202 problem = stub(Problem,
214 problem = stub(Problem,
203 :id => 1, :name => 'test_normal')
215 :id => 1, :name => 'test_normal')
204 grader_should(:grade => 'test1_correct.c',
216 grader_should(:grade => 'test1_correct.c',
205 :on => problem,
217 :on => problem,
206 :with => 'in1.txt',
218 :with => 'in1.txt',
207 :and_report => {
219 :and_report => {
208 :graded_at= => nil,
220 :graded_at= => nil,
209 :compiler_message= => '',
221 :compiler_message= => '',
210 :grader_comment= => '',
222 :grader_comment= => '',
211 :running_stat= => nil,
223 :running_stat= => nil,
212 :output_file_name= => nil,
224 :output_file_name= => nil,
213 :running_time= => nil,
225 :running_time= => nil,
214 :exit_status= => nil,
226 :exit_status= => nil,
215 :memory_usage= => nil,
227 :memory_usage= => nil,
216 :save => nil})
228 :save => nil})
217 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
229 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
218 end
230 end
219
231
220 it "should compile test request with error and report compilation error" do
232 it "should compile test request with error and report compilation error" do
221 problem = stub(Problem,
233 problem = stub(Problem,
222 :id => 1, :name => 'test_normal')
234 :id => 1, :name => 'test_normal')
223 grader_should(:grade => 'test1_compile_error.c',
235 grader_should(:grade => 'test1_compile_error.c',
224 :on => problem,
236 :on => problem,
225 :with => 'in1.txt',
237 :with => 'in1.txt',
226 :and_report => {
238 :and_report => {
227 :graded_at= => nil,
239 :graded_at= => nil,
228 :compiler_message= => /.+/,
240 :compiler_message= => /.+/,
229 :grader_comment= => /[Cc]ompil.*error/,
241 :grader_comment= => /[Cc]ompil.*error/,
230 :running_stat= => '',
242 :running_stat= => '',
231 :save => nil})
243 :save => nil})
232 end
244 end
233
245
234 it "should report exit status" do
246 it "should report exit status" do
235 problem = stub(Problem,
247 problem = stub(Problem,
236 :id => 1, :name => 'test_normal')
248 :id => 1, :name => 'test_normal')
237 grader_should(:grade => 'add_nonzero_exit_status.c',
249 grader_should(:grade => 'add_nonzero_exit_status.c',
238 :on => problem,
250 :on => problem,
239 :with => 'in1.txt',
251 :with => 'in1.txt',
240 :and_report => {
252 :and_report => {
241 :graded_at= => nil,
253 :graded_at= => nil,
242 :compiler_message= => '',
254 :compiler_message= => '',
243 :grader_comment= => '',
255 :grader_comment= => '',
244 :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
256 :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
245 :output_file_name= => lambda { |fname|
257 :output_file_name= => lambda { |fname|
246 File.exists?(fname).should be_true
258 File.exists?(fname).should be_true
247 },
259 },
248 :running_time= => nil,
260 :running_time= => nil,
249 :exit_status= => /10/,
261 :exit_status= => /10/,
250 :memory_usage= => nil,
262 :memory_usage= => nil,
251 :save => nil})
263 :save => nil})
252 end
264 end
253
265
254 it "should produce running statistics for normal submission" do
266 it "should produce running statistics for normal submission" do
255 problem = stub(Problem,
267 problem = stub(Problem,
256 :id => 1, :name => 'test_normal')
268 :id => 1, :name => 'test_normal')
257 grader_should(:grade => 'test_run_stat.c',
269 grader_should(:grade => 'test_run_stat.c',
258 :on => problem,
270 :on => problem,
259 :with => 'in1.txt',
271 :with => 'in1.txt',
260 :and_report => {
272 :and_report => {
261 :graded_at= => nil,
273 :graded_at= => nil,
262 :compiler_message= => '',
274 :compiler_message= => '',
263 :grader_comment= => '',
275 :grader_comment= => '',
264 :running_stat= => nil,
276 :running_stat= => nil,
265 :output_file_name= => lambda { |fname|
277 :output_file_name= => lambda { |fname|
266 File.exists?(fname).should be_true
278 File.exists?(fname).should be_true
267 },
279 },
268 :running_time= => lambda { |t|
280 :running_time= => lambda { |t|
269 (t>=0.14) and (t<=0.16)
281 (t>=0.14) and (t<=0.16)
270 },
282 },
271 :exit_status= => nil,
283 :exit_status= => nil,
272 :memory_usage= => lambda { |s|
284 :memory_usage= => lambda { |s|
273 (s>=500) and (s<=1000)
285 (s>=500) and (s<=1000)
274 },
286 },
275 :save => nil})
287 :save => nil})
276 end
288 end
277
289
278 protected
290 protected
279 def grader_should(args)
291 def grader_should(args)
280 @user1 = stub(User,
292 @user1 = stub(User,
281 :id => 1, :login => 'user1')
293 :id => 1, :login => 'user1')
282
294
283 problem = args[:on]
295 problem = args[:on]
284 input_file = @config.test_request_input_base_dir + "/" + args[:with]
296 input_file = @config.test_request_input_base_dir + "/" + args[:with]
285
297
286 submission =
298 submission =
287 create_submission_from_file(1, @user1, args[:on], args[:grade])
299 create_submission_from_file(1, @user1, args[:on], args[:grade])
288
300
289 test_request = stub(TestRequest,
301 test_request = stub(TestRequest,
290 :id => 1,
302 :id => 1,
291 :user => @user1,
303 :user => @user1,
292 :problem => problem,
304 :problem => problem,
293 :submission => submission,
305 :submission => submission,
294 :input_file_name => input_file,
306 :input_file_name => input_file,
295 :language => submission.language,
307 :language => submission.language,
296 :problem_name => problem.name)
308 :problem_name => problem.name)
297
309
298 expectations = args[:and_report]
310 expectations = args[:and_report]
299
311
300 expectations.each do |key,val|
312 expectations.each do |key,val|
301 if val==nil
313 if val==nil
302 test_request.should_receive(key)
314 test_request.should_receive(key)
303 elsif val.class == Proc
315 elsif val.class == Proc
304 test_request.should_receive(key) { |fname|
316 test_request.should_receive(key) { |fname|
305 val.call(fname)
317 val.call(fname)
306 }
318 }
307 else
319 else
308 test_request.should_receive(key).with(val)
320 test_request.should_receive(key).with(val)
309 end
321 end
310 end
322 end
311
323
312 @engine.grade(test_request)
324 @engine.grade(test_request)
313 end
325 end
314
326
315 end
327 end
316
328
You need to be logged in to leave comments. Login now