cafe-grader-judge-scripts

Commit r126:be832795a805 public

parent | child
Description:
Merge branch 'master' into windows
Commit status:
[Not Reviewed]
References:
merge default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r126:be832795a805 - - 2 files changed: 26 inserted, 1 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -0,0 +1,13
1 + /*
2 + LANG: C++
3 + */
4 + #include <cstdio>
5 +
6 + int main()
7 + {
8 + int a,b,r;
9 + r = scanf("%d %d",&a,&b);
10 + printf("%d\n",a+b);
11 + return 0;
12 + }
13 +
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,316 +1,328
1 1 require File.join(File.dirname(__FILE__),'spec_helper')
2 2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3 3
4 4 describe "A grader engine, when grading submissions" do
5 5
6 6 include GraderEngineHelperMethods
7 7
8 8 before(:each) do
9 9 @config = Grader::Configuration.get_instance
10 10
11 11 # this test is from Pong
12 12 @problem_test_normal = stub(Problem,
13 13 :id => 1, :name => 'test_normal',
14 14 :full_score => 135)
15 15 @user_user1 = stub(User,
16 16 :id => 1, :login => 'user1')
17 17
18 18 @engine = Grader::Engine.new
19 19 init_sandbox
20 20 end
21 21
22 22 it "should grade normal submission" do
23 23 grader_should(:grade => "test1_correct.c",
24 24 :on => @problem_test_normal,
25 25 :and_report => {
26 26 :score => 135,
27 27 :comment => /^(\[|P|\])+/})
28 28 end
29 29
30 30
31 + it "should grade normal submission in C++" do
32 + cpp_lang = stub(Language, :name => 'cpp', :ext => 'cpp')
33 +
34 + grader_should(:grade => "test1_correct_cc.cc",
35 + :language => cpp_lang,
36 + :on => @problem_test_normal,
37 + :and_report => {
38 + :score => 135,
39 + :comment => /^(\[|P|\])+/})
40 + end
41 +
31 42 it "should produce error message when submission cannot compile" do
32 43 grader_should(:grade => "test1_compile_error.c",
33 44 :on => @problem_test_normal,
34 45 :and_report => {
35 46 :score => 0,
36 47 :comment => 'compilation error',
37 48 :compiler_message => /[Ee]rror/})
38 49 end
39 50
40 51 it "should produce timeout error when submission runs forever" do
41 52 @problem_test_timeout = stub(Problem,
42 53 :id => 1, :name => 'test_timeout',
43 54 :full_score => 10)
44 55 grader_should(:grade => "test2_timeout.c",
45 56 :on => @problem_test_timeout,
46 57 :and_report => {
47 58 :score => 0,
48 59 :comment => 'TT'})
49 60 end
50 61
51 62 it "should produce timeout error correctly with fractional running time and fractional time limits" do
52 63 @problem_test_timeout = stub(Problem,
53 64 :id => 1, :name => 'test_timeout',
54 65 :full_score => 20)
55 66 grader_should(:grade => "test2_05sec.c",
56 67 :on => @problem_test_timeout,
57 68 :and_report => {
58 69 :score => 10,
59 70 :comment => 'TP'})
60 71 end
61 72
62 73 it "should produce runtime error when submission uses too much static memory" do
63 74 @problem_test_memory = stub(Problem,
64 75 :id => 1, :name => 'test_memory',
65 76 :full_score => 20)
66 77 grader_should(:grade => "add_too_much_memory_static.c",
67 78 :on => @problem_test_memory,
68 79 :and_report => {
69 80 :score => 10,
70 81 :comment => /[^P]P/})
71 82 end
72 83
73 84 it "should not allow submission to allocate too much dynamic memory" do
74 85 @problem_test_memory = stub(Problem,
75 86 :id => 1, :name => 'test_memory',
76 87 :full_score => 20)
77 88 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 89 :on => @problem_test_memory,
79 90 :and_report => {
80 91 :score => 10,
81 92 :comment => /[^P]P/})
82 93 end
83 94
84 95 it "should score test runs correctly when submission fails in some test case" do
85 96 grader_should(:grade => "add_fail_test_case_1.c",
86 97 :on => @problem_test_normal,
87 98 :and_report => {
88 99 :score => 105,
89 100 :comment => /^.*(-|x).*$/})
90 101 end
91 102
92 103 it "should fail submission with non-zero exit status" do
93 104 grader_should(:grade => "add_nonzero_exit_status.c",
94 105 :on => @problem_test_normal,
95 106 :and_report => {
96 107 :score => 0,
97 108 :comment => /^(\[|x|\])+$/})
98 109 end
99 110
100 111 it "should not allow malicious submission to see PROBLEM_HOME" do
101 112 problem_test_yesno = stub(Problem,
102 113 :id => 1, :name => 'test_yesno',
103 114 :full_score => 10)
104 115 grader_should(:grade => "yesno_access_problem_home.c",
105 116 :on => problem_test_yesno,
106 117 :and_report => {
107 118 :score => 0,
108 119 :comment => /(-|x)/})
109 120 end
110 121
111 122 it "should not allow malicious submission to open files" do
112 123 problem_test_yesno = stub(Problem,
113 124 :id => 1, :name => 'test_yesno',
114 125 :full_score => 10)
115 126 grader_should(:grade => "yesno_open_file.c",
116 127 :on => problem_test_yesno,
117 128 :and_report => {
118 129 :score => 0,
119 130 :comment => /(-|x)/})
120 131 end
121 132
122 133 def grader_should(args)
123 134 @user1 = stub(User,
124 135 :id => 1, :login => 'user1')
136 +
125 137 submission =
126 - create_submission_from_file(1, @user1, args[:on], args[:grade])
138 + create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language])
127 139 submission.should_receive(:graded_at=)
128 140
129 141 expected_score = args[:and_report][:score]
130 142 expected_comment = args[:and_report][:comment]
131 143 if args[:and_report][:compiler_message]!=nil
132 144 expected_compiler_message = args[:and_report][:compiler_message]
133 145 else
134 146 expected_compiler_message = ''
135 147 end
136 148
137 149 submission.should_receive(:points=).with(expected_score)
138 150 submission.should_receive(:grader_comment=).with(expected_comment)
139 151 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 152 submission.should_receive(:save)
141 153
142 154 @engine.grade(submission)
143 155 end
144 156
145 157 protected
146 158
147 159 def create_normal_submission_mock_from_file(source_fname)
148 160 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 161 end
150 162
151 163 end
152 164
153 165 describe "A grader engine, when grading test requests" do
154 166
155 167 include GraderEngineHelperMethods
156 168
157 169 before(:each) do
158 170 @config = Grader::Configuration.get_instance
159 171 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
160 172 :reporter => Grader::TestRequestReporter.new)
161 173 init_sandbox
162 174 end
163 175
164 176 it "should report error if there is no problem template" do
165 177 problem = stub(Problem,
166 178 :id => 1, :name => 'nothing')
167 179 grader_should(:grade => 'test1_correct.c',
168 180 :on => problem,
169 181 :with => 'in1.txt',
170 182 :and_report => {
171 183 :graded_at= => nil,
172 184 :compiler_message= => '',
173 185 :grader_comment= => '',
174 186 :running_stat= => /template not found/,
175 187 :running_time= => nil,
176 188 :exit_status= => nil,
177 189 :memory_usage= => nil,
178 190 :save => nil})
179 191 end
180 192
181 193 it "should run test request and produce output file" do
182 194 problem = stub(Problem,
183 195 :id => 1, :name => 'test_normal')
184 196 grader_should(:grade => 'test1_correct.c',
185 197 :on => problem,
186 198 :with => 'in1.txt',
187 199 :and_report => {
188 200 :graded_at= => nil,
189 201 :compiler_message= => '',
190 202 :grader_comment= => '',
191 203 :running_stat= => /0.0\d* sec./,
192 204 :output_file_name= => lambda { |fname|
193 205 File.exists?(fname).should be_true
194 206 },
195 207 :running_time= => nil,
196 208 :exit_status= => nil,
197 209 :memory_usage= => nil,
198 210 :save => nil})
199 211 end
200 212
201 213 it "should clean up problem directory after running test request" do
202 214 problem = stub(Problem,
203 215 :id => 1, :name => 'test_normal')
204 216 grader_should(:grade => 'test1_correct.c',
205 217 :on => problem,
206 218 :with => 'in1.txt',
207 219 :and_report => {
208 220 :graded_at= => nil,
209 221 :compiler_message= => '',
210 222 :grader_comment= => '',
211 223 :running_stat= => nil,
212 224 :output_file_name= => nil,
213 225 :running_time= => nil,
214 226 :exit_status= => nil,
215 227 :memory_usage= => nil,
216 228 :save => nil})
217 229 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
218 230 end
219 231
220 232 it "should compile test request with error and report compilation error" do
221 233 problem = stub(Problem,
222 234 :id => 1, :name => 'test_normal')
223 235 grader_should(:grade => 'test1_compile_error.c',
224 236 :on => problem,
225 237 :with => 'in1.txt',
226 238 :and_report => {
227 239 :graded_at= => nil,
228 240 :compiler_message= => /.+/,
229 241 :grader_comment= => /[Cc]ompil.*error/,
230 242 :running_stat= => '',
231 243 :save => nil})
232 244 end
233 245
234 246 it "should report exit status" do
235 247 problem = stub(Problem,
236 248 :id => 1, :name => 'test_normal')
237 249 grader_should(:grade => 'add_nonzero_exit_status.c',
238 250 :on => problem,
239 251 :with => 'in1.txt',
240 252 :and_report => {
241 253 :graded_at= => nil,
242 254 :compiler_message= => '',
243 255 :grader_comment= => '',
244 256 :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
245 257 :output_file_name= => lambda { |fname|
246 258 File.exists?(fname).should be_true
247 259 },
248 260 :running_time= => nil,
249 261 :exit_status= => /10/,
250 262 :memory_usage= => nil,
251 263 :save => nil})
252 264 end
253 265
254 266 it "should produce running statistics for normal submission" do
255 267 problem = stub(Problem,
256 268 :id => 1, :name => 'test_normal')
257 269 grader_should(:grade => 'test_run_stat.c',
258 270 :on => problem,
259 271 :with => 'in1.txt',
260 272 :and_report => {
261 273 :graded_at= => nil,
262 274 :compiler_message= => '',
263 275 :grader_comment= => '',
264 276 :running_stat= => nil,
265 277 :output_file_name= => lambda { |fname|
266 278 File.exists?(fname).should be_true
267 279 },
268 280 :running_time= => lambda { |t|
269 281 (t>=0.14) and (t<=0.16)
270 282 },
271 283 :exit_status= => nil,
272 284 :memory_usage= => lambda { |s|
273 285 (s>=500) and (s<=1000)
274 286 },
275 287 :save => nil})
276 288 end
277 289
278 290 protected
279 291 def grader_should(args)
280 292 @user1 = stub(User,
281 293 :id => 1, :login => 'user1')
282 294
283 295 problem = args[:on]
284 296 input_file = @config.test_request_input_base_dir + "/" + args[:with]
285 297
286 298 submission =
287 299 create_submission_from_file(1, @user1, args[:on], args[:grade])
288 300
289 301 test_request = stub(TestRequest,
290 302 :id => 1,
291 303 :user => @user1,
292 304 :problem => problem,
293 305 :submission => submission,
294 306 :input_file_name => input_file,
295 307 :language => submission.language,
296 308 :problem_name => problem.name)
297 309
298 310 expectations = args[:and_report]
299 311
300 312 expectations.each do |key,val|
301 313 if val==nil
302 314 test_request.should_receive(key)
303 315 elsif val.class == Proc
304 316 test_request.should_receive(key) { |fname|
305 317 val.call(fname)
306 318 }
307 319 else
308 320 test_request.should_receive(key).with(val)
309 321 end
310 322 end
311 323
312 324 @engine.grade(test_request)
313 325 end
314 326
315 327 end
316 328
You need to be logged in to leave comments. Login now