cafe-grader-judge-scripts

Commit r205:825a5364bee6 public

parent | child
Description:
Updated `be_true` and `be_false` to `be true` and `be false` respectively
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r205:825a5364bee6 - - 1 file changed: 4 inserted, 4 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,328 +1,328
1 1 require File.join(File.dirname(__FILE__),'spec_helper')
2 2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3 3
4 4 describe "A grader engine, when grading submissions" do
5 5
6 6 include GraderEngineHelperMethods
7 7
8 8 before(:each) do
9 9 @config = Grader::Configuration.get_instance
10 10
11 11 # this test is from Pong
12 12 @problem_test_normal = stub(Problem,
13 13 :id => 1, :name => 'test_normal',
14 14 :full_score => 135)
15 15 @user_user1 = stub(User,
16 16 :id => 1, :login => 'user1')
17 17
18 18 @engine = Grader::Engine.new
19 19 init_sandbox
20 20 end
21 21
22 22 it "should grade normal submission" do
23 23 grader_should(:grade => "test1_correct.c",
24 24 :on => @problem_test_normal,
25 25 :and_report => {
26 26 :score => 135,
27 27 :comment => /^(\[|P|\])+/})
28 28 end
29 29
30 30
31 31 it "should grade normal submission in C++" do
32 32 cpp_lang = stub(Language, :name => 'cpp', :ext => 'cpp')
33 33
34 34 grader_should(:grade => "test1_correct_cc.cc",
35 35 :language => cpp_lang,
36 36 :on => @problem_test_normal,
37 37 :and_report => {
38 38 :score => 135,
39 39 :comment => /^(\[|P|\])+/})
40 40 end
41 41
42 42 it "should produce error message when submission cannot compile" do
43 43 grader_should(:grade => "test1_compile_error.c",
44 44 :on => @problem_test_normal,
45 45 :and_report => {
46 46 :score => 0,
47 47 :comment => 'compilation error',
48 48 :compiler_message => /[Ee]rror/})
49 49 end
50 50
51 51 it "should produce timeout error when submission runs forever" do
52 52 @problem_test_timeout = stub(Problem,
53 53 :id => 1, :name => 'test_timeout',
54 54 :full_score => 10)
55 55 grader_should(:grade => "test2_timeout.c",
56 56 :on => @problem_test_timeout,
57 57 :and_report => {
58 58 :score => 0,
59 59 :comment => 'TT'})
60 60 end
61 61
62 62 it "should produce timeout error correctly with fractional running time and fractional time limits" do
63 63 @problem_test_timeout = stub(Problem,
64 64 :id => 1, :name => 'test_timeout',
65 65 :full_score => 20)
66 66 grader_should(:grade => "test2_05sec.c",
67 67 :on => @problem_test_timeout,
68 68 :and_report => {
69 69 :score => 10,
70 70 :comment => 'TP'})
71 71 end
72 72
73 73 it "should produce runtime error when submission uses too much static memory" do
74 74 @problem_test_memory = stub(Problem,
75 75 :id => 1, :name => 'test_memory',
76 76 :full_score => 20)
77 77 grader_should(:grade => "add_too_much_memory_static.c",
78 78 :on => @problem_test_memory,
79 79 :and_report => {
80 80 :score => 10,
81 81 :comment => /[^P]P/})
82 82 end
83 83
84 84 it "should not allow submission to allocate too much dynamic memory" do
85 85 @problem_test_memory = stub(Problem,
86 86 :id => 1, :name => 'test_memory',
87 87 :full_score => 20)
88 88 grader_should(:grade => "add_too_much_memory_dynamic.c",
89 89 :on => @problem_test_memory,
90 90 :and_report => {
91 91 :score => 10,
92 92 :comment => /[^P]P/})
93 93 end
94 94
95 95 it "should score test runs correctly when submission fails in some test case" do
96 96 grader_should(:grade => "add_fail_test_case_1.c",
97 97 :on => @problem_test_normal,
98 98 :and_report => {
99 99 :score => 105,
100 100 :comment => /^.*(-|x).*$/})
101 101 end
102 102
103 103 it "should fail submission with non-zero exit status" do
104 104 grader_should(:grade => "add_nonzero_exit_status.c",
105 105 :on => @problem_test_normal,
106 106 :and_report => {
107 107 :score => 0,
108 108 :comment => /^(\[|x|\])+$/})
109 109 end
110 110
111 111 it "should not allow malicious submission to see PROBLEM_HOME" do
112 112 problem_test_yesno = stub(Problem,
113 113 :id => 1, :name => 'test_yesno',
114 114 :full_score => 10)
115 115 grader_should(:grade => "yesno_access_problem_home.c",
116 116 :on => problem_test_yesno,
117 117 :and_report => {
118 118 :score => 0,
119 119 :comment => /(-|x)/})
120 120 end
121 121
122 122 it "should not allow malicious submission to open files" do
123 123 problem_test_yesno = stub(Problem,
124 124 :id => 1, :name => 'test_yesno',
125 125 :full_score => 10)
126 126 grader_should(:grade => "yesno_open_file.c",
127 127 :on => problem_test_yesno,
128 128 :and_report => {
129 129 :score => 0,
130 130 :comment => /(-|x)/})
131 131 end
132 132
133 133 def grader_should(args)
134 134 @user1 = stub(User,
135 135 :id => 1, :login => 'user1')
136 136
137 137 submission =
138 138 create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language])
139 139 submission.should_receive(:graded_at=)
140 140
141 141 expected_score = args[:and_report][:score]
142 142 expected_comment = args[:and_report][:comment]
143 143 if args[:and_report][:compiler_message]!=nil
144 144 expected_compiler_message = args[:and_report][:compiler_message]
145 145 else
146 146 expected_compiler_message = ''
147 147 end
148 148
149 149 submission.should_receive(:points=).with(expected_score)
150 150 submission.should_receive(:grader_comment=).with(expected_comment)
151 151 submission.should_receive(:compiler_message=).with(expected_compiler_message)
152 152 submission.should_receive(:save)
153 153
154 154 @engine.grade(submission)
155 155 end
156 156
157 157 protected
158 158
159 159 def create_normal_submission_mock_from_file(source_fname)
160 160 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
161 161 end
162 162
163 163 end
164 164
165 165 describe "A grader engine, when grading test requests" do
166 166
167 167 include GraderEngineHelperMethods
168 168
169 169 before(:each) do
170 170 @config = Grader::Configuration.get_instance
171 171 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
172 172 :reporter => Grader::TestRequestReporter.new)
173 173 init_sandbox
174 174 end
175 175
176 176 it "should report error if there is no problem template" do
177 177 problem = stub(Problem,
178 178 :id => 1, :name => 'nothing')
179 179 grader_should(:grade => 'test1_correct.c',
180 180 :on => problem,
181 181 :with => 'in1.txt',
182 182 :and_report => {
183 183 :graded_at= => nil,
184 184 :compiler_message= => '',
185 185 :grader_comment= => '',
186 186 :running_stat= => /template not found/,
187 187 :running_time= => nil,
188 188 :exit_status= => nil,
189 189 :memory_usage= => nil,
190 190 :save => nil})
191 191 end
192 192
193 193 it "should run test request and produce output file" do
194 194 problem = stub(Problem,
195 195 :id => 1, :name => 'test_normal')
196 196 grader_should(:grade => 'test1_correct.c',
197 197 :on => problem,
198 198 :with => 'in1.txt',
199 199 :and_report => {
200 200 :graded_at= => nil,
201 201 :compiler_message= => '',
202 202 :grader_comment= => '',
203 203 :running_stat= => /0.0\d* sec./,
204 204 :output_file_name= => lambda { |fname|
205 - File.exists?(fname).should be_true
205 + File.exists?(fname).should be true
206 206 },
207 207 :running_time= => nil,
208 208 :exit_status= => nil,
209 209 :memory_usage= => nil,
210 210 :save => nil})
211 211 end
212 212
213 213 it "should clean up problem directory after running test request" do
214 214 problem = stub(Problem,
215 215 :id => 1, :name => 'test_normal')
216 216 grader_should(:grade => 'test1_correct.c',
217 217 :on => problem,
218 218 :with => 'in1.txt',
219 219 :and_report => {
220 220 :graded_at= => nil,
221 221 :compiler_message= => '',
222 222 :grader_comment= => '',
223 223 :running_stat= => nil,
224 224 :output_file_name= => nil,
225 225 :running_time= => nil,
226 226 :exit_status= => nil,
227 227 :memory_usage= => nil,
228 228 :save => nil})
229 - File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
229 + File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be false
230 230 end
231 231
232 232 it "should compile test request with error and report compilation error" do
233 233 problem = stub(Problem,
234 234 :id => 1, :name => 'test_normal')
235 235 grader_should(:grade => 'test1_compile_error.c',
236 236 :on => problem,
237 237 :with => 'in1.txt',
238 238 :and_report => {
239 239 :graded_at= => nil,
240 240 :compiler_message= => /.+/,
241 241 :grader_comment= => /[Cc]ompil.*error/,
242 242 :running_stat= => '',
243 243 :save => nil})
244 244 end
245 245
246 246 it "should report exit status" do
247 247 problem = stub(Problem,
248 248 :id => 1, :name => 'test_normal')
249 249 grader_should(:grade => 'add_nonzero_exit_status.c',
250 250 :on => problem,
251 251 :with => 'in1.txt',
252 252 :and_report => {
253 253 :graded_at= => nil,
254 254 :compiler_message= => '',
255 255 :grader_comment= => '',
256 256 :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
257 257 :output_file_name= => lambda { |fname|
258 - File.exists?(fname).should be_true
258 + File.exists?(fname).should be true
259 259 },
260 260 :running_time= => nil,
261 261 :exit_status= => /10/,
262 262 :memory_usage= => nil,
263 263 :save => nil})
264 264 end
265 265
266 266 it "should produce running statistics for normal submission" do
267 267 problem = stub(Problem,
268 268 :id => 1, :name => 'test_normal')
269 269 grader_should(:grade => 'test_run_stat.c',
270 270 :on => problem,
271 271 :with => 'in1.txt',
272 272 :and_report => {
273 273 :graded_at= => nil,
274 274 :compiler_message= => '',
275 275 :grader_comment= => '',
276 276 :running_stat= => nil,
277 277 :output_file_name= => lambda { |fname|
278 - File.exists?(fname).should be_true
278 + File.exists?(fname).should be true
279 279 },
280 280 :running_time= => lambda { |t|
281 281 (t>=0.14) and (t<=0.16)
282 282 },
283 283 :exit_status= => nil,
284 284 :memory_usage= => lambda { |s|
285 285 (s>=500) and (s<=1000)
286 286 },
287 287 :save => nil})
288 288 end
289 289
290 290 protected
291 291 def grader_should(args)
292 292 @user1 = stub(User,
293 293 :id => 1, :login => 'user1')
294 294
295 295 problem = args[:on]
296 296 input_file = @config.test_request_input_base_dir + "/" + args[:with]
297 297
298 298 submission =
299 299 create_submission_from_file(1, @user1, args[:on], args[:grade])
300 300
301 301 test_request = stub(TestRequest,
302 302 :id => 1,
303 303 :user => @user1,
304 304 :problem => problem,
305 305 :submission => submission,
306 306 :input_file_name => input_file,
307 307 :language => submission.language,
308 308 :problem_name => problem.name)
309 309
310 310 expectations = args[:and_report]
311 311
312 312 expectations.each do |key,val|
313 313 if val==nil
314 314 test_request.should_receive(key)
315 315 elsif val.class == Proc
316 316 test_request.should_receive(key) { |fname|
317 317 val.call(fname)
318 318 }
319 319 else
320 320 test_request.should_receive(key).with(val)
321 321 end
322 322 end
323 323
324 324 @engine.grade(test_request)
325 325 end
326 326
327 327 end
328 328
You need to be logged in to leave comments. Login now