cafe-grader-judge-scripts

Commit r44:a07593cb07c4 public

parent | child
Description:
[grader] small tweak on one test case git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@168 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r44:a07593cb07c4 - - 1 file changed: 1 inserted, 1 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,316 +1,316
1 1 require File.join(File.dirname(__FILE__),'spec_helper')
2 2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3 3
4 4 describe "A grader engine, when grading submissions" do
5 5
6 6 include GraderEngineHelperMethods
7 7
8 8 before(:each) do
9 9 @config = Grader::Configuration.get_instance
10 10
11 11 # this test is from Pong
12 12 @problem_test_normal = stub(Problem,
13 13 :id => 1, :name => 'test_normal',
14 14 :full_score => 135)
15 15 @user_user1 = stub(User,
16 16 :id => 1, :login => 'user1')
17 17
18 18 @engine = Grader::Engine.new
19 19 init_sandbox
20 20 end
21 21
22 22 it "should grade normal submission" do
23 23 grader_should(:grade => "test1_correct.c",
24 24 :on => @problem_test_normal,
25 25 :and_report => {
26 26 :score => 135,
27 27 :comment => /^PASSED/})
28 28 end
29 29
30 30
31 31 it "should produce error message when submission cannot compile" do
32 32 grader_should(:grade => "test1_compile_error.c",
33 33 :on => @problem_test_normal,
34 34 :and_report => {
35 35 :score => 0,
36 36 :comment => 'FAILED: compilation error',
37 37 :compiler_message => /[Ee]rror/})
38 38 end
39 39
40 40 it "should produce timeout error when submission runs forever" do
41 41 @problem_test_timeout = stub(Problem,
42 42 :id => 1, :name => 'test_timeout',
43 43 :full_score => 10)
44 44 grader_should(:grade => "test2_timeout.c",
45 45 :on => @problem_test_timeout,
46 46 :and_report => {
47 47 :score => 0,
48 48 :comment => 'FAILED: TT'})
49 49 end
50 50
51 51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
52 52 @problem_test_timeout = stub(Problem,
53 53 :id => 1, :name => 'test_timeout',
54 54 :full_score => 20)
55 55 grader_should(:grade => "test2_1-5sec.c",
56 56 :on => @problem_test_timeout,
57 57 :and_report => {
58 58 :score => 10,
59 59 :comment => 'FAILED: TP'})
60 60 end
61 61
62 62 it "should produce runtime error when submission uses too much static memory" do
63 63 @problem_test_memory = stub(Problem,
64 64 :id => 1, :name => 'test_memory',
65 65 :full_score => 20)
66 66 grader_should(:grade => "add_too_much_memory_static.c",
67 67 :on => @problem_test_memory,
68 68 :and_report => {
69 69 :score => 10,
70 70 :comment => /FAILED: [^P]P/})
71 71 end
72 72
73 73 it "should not allow submission to allocate too much dynamic memory" do
74 74 @problem_test_memory = stub(Problem,
75 75 :id => 1, :name => 'test_memory',
76 76 :full_score => 20)
77 77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 78 :on => @problem_test_memory,
79 79 :and_report => {
80 80 :score => 10,
81 81 :comment => /FAILED: [^P]P/})
82 82 end
83 83
84 84 it "should score test runs correctly when submission fails in some test case" do
85 85 grader_should(:grade => "add_fail_test_case_1.c",
86 86 :on => @problem_test_normal,
87 87 :and_report => {
88 88 :score => 105,
89 89 :comment => /^FAILED:/})
90 90 end
91 91
92 92 it "should fail submission with non-zero exit status" do
93 93 grader_should(:grade => "add_nonzero_exit_status.c",
94 94 :on => @problem_test_normal,
95 95 :and_report => {
96 96 :score => 0,
97 97 :comment => /^FAILED:/})
98 98 end
99 99
100 100 it "should not allow malicious submission to see PROBLEM_HOME" do
101 101 problem_test_yesno = stub(Problem,
102 102 :id => 1, :name => 'test_yesno',
103 103 :full_score => 10)
104 104 grader_should(:grade => "yesno_access_problem_home.c",
105 105 :on => problem_test_yesno,
106 106 :and_report => {
107 107 :score => 0,
108 108 :comment => /^FAILED:/})
109 109 end
110 110
111 111 it "should not allow malicious submission to open files" do
112 112 problem_test_yesno = stub(Problem,
113 113 :id => 1, :name => 'test_yesno',
114 114 :full_score => 10)
115 115 grader_should(:grade => "yesno_open_file.c",
116 116 :on => problem_test_yesno,
117 117 :and_report => {
118 118 :score => 0,
119 119 :comment => /^FAILED:/})
120 120 end
121 121
122 122 def grader_should(args)
123 123 @user1 = stub(User,
124 124 :id => 1, :login => 'user1')
125 125 submission =
126 126 create_submission_from_file(1, @user1, args[:on], args[:grade])
127 127 submission.should_receive(:graded_at=)
128 128
129 129 expected_score = args[:and_report][:score]
130 130 expected_comment = args[:and_report][:comment]
131 131 if args[:and_report][:compiler_message]!=nil
132 132 expected_compiler_message = args[:and_report][:compiler_message]
133 133 else
134 134 expected_compiler_message = ''
135 135 end
136 136
137 137 submission.should_receive(:points=).with(expected_score)
138 138 submission.should_receive(:grader_comment=).with(expected_comment)
139 139 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 140 submission.should_receive(:save)
141 141
142 142 @engine.grade(submission)
143 143 end
144 144
145 145 protected
146 146
147 147 def create_normal_submission_mock_from_file(source_fname)
148 148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 149 end
150 150
151 151 end
152 152
153 153 describe "A grader engine, when grading test requests" do
154 154
155 155 include GraderEngineHelperMethods
156 156
157 157 before(:each) do
158 158 @config = Grader::Configuration.get_instance
159 159 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
160 160 Grader::TestRequestReporter.new)
161 161 init_sandbox
162 162 end
163 163
164 164 it "should report error if there is no problem template" do
165 165 problem = stub(Problem,
166 166 :id => 1, :name => 'nothing')
167 167 grader_should(:grade => 'test1_correct.c',
168 168 :on => problem,
169 169 :with => 'in1.txt',
170 170 :and_report => {
171 171 :graded_at= => nil,
172 172 :compiler_message= => '',
173 173 :grader_comment= => '',
174 174 :running_stat= => /template not found/,
175 175 :running_time= => nil,
176 176 :exit_status= => nil,
177 177 :memory_usage= => nil,
178 178 :save => nil})
179 179 end
180 180
181 181 it "should run test request and produce output file" do
182 182 problem = stub(Problem,
183 183 :id => 1, :name => 'test_normal')
184 184 grader_should(:grade => 'test1_correct.c',
185 185 :on => problem,
186 186 :with => 'in1.txt',
187 187 :and_report => {
188 188 :graded_at= => nil,
189 189 :compiler_message= => '',
190 190 :grader_comment= => '',
191 - :running_stat= => /0.0 sec./,
191 + :running_stat= => /0.0\d* sec./,
192 192 :output_file_name= => lambda { |fname|
193 193 File.exists?(fname).should be_true
194 194 },
195 195 :running_time= => nil,
196 196 :exit_status= => nil,
197 197 :memory_usage= => nil,
198 198 :save => nil})
199 199 end
200 200
201 201 it "should clean up problem directory after running test request" do
202 202 problem = stub(Problem,
203 203 :id => 1, :name => 'test_normal')
204 204 grader_should(:grade => 'test1_correct.c',
205 205 :on => problem,
206 206 :with => 'in1.txt',
207 207 :and_report => {
208 208 :graded_at= => nil,
209 209 :compiler_message= => '',
210 210 :grader_comment= => '',
211 211 :running_stat= => nil,
212 212 :output_file_name= => nil,
213 213 :running_time= => nil,
214 214 :exit_status= => nil,
215 215 :memory_usage= => nil,
216 216 :save => nil})
217 217 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
218 218 end
219 219
220 220 it "should compile test request with error and report compilation error" do
221 221 problem = stub(Problem,
222 222 :id => 1, :name => 'test_normal')
223 223 grader_should(:grade => 'test1_compile_error.c',
224 224 :on => problem,
225 225 :with => 'in1.txt',
226 226 :and_report => {
227 227 :graded_at= => nil,
228 228 :compiler_message= => /.+/,
229 229 :grader_comment= => /[Cc]ompil.*error/,
230 230 :running_stat= => '',
231 231 :save => nil})
232 232 end
233 233
234 234 it "should report exit status" do
235 235 problem = stub(Problem,
236 236 :id => 1, :name => 'test_normal')
237 237 grader_should(:grade => 'add_nonzero_exit_status.c',
238 238 :on => problem,
239 239 :with => 'in1.txt',
240 240 :and_report => {
241 241 :graded_at= => nil,
242 242 :compiler_message= => '',
243 243 :grader_comment= => '',
244 244 :running_stat= => /[Ee]xit.*status.*10.*0\.0 sec/m,
245 245 :output_file_name= => lambda { |fname|
246 246 File.exists?(fname).should be_true
247 247 },
248 248 :running_time= => nil,
249 249 :exit_status= => /10/,
250 250 :memory_usage= => nil,
251 251 :save => nil})
252 252 end
253 253
254 254 it "should produce running statistics for normal submission" do
255 255 problem = stub(Problem,
256 256 :id => 1, :name => 'test_normal')
257 257 grader_should(:grade => 'test_run_stat.c',
258 258 :on => problem,
259 259 :with => 'in1.txt',
260 260 :and_report => {
261 261 :graded_at= => nil,
262 262 :compiler_message= => '',
263 263 :grader_comment= => '',
264 264 :running_stat= => nil,
265 265 :output_file_name= => lambda { |fname|
266 266 File.exists?(fname).should be_true
267 267 },
268 268 :running_time= => lambda { |t|
269 269 (t>=0.14) and (t<=0.16)
270 270 },
271 271 :exit_status= => nil,
272 272 :memory_usage= => lambda { |s|
273 273 (s>=500) and (s<=1000)
274 274 },
275 275 :save => nil})
276 276 end
277 277
278 278 protected
279 279 def grader_should(args)
280 280 @user1 = stub(User,
281 281 :id => 1, :login => 'user1')
282 282
283 283 problem = args[:on]
284 284 input_file = @config.test_request_input_base_dir + "/" + args[:with]
285 285
286 286 submission =
287 287 create_submission_from_file(1, @user1, args[:on], args[:grade])
288 288
289 289 test_request = stub(TestRequest,
290 290 :id => 1,
291 291 :user => @user1,
292 292 :problem => problem,
293 293 :submission => submission,
294 294 :input_file_name => input_file,
295 295 :language => submission.language,
296 296 :problem_name => problem.name)
297 297
298 298 expectations = args[:and_report]
299 299
300 300 expectations.each do |key,val|
301 301 if val==nil
302 302 test_request.should_receive(key)
303 303 elsif val.class == Proc
304 304 test_request.should_receive(key) { |fname|
305 305 val.call(fname)
306 306 }
307 307 else
308 308 test_request.should_receive(key).with(val)
309 309 end
310 310 end
311 311
312 312 @engine.grade(test_request)
313 313 end
314 314
315 315 end
316 316
You need to be logged in to leave comments. Login now