cafe-grader-judge-scripts

Commit r33:e7a6930e8c87 public

parent | child
Description:
[grader] finish engine spec on grading non-zero exit status git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@127 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r33:e7a6930e8c87 - - 1 file changed: 5 inserted, 1 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,254 +1,258
1 1 require File.join(File.dirname(__FILE__),'spec_helper')
2 2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3 3
4 4 describe "A grader engine, when grading submissions" do
5 5
6 6 include GraderEngineHelperMethods
7 7
8 8 before(:each) do
9 9 @config = Grader::Configuration.get_instance
10 10
11 11 # this test is from Pong
12 12 @problem_test_normal = stub(Problem,
13 13 :id => 1, :name => 'test_normal',
14 14 :full_score => 135)
15 15 @user_user1 = stub(User,
16 16 :id => 1, :login => 'user1')
17 17
18 18 @engine = Grader::Engine.new
19 19 init_sandbox
20 20 end
21 21
22 22 it "should grade normal submission" do
23 23 grader_should(:grade => "test1_correct.c",
24 24 :on => @problem_test_normal,
25 25 :and_report => {
26 26 :score => 135,
27 27 :comment => /^PASSED/})
28 28 end
29 29
30 30
31 31 it "should produce error message when submission cannot compile" do
32 32 grader_should(:grade => "test1_compile_error.c",
33 33 :on => @problem_test_normal,
34 34 :and_report => {
35 35 :score => 0,
36 36 :comment => 'FAILED: compile error',
37 37 :compiler_message => /[Ee]rror/})
38 38 end
39 39
40 40 it "should produce timeout error when submission runs forever" do
41 41 @problem_test_timeout = stub(Problem,
42 42 :id => 1, :name => 'test_timeout',
43 43 :full_score => 10)
44 44 grader_should(:grade => "test2_timeout.c",
45 45 :on => @problem_test_timeout,
46 46 :and_report => {
47 47 :score => 0,
48 48 :comment => 'FAILED: TT'})
49 49 end
50 50
51 51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
52 52 @problem_test_timeout = stub(Problem,
53 53 :id => 1, :name => 'test_timeout',
54 54 :full_score => 20)
55 55 grader_should(:grade => "test2_1-5sec.c",
56 56 :on => @problem_test_timeout,
57 57 :and_report => {
58 58 :score => 10,
59 59 :comment => 'FAILED: TP'})
60 60 end
61 61
62 62 it "should produce runtime error when submission uses too much static memory" do
63 63 @problem_test_memory = stub(Problem,
64 64 :id => 1, :name => 'test_memory',
65 65 :full_score => 20)
66 66 grader_should(:grade => "add_too_much_memory_static.c",
67 67 :on => @problem_test_memory,
68 68 :and_report => {
69 69 :score => 10,
70 70 :comment => /FAILED: [^P]P/})
71 71 end
72 72
73 73 it "should not allow submission to allocate too much dynamic memory" do
74 74 @problem_test_memory = stub(Problem,
75 75 :id => 1, :name => 'test_memory',
76 76 :full_score => 20)
77 77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 78 :on => @problem_test_memory,
79 79 :and_report => {
80 80 :score => 10,
81 81 :comment => /FAILED: [^P]P/})
82 82 end
83 83
84 84 it "should score test runs correctly when submission fails in some test case" do
85 85 grader_should(:grade => "add_fail_test_case_1.c",
86 86 :on => @problem_test_normal,
87 87 :and_report => {
88 88 :score => 105,
89 89 :comment => /^FAILED:/})
90 90 end
91 91
92 92 it "should fail submission with non-zero exit status" do
93 - violated("has not been implemented")
93 + grader_should(:grade => "add_nonzero_exit_status.c",
94 + :on => @problem_test_normal,
95 + :and_report => {
96 + :score => 0,
97 + :comment => /^FAILED:/})
94 98 end
95 99
96 100 def grader_should(args)
97 101 @user1 = stub(User,
98 102 :id => 1, :login => 'user1')
99 103 submission =
100 104 create_submission_from_file(1, @user1, args[:on], args[:grade])
101 105 submission.should_receive(:graded_at=)
102 106
103 107 expected_score = args[:and_report][:score]
104 108 expected_comment = args[:and_report][:comment]
105 109 if args[:and_report][:compiler_message]!=nil
106 110 expected_compiler_message = args[:and_report][:compiler_message]
107 111 else
108 112 expected_compiler_message = ''
109 113 end
110 114
111 115 submission.should_receive(:points=).with(expected_score)
112 116 submission.should_receive(:grader_comment=).with(expected_comment)
113 117 submission.should_receive(:compiler_message=).with(expected_compiler_message)
114 118 submission.should_receive(:save)
115 119
116 120 @engine.grade(submission)
117 121 end
118 122
119 123 protected
120 124
121 125 def create_normal_submission_mock_from_file(source_fname)
122 126 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
123 127 end
124 128
125 129 end
126 130
127 131 describe "A grader engine, when grading test requests" do
128 132
129 133 include GraderEngineHelperMethods
130 134
131 135 before(:each) do
132 136 @config = Grader::Configuration.get_instance
133 137 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
134 138 Grader::TestRequestReporter.new)
135 139 init_sandbox
136 140 end
137 141
138 142 it "should report error if there is no problem template" do
139 143 problem = stub(Problem,
140 144 :id => 1, :name => 'nothing')
141 145 grader_should(:grade => 'test1_correct.c',
142 146 :on => problem,
143 147 :with => 'in1.txt',
144 148 :and_report => {
145 149 :graded_at= => nil,
146 150 :compiler_message= => '',
147 151 :grader_comment= => '',
148 152 :running_stat= => /template not found/,
149 153 :save => nil})
150 154 end
151 155
152 156 it "should run test request and produce output file" do
153 157 problem = stub(Problem,
154 158 :id => 1, :name => 'test_normal')
155 159 grader_should(:grade => 'test1_correct.c',
156 160 :on => problem,
157 161 :with => 'in1.txt',
158 162 :and_report => {
159 163 :graded_at= => nil,
160 164 :compiler_message= => '',
161 165 :grader_comment= => '',
162 166 :running_stat= => /0.0 sec./,
163 167 :output_file_name= => lambda { |fname|
164 168 File.exists?(fname).should be_true
165 169 },
166 170 :save => nil})
167 171 end
168 172
169 173 it "should clean up problem directory after running test request" do
170 174 problem = stub(Problem,
171 175 :id => 1, :name => 'test_normal')
172 176 grader_should(:grade => 'test1_correct.c',
173 177 :on => problem,
174 178 :with => 'in1.txt',
175 179 :and_report => {
176 180 :graded_at= => nil,
177 181 :compiler_message= => '',
178 182 :grader_comment= => '',
179 183 :running_stat= => nil,
180 184 :output_file_name= => nil,
181 185 :save => nil})
182 186 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
183 187 end
184 188
185 189 it "should compile test request with error and report compilation error" do
186 190 problem = stub(Problem,
187 191 :id => 1, :name => 'test_normal')
188 192 grader_should(:grade => 'test1_compile_error.c',
189 193 :on => problem,
190 194 :with => 'in1.txt',
191 195 :and_report => {
192 196 :graded_at= => nil,
193 197 :compiler_message= => /.+/,
194 198 :grader_comment= => /[Cc]ompil.*error/,
195 199 :running_stat= => '',
196 200 :save => nil})
197 201 end
198 202
199 203 it "should report exit status" do
200 204 problem = stub(Problem,
201 205 :id => 1, :name => 'test_normal')
202 206 grader_should(:grade => 'add_nonzero_exit_status.c',
203 207 :on => problem,
204 208 :with => 'in1.txt',
205 209 :and_report => {
206 210 :graded_at= => nil,
207 211 :compiler_message= => '',
208 212 :grader_comment= => '',
209 213 :running_stat= => /[Ee]xit.*status.*10.*0.0 sec./,
210 214 :output_file_name= => lambda { |fname|
211 215 File.exists?(fname).should be_true
212 216 },
213 217 :save => nil})
214 218 end
215 219
216 220 protected
217 221 def grader_should(args)
218 222 @user1 = stub(User,
219 223 :id => 1, :login => 'user1')
220 224
221 225 problem = args[:on]
222 226 input_file = @config.test_request_input_base_dir + "/" + args[:with]
223 227
224 228 submission =
225 229 create_submission_from_file(1, @user1, args[:on], args[:grade])
226 230
227 231 test_request = stub(TestRequest,
228 232 :id => 1,
229 233 :user => @user1,
230 234 :problem => problem,
231 235 :submission => submission,
232 236 :input_file_name => input_file,
233 237 :language => submission.language,
234 238 :problem_name => problem.name)
235 239
236 240 expectations = args[:and_report]
237 241
238 242 expectations.each do |key,val|
239 243 if val==nil
240 244 test_request.should_receive(key)
241 245 elsif val.class == Proc
242 246 test_request.should_receive(key) { |fname|
243 247 val.call(fname)
244 248 }
245 249 else
246 250 test_request.should_receive(key).with(val)
247 251 end
248 252 end
249 253
250 254 @engine.grade(test_request)
251 255 end
252 256
253 257 end
254 258
You need to be logged in to leave comments. Login now