cafe-grader-judge-scripts

Commit r33:e7a6930e8c87 public

parent | child
Description:
[grader] finish engine spec on grading non-zero exit status git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@127 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r33:e7a6930e8c87 - - 1 file changed: 5 inserted, 1 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,254 +1,258
1 require File.join(File.dirname(__FILE__),'spec_helper')
1 require File.join(File.dirname(__FILE__),'spec_helper')
2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3
3
4 describe "A grader engine, when grading submissions" do
4 describe "A grader engine, when grading submissions" do
5
5
6 include GraderEngineHelperMethods
6 include GraderEngineHelperMethods
7
7
8 before(:each) do
8 before(:each) do
9 @config = Grader::Configuration.get_instance
9 @config = Grader::Configuration.get_instance
10
10
11 # this test is from Pong
11 # this test is from Pong
12 @problem_test_normal = stub(Problem,
12 @problem_test_normal = stub(Problem,
13 :id => 1, :name => 'test_normal',
13 :id => 1, :name => 'test_normal',
14 :full_score => 135)
14 :full_score => 135)
15 @user_user1 = stub(User,
15 @user_user1 = stub(User,
16 :id => 1, :login => 'user1')
16 :id => 1, :login => 'user1')
17
17
18 @engine = Grader::Engine.new
18 @engine = Grader::Engine.new
19 init_sandbox
19 init_sandbox
20 end
20 end
21
21
22 it "should grade normal submission" do
22 it "should grade normal submission" do
23 grader_should(:grade => "test1_correct.c",
23 grader_should(:grade => "test1_correct.c",
24 :on => @problem_test_normal,
24 :on => @problem_test_normal,
25 :and_report => {
25 :and_report => {
26 :score => 135,
26 :score => 135,
27 :comment => /^PASSED/})
27 :comment => /^PASSED/})
28 end
28 end
29
29
30
30
31 it "should produce error message when submission cannot compile" do
31 it "should produce error message when submission cannot compile" do
32 grader_should(:grade => "test1_compile_error.c",
32 grader_should(:grade => "test1_compile_error.c",
33 :on => @problem_test_normal,
33 :on => @problem_test_normal,
34 :and_report => {
34 :and_report => {
35 :score => 0,
35 :score => 0,
36 :comment => 'FAILED: compile error',
36 :comment => 'FAILED: compile error',
37 :compiler_message => /[Ee]rror/})
37 :compiler_message => /[Ee]rror/})
38 end
38 end
39
39
40 it "should produce timeout error when submission runs forever" do
40 it "should produce timeout error when submission runs forever" do
41 @problem_test_timeout = stub(Problem,
41 @problem_test_timeout = stub(Problem,
42 :id => 1, :name => 'test_timeout',
42 :id => 1, :name => 'test_timeout',
43 :full_score => 10)
43 :full_score => 10)
44 grader_should(:grade => "test2_timeout.c",
44 grader_should(:grade => "test2_timeout.c",
45 :on => @problem_test_timeout,
45 :on => @problem_test_timeout,
46 :and_report => {
46 :and_report => {
47 :score => 0,
47 :score => 0,
48 :comment => 'FAILED: TT'})
48 :comment => 'FAILED: TT'})
49 end
49 end
50
50
51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
52 @problem_test_timeout = stub(Problem,
52 @problem_test_timeout = stub(Problem,
53 :id => 1, :name => 'test_timeout',
53 :id => 1, :name => 'test_timeout',
54 :full_score => 20)
54 :full_score => 20)
55 grader_should(:grade => "test2_1-5sec.c",
55 grader_should(:grade => "test2_1-5sec.c",
56 :on => @problem_test_timeout,
56 :on => @problem_test_timeout,
57 :and_report => {
57 :and_report => {
58 :score => 10,
58 :score => 10,
59 :comment => 'FAILED: TP'})
59 :comment => 'FAILED: TP'})
60 end
60 end
61
61
62 it "should produce runtime error when submission uses too much static memory" do
62 it "should produce runtime error when submission uses too much static memory" do
63 @problem_test_memory = stub(Problem,
63 @problem_test_memory = stub(Problem,
64 :id => 1, :name => 'test_memory',
64 :id => 1, :name => 'test_memory',
65 :full_score => 20)
65 :full_score => 20)
66 grader_should(:grade => "add_too_much_memory_static.c",
66 grader_should(:grade => "add_too_much_memory_static.c",
67 :on => @problem_test_memory,
67 :on => @problem_test_memory,
68 :and_report => {
68 :and_report => {
69 :score => 10,
69 :score => 10,
70 :comment => /FAILED: [^P]P/})
70 :comment => /FAILED: [^P]P/})
71 end
71 end
72
72
73 it "should not allow submission to allocate too much dynamic memory" do
73 it "should not allow submission to allocate too much dynamic memory" do
74 @problem_test_memory = stub(Problem,
74 @problem_test_memory = stub(Problem,
75 :id => 1, :name => 'test_memory',
75 :id => 1, :name => 'test_memory',
76 :full_score => 20)
76 :full_score => 20)
77 grader_should(:grade => "add_too_much_memory_dynamic.c",
77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 :on => @problem_test_memory,
78 :on => @problem_test_memory,
79 :and_report => {
79 :and_report => {
80 :score => 10,
80 :score => 10,
81 :comment => /FAILED: [^P]P/})
81 :comment => /FAILED: [^P]P/})
82 end
82 end
83
83
84 it "should score test runs correctly when submission fails in some test case" do
84 it "should score test runs correctly when submission fails in some test case" do
85 grader_should(:grade => "add_fail_test_case_1.c",
85 grader_should(:grade => "add_fail_test_case_1.c",
86 :on => @problem_test_normal,
86 :on => @problem_test_normal,
87 :and_report => {
87 :and_report => {
88 :score => 105,
88 :score => 105,
89 :comment => /^FAILED:/})
89 :comment => /^FAILED:/})
90 end
90 end
91
91
92 it "should fail submission with non-zero exit status" do
92 it "should fail submission with non-zero exit status" do
93 - violated("has not been implemented")
93 + grader_should(:grade => "add_nonzero_exit_status.c",
94 + :on => @problem_test_normal,
95 + :and_report => {
96 + :score => 0,
97 + :comment => /^FAILED:/})
94 end
98 end
95
99
96 def grader_should(args)
100 def grader_should(args)
97 @user1 = stub(User,
101 @user1 = stub(User,
98 :id => 1, :login => 'user1')
102 :id => 1, :login => 'user1')
99 submission =
103 submission =
100 create_submission_from_file(1, @user1, args[:on], args[:grade])
104 create_submission_from_file(1, @user1, args[:on], args[:grade])
101 submission.should_receive(:graded_at=)
105 submission.should_receive(:graded_at=)
102
106
103 expected_score = args[:and_report][:score]
107 expected_score = args[:and_report][:score]
104 expected_comment = args[:and_report][:comment]
108 expected_comment = args[:and_report][:comment]
105 if args[:and_report][:compiler_message]!=nil
109 if args[:and_report][:compiler_message]!=nil
106 expected_compiler_message = args[:and_report][:compiler_message]
110 expected_compiler_message = args[:and_report][:compiler_message]
107 else
111 else
108 expected_compiler_message = ''
112 expected_compiler_message = ''
109 end
113 end
110
114
111 submission.should_receive(:points=).with(expected_score)
115 submission.should_receive(:points=).with(expected_score)
112 submission.should_receive(:grader_comment=).with(expected_comment)
116 submission.should_receive(:grader_comment=).with(expected_comment)
113 submission.should_receive(:compiler_message=).with(expected_compiler_message)
117 submission.should_receive(:compiler_message=).with(expected_compiler_message)
114 submission.should_receive(:save)
118 submission.should_receive(:save)
115
119
116 @engine.grade(submission)
120 @engine.grade(submission)
117 end
121 end
118
122
119 protected
123 protected
120
124
121 def create_normal_submission_mock_from_file(source_fname)
125 def create_normal_submission_mock_from_file(source_fname)
122 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
126 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
123 end
127 end
124
128
125 end
129 end
126
130
127 describe "A grader engine, when grading test requests" do
131 describe "A grader engine, when grading test requests" do
128
132
129 include GraderEngineHelperMethods
133 include GraderEngineHelperMethods
130
134
131 before(:each) do
135 before(:each) do
132 @config = Grader::Configuration.get_instance
136 @config = Grader::Configuration.get_instance
133 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
137 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
134 Grader::TestRequestReporter.new)
138 Grader::TestRequestReporter.new)
135 init_sandbox
139 init_sandbox
136 end
140 end
137
141
138 it "should report error if there is no problem template" do
142 it "should report error if there is no problem template" do
139 problem = stub(Problem,
143 problem = stub(Problem,
140 :id => 1, :name => 'nothing')
144 :id => 1, :name => 'nothing')
141 grader_should(:grade => 'test1_correct.c',
145 grader_should(:grade => 'test1_correct.c',
142 :on => problem,
146 :on => problem,
143 :with => 'in1.txt',
147 :with => 'in1.txt',
144 :and_report => {
148 :and_report => {
145 :graded_at= => nil,
149 :graded_at= => nil,
146 :compiler_message= => '',
150 :compiler_message= => '',
147 :grader_comment= => '',
151 :grader_comment= => '',
148 :running_stat= => /template not found/,
152 :running_stat= => /template not found/,
149 :save => nil})
153 :save => nil})
150 end
154 end
151
155
152 it "should run test request and produce output file" do
156 it "should run test request and produce output file" do
153 problem = stub(Problem,
157 problem = stub(Problem,
154 :id => 1, :name => 'test_normal')
158 :id => 1, :name => 'test_normal')
155 grader_should(:grade => 'test1_correct.c',
159 grader_should(:grade => 'test1_correct.c',
156 :on => problem,
160 :on => problem,
157 :with => 'in1.txt',
161 :with => 'in1.txt',
158 :and_report => {
162 :and_report => {
159 :graded_at= => nil,
163 :graded_at= => nil,
160 :compiler_message= => '',
164 :compiler_message= => '',
161 :grader_comment= => '',
165 :grader_comment= => '',
162 :running_stat= => /0.0 sec./,
166 :running_stat= => /0.0 sec./,
163 :output_file_name= => lambda { |fname|
167 :output_file_name= => lambda { |fname|
164 File.exists?(fname).should be_true
168 File.exists?(fname).should be_true
165 },
169 },
166 :save => nil})
170 :save => nil})
167 end
171 end
168
172
169 it "should clean up problem directory after running test request" do
173 it "should clean up problem directory after running test request" do
170 problem = stub(Problem,
174 problem = stub(Problem,
171 :id => 1, :name => 'test_normal')
175 :id => 1, :name => 'test_normal')
172 grader_should(:grade => 'test1_correct.c',
176 grader_should(:grade => 'test1_correct.c',
173 :on => problem,
177 :on => problem,
174 :with => 'in1.txt',
178 :with => 'in1.txt',
175 :and_report => {
179 :and_report => {
176 :graded_at= => nil,
180 :graded_at= => nil,
177 :compiler_message= => '',
181 :compiler_message= => '',
178 :grader_comment= => '',
182 :grader_comment= => '',
179 :running_stat= => nil,
183 :running_stat= => nil,
180 :output_file_name= => nil,
184 :output_file_name= => nil,
181 :save => nil})
185 :save => nil})
182 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
186 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
183 end
187 end
184
188
185 it "should compile test request with error and report compilation error" do
189 it "should compile test request with error and report compilation error" do
186 problem = stub(Problem,
190 problem = stub(Problem,
187 :id => 1, :name => 'test_normal')
191 :id => 1, :name => 'test_normal')
188 grader_should(:grade => 'test1_compile_error.c',
192 grader_should(:grade => 'test1_compile_error.c',
189 :on => problem,
193 :on => problem,
190 :with => 'in1.txt',
194 :with => 'in1.txt',
191 :and_report => {
195 :and_report => {
192 :graded_at= => nil,
196 :graded_at= => nil,
193 :compiler_message= => /.+/,
197 :compiler_message= => /.+/,
194 :grader_comment= => /[Cc]ompil.*error/,
198 :grader_comment= => /[Cc]ompil.*error/,
195 :running_stat= => '',
199 :running_stat= => '',
196 :save => nil})
200 :save => nil})
197 end
201 end
198
202
199 it "should report exit status" do
203 it "should report exit status" do
200 problem = stub(Problem,
204 problem = stub(Problem,
201 :id => 1, :name => 'test_normal')
205 :id => 1, :name => 'test_normal')
202 grader_should(:grade => 'add_nonzero_exit_status.c',
206 grader_should(:grade => 'add_nonzero_exit_status.c',
203 :on => problem,
207 :on => problem,
204 :with => 'in1.txt',
208 :with => 'in1.txt',
205 :and_report => {
209 :and_report => {
206 :graded_at= => nil,
210 :graded_at= => nil,
207 :compiler_message= => '',
211 :compiler_message= => '',
208 :grader_comment= => '',
212 :grader_comment= => '',
209 :running_stat= => /[Ee]xit.*status.*10.*0.0 sec./,
213 :running_stat= => /[Ee]xit.*status.*10.*0.0 sec./,
210 :output_file_name= => lambda { |fname|
214 :output_file_name= => lambda { |fname|
211 File.exists?(fname).should be_true
215 File.exists?(fname).should be_true
212 },
216 },
213 :save => nil})
217 :save => nil})
214 end
218 end
215
219
216 protected
220 protected
217 def grader_should(args)
221 def grader_should(args)
218 @user1 = stub(User,
222 @user1 = stub(User,
219 :id => 1, :login => 'user1')
223 :id => 1, :login => 'user1')
220
224
221 problem = args[:on]
225 problem = args[:on]
222 input_file = @config.test_request_input_base_dir + "/" + args[:with]
226 input_file = @config.test_request_input_base_dir + "/" + args[:with]
223
227
224 submission =
228 submission =
225 create_submission_from_file(1, @user1, args[:on], args[:grade])
229 create_submission_from_file(1, @user1, args[:on], args[:grade])
226
230
227 test_request = stub(TestRequest,
231 test_request = stub(TestRequest,
228 :id => 1,
232 :id => 1,
229 :user => @user1,
233 :user => @user1,
230 :problem => problem,
234 :problem => problem,
231 :submission => submission,
235 :submission => submission,
232 :input_file_name => input_file,
236 :input_file_name => input_file,
233 :language => submission.language,
237 :language => submission.language,
234 :problem_name => problem.name)
238 :problem_name => problem.name)
235
239
236 expectations = args[:and_report]
240 expectations = args[:and_report]
237
241
238 expectations.each do |key,val|
242 expectations.each do |key,val|
239 if val==nil
243 if val==nil
240 test_request.should_receive(key)
244 test_request.should_receive(key)
241 elsif val.class == Proc
245 elsif val.class == Proc
242 test_request.should_receive(key) { |fname|
246 test_request.should_receive(key) { |fname|
243 val.call(fname)
247 val.call(fname)
244 }
248 }
245 else
249 else
246 test_request.should_receive(key).with(val)
250 test_request.should_receive(key).with(val)
247 end
251 end
248 end
252 end
249
253
250 @engine.grade(test_request)
254 @engine.grade(test_request)
251 end
255 end
252
256
253 end
257 end
254
258
You need to be logged in to leave comments. Login now