cafe-grader-judge-scripts

Commit r44:a07593cb07c4 public

parent | child
Description:
[grader] small tweak on one test case git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@168 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r44:a07593cb07c4 - - 1 file changed: 1 inserted, 1 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -95,193 +95,193
95 :and_report => {
95 :and_report => {
96 :score => 0,
96 :score => 0,
97 :comment => /^FAILED:/})
97 :comment => /^FAILED:/})
98 end
98 end
99
99
100 it "should not allow malicious submission to see PROBLEM_HOME" do
100 it "should not allow malicious submission to see PROBLEM_HOME" do
101 problem_test_yesno = stub(Problem,
101 problem_test_yesno = stub(Problem,
102 :id => 1, :name => 'test_yesno',
102 :id => 1, :name => 'test_yesno',
103 :full_score => 10)
103 :full_score => 10)
104 grader_should(:grade => "yesno_access_problem_home.c",
104 grader_should(:grade => "yesno_access_problem_home.c",
105 :on => problem_test_yesno,
105 :on => problem_test_yesno,
106 :and_report => {
106 :and_report => {
107 :score => 0,
107 :score => 0,
108 :comment => /^FAILED:/})
108 :comment => /^FAILED:/})
109 end
109 end
110
110
111 it "should not allow malicious submission to open files" do
111 it "should not allow malicious submission to open files" do
112 problem_test_yesno = stub(Problem,
112 problem_test_yesno = stub(Problem,
113 :id => 1, :name => 'test_yesno',
113 :id => 1, :name => 'test_yesno',
114 :full_score => 10)
114 :full_score => 10)
115 grader_should(:grade => "yesno_open_file.c",
115 grader_should(:grade => "yesno_open_file.c",
116 :on => problem_test_yesno,
116 :on => problem_test_yesno,
117 :and_report => {
117 :and_report => {
118 :score => 0,
118 :score => 0,
119 :comment => /^FAILED:/})
119 :comment => /^FAILED:/})
120 end
120 end
121
121
122 def grader_should(args)
122 def grader_should(args)
123 @user1 = stub(User,
123 @user1 = stub(User,
124 :id => 1, :login => 'user1')
124 :id => 1, :login => 'user1')
125 submission =
125 submission =
126 create_submission_from_file(1, @user1, args[:on], args[:grade])
126 create_submission_from_file(1, @user1, args[:on], args[:grade])
127 submission.should_receive(:graded_at=)
127 submission.should_receive(:graded_at=)
128
128
129 expected_score = args[:and_report][:score]
129 expected_score = args[:and_report][:score]
130 expected_comment = args[:and_report][:comment]
130 expected_comment = args[:and_report][:comment]
131 if args[:and_report][:compiler_message]!=nil
131 if args[:and_report][:compiler_message]!=nil
132 expected_compiler_message = args[:and_report][:compiler_message]
132 expected_compiler_message = args[:and_report][:compiler_message]
133 else
133 else
134 expected_compiler_message = ''
134 expected_compiler_message = ''
135 end
135 end
136
136
137 submission.should_receive(:points=).with(expected_score)
137 submission.should_receive(:points=).with(expected_score)
138 submission.should_receive(:grader_comment=).with(expected_comment)
138 submission.should_receive(:grader_comment=).with(expected_comment)
139 submission.should_receive(:compiler_message=).with(expected_compiler_message)
139 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 submission.should_receive(:save)
140 submission.should_receive(:save)
141
141
142 @engine.grade(submission)
142 @engine.grade(submission)
143 end
143 end
144
144
145 protected
145 protected
146
146
147 def create_normal_submission_mock_from_file(source_fname)
147 def create_normal_submission_mock_from_file(source_fname)
148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 end
149 end
150
150
151 end
151 end
152
152
153 describe "A grader engine, when grading test requests" do
153 describe "A grader engine, when grading test requests" do
154
154
155 include GraderEngineHelperMethods
155 include GraderEngineHelperMethods
156
156
157 before(:each) do
157 before(:each) do
158 @config = Grader::Configuration.get_instance
158 @config = Grader::Configuration.get_instance
159 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
159 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
160 Grader::TestRequestReporter.new)
160 Grader::TestRequestReporter.new)
161 init_sandbox
161 init_sandbox
162 end
162 end
163
163
164 it "should report error if there is no problem template" do
164 it "should report error if there is no problem template" do
165 problem = stub(Problem,
165 problem = stub(Problem,
166 :id => 1, :name => 'nothing')
166 :id => 1, :name => 'nothing')
167 grader_should(:grade => 'test1_correct.c',
167 grader_should(:grade => 'test1_correct.c',
168 :on => problem,
168 :on => problem,
169 :with => 'in1.txt',
169 :with => 'in1.txt',
170 :and_report => {
170 :and_report => {
171 :graded_at= => nil,
171 :graded_at= => nil,
172 :compiler_message= => '',
172 :compiler_message= => '',
173 :grader_comment= => '',
173 :grader_comment= => '',
174 :running_stat= => /template not found/,
174 :running_stat= => /template not found/,
175 :running_time= => nil,
175 :running_time= => nil,
176 :exit_status= => nil,
176 :exit_status= => nil,
177 :memory_usage= => nil,
177 :memory_usage= => nil,
178 :save => nil})
178 :save => nil})
179 end
179 end
180
180
181 it "should run test request and produce output file" do
181 it "should run test request and produce output file" do
182 problem = stub(Problem,
182 problem = stub(Problem,
183 :id => 1, :name => 'test_normal')
183 :id => 1, :name => 'test_normal')
184 grader_should(:grade => 'test1_correct.c',
184 grader_should(:grade => 'test1_correct.c',
185 :on => problem,
185 :on => problem,
186 :with => 'in1.txt',
186 :with => 'in1.txt',
187 :and_report => {
187 :and_report => {
188 :graded_at= => nil,
188 :graded_at= => nil,
189 :compiler_message= => '',
189 :compiler_message= => '',
190 :grader_comment= => '',
190 :grader_comment= => '',
191 - :running_stat= => /0.0 sec./,
191 + :running_stat= => /0.0\d* sec./,
192 :output_file_name= => lambda { |fname|
192 :output_file_name= => lambda { |fname|
193 File.exists?(fname).should be_true
193 File.exists?(fname).should be_true
194 },
194 },
195 :running_time= => nil,
195 :running_time= => nil,
196 :exit_status= => nil,
196 :exit_status= => nil,
197 :memory_usage= => nil,
197 :memory_usage= => nil,
198 :save => nil})
198 :save => nil})
199 end
199 end
200
200
201 it "should clean up problem directory after running test request" do
201 it "should clean up problem directory after running test request" do
202 problem = stub(Problem,
202 problem = stub(Problem,
203 :id => 1, :name => 'test_normal')
203 :id => 1, :name => 'test_normal')
204 grader_should(:grade => 'test1_correct.c',
204 grader_should(:grade => 'test1_correct.c',
205 :on => problem,
205 :on => problem,
206 :with => 'in1.txt',
206 :with => 'in1.txt',
207 :and_report => {
207 :and_report => {
208 :graded_at= => nil,
208 :graded_at= => nil,
209 :compiler_message= => '',
209 :compiler_message= => '',
210 :grader_comment= => '',
210 :grader_comment= => '',
211 :running_stat= => nil,
211 :running_stat= => nil,
212 :output_file_name= => nil,
212 :output_file_name= => nil,
213 :running_time= => nil,
213 :running_time= => nil,
214 :exit_status= => nil,
214 :exit_status= => nil,
215 :memory_usage= => nil,
215 :memory_usage= => nil,
216 :save => nil})
216 :save => nil})
217 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
217 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
218 end
218 end
219
219
220 it "should compile test request with error and report compilation error" do
220 it "should compile test request with error and report compilation error" do
221 problem = stub(Problem,
221 problem = stub(Problem,
222 :id => 1, :name => 'test_normal')
222 :id => 1, :name => 'test_normal')
223 grader_should(:grade => 'test1_compile_error.c',
223 grader_should(:grade => 'test1_compile_error.c',
224 :on => problem,
224 :on => problem,
225 :with => 'in1.txt',
225 :with => 'in1.txt',
226 :and_report => {
226 :and_report => {
227 :graded_at= => nil,
227 :graded_at= => nil,
228 :compiler_message= => /.+/,
228 :compiler_message= => /.+/,
229 :grader_comment= => /[Cc]ompil.*error/,
229 :grader_comment= => /[Cc]ompil.*error/,
230 :running_stat= => '',
230 :running_stat= => '',
231 :save => nil})
231 :save => nil})
232 end
232 end
233
233
234 it "should report exit status" do
234 it "should report exit status" do
235 problem = stub(Problem,
235 problem = stub(Problem,
236 :id => 1, :name => 'test_normal')
236 :id => 1, :name => 'test_normal')
237 grader_should(:grade => 'add_nonzero_exit_status.c',
237 grader_should(:grade => 'add_nonzero_exit_status.c',
238 :on => problem,
238 :on => problem,
239 :with => 'in1.txt',
239 :with => 'in1.txt',
240 :and_report => {
240 :and_report => {
241 :graded_at= => nil,
241 :graded_at= => nil,
242 :compiler_message= => '',
242 :compiler_message= => '',
243 :grader_comment= => '',
243 :grader_comment= => '',
244 :running_stat= => /[Ee]xit.*status.*10.*0\.0 sec/m,
244 :running_stat= => /[Ee]xit.*status.*10.*0\.0 sec/m,
245 :output_file_name= => lambda { |fname|
245 :output_file_name= => lambda { |fname|
246 File.exists?(fname).should be_true
246 File.exists?(fname).should be_true
247 },
247 },
248 :running_time= => nil,
248 :running_time= => nil,
249 :exit_status= => /10/,
249 :exit_status= => /10/,
250 :memory_usage= => nil,
250 :memory_usage= => nil,
251 :save => nil})
251 :save => nil})
252 end
252 end
253
253
254 it "should produce running statistics for normal submission" do
254 it "should produce running statistics for normal submission" do
255 problem = stub(Problem,
255 problem = stub(Problem,
256 :id => 1, :name => 'test_normal')
256 :id => 1, :name => 'test_normal')
257 grader_should(:grade => 'test_run_stat.c',
257 grader_should(:grade => 'test_run_stat.c',
258 :on => problem,
258 :on => problem,
259 :with => 'in1.txt',
259 :with => 'in1.txt',
260 :and_report => {
260 :and_report => {
261 :graded_at= => nil,
261 :graded_at= => nil,
262 :compiler_message= => '',
262 :compiler_message= => '',
263 :grader_comment= => '',
263 :grader_comment= => '',
264 :running_stat= => nil,
264 :running_stat= => nil,
265 :output_file_name= => lambda { |fname|
265 :output_file_name= => lambda { |fname|
266 File.exists?(fname).should be_true
266 File.exists?(fname).should be_true
267 },
267 },
268 :running_time= => lambda { |t|
268 :running_time= => lambda { |t|
269 (t>=0.14) and (t<=0.16)
269 (t>=0.14) and (t<=0.16)
270 },
270 },
271 :exit_status= => nil,
271 :exit_status= => nil,
272 :memory_usage= => lambda { |s|
272 :memory_usage= => lambda { |s|
273 (s>=500) and (s<=1000)
273 (s>=500) and (s<=1000)
274 },
274 },
275 :save => nil})
275 :save => nil})
276 end
276 end
277
277
278 protected
278 protected
279 def grader_should(args)
279 def grader_should(args)
280 @user1 = stub(User,
280 @user1 = stub(User,
281 :id => 1, :login => 'user1')
281 :id => 1, :login => 'user1')
282
282
283 problem = args[:on]
283 problem = args[:on]
284 input_file = @config.test_request_input_base_dir + "/" + args[:with]
284 input_file = @config.test_request_input_base_dir + "/" + args[:with]
285
285
286 submission =
286 submission =
287 create_submission_from_file(1, @user1, args[:on], args[:grade])
287 create_submission_from_file(1, @user1, args[:on], args[:grade])
You need to be logged in to leave comments. Login now