cafe-grader-judge-scripts

Commit r39:7a9b64601b9d public

parent | child
Description:
[grader] engine_spec: fix reg-ex new line problem, change test case for 1.5 sec submission (as the machine is a little faster) git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@150 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r39:7a9b64601b9d - - 3 files changed: 3 inserted, 3 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,21 +1,21
1 1 #include <stdio.h>
2 2 #include <stdlib.h>
3 3
4 4 int main()
5 5 {
6 6 int a,b;
7 7
8 8 int c=0;
9 9
10 10 scanf("%d %d",&a,&b);
11 11 printf("%d\n",a+b);
12 12
13 13 for(a=0; a<2; a++) {
14 - while(c<1150000000) {
14 + while(c<1550000000) {
15 15 c++;
16 16 b+=c;
17 17 }
18 18 }
19 19 exit(0);
20 20 }
21 21
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -43,238 +43,238
43 43 :full_score => 10)
44 44 grader_should(:grade => "test2_timeout.c",
45 45 :on => @problem_test_timeout,
46 46 :and_report => {
47 47 :score => 0,
48 48 :comment => 'FAILED: TT'})
49 49 end
50 50
51 51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
52 52 @problem_test_timeout = stub(Problem,
53 53 :id => 1, :name => 'test_timeout',
54 54 :full_score => 20)
55 55 grader_should(:grade => "test2_1-5sec.c",
56 56 :on => @problem_test_timeout,
57 57 :and_report => {
58 58 :score => 10,
59 59 :comment => 'FAILED: TP'})
60 60 end
61 61
62 62 it "should produce runtime error when submission uses too much static memory" do
63 63 @problem_test_memory = stub(Problem,
64 64 :id => 1, :name => 'test_memory',
65 65 :full_score => 20)
66 66 grader_should(:grade => "add_too_much_memory_static.c",
67 67 :on => @problem_test_memory,
68 68 :and_report => {
69 69 :score => 10,
70 70 :comment => /FAILED: [^P]P/})
71 71 end
72 72
73 73 it "should not allow submission to allocate too much dynamic memory" do
74 74 @problem_test_memory = stub(Problem,
75 75 :id => 1, :name => 'test_memory',
76 76 :full_score => 20)
77 77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 78 :on => @problem_test_memory,
79 79 :and_report => {
80 80 :score => 10,
81 81 :comment => /FAILED: [^P]P/})
82 82 end
83 83
84 84 it "should score test runs correctly when submission fails in some test case" do
85 85 grader_should(:grade => "add_fail_test_case_1.c",
86 86 :on => @problem_test_normal,
87 87 :and_report => {
88 88 :score => 105,
89 89 :comment => /^FAILED:/})
90 90 end
91 91
92 92 it "should fail submission with non-zero exit status" do
93 93 grader_should(:grade => "add_nonzero_exit_status.c",
94 94 :on => @problem_test_normal,
95 95 :and_report => {
96 96 :score => 0,
97 97 :comment => /^FAILED:/})
98 98 end
99 99
100 100 it "should not allow malicious submission to see PROBLEM_HOME" do
101 101 problem_test_yesno = stub(Problem,
102 102 :id => 1, :name => 'test_yesno',
103 103 :full_score => 10)
104 104 grader_should(:grade => "yesno_access_problem_home.c",
105 105 :on => problem_test_yesno,
106 106 :and_report => {
107 107 :score => 0,
108 108 :comment => /^FAILED:/})
109 109 end
110 110
111 111 it "should not allow malicious submission to open files" do
112 112 problem_test_yesno = stub(Problem,
113 113 :id => 1, :name => 'test_yesno',
114 114 :full_score => 10)
115 115 grader_should(:grade => "yesno_open_file.c",
116 116 :on => problem_test_yesno,
117 117 :and_report => {
118 118 :score => 0,
119 119 :comment => /^FAILED:/})
120 120 end
121 121
122 122 def grader_should(args)
123 123 @user1 = stub(User,
124 124 :id => 1, :login => 'user1')
125 125 submission =
126 126 create_submission_from_file(1, @user1, args[:on], args[:grade])
127 127 submission.should_receive(:graded_at=)
128 128
129 129 expected_score = args[:and_report][:score]
130 130 expected_comment = args[:and_report][:comment]
131 131 if args[:and_report][:compiler_message]!=nil
132 132 expected_compiler_message = args[:and_report][:compiler_message]
133 133 else
134 134 expected_compiler_message = ''
135 135 end
136 136
137 137 submission.should_receive(:points=).with(expected_score)
138 138 submission.should_receive(:grader_comment=).with(expected_comment)
139 139 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 140 submission.should_receive(:save)
141 141
142 142 @engine.grade(submission)
143 143 end
144 144
145 145 protected
146 146
147 147 def create_normal_submission_mock_from_file(source_fname)
148 148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 149 end
150 150
151 151 end
152 152
153 153 describe "A grader engine, when grading test requests" do
154 154
155 155 include GraderEngineHelperMethods
156 156
157 157 before(:each) do
158 158 @config = Grader::Configuration.get_instance
159 159 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
160 160 Grader::TestRequestReporter.new)
161 161 init_sandbox
162 162 end
163 163
164 164 it "should report error if there is no problem template" do
165 165 problem = stub(Problem,
166 166 :id => 1, :name => 'nothing')
167 167 grader_should(:grade => 'test1_correct.c',
168 168 :on => problem,
169 169 :with => 'in1.txt',
170 170 :and_report => {
171 171 :graded_at= => nil,
172 172 :compiler_message= => '',
173 173 :grader_comment= => '',
174 174 :running_stat= => /template not found/,
175 175 :save => nil})
176 176 end
177 177
178 178 it "should run test request and produce output file" do
179 179 problem = stub(Problem,
180 180 :id => 1, :name => 'test_normal')
181 181 grader_should(:grade => 'test1_correct.c',
182 182 :on => problem,
183 183 :with => 'in1.txt',
184 184 :and_report => {
185 185 :graded_at= => nil,
186 186 :compiler_message= => '',
187 187 :grader_comment= => '',
188 188 :running_stat= => /0.0 sec./,
189 189 :output_file_name= => lambda { |fname|
190 190 File.exists?(fname).should be_true
191 191 },
192 192 :save => nil})
193 193 end
194 194
195 195 it "should clean up problem directory after running test request" do
196 196 problem = stub(Problem,
197 197 :id => 1, :name => 'test_normal')
198 198 grader_should(:grade => 'test1_correct.c',
199 199 :on => problem,
200 200 :with => 'in1.txt',
201 201 :and_report => {
202 202 :graded_at= => nil,
203 203 :compiler_message= => '',
204 204 :grader_comment= => '',
205 205 :running_stat= => nil,
206 206 :output_file_name= => nil,
207 207 :save => nil})
208 208 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
209 209 end
210 210
211 211 it "should compile test request with error and report compilation error" do
212 212 problem = stub(Problem,
213 213 :id => 1, :name => 'test_normal')
214 214 grader_should(:grade => 'test1_compile_error.c',
215 215 :on => problem,
216 216 :with => 'in1.txt',
217 217 :and_report => {
218 218 :graded_at= => nil,
219 219 :compiler_message= => /.+/,
220 220 :grader_comment= => /[Cc]ompil.*error/,
221 221 :running_stat= => '',
222 222 :save => nil})
223 223 end
224 224
225 225 it "should report exit status" do
226 226 problem = stub(Problem,
227 227 :id => 1, :name => 'test_normal')
228 228 grader_should(:grade => 'add_nonzero_exit_status.c',
229 229 :on => problem,
230 230 :with => 'in1.txt',
231 231 :and_report => {
232 232 :graded_at= => nil,
233 233 :compiler_message= => '',
234 234 :grader_comment= => '',
235 - :running_stat= => /[Ee]xit.*status.*10.*0.0 sec./,
235 + :running_stat= => /[Ee]xit.*status.*10.*0\.0 sec/m,
236 236 :output_file_name= => lambda { |fname|
237 237 File.exists?(fname).should be_true
238 238 },
239 239 :save => nil})
240 240 end
241 241
242 242 protected
243 243 def grader_should(args)
244 244 @user1 = stub(User,
245 245 :id => 1, :login => 'user1')
246 246
247 247 problem = args[:on]
248 248 input_file = @config.test_request_input_base_dir + "/" + args[:with]
249 249
250 250 submission =
251 251 create_submission_from_file(1, @user1, args[:on], args[:grade])
252 252
253 253 test_request = stub(TestRequest,
254 254 :id => 1,
255 255 :user => @user1,
256 256 :problem => problem,
257 257 :submission => submission,
258 258 :input_file_name => input_file,
259 259 :language => submission.language,
260 260 :problem_name => problem.name)
261 261
262 262 expectations = args[:and_report]
263 263
264 264 expectations.each do |key,val|
265 265 if val==nil
266 266 test_request.should_receive(key)
267 267 elsif val.class == Proc
268 268 test_request.should_receive(key) { |fname|
269 269 val.call(fname)
270 270 }
271 271 else
272 272 test_request.should_receive(key).with(val)
273 273 end
274 274 end
275 275
276 276 @engine.grade(test_request)
277 277 end
278 278
279 279 end
280 280
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,30 +1,30
1 1 module GraderEngineHelperMethods
2 2
3 3 def clear_sandbox
4 4 config = Grader::Configuration.get_instance
5 5 clear_cmd = "rm -rf #{config.test_sandbox_dir}/*"
6 6 system(clear_cmd)
7 7 end
8 8
9 9 def init_sandbox
10 10 config = Grader::Configuration.get_instance
11 11 clear_sandbox
12 - Dir.mkdir config.user_result_dir
12 + FileUtils.mkdir_p config.user_result_dir
13 13 cp_cmd = "cp -R #{config.test_data_dir}/ev #{config.test_sandbox_dir}"
14 14 system(cp_cmd)
15 15 end
16 16
17 17 def create_submission_from_file(id, user, problem,
18 18 source_fname, language=nil)
19 19
20 20 language = stub(Language, :name => 'c', :ext => 'c') if language==nil
21 21
22 22 config = Grader::Configuration.get_instance
23 23 source = File.open(config.test_data_dir + "/" + source_fname).read
24 24 stub(Submission,
25 25 :id => id, :user => user, :problem => problem,
26 26 :source => source, :language => language)
27 27 end
28 28
29 29 end
30 30
You need to be logged in to leave comments. Login now