cafe-grader-judge-scripts

Commit r41:d4e37450ddbc public

parent | child
Description:
[grader] extracted test_request info into new stat columns git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@163 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r41:d4e37450ddbc - - 3 files changed: 37 inserted, 7 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -11,180 +11,196
11 11 # - prepare problem configuration for grading --- basically it copy
12 12 # all config files, and copy user's input into the testcase
13 13 # directory. First, it finds the template from problem template
14 14 # directory; if it can't find a template, it'll use the template
15 15 # from default template.
16 16 class TestRequestRoomMaker
17 17 def initialize
18 18 @config = Grader::Configuration.get_instance
19 19 end
20 20
21 21 def produce_grading_room(test_request)
22 22 grading_room = grading_room_dir(test_request)
23 23 FileUtils.mkdir_p(grading_room)
24 24 grading_room
25 25 end
26 26
27 27 def find_problem_home(test_request)
28 28 problem_name = test_request.problem_name
29 29
30 30 template_dir = "#{@config.test_request_problem_templates_dir}/" + problem_name
31 31
32 32 raise "Test Request: error template not found" if !File.exists?(template_dir)
33 33
34 34 problem_home = problem_home_dir(test_request)
35 35 FileUtils.mkdir_p(problem_home)
36 36
37 37 copy_problem_template(template_dir,problem_home)
38 38 link_input_file(test_request,problem_home)
39 39
40 40 problem_home
41 41 end
42 42
43 43 def save_source(test_request,source_name)
44 44 dir = self.produce_grading_room(test_request)
45 45 submission = test_request.submission
46 46 f = File.open("#{dir}/#{source_name}","w")
47 47 f.write(submission.source)
48 48 f.close
49 49 end
50 50
51 51 def clean_up(test_request)
52 52 problem_home = problem_home_dir(test_request)
53 53 remove_data_files(problem_home)
54 54 end
55 55
56 56 protected
57 57 def grading_room_dir(test_request)
58 58 problem_name = test_request.problem_name
59 59 user = test_request.user
60 60 "#{@config.user_result_dir}" +
61 61 "/#{user.login}/test_request" +
62 62 "/#{problem_name}/#{test_request.id}"
63 63 end
64 64
65 65 def problem_home_dir(test_request)
66 66 problem_name = test_request.problem_name
67 67 user = test_request.user
68 68 "#{@config.user_result_dir}" +
69 69 "/#{user.login}/test_request/#{problem_name}"
70 70 end
71 71
72 72 def copy_problem_template(template_dir,problem_home)
73 73 cmd = "cp -R #{template_dir}/* #{problem_home}"
74 74 system_and_raise_when_fail(cmd,"Test Request: cannot copy problem template")
75 75 end
76 76
77 77 def link_input_file(test_request,problem_home)
78 78 cmd = "ln -s #{test_request.input_file_name} #{problem_home}/test_cases/1/input-1.txt"
79 79 system_and_raise_when_fail(cmd,"Test Request: cannot link input file")
80 80 end
81 81
82 82 def remove_data_files(problem_home)
83 83 if File.exists?("#{problem_home}/test_cases/1/input-1.txt")
84 84 cmd = "rm #{problem_home}/test_cases/1/*"
85 85 system_and_raise_when_fail(cmd,"Test Request: cannot remove data files")
86 86 end
87 87 end
88 88
89 89 def system_and_raise_when_fail(cmd,msg)
90 90 if !system(cmd)
91 91 raise msg
92 92 end
93 93 end
94 94
95 95 end
96 96
97 97 class TestRequestReporter
98 98 def initialize
99 99 @config = Grader::Configuration.get_instance
100 100 end
101 101
102 102 def report(test_request,test_result_dir)
103 103 save_result(test_request,read_result(test_result_dir))
104 104 end
105 105
106 106 def report_error(test_request, msg)
107 - save_result(test_request, {:running_stat => "#{msg}"})
107 + save_result(test_request, {:running_stat => {:msg => "#{msg}"}})
108 108 end
109 109
110 110 protected
111 111 def read_result(test_result_dir)
112 112 # TODO:
113 113 cmp_msg_fname = "#{test_result_dir}/compiler_message"
114 114 cmp_file = File.open(cmp_msg_fname)
115 115 cmp_msg = cmp_file.read
116 116 cmp_file.close
117 117
118 118 result_file_name = "#{test_result_dir}/1/result"
119 119
120 120 if File.exists?(result_file_name)
121 121 output_file_name = "#{test_result_dir}/1/output.txt"
122 122 results = File.open("#{test_result_dir}/1/result").readlines
123 123 stat = format_running_stat(results)
124 124
125 125 return {
126 126 :output_file_name => output_file_name,
127 127 :running_stat => stat,
128 128 :comment => "",
129 129 :cmp_msg => cmp_msg}
130 130 else
131 131 return {
132 - :running_stat => "",
132 + :running_stat => nil,
133 133 :comment => "Compilation error",
134 134 :cmp_msg => cmp_msg}
135 135 end
136 136 end
137 137
138 138 def format_running_stat(results)
139 139 running_time_line = results[-1]
140 140
141 + # extract exit status line
141 142 run_stat = ""
142 143 if !(/[Cc]orrect/.match(results[0]))
143 144 run_stat = results[0].chomp
145 + else
146 + run_stat = 'Program exited normally'
144 147 end
145 148
149 + # extract running time
146 150 if res = /r(.*)u(.*)s/.match(running_time_line)
147 151 seconds = (res[1].to_f + res[2].to_f)
148 152 time_stat = "Time used: #{seconds} sec."
149 153 else
154 + seconds = nil
150 155 time_stat = "Time used: n/a sec."
151 156 end
152 - return "#{run_stat}\n#{time_stat}"
157 + return {
158 + :msg => "#{run_stat}\n#{time_stat}",
159 + :running_time => seconds,
160 + :exit_status => run_stat
161 + }
153 162 end
154 163
155 164 def save_result(test_request,result)
156 165 if result[:output_file_name]!=nil
157 166 test_request.output_file_name = link_output_file(test_request,
158 167 result[:output_file_name])
159 168 end
160 169 test_request.graded_at = Time.now
161 170 test_request.compiler_message = (result[:cmp_msg] or '')
162 171 test_request.grader_comment = (result[:comment] or '')
163 - test_request.running_stat = (result[:running_stat] or '')
172 + if result[:running_stat]!=nil
173 + test_request.running_stat = (result[:running_stat][:msg] or '')
174 + test_request.running_time = (result[:running_stat][:running_time] or nil)
175 + test_request.exit_status = (result[:running_stat][:exit_status])
176 + test_request.memory_usage = nil # should be added later
177 + else
178 + test_request.running_stat = ''
179 + end
164 180 test_request.save
165 181 end
166 182
167 183 protected
168 184 def link_output_file(test_request, fname)
169 185 target_file_name = random_output_file_name(test_request.user,
170 186 test_request.problem)
171 187 FileUtils.mkdir_p(File.dirname(target_file_name))
172 188 cmd = "ln -s #{fname} #{target_file_name}"
173 189 if !system(cmd)
174 190 raise "TestRequestReporter: cannot move output file"
175 191 end
176 192 return target_file_name
177 193 end
178 194
179 195 def random_output_file_name(user,problem)
180 196 problem_name = TestRequest.name_of(problem)
181 197 begin
182 198 tmpname = "#{@config.test_request_output_base_dir}" +
183 199 "/#{user.login}/#{problem_name}/#{rand(10000)}"
184 200 end while File.exists?(tmpname)
185 201 tmpname
186 202 end
187 203
188 204 end
189 205
190 206 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,21 +1,23
1 1 #include <stdio.h>
2 2 #include <stdlib.h>
3 + #include <unistd.h>
3 4
4 5 int main()
5 6 {
6 7 int a,b;
7 8
8 9 int c=0;
9 10
10 11 scanf("%d %d",&a,&b);
11 12 printf("%d\n",a+b);
12 13
13 - for(a=0; a<2; a++) {
14 - while(c<1550000000) {
14 + sleep(1);
15 +
16 + c = 0;
17 + while(c<1000000000) {
15 18 c++;
16 19 b+=c;
17 20 }
18 - }
19 21 exit(0);
20 22 }
21 23
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -79,202 +79,214
79 79 :and_report => {
80 80 :score => 10,
81 81 :comment => /FAILED: [^P]P/})
82 82 end
83 83
84 84 it "should score test runs correctly when submission fails in some test case" do
85 85 grader_should(:grade => "add_fail_test_case_1.c",
86 86 :on => @problem_test_normal,
87 87 :and_report => {
88 88 :score => 105,
89 89 :comment => /^FAILED:/})
90 90 end
91 91
92 92 it "should fail submission with non-zero exit status" do
93 93 grader_should(:grade => "add_nonzero_exit_status.c",
94 94 :on => @problem_test_normal,
95 95 :and_report => {
96 96 :score => 0,
97 97 :comment => /^FAILED:/})
98 98 end
99 99
100 100 it "should not allow malicious submission to see PROBLEM_HOME" do
101 101 problem_test_yesno = stub(Problem,
102 102 :id => 1, :name => 'test_yesno',
103 103 :full_score => 10)
104 104 grader_should(:grade => "yesno_access_problem_home.c",
105 105 :on => problem_test_yesno,
106 106 :and_report => {
107 107 :score => 0,
108 108 :comment => /^FAILED:/})
109 109 end
110 110
111 111 it "should not allow malicious submission to open files" do
112 112 problem_test_yesno = stub(Problem,
113 113 :id => 1, :name => 'test_yesno',
114 114 :full_score => 10)
115 115 grader_should(:grade => "yesno_open_file.c",
116 116 :on => problem_test_yesno,
117 117 :and_report => {
118 118 :score => 0,
119 119 :comment => /^FAILED:/})
120 120 end
121 121
122 122 def grader_should(args)
123 123 @user1 = stub(User,
124 124 :id => 1, :login => 'user1')
125 125 submission =
126 126 create_submission_from_file(1, @user1, args[:on], args[:grade])
127 127 submission.should_receive(:graded_at=)
128 128
129 129 expected_score = args[:and_report][:score]
130 130 expected_comment = args[:and_report][:comment]
131 131 if args[:and_report][:compiler_message]!=nil
132 132 expected_compiler_message = args[:and_report][:compiler_message]
133 133 else
134 134 expected_compiler_message = ''
135 135 end
136 136
137 137 submission.should_receive(:points=).with(expected_score)
138 138 submission.should_receive(:grader_comment=).with(expected_comment)
139 139 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 140 submission.should_receive(:save)
141 141
142 142 @engine.grade(submission)
143 143 end
144 144
145 145 protected
146 146
147 147 def create_normal_submission_mock_from_file(source_fname)
148 148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 149 end
150 150
151 151 end
152 152
153 153 describe "A grader engine, when grading test requests" do
154 154
155 155 include GraderEngineHelperMethods
156 156
157 157 before(:each) do
158 158 @config = Grader::Configuration.get_instance
159 159 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
160 160 Grader::TestRequestReporter.new)
161 161 init_sandbox
162 162 end
163 163
164 164 it "should report error if there is no problem template" do
165 165 problem = stub(Problem,
166 166 :id => 1, :name => 'nothing')
167 167 grader_should(:grade => 'test1_correct.c',
168 168 :on => problem,
169 169 :with => 'in1.txt',
170 170 :and_report => {
171 171 :graded_at= => nil,
172 172 :compiler_message= => '',
173 173 :grader_comment= => '',
174 174 :running_stat= => /template not found/,
175 + :running_time= => nil,
176 + :exit_status= => nil,
177 + :memory_usage= => nil,
175 178 :save => nil})
176 179 end
177 180
178 181 it "should run test request and produce output file" do
179 182 problem = stub(Problem,
180 183 :id => 1, :name => 'test_normal')
181 184 grader_should(:grade => 'test1_correct.c',
182 185 :on => problem,
183 186 :with => 'in1.txt',
184 187 :and_report => {
185 188 :graded_at= => nil,
186 189 :compiler_message= => '',
187 190 :grader_comment= => '',
188 191 :running_stat= => /0.0 sec./,
189 192 :output_file_name= => lambda { |fname|
190 193 File.exists?(fname).should be_true
191 194 },
195 + :running_time= => nil,
196 + :exit_status= => nil,
197 + :memory_usage= => nil,
192 198 :save => nil})
193 199 end
194 200
195 201 it "should clean up problem directory after running test request" do
196 202 problem = stub(Problem,
197 203 :id => 1, :name => 'test_normal')
198 204 grader_should(:grade => 'test1_correct.c',
199 205 :on => problem,
200 206 :with => 'in1.txt',
201 207 :and_report => {
202 208 :graded_at= => nil,
203 209 :compiler_message= => '',
204 210 :grader_comment= => '',
205 211 :running_stat= => nil,
206 212 :output_file_name= => nil,
213 + :running_time= => nil,
214 + :exit_status= => nil,
215 + :memory_usage= => nil,
207 216 :save => nil})
208 217 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
209 218 end
210 219
211 220 it "should compile test request with error and report compilation error" do
212 221 problem = stub(Problem,
213 222 :id => 1, :name => 'test_normal')
214 223 grader_should(:grade => 'test1_compile_error.c',
215 224 :on => problem,
216 225 :with => 'in1.txt',
217 226 :and_report => {
218 227 :graded_at= => nil,
219 228 :compiler_message= => /.+/,
220 229 :grader_comment= => /[Cc]ompil.*error/,
221 230 :running_stat= => '',
222 231 :save => nil})
223 232 end
224 233
225 234 it "should report exit status" do
226 235 problem = stub(Problem,
227 236 :id => 1, :name => 'test_normal')
228 237 grader_should(:grade => 'add_nonzero_exit_status.c',
229 238 :on => problem,
230 239 :with => 'in1.txt',
231 240 :and_report => {
232 241 :graded_at= => nil,
233 242 :compiler_message= => '',
234 243 :grader_comment= => '',
235 244 :running_stat= => /[Ee]xit.*status.*10.*0\.0 sec/m,
236 245 :output_file_name= => lambda { |fname|
237 246 File.exists?(fname).should be_true
238 247 },
248 + :running_time= => nil,
249 + :exit_status= => /10/,
250 + :memory_usage= => nil,
239 251 :save => nil})
240 252 end
241 253
242 254 protected
243 255 def grader_should(args)
244 256 @user1 = stub(User,
245 257 :id => 1, :login => 'user1')
246 258
247 259 problem = args[:on]
248 260 input_file = @config.test_request_input_base_dir + "/" + args[:with]
249 261
250 262 submission =
251 263 create_submission_from_file(1, @user1, args[:on], args[:grade])
252 264
253 265 test_request = stub(TestRequest,
254 266 :id => 1,
255 267 :user => @user1,
256 268 :problem => problem,
257 269 :submission => submission,
258 270 :input_file_name => input_file,
259 271 :language => submission.language,
260 272 :problem_name => problem.name)
261 273
262 274 expectations = args[:and_report]
263 275
264 276 expectations.each do |key,val|
265 277 if val==nil
266 278 test_request.should_receive(key)
267 279 elsif val.class == Proc
268 280 test_request.should_receive(key) { |fname|
269 281 val.call(fname)
270 282 }
271 283 else
272 284 test_request.should_receive(key).with(val)
273 285 end
274 286 end
275 287
276 288 @engine.grade(test_request)
277 289 end
278 290
279 291 end
280 292
You need to be logged in to leave comments. Login now