Description:
[grader] extracted test_request info into new stat columns
git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@163 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
Diff options:
Comments:
0 Commit comments
0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Author
jittat
-
r41:d4e37450ddbc - - 3 files changed: 39 inserted, 9 deleted
| @@ -83,105 +83,121 | |||
|
|
83 | 83 | if File.exists?("#{problem_home}/test_cases/1/input-1.txt") |
|
|
84 | 84 | cmd = "rm #{problem_home}/test_cases/1/*" |
|
|
85 | 85 | system_and_raise_when_fail(cmd,"Test Request: cannot remove data files") |
|
|
86 | 86 | end |
|
|
87 | 87 | end |
|
|
88 | 88 | |
|
|
89 | 89 | def system_and_raise_when_fail(cmd,msg) |
|
|
90 | 90 | if !system(cmd) |
|
|
91 | 91 | raise msg |
|
|
92 | 92 | end |
|
|
93 | 93 | end |
|
|
94 | 94 | |
|
|
95 | 95 | end |
|
|
96 | 96 | |
|
|
97 | 97 | class TestRequestReporter |
|
|
98 | 98 | def initialize |
|
|
99 | 99 | @config = Grader::Configuration.get_instance |
|
|
100 | 100 | end |
|
|
101 | 101 | |
|
|
102 | 102 | def report(test_request,test_result_dir) |
|
|
103 | 103 | save_result(test_request,read_result(test_result_dir)) |
|
|
104 | 104 | end |
|
|
105 | 105 | |
|
|
106 | 106 | def report_error(test_request, msg) |
|
|
107 | - save_result(test_request, {:running_stat => "#{msg}"}) | |
|
|
107 | + save_result(test_request, {:running_stat => {:msg => "#{msg}"}}) | |
|
|
108 | 108 | end |
|
|
109 | 109 | |
|
|
110 | 110 | protected |
|
|
111 | 111 | def read_result(test_result_dir) |
|
|
112 | 112 | # TODO: |
|
|
113 | 113 | cmp_msg_fname = "#{test_result_dir}/compiler_message" |
|
|
114 | 114 | cmp_file = File.open(cmp_msg_fname) |
|
|
115 | 115 | cmp_msg = cmp_file.read |
|
|
116 | 116 | cmp_file.close |
|
|
117 | 117 | |
|
|
118 | 118 | result_file_name = "#{test_result_dir}/1/result" |
|
|
119 | 119 | |
|
|
120 | 120 | if File.exists?(result_file_name) |
|
|
121 | 121 | output_file_name = "#{test_result_dir}/1/output.txt" |
|
|
122 | 122 | results = File.open("#{test_result_dir}/1/result").readlines |
|
|
123 | 123 | stat = format_running_stat(results) |
|
|
124 | 124 | |
|
|
125 | 125 | return { |
|
|
126 | 126 | :output_file_name => output_file_name, |
|
|
127 | 127 | :running_stat => stat, |
|
|
128 | 128 | :comment => "", |
|
|
129 | 129 | :cmp_msg => cmp_msg} |
|
|
130 | 130 | else |
|
|
131 | 131 | return { |
|
|
132 |
- :running_stat => |
|
|
|
132 | + :running_stat => nil, | |
|
|
133 | 133 | :comment => "Compilation error", |
|
|
134 | 134 | :cmp_msg => cmp_msg} |
|
|
135 | 135 | end |
|
|
136 | 136 | end |
|
|
137 | 137 | |
|
|
138 | 138 | def format_running_stat(results) |
|
|
139 | 139 | running_time_line = results[-1] |
|
|
140 | 140 | |
|
|
141 | + # extract exit status line | |
|
|
141 | 142 | run_stat = "" |
|
|
142 | 143 | if !(/[Cc]orrect/.match(results[0])) |
|
|
143 | 144 | run_stat = results[0].chomp |
|
|
145 | + else | |
|
|
146 | + run_stat = 'Program exited normally' | |
|
|
144 | 147 | end |
|
|
145 | 148 | |
|
|
149 | + # extract running time | |
|
|
146 | 150 | if res = /r(.*)u(.*)s/.match(running_time_line) |
|
|
147 | 151 | seconds = (res[1].to_f + res[2].to_f) |
|
|
148 | 152 | time_stat = "Time used: #{seconds} sec." |
|
|
149 | 153 | else |
|
|
154 | + seconds = nil | |
|
|
150 | 155 | time_stat = "Time used: n/a sec." |
|
|
151 | 156 | end |
|
|
152 | - return "#{run_stat}\n#{time_stat}" | |
|
|
157 | + return { | |
|
|
158 | + :msg => "#{run_stat}\n#{time_stat}", | |
|
|
159 | + :running_time => seconds, | |
|
|
160 | + :exit_status => run_stat | |
|
|
161 | + } | |
|
|
153 | 162 | end |
|
|
154 | 163 | |
|
|
155 | 164 | def save_result(test_request,result) |
|
|
156 | 165 | if result[:output_file_name]!=nil |
|
|
157 | 166 | test_request.output_file_name = link_output_file(test_request, |
|
|
158 | 167 | result[:output_file_name]) |
|
|
159 | 168 | end |
|
|
160 | 169 | test_request.graded_at = Time.now |
|
|
161 | 170 | test_request.compiler_message = (result[:cmp_msg] or '') |
|
|
162 | 171 | test_request.grader_comment = (result[:comment] or '') |
|
|
163 | - test_request.running_stat = (result[:running_stat] or '') | |
|
|
172 | + if result[:running_stat]!=nil | |
|
|
173 | + test_request.running_stat = (result[:running_stat][:msg] or '') | |
|
|
174 | + test_request.running_time = (result[:running_stat][:running_time] or nil) | |
|
|
175 | + test_request.exit_status = (result[:running_stat][:exit_status]) | |
|
|
176 | + test_request.memory_usage = nil # should be added later | |
|
|
177 | + else | |
|
|
178 | + test_request.running_stat = '' | |
|
|
179 | + end | |
|
|
164 | 180 | test_request.save |
|
|
165 | 181 | end |
|
|
166 | 182 | |
|
|
167 | 183 | protected |
|
|
168 | 184 | def link_output_file(test_request, fname) |
|
|
169 | 185 | target_file_name = random_output_file_name(test_request.user, |
|
|
170 | 186 | test_request.problem) |
|
|
171 | 187 | FileUtils.mkdir_p(File.dirname(target_file_name)) |
|
|
172 | 188 | cmd = "ln -s #{fname} #{target_file_name}" |
|
|
173 | 189 | if !system(cmd) |
|
|
174 | 190 | raise "TestRequestReporter: cannot move output file" |
|
|
175 | 191 | end |
|
|
176 | 192 | return target_file_name |
|
|
177 | 193 | end |
|
|
178 | 194 | |
|
|
179 | 195 | def random_output_file_name(user,problem) |
|
|
180 | 196 | problem_name = TestRequest.name_of(problem) |
|
|
181 | 197 | begin |
|
|
182 | 198 | tmpname = "#{@config.test_request_output_base_dir}" + |
|
|
183 | 199 | "/#{user.login}/#{problem_name}/#{rand(10000)}" |
|
|
184 | 200 | end while File.exists?(tmpname) |
|
|
185 | 201 | tmpname |
|
|
186 | 202 | end |
|
|
187 | 203 | |
| @@ -1,21 +1,23 | |||
|
|
1 | 1 | #include <stdio.h> |
|
|
2 | 2 | #include <stdlib.h> |
|
|
3 | + #include <unistd.h> | |
|
|
3 | 4 | |
|
|
4 | 5 | int main() |
|
|
5 | 6 | { |
|
|
6 | 7 | int a,b; |
|
|
7 | 8 | |
|
|
8 | 9 | int c=0; |
|
|
9 | 10 | |
|
|
10 | 11 | scanf("%d %d",&a,&b); |
|
|
11 | 12 | printf("%d\n",a+b); |
|
|
12 | 13 | |
|
|
13 | - for(a=0; a<2; a++) { | |
|
|
14 | - while(c<1550000000) { | |
|
|
15 | - c++; | |
|
|
16 | - b+=c; | |
|
|
17 |
- |
|
|
|
14 | + sleep(1); | |
|
|
15 | + | |
|
|
16 | + c = 0; | |
|
|
17 | + while(c<1000000000) { | |
|
|
18 | + c++; | |
|
|
19 | + b+=c; | |
|
|
18 | 20 | } |
|
|
19 | 21 | exit(0); |
|
|
20 | 22 | } |
|
|
21 | 23 | |
| @@ -151,112 +151,124 | |||
|
|
151 | 151 | end |
|
|
152 | 152 | |
|
|
153 | 153 | describe "A grader engine, when grading test requests" do |
|
|
154 | 154 | |
|
|
155 | 155 | include GraderEngineHelperMethods |
|
|
156 | 156 | |
|
|
157 | 157 | before(:each) do |
|
|
158 | 158 | @config = Grader::Configuration.get_instance |
|
|
159 | 159 | @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new, |
|
|
160 | 160 | Grader::TestRequestReporter.new) |
|
|
161 | 161 | init_sandbox |
|
|
162 | 162 | end |
|
|
163 | 163 | |
|
|
164 | 164 | it "should report error if there is no problem template" do |
|
|
165 | 165 | problem = stub(Problem, |
|
|
166 | 166 | :id => 1, :name => 'nothing') |
|
|
167 | 167 | grader_should(:grade => 'test1_correct.c', |
|
|
168 | 168 | :on => problem, |
|
|
169 | 169 | :with => 'in1.txt', |
|
|
170 | 170 | :and_report => { |
|
|
171 | 171 | :graded_at= => nil, |
|
|
172 | 172 | :compiler_message= => '', |
|
|
173 | 173 | :grader_comment= => '', |
|
|
174 | 174 | :running_stat= => /template not found/, |
|
|
175 | + :running_time= => nil, | |
|
|
176 | + :exit_status= => nil, | |
|
|
177 | + :memory_usage= => nil, | |
|
|
175 | 178 | :save => nil}) |
|
|
176 | 179 | end |
|
|
177 | 180 | |
|
|
178 | 181 | it "should run test request and produce output file" do |
|
|
179 | 182 | problem = stub(Problem, |
|
|
180 | 183 | :id => 1, :name => 'test_normal') |
|
|
181 | 184 | grader_should(:grade => 'test1_correct.c', |
|
|
182 | 185 | :on => problem, |
|
|
183 | 186 | :with => 'in1.txt', |
|
|
184 | 187 | :and_report => { |
|
|
185 | 188 | :graded_at= => nil, |
|
|
186 | 189 | :compiler_message= => '', |
|
|
187 | 190 | :grader_comment= => '', |
|
|
188 | 191 | :running_stat= => /0.0 sec./, |
|
|
189 | 192 | :output_file_name= => lambda { |fname| |
|
|
190 | 193 | File.exists?(fname).should be_true |
|
|
191 | 194 | }, |
|
|
195 | + :running_time= => nil, | |
|
|
196 | + :exit_status= => nil, | |
|
|
197 | + :memory_usage= => nil, | |
|
|
192 | 198 | :save => nil}) |
|
|
193 | 199 | end |
|
|
194 | 200 | |
|
|
195 | 201 | it "should clean up problem directory after running test request" do |
|
|
196 | 202 | problem = stub(Problem, |
|
|
197 | 203 | :id => 1, :name => 'test_normal') |
|
|
198 | 204 | grader_should(:grade => 'test1_correct.c', |
|
|
199 | 205 | :on => problem, |
|
|
200 | 206 | :with => 'in1.txt', |
|
|
201 | 207 | :and_report => { |
|
|
202 | 208 | :graded_at= => nil, |
|
|
203 | 209 | :compiler_message= => '', |
|
|
204 | 210 | :grader_comment= => '', |
|
|
205 | 211 | :running_stat= => nil, |
|
|
206 | 212 | :output_file_name= => nil, |
|
|
213 | + :running_time= => nil, | |
|
|
214 | + :exit_status= => nil, | |
|
|
215 | + :memory_usage= => nil, | |
|
|
207 | 216 | :save => nil}) |
|
|
208 | 217 | File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false |
|
|
209 | 218 | end |
|
|
210 | 219 | |
|
|
211 | 220 | it "should compile test request with error and report compilation error" do |
|
|
212 | 221 | problem = stub(Problem, |
|
|
213 | 222 | :id => 1, :name => 'test_normal') |
|
|
214 | 223 | grader_should(:grade => 'test1_compile_error.c', |
|
|
215 | 224 | :on => problem, |
|
|
216 | 225 | :with => 'in1.txt', |
|
|
217 | 226 | :and_report => { |
|
|
218 | 227 | :graded_at= => nil, |
|
|
219 | 228 | :compiler_message= => /.+/, |
|
|
220 | 229 | :grader_comment= => /[Cc]ompil.*error/, |
|
|
221 | 230 | :running_stat= => '', |
|
|
222 | 231 | :save => nil}) |
|
|
223 | 232 | end |
|
|
224 | 233 | |
|
|
225 | 234 | it "should report exit status" do |
|
|
226 | 235 | problem = stub(Problem, |
|
|
227 | 236 | :id => 1, :name => 'test_normal') |
|
|
228 | 237 | grader_should(:grade => 'add_nonzero_exit_status.c', |
|
|
229 | 238 | :on => problem, |
|
|
230 | 239 | :with => 'in1.txt', |
|
|
231 | 240 | :and_report => { |
|
|
232 | 241 | :graded_at= => nil, |
|
|
233 | 242 | :compiler_message= => '', |
|
|
234 | 243 | :grader_comment= => '', |
|
|
235 | 244 | :running_stat= => /[Ee]xit.*status.*10.*0\.0 sec/m, |
|
|
236 | 245 | :output_file_name= => lambda { |fname| |
|
|
237 | 246 | File.exists?(fname).should be_true |
|
|
238 | 247 | }, |
|
|
248 | + :running_time= => nil, | |
|
|
249 | + :exit_status= => /10/, | |
|
|
250 | + :memory_usage= => nil, | |
|
|
239 | 251 | :save => nil}) |
|
|
240 | 252 | end |
|
|
241 | 253 | |
|
|
242 | 254 | protected |
|
|
243 | 255 | def grader_should(args) |
|
|
244 | 256 | @user1 = stub(User, |
|
|
245 | 257 | :id => 1, :login => 'user1') |
|
|
246 | 258 | |
|
|
247 | 259 | problem = args[:on] |
|
|
248 | 260 | input_file = @config.test_request_input_base_dir + "/" + args[:with] |
|
|
249 | 261 | |
|
|
250 | 262 | submission = |
|
|
251 | 263 | create_submission_from_file(1, @user1, args[:on], args[:grade]) |
|
|
252 | 264 | |
|
|
253 | 265 | test_request = stub(TestRequest, |
|
|
254 | 266 | :id => 1, |
|
|
255 | 267 | :user => @user1, |
|
|
256 | 268 | :problem => problem, |
|
|
257 | 269 | :submission => submission, |
|
|
258 | 270 | :input_file_name => input_file, |
|
|
259 | 271 | :language => submission.language, |
|
|
260 | 272 | :problem_name => problem.name) |
|
|
261 | 273 | |
|
|
262 | 274 | expectations = args[:and_report] |
You need to be logged in to leave comments.
Login now