cafe-grader-judge-scripts

Commit r181:5c6f0b732e70 public

parent | child
Description:
handle the case when problem id or submission id is null. Grader will simply skip such request. Add more report on console (for command line grading) (mercurial grafted from d233105d3965c5368c9b33125f390e39b25f910e)
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r181:5c6f0b732e70 - - 2 files changed: 30 inserted, 30 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,443 +1,443
1 1 #!/usr/bin/env ruby
2 2
3 3 def stop_grader(id)
4 4 if id==:all
5 5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
6 6 else
7 7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
8 8 end
9 9 end
10 10
11 11 def check_stopfile
12 12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
13 13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
14 14 end
15 15
16 16 def clear_stopfile
17 17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 18 File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
19 19 end
20 20 end
21 21
22 22 def config
23 23 Grader::Configuration.get_instance
24 24 end
25 25
26 26 def log_file_name
27 27 if !File.exists?(config.log_dir)
28 28 raise "Log directory does not exist: #{config.log_dir}"
29 29 end
30 30 config.log_dir +
31 31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
32 32 end
33 33
34 34 def log(str)
35 35 if config.talkative
36 36 puts str
37 37 end
38 38 if config.logging
39 39 fp = File.open(log_file_name,"a")
40 40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
41 41 fp.close
42 42 end
43 43 end
44 44
45 45 def display_manual
46 46 puts <<USAGE
47 47 Grader.
48 48 using: (1) grader
49 49 (2) grader environment [mode]
50 50 (3) grader stop [all|pids-list]
51 51 (4) grader --help
52 52 (1) call grader with environment = 'exam', mode = 'queue'
53 53 (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
54 54 (3) create stop-file to stop running grader in queue mode
55 55 (4) You are here.
56 56 USAGE
57 57 end
58 58
59 59 def process_options_and_stop_file
60 60 # The list of options are:
61 61 # - stop [all|process ids]
62 62 # -
63 63
64 64 # Process 'help' option
65 65 if (ARGV.length==1) and (/help/.match(ARGV[0]))
66 66 display_manual
67 67 exit(0)
68 68 end
69 69
70 70 # Process 'stop' option.
71 71 if (ARGV.length >= 1) and (ARGV[0]=='stop')
72 72 if ARGV.length==1
73 73 puts "you should specify pid-list or 'all'"
74 74 display_manual
75 75 elsif (ARGV.length==2) and (ARGV[1]=='all')
76 76 stop_grader(:all)
77 77 puts "A global stop file ('stop.all') created."
78 78 puts "You should remove it manually later."
79 79 else
80 80 (1..ARGV.length-1).each do |i|
81 81 stop_grader(ARGV[i])
82 82 end
83 83 puts "stop file(s) created"
84 84 end
85 85 exit(0)
86 86 end
87 87
88 88 # Check stop file.
89 89 if check_stopfile
90 90 puts "Stop file exists. Terminated."
91 91 clear_stopfile
92 92 exit(0)
93 93 end
94 94
95 95 #default options
96 96 options = {
97 97 :mode => 'queue',
98 98 :environment => 'exam',
99 99 :dry_run => false,
100 100 }
101 101
102 102 # Process mode and environment option
103 103 if ARGV.length >= 1
104 104 options[:environment] = ARGV.shift
105 105 if ARGV.length >=1
106 106 options[:mode] = ARGV.shift
107 107 end
108 108 end
109 109
110 110 options[:dry_run] = (ARGV.delete('--dry') != nil)
111 111 if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
112 112 puts "Dry run currently works only for 'prob' or 'contest' modes."
113 113 exit(0)
114 114 end
115 115
116 116 options[:report] = (ARGV.delete('--report') != nil)
117 117 if options[:report] and (not ['prob','contest','autonew'].include? options[:mode])
118 118 puts "Report currently works only for 'prob' or 'contest' modes."
119 119 exit(0)
120 120 end
121 121
122 122 return options
123 123 end
124 124
125 125 class ResultCollector
126 126 def initialize
127 127 @results = {}
128 128 @problems = {}
129 129 @users = {}
130 130 end
131 131
132 132 def after_save_hook(submission, grading_result)
133 133 end
134 134
135 135 def save(submission, grading_result)
136 136 user = submission.user
137 137 problem = submission.problem
138 138 if not @problems.has_key? problem.id
139 139 @problems[problem.id] = problem
140 140 end
141 141 if not @users.has_key? user.id
142 142 @users[user.id] = user
143 143 end
144 144 @results[[user.id, problem.id]] = grading_result
145 145
146 146 after_save_hook(submission, grading_result)
147 147 end
148 148
149 149 def print_report_by_user
150 150 puts "---------------------"
151 151 puts " REPORT"
152 152 puts "---------------------"
153 153
154 154 print "login,email"
155 155 @problems.each_value do |problem|
156 156 print ",#{problem.name}"
157 157 end
158 158 print "\n"
159 159
160 160 @users.each_value do |user|
161 161 print "#{user.login},#{user.email}"
162 162 @problems.each_value do |problem|
163 163 if @results.has_key? [user.id, problem.id]
164 164 print ",#{@results[[user.id,problem.id]][:points]}"
165 165 else
166 166 print ","
167 167 end
168 168 end
169 169 print "\n"
170 170 end
171 171 end
172 172 end
173 173
174 174 def grader_general_loop(engine, grader_proc, options)
175 175 runner = Grader::Runner.new(engine, grader_proc)
176 176 while true
177 177
178 178 if check_stopfile # created by calling grader stop
179 179 clear_stopfile
180 180 log "stopped (with stop file)"
181 181 break
182 182 end
183 183
184 184 task = yield(runner)
185 185
186 186 if task==nil
187 187 sleep(1)
188 188 end
189 189 end
190 190 end
191 191
192 192 def grader_queue_loop(grader_proc, options)
193 193 log "Grader: queue"
194 194 engine = Grader::Engine.new
195 195 grader_general_loop(engine, grader_proc, options) do |runner|
196 196 runner.grade_oldest_task
197 197 end
198 198 end
199 199
200 200 def grader_test_request_loop(grader_proc, options)
201 201 log "Grader: test_request"
202 202 engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
203 203 :reporter => Grader::TestRequestReporter.new)
204 204 grader_general_loop(engine, grader_proc, options) do |runner|
205 205 runner.grade_oldest_test_request
206 206 end
207 207 end
208 208
209 209 def grader_autonew_loop(grader_proc, options)
210 210 log "Grader: autonew"
211 211
212 212 if options[:report]
213 213 result_collector = ResultCollector.new
214 214 else
215 215 result_collector = nil
216 216 end
217 217
218 218 if options[:dry_run]
219 219 puts "Running in dry mode"
220 220 end
221 221
222 222 prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
223 223 :result_collector => result_collector)
224 224
225 225 engine = Grader::Engine.new(:reporter => prob_reporter)
226 226 runner = Grader::Runner.new(engine, grader_proc)
227 227
228 228 grader_proc.report_active if grader_proc!=nil
229 229
230 230 latest_submitted_at = nil
231 231 graded_submission_ids = {}
232 232
233 233 while true
234 234
235 235 if check_stopfile # created by calling grader stop
236 236 clear_stopfile
237 237 log "stopped (with stop file)"
238 238 break
239 239 end
240 240
241 241 if latest_submitted_at==nil
242 242 submissions = Submission.all
243 243 else
244 244 submissions = Submission.all(:conditions => ["submitted_at >= :latest",
245 245 {:latest => latest_submitted_at}])
246 246 end
247 247
248 248 graded_any = false
249 249
250 250 if submissions.length != 0
251 251 submissions.each do |submission|
252 252 if (submission.problem == nil) or (!submission.problem.available)
253 253 next
254 254 end
255 255 if ! graded_submission_ids[submission.id]
256 256 runner.grade_submission(submission)
257 257 graded_submission_ids[submission.id] = true
258 258 if (!latest_submitted_at or
259 259 latest_submitted_at < submission.submitted_at)
260 260 latest_submitted_at = submission.submitted_at
261 261 end
262 262 puts "graded: #{submission.id}"
263 263 puts "latest: #{latest_submitted_at}"
264 264 graded_any = true
265 265 end
266 266 end
267 267 end
268 268
269 269 if ! graded_any
270 270 sleep(1)
271 271 end
272 272 end
273 273 end
274 274
275 275 def grader_grade_problems(grader_proc, options)
276 276 if options[:report]
277 277 result_collector = ResultCollector.new
278 278 else
279 279 result_collector = nil
280 280 end
281 281
282 282 if options[:dry_run]
283 283 puts "Running in dry mode"
284 284 end
285 285
286 286 prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
287 287 :result_collector => result_collector)
288 288 engine = Grader::Engine.new(:reporter => prob_reporter)
289 289 runner = Grader::Runner.new(engine, grader_proc)
290 290
291 291 grader_proc.report_active if grader_proc!=nil
292 292
293 293 ARGV.each do |prob_name|
294 294 prob = Problem.find_by_name(prob_name)
295 295 if prob==nil
296 296 puts "cannot find problem: #{prob_name}"
297 297 else
298 298 runner.grade_problem(prob)
299 299 end
300 300 end
301 301
302 302 if options[:report]
303 303 result_collector.print_report_by_user
304 304 end
305 305 end
306 306
307 307 def grader_grade_contests(grader_proc, options)
308 308 # always use dry run when grading during contest
309 309 dry_run = options[:dry_run] = true
310 310
311 311 contest_name = ARGV.shift
312 312
313 313 contest = Contest.find_by_name(contest_name)
314 314 if contest==nil
315 315 puts "cannot find contest: #{contest_name}"
316 316 exit(0)
317 317 end
318 318
319 319 if options[:report]
320 320 result_collector = ResultCollector.new
321 321 else
322 322 result_collector = nil
323 323 end
324 324
325 325 if options[:dry_run]
326 326 puts "Running in dry mode"
327 327 end
328 328
329 329 prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
330 330 :result_collector => result_collector)
331 331 engine = Grader::Engine.new(:reporter => prob_reporter)
332 332 runner = Grader::Runner.new(engine, grader_proc)
333 333
334 334 grader_proc.report_active if grader_proc!=nil
335 335
336 336 contest.problems.each do |problem|
337 337 puts "Grading: #{problem.name}"
338 338 runner.grade_problem(problem,
339 339 :user_conditions => lambda do |u|
340 340 u.contest_finished? and
341 341 u.contest_ids.include?(contest.id)
342 342 end)
343 343 end
344 344
345 345 if options[:report]
346 346 result_collector.print_report_by_user
347 347 end
348 348 end
349 349
350 350 def grader_grade_submissions(grader_proc, options)
351 351 engine = Grader::Engine.new
352 352 runner = Grader::Runner.new(engine, grader_proc)
353 353
354 354 grader_proc.report_active if grader_proc!=nil
355 355
356 356 ARGV.each do |sub_id|
357 357 puts "Grading #{sub_id}"
358 358 begin
359 359 submission = Submission.find(sub_id.to_i)
360 360 rescue ActiveRecord::RecordNotFound
361 - puts "Record not found"
361 + puts "Submission #{sub_id} not found"
362 362 submission = nil
363 363 end
364 364
365 365 if submission!=nil
366 366 runner.grade_submission(submission)
367 367 end
368 368 end
369 369 end
370 370
371 371 #########################################
372 372 # main program
373 373 #########################################
374 374
375 375 options = process_options_and_stop_file
376 376 GRADER_ENV = options[:environment]
377 377 grader_mode = options[:mode]
378 378 dry_run = options[:dry_run]
379 379
380 380 puts "environment: #{GRADER_ENV}"
381 381 require File.join(File.dirname(__FILE__),'config/environment')
382 382
383 383 # add grader_mode to config
384 384 # this is needed because method log needs it. TODO: clean this up
385 385 class << config
386 386 attr_accessor :grader_mode
387 387 end
388 388 config.grader_mode = grader_mode
389 389
390 390 # reading rails environment
391 391 log 'Reading rails environment'
392 392
393 393 RAILS_ENV = config.rails_env
394 394 require RAILS_ROOT + '/config/environment'
395 395
396 396 # register grader process
397 397 if config.report_grader
398 398 grader_proc = GraderProcess.register(config.grader_hostname,
399 399 Process.pid,
400 400 grader_mode)
401 401 else
402 402 grader_proc = nil
403 403 end
404 404
405 405 #set loggin environment
406 406 ENV['GRADER_LOGGING'] = log_file_name
407 407
408 408 # register exit handler to report inactive, and terminated
409 409 at_exit do
410 410 if grader_proc!=nil
411 411 grader_proc.report_inactive
412 412 grader_proc.terminate
413 413 end
414 414 end
415 415
416 416 #
417 417 # MAIN LOOP
418 418 #
419 419
420 420 case grader_mode
421 421 when "queue"
422 422 grader_queue_loop(grader_proc, options)
423 423
424 424 when "test_request"
425 425 grader_test_request_loop(grader_proc, options)
426 426
427 427 when "prob"
428 428 grader_grade_problems(grader_proc, options)
429 429
430 430 when "contest"
431 431 grader_grade_contests(grader_proc, options)
432 432
433 433 when "sub"
434 434 grader_grade_submissions(grader_proc, options)
435 435
436 436 when "autonew"
437 437 grader_autonew_loop(grader_proc, options)
438 438
439 439 else
440 440 display_manual
441 441 exit(0)
442 442 end
443 443
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,190 +1,190
1 1 require 'fileutils'
2 2 require File.join(File.dirname(__FILE__),'dir_init')
3 3
4 4 module Grader
5 5
6 6 #
7 7 # A grader engine grades a submission, against anything: a test
8 8 # data, or a user submitted test data. It uses two helpers objects:
9 9 # room_maker and reporter.
10 10 #
11 11 class Engine
12 12
13 13 attr_writer :room_maker
14 14 attr_writer :reporter
15 15
16 16 def initialize(options={})
17 17 # default options
18 18 if not options.include? :room_maker
19 19 options[:room_maker] = Grader::SubmissionRoomMaker.new
20 20 end
21 21 if not options.include? :reporter
22 22 options[:reporter] = Grader::SubmissionReporter.new
23 23 end
24 24
25 25 @config = Grader::Configuration.get_instance
26 26
27 27 @room_maker = options[:room_maker]
28 28 @reporter = options[:reporter]
29 29 end
30 30
31 31 # takes a submission, asks room_maker to produce grading directories,
32 32 # calls grader scripts, and asks reporter to save the result
33 33 def grade(submission)
34 34 current_dir = FileUtils.pwd
35 35
36 36 user = submission.user
37 37 problem = submission.problem
38 38
39 - # TODO: will have to create real exception for this
40 - if user==nil or problem == nil
41 - @reporter.report_error(submission,"Grading error: problem with submission")
42 - #raise "engine: user or problem is nil"
43 - end
44 -
45 - # TODO: this is another hack so that output only task can be judged
46 - if submission.language!=nil
47 - language = submission.language.name
48 - lang_ext = submission.language.ext
49 - else
50 - language = 'c'
51 - lang_ext = 'c'
52 - end
39 + begin
40 + # TODO: will have to create real exception for this
41 + if user==nil or problem == nil
42 + @reporter.report_error(submission,"Grading error: problem with submission")
43 + raise "engine: user or problem is nil"
44 + end
53 45
54 - # This is needed because older version of std-scripts/compile
55 - # only look for c++.
56 - if language == 'cpp'
57 - language = 'c++'
58 - end
46 + # TODO: this is another hack so that output only task can be judged
47 + if submission.language!=nil
48 + language = submission.language.name
49 + lang_ext = submission.language.ext
50 + else
51 + language = 'c'
52 + lang_ext = 'c'
53 + end
59 54
60 - # COMMENT: should it be only source.ext?
61 - if problem!=nil
62 - source_name = "#{problem.name}.#{lang_ext}"
63 - else
64 - source_name = "source.#{lang_ext}"
65 - end
55 + # This is needed because older version of std-scripts/compile
56 + # only look for c++.
57 + if language == 'cpp'
58 + language = 'c++'
59 + end
66 60
67 - begin
61 + # COMMENT: should it be only source.ext?
62 + if problem!=nil
63 + source_name = "#{problem.name}.#{lang_ext}"
64 + else
65 + source_name = "source.#{lang_ext}"
66 + end
67 +
68 68 grading_dir = @room_maker.produce_grading_room(submission)
69 69 @room_maker.save_source(submission,source_name)
70 70 problem_home = @room_maker.find_problem_home(submission)
71 71
72 72 # puts "GRADING DIR: #{grading_dir}"
73 73 # puts "PROBLEM DIR: #{problem_home}"
74 74
75 75 if !FileTest.exist?(problem_home)
76 76 puts "PROBLEM DIR: #{problem_home}"
77 - puts "No test data. (check problem dir)"
78 - raise "No test data."
77 + raise "engine: No test data."
79 78 end
80 79
81 80 dinit = DirInit::Manager.new(problem_home)
82 81
83 82 dinit.setup do
84 83 copy_log = copy_script(problem_home)
85 84 save_copy_log(problem_home,copy_log)
86 85 end
87 86
88 87 call_judge(problem_home,language,grading_dir,source_name)
89 88
90 89 @reporter.report(submission,"#{grading_dir}/test-result")
91 90
92 91 dinit.teardown do
93 92 copy_log = load_copy_log(problem_home)
94 93 clear_copy_log(problem_home)
95 94 clear_script(copy_log,problem_home)
96 95 end
97 96
98 97 rescue RuntimeError => msg
99 98 @reporter.report_error(submission, msg)
99 + puts "ERROR: #{msg}"
100 100
101 101 ensure
102 102 @room_maker.clean_up(submission)
103 103 Dir.chdir(current_dir) # this is really important
104 104 end
105 105 end
106 106
107 107 protected
108 108
109 109 def talk(str)
110 110 if @config.talkative
111 111 puts str
112 112 end
113 113 end
114 114
115 115 def call_judge(problem_home,language,grading_dir,fname)
116 116 ENV['PROBLEM_HOME'] = problem_home
117 117 ENV['RUBYOPT'] = ''
118 118
119 119 talk grading_dir
120 120 Dir.chdir grading_dir
121 121 cmd = "#{problem_home}/script/judge #{language} #{fname}"
122 122 talk "CMD: #{cmd}"
123 123 system(cmd)
124 124 end
125 125
126 126 def get_std_script_dir
127 127 GRADER_ROOT + '/std-script'
128 128 end
129 129
130 130 def copy_script(problem_home)
131 131 script_dir = "#{problem_home}/script"
132 132 std_script_dir = get_std_script_dir
133 133
134 - raise "std-script directory not found" if !FileTest.exist?(std_script_dir)
134 + raise "engine: std-script directory not found" if !FileTest.exist?(std_script_dir)
135 135
136 136 scripts = Dir[std_script_dir + '/*']
137 137
138 138 copied = []
139 139
140 140 scripts.each do |s|
141 141 fname = File.basename(s)
142 142 next if FileTest.directory?(s)
143 143 if !FileTest.exist?("#{script_dir}/#{fname}")
144 144 copied << fname
145 145 FileUtils.cp(s, "#{script_dir}", :preserve => true)
146 146 end
147 147 end
148 148
149 149 return copied
150 150 end
151 151
152 152 def copy_log_filename(problem_home)
153 153 return File.join(problem_home, '.scripts_copied')
154 154 end
155 155
156 156 def save_copy_log(problem_home, log)
157 157 f = File.new(copy_log_filename(problem_home),"w")
158 158 log.each do |fname|
159 159 f.write("#{fname}\n")
160 160 end
161 161 f.close
162 162 end
163 163
164 164 def load_copy_log(problem_home)
165 165 f = File.new(copy_log_filename(problem_home),"r")
166 166 log = []
167 167 f.readlines.each do |line|
168 168 log << line.strip
169 169 end
170 170 f.close
171 171 log
172 172 end
173 173
174 174 def clear_copy_log(problem_home)
175 175 File.delete(copy_log_filename(problem_home))
176 176 end
177 177
178 178 def clear_script(log,problem_home)
179 179 log.each do |s|
180 180 FileUtils.rm("#{problem_home}/script/#{s}")
181 181 end
182 182 end
183 183
184 184 def mkdir_if_does_not_exist(dirname)
185 185 Dir.mkdir(dirname) if !FileTest.exist?(dirname)
186 186 end
187 187
188 188 end
189 189
190 190 end
You need to be logged in to leave comments. Login now