cafe-grader-judge-scripts

Commit r175:29d3adfcaa1c public

parent | child
Description:
merge
Commit status:
[Not Reviewed]
References:
merge java
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r175:29d3adfcaa1c - - 3 files changed: 17 inserted, 3 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,232 +1,237
1 1 #!/usr/bin/env ruby
2 2
3 3 def stop_grader(id)
4 4 if id==:all
5 5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
6 6 else
7 7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
8 8 end
9 9 end
10 10
11 11 def check_stopfile
12 12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
13 13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
14 14 end
15 15
16 16 def clear_stopfile
17 17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 18 File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
19 19 end
20 20 end
21 21
22 22 def config
23 23 Grader::Configuration.get_instance
24 24 end
25 25
26 26 def log_file_name
27 27 if !File.exists?(config.log_dir)
28 28 raise "Log directory does not exist: #{config.log_dir}"
29 29 end
30 30 config.log_dir +
31 31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
32 32 end
33 33
34 34 def log(str)
35 35 if config.talkative
36 36 puts str
37 37 end
38 38 if config.logging
39 39 fp = File.open(log_file_name,"a")
40 40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
41 41 fp.close
42 42 end
43 43 end
44 44
45 45 def display_manual
46 46 puts <<USAGE
47 47 Grader.
48 48 using: (1) grader
49 49 (2) grader environment [mode] [options]
50 50 (3) grader stop [all|pids-list]
51 51 (4) grader --help
52 52 (1) call grader with environment = 'exam', mode = 'queue'
53 53 (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
54 54 queue: repeatedly check the task queue and grade any available tasks
55 55
56 56 prob: re-grade every user latest submission of the specific problem.
57 57 the problem name must be specified by the next argument.
58 58
59 59 additional options:
60 + --all-sub re-grade every submissions instead of just the latest submission of each user.
60 61
61 - --all-sub re-grade every submissions instead of just the latest submission of each user.
62 62 sub: re-grader the specified submission.
63 63 The submission ID to be re-graded must be specified by the next argument.
64 64
65 + options:
66 + --err-log log error to a file in the log dir
67 +
65 68 (3) create stop-file to stop running grader in queue mode
66 69 (4) You are here.
67 70 USAGE
68 71 end
69 72
70 73 def process_options_and_stop_file
71 74 # The list of options are:
72 75 # - stop [all|process ids]
73 76 # -
74 77
75 78 # Process 'help' option
76 79 if (ARGV.length==1) and (/help/.match(ARGV[0]))
77 80 display_manual
78 81 exit(0)
79 82 end
80 83
81 84 # Process 'stop' option.
82 85 if (ARGV.length >= 1) and (ARGV[0]=='stop')
83 86 if ARGV.length==1
84 87 puts "you should specify pid-list or 'all'"
85 88 display_manual
86 89 elsif (ARGV.length==2) and (ARGV[1]=='all')
87 90 stop_grader(:all)
88 91 puts "A global stop file ('stop.all') created."
89 92 puts "You should remove it manually later."
90 93 else
91 94 (1..ARGV.length-1).each do |i|
92 95 stop_grader(ARGV[i])
93 96 end
94 97 puts "stop file(s) created"
95 98 end
96 99 exit(0)
97 100 end
98 101
99 102 # Check stop file.
100 103 if check_stopfile
101 104 puts "Stop file exists. Terminated."
102 105 clear_stopfile
103 106 exit(0)
104 107 end
105 108
106 109 #default options
107 110 options = {
108 111 :mode => 'queue',
109 112 :environment => 'exam',
110 113 :dry_run => false,
111 114 }
112 115
113 116 # Process mode and environment option
114 117 if ARGV.length >= 1
115 118 options[:environment] = ARGV.shift
116 119 if ARGV.length >=1
117 120 options[:mode] = ARGV.shift
118 121 end
119 122 else
120 123 puts 'no argument specified, using default mode and environment.'
121 124 end
122 125
123 126 options[:dry_run] = (ARGV.delete('--dry') != nil)
124 127 if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
125 128 puts "Dry run currently works only for 'prob' or 'contest' modes."
126 129 exit(0)
127 130 end
128 131
129 132 options[:report] = (ARGV.delete('--report') != nil)
130 133 if options[:report] and (not ['prob','contest','autonew'].include? options[:mode])
131 134 puts "Report currently works only for 'prob' or 'contest' modes."
132 135 exit(0)
133 136 end
134 137
135 138 options[:all_sub] = (ARGV.delete('--all-sub') != nil)
136 139
140 + options[:err_log] = (ARGV.delete('--err-log') != nil)
141 +
137 142 return options
138 143 end
139 144
140 145 class ResultCollector
141 146 def initialize
142 147 @results = {}
143 148 @problems = {}
144 149 @users = {}
145 150 end
146 151
147 152 def after_save_hook(submission, grading_result)
148 153 end
149 154
150 155 def save(submission, grading_result)
151 156 user = submission.user
152 157 problem = submission.problem
153 158 if not @problems.has_key? problem.id
154 159 @problems[problem.id] = problem
155 160 end
156 161 if not @users.has_key? user.id
157 162 @users[user.id] = user
158 163 end
159 164 @results[[user.id, problem.id]] = grading_result
160 165
161 166 after_save_hook(submission, grading_result)
162 167 end
163 168
164 169 def print_report_by_user
165 170 puts "---------------------"
166 171 puts " REPORT"
167 172 puts "---------------------"
168 173
169 174 print "login,email"
170 175 @problems.each_value do |problem|
171 176 print ",#{problem.name}"
172 177 end
173 178 print "\n"
174 179
175 180 @users.each_value do |user|
176 181 print "#{user.login},#{user.email}"
177 182 @problems.each_value do |problem|
178 183 if @results.has_key? [user.id, problem.id]
179 184 print ",#{@results[[user.id,problem.id]][:points]}"
180 185 else
181 186 print ","
182 187 end
183 188 end
184 189 print "\n"
185 190 end
186 191 end
187 192 end
188 193
189 194 def grader_general_loop(engine, grader_proc, options)
190 195 runner = Grader::Runner.new(engine, grader_proc)
191 196 while true
192 197
193 198 if check_stopfile # created by calling grader stop
194 199 clear_stopfile
195 200 log "stopped (with stop file)"
196 201 break
197 202 end
198 203
199 204 task = yield(runner)
200 205
201 206 if task==nil
202 207 sleep(1)
203 208 end
204 209 end
205 210 end
206 211
207 212 def grader_queue_loop(grader_proc, options)
208 213 log "Grader: queue"
209 214 engine = Grader::Engine.new
210 215 grader_general_loop(engine, grader_proc, options) do |runner|
211 216 runner.grade_oldest_task
212 217 end
213 218 end
214 219
215 220 def grader_test_request_loop(grader_proc, options)
216 221 log "Grader: test_request"
217 222 engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
218 223 :reporter => Grader::TestRequestReporter.new)
219 224 grader_general_loop(engine, grader_proc, options) do |runner|
220 225 runner.grade_oldest_test_request
221 226 end
222 227 end
223 228
224 229 def grader_autonew_loop(grader_proc, options)
225 230 log "Grader: autonew"
226 231
227 232 if options[:report]
228 233 result_collector = ResultCollector.new
229 234 else
230 235 result_collector = nil
231 236 end
232 237
@@ -327,133 +332,140
327 332
328 333 contest = Contest.find_by_name(contest_name)
329 334 if contest==nil
330 335 puts "cannot find contest: #{contest_name}"
331 336 exit(0)
332 337 end
333 338
334 339 if options[:report]
335 340 result_collector = ResultCollector.new
336 341 else
337 342 result_collector = nil
338 343 end
339 344
340 345 if options[:dry_run]
341 346 puts "Running in dry mode"
342 347 end
343 348
344 349 prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
345 350 :result_collector => result_collector)
346 351 engine = Grader::Engine.new(:reporter => prob_reporter)
347 352 runner = Grader::Runner.new(engine, grader_proc)
348 353
349 354 grader_proc.report_active if grader_proc!=nil
350 355
351 356 contest.problems.each do |problem|
352 357 puts "Grading: #{problem.name}"
353 358 runner.grade_problem(problem,
354 359 :user_conditions => lambda do |u|
355 360 u.contest_finished? and
356 361 u.contest_ids.include?(contest.id)
357 362 end)
358 363 end
359 364
360 365 if options[:report]
361 366 result_collector.print_report_by_user
362 367 end
363 368 end
364 369
365 370 def grader_grade_submissions(grader_proc, options)
366 371 engine = Grader::Engine.new
367 372 runner = Grader::Runner.new(engine, grader_proc)
368 373
369 374 grader_proc.report_active if grader_proc!=nil
370 375
371 376 ARGV.each do |sub_id|
372 377 puts "Grading #{sub_id}"
373 378 begin
374 379 submission = Submission.find(sub_id.to_i)
375 380 rescue ActiveRecord::RecordNotFound
376 381 puts "Submission #{sub_id} not found"
377 382 submission = nil
378 383 end
379 384
380 385 if submission!=nil
381 386 runner.grade_submission(submission)
382 387 end
383 388 end
384 389 end
385 390
386 391 #########################################
387 392 # main program
388 393 #########################################
389 394
390 395 options = process_options_and_stop_file
391 396 GRADER_ENV = options[:environment]
392 397 grader_mode = options[:mode]
393 398 dry_run = options[:dry_run]
394 399
395 400 puts "environment: #{GRADER_ENV}"
396 401 puts "grader mode: #{grader_mode}"
397 402 require File.join(File.dirname(__FILE__),'config/environment')
398 403
399 404 # add grader_mode to config
400 405 # this is needed because method log needs it. TODO: clean this up
401 406 class << config
402 407 attr_accessor :grader_mode
403 408 end
404 409 config.grader_mode = grader_mode
405 410
406 411 # reading rails environment
407 412 log 'Reading rails environment'
408 413
409 414 RAILS_ENV = config.rails_env
410 415 require RAILS_ROOT + '/config/environment'
411 416
412 417 # register grader process
413 418 if config.report_grader
414 419 grader_proc = GraderProcess.register(config.grader_hostname,
415 420 Process.pid,
416 421 grader_mode)
417 422 else
418 423 grader_proc = nil
419 424 end
420 425
421 426 #set loggin environment
422 427 ENV['GRADER_LOGGING'] = log_file_name
428 + if options[:err_log]
429 + err_file_name = log_file_name + '.err'
430 + $stderr.reopen(err_file_name,"a")
431 + log "STDERR log to file [#{err_file_name}]"
432 + warn "start logging for grader PID #{Process.id} on #{Time.now.in_time_zone}"
433 + end
434 +
423 435
424 436 # register exit handler to report inactive, and terminated
425 437 at_exit do
426 438 if grader_proc!=nil
427 439 grader_proc.report_inactive
428 440 grader_proc.terminate
429 441 end
430 442 end
431 443
432 444 #
433 445 # MAIN LOOP
434 446 #
435 447
436 448 case grader_mode
437 449 when "queue"
438 450 grader_queue_loop(grader_proc, options)
439 451
440 452 when "test_request"
441 453 grader_test_request_loop(grader_proc, options)
442 454
443 455 when "prob"
444 456 grader_grade_problems(grader_proc, options)
445 457
446 458 when "contest"
447 459 grader_grade_contests(grader_proc, options)
448 460
449 461 when "sub"
450 462 grader_grade_submissions(grader_proc, options)
451 463
452 464 when "autonew"
453 465 grader_autonew_loop(grader_proc, options)
454 466
455 467 else
456 468 display_manual
457 469 exit(0)
458 470 end
459 471
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -25,166 +25,168
25 25 @config = Grader::Configuration.get_instance
26 26
27 27 @room_maker = options[:room_maker]
28 28 @reporter = options[:reporter]
29 29 end
30 30
31 31 # takes a submission, asks room_maker to produce grading directories,
32 32 # calls grader scripts, and asks reporter to save the result
33 33 def grade(submission)
34 34 current_dir = FileUtils.pwd
35 35
36 36 user = submission.user
37 37 problem = submission.problem
38 38
39 39 begin
40 40 # TODO: will have to create real exception for this
41 41 if user==nil or problem == nil
42 42 @reporter.report_error(submission,"Grading error: problem with submission")
43 43 raise "engine: user or problem is nil"
44 44 end
45 45
46 46 # TODO: this is another hack so that output only task can be judged
47 47 if submission.language!=nil
48 48 language = submission.language.name
49 49 lang_ext = submission.language.ext
50 50 else
51 51 language = 'c'
52 52 lang_ext = 'c'
53 53 end
54 54
55 55 # This is needed because older version of std-scripts/compile
56 56 # only look for c++.
57 57 if language == 'cpp'
58 58 language = 'c++'
59 59 end
60 60
61 61 # COMMENT: should it be only source.ext?
62 62 if problem!=nil
63 63 source_name = "#{problem.name}.#{lang_ext}"
64 64 else
65 65 source_name = "source.#{lang_ext}"
66 66 end
67 67
68 68 grading_dir = @room_maker.produce_grading_room(submission)
69 69 @room_maker.save_source(submission,source_name)
70 70 problem_home = @room_maker.find_problem_home(submission)
71 71
72 72 # puts "GRADING DIR: #{grading_dir}"
73 73 # puts "PROBLEM DIR: #{problem_home}"
74 74
75 75 if !FileTest.exist?(problem_home)
76 76 puts "PROBLEM DIR: #{problem_home}"
77 77 raise "engine: No test data."
78 78 end
79 79
80 80 dinit = DirInit::Manager.new(problem_home)
81 81
82 82 dinit.setup do
83 83 copy_log = copy_script(problem_home)
84 84 save_copy_log(problem_home,copy_log)
85 85 end
86 86
87 87 call_judge(problem_home,language,grading_dir,source_name)
88 88
89 89 @reporter.report(submission,"#{grading_dir}/test-result")
90 90
91 91 dinit.teardown do
92 92 copy_log = load_copy_log(problem_home)
93 93 clear_copy_log(problem_home)
94 94 clear_script(copy_log,problem_home)
95 95 end
96 96
97 97 rescue RuntimeError => msg
98 98 @reporter.report_error(submission, msg)
99 99 puts "ERROR: #{msg}"
100 100
101 101 ensure
102 102 @room_maker.clean_up(submission)
103 103 Dir.chdir(current_dir) # this is really important
104 104 end
105 105 end
106 106
107 107 protected
108 108
109 109 def talk(str)
110 110 if @config.talkative
111 111 puts str
112 112 end
113 113 end
114 114
115 115 def call_judge(problem_home,language,grading_dir,fname)
116 116 ENV['PROBLEM_HOME'] = problem_home
117 117 ENV['RUBYOPT'] = ''
118 118
119 119 talk grading_dir
120 120 Dir.chdir grading_dir
121 - cmd = "#{problem_home}/script/judge #{language} #{fname}"
121 + script_name = "#{problem_home}/script/judge"
122 + cmd = "#{script_name} #{language} #{fname}"
122 123 talk "CMD: #{cmd}"
124 + warn "ERROR: file does not exists #{script_name}" unless File.exists? script_name
123 125 system(cmd)
124 126 end
125 127
126 128 def get_std_script_dir
127 129 GRADER_ROOT + '/std-script'
128 130 end
129 131
130 132 def copy_script(problem_home)
131 133 script_dir = "#{problem_home}/script"
132 134 std_script_dir = get_std_script_dir
133 135
134 136 raise "engine: std-script directory not found" if !FileTest.exist?(std_script_dir)
135 137
136 138 scripts = Dir[std_script_dir + '/*']
137 139
138 140 copied = []
139 141
140 142 scripts.each do |s|
141 143 fname = File.basename(s)
142 144 next if FileTest.directory?(s)
143 145 if !FileTest.exist?("#{script_dir}/#{fname}")
144 146 copied << fname
145 147 FileUtils.cp(s, "#{script_dir}", :preserve => true)
146 148 end
147 149 end
148 150
149 151 return copied
150 152 end
151 153
152 154 def copy_log_filename(problem_home)
153 155 return File.join(problem_home, '.scripts_copied')
154 156 end
155 157
156 158 def save_copy_log(problem_home, log)
157 159 f = File.new(copy_log_filename(problem_home),"w")
158 160 log.each do |fname|
159 161 f.write("#{fname}\n")
160 162 end
161 163 f.close
162 164 end
163 165
164 166 def load_copy_log(problem_home)
165 167 f = File.new(copy_log_filename(problem_home),"r")
166 168 log = []
167 169 f.readlines.each do |line|
168 170 log << line.strip
169 171 end
170 172 f.close
171 173 log
172 174 end
173 175
174 176 def clear_copy_log(problem_home)
175 177 File.delete(copy_log_filename(problem_home))
176 178 end
177 179
178 180 def clear_script(log,problem_home)
179 181 log.each do |s|
180 182 FileUtils.rm("#{problem_home}/script/#{s}")
181 183 end
182 184 end
183 185
184 186 def mkdir_if_does_not_exist(dirname)
185 187 Dir.mkdir(dirname) if !FileTest.exist?(dirname)
186 188 end
187 189
188 190 end
189 191
190 192 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,67 +1,67
1 1 #
2 2 # A runner drives the engine into various tasks.
3 3 #
4 4
5 5 module Grader
6 6
7 7 class Runner
8 8
9 9 def initialize(engine, grader_process=nil)
10 10 @engine = engine
11 11 @grader_process = grader_process
12 12 end
13 13
14 14 def grade_oldest_task
15 15 task = Task.get_inqueue_and_change_status(Task::STATUS_GRADING)
16 16 if task!=nil
17 17 @grader_process.report_active(task) if @grader_process!=nil
18 18
19 19 submission = Submission.find(task.submission_id)
20 20 @engine.grade(submission)
21 21 task.status_complete!
22 22 @grader_process.report_inactive(task) if @grader_process!=nil
23 23 end
24 24 return task
25 25 end
26 26
27 27 def grade_problem(problem, options={})
28 28 User.find_each do |u|
29 29 puts "user: #{u.login}"
30 30 if options[:user_conditions]!=nil
31 31 con_proc = options[:user_conditions]
32 32 next if not con_proc.call(u)
33 33 end
34 34 if options[:all_sub]
35 35 Submission.where(user_id: u.id,problem_id: problem.id).find_each do |sub|
36 36 @engine.grade(sub)
37 37 end
38 38 else
39 39 last_sub = Submission.find_last_by_user_and_problem(u.id,problem.id)
40 40 if last_sub!=nil
41 41 @engine.grade(last_sub)
42 42 end
43 43 end
44 44 end
45 45 end
46 46
47 47 def grade_submission(submission)
48 - puts "Submission: #{submission.id} by #{submission.user.full_name}"
48 + puts "Submission: #{submission.id} by #{submission.try(:user).try(:full_name)}"
49 49 @engine.grade(submission)
50 50 end
51 51
52 52 def grade_oldest_test_request
53 53 test_request = TestRequest.get_inqueue_and_change_status(Task::STATUS_GRADING)
54 54 if test_request!=nil
55 55 @grader_process.report_active(test_request) if @grader_process!=nil
56 56
57 57 @engine.grade(test_request)
58 58 test_request.status_complete!
59 59 @grader_process.report_inactive(test_request) if @grader_process!=nil
60 60 end
61 61 return test_request
62 62 end
63 63
64 64 end
65 65
66 66 end
67 67
You need to be logged in to leave comments. Login now