cafe-grader-judge-scripts

Commit r121:6d40867ff9f2 public

parent | child
Description:
refactored grader, added autonew mode
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r121:6d40867ff9f2 - - 1 file changed: 197 inserted, 116 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,344 +1,425
1 #!/usr/bin/ruby
1 #!/usr/bin/ruby
2
2
3 def stop_grader(id)
3 def stop_grader(id)
4 if id==:all
4 if id==:all
5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
6 else
6 else
7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
8 end
8 end
9 end
9 end
10
10
11 def check_stopfile
11 def check_stopfile
12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
14 end
14 end
15
15
16 def clear_stopfile
16 def clear_stopfile
17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
19 end
19 end
20 end
20 end
21
21
22 def config
22 def config
23 Grader::Configuration.get_instance
23 Grader::Configuration.get_instance
24 end
24 end
25
25
26 def log_file_name
26 def log_file_name
27 if !File.exists?(config.log_dir)
27 if !File.exists?(config.log_dir)
28 raise "Log directory does not exist: #{config.log_dir}"
28 raise "Log directory does not exist: #{config.log_dir}"
29 end
29 end
30 config.log_dir +
30 config.log_dir +
31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
32 end
32 end
33
33
34 def log(str)
34 def log(str)
35 if config.talkative
35 if config.talkative
36 puts str
36 puts str
37 end
37 end
38 if config.logging
38 if config.logging
39 fp = File.open(log_file_name,"a")
39 fp = File.open(log_file_name,"a")
40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
41 fp.close
41 fp.close
42 end
42 end
43 end
43 end
44
44
45 def display_manual
45 def display_manual
46 puts <<USAGE
46 puts <<USAGE
47 Grader.
47 Grader.
48 using: (1) grader
48 using: (1) grader
49 (2) grader environment [mode]
49 (2) grader environment [mode]
50 (3) grader stop [all|pids-list]
50 (3) grader stop [all|pids-list]
51 (4) grader --help
51 (4) grader --help
52 (1) call grader with environment = 'exam', mode = 'queue'
52 (1) call grader with environment = 'exam', mode = 'queue'
53 - (2) possible modes are: 'queue', 'prob', 'test_request'
53 + (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
54 (3) create stop-file to stop running grader in queue mode
54 (3) create stop-file to stop running grader in queue mode
55 (4) You are here.
55 (4) You are here.
56 USAGE
56 USAGE
57 end
57 end
58
58
59 def process_options_and_stop_file
59 def process_options_and_stop_file
60 # The list of options are:
60 # The list of options are:
61 # - stop [all|process ids]
61 # - stop [all|process ids]
62 # -
62 # -
63
63
64 # Process 'help' option
64 # Process 'help' option
65 if (ARGV.length==1) and (/help/.match(ARGV[0]))
65 if (ARGV.length==1) and (/help/.match(ARGV[0]))
66 display_manual
66 display_manual
67 exit(0)
67 exit(0)
68 end
68 end
69
69
70 # Process 'stop' option.
70 # Process 'stop' option.
71 if (ARGV.length >= 1) and (ARGV[0]=='stop')
71 if (ARGV.length >= 1) and (ARGV[0]=='stop')
72 if ARGV.length==1
72 if ARGV.length==1
73 puts "you should specify pid-list or 'all'"
73 puts "you should specify pid-list or 'all'"
74 display_manual
74 display_manual
75 elsif (ARGV.length==2) and (ARGV[1]=='all')
75 elsif (ARGV.length==2) and (ARGV[1]=='all')
76 stop_grader(:all)
76 stop_grader(:all)
77 puts "A global stop file ('stop.all') created."
77 puts "A global stop file ('stop.all') created."
78 puts "You should remove it manually later."
78 puts "You should remove it manually later."
79 else
79 else
80 (1..ARGV.length-1).each do |i|
80 (1..ARGV.length-1).each do |i|
81 stop_grader(ARGV[i])
81 stop_grader(ARGV[i])
82 end
82 end
83 puts "stop file(s) created"
83 puts "stop file(s) created"
84 end
84 end
85 exit(0)
85 exit(0)
86 end
86 end
87
87
88 # Check stop file.
88 # Check stop file.
89 if check_stopfile
89 if check_stopfile
90 puts "Stop file exists. Terminated."
90 puts "Stop file exists. Terminated."
91 clear_stopfile
91 clear_stopfile
92 exit(0)
92 exit(0)
93 end
93 end
94
94
95 #default options
95 #default options
96 options = {
96 options = {
97 :mode => 'queue',
97 :mode => 'queue',
98 :environment => 'exam',
98 :environment => 'exam',
99 :dry_run => false,
99 :dry_run => false,
100 }
100 }
101
101
102 # Process mode and environment option
102 # Process mode and environment option
103 if ARGV.length >= 1
103 if ARGV.length >= 1
104 options[:environment] = ARGV.shift
104 options[:environment] = ARGV.shift
105 if ARGV.length >=1
105 if ARGV.length >=1
106 options[:mode] = ARGV.shift
106 options[:mode] = ARGV.shift
107 end
107 end
108 end
108 end
109
109
110 options[:dry_run] = (ARGV.delete('--dry') != nil)
110 options[:dry_run] = (ARGV.delete('--dry') != nil)
111 - if options[:dry_run] and (not ['prob','contest'].include? options[:mode])
111 + if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
112 puts "Dry run currently works only for 'prob' or 'contest' modes."
112 puts "Dry run currently works only for 'prob' or 'contest' modes."
113 exit(0)
113 exit(0)
114 end
114 end
115
115
116 options[:report] = (ARGV.delete('--report') != nil)
116 options[:report] = (ARGV.delete('--report') != nil)
117 if options[:report] and (not ['prob','contest'].include? options[:mode])
117 if options[:report] and (not ['prob','contest'].include? options[:mode])
118 puts "Report currently works only for 'prob' or 'contest' modes."
118 puts "Report currently works only for 'prob' or 'contest' modes."
119 exit(0)
119 exit(0)
120 end
120 end
121
121
122 return options
122 return options
123 end
123 end
124
124
125 class ResultCollector
125 class ResultCollector
126 def initialize
126 def initialize
127 @results = {}
127 @results = {}
128 @problems = {}
128 @problems = {}
129 @users = {}
129 @users = {}
130 end
130 end
131
131
132 def save(user, problem, grading_result)
132 def save(user, problem, grading_result)
133 if not @problems.has_key? problem.id
133 if not @problems.has_key? problem.id
134 @problems[problem.id] = problem
134 @problems[problem.id] = problem
135 end
135 end
136 if not @users.has_key? user.id
136 if not @users.has_key? user.id
137 @users[user.id] = user
137 @users[user.id] = user
138 end
138 end
139 @results[[user.id, problem.id]] = grading_result
139 @results[[user.id, problem.id]] = grading_result
140 end
140 end
141
141
142 def print_report_by_user
142 def print_report_by_user
143 puts "---------------------"
143 puts "---------------------"
144 puts " REPORT"
144 puts " REPORT"
145 puts "---------------------"
145 puts "---------------------"
146
146
147 print "login,email"
147 print "login,email"
148 @problems.each_value do |problem|
148 @problems.each_value do |problem|
149 print ",#{problem.name}"
149 print ",#{problem.name}"
150 end
150 end
151 print "\n"
151 print "\n"
152
152
153 @users.each_value do |user|
153 @users.each_value do |user|
154 print "#{user.login},#{user.email}"
154 print "#{user.login},#{user.email}"
155 @problems.each_value do |problem|
155 @problems.each_value do |problem|
156 if @results.has_key? [user.id, problem.id]
156 if @results.has_key? [user.id, problem.id]
157 print ",#{@results[[user.id,problem.id]][:points]}"
157 print ",#{@results[[user.id,problem.id]][:points]}"
158 else
158 else
159 print ","
159 print ","
160 end
160 end
161 end
161 end
162 print "\n"
162 print "\n"
163 end
163 end
164 end
164 end
165 end
165 end
166
166
167 + def grader_general_loop(engine, grader_proc, options)
168 + runner = Grader::Runner.new(engine, grader_proc)
169 + while true
170 +
171 + if check_stopfile # created by calling grader stop
172 + clear_stopfile
173 + log "stopped (with stop file)"
174 + break
175 + end
176 +
177 + task = yield(runner)
178 +
179 + if task==nil
180 + sleep(1)
181 + end
182 + end
183 + end
184 +
185 + def grader_queue_loop(grader_proc, options)
186 + log "Grader: queue"
187 + engine = Grader::Engine.new
188 + grader_general_loop(engine, grader_proc, options) do |runner|
189 + runner.grade_oldest_task
190 + end
191 + end
192 +
193 + def grader_test_request_loop(grader_proc, options)
194 + log "Grader: test_request"
195 + engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
196 + :reporter => Grader::TestRequestReporter.new)
197 + grader_general_loop(engine, grader_proc, options) do |runner|
198 + runner.grade_oldest_test_request
199 + end
200 + end
201 +
202 + def grader_autonew_loop(grader_proc, options)
203 + log "Grader: autonew"
204 +
205 + if options[:dry_run]
206 + puts "Running in dry mode"
207 + end
208 +
209 + prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run])
210 + engine = Grader::Engine.new(:reporter => prob_reporter)
211 + runner = Grader::Runner.new(engine, grader_proc)
212 +
213 + grader_proc.report_active if grader_proc!=nil
214 +
215 + latest_submitted_at = nil
216 + graded_submission_ids = {}
217 +
218 + while true
219 +
220 + if check_stopfile # created by calling grader stop
221 + clear_stopfile
222 + log "stopped (with stop file)"
223 + break
224 + end
225 +
226 + if latest_submitted_at==nil
227 + submissions = Submission.all
228 + else
229 + submissions = Submission.all(:conditions => ["submitted_at >= :latest",
230 + {:latest => latest_submitted_at}])
231 + end
232 +
233 + graded_any = false
234 +
235 + if submissions.length != 0
236 + submissions.each do |submission|
237 + if ! graded_submission_ids[submission.id]
238 + runner.grade_submission(submission)
239 + graded_submission_ids[submission.id] = true
240 + if (!latest_submitted_at or
241 + latest_submitted_at < submission.submitted_at)
242 + latest_submitted_at = submission.submitted_at
243 + end
244 + puts "graded: #{submission.id}"
245 + puts "latest: #{latest_submitted_at}"
246 + graded_any = true
247 + end
248 + end
249 + end
250 +
251 + if ! graded_any
252 + sleep(1)
253 + end
254 + end
255 + end
256 +
257 + def grader_grade_problems(grader_proc, options)
258 + if options[:report]
259 + result_collector = ResultCollector.new
260 + else
261 + result_collector = nil
262 + end
263 +
264 + if options[:dry_run]
265 + puts "Running in dry mode"
266 + end
267 +
268 + prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
269 + :result_collector => result_collector)
270 + engine = Grader::Engine.new(:reporter => prob_reporter)
271 + runner = Grader::Runner.new(engine, grader_proc)
272 +
273 + grader_proc.report_active if grader_proc!=nil
274 +
275 + ARGV.each do |prob_name|
276 + prob = Problem.find_by_name(prob_name)
277 + if prob==nil
278 + puts "cannot find problem: #{prob_name}"
279 + else
280 + runner.grade_problem(prob)
281 + end
282 + end
283 +
284 + if options[:report]
285 + result_collector.print_report_by_user
286 + end
287 + end
288 +
289 + def grader_grade_contests(grader_proc, options)
290 + # always use dry run when grading during contest
291 + dry_run = options[:dry_run] = true
292 +
293 + contest_name = ARGV.shift
294 +
295 + contest = Contest.find_by_name(contest_name)
296 + if contest==nil
297 + puts "cannot find contest: #{contest_name}"
298 + exit(0)
299 + end
300 +
301 + if options[:report]
302 + result_collector = ResultCollector.new
303 + else
304 + result_collector = nil
305 + end
306 +
307 + if options[:dry_run]
308 + puts "Running in dry mode"
309 + end
310 +
311 + prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
312 + :result_collector => result_collector)
313 + engine = Grader::Engine.new(:reporter => prob_reporter)
314 + runner = Grader::Runner.new(engine, grader_proc)
315 +
316 + grader_proc.report_active if grader_proc!=nil
317 +
318 + contest.problems.each do |problem|
319 + puts "Grading: #{problem.name}"
320 + runner.grade_problem(problem,
321 + :user_conditions => lambda do |u|
322 + u.contest_finished? and
323 + u.contest_ids.include?(contest.id)
324 + end)
325 + end
326 +
327 + if options[:report]
328 + result_collector.print_report_by_user
329 + end
330 + end
331 +
332 + def grader_grade_submissions(grader_proc, options)
333 + engine = Grader::Engine.new
334 + runner = Grader::Runner.new(engine, grader_proc)
335 +
336 + grader_proc.report_active if grader_proc!=nil
337 +
338 + ARGV.each do |sub_id|
339 + puts "Grading #{sub_id}"
340 + begin
341 + submission = Submission.find(sub_id.to_i)
342 + rescue ActiveRecord::RecordNotFound
343 + puts "Record not found"
344 + submission = nil
345 + end
346 +
347 + if submission!=nil
348 + runner.grade_submission(submission)
349 + end
350 + end
351 + end
352 +
167 #########################################
353 #########################################
168 # main program
354 # main program
169 #########################################
355 #########################################
170
356
171 options = process_options_and_stop_file
357 options = process_options_and_stop_file
172 GRADER_ENV = options[:environment]
358 GRADER_ENV = options[:environment]
173 grader_mode = options[:mode]
359 grader_mode = options[:mode]
174 dry_run = options[:dry_run]
360 dry_run = options[:dry_run]
175
361
176 puts "environment: #{GRADER_ENV}"
362 puts "environment: #{GRADER_ENV}"
177 require File.join(File.dirname(__FILE__),'config/environment')
363 require File.join(File.dirname(__FILE__),'config/environment')
178
364
179 # add grader_mode to config
365 # add grader_mode to config
180 # this is needed because method log needs it. TODO: clean this up
366 # this is needed because method log needs it. TODO: clean this up
181 class << config
367 class << config
182 attr_accessor :grader_mode
368 attr_accessor :grader_mode
183 end
369 end
184 config.grader_mode = grader_mode
370 config.grader_mode = grader_mode
185
371
186 # reading rails environment
372 # reading rails environment
187 log 'Reading rails environment'
373 log 'Reading rails environment'
188
374
189 RAILS_ENV = config.rails_env
375 RAILS_ENV = config.rails_env
190 require RAILS_ROOT + '/config/environment'
376 require RAILS_ROOT + '/config/environment'
191
377
192 # register grader process
378 # register grader process
193 if config.report_grader
379 if config.report_grader
194 grader_proc = GraderProcess.register(config.grader_hostname,
380 grader_proc = GraderProcess.register(config.grader_hostname,
195 Process.pid,
381 Process.pid,
196 grader_mode)
382 grader_mode)
197 else
383 else
198 grader_proc = nil
384 grader_proc = nil
199 end
385 end
200
386
201 #set loggin environment
387 #set loggin environment
202 ENV['GRADER_LOGGING'] = log_file_name
388 ENV['GRADER_LOGGING'] = log_file_name
203
389
204 # register exit handler to report inactive, and terminated
390 # register exit handler to report inactive, and terminated
205 at_exit do
391 at_exit do
206 if grader_proc!=nil
392 if grader_proc!=nil
207 grader_proc.report_inactive
393 grader_proc.report_inactive
208 grader_proc.terminate
394 grader_proc.terminate
209 end
395 end
210 end
396 end
211
397
212 #
398 #
213 # MAIN LOOP
399 # MAIN LOOP
214 #
400 #
215
401
216 case grader_mode
402 case grader_mode
217 - when "queue", "test_request"
403 + when "queue"
218 - log "Grader: #{grader_mode}"
404 + grader_queue_loop(grader_proc, options)
219 - if grader_mode=="queue"
220 - engine = Grader::Engine.new
221 - else
222 - engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
223 - :reporter => Grader::TestRequestReporter.new)
224 - end
225 -
226 - runner = Grader::Runner.new(engine, grader_proc)
227 - while true
228
405
229 - if check_stopfile # created by calling grader stop
406 + when "test_request"
230 - clear_stopfile
407 + grader_test_request_loop(grader_proc, options)
231 - log "stopped (with stop file)"
232 - break
233 - end
234 -
235 - if grader_mode=="queue"
236 - task = runner.grade_oldest_task
237 - else
238 - task = runner.grade_oldest_test_request
239 - end
240 - if task==nil
241 - sleep(1)
242 - end
243 - end
244
408
245 when "prob"
409 when "prob"
246 - if options[:report]
410 + grader_grade_problems(grader_proc, options)
247 - result_collector = ResultCollector.new
248 - else
249 - result_collector = nil
250 - end
251 -
252 - if options[:dry_run]
253 - puts "Running in dry mode"
254 - end
255 -
256 - prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
257 - :result_collector => result_collector)
258 - engine = Grader::Engine.new(:reporter => prob_reporter)
259 - runner = Grader::Runner.new(engine, grader_proc)
260 -
261 - grader_proc.report_active if grader_proc!=nil
262 -
263 - ARGV.each do |prob_name|
264 - prob = Problem.find_by_name(prob_name)
265 - if prob==nil
266 - puts "cannot find problem: #{prob_name}"
267 - else
268 - runner.grade_problem(prob)
269 - end
270 - end
271 -
272 - if options[:report]
273 - result_collector.print_report_by_user
274 - end
275
411
276 when "contest"
412 when "contest"
277 - # always use dry run when grading during contest
413 + grader_grade_contests(grader_proc, options)
278 - contest_name = ARGV.shift
279 -
280 - dry_run = options[:dry_run] = true
281 -
282 - contest = Contest.find_by_name(contest_name)
283 - if contest==nil
284 - puts "cannot find contest: #{contest_name}"
285 - exit(0)
286 - end
287 -
288 - if options[:report]
289 - result_collector = ResultCollector.new
290 - else
291 - result_collector = nil
292 - end
293 -
294 - if options[:dry_run]
295 - puts "Running in dry mode"
296 - end
297 -
298 - prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
299 - :result_collector => result_collector)
300 - engine = Grader::Engine.new(:reporter => prob_reporter)
301 - runner = Grader::Runner.new(engine, grader_proc)
302 -
303 - grader_proc.report_active if grader_proc!=nil
304 -
305 - contest.problems.each do |problem|
306 - puts "Grading: #{problem.name}"
307 - runner.grade_problem(problem,
308 - :user_conditions => lambda do |u|
309 - u.contest_finished? and
310 - u.contest_ids.include?(contest.id)
311 - end)
312 - end
313 -
314 - if options[:report]
315 - result_collector.print_report_by_user
316 - end
317
414
318 when "sub"
415 when "sub"
319 - engine = Grader::Engine.new
416 + grader_grade_submissions(grader_proc, options)
320 - runner = Grader::Runner.new(engine, grader_proc)
321 -
322 - grader_proc.report_active if grader_proc!=nil
323
417
324 - ARGV.each do |sub_id|
418 + when "autonew"
325 - puts "Grading #{sub_id}"
419 + grader_autonew_loop(grader_proc, options)
326 - begin
327 - submission = Submission.find(sub_id.to_i)
328 - rescue ActiveRecord::RecordNotFound
329 - puts "Record not found"
330 - submission = nil
331 - end
332 -
333 - if submission!=nil
334 - runner.grade_submission(submission)
335 - end
336 - end
337 -
338 -
339
420
340 else
421 else
341 display_manual
422 display_manual
342 exit(0)
423 exit(0)
343 end
424 end
344
425
You need to be logged in to leave comments. Login now