cafe-grader-judge-scripts

Commit r122:ed471e6e96ca public

parent | child
Description:
added after_save_hook to ResultCollector
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r122:ed471e6e96ca - - 1 file changed: 15 inserted, 2 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,425 +1,438
1 #!/usr/bin/ruby
1 #!/usr/bin/ruby
2
2
3 def stop_grader(id)
3 def stop_grader(id)
4 if id==:all
4 if id==:all
5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
6 else
6 else
7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
8 end
8 end
9 end
9 end
10
10
11 def check_stopfile
11 def check_stopfile
12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
14 end
14 end
15
15
16 def clear_stopfile
16 def clear_stopfile
17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
19 end
19 end
20 end
20 end
21
21
22 def config
22 def config
23 Grader::Configuration.get_instance
23 Grader::Configuration.get_instance
24 end
24 end
25
25
26 def log_file_name
26 def log_file_name
27 if !File.exists?(config.log_dir)
27 if !File.exists?(config.log_dir)
28 raise "Log directory does not exist: #{config.log_dir}"
28 raise "Log directory does not exist: #{config.log_dir}"
29 end
29 end
30 config.log_dir +
30 config.log_dir +
31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
32 end
32 end
33
33
34 def log(str)
34 def log(str)
35 if config.talkative
35 if config.talkative
36 puts str
36 puts str
37 end
37 end
38 if config.logging
38 if config.logging
39 fp = File.open(log_file_name,"a")
39 fp = File.open(log_file_name,"a")
40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
41 fp.close
41 fp.close
42 end
42 end
43 end
43 end
44
44
45 def display_manual
45 def display_manual
46 puts <<USAGE
46 puts <<USAGE
47 Grader.
47 Grader.
48 using: (1) grader
48 using: (1) grader
49 (2) grader environment [mode]
49 (2) grader environment [mode]
50 (3) grader stop [all|pids-list]
50 (3) grader stop [all|pids-list]
51 (4) grader --help
51 (4) grader --help
52 (1) call grader with environment = 'exam', mode = 'queue'
52 (1) call grader with environment = 'exam', mode = 'queue'
53 (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
53 (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
54 (3) create stop-file to stop running grader in queue mode
54 (3) create stop-file to stop running grader in queue mode
55 (4) You are here.
55 (4) You are here.
56 USAGE
56 USAGE
57 end
57 end
58
58
59 def process_options_and_stop_file
59 def process_options_and_stop_file
60 # The list of options are:
60 # The list of options are:
61 # - stop [all|process ids]
61 # - stop [all|process ids]
62 # -
62 # -
63
63
64 # Process 'help' option
64 # Process 'help' option
65 if (ARGV.length==1) and (/help/.match(ARGV[0]))
65 if (ARGV.length==1) and (/help/.match(ARGV[0]))
66 display_manual
66 display_manual
67 exit(0)
67 exit(0)
68 end
68 end
69
69
70 # Process 'stop' option.
70 # Process 'stop' option.
71 if (ARGV.length >= 1) and (ARGV[0]=='stop')
71 if (ARGV.length >= 1) and (ARGV[0]=='stop')
72 if ARGV.length==1
72 if ARGV.length==1
73 puts "you should specify pid-list or 'all'"
73 puts "you should specify pid-list or 'all'"
74 display_manual
74 display_manual
75 elsif (ARGV.length==2) and (ARGV[1]=='all')
75 elsif (ARGV.length==2) and (ARGV[1]=='all')
76 stop_grader(:all)
76 stop_grader(:all)
77 puts "A global stop file ('stop.all') created."
77 puts "A global stop file ('stop.all') created."
78 puts "You should remove it manually later."
78 puts "You should remove it manually later."
79 else
79 else
80 (1..ARGV.length-1).each do |i|
80 (1..ARGV.length-1).each do |i|
81 stop_grader(ARGV[i])
81 stop_grader(ARGV[i])
82 end
82 end
83 puts "stop file(s) created"
83 puts "stop file(s) created"
84 end
84 end
85 exit(0)
85 exit(0)
86 end
86 end
87
87
88 # Check stop file.
88 # Check stop file.
89 if check_stopfile
89 if check_stopfile
90 puts "Stop file exists. Terminated."
90 puts "Stop file exists. Terminated."
91 clear_stopfile
91 clear_stopfile
92 exit(0)
92 exit(0)
93 end
93 end
94
94
95 #default options
95 #default options
96 options = {
96 options = {
97 :mode => 'queue',
97 :mode => 'queue',
98 :environment => 'exam',
98 :environment => 'exam',
99 :dry_run => false,
99 :dry_run => false,
100 }
100 }
101
101
102 # Process mode and environment option
102 # Process mode and environment option
103 if ARGV.length >= 1
103 if ARGV.length >= 1
104 options[:environment] = ARGV.shift
104 options[:environment] = ARGV.shift
105 if ARGV.length >=1
105 if ARGV.length >=1
106 options[:mode] = ARGV.shift
106 options[:mode] = ARGV.shift
107 end
107 end
108 end
108 end
109
109
110 options[:dry_run] = (ARGV.delete('--dry') != nil)
110 options[:dry_run] = (ARGV.delete('--dry') != nil)
111 if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
111 if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
112 puts "Dry run currently works only for 'prob' or 'contest' modes."
112 puts "Dry run currently works only for 'prob' or 'contest' modes."
113 exit(0)
113 exit(0)
114 end
114 end
115
115
116 options[:report] = (ARGV.delete('--report') != nil)
116 options[:report] = (ARGV.delete('--report') != nil)
117 - if options[:report] and (not ['prob','contest'].include? options[:mode])
117 + if options[:report] and (not ['prob','contest','autonew'].include? options[:mode])
118 puts "Report currently works only for 'prob' or 'contest' modes."
118 puts "Report currently works only for 'prob' or 'contest' modes."
119 exit(0)
119 exit(0)
120 end
120 end
121
121
122 return options
122 return options
123 end
123 end
124
124
125 class ResultCollector
125 class ResultCollector
126 def initialize
126 def initialize
127 @results = {}
127 @results = {}
128 @problems = {}
128 @problems = {}
129 @users = {}
129 @users = {}
130 end
130 end
131
131
132 + def after_save_hook(user, problem, grading_result)
133 + end
134 +
132 def save(user, problem, grading_result)
135 def save(user, problem, grading_result)
133 if not @problems.has_key? problem.id
136 if not @problems.has_key? problem.id
134 @problems[problem.id] = problem
137 @problems[problem.id] = problem
135 end
138 end
136 if not @users.has_key? user.id
139 if not @users.has_key? user.id
137 @users[user.id] = user
140 @users[user.id] = user
138 end
141 end
139 @results[[user.id, problem.id]] = grading_result
142 @results[[user.id, problem.id]] = grading_result
143 +
144 + after_save_hook(user, problem, grading_result)
140 end
145 end
141
146
142 def print_report_by_user
147 def print_report_by_user
143 puts "---------------------"
148 puts "---------------------"
144 puts " REPORT"
149 puts " REPORT"
145 puts "---------------------"
150 puts "---------------------"
146
151
147 print "login,email"
152 print "login,email"
148 @problems.each_value do |problem|
153 @problems.each_value do |problem|
149 print ",#{problem.name}"
154 print ",#{problem.name}"
150 end
155 end
151 print "\n"
156 print "\n"
152
157
153 @users.each_value do |user|
158 @users.each_value do |user|
154 print "#{user.login},#{user.email}"
159 print "#{user.login},#{user.email}"
155 @problems.each_value do |problem|
160 @problems.each_value do |problem|
156 if @results.has_key? [user.id, problem.id]
161 if @results.has_key? [user.id, problem.id]
157 print ",#{@results[[user.id,problem.id]][:points]}"
162 print ",#{@results[[user.id,problem.id]][:points]}"
158 else
163 else
159 print ","
164 print ","
160 end
165 end
161 end
166 end
162 print "\n"
167 print "\n"
163 end
168 end
164 end
169 end
165 end
170 end
166
171
167 def grader_general_loop(engine, grader_proc, options)
172 def grader_general_loop(engine, grader_proc, options)
168 runner = Grader::Runner.new(engine, grader_proc)
173 runner = Grader::Runner.new(engine, grader_proc)
169 while true
174 while true
170
175
171 if check_stopfile # created by calling grader stop
176 if check_stopfile # created by calling grader stop
172 clear_stopfile
177 clear_stopfile
173 log "stopped (with stop file)"
178 log "stopped (with stop file)"
174 break
179 break
175 end
180 end
176
181
177 task = yield(runner)
182 task = yield(runner)
178
183
179 if task==nil
184 if task==nil
180 sleep(1)
185 sleep(1)
181 end
186 end
182 end
187 end
183 end
188 end
184
189
185 def grader_queue_loop(grader_proc, options)
190 def grader_queue_loop(grader_proc, options)
186 log "Grader: queue"
191 log "Grader: queue"
187 engine = Grader::Engine.new
192 engine = Grader::Engine.new
188 grader_general_loop(engine, grader_proc, options) do |runner|
193 grader_general_loop(engine, grader_proc, options) do |runner|
189 runner.grade_oldest_task
194 runner.grade_oldest_task
190 end
195 end
191 end
196 end
192
197
193 def grader_test_request_loop(grader_proc, options)
198 def grader_test_request_loop(grader_proc, options)
194 log "Grader: test_request"
199 log "Grader: test_request"
195 engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
200 engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
196 :reporter => Grader::TestRequestReporter.new)
201 :reporter => Grader::TestRequestReporter.new)
197 grader_general_loop(engine, grader_proc, options) do |runner|
202 grader_general_loop(engine, grader_proc, options) do |runner|
198 runner.grade_oldest_test_request
203 runner.grade_oldest_test_request
199 end
204 end
200 end
205 end
201
206
202 def grader_autonew_loop(grader_proc, options)
207 def grader_autonew_loop(grader_proc, options)
203 log "Grader: autonew"
208 log "Grader: autonew"
204
209
210 + if options[:report]
211 + result_collector = ResultCollector.new
212 + else
213 + result_collector = nil
214 + end
215 +
205 if options[:dry_run]
216 if options[:dry_run]
206 puts "Running in dry mode"
217 puts "Running in dry mode"
207 end
218 end
208
219
209 - prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run])
220 + prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
221 + :result_collector => result_collector)
222 +
210 engine = Grader::Engine.new(:reporter => prob_reporter)
223 engine = Grader::Engine.new(:reporter => prob_reporter)
211 runner = Grader::Runner.new(engine, grader_proc)
224 runner = Grader::Runner.new(engine, grader_proc)
212
225
213 grader_proc.report_active if grader_proc!=nil
226 grader_proc.report_active if grader_proc!=nil
214
227
215 latest_submitted_at = nil
228 latest_submitted_at = nil
216 graded_submission_ids = {}
229 graded_submission_ids = {}
217
230
218 while true
231 while true
219
232
220 if check_stopfile # created by calling grader stop
233 if check_stopfile # created by calling grader stop
221 clear_stopfile
234 clear_stopfile
222 log "stopped (with stop file)"
235 log "stopped (with stop file)"
223 break
236 break
224 end
237 end
225
238
226 if latest_submitted_at==nil
239 if latest_submitted_at==nil
227 submissions = Submission.all
240 submissions = Submission.all
228 else
241 else
229 submissions = Submission.all(:conditions => ["submitted_at >= :latest",
242 submissions = Submission.all(:conditions => ["submitted_at >= :latest",
230 {:latest => latest_submitted_at}])
243 {:latest => latest_submitted_at}])
231 end
244 end
232
245
233 graded_any = false
246 graded_any = false
234
247
235 if submissions.length != 0
248 if submissions.length != 0
236 submissions.each do |submission|
249 submissions.each do |submission|
237 if ! graded_submission_ids[submission.id]
250 if ! graded_submission_ids[submission.id]
238 runner.grade_submission(submission)
251 runner.grade_submission(submission)
239 graded_submission_ids[submission.id] = true
252 graded_submission_ids[submission.id] = true
240 if (!latest_submitted_at or
253 if (!latest_submitted_at or
241 latest_submitted_at < submission.submitted_at)
254 latest_submitted_at < submission.submitted_at)
242 latest_submitted_at = submission.submitted_at
255 latest_submitted_at = submission.submitted_at
243 end
256 end
244 puts "graded: #{submission.id}"
257 puts "graded: #{submission.id}"
245 puts "latest: #{latest_submitted_at}"
258 puts "latest: #{latest_submitted_at}"
246 graded_any = true
259 graded_any = true
247 end
260 end
248 end
261 end
249 end
262 end
250
263
251 if ! graded_any
264 if ! graded_any
252 sleep(1)
265 sleep(1)
253 end
266 end
254 end
267 end
255 end
268 end
256
269
257 def grader_grade_problems(grader_proc, options)
270 def grader_grade_problems(grader_proc, options)
258 if options[:report]
271 if options[:report]
259 result_collector = ResultCollector.new
272 result_collector = ResultCollector.new
260 else
273 else
261 result_collector = nil
274 result_collector = nil
262 end
275 end
263
276
264 if options[:dry_run]
277 if options[:dry_run]
265 puts "Running in dry mode"
278 puts "Running in dry mode"
266 end
279 end
267
280
268 prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
281 prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
269 :result_collector => result_collector)
282 :result_collector => result_collector)
270 engine = Grader::Engine.new(:reporter => prob_reporter)
283 engine = Grader::Engine.new(:reporter => prob_reporter)
271 runner = Grader::Runner.new(engine, grader_proc)
284 runner = Grader::Runner.new(engine, grader_proc)
272
285
273 grader_proc.report_active if grader_proc!=nil
286 grader_proc.report_active if grader_proc!=nil
274
287
275 ARGV.each do |prob_name|
288 ARGV.each do |prob_name|
276 prob = Problem.find_by_name(prob_name)
289 prob = Problem.find_by_name(prob_name)
277 if prob==nil
290 if prob==nil
278 puts "cannot find problem: #{prob_name}"
291 puts "cannot find problem: #{prob_name}"
279 else
292 else
280 runner.grade_problem(prob)
293 runner.grade_problem(prob)
281 end
294 end
282 end
295 end
283
296
284 if options[:report]
297 if options[:report]
285 result_collector.print_report_by_user
298 result_collector.print_report_by_user
286 end
299 end
287 end
300 end
288
301
289 def grader_grade_contests(grader_proc, options)
302 def grader_grade_contests(grader_proc, options)
290 # always use dry run when grading during contest
303 # always use dry run when grading during contest
291 dry_run = options[:dry_run] = true
304 dry_run = options[:dry_run] = true
292
305
293 contest_name = ARGV.shift
306 contest_name = ARGV.shift
294
307
295 contest = Contest.find_by_name(contest_name)
308 contest = Contest.find_by_name(contest_name)
296 if contest==nil
309 if contest==nil
297 puts "cannot find contest: #{contest_name}"
310 puts "cannot find contest: #{contest_name}"
298 exit(0)
311 exit(0)
299 end
312 end
300
313
301 if options[:report]
314 if options[:report]
302 result_collector = ResultCollector.new
315 result_collector = ResultCollector.new
303 else
316 else
304 result_collector = nil
317 result_collector = nil
305 end
318 end
306
319
307 if options[:dry_run]
320 if options[:dry_run]
308 puts "Running in dry mode"
321 puts "Running in dry mode"
309 end
322 end
310
323
311 prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
324 prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
312 :result_collector => result_collector)
325 :result_collector => result_collector)
313 engine = Grader::Engine.new(:reporter => prob_reporter)
326 engine = Grader::Engine.new(:reporter => prob_reporter)
314 runner = Grader::Runner.new(engine, grader_proc)
327 runner = Grader::Runner.new(engine, grader_proc)
315
328
316 grader_proc.report_active if grader_proc!=nil
329 grader_proc.report_active if grader_proc!=nil
317
330
318 contest.problems.each do |problem|
331 contest.problems.each do |problem|
319 puts "Grading: #{problem.name}"
332 puts "Grading: #{problem.name}"
320 runner.grade_problem(problem,
333 runner.grade_problem(problem,
321 :user_conditions => lambda do |u|
334 :user_conditions => lambda do |u|
322 u.contest_finished? and
335 u.contest_finished? and
323 u.contest_ids.include?(contest.id)
336 u.contest_ids.include?(contest.id)
324 end)
337 end)
325 end
338 end
326
339
327 if options[:report]
340 if options[:report]
328 result_collector.print_report_by_user
341 result_collector.print_report_by_user
329 end
342 end
330 end
343 end
331
344
332 def grader_grade_submissions(grader_proc, options)
345 def grader_grade_submissions(grader_proc, options)
333 engine = Grader::Engine.new
346 engine = Grader::Engine.new
334 runner = Grader::Runner.new(engine, grader_proc)
347 runner = Grader::Runner.new(engine, grader_proc)
335
348
336 grader_proc.report_active if grader_proc!=nil
349 grader_proc.report_active if grader_proc!=nil
337
350
338 ARGV.each do |sub_id|
351 ARGV.each do |sub_id|
339 puts "Grading #{sub_id}"
352 puts "Grading #{sub_id}"
340 begin
353 begin
341 submission = Submission.find(sub_id.to_i)
354 submission = Submission.find(sub_id.to_i)
342 rescue ActiveRecord::RecordNotFound
355 rescue ActiveRecord::RecordNotFound
343 puts "Record not found"
356 puts "Record not found"
344 submission = nil
357 submission = nil
345 end
358 end
346
359
347 if submission!=nil
360 if submission!=nil
348 runner.grade_submission(submission)
361 runner.grade_submission(submission)
349 end
362 end
350 end
363 end
351 end
364 end
352
365
353 #########################################
366 #########################################
354 # main program
367 # main program
355 #########################################
368 #########################################
356
369
357 options = process_options_and_stop_file
370 options = process_options_and_stop_file
358 GRADER_ENV = options[:environment]
371 GRADER_ENV = options[:environment]
359 grader_mode = options[:mode]
372 grader_mode = options[:mode]
360 dry_run = options[:dry_run]
373 dry_run = options[:dry_run]
361
374
362 puts "environment: #{GRADER_ENV}"
375 puts "environment: #{GRADER_ENV}"
363 require File.join(File.dirname(__FILE__),'config/environment')
376 require File.join(File.dirname(__FILE__),'config/environment')
364
377
365 # add grader_mode to config
378 # add grader_mode to config
366 # this is needed because method log needs it. TODO: clean this up
379 # this is needed because method log needs it. TODO: clean this up
367 class << config
380 class << config
368 attr_accessor :grader_mode
381 attr_accessor :grader_mode
369 end
382 end
370 config.grader_mode = grader_mode
383 config.grader_mode = grader_mode
371
384
372 # reading rails environment
385 # reading rails environment
373 log 'Reading rails environment'
386 log 'Reading rails environment'
374
387
375 RAILS_ENV = config.rails_env
388 RAILS_ENV = config.rails_env
376 require RAILS_ROOT + '/config/environment'
389 require RAILS_ROOT + '/config/environment'
377
390
378 # register grader process
391 # register grader process
379 if config.report_grader
392 if config.report_grader
380 grader_proc = GraderProcess.register(config.grader_hostname,
393 grader_proc = GraderProcess.register(config.grader_hostname,
381 Process.pid,
394 Process.pid,
382 grader_mode)
395 grader_mode)
383 else
396 else
384 grader_proc = nil
397 grader_proc = nil
385 end
398 end
386
399
387 #set loggin environment
400 #set loggin environment
388 ENV['GRADER_LOGGING'] = log_file_name
401 ENV['GRADER_LOGGING'] = log_file_name
389
402
390 # register exit handler to report inactive, and terminated
403 # register exit handler to report inactive, and terminated
391 at_exit do
404 at_exit do
392 if grader_proc!=nil
405 if grader_proc!=nil
393 grader_proc.report_inactive
406 grader_proc.report_inactive
394 grader_proc.terminate
407 grader_proc.terminate
395 end
408 end
396 end
409 end
397
410
398 #
411 #
399 # MAIN LOOP
412 # MAIN LOOP
400 #
413 #
401
414
402 case grader_mode
415 case grader_mode
403 when "queue"
416 when "queue"
404 grader_queue_loop(grader_proc, options)
417 grader_queue_loop(grader_proc, options)
405
418
406 when "test_request"
419 when "test_request"
407 grader_test_request_loop(grader_proc, options)
420 grader_test_request_loop(grader_proc, options)
408
421
409 when "prob"
422 when "prob"
410 grader_grade_problems(grader_proc, options)
423 grader_grade_problems(grader_proc, options)
411
424
412 when "contest"
425 when "contest"
413 grader_grade_contests(grader_proc, options)
426 grader_grade_contests(grader_proc, options)
414
427
415 when "sub"
428 when "sub"
416 grader_grade_submissions(grader_proc, options)
429 grader_grade_submissions(grader_proc, options)
417
430
418 when "autonew"
431 when "autonew"
419 grader_autonew_loop(grader_proc, options)
432 grader_autonew_loop(grader_proc, options)
420
433
421 else
434 else
422 display_manual
435 display_manual
423 exit(0)
436 exit(0)
424 end
437 end
425
438
You need to be logged in to leave comments. Login now