cafe-grader-judge-scripts

Commit r200:3db773289a2d public

parent | child
Description:
Merge pull request #6 from nattee/master add options for grading all submissions of a specific problem
Commit status:
[Not Reviewed]
References:
merge default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r200:3db773289a2d - - 2 files changed: 28 inserted, 7 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,169 +1,184
1 #!/usr/bin/env ruby
1 #!/usr/bin/env ruby
2
2
3 def stop_grader(id)
3 def stop_grader(id)
4 if id==:all
4 if id==:all
5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
6 else
6 else
7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
8 end
8 end
9 end
9 end
10
10
11 def check_stopfile
11 def check_stopfile
12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
14 end
14 end
15
15
16 def clear_stopfile
16 def clear_stopfile
17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
19 end
19 end
20 end
20 end
21
21
22 def config
22 def config
23 Grader::Configuration.get_instance
23 Grader::Configuration.get_instance
24 end
24 end
25
25
26 def log_file_name
26 def log_file_name
27 if !File.exists?(config.log_dir)
27 if !File.exists?(config.log_dir)
28 raise "Log directory does not exist: #{config.log_dir}"
28 raise "Log directory does not exist: #{config.log_dir}"
29 end
29 end
30 config.log_dir +
30 config.log_dir +
31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
32 end
32 end
33
33
34 def log(str)
34 def log(str)
35 if config.talkative
35 if config.talkative
36 puts str
36 puts str
37 end
37 end
38 if config.logging
38 if config.logging
39 fp = File.open(log_file_name,"a")
39 fp = File.open(log_file_name,"a")
40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
41 fp.close
41 fp.close
42 end
42 end
43 end
43 end
44
44
45 def display_manual
45 def display_manual
46 puts <<USAGE
46 puts <<USAGE
47 Grader.
47 Grader.
48 using: (1) grader
48 using: (1) grader
49 - (2) grader environment [mode]
49 + (2) grader environment [mode] [options]
50 (3) grader stop [all|pids-list]
50 (3) grader stop [all|pids-list]
51 (4) grader --help
51 (4) grader --help
52 (1) call grader with environment = 'exam', mode = 'queue'
52 (1) call grader with environment = 'exam', mode = 'queue'
53 (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
53 (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
54 + queue: repeatedly check the task queue and grade any available tasks
55 +
56 + prob: re-grade every user latest submission of the specific problem.
57 + the problem name must be specified by the next argument.
58 +
59 + additional options:
60 +
61 + --all-sub re-grade every submissions instead of just the latest submission of each user.
62 + sub: re-grader the specified submission.
63 + The submission ID to be re-graded must be specified by the next argument.
64 +
54 (3) create stop-file to stop running grader in queue mode
65 (3) create stop-file to stop running grader in queue mode
55 (4) You are here.
66 (4) You are here.
56 USAGE
67 USAGE
57 end
68 end
58
69
59 def process_options_and_stop_file
70 def process_options_and_stop_file
60 # The list of options are:
71 # The list of options are:
61 # - stop [all|process ids]
72 # - stop [all|process ids]
62 # -
73 # -
63
74
64 # Process 'help' option
75 # Process 'help' option
65 if (ARGV.length==1) and (/help/.match(ARGV[0]))
76 if (ARGV.length==1) and (/help/.match(ARGV[0]))
66 display_manual
77 display_manual
67 exit(0)
78 exit(0)
68 end
79 end
69
80
70 # Process 'stop' option.
81 # Process 'stop' option.
71 if (ARGV.length >= 1) and (ARGV[0]=='stop')
82 if (ARGV.length >= 1) and (ARGV[0]=='stop')
72 if ARGV.length==1
83 if ARGV.length==1
73 puts "you should specify pid-list or 'all'"
84 puts "you should specify pid-list or 'all'"
74 display_manual
85 display_manual
75 elsif (ARGV.length==2) and (ARGV[1]=='all')
86 elsif (ARGV.length==2) and (ARGV[1]=='all')
76 stop_grader(:all)
87 stop_grader(:all)
77 puts "A global stop file ('stop.all') created."
88 puts "A global stop file ('stop.all') created."
78 puts "You should remove it manually later."
89 puts "You should remove it manually later."
79 else
90 else
80 (1..ARGV.length-1).each do |i|
91 (1..ARGV.length-1).each do |i|
81 stop_grader(ARGV[i])
92 stop_grader(ARGV[i])
82 end
93 end
83 puts "stop file(s) created"
94 puts "stop file(s) created"
84 end
95 end
85 exit(0)
96 exit(0)
86 end
97 end
87
98
88 # Check stop file.
99 # Check stop file.
89 if check_stopfile
100 if check_stopfile
90 puts "Stop file exists. Terminated."
101 puts "Stop file exists. Terminated."
91 clear_stopfile
102 clear_stopfile
92 exit(0)
103 exit(0)
93 end
104 end
94
105
95 #default options
106 #default options
96 options = {
107 options = {
97 :mode => 'queue',
108 :mode => 'queue',
98 :environment => 'exam',
109 :environment => 'exam',
99 :dry_run => false,
110 :dry_run => false,
100 }
111 }
101
112
102 # Process mode and environment option
113 # Process mode and environment option
103 if ARGV.length >= 1
114 if ARGV.length >= 1
104 options[:environment] = ARGV.shift
115 options[:environment] = ARGV.shift
105 if ARGV.length >=1
116 if ARGV.length >=1
106 options[:mode] = ARGV.shift
117 options[:mode] = ARGV.shift
107 end
118 end
119 + else
120 + puts 'no argument specified, using default mode and environment.'
108 end
121 end
109
122
110 options[:dry_run] = (ARGV.delete('--dry') != nil)
123 options[:dry_run] = (ARGV.delete('--dry') != nil)
111 if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
124 if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
112 puts "Dry run currently works only for 'prob' or 'contest' modes."
125 puts "Dry run currently works only for 'prob' or 'contest' modes."
113 exit(0)
126 exit(0)
114 end
127 end
115
128
116 options[:report] = (ARGV.delete('--report') != nil)
129 options[:report] = (ARGV.delete('--report') != nil)
117 if options[:report] and (not ['prob','contest','autonew'].include? options[:mode])
130 if options[:report] and (not ['prob','contest','autonew'].include? options[:mode])
118 puts "Report currently works only for 'prob' or 'contest' modes."
131 puts "Report currently works only for 'prob' or 'contest' modes."
119 exit(0)
132 exit(0)
120 end
133 end
121
134
135 + options[:all_sub] = (ARGV.delete('--all-sub') != nil)
136 +
122 return options
137 return options
123 end
138 end
124
139
125 class ResultCollector
140 class ResultCollector
126 def initialize
141 def initialize
127 @results = {}
142 @results = {}
128 @problems = {}
143 @problems = {}
129 @users = {}
144 @users = {}
130 end
145 end
131
146
132 def after_save_hook(submission, grading_result)
147 def after_save_hook(submission, grading_result)
133 end
148 end
134
149
135 def save(submission, grading_result)
150 def save(submission, grading_result)
136 user = submission.user
151 user = submission.user
137 problem = submission.problem
152 problem = submission.problem
138 if not @problems.has_key? problem.id
153 if not @problems.has_key? problem.id
139 @problems[problem.id] = problem
154 @problems[problem.id] = problem
140 end
155 end
141 if not @users.has_key? user.id
156 if not @users.has_key? user.id
142 @users[user.id] = user
157 @users[user.id] = user
143 end
158 end
144 @results[[user.id, problem.id]] = grading_result
159 @results[[user.id, problem.id]] = grading_result
145
160
146 after_save_hook(submission, grading_result)
161 after_save_hook(submission, grading_result)
147 end
162 end
148
163
149 def print_report_by_user
164 def print_report_by_user
150 puts "---------------------"
165 puts "---------------------"
151 puts " REPORT"
166 puts " REPORT"
152 puts "---------------------"
167 puts "---------------------"
153
168
154 print "login,email"
169 print "login,email"
155 @problems.each_value do |problem|
170 @problems.each_value do |problem|
156 print ",#{problem.name}"
171 print ",#{problem.name}"
157 end
172 end
158 print "\n"
173 print "\n"
159
174
160 @users.each_value do |user|
175 @users.each_value do |user|
161 print "#{user.login},#{user.email}"
176 print "#{user.login},#{user.email}"
162 @problems.each_value do |problem|
177 @problems.each_value do |problem|
163 if @results.has_key? [user.id, problem.id]
178 if @results.has_key? [user.id, problem.id]
164 print ",#{@results[[user.id,problem.id]][:points]}"
179 print ",#{@results[[user.id,problem.id]][:points]}"
165 else
180 else
166 print ","
181 print ","
167 end
182 end
168 end
183 end
169 print "\n"
184 print "\n"
@@ -250,179 +265,180
250 if submissions.length != 0
265 if submissions.length != 0
251 submissions.each do |submission|
266 submissions.each do |submission|
252 if (submission.problem == nil) or (!submission.problem.available)
267 if (submission.problem == nil) or (!submission.problem.available)
253 next
268 next
254 end
269 end
255 if ! graded_submission_ids[submission.id]
270 if ! graded_submission_ids[submission.id]
256 runner.grade_submission(submission)
271 runner.grade_submission(submission)
257 graded_submission_ids[submission.id] = true
272 graded_submission_ids[submission.id] = true
258 if (!latest_submitted_at or
273 if (!latest_submitted_at or
259 latest_submitted_at < submission.submitted_at)
274 latest_submitted_at < submission.submitted_at)
260 latest_submitted_at = submission.submitted_at
275 latest_submitted_at = submission.submitted_at
261 end
276 end
262 puts "graded: #{submission.id}"
277 puts "graded: #{submission.id}"
263 puts "latest: #{latest_submitted_at}"
278 puts "latest: #{latest_submitted_at}"
264 graded_any = true
279 graded_any = true
265 end
280 end
266 end
281 end
267 end
282 end
268
283
269 if ! graded_any
284 if ! graded_any
270 sleep(1)
285 sleep(1)
271 end
286 end
272 end
287 end
273 end
288 end
274
289
275 def grader_grade_problems(grader_proc, options)
290 def grader_grade_problems(grader_proc, options)
276 if options[:report]
291 if options[:report]
277 result_collector = ResultCollector.new
292 result_collector = ResultCollector.new
278 else
293 else
279 result_collector = nil
294 result_collector = nil
280 end
295 end
281
296
282 if options[:dry_run]
297 if options[:dry_run]
283 puts "Running in dry mode"
298 puts "Running in dry mode"
284 end
299 end
285
300
286 prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
301 prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
287 :result_collector => result_collector)
302 :result_collector => result_collector)
288 engine = Grader::Engine.new(:reporter => prob_reporter)
303 engine = Grader::Engine.new(:reporter => prob_reporter)
289 runner = Grader::Runner.new(engine, grader_proc)
304 runner = Grader::Runner.new(engine, grader_proc)
290
305
291 grader_proc.report_active if grader_proc!=nil
306 grader_proc.report_active if grader_proc!=nil
292
307
293 ARGV.each do |prob_name|
308 ARGV.each do |prob_name|
294 prob = Problem.find_by_name(prob_name)
309 prob = Problem.find_by_name(prob_name)
295 if prob==nil
310 if prob==nil
296 puts "cannot find problem: #{prob_name}"
311 puts "cannot find problem: #{prob_name}"
297 else
312 else
298 - runner.grade_problem(prob)
313 + runner.grade_problem(prob,options)
299 end
314 end
300 end
315 end
301
316
302 if options[:report]
317 if options[:report]
303 result_collector.print_report_by_user
318 result_collector.print_report_by_user
304 end
319 end
305 end
320 end
306
321
307 def grader_grade_contests(grader_proc, options)
322 def grader_grade_contests(grader_proc, options)
308 # always use dry run when grading during contest
323 # always use dry run when grading during contest
309 dry_run = options[:dry_run] = true
324 dry_run = options[:dry_run] = true
310
325
311 contest_name = ARGV.shift
326 contest_name = ARGV.shift
312
327
313 contest = Contest.find_by_name(contest_name)
328 contest = Contest.find_by_name(contest_name)
314 if contest==nil
329 if contest==nil
315 puts "cannot find contest: #{contest_name}"
330 puts "cannot find contest: #{contest_name}"
316 exit(0)
331 exit(0)
317 end
332 end
318
333
319 if options[:report]
334 if options[:report]
320 result_collector = ResultCollector.new
335 result_collector = ResultCollector.new
321 else
336 else
322 result_collector = nil
337 result_collector = nil
323 end
338 end
324
339
325 if options[:dry_run]
340 if options[:dry_run]
326 puts "Running in dry mode"
341 puts "Running in dry mode"
327 end
342 end
328
343
329 prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
344 prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
330 :result_collector => result_collector)
345 :result_collector => result_collector)
331 engine = Grader::Engine.new(:reporter => prob_reporter)
346 engine = Grader::Engine.new(:reporter => prob_reporter)
332 runner = Grader::Runner.new(engine, grader_proc)
347 runner = Grader::Runner.new(engine, grader_proc)
333
348
334 grader_proc.report_active if grader_proc!=nil
349 grader_proc.report_active if grader_proc!=nil
335
350
336 contest.problems.each do |problem|
351 contest.problems.each do |problem|
337 puts "Grading: #{problem.name}"
352 puts "Grading: #{problem.name}"
338 runner.grade_problem(problem,
353 runner.grade_problem(problem,
339 :user_conditions => lambda do |u|
354 :user_conditions => lambda do |u|
340 u.contest_finished? and
355 u.contest_finished? and
341 u.contest_ids.include?(contest.id)
356 u.contest_ids.include?(contest.id)
342 end)
357 end)
343 end
358 end
344
359
345 if options[:report]
360 if options[:report]
346 result_collector.print_report_by_user
361 result_collector.print_report_by_user
347 end
362 end
348 end
363 end
349
364
350 def grader_grade_submissions(grader_proc, options)
365 def grader_grade_submissions(grader_proc, options)
351 engine = Grader::Engine.new
366 engine = Grader::Engine.new
352 runner = Grader::Runner.new(engine, grader_proc)
367 runner = Grader::Runner.new(engine, grader_proc)
353
368
354 grader_proc.report_active if grader_proc!=nil
369 grader_proc.report_active if grader_proc!=nil
355
370
356 ARGV.each do |sub_id|
371 ARGV.each do |sub_id|
357 puts "Grading #{sub_id}"
372 puts "Grading #{sub_id}"
358 begin
373 begin
359 submission = Submission.find(sub_id.to_i)
374 submission = Submission.find(sub_id.to_i)
360 rescue ActiveRecord::RecordNotFound
375 rescue ActiveRecord::RecordNotFound
361 puts "Submission #{sub_id} not found"
376 puts "Submission #{sub_id} not found"
362 submission = nil
377 submission = nil
363 end
378 end
364
379
365 if submission!=nil
380 if submission!=nil
366 runner.grade_submission(submission)
381 runner.grade_submission(submission)
367 end
382 end
368 end
383 end
369 end
384 end
370
385
371 #########################################
386 #########################################
372 # main program
387 # main program
373 #########################################
388 #########################################
374
389
375 options = process_options_and_stop_file
390 options = process_options_and_stop_file
376 GRADER_ENV = options[:environment]
391 GRADER_ENV = options[:environment]
377 grader_mode = options[:mode]
392 grader_mode = options[:mode]
378 dry_run = options[:dry_run]
393 dry_run = options[:dry_run]
379
394
380 puts "environment: #{GRADER_ENV}"
395 puts "environment: #{GRADER_ENV}"
396 + puts "grader mode: #{grader_mode}"
381 require File.join(File.dirname(__FILE__),'config/environment')
397 require File.join(File.dirname(__FILE__),'config/environment')
382
398
383 # add grader_mode to config
399 # add grader_mode to config
384 # this is needed because method log needs it. TODO: clean this up
400 # this is needed because method log needs it. TODO: clean this up
385 class << config
401 class << config
386 attr_accessor :grader_mode
402 attr_accessor :grader_mode
387 end
403 end
388 config.grader_mode = grader_mode
404 config.grader_mode = grader_mode
389
405
390 # reading rails environment
406 # reading rails environment
391 log 'Reading rails environment'
407 log 'Reading rails environment'
392
408
393 RAILS_ENV = config.rails_env
409 RAILS_ENV = config.rails_env
394 require RAILS_ROOT + '/config/environment'
410 require RAILS_ROOT + '/config/environment'
395
411
396 # register grader process
412 # register grader process
397 if config.report_grader
413 if config.report_grader
398 grader_proc = GraderProcess.register(config.grader_hostname,
414 grader_proc = GraderProcess.register(config.grader_hostname,
399 Process.pid,
415 Process.pid,
400 grader_mode)
416 grader_mode)
401 else
417 else
402 grader_proc = nil
418 grader_proc = nil
403 end
419 end
404
420
405 #set loggin environment
421 #set loggin environment
406 ENV['GRADER_LOGGING'] = log_file_name
422 ENV['GRADER_LOGGING'] = log_file_name
407
423
408 # register exit handler to report inactive, and terminated
424 # register exit handler to report inactive, and terminated
409 at_exit do
425 at_exit do
410 if grader_proc!=nil
426 if grader_proc!=nil
411 grader_proc.report_inactive
427 grader_proc.report_inactive
412 grader_proc.terminate
428 grader_proc.terminate
413 end
429 end
414 end
430 end
415
431
416 #
432 #
417 # MAIN LOOP
433 # MAIN LOOP
418 #
434 #
419
435
420 case grader_mode
436 case grader_mode
421 when "queue"
437 when "queue"
422 grader_queue_loop(grader_proc, options)
438 grader_queue_loop(grader_proc, options)
423
439
424 when "test_request"
440 when "test_request"
425 grader_test_request_loop(grader_proc, options)
441 grader_test_request_loop(grader_proc, options)
426
442
427 when "prob"
443 when "prob"
428 grader_grade_problems(grader_proc, options)
444 grader_grade_problems(grader_proc, options)
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,62 +1,67
1 #
1 #
2 # A runner drives the engine into various tasks.
2 # A runner drives the engine into various tasks.
3 #
3 #
4
4
5 module Grader
5 module Grader
6
6
7 class Runner
7 class Runner
8
8
9 def initialize(engine, grader_process=nil)
9 def initialize(engine, grader_process=nil)
10 @engine = engine
10 @engine = engine
11 @grader_process = grader_process
11 @grader_process = grader_process
12 end
12 end
13
13
14 def grade_oldest_task
14 def grade_oldest_task
15 task = Task.get_inqueue_and_change_status(Task::STATUS_GRADING)
15 task = Task.get_inqueue_and_change_status(Task::STATUS_GRADING)
16 if task!=nil
16 if task!=nil
17 @grader_process.report_active(task) if @grader_process!=nil
17 @grader_process.report_active(task) if @grader_process!=nil
18
18
19 submission = Submission.find(task.submission_id)
19 submission = Submission.find(task.submission_id)
20 @engine.grade(submission)
20 @engine.grade(submission)
21 task.status_complete!
21 task.status_complete!
22 @grader_process.report_inactive(task) if @grader_process!=nil
22 @grader_process.report_inactive(task) if @grader_process!=nil
23 end
23 end
24 return task
24 return task
25 end
25 end
26
26
27 def grade_problem(problem, options={})
27 def grade_problem(problem, options={})
28 - users = User.find(:all)
28 + User.find_each do |u|
29 - users.each do |u|
30 puts "user: #{u.login}"
29 puts "user: #{u.login}"
31 if options[:user_conditions]!=nil
30 if options[:user_conditions]!=nil
32 con_proc = options[:user_conditions]
31 con_proc = options[:user_conditions]
33 next if not con_proc.call(u)
32 next if not con_proc.call(u)
34 end
33 end
35 - last_sub = Submission.find_last_by_user_and_problem(u.id,problem.id)
34 + if options[:all_sub]
36 - if last_sub!=nil
35 + Submission.where(user_id: u.id,problem_id: problem.id).find_each do |sub|
37 - @engine.grade(last_sub)
36 + @engine.grade(sub)
37 + end
38 + else
39 + last_sub = Submission.find_last_by_user_and_problem(u.id,problem.id)
40 + if last_sub!=nil
41 + @engine.grade(last_sub)
42 + end
38 end
43 end
39 end
44 end
40 end
45 end
41
46
42 def grade_submission(submission)
47 def grade_submission(submission)
43 puts "Submission: #{submission.id} by #{submission.user.full_name}"
48 puts "Submission: #{submission.id} by #{submission.user.full_name}"
44 @engine.grade(submission)
49 @engine.grade(submission)
45 end
50 end
46
51
47 def grade_oldest_test_request
52 def grade_oldest_test_request
48 test_request = TestRequest.get_inqueue_and_change_status(Task::STATUS_GRADING)
53 test_request = TestRequest.get_inqueue_and_change_status(Task::STATUS_GRADING)
49 if test_request!=nil
54 if test_request!=nil
50 @grader_process.report_active(test_request) if @grader_process!=nil
55 @grader_process.report_active(test_request) if @grader_process!=nil
51
56
52 @engine.grade(test_request)
57 @engine.grade(test_request)
53 test_request.status_complete!
58 test_request.status_complete!
54 @grader_process.report_inactive(test_request) if @grader_process!=nil
59 @grader_process.report_inactive(test_request) if @grader_process!=nil
55 end
60 end
56 return test_request
61 return test_request
57 end
62 end
58
63
59 end
64 end
60
65
61 end
66 end
62
67
You need to be logged in to leave comments. Login now