cafe-grader-judge-scripts

Commit r121:6d40867ff9f2 public

parent | child
Description:
refactored grader, added autonew mode
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r121:6d40867ff9f2 - - 1 file changed: 198 inserted, 117 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,344 +1,425
1 1 #!/usr/bin/ruby
2 2
3 3 def stop_grader(id)
4 4 if id==:all
5 5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
6 6 else
7 7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
8 8 end
9 9 end
10 10
11 11 def check_stopfile
12 12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
13 13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
14 14 end
15 15
16 16 def clear_stopfile
17 17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 18 File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
19 19 end
20 20 end
21 21
22 22 def config
23 23 Grader::Configuration.get_instance
24 24 end
25 25
26 26 def log_file_name
27 27 if !File.exists?(config.log_dir)
28 28 raise "Log directory does not exist: #{config.log_dir}"
29 29 end
30 30 config.log_dir +
31 31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
32 32 end
33 33
34 34 def log(str)
35 35 if config.talkative
36 36 puts str
37 37 end
38 38 if config.logging
39 39 fp = File.open(log_file_name,"a")
40 40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
41 41 fp.close
42 42 end
43 43 end
44 44
45 45 def display_manual
46 46 puts <<USAGE
47 47 Grader.
48 48 using: (1) grader
49 49 (2) grader environment [mode]
50 50 (3) grader stop [all|pids-list]
51 51 (4) grader --help
52 52 (1) call grader with environment = 'exam', mode = 'queue'
53 - (2) possible modes are: 'queue', 'prob', 'test_request'
53 + (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
54 54 (3) create stop-file to stop running grader in queue mode
55 55 (4) You are here.
56 56 USAGE
57 57 end
58 58
59 59 def process_options_and_stop_file
60 60 # The list of options are:
61 61 # - stop [all|process ids]
62 62 # -
63 63
64 64 # Process 'help' option
65 65 if (ARGV.length==1) and (/help/.match(ARGV[0]))
66 66 display_manual
67 67 exit(0)
68 68 end
69 69
70 70 # Process 'stop' option.
71 71 if (ARGV.length >= 1) and (ARGV[0]=='stop')
72 72 if ARGV.length==1
73 73 puts "you should specify pid-list or 'all'"
74 74 display_manual
75 75 elsif (ARGV.length==2) and (ARGV[1]=='all')
76 76 stop_grader(:all)
77 77 puts "A global stop file ('stop.all') created."
78 78 puts "You should remove it manually later."
79 79 else
80 80 (1..ARGV.length-1).each do |i|
81 81 stop_grader(ARGV[i])
82 82 end
83 83 puts "stop file(s) created"
84 84 end
85 85 exit(0)
86 86 end
87 87
88 88 # Check stop file.
89 89 if check_stopfile
90 90 puts "Stop file exists. Terminated."
91 91 clear_stopfile
92 92 exit(0)
93 93 end
94 94
95 95 #default options
96 96 options = {
97 97 :mode => 'queue',
98 98 :environment => 'exam',
99 99 :dry_run => false,
100 100 }
101 101
102 102 # Process mode and environment option
103 103 if ARGV.length >= 1
104 104 options[:environment] = ARGV.shift
105 105 if ARGV.length >=1
106 106 options[:mode] = ARGV.shift
107 107 end
108 108 end
109 109
110 110 options[:dry_run] = (ARGV.delete('--dry') != nil)
111 - if options[:dry_run] and (not ['prob','contest'].include? options[:mode])
111 + if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
112 112 puts "Dry run currently works only for 'prob' or 'contest' modes."
113 113 exit(0)
114 114 end
115 115
116 116 options[:report] = (ARGV.delete('--report') != nil)
117 117 if options[:report] and (not ['prob','contest'].include? options[:mode])
118 118 puts "Report currently works only for 'prob' or 'contest' modes."
119 119 exit(0)
120 120 end
121 121
122 122 return options
123 123 end
124 124
125 125 class ResultCollector
126 126 def initialize
127 127 @results = {}
128 128 @problems = {}
129 129 @users = {}
130 130 end
131 131
132 132 def save(user, problem, grading_result)
133 133 if not @problems.has_key? problem.id
134 134 @problems[problem.id] = problem
135 135 end
136 136 if not @users.has_key? user.id
137 137 @users[user.id] = user
138 138 end
139 139 @results[[user.id, problem.id]] = grading_result
140 140 end
141 141
142 142 def print_report_by_user
143 143 puts "---------------------"
144 144 puts " REPORT"
145 145 puts "---------------------"
146 146
147 147 print "login,email"
148 148 @problems.each_value do |problem|
149 149 print ",#{problem.name}"
150 150 end
151 151 print "\n"
152 152
153 153 @users.each_value do |user|
154 154 print "#{user.login},#{user.email}"
155 155 @problems.each_value do |problem|
156 156 if @results.has_key? [user.id, problem.id]
157 157 print ",#{@results[[user.id,problem.id]][:points]}"
158 158 else
159 159 print ","
160 160 end
161 161 end
162 162 print "\n"
163 163 end
164 164 end
165 165 end
166 166
167 + def grader_general_loop(engine, grader_proc, options)
168 + runner = Grader::Runner.new(engine, grader_proc)
169 + while true
170 +
171 + if check_stopfile # created by calling grader stop
172 + clear_stopfile
173 + log "stopped (with stop file)"
174 + break
175 + end
176 +
177 + task = yield(runner)
178 +
179 + if task==nil
180 + sleep(1)
181 + end
182 + end
183 + end
184 +
185 + def grader_queue_loop(grader_proc, options)
186 + log "Grader: queue"
187 + engine = Grader::Engine.new
188 + grader_general_loop(engine, grader_proc, options) do |runner|
189 + runner.grade_oldest_task
190 + end
191 + end
192 +
193 + def grader_test_request_loop(grader_proc, options)
194 + log "Grader: test_request"
195 + engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
196 + :reporter => Grader::TestRequestReporter.new)
197 + grader_general_loop(engine, grader_proc, options) do |runner|
198 + runner.grade_oldest_test_request
199 + end
200 + end
201 +
202 + def grader_autonew_loop(grader_proc, options)
203 + log "Grader: autonew"
204 +
205 + if options[:dry_run]
206 + puts "Running in dry mode"
207 + end
208 +
209 + prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run])
210 + engine = Grader::Engine.new(:reporter => prob_reporter)
211 + runner = Grader::Runner.new(engine, grader_proc)
212 +
213 + grader_proc.report_active if grader_proc!=nil
214 +
215 + latest_submitted_at = nil
216 + graded_submission_ids = {}
217 +
218 + while true
219 +
220 + if check_stopfile # created by calling grader stop
221 + clear_stopfile
222 + log "stopped (with stop file)"
223 + break
224 + end
225 +
226 + if latest_submitted_at==nil
227 + submissions = Submission.all
228 + else
229 + submissions = Submission.all(:conditions => ["submitted_at >= :latest",
230 + {:latest => latest_submitted_at}])
231 + end
232 +
233 + graded_any = false
234 +
235 + if submissions.length != 0
236 + submissions.each do |submission|
237 + if ! graded_submission_ids[submission.id]
238 + runner.grade_submission(submission)
239 + graded_submission_ids[submission.id] = true
240 + if (!latest_submitted_at or
241 + latest_submitted_at < submission.submitted_at)
242 + latest_submitted_at = submission.submitted_at
243 + end
244 + puts "graded: #{submission.id}"
245 + puts "latest: #{latest_submitted_at}"
246 + graded_any = true
247 + end
248 + end
249 + end
250 +
251 + if ! graded_any
252 + sleep(1)
253 + end
254 + end
255 + end
256 +
257 + def grader_grade_problems(grader_proc, options)
258 + if options[:report]
259 + result_collector = ResultCollector.new
260 + else
261 + result_collector = nil
262 + end
263 +
264 + if options[:dry_run]
265 + puts "Running in dry mode"
266 + end
267 +
268 + prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
269 + :result_collector => result_collector)
270 + engine = Grader::Engine.new(:reporter => prob_reporter)
271 + runner = Grader::Runner.new(engine, grader_proc)
272 +
273 + grader_proc.report_active if grader_proc!=nil
274 +
275 + ARGV.each do |prob_name|
276 + prob = Problem.find_by_name(prob_name)
277 + if prob==nil
278 + puts "cannot find problem: #{prob_name}"
279 + else
280 + runner.grade_problem(prob)
281 + end
282 + end
283 +
284 + if options[:report]
285 + result_collector.print_report_by_user
286 + end
287 + end
288 +
289 + def grader_grade_contests(grader_proc, options)
290 + # always use dry run when grading during contest
291 + dry_run = options[:dry_run] = true
292 +
293 + contest_name = ARGV.shift
294 +
295 + contest = Contest.find_by_name(contest_name)
296 + if contest==nil
297 + puts "cannot find contest: #{contest_name}"
298 + exit(0)
299 + end
300 +
301 + if options[:report]
302 + result_collector = ResultCollector.new
303 + else
304 + result_collector = nil
305 + end
306 +
307 + if options[:dry_run]
308 + puts "Running in dry mode"
309 + end
310 +
311 + prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
312 + :result_collector => result_collector)
313 + engine = Grader::Engine.new(:reporter => prob_reporter)
314 + runner = Grader::Runner.new(engine, grader_proc)
315 +
316 + grader_proc.report_active if grader_proc!=nil
317 +
318 + contest.problems.each do |problem|
319 + puts "Grading: #{problem.name}"
320 + runner.grade_problem(problem,
321 + :user_conditions => lambda do |u|
322 + u.contest_finished? and
323 + u.contest_ids.include?(contest.id)
324 + end)
325 + end
326 +
327 + if options[:report]
328 + result_collector.print_report_by_user
329 + end
330 + end
331 +
332 + def grader_grade_submissions(grader_proc, options)
333 + engine = Grader::Engine.new
334 + runner = Grader::Runner.new(engine, grader_proc)
335 +
336 + grader_proc.report_active if grader_proc!=nil
337 +
338 + ARGV.each do |sub_id|
339 + puts "Grading #{sub_id}"
340 + begin
341 + submission = Submission.find(sub_id.to_i)
342 + rescue ActiveRecord::RecordNotFound
343 + puts "Record not found"
344 + submission = nil
345 + end
346 +
347 + if submission!=nil
348 + runner.grade_submission(submission)
349 + end
350 + end
351 + end
352 +
167 353 #########################################
168 354 # main program
169 355 #########################################
170 356
171 357 options = process_options_and_stop_file
172 358 GRADER_ENV = options[:environment]
173 359 grader_mode = options[:mode]
174 360 dry_run = options[:dry_run]
175 361
176 362 puts "environment: #{GRADER_ENV}"
177 363 require File.join(File.dirname(__FILE__),'config/environment')
178 364
179 365 # add grader_mode to config
180 366 # this is needed because method log needs it. TODO: clean this up
181 367 class << config
182 368 attr_accessor :grader_mode
183 369 end
184 370 config.grader_mode = grader_mode
185 371
186 372 # reading rails environment
187 373 log 'Reading rails environment'
188 374
189 375 RAILS_ENV = config.rails_env
190 376 require RAILS_ROOT + '/config/environment'
191 377
192 378 # register grader process
193 379 if config.report_grader
194 380 grader_proc = GraderProcess.register(config.grader_hostname,
195 381 Process.pid,
196 382 grader_mode)
197 383 else
198 384 grader_proc = nil
199 385 end
200 386
201 387 #set loggin environment
202 388 ENV['GRADER_LOGGING'] = log_file_name
203 389
204 390 # register exit handler to report inactive, and terminated
205 391 at_exit do
206 392 if grader_proc!=nil
207 393 grader_proc.report_inactive
208 394 grader_proc.terminate
209 395 end
210 396 end
211 397
212 398 #
213 399 # MAIN LOOP
214 400 #
215 401
216 402 case grader_mode
217 - when "queue", "test_request"
218 - log "Grader: #{grader_mode}"
219 - if grader_mode=="queue"
220 - engine = Grader::Engine.new
221 - else
222 - engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
223 - :reporter => Grader::TestRequestReporter.new)
224 - end
403 + when "queue"
404 + grader_queue_loop(grader_proc, options)
225 405
226 - runner = Grader::Runner.new(engine, grader_proc)
227 - while true
228 -
229 - if check_stopfile # created by calling grader stop
230 - clear_stopfile
231 - log "stopped (with stop file)"
232 - break
233 - end
234 -
235 - if grader_mode=="queue"
236 - task = runner.grade_oldest_task
237 - else
238 - task = runner.grade_oldest_test_request
239 - end
240 - if task==nil
241 - sleep(1)
242 - end
243 - end
244 -
406 + when "test_request"
407 + grader_test_request_loop(grader_proc, options)
408 +
245 409 when "prob"
246 - if options[:report]
247 - result_collector = ResultCollector.new
248 - else
249 - result_collector = nil
250 - end
251 -
252 - if options[:dry_run]
253 - puts "Running in dry mode"
254 - end
255 -
256 - prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
257 - :result_collector => result_collector)
258 - engine = Grader::Engine.new(:reporter => prob_reporter)
259 - runner = Grader::Runner.new(engine, grader_proc)
260 -
261 - grader_proc.report_active if grader_proc!=nil
262 -
263 - ARGV.each do |prob_name|
264 - prob = Problem.find_by_name(prob_name)
265 - if prob==nil
266 - puts "cannot find problem: #{prob_name}"
267 - else
268 - runner.grade_problem(prob)
269 - end
270 - end
271 -
272 - if options[:report]
273 - result_collector.print_report_by_user
274 - end
410 + grader_grade_problems(grader_proc, options)
275 411
276 412 when "contest"
277 - # always use dry run when grading during contest
278 - contest_name = ARGV.shift
279 -
280 - dry_run = options[:dry_run] = true
281 -
282 - contest = Contest.find_by_name(contest_name)
283 - if contest==nil
284 - puts "cannot find contest: #{contest_name}"
285 - exit(0)
286 - end
287 -
288 - if options[:report]
289 - result_collector = ResultCollector.new
290 - else
291 - result_collector = nil
292 - end
293 -
294 - if options[:dry_run]
295 - puts "Running in dry mode"
296 - end
297 -
298 - prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
299 - :result_collector => result_collector)
300 - engine = Grader::Engine.new(:reporter => prob_reporter)
301 - runner = Grader::Runner.new(engine, grader_proc)
302 -
303 - grader_proc.report_active if grader_proc!=nil
304 -
305 - contest.problems.each do |problem|
306 - puts "Grading: #{problem.name}"
307 - runner.grade_problem(problem,
308 - :user_conditions => lambda do |u|
309 - u.contest_finished? and
310 - u.contest_ids.include?(contest.id)
311 - end)
312 - end
313 -
314 - if options[:report]
315 - result_collector.print_report_by_user
316 - end
413 + grader_grade_contests(grader_proc, options)
317 414
318 415 when "sub"
319 - engine = Grader::Engine.new
320 - runner = Grader::Runner.new(engine, grader_proc)
321 -
322 - grader_proc.report_active if grader_proc!=nil
416 + grader_grade_submissions(grader_proc, options)
323 417
324 - ARGV.each do |sub_id|
325 - puts "Grading #{sub_id}"
326 - begin
327 - submission = Submission.find(sub_id.to_i)
328 - rescue ActiveRecord::RecordNotFound
329 - puts "Record not found"
330 - submission = nil
331 - end
332 -
333 - if submission!=nil
334 - runner.grade_submission(submission)
335 - end
336 - end
337 -
338 -
418 + when "autonew"
419 + grader_autonew_loop(grader_proc, options)
339 420
340 421 else
341 422 display_manual
342 423 exit(0)
343 424 end
344 425
You need to be logged in to leave comments. Login now