cafe-grader-judge-scripts

Commit r184:4e43dab877a1 public

parent | child
Description:
add options for grading all submissions of a specific problem (grafted from branch algo-bm 8ed1c0aa59eaf8e22e40fc765c1fba4ac88a34b5)
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r184:4e43dab877a1 - - 2 files changed: 25 inserted, 4 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,443 +1,459
1 1 #!/usr/bin/env ruby
2 2
3 3 def stop_grader(id)
4 4 if id==:all
5 5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
6 6 else
7 7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
8 8 end
9 9 end
10 10
11 11 def check_stopfile
12 12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
13 13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
14 14 end
15 15
16 16 def clear_stopfile
17 17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 18 File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
19 19 end
20 20 end
21 21
22 22 def config
23 23 Grader::Configuration.get_instance
24 24 end
25 25
26 26 def log_file_name
27 27 if !File.exists?(config.log_dir)
28 28 raise "Log directory does not exist: #{config.log_dir}"
29 29 end
30 30 config.log_dir +
31 31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
32 32 end
33 33
34 34 def log(str)
35 35 if config.talkative
36 36 puts str
37 37 end
38 38 if config.logging
39 39 fp = File.open(log_file_name,"a")
40 40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
41 41 fp.close
42 42 end
43 43 end
44 44
45 45 def display_manual
46 46 puts <<USAGE
47 47 Grader.
48 48 using: (1) grader
49 - (2) grader environment [mode]
49 + (2) grader environment [mode] [options]
50 50 (3) grader stop [all|pids-list]
51 51 (4) grader --help
52 52 (1) call grader with environment = 'exam', mode = 'queue'
53 53 (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
54 + queue: repeatedly check the task queue and grade any available tasks
55 +
56 + prob: re-grade every user latest submission of the specific problem.
57 + the problem name must be specified by the next argument.
58 +
59 + additional options:
60 +
61 + --all-sub re-grade every submissions instead of just the latest submission of each user.
62 + sub: re-grader the specified submission.
63 + The submission ID to be re-graded must be specified by the next argument.
64 +
54 65 (3) create stop-file to stop running grader in queue mode
55 66 (4) You are here.
56 67 USAGE
57 68 end
58 69
59 70 def process_options_and_stop_file
60 71 # The list of options are:
61 72 # - stop [all|process ids]
62 73 # -
63 74
64 75 # Process 'help' option
65 76 if (ARGV.length==1) and (/help/.match(ARGV[0]))
66 77 display_manual
67 78 exit(0)
68 79 end
69 80
70 81 # Process 'stop' option.
71 82 if (ARGV.length >= 1) and (ARGV[0]=='stop')
72 83 if ARGV.length==1
73 84 puts "you should specify pid-list or 'all'"
74 85 display_manual
75 86 elsif (ARGV.length==2) and (ARGV[1]=='all')
76 87 stop_grader(:all)
77 88 puts "A global stop file ('stop.all') created."
78 89 puts "You should remove it manually later."
79 90 else
80 91 (1..ARGV.length-1).each do |i|
81 92 stop_grader(ARGV[i])
82 93 end
83 94 puts "stop file(s) created"
84 95 end
85 96 exit(0)
86 97 end
87 98
88 99 # Check stop file.
89 100 if check_stopfile
90 101 puts "Stop file exists. Terminated."
91 102 clear_stopfile
92 103 exit(0)
93 104 end
94 105
95 106 #default options
96 107 options = {
97 108 :mode => 'queue',
98 109 :environment => 'exam',
99 110 :dry_run => false,
100 111 }
101 112
102 113 # Process mode and environment option
103 114 if ARGV.length >= 1
104 115 options[:environment] = ARGV.shift
105 116 if ARGV.length >=1
106 117 options[:mode] = ARGV.shift
107 118 end
119 + else
120 + puts 'no argument specified, using default mode and environment.'
108 121 end
109 122
110 123 options[:dry_run] = (ARGV.delete('--dry') != nil)
111 124 if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
112 125 puts "Dry run currently works only for 'prob' or 'contest' modes."
113 126 exit(0)
114 127 end
115 128
116 129 options[:report] = (ARGV.delete('--report') != nil)
117 130 if options[:report] and (not ['prob','contest','autonew'].include? options[:mode])
118 131 puts "Report currently works only for 'prob' or 'contest' modes."
119 132 exit(0)
120 133 end
121 134
135 + options[:all_sub] = (ARGV.delete('--all-sub') != nil)
136 +
122 137 return options
123 138 end
124 139
125 140 class ResultCollector
126 141 def initialize
127 142 @results = {}
128 143 @problems = {}
129 144 @users = {}
130 145 end
131 146
132 147 def after_save_hook(submission, grading_result)
133 148 end
134 149
135 150 def save(submission, grading_result)
136 151 user = submission.user
137 152 problem = submission.problem
138 153 if not @problems.has_key? problem.id
139 154 @problems[problem.id] = problem
140 155 end
141 156 if not @users.has_key? user.id
142 157 @users[user.id] = user
143 158 end
144 159 @results[[user.id, problem.id]] = grading_result
145 160
146 161 after_save_hook(submission, grading_result)
147 162 end
148 163
149 164 def print_report_by_user
150 165 puts "---------------------"
151 166 puts " REPORT"
152 167 puts "---------------------"
153 168
154 169 print "login,email"
155 170 @problems.each_value do |problem|
156 171 print ",#{problem.name}"
157 172 end
158 173 print "\n"
159 174
160 175 @users.each_value do |user|
161 176 print "#{user.login},#{user.email}"
162 177 @problems.each_value do |problem|
163 178 if @results.has_key? [user.id, problem.id]
164 179 print ",#{@results[[user.id,problem.id]][:points]}"
165 180 else
166 181 print ","
167 182 end
168 183 end
169 184 print "\n"
170 185 end
171 186 end
172 187 end
173 188
174 189 def grader_general_loop(engine, grader_proc, options)
175 190 runner = Grader::Runner.new(engine, grader_proc)
176 191 while true
177 192
178 193 if check_stopfile # created by calling grader stop
179 194 clear_stopfile
180 195 log "stopped (with stop file)"
181 196 break
182 197 end
183 198
184 199 task = yield(runner)
185 200
186 201 if task==nil
187 202 sleep(1)
188 203 end
189 204 end
190 205 end
191 206
192 207 def grader_queue_loop(grader_proc, options)
193 208 log "Grader: queue"
194 209 engine = Grader::Engine.new
195 210 grader_general_loop(engine, grader_proc, options) do |runner|
196 211 runner.grade_oldest_task
197 212 end
198 213 end
199 214
200 215 def grader_test_request_loop(grader_proc, options)
201 216 log "Grader: test_request"
202 217 engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
203 218 :reporter => Grader::TestRequestReporter.new)
204 219 grader_general_loop(engine, grader_proc, options) do |runner|
205 220 runner.grade_oldest_test_request
206 221 end
207 222 end
208 223
209 224 def grader_autonew_loop(grader_proc, options)
210 225 log "Grader: autonew"
211 226
212 227 if options[:report]
213 228 result_collector = ResultCollector.new
214 229 else
215 230 result_collector = nil
216 231 end
217 232
218 233 if options[:dry_run]
219 234 puts "Running in dry mode"
220 235 end
221 236
222 237 prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
223 238 :result_collector => result_collector)
224 239
225 240 engine = Grader::Engine.new(:reporter => prob_reporter)
226 241 runner = Grader::Runner.new(engine, grader_proc)
227 242
228 243 grader_proc.report_active if grader_proc!=nil
229 244
230 245 latest_submitted_at = nil
231 246 graded_submission_ids = {}
232 247
233 248 while true
234 249
235 250 if check_stopfile # created by calling grader stop
236 251 clear_stopfile
237 252 log "stopped (with stop file)"
238 253 break
239 254 end
240 255
241 256 if latest_submitted_at==nil
242 257 submissions = Submission.all
243 258 else
244 259 submissions = Submission.all(:conditions => ["submitted_at >= :latest",
245 260 {:latest => latest_submitted_at}])
246 261 end
247 262
248 263 graded_any = false
249 264
250 265 if submissions.length != 0
251 266 submissions.each do |submission|
252 267 if (submission.problem == nil) or (!submission.problem.available)
253 268 next
254 269 end
255 270 if ! graded_submission_ids[submission.id]
256 271 runner.grade_submission(submission)
257 272 graded_submission_ids[submission.id] = true
258 273 if (!latest_submitted_at or
259 274 latest_submitted_at < submission.submitted_at)
260 275 latest_submitted_at = submission.submitted_at
261 276 end
262 277 puts "graded: #{submission.id}"
263 278 puts "latest: #{latest_submitted_at}"
264 279 graded_any = true
265 280 end
266 281 end
267 282 end
268 283
269 284 if ! graded_any
270 285 sleep(1)
271 286 end
272 287 end
273 288 end
274 289
275 290 def grader_grade_problems(grader_proc, options)
276 291 if options[:report]
277 292 result_collector = ResultCollector.new
278 293 else
279 294 result_collector = nil
280 295 end
281 296
282 297 if options[:dry_run]
283 298 puts "Running in dry mode"
284 299 end
285 300
286 301 prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
287 302 :result_collector => result_collector)
288 303 engine = Grader::Engine.new(:reporter => prob_reporter)
289 304 runner = Grader::Runner.new(engine, grader_proc)
290 305
291 306 grader_proc.report_active if grader_proc!=nil
292 307
293 308 ARGV.each do |prob_name|
294 309 prob = Problem.find_by_name(prob_name)
295 310 if prob==nil
296 311 puts "cannot find problem: #{prob_name}"
297 312 else
298 - runner.grade_problem(prob)
313 + runner.grade_problem(prob,options)
299 314 end
300 315 end
301 316
302 317 if options[:report]
303 318 result_collector.print_report_by_user
304 319 end
305 320 end
306 321
307 322 def grader_grade_contests(grader_proc, options)
308 323 # always use dry run when grading during contest
309 324 dry_run = options[:dry_run] = true
310 325
311 326 contest_name = ARGV.shift
312 327
313 328 contest = Contest.find_by_name(contest_name)
314 329 if contest==nil
315 330 puts "cannot find contest: #{contest_name}"
316 331 exit(0)
317 332 end
318 333
319 334 if options[:report]
320 335 result_collector = ResultCollector.new
321 336 else
322 337 result_collector = nil
323 338 end
324 339
325 340 if options[:dry_run]
326 341 puts "Running in dry mode"
327 342 end
328 343
329 344 prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
330 345 :result_collector => result_collector)
331 346 engine = Grader::Engine.new(:reporter => prob_reporter)
332 347 runner = Grader::Runner.new(engine, grader_proc)
333 348
334 349 grader_proc.report_active if grader_proc!=nil
335 350
336 351 contest.problems.each do |problem|
337 352 puts "Grading: #{problem.name}"
338 353 runner.grade_problem(problem,
339 354 :user_conditions => lambda do |u|
340 355 u.contest_finished? and
341 356 u.contest_ids.include?(contest.id)
342 357 end)
343 358 end
344 359
345 360 if options[:report]
346 361 result_collector.print_report_by_user
347 362 end
348 363 end
349 364
350 365 def grader_grade_submissions(grader_proc, options)
351 366 engine = Grader::Engine.new
352 367 runner = Grader::Runner.new(engine, grader_proc)
353 368
354 369 grader_proc.report_active if grader_proc!=nil
355 370
356 371 ARGV.each do |sub_id|
357 372 puts "Grading #{sub_id}"
358 373 begin
359 374 submission = Submission.find(sub_id.to_i)
360 375 rescue ActiveRecord::RecordNotFound
361 376 puts "Submission #{sub_id} not found"
362 377 submission = nil
363 378 end
364 379
365 380 if submission!=nil
366 381 runner.grade_submission(submission)
367 382 end
368 383 end
369 384 end
370 385
371 386 #########################################
372 387 # main program
373 388 #########################################
374 389
375 390 options = process_options_and_stop_file
376 391 GRADER_ENV = options[:environment]
377 392 grader_mode = options[:mode]
378 393 dry_run = options[:dry_run]
379 394
380 395 puts "environment: #{GRADER_ENV}"
396 + puts "grader mode: #{grader_mode}"
381 397 require File.join(File.dirname(__FILE__),'config/environment')
382 398
383 399 # add grader_mode to config
384 400 # this is needed because method log needs it. TODO: clean this up
385 401 class << config
386 402 attr_accessor :grader_mode
387 403 end
388 404 config.grader_mode = grader_mode
389 405
390 406 # reading rails environment
391 407 log 'Reading rails environment'
392 408
393 409 RAILS_ENV = config.rails_env
394 410 require RAILS_ROOT + '/config/environment'
395 411
396 412 # register grader process
397 413 if config.report_grader
398 414 grader_proc = GraderProcess.register(config.grader_hostname,
399 415 Process.pid,
400 416 grader_mode)
401 417 else
402 418 grader_proc = nil
403 419 end
404 420
405 421 #set loggin environment
406 422 ENV['GRADER_LOGGING'] = log_file_name
407 423
408 424 # register exit handler to report inactive, and terminated
409 425 at_exit do
410 426 if grader_proc!=nil
411 427 grader_proc.report_inactive
412 428 grader_proc.terminate
413 429 end
414 430 end
415 431
416 432 #
417 433 # MAIN LOOP
418 434 #
419 435
420 436 case grader_mode
421 437 when "queue"
422 438 grader_queue_loop(grader_proc, options)
423 439
424 440 when "test_request"
425 441 grader_test_request_loop(grader_proc, options)
426 442
427 443 when "prob"
428 444 grader_grade_problems(grader_proc, options)
429 445
430 446 when "contest"
431 447 grader_grade_contests(grader_proc, options)
432 448
433 449 when "sub"
434 450 grader_grade_submissions(grader_proc, options)
435 451
436 452 when "autonew"
437 453 grader_autonew_loop(grader_proc, options)
438 454
439 455 else
440 456 display_manual
441 457 exit(0)
442 458 end
443 459
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,62 +1,67
1 1 #
2 2 # A runner drives the engine into various tasks.
3 3 #
4 4
5 5 module Grader
6 6
7 7 class Runner
8 8
9 9 def initialize(engine, grader_process=nil)
10 10 @engine = engine
11 11 @grader_process = grader_process
12 12 end
13 13
14 14 def grade_oldest_task
15 15 task = Task.get_inqueue_and_change_status(Task::STATUS_GRADING)
16 16 if task!=nil
17 17 @grader_process.report_active(task) if @grader_process!=nil
18 18
19 19 submission = Submission.find(task.submission_id)
20 20 @engine.grade(submission)
21 21 task.status_complete!
22 22 @grader_process.report_inactive(task) if @grader_process!=nil
23 23 end
24 24 return task
25 25 end
26 26
27 27 def grade_problem(problem, options={})
28 - users = User.find(:all)
29 - users.each do |u|
28 + User.find_each do |u|
30 29 puts "user: #{u.login}"
31 30 if options[:user_conditions]!=nil
32 31 con_proc = options[:user_conditions]
33 32 next if not con_proc.call(u)
34 33 end
34 + if options[:all_sub]
35 + Submission.where(user_id: u.id,problem_id: problem.id).find_each do |sub|
36 + @engine.grade(sub)
37 + end
38 + else
35 39 last_sub = Submission.find_last_by_user_and_problem(u.id,problem.id)
36 40 if last_sub!=nil
37 41 @engine.grade(last_sub)
38 42 end
39 43 end
40 44 end
45 + end
41 46
42 47 def grade_submission(submission)
43 48 puts "Submission: #{submission.id} by #{submission.user.full_name}"
44 49 @engine.grade(submission)
45 50 end
46 51
47 52 def grade_oldest_test_request
48 53 test_request = TestRequest.get_inqueue_and_change_status(Task::STATUS_GRADING)
49 54 if test_request!=nil
50 55 @grader_process.report_active(test_request) if @grader_process!=nil
51 56
52 57 @engine.grade(test_request)
53 58 test_request.status_complete!
54 59 @grader_process.report_inactive(test_request) if @grader_process!=nil
55 60 end
56 61 return test_request
57 62 end
58 63
59 64 end
60 65
61 66 end
62 67
You need to be logged in to leave comments. Login now