cafe-grader-judge-scripts

Commit r123:57e6d251108e public

parent | child
Description:
sends whole submission to result collecter in submission reporter, instead of just user and problem
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r123:57e6d251108e - - 2 files changed: 6 inserted, 5 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,336 +1,338
1 1 #!/usr/bin/ruby
2 2
3 3 def stop_grader(id)
4 4 if id==:all
5 5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
6 6 else
7 7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
8 8 end
9 9 end
10 10
11 11 def check_stopfile
12 12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
13 13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
14 14 end
15 15
16 16 def clear_stopfile
17 17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 18 File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
19 19 end
20 20 end
21 21
22 22 def config
23 23 Grader::Configuration.get_instance
24 24 end
25 25
26 26 def log_file_name
27 27 if !File.exists?(config.log_dir)
28 28 raise "Log directory does not exist: #{config.log_dir}"
29 29 end
30 30 config.log_dir +
31 31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
32 32 end
33 33
34 34 def log(str)
35 35 if config.talkative
36 36 puts str
37 37 end
38 38 if config.logging
39 39 fp = File.open(log_file_name,"a")
40 40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
41 41 fp.close
42 42 end
43 43 end
44 44
45 45 def display_manual
46 46 puts <<USAGE
47 47 Grader.
48 48 using: (1) grader
49 49 (2) grader environment [mode]
50 50 (3) grader stop [all|pids-list]
51 51 (4) grader --help
52 52 (1) call grader with environment = 'exam', mode = 'queue'
53 53 (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
54 54 (3) create stop-file to stop running grader in queue mode
55 55 (4) You are here.
56 56 USAGE
57 57 end
58 58
59 59 def process_options_and_stop_file
60 60 # The list of options are:
61 61 # - stop [all|process ids]
62 62 # -
63 63
64 64 # Process 'help' option
65 65 if (ARGV.length==1) and (/help/.match(ARGV[0]))
66 66 display_manual
67 67 exit(0)
68 68 end
69 69
70 70 # Process 'stop' option.
71 71 if (ARGV.length >= 1) and (ARGV[0]=='stop')
72 72 if ARGV.length==1
73 73 puts "you should specify pid-list or 'all'"
74 74 display_manual
75 75 elsif (ARGV.length==2) and (ARGV[1]=='all')
76 76 stop_grader(:all)
77 77 puts "A global stop file ('stop.all') created."
78 78 puts "You should remove it manually later."
79 79 else
80 80 (1..ARGV.length-1).each do |i|
81 81 stop_grader(ARGV[i])
82 82 end
83 83 puts "stop file(s) created"
84 84 end
85 85 exit(0)
86 86 end
87 87
88 88 # Check stop file.
89 89 if check_stopfile
90 90 puts "Stop file exists. Terminated."
91 91 clear_stopfile
92 92 exit(0)
93 93 end
94 94
95 95 #default options
96 96 options = {
97 97 :mode => 'queue',
98 98 :environment => 'exam',
99 99 :dry_run => false,
100 100 }
101 101
102 102 # Process mode and environment option
103 103 if ARGV.length >= 1
104 104 options[:environment] = ARGV.shift
105 105 if ARGV.length >=1
106 106 options[:mode] = ARGV.shift
107 107 end
108 108 end
109 109
110 110 options[:dry_run] = (ARGV.delete('--dry') != nil)
111 111 if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
112 112 puts "Dry run currently works only for 'prob' or 'contest' modes."
113 113 exit(0)
114 114 end
115 115
116 116 options[:report] = (ARGV.delete('--report') != nil)
117 117 if options[:report] and (not ['prob','contest','autonew'].include? options[:mode])
118 118 puts "Report currently works only for 'prob' or 'contest' modes."
119 119 exit(0)
120 120 end
121 121
122 122 return options
123 123 end
124 124
125 125 class ResultCollector
126 126 def initialize
127 127 @results = {}
128 128 @problems = {}
129 129 @users = {}
130 130 end
131 131
132 - def after_save_hook(user, problem, grading_result)
132 + def after_save_hook(submission, grading_result)
133 133 end
134 134
135 - def save(user, problem, grading_result)
135 + def save(submission, grading_result)
136 + user = submission.user
137 + problem = submission.problem
136 138 if not @problems.has_key? problem.id
137 139 @problems[problem.id] = problem
138 140 end
139 141 if not @users.has_key? user.id
140 142 @users[user.id] = user
141 143 end
142 144 @results[[user.id, problem.id]] = grading_result
143 145
144 - after_save_hook(user, problem, grading_result)
146 + after_save_hook(submission, grading_result)
145 147 end
146 148
147 149 def print_report_by_user
148 150 puts "---------------------"
149 151 puts " REPORT"
150 152 puts "---------------------"
151 153
152 154 print "login,email"
153 155 @problems.each_value do |problem|
154 156 print ",#{problem.name}"
155 157 end
156 158 print "\n"
157 159
158 160 @users.each_value do |user|
159 161 print "#{user.login},#{user.email}"
160 162 @problems.each_value do |problem|
161 163 if @results.has_key? [user.id, problem.id]
162 164 print ",#{@results[[user.id,problem.id]][:points]}"
163 165 else
164 166 print ","
165 167 end
166 168 end
167 169 print "\n"
168 170 end
169 171 end
170 172 end
171 173
172 174 def grader_general_loop(engine, grader_proc, options)
173 175 runner = Grader::Runner.new(engine, grader_proc)
174 176 while true
175 177
176 178 if check_stopfile # created by calling grader stop
177 179 clear_stopfile
178 180 log "stopped (with stop file)"
179 181 break
180 182 end
181 183
182 184 task = yield(runner)
183 185
184 186 if task==nil
185 187 sleep(1)
186 188 end
187 189 end
188 190 end
189 191
190 192 def grader_queue_loop(grader_proc, options)
191 193 log "Grader: queue"
192 194 engine = Grader::Engine.new
193 195 grader_general_loop(engine, grader_proc, options) do |runner|
194 196 runner.grade_oldest_task
195 197 end
196 198 end
197 199
198 200 def grader_test_request_loop(grader_proc, options)
199 201 log "Grader: test_request"
200 202 engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
201 203 :reporter => Grader::TestRequestReporter.new)
202 204 grader_general_loop(engine, grader_proc, options) do |runner|
203 205 runner.grade_oldest_test_request
204 206 end
205 207 end
206 208
207 209 def grader_autonew_loop(grader_proc, options)
208 210 log "Grader: autonew"
209 211
210 212 if options[:report]
211 213 result_collector = ResultCollector.new
212 214 else
213 215 result_collector = nil
214 216 end
215 217
216 218 if options[:dry_run]
217 219 puts "Running in dry mode"
218 220 end
219 221
220 222 prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
221 223 :result_collector => result_collector)
222 224
223 225 engine = Grader::Engine.new(:reporter => prob_reporter)
224 226 runner = Grader::Runner.new(engine, grader_proc)
225 227
226 228 grader_proc.report_active if grader_proc!=nil
227 229
228 230 latest_submitted_at = nil
229 231 graded_submission_ids = {}
230 232
231 233 while true
232 234
233 235 if check_stopfile # created by calling grader stop
234 236 clear_stopfile
235 237 log "stopped (with stop file)"
236 238 break
237 239 end
238 240
239 241 if latest_submitted_at==nil
240 242 submissions = Submission.all
241 243 else
242 244 submissions = Submission.all(:conditions => ["submitted_at >= :latest",
243 245 {:latest => latest_submitted_at}])
244 246 end
245 247
246 248 graded_any = false
247 249
248 250 if submissions.length != 0
249 251 submissions.each do |submission|
250 252 if ! graded_submission_ids[submission.id]
251 253 runner.grade_submission(submission)
252 254 graded_submission_ids[submission.id] = true
253 255 if (!latest_submitted_at or
254 256 latest_submitted_at < submission.submitted_at)
255 257 latest_submitted_at = submission.submitted_at
256 258 end
257 259 puts "graded: #{submission.id}"
258 260 puts "latest: #{latest_submitted_at}"
259 261 graded_any = true
260 262 end
261 263 end
262 264 end
263 265
264 266 if ! graded_any
265 267 sleep(1)
266 268 end
267 269 end
268 270 end
269 271
270 272 def grader_grade_problems(grader_proc, options)
271 273 if options[:report]
272 274 result_collector = ResultCollector.new
273 275 else
274 276 result_collector = nil
275 277 end
276 278
277 279 if options[:dry_run]
278 280 puts "Running in dry mode"
279 281 end
280 282
281 283 prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
282 284 :result_collector => result_collector)
283 285 engine = Grader::Engine.new(:reporter => prob_reporter)
284 286 runner = Grader::Runner.new(engine, grader_proc)
285 287
286 288 grader_proc.report_active if grader_proc!=nil
287 289
288 290 ARGV.each do |prob_name|
289 291 prob = Problem.find_by_name(prob_name)
290 292 if prob==nil
291 293 puts "cannot find problem: #{prob_name}"
292 294 else
293 295 runner.grade_problem(prob)
294 296 end
295 297 end
296 298
297 299 if options[:report]
298 300 result_collector.print_report_by_user
299 301 end
300 302 end
301 303
302 304 def grader_grade_contests(grader_proc, options)
303 305 # always use dry run when grading during contest
304 306 dry_run = options[:dry_run] = true
305 307
306 308 contest_name = ARGV.shift
307 309
308 310 contest = Contest.find_by_name(contest_name)
309 311 if contest==nil
310 312 puts "cannot find contest: #{contest_name}"
311 313 exit(0)
312 314 end
313 315
314 316 if options[:report]
315 317 result_collector = ResultCollector.new
316 318 else
317 319 result_collector = nil
318 320 end
319 321
320 322 if options[:dry_run]
321 323 puts "Running in dry mode"
322 324 end
323 325
324 326 prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
325 327 :result_collector => result_collector)
326 328 engine = Grader::Engine.new(:reporter => prob_reporter)
327 329 runner = Grader::Runner.new(engine, grader_proc)
328 330
329 331 grader_proc.report_active if grader_proc!=nil
330 332
331 333 contest.problems.each do |problem|
332 334 puts "Grading: #{problem.name}"
333 335 runner.grade_problem(problem,
334 336 :user_conditions => lambda do |u|
335 337 u.contest_finished? and
336 338 u.contest_ids.include?(contest.id)
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,135 +1,134
1 1 module Grader
2 2
3 3 class SubmissionRoomMaker
4 4 def initialize
5 5 @config = Grader::Configuration.get_instance
6 6 end
7 7
8 8 def produce_grading_room(submission)
9 9 user = submission.user
10 10 problem = submission.problem
11 11 grading_room = "#{@config.user_result_dir}/" +
12 12 "#{user.login}/#{problem.name}/#{submission.id}"
13 13
14 14 FileUtils.mkdir_p(grading_room)
15 15 grading_room
16 16 end
17 17
18 18 def find_problem_home(submission)
19 19 problem = submission.problem
20 20 "#{@config.problems_dir}/#{problem.name}"
21 21 end
22 22
23 23 def save_source(submission,source_name)
24 24 dir = self.produce_grading_room(submission)
25 25 f = File.open("#{dir}/#{source_name}","w")
26 26 f.write(submission.source)
27 27 f.close
28 28 end
29 29
30 30 def clean_up(submission)
31 31 end
32 32 end
33 33
34 34 class SubmissionReporter
35 35 def initialize(options={})
36 36 options = {:dry_run => false, :result_collector => nil}.merge(options)
37 37 @config = Grader::Configuration.get_instance
38 38 @dry_run = options[:dry_run]
39 39 @result_collector = options[:result_collector]
40 40 end
41 41
42 42 def report(sub,test_result_dir)
43 43 result = read_result(test_result_dir)
44 44 if @result_collector
45 - @result_collector.save(sub.user,
46 - sub.problem,
45 + @result_collector.save(sub,
47 46 result)
48 47 end
49 48 save_result(sub,result)
50 49 end
51 50
52 51 def report_error(sub,msg)
53 52 save_result(sub,{:points => 0,
54 53 :comment => "Grading error: #{msg}" })
55 54 end
56 55
57 56 protected
58 57 def read_result(test_result_dir)
59 58 cmp_msg_fname = "#{test_result_dir}/compiler_message"
60 59 if FileTest.exist?(cmp_msg_fname)
61 60 cmp_file = File.open(cmp_msg_fname)
62 61 cmp_msg = cmp_file.read
63 62 cmp_file.close
64 63 else
65 64 cmp_msg = ""
66 65 end
67 66
68 67 result_fname = "#{test_result_dir}/result"
69 68 comment_fname = "#{test_result_dir}/comment"
70 69 if FileTest.exist?(result_fname)
71 70 comment = ""
72 71 begin
73 72 result_file = File.open(result_fname)
74 73 result = result_file.readline.to_i
75 74 result_file.close
76 75 rescue
77 76 result = 0
78 77 comment = "error reading result file."
79 78 end
80 79
81 80 begin
82 81 comment_file = File.open(comment_fname)
83 82 comment += comment_file.readline.chomp
84 83 comment_file.close
85 84 rescue
86 85 comment += ""
87 86 end
88 87
89 88 return {:points => result,
90 89 :comment => comment,
91 90 :cmp_msg => cmp_msg}
92 91 else
93 92 if FileTest.exist?("#{test_result_dir}/a.out")
94 93 return {:points => 0,
95 94 :comment => 'error during grading',
96 95 :cmp_msg => cmp_msg}
97 96 else
98 97 return {:points => 0,
99 98 :comment => 'compilation error',
100 99 :cmp_msg => cmp_msg}
101 100 end
102 101 end
103 102 end
104 103
105 104 def save_result(submission,result)
106 105 problem = submission.problem
107 106 submission.graded_at = Time.now.gmtime
108 107 points = result[:points]
109 108 submission.points = points
110 109 comment = @config.report_comment(result[:comment])
111 110
112 111 #
113 112 # TODO: FIX THIS MESSAGE
114 113 #
115 114 if problem == nil
116 115 submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
117 116 elsif points == problem.full_score
118 117 #submission.grader_comment = 'PASSED: ' + comment
119 118 submission.grader_comment = comment
120 119 elsif result[:comment].chomp =~ /^[\[\]P]+$/
121 120 submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
122 121 else
123 122 #submission.grader_comment = 'FAILED: ' + comment
124 123 submission.grader_comment = comment
125 124 end
126 125 submission.compiler_message = result[:cmp_msg] or ''
127 126
128 127 if not @dry_run
129 128 submission.save
130 129 end
131 130 end
132 131
133 132 end
134 133
135 134 end
You need to be logged in to leave comments. Login now