cafe-grader-judge-scripts

Commit r124:20e4101950cb public

parent | child
Description:
Merge branch 'master' of gitorious.org:cafe-grader/cafe-grader-judge-scripts into windows
Commit status:
[Not Reviewed]
References:
merge default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r124:20e4101950cb - - 8 files changed: 264 inserted, 123 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -0,0 +1,3
1 + sandbox
2 + coverage
3 +
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -0,0 +1,13
1 + /*
2 + LANG: C++
3 + */
4 + #include <cstdio>
5 +
6 + int main()
7 + {
8 + int a,b,r;
9 + r = scanf("%d %d",&a,&b);
10 + printf("%d\n",a+b);
11 + return 0;
12 + }
13 +
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -0,0 +1,14
1 + require 'rake'
2 + require 'spec/rake/spectask'
3 +
4 + desc "Run all examples"
5 + Spec::Rake::SpecTask.new('spec') do |t|
6 + t.spec_files = FileList['*spec.rb']
7 + end
8 +
9 + desc "Run all examples with RCov"
10 + Spec::Rake::SpecTask.new('spec_with_rcov') do |t|
11 + t.spec_files = FileList['*spec.rb']
12 + t.rcov = true
13 + t.rcov_opts = ['--exclude', '.*_spec\.rb']
14 + end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,344 +1,440
1 1 #!/usr/bin/ruby
2 2
3 3 def stop_grader(id)
4 4 if id==:all
5 5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
6 6 else
7 7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
8 8 end
9 9 end
10 10
11 11 def check_stopfile
12 12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
13 13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
14 14 end
15 15
16 16 def clear_stopfile
17 17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 18 File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}")
19 19 end
20 20 end
21 21
22 22 def config
23 23 Grader::Configuration.get_instance
24 24 end
25 25
26 26 def log_file_name
27 27 if !File.exists?(config.log_dir)
28 28 raise "Log directory does not exist: #{config.log_dir}"
29 29 end
30 30 config.log_dir +
31 31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
32 32 end
33 33
34 34 def log(str)
35 35 if config.talkative
36 36 puts str
37 37 end
38 38 if config.logging
39 39 fp = File.open(log_file_name,"a")
40 40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
41 41 fp.close
42 42 end
43 43 end
44 44
45 45 def display_manual
46 46 puts <<USAGE
47 47 Grader.
48 48 using: (1) grader
49 49 (2) grader environment [mode]
50 50 (3) grader stop [all|pids-list]
51 51 (4) grader --help
52 52 (1) call grader with environment = 'exam', mode = 'queue'
53 - (2) possible modes are: 'queue', 'prob', 'test_request'
53 + (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew'
54 54 (3) create stop-file to stop running grader in queue mode
55 55 (4) You are here.
56 56 USAGE
57 57 end
58 58
59 59 def process_options_and_stop_file
60 60 # The list of options are:
61 61 # - stop [all|process ids]
62 62 # -
63 63
64 64 # Process 'help' option
65 65 if (ARGV.length==1) and (/help/.match(ARGV[0]))
66 66 display_manual
67 67 exit(0)
68 68 end
69 69
70 70 # Process 'stop' option.
71 71 if (ARGV.length >= 1) and (ARGV[0]=='stop')
72 72 if ARGV.length==1
73 73 puts "you should specify pid-list or 'all'"
74 74 display_manual
75 75 elsif (ARGV.length==2) and (ARGV[1]=='all')
76 76 stop_grader(:all)
77 77 puts "A global stop file ('stop.all') created."
78 78 puts "You should remove it manually later."
79 79 else
80 80 (1..ARGV.length-1).each do |i|
81 81 stop_grader(ARGV[i])
82 82 end
83 83 puts "stop file(s) created"
84 84 end
85 85 exit(0)
86 86 end
87 87
88 88 # Check stop file.
89 89 if check_stopfile
90 90 puts "Stop file exists. Terminated."
91 91 clear_stopfile
92 92 exit(0)
93 93 end
94 94
95 95 #default options
96 96 options = {
97 97 :mode => 'queue',
98 98 :environment => 'exam',
99 99 :dry_run => false,
100 100 }
101 101
102 102 # Process mode and environment option
103 103 if ARGV.length >= 1
104 104 options[:environment] = ARGV.shift
105 105 if ARGV.length >=1
106 106 options[:mode] = ARGV.shift
107 107 end
108 108 end
109 109
110 110 options[:dry_run] = (ARGV.delete('--dry') != nil)
111 - if options[:dry_run] and (not ['prob','contest'].include? options[:mode])
111 + if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode])
112 112 puts "Dry run currently works only for 'prob' or 'contest' modes."
113 113 exit(0)
114 114 end
115 115
116 116 options[:report] = (ARGV.delete('--report') != nil)
117 - if options[:report] and (not ['prob','contest'].include? options[:mode])
117 + if options[:report] and (not ['prob','contest','autonew'].include? options[:mode])
118 118 puts "Report currently works only for 'prob' or 'contest' modes."
119 119 exit(0)
120 120 end
121 121
122 122 return options
123 123 end
124 124
125 125 class ResultCollector
126 126 def initialize
127 127 @results = {}
128 128 @problems = {}
129 129 @users = {}
130 130 end
131 131
132 - def save(user, problem, grading_result)
132 + def after_save_hook(submission, grading_result)
133 + end
134 +
135 + def save(submission, grading_result)
136 + user = submission.user
137 + problem = submission.problem
133 138 if not @problems.has_key? problem.id
134 139 @problems[problem.id] = problem
135 140 end
136 141 if not @users.has_key? user.id
137 142 @users[user.id] = user
138 143 end
139 144 @results[[user.id, problem.id]] = grading_result
145 +
146 + after_save_hook(submission, grading_result)
140 147 end
141 148
142 149 def print_report_by_user
143 150 puts "---------------------"
144 151 puts " REPORT"
145 152 puts "---------------------"
146 153
147 154 print "login,email"
148 155 @problems.each_value do |problem|
149 156 print ",#{problem.name}"
150 157 end
151 158 print "\n"
152 159
153 160 @users.each_value do |user|
154 161 print "#{user.login},#{user.email}"
155 162 @problems.each_value do |problem|
156 163 if @results.has_key? [user.id, problem.id]
157 164 print ",#{@results[[user.id,problem.id]][:points]}"
158 165 else
159 166 print ","
160 167 end
161 168 end
162 169 print "\n"
163 170 end
164 171 end
165 172 end
166 173
174 + def grader_general_loop(engine, grader_proc, options)
175 + runner = Grader::Runner.new(engine, grader_proc)
176 + while true
177 +
178 + if check_stopfile # created by calling grader stop
179 + clear_stopfile
180 + log "stopped (with stop file)"
181 + break
182 + end
183 +
184 + task = yield(runner)
185 +
186 + if task==nil
187 + sleep(1)
188 + end
189 + end
190 + end
191 +
192 + def grader_queue_loop(grader_proc, options)
193 + log "Grader: queue"
194 + engine = Grader::Engine.new
195 + grader_general_loop(engine, grader_proc, options) do |runner|
196 + runner.grade_oldest_task
197 + end
198 + end
199 +
200 + def grader_test_request_loop(grader_proc, options)
201 + log "Grader: test_request"
202 + engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
203 + :reporter => Grader::TestRequestReporter.new)
204 + grader_general_loop(engine, grader_proc, options) do |runner|
205 + runner.grade_oldest_test_request
206 + end
207 + end
208 +
209 + def grader_autonew_loop(grader_proc, options)
210 + log "Grader: autonew"
211 +
212 + if options[:report]
213 + result_collector = ResultCollector.new
214 + else
215 + result_collector = nil
216 + end
217 +
218 + if options[:dry_run]
219 + puts "Running in dry mode"
220 + end
221 +
222 + prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
223 + :result_collector => result_collector)
224 +
225 + engine = Grader::Engine.new(:reporter => prob_reporter)
226 + runner = Grader::Runner.new(engine, grader_proc)
227 +
228 + grader_proc.report_active if grader_proc!=nil
229 +
230 + latest_submitted_at = nil
231 + graded_submission_ids = {}
232 +
233 + while true
234 +
235 + if check_stopfile # created by calling grader stop
236 + clear_stopfile
237 + log "stopped (with stop file)"
238 + break
239 + end
240 +
241 + if latest_submitted_at==nil
242 + submissions = Submission.all
243 + else
244 + submissions = Submission.all(:conditions => ["submitted_at >= :latest",
245 + {:latest => latest_submitted_at}])
246 + end
247 +
248 + graded_any = false
249 +
250 + if submissions.length != 0
251 + submissions.each do |submission|
252 + if ! graded_submission_ids[submission.id]
253 + runner.grade_submission(submission)
254 + graded_submission_ids[submission.id] = true
255 + if (!latest_submitted_at or
256 + latest_submitted_at < submission.submitted_at)
257 + latest_submitted_at = submission.submitted_at
258 + end
259 + puts "graded: #{submission.id}"
260 + puts "latest: #{latest_submitted_at}"
261 + graded_any = true
262 + end
263 + end
264 + end
265 +
266 + if ! graded_any
267 + sleep(1)
268 + end
269 + end
270 + end
271 +
272 + def grader_grade_problems(grader_proc, options)
273 + if options[:report]
274 + result_collector = ResultCollector.new
275 + else
276 + result_collector = nil
277 + end
278 +
279 + if options[:dry_run]
280 + puts "Running in dry mode"
281 + end
282 +
283 + prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run],
284 + :result_collector => result_collector)
285 + engine = Grader::Engine.new(:reporter => prob_reporter)
286 + runner = Grader::Runner.new(engine, grader_proc)
287 +
288 + grader_proc.report_active if grader_proc!=nil
289 +
290 + ARGV.each do |prob_name|
291 + prob = Problem.find_by_name(prob_name)
292 + if prob==nil
293 + puts "cannot find problem: #{prob_name}"
294 + else
295 + runner.grade_problem(prob)
296 + end
297 + end
298 +
299 + if options[:report]
300 + result_collector.print_report_by_user
301 + end
302 + end
303 +
304 + def grader_grade_contests(grader_proc, options)
305 + # always use dry run when grading during contest
306 + dry_run = options[:dry_run] = true
307 +
308 + contest_name = ARGV.shift
309 +
310 + contest = Contest.find_by_name(contest_name)
311 + if contest==nil
312 + puts "cannot find contest: #{contest_name}"
313 + exit(0)
314 + end
315 +
316 + if options[:report]
317 + result_collector = ResultCollector.new
318 + else
319 + result_collector = nil
320 + end
321 +
322 + if options[:dry_run]
323 + puts "Running in dry mode"
324 + end
325 +
326 + prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
327 + :result_collector => result_collector)
328 + engine = Grader::Engine.new(:reporter => prob_reporter)
329 + runner = Grader::Runner.new(engine, grader_proc)
330 +
331 + grader_proc.report_active if grader_proc!=nil
332 +
333 + contest.problems.each do |problem|
334 + puts "Grading: #{problem.name}"
335 + runner.grade_problem(problem,
336 + :user_conditions => lambda do |u|
337 + u.contest_finished? and
338 + u.contest_ids.include?(contest.id)
339 + end)
340 + end
341 +
342 + if options[:report]
343 + result_collector.print_report_by_user
344 + end
345 + end
346 +
347 + def grader_grade_submissions(grader_proc, options)
348 + engine = Grader::Engine.new
349 + runner = Grader::Runner.new(engine, grader_proc)
350 +
351 + grader_proc.report_active if grader_proc!=nil
352 +
353 + ARGV.each do |sub_id|
354 + puts "Grading #{sub_id}"
355 + begin
356 + submission = Submission.find(sub_id.to_i)
357 + rescue ActiveRecord::RecordNotFound
358 + puts "Record not found"
359 + submission = nil
360 + end
361 +
362 + if submission!=nil
363 + runner.grade_submission(submission)
364 + end
365 + end
366 + end
367 +
167 368 #########################################
168 369 # main program
169 370 #########################################
170 371
171 372 options = process_options_and_stop_file
172 373 GRADER_ENV = options[:environment]
173 374 grader_mode = options[:mode]
174 375 dry_run = options[:dry_run]
175 376
176 377 puts "environment: #{GRADER_ENV}"
177 378 require File.join(File.dirname(__FILE__),'config/environment')
178 379
179 380 # add grader_mode to config
180 381 # this is needed because method log needs it. TODO: clean this up
181 382 class << config
182 383 attr_accessor :grader_mode
183 384 end
184 385 config.grader_mode = grader_mode
185 386
186 387 # reading rails environment
187 388 log 'Reading rails environment'
188 389
189 390 RAILS_ENV = config.rails_env
190 391 require RAILS_ROOT + '/config/environment'
191 392
192 393 # register grader process
193 394 if config.report_grader
194 395 grader_proc = GraderProcess.register(config.grader_hostname,
195 396 Process.pid,
196 397 grader_mode)
197 398 else
198 399 grader_proc = nil
199 400 end
200 401
201 402 #set loggin environment
202 403 ENV['GRADER_LOGGING'] = log_file_name
203 404
204 405 # register exit handler to report inactive, and terminated
205 406 at_exit do
206 407 if grader_proc!=nil
207 408 grader_proc.report_inactive
208 409 grader_proc.terminate
209 410 end
210 411 end
211 412
212 413 #
213 414 # MAIN LOOP
214 415 #
215 416
216 417 case grader_mode
217 - when "queue", "test_request"
218 - log "Grader: #{grader_mode}"
219 - if grader_mode=="queue"
220 - engine = Grader::Engine.new
221 - else
222 - engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
223 - :reporter => Grader::TestRequestReporter.new)
224 - end
225 -
226 - runner = Grader::Runner.new(engine, grader_proc)
227 - while true
418 + when "queue"
419 + grader_queue_loop(grader_proc, options)
228 420
229 - if check_stopfile # created by calling grader stop
230 - clear_stopfile
231 - log "stopped (with stop file)"
232 - break
233 - end
234 -
235 - if grader_mode=="queue"
236 - task = runner.grade_oldest_task
237 - else
238 - task = runner.grade_oldest_test_request
239 - end
240 - if task==nil
241 - sleep(1)
242 - end
243 - end
421 + when "test_request"
422 + grader_test_request_loop(grader_proc, options)
244 423
245 424 when "prob"
246 - if options[:report]
247 - result_collector = ResultCollector.new
248 - else
249 - result_collector = nil
250 - end
251 -
252 - if options[:dry_run]
253 - puts "Running in dry mode"
254 - end
255 -
256 - prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
257 - :result_collector => result_collector)
258 - engine = Grader::Engine.new(:reporter => prob_reporter)
259 - runner = Grader::Runner.new(engine, grader_proc)
260 -
261 - grader_proc.report_active if grader_proc!=nil
262 -
263 - ARGV.each do |prob_name|
264 - prob = Problem.find_by_name(prob_name)
265 - if prob==nil
266 - puts "cannot find problem: #{prob_name}"
267 - else
268 - runner.grade_problem(prob)
269 - end
270 - end
271 -
272 - if options[:report]
273 - result_collector.print_report_by_user
274 - end
425 + grader_grade_problems(grader_proc, options)
275 426
276 427 when "contest"
277 - # always use dry run when grading during contest
278 - contest_name = ARGV.shift
279 -
280 - dry_run = options[:dry_run] = true
281 -
282 - contest = Contest.find_by_name(contest_name)
283 - if contest==nil
284 - puts "cannot find contest: #{contest_name}"
285 - exit(0)
286 - end
287 -
288 - if options[:report]
289 - result_collector = ResultCollector.new
290 - else
291 - result_collector = nil
292 - end
293 -
294 - if options[:dry_run]
295 - puts "Running in dry mode"
296 - end
297 -
298 - prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run,
299 - :result_collector => result_collector)
300 - engine = Grader::Engine.new(:reporter => prob_reporter)
301 - runner = Grader::Runner.new(engine, grader_proc)
302 -
303 - grader_proc.report_active if grader_proc!=nil
304 -
305 - contest.problems.each do |problem|
306 - puts "Grading: #{problem.name}"
307 - runner.grade_problem(problem,
308 - :user_conditions => lambda do |u|
309 - u.contest_finished? and
310 - u.contest_ids.include?(contest.id)
311 - end)
312 - end
313 -
314 - if options[:report]
315 - result_collector.print_report_by_user
316 - end
428 + grader_grade_contests(grader_proc, options)
317 429
318 430 when "sub"
319 - engine = Grader::Engine.new
320 - runner = Grader::Runner.new(engine, grader_proc)
321 -
322 - grader_proc.report_active if grader_proc!=nil
431 + grader_grade_submissions(grader_proc, options)
323 432
324 - ARGV.each do |sub_id|
325 - puts "Grading #{sub_id}"
326 - begin
327 - submission = Submission.find(sub_id.to_i)
328 - rescue ActiveRecord::RecordNotFound
329 - puts "Record not found"
330 - submission = nil
331 - end
332 -
333 - if submission!=nil
334 - runner.grade_submission(submission)
335 - end
336 - end
337 -
338 -
433 + when "autonew"
434 + grader_autonew_loop(grader_proc, options)
339 435
340 436 else
341 437 display_manual
342 438 exit(0)
343 439 end
344 440
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,151 +1,151
1 1 require 'fileutils'
2 2 require File.join(File.dirname(__FILE__),'dir_init')
3 3
4 4 module Grader
5 5
6 6 #
7 7 # A grader engine grades a submission, against anything: a test
8 8 # data, or a user submitted test data. It uses two helpers objects:
9 9 # room_maker and reporter.
10 10 #
11 11 class Engine
12 12
13 13 attr_writer :room_maker
14 14 attr_writer :reporter
15 15
16 16 def initialize(options={})
17 17 # default options
18 18 if not options.include? :room_maker
19 19 options[:room_maker] = Grader::SubmissionRoomMaker.new
20 20 end
21 21 if not options.include? :reporter
22 22 options[:reporter] = Grader::SubmissionReporter.new
23 23 end
24 24
25 25 @config = Grader::Configuration.get_instance
26 26
27 27 @room_maker = options[:room_maker]
28 28 @reporter = options[:reporter]
29 29 end
30 30
31 31 # takes a submission, asks room_maker to produce grading directories,
32 32 # calls grader scripts, and asks reporter to save the result
33 33 def grade(submission)
34 34 current_dir = FileUtils.pwd
35 35
36 36 user = submission.user
37 37 problem = submission.problem
38 38
39 39 # TODO: will have to create real exception for this
40 40 if user==nil or problem == nil
41 41 @reporter.report_error(submission,"Grading error: problem with submission")
42 42 #raise "engine: user or problem is nil"
43 43 end
44 44
45 45 # TODO: this is another hack so that output only task can be judged
46 46 if submission.language!=nil
47 47 language = submission.language.name
48 48 lang_ext = submission.language.ext
49 49 else
50 50 language = 'c'
51 51 lang_ext = 'c'
52 52 end
53 53
54 - # FIX THIS
55 - talk 'some hack on language'
54 + # This is needed because older version of std-scripts/compile
55 + # only look for c++.
56 56 if language == 'cpp'
57 57 language = 'c++'
58 58 end
59 59
60 60 # COMMENT: should it be only source.ext?
61 61 if problem!=nil
62 62 source_name = "#{problem.name}.#{lang_ext}"
63 63 else
64 64 source_name = "source.#{lang_ext}"
65 65 end
66 66
67 67 begin
68 68 grading_dir = @room_maker.produce_grading_room(submission)
69 69 @room_maker.save_source(submission,source_name)
70 70 problem_home = @room_maker.find_problem_home(submission)
71 71
72 72 # puts "GRADING DIR: #{grading_dir}"
73 73 # puts "PROBLEM DIR: #{problem_home}"
74 74
75 75 if !FileTest.exist?(problem_home)
76 76 raise "No test data."
77 77 end
78 78
79 79 dinit = DirInit::Manager.new(problem_home)
80 80
81 81 dinit.setup do
82 82 copy_log = copy_script(problem_home)
83 83 save_copy_log(problem_home,copy_log)
84 84 end
85 85
86 86 call_judge(problem_home,language,grading_dir,source_name)
87 87
88 88 @reporter.report(submission,"#{grading_dir}/test-result")
89 89
90 90 dinit.teardown do
91 91 copy_log = load_copy_log(problem_home)
92 92 clear_copy_log(problem_home)
93 93 clear_script(copy_log,problem_home)
94 94 end
95 95
96 96 rescue RuntimeError => msg
97 97 @reporter.report_error(submission, msg)
98 98
99 99 ensure
100 100 @room_maker.clean_up(submission)
101 101 Dir.chdir(current_dir) # this is really important
102 102 end
103 103 end
104 104
105 105 protected
106 106
107 107 def talk(str)
108 108 if @config.talkative
109 109 puts str
110 110 end
111 111 end
112 112
113 113 def call_judge(problem_home,language,grading_dir,fname)
114 114 ENV['PROBLEM_HOME'] = problem_home
115 115
116 116 talk grading_dir
117 117 Dir.chdir grading_dir
118 118 cmd = "#{problem_home}/script/judge #{language} #{fname}"
119 119 talk "CMD: #{cmd}"
120 120 system("ruby " + cmd)
121 121 end
122 122
123 123 def get_std_script_dir
124 124 GRADER_ROOT + '/std-script'
125 125 end
126 126
127 127 def copy_script(problem_home)
128 128 script_dir = "#{problem_home}/script"
129 129 std_script_dir = get_std_script_dir
130 130
131 131 raise "std-script directory not found" if !FileTest.exist?(std_script_dir)
132 132
133 133 scripts = Dir[std_script_dir + '/*']
134 134
135 135 copied = []
136 136
137 137 scripts.each do |s|
138 138 fname = File.basename(s)
139 139 next if FileTest.directory?(s)
140 140 if !FileTest.exist?("#{script_dir}/#{fname}")
141 141 copied << fname
142 142 FileUtils.cp(s, "#{script_dir}")
143 143 end
144 144 end
145 145
146 146 return copied
147 147 end
148 148
149 149 def copy_log_filename(problem_home)
150 150 return File.join(problem_home, '.scripts_copied')
151 151 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,135 +1,134
1 1 module Grader
2 2
3 3 class SubmissionRoomMaker
4 4 def initialize
5 5 @config = Grader::Configuration.get_instance
6 6 end
7 7
8 8 def produce_grading_room(submission)
9 9 user = submission.user
10 10 problem = submission.problem
11 11 grading_room = "#{@config.user_result_dir}/" +
12 12 "#{user.login}/#{problem.name}/#{submission.id}"
13 13
14 14 FileUtils.mkdir_p(grading_room)
15 15 grading_room
16 16 end
17 17
18 18 def find_problem_home(submission)
19 19 problem = submission.problem
20 20 "#{@config.problems_dir}/#{problem.name}"
21 21 end
22 22
23 23 def save_source(submission,source_name)
24 24 dir = self.produce_grading_room(submission)
25 25 f = File.open("#{dir}/#{source_name}","w")
26 26 f.write(submission.source)
27 27 f.close
28 28 end
29 29
30 30 def clean_up(submission)
31 31 end
32 32 end
33 33
34 34 class SubmissionReporter
35 35 def initialize(options={})
36 36 options = {:dry_run => false, :result_collector => nil}.merge(options)
37 37 @config = Grader::Configuration.get_instance
38 38 @dry_run = options[:dry_run]
39 39 @result_collector = options[:result_collector]
40 40 end
41 41
42 42 def report(sub,test_result_dir)
43 43 result = read_result(test_result_dir)
44 44 if @result_collector
45 - @result_collector.save(sub.user,
46 - sub.problem,
45 + @result_collector.save(sub,
47 46 result)
48 47 end
49 48 save_result(sub,result)
50 49 end
51 50
52 51 def report_error(sub,msg)
53 52 save_result(sub,{:points => 0,
54 53 :comment => "Grading error: #{msg}" })
55 54 end
56 55
57 56 protected
58 57 def read_result(test_result_dir)
59 58 cmp_msg_fname = "#{test_result_dir}/compiler_message"
60 59 if FileTest.exist?(cmp_msg_fname)
61 60 cmp_file = File.open(cmp_msg_fname)
62 61 cmp_msg = cmp_file.read
63 62 cmp_file.close
64 63 else
65 64 cmp_msg = ""
66 65 end
67 66
68 67 result_fname = "#{test_result_dir}/result"
69 68 comment_fname = "#{test_result_dir}/comment"
70 69 if FileTest.exist?(result_fname)
71 70 comment = ""
72 71 begin
73 72 result_file = File.open(result_fname)
74 73 result = result_file.readline.to_i
75 74 result_file.close
76 75 rescue
77 76 result = 0
78 77 comment = "error reading result file."
79 78 end
80 79
81 80 begin
82 81 comment_file = File.open(comment_fname)
83 82 comment += comment_file.readline.chomp
84 83 comment_file.close
85 84 rescue
86 85 comment += ""
87 86 end
88 87
89 88 return {:points => result,
90 89 :comment => comment,
91 90 :cmp_msg => cmp_msg}
92 91 else
93 92 if FileTest.exist?("#{test_result_dir}/a.out")
94 93 return {:points => 0,
95 94 :comment => 'error during grading',
96 95 :cmp_msg => cmp_msg}
97 96 else
98 97 return {:points => 0,
99 98 :comment => 'compilation error',
100 99 :cmp_msg => cmp_msg}
101 100 end
102 101 end
103 102 end
104 103
105 104 def save_result(submission,result)
106 105 problem = submission.problem
107 106 submission.graded_at = Time.now.gmtime
108 107 points = result[:points]
109 108 submission.points = points
110 109 comment = @config.report_comment(result[:comment])
111 110
112 111 #
113 112 # TODO: FIX THIS MESSAGE
114 113 #
115 114 if problem == nil
116 115 submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
117 116 elsif points == problem.full_score
118 117 #submission.grader_comment = 'PASSED: ' + comment
119 118 submission.grader_comment = comment
120 119 elsif result[:comment].chomp =~ /^[\[\]P]+$/
121 120 submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
122 121 else
123 122 #submission.grader_comment = 'FAILED: ' + comment
124 123 submission.grader_comment = comment
125 124 end
126 125 submission.compiler_message = result[:cmp_msg] or ''
127 126
128 127 if not @dry_run
129 128 submission.save
130 129 end
131 130 end
132 131
133 132 end
134 133
135 134 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,109 +1,113
1 1 #!/usr/bin/ruby
2 2
3 3 require 'fileutils'
4 4
5 5 ##############################
6 6 #
7 7 # Standard Compile Script
8 8 #
9 9 # Supported compilers:
10 10 # gcc, g++, and fpc.
11 11 #
12 12 ##############################
13 13
14 14 def talk(msg)
15 15 if ENV['TALKATIVE']!=nil
16 16 puts str
17 17 end
18 18 if ENV['GRADER_LOGGING']!=nil
19 19 log_fname = ENV['GRADER_LOGGING']
20 20 fp = File.open(log_fname,"a")
21 21 fp.puts("run: #{Time.new.strftime("%H:%M")} #{msg}")
22 22 fp.close
23 23 end
24 24 end
25 25
26 26 C_COMPILER = "gcc"
27 27 CPLUSPLUS_COMPILER = "g++"
28 28 PASCAL_COMPILER = "fpc"
29 29
30 30 C_OPTIONS = "-O2 -s -static -std=c99 -DCONTEST -lm -Wall"
31 31 CPLUSPLUS_OPTIONS = "-O2 -s -static -DCONTEST -lm -Wall"
32 32 PASCAL_OPTIONS = "-O1 -XS -dCONTEST"
33 33
34 34 # Check for the correct number of arguments. Otherwise, print usage.
35 35 if ARGV.length == 0 or ARGV.length > 4
36 36 puts "Usage: compile <language> [<source-file>] [<output-file>] [<message-file>]"
37 37 puts
38 38 puts "<source-file> is defaulted to \"source\"."
39 39 puts "<output-file> is defaulted to \"a.out\"."
40 40 puts "<message-file> is defaulted to \"compiler_message\"."
41 41 puts
42 42 exit(127)
43 43 end
44 44
45 45 PARAMS = {
46 46 :source_file => [1,'source'],
47 47 :output_file => [2,'a.out'],
48 48 :message_file => [3,'compiler_message']
49 49 }
50 50
51 51 params = {}
52 52 params[:prog_lang] = ARGV[0]
53 53 PARAMS.each_key do |param_name|
54 54 index, default = PARAMS[param_name]
55 55 if ARGV.length > index
56 56 params[param_name] = ARGV[index]
57 57 else
58 58 params[param_name] = default
59 59 end
60 60 talk "#{param_name}: #{params[param_name]}"
61 61 end
62 62
63 63 # Remove any remaining output files or message files.
64 64 if FileTest.exists? params[:output_file]
65 65 FileUtils.rm(params[:output_file])
66 66 end
67 67 if FileTest.exists? params[:message_file]
68 68 FileUtils.rm(params[:message_file])
69 69 end
70 70
71 71 # Check if the source file exists before attempt compiling.
72 72 if !FileTest.exists? params[:source_file]
73 73 talk("ERROR: The source file does not exist!")
74 74 open(params[:message_file],"w") do |f|
75 75 f.puts "ERROR: The source file did not exist."
76 76 end
77 77 exit(127)
78 78 end
79 79
80 + if params[:prog_lang]=='cpp':
81 + params[:prog_lang] = 'c++'
82 + end
83 +
80 84 # Compile.
81 85 case params[:prog_lang]
82 86
83 87 when "c"
84 88 command = "#{C_COMPILER} #{params[:source_file]} -o #{params[:output_file]} #{C_OPTIONS} 2> #{params[:message_file]}"
85 89 system(command)
86 90
87 91 when "c++"
88 92 command = "#{CPLUSPLUS_COMPILER} #{params[:source_file]} -o #{params[:output_file]} #{CPLUSPLUS_OPTIONS} 2> #{params[:message_file]}"
89 93 system(command)
90 94
91 95 when "pas"
92 96 command = "#{PASCAL_COMPILER} #{params[:source_file]} -ooutpas #{PASCAL_OPTIONS} > #{params[:message_file]}"
93 97 system(command)
94 98 FileUtils.mv("output", params[:output_file])
95 99
96 100 else
97 101 talk("ERROR: Invalid language specified!")
98 102 open(params[:message_file],"w") do |f|
99 103 f.puts "ERROR: Invalid language specified!"
100 104 end
101 105 exit(127)
102 106 end
103 107
104 108 # Report success or failure.
105 109 if FileTest.exists? params[:output_file]
106 110 talk "Compilation was successful!"
107 111 else
108 112 talk "ERROR: Something was wrong during the compilation!"
109 113 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,222 +1,234
1 1 require File.join(File.dirname(__FILE__),'spec_helper')
2 2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3 3
4 4 describe "A grader engine, when grading submissions" do
5 5
6 6 include GraderEngineHelperMethods
7 7
8 8 before(:each) do
9 9 @config = Grader::Configuration.get_instance
10 10
11 11 # this test is from Pong
12 12 @problem_test_normal = stub(Problem,
13 13 :id => 1, :name => 'test_normal',
14 14 :full_score => 135)
15 15 @user_user1 = stub(User,
16 16 :id => 1, :login => 'user1')
17 17
18 18 @engine = Grader::Engine.new
19 19 init_sandbox
20 20 end
21 21
22 22 it "should grade normal submission" do
23 23 grader_should(:grade => "test1_correct.c",
24 24 :on => @problem_test_normal,
25 25 :and_report => {
26 26 :score => 135,
27 27 :comment => /^(\[|P|\])+/})
28 28 end
29 29
30 30
31 + it "should grade normal submission in C++" do
32 + cpp_lang = stub(Language, :name => 'cpp', :ext => 'cpp')
33 +
34 + grader_should(:grade => "test1_correct_cc.cc",
35 + :language => cpp_lang,
36 + :on => @problem_test_normal,
37 + :and_report => {
38 + :score => 135,
39 + :comment => /^(\[|P|\])+/})
40 + end
41 +
31 42 it "should produce error message when submission cannot compile" do
32 43 grader_should(:grade => "test1_compile_error.c",
33 44 :on => @problem_test_normal,
34 45 :and_report => {
35 46 :score => 0,
36 47 :comment => 'compilation error',
37 48 :compiler_message => /[Ee]rror/})
38 49 end
39 50
40 51 it "should produce timeout error when submission runs forever" do
41 52 @problem_test_timeout = stub(Problem,
42 53 :id => 1, :name => 'test_timeout',
43 54 :full_score => 10)
44 55 grader_should(:grade => "test2_timeout.c",
45 56 :on => @problem_test_timeout,
46 57 :and_report => {
47 58 :score => 0,
48 59 :comment => 'TT'})
49 60 end
50 61
51 62 it "should produce timeout error correctly with fractional running time and fractional time limits" do
52 63 @problem_test_timeout = stub(Problem,
53 64 :id => 1, :name => 'test_timeout',
54 65 :full_score => 20)
55 66 grader_should(:grade => "test2_05sec.c",
56 67 :on => @problem_test_timeout,
57 68 :and_report => {
58 69 :score => 10,
59 70 :comment => 'TP'})
60 71 end
61 72
62 73 it "should produce runtime error when submission uses too much static memory" do
63 74 @problem_test_memory = stub(Problem,
64 75 :id => 1, :name => 'test_memory',
65 76 :full_score => 20)
66 77 grader_should(:grade => "add_too_much_memory_static.c",
67 78 :on => @problem_test_memory,
68 79 :and_report => {
69 80 :score => 10,
70 81 :comment => /[^P]P/})
71 82 end
72 83
73 84 it "should not allow submission to allocate too much dynamic memory" do
74 85 @problem_test_memory = stub(Problem,
75 86 :id => 1, :name => 'test_memory',
76 87 :full_score => 20)
77 88 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 89 :on => @problem_test_memory,
79 90 :and_report => {
80 91 :score => 10,
81 92 :comment => /[^P]P/})
82 93 end
83 94
84 95 it "should score test runs correctly when submission fails in some test case" do
85 96 grader_should(:grade => "add_fail_test_case_1.c",
86 97 :on => @problem_test_normal,
87 98 :and_report => {
88 99 :score => 105,
89 100 :comment => /^.*(-|x).*$/})
90 101 end
91 102
92 103 it "should fail submission with non-zero exit status" do
93 104 grader_should(:grade => "add_nonzero_exit_status.c",
94 105 :on => @problem_test_normal,
95 106 :and_report => {
96 107 :score => 0,
97 108 :comment => /^(\[|x|\])+$/})
98 109 end
99 110
100 111 it "should not allow malicious submission to see PROBLEM_HOME" do
101 112 problem_test_yesno = stub(Problem,
102 113 :id => 1, :name => 'test_yesno',
103 114 :full_score => 10)
104 115 grader_should(:grade => "yesno_access_problem_home.c",
105 116 :on => problem_test_yesno,
106 117 :and_report => {
107 118 :score => 0,
108 119 :comment => /(-|x)/})
109 120 end
110 121
111 122 it "should not allow malicious submission to open files" do
112 123 problem_test_yesno = stub(Problem,
113 124 :id => 1, :name => 'test_yesno',
114 125 :full_score => 10)
115 126 grader_should(:grade => "yesno_open_file.c",
116 127 :on => problem_test_yesno,
117 128 :and_report => {
118 129 :score => 0,
119 130 :comment => /(-|x)/})
120 131 end
121 132
122 133 def grader_should(args)
123 134 @user1 = stub(User,
124 135 :id => 1, :login => 'user1')
136 +
125 137 submission =
126 - create_submission_from_file(1, @user1, args[:on], args[:grade])
138 + create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language])
127 139 submission.should_receive(:graded_at=)
128 140
129 141 expected_score = args[:and_report][:score]
130 142 expected_comment = args[:and_report][:comment]
131 143 if args[:and_report][:compiler_message]!=nil
132 144 expected_compiler_message = args[:and_report][:compiler_message]
133 145 else
134 146 expected_compiler_message = ''
135 147 end
136 148
137 149 submission.should_receive(:points=).with(expected_score)
138 150 submission.should_receive(:grader_comment=).with(expected_comment)
139 151 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 152 submission.should_receive(:save)
141 153
142 154 @engine.grade(submission)
143 155 end
144 156
145 157 protected
146 158
147 159 def create_normal_submission_mock_from_file(source_fname)
148 160 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 161 end
150 162
151 163 end
152 164
153 165 describe "A grader engine, when grading test requests" do
154 166
155 167 include GraderEngineHelperMethods
156 168
157 169 before(:each) do
158 170 @config = Grader::Configuration.get_instance
159 171 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
160 172 :reporter => Grader::TestRequestReporter.new)
161 173 init_sandbox
162 174 end
163 175
164 176 it "should report error if there is no problem template" do
165 177 problem = stub(Problem,
166 178 :id => 1, :name => 'nothing')
167 179 grader_should(:grade => 'test1_correct.c',
168 180 :on => problem,
169 181 :with => 'in1.txt',
170 182 :and_report => {
171 183 :graded_at= => nil,
172 184 :compiler_message= => '',
173 185 :grader_comment= => '',
174 186 :running_stat= => /template not found/,
175 187 :running_time= => nil,
176 188 :exit_status= => nil,
177 189 :memory_usage= => nil,
178 190 :save => nil})
179 191 end
180 192
181 193 it "should run test request and produce output file" do
182 194 problem = stub(Problem,
183 195 :id => 1, :name => 'test_normal')
184 196 grader_should(:grade => 'test1_correct.c',
185 197 :on => problem,
186 198 :with => 'in1.txt',
187 199 :and_report => {
188 200 :graded_at= => nil,
189 201 :compiler_message= => '',
190 202 :grader_comment= => '',
191 203 :running_stat= => /0.0\d* sec./,
192 204 :output_file_name= => lambda { |fname|
193 205 File.exists?(fname).should be_true
194 206 },
195 207 :running_time= => nil,
196 208 :exit_status= => nil,
197 209 :memory_usage= => nil,
198 210 :save => nil})
199 211 end
200 212
201 213 it "should clean up problem directory after running test request" do
202 214 problem = stub(Problem,
203 215 :id => 1, :name => 'test_normal')
204 216 grader_should(:grade => 'test1_correct.c',
205 217 :on => problem,
206 218 :with => 'in1.txt',
207 219 :and_report => {
208 220 :graded_at= => nil,
209 221 :compiler_message= => '',
210 222 :grader_comment= => '',
211 223 :running_stat= => nil,
212 224 :output_file_name= => nil,
213 225 :running_time= => nil,
214 226 :exit_status= => nil,
215 227 :memory_usage= => nil,
216 228 :save => nil})
217 229 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
218 230 end
219 231
220 232 it "should compile test request with error and report compilation error" do
221 233 problem = stub(Problem,
222 234 :id => 1, :name => 'test_normal')
You need to be logged in to leave comments. Login now