Description:
Merge branch 'master' of gitorious.org:cafe-grader/cafe-grader-judge-scripts into windows
Commit status:
[Not Reviewed]
References:
Diff options:
Comments:
0 Commit comments
0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Author
Jittat Fakcharoenphol
-
r124:20e4101950cb - - 8 files changed: 264 inserted, 123 deleted
| @@ -0,0 +1,13 | |||||
|
|
|
1 | + /* | ||
|
|
|
2 | + LANG: C++ | ||
|
|
|
3 | + */ | ||
|
|
|
4 | + #include <cstdio> | ||
|
|
|
5 | + | ||
|
|
|
6 | + int main() | ||
|
|
|
7 | + { | ||
|
|
|
8 | + int a,b,r; | ||
|
|
|
9 | + r = scanf("%d %d",&a,&b); | ||
|
|
|
10 | + printf("%d\n",a+b); | ||
|
|
|
11 | + return 0; | ||
|
|
|
12 | + } | ||
|
|
|
13 | + | ||
| @@ -0,0 +1,14 | |||||
|
|
|
1 | + require 'rake' | ||
|
|
|
2 | + require 'spec/rake/spectask' | ||
|
|
|
3 | + | ||
|
|
|
4 | + desc "Run all examples" | ||
|
|
|
5 | + Spec::Rake::SpecTask.new('spec') do |t| | ||
|
|
|
6 | + t.spec_files = FileList['*spec.rb'] | ||
|
|
|
7 | + end | ||
|
|
|
8 | + | ||
|
|
|
9 | + desc "Run all examples with RCov" | ||
|
|
|
10 | + Spec::Rake::SpecTask.new('spec_with_rcov') do |t| | ||
|
|
|
11 | + t.spec_files = FileList['*spec.rb'] | ||
|
|
|
12 | + t.rcov = true | ||
|
|
|
13 | + t.rcov_opts = ['--exclude', '.*_spec\.rb'] | ||
|
|
|
14 | + end | ||
| @@ -1,344 +1,440 | |||||
|
|
1 | #!/usr/bin/ruby |
|
1 | #!/usr/bin/ruby |
|
|
2 |
|
2 | ||
|
|
3 | def stop_grader(id) |
|
3 | def stop_grader(id) |
|
|
4 | if id==:all |
|
4 | if id==:all |
|
|
5 | File.open(File.dirname(__FILE__) + "/stop.all",'w').close |
|
5 | File.open(File.dirname(__FILE__) + "/stop.all",'w').close |
|
|
6 | else |
|
6 | else |
|
|
7 | File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close |
|
7 | File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close |
|
|
8 | end |
|
8 | end |
|
|
9 | end |
|
9 | end |
|
|
10 |
|
10 | ||
|
|
11 | def check_stopfile |
|
11 | def check_stopfile |
|
|
12 | FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or |
|
12 | FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or |
|
|
13 | FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}") |
|
13 | FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}") |
|
|
14 | end |
|
14 | end |
|
|
15 |
|
15 | ||
|
|
16 | def clear_stopfile |
|
16 | def clear_stopfile |
|
|
17 | if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}") |
|
17 | if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}") |
|
|
18 | File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}") |
|
18 | File.delete(File.dirname(__FILE__) + "/stop.#{Process.pid}") |
|
|
19 | end |
|
19 | end |
|
|
20 | end |
|
20 | end |
|
|
21 |
|
21 | ||
|
|
22 | def config |
|
22 | def config |
|
|
23 | Grader::Configuration.get_instance |
|
23 | Grader::Configuration.get_instance |
|
|
24 | end |
|
24 | end |
|
|
25 |
|
25 | ||
|
|
26 | def log_file_name |
|
26 | def log_file_name |
|
|
27 | if !File.exists?(config.log_dir) |
|
27 | if !File.exists?(config.log_dir) |
|
|
28 | raise "Log directory does not exist: #{config.log_dir}" |
|
28 | raise "Log directory does not exist: #{config.log_dir}" |
|
|
29 | end |
|
29 | end |
|
|
30 | config.log_dir + |
|
30 | config.log_dir + |
|
|
31 | "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}" |
|
31 | "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}" |
|
|
32 | end |
|
32 | end |
|
|
33 |
|
33 | ||
|
|
34 | def log(str) |
|
34 | def log(str) |
|
|
35 | if config.talkative |
|
35 | if config.talkative |
|
|
36 | puts str |
|
36 | puts str |
|
|
37 | end |
|
37 | end |
|
|
38 | if config.logging |
|
38 | if config.logging |
|
|
39 | fp = File.open(log_file_name,"a") |
|
39 | fp = File.open(log_file_name,"a") |
|
|
40 | fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}") |
|
40 | fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}") |
|
|
41 | fp.close |
|
41 | fp.close |
|
|
42 | end |
|
42 | end |
|
|
43 | end |
|
43 | end |
|
|
44 |
|
44 | ||
|
|
45 | def display_manual |
|
45 | def display_manual |
|
|
46 | puts <<USAGE |
|
46 | puts <<USAGE |
|
|
47 | Grader. |
|
47 | Grader. |
|
|
48 | using: (1) grader |
|
48 | using: (1) grader |
|
|
49 | (2) grader environment [mode] |
|
49 | (2) grader environment [mode] |
|
|
50 | (3) grader stop [all|pids-list] |
|
50 | (3) grader stop [all|pids-list] |
|
|
51 | (4) grader --help |
|
51 | (4) grader --help |
|
|
52 | (1) call grader with environment = 'exam', mode = 'queue' |
|
52 | (1) call grader with environment = 'exam', mode = 'queue' |
|
|
53 |
- (2) possible modes are: 'queue', 'prob', ' |
|
53 | + (2) possible modes are: 'queue', 'test_request', 'prob', 'sub', 'contest', and 'autonew' |
|
|
54 | (3) create stop-file to stop running grader in queue mode |
|
54 | (3) create stop-file to stop running grader in queue mode |
|
|
55 | (4) You are here. |
|
55 | (4) You are here. |
|
|
56 | USAGE |
|
56 | USAGE |
|
|
57 | end |
|
57 | end |
|
|
58 |
|
58 | ||
|
|
59 | def process_options_and_stop_file |
|
59 | def process_options_and_stop_file |
|
|
60 | # The list of options are: |
|
60 | # The list of options are: |
|
|
61 | # - stop [all|process ids] |
|
61 | # - stop [all|process ids] |
|
|
62 | # - |
|
62 | # - |
|
|
63 |
|
63 | ||
|
|
64 | # Process 'help' option |
|
64 | # Process 'help' option |
|
|
65 | if (ARGV.length==1) and (/help/.match(ARGV[0])) |
|
65 | if (ARGV.length==1) and (/help/.match(ARGV[0])) |
|
|
66 | display_manual |
|
66 | display_manual |
|
|
67 | exit(0) |
|
67 | exit(0) |
|
|
68 | end |
|
68 | end |
|
|
69 |
|
69 | ||
|
|
70 | # Process 'stop' option. |
|
70 | # Process 'stop' option. |
|
|
71 | if (ARGV.length >= 1) and (ARGV[0]=='stop') |
|
71 | if (ARGV.length >= 1) and (ARGV[0]=='stop') |
|
|
72 | if ARGV.length==1 |
|
72 | if ARGV.length==1 |
|
|
73 | puts "you should specify pid-list or 'all'" |
|
73 | puts "you should specify pid-list or 'all'" |
|
|
74 | display_manual |
|
74 | display_manual |
|
|
75 | elsif (ARGV.length==2) and (ARGV[1]=='all') |
|
75 | elsif (ARGV.length==2) and (ARGV[1]=='all') |
|
|
76 | stop_grader(:all) |
|
76 | stop_grader(:all) |
|
|
77 | puts "A global stop file ('stop.all') created." |
|
77 | puts "A global stop file ('stop.all') created." |
|
|
78 | puts "You should remove it manually later." |
|
78 | puts "You should remove it manually later." |
|
|
79 | else |
|
79 | else |
|
|
80 | (1..ARGV.length-1).each do |i| |
|
80 | (1..ARGV.length-1).each do |i| |
|
|
81 | stop_grader(ARGV[i]) |
|
81 | stop_grader(ARGV[i]) |
|
|
82 | end |
|
82 | end |
|
|
83 | puts "stop file(s) created" |
|
83 | puts "stop file(s) created" |
|
|
84 | end |
|
84 | end |
|
|
85 | exit(0) |
|
85 | exit(0) |
|
|
86 | end |
|
86 | end |
|
|
87 |
|
87 | ||
|
|
88 | # Check stop file. |
|
88 | # Check stop file. |
|
|
89 | if check_stopfile |
|
89 | if check_stopfile |
|
|
90 | puts "Stop file exists. Terminated." |
|
90 | puts "Stop file exists. Terminated." |
|
|
91 | clear_stopfile |
|
91 | clear_stopfile |
|
|
92 | exit(0) |
|
92 | exit(0) |
|
|
93 | end |
|
93 | end |
|
|
94 |
|
94 | ||
|
|
95 | #default options |
|
95 | #default options |
|
|
96 | options = { |
|
96 | options = { |
|
|
97 | :mode => 'queue', |
|
97 | :mode => 'queue', |
|
|
98 | :environment => 'exam', |
|
98 | :environment => 'exam', |
|
|
99 | :dry_run => false, |
|
99 | :dry_run => false, |
|
|
100 | } |
|
100 | } |
|
|
101 |
|
101 | ||
|
|
102 | # Process mode and environment option |
|
102 | # Process mode and environment option |
|
|
103 | if ARGV.length >= 1 |
|
103 | if ARGV.length >= 1 |
|
|
104 | options[:environment] = ARGV.shift |
|
104 | options[:environment] = ARGV.shift |
|
|
105 | if ARGV.length >=1 |
|
105 | if ARGV.length >=1 |
|
|
106 | options[:mode] = ARGV.shift |
|
106 | options[:mode] = ARGV.shift |
|
|
107 | end |
|
107 | end |
|
|
108 | end |
|
108 | end |
|
|
109 |
|
109 | ||
|
|
110 | options[:dry_run] = (ARGV.delete('--dry') != nil) |
|
110 | options[:dry_run] = (ARGV.delete('--dry') != nil) |
|
|
111 | - if options[:dry_run] and (not ['prob','contest'].include? options[:mode]) |
|
111 | + if options[:dry_run] and (not ['prob','contest','autonew'].include? options[:mode]) |
|
|
112 | puts "Dry run currently works only for 'prob' or 'contest' modes." |
|
112 | puts "Dry run currently works only for 'prob' or 'contest' modes." |
|
|
113 | exit(0) |
|
113 | exit(0) |
|
|
114 | end |
|
114 | end |
|
|
115 |
|
115 | ||
|
|
116 | options[:report] = (ARGV.delete('--report') != nil) |
|
116 | options[:report] = (ARGV.delete('--report') != nil) |
|
|
117 | - if options[:report] and (not ['prob','contest'].include? options[:mode]) |
|
117 | + if options[:report] and (not ['prob','contest','autonew'].include? options[:mode]) |
|
|
118 | puts "Report currently works only for 'prob' or 'contest' modes." |
|
118 | puts "Report currently works only for 'prob' or 'contest' modes." |
|
|
119 | exit(0) |
|
119 | exit(0) |
|
|
120 | end |
|
120 | end |
|
|
121 |
|
121 | ||
|
|
122 | return options |
|
122 | return options |
|
|
123 | end |
|
123 | end |
|
|
124 |
|
124 | ||
|
|
125 | class ResultCollector |
|
125 | class ResultCollector |
|
|
126 | def initialize |
|
126 | def initialize |
|
|
127 | @results = {} |
|
127 | @results = {} |
|
|
128 | @problems = {} |
|
128 | @problems = {} |
|
|
129 | @users = {} |
|
129 | @users = {} |
|
|
130 | end |
|
130 | end |
|
|
131 |
|
131 | ||
|
|
132 |
- def |
|
132 | + def after_save_hook(submission, grading_result) |
|
|
|
133 | + end | ||
|
|
|
134 | + | ||
|
|
|
135 | + def save(submission, grading_result) | ||
|
|
|
136 | + user = submission.user | ||
|
|
|
137 | + problem = submission.problem | ||
|
|
133 | if not @problems.has_key? problem.id |
|
138 | if not @problems.has_key? problem.id |
|
|
134 | @problems[problem.id] = problem |
|
139 | @problems[problem.id] = problem |
|
|
135 | end |
|
140 | end |
|
|
136 | if not @users.has_key? user.id |
|
141 | if not @users.has_key? user.id |
|
|
137 | @users[user.id] = user |
|
142 | @users[user.id] = user |
|
|
138 | end |
|
143 | end |
|
|
139 | @results[[user.id, problem.id]] = grading_result |
|
144 | @results[[user.id, problem.id]] = grading_result |
|
|
|
145 | + | ||
|
|
|
146 | + after_save_hook(submission, grading_result) | ||
|
|
140 | end |
|
147 | end |
|
|
141 |
|
148 | ||
|
|
142 | def print_report_by_user |
|
149 | def print_report_by_user |
|
|
143 | puts "---------------------" |
|
150 | puts "---------------------" |
|
|
144 | puts " REPORT" |
|
151 | puts " REPORT" |
|
|
145 | puts "---------------------" |
|
152 | puts "---------------------" |
|
|
146 |
|
153 | ||
|
|
147 | print "login,email" |
|
154 | print "login,email" |
|
|
148 | @problems.each_value do |problem| |
|
155 | @problems.each_value do |problem| |
|
|
149 | print ",#{problem.name}" |
|
156 | print ",#{problem.name}" |
|
|
150 | end |
|
157 | end |
|
|
151 | print "\n" |
|
158 | print "\n" |
|
|
152 |
|
159 | ||
|
|
153 | @users.each_value do |user| |
|
160 | @users.each_value do |user| |
|
|
154 | print "#{user.login},#{user.email}" |
|
161 | print "#{user.login},#{user.email}" |
|
|
155 | @problems.each_value do |problem| |
|
162 | @problems.each_value do |problem| |
|
|
156 | if @results.has_key? [user.id, problem.id] |
|
163 | if @results.has_key? [user.id, problem.id] |
|
|
157 | print ",#{@results[[user.id,problem.id]][:points]}" |
|
164 | print ",#{@results[[user.id,problem.id]][:points]}" |
|
|
158 | else |
|
165 | else |
|
|
159 | print "," |
|
166 | print "," |
|
|
160 | end |
|
167 | end |
|
|
161 | end |
|
168 | end |
|
|
162 | print "\n" |
|
169 | print "\n" |
|
|
163 | end |
|
170 | end |
|
|
164 | end |
|
171 | end |
|
|
165 | end |
|
172 | end |
|
|
166 |
|
173 | ||
|
|
|
174 | + def grader_general_loop(engine, grader_proc, options) | ||
|
|
|
175 | + runner = Grader::Runner.new(engine, grader_proc) | ||
|
|
|
176 | + while true | ||
|
|
|
177 | + | ||
|
|
|
178 | + if check_stopfile # created by calling grader stop | ||
|
|
|
179 | + clear_stopfile | ||
|
|
|
180 | + log "stopped (with stop file)" | ||
|
|
|
181 | + break | ||
|
|
|
182 | + end | ||
|
|
|
183 | + | ||
|
|
|
184 | + task = yield(runner) | ||
|
|
|
185 | + | ||
|
|
|
186 | + if task==nil | ||
|
|
|
187 | + sleep(1) | ||
|
|
|
188 | + end | ||
|
|
|
189 | + end | ||
|
|
|
190 | + end | ||
|
|
|
191 | + | ||
|
|
|
192 | + def grader_queue_loop(grader_proc, options) | ||
|
|
|
193 | + log "Grader: queue" | ||
|
|
|
194 | + engine = Grader::Engine.new | ||
|
|
|
195 | + grader_general_loop(engine, grader_proc, options) do |runner| | ||
|
|
|
196 | + runner.grade_oldest_task | ||
|
|
|
197 | + end | ||
|
|
|
198 | + end | ||
|
|
|
199 | + | ||
|
|
|
200 | + def grader_test_request_loop(grader_proc, options) | ||
|
|
|
201 | + log "Grader: test_request" | ||
|
|
|
202 | + engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new, | ||
|
|
|
203 | + :reporter => Grader::TestRequestReporter.new) | ||
|
|
|
204 | + grader_general_loop(engine, grader_proc, options) do |runner| | ||
|
|
|
205 | + runner.grade_oldest_test_request | ||
|
|
|
206 | + end | ||
|
|
|
207 | + end | ||
|
|
|
208 | + | ||
|
|
|
209 | + def grader_autonew_loop(grader_proc, options) | ||
|
|
|
210 | + log "Grader: autonew" | ||
|
|
|
211 | + | ||
|
|
|
212 | + if options[:report] | ||
|
|
|
213 | + result_collector = ResultCollector.new | ||
|
|
|
214 | + else | ||
|
|
|
215 | + result_collector = nil | ||
|
|
|
216 | + end | ||
|
|
|
217 | + | ||
|
|
|
218 | + if options[:dry_run] | ||
|
|
|
219 | + puts "Running in dry mode" | ||
|
|
|
220 | + end | ||
|
|
|
221 | + | ||
|
|
|
222 | + prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run], | ||
|
|
|
223 | + :result_collector => result_collector) | ||
|
|
|
224 | + | ||
|
|
|
225 | + engine = Grader::Engine.new(:reporter => prob_reporter) | ||
|
|
|
226 | + runner = Grader::Runner.new(engine, grader_proc) | ||
|
|
|
227 | + | ||
|
|
|
228 | + grader_proc.report_active if grader_proc!=nil | ||
|
|
|
229 | + | ||
|
|
|
230 | + latest_submitted_at = nil | ||
|
|
|
231 | + graded_submission_ids = {} | ||
|
|
|
232 | + | ||
|
|
|
233 | + while true | ||
|
|
|
234 | + | ||
|
|
|
235 | + if check_stopfile # created by calling grader stop | ||
|
|
|
236 | + clear_stopfile | ||
|
|
|
237 | + log "stopped (with stop file)" | ||
|
|
|
238 | + break | ||
|
|
|
239 | + end | ||
|
|
|
240 | + | ||
|
|
|
241 | + if latest_submitted_at==nil | ||
|
|
|
242 | + submissions = Submission.all | ||
|
|
|
243 | + else | ||
|
|
|
244 | + submissions = Submission.all(:conditions => ["submitted_at >= :latest", | ||
|
|
|
245 | + {:latest => latest_submitted_at}]) | ||
|
|
|
246 | + end | ||
|
|
|
247 | + | ||
|
|
|
248 | + graded_any = false | ||
|
|
|
249 | + | ||
|
|
|
250 | + if submissions.length != 0 | ||
|
|
|
251 | + submissions.each do |submission| | ||
|
|
|
252 | + if ! graded_submission_ids[submission.id] | ||
|
|
|
253 | + runner.grade_submission(submission) | ||
|
|
|
254 | + graded_submission_ids[submission.id] = true | ||
|
|
|
255 | + if (!latest_submitted_at or | ||
|
|
|
256 | + latest_submitted_at < submission.submitted_at) | ||
|
|
|
257 | + latest_submitted_at = submission.submitted_at | ||
|
|
|
258 | + end | ||
|
|
|
259 | + puts "graded: #{submission.id}" | ||
|
|
|
260 | + puts "latest: #{latest_submitted_at}" | ||
|
|
|
261 | + graded_any = true | ||
|
|
|
262 | + end | ||
|
|
|
263 | + end | ||
|
|
|
264 | + end | ||
|
|
|
265 | + | ||
|
|
|
266 | + if ! graded_any | ||
|
|
|
267 | + sleep(1) | ||
|
|
|
268 | + end | ||
|
|
|
269 | + end | ||
|
|
|
270 | + end | ||
|
|
|
271 | + | ||
|
|
|
272 | + def grader_grade_problems(grader_proc, options) | ||
|
|
|
273 | + if options[:report] | ||
|
|
|
274 | + result_collector = ResultCollector.new | ||
|
|
|
275 | + else | ||
|
|
|
276 | + result_collector = nil | ||
|
|
|
277 | + end | ||
|
|
|
278 | + | ||
|
|
|
279 | + if options[:dry_run] | ||
|
|
|
280 | + puts "Running in dry mode" | ||
|
|
|
281 | + end | ||
|
|
|
282 | + | ||
|
|
|
283 | + prob_reporter = Grader::SubmissionReporter.new(:dry_run => options[:dry_run], | ||
|
|
|
284 | + :result_collector => result_collector) | ||
|
|
|
285 | + engine = Grader::Engine.new(:reporter => prob_reporter) | ||
|
|
|
286 | + runner = Grader::Runner.new(engine, grader_proc) | ||
|
|
|
287 | + | ||
|
|
|
288 | + grader_proc.report_active if grader_proc!=nil | ||
|
|
|
289 | + | ||
|
|
|
290 | + ARGV.each do |prob_name| | ||
|
|
|
291 | + prob = Problem.find_by_name(prob_name) | ||
|
|
|
292 | + if prob==nil | ||
|
|
|
293 | + puts "cannot find problem: #{prob_name}" | ||
|
|
|
294 | + else | ||
|
|
|
295 | + runner.grade_problem(prob) | ||
|
|
|
296 | + end | ||
|
|
|
297 | + end | ||
|
|
|
298 | + | ||
|
|
|
299 | + if options[:report] | ||
|
|
|
300 | + result_collector.print_report_by_user | ||
|
|
|
301 | + end | ||
|
|
|
302 | + end | ||
|
|
|
303 | + | ||
|
|
|
304 | + def grader_grade_contests(grader_proc, options) | ||
|
|
|
305 | + # always use dry run when grading during contest | ||
|
|
|
306 | + dry_run = options[:dry_run] = true | ||
|
|
|
307 | + | ||
|
|
|
308 | + contest_name = ARGV.shift | ||
|
|
|
309 | + | ||
|
|
|
310 | + contest = Contest.find_by_name(contest_name) | ||
|
|
|
311 | + if contest==nil | ||
|
|
|
312 | + puts "cannot find contest: #{contest_name}" | ||
|
|
|
313 | + exit(0) | ||
|
|
|
314 | + end | ||
|
|
|
315 | + | ||
|
|
|
316 | + if options[:report] | ||
|
|
|
317 | + result_collector = ResultCollector.new | ||
|
|
|
318 | + else | ||
|
|
|
319 | + result_collector = nil | ||
|
|
|
320 | + end | ||
|
|
|
321 | + | ||
|
|
|
322 | + if options[:dry_run] | ||
|
|
|
323 | + puts "Running in dry mode" | ||
|
|
|
324 | + end | ||
|
|
|
325 | + | ||
|
|
|
326 | + prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run, | ||
|
|
|
327 | + :result_collector => result_collector) | ||
|
|
|
328 | + engine = Grader::Engine.new(:reporter => prob_reporter) | ||
|
|
|
329 | + runner = Grader::Runner.new(engine, grader_proc) | ||
|
|
|
330 | + | ||
|
|
|
331 | + grader_proc.report_active if grader_proc!=nil | ||
|
|
|
332 | + | ||
|
|
|
333 | + contest.problems.each do |problem| | ||
|
|
|
334 | + puts "Grading: #{problem.name}" | ||
|
|
|
335 | + runner.grade_problem(problem, | ||
|
|
|
336 | + :user_conditions => lambda do |u| | ||
|
|
|
337 | + u.contest_finished? and | ||
|
|
|
338 | + u.contest_ids.include?(contest.id) | ||
|
|
|
339 | + end) | ||
|
|
|
340 | + end | ||
|
|
|
341 | + | ||
|
|
|
342 | + if options[:report] | ||
|
|
|
343 | + result_collector.print_report_by_user | ||
|
|
|
344 | + end | ||
|
|
|
345 | + end | ||
|
|
|
346 | + | ||
|
|
|
347 | + def grader_grade_submissions(grader_proc, options) | ||
|
|
|
348 | + engine = Grader::Engine.new | ||
|
|
|
349 | + runner = Grader::Runner.new(engine, grader_proc) | ||
|
|
|
350 | + | ||
|
|
|
351 | + grader_proc.report_active if grader_proc!=nil | ||
|
|
|
352 | + | ||
|
|
|
353 | + ARGV.each do |sub_id| | ||
|
|
|
354 | + puts "Grading #{sub_id}" | ||
|
|
|
355 | + begin | ||
|
|
|
356 | + submission = Submission.find(sub_id.to_i) | ||
|
|
|
357 | + rescue ActiveRecord::RecordNotFound | ||
|
|
|
358 | + puts "Record not found" | ||
|
|
|
359 | + submission = nil | ||
|
|
|
360 | + end | ||
|
|
|
361 | + | ||
|
|
|
362 | + if submission!=nil | ||
|
|
|
363 | + runner.grade_submission(submission) | ||
|
|
|
364 | + end | ||
|
|
|
365 | + end | ||
|
|
|
366 | + end | ||
|
|
|
367 | + | ||
|
|
167 | ######################################### |
|
368 | ######################################### |
|
|
168 | # main program |
|
369 | # main program |
|
|
169 | ######################################### |
|
370 | ######################################### |
|
|
170 |
|
371 | ||
|
|
171 | options = process_options_and_stop_file |
|
372 | options = process_options_and_stop_file |
|
|
172 | GRADER_ENV = options[:environment] |
|
373 | GRADER_ENV = options[:environment] |
|
|
173 | grader_mode = options[:mode] |
|
374 | grader_mode = options[:mode] |
|
|
174 | dry_run = options[:dry_run] |
|
375 | dry_run = options[:dry_run] |
|
|
175 |
|
376 | ||
|
|
176 | puts "environment: #{GRADER_ENV}" |
|
377 | puts "environment: #{GRADER_ENV}" |
|
|
177 | require File.join(File.dirname(__FILE__),'config/environment') |
|
378 | require File.join(File.dirname(__FILE__),'config/environment') |
|
|
178 |
|
379 | ||
|
|
179 | # add grader_mode to config |
|
380 | # add grader_mode to config |
|
|
180 | # this is needed because method log needs it. TODO: clean this up |
|
381 | # this is needed because method log needs it. TODO: clean this up |
|
|
181 | class << config |
|
382 | class << config |
|
|
182 | attr_accessor :grader_mode |
|
383 | attr_accessor :grader_mode |
|
|
183 | end |
|
384 | end |
|
|
184 | config.grader_mode = grader_mode |
|
385 | config.grader_mode = grader_mode |
|
|
185 |
|
386 | ||
|
|
186 | # reading rails environment |
|
387 | # reading rails environment |
|
|
187 | log 'Reading rails environment' |
|
388 | log 'Reading rails environment' |
|
|
188 |
|
389 | ||
|
|
189 | RAILS_ENV = config.rails_env |
|
390 | RAILS_ENV = config.rails_env |
|
|
190 | require RAILS_ROOT + '/config/environment' |
|
391 | require RAILS_ROOT + '/config/environment' |
|
|
191 |
|
392 | ||
|
|
192 | # register grader process |
|
393 | # register grader process |
|
|
193 | if config.report_grader |
|
394 | if config.report_grader |
|
|
194 | grader_proc = GraderProcess.register(config.grader_hostname, |
|
395 | grader_proc = GraderProcess.register(config.grader_hostname, |
|
|
195 | Process.pid, |
|
396 | Process.pid, |
|
|
196 | grader_mode) |
|
397 | grader_mode) |
|
|
197 | else |
|
398 | else |
|
|
198 | grader_proc = nil |
|
399 | grader_proc = nil |
|
|
199 | end |
|
400 | end |
|
|
200 |
|
401 | ||
|
|
201 | #set loggin environment |
|
402 | #set loggin environment |
|
|
202 | ENV['GRADER_LOGGING'] = log_file_name |
|
403 | ENV['GRADER_LOGGING'] = log_file_name |
|
|
203 |
|
404 | ||
|
|
204 | # register exit handler to report inactive, and terminated |
|
405 | # register exit handler to report inactive, and terminated |
|
|
205 | at_exit do |
|
406 | at_exit do |
|
|
206 | if grader_proc!=nil |
|
407 | if grader_proc!=nil |
|
|
207 | grader_proc.report_inactive |
|
408 | grader_proc.report_inactive |
|
|
208 | grader_proc.terminate |
|
409 | grader_proc.terminate |
|
|
209 | end |
|
410 | end |
|
|
210 | end |
|
411 | end |
|
|
211 |
|
412 | ||
|
|
212 | # |
|
413 | # |
|
|
213 | # MAIN LOOP |
|
414 | # MAIN LOOP |
|
|
214 | # |
|
415 | # |
|
|
215 |
|
416 | ||
|
|
216 | case grader_mode |
|
417 | case grader_mode |
|
|
217 |
- when "queue" |
|
418 | + when "queue" |
|
|
218 | - log "Grader: #{grader_mode}" |
|
419 | + grader_queue_loop(grader_proc, options) |
|
|
219 | - if grader_mode=="queue" |
|
||
|
|
220 | - engine = Grader::Engine.new |
|
||
|
|
221 | - else |
|
||
|
|
222 | - engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new, |
|
||
|
|
223 | - :reporter => Grader::TestRequestReporter.new) |
|
||
|
|
224 | - end |
|
||
|
|
225 | - |
|
||
|
|
226 | - runner = Grader::Runner.new(engine, grader_proc) |
|
||
|
|
227 | - while true |
|
||
|
|
228 |
|
420 | ||
|
|
229 | - if check_stopfile # created by calling grader stop |
|
421 | + when "test_request" |
|
|
230 | - clear_stopfile |
|
422 | + grader_test_request_loop(grader_proc, options) |
|
|
231 | - log "stopped (with stop file)" |
|
||
|
|
232 | - break |
|
||
|
|
233 | - end |
|
||
|
|
234 | - |
|
||
|
|
235 | - if grader_mode=="queue" |
|
||
|
|
236 | - task = runner.grade_oldest_task |
|
||
|
|
237 | - else |
|
||
|
|
238 | - task = runner.grade_oldest_test_request |
|
||
|
|
239 | - end |
|
||
|
|
240 | - if task==nil |
|
||
|
|
241 | - sleep(1) |
|
||
|
|
242 | - end |
|
||
|
|
243 | - end |
|
||
|
|
244 |
|
423 | ||
|
|
245 | when "prob" |
|
424 | when "prob" |
|
|
246 | - if options[:report] |
|
425 | + grader_grade_problems(grader_proc, options) |
|
|
247 | - result_collector = ResultCollector.new |
|
||
|
|
248 | - else |
|
||
|
|
249 | - result_collector = nil |
|
||
|
|
250 | - end |
|
||
|
|
251 | - |
|
||
|
|
252 | - if options[:dry_run] |
|
||
|
|
253 | - puts "Running in dry mode" |
|
||
|
|
254 | - end |
|
||
|
|
255 | - |
|
||
|
|
256 | - prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run, |
|
||
|
|
257 | - :result_collector => result_collector) |
|
||
|
|
258 | - engine = Grader::Engine.new(:reporter => prob_reporter) |
|
||
|
|
259 | - runner = Grader::Runner.new(engine, grader_proc) |
|
||
|
|
260 | - |
|
||
|
|
261 | - grader_proc.report_active if grader_proc!=nil |
|
||
|
|
262 | - |
|
||
|
|
263 | - ARGV.each do |prob_name| |
|
||
|
|
264 | - prob = Problem.find_by_name(prob_name) |
|
||
|
|
265 | - if prob==nil |
|
||
|
|
266 | - puts "cannot find problem: #{prob_name}" |
|
||
|
|
267 | - else |
|
||
|
|
268 | - runner.grade_problem(prob) |
|
||
|
|
269 | - end |
|
||
|
|
270 | - end |
|
||
|
|
271 | - |
|
||
|
|
272 | - if options[:report] |
|
||
|
|
273 | - result_collector.print_report_by_user |
|
||
|
|
274 | - end |
|
||
|
|
275 |
|
426 | ||
|
|
276 | when "contest" |
|
427 | when "contest" |
|
|
277 | - # always use dry run when grading during contest |
|
428 | + grader_grade_contests(grader_proc, options) |
|
|
278 | - contest_name = ARGV.shift |
|
||
|
|
279 | - |
|
||
|
|
280 | - dry_run = options[:dry_run] = true |
|
||
|
|
281 | - |
|
||
|
|
282 | - contest = Contest.find_by_name(contest_name) |
|
||
|
|
283 | - if contest==nil |
|
||
|
|
284 | - puts "cannot find contest: #{contest_name}" |
|
||
|
|
285 | - exit(0) |
|
||
|
|
286 | - end |
|
||
|
|
287 | - |
|
||
|
|
288 | - if options[:report] |
|
||
|
|
289 | - result_collector = ResultCollector.new |
|
||
|
|
290 | - else |
|
||
|
|
291 | - result_collector = nil |
|
||
|
|
292 | - end |
|
||
|
|
293 | - |
|
||
|
|
294 | - if options[:dry_run] |
|
||
|
|
295 | - puts "Running in dry mode" |
|
||
|
|
296 | - end |
|
||
|
|
297 | - |
|
||
|
|
298 | - prob_reporter = Grader::SubmissionReporter.new(:dry_run => dry_run, |
|
||
|
|
299 | - :result_collector => result_collector) |
|
||
|
|
300 | - engine = Grader::Engine.new(:reporter => prob_reporter) |
|
||
|
|
301 | - runner = Grader::Runner.new(engine, grader_proc) |
|
||
|
|
302 | - |
|
||
|
|
303 | - grader_proc.report_active if grader_proc!=nil |
|
||
|
|
304 | - |
|
||
|
|
305 | - contest.problems.each do |problem| |
|
||
|
|
306 | - puts "Grading: #{problem.name}" |
|
||
|
|
307 | - runner.grade_problem(problem, |
|
||
|
|
308 | - :user_conditions => lambda do |u| |
|
||
|
|
309 | - u.contest_finished? and |
|
||
|
|
310 | - u.contest_ids.include?(contest.id) |
|
||
|
|
311 | - end) |
|
||
|
|
312 | - end |
|
||
|
|
313 | - |
|
||
|
|
314 | - if options[:report] |
|
||
|
|
315 | - result_collector.print_report_by_user |
|
||
|
|
316 | - end |
|
||
|
|
317 |
|
429 | ||
|
|
318 | when "sub" |
|
430 | when "sub" |
|
|
319 | - engine = Grader::Engine.new |
|
431 | + grader_grade_submissions(grader_proc, options) |
|
|
320 | - runner = Grader::Runner.new(engine, grader_proc) |
|
||
|
|
321 | - |
|
||
|
|
322 | - grader_proc.report_active if grader_proc!=nil |
|
||
|
|
323 |
|
432 | ||
|
|
324 | - ARGV.each do |sub_id| |
|
433 | + when "autonew" |
|
|
325 | - puts "Grading #{sub_id}" |
|
434 | + grader_autonew_loop(grader_proc, options) |
|
|
326 | - begin |
|
||
|
|
327 | - submission = Submission.find(sub_id.to_i) |
|
||
|
|
328 | - rescue ActiveRecord::RecordNotFound |
|
||
|
|
329 | - puts "Record not found" |
|
||
|
|
330 | - submission = nil |
|
||
|
|
331 | - end |
|
||
|
|
332 | - |
|
||
|
|
333 | - if submission!=nil |
|
||
|
|
334 | - runner.grade_submission(submission) |
|
||
|
|
335 | - end |
|
||
|
|
336 | - end |
|
||
|
|
337 | - |
|
||
|
|
338 | - |
|
||
|
|
339 |
|
435 | ||
|
|
340 | else |
|
436 | else |
|
|
341 | display_manual |
|
437 | display_manual |
|
|
342 | exit(0) |
|
438 | exit(0) |
|
|
343 | end |
|
439 | end |
|
|
344 |
|
440 | ||
| @@ -1,151 +1,151 | |||||
|
|
1 | require 'fileutils' |
|
1 | require 'fileutils' |
|
|
2 | require File.join(File.dirname(__FILE__),'dir_init') |
|
2 | require File.join(File.dirname(__FILE__),'dir_init') |
|
|
3 |
|
3 | ||
|
|
4 | module Grader |
|
4 | module Grader |
|
|
5 |
|
5 | ||
|
|
6 | # |
|
6 | # |
|
|
7 | # A grader engine grades a submission, against anything: a test |
|
7 | # A grader engine grades a submission, against anything: a test |
|
|
8 | # data, or a user submitted test data. It uses two helpers objects: |
|
8 | # data, or a user submitted test data. It uses two helpers objects: |
|
|
9 | # room_maker and reporter. |
|
9 | # room_maker and reporter. |
|
|
10 | # |
|
10 | # |
|
|
11 | class Engine |
|
11 | class Engine |
|
|
12 |
|
12 | ||
|
|
13 | attr_writer :room_maker |
|
13 | attr_writer :room_maker |
|
|
14 | attr_writer :reporter |
|
14 | attr_writer :reporter |
|
|
15 |
|
15 | ||
|
|
16 | def initialize(options={}) |
|
16 | def initialize(options={}) |
|
|
17 | # default options |
|
17 | # default options |
|
|
18 | if not options.include? :room_maker |
|
18 | if not options.include? :room_maker |
|
|
19 | options[:room_maker] = Grader::SubmissionRoomMaker.new |
|
19 | options[:room_maker] = Grader::SubmissionRoomMaker.new |
|
|
20 | end |
|
20 | end |
|
|
21 | if not options.include? :reporter |
|
21 | if not options.include? :reporter |
|
|
22 | options[:reporter] = Grader::SubmissionReporter.new |
|
22 | options[:reporter] = Grader::SubmissionReporter.new |
|
|
23 | end |
|
23 | end |
|
|
24 |
|
24 | ||
|
|
25 | @config = Grader::Configuration.get_instance |
|
25 | @config = Grader::Configuration.get_instance |
|
|
26 |
|
26 | ||
|
|
27 | @room_maker = options[:room_maker] |
|
27 | @room_maker = options[:room_maker] |
|
|
28 | @reporter = options[:reporter] |
|
28 | @reporter = options[:reporter] |
|
|
29 | end |
|
29 | end |
|
|
30 |
|
30 | ||
|
|
31 | # takes a submission, asks room_maker to produce grading directories, |
|
31 | # takes a submission, asks room_maker to produce grading directories, |
|
|
32 | # calls grader scripts, and asks reporter to save the result |
|
32 | # calls grader scripts, and asks reporter to save the result |
|
|
33 | def grade(submission) |
|
33 | def grade(submission) |
|
|
34 | current_dir = FileUtils.pwd |
|
34 | current_dir = FileUtils.pwd |
|
|
35 |
|
35 | ||
|
|
36 | user = submission.user |
|
36 | user = submission.user |
|
|
37 | problem = submission.problem |
|
37 | problem = submission.problem |
|
|
38 |
|
38 | ||
|
|
39 | # TODO: will have to create real exception for this |
|
39 | # TODO: will have to create real exception for this |
|
|
40 | if user==nil or problem == nil |
|
40 | if user==nil or problem == nil |
|
|
41 | @reporter.report_error(submission,"Grading error: problem with submission") |
|
41 | @reporter.report_error(submission,"Grading error: problem with submission") |
|
|
42 | #raise "engine: user or problem is nil" |
|
42 | #raise "engine: user or problem is nil" |
|
|
43 | end |
|
43 | end |
|
|
44 |
|
44 | ||
|
|
45 | # TODO: this is another hack so that output only task can be judged |
|
45 | # TODO: this is another hack so that output only task can be judged |
|
|
46 | if submission.language!=nil |
|
46 | if submission.language!=nil |
|
|
47 | language = submission.language.name |
|
47 | language = submission.language.name |
|
|
48 | lang_ext = submission.language.ext |
|
48 | lang_ext = submission.language.ext |
|
|
49 | else |
|
49 | else |
|
|
50 | language = 'c' |
|
50 | language = 'c' |
|
|
51 | lang_ext = 'c' |
|
51 | lang_ext = 'c' |
|
|
52 | end |
|
52 | end |
|
|
53 |
|
53 | ||
|
|
54 | - # FIX THIS |
|
54 | + # This is needed because older version of std-scripts/compile |
|
|
55 | - talk 'some hack on language' |
|
55 | + # only look for c++. |
|
|
56 | if language == 'cpp' |
|
56 | if language == 'cpp' |
|
|
57 | language = 'c++' |
|
57 | language = 'c++' |
|
|
58 | end |
|
58 | end |
|
|
59 |
|
59 | ||
|
|
60 | # COMMENT: should it be only source.ext? |
|
60 | # COMMENT: should it be only source.ext? |
|
|
61 | if problem!=nil |
|
61 | if problem!=nil |
|
|
62 | source_name = "#{problem.name}.#{lang_ext}" |
|
62 | source_name = "#{problem.name}.#{lang_ext}" |
|
|
63 | else |
|
63 | else |
|
|
64 | source_name = "source.#{lang_ext}" |
|
64 | source_name = "source.#{lang_ext}" |
|
|
65 | end |
|
65 | end |
|
|
66 |
|
66 | ||
|
|
67 | begin |
|
67 | begin |
|
|
68 | grading_dir = @room_maker.produce_grading_room(submission) |
|
68 | grading_dir = @room_maker.produce_grading_room(submission) |
|
|
69 | @room_maker.save_source(submission,source_name) |
|
69 | @room_maker.save_source(submission,source_name) |
|
|
70 | problem_home = @room_maker.find_problem_home(submission) |
|
70 | problem_home = @room_maker.find_problem_home(submission) |
|
|
71 |
|
71 | ||
|
|
72 | # puts "GRADING DIR: #{grading_dir}" |
|
72 | # puts "GRADING DIR: #{grading_dir}" |
|
|
73 | # puts "PROBLEM DIR: #{problem_home}" |
|
73 | # puts "PROBLEM DIR: #{problem_home}" |
|
|
74 |
|
74 | ||
|
|
75 | if !FileTest.exist?(problem_home) |
|
75 | if !FileTest.exist?(problem_home) |
|
|
76 | raise "No test data." |
|
76 | raise "No test data." |
|
|
77 | end |
|
77 | end |
|
|
78 |
|
78 | ||
|
|
79 | dinit = DirInit::Manager.new(problem_home) |
|
79 | dinit = DirInit::Manager.new(problem_home) |
|
|
80 |
|
80 | ||
|
|
81 | dinit.setup do |
|
81 | dinit.setup do |
|
|
82 | copy_log = copy_script(problem_home) |
|
82 | copy_log = copy_script(problem_home) |
|
|
83 | save_copy_log(problem_home,copy_log) |
|
83 | save_copy_log(problem_home,copy_log) |
|
|
84 | end |
|
84 | end |
|
|
85 |
|
85 | ||
|
|
86 | call_judge(problem_home,language,grading_dir,source_name) |
|
86 | call_judge(problem_home,language,grading_dir,source_name) |
|
|
87 |
|
87 | ||
|
|
88 | @reporter.report(submission,"#{grading_dir}/test-result") |
|
88 | @reporter.report(submission,"#{grading_dir}/test-result") |
|
|
89 |
|
89 | ||
|
|
90 | dinit.teardown do |
|
90 | dinit.teardown do |
|
|
91 | copy_log = load_copy_log(problem_home) |
|
91 | copy_log = load_copy_log(problem_home) |
|
|
92 | clear_copy_log(problem_home) |
|
92 | clear_copy_log(problem_home) |
|
|
93 | clear_script(copy_log,problem_home) |
|
93 | clear_script(copy_log,problem_home) |
|
|
94 | end |
|
94 | end |
|
|
95 |
|
95 | ||
|
|
96 | rescue RuntimeError => msg |
|
96 | rescue RuntimeError => msg |
|
|
97 | @reporter.report_error(submission, msg) |
|
97 | @reporter.report_error(submission, msg) |
|
|
98 |
|
98 | ||
|
|
99 | ensure |
|
99 | ensure |
|
|
100 | @room_maker.clean_up(submission) |
|
100 | @room_maker.clean_up(submission) |
|
|
101 | Dir.chdir(current_dir) # this is really important |
|
101 | Dir.chdir(current_dir) # this is really important |
|
|
102 | end |
|
102 | end |
|
|
103 | end |
|
103 | end |
|
|
104 |
|
104 | ||
|
|
105 | protected |
|
105 | protected |
|
|
106 |
|
106 | ||
|
|
107 | def talk(str) |
|
107 | def talk(str) |
|
|
108 | if @config.talkative |
|
108 | if @config.talkative |
|
|
109 | puts str |
|
109 | puts str |
|
|
110 | end |
|
110 | end |
|
|
111 | end |
|
111 | end |
|
|
112 |
|
112 | ||
|
|
113 | def call_judge(problem_home,language,grading_dir,fname) |
|
113 | def call_judge(problem_home,language,grading_dir,fname) |
|
|
114 | ENV['PROBLEM_HOME'] = problem_home |
|
114 | ENV['PROBLEM_HOME'] = problem_home |
|
|
115 |
|
115 | ||
|
|
116 | talk grading_dir |
|
116 | talk grading_dir |
|
|
117 | Dir.chdir grading_dir |
|
117 | Dir.chdir grading_dir |
|
|
118 | cmd = "#{problem_home}/script/judge #{language} #{fname}" |
|
118 | cmd = "#{problem_home}/script/judge #{language} #{fname}" |
|
|
119 | talk "CMD: #{cmd}" |
|
119 | talk "CMD: #{cmd}" |
|
|
120 | system("ruby " + cmd) |
|
120 | system("ruby " + cmd) |
|
|
121 | end |
|
121 | end |
|
|
122 |
|
122 | ||
|
|
123 | def get_std_script_dir |
|
123 | def get_std_script_dir |
|
|
124 | GRADER_ROOT + '/std-script' |
|
124 | GRADER_ROOT + '/std-script' |
|
|
125 | end |
|
125 | end |
|
|
126 |
|
126 | ||
|
|
127 | def copy_script(problem_home) |
|
127 | def copy_script(problem_home) |
|
|
128 | script_dir = "#{problem_home}/script" |
|
128 | script_dir = "#{problem_home}/script" |
|
|
129 | std_script_dir = get_std_script_dir |
|
129 | std_script_dir = get_std_script_dir |
|
|
130 |
|
130 | ||
|
|
131 | raise "std-script directory not found" if !FileTest.exist?(std_script_dir) |
|
131 | raise "std-script directory not found" if !FileTest.exist?(std_script_dir) |
|
|
132 |
|
132 | ||
|
|
133 | scripts = Dir[std_script_dir + '/*'] |
|
133 | scripts = Dir[std_script_dir + '/*'] |
|
|
134 |
|
134 | ||
|
|
135 | copied = [] |
|
135 | copied = [] |
|
|
136 |
|
136 | ||
|
|
137 | scripts.each do |s| |
|
137 | scripts.each do |s| |
|
|
138 | fname = File.basename(s) |
|
138 | fname = File.basename(s) |
|
|
139 | next if FileTest.directory?(s) |
|
139 | next if FileTest.directory?(s) |
|
|
140 | if !FileTest.exist?("#{script_dir}/#{fname}") |
|
140 | if !FileTest.exist?("#{script_dir}/#{fname}") |
|
|
141 | copied << fname |
|
141 | copied << fname |
|
|
142 | FileUtils.cp(s, "#{script_dir}") |
|
142 | FileUtils.cp(s, "#{script_dir}") |
|
|
143 | end |
|
143 | end |
|
|
144 | end |
|
144 | end |
|
|
145 |
|
145 | ||
|
|
146 | return copied |
|
146 | return copied |
|
|
147 | end |
|
147 | end |
|
|
148 |
|
148 | ||
|
|
149 | def copy_log_filename(problem_home) |
|
149 | def copy_log_filename(problem_home) |
|
|
150 | return File.join(problem_home, '.scripts_copied') |
|
150 | return File.join(problem_home, '.scripts_copied') |
|
|
151 | end |
|
151 | end |
| @@ -1,135 +1,134 | |||||
|
|
1 | module Grader |
|
1 | module Grader |
|
|
2 |
|
2 | ||
|
|
3 | class SubmissionRoomMaker |
|
3 | class SubmissionRoomMaker |
|
|
4 | def initialize |
|
4 | def initialize |
|
|
5 | @config = Grader::Configuration.get_instance |
|
5 | @config = Grader::Configuration.get_instance |
|
|
6 | end |
|
6 | end |
|
|
7 |
|
7 | ||
|
|
8 | def produce_grading_room(submission) |
|
8 | def produce_grading_room(submission) |
|
|
9 | user = submission.user |
|
9 | user = submission.user |
|
|
10 | problem = submission.problem |
|
10 | problem = submission.problem |
|
|
11 | grading_room = "#{@config.user_result_dir}/" + |
|
11 | grading_room = "#{@config.user_result_dir}/" + |
|
|
12 | "#{user.login}/#{problem.name}/#{submission.id}" |
|
12 | "#{user.login}/#{problem.name}/#{submission.id}" |
|
|
13 |
|
13 | ||
|
|
14 | FileUtils.mkdir_p(grading_room) |
|
14 | FileUtils.mkdir_p(grading_room) |
|
|
15 | grading_room |
|
15 | grading_room |
|
|
16 | end |
|
16 | end |
|
|
17 |
|
17 | ||
|
|
18 | def find_problem_home(submission) |
|
18 | def find_problem_home(submission) |
|
|
19 | problem = submission.problem |
|
19 | problem = submission.problem |
|
|
20 | "#{@config.problems_dir}/#{problem.name}" |
|
20 | "#{@config.problems_dir}/#{problem.name}" |
|
|
21 | end |
|
21 | end |
|
|
22 |
|
22 | ||
|
|
23 | def save_source(submission,source_name) |
|
23 | def save_source(submission,source_name) |
|
|
24 | dir = self.produce_grading_room(submission) |
|
24 | dir = self.produce_grading_room(submission) |
|
|
25 | f = File.open("#{dir}/#{source_name}","w") |
|
25 | f = File.open("#{dir}/#{source_name}","w") |
|
|
26 | f.write(submission.source) |
|
26 | f.write(submission.source) |
|
|
27 | f.close |
|
27 | f.close |
|
|
28 | end |
|
28 | end |
|
|
29 |
|
29 | ||
|
|
30 | def clean_up(submission) |
|
30 | def clean_up(submission) |
|
|
31 | end |
|
31 | end |
|
|
32 | end |
|
32 | end |
|
|
33 |
|
33 | ||
|
|
34 | class SubmissionReporter |
|
34 | class SubmissionReporter |
|
|
35 | def initialize(options={}) |
|
35 | def initialize(options={}) |
|
|
36 | options = {:dry_run => false, :result_collector => nil}.merge(options) |
|
36 | options = {:dry_run => false, :result_collector => nil}.merge(options) |
|
|
37 | @config = Grader::Configuration.get_instance |
|
37 | @config = Grader::Configuration.get_instance |
|
|
38 | @dry_run = options[:dry_run] |
|
38 | @dry_run = options[:dry_run] |
|
|
39 | @result_collector = options[:result_collector] |
|
39 | @result_collector = options[:result_collector] |
|
|
40 | end |
|
40 | end |
|
|
41 |
|
41 | ||
|
|
42 | def report(sub,test_result_dir) |
|
42 | def report(sub,test_result_dir) |
|
|
43 | result = read_result(test_result_dir) |
|
43 | result = read_result(test_result_dir) |
|
|
44 | if @result_collector |
|
44 | if @result_collector |
|
|
45 |
- @result_collector.save(sub |
|
45 | + @result_collector.save(sub, |
|
|
46 | - sub.problem, |
|
||
|
|
47 | result) |
|
46 | result) |
|
|
48 | end |
|
47 | end |
|
|
49 | save_result(sub,result) |
|
48 | save_result(sub,result) |
|
|
50 | end |
|
49 | end |
|
|
51 |
|
50 | ||
|
|
52 | def report_error(sub,msg) |
|
51 | def report_error(sub,msg) |
|
|
53 | save_result(sub,{:points => 0, |
|
52 | save_result(sub,{:points => 0, |
|
|
54 | :comment => "Grading error: #{msg}" }) |
|
53 | :comment => "Grading error: #{msg}" }) |
|
|
55 | end |
|
54 | end |
|
|
56 |
|
55 | ||
|
|
57 | protected |
|
56 | protected |
|
|
58 | def read_result(test_result_dir) |
|
57 | def read_result(test_result_dir) |
|
|
59 | cmp_msg_fname = "#{test_result_dir}/compiler_message" |
|
58 | cmp_msg_fname = "#{test_result_dir}/compiler_message" |
|
|
60 | if FileTest.exist?(cmp_msg_fname) |
|
59 | if FileTest.exist?(cmp_msg_fname) |
|
|
61 | cmp_file = File.open(cmp_msg_fname) |
|
60 | cmp_file = File.open(cmp_msg_fname) |
|
|
62 | cmp_msg = cmp_file.read |
|
61 | cmp_msg = cmp_file.read |
|
|
63 | cmp_file.close |
|
62 | cmp_file.close |
|
|
64 | else |
|
63 | else |
|
|
65 | cmp_msg = "" |
|
64 | cmp_msg = "" |
|
|
66 | end |
|
65 | end |
|
|
67 |
|
66 | ||
|
|
68 | result_fname = "#{test_result_dir}/result" |
|
67 | result_fname = "#{test_result_dir}/result" |
|
|
69 | comment_fname = "#{test_result_dir}/comment" |
|
68 | comment_fname = "#{test_result_dir}/comment" |
|
|
70 | if FileTest.exist?(result_fname) |
|
69 | if FileTest.exist?(result_fname) |
|
|
71 | comment = "" |
|
70 | comment = "" |
|
|
72 | begin |
|
71 | begin |
|
|
73 | result_file = File.open(result_fname) |
|
72 | result_file = File.open(result_fname) |
|
|
74 | result = result_file.readline.to_i |
|
73 | result = result_file.readline.to_i |
|
|
75 | result_file.close |
|
74 | result_file.close |
|
|
76 | rescue |
|
75 | rescue |
|
|
77 | result = 0 |
|
76 | result = 0 |
|
|
78 | comment = "error reading result file." |
|
77 | comment = "error reading result file." |
|
|
79 | end |
|
78 | end |
|
|
80 |
|
79 | ||
|
|
81 | begin |
|
80 | begin |
|
|
82 | comment_file = File.open(comment_fname) |
|
81 | comment_file = File.open(comment_fname) |
|
|
83 | comment += comment_file.readline.chomp |
|
82 | comment += comment_file.readline.chomp |
|
|
84 | comment_file.close |
|
83 | comment_file.close |
|
|
85 | rescue |
|
84 | rescue |
|
|
86 | comment += "" |
|
85 | comment += "" |
|
|
87 | end |
|
86 | end |
|
|
88 |
|
87 | ||
|
|
89 | return {:points => result, |
|
88 | return {:points => result, |
|
|
90 | :comment => comment, |
|
89 | :comment => comment, |
|
|
91 | :cmp_msg => cmp_msg} |
|
90 | :cmp_msg => cmp_msg} |
|
|
92 | else |
|
91 | else |
|
|
93 | if FileTest.exist?("#{test_result_dir}/a.out") |
|
92 | if FileTest.exist?("#{test_result_dir}/a.out") |
|
|
94 | return {:points => 0, |
|
93 | return {:points => 0, |
|
|
95 | :comment => 'error during grading', |
|
94 | :comment => 'error during grading', |
|
|
96 | :cmp_msg => cmp_msg} |
|
95 | :cmp_msg => cmp_msg} |
|
|
97 | else |
|
96 | else |
|
|
98 | return {:points => 0, |
|
97 | return {:points => 0, |
|
|
99 | :comment => 'compilation error', |
|
98 | :comment => 'compilation error', |
|
|
100 | :cmp_msg => cmp_msg} |
|
99 | :cmp_msg => cmp_msg} |
|
|
101 | end |
|
100 | end |
|
|
102 | end |
|
101 | end |
|
|
103 | end |
|
102 | end |
|
|
104 |
|
103 | ||
|
|
105 | def save_result(submission,result) |
|
104 | def save_result(submission,result) |
|
|
106 | problem = submission.problem |
|
105 | problem = submission.problem |
|
|
107 | submission.graded_at = Time.now.gmtime |
|
106 | submission.graded_at = Time.now.gmtime |
|
|
108 | points = result[:points] |
|
107 | points = result[:points] |
|
|
109 | submission.points = points |
|
108 | submission.points = points |
|
|
110 | comment = @config.report_comment(result[:comment]) |
|
109 | comment = @config.report_comment(result[:comment]) |
|
|
111 |
|
110 | ||
|
|
112 | # |
|
111 | # |
|
|
113 | # TODO: FIX THIS MESSAGE |
|
112 | # TODO: FIX THIS MESSAGE |
|
|
114 | # |
|
113 | # |
|
|
115 | if problem == nil |
|
114 | if problem == nil |
|
|
116 | submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)' |
|
115 | submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)' |
|
|
117 | elsif points == problem.full_score |
|
116 | elsif points == problem.full_score |
|
|
118 | #submission.grader_comment = 'PASSED: ' + comment |
|
117 | #submission.grader_comment = 'PASSED: ' + comment |
|
|
119 | submission.grader_comment = comment |
|
118 | submission.grader_comment = comment |
|
|
120 | elsif result[:comment].chomp =~ /^[\[\]P]+$/ |
|
119 | elsif result[:comment].chomp =~ /^[\[\]P]+$/ |
|
|
121 | submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)' |
|
120 | submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)' |
|
|
122 | else |
|
121 | else |
|
|
123 | #submission.grader_comment = 'FAILED: ' + comment |
|
122 | #submission.grader_comment = 'FAILED: ' + comment |
|
|
124 | submission.grader_comment = comment |
|
123 | submission.grader_comment = comment |
|
|
125 | end |
|
124 | end |
|
|
126 | submission.compiler_message = result[:cmp_msg] or '' |
|
125 | submission.compiler_message = result[:cmp_msg] or '' |
|
|
127 |
|
126 | ||
|
|
128 | if not @dry_run |
|
127 | if not @dry_run |
|
|
129 | submission.save |
|
128 | submission.save |
|
|
130 | end |
|
129 | end |
|
|
131 | end |
|
130 | end |
|
|
132 |
|
131 | ||
|
|
133 | end |
|
132 | end |
|
|
134 |
|
133 | ||
|
|
135 | end |
|
134 | end |
| @@ -1,109 +1,113 | |||||
|
|
1 | #!/usr/bin/ruby |
|
1 | #!/usr/bin/ruby |
|
|
2 |
|
2 | ||
|
|
3 | require 'fileutils' |
|
3 | require 'fileutils' |
|
|
4 |
|
4 | ||
|
|
5 | ############################## |
|
5 | ############################## |
|
|
6 | # |
|
6 | # |
|
|
7 | # Standard Compile Script |
|
7 | # Standard Compile Script |
|
|
8 | # |
|
8 | # |
|
|
9 | # Supported compilers: |
|
9 | # Supported compilers: |
|
|
10 | # gcc, g++, and fpc. |
|
10 | # gcc, g++, and fpc. |
|
|
11 | # |
|
11 | # |
|
|
12 | ############################## |
|
12 | ############################## |
|
|
13 |
|
13 | ||
|
|
14 | def talk(msg) |
|
14 | def talk(msg) |
|
|
15 | if ENV['TALKATIVE']!=nil |
|
15 | if ENV['TALKATIVE']!=nil |
|
|
16 | puts str |
|
16 | puts str |
|
|
17 | end |
|
17 | end |
|
|
18 | if ENV['GRADER_LOGGING']!=nil |
|
18 | if ENV['GRADER_LOGGING']!=nil |
|
|
19 | log_fname = ENV['GRADER_LOGGING'] |
|
19 | log_fname = ENV['GRADER_LOGGING'] |
|
|
20 | fp = File.open(log_fname,"a") |
|
20 | fp = File.open(log_fname,"a") |
|
|
21 | fp.puts("run: #{Time.new.strftime("%H:%M")} #{msg}") |
|
21 | fp.puts("run: #{Time.new.strftime("%H:%M")} #{msg}") |
|
|
22 | fp.close |
|
22 | fp.close |
|
|
23 | end |
|
23 | end |
|
|
24 | end |
|
24 | end |
|
|
25 |
|
25 | ||
|
|
26 | C_COMPILER = "gcc" |
|
26 | C_COMPILER = "gcc" |
|
|
27 | CPLUSPLUS_COMPILER = "g++" |
|
27 | CPLUSPLUS_COMPILER = "g++" |
|
|
28 | PASCAL_COMPILER = "fpc" |
|
28 | PASCAL_COMPILER = "fpc" |
|
|
29 |
|
29 | ||
|
|
30 | C_OPTIONS = "-O2 -s -static -std=c99 -DCONTEST -lm -Wall" |
|
30 | C_OPTIONS = "-O2 -s -static -std=c99 -DCONTEST -lm -Wall" |
|
|
31 | CPLUSPLUS_OPTIONS = "-O2 -s -static -DCONTEST -lm -Wall" |
|
31 | CPLUSPLUS_OPTIONS = "-O2 -s -static -DCONTEST -lm -Wall" |
|
|
32 | PASCAL_OPTIONS = "-O1 -XS -dCONTEST" |
|
32 | PASCAL_OPTIONS = "-O1 -XS -dCONTEST" |
|
|
33 |
|
33 | ||
|
|
34 | # Check for the correct number of arguments. Otherwise, print usage. |
|
34 | # Check for the correct number of arguments. Otherwise, print usage. |
|
|
35 | if ARGV.length == 0 or ARGV.length > 4 |
|
35 | if ARGV.length == 0 or ARGV.length > 4 |
|
|
36 | puts "Usage: compile <language> [<source-file>] [<output-file>] [<message-file>]" |
|
36 | puts "Usage: compile <language> [<source-file>] [<output-file>] [<message-file>]" |
|
|
37 | puts |
|
37 | puts |
|
|
38 | puts "<source-file> is defaulted to \"source\"." |
|
38 | puts "<source-file> is defaulted to \"source\"." |
|
|
39 | puts "<output-file> is defaulted to \"a.out\"." |
|
39 | puts "<output-file> is defaulted to \"a.out\"." |
|
|
40 | puts "<message-file> is defaulted to \"compiler_message\"." |
|
40 | puts "<message-file> is defaulted to \"compiler_message\"." |
|
|
41 | puts |
|
41 | puts |
|
|
42 | exit(127) |
|
42 | exit(127) |
|
|
43 | end |
|
43 | end |
|
|
44 |
|
44 | ||
|
|
45 | PARAMS = { |
|
45 | PARAMS = { |
|
|
46 | :source_file => [1,'source'], |
|
46 | :source_file => [1,'source'], |
|
|
47 | :output_file => [2,'a.out'], |
|
47 | :output_file => [2,'a.out'], |
|
|
48 | :message_file => [3,'compiler_message'] |
|
48 | :message_file => [3,'compiler_message'] |
|
|
49 | } |
|
49 | } |
|
|
50 |
|
50 | ||
|
|
51 | params = {} |
|
51 | params = {} |
|
|
52 | params[:prog_lang] = ARGV[0] |
|
52 | params[:prog_lang] = ARGV[0] |
|
|
53 | PARAMS.each_key do |param_name| |
|
53 | PARAMS.each_key do |param_name| |
|
|
54 | index, default = PARAMS[param_name] |
|
54 | index, default = PARAMS[param_name] |
|
|
55 | if ARGV.length > index |
|
55 | if ARGV.length > index |
|
|
56 | params[param_name] = ARGV[index] |
|
56 | params[param_name] = ARGV[index] |
|
|
57 | else |
|
57 | else |
|
|
58 | params[param_name] = default |
|
58 | params[param_name] = default |
|
|
59 | end |
|
59 | end |
|
|
60 | talk "#{param_name}: #{params[param_name]}" |
|
60 | talk "#{param_name}: #{params[param_name]}" |
|
|
61 | end |
|
61 | end |
|
|
62 |
|
62 | ||
|
|
63 | # Remove any remaining output files or message files. |
|
63 | # Remove any remaining output files or message files. |
|
|
64 | if FileTest.exists? params[:output_file] |
|
64 | if FileTest.exists? params[:output_file] |
|
|
65 | FileUtils.rm(params[:output_file]) |
|
65 | FileUtils.rm(params[:output_file]) |
|
|
66 | end |
|
66 | end |
|
|
67 | if FileTest.exists? params[:message_file] |
|
67 | if FileTest.exists? params[:message_file] |
|
|
68 | FileUtils.rm(params[:message_file]) |
|
68 | FileUtils.rm(params[:message_file]) |
|
|
69 | end |
|
69 | end |
|
|
70 |
|
70 | ||
|
|
71 | # Check if the source file exists before attempt compiling. |
|
71 | # Check if the source file exists before attempt compiling. |
|
|
72 | if !FileTest.exists? params[:source_file] |
|
72 | if !FileTest.exists? params[:source_file] |
|
|
73 | talk("ERROR: The source file does not exist!") |
|
73 | talk("ERROR: The source file does not exist!") |
|
|
74 | open(params[:message_file],"w") do |f| |
|
74 | open(params[:message_file],"w") do |f| |
|
|
75 | f.puts "ERROR: The source file did not exist." |
|
75 | f.puts "ERROR: The source file did not exist." |
|
|
76 | end |
|
76 | end |
|
|
77 | exit(127) |
|
77 | exit(127) |
|
|
78 | end |
|
78 | end |
|
|
79 |
|
79 | ||
|
|
|
80 | + if params[:prog_lang]=='cpp': | ||
|
|
|
81 | + params[:prog_lang] = 'c++' | ||
|
|
|
82 | + end | ||
|
|
|
83 | + | ||
|
|
80 | # Compile. |
|
84 | # Compile. |
|
|
81 | case params[:prog_lang] |
|
85 | case params[:prog_lang] |
|
|
82 |
|
86 | ||
|
|
83 | when "c" |
|
87 | when "c" |
|
|
84 | command = "#{C_COMPILER} #{params[:source_file]} -o #{params[:output_file]} #{C_OPTIONS} 2> #{params[:message_file]}" |
|
88 | command = "#{C_COMPILER} #{params[:source_file]} -o #{params[:output_file]} #{C_OPTIONS} 2> #{params[:message_file]}" |
|
|
85 | system(command) |
|
89 | system(command) |
|
|
86 |
|
90 | ||
|
|
87 | when "c++" |
|
91 | when "c++" |
|
|
88 | command = "#{CPLUSPLUS_COMPILER} #{params[:source_file]} -o #{params[:output_file]} #{CPLUSPLUS_OPTIONS} 2> #{params[:message_file]}" |
|
92 | command = "#{CPLUSPLUS_COMPILER} #{params[:source_file]} -o #{params[:output_file]} #{CPLUSPLUS_OPTIONS} 2> #{params[:message_file]}" |
|
|
89 | system(command) |
|
93 | system(command) |
|
|
90 |
|
94 | ||
|
|
91 | when "pas" |
|
95 | when "pas" |
|
|
92 | command = "#{PASCAL_COMPILER} #{params[:source_file]} -ooutpas #{PASCAL_OPTIONS} > #{params[:message_file]}" |
|
96 | command = "#{PASCAL_COMPILER} #{params[:source_file]} -ooutpas #{PASCAL_OPTIONS} > #{params[:message_file]}" |
|
|
93 | system(command) |
|
97 | system(command) |
|
|
94 | FileUtils.mv("output", params[:output_file]) |
|
98 | FileUtils.mv("output", params[:output_file]) |
|
|
95 |
|
99 | ||
|
|
96 | else |
|
100 | else |
|
|
97 | talk("ERROR: Invalid language specified!") |
|
101 | talk("ERROR: Invalid language specified!") |
|
|
98 | open(params[:message_file],"w") do |f| |
|
102 | open(params[:message_file],"w") do |f| |
|
|
99 | f.puts "ERROR: Invalid language specified!" |
|
103 | f.puts "ERROR: Invalid language specified!" |
|
|
100 | end |
|
104 | end |
|
|
101 | exit(127) |
|
105 | exit(127) |
|
|
102 | end |
|
106 | end |
|
|
103 |
|
107 | ||
|
|
104 | # Report success or failure. |
|
108 | # Report success or failure. |
|
|
105 | if FileTest.exists? params[:output_file] |
|
109 | if FileTest.exists? params[:output_file] |
|
|
106 | talk "Compilation was successful!" |
|
110 | talk "Compilation was successful!" |
|
|
107 | else |
|
111 | else |
|
|
108 | talk "ERROR: Something was wrong during the compilation!" |
|
112 | talk "ERROR: Something was wrong during the compilation!" |
|
|
109 | end |
|
113 | end |
| @@ -1,222 +1,234 | |||||
|
|
1 | require File.join(File.dirname(__FILE__),'spec_helper') |
|
1 | require File.join(File.dirname(__FILE__),'spec_helper') |
|
|
2 | require File.join(File.dirname(__FILE__),'engine_spec_helper') |
|
2 | require File.join(File.dirname(__FILE__),'engine_spec_helper') |
|
|
3 |
|
3 | ||
|
|
4 | describe "A grader engine, when grading submissions" do |
|
4 | describe "A grader engine, when grading submissions" do |
|
|
5 |
|
5 | ||
|
|
6 | include GraderEngineHelperMethods |
|
6 | include GraderEngineHelperMethods |
|
|
7 |
|
7 | ||
|
|
8 | before(:each) do |
|
8 | before(:each) do |
|
|
9 | @config = Grader::Configuration.get_instance |
|
9 | @config = Grader::Configuration.get_instance |
|
|
10 |
|
10 | ||
|
|
11 | # this test is from Pong |
|
11 | # this test is from Pong |
|
|
12 | @problem_test_normal = stub(Problem, |
|
12 | @problem_test_normal = stub(Problem, |
|
|
13 | :id => 1, :name => 'test_normal', |
|
13 | :id => 1, :name => 'test_normal', |
|
|
14 | :full_score => 135) |
|
14 | :full_score => 135) |
|
|
15 | @user_user1 = stub(User, |
|
15 | @user_user1 = stub(User, |
|
|
16 | :id => 1, :login => 'user1') |
|
16 | :id => 1, :login => 'user1') |
|
|
17 |
|
17 | ||
|
|
18 | @engine = Grader::Engine.new |
|
18 | @engine = Grader::Engine.new |
|
|
19 | init_sandbox |
|
19 | init_sandbox |
|
|
20 | end |
|
20 | end |
|
|
21 |
|
21 | ||
|
|
22 | it "should grade normal submission" do |
|
22 | it "should grade normal submission" do |
|
|
23 | grader_should(:grade => "test1_correct.c", |
|
23 | grader_should(:grade => "test1_correct.c", |
|
|
24 | :on => @problem_test_normal, |
|
24 | :on => @problem_test_normal, |
|
|
25 | :and_report => { |
|
25 | :and_report => { |
|
|
26 | :score => 135, |
|
26 | :score => 135, |
|
|
27 | :comment => /^(\[|P|\])+/}) |
|
27 | :comment => /^(\[|P|\])+/}) |
|
|
28 | end |
|
28 | end |
|
|
29 |
|
29 | ||
|
|
30 |
|
30 | ||
|
|
|
31 | + it "should grade normal submission in C++" do | ||
|
|
|
32 | + cpp_lang = stub(Language, :name => 'cpp', :ext => 'cpp') | ||
|
|
|
33 | + | ||
|
|
|
34 | + grader_should(:grade => "test1_correct_cc.cc", | ||
|
|
|
35 | + :language => cpp_lang, | ||
|
|
|
36 | + :on => @problem_test_normal, | ||
|
|
|
37 | + :and_report => { | ||
|
|
|
38 | + :score => 135, | ||
|
|
|
39 | + :comment => /^(\[|P|\])+/}) | ||
|
|
|
40 | + end | ||
|
|
|
41 | + | ||
|
|
31 | it "should produce error message when submission cannot compile" do |
|
42 | it "should produce error message when submission cannot compile" do |
|
|
32 | grader_should(:grade => "test1_compile_error.c", |
|
43 | grader_should(:grade => "test1_compile_error.c", |
|
|
33 | :on => @problem_test_normal, |
|
44 | :on => @problem_test_normal, |
|
|
34 | :and_report => { |
|
45 | :and_report => { |
|
|
35 | :score => 0, |
|
46 | :score => 0, |
|
|
36 | :comment => 'compilation error', |
|
47 | :comment => 'compilation error', |
|
|
37 | :compiler_message => /[Ee]rror/}) |
|
48 | :compiler_message => /[Ee]rror/}) |
|
|
38 | end |
|
49 | end |
|
|
39 |
|
50 | ||
|
|
40 | it "should produce timeout error when submission runs forever" do |
|
51 | it "should produce timeout error when submission runs forever" do |
|
|
41 | @problem_test_timeout = stub(Problem, |
|
52 | @problem_test_timeout = stub(Problem, |
|
|
42 | :id => 1, :name => 'test_timeout', |
|
53 | :id => 1, :name => 'test_timeout', |
|
|
43 | :full_score => 10) |
|
54 | :full_score => 10) |
|
|
44 | grader_should(:grade => "test2_timeout.c", |
|
55 | grader_should(:grade => "test2_timeout.c", |
|
|
45 | :on => @problem_test_timeout, |
|
56 | :on => @problem_test_timeout, |
|
|
46 | :and_report => { |
|
57 | :and_report => { |
|
|
47 | :score => 0, |
|
58 | :score => 0, |
|
|
48 | :comment => 'TT'}) |
|
59 | :comment => 'TT'}) |
|
|
49 | end |
|
60 | end |
|
|
50 |
|
61 | ||
|
|
51 | it "should produce timeout error correctly with fractional running time and fractional time limits" do |
|
62 | it "should produce timeout error correctly with fractional running time and fractional time limits" do |
|
|
52 | @problem_test_timeout = stub(Problem, |
|
63 | @problem_test_timeout = stub(Problem, |
|
|
53 | :id => 1, :name => 'test_timeout', |
|
64 | :id => 1, :name => 'test_timeout', |
|
|
54 | :full_score => 20) |
|
65 | :full_score => 20) |
|
|
55 | grader_should(:grade => "test2_05sec.c", |
|
66 | grader_should(:grade => "test2_05sec.c", |
|
|
56 | :on => @problem_test_timeout, |
|
67 | :on => @problem_test_timeout, |
|
|
57 | :and_report => { |
|
68 | :and_report => { |
|
|
58 | :score => 10, |
|
69 | :score => 10, |
|
|
59 | :comment => 'TP'}) |
|
70 | :comment => 'TP'}) |
|
|
60 | end |
|
71 | end |
|
|
61 |
|
72 | ||
|
|
62 | it "should produce runtime error when submission uses too much static memory" do |
|
73 | it "should produce runtime error when submission uses too much static memory" do |
|
|
63 | @problem_test_memory = stub(Problem, |
|
74 | @problem_test_memory = stub(Problem, |
|
|
64 | :id => 1, :name => 'test_memory', |
|
75 | :id => 1, :name => 'test_memory', |
|
|
65 | :full_score => 20) |
|
76 | :full_score => 20) |
|
|
66 | grader_should(:grade => "add_too_much_memory_static.c", |
|
77 | grader_should(:grade => "add_too_much_memory_static.c", |
|
|
67 | :on => @problem_test_memory, |
|
78 | :on => @problem_test_memory, |
|
|
68 | :and_report => { |
|
79 | :and_report => { |
|
|
69 | :score => 10, |
|
80 | :score => 10, |
|
|
70 | :comment => /[^P]P/}) |
|
81 | :comment => /[^P]P/}) |
|
|
71 | end |
|
82 | end |
|
|
72 |
|
83 | ||
|
|
73 | it "should not allow submission to allocate too much dynamic memory" do |
|
84 | it "should not allow submission to allocate too much dynamic memory" do |
|
|
74 | @problem_test_memory = stub(Problem, |
|
85 | @problem_test_memory = stub(Problem, |
|
|
75 | :id => 1, :name => 'test_memory', |
|
86 | :id => 1, :name => 'test_memory', |
|
|
76 | :full_score => 20) |
|
87 | :full_score => 20) |
|
|
77 | grader_should(:grade => "add_too_much_memory_dynamic.c", |
|
88 | grader_should(:grade => "add_too_much_memory_dynamic.c", |
|
|
78 | :on => @problem_test_memory, |
|
89 | :on => @problem_test_memory, |
|
|
79 | :and_report => { |
|
90 | :and_report => { |
|
|
80 | :score => 10, |
|
91 | :score => 10, |
|
|
81 | :comment => /[^P]P/}) |
|
92 | :comment => /[^P]P/}) |
|
|
82 | end |
|
93 | end |
|
|
83 |
|
94 | ||
|
|
84 | it "should score test runs correctly when submission fails in some test case" do |
|
95 | it "should score test runs correctly when submission fails in some test case" do |
|
|
85 | grader_should(:grade => "add_fail_test_case_1.c", |
|
96 | grader_should(:grade => "add_fail_test_case_1.c", |
|
|
86 | :on => @problem_test_normal, |
|
97 | :on => @problem_test_normal, |
|
|
87 | :and_report => { |
|
98 | :and_report => { |
|
|
88 | :score => 105, |
|
99 | :score => 105, |
|
|
89 | :comment => /^.*(-|x).*$/}) |
|
100 | :comment => /^.*(-|x).*$/}) |
|
|
90 | end |
|
101 | end |
|
|
91 |
|
102 | ||
|
|
92 | it "should fail submission with non-zero exit status" do |
|
103 | it "should fail submission with non-zero exit status" do |
|
|
93 | grader_should(:grade => "add_nonzero_exit_status.c", |
|
104 | grader_should(:grade => "add_nonzero_exit_status.c", |
|
|
94 | :on => @problem_test_normal, |
|
105 | :on => @problem_test_normal, |
|
|
95 | :and_report => { |
|
106 | :and_report => { |
|
|
96 | :score => 0, |
|
107 | :score => 0, |
|
|
97 | :comment => /^(\[|x|\])+$/}) |
|
108 | :comment => /^(\[|x|\])+$/}) |
|
|
98 | end |
|
109 | end |
|
|
99 |
|
110 | ||
|
|
100 | it "should not allow malicious submission to see PROBLEM_HOME" do |
|
111 | it "should not allow malicious submission to see PROBLEM_HOME" do |
|
|
101 | problem_test_yesno = stub(Problem, |
|
112 | problem_test_yesno = stub(Problem, |
|
|
102 | :id => 1, :name => 'test_yesno', |
|
113 | :id => 1, :name => 'test_yesno', |
|
|
103 | :full_score => 10) |
|
114 | :full_score => 10) |
|
|
104 | grader_should(:grade => "yesno_access_problem_home.c", |
|
115 | grader_should(:grade => "yesno_access_problem_home.c", |
|
|
105 | :on => problem_test_yesno, |
|
116 | :on => problem_test_yesno, |
|
|
106 | :and_report => { |
|
117 | :and_report => { |
|
|
107 | :score => 0, |
|
118 | :score => 0, |
|
|
108 | :comment => /(-|x)/}) |
|
119 | :comment => /(-|x)/}) |
|
|
109 | end |
|
120 | end |
|
|
110 |
|
121 | ||
|
|
111 | it "should not allow malicious submission to open files" do |
|
122 | it "should not allow malicious submission to open files" do |
|
|
112 | problem_test_yesno = stub(Problem, |
|
123 | problem_test_yesno = stub(Problem, |
|
|
113 | :id => 1, :name => 'test_yesno', |
|
124 | :id => 1, :name => 'test_yesno', |
|
|
114 | :full_score => 10) |
|
125 | :full_score => 10) |
|
|
115 | grader_should(:grade => "yesno_open_file.c", |
|
126 | grader_should(:grade => "yesno_open_file.c", |
|
|
116 | :on => problem_test_yesno, |
|
127 | :on => problem_test_yesno, |
|
|
117 | :and_report => { |
|
128 | :and_report => { |
|
|
118 | :score => 0, |
|
129 | :score => 0, |
|
|
119 | :comment => /(-|x)/}) |
|
130 | :comment => /(-|x)/}) |
|
|
120 | end |
|
131 | end |
|
|
121 |
|
132 | ||
|
|
122 | def grader_should(args) |
|
133 | def grader_should(args) |
|
|
123 | @user1 = stub(User, |
|
134 | @user1 = stub(User, |
|
|
124 | :id => 1, :login => 'user1') |
|
135 | :id => 1, :login => 'user1') |
|
|
|
136 | + | ||
|
|
125 | submission = |
|
137 | submission = |
|
|
126 | - create_submission_from_file(1, @user1, args[:on], args[:grade]) |
|
138 | + create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language]) |
|
|
127 | submission.should_receive(:graded_at=) |
|
139 | submission.should_receive(:graded_at=) |
|
|
128 |
|
140 | ||
|
|
129 | expected_score = args[:and_report][:score] |
|
141 | expected_score = args[:and_report][:score] |
|
|
130 | expected_comment = args[:and_report][:comment] |
|
142 | expected_comment = args[:and_report][:comment] |
|
|
131 | if args[:and_report][:compiler_message]!=nil |
|
143 | if args[:and_report][:compiler_message]!=nil |
|
|
132 | expected_compiler_message = args[:and_report][:compiler_message] |
|
144 | expected_compiler_message = args[:and_report][:compiler_message] |
|
|
133 | else |
|
145 | else |
|
|
134 | expected_compiler_message = '' |
|
146 | expected_compiler_message = '' |
|
|
135 | end |
|
147 | end |
|
|
136 |
|
148 | ||
|
|
137 | submission.should_receive(:points=).with(expected_score) |
|
149 | submission.should_receive(:points=).with(expected_score) |
|
|
138 | submission.should_receive(:grader_comment=).with(expected_comment) |
|
150 | submission.should_receive(:grader_comment=).with(expected_comment) |
|
|
139 | submission.should_receive(:compiler_message=).with(expected_compiler_message) |
|
151 | submission.should_receive(:compiler_message=).with(expected_compiler_message) |
|
|
140 | submission.should_receive(:save) |
|
152 | submission.should_receive(:save) |
|
|
141 |
|
153 | ||
|
|
142 | @engine.grade(submission) |
|
154 | @engine.grade(submission) |
|
|
143 | end |
|
155 | end |
|
|
144 |
|
156 | ||
|
|
145 | protected |
|
157 | protected |
|
|
146 |
|
158 | ||
|
|
147 | def create_normal_submission_mock_from_file(source_fname) |
|
159 | def create_normal_submission_mock_from_file(source_fname) |
|
|
148 | create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname) |
|
160 | create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname) |
|
|
149 | end |
|
161 | end |
|
|
150 |
|
162 | ||
|
|
151 | end |
|
163 | end |
|
|
152 |
|
164 | ||
|
|
153 | describe "A grader engine, when grading test requests" do |
|
165 | describe "A grader engine, when grading test requests" do |
|
|
154 |
|
166 | ||
|
|
155 | include GraderEngineHelperMethods |
|
167 | include GraderEngineHelperMethods |
|
|
156 |
|
168 | ||
|
|
157 | before(:each) do |
|
169 | before(:each) do |
|
|
158 | @config = Grader::Configuration.get_instance |
|
170 | @config = Grader::Configuration.get_instance |
|
|
159 | @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new, |
|
171 | @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new, |
|
|
160 | :reporter => Grader::TestRequestReporter.new) |
|
172 | :reporter => Grader::TestRequestReporter.new) |
|
|
161 | init_sandbox |
|
173 | init_sandbox |
|
|
162 | end |
|
174 | end |
|
|
163 |
|
175 | ||
|
|
164 | it "should report error if there is no problem template" do |
|
176 | it "should report error if there is no problem template" do |
|
|
165 | problem = stub(Problem, |
|
177 | problem = stub(Problem, |
|
|
166 | :id => 1, :name => 'nothing') |
|
178 | :id => 1, :name => 'nothing') |
|
|
167 | grader_should(:grade => 'test1_correct.c', |
|
179 | grader_should(:grade => 'test1_correct.c', |
|
|
168 | :on => problem, |
|
180 | :on => problem, |
|
|
169 | :with => 'in1.txt', |
|
181 | :with => 'in1.txt', |
|
|
170 | :and_report => { |
|
182 | :and_report => { |
|
|
171 | :graded_at= => nil, |
|
183 | :graded_at= => nil, |
|
|
172 | :compiler_message= => '', |
|
184 | :compiler_message= => '', |
|
|
173 | :grader_comment= => '', |
|
185 | :grader_comment= => '', |
|
|
174 | :running_stat= => /template not found/, |
|
186 | :running_stat= => /template not found/, |
|
|
175 | :running_time= => nil, |
|
187 | :running_time= => nil, |
|
|
176 | :exit_status= => nil, |
|
188 | :exit_status= => nil, |
|
|
177 | :memory_usage= => nil, |
|
189 | :memory_usage= => nil, |
|
|
178 | :save => nil}) |
|
190 | :save => nil}) |
|
|
179 | end |
|
191 | end |
|
|
180 |
|
192 | ||
|
|
181 | it "should run test request and produce output file" do |
|
193 | it "should run test request and produce output file" do |
|
|
182 | problem = stub(Problem, |
|
194 | problem = stub(Problem, |
|
|
183 | :id => 1, :name => 'test_normal') |
|
195 | :id => 1, :name => 'test_normal') |
|
|
184 | grader_should(:grade => 'test1_correct.c', |
|
196 | grader_should(:grade => 'test1_correct.c', |
|
|
185 | :on => problem, |
|
197 | :on => problem, |
|
|
186 | :with => 'in1.txt', |
|
198 | :with => 'in1.txt', |
|
|
187 | :and_report => { |
|
199 | :and_report => { |
|
|
188 | :graded_at= => nil, |
|
200 | :graded_at= => nil, |
|
|
189 | :compiler_message= => '', |
|
201 | :compiler_message= => '', |
|
|
190 | :grader_comment= => '', |
|
202 | :grader_comment= => '', |
|
|
191 | :running_stat= => /0.0\d* sec./, |
|
203 | :running_stat= => /0.0\d* sec./, |
|
|
192 | :output_file_name= => lambda { |fname| |
|
204 | :output_file_name= => lambda { |fname| |
|
|
193 | File.exists?(fname).should be_true |
|
205 | File.exists?(fname).should be_true |
|
|
194 | }, |
|
206 | }, |
|
|
195 | :running_time= => nil, |
|
207 | :running_time= => nil, |
|
|
196 | :exit_status= => nil, |
|
208 | :exit_status= => nil, |
|
|
197 | :memory_usage= => nil, |
|
209 | :memory_usage= => nil, |
|
|
198 | :save => nil}) |
|
210 | :save => nil}) |
|
|
199 | end |
|
211 | end |
|
|
200 |
|
212 | ||
|
|
201 | it "should clean up problem directory after running test request" do |
|
213 | it "should clean up problem directory after running test request" do |
|
|
202 | problem = stub(Problem, |
|
214 | problem = stub(Problem, |
|
|
203 | :id => 1, :name => 'test_normal') |
|
215 | :id => 1, :name => 'test_normal') |
|
|
204 | grader_should(:grade => 'test1_correct.c', |
|
216 | grader_should(:grade => 'test1_correct.c', |
|
|
205 | :on => problem, |
|
217 | :on => problem, |
|
|
206 | :with => 'in1.txt', |
|
218 | :with => 'in1.txt', |
|
|
207 | :and_report => { |
|
219 | :and_report => { |
|
|
208 | :graded_at= => nil, |
|
220 | :graded_at= => nil, |
|
|
209 | :compiler_message= => '', |
|
221 | :compiler_message= => '', |
|
|
210 | :grader_comment= => '', |
|
222 | :grader_comment= => '', |
|
|
211 | :running_stat= => nil, |
|
223 | :running_stat= => nil, |
|
|
212 | :output_file_name= => nil, |
|
224 | :output_file_name= => nil, |
|
|
213 | :running_time= => nil, |
|
225 | :running_time= => nil, |
|
|
214 | :exit_status= => nil, |
|
226 | :exit_status= => nil, |
|
|
215 | :memory_usage= => nil, |
|
227 | :memory_usage= => nil, |
|
|
216 | :save => nil}) |
|
228 | :save => nil}) |
|
|
217 | File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false |
|
229 | File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false |
|
|
218 | end |
|
230 | end |
|
|
219 |
|
231 | ||
|
|
220 | it "should compile test request with error and report compilation error" do |
|
232 | it "should compile test request with error and report compilation error" do |
|
|
221 | problem = stub(Problem, |
|
233 | problem = stub(Problem, |
|
|
222 | :id => 1, :name => 'test_normal') |
|
234 | :id => 1, :name => 'test_normal') |
You need to be logged in to leave comments.
Login now