cafe-grader-judge-scripts

Commit r92:274af5f15b1f public

parent | child
Description:
added grading report
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r92:274af5f15b1f - - 4 files changed: 93 inserted, 19 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,239 +1,297
1 1 #!/usr/bin/ruby
2 2
3 3 def stop_grader(id)
4 4 if id==:all
5 5 File.open(File.dirname(__FILE__) + "/stop.all",'w').close
6 6 else
7 7 File.open(File.dirname(__FILE__) + "/stop.#{id}",'w').close
8 8 end
9 9 end
10 10
11 11 def check_stopfile
12 12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
13 13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
14 14 end
15 15
16 16 def clear_stopfile
17 17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 18 system("rm " + File.dirname(__FILE__) + "/stop.#{Process.pid}")
19 19 end
20 20 end
21 21
22 22 def config
23 23 Grader::Configuration.get_instance
24 24 end
25 25
26 26 def log_file_name
27 27 if !File.exists?(config.log_dir)
28 28 raise "Log directory does not exist: #{config.log_dir}"
29 29 end
30 30 config.log_dir +
31 31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
32 32 end
33 33
34 34 def log(str)
35 35 if config.talkative
36 36 puts str
37 37 end
38 38 if config.logging
39 39 fp = File.open(log_file_name,"a")
40 40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
41 41 fp.close
42 42 end
43 43 end
44 44
45 45 def display_manual
46 46 puts <<USAGE
47 47 Grader.
48 48 using: (1) grader
49 49 (2) grader environment [mode]
50 50 (3) grader stop [all|pids-list]
51 51 (4) grader --help
52 52 (1) call grader with environment = 'exam', mode = 'queue'
53 53 (2) possible modes are: 'queue', 'prob', 'test_request'
54 54 (3) create stop-file to stop running grader in queue mode
55 55 (4) You are here.
56 56 USAGE
57 57 end
58 58
59 - #########################################
60 - # main program
61 - #########################################
62 -
63 - # Reading environment and options.
64 -
65 59 def process_options_and_stop_file
66 60 # The list of options are:
67 61 # - stop [all|process ids]
68 62 # -
69 63
70 64 # Process 'help' option
71 65 if (ARGV.length==1) and (/help/.match(ARGV[0]))
72 66 display_manual
73 67 exit(0)
74 68 end
75 69
76 70 # Process 'stop' option.
77 71 if (ARGV.length >= 1) and (ARGV[0]=='stop')
78 72 if ARGV.length==1
79 73 puts "you should specify pid-list or 'all'"
80 74 display_manual
81 75 elsif (ARGV.length==2) and (ARGV[1]=='all')
82 76 stop_grader(:all)
83 77 puts "A global stop file ('stop.all') created."
84 78 puts "You should remove it manually later."
85 79 else
86 80 (1..ARGV.length-1).each do |i|
87 81 stop_grader(ARGV[i])
88 82 end
89 83 puts "stop file(s) created"
90 84 end
91 85 exit(0)
92 86 end
93 87
94 88 # Check stop file.
95 89 if check_stopfile
96 90 puts "Stop file exists. Terminated."
97 91 clear_stopfile
98 92 exit(0)
99 93 end
100 94
101 95 #default options
102 96 options = {
103 97 :mode => 'queue',
104 98 :environment => 'exam',
105 99 :dry_run => false,
106 100 }
107 101
108 102 # Process mode and environment option
109 103 if ARGV.length >= 1
110 104 options[:environment] = ARGV.shift
111 105 if ARGV.length >=1
112 106 options[:mode] = ARGV.shift
113 107 end
114 108 end
115 109
116 110 options[:dry_run] = (ARGV.delete('--dry') != nil)
117 111 if options[:dry_run] and (not options[:mode] == 'prob')
118 - puts "Dry run currently works for 'prob' mode."
112 + puts "Dry run currently works only for 'prob' mode."
113 + exit(0)
114 + end
115 +
116 + options[:report] = (ARGV.delete('--report') != nil)
117 + if options[:report] and (not options[:mode] == 'prob')
118 + puts "Report currently works only for 'prob' mode."
119 119 exit(0)
120 120 end
121 121
122 122 return options
123 123 end
124 124
125 - # ======= Main ========
125 + class ResultCollector
126 + def initialize
127 + @results = {}
128 + @problems = {}
129 + @users = {}
130 + end
131 +
132 + def save(user, problem, grading_result)
133 + if not @problems.has_key? problem.id
134 + @problems[problem.id] = problem
135 + end
136 + if not @users.has_key? user.id
137 + @users[user.id] = user
138 + end
139 + @results[[user.id, problem.id]] = grading_result
140 + end
141 +
142 + def print_report_by_user
143 + puts "---------------------"
144 + puts " REPORT"
145 + puts "---------------------"
146 +
147 + print "login"
148 + @problems.each_value do |problem|
149 + print ",#{problem.name}"
150 + end
151 + print "\n"
152 +
153 + @users.each_value do |user|
154 + print "#{user.login}"
155 + @problems.each_value do |problem|
156 + if @results.has_key? [user.id, problem.id]
157 + print ",#{@results[[user.id,problem.id]][:points]}"
158 + else
159 + print ","
160 + end
161 + end
162 + print "\n"
163 + end
164 + end
165 + end
166 +
167 + #########################################
168 + # main program
169 + #########################################
170 +
126 171 options = process_options_and_stop_file
127 172 GRADER_ENV = options[:environment]
128 173 grader_mode = options[:mode]
129 174 dry_run = options[:dry_run]
130 175
131 176 puts "environment: #{GRADER_ENV}"
132 177 require File.join(File.dirname(__FILE__),'config/environment')
133 178
134 179 # add grader_mode to config
135 180 # this is needed because method log needs it. TODO: clean this up
136 181 class << config
137 182 attr_accessor :grader_mode
138 183 end
139 184 config.grader_mode = grader_mode
140 185
141 186 # reading rails environment
142 187 log 'Reading rails environment'
143 188
144 189 RAILS_ENV = config.rails_env
145 190 require RAILS_ROOT + '/config/environment'
146 191
147 192 # register grader process
148 193 if config.report_grader
149 194 grader_proc = GraderProcess.register(config.grader_hostname,
150 195 Process.pid,
151 196 grader_mode)
152 197 else
153 198 grader_proc = nil
154 199 end
155 200
156 201 #set loggin environment
157 202 ENV['GRADER_LOGGING'] = log_file_name
158 203
159 204 # register exit handler to report inactive, and terminated
160 205 at_exit do
161 206 if grader_proc!=nil
162 207 grader_proc.report_inactive
163 208 grader_proc.terminate
164 209 end
165 210 end
166 211
167 212 #
168 213 # MAIN LOOP
169 214 #
170 215
171 216 case grader_mode
172 217 when "queue", "test_request"
173 218 log "Grader: #{grader_mode}"
174 219 if grader_mode=="queue"
175 220 engine = Grader::Engine.new
176 221 else
177 - engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
178 - Grader::TestRequestReporter.new)
222 + engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
223 + :reporter => Grader::TestRequestReporter.new)
179 224 end
180 225
181 226 runner = Grader::Runner.new(engine, grader_proc)
182 227 while true
183 228
184 229 if check_stopfile # created by calling grader stop
185 230 clear_stopfile
186 231 log "stopped (with stop file)"
187 232 break
188 233 end
189 234
190 235 if grader_mode=="queue"
191 236 task = runner.grade_oldest_task
192 237 else
193 238 task = runner.grade_oldest_test_request
194 239 end
195 240 if task==nil
196 241 sleep(1)
197 242 end
198 243 end
199 244
200 245 when "prob"
201 - engine = Grader::Engine.new(nil, Grader::SubmissionReporter.new(dry_run))
246 + if options[:report]
247 + result_collector = ResultCollector.new
248 + else
249 + result_collector = nil
250 + end
251 +
252 + engine = (Grader::Engine.
253 + new(:reporter =>
254 + Grader::SubmissionReporter.new(:dry_run => dry_run,
255 + :result_collector => result_collector)))
202 256 runner = Grader::Runner.new(engine, grader_proc)
203 257
204 258 grader_proc.report_active if grader_proc!=nil
205 259
206 260 ARGV.each do |prob_name|
207 261 prob = Problem.find_by_name(prob_name)
208 262 if prob==nil
209 263 puts "cannot find problem: #{prob_name}"
210 264 else
211 265 runner.grade_problem(prob)
212 266 end
213 267 end
214 268
269 + if options[:report]
270 + result_collector.print_report_by_user
271 + end
272 +
215 273 when "sub"
216 274 engine = Grader::Engine.new
217 275 runner = Grader::Runner.new(engine, grader_proc)
218 276
219 277 grader_proc.report_active if grader_proc!=nil
220 278
221 279 ARGV.each do |sub_id|
222 280 puts "Grading #{sub_id}"
223 281 begin
224 282 submission = Submission.find(sub_id.to_i)
225 283 rescue ActiveRecord::RecordNotFound
226 284 puts "Record not found"
227 285 submission = nil
228 286 end
229 287
230 288 if submission!=nil
231 289 runner.grade_submission(submission)
232 290 end
233 291 end
234 292
235 293 else
236 294 display_manual
237 295 exit(0)
238 296 end
239 297
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,179 +1,187
1 1 require 'fileutils'
2 2 require 'ftools'
3 3 require File.join(File.dirname(__FILE__),'dir_init')
4 4
5 5 module Grader
6 6
7 7 #
8 8 # A grader engine grades a submission, against anything: a test
9 9 # data, or a user submitted test data. It uses two helpers objects:
10 10 # room_maker and reporter.
11 11 #
12 12 class Engine
13 13
14 14 attr_writer :room_maker
15 15 attr_writer :reporter
16 16
17 - def initialize(room_maker=nil, reporter=nil)
17 + def initialize(options={})
18 + # default options
19 + if not options.include? :room_maker
20 + options[:room_maker] = Grader::SubmissionRoomMaker.new
21 + end
22 + if not options.include? :reporter
23 + options[:reporter] = Grader::SubmissionReporter.new
24 + end
25 +
18 26 @config = Grader::Configuration.get_instance
19 27
20 - @room_maker = room_maker || Grader::SubmissionRoomMaker.new
21 - @reporter = reporter || Grader::SubmissionReporter.new
28 + @room_maker = options[:room_maker]
29 + @reporter = options[:reporter]
22 30 end
23 31
24 32 # takes a submission, asks room_maker to produce grading directories,
25 33 # calls grader scripts, and asks reporter to save the result
26 34 def grade(submission)
27 35 current_dir = `pwd`.chomp
28 36
29 37 user = submission.user
30 38 problem = submission.problem
31 39
32 40 # TODO: will have to create real exception for this
33 41 if user==nil or problem == nil
34 42 @reporter.report_error(submission,"Grading error: problem with submission")
35 43 #raise "engine: user or problem is nil"
36 44 end
37 45
38 46 # TODO: this is another hack so that output only task can be judged
39 47 if submission.language!=nil
40 48 language = submission.language.name
41 49 lang_ext = submission.language.ext
42 50 else
43 51 language = 'c'
44 52 lang_ext = 'c'
45 53 end
46 54
47 55 # FIX THIS
48 56 talk 'some hack on language'
49 57 if language == 'cpp'
50 58 language = 'c++'
51 59 end
52 60
53 61 # COMMENT: should it be only source.ext?
54 62 if problem!=nil
55 63 source_name = "#{problem.name}.#{lang_ext}"
56 64 else
57 65 source_name = "source.#{lang_ext}"
58 66 end
59 67
60 68 begin
61 69 grading_dir = @room_maker.produce_grading_room(submission)
62 70 @room_maker.save_source(submission,source_name)
63 71 problem_home = @room_maker.find_problem_home(submission)
64 72
65 73 # puts "GRADING DIR: #{grading_dir}"
66 74 # puts "PROBLEM DIR: #{problem_home}"
67 75
68 76 if !FileTest.exist?(problem_home)
69 77 raise "No test data."
70 78 end
71 79
72 80 dinit = DirInit::Manager.new(problem_home)
73 81
74 82 dinit.setup do
75 83 copy_log = copy_script(problem_home)
76 84 save_copy_log(problem_home,copy_log)
77 85 end
78 86
79 87 call_judge(problem_home,language,grading_dir,source_name)
80 88
81 89 @reporter.report(submission,"#{grading_dir}/test-result")
82 90
83 91 dinit.teardown do
84 92 copy_log = load_copy_log(problem_home)
85 93 clear_copy_log(problem_home)
86 94 clear_script(copy_log,problem_home)
87 95 end
88 96
89 97 rescue RuntimeError => msg
90 98 @reporter.report_error(submission, msg)
91 99
92 100 ensure
93 101 @room_maker.clean_up(submission)
94 102 Dir.chdir(current_dir) # this is really important
95 103 end
96 104 end
97 105
98 106 protected
99 107
100 108 def talk(str)
101 109 if @config.talkative
102 110 puts str
103 111 end
104 112 end
105 113
106 114 def call_judge(problem_home,language,grading_dir,fname)
107 115 ENV['PROBLEM_HOME'] = problem_home
108 116
109 117 talk grading_dir
110 118 Dir.chdir grading_dir
111 119 cmd = "#{problem_home}/script/judge #{language} #{fname}"
112 120 talk "CMD: #{cmd}"
113 121 system(cmd)
114 122 end
115 123
116 124 def get_std_script_dir
117 125 GRADER_ROOT + '/std-script'
118 126 end
119 127
120 128 def copy_script(problem_home)
121 129 script_dir = "#{problem_home}/script"
122 130 std_script_dir = get_std_script_dir
123 131
124 132 raise "std-script directory not found" if !FileTest.exist?(std_script_dir)
125 133
126 134 scripts = Dir[std_script_dir + '/*']
127 135
128 136 copied = []
129 137
130 138 scripts.each do |s|
131 139 fname = File.basename(s)
132 140 if !FileTest.exist?("#{script_dir}/#{fname}")
133 141 copied << fname
134 142 system("cp #{s} #{script_dir}")
135 143 end
136 144 end
137 145
138 146 return copied
139 147 end
140 148
141 149 def copy_log_filename(problem_home)
142 150 return File.join(problem_home, '.scripts_copied')
143 151 end
144 152
145 153 def save_copy_log(problem_home, log)
146 154 f = File.new(copy_log_filename(problem_home),"w")
147 155 log.each do |fname|
148 156 f.write("#{fname}\n")
149 157 end
150 158 f.close
151 159 end
152 160
153 161 def load_copy_log(problem_home)
154 162 f = File.new(copy_log_filename(problem_home),"r")
155 163 log = []
156 164 f.readlines.each do |line|
157 165 log << line.strip
158 166 end
159 167 f.close
160 168 log
161 169 end
162 170
163 171 def clear_copy_log(problem_home)
164 172 File.delete(copy_log_filename(problem_home))
165 173 end
166 174
167 175 def clear_script(log,problem_home)
168 176 log.each do |s|
169 177 system("rm #{problem_home}/script/#{s}")
170 178 end
171 179 end
172 180
173 181 def mkdir_if_does_not_exist(dirname)
174 182 Dir.mkdir(dirname) if !FileTest.exist?(dirname)
175 183 end
176 184
177 185 end
178 186
179 187 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,127 +1,135
1 1 module Grader
2 2
3 3 class SubmissionRoomMaker
4 4 def initialize
5 5 @config = Grader::Configuration.get_instance
6 6 end
7 7
8 8 def produce_grading_room(submission)
9 9 user = submission.user
10 10 problem = submission.problem
11 11 grading_room = "#{@config.user_result_dir}/" +
12 12 "#{user.login}/#{problem.name}/#{submission.id}"
13 13
14 14 FileUtils.mkdir_p(grading_room)
15 15 grading_room
16 16 end
17 17
18 18 def find_problem_home(submission)
19 19 problem = submission.problem
20 20 "#{@config.problems_dir}/#{problem.name}"
21 21 end
22 22
23 23 def save_source(submission,source_name)
24 24 dir = self.produce_grading_room(submission)
25 25 f = File.open("#{dir}/#{source_name}","w")
26 26 f.write(submission.source)
27 27 f.close
28 28 end
29 29
30 30 def clean_up(submission)
31 31 end
32 32 end
33 33
34 34 class SubmissionReporter
35 - def initialize(dry_run=false)
35 + def initialize(options={})
36 + options = {:dry_run => false, :result_collector => nil}.merge(options)
36 37 @config = Grader::Configuration.get_instance
37 - @dry_run = dry_run
38 + @dry_run = options[:dry_run]
39 + @result_collector = options[:result_collector]
38 40 end
39 41
40 42 def report(sub,test_result_dir)
41 - save_result(sub,read_result(test_result_dir))
43 + result = read_result(test_result_dir)
44 + if @result_collector
45 + @result_collector.save(sub.user,
46 + sub.problem,
47 + result)
48 + end
49 + save_result(sub,result)
42 50 end
43 51
44 52 def report_error(sub,msg)
45 53 save_result(sub,{:points => 0,
46 54 :comment => "Grading error: #{msg}" })
47 55 end
48 56
49 57 protected
50 58 def read_result(test_result_dir)
51 59 cmp_msg_fname = "#{test_result_dir}/compiler_message"
52 60 if FileTest.exist?(cmp_msg_fname)
53 61 cmp_file = File.open(cmp_msg_fname)
54 62 cmp_msg = cmp_file.read
55 63 cmp_file.close
56 64 else
57 65 cmp_msg = ""
58 66 end
59 67
60 68 result_fname = "#{test_result_dir}/result"
61 69 comment_fname = "#{test_result_dir}/comment"
62 70 if FileTest.exist?(result_fname)
63 71 comment = ""
64 72 begin
65 73 result_file = File.open(result_fname)
66 74 result = result_file.readline.to_i
67 75 result_file.close
68 76 rescue
69 77 result = 0
70 78 comment = "error reading result file."
71 79 end
72 80
73 81 begin
74 82 comment_file = File.open(comment_fname)
75 83 comment += comment_file.readline.chomp
76 84 comment_file.close
77 85 rescue
78 86 comment += ""
79 87 end
80 88
81 89 return {:points => result,
82 90 :comment => comment,
83 91 :cmp_msg => cmp_msg}
84 92 else
85 93 if FileTest.exist?("#{test_result_dir}/a.out")
86 94 return {:points => 0,
87 95 :comment => 'error during grading',
88 96 :cmp_msg => cmp_msg}
89 97 else
90 98 return {:points => 0,
91 99 :comment => 'compilation error',
92 100 :cmp_msg => cmp_msg}
93 101 end
94 102 end
95 103 end
96 104
97 105 def save_result(submission,result)
98 106 problem = submission.problem
99 107 submission.graded_at = Time.now.gmtime
100 108 points = result[:points]
101 109 submission.points = points
102 110 comment = @config.report_comment(result[:comment])
103 111
104 112 #
105 113 # TODO: FIX THIS MESSAGE
106 114 #
107 115 if problem == nil
108 116 submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
109 117 elsif points == problem.full_score
110 118 #submission.grader_comment = 'PASSED: ' + comment
111 119 submission.grader_comment = comment
112 120 elsif result[:comment].chomp =~ /^[\[\]P]+$/
113 121 submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
114 122 else
115 123 #submission.grader_comment = 'FAILED: ' + comment
116 124 submission.grader_comment = comment
117 125 end
118 126 submission.compiler_message = result[:cmp_msg] or ''
119 127
120 128 if not @dry_run
121 129 submission.save
122 130 end
123 131 end
124 132
125 133 end
126 134
127 135 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,316 +1,316
1 1 require File.join(File.dirname(__FILE__),'spec_helper')
2 2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3 3
4 4 describe "A grader engine, when grading submissions" do
5 5
6 6 include GraderEngineHelperMethods
7 7
8 8 before(:each) do
9 9 @config = Grader::Configuration.get_instance
10 10
11 11 # this test is from Pong
12 12 @problem_test_normal = stub(Problem,
13 13 :id => 1, :name => 'test_normal',
14 14 :full_score => 135)
15 15 @user_user1 = stub(User,
16 16 :id => 1, :login => 'user1')
17 17
18 18 @engine = Grader::Engine.new
19 19 init_sandbox
20 20 end
21 21
22 22 it "should grade normal submission" do
23 23 grader_should(:grade => "test1_correct.c",
24 24 :on => @problem_test_normal,
25 25 :and_report => {
26 26 :score => 135,
27 27 :comment => /^(\[|P|\])+/})
28 28 end
29 29
30 30
31 31 it "should produce error message when submission cannot compile" do
32 32 grader_should(:grade => "test1_compile_error.c",
33 33 :on => @problem_test_normal,
34 34 :and_report => {
35 35 :score => 0,
36 36 :comment => 'compilation error',
37 37 :compiler_message => /[Ee]rror/})
38 38 end
39 39
40 40 it "should produce timeout error when submission runs forever" do
41 41 @problem_test_timeout = stub(Problem,
42 42 :id => 1, :name => 'test_timeout',
43 43 :full_score => 10)
44 44 grader_should(:grade => "test2_timeout.c",
45 45 :on => @problem_test_timeout,
46 46 :and_report => {
47 47 :score => 0,
48 48 :comment => 'TT'})
49 49 end
50 50
51 51 it "should produce timeout error correctly with fractional running time and fractional time limits" do
52 52 @problem_test_timeout = stub(Problem,
53 53 :id => 1, :name => 'test_timeout',
54 54 :full_score => 20)
55 55 grader_should(:grade => "test2_05sec.c",
56 56 :on => @problem_test_timeout,
57 57 :and_report => {
58 58 :score => 10,
59 59 :comment => 'TP'})
60 60 end
61 61
62 62 it "should produce runtime error when submission uses too much static memory" do
63 63 @problem_test_memory = stub(Problem,
64 64 :id => 1, :name => 'test_memory',
65 65 :full_score => 20)
66 66 grader_should(:grade => "add_too_much_memory_static.c",
67 67 :on => @problem_test_memory,
68 68 :and_report => {
69 69 :score => 10,
70 70 :comment => /[^P]P/})
71 71 end
72 72
73 73 it "should not allow submission to allocate too much dynamic memory" do
74 74 @problem_test_memory = stub(Problem,
75 75 :id => 1, :name => 'test_memory',
76 76 :full_score => 20)
77 77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 78 :on => @problem_test_memory,
79 79 :and_report => {
80 80 :score => 10,
81 81 :comment => /[^P]P/})
82 82 end
83 83
84 84 it "should score test runs correctly when submission fails in some test case" do
85 85 grader_should(:grade => "add_fail_test_case_1.c",
86 86 :on => @problem_test_normal,
87 87 :and_report => {
88 88 :score => 105,
89 89 :comment => /^.*(-|x).*$/})
90 90 end
91 91
92 92 it "should fail submission with non-zero exit status" do
93 93 grader_should(:grade => "add_nonzero_exit_status.c",
94 94 :on => @problem_test_normal,
95 95 :and_report => {
96 96 :score => 0,
97 97 :comment => /^(\[|x|\])+$/})
98 98 end
99 99
100 100 it "should not allow malicious submission to see PROBLEM_HOME" do
101 101 problem_test_yesno = stub(Problem,
102 102 :id => 1, :name => 'test_yesno',
103 103 :full_score => 10)
104 104 grader_should(:grade => "yesno_access_problem_home.c",
105 105 :on => problem_test_yesno,
106 106 :and_report => {
107 107 :score => 0,
108 108 :comment => /(-|x)/})
109 109 end
110 110
111 111 it "should not allow malicious submission to open files" do
112 112 problem_test_yesno = stub(Problem,
113 113 :id => 1, :name => 'test_yesno',
114 114 :full_score => 10)
115 115 grader_should(:grade => "yesno_open_file.c",
116 116 :on => problem_test_yesno,
117 117 :and_report => {
118 118 :score => 0,
119 119 :comment => /(-|x)/})
120 120 end
121 121
122 122 def grader_should(args)
123 123 @user1 = stub(User,
124 124 :id => 1, :login => 'user1')
125 125 submission =
126 126 create_submission_from_file(1, @user1, args[:on], args[:grade])
127 127 submission.should_receive(:graded_at=)
128 128
129 129 expected_score = args[:and_report][:score]
130 130 expected_comment = args[:and_report][:comment]
131 131 if args[:and_report][:compiler_message]!=nil
132 132 expected_compiler_message = args[:and_report][:compiler_message]
133 133 else
134 134 expected_compiler_message = ''
135 135 end
136 136
137 137 submission.should_receive(:points=).with(expected_score)
138 138 submission.should_receive(:grader_comment=).with(expected_comment)
139 139 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 140 submission.should_receive(:save)
141 141
142 142 @engine.grade(submission)
143 143 end
144 144
145 145 protected
146 146
147 147 def create_normal_submission_mock_from_file(source_fname)
148 148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 149 end
150 150
151 151 end
152 152
153 153 describe "A grader engine, when grading test requests" do
154 154
155 155 include GraderEngineHelperMethods
156 156
157 157 before(:each) do
158 158 @config = Grader::Configuration.get_instance
159 - @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
160 - Grader::TestRequestReporter.new)
159 + @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
160 + :reporter => Grader::TestRequestReporter.new)
161 161 init_sandbox
162 162 end
163 163
164 164 it "should report error if there is no problem template" do
165 165 problem = stub(Problem,
166 166 :id => 1, :name => 'nothing')
167 167 grader_should(:grade => 'test1_correct.c',
168 168 :on => problem,
169 169 :with => 'in1.txt',
170 170 :and_report => {
171 171 :graded_at= => nil,
172 172 :compiler_message= => '',
173 173 :grader_comment= => '',
174 174 :running_stat= => /template not found/,
175 175 :running_time= => nil,
176 176 :exit_status= => nil,
177 177 :memory_usage= => nil,
178 178 :save => nil})
179 179 end
180 180
181 181 it "should run test request and produce output file" do
182 182 problem = stub(Problem,
183 183 :id => 1, :name => 'test_normal')
184 184 grader_should(:grade => 'test1_correct.c',
185 185 :on => problem,
186 186 :with => 'in1.txt',
187 187 :and_report => {
188 188 :graded_at= => nil,
189 189 :compiler_message= => '',
190 190 :grader_comment= => '',
191 191 :running_stat= => /0.0\d* sec./,
192 192 :output_file_name= => lambda { |fname|
193 193 File.exists?(fname).should be_true
194 194 },
195 195 :running_time= => nil,
196 196 :exit_status= => nil,
197 197 :memory_usage= => nil,
198 198 :save => nil})
199 199 end
200 200
201 201 it "should clean up problem directory after running test request" do
202 202 problem = stub(Problem,
203 203 :id => 1, :name => 'test_normal')
204 204 grader_should(:grade => 'test1_correct.c',
205 205 :on => problem,
206 206 :with => 'in1.txt',
207 207 :and_report => {
208 208 :graded_at= => nil,
209 209 :compiler_message= => '',
210 210 :grader_comment= => '',
211 211 :running_stat= => nil,
212 212 :output_file_name= => nil,
213 213 :running_time= => nil,
214 214 :exit_status= => nil,
215 215 :memory_usage= => nil,
216 216 :save => nil})
217 217 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
218 218 end
219 219
220 220 it "should compile test request with error and report compilation error" do
221 221 problem = stub(Problem,
222 222 :id => 1, :name => 'test_normal')
223 223 grader_should(:grade => 'test1_compile_error.c',
224 224 :on => problem,
225 225 :with => 'in1.txt',
226 226 :and_report => {
227 227 :graded_at= => nil,
228 228 :compiler_message= => /.+/,
229 229 :grader_comment= => /[Cc]ompil.*error/,
230 230 :running_stat= => '',
231 231 :save => nil})
232 232 end
233 233
234 234 it "should report exit status" do
235 235 problem = stub(Problem,
236 236 :id => 1, :name => 'test_normal')
237 237 grader_should(:grade => 'add_nonzero_exit_status.c',
238 238 :on => problem,
239 239 :with => 'in1.txt',
240 240 :and_report => {
241 241 :graded_at= => nil,
242 242 :compiler_message= => '',
243 243 :grader_comment= => '',
244 244 :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
245 245 :output_file_name= => lambda { |fname|
246 246 File.exists?(fname).should be_true
247 247 },
248 248 :running_time= => nil,
249 249 :exit_status= => /10/,
250 250 :memory_usage= => nil,
251 251 :save => nil})
252 252 end
253 253
254 254 it "should produce running statistics for normal submission" do
255 255 problem = stub(Problem,
256 256 :id => 1, :name => 'test_normal')
257 257 grader_should(:grade => 'test_run_stat.c',
258 258 :on => problem,
259 259 :with => 'in1.txt',
260 260 :and_report => {
261 261 :graded_at= => nil,
262 262 :compiler_message= => '',
263 263 :grader_comment= => '',
264 264 :running_stat= => nil,
265 265 :output_file_name= => lambda { |fname|
266 266 File.exists?(fname).should be_true
267 267 },
268 268 :running_time= => lambda { |t|
269 269 (t>=0.14) and (t<=0.16)
270 270 },
271 271 :exit_status= => nil,
272 272 :memory_usage= => lambda { |s|
273 273 (s>=500) and (s<=1000)
274 274 },
275 275 :save => nil})
276 276 end
277 277
278 278 protected
279 279 def grader_should(args)
280 280 @user1 = stub(User,
281 281 :id => 1, :login => 'user1')
282 282
283 283 problem = args[:on]
284 284 input_file = @config.test_request_input_base_dir + "/" + args[:with]
285 285
286 286 submission =
287 287 create_submission_from_file(1, @user1, args[:on], args[:grade])
288 288
289 289 test_request = stub(TestRequest,
290 290 :id => 1,
291 291 :user => @user1,
292 292 :problem => problem,
293 293 :submission => submission,
294 294 :input_file_name => input_file,
295 295 :language => submission.language,
296 296 :problem_name => problem.name)
297 297
298 298 expectations = args[:and_report]
299 299
300 300 expectations.each do |key,val|
301 301 if val==nil
302 302 test_request.should_receive(key)
303 303 elsif val.class == Proc
304 304 test_request.should_receive(key) { |fname|
305 305 val.call(fname)
306 306 }
307 307 else
308 308 test_request.should_receive(key).with(val)
309 309 end
310 310 end
311 311
312 312 @engine.grade(test_request)
313 313 end
314 314
315 315 end
316 316
You need to be logged in to leave comments. Login now