cafe-grader-judge-scripts

Commit r92:274af5f15b1f public

parent | child
Description:
added grading report
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r92:274af5f15b1f - - 4 files changed: 93 inserted, 19 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -11,229 +11,287
11 11 def check_stopfile
12 12 FileTest.exist?(File.dirname(__FILE__) + "/stop.all") or
13 13 FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
14 14 end
15 15
16 16 def clear_stopfile
17 17 if FileTest.exist?(File.dirname(__FILE__) + "/stop.#{Process.pid}")
18 18 system("rm " + File.dirname(__FILE__) + "/stop.#{Process.pid}")
19 19 end
20 20 end
21 21
22 22 def config
23 23 Grader::Configuration.get_instance
24 24 end
25 25
26 26 def log_file_name
27 27 if !File.exists?(config.log_dir)
28 28 raise "Log directory does not exist: #{config.log_dir}"
29 29 end
30 30 config.log_dir +
31 31 "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
32 32 end
33 33
34 34 def log(str)
35 35 if config.talkative
36 36 puts str
37 37 end
38 38 if config.logging
39 39 fp = File.open(log_file_name,"a")
40 40 fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
41 41 fp.close
42 42 end
43 43 end
44 44
45 45 def display_manual
46 46 puts <<USAGE
47 47 Grader.
48 48 using: (1) grader
49 49 (2) grader environment [mode]
50 50 (3) grader stop [all|pids-list]
51 51 (4) grader --help
52 52 (1) call grader with environment = 'exam', mode = 'queue'
53 53 (2) possible modes are: 'queue', 'prob', 'test_request'
54 54 (3) create stop-file to stop running grader in queue mode
55 55 (4) You are here.
56 56 USAGE
57 57 end
58 58
59 - #########################################
60 - # main program
61 - #########################################
62 -
63 - # Reading environment and options.
64 -
65 59 def process_options_and_stop_file
66 60 # The list of options are:
67 61 # - stop [all|process ids]
68 62 # -
69 63
70 64 # Process 'help' option
71 65 if (ARGV.length==1) and (/help/.match(ARGV[0]))
72 66 display_manual
73 67 exit(0)
74 68 end
75 69
76 70 # Process 'stop' option.
77 71 if (ARGV.length >= 1) and (ARGV[0]=='stop')
78 72 if ARGV.length==1
79 73 puts "you should specify pid-list or 'all'"
80 74 display_manual
81 75 elsif (ARGV.length==2) and (ARGV[1]=='all')
82 76 stop_grader(:all)
83 77 puts "A global stop file ('stop.all') created."
84 78 puts "You should remove it manually later."
85 79 else
86 80 (1..ARGV.length-1).each do |i|
87 81 stop_grader(ARGV[i])
88 82 end
89 83 puts "stop file(s) created"
90 84 end
91 85 exit(0)
92 86 end
93 87
94 88 # Check stop file.
95 89 if check_stopfile
96 90 puts "Stop file exists. Terminated."
97 91 clear_stopfile
98 92 exit(0)
99 93 end
100 94
101 95 #default options
102 96 options = {
103 97 :mode => 'queue',
104 98 :environment => 'exam',
105 99 :dry_run => false,
106 100 }
107 101
108 102 # Process mode and environment option
109 103 if ARGV.length >= 1
110 104 options[:environment] = ARGV.shift
111 105 if ARGV.length >=1
112 106 options[:mode] = ARGV.shift
113 107 end
114 108 end
115 109
116 110 options[:dry_run] = (ARGV.delete('--dry') != nil)
117 111 if options[:dry_run] and (not options[:mode] == 'prob')
118 - puts "Dry run currently works for 'prob' mode."
112 + puts "Dry run currently works only for 'prob' mode."
113 + exit(0)
114 + end
115 +
116 + options[:report] = (ARGV.delete('--report') != nil)
117 + if options[:report] and (not options[:mode] == 'prob')
118 + puts "Report currently works only for 'prob' mode."
119 119 exit(0)
120 120 end
121 121
122 122 return options
123 123 end
124 124
125 - # ======= Main ========
125 + class ResultCollector
126 + def initialize
127 + @results = {}
128 + @problems = {}
129 + @users = {}
130 + end
131 +
132 + def save(user, problem, grading_result)
133 + if not @problems.has_key? problem.id
134 + @problems[problem.id] = problem
135 + end
136 + if not @users.has_key? user.id
137 + @users[user.id] = user
138 + end
139 + @results[[user.id, problem.id]] = grading_result
140 + end
141 +
142 + def print_report_by_user
143 + puts "---------------------"
144 + puts " REPORT"
145 + puts "---------------------"
146 +
147 + print "login"
148 + @problems.each_value do |problem|
149 + print ",#{problem.name}"
150 + end
151 + print "\n"
152 +
153 + @users.each_value do |user|
154 + print "#{user.login}"
155 + @problems.each_value do |problem|
156 + if @results.has_key? [user.id, problem.id]
157 + print ",#{@results[[user.id,problem.id]][:points]}"
158 + else
159 + print ","
160 + end
161 + end
162 + print "\n"
163 + end
164 + end
165 + end
166 +
167 + #########################################
168 + # main program
169 + #########################################
170 +
126 171 options = process_options_and_stop_file
127 172 GRADER_ENV = options[:environment]
128 173 grader_mode = options[:mode]
129 174 dry_run = options[:dry_run]
130 175
131 176 puts "environment: #{GRADER_ENV}"
132 177 require File.join(File.dirname(__FILE__),'config/environment')
133 178
134 179 # add grader_mode to config
135 180 # this is needed because method log needs it. TODO: clean this up
136 181 class << config
137 182 attr_accessor :grader_mode
138 183 end
139 184 config.grader_mode = grader_mode
140 185
141 186 # reading rails environment
142 187 log 'Reading rails environment'
143 188
144 189 RAILS_ENV = config.rails_env
145 190 require RAILS_ROOT + '/config/environment'
146 191
147 192 # register grader process
148 193 if config.report_grader
149 194 grader_proc = GraderProcess.register(config.grader_hostname,
150 195 Process.pid,
151 196 grader_mode)
152 197 else
153 198 grader_proc = nil
154 199 end
155 200
156 201 #set loggin environment
157 202 ENV['GRADER_LOGGING'] = log_file_name
158 203
159 204 # register exit handler to report inactive, and terminated
160 205 at_exit do
161 206 if grader_proc!=nil
162 207 grader_proc.report_inactive
163 208 grader_proc.terminate
164 209 end
165 210 end
166 211
167 212 #
168 213 # MAIN LOOP
169 214 #
170 215
171 216 case grader_mode
172 217 when "queue", "test_request"
173 218 log "Grader: #{grader_mode}"
174 219 if grader_mode=="queue"
175 220 engine = Grader::Engine.new
176 221 else
177 - engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
178 - Grader::TestRequestReporter.new)
222 + engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
223 + :reporter => Grader::TestRequestReporter.new)
179 224 end
180 225
181 226 runner = Grader::Runner.new(engine, grader_proc)
182 227 while true
183 228
184 229 if check_stopfile # created by calling grader stop
185 230 clear_stopfile
186 231 log "stopped (with stop file)"
187 232 break
188 233 end
189 234
190 235 if grader_mode=="queue"
191 236 task = runner.grade_oldest_task
192 237 else
193 238 task = runner.grade_oldest_test_request
194 239 end
195 240 if task==nil
196 241 sleep(1)
197 242 end
198 243 end
199 244
200 245 when "prob"
201 - engine = Grader::Engine.new(nil, Grader::SubmissionReporter.new(dry_run))
246 + if options[:report]
247 + result_collector = ResultCollector.new
248 + else
249 + result_collector = nil
250 + end
251 +
252 + engine = (Grader::Engine.
253 + new(:reporter =>
254 + Grader::SubmissionReporter.new(:dry_run => dry_run,
255 + :result_collector => result_collector)))
202 256 runner = Grader::Runner.new(engine, grader_proc)
203 257
204 258 grader_proc.report_active if grader_proc!=nil
205 259
206 260 ARGV.each do |prob_name|
207 261 prob = Problem.find_by_name(prob_name)
208 262 if prob==nil
209 263 puts "cannot find problem: #{prob_name}"
210 264 else
211 265 runner.grade_problem(prob)
212 266 end
213 267 end
214 268
269 + if options[:report]
270 + result_collector.print_report_by_user
271 + end
272 +
215 273 when "sub"
216 274 engine = Grader::Engine.new
217 275 runner = Grader::Runner.new(engine, grader_proc)
218 276
219 277 grader_proc.report_active if grader_proc!=nil
220 278
221 279 ARGV.each do |sub_id|
222 280 puts "Grading #{sub_id}"
223 281 begin
224 282 submission = Submission.find(sub_id.to_i)
225 283 rescue ActiveRecord::RecordNotFound
226 284 puts "Record not found"
227 285 submission = nil
228 286 end
229 287
230 288 if submission!=nil
231 289 runner.grade_submission(submission)
232 290 end
233 291 end
234 292
235 293 else
236 294 display_manual
237 295 exit(0)
238 296 end
239 297
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,69 +1,77
1 1 require 'fileutils'
2 2 require 'ftools'
3 3 require File.join(File.dirname(__FILE__),'dir_init')
4 4
5 5 module Grader
6 6
7 7 #
8 8 # A grader engine grades a submission, against anything: a test
9 9 # data, or a user submitted test data. It uses two helpers objects:
10 10 # room_maker and reporter.
11 11 #
12 12 class Engine
13 13
14 14 attr_writer :room_maker
15 15 attr_writer :reporter
16 16
17 - def initialize(room_maker=nil, reporter=nil)
17 + def initialize(options={})
18 + # default options
19 + if not options.include? :room_maker
20 + options[:room_maker] = Grader::SubmissionRoomMaker.new
21 + end
22 + if not options.include? :reporter
23 + options[:reporter] = Grader::SubmissionReporter.new
24 + end
25 +
18 26 @config = Grader::Configuration.get_instance
19 27
20 - @room_maker = room_maker || Grader::SubmissionRoomMaker.new
21 - @reporter = reporter || Grader::SubmissionReporter.new
28 + @room_maker = options[:room_maker]
29 + @reporter = options[:reporter]
22 30 end
23 31
24 32 # takes a submission, asks room_maker to produce grading directories,
25 33 # calls grader scripts, and asks reporter to save the result
26 34 def grade(submission)
27 35 current_dir = `pwd`.chomp
28 36
29 37 user = submission.user
30 38 problem = submission.problem
31 39
32 40 # TODO: will have to create real exception for this
33 41 if user==nil or problem == nil
34 42 @reporter.report_error(submission,"Grading error: problem with submission")
35 43 #raise "engine: user or problem is nil"
36 44 end
37 45
38 46 # TODO: this is another hack so that output only task can be judged
39 47 if submission.language!=nil
40 48 language = submission.language.name
41 49 lang_ext = submission.language.ext
42 50 else
43 51 language = 'c'
44 52 lang_ext = 'c'
45 53 end
46 54
47 55 # FIX THIS
48 56 talk 'some hack on language'
49 57 if language == 'cpp'
50 58 language = 'c++'
51 59 end
52 60
53 61 # COMMENT: should it be only source.ext?
54 62 if problem!=nil
55 63 source_name = "#{problem.name}.#{lang_ext}"
56 64 else
57 65 source_name = "source.#{lang_ext}"
58 66 end
59 67
60 68 begin
61 69 grading_dir = @room_maker.produce_grading_room(submission)
62 70 @room_maker.save_source(submission,source_name)
63 71 problem_home = @room_maker.find_problem_home(submission)
64 72
65 73 # puts "GRADING DIR: #{grading_dir}"
66 74 # puts "PROBLEM DIR: #{problem_home}"
67 75
68 76 if !FileTest.exist?(problem_home)
69 77 raise "No test data."
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,89 +1,97
1 1 module Grader
2 2
3 3 class SubmissionRoomMaker
4 4 def initialize
5 5 @config = Grader::Configuration.get_instance
6 6 end
7 7
8 8 def produce_grading_room(submission)
9 9 user = submission.user
10 10 problem = submission.problem
11 11 grading_room = "#{@config.user_result_dir}/" +
12 12 "#{user.login}/#{problem.name}/#{submission.id}"
13 13
14 14 FileUtils.mkdir_p(grading_room)
15 15 grading_room
16 16 end
17 17
18 18 def find_problem_home(submission)
19 19 problem = submission.problem
20 20 "#{@config.problems_dir}/#{problem.name}"
21 21 end
22 22
23 23 def save_source(submission,source_name)
24 24 dir = self.produce_grading_room(submission)
25 25 f = File.open("#{dir}/#{source_name}","w")
26 26 f.write(submission.source)
27 27 f.close
28 28 end
29 29
30 30 def clean_up(submission)
31 31 end
32 32 end
33 33
34 34 class SubmissionReporter
35 - def initialize(dry_run=false)
35 + def initialize(options={})
36 + options = {:dry_run => false, :result_collector => nil}.merge(options)
36 37 @config = Grader::Configuration.get_instance
37 - @dry_run = dry_run
38 + @dry_run = options[:dry_run]
39 + @result_collector = options[:result_collector]
38 40 end
39 41
40 42 def report(sub,test_result_dir)
41 - save_result(sub,read_result(test_result_dir))
43 + result = read_result(test_result_dir)
44 + if @result_collector
45 + @result_collector.save(sub.user,
46 + sub.problem,
47 + result)
48 + end
49 + save_result(sub,result)
42 50 end
43 51
44 52 def report_error(sub,msg)
45 53 save_result(sub,{:points => 0,
46 54 :comment => "Grading error: #{msg}" })
47 55 end
48 56
49 57 protected
50 58 def read_result(test_result_dir)
51 59 cmp_msg_fname = "#{test_result_dir}/compiler_message"
52 60 if FileTest.exist?(cmp_msg_fname)
53 61 cmp_file = File.open(cmp_msg_fname)
54 62 cmp_msg = cmp_file.read
55 63 cmp_file.close
56 64 else
57 65 cmp_msg = ""
58 66 end
59 67
60 68 result_fname = "#{test_result_dir}/result"
61 69 comment_fname = "#{test_result_dir}/comment"
62 70 if FileTest.exist?(result_fname)
63 71 comment = ""
64 72 begin
65 73 result_file = File.open(result_fname)
66 74 result = result_file.readline.to_i
67 75 result_file.close
68 76 rescue
69 77 result = 0
70 78 comment = "error reading result file."
71 79 end
72 80
73 81 begin
74 82 comment_file = File.open(comment_fname)
75 83 comment += comment_file.readline.chomp
76 84 comment_file.close
77 85 rescue
78 86 comment += ""
79 87 end
80 88
81 89 return {:points => result,
82 90 :comment => comment,
83 91 :cmp_msg => cmp_msg}
84 92 else
85 93 if FileTest.exist?("#{test_result_dir}/a.out")
86 94 return {:points => 0,
87 95 :comment => 'error during grading',
88 96 :cmp_msg => cmp_msg}
89 97 else
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -111,98 +111,98
111 111 it "should not allow malicious submission to open files" do
112 112 problem_test_yesno = stub(Problem,
113 113 :id => 1, :name => 'test_yesno',
114 114 :full_score => 10)
115 115 grader_should(:grade => "yesno_open_file.c",
116 116 :on => problem_test_yesno,
117 117 :and_report => {
118 118 :score => 0,
119 119 :comment => /(-|x)/})
120 120 end
121 121
122 122 def grader_should(args)
123 123 @user1 = stub(User,
124 124 :id => 1, :login => 'user1')
125 125 submission =
126 126 create_submission_from_file(1, @user1, args[:on], args[:grade])
127 127 submission.should_receive(:graded_at=)
128 128
129 129 expected_score = args[:and_report][:score]
130 130 expected_comment = args[:and_report][:comment]
131 131 if args[:and_report][:compiler_message]!=nil
132 132 expected_compiler_message = args[:and_report][:compiler_message]
133 133 else
134 134 expected_compiler_message = ''
135 135 end
136 136
137 137 submission.should_receive(:points=).with(expected_score)
138 138 submission.should_receive(:grader_comment=).with(expected_comment)
139 139 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 140 submission.should_receive(:save)
141 141
142 142 @engine.grade(submission)
143 143 end
144 144
145 145 protected
146 146
147 147 def create_normal_submission_mock_from_file(source_fname)
148 148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 149 end
150 150
151 151 end
152 152
153 153 describe "A grader engine, when grading test requests" do
154 154
155 155 include GraderEngineHelperMethods
156 156
157 157 before(:each) do
158 158 @config = Grader::Configuration.get_instance
159 - @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
160 - Grader::TestRequestReporter.new)
159 + @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
160 + :reporter => Grader::TestRequestReporter.new)
161 161 init_sandbox
162 162 end
163 163
164 164 it "should report error if there is no problem template" do
165 165 problem = stub(Problem,
166 166 :id => 1, :name => 'nothing')
167 167 grader_should(:grade => 'test1_correct.c',
168 168 :on => problem,
169 169 :with => 'in1.txt',
170 170 :and_report => {
171 171 :graded_at= => nil,
172 172 :compiler_message= => '',
173 173 :grader_comment= => '',
174 174 :running_stat= => /template not found/,
175 175 :running_time= => nil,
176 176 :exit_status= => nil,
177 177 :memory_usage= => nil,
178 178 :save => nil})
179 179 end
180 180
181 181 it "should run test request and produce output file" do
182 182 problem = stub(Problem,
183 183 :id => 1, :name => 'test_normal')
184 184 grader_should(:grade => 'test1_correct.c',
185 185 :on => problem,
186 186 :with => 'in1.txt',
187 187 :and_report => {
188 188 :graded_at= => nil,
189 189 :compiler_message= => '',
190 190 :grader_comment= => '',
191 191 :running_stat= => /0.0\d* sec./,
192 192 :output_file_name= => lambda { |fname|
193 193 File.exists?(fname).should be_true
194 194 },
195 195 :running_time= => nil,
196 196 :exit_status= => nil,
197 197 :memory_usage= => nil,
198 198 :save => nil})
199 199 end
200 200
201 201 it "should clean up problem directory after running test request" do
202 202 problem = stub(Problem,
203 203 :id => 1, :name => 'test_normal')
204 204 grader_should(:grade => 'test1_correct.c',
205 205 :on => problem,
206 206 :with => 'in1.txt',
207 207 :and_report => {
208 208 :graded_at= => nil,
You need to be logged in to leave comments. Login now