cafe-grader-judge-scripts

Commit r117:cc3e1102f8c6 public

parent | child
Description:
translates test_request input/output file paths for remote grading test_request
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r117:cc3e1102f8c6 - - 5 files changed: 36 inserted, 3 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,20 +1,23
1 1 #
2 2 # See documentation in lib/configuration.rb
3 3 #
4 4 Grader::Initializer.run do |config|
5 5
6 6 config.problems_dir = GRADER_ROOT + "/../ev-exam"
7 7 config.user_result_dir = GRADER_ROOT + "/../result"
8 8
9 9 config.talkative = true
10 10 config.logging = true
11 11 config.log_dir = GRADER_ROOT + "/../log"
12 12
13 13 config.report_grader = true
14 14
15 15 config.test_request_input_base_dir = RAILS_ROOT + "/data/test_request/input"
16 16 config.test_request_output_base_dir = RAILS_ROOT + "/data/test_request/output"
17 17 config.test_request_problem_templates_dir = config.problems_dir + "/test_request"
18 18
19 + # change this if you want the path on the output to be translated
20 + config.test_request_org_output_base_dir = config.test_request_output_base_dir
21 +
19 22 config.comment_report_style = :short
20 23 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,19 +1,22
1 1 #
2 2 # See documentation in lib/configuration.rb
3 3 #
4 4 Grader::Initializer.run do |config|
5 5 config.problems_dir = GRADER_ROOT + "/../ev"
6 6 config.user_result_dir = GRADER_ROOT + "/../result"
7 7
8 8 config.talkative = true
9 9 config.logging = true
10 10 config.log_dir = GRADER_ROOT + "/../log"
11 11
12 12 config.report_grader = true
13 13
14 14 config.test_request_input_base_dir = RAILS_ROOT + "/data/test_request/input"
15 15 config.test_request_output_base_dir = RAILS_ROOT + "/data/test_request/output"
16 16 config.test_request_problem_templates_dir = config.problems_dir + "/test_request"
17 +
18 + # change this if you want the path on the output to be translated
19 + config.test_request_org_output_base_dir = config.test_request_output_base_dir
17 20
18 21 config.comment_report_style = :full
19 22 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,29 +1,32
1 1 #
2 2 # See documentation in lib/configuration.rb
3 3 #
4 4 Grader::Initializer.run do |config|
5 5 config.problems_dir = GRADER_ROOT + "/test/sandbox/ev"
6 6 config.user_result_dir = GRADER_ROOT + "/test/sandbox/result"
7 7
8 8 config.talkative = false
9 9
10 10 config.report_grader = false
11 11
12 12 config.rails_env = 'test'
13 13
14 14 config.comment_report_style = :full
15 15
16 16 config.test_request_input_base_dir = GRADER_ROOT + "/test/data/test_request/input"
17 17 config.test_request_output_base_dir = GRADER_ROOT + "/test/sandbox/test_request/output"
18 18 config.test_request_problem_templates_dir = GRADER_ROOT + "/test/data/test_request/problems"
19 19
20 + # change this if you want the path on the output to be translated
21 + config.test_request_org_output_base_dir = config.test_request_output_base_dir
22 +
20 23 #
21 24 # These options are for testing
22 25 #
23 26 class << config
24 27 attr_accessor :test_data_dir, :test_sandbox_dir
25 28 end
26 29
27 30 config.test_data_dir = GRADER_ROOT + "/test/data"
28 31 config.test_sandbox_dir = GRADER_ROOT + "/test/sandbox"
29 32 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,84 +1,88
1 1 module Grader
2 2
3 3 # This singleton class holds basic configurations for grader. When
4 4 # running in each mode, grader uses resources from different
5 5 # directories and outputs differently. Usually the attributes name
6 6 # are descriptive; below we explain more on each attributes.
7 7 class Configuration
8 8 # Rails' environment: "development", "production"
9 9 attr_accessor :rails_env
10 10
11 11 # Grader looks for problem [prob] in problem_dir/[prob], and store
12 12 # execution results for submission [x] of user [u] in directory
13 13 # user_result_dir/[u]/[x]
14 14 attr_accessor :problems_dir
15 15 attr_accessor :user_result_dir
16 16
17 17 # If report_grader=true, the grader would add a row in model
18 18 # GraderProcess. It would report itself with grader_hostname and
19 19 # process id.
20 20 attr_accessor :report_grader
21 21 attr_accessor :grader_hostname
22 22
23 23 # If talkative=true, grader would report status to console. If
24 24 # logging=true, grader would report status to a log file located
25 25 # in log_dir, in a file name mode.options.pid. TODO: defined
26 26 # log file naming.
27 27 attr_accessor :talkative
28 28 attr_accessor :logging
29 29 attr_accessor :log_dir
30 30
31 31 # These are directories related to the test interface.
32 32 attr_accessor :test_request_input_base_dir
33 33 attr_accessor :test_request_output_base_dir
34 34 attr_accessor :test_request_problem_templates_dir
35 35
36 + # this is for linking output from test request
37 + # TODO: find a cleaner way to do this.
38 + attr_accessor :test_request_org_output_base_dir
39 +
36 40 # Comment received from the grading script will be filtered
37 41 # through Configuration#report_comment. How this method behave
38 42 # depends on this option; right now only two formats, :short and
39 43 # :long
40 44 attr_accessor :comment_report_style
41 45
42 46 def report_comment(comment)
43 47 case comment_report_style
44 48 when :short
45 49 if comment.chomp =~ /^[\[\]P]+$/ # all P's
46 50 'passed'
47 51 elsif comment.chomp =~ /[Cc]ompil.*[Ee]rror/
48 52 'compilation error'
49 53 else
50 54 'failed'
51 55 end
52 56
53 57 when :full
54 58 comment.chomp
55 59 end
56 60 end
57 61
58 62 # Codes for singleton
59 63 private_class_method :new
60 64
61 65 @@instance = nil
62 66
63 67 def self.get_instance
64 68 if @@instance==nil
65 69 @@instance = new
66 70 end
67 71 @@instance
68 72 end
69 73
70 74 private
71 75 def initialize
72 76 @talkative = false
73 77 @log_file = nil
74 78 @report_grader = false
75 79 @grader_hostname = `hostname`.chomp
76 80
77 81 @rails_env = 'development'
78 82
79 83 @comment_report_style = :full
80 84 end
81 85
82 86 end
83 87
84 88 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,254 +1,274
1 1 #
2 2 # This part contains various test_request helpers for interfacing
3 3 # with Grader::Engine. There are TestRequestRoomMaker and
4 4 # TestRequestReporter.
5 5
6 6 module Grader
7 7
8 + def self.translate_filepath(filename, marker, new_base_path)
9 + p = filename.index(marker)
10 + end_path = filename[(p+marker.length)..-1]
11 + return new_base_path + end_path
12 + end
13 +
8 14 def self.link_or_copy(src, des)
9 15 begin
10 16 FileUtils.ln_s(src, des)
11 17 rescue NotImplementedError
12 18 FileUtils.cp(src,des)
13 19 end
14 20 end
15 21
16 22 def self.call_and_log(error_message)
17 23 begin
18 24 yield
19 25 rescue
20 26 msg = "ERROR: #{error_message}"
21 27 raise msg
22 28 end
23 29 end
24 30
25 31 #
26 32 # A TestRequestRoomMaker is a helper object for Engine
27 33 # - finds grading room: in user_result_dir/(user)/test_request/ ...
28 34 # - prepare problem configuration for grading --- basically it copy
29 35 # all config files, and copy user's input into the testcase
30 36 # directory. First, it finds the template from problem template
31 37 # directory; if it can't find a template, it'll use the template
32 38 # from default template.
33 39 class TestRequestRoomMaker
34 40 def initialize
35 41 @config = Grader::Configuration.get_instance
36 42 end
37 43
38 44 def produce_grading_room(test_request)
39 45 grading_room = grading_room_dir(test_request)
40 46 FileUtils.mkdir_p(grading_room)
41 47
42 48 #
43 49 # Also copy additional submitted file to this directory as well.
44 50 # The program would see this file only if it is copied
45 51 # to the sandbox directory later. The run script should do it.
46 52 #
47 53 if FileTest.exists?("#{test_request.input_file_name}.files")
48 54 FileUtils.cp_r("#{test_request.input_file_name}.files/.",
49 55 "#{grading_room}")
50 56 end
51 57
52 58 grading_room
53 59 end
54 60
55 61 def find_problem_home(test_request)
56 62 problem_name = test_request.problem_name
57 63
58 64 template_dir = "#{@config.test_request_problem_templates_dir}/" + problem_name
59 65
60 66 raise "Test Request: error template not found" if !File.exists?(template_dir)
61 67
62 68 problem_home = problem_home_dir(test_request)
63 69 FileUtils.mkdir_p(problem_home)
64 70
65 71 copy_problem_template(template_dir,problem_home)
66 72 link_input_file(test_request,problem_home)
67 73
68 74 problem_home
69 75 end
70 76
71 77 def save_source(test_request,source_name)
72 78 dir = self.produce_grading_room(test_request)
73 79 submission = test_request.submission
74 80 f = File.open("#{dir}/#{source_name}","w")
75 81 f.write(submission.source)
76 82 f.close
77 83 end
78 84
79 85 def clean_up(test_request)
80 86 problem_home = problem_home_dir(test_request)
81 87 remove_data_files(problem_home)
82 88 end
83 89
84 90 protected
85 91 def grading_room_dir(test_request)
86 92 problem_name = test_request.problem_name
87 93 user = test_request.user
88 94 grading_room = "#{@config.user_result_dir}" +
89 95 "/#{user.login}/test_request" +
90 96 "/#{problem_name}/#{test_request.id}"
91 97 grading_room
92 98 end
93 99
94 100 def problem_home_dir(test_request)
95 101 problem_name = test_request.problem_name
96 102 user = test_request.user
97 103 "#{@config.user_result_dir}" +
98 104 "/#{user.login}/test_request/#{problem_name}"
99 105 end
100 106
101 107 def copy_problem_template(template_dir,problem_home)
102 108 Grader::call_and_log("Test Request: cannot copy problem template") {
103 109 FileUtils.cp_r("#{template_dir}/.","#{problem_home}")
104 110 }
105 111 end
106 112
113 + def translate_input_filename(filename)
114 + return Grader::translate_filepath(filename,
115 + 'input',
116 + @config.test_request_input_base_dir)
117 + end
118 +
107 119 def link_input_file(test_request, problem_home)
108 - input_fname = "#{test_request.input_file_name}"
120 + input_fname = translate_input_filename(test_request.input_file_name)
121 +
109 122 if !File.exists?(input_fname)
110 123 raise "Test Request: input file not found."
111 124 end
112 125
113 126 input_fname_problem_home = "#{problem_home}/test_cases/1/input-1.txt"
114 127 if File.exists?(input_fname_problem_home)
115 128 FileUtils.rm([input_fname_problem_home], :force => true)
116 129 end
117 130
118 131 Grader::link_or_copy("#{input_fname}", "#{input_fname_problem_home}")
119 132 end
120 133
121 134 def remove_data_files(problem_home)
122 135 if File.exists?("#{problem_home}/test_cases/1/input-1.txt")
123 136 Grader::call_and_log("Test Request: cannot remove data files") {
124 137 FileUtils.rm Dir.glob("#{problem_home}/test_cases/1/*")
125 138 }
126 139 end
127 140 end
128 141
129 142 end
130 143
131 144 class TestRequestReporter
132 145 def initialize
133 146 @config = Grader::Configuration.get_instance
134 147 end
135 148
136 149 def report(test_request,test_result_dir)
137 150 save_result(test_request,read_result(test_result_dir))
138 151 end
139 152
140 153 def report_error(test_request, msg)
141 154 save_result(test_request, {:running_stat => {
142 155 :msg => "#{msg}",
143 156 :running_time => nil,
144 157 :exit_status => "Some error occured. Program did not run",
145 158 :memory_usage => nil
146 159 }})
147 160 end
148 161
149 162 protected
150 163 def read_result(test_result_dir)
151 164 # TODO:
152 165 cmp_msg_fname = "#{test_result_dir}/compiler_message"
153 166 cmp_file = File.open(cmp_msg_fname)
154 167 cmp_msg = cmp_file.read
155 168 cmp_file.close
156 169
157 170 result_file_name = "#{test_result_dir}/1/result"
158 171
159 172 if File.exists?(result_file_name)
160 173 output_file_name = "#{test_result_dir}/1/output.txt"
161 174 results = []
162 175 File.open("#{test_result_dir}/1/result") do |f|
163 176 results = f.readlines
164 177 end
165 178 stat = extract_running_stat(results)
166 179
167 180 return {
168 181 :output_file_name => output_file_name,
169 182 :running_stat => stat,
170 183 :comment => "",
171 184 :cmp_msg => cmp_msg}
172 185 else
173 186 return {
174 187 :running_stat => nil,
175 188 :comment => "Compilation error",
176 189 :cmp_msg => cmp_msg}
177 190 end
178 191 end
179 192
180 193 def extract_running_stat(results)
181 194 running_stat_line = results[-1]
182 195
183 196 # extract exit status line
184 197 run_stat = ""
185 198 if !(/[Cc]orrect/.match(results[0]))
186 199 run_stat = results[0].chomp
187 200 else
188 201 run_stat = 'Program exited normally'
189 202 end
190 203
191 204 # extract running time
192 205 if res = /r(.*)u(.*)s/.match(running_stat_line)
193 206 seconds = (res[1].to_f + res[2].to_f)
194 207 time_stat = "Time used: #{seconds} sec."
195 208 else
196 209 seconds = nil
197 210 time_stat = "Time used: n/a sec."
198 211 end
199 212
200 213 # extract memory usage
201 214 if res = /s(.*)m/.match(running_stat_line)
202 215 memory_used = res[1].to_i
203 216 else
204 217 memory_used = -1
205 218 end
206 219
207 220 return {
208 221 :msg => "#{run_stat}\n#{time_stat}",
209 222 :running_time => seconds,
210 223 :exit_status => run_stat,
211 224 :memory_usage => memory_used
212 225 }
213 226 end
214 227
215 228 def save_result(test_request,result)
216 229 if result[:output_file_name]!=nil
217 - test_request.output_file_name = link_output_file(test_request,
218 - result[:output_file_name])
230 + org_filename = link_output_file(test_request,
231 + result[:output_file_name])
232 + test_request.output_file_name = translate_output_filename(org_filename)
219 233 end
220 234 test_request.graded_at = Time.now
221 235 test_request.compiler_message = (result[:cmp_msg] or '')
222 236 test_request.grader_comment = (result[:comment] or '')
223 237 if result[:running_stat]!=nil
224 238 test_request.running_stat = (result[:running_stat][:msg] or '')
225 239 test_request.running_time = (result[:running_stat][:running_time] or nil)
226 240 test_request.exit_status = result[:running_stat][:exit_status]
227 241 test_request.memory_usage = result[:running_stat][:memory_usage]
228 242 else
229 243 test_request.running_stat = ''
230 244 end
231 245 test_request.save
232 246 end
233 247
234 248 protected
249 + def translate_output_filename(filename)
250 + return Grader::translate_filepath(filename,
251 + 'output',
252 + @config.test_request_org_output_base_dir)
253 + end
254 +
235 255 def link_output_file(test_request, fname)
236 256 target_file_name = random_output_file_name(test_request.user,
237 257 test_request.problem)
238 258 FileUtils.mkdir_p(File.dirname(target_file_name))
239 259 Grader::link_or_copy("#{fname}", "#{target_file_name}")
240 260 return target_file_name
241 261 end
242 262
243 263 def random_output_file_name(user,problem)
244 264 problem_name = TestRequest.name_of(problem)
245 265 begin
246 266 tmpname = "#{@config.test_request_output_base_dir}" +
247 267 "/#{user.login}/#{problem_name}/#{rand(10000)}"
248 268 end while File.exists?(tmpname)
249 269 tmpname
250 270 end
251 271
252 272 end
253 273
254 274 end
You need to be logged in to leave comments. Login now